From phk at varnish-cache.org  Mon Jan  2 10:02:17 2012
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Mon, 02 Jan 2012 11:02:17 +0100
Subject: [master] 473e58c Collapse two VTAILQ_ENTRY() in struct sess,
	saving 16 bytes.
Message-ID: <E1Rheif-00037q-2H@project.varnish-software.com>

commit 473e58c0b53edde3795bfc4637d2b71fb3166383
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Jan 2 10:01:47 2012 +0000

    Collapse two VTAILQ_ENTRY() in struct sess, saving 16 bytes.

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index 42562b5..a15f272 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -636,6 +636,8 @@ struct sess {
 	struct worker		*wrk;
 	struct req		*req;
 
+	VTAILQ_ENTRY(sess)	list;
+
 	/* Session related fields ------------------------------------*/
 
 	int			fd;
@@ -651,11 +653,8 @@ struct sess {
 	char			addr[ADDR_BUFSIZE];
 	char			port[PORT_BUFSIZE];
 
-	VTAILQ_ENTRY(sess)	poollist;
 	struct acct		acct_ses;
 
-	VTAILQ_ENTRY(sess)	list;
-
 	/* Timestamps, all on TIM_real() timescale */
 	double			t_open;		/* fd accepted */
 	double			t_idle;		/* fd accepted or resp sent */
diff --git a/bin/varnishd/cache/cache_pool.c b/bin/varnishd/cache/cache_pool.c
index 9b0760c..84a6cfd 100644
--- a/bin/varnishd/cache/cache_pool.c
+++ b/bin/varnishd/cache/cache_pool.c
@@ -201,7 +201,7 @@ Pool_Work_Thread(void *priv, struct worker *wrk)
 		if (wrk->sp != NULL) {
 			/* Process queued requests, if any */
 			assert(pp->lqueue > 0);
-			VTAILQ_REMOVE(&pp->queue, wrk->sp, poollist);
+			VTAILQ_REMOVE(&pp->queue, wrk->sp, list);
 			wrk->do_what = pool_do_sess;
 			pp->lqueue--;
 		} else if (!VTAILQ_EMPTY(&pp->socks)) {
@@ -314,7 +314,7 @@ pool_queue(struct pool *pp, struct sess *sp)
 		return (-1);
 	}
 
-	VTAILQ_INSERT_TAIL(&pp->queue, sp, poollist);
+	VTAILQ_INSERT_TAIL(&pp->queue, sp, list);
 	pp->nqueued++;
 	pp->lqueue++;
 	Lck_Unlock(&pp->mtx);



From phk at varnish-cache.org  Mon Jan  2 10:23:06 2012
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Mon, 02 Jan 2012 11:23:06 +0100
Subject: [master] c632af8 Pack the memitem a bit smarter and drop the payload
	element which costs us 8 bytes per allocation.
Message-ID: <E1Rhf2o-0002Cg-A6@project.varnish-software.com>

commit c632af8afa5976b05fe499c71d83f0bacde7de8d
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Jan 2 10:22:37 2012 +0000

    Pack the memitem a bit smarter and drop the payload element which costs
    us 8 bytes per allocation.

diff --git a/bin/varnishd/cache/cache_mempool.c b/bin/varnishd/cache/cache_mempool.c
index a9d4249..468d5a8 100644
--- a/bin/varnishd/cache/cache_mempool.c
+++ b/bin/varnishd/cache/cache_mempool.c
@@ -29,6 +29,7 @@
  */
 
 #include <stddef.h>
+#include <stdio.h>
 #include <stdlib.h>
 
 #include "config.h"
@@ -40,10 +41,9 @@
 struct memitem {
 	unsigned			magic;
 #define MEMITEM_MAGIC			0x42e55401
-	VTAILQ_ENTRY(memitem)		list;
 	unsigned			size;
+	VTAILQ_ENTRY(memitem)		list;
 	double				touched;
-	char				payload;
 };
 
 VTAILQ_HEAD(memhead_s, memitem);
@@ -92,8 +92,6 @@ mpl_alloc(const struct mempool *mpl)
  *   of the list not getting too old.
  */
 
-#include <stdio.h>
-
 static void *
 mpl_guard(void *priv)
 {
@@ -309,7 +307,7 @@ MPL_Get(struct mempool *mpl, unsigned *size)
 
 	CHECK_OBJ_NOTNULL(mi, MEMITEM_MAGIC);
 	/* Throw away sizeof info for FlexeLint: */
-	return ((void*)(uintptr_t)&mi->payload);
+	return ((void*)(uintptr_t)(mi+1));
 }
 
 void
@@ -320,7 +318,7 @@ MPL_Free(struct mempool *mpl, void *item)
 	CHECK_OBJ_NOTNULL(mpl, MEMPOOL_MAGIC);
 	AN(item);
 
-	mi = (void*)((uintptr_t)item - offsetof(struct memitem, payload));
+	mi = (void*)((uintptr_t)item - sizeof(*mi));
 	CHECK_OBJ_NOTNULL(mi, MEMITEM_MAGIC);
 	memset(item, 0, mi->size);
 
@@ -348,6 +346,6 @@ void
 MPL_AssertSane(void *item)
 {
 	struct memitem *mi;
-	mi = (void*)((uintptr_t)item - offsetof(struct memitem, payload));
+	mi = (void*)((uintptr_t)item - sizeof(*mi));
 	CHECK_OBJ_NOTNULL(mi, MEMITEM_MAGIC);
 }



From phk at varnish-cache.org  Mon Jan  2 10:23:58 2012
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Mon, 02 Jan 2012 11:23:58 +0100
Subject: [master] 58a062a Pack struct sess slightly smarter,
	and get its allocation size (incl memitem) down to 512 bytes,
	despite ridiculously large sockaddr_storage structs.
Message-ID: <E1Rhf3e-0002an-FC@project.varnish-software.com>

commit 58a062a6e39a11a3dd436be5c03792ef1f868f05
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Jan 2 10:23:18 2012 +0000

    Pack struct sess slightly smarter, and get its allocation size
    (incl memitem) down to 512 bytes, despite ridiculously large
    sockaddr_storage structs.

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index a15f272..75bc803 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -630,6 +630,10 @@ struct sess {
 	unsigned		magic;
 #define SESS_MAGIC		0x2c2f9c5a
 
+	enum step		step;
+	int			fd;
+	unsigned		vsl_id;
+
 	/* Cross references ------------------------------------------*/
 
 	struct sesspool		*sesspool;
@@ -640,9 +644,6 @@ struct sess {
 
 	/* Session related fields ------------------------------------*/
 
-	int			fd;
-	unsigned		vsl_id;
-
 	socklen_t		sockaddrlen;
 	socklen_t		mysockaddrlen;
 	struct sockaddr_storage	sockaddr;
@@ -658,18 +659,11 @@ struct sess {
 	/* Timestamps, all on TIM_real() timescale */
 	double			t_open;		/* fd accepted */
 	double			t_idle;		/* fd accepted or resp sent */
+	double			t_req;
 
 #if defined(HAVE_EPOLL_CTL)
 	struct epoll_event ev;
 #endif
-	enum step		step;
-
-	/* Request related fields ------------------------------------*/
-
-
-	/* Timestamps, all on TIM_real() timescale */
-	double			t_req;
-
 };
 
 /* Prototypes etc ----------------------------------------------------*/



From phk at varnish-cache.org  Mon Jan  2 10:49:46 2012
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Mon, 02 Jan 2012 11:49:46 +0100
Subject: [master] de08ec0 Shave 16 bytes of the ADDR_BUFSIZE to push struct
	sess(+memitem) down to 512 bytes on epoll systems.
Message-ID: <E1RhfSc-0000cZ-Ih@project.varnish-software.com>

commit de08ec088e0a8ca1264d0df46d037b01bdaeb091
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Jan 2 10:48:57 2012 +0000

    Shave 16 bytes of the ADDR_BUFSIZE to push struct sess(+memitem) down
    to 512 bytes on epoll systems.

diff --git a/bin/varnishd/common/common.h b/bin/varnishd/common/common.h
index 7fcc7ec..f665e52 100644
--- a/bin/varnishd/common/common.h
+++ b/bin/varnishd/common/common.h
@@ -63,9 +63,10 @@ struct cli;
 /**********************************************************************
  * NI_MAXHOST and less so NI_MAXSERV, are ridiculously large for numeric
  * representations of TCP/IP socket addresses, so we use our own.
+ * <netinet/in6.h>::INET6_ADDRSTRLEN is 46
  */
 
-#define ADDR_BUFSIZE	64
+#define ADDR_BUFSIZE	48
 #define PORT_BUFSIZE	8
 
 



From phk at varnish-cache.org  Mon Jan  2 11:04:40 2012
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Mon, 02 Jan 2012 12:04:40 +0100
Subject: [master] 8a37b31 Comment on struct sess size issues.
Message-ID: <E1Rhfh2-0006IU-8o@project.varnish-software.com>

commit 8a37b3122217fde89d98cc3b5ce0c9452ed843a1
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Jan 2 11:04:29 2012 +0000

    Comment on struct sess size issues.

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index 75bc803..6b0d0a9 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -624,7 +624,15 @@ struct req {
 
 };
 
-/*--------------------------------------------------------------------*/
+/*--------------------------------------------------------------------
+ * Struct sess is a high memory-load structure because sessions typically
+ * hang around the waiter for relatively long time.
+ *
+ * The size goal for struct sess + struct memitem is <512 bytes
+ *
+ * Getting down to the next relevant size (<256 bytes because of how malloc
+ * works, is not realistic without a lot of code changes.
+ */
 
 struct sess {
 	unsigned		magic;



From phk at varnish-cache.org  Mon Jan  2 12:11:09 2012
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Mon, 02 Jan 2012 13:11:09 +0100
Subject: [master] a4e2d47 All relevant BSD's have a testable kqueue now.
Message-ID: <E1RhgjN-0001AX-Ri@project.varnish-software.com>

commit a4e2d478d5d8bab31a654248b9509c84adc9f1f1
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Jan 2 12:10:57 2012 +0000

    All relevant BSD's have a testable kqueue now.

diff --git a/configure.ac b/configure.ac
index 8404de4..01eac28 100644
--- a/configure.ac
+++ b/configure.ac
@@ -334,16 +334,7 @@ AC_ARG_ENABLE(kqueue,
     [enable_kqueue=yes])
 
 if test "$enable_kqueue" = yes; then
-	case $target in
-	*-*-freebsd* | *-*-darwin9* | *-*-darwin11* | *-*-netbsd* )
-		AC_CHECK_FUNCS([kqueue])
-		;;
-	*-*-bsd*)
-		# No other BSD has a sufficiently recent implementation
-		AC_MSG_WARN([won't look for kqueue() on $target])
-		ac_cv_func_kqueue=no
-		;;
-	esac
+	AC_CHECK_FUNCS([kqueue])
 else
 	ac_cv_func_kqueue=no
 fi



From tfheen at varnish-cache.org  Mon Jan  2 12:40:12 2012
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Mon, 02 Jan 2012 13:40:12 +0100
Subject: [master] b70a93c Fix typo
Message-ID: <E1RhhBU-0004Yq-59@project.varnish-software.com>

commit b70a93cff1df0763a99a6b0fbad952ee86c24be3
Author: Tollef Fog Heen <tfheen at varnish-software.com>
Date:   Mon Jan 2 12:59:42 2012 +0100

    Fix typo

diff --git a/doc/sphinx/reference/varnish-cli.rst b/doc/sphinx/reference/varnish-cli.rst
index 69b6546..a097248 100644
--- a/doc/sphinx/reference/varnish-cli.rst
+++ b/doc/sphinx/reference/varnish-cli.rst
@@ -196,7 +196,7 @@ consists of a field, an operator, and an argument.  Conditions can be
 ANDed together with "&&".
 
 A field can be any of the variables from VCL, for instance req.url,
-req.http.host or obj.set-cookie.
+req.http.host or obj.http.set-cookie.
 
 Operators are "==" for direct comparision, "~" for a regular
 expression match, and ">" or "<" for size comparisons.  Prepending



From tfheen at varnish-cache.org  Mon Jan  2 12:40:12 2012
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Mon, 02 Jan 2012 13:40:12 +0100
Subject: [master] 3d43f62 Add support for banning on http.status
Message-ID: <E1RhhBU-0004Yl-39@project.varnish-software.com>

commit 3d43f62b2e886ff115caa803eb45bccf72c806ce
Author: Tollef Fog Heen <tfheen at varnish-software.com>
Date:   Mon Jan 2 12:58:03 2012 +0100

    Add support for banning on http.status
    
    Fixes: #1076

diff --git a/bin/varnishd/cache/cache_ban.c b/bin/varnishd/cache/cache_ban.c
index a884983..451e504 100644
--- a/bin/varnishd/cache/cache_ban.c
+++ b/bin/varnishd/cache/cache_ban.c
@@ -113,9 +113,10 @@ static bgthread_t ban_lurker;
 #define	BAN_OPER_MATCH	0x12
 #define	BAN_OPER_NMATCH	0x13
 
-#define BAN_ARG_URL	0x18
-#define BAN_ARG_REQHTTP	0x19
-#define BAN_ARG_OBJHTTP	0x1a
+#define BAN_ARG_URL		0x18
+#define BAN_ARG_REQHTTP		0x19
+#define BAN_ARG_OBJHTTP		0x1a
+#define BAN_ARG_OBJSTATUS	0x1b
 
 /*--------------------------------------------------------------------
  * Variables we can purge on
@@ -609,6 +610,7 @@ ban_evaluate(const uint8_t *bs, const struct http *objhttp,
 	struct ban_test bt;
 	const uint8_t *be;
 	char *arg1;
+	char buf[10];
 
 	be = bs + ban_len(bs);
 	bs += 13;
@@ -628,6 +630,10 @@ ban_evaluate(const uint8_t *bs, const struct http *objhttp,
 		case BAN_ARG_OBJHTTP:
 			(void)http_GetHdr(objhttp, bt.arg1_spec, &arg1);
 			break;
+		case BAN_ARG_OBJSTATUS:
+			arg1 = buf;
+			sprintf(buf, "%d", objhttp->status);
+			break;
 		default:
 			INCOMPL();
 		}
diff --git a/include/tbl/ban_vars.h b/include/tbl/ban_vars.h
index 669b9b1..65b17e0 100644
--- a/include/tbl/ban_vars.h
+++ b/include/tbl/ban_vars.h
@@ -35,3 +35,4 @@
 PVAR("req.url",		PVAR_REQ,		BAN_ARG_URL)
 PVAR("req.http.",	PVAR_REQ|PVAR_HTTP,	BAN_ARG_REQHTTP)
 PVAR("obj.http.",	PVAR_HTTP,		BAN_ARG_OBJHTTP)
+PVAR("obj.status",	0,			BAN_ARG_OBJSTATUS)



From phk at varnish-cache.org  Tue Jan  3 18:11:03 2012
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Tue, 03 Jan 2012 19:11:03 +0100
Subject: [master] b6efb69 Remove outdated comment
Message-ID: <E1Ri8pD-0006f7-0C@project.varnish-software.com>

commit b6efb69ef5ecfd82d35adf85134d553381297325
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Jan 3 08:15:12 2012 +0000

    Remove outdated comment

diff --git a/configure.ac b/configure.ac
index 01eac28..72d2b7c 100644
--- a/configure.ac
+++ b/configure.ac
@@ -320,12 +320,6 @@ AC_CHECK_FUNCS([clock_gettime])
 AC_CHECK_FUNCS([gethrtime]) 
 LIBS="${save_LIBS}"
 
-# Check which mechanism to use for the acceptor.  We look for kqueue
-# only on platforms on which we know that it works, because there are
-# platforms where a simple AC_CHECK_FUNCS([kqueue]) would succeed but
-# the build would fail.  We also allow the user to disable mechanisms
-# he doesn't want to use.
-
 # --enable-kqueue
 AC_ARG_ENABLE(kqueue,
     AS_HELP_STRING([--enable-kqueue],



From phk at varnish-cache.org  Tue Jan  3 18:11:03 2012
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Tue, 03 Jan 2012 19:11:03 +0100
Subject: [master] 0a96b48 Add explanation for Varnish "hashing" in light of
	advisories.
Message-ID: <E1Ri8pD-0006fJ-23@project.varnish-software.com>

commit 0a96b4875c06f2320b43cd0833d35e33ba047dc7
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Jan 3 18:10:29 2012 +0000

    Add explanation for Varnish "hashing" in light of advisories.

diff --git a/doc/sphinx/phk/index.rst b/doc/sphinx/phk/index.rst
index 822434c..58935c7 100644
--- a/doc/sphinx/phk/index.rst
+++ b/doc/sphinx/phk/index.rst
@@ -8,6 +8,7 @@ You may or may not want to know what Poul-Henning think.
 
 .. toctree::
 
+	varnish_does_not_hash.rst
 	thetoolsweworkwith.rst
 	three-zero.rst
 	ssl.rst
diff --git a/doc/sphinx/phk/varnish_does_not_hash.rst b/doc/sphinx/phk/varnish_does_not_hash.rst
new file mode 100644
index 0000000..8393c87
--- /dev/null
+++ b/doc/sphinx/phk/varnish_does_not_hash.rst
@@ -0,0 +1,145 @@
+.. _phk_varnish_does_not_hash:
+
+=====================
+Varnish Does Not Hash
+=====================
+
+A spate of security advisories related to hash-collisions have made
+a lot of people stare at Varnish and wonder if it is affected.
+
+The answer is no, but the explanation is probably not what most of
+you expected:
+
+Varnish does not hash, at least not by default, and
+even if it does, it's still as immune to the attacks as can be.
+
+To understand what is going on, I have to introduce a concept from
+Shannons information theory: "entropy."
+
+Entropy is hard to explain, and according to legend, that is exactly
+why Shannon recycled that term from thermodynamics.
+
+In this context, we can get away with thinking about entropy as how
+much our "keys" differ::
+
+	Low entropy (1 bit):
+		/foo/bar/barf/some/cms/content/article?article=2
+		/foo/bar/barf/some/cms/content/article?article=3
+
+	High entropy (65 bits):
+		/i?ee30d0770eb460634e9d5dcfb562a2c5.html
+		/i?bca3633d52607f38a107cb5297fd66e5.html
+
+Hashing consists of calculating a hash-index from the key and
+storing the objects in an array indexed by that key.
+
+Typically, but not always, the key is a string and the index is a
+(smallish) integer, and the job of the hash-function is to squeeze
+the key into the integer, without loosing any of the entropy.
+
+Needless to say, the more entropy you have to begin with, the more
+of it you can afford to loose, and loose some you almost invariably
+will.
+
+There are two families of hash-functions, the fast ones, and the good
+ones, and the security advisories are about the fast ones.
+
+The good ones are slower, but probably not so much slower that you
+care, and therefore, if you want to fix your web-app:
+
+Change::
+	foo=somedict[$somekey]
+To::
+	foo=somedict[md5($somekey)]
+
+and forget about the advisories.
+
+Yes, that's right: Cryptographic hash algorithms are the good ones,
+they are built to not throw any entropy away, and they are built to
+have very hard to predict collisions, which is exactly the problem
+with the fast hash-functions in the advisories.
+
+-----------------
+What Varnish Does
+-----------------
+
+The way to avoid having hash-collisions is to not use a hash:  Use a
+tree instead, there every object has its own place and there are no
+collisions.
+
+Varnish does that, but with a twist.
+
+The "keys" in varnish can be very long, by default they consist of::
+
+	sub vcl_hash {
+	    hash_data(req.url);
+	    if (req.http.host) {
+		hash_data(req.http.host);
+	    } else {
+		hash_data(server.ip);
+	    }
+	    return (hash);
+	}
+
+But some users will add cookies, user identification and many other
+bits and pieces of string in there, and in the end the keys can be
+kilobytes in length, and quite often, as in the first example above,
+the first difference may not come until pretty far into the keys.
+
+Trees generally need to have a copy of the key around to be able
+to tell if they have a match, and more importantly to compare
+tree-leaves in order to "re-balance" the tree and other such arcanae
+of data structures.
+
+This would add another per-object memory load to Varnish, and it
+would feel particularly silly to store 48 identical characters for
+each object in the far too common case seen above.
+
+But furthermore, we want the tree to be very fast to do lookups in,
+preferably it should be lockless for lookups, and that means that
+we cannot (realistically) use any of the "smart" trees which
+automatically balance themselves etc.
+
+You (generally) don't need a "smart" tree if your keys look
+like random data in the order they arrive, but we can pretty
+much expect the opposite as article number 4, 5, 6 etc are added
+to the CMS in the first example.
+
+But we can make the keys look random, and make them small and fixed
+size at the same time, and the perfect functions designed for just
+that task are the "good" hash-functions, the cryptographic ones.
+
+So what Varnish does is "key-compression":  All the strings hash_data()
+are fed, are pushed through a cryptographic hash algorithm called
+SHA256, which, as the name says, always spits out 256 bits (= 32
+bytes), no matter how many bits you feed it.
+
+This does not eliminate the key-storage requirement, but now all
+the keys are 32 bytes and can be put directly into the data structure::
+
+	struct objhead {
+		[...]
+		unsigned char           digest[DIGEST_LEN];
+	};
+
+In the example above, the output of SHA256 for the 1 bit difference
+in entropy becomes::
+
+	/foo/bar/barf/some/cms/content/article?article=2
+	-> 14f0553caa5c796650ec82256e3f111ae2f20020a4b9029f135a01610932054e
+	/foo/bar/barf/some/cms/content/article?article=3
+	-> 4d45b9544077921575c3c5a2a14c779bff6c4830d1fbafe4bd7e03e5dd93ca05
+
+That should be random enough.
+
+But the key-compression does introduce a risk of collisions, since
+not even SHA256 can guarantee different outputs for all possible
+inputs:  Try pushing all the possible 33 bytes long files through
+SHA256 and sooner or later you will get collisions.
+
+The risk of collision is very small however, and I can all but
+promise you, that you will be fully offset in fame and money for
+any inconvenience a collision might cause, because you will
+be the first person to find a SHA256 collision.
+
+Poul-Henning, 2012-01-03



From phk at varnish-cache.org  Tue Jan  3 18:11:03 2012
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Tue, 03 Jan 2012 19:11:03 +0100
Subject: [master] 1fbff54 Merge branch 'master' of
	ssh://git.varnish-cache.org/git/varnish-cache
Message-ID: <E1Ri8pD-0006fW-3Y@project.varnish-software.com>

commit 1fbff541f44cda7b9c3bd07739372099bc8df9e5
Merge: 0a96b48 b70a93c
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Jan 3 18:11:00 2012 +0000

    Merge branch 'master' of ssh://git.varnish-cache.org/git/varnish-cache



From tfheen at varnish-cache.org  Mon Jan  9 13:00:49 2012
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Mon, 09 Jan 2012 14:00:49 +0100
Subject: [master] 84f098e Get rid of unused variable,
	fixes compilation problem with newer gcc
Message-ID: <E1RkEqH-0004S4-IT@project.varnish-software.com>

commit 84f098e0b70a7aca500fd8969929438c821bd392
Author: Tollef Fog Heen <tfheen at varnish-software.com>
Date:   Mon Jan 9 14:00:30 2012 +0100

    Get rid of unused variable, fixes compilation problem with newer gcc

diff --git a/bin/varnishstat/varnishstat_curses.c b/bin/varnishstat/varnishstat_curses.c
index 3a672e1..5997096 100644
--- a/bin/varnishstat/varnishstat_curses.c
+++ b/bin/varnishstat/varnishstat_curses.c
@@ -145,7 +145,6 @@ do_curses(struct VSM_data *vd, const struct VSC_C_main *VSC_C_main,
 	time_t rt;
 	int ch, line;
 	struct pt *pt;
-	double act, lact;
 
 	(void)initscr();
 	AC(raw());
@@ -167,17 +166,12 @@ do_curses(struct VSM_data *vd, const struct VSC_C_main *VSC_C_main,
 		lt = 0;
 		lhit = 0;
 		lmiss = 0;
-		lact = 0;
 
 		while (1) {
 			/*
 			 * Break to outher loop if we need to re-read file.
 			 * Only check if it looks like nothing is happening.
 			 */
-			act = VSC_C_main->cache_hit +
-			    VSC_C_main->cache_miss + 1;
-			lact = act;
-
 			AZ(gettimeofday(&tv, NULL));
 			tt = tv.tv_usec * 1e-6 + tv.tv_sec;
 			lt = tt - lt;



From phk at varnish-cache.org  Mon Jan  9 15:22:31 2012
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Mon, 09 Jan 2012 16:22:31 +0100
Subject: [master] 6bec2c8 Fix a buglet in handling non-ascii strings in VCL,
	now that we (I) have decided what the strategy is going to be for
	synthetic in the future.
Message-ID: <E1RkH3P-0005b0-Vu@project.varnish-software.com>

commit 6bec2c8361f4bb87e84e7760d0970f3d96e9d07f
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Jan 9 15:21:35 2012 +0000

    Fix a buglet in handling non-ascii strings in VCL, now that we (I)
    have decided what the strategy is going to be for synthetic in
    the future.
    
    Fixes	#545

diff --git a/bin/varnishtest/tests/r00545.vtc b/bin/varnishtest/tests/r00545.vtc
new file mode 100644
index 0000000..0888af1
--- /dev/null
+++ b/bin/varnishtest/tests/r00545.vtc
@@ -0,0 +1,18 @@
+varnishtest "High-bit chars"
+
+server s1 {
+	rxreq
+	txresp
+} -start
+
+varnish v1 -vcl+backend {
+	sub vcl_deliver {
+		set resp.http.foo = "???";
+	}
+} -start
+
+client c1 {
+	txreq
+	rxresp
+	expect resp.http.foo == "???"
+} -run
diff --git a/lib/libvcl/vcc_compile.c b/lib/libvcl/vcc_compile.c
index fddd019..27447fe 100644
--- a/lib/libvcl/vcc_compile.c
+++ b/lib/libvcl/vcc_compile.c
@@ -229,7 +229,7 @@ EncString(struct vsb *sb, const char *b, const char *e, int mode)
 			if (isgraph(*b))
 				VSB_printf(sb, "%c", *b);
 			else
-				VSB_printf(sb, "\\%03o", *b);
+				VSB_printf(sb, "\\%03o", (uint8_t)*b);
 			break;
 		}
 	}
@@ -241,7 +241,7 @@ EncToken(struct vsb *sb, const struct token *t)
 {
 
 	assert(t->tok == CSTR);
-	EncString(sb, t->dec, NULL, 0);
+	EncString(sb, t->dec, NULL, 1);
 }
 
 /*--------------------------------------------------------------------



From geoff at varnish-cache.org  Mon Jan  9 20:51:44 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:51:44 +0100
Subject: [experimental-ims] 6f7944c Document various director retry-mechanisms
Message-ID: <E1RkMC0-0006h2-Fr@project.varnish-software.com>

commit 6f7944c06c57cf40d0c1e149e15e18d639fbbbd6
Author: Kristian Lyngstol <kristian at bohemians.org>
Date:   Wed Aug 31 15:30:08 2011 +0200

    Document various director retry-mechanisms

diff --git a/doc/sphinx/reference/vcl.rst b/doc/sphinx/reference/vcl.rst
index 4a211a1..4f005d0 100644
--- a/doc/sphinx/reference/vcl.rst
+++ b/doc/sphinx/reference/vcl.rst
@@ -148,58 +148,69 @@ Configuring a director may look like this:::
     }
   } 
 
-The random director
-~~~~~~~~~~~~~~~~~~~
-
-The random director takes one per director option .retries.  This
-specifies how many tries it will use to find a working backend.  The
-default is the same as the number of backends defined for the
-director.
-
-There is also a per-backend option: weight which defines the portion
-of traffic to send to the particular backend.
-
-The round-robin director
-~~~~~~~~~~~~~~~~~~~~~~~~
+The family of random directors
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+There are three directors that share the same logic, called the random
+director, client director and hash director. They each distribute traffic
+among the backends assigned to it using a random distribution seeded with
+either the client identity, a random number or the cache hash (typically
+url). Beyond the initial seed, they act the same.
+
+Each backend requires a .weight option which sets the amount of traffic
+each backend will get compared to the others. Equal weight means equal
+traffic. A backend with lower weight than an other will get proportionally
+less traffic.
+
+The director has an optional .retries option which defaults to the number
+of backends the director has. The director will attempt .retries times to
+find a healthy backend if the first attempt fails. Each attempt re-uses the
+previous seed in an iterative manner. For the random director this detail
+is of no importance as it will give different results each time. For the
+hash and client director, this means the same URL or the same client will
+fail to the same server consistently.
 
-The round-robin director does not take any options.
+The random director
+...................
 
+This uses a random number to seed the backend selection.
 
 The client director
-~~~~~~~~~~~~~~~~~~~
+...................
 
 The client director picks a backend based on the clients
 *identity*. You can set the VCL variable *client.identity* to identify
 the client by picking up the value of a session cookie or similar.
 
-Note: from 2.1.0 to 2.1.3 *client.identity* isn't available and the
-director will use automatically set the idenity based on client.ip In
-2.1.4 and onwards you can set client.identity to any string available.
-
-The client director takes one option - *retries* which set the number
-of retries the director should take in order to find a healthy
-backend.
-
-
-
 The hash director
-~~~~~~~~~~~~~~~~~
+.................
 
 The hash director will pick a backend based on the URL hash
-value. 
+value.
 
 This is useful is you are using Varnish to load balance in front of
 other Varnish caches or other web accelerators as objects won't be
 duplicated across caches.
 
-The client director takes one option - *retries* which set the number
-of retries the director should take in order to find a healthy
-backend.
+It will use the value of req.hash, just as the normal cache-lookup methods.
+
+
+The round-robin director
+~~~~~~~~~~~~~~~~~~~~~~~~
+
+The round-robin director does not take any options.
+
+It will use the first backend for the first request, the second backend for
+the second request and so on, and start from the top again when it gets to
+the end.
+
+If a backend is unhealthy or Varnish fails to connect, it will be skipped.
+The round-robin director will try all the backends once before giving up.
 
 The DNS director
 ~~~~~~~~~~~~~~~~
 
-The DNS director can use backends in three different ways. Either like the
+The DNS director can use backends in two different ways. Either like the
 random or round-robin director or using .list::
 
   director directorname dns {
@@ -216,6 +227,9 @@ random or round-robin director or using .list::
 
 This will specify 384 backends, all using port 80 and a connection timeout
 of 0.4s. Options must come before the list of IPs in the .list statement.
+The .list-method does not support IPv6. It is not a white-list, it is an
+actual list of backends that will be created internally in Varnish - the
+larger subnet the more overhead.
 
 The .ttl defines the cache duration of the DNS lookups.
 
@@ -223,6 +237,12 @@ The above example will append "internal.example.net" to the incoming Host
 header supplied by the client, before looking it up. All settings are
 optional.
 
+Health checks are not thoroughly supported.
+
+DNS round robin balancing is supported. If a hostname resolves to multiple
+backends, the director will divide the traffic between all of them in a
+round-robin manner.
+
 The fallback director
 ~~~~~~~~~~~~~~~~~~~~~
 



From geoff at varnish-cache.org  Mon Jan  9 20:51:44 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:51:44 +0100
Subject: [experimental-ims] 863c22b Allow relational comparisons on REAL type.
Message-ID: <E1RkMC0-0006h8-HN@project.varnish-software.com>

commit 863c22b821fae10287e110adb4aa6ba2f86494f6
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Aug 31 15:17:43 2011 +0000

    Allow relational comparisons on REAL type.
    
    Fixes	#1002

diff --git a/bin/varnishtest/tests/r01002.vtc b/bin/varnishtest/tests/r01002.vtc
new file mode 100644
index 0000000..12eb636
--- /dev/null
+++ b/bin/varnishtest/tests/r01002.vtc
@@ -0,0 +1,12 @@
+varnishtest "Real relational comparisons"
+
+varnish v1 -vcl {
+	import std from "${topbuild}/lib/libvmod_std/.libs/libvmod_std.so" ;
+
+	backend foo { .host = "${bad_ip}"; }
+	sub vcl_recv {
+		if (std.random(0,5) < 1.0) {
+			return (pipe);
+		}
+	}
+}
diff --git a/lib/libvcl/vcc_expr.c b/lib/libvcl/vcc_expr.c
index 0426299..3883dd2 100644
--- a/lib/libvcl/vcc_expr.c
+++ b/lib/libvcl/vcc_expr.c
@@ -886,6 +886,7 @@ static const struct cmps {
 	NUM_REL(INT),
 	NUM_REL(DURATION),
 	NUM_REL(BYTES),
+	NUM_REL(REAL),
 
 	{STRING,	T_EQ,	"!VRT_strcmp(\v1, \v2)" },
 	{STRING,	T_NEQ,	"VRT_strcmp(\v1, \v2)" },



From geoff at varnish-cache.org  Mon Jan  9 20:51:44 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:51:44 +0100
Subject: [experimental-ims] 16d58af Merge branch 'master' of
	ssh://git.varnish-cache.org/git/varnish-cache
Message-ID: <E1RkMC0-0006hE-IW@project.varnish-software.com>

commit 16d58afeeb41da5203c7793cdd09798364018217
Merge: 863c22b 6f7944c
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Aug 31 15:18:03 2011 +0000

    Merge branch 'master' of ssh://git.varnish-cache.org/git/varnish-cache



From geoff at varnish-cache.org  Mon Jan  9 20:51:44 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:51:44 +0100
Subject: [experimental-ims] c0e2acc When headers are surplus to limits,
	only log the first 20 char.
Message-ID: <E1RkMC0-0006hO-K7@project.varnish-software.com>

commit c0e2accfb3ce6bc46696ad22ad78b886d064be96
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Aug 31 15:27:41 2011 +0000

    When headers are surplus to limits, only log the first 20 char.

diff --git a/bin/varnishd/cache_http.c b/bin/varnishd/cache_http.c
index 7fe0a8a..f77ab74 100644
--- a/bin/varnishd/cache_http.c
+++ b/bin/varnishd/cache_http.c
@@ -522,7 +522,8 @@ http_dissect_hdrs(struct worker *w, struct http *hp, int fd, char *p,
 
 		if (q - p > htc->maxhdr) {
 			VSC_C_main->losthdr++;
-			WSL(w, SLT_LostHeader, fd, "%.*s", q - p, p);
+			WSL(w, SLT_LostHeader, fd, "%.*s",
+			    q - p > 20 ? 20 : q - p, p);
 			return (413);
 		}
 
@@ -547,7 +548,8 @@ http_dissect_hdrs(struct worker *w, struct http *hp, int fd, char *p,
 			hp->nhd++;
 		} else {
 			VSC_C_main->losthdr++;
-			WSL(w, SLT_LostHeader, fd, "%.*s", q - p, p);
+			WSL(w, SLT_LostHeader, fd, "%.*s",
+			    q - p > 20 ? 20 : q - p, p);
 			return (413);
 		}
 	}



From geoff at varnish-cache.org  Mon Jan  9 20:51:44 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:51:44 +0100
Subject: [experimental-ims] 0d079cc Reset the "built vary spec" (also) if we
	came back from the waiting list, otherwise it might turn into garbage.
Message-ID: <E1RkMC0-0006hb-Mb@project.varnish-software.com>

commit 0d079cc0576fa5bb9c84f34131c61ebb51dcc89c
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu Sep 1 07:55:46 2011 +0000

    Reset the "built vary spec" (also) if we came back from the waiting
    list, otherwise it might turn into garbage.
    
    Fixes	#994
    Fixes	#1001

diff --git a/bin/varnishd/cache_center.c b/bin/varnishd/cache_center.c
index df9941c..0f69861 100644
--- a/bin/varnishd/cache_center.c
+++ b/bin/varnishd/cache_center.c
@@ -1080,10 +1080,12 @@ cnt_lookup(struct sess *sp)
 		AZ(sp->vary_l);
 		AZ(sp->vary_e);
 		(void)WS_Reserve(sp->ws, 0);
-		sp->vary_b = (void*)sp->ws->f;
-		sp->vary_e = (void*)sp->ws->r;
-		sp->vary_b[2] = '\0';
+	} else {
+		AN(sp->ws->r);
 	}
+	sp->vary_b = (void*)sp->ws->f;
+	sp->vary_e = (void*)sp->ws->r;
+	sp->vary_b[2] = '\0';
 
 	oc = HSH_Lookup(sp, &oh);
 



From geoff at varnish-cache.org  Mon Jan  9 20:51:44 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:51:44 +0100
Subject: [experimental-ims] 69e5bd5 Also remove DIAGNISTICS around the
	prototype for MCF_DumpRst()
Message-ID: <E1RkMC0-0006hk-OW@project.varnish-software.com>

commit 69e5bd536a561f56cf52926328716c000e35a5d3
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu Sep 1 08:37:08 2011 +0000

    Also remove DIAGNISTICS around the prototype for MCF_DumpRst()

diff --git a/bin/varnishd/mgt.h b/bin/varnishd/mgt.h
index 30b90ba..485de26 100644
--- a/bin/varnishd/mgt.h
+++ b/bin/varnishd/mgt.h
@@ -66,9 +66,7 @@ void mgt_cli_close_all(void);
 void MCF_ParamSync(void);
 void MCF_ParamInit(struct cli *);
 void MCF_ParamSet(struct cli *, const char *param, const char *val);
-#ifdef DIAGNOSTICS
 void MCF_DumpRst(void);
-#endif
 
 /* mgt_sandbox.c */
 void mgt_sandbox(void);



From geoff at varnish-cache.org  Mon Jan  9 20:51:44 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:51:44 +0100
Subject: [experimental-ims] d600ab8 Move acct_tmp to worker instead of session
	and report # gunziped bytes rather than storage size when we
	gunzip for delivery.
Message-ID: <E1RkMC0-0006hx-RI@project.varnish-software.com>

commit d600ab8745c72f08272250cdb86dd4254f3feacd
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu Sep 1 08:38:33 2011 +0000

    Move acct_tmp to worker instead of session and report # gunziped
    bytes rather than storage size when we gunzip for delivery.
    
    Fixes	#992

diff --git a/bin/varnishd/cache.h b/bin/varnishd/cache.h
index 0509d99..43a0c41 100644
--- a/bin/varnishd/cache.h
+++ b/bin/varnishd/cache.h
@@ -353,6 +353,8 @@ struct worker {
 #define RES_ESI_CHILD		(1<<5)
 #define RES_GUNZIP		(1<<6)
 
+	/* Temporary accounting */
+	struct acct		acct_tmp;
 };
 
 /* Work Request for worker thread ------------------------------------*/
@@ -601,7 +603,6 @@ struct sess {
 	struct sessmem		*mem;
 
 	struct workreq		workreq;
-	struct acct		acct_tmp;
 	struct acct		acct_req;
 	struct acct		acct_ses;
 
diff --git a/bin/varnishd/cache_center.c b/bin/varnishd/cache_center.c
index 0f69861..a4e5051 100644
--- a/bin/varnishd/cache_center.c
+++ b/bin/varnishd/cache_center.c
@@ -869,7 +869,7 @@ cnt_fetchbody(struct sess *sp)
 		AN(sp->obj->objcore->ban);
 		HSH_Unbusy(sp);
 	}
-	sp->acct_tmp.fetch++;
+	sp->wrk->acct_tmp.fetch++;
 	sp->step = STP_PREPRESP;
 	return (0);
 }
@@ -926,7 +926,7 @@ cnt_streambody(struct sess *sp)
 	} else {
 		sp->doclose = "Stream error";
 	}
-	sp->acct_tmp.fetch++;
+	sp->wrk->acct_tmp.fetch++;
 	sp->director = NULL;
 	sp->restarts = 0;
 
@@ -968,7 +968,7 @@ cnt_first(struct sess *sp)
 	HTC_Init(sp->htc, sp->ws, sp->fd, params->http_req_size,
 	    params->http_req_hdr_len);
 	sp->wrk->lastused = sp->t_open;
-	sp->acct_tmp.sess++;
+	sp->wrk->acct_tmp.sess++;
 
 	sp->step = STP_WAIT;
 	return (0);
@@ -1272,7 +1272,7 @@ cnt_pass(struct sess *sp)
 		return (0);
 	}
 	assert(sp->handling == VCL_RET_PASS);
-	sp->acct_tmp.pass++;
+	sp->wrk->acct_tmp.pass++;
 	sp->sendbody = 1;
 	sp->step = STP_FETCH;
 	return (0);
@@ -1310,7 +1310,7 @@ cnt_pipe(struct sess *sp)
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 	CHECK_OBJ_NOTNULL(sp->vcl, VCL_CONF_MAGIC);
 
-	sp->acct_tmp.pipe++;
+	sp->wrk->acct_tmp.pipe++;
 	WS_Reset(sp->wrk->ws, NULL);
 	http_Setup(sp->wrk->bereq, sp->wrk->ws);
 	http_FilterHeader(sp, HTTPH_R_PIPE);
@@ -1459,7 +1459,7 @@ cnt_start(struct sess *sp)
 	sp->wrk->stats.client_req++;
 	sp->t_req = TIM_real();
 	sp->wrk->lastused = sp->t_req;
-	sp->acct_tmp.req++;
+	sp->wrk->acct_tmp.req++;
 
 	/* Assign XID and log */
 	sp->xid = ++xids;				/* XXX not locked */
@@ -1614,6 +1614,9 @@ CNT_Session(struct sess *sp)
 	AZ(w->is_gunzip);
 	AZ(w->do_gunzip);
 	AZ(w->do_esi);
+#define ACCT(foo)	AZ(w->acct_tmp.foo);
+#include "acct_fields.h"
+#undef ACCT
 	assert(WRW_IsReleased(w));
 }
 
diff --git a/bin/varnishd/cache_gzip.c b/bin/varnishd/cache_gzip.c
index 40a8bdf..bed3104 100644
--- a/bin/varnishd/cache_gzip.c
+++ b/bin/varnishd/cache_gzip.c
@@ -378,6 +378,7 @@ VGZ_WrwGunzip(const struct sess *sp, struct vgz *vg, const void *ibuf,
 			return (-1);
 		}
 		if (obufl == *obufp || i == VGZ_STUCK) {
+			sp->wrk->acct_tmp.bodybytes += *obufp;
 			(void)WRW_Write(sp->wrk, obuf, *obufp);
 			(void)WRW_Flush(sp->wrk);
 			*obufp = 0;
diff --git a/bin/varnishd/cache_response.c b/bin/varnishd/cache_response.c
index 17d1525..c592909 100644
--- a/bin/varnishd/cache_response.c
+++ b/bin/varnishd/cache_response.c
@@ -175,7 +175,6 @@ res_WriteGunzipObj(struct sess *sp)
 		CHECK_OBJ_NOTNULL(st, STORAGE_MAGIC);
 		u += st->len;
 
-		sp->acct_tmp.bodybytes += st->len;	/* XXX ? */
 		VSC_C_main->n_objwrite++;
 
 		i = VGZ_WrwGunzip(sp, vg,
@@ -195,7 +194,7 @@ res_WriteGunzipObj(struct sess *sp)
 /*--------------------------------------------------------------------*/
 
 static void
-res_WriteDirObj(struct sess *sp, ssize_t low, ssize_t high)
+res_WriteDirObj(const struct sess *sp, ssize_t low, ssize_t high)
 {
 	ssize_t u = 0;
 	size_t ptr, off, len;
@@ -227,7 +226,7 @@ res_WriteDirObj(struct sess *sp, ssize_t low, ssize_t high)
 
 		ptr += len;
 
-		sp->acct_tmp.bodybytes += len;
+		sp->wrk->acct_tmp.bodybytes += len;
 #ifdef SENDFILE_WORKS
 		/*
 		 * XXX: the overhead of setting up sendfile is not
@@ -292,7 +291,7 @@ RES_WriteObj(struct sess *sp)
 	 * Send HTTP protocol header, unless interior ESI object
 	 */
 	if (!(sp->wrk->res_mode & RES_ESI_CHILD))
-		sp->acct_tmp.hdrbytes +=
+		sp->wrk->acct_tmp.hdrbytes +=
 		    http_Write(sp->wrk, sp->wrk->resp, 1);
 
 	if (!sp->wantbody)
@@ -348,7 +347,7 @@ RES_StreamStart(struct sess *sp)
 		http_PrintfHeader(sp->wrk, sp->fd, sp->wrk->resp,
 		    "Content-Length: %s", sp->wrk->h_content_length);
 
-	sp->acct_tmp.hdrbytes +=
+	sp->wrk->acct_tmp.hdrbytes +=
 	    http_Write(sp->wrk, sp->wrk->resp, 1);
 
 	if (sp->wrk->res_mode & RES_CHUNKED)
diff --git a/bin/varnishd/cache_session.c b/bin/varnishd/cache_session.c
index af422a6..1ebca44 100644
--- a/bin/varnishd/cache_session.c
+++ b/bin/varnishd/cache_session.c
@@ -80,7 +80,7 @@ static struct lock		stat_mtx;
 void
 SES_Charge(struct sess *sp)
 {
-	struct acct *a = &sp->acct_tmp;
+	struct acct *a = &sp->wrk->acct_tmp;
 
 #define ACCT(foo)				\
 	sp->wrk->stats.s_##foo += a->foo;	\



From geoff at varnish-cache.org  Mon Jan  9 20:51:44 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:51:44 +0100
Subject: [experimental-ims] c14ced8 Take another stab at #1001/#994
Message-ID: <E1RkMC0-0006iE-UI@project.varnish-software.com>

commit c14ced83338ed506318c98d167b28787d24669d8
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu Sep 1 09:59:22 2011 +0000

    Take another stab at #1001/#994

diff --git a/bin/varnishd/cache.h b/bin/varnishd/cache.h
index 43a0c41..f9989fb 100644
--- a/bin/varnishd/cache.h
+++ b/bin/varnishd/cache.h
@@ -899,6 +899,7 @@ void RES_StreamPoll(const struct sess *sp);
 /* cache_vary.c */
 struct vsb *VRY_Create(const struct sess *sp, const struct http *hp);
 int VRY_Match(struct sess *sp, const uint8_t *vary);
+void VRY_Validate(const uint8_t *vary);
 
 /* cache_vcl.c */
 void VCL_Init(void);
diff --git a/bin/varnishd/cache_center.c b/bin/varnishd/cache_center.c
index a4e5051..6bb8324 100644
--- a/bin/varnishd/cache_center.c
+++ b/bin/varnishd/cache_center.c
@@ -802,6 +802,7 @@ cnt_fetchbody(struct sess *sp)
 		    (void *)WS_Alloc(sp->obj->http->ws, varyl);
 		AN(sp->obj->vary);
 		memcpy(sp->obj->vary, VSB_data(vary), varyl);
+		VRY_Validate(sp->obj->vary);
 		VSB_delete(vary);
 	}
 
@@ -1107,10 +1108,14 @@ cnt_lookup(struct sess *sp)
 	if (oc->flags & OC_F_BUSY) {
 		sp->wrk->stats.cache_miss++;
 
-		if (sp->vary_l != NULL)
+		if (sp->vary_l != NULL) {
+			assert(oc->busyobj->vary == sp->vary_b);
+			VRY_Validate(oc->busyobj->vary);
 			WS_ReleaseP(sp->ws, (void*)sp->vary_l);
-		else
-			WS_Release(sp->ws, 0);
+		} else {
+			AZ(oc->busyobj->vary);
+			WS_Release(sp->ws, 0); 
+		}
 		sp->vary_b = NULL;
 		sp->vary_l = NULL;
 		sp->vary_e = NULL;
diff --git a/bin/varnishd/cache_hash.c b/bin/varnishd/cache_hash.c
index b070b38..328964a 100644
--- a/bin/varnishd/cache_hash.c
+++ b/bin/varnishd/cache_hash.c
@@ -461,7 +461,12 @@ HSH_Lookup(struct sess *sp, struct objhead **poh)
 	AN(oc->flags & OC_F_BUSY);
 	oc->refcnt = 1;
 
-	w->nbusyobj->vary = sp->vary_b;
+	/* XXX: clear w->nbusyobj before use */
+	VRY_Validate(sp->vary_b);
+	if (sp->vary_l != NULL)
+		w->nbusyobj->vary = sp->vary_b;
+	else
+		w->nbusyobj->vary = NULL;
 	oc->busyobj = w->nbusyobj;
 	w->nbusyobj = NULL;
 
diff --git a/bin/varnishd/cache_vary.c b/bin/varnishd/cache_vary.c
index 73d6354..b7a6b01 100644
--- a/bin/varnishd/cache_vary.c
+++ b/bin/varnishd/cache_vary.c
@@ -247,3 +247,13 @@ VRY_Match(struct sess *sp, const uint8_t *vary)
 		sp->vary_l = vsp + 3;
 	return (retval);
 }
+
+void
+VRY_Validate(const uint8_t *vary)
+{
+
+	while (vary[2] != 0) {
+		assert(strlen((const char*)vary+3) == vary[2]);
+		vary += vry_len(vary);
+	}
+}



From geoff at varnish-cache.org  Mon Jan  9 20:51:45 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:51:45 +0100
Subject: [experimental-ims] 1bddcfa Document hit_for_pass in the tutorial
Message-ID: <E1RkMC1-0006iP-0g@project.varnish-software.com>

commit 1bddcfa742f5a48808429681b5df21520fadd0b4
Author: Kristian Lyngstol <kristian at bohemians.org>
Date:   Thu Sep 1 15:20:36 2011 +0200

    Document hit_for_pass in the tutorial

diff --git a/doc/sphinx/tutorial/vcl.rst b/doc/sphinx/tutorial/vcl.rst
index f52c1ed..3ed78c5 100644
--- a/doc/sphinx/tutorial/vcl.rst
+++ b/doc/sphinx/tutorial/vcl.rst
@@ -58,7 +58,15 @@ The most common actions to return are these:
 *pass*
  When you return pass the request and subsequent response will be passed to
  and from the backend server. It won't be cached. pass can be returned from
- both vcl_recv and vcl_fetch.
+ vcl_recv
+
+*hit_for_pass*
+  Similar to pass, but accessible from vcl_fetch. Unlike pass, hit_for_pass
+  will create a hitforpass object in the cache. This has the side-effect of
+  caching the decision not to cache. This is to allow would-be uncachable
+  requests to be passed to the backend at the same time. The same logic is
+  not necessary in vcl_recv because this happens before any potential
+  queueing for an object takes place.
 
 *lookup*
   When you return lookup from vcl_recv you tell Varnish to deliver content 



From geoff at varnish-cache.org  Mon Jan  9 20:51:45 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:51:45 +0100
Subject: [experimental-ims] 2c28590 more on hit_for_pass
Message-ID: <E1RkMC1-0006ia-2u@project.varnish-software.com>

commit 2c28590561557aa2cb181296e0b63255e81fb24b
Author: Per Buer <perbu at varnish-software.com>
Date:   Thu Sep 1 15:41:39 2011 +0200

    more on hit_for_pass

diff --git a/doc/sphinx/reference/vcl.rst b/doc/sphinx/reference/vcl.rst
index 4f005d0..14d0450 100644
--- a/doc/sphinx/reference/vcl.rst
+++ b/doc/sphinx/reference/vcl.rst
@@ -532,12 +532,18 @@ vcl_fetch
   error code [reason]
     Return the specified error code to the client and abandon the request.
 
-  hit_for_pass
-    Pass in fetch. This will create a hit_for_pass object. Note that
-    the TTL for the hit_for_pass object will be set to what the
-    current value of beresp.ttl. Control will be handled to
-    vcl_deliver on the current request, but subsequent requests will
-    go directly to vcl_pass based on the hit_for_pass object.
+  hit_for_pass 
+    Pass in fetch. Passes the object without caching it. This will
+    create a socalled hit_for_pass object which has the side effect
+    that the decision not to cache will be cached. This is to allow
+    would-be uncachable requests to be passed to the backend at the
+    same time. The same logic is not necessary in vcl_recv because
+    this happens before any potential queueing for an object takes
+    place.  Note that the TTL for the hit_for_pass object will be set
+    to what the current value of beresp.ttl is. Control will be
+    handled to vcl_deliver on the current request, but subsequent
+    requests will go directly to vcl_pass based on the hit_for_pass
+    object.
 
   restart
     Restart the transaction. Increases the restart counter. If the number 



From geoff at varnish-cache.org  Mon Jan  9 20:51:45 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:51:45 +0100
Subject: [experimental-ims] cb07c41 A bit of spit and polish inspired by
	DocWilcos varnishtest patch.
Message-ID: <E1RkMC1-0006il-4P@project.varnish-software.com>

commit cb07c41dfa1a32ddefaece2850386618820e19a6
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu Sep 1 15:19:33 2011 +0000

    A bit of spit and polish inspired by DocWilcos varnishtest patch.

diff --git a/bin/varnishlog/flint.lnt b/bin/varnishlog/flint.lnt
index 149eec1..3a0395c 100644
--- a/bin/varnishlog/flint.lnt
+++ b/bin/varnishlog/flint.lnt
@@ -1,4 +1,5 @@
-
+-efile(451, "../../include/vsc_all.h")
+-e835   // A zero has been given as ___ argument to operator '___'  (<<)
 -e712	// 14      Info    712     Loss of precision (___) (___ to ___)
 -e747	// 16      Info    747     Significant prototype coercion (___) ___ to ___
 
@@ -17,3 +18,4 @@
 -e788	// enum constant '___' not used within defaulted switch
 -e641	// Converting enum '___' to '___'
 
+-esym(785,VSL_tags);	// Sparse array
diff --git a/bin/varnishlog/flint.sh b/bin/varnishlog/flint.sh
index a94710d..dc9e9ba 100755
--- a/bin/varnishlog/flint.sh
+++ b/bin/varnishlog/flint.sh
@@ -1,11 +1,28 @@
 #!/bin/sh
 
+if [ "x$1" = "x-ok" -a -f _.fl ] ; then
+	echo "Saved as reference"
+	mv _.fl _.fl.old
+	exit 0
+fi
+
 flexelint \
-	-I/usr/include \
+	../flint.lnt \
+	flint.lnt \
 	-I. \
 	-I../../include \
 	-I../.. \
-	../flint.lnt \
-	flint.lnt \
+	-I/usr/local/include \
+	-DVARNISH_STATE_DIR=\"foo\" \
 	*.c \
-	../../lib/libvarnishapi/*.c
+	../../lib/libvarnishapi/*.c \
+	2>&1 | tee _.fl
+
+if [ -f _.fl.old ] ; then
+	diff -u _.fl.old _.fl
+fi
+
+if [ "x$1" = "x-ok" ] ; then
+	echo "Saved as reference"
+	mv _.fl _.fl.old
+fi
diff --git a/bin/varnishlog/varnishlog.c b/bin/varnishlog/varnishlog.c
index ed29865..c7749cb 100644
--- a/bin/varnishlog/varnishlog.c
+++ b/bin/varnishlog/varnishlog.c
@@ -46,7 +46,6 @@
 
 #include "libvarnish.h"
 #include "vsl.h"
-#include "vre.h"
 #include "varnishapi.h"
 
 static int	b_flag, c_flag;
@@ -60,7 +59,7 @@ static uint64_t       bitmap[65536];
 #define F_INVCL		(1 << 0)
 
 static void
-h_order_finish(int fd, struct VSM_data *vd)
+h_order_finish(int fd, const struct VSM_data *vd)
 {
 
 	AZ(VSB_finish(ob[fd]));
@@ -72,7 +71,7 @@ h_order_finish(int fd, struct VSM_data *vd)
 }
 
 static void
-clean_order(struct VSM_data *vd)
+clean_order(const struct VSM_data *vd)
 {
 	unsigned u;
 
@@ -321,7 +320,8 @@ main(int argc, char * const *argv)
 			w_arg = optarg;
 			break;
 		case 'm':
-			m_flag = 1; /* fall through */
+			m_flag = 1;
+			/* FALLTHROUGH */
 		default:
 			if (VSL_Arg(vd, c, optarg) > 0)
 				break;
diff --git a/include/varnishapi.h b/include/varnishapi.h
index 0e97755..dcdd74f 100644
--- a/include/varnishapi.h
+++ b/include/varnishapi.h
@@ -109,7 +109,7 @@ int VSM_ReOpen(struct VSM_data *vd, int diag);
 	 *	-1 failure to reopen.
 	 */
 
-unsigned VSM_Seq(struct VSM_data *vd);
+unsigned VSM_Seq(const struct VSM_data *vd);
 	/*
 	 * Return the allocation sequence number
 	 */
@@ -260,6 +260,7 @@ void VSL_NonBlocking(const struct VSM_data *vd, int nb);
 int VSL_Dispatch(struct VSM_data *vd, VSL_handler_f *func, void *priv);
 int VSL_NextLog(const struct VSM_data *lh, uint32_t **pp, uint64_t *bitmap);
 int VSL_Matched(const struct VSM_data *vd, uint64_t bitmap);
+int VSL_Name2Tag(const char *name, int l);
 extern const char *VSL_tags[256];
 
 #endif
diff --git a/lib/libvarnishapi/vsc.c b/lib/libvarnishapi/vsc.c
index d9624b9..fc18cb3 100644
--- a/lib/libvarnishapi/vsc.c
+++ b/lib/libvarnishapi/vsc.c
@@ -330,7 +330,7 @@ VSC_Iter(struct VSM_data *vd, VSC_iter_f *func, void *priv)
 		CHECK_OBJ_NOTNULL(sha, VSM_CHUNK_MAGIC);
 		if (strcmp(sha->class, VSC_CLASS))
 			continue;
-
+		/*lint -save -e525 -e539 */
 #define VSC_F(a,b,c,d,e)
 #define VSC_DONE(a,b,c)
 #define VSC_DO(U,l,t)						\
@@ -343,6 +343,7 @@ VSC_Iter(struct VSM_data *vd, VSC_iter_f *func, void *priv)
 #undef VSC_F
 #undef VSC_DO
 #undef VSC_DONE
+		/*lint -restore */
 		break;
 	}
 	return (i);
diff --git a/lib/libvarnishapi/vsl.c b/lib/libvarnishapi/vsl.c
index f4e633b..6791888 100644
--- a/lib/libvarnishapi/vsl.c
+++ b/lib/libvarnishapi/vsl.c
@@ -206,7 +206,7 @@ vsl_nextlog(struct vsl *vsl, uint32_t **pp)
 }
 
 int
-VSL_NextLog(const struct VSM_data *vd, uint32_t **pp, uint64_t *mb)
+VSL_NextLog(const struct VSM_data *vd, uint32_t **pp, uint64_t *bits)
 {
 	struct vsl *vsl;
 	uint32_t *p;
@@ -268,7 +268,7 @@ VSL_NextLog(const struct VSM_data *vd, uint32_t **pp, uint64_t *mb)
 			if (i != VRE_ERROR_NOMATCH)
 				continue;
 		}
-		if (mb != NULL) {
+		if (bits != NULL) {
 			struct vsl_re_match *vrm;
 			int j = 0;
 			VTAILQ_FOREACH(vrm, &vsl->matchers, next) {
@@ -276,7 +276,7 @@ VSL_NextLog(const struct VSM_data *vd, uint32_t **pp, uint64_t *mb)
 					i = VRE_exec(vrm->re, VSL_DATA(p),
 						     VSL_LEN(p), 0, 0, NULL, 0);
 					if (i >= 0)
-						*mb |= 1 << j;
+						*bits |= (uintmax_t)1 << j;
 				}
 				j++;
 			}
diff --git a/lib/libvarnishapi/vsl_arg.c b/lib/libvarnishapi/vsl_arg.c
index 49930c8..21cd8e9 100644
--- a/lib/libvarnishapi/vsl_arg.c
+++ b/lib/libvarnishapi/vsl_arg.c
@@ -41,7 +41,6 @@
 #include <unistd.h>
 
 #include "vas.h"
-#include "vin.h"
 #include "vre.h"
 #include "vbm.h"
 #include "miniobj.h"
@@ -50,6 +49,34 @@
 #include "vsm_api.h"
 #include "vsl_api.h"
 
+/*--------------------------------------------------------------------
+ * Look up a tag
+ *   0..255	tag number
+ *   -1		no tag matches
+ *   -2		multiple tags match
+ */
+
+int
+VSL_Name2Tag(const char *name, int l)
+{
+	int i, n;
+
+	if (l == -1)
+		l = strlen(name);
+	n = -1;
+	for (i = 0; i < 256; i++) {
+		if (VSL_tags[i] != NULL &&
+		    !strncasecmp(name, VSL_tags[i], l)) {
+			if (n == -1)
+				n = i;
+			else
+				n = -2;
+		}
+	}
+	return (n);
+}
+
+
 /*--------------------------------------------------------------------*/
 
 static int
@@ -99,8 +126,8 @@ vsl_IX_arg(const struct VSM_data *vd, const char *opt, int arg)
 static int
 vsl_ix_arg(const struct VSM_data *vd, const char *opt, int arg)
 {
-	int i, j, l;
-	const char *b, *e, *p, *q;
+	int i, l;
+	const char *b, *e;
 
 	CHECK_OBJ_NOTNULL(vd, VSM_MAGIC);
 	/* If first option is 'i', set all bits for supression */
@@ -120,24 +147,17 @@ vsl_ix_arg(const struct VSM_data *vd, const char *opt, int arg)
 			e++;
 		while (isspace(b[l - 1]))
 			l--;
-		for (i = 0; i < 256; i++) {
-			if (VSL_tags[i] == NULL)
-				continue;
-			p = VSL_tags[i];
-			q = b;
-			for (j = 0; j < l; j++)
-				if (tolower(*q++) != tolower(*p++))
-					break;
-			if (j != l || *p != '\0')
-				continue;
-
+		i = VSL_Name2Tag(b, l);
+		if (i >= 0) {
 			if (arg == 'x')
 				vbit_set(vd->vsl->vbm_supress, i);
 			else
 				vbit_clr(vd->vsl->vbm_supress, i);
-			break;
-		}
-		if (i == 256) {
+		} else if (i == -2) {
+			fprintf(stderr,
+			    "\"%*.*s\" matches multiple tags\n", l, l, b);
+			return (-1);
+		} else {
 			fprintf(stderr,
 			    "Could not match \"%*.*s\" to any tag\n", l, l, b);
 			return (-1);
@@ -148,19 +168,6 @@ vsl_ix_arg(const struct VSM_data *vd, const char *opt, int arg)
 
 /*--------------------------------------------------------------------*/
 
-static int
-name2tag(const char *n)
-{
-	int i;
-
-	for (i = 0; i < 256; i++) {
-		if (VSL_tags[i] == NULL)
-			continue;
-		if (!strcasecmp(n, VSL_tags[i]))
-			return (i);
-	}
-	return (-1);
-}
 
 static int
 vsl_m_arg(const struct VSM_data *vd, const char *opt)
@@ -171,8 +178,6 @@ vsl_m_arg(const struct VSM_data *vd, const char *opt)
 	int erroroffset;
 
 	CHECK_OBJ_NOTNULL(vd, VSM_MAGIC);
-	ALLOC_OBJ(m, VSL_RE_MATCH_MAGIC);
-	AN(m);
 
 	if (!strchr(opt, ':')) {
 		fprintf(stderr, "No : found in -o option %s\n", opt);
@@ -185,10 +190,13 @@ vsl_m_arg(const struct VSM_data *vd, const char *opt)
 	*regex = '\0';
 	regex++;
 
-	m->tag = name2tag(o);
-	if (m->tag == -1) {
+	ALLOC_OBJ(m, VSL_RE_MATCH_MAGIC);
+	AN(m);
+	m->tag = VSL_Name2Tag(o, -1);
+	if (m->tag < 0) {
 		fprintf(stderr, "Illegal tag %s specified\n", o);
 		free(o);
+		FREE_OBJ(m);
 		return (-1);
 	}
 	/* Get tag, regex */
@@ -196,6 +204,7 @@ vsl_m_arg(const struct VSM_data *vd, const char *opt)
 	if (m->re == NULL) {
 		fprintf(stderr, "Illegal regex: %s\n", error);
 		free(o);
+		FREE_OBJ(m);
 		return (-1);
 	}
 	vd->vsl->num_matchers++;
diff --git a/lib/libvarnishapi/vsm.c b/lib/libvarnishapi/vsm.c
index efb7592..888a0d7 100644
--- a/lib/libvarnishapi/vsm.c
+++ b/lib/libvarnishapi/vsm.c
@@ -224,7 +224,7 @@ VSM_Open(struct VSM_data *vd, int diag)
 	CHECK_OBJ_NOTNULL(vd, VSM_MAGIC);
 	AZ(vd->VSM_head);
 	if (!vd->n_opt)
-		VSM_n_Arg(vd, "");
+		(void)VSM_n_Arg(vd, "");
 	return (vsm_open(vd, diag));
 }
 
@@ -331,7 +331,7 @@ VSM_iter0(struct VSM_data *vd)
 	CHECK_OBJ_NOTNULL(vd, VSM_MAGIC);
 	vd->alloc_seq = vd->VSM_head->alloc_seq;
 	while (vd->alloc_seq == 0) {
-		usleep(50000);
+		(void)usleep(50000);
 		vd->alloc_seq = vd->VSM_head->alloc_seq;
 	}
 	CHECK_OBJ_NOTNULL(&vd->VSM_head->head, VSM_CHUNK_MAGIC);
@@ -358,7 +358,7 @@ VSM_itern(const struct VSM_data *vd, struct VSM_chunk **pp)
 
 /*--------------------------------------------------------------------*/
 unsigned
-VSM_Seq(struct VSM_data *vd)
+VSM_Seq(const struct VSM_data *vd)
 {
 
 	CHECK_OBJ_NOTNULL(vd, VSM_MAGIC);



From geoff at varnish-cache.org  Mon Jan  9 20:51:45 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:51:45 +0100
Subject: [experimental-ims] 33774e3 Merge branch 'master' of
	ssh://git.varnish-cache.org/git/varnish-cache
Message-ID: <E1RkMC1-0006is-6J@project.varnish-software.com>

commit 33774e3dc2d661598839db95444cd759b2ebdd72
Merge: cb07c41 1bddcfa
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu Sep 1 15:20:22 2011 +0000

    Merge branch 'master' of ssh://git.varnish-cache.org/git/varnish-cache



From geoff at varnish-cache.org  Mon Jan  9 20:51:45 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:51:45 +0100
Subject: [experimental-ims] e14ade5 Merge branch 'master' of
	ssh://git.varnish-cache.org/git/varnish-cache
Message-ID: <E1RkMC1-0006jG-A4@project.varnish-software.com>

commit e14ade5cf2d370c737122f1fc63b8a6883016fa2
Merge: 2c28590 e000786
Author: Per Buer <perbu at varnish-software.com>
Date:   Mon Sep 5 13:01:44 2011 +0200

    Merge branch 'master' of ssh://git.varnish-cache.org/git/varnish-cache



From geoff at varnish-cache.org  Mon Jan  9 20:51:45 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:51:45 +0100
Subject: [experimental-ims] e000786 Ignore invalid HTTP headers
Message-ID: <E1RkMC1-0006j3-8O@project.varnish-software.com>

commit e0007862dee9352d63bc2da613304f4652810b9d
Author: Andreas Plesner Jacobsen <apj at mutt.dk>
Date:   Mon Sep 5 12:05:24 2011 +0200

    Ignore invalid HTTP headers

diff --git a/bin/varnishncsa/varnishncsa.c b/bin/varnishncsa/varnishncsa.c
index 9ebb096..25c63ee 100644
--- a/bin/varnishncsa/varnishncsa.c
+++ b/bin/varnishncsa/varnishncsa.c
@@ -250,7 +250,7 @@ static int
 collect_backend(struct logline *lp, enum VSL_tag_e tag, unsigned spec,
     const char *ptr, unsigned len)
 {
-	const char *end, *next;
+	const char *end, *next, *split;
 
 	assert(spec & VSL_S_BACKEND);
 	end = ptr + len;
@@ -330,18 +330,19 @@ collect_backend(struct logline *lp, enum VSL_tag_e tag, unsigned spec,
 		break;
 
 	case SLT_TxHeader:
+		split = strchr(ptr, ':');
 		if (!lp->active)
 			break;
+		if (split == NULL)
+			break;
 		if (isprefix(ptr, "authorization:", end, &next) &&
 		    isprefix(next, "basic", end, &next)) {
 			lp->df_u = trimline(next, end);
 		} else {
 			struct hdr *h;
-			const char *split;
 			size_t l;
 			h = malloc(sizeof(struct hdr));
 			AN(h);
-			split = strchr(ptr, ':');
 			AN(split);
 			l = strlen(split);
 			h->key = trimline(ptr, split-1);
@@ -369,7 +370,7 @@ static int
 collect_client(struct logline *lp, enum VSL_tag_e tag, unsigned spec,
     const char *ptr, unsigned len)
 {
-	const char *end, *next;
+	const char *end, *next, *split;
 	long l;
 	time_t t;
 
@@ -437,8 +438,11 @@ collect_client(struct logline *lp, enum VSL_tag_e tag, unsigned spec,
 
 	case SLT_TxHeader:
 	case SLT_RxHeader:
+		split = strchr(ptr, ':');
 		if (!lp->active)
 			break;
+		if (split == NULL)
+			break;
 		if (tag == SLT_RxHeader &&
 		    isprefix(ptr, "authorization:", end, &next) &&
 		    isprefix(next, "basic", end, &next)) {
@@ -446,10 +450,8 @@ collect_client(struct logline *lp, enum VSL_tag_e tag, unsigned spec,
 			lp->df_u = trimline(next, end);
 		} else {
 			struct hdr *h;
-			const char *split;
 			h = malloc(sizeof(struct hdr));
 			AN(h);
-			split = strchr(ptr, ':');
 			AN(split);
 			h->key = trimline(ptr, split);
 			h->value = trimline(split+1, end);



From geoff at varnish-cache.org  Mon Jan  9 20:51:45 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:51:45 +0100
Subject: [experimental-ims] dce01d1 Small optimization: don't process header
	when inactive
Message-ID: <E1RkMC1-0006jQ-Ca@project.varnish-software.com>

commit dce01d1ec45c5e0bc57e7e4a2a9ea7ec097a07a2
Author: Andreas Plesner Jacobsen <apj at mutt.dk>
Date:   Mon Sep 5 13:04:37 2011 +0200

    Small optimization: don't process header when inactive

diff --git a/bin/varnishncsa/varnishncsa.c b/bin/varnishncsa/varnishncsa.c
index 25c63ee..28f6f21 100644
--- a/bin/varnishncsa/varnishncsa.c
+++ b/bin/varnishncsa/varnishncsa.c
@@ -330,9 +330,9 @@ collect_backend(struct logline *lp, enum VSL_tag_e tag, unsigned spec,
 		break;
 
 	case SLT_TxHeader:
-		split = strchr(ptr, ':');
 		if (!lp->active)
 			break;
+		split = strchr(ptr, ':');
 		if (split == NULL)
 			break;
 		if (isprefix(ptr, "authorization:", end, &next) &&
@@ -438,9 +438,9 @@ collect_client(struct logline *lp, enum VSL_tag_e tag, unsigned spec,
 
 	case SLT_TxHeader:
 	case SLT_RxHeader:
-		split = strchr(ptr, ':');
 		if (!lp->active)
 			break;
+		split = strchr(ptr, ':');
 		if (split == NULL)
 			break;
 		if (tag == SLT_RxHeader &&



From geoff at varnish-cache.org  Mon Jan  9 20:51:45 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:51:45 +0100
Subject: [experimental-ims] db1a21f Add documentation of string syntax,
	rollback, panic and synthetic keywords.
Message-ID: <E1RkMC1-0006jc-Et@project.varnish-software.com>

commit db1a21ffba8f34776c0c971de80e29c7fd0766d6
Author: Andreas Plesner Jacobsen <apj at mutt.dk>
Date:   Mon Sep 5 20:09:59 2011 +0200

    Add documentation of string syntax, rollback, panic and synthetic keywords.
    
    Fixes: #991

diff --git a/doc/sphinx/reference/vcl.rst b/doc/sphinx/reference/vcl.rst
index 4f005d0..82e20f4 100644
--- a/doc/sphinx/reference/vcl.rst
+++ b/doc/sphinx/reference/vcl.rst
@@ -39,6 +39,12 @@ In addition to the C-like assignment (=), comparison (==, !=) and
 boolean (!, && and \|\|) operators, VCL supports both regular
 expression and ACL matching using the ~ and the !~ operators.
 
+Basic strings are enclosed in " ... " and uses URL-style %-escapes.
+
+Long strings are enclosed in {" ... "} and do not have an escape
+character. They may contain any character including ", newline and
+other control characters except for the NUL (0x00) character.
+
 Unlike C and Perl, the backslash (\) character has no special meaning
 in strings in VCL, so it can be freely used in regular expressions
 without doubling.
@@ -55,6 +61,15 @@ You can use the *set* keyword to arbitrary HTTP headers. You can
 remove headers with the *remove* or *unset* keywords, which are
 synonym.
 
+You can use the *rollback* keyword to revert any changes to req at
+any time.
+
+The *synthetic* keyword is used to produce a synthetic response
+body in vcl_error. It takes a single string as argument.
+
+You can force a crash of the client process with the *panic* keyword.
+*panic* takes a string as argument.
+
 The ``return(action)`` keyword terminates the subroutine. *action* can be,
 depending on context one of
 



From geoff at varnish-cache.org  Mon Jan  9 20:51:45 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:51:45 +0100
Subject: [experimental-ims] 8548216 I have not been able to reproduce this
	exact circumstance, but the fix is readily obvious,
	so:  Tighten requirement for range delivery.
Message-ID: <E1RkMC1-0006jm-Gp@project.varnish-software.com>

commit 8548216821a2710953f2063be75490bb0d9d7eba
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Sep 6 08:15:06 2011 +0000

    I have not been able to reproduce this exact circumstance, but the
    fix is readily obvious, so:  Tighten requirement for range delivery.
    
    Fixes	#1007

diff --git a/bin/varnishd/cache_response.c b/bin/varnishd/cache_response.c
index c592909..73338f7 100644
--- a/bin/varnishd/cache_response.c
+++ b/bin/varnishd/cache_response.c
@@ -276,9 +276,13 @@ RES_WriteObj(struct sess *sp)
 	 */
 	low = 0;
 	high = sp->obj->len - 1;
-	if (!(sp->wrk->res_mode & (RES_ESI|RES_ESI_CHILD|RES_GUNZIP)) &&
-	    params->http_range_support && sp->obj->response == 200 &&
-	    sp->wantbody && http_GetHdr(sp->http, H_Range, &r))
+	if (
+	    sp->wantbody &&
+	    (sp->wrk->res_mode & RES_LEN) &&
+	    !(sp->wrk->res_mode & (RES_ESI|RES_ESI_CHILD|RES_GUNZIP)) &&
+	    params->http_range_support &&
+	    sp->obj->response == 200 &&
+	    http_GetHdr(sp->http, H_Range, &r))
 		res_dorange(sp, r, &low, &high);
 
 	/*



From geoff at varnish-cache.org  Mon Jan  9 20:51:45 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:51:45 +0100
Subject: [experimental-ims] daa446e Unlink -s file with autogenerated names
Message-ID: <E1RkMC1-0006jx-Io@project.varnish-software.com>

commit daa446eb29a1d90b452dc8ccf31bf28b6c4c516b
Author: Tollef Fog Heen <tfheen at varnish-software.com>
Date:   Tue Sep 6 13:49:59 2011 +0200

    Unlink -s file with autogenerated names
    
    If just -s file or -s file,/path/to/directory is given, the file
    stevedore will generate a name.  Make sure to unlink this to not leak
    disk space over time.
    
    Fixes: #1008

diff --git a/bin/varnishd/stevedore_utils.c b/bin/varnishd/stevedore_utils.c
index a38588d..bd4e368 100644
--- a/bin/varnishd/stevedore_utils.c
+++ b/bin/varnishd/stevedore_utils.c
@@ -108,6 +108,7 @@ STV_GetFile(const char *fn, int *fdp, const char **fnp, const char *ctx)
 		if (fd < 0)
 			ARGV_ERR("(%s) \"%s\" mkstemp(%s) failed (%s)\n",
 			    ctx, fn, buf, strerror(errno));
+		AZ(unlink(buf));
 		*fnp = strdup(buf);
 		AN(*fnp);
 		retval = 2;



From geoff at varnish-cache.org  Mon Jan  9 20:51:45 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:51:45 +0100
Subject: [experimental-ims] 5facfd3 Make the default -s file size 100MB rather
	than 50% of free disk space
Message-ID: <E1RkMC1-0006k7-Kd@project.varnish-software.com>

commit 5facfd3ce2639309a7a02a9e21cea738cc43f12d
Author: Tollef Fog Heen <tfheen at varnish-software.com>
Date:   Tue Sep 6 13:51:04 2011 +0200

    Make the default -s file size 100MB rather than 50% of free disk space

diff --git a/bin/varnishd/storage_file.c b/bin/varnishd/storage_file.c
index ea78526..7687179 100644
--- a/bin/varnishd/storage_file.c
+++ b/bin/varnishd/storage_file.c
@@ -116,7 +116,7 @@ smf_initfile(struct stevedore *st, struct smf_sc *sc, const char *size)
 	/* XXX: force block allocation here or in open ? */
 }
 
-static const char default_size[] = "50%";
+static const char default_size[] = "100M";
 static const char default_filename[] = ".";
 
 static void



From geoff at varnish-cache.org  Mon Jan  9 20:51:45 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:51:45 +0100
Subject: [experimental-ims] efaa48a Fix handling vs Varnish:handling in
	varnishncsa
Message-ID: <E1RkMC1-0006kI-Mh@project.varnish-software.com>

commit efaa48a908806f9d0a0969f840393e5b77771036
Author: Tollef Fog Heen <tfheen at varnish-software.com>
Date:   Tue Sep 6 15:07:03 2011 +0200

    Fix handling vs Varnish:handling in varnishncsa
    
    Thanks to Kai for patch
    Fixes: #999

diff --git a/bin/varnishncsa/varnishncsa.c b/bin/varnishncsa/varnishncsa.c
index 28f6f21..70e915f 100644
--- a/bin/varnishncsa/varnishncsa.c
+++ b/bin/varnishncsa/varnishncsa.c
@@ -716,7 +716,7 @@ h_ncsa(void *priv, enum VSL_tag_e tag, unsigned fd,
 					VSB_cat(os, (lp->df_hitmiss ? lp->df_hitmiss : "-"));
 					p = tmp;
 					break;
-				} else if (strcmp(fname, "handling") == 0) {
+				} else if (strcmp(fname, "Varnish:handling") == 0) {
 					VSB_cat(os, (lp->df_handling ? lp->df_handling : "-"));
 					p = tmp;
 					break;



From geoff at varnish-cache.org  Mon Jan  9 20:51:45 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:51:45 +0100
Subject: [experimental-ims] 4633d00 Fix up default format string in man page
	and code comments
Message-ID: <E1RkMC1-0006kV-Pl@project.varnish-software.com>

commit 4633d008fac58813b06bdc421dd31787851b6c50
Author: Tollef Fog Heen <tfheen at varnish-software.com>
Date:   Tue Sep 6 15:09:14 2011 +0200

    Fix up default format string in man page and code comments
    
    We're actually using %s, not %>s, since we don't support >.
    
    Thanks to Kai for patch
    Fixes: #999

diff --git a/bin/varnishncsa/varnishncsa.c b/bin/varnishncsa/varnishncsa.c
index 70e915f..7c35c3e 100644
--- a/bin/varnishncsa/varnishncsa.c
+++ b/bin/varnishncsa/varnishncsa.c
@@ -31,7 +31,7 @@
  * Obtain log data from the shared memory log, order it by session ID, and
  * display it in Apache / NCSA combined log format:
  *
- *	%h %l %u %t "%r" %>s %b "%{Referer}i" "%{User-agent}i"
+ *	%h %l %u %t "%r" %s %b "%{Referer}i" "%{User-agent}i"
  *
  * where the fields are defined as follows:
  *
diff --git a/doc/sphinx/reference/varnishncsa.rst b/doc/sphinx/reference/varnishncsa.rst
index 65f08dd..a4bac0a 100644
--- a/doc/sphinx/reference/varnishncsa.rst
+++ b/doc/sphinx/reference/varnishncsa.rst
@@ -52,7 +52,7 @@ The following options are available:
 -F format   Specify the log format used. If no format is specified the  
    	    default log format is used. Currently it is:
 
-            %h %l %u %t "%r" %>s %b "%{Referer}i" "%{User-agent}i"
+            %h %l %u %t "%r" %s %b "%{Referer}i" "%{User-agent}i"
 
 	    Supported formatters are:
 



From geoff at varnish-cache.org  Mon Jan  9 20:51:45 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:51:45 +0100
Subject: [experimental-ims] b09f55e Fix string documentation for 3.0
Message-ID: <E1RkMC1-0006kf-Sc@project.varnish-software.com>

commit b09f55ec50e4fcfaa5ee122a39b2a4c31ae3e9cf
Author: Andreas Plesner Jacobsen <apj at mutt.dk>
Date:   Tue Sep 6 19:30:40 2011 +0200

    Fix string documentation for 3.0

diff --git a/doc/sphinx/installation/upgrade.rst b/doc/sphinx/installation/upgrade.rst
index ee57df4..d00e9b0 100644
--- a/doc/sphinx/installation/upgrade.rst
+++ b/doc/sphinx/installation/upgrade.rst
@@ -13,6 +13,10 @@ string concatenation operator
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 String concatenation did not have an operator previously, but this has now been changed to ``+``.
 
+no more %-escapes in strings
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+To simplify strings, the %-encoding has been removed. If you need non-printable characters, you need to use inline C.
+
 ``log`` moved to the std vmod
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
diff --git a/doc/sphinx/reference/vcl.rst b/doc/sphinx/reference/vcl.rst
index 82e20f4..7359e6f 100644
--- a/doc/sphinx/reference/vcl.rst
+++ b/doc/sphinx/reference/vcl.rst
@@ -39,11 +39,11 @@ In addition to the C-like assignment (=), comparison (==, !=) and
 boolean (!, && and \|\|) operators, VCL supports both regular
 expression and ACL matching using the ~ and the !~ operators.
 
-Basic strings are enclosed in " ... " and uses URL-style %-escapes.
+Basic strings are enclosed in " ... ", and may not contain newlines.
 
-Long strings are enclosed in {" ... "} and do not have an escape
-character. They may contain any character including ", newline and
-other control characters except for the NUL (0x00) character.
+Long strings are enclosed in {" ... "}. They may contain any
+character including ", newline and other control characters except
+for the NUL (0x00) character.
 
 Unlike C and Perl, the backslash (\) character has no special meaning
 in strings in VCL, so it can be freely used in regular expressions
diff --git a/lib/libvcl/vcc_token.c b/lib/libvcl/vcc_token.c
index 5a1b00c..c5766c0 100644
--- a/lib/libvcl/vcc_token.c
+++ b/lib/libvcl/vcc_token.c
@@ -492,7 +492,7 @@ vcc_Lexer(struct vcc *tl, struct source *sp)
 			continue;
 		}
 
-		/* Match strings, with \\ and \" escapes */
+		/* Match strings */
 		if (*p == '"') {
 			for (q = p + 1; q < sp->e; q++) {
 				if (*q == '"') {



From geoff at varnish-cache.org  Mon Jan  9 20:51:46 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:51:46 +0100
Subject: [experimental-ims] b3690c6 Make it possible for VCL initializtion to
	fail gracefully,
	particularly so VMOD loading can emit sensible diagnostics.
Message-ID: <E1RkMC2-0006lR-96@project.varnish-software.com>

commit b3690c63fdbb06caf6b33aa55b8ca83a1b8455d2
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Sep 7 19:30:48 2011 +0000

    Make it possible for VCL initializtion to fail gracefully, particularly
    so VMOD loading can emit sensible diagnostics.

diff --git a/bin/varnishd/cache_vcl.c b/bin/varnishd/cache_vcl.c
index 49f3fbb..ac712fb 100644
--- a/bin/varnishd/cache_vcl.c
+++ b/bin/varnishd/cache_vcl.c
@@ -170,10 +170,15 @@ VCL_Load(const char *fn, const char *name, struct cli *cli)
 		FREE_OBJ(vcl);
 		return (1);
 	}
+	if (vcl->conf->init_vcl(cli)) {
+		VCLI_Out(cli, "VCL \"%s\" Failed to initialize", name);
+		(void)dlclose(vcl->dlh);
+		FREE_OBJ(vcl);
+		return (1);
+	}
 	REPLACE(vcl->name, name);
-	VTAILQ_INSERT_TAIL(&vcl_head, vcl, list);
 	VCLI_Out(cli, "Loaded \"%s\" as \"%s\"", fn , name);
-	vcl->conf->init_vcl(cli);
+	VTAILQ_INSERT_TAIL(&vcl_head, vcl, list);
 	(void)vcl->conf->init_func(NULL);
 	Lck_Lock(&vcl_mtx);
 	if (vcl_active == NULL)
diff --git a/bin/varnishd/cache_vrt_vmod.c b/bin/varnishd/cache_vrt_vmod.c
index 4a4d230..84a2bb7 100644
--- a/bin/varnishd/cache_vrt_vmod.c
+++ b/bin/varnishd/cache_vrt_vmod.c
@@ -60,12 +60,12 @@ struct vmod {
 
 static VTAILQ_HEAD(,vmod)	vmods = VTAILQ_HEAD_INITIALIZER(vmods);
 
-void
-VRT_Vmod_Init(void **hdl, void *ptr, int len, const char *nm, const char *path)
+int
+VRT_Vmod_Init(void **hdl, void *ptr, int len, const char *nm,
+    const char *path, struct cli *cli)
 {
 	struct vmod *v;
-	void *x;
-	const int *i;
+	void *x, *y, *z;
 
 	ASSERT_CLI();
 
@@ -76,31 +76,46 @@ VRT_Vmod_Init(void **hdl, void *ptr, int len, const char *nm, const char *path)
 		ALLOC_OBJ(v, VMOD_MAGIC);
 		AN(v);
 
-		VTAILQ_INSERT_TAIL(&vmods, v, list);
-		VSC_C_main->vmods++;
-
-		REPLACE(v->nm, nm);
-		REPLACE(v->path, path);
-
-		v->hdl = dlopen(v->path, RTLD_NOW | RTLD_LOCAL);
-		if (! v->hdl) {
-			char buf[1024];
-			sprintf(buf, "dlopen failed (child process lacks permission?): %.512s", dlerror());
-			VAS_Fail(__func__, __FILE__, __LINE__, buf, 0, 0);
+		v->hdl = dlopen(path, RTLD_NOW | RTLD_LOCAL);
+		if (v->hdl == NULL) {
+			VCLI_Out(cli, "Loading VMOD %s from %s:\n", nm, path);
+			VCLI_Out(cli, "dlopen() failed: %s\n", dlerror());
+			VCLI_Out(cli, "Check child process permissions.\n"); 
+			FREE_OBJ(v);
+			return (1);
 		}
 
 		x = dlsym(v->hdl, "Vmod_Name");
+		y = dlsym(v->hdl, "Vmod_Len");
+		z = dlsym(v->hdl, "Vmod_Func");
+		if (x == NULL || y == NULL || z == NULL) {
+			VCLI_Out(cli, "Loading VMOD %s from %s:\n", nm, path);
+			VCLI_Out(cli, "VMOD symbols not found\n");
+			VCLI_Out(cli, "Check relative pathnames.\n");
+			(void)dlclose(v->hdl);
+			FREE_OBJ(v);
+			return (1);
+		}
 		AN(x);
-		/* XXX: check that name is correct */
+		AN(y);
+		AN(z);
+		if (strcmp(x, nm)) {
+			VCLI_Out(cli, "Loading VMOD %s from %s:\n", nm, path);
+			VCLI_Out(cli, "File contain wrong VMOD (\"%s\")\n", x);
+			VCLI_Out(cli, "Check relative pathnames ?.\n");
+			(void)dlclose(v->hdl);
+			FREE_OBJ(v);
+			return (1);
+		}
 
-		x = dlsym(v->hdl, "Vmod_Len");
-		AN(x);
-		i = x;
-		v->funclen = *i;
+		v->funclen = *(const int *)y;
+		v->funcs = z;
 
-		x = dlsym(v->hdl, "Vmod_Func");
-		AN(x);
-		v->funcs = x;
+		REPLACE(v->nm, nm);
+		REPLACE(v->path, path);
+
+		VSC_C_main->vmods++;
+		VTAILQ_INSERT_TAIL(&vmods, v, list);
 	}
 
 	assert(len == v->funclen);
@@ -108,6 +123,7 @@ VRT_Vmod_Init(void **hdl, void *ptr, int len, const char *nm, const char *path)
 	v->ref++;
 
 	*hdl = v;
+	return (0);
 }
 
 void
diff --git a/bin/varnishd/mgt_child.c b/bin/varnishd/mgt_child.c
index 2664931..ebdf57b 100644
--- a/bin/varnishd/mgt_child.c
+++ b/bin/varnishd/mgt_child.c
@@ -385,7 +385,7 @@ start_child(struct cli *cli)
 	mgt_cli_start_child(child_cli_in, child_VCLI_Out);
 	child_pid = pid;
 	if (mgt_push_vcls_and_start(&u, &p)) {
-		REPORT(LOG_ERR, "Pushing vcls failed: %s", p);
+		REPORT(LOG_ERR, "Pushing vcls failed:\n%s", p);
 		free(p);
 		child_state = CH_RUNNING;
 		mgt_stop_child();
diff --git a/include/vrt.h b/include/vrt.h
index cfafe75..9f0228f 100644
--- a/include/vrt.h
+++ b/include/vrt.h
@@ -183,8 +183,8 @@ void VRT_init_dir(struct cli *, struct director **, const char *name,
 void VRT_fini_dir(struct cli *, struct director *);
 
 /* VMOD/Modules related */
-void VRT_Vmod_Init(void **hdl, void *ptr, int len, const char *nm,
-    const char *path);
+int VRT_Vmod_Init(void **hdl, void *ptr, int len, const char *nm,
+    const char *path, struct cli *cli);
 void VRT_Vmod_Fini(void **hdl);
 
 struct vmod_priv;
diff --git a/lib/libvcl/generate.py b/lib/libvcl/generate.py
index 5407a51..fde8bb2 100755
--- a/lib/libvcl/generate.py
+++ b/lib/libvcl/generate.py
@@ -712,7 +712,7 @@ fo.write("""
 struct sess;
 struct cli;
 
-typedef void vcl_init_f(struct cli *);
+typedef int vcl_init_f(struct cli *);
 typedef void vcl_fini_f(struct cli *);
 typedef int vcl_func_f(struct sess *sp);
 """)
diff --git a/lib/libvcl/vcc_compile.c b/lib/libvcl/vcc_compile.c
index b0019ca..c2a52d4 100644
--- a/lib/libvcl/vcc_compile.c
+++ b/lib/libvcl/vcc_compile.c
@@ -310,9 +310,10 @@ static void
 EmitInitFunc(const struct vcc *tl)
 {
 
-	Fc(tl, 0, "\nstatic void\nVGC_Init(struct cli *cli)\n{\n\n");
+	Fc(tl, 0, "\nstatic int\nVGC_Init(struct cli *cli)\n{\n\n");
 	AZ(VSB_finish(tl->fi));
 	VSB_cat(tl->fc, VSB_data(tl->fi));
+	Fc(tl, 0, "\treturn(0);\n");
 	Fc(tl, 0, "}\n");
 }
 
@@ -321,7 +322,7 @@ EmitFiniFunc(const struct vcc *tl)
 {
 	unsigned u;
 
-	Fc(tl, 0, "\nstatic void\nVGC_Fini(struct cli *cli)\n{\n\n");
+	Fc(tl, 0, "\nstatic int\nVGC_Fini(struct cli *cli)\n{\n\n");
 
 	/*
 	 * We do this here, so we are sure they happen before any
@@ -332,6 +333,7 @@ EmitFiniFunc(const struct vcc *tl)
 
 	AZ(VSB_finish(tl->ff));
 	VSB_cat(tl->fc, VSB_data(tl->ff));
+	Fc(tl, 0, "\treturn(0);\n");
 	Fc(tl, 0, "}\n");
 }
 
diff --git a/lib/libvcl/vcc_vmod.c b/lib/libvcl/vcc_vmod.c
index 35cc702..f74f874 100644
--- a/lib/libvcl/vcc_vmod.c
+++ b/lib/libvcl/vcc_vmod.c
@@ -102,13 +102,15 @@ vcc_ParseImport(struct vcc *tl)
 
 	Fh(tl, 0, "static void *VGC_vmod_%.*s;\n", PF(mod));
 
-	Fi(tl, 0, "\tVRT_Vmod_Init(&VGC_vmod_%.*s,\n", PF(mod));
+	Fi(tl, 0, "\tif (VRT_Vmod_Init(&VGC_vmod_%.*s,\n", PF(mod));
 	Fi(tl, 0, "\t    &Vmod_Func_%.*s,\n", PF(mod));
 	Fi(tl, 0, "\t    sizeof(Vmod_Func_%.*s),\n", PF(mod));
 	Fi(tl, 0, "\t    \"%.*s\",\n", PF(mod));
 	Fi(tl, 0, "\t    ");
 	EncString(tl->fi, fn, NULL, 0);
-	Fi(tl, 0, ");\n");
+	Fi(tl, 0, ",\n\t    ");
+	Fi(tl, 0, "cli))\n");
+	Fi(tl, 0, "\t\treturn(1);\n");
 
 	SkipToken(tl, ';');
 



From geoff at varnish-cache.org  Mon Jan  9 20:51:47 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:51:47 +0100
Subject: [experimental-ims] 3a08c3d Add Vmod_Id handle which VMOD can identify
	themselves to VRT with.
Message-ID: <E1RkMC3-0006mE-0x@project.varnish-software.com>

commit 3a08c3d95cace24133756e5e204ef73545921171
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu Sep 8 11:53:05 2011 +0000

    Add Vmod_Id handle which VMOD can identify themselves to VRT with.

diff --git a/bin/varnishd/cache_vrt_vmod.c b/bin/varnishd/cache_vrt_vmod.c
index 84a2bb7..bd2e172 100644
--- a/bin/varnishd/cache_vrt_vmod.c
+++ b/bin/varnishd/cache_vrt_vmod.c
@@ -56,6 +56,7 @@ struct vmod {
 	void			*hdl;
 	const void		*funcs;
 	int			funclen;
+	const void		*idptr;
 };
 
 static VTAILQ_HEAD(,vmod)	vmods = VTAILQ_HEAD_INITIALIZER(vmods);
@@ -65,12 +66,12 @@ VRT_Vmod_Init(void **hdl, void *ptr, int len, const char *nm,
     const char *path, struct cli *cli)
 {
 	struct vmod *v;
-	void *x, *y, *z;
+	void *x, *y, *z, *w;
 
 	ASSERT_CLI();
 
 	VTAILQ_FOREACH(v, &vmods, list)
-		if (!strcmp(v->nm, nm))
+		if (!strcmp(v->nm, nm))	// Also path, len ?
 			break;
 	if (v == NULL) {
 		ALLOC_OBJ(v, VMOD_MAGIC);
@@ -88,7 +89,8 @@ VRT_Vmod_Init(void **hdl, void *ptr, int len, const char *nm,
 		x = dlsym(v->hdl, "Vmod_Name");
 		y = dlsym(v->hdl, "Vmod_Len");
 		z = dlsym(v->hdl, "Vmod_Func");
-		if (x == NULL || y == NULL || z == NULL) {
+		w = dlsym(v->hdl, "Vmod_Id");
+		if (x == NULL || y == NULL || z == NULL || w == NULL) {
 			VCLI_Out(cli, "Loading VMOD %s from %s:\n", nm, path);
 			VCLI_Out(cli, "VMOD symbols not found\n");
 			VCLI_Out(cli, "Check relative pathnames.\n");
@@ -99,6 +101,7 @@ VRT_Vmod_Init(void **hdl, void *ptr, int len, const char *nm,
 		AN(x);
 		AN(y);
 		AN(z);
+		AN(w);
 		if (strcmp(x, nm)) {
 			VCLI_Out(cli, "Loading VMOD %s from %s:\n", nm, path);
 			VCLI_Out(cli, "File contain wrong VMOD (\"%s\")\n", x);
@@ -116,6 +119,7 @@ VRT_Vmod_Init(void **hdl, void *ptr, int len, const char *nm,
 
 		VSC_C_main->vmods++;
 		VTAILQ_INSERT_TAIL(&vmods, v, list);
+		v->idptr = w;
 	}
 
 	assert(len == v->funclen);
diff --git a/lib/libvmod_std/vmod.py b/lib/libvmod_std/vmod.py
index 83f8eca..1b0f1c0 100755
--- a/lib/libvmod_std/vmod.py
+++ b/lib/libvmod_std/vmod.py
@@ -308,5 +308,9 @@ fc.write("\n");
 fc.write('const char * const Vmod_Spec[] = {\n' + slist + '\t0\n};\n')
 
 fc.write('const char Vmod_Varnish_ABI[] = VMOD_ABI_Version;\n')
+
+fh.write('extern const void * const Vmod_Id;\n')
+fc.write('const void * const Vmod_Id = &Vmod_Id;\n')
+
 fc.write("\n")
 



From geoff at varnish-cache.org  Mon Jan  9 20:51:47 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:51:47 +0100
Subject: [experimental-ims] 4213531 Merge branch 'master' of
	ssh://git.varnish-cache.org/git/varnish-cache
Message-ID: <E1RkMC3-0006mQ-4N@project.varnish-software.com>

commit 4213531e8043bf382f9070e0bd6fac2aa4c8d547
Merge: 3a08c3d 20732d8
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu Sep 8 21:50:34 2011 +0000

    Merge branch 'master' of ssh://git.varnish-cache.org/git/varnish-cache



From geoff at varnish-cache.org  Mon Jan  9 20:51:47 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:51:47 +0100
Subject: [experimental-ims] 8156c29 Restore old nuke_limit,
	as 10 seems to be too low
Message-ID: <E1RkMC3-0006md-7G@project.varnish-software.com>

commit 8156c297082a43396324f062bdc97aba2a9af2ec
Author: Andreas Plesner Jacobsen <apj at marvin.nerd.dk>
Date:   Fri Sep 9 13:33:38 2011 +0200

    Restore old nuke_limit, as 10 seems to be too low
    
    Fixes #1012

diff --git a/bin/varnishd/mgt_param.c b/bin/varnishd/mgt_param.c
index de93679..33101bb 100644
--- a/bin/varnishd/mgt_param.c
+++ b/bin/varnishd/mgt_param.c
@@ -622,7 +622,7 @@ static const struct parspec input_parspec[] = {
 		"Maximum number of objects we attempt to nuke in order"
 		"to make space for a object body.",
 		EXPERIMENTAL,
-		"10", "allocations" },
+		"50", "allocations" },
 	{ "fetch_chunksize",
 		tweak_uint, &master.fetch_chunksize, 4, UINT_MAX / 1024.,
 		"The default chunksize used by fetcher. "



From geoff at varnish-cache.org  Mon Jan  9 20:51:47 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:51:47 +0100
Subject: [experimental-ims] ce9dd3b Add type to connect_timeout
Message-ID: <E1RkMC3-0006n6-CM@project.varnish-software.com>

commit ce9dd3bb3b887731a203781e6d23b85010497191
Author: Andreas Plesner Jacobsen <apj at mutt.dk>
Date:   Fri Sep 9 15:30:55 2011 +0200

    Add type to connect_timeout
    
    References #971

diff --git a/doc/sphinx/reference/vcl.rst b/doc/sphinx/reference/vcl.rst
index 0e39187..aae16bc 100644
--- a/doc/sphinx/reference/vcl.rst
+++ b/doc/sphinx/reference/vcl.rst
@@ -232,7 +232,7 @@ random or round-robin director or using .list::
           .list = {
                   .host_header = "www.example.com";
                   .port = "80";
-                  .connect_timeout = 0.4;
+                  .connect_timeout = 0.4s;
                   "192.168.15.0"/24;
                   "192.168.16.128"/25;
           }



From geoff at varnish-cache.org  Mon Jan  9 20:51:47 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:51:47 +0100
Subject: [experimental-ims] f7c36f9 Tabulate probe properties and add all
	missing ones
Message-ID: <E1RkMC3-0006nH-FD@project.varnish-software.com>

commit f7c36f94d4be6f1b92c93d9c723267700a8c5b97
Author: Andreas Plesner Jacobsen <apj at mutt.dk>
Date:   Fri Sep 9 17:43:31 2011 +0200

    Tabulate probe properties and add all missing ones
    
    Really fixes #997

diff --git a/doc/sphinx/reference/vcl.rst b/doc/sphinx/reference/vcl.rst
index aae16bc..517f4fb 100644
--- a/doc/sphinx/reference/vcl.rst
+++ b/doc/sphinx/reference/vcl.rst
@@ -280,12 +280,36 @@ Backend probes
 
 Backends can be probed to see whether they should be considered
 healthy or not.  The return status can also be checked by using
-req.backend.healthy .window is how many of the latest polls we
-examine, while .threshold is how many of those must have succeeded for
-us to consider the backend healthy.  .initial is how many of the
-probes are considered good when Varnish starts - defaults to the same
-amount as the threshold. .expected_response is the expected backend
-HTTP response code.
+req.backend.healthy.
+
+Probes take the following parameters:
+
+.url
+  Specify a URL to request from the backend.
+  Defaults to "/".
+.request
+  Specify a full HTTP request using multiple strings.
+  .request will have \r\n automatically inserted after every string.
+  If specified, .request will take precedence over .url.
+.window
+  How many of the latest polls we examine to determine backend health.
+  Defaults to 8.
+.threshold 
+  How many of the polls in .window must have succeeded for us to consider
+  the backend healthy.
+  Defaults to 3.
+.initial
+  How many of the probes are considered good when Varnish starts.
+  Defaults to the same amount as the threshold.
+.expected_response
+  The expected backend HTTP response code.
+  Defaults to 200.
+.interval
+  Defines how often the probe should check the backend.
+  Default is every 5 seconds.
+.timeout
+  How fast each probe times out.
+  Default is 2 seconds.
 
 A backend with a probe can be defined like this, together with the
 backend or director:::



From geoff at varnish-cache.org  Mon Jan  9 20:51:48 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:51:48 +0100
Subject: [experimental-ims] 82344cb Merge branch 'master' of
	ssh://git.varnish-cache.org/git/varnish-cache
Message-ID: <E1RkMC4-0006oR-Dc@project.varnish-software.com>

commit 82344cb91b320ae7720895fc636cd6cbd34c388c
Merge: e14ade5 87ae60f
Author: Per Buer <perbu at varnish-software.com>
Date:   Mon Sep 12 10:37:47 2011 +0200

    Merge branch 'master' of ssh://git.varnish-cache.org/git/varnish-cache



From geoff at varnish-cache.org  Mon Jan  9 20:51:48 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:51:48 +0100
Subject: [experimental-ims] b16790a Avoid 32 bit variables in stats
	aggregation.
Message-ID: <E1RkMC4-0006od-Gk@project.varnish-software.com>

commit b16790a74f5e5ffb34d2f27027778399a90a3f71
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Sep 12 10:13:35 2011 +0000

    Avoid 32 bit variables in stats aggregation.
    
    Fixes	#993

diff --git a/bin/varnishd/cache.h b/bin/varnishd/cache.h
index 1c1952d..e2faf30 100644
--- a/bin/varnishd/cache.h
+++ b/bin/varnishd/cache.h
@@ -218,7 +218,7 @@ struct acct {
 /*--------------------------------------------------------------------*/
 
 #define L0(n)
-#define L1(n)			int n;
+#define L1(n)			uint64_t n;
 #define VSC_F(n, t, l, f, e)	L##l(n)
 #define VSC_DO_MAIN
 struct dstat {



From geoff at varnish-cache.org  Mon Jan  9 20:51:48 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:51:48 +0100
Subject: [experimental-ims] f336ffe Commit the actual fix for #1014 and not
	just the test-case.
Message-ID: <E1RkMC4-0006ou-Jq@project.varnish-software.com>

commit f336ffe66fe7f0a565539cf83e46aa93ba96f5ef
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Sep 12 10:14:27 2011 +0000

    Commit the actual fix for #1014 and not just the test-case.
    
    Fixes	#1014

diff --git a/bin/varnishd/cache_fetch.c b/bin/varnishd/cache_fetch.c
index 36df887..73edaa1 100644
--- a/bin/varnishd/cache_fetch.c
+++ b/bin/varnishd/cache_fetch.c
@@ -202,20 +202,15 @@ fetch_straight(struct sess *sp, struct http_conn *htc, const char *b)
 	ssize_t cl;
 
 	assert(sp->wrk->body_status == BS_LENGTH);
+
 	cl = fetch_number(b, 10);
+	sp->wrk->vfp->begin(sp, cl > 0 ? cl : 0);
 	if (cl < 0) {
 		WSP(sp, SLT_FetchError, "straight length field bogus");
 		return (-1);
-	}
-	/*
-	 * XXX: we shouldn't need this if we have cl==0
-	 * XXX: but we must also conditionalize the vfp->end()
-	 */
-	sp->wrk->vfp->begin(sp, cl);
-	if (cl == 0)
+	} else if (cl == 0)
 		return (0);
 
-
 	i = sp->wrk->vfp->bytes(sp, htc, cl);
 	if (i <= 0) {
 		WSP(sp, SLT_FetchError, "straight read_error: %d %d (%s)",
diff --git a/bin/varnishd/cache_gzip.c b/bin/varnishd/cache_gzip.c
index 675718a..b32b5cf 100644
--- a/bin/varnishd/cache_gzip.c
+++ b/bin/varnishd/cache_gzip.c
@@ -600,6 +600,7 @@ vfp_testgzip_begin(struct sess *sp, size_t estimate)
 {
 	(void)estimate;
 	sp->wrk->vgz_rx = VGZ_NewUngzip(sp, "u F -");
+	CHECK_OBJ_NOTNULL(sp->wrk->vgz_rx, VGZ_MAGIC);
 }
 
 static int __match_proto__()



From geoff at varnish-cache.org  Mon Jan  9 20:51:48 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:51:48 +0100
Subject: [experimental-ims] 6447d64 Updating configure line for ppc64,
	as --disabled-jemalloc has changed to --without-jemalloc
Message-ID: <E1RkMC4-0006pA-N8@project.varnish-software.com>

commit 6447d64122b337c6c6ef662a398eac6babc1dd80
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Mon Sep 12 12:31:13 2011 +0200

    Updating configure line for ppc64, as --disabled-jemalloc has changed
    to --without-jemalloc
    
    Patch from P?l-Eivind Johnsen
    
    Fixes: #1011

diff --git a/redhat/varnish.spec b/redhat/varnish.spec
index ca282e2..0d66765 100644
--- a/redhat/varnish.spec
+++ b/redhat/varnish.spec
@@ -87,7 +87,7 @@ cp bin/varnishd/default.vcl etc/zope-plone.vcl examples
 # Remove "--disable static" if you want to build static libraries 
 # jemalloc is not compatible with Red Hat's ppc64 RHEL kernel :-(
 %ifarch ppc64 ppc
-	%configure --disable-static --localstatedir=/var/lib --disable-jemalloc
+	%configure --disable-static --localstatedir=/var/lib --without-jemalloc
 %else
 	%configure --disable-static --localstatedir=/var/lib
 %endif



From geoff at varnish-cache.org  Mon Jan  9 20:51:48 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:51:48 +0100
Subject: [experimental-ims] 083d19a Dump the VSL into the varnishtest output.
Message-ID: <E1RkMC4-0006pL-QJ@project.varnish-software.com>

commit 083d19a60d7df575333736302f8bd91cd7b50f3e
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Sep 14 07:44:17 2011 +0000

    Dump the VSL into the varnishtest output.
    
    Based on code from:	DocWilco

diff --git a/bin/varnishtest/vtc_varnish.c b/bin/varnishtest/vtc_varnish.c
index 2b6e3d4..c53482e 100644
--- a/bin/varnishtest/vtc_varnish.c
+++ b/bin/varnishtest/vtc_varnish.c
@@ -47,7 +47,6 @@
 #include "libvarnish.h"
 #include "varnishapi.h"
 #include "vcli.h"
-#include "cli_common.h"
 #include "vss.h"
 #include "vsb.h"
 
@@ -58,7 +57,6 @@ struct varnish {
 #define VARNISH_MAGIC		0x208cd8e3
 	char			*name;
 	struct vtclog		*vl;
-	struct vtclog		*vl1;
 	VTAILQ_ENTRY(varnish)	list;
 
 	struct vsb		*storage;
@@ -68,16 +66,22 @@ struct varnish {
 	pid_t			pid;
 
 	pthread_t		tp;
+	pthread_t		tp_vsl;
 
 	int			cli_fd;
 	int			vcl_nbr;
 	char			*workdir;
 
-	struct VSM_data		*vd;
+	struct VSM_data		*vd;		/* vsc use */
+
+	unsigned		vsl_tag_count[256];
+	unsigned		vsl_sleep;
 };
 
 #define NONSENSE	"%XJEIFLH|)Xspa8P"
 
+#define VSL_SLEEP_USEC	(50*1000)
+
 static VTAILQ_HEAD(, varnish)	varnishes =
     VTAILQ_HEAD_INITIALIZER(varnishes);
 
@@ -116,6 +120,57 @@ varnish_ask_cli(const struct varnish *v, const char *cmd, char **repl)
 }
 
 /**********************************************************************
+ * Varnishlog gatherer + thread
+ */
+
+static int
+h_addlog(void *priv, enum VSL_tag_e tag, unsigned fd, unsigned len,
+    unsigned spec, const char *ptr, uint64_t bitmap)
+{
+	struct varnish *v;
+	int type;
+
+	(void) bitmap;
+
+	type = (spec & VSL_S_CLIENT) ? 'c' :
+	    (spec & VSL_S_BACKEND) ? 'b' : '-';
+	CAST_OBJ_NOTNULL(v, priv, VARNISH_MAGIC);
+
+	v->vsl_tag_count[tag]++;
+
+	vtc_log(v->vl, 4, "vsl| %5u %-12s %c %.*s", fd,
+	    VSL_tags[tag], type, len, ptr);
+	v->vsl_sleep = 100;
+	return (0);
+}
+
+static void *
+varnishlog_thread(void *priv)
+{
+	struct varnish *v;
+	struct VSM_data	*vsl;
+
+	CAST_OBJ_NOTNULL(v, priv, VARNISH_MAGIC);
+	vsl = VSM_New();
+	VSL_Setup(vsl);
+	(void)VSL_Arg(vsl, 'n', v->workdir);
+	VSL_NonBlocking(vsl, 1);
+	while (v->pid  && VSL_Open(vsl, 0) != 0) {
+		assert(usleep(VSL_SLEEP_USEC) == 0 || errno == EINTR);
+	}
+	while (v->pid) {
+		if (VSL_Dispatch(vsl, h_addlog, v) < 0) {
+			assert(usleep(v->vsl_sleep) == 0 || errno == EINTR);
+			v->vsl_sleep += v->vsl_sleep;
+			if (v->vsl_sleep > VSL_SLEEP_USEC)
+				v->vsl_sleep = VSL_SLEEP_USEC;
+		}
+	}
+	VSM_Delete(vsl);
+	return (NULL);
+}
+
+/**********************************************************************
  * Allocate and initialize a varnish
  */
 
@@ -145,9 +200,6 @@ varnish_new(const char *name)
 	    v->workdir, v->workdir, random(), v->workdir);
 	AZ(system(buf));
 
-	v->vl1 = vtc_logopen(name);
-	AN(v->vl1);
-
 	if (*v->name != 'v')
 		vtc_log(v->vl, 0, "Varnish name must start with 'v'");
 
@@ -218,7 +270,7 @@ varnish_thread(void *priv)
 		if (i <= 0)
 			break;
 		buf[i] = '\0';
-		vtc_dump(v->vl1, 3, "debug", buf, -1);
+		vtc_dump(v->vl, 3, "debug", buf, -1);
 	}
 	return (NULL);
 }
@@ -294,6 +346,7 @@ varnish_launch(struct varnish *v)
 	v->fds[2] = v->fds[3] = -1;
 	VSB_delete(vsb);
 	AZ(pthread_create(&v->tp, NULL, varnish_thread, v));
+	AZ(pthread_create(&v->tp_vsl, NULL, varnishlog_thread, v));
 
 	/* Wait for the varnish to call home */
 	fd[0].fd = v->cli_fd;
@@ -458,7 +511,9 @@ varnish_wait(struct varnish *v)
 	AZ(pthread_join(v->tp, &p));
 	AZ(close(v->fds[0]));
 	r = wait4(v->pid, &status, 0, NULL);
+	v->pid = 0;
 	vtc_log(v->vl, 2, "R %d Status: %04x", r, status);
+	AZ(pthread_join(v->tp_vsl, &p));
 	if (WIFEXITED(status) && WEXITSTATUS(status) == 0)
 		return;
 #ifdef WCOREDUMP



From geoff at varnish-cache.org  Mon Jan  9 20:51:48 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:51:48 +0100
Subject: [experimental-ims] 0bda0cb Minor flexelint'ing
Message-ID: <E1RkMC4-0006pX-TB@project.varnish-software.com>

commit 0bda0cb32712fa873d771ffc6f1ed593fd640de6
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Sep 14 07:50:40 2011 +0000

    Minor flexelint'ing

diff --git a/bin/varnishtest/flint.lnt b/bin/varnishtest/flint.lnt
index f50040c..8f3d880 100644
--- a/bin/varnishtest/flint.lnt
+++ b/bin/varnishtest/flint.lnt
@@ -22,3 +22,4 @@
 -e737	// [45]  Loss of sign in promotion from int to unsigned
 -e713	// Loss of precision (assignment) (unsigned long long to long long)
 -e574	// Signed-unsigned mix with relational
+-e835	//  A zero has been given as ___ argument to operator '___'  (<<)
diff --git a/bin/varnishtest/vtc.c b/bin/varnishtest/vtc.c
index 5b14edc..a034fef 100644
--- a/bin/varnishtest/vtc.c
+++ b/bin/varnishtest/vtc.c
@@ -60,7 +60,6 @@ volatile sig_atomic_t	vtc_error;	/* Error encountered */
 int			vtc_stop;	/* Stops current test without error */
 pthread_t		vtc_thread;
 static struct vtclog	*vltop;
-int			in_tree = 0;	/* Are we running in-tree */
 
 /**********************************************************************
  * Macro facility
@@ -238,18 +237,6 @@ extmacro_def(const char *name, const char *fmt, ...)
 	}
 }
 
-const char *
-extmacro_get(const char *name)
-{
-	struct extmacro *m;
-
-	VTAILQ_FOREACH(m, &extmacro_list, list)
-		if (!strcmp(name, m->name))
-			return (m->val);
-
-	return (NULL);
-}
-
 /**********************************************************************
  * Execute a file
  */
diff --git a/bin/varnishtest/vtc.h b/bin/varnishtest/vtc.h
index 16d577f..eb9a02a 100644
--- a/bin/varnishtest/vtc.h
+++ b/bin/varnishtest/vtc.h
@@ -85,4 +85,3 @@ void macro_def(struct vtclog *vl, const char *instance, const char *name,
 struct vsb *macro_expand(struct vtclog *vl, const char *text);
 
 void extmacro_def(const char *name, const char *fmt, ...);
-const char *extmacro_get(const char *name);
diff --git a/bin/varnishtest/vtc_http.c b/bin/varnishtest/vtc_http.c
index 758113b..9c0f175 100644
--- a/bin/varnishtest/vtc_http.c
+++ b/bin/varnishtest/vtc_http.c
@@ -586,7 +586,7 @@ cmd_http_gunzip_body(CMD_ARGS)
  */
 
 static void
-gzip_body(struct http *hp, const char *txt, char **body, int *bodylen)
+gzip_body(const struct http *hp, const char *txt, char **body, int *bodylen)
 {
 	int l, i;
 	z_stream vz;



From geoff at varnish-cache.org  Mon Jan  9 20:51:49 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:51:49 +0100
Subject: [experimental-ims] 2a754e9 Rename the pool initializer to Pool_Init()
Message-ID: <E1RkMC5-0006q2-3Z@project.varnish-software.com>

commit 2a754e9eb282cdf5bf65fd77f4b2479a383b6f5b
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sat Sep 17 09:40:20 2011 +0000

    Rename the pool initializer to Pool_Init()

diff --git a/bin/varnishd/cache.h b/bin/varnishd/cache.h
index 1abd6db..5b4b575 100644
--- a/bin/varnishd/cache.h
+++ b/bin/varnishd/cache.h
@@ -839,8 +839,7 @@ void PAN_Init(void);
 void PipeSession(struct sess *sp);
 
 /* cache_pool.c */
-void WRK_Init(void);
-void WRK2_Init(void);
+void Pool_Init(void);
 int WRK_QueueSession(struct sess *sp);
 void WRK_SumStat(struct worker *w);
 int WRK_TrySumStat(struct worker *w);
@@ -937,6 +936,7 @@ void VMOD_Init(void);
 
 /* cache_wrk.c */
 
+void WRK_Init(void);
 void *WRK_thread(void *priv);
 
 /* cache_ws.c */
diff --git a/bin/varnishd/cache_main.c b/bin/varnishd/cache_main.c
index f03604d..4312393 100644
--- a/bin/varnishd/cache_main.c
+++ b/bin/varnishd/cache_main.c
@@ -116,6 +116,7 @@ child_main(void)
 	VBE_Init();
 	VBP_Init();
 	WRK_Init();
+	Pool_Init();
 
 	EXP_Init();
 	HSH_Init();
diff --git a/bin/varnishd/cache_pool.c b/bin/varnishd/cache_pool.c
index 8286162..b7f1f41 100644
--- a/bin/varnishd/cache_pool.c
+++ b/bin/varnishd/cache_pool.c
@@ -511,7 +511,7 @@ WRK_BgThread(pthread_t *thr, const char *name, bgthread_t *func, void *priv)
 /*--------------------------------------------------------------------*/
 
 void
-WRK2_Init(void)
+Pool_Init(void)
 {
 	pthread_t tp;
 
diff --git a/bin/varnishd/cache_wrk.c b/bin/varnishd/cache_wrk.c
index 5b78c19..9b3bc2b 100644
--- a/bin/varnishd/cache_wrk.c
+++ b/bin/varnishd/cache_wrk.c
@@ -156,5 +156,4 @@ void
 WRK_Init(void)
 {
 	Lck_New(&wstat_mtx, lck_wstat);
-	WRK2_Init();
 }



From geoff at varnish-cache.org  Mon Jan  9 20:51:49 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:51:49 +0100
Subject: [experimental-ims] 685d9b9 Missed renaming these two bits.
Message-ID: <E1RkMC5-0006rT-QQ@project.varnish-software.com>

commit 685d9b962246d7dbad53d5076b6aa547809c2b06
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sat Sep 17 11:02:31 2011 +0000

    Missed renaming these two bits.

diff --git a/bin/varnishd/cache_waiter_kqueue.c b/bin/varnishd/cache_waiter_kqueue.c
index cbea248..8febec7 100644
--- a/bin/varnishd/cache_waiter_kqueue.c
+++ b/bin/varnishd/cache_waiter_kqueue.c
@@ -205,7 +205,7 @@ vwk_thread(void *priv)
 /*--------------------------------------------------------------------*/
 
 static void
-vca_kqueue_pass(void *priv, const struct sess *sp)
+vwk_pass(void *priv, const struct sess *sp)
 {
 	struct vwk *vwk;
 
@@ -216,7 +216,7 @@ vca_kqueue_pass(void *priv, const struct sess *sp)
 /*--------------------------------------------------------------------*/
 
 static void *
-vca_kqueue_init(void)
+vwk_init(void)
 {
 	int i;
 	struct vwk *vwk;
@@ -241,8 +241,8 @@ vca_kqueue_init(void)
 
 struct waiter waiter_kqueue = {
 	.name =		"kqueue",
-	.init =		vca_kqueue_init,
-	.pass =		vca_kqueue_pass,
+	.init =		vwk_init,
+	.pass =		vwk_pass,
 };
 
 #endif /* defined(HAVE_KQUEUE) */



From geoff at varnish-cache.org  Mon Jan  9 20:51:49 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:51:49 +0100
Subject: [experimental-ims] b190a57 Allocate VTLA VWE to the Epoll Waiter and
	make it an object
Message-ID: <E1RkMC5-0006ri-T6@project.varnish-software.com>

commit b190a57e1a3ca8f60ec7c726a80a257147b8e891
Author: Poul-Henning Kamp <phk at varnish-cache.org>
Date:   Sat Sep 17 13:10:36 2011 +0200

    Allocate VTLA VWE to the Epoll Waiter and make it an object

diff --git a/bin/varnishd/cache_waiter_epoll.c b/bin/varnishd/cache_waiter_epoll.c
index c405bfc..05c8bc2 100644
--- a/bin/varnishd/cache_waiter_epoll.c
+++ b/bin/varnishd/cache_waiter_epoll.c
@@ -53,15 +53,21 @@
 
 #define NEEV	100
 
-static pthread_t vca_epoll_thread;
-static pthread_t vca_epoll_timeout_thread;;
-static int epfd = -1;
+struct vwe {
+	unsigned		magic;
+#define VWE_MAGIC		0x6bd73424
 
-static VTAILQ_HEAD(,sess) sesshead = VTAILQ_HEAD_INITIALIZER(sesshead);
-int dotimer_pipe[2];
+	pthread_t		epoll_thread;
+	pthread_t		timer_thread;
+	int			epfd;
+
+	VTAILQ_HEAD(,sess)	sesshead;
+	int			pipes[2];
+	int			timer_pipes[2];
+};
 
 static void
-vca_modadd(int fd, void *data, short arm)
+vwe_modadd(struct vwe *vwe, int fd, void *data, short arm)
 {
 
 	/* XXX: EPOLLET (edge triggered) can cause rather Bad Things to
@@ -69,55 +75,55 @@ vca_modadd(int fd, void *data, short arm)
 	 * XXX: will hang. See #644.
 	 */
 	assert(fd >= 0);
-	if (data == vca_pipes || data == dotimer_pipe) {
+	if (data == vwe->pipes || data == vwe->timer_pipes) {
 		struct epoll_event ev = {
 		    EPOLLIN | EPOLLPRI , { data }
 		};
-		AZ(epoll_ctl(epfd, arm, fd, &ev));
+		AZ(epoll_ctl(vwe->epfd, arm, fd, &ev));
 	} else {
 		struct sess *sp = (struct sess *)data;
 		CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 		sp->ev.data.ptr = data;
 		sp->ev.events = EPOLLIN | EPOLLPRI | EPOLLONESHOT | EPOLLRDHUP;
-		AZ(epoll_ctl(epfd, arm, fd, &sp->ev));
+		AZ(epoll_ctl(vwe->epfd, arm, fd, &sp->ev));
 	}
 }
 
 static void
-vca_cond_modadd(int fd, void *data)
+vwe_cond_modadd(struct vwe *vwe, int fd, void *data)
 {
 	struct sess *sp = (struct sess *)data;
 
 	assert(fd >= 0);
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 	if (sp->ev.data.ptr)
-		AZ(epoll_ctl(epfd, EPOLL_CTL_MOD, fd, &sp->ev));
+		AZ(epoll_ctl(vwe->epfd, EPOLL_CTL_MOD, fd, &sp->ev));
 	else {
 		sp->ev.data.ptr = data;
 		sp->ev.events = EPOLLIN | EPOLLPRI | EPOLLONESHOT | EPOLLRDHUP;
-		AZ(epoll_ctl(epfd, EPOLL_CTL_ADD, fd, &sp->ev));
+		AZ(epoll_ctl(vwe->epfd, EPOLL_CTL_ADD, fd, &sp->ev));
 	}
 }
 
 static void
-vca_eev(const struct epoll_event *ep)
+vwe_eev(struct vwe *vwe, const struct epoll_event *ep)
 {
 	struct sess *ss[NEEV], *sp;
 	int i, j;
 
 	AN(ep->data.ptr);
-	if (ep->data.ptr == vca_pipes) {
+	if (ep->data.ptr == vwe->pipes) {
 		if (ep->events & EPOLLIN || ep->events & EPOLLPRI) {
 			j = 0;
-			i = read(vca_pipes[0], ss, sizeof ss);
+			i = read(vwe->pipes[0], ss, sizeof ss);
 			if (i == -1 && errno == EAGAIN)
 				return;
 			while (i >= sizeof ss[0]) {
 				CHECK_OBJ_NOTNULL(ss[j], SESS_MAGIC);
 				assert(ss[j]->fd >= 0);
 				AZ(ss[j]->obj);
-				VTAILQ_INSERT_TAIL(&sesshead, ss[j], list);
-				vca_cond_modadd(ss[j]->fd, ss[j]);
+				VTAILQ_INSERT_TAIL(&vwe->sesshead, ss[j], list);
+				vwe_cond_modadd(vwe, ss[j]->fd, ss[j]);
 				j++;
 				i -= sizeof ss[0];
 			}
@@ -128,21 +134,21 @@ vca_eev(const struct epoll_event *ep)
 		if (ep->events & EPOLLIN || ep->events & EPOLLPRI) {
 			i = HTC_Rx(sp->htc);
 			if (i == 0) {
-				vca_modadd(sp->fd, sp, EPOLL_CTL_MOD);
+				vwe_modadd(vwe, sp->fd, sp, EPOLL_CTL_MOD);
 				return;	/* more needed */
 			}
-			VTAILQ_REMOVE(&sesshead, sp, list);
+			VTAILQ_REMOVE(&vwe->sesshead, sp, list);
 			vca_handover(sp, i);
 		} else if (ep->events & EPOLLERR) {
-			VTAILQ_REMOVE(&sesshead, sp, list);
+			VTAILQ_REMOVE(&vwe->sesshead, sp, list);
 			vca_close_session(sp, "ERR");
 			SES_Delete(sp);
 		} else if (ep->events & EPOLLHUP) {
-			VTAILQ_REMOVE(&sesshead, sp, list);
+			VTAILQ_REMOVE(&vwe->sesshead, sp, list);
 			vca_close_session(sp, "HUP");
 			SES_Delete(sp);
 		} else if (ep->events & EPOLLRDHUP) {
-			VTAILQ_REMOVE(&sesshead, sp, list);
+			VTAILQ_REMOVE(&vwe->sesshead, sp, list);
 			vca_close_session(sp, "RHUP");
 			SES_Delete(sp);
 		}
@@ -152,34 +158,36 @@ vca_eev(const struct epoll_event *ep)
 /*--------------------------------------------------------------------*/
 
 static void *
-vca_main(void *arg)
+vwe_thread(void *priv)
 {
 	struct epoll_event ev[NEEV], *ep;
 	struct sess *sp;
 	char junk;
 	double deadline;
 	int dotimer, i, n;
+	struct vwe *vwe;
+
+	CAST_OBJ_NOTNULL(vwe, priv, VWE_MAGIC);
 
 	THR_SetName("cache-epoll");
-	(void)arg;
 
-	epfd = epoll_create(1);
-	assert(epfd >= 0);
+	vwe->epfd = epoll_create(1);
+	assert(vwe->epfd >= 0);
 
-	vca_modadd(vca_pipes[0], vca_pipes, EPOLL_CTL_ADD);
-	vca_modadd(dotimer_pipe[0], dotimer_pipe, EPOLL_CTL_ADD);
+	vwe_modadd(vwe, vwe->pipes[0], vwe->pipes, EPOLL_CTL_ADD);
+	vwe_modadd(vwe, vwe->timer_pipes[0], vwe->timer_pipes, EPOLL_CTL_ADD);
 
 	while (1) {
 		dotimer = 0;
-		n = epoll_wait(epfd, ev, NEEV, -1);
+		n = epoll_wait(vwe->epfd, ev, NEEV, -1);
 		for (ep = ev, i = 0; i < n; i++, ep++) {
-			if (ep->data.ptr == dotimer_pipe &&
+			if (ep->data.ptr == vwe->timer_pipes &&
 			    (ep->events == EPOLLIN || ep->events == EPOLLPRI))
 			{
-				assert(read(dotimer_pipe[0], &junk, 1));
+				assert(read(vwe->timer_pipes[0], &junk, 1));
 				dotimer = 1;
 			} else
-				vca_eev(ep);
+				vwe_eev(vwe, ep);
 		}
 		if (!dotimer)
 			continue;
@@ -187,12 +195,12 @@ vca_main(void *arg)
 		/* check for timeouts */
 		deadline = TIM_real() - params->sess_timeout;
 		for (;;) {
-			sp = VTAILQ_FIRST(&sesshead);
+			sp = VTAILQ_FIRST(&vwe->sesshead);
 			if (sp == NULL)
 				break;
 			if (sp->t_open > deadline)
 				break;
-			VTAILQ_REMOVE(&sesshead, sp, list);
+			VTAILQ_REMOVE(&vwe->sesshead, sp, list);
 			// XXX: not yet VTCP_linger(sp->fd, 0);
 			vca_close_session(sp, "timeout");
 			SES_Delete(sp);
@@ -204,16 +212,17 @@ vca_main(void *arg)
 /*--------------------------------------------------------------------*/
 
 static void *
-vca_sess_timeout_ticker(void *arg)
+vwe_sess_timeout_ticker(void *priv)
 {
 	char ticker = 'R';
+	struct vwe *vwe;
 
+	CAST_OBJ_NOTNULL(vwe, priv, VWE_MAGIC);
 	THR_SetName("cache-epoll-sess_timeout_ticker");
-	(void)arg;
 
 	while (1) {
 		/* ticking */
-		assert(write(dotimer_pipe[1], &ticker, 1));
+		assert(write(vwe->timer_pipes[1], &ticker, 1));
 		TIM_sleep(100 * 1e-3);
 	}
 	return NULL;
@@ -222,45 +231,52 @@ vca_sess_timeout_ticker(void *arg)
 /*--------------------------------------------------------------------*/
 
 static void
-vca_epoll_pass(void *priv, const struct sess *sp)
+vwe_pass(void *priv, const struct sess *sp)
 {
+	struct vwe *vwe;
 
-	(void)priv;
-	assert(sizeof sp == write(vca_pipes[1], &sp, sizeof sp));
+	CAST_OBJ_NOTNULL(vwe, priv, VWE_MAGIC);
+	assert(sizeof sp == write(vwe->pipes[1], &sp, sizeof sp));
 }
 
 /*--------------------------------------------------------------------*/
 
 static void *
-vca_epoll_init(void)
+vwe_init(void)
 {
 	int i;
+	struct vwe *vwe;
+
+	ALLOC_OBJ(vwe, VWE_MAGIC);
+	AN(vwe);
+	VTAILQ_INIT(&vwe->sesshead);
+	AZ(pipe(vwe->pipes));
+	AZ(pipe(vwe->timer_pipes));
 
-	i = fcntl(vca_pipes[0], F_GETFL);
+	i = fcntl(vwe->pipes[0], F_GETFL);
 	assert(i != -1);
 	i |= O_NONBLOCK;
-	i = fcntl(vca_pipes[0], F_SETFL, i);
+	i = fcntl(vwe->pipes[0], F_SETFL, i);
 	assert(i != -1);
 
-	AZ(pipe(dotimer_pipe));
-	i = fcntl(dotimer_pipe[0], F_GETFL);
+	i = fcntl(vwe->timer_pipes[0], F_GETFL);
 	assert(i != -1);
 	i |= O_NONBLOCK;
-	i = fcntl(dotimer_pipe[0], F_SETFL, i);
+	i = fcntl(vwe->timer_pipes[0], F_SETFL, i);
 	assert(i != -1);
 
-	AZ(pthread_create(&vca_epoll_timeout_thread,
-	    NULL, vca_sess_timeout_ticker, NULL));
-	AZ(pthread_create(&vca_epoll_thread, NULL, vca_main, NULL));
-	return(NULL);
+	AZ(pthread_create(&vwe->timer_thread,
+	    NULL, vwe_sess_timeout_ticker, vwe));
+	AZ(pthread_create(&vwe->epoll_thread, NULL, vwe_thread, vwe));
+	return(vwe);
 }
 
 /*--------------------------------------------------------------------*/
 
 struct waiter waiter_epoll = {
 	.name =		"epoll",
-	.init =		vca_epoll_init,
-	.pass =		vca_epoll_pass,
+	.init =		vwe_init,
+	.pass =		vwe_pass,
 };
 
 #endif /* defined(HAVE_EPOLL_CTL) */



From geoff at varnish-cache.org  Mon Jan  9 20:51:54 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:51:54 +0100
Subject: [experimental-ims] 012200a Eliminate the global vca_pipes.
Message-ID: <E1RkMCA-0006rv-Ve@project.varnish-software.com>

commit 012200abc78909d6c5b511a39895659897eab09d
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sat Sep 17 11:43:39 2011 +0000

    Eliminate the global vca_pipes.

diff --git a/bin/varnishd/cache_acceptor.c b/bin/varnishd/cache_acceptor.c
index 2afc975..f7147cc 100644
--- a/bin/varnishd/cache_acceptor.c
+++ b/bin/varnishd/cache_acceptor.c
@@ -95,8 +95,6 @@ static const struct linger linger = {
 
 static unsigned char	need_sndtimeo, need_rcvtimeo, need_linger, need_test;
 
-int vca_pipes[2] = { -1, -1 };
-
 static void
 sock_test(int fd)
 {
@@ -407,7 +405,6 @@ ccf_start(struct cli *cli, const char * const *av, void *priv)
 	AN(vca_act->init);
 	AN(vca_act->pass);
 
-	AZ(pipe(vca_pipes)); 	/* XXX */
 	waiter_priv = vca_act->init();
 	AZ(pthread_create(&VCA_thread, NULL, vca_acct, NULL));
 	VSL(SLT_Debug, 0, "Acceptor is %s", vca_act->name);
diff --git a/bin/varnishd/cache_waiter.h b/bin/varnishd/cache_waiter.h
index 9bf139c..ab54fe7 100644
--- a/bin/varnishd/cache_waiter.h
+++ b/bin/varnishd/cache_waiter.h
@@ -33,8 +33,6 @@ struct sess;
 typedef void* waiter_init_f(void);
 typedef void waiter_pass_f(void *priv, const struct sess *);
 
-extern int vca_pipes[2];
-
 struct waiter {
 	const char		*name;
 	waiter_init_f		*init;



From geoff at varnish-cache.org  Mon Jan  9 20:51:55 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:51:55 +0100
Subject: [experimental-ims] 74a0b31 Attempt to convert to object (VTLA = VWS)
Message-ID: <E1RkMCB-0006xS-2m@project.varnish-software.com>

commit 74a0b3112158ac4eea3b74893bde55c1b4808200
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sat Sep 17 12:43:22 2011 +0000

    Attempt to convert to object (VTLA = VWS)

diff --git a/bin/varnishd/cache_waiter_ports.c b/bin/varnishd/cache_waiter_ports.c
index 6a587cc..d607b8a 100644
--- a/bin/varnishd/cache_waiter_ports.c
+++ b/bin/varnishd/cache_waiter_ports.c
@@ -47,44 +47,48 @@
 #include "cache_waiter.h"
 
 #define MAX_EVENTS 256
-static pthread_t vca_ports_thread;
-int solaris_dport = -1;
 
-static VTAILQ_HEAD(,sess) sesshead = VTAILQ_HEAD_INITIALIZER(sesshead);
+struct vws {
+	unsigned		magic;
+#define VWS_MAGIC		0x0b771473
+	pthread_t		ports_thread;
+	int			dport;
+	VTAILQ_HEAD(,sess)	sesshead;
+};
 
 static inline void
-vca_add(int fd, void *data)
+vws_add(struct vws *vws, int fd, void *data)
 {
 	/*
 	 * POLLIN should be all we need here
 	 *
 	 */
-	AZ(port_associate(solaris_dport, PORT_SOURCE_FD, fd, POLLIN, data));
+	AZ(port_associate(vws->dport, PORT_SOURCE_FD, fd, POLLIN, data));
 }
 
 static inline void
-vca_del(int fd)
+vws_del(struct vws *vws, int fd)
 {
-	port_dissociate(solaris_dport, PORT_SOURCE_FD, fd);
+	port_dissociate(vws->dport, PORT_SOURCE_FD, fd);
 }
 
 static inline void
-vca_port_ev(port_event_t *ev) {
+vws_port_ev(struct vws *vws, port_event_t *ev) {
 	struct sess *sp;
 	if(ev->portev_source == PORT_SOURCE_USER) {
 		CAST_OBJ_NOTNULL(sp, ev->portev_user, SESS_MAGIC);
 		assert(sp->fd >= 0);
 		AZ(sp->obj);
-		VTAILQ_INSERT_TAIL(&sesshead, sp, list);
-		vca_add(sp->fd, sp);
+		VTAILQ_INSERT_TAIL(&vws->sesshead, sp, list);
+		vws_add(vw, sp->fd, sp);
 	} else {
 		int i;
 		assert(ev->portev_source == PORT_SOURCE_FD);
 		CAST_OBJ_NOTNULL(sp, ev->portev_user, SESS_MAGIC);
 		assert(sp->fd >= 0);
 		if(ev->portev_events & POLLERR) {
-			vca_del(sp->fd);
-			VTAILQ_REMOVE(&sesshead, sp, list);
+			vws_del(vws->sp->fd);
+			VTAILQ_REMOVE(&vws->sesshead, sp, list);
 			vca_close_session(sp, "EOF");
 			SES_Delete(sp);
 			return;
@@ -93,7 +97,7 @@ vca_port_ev(port_event_t *ev) {
 
 		if (i == 0) {
 			/* incomplete header, wait for more data */
-			vca_add(sp->fd, sp);
+			vws_add(sp->fd, sp);
 			return;
 		}
 
@@ -109,8 +113,8 @@ vca_port_ev(port_event_t *ev) {
 		 *
 		 * Ref: http://opensolaris.org/jive/thread.jspa?threadID=129476&tstart=0
 		 */
-		vca_del(sp->fd);
-		VTAILQ_REMOVE(&sesshead, sp, list);
+		vws_del(vws->sp->fd);
+		VTAILQ_REMOVE(&vws->sesshead, sp, list);
 
 		/* vca_handover will also handle errors */
 		vca_handover(sp, i);
@@ -119,10 +123,12 @@ vca_port_ev(port_event_t *ev) {
 }
 
 static void *
-vca_main(void *arg)
+vws_thread(void *priv)
 {
 	struct sess *sp;
+	struct vws *vws;
 
+	CAST_OBJ_NOTNULL(vwp, priv, VWP_MAGIC);
 	/*
 	 * timeouts:
 	 *
@@ -147,13 +153,12 @@ vca_main(void *arg)
 	static struct timespec max_ts = {1L, 0L};		/* 1 second */
 	static double	       max_t  = 1.0;			/* 1 second */
 
+	/* XXX: These should probably go in vws ? */
 	struct timespec ts;
 	struct timespec *timeout;
 
-	(void)arg;
-
-	solaris_dport = port_create();
-	assert(solaris_dport >= 0);
+	vws->dport = port_create();
+	assert(vws->dport >= 0);
 
 	timeout = &max_ts;
 
@@ -184,14 +189,13 @@ vca_main(void *arg)
 		 *
 		 */
 
-		ret = port_getn(solaris_dport, ev, MAX_EVENTS, &nevents, timeout);
+		ret = port_getn(vws->dport, ev, MAX_EVENTS, &nevents, timeout);
 
 		if (ret < 0)
 			assert((errno == EINTR) || (errno == ETIME));
 
-		for (ei=0; ei<nevents; ei++) {
-			vca_port_ev(ev + ei);
-		}
+		for (ei = 0; ei < nevents; ei++)
+			vws_port_ev(vws, ev + ei);
 
 		/* check for timeouts */
 		now = TIM_real();
@@ -205,15 +209,15 @@ vca_main(void *arg)
 		 */
 
 		for (;;) {
-			sp = VTAILQ_FIRST(&sesshead);
+			sp = VTAILQ_FIRST(&vws->sesshead);
 			if (sp == NULL)
 				break;
 			if (sp->t_open > deadline) {
 				break;
 			}
-			VTAILQ_REMOVE(&sesshead, sp, list);
+			VTAILQ_REMOVE(&vws->sesshead, sp, list);
 			if(sp->fd != -1) {
-				vca_del(sp->fd);
+				vws_del(vws, sp->fd);
 			}
 			vca_close_session(sp, "timeout");
 			SES_Delete(sp);
@@ -246,12 +250,13 @@ vca_main(void *arg)
 /*--------------------------------------------------------------------*/
 
 static void
-vca_ports_pass(void *priv, const struct sess *sp)
+vws_pass(void *priv, const struct sess *sp)
 {
 	int r;
+	struct vws *vws;
 
-	(void)priv;
-	while((r = port_send(solaris_dport, 0, sp)) == -1 &&
+	CAST_OBJ_NOTNULL(vws, priv, VWS_MAGIC);
+	while((r = port_send(vws->dport, 0, TRUST_ME(sp))) == -1 &&
 		errno == EAGAIN);
 	AZ(r);
 }
@@ -259,19 +264,23 @@ vca_ports_pass(void *priv, const struct sess *sp)
 /*--------------------------------------------------------------------*/
 
 static void *
-vca_ports_init(void)
+vws_init(void)
 {
+	struct vws *vws;
 
-	AZ(pthread_create(&vca_ports_thread, NULL, vca_main, NULL));
-	return (NULL);
+	ALLOC_OBJ(vws, VWS_MAGIC);
+	AN(vws);
+	VTAILQ_INIT(&vws->sesshead);
+	AZ(pthread_create(&vws_ports_thread, NULL, vws_thread, vws));
+	return (vws);
 }
 
 /*--------------------------------------------------------------------*/
 
 struct waiter waiter_ports = {
 	.name =		"ports",
-	.init =		vca_ports_init,
-	.pass =		vca_ports_pass
+	.init =		vws_init,
+	.pass =		vws_pass
 };
 
 #endif /* defined(HAVE_PORT_CREATE) */



From geoff at varnish-cache.org  Mon Jan  9 20:51:55 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:51:55 +0100
Subject: [experimental-ims] 0ad2c29 Move vca_handover() to SES_Handle() Move
	vca_close_session() to SES_Close() If SES_Delete() gets a
	reason, it calls SES_Close()
Message-ID: <E1RkMCB-0006xi-6u@project.varnish-software.com>

commit 0ad2c29e9716f357f1000e40198aa3bbaf2b0a19
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sat Sep 17 13:19:19 2011 +0000

    Move vca_handover() to SES_Handle()
    Move vca_close_session() to SES_Close()
    If SES_Delete() gets a reason, it calls SES_Close()
    
    More work on the solaris ports waiter.

diff --git a/bin/varnishd/cache.h b/bin/varnishd/cache.h
index a6b1786..2f43ff8 100644
--- a/bin/varnishd/cache.h
+++ b/bin/varnishd/cache.h
@@ -646,7 +646,6 @@ struct vbc {
 
 /* cache_acceptor.c */
 void vca_return_session(struct sess *sp);
-void vca_close_session(struct sess *sp, const char *why);
 void VCA_Prep(struct sess *sp);
 void VCA_Init(void);
 void VCA_Shutdown(void);
@@ -861,7 +860,8 @@ void WRW_Sendfile(struct worker *w, int fd, off_t off, unsigned len);
 void SES_Init(void);
 struct sess *SES_New(void);
 struct sess *SES_Alloc(void);
-void SES_Delete(struct sess *sp);
+void SES_Close(struct sess *sp, const char *reason);
+void SES_Delete(struct sess *sp, const char *reason);
 void SES_Charge(struct sess *sp);
 
 /* cache_shmlog.c */
diff --git a/bin/varnishd/cache_acceptor.c b/bin/varnishd/cache_acceptor.c
index f7147cc..46f955b 100644
--- a/bin/varnishd/cache_acceptor.c
+++ b/bin/varnishd/cache_acceptor.c
@@ -330,47 +330,10 @@ vca_acct(void *arg)
 	NEEDLESS_RETURN(NULL);
 }
 
-/*--------------------------------------------------------------------*/
-
-void
-vca_handover(struct sess *sp, int status)
-{
-
-	switch (status) {
-	case -2:
-		vca_close_session(sp, "blast");
-		SES_Delete(sp);
-		break;
-	case -1:
-		vca_close_session(sp, "no request");
-		SES_Delete(sp);
-		break;
-	case 1:
-		sp->step = STP_START;
-		if (Pool_QueueSession(sp))
-			VSC_C_main->client_drop_late++;
-		break;
-	default:
-		INCOMPL();
-	}
-}
 
 /*--------------------------------------------------------------------*/
 
 void
-vca_close_session(struct sess *sp, const char *why)
-{
-	int i;
-
-	VSL(SLT_SessionClose, sp->id, "%s", why);
-	if (sp->fd >= 0) {
-		i = close(sp->fd);
-		assert(i == 0 || errno != EBADF);	/* XXX EINVAL seen */
-	}
-	sp->fd = -1;
-}
-
-void
 vca_return_session(struct sess *sp)
 {
 
@@ -383,7 +346,7 @@ vca_return_session(struct sess *sp)
 	 * acceptor thread, to reduce syscall density of the latter.
 	 */
 	if (VTCP_nonblocking(sp->fd))
-		vca_close_session(sp, "remote closed");
+		SES_Close(sp, "remote closed");
 	vca_act->pass(waiter_priv, sp);
 }
 
diff --git a/bin/varnishd/cache_center.c b/bin/varnishd/cache_center.c
index 39a2104..24b0ddc 100644
--- a/bin/varnishd/cache_center.c
+++ b/bin/varnishd/cache_center.c
@@ -116,14 +116,14 @@ cnt_wait(struct sess *sp)
 		return (0);
 	}
 	if (i == -2) {
-		vca_close_session(sp, "overflow");
+		SES_Close(sp, "overflow");
 		return (0);
 	}
 	if (i == -1 && Tlen(sp->htc->rxbuf) == 0 &&
 	    (errno == 0 || errno == ECONNRESET))
-		vca_close_session(sp, "EOF");
+		SES_Close(sp, "EOF");
 	else
-		vca_close_session(sp, "error");
+		SES_Close(sp, "error");
 	sp->step = STP_DONE;
 	return (0);
 }
@@ -364,13 +364,13 @@ cnt_done(struct sess *sp)
 		 * before we close, to get queued data transmitted.
 		 */
 		// XXX: not yet (void)VTCP_linger(sp->fd, 0);
-		vca_close_session(sp, sp->doclose);
+		SES_Close(sp, sp->doclose);
 	}
 
 	if (sp->fd < 0) {
 		sp->wrk->stats.sess_closed++;
 		sp->wrk = NULL;
-		SES_Delete(sp);
+		SES_Delete(sp, NULL);
 		return (1);
 	}
 
@@ -1483,7 +1483,7 @@ cnt_start(struct sess *sp)
 	/* If we could not even parse the request, just close */
 	if (done == 400) {
 		sp->step = STP_DONE;
-		vca_close_session(sp, "junk");
+		SES_Close(sp, "junk");
 		return (0);
 	}
 
diff --git a/bin/varnishd/cache_esi_deliver.c b/bin/varnishd/cache_esi_deliver.c
index 61555bf..ea4951f 100644
--- a/bin/varnishd/cache_esi_deliver.c
+++ b/bin/varnishd/cache_esi_deliver.c
@@ -335,8 +335,7 @@ ESI_Deliver(struct sess *sp)
 						st->ptr + off, l2,
 						obuf, sizeof obuf, &obufl);
 					if (WRW_Error(sp->wrk)) {
-						vca_close_session(sp,
-						    "remote closed");
+						SES_Close(sp, "remote closed");
 						p = e;
 						break;
 					}
@@ -388,7 +387,7 @@ ESI_Deliver(struct sess *sp)
 				obufl = 0;
 			}
 			if (WRW_Flush(sp->wrk)) {
-				vca_close_session(sp, "remote closed");
+				SES_Close(sp, "remote closed");
 				p = e;
 				break;
 			}
diff --git a/bin/varnishd/cache_pipe.c b/bin/varnishd/cache_pipe.c
index 0175937..ddb5c15 100644
--- a/bin/varnishd/cache_pipe.c
+++ b/bin/varnishd/cache_pipe.c
@@ -87,7 +87,7 @@ PipeSession(struct sess *sp)
 	i = WRW_FlushRelease(w);
 
 	if (i) {
-		vca_close_session(sp, "pipe");
+		SES_Close(sp, "pipe");
 		VDI_CloseFd(sp);
 		return;
 	}
@@ -127,6 +127,6 @@ PipeSession(struct sess *sp)
 			fds[1].fd = -1;
 		}
 	}
-	vca_close_session(sp, "pipe");
+	SES_Close(sp, "pipe");
 	VDI_CloseFd(sp);
 }
diff --git a/bin/varnishd/cache_pool.c b/bin/varnishd/cache_pool.c
index a80cd9b..92e81b1 100644
--- a/bin/varnishd/cache_pool.c
+++ b/bin/varnishd/cache_pool.c
@@ -232,7 +232,6 @@ Pool_QueueSession(struct sess *sp)
 	 * XXX: sleep whichever thread got us here
 	 */
 	sp->t_end = TIM_real();
-	vca_close_session(sp, "dropped");
 	if (sp->vcl != NULL) {
 		/*
 		 * A session parked on a busy object can come here
@@ -240,7 +239,7 @@ Pool_QueueSession(struct sess *sp)
 		 */
 		VCL_Rel(&sp->vcl);
 	}
-	SES_Delete(sp);
+	SES_Delete(sp, "dropped");
 	return (1);
 }
 
diff --git a/bin/varnishd/cache_response.c b/bin/varnishd/cache_response.c
index 73338f7..982d8fe 100644
--- a/bin/varnishd/cache_response.c
+++ b/bin/varnishd/cache_response.c
@@ -323,7 +323,7 @@ RES_WriteObj(struct sess *sp)
 		WRW_EndChunk(sp->wrk);
 
 	if (WRW_FlushRelease(sp->wrk))
-		vca_close_session(sp, "remote closed");
+		SES_Close(sp, "remote closed");
 }
 
 /*--------------------------------------------------------------------*/
@@ -423,5 +423,5 @@ RES_StreamEnd(struct sess *sp)
 	    !(sp->wrk->res_mode & RES_ESI_CHILD))
 		WRW_EndChunk(sp->wrk);
 	if (WRW_FlushRelease(sp->wrk))
-		vca_close_session(sp, "remote closed");
+		SES_Close(sp, "remote closed");
 }
diff --git a/bin/varnishd/cache_session.c b/bin/varnishd/cache_session.c
index 1ebca44..15373a4 100644
--- a/bin/varnishd/cache_session.c
+++ b/bin/varnishd/cache_session.c
@@ -48,6 +48,7 @@
 
 #include "cache.h"
 #include "cache_backend.h"
+#include "cache_waiter.h"
 
 /*--------------------------------------------------------------------*/
 
@@ -238,12 +239,55 @@ SES_Alloc(void)
 }
 
 /*--------------------------------------------------------------------
- * Recycle a session.  If the workspace has changed, deleted it,
+ * Handle a session (from waiter)
+ *
+ * Status: see HTC_Rx()
+ */
+
+void
+SES_Handle(struct sess *sp, int status)
+{
+
+	switch (status) {
+	case -2:
+		SES_Delete(sp, "blast");
+		break;
+	case -1:
+		SES_Delete(sp, "no request");
+		break;
+	case 1:
+		sp->step = STP_START;
+		if (Pool_QueueSession(sp))
+			VSC_C_main->client_drop_late++;
+		break;
+	default:
+		WRONG("Unexpected return from HTC_Rx()");
+	}
+}
+
+/*--------------------------------------------------------------------
+ * Close a sessions connection.
+ */
+
+void
+SES_Close(struct sess *sp, const char *reason)
+{
+	int i;
+
+	assert(sp->fd >= 0);
+	VSL(SLT_SessionClose, sp->id, "%s", reason);
+	i = close(sp->fd);
+	assert(i == 0 || errno != EBADF); /* XXX EINVAL seen */
+	sp->fd = -1;
+}
+
+/*--------------------------------------------------------------------
+ * (Close &) Recycle a session.  If the workspace has changed, deleted it,
  * otherwise wash it, and put it up for adoption.
  */
 
 void
-SES_Delete(struct sess *sp)
+SES_Delete(struct sess *sp, const char *reason)
 {
 	struct acct *b = &sp->acct_ses;
 	struct sessmem *sm;
@@ -253,6 +297,10 @@ SES_Delete(struct sess *sp)
 	sm = sp->mem;
 	CHECK_OBJ_NOTNULL(sm, SESSMEM_MAGIC);
 
+	if (reason != NULL)
+		SES_Close(sp, reason);
+	assert(sp->fd < 0);
+
 	AZ(sp->obj);
 	AZ(sp->vcl);
 	VSC_C_main->n_sess--;			/* XXX: locking ? */
diff --git a/bin/varnishd/cache_waiter.h b/bin/varnishd/cache_waiter.h
index ab54fe7..86eb2cb 100644
--- a/bin/varnishd/cache_waiter.h
+++ b/bin/varnishd/cache_waiter.h
@@ -53,5 +53,6 @@ extern struct waiter waiter_poll;
 extern struct waiter waiter_ports;
 #endif
 
-/* vca_acceptor.c */
-void vca_handover(struct sess *sp, int bad);
+/* cache_session.c */
+void SES_Handle(struct sess *sp, int status);
+
diff --git a/bin/varnishd/cache_waiter_epoll.c b/bin/varnishd/cache_waiter_epoll.c
index 05c8bc2..acb8e13 100644
--- a/bin/varnishd/cache_waiter_epoll.c
+++ b/bin/varnishd/cache_waiter_epoll.c
@@ -138,19 +138,16 @@ vwe_eev(struct vwe *vwe, const struct epoll_event *ep)
 				return;	/* more needed */
 			}
 			VTAILQ_REMOVE(&vwe->sesshead, sp, list);
-			vca_handover(sp, i);
+			SES_Handle(sp, i);
 		} else if (ep->events & EPOLLERR) {
 			VTAILQ_REMOVE(&vwe->sesshead, sp, list);
-			vca_close_session(sp, "ERR");
-			SES_Delete(sp);
+			SES_Delete(sp, "ERR");
 		} else if (ep->events & EPOLLHUP) {
 			VTAILQ_REMOVE(&vwe->sesshead, sp, list);
-			vca_close_session(sp, "HUP");
-			SES_Delete(sp);
+			SES_Delete(sp, "HUP");
 		} else if (ep->events & EPOLLRDHUP) {
 			VTAILQ_REMOVE(&vwe->sesshead, sp, list);
-			vca_close_session(sp, "RHUP");
-			SES_Delete(sp);
+			SES_Delete(sp, "RHUP");
 		}
 	}
 }
@@ -202,8 +199,7 @@ vwe_thread(void *priv)
 				break;
 			VTAILQ_REMOVE(&vwe->sesshead, sp, list);
 			// XXX: not yet VTCP_linger(sp->fd, 0);
-			vca_close_session(sp, "timeout");
-			SES_Delete(sp);
+			SES_Delete(sp, "timeout");
 		}
 	}
 	return NULL;
diff --git a/bin/varnishd/cache_waiter_kqueue.c b/bin/varnishd/cache_waiter_kqueue.c
index 8febec7..7e6b10f 100644
--- a/bin/varnishd/cache_waiter_kqueue.c
+++ b/bin/varnishd/cache_waiter_kqueue.c
@@ -125,12 +125,11 @@ vwk_kev(struct vwk *vwk, const struct kevent *kp)
 			return;	/* more needed */
 		}
 		VTAILQ_REMOVE(&vwk->sesshead, sp, list);
-		vca_handover(sp, i);
+		SES_Handle(sp, i);
 		return;
 	} else if (kp->flags & EV_EOF) {
 		VTAILQ_REMOVE(&vwk->sesshead, sp, list);
-		vca_close_session(sp, "EOF");
-		SES_Delete(sp);
+		SES_Delete(sp, "EOF");
 		return;
 	} else {
 		VSL(SLT_Debug, sp->id, "KQ: sp %p kev data %lu flags 0x%x%s",
@@ -196,8 +195,7 @@ vwk_thread(void *priv)
 				break;
 			VTAILQ_REMOVE(&vwk->sesshead, sp, list);
 			// XXX: not yet (void)VTCP_linger(sp->fd, 0);
-			vca_close_session(sp, "timeout");
-			SES_Delete(sp);
+			SES_Delete(sp, "timeout");
 		}
 	}
 }
diff --git a/bin/varnishd/cache_waiter_poll.c b/bin/varnishd/cache_waiter_poll.c
index c51776d..f44daa9 100644
--- a/bin/varnishd/cache_waiter_poll.c
+++ b/bin/varnishd/cache_waiter_poll.c
@@ -165,14 +165,13 @@ vwp_main(void *priv)
 					VTAILQ_INSERT_HEAD(&vwp->sesshead, sp, list);
 				} else {
 					vwp_unpoll(vwp, fd);
-					vca_handover(sp, i);
+					SES_Handle(sp, i);
 				}
 			} else if (sp->t_open <= deadline) {
 				VTAILQ_REMOVE(&vwp->sesshead, sp, list);
 				vwp_unpoll(vwp, fd);
 				// XXX: not yet (void)VTCP_linger(sp->fd, 0);
-				vca_close_session(sp, "timeout");
-				SES_Delete(sp);
+				SES_Delete(sp, "timeout");
 			}
 		}
 		if (v && vwp->pollfd[vwp->pipes[0]].revents) {
diff --git a/bin/varnishd/cache_waiter_ports.c b/bin/varnishd/cache_waiter_ports.c
index d607b8a..329ee23 100644
--- a/bin/varnishd/cache_waiter_ports.c
+++ b/bin/varnishd/cache_waiter_ports.c
@@ -80,24 +80,23 @@ vws_port_ev(struct vws *vws, port_event_t *ev) {
 		assert(sp->fd >= 0);
 		AZ(sp->obj);
 		VTAILQ_INSERT_TAIL(&vws->sesshead, sp, list);
-		vws_add(vw, sp->fd, sp);
+		vws_add(vws, sp->fd, sp);
 	} else {
 		int i;
 		assert(ev->portev_source == PORT_SOURCE_FD);
 		CAST_OBJ_NOTNULL(sp, ev->portev_user, SESS_MAGIC);
 		assert(sp->fd >= 0);
 		if(ev->portev_events & POLLERR) {
-			vws_del(vws->sp->fd);
+			vws_del(vws, sp->fd);
 			VTAILQ_REMOVE(&vws->sesshead, sp, list);
-			vca_close_session(sp, "EOF");
-			SES_Delete(sp);
+			SES_Delete(sp, "EOF");
 			return;
 		}
 		i = HTC_Rx(sp->htc);
 
 		if (i == 0) {
 			/* incomplete header, wait for more data */
-			vws_add(sp->fd, sp);
+			vws_add(vws, sp->fd, sp);
 			return;
 		}
 
@@ -113,11 +112,11 @@ vws_port_ev(struct vws *vws, port_event_t *ev) {
 		 *
 		 * Ref: http://opensolaris.org/jive/thread.jspa?threadID=129476&tstart=0
 		 */
-		vws_del(vws->sp->fd);
+		vws_del(vws, sp->fd);
 		VTAILQ_REMOVE(&vws->sesshead, sp, list);
 
-		/* vca_handover will also handle errors */
-		vca_handover(sp, i);
+		/* SES_Handle will also handle errors */
+		SES_Handle(sp, i);
 	}
 	return;
 }
@@ -128,7 +127,7 @@ vws_thread(void *priv)
 	struct sess *sp;
 	struct vws *vws;
 
-	CAST_OBJ_NOTNULL(vwp, priv, VWP_MAGIC);
+	CAST_OBJ_NOTNULL(vws, priv, VWS_MAGIC);
 	/*
 	 * timeouts:
 	 *
@@ -219,8 +218,7 @@ vws_thread(void *priv)
 			if(sp->fd != -1) {
 				vws_del(vws, sp->fd);
 			}
-			vca_close_session(sp, "timeout");
-			SES_Delete(sp);
+			SES_Delete(sp, "timeout");
 		}
 
 		/*
@@ -271,7 +269,7 @@ vws_init(void)
 	ALLOC_OBJ(vws, VWS_MAGIC);
 	AN(vws);
 	VTAILQ_INIT(&vws->sesshead);
-	AZ(pthread_create(&vws_ports_thread, NULL, vws_thread, vws));
+	AZ(pthread_create(&vws->ports_thread, NULL, vws_thread, vws));
 	return (vws);
 }
 
diff --git a/bin/varnishd/storage_file.c b/bin/varnishd/storage_file.c
index 45dfac4..30203f8 100644
--- a/bin/varnishd/storage_file.c
+++ b/bin/varnishd/storage_file.c
@@ -557,7 +557,6 @@ const struct stevedore smf_stevedore = {
 #ifdef INCLUDE_TEST_DRIVER
 
 void vca_flush(struct sess *sp) {}
-void vca_close_session(struct sess *sp, const char *why) {}
 
 #define N	100
 #define M	(128*1024)



From geoff at varnish-cache.org  Mon Jan  9 20:51:55 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:51:55 +0100
Subject: [experimental-ims] 12f5366 Rename struct wq to struct pool,
	and tag worker threads with it.
Message-ID: <E1RkMCB-0006yc-Gq@project.varnish-software.com>

commit 12f5366e05dd0b1cb52ceea1e75f80cd8858b642
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sat Sep 17 15:59:24 2011 +0000

    Rename struct wq to struct pool, and tag worker threads with it.

diff --git a/bin/varnishd/cache.h b/bin/varnishd/cache.h
index 4fd7086..0a566f2 100644
--- a/bin/varnishd/cache.h
+++ b/bin/varnishd/cache.h
@@ -116,6 +116,7 @@ struct SHA256Context;
 struct VSC_C_lck;
 struct waitinglist;
 struct vef_priv;
+struct pool;
 
 #define DIGEST_LEN		32
 
@@ -291,6 +292,7 @@ struct stream_ctx {
 struct worker {
 	unsigned		magic;
 #define WORKER_MAGIC		0x6391adcf
+	struct pool		*pool;
 	struct objhead		*nobjhead;
 	struct objcore		*nobjcore;
 	struct waitinglist	*nwaitinglist;
diff --git a/bin/varnishd/cache_pool.c b/bin/varnishd/cache_pool.c
index 0bbb76d..63245d6 100644
--- a/bin/varnishd/cache_pool.c
+++ b/bin/varnishd/cache_pool.c
@@ -64,9 +64,9 @@ VTAILQ_HEAD(workerhead, worker);
 
 /* Number of work requests queued in excess of worker threads available */
 
-struct wq {
+struct pool {
 	unsigned		magic;
-#define WQ_MAGIC		0x606658fa
+#define POOL_MAGIC		0x606658fa
 	struct lock		mtx;
 	struct workerhead	idle;
 	VTAILQ_HEAD(, workreq)	queue;
@@ -77,7 +77,7 @@ struct wq {
 	uintmax_t		nqueue;
 };
 
-static struct wq		**wq;
+static struct pool		**wq;
 static unsigned			nwq;
 static unsigned			queue_max;
 static unsigned			nthr_max;
@@ -90,10 +90,11 @@ static struct lock		herder_mtx;
 void
 Pool_Work_Thread(void *priv, struct worker *w)
 {
-	struct wq *qp;
+	struct pool *qp;
 	int stats_clean;
 
-	CAST_OBJ_NOTNULL(qp, priv, WQ_MAGIC);
+	CAST_OBJ_NOTNULL(qp, priv, POOL_MAGIC);
+	w->pool = qp;
 	Lck_Lock(&qp->mtx);
 	qp->nthr++;
 	stats_clean = 1;
@@ -145,6 +146,7 @@ Pool_Work_Thread(void *priv, struct worker *w)
 	}
 	qp->nthr--;
 	Lck_Unlock(&qp->mtx);
+	w->pool = NULL;
 }
 
 /*--------------------------------------------------------------------
@@ -157,7 +159,7 @@ static int
 WRK_Queue(struct workreq *wrq)
 {
 	struct worker *w;
-	struct wq *qp;
+	struct pool *qp;
 	static unsigned nq = 0;
 	unsigned onq;
 
@@ -253,7 +255,7 @@ Pool_QueueSession(struct sess *sp)
 static void
 wrk_addpools(const unsigned pools)
 {
-	struct wq **pwq, **owq;
+	struct pool **pwq, **owq;
 	unsigned u;
 
 	pwq = calloc(sizeof *pwq, pools);
@@ -266,7 +268,7 @@ wrk_addpools(const unsigned pools)
 	for (u = nwq; u < pools; u++) {
 		wq[u] = calloc(sizeof *wq[0], 1);
 		XXXAN(wq[u]);
-		wq[u]->magic = WQ_MAGIC;
+		wq[u]->magic = POOL_MAGIC;
 		Lck_New(&wq[u]->mtx, lck_wq);
 		VTAILQ_INIT(&wq[u]->queue);
 		VTAILQ_INIT(&wq[u]->idle);
@@ -280,7 +282,7 @@ wrk_addpools(const unsigned pools)
  */
 
 static void
-wrk_decimate_flock(struct wq *qp, double t_idle, struct VSC_C_main *vs)
+wrk_decimate_flock(struct pool *qp, double t_idle, struct VSC_C_main *vs)
 {
 	struct worker *w = NULL;
 
@@ -380,7 +382,7 @@ wrk_herdtimer_thread(void *priv)
  */
 
 static void
-wrk_breed_flock(struct wq *qp, const pthread_attr_t *tp_attr)
+wrk_breed_flock(struct pool *qp, const pthread_attr_t *tp_attr)
 {
 	pthread_t tp;
 
diff --git a/bin/varnishd/cache_wrk.c b/bin/varnishd/cache_wrk.c
index 66ba04b..7b731ba 100644
--- a/bin/varnishd/cache_wrk.c
+++ b/bin/varnishd/cache_wrk.c
@@ -175,6 +175,7 @@ wrk_thread_real(void *priv, unsigned shm_workspace, unsigned sess_workspace,
 	VSL(SLT_WorkThread, 0, "%p start", w);
 
 	Pool_Work_Thread(priv, w);
+	AZ(w->pool);
 
 	VSL(SLT_WorkThread, 0, "%p end", w);
 	if (w->vcl != NULL)



From geoff at varnish-cache.org  Mon Jan  9 20:51:55 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:51:55 +0100
Subject: [experimental-ims] 738cafa Drop struct workreq, we don't use it.
Message-ID: <E1RkMCB-0006yk-Jx@project.varnish-software.com>

commit 738cafa04be5cca08cbb472603974732357ce112
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sat Sep 17 16:45:07 2011 +0000

    Drop struct workreq, we don't use it.

diff --git a/bin/varnishd/cache.h b/bin/varnishd/cache.h
index 0a566f2..65d1886 100644
--- a/bin/varnishd/cache.h
+++ b/bin/varnishd/cache.h
@@ -108,7 +108,6 @@ struct objhead;
 struct objcore;
 struct busyobj;
 struct storage;
-struct workreq;
 struct vrt_backend;
 struct cli_proto;
 struct ban;
@@ -307,7 +306,7 @@ struct worker {
 	pthread_cond_t		cond;
 
 	VTAILQ_ENTRY(worker)	list;
-	struct workreq		*wrq;
+	struct sess		*sp;
 
 	struct VCL_conf		*vcl;
 
@@ -369,21 +368,6 @@ struct worker {
 	struct acct		acct_tmp;
 };
 
-/* Work Request for worker thread ------------------------------------*/
-
-/*
- * This is a worker-function.
- * XXX: typesafety is probably not worth fighting for
- */
-
-typedef void workfunc(struct worker *, void *priv);
-
-struct workreq {
-	VTAILQ_ENTRY(workreq)	list;
-	workfunc		*func;
-	void			*priv;
-};
-
 /* Storage -----------------------------------------------------------*/
 
 struct storage {
@@ -614,7 +598,7 @@ struct sess {
 	/* Various internal stuff */
 	struct sessmem		*mem;
 
-	struct workreq		workreq;
+	VTAILQ_ENTRY(sess)	poollist;
 	struct acct		acct_req;
 	struct acct		acct_ses;
 
diff --git a/bin/varnishd/cache_cli.c b/bin/varnishd/cache_cli.c
index a3d1851..dd26303 100644
--- a/bin/varnishd/cache_cli.c
+++ b/bin/varnishd/cache_cli.c
@@ -131,7 +131,6 @@ cli_debug_sizeof(struct cli *cli, const char * const *av, void *priv)
 	SZOF(struct http_conn);
 	SZOF(struct acct);
 	SZOF(struct worker);
-	SZOF(struct workreq);
 	SZOF(struct storage);
 	SZOF(struct object);
 	SZOF(struct objcore);
diff --git a/bin/varnishd/cache_pool.c b/bin/varnishd/cache_pool.c
index 63245d6..6fc7737 100644
--- a/bin/varnishd/cache_pool.c
+++ b/bin/varnishd/cache_pool.c
@@ -69,7 +69,7 @@ struct pool {
 #define POOL_MAGIC		0x606658fa
 	struct lock		mtx;
 	struct workerhead	idle;
-	VTAILQ_HEAD(, workreq)	queue;
+	VTAILQ_HEAD(, sess)	queue;
 	unsigned		nthr;
 	unsigned		lqueue;
 	unsigned		last_lqueue;
@@ -105,9 +105,9 @@ Pool_Work_Thread(void *priv, struct worker *w)
 		CHECK_OBJ_NOTNULL(w, WORKER_MAGIC);
 
 		/* Process queued requests, if any */
-		w->wrq = VTAILQ_FIRST(&qp->queue);
-		if (w->wrq != NULL) {
-			VTAILQ_REMOVE(&qp->queue, w->wrq, list);
+		w->sp = VTAILQ_FIRST(&qp->queue);
+		if (w->sp != NULL) {
+			VTAILQ_REMOVE(&qp->queue, w->sp, poollist);
 			qp->lqueue--;
 		} else {
 			if (isnan(w->lastused))
@@ -117,17 +117,21 @@ Pool_Work_Thread(void *priv, struct worker *w)
 				WRK_SumStat(w);
 			Lck_CondWait(&w->cond, &qp->mtx);
 		}
-		if (w->wrq == NULL)
+		if (w->sp == NULL)
 			break;
 		Lck_Unlock(&qp->mtx);
 		stats_clean = 0;
-		AN(w->wrq);
-		AN(w->wrq->func);
 		w->lastused = NAN;
 		WS_Reset(w->ws, NULL);
 		w->storage_hint = NULL;
 
-		w->wrq->func(w, w->wrq->priv);
+		AZ(w->sp->wrk);
+		THR_SetSession(w->sp);
+		w->sp->wrk = w;
+		CHECK_OBJ_ORNULL(w->nobjhead, OBJHEAD_MAGIC);
+		CNT_Session(w->sp);
+		CHECK_OBJ_ORNULL(w->nobjhead, OBJHEAD_MAGIC);
+		THR_SetSession(NULL);
 
 		WS_Assert(w->ws);
 		AZ(w->bereq->ws);
@@ -136,7 +140,7 @@ Pool_Work_Thread(void *priv, struct worker *w)
 		AZ(w->wrw.wfd);
 		AZ(w->storage_hint);
 		assert(w->wlp == w->wlb);
-		w->wrq = NULL;
+		w->sp = NULL;
 		if (params->diag_bitmap & 0x00040000) {
 			if (w->vcl != NULL)
 				VCL_Rel(&w->vcl);
@@ -156,7 +160,7 @@ Pool_Work_Thread(void *priv, struct worker *w)
  */
 
 static int
-WRK_Queue(struct workreq *wrq)
+WRK_Queue(struct sess *sp)
 {
 	struct worker *w;
 	struct pool *qp;
@@ -181,7 +185,7 @@ WRK_Queue(struct workreq *wrq)
 	if (w != NULL) {
 		VTAILQ_REMOVE(&qp->idle, w, list);
 		Lck_Unlock(&qp->mtx);
-		w->wrq = wrq;
+		w->sp = sp;
 		AZ(pthread_cond_signal(&w->cond));
 		return (0);
 	}
@@ -193,7 +197,7 @@ WRK_Queue(struct workreq *wrq)
 		return (-1);
 	}
 
-	VTAILQ_INSERT_TAIL(&qp->queue, wrq, list);
+	VTAILQ_INSERT_TAIL(&qp->queue, sp, poollist);
 	qp->nqueue++;
 	qp->lqueue++;
 	Lck_Unlock(&qp->mtx);
@@ -203,31 +207,12 @@ WRK_Queue(struct workreq *wrq)
 
 /*--------------------------------------------------------------------*/
 
-static void
-wrk_do_cnt_sess(struct worker *w, void *priv)
-{
-	struct sess *sess;
-
-	CAST_OBJ_NOTNULL(sess, priv, SESS_MAGIC);
-	AZ(sess->wrk);
-	THR_SetSession(sess);
-	sess->wrk = w;
-	CHECK_OBJ_ORNULL(w->nobjhead, OBJHEAD_MAGIC);
-	CNT_Session(sess);
-	CHECK_OBJ_ORNULL(w->nobjhead, OBJHEAD_MAGIC);
-	THR_SetSession(NULL);
-}
-
-/*--------------------------------------------------------------------*/
-
 int
 Pool_QueueSession(struct sess *sp)
 {
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 	AZ(sp->wrk);
-	sp->workreq.func = wrk_do_cnt_sess;
-	sp->workreq.priv = sp;
-	if (WRK_Queue(&sp->workreq) == 0)
+	if (WRK_Queue(sp) == 0)
 		return (0);
 
 	/*
@@ -303,7 +288,7 @@ wrk_decimate_flock(struct pool *qp, double t_idle, struct VSC_C_main *vs)
 
 	/* And give it a kiss on the cheek... */
 	if (w != NULL) {
-		AZ(w->wrq);
+		AZ(w->sp);
 		AZ(pthread_cond_signal(&w->cond));
 		TIM_sleep(params->wthread_purge_delay * 1e-3);
 	}



From geoff at varnish-cache.org  Mon Jan  9 20:51:56 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:51:56 +0100
Subject: [experimental-ims] 6810e7c First cleanup sweep over new per-pool
	acceptor code:
Message-ID: <E1RkMCC-00071B-K9@project.varnish-software.com>

commit 6810e7c339f46aadd5b1e6676f99663cc4b2ccd5
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sun Sep 18 10:46:08 2011 +0000

    First cleanup sweep over new per-pool acceptor code:
    
    Store the data from accept(2) on the worker->ws, we only need it for
    a brief moment until the worker configures a session with it.
    
    If we fail to allocate a session after accepting the connection,
    we silently close the connection again:  Presumably it is a DoS
    situation.  (false positive:  Extremely popular busy objects)
    
    Make three clearly defined counters for sessions:  Accepted, Dropped
    and failed (accept(2) failure).
    
    Lots of spit&polish.

diff --git a/bin/varnishd/cache.h b/bin/varnishd/cache.h
index 2a7578d..d5e4311 100644
--- a/bin/varnishd/cache.h
+++ b/bin/varnishd/cache.h
@@ -289,6 +289,20 @@ struct stream_ctx {
 };
 
 /*--------------------------------------------------------------------*/
+
+struct wrk_accept {
+	unsigned		magic;
+#define WRK_ACCEPT_MAGIC	0x8c4b4d59
+
+	/* Accept stuff */
+	struct sockaddr_storage	acceptaddr;
+	socklen_t		acceptaddrlen;
+	int			acceptsock;
+	struct listen_sock	*acceptlsock;
+};
+
+/*--------------------------------------------------------------------*/
+
 struct worker {
 	unsigned		magic;
 #define WORKER_MAGIC		0x6391adcf
@@ -303,12 +317,6 @@ struct worker {
 	/* Pool stuff */
 	double			lastused;
 
-	/* Accept stuff */
-	struct sockaddr_storage	acceptaddr;
-	socklen_t		acceptaddrlen;
-	int			acceptsock;
-	struct listen_sock	*acceptlsock;
-
 	struct wrw		wrw;
 
 	pthread_cond_t		cond;
@@ -642,8 +650,9 @@ struct vbc {
 void VCA_Prep(struct sess *sp);
 void VCA_Init(void);
 void VCA_Shutdown(void);
-int VCA_Accept(int sock, socklen_t *slp, struct sockaddr_storage *sap);
+int VCA_Accept(struct listen_sock *ls, struct wrk_accept *wa);
 void VCA_SetupSess(struct worker *w);
+void VCA_FailSess(struct worker *w);
 
 /* cache_backend.c */
 void VBE_UseHealth(const struct director *vdi);
diff --git a/bin/varnishd/cache_acceptor.c b/bin/varnishd/cache_acceptor.c
index 50cc824..ba4e84a 100644
--- a/bin/varnishd/cache_acceptor.c
+++ b/bin/varnishd/cache_acceptor.c
@@ -167,7 +167,7 @@ vca_pace_check(void)
 {
 	double p;
 
-	if (vca_pace == 0.0) 
+	if (vca_pace == 0.0)
 		return;
 	Lck_Lock(&pace_mtx);
 	p = vca_pace;
@@ -191,7 +191,7 @@ static void
 vca_pace_good(void)
 {
 
-	if (vca_pace == 0.0) 
+	if (vca_pace == 0.0)
 		return;
 	Lck_Lock(&pace_mtx);
 	vca_pace *= params->acceptor_sleep_decay;
@@ -207,69 +207,89 @@ vca_pace_good(void)
 static int hack_ready;
 
 int
-VCA_Accept(int sock, socklen_t *slp, struct sockaddr_storage *sap)
+VCA_Accept(struct listen_sock *ls, struct wrk_accept *wa)
 {
 	int i;
 
-	assert(sock >= 0);
+	CHECK_OBJ_NOTNULL(ls, LISTEN_SOCK_MAGIC);
+	assert(ls->sock >= 0);
 	vca_pace_check();
 
 	while(!hack_ready)
 		(void)usleep(100*1000);
 
-	*slp = sizeof *sap;
-	i = accept(sock, (void*)sap, slp);
+	wa->acceptaddrlen = sizeof wa->acceptaddr;
+	i = accept(ls->sock, (void*)&wa->acceptaddr, &wa->acceptaddrlen);
 
 	if (i < 0) {
-		VSC_C_main->accept_fail++;
 		switch (errno) {
-		case EAGAIN:
 		case ECONNABORTED:
 			break;
 		case EMFILE:
-			VSL(SLT_Debug, sock, "Too many open files");
+			VSL(SLT_Debug, ls->sock, "Too many open files");
 			vca_pace_bad();
 			break;
 		default:
-			VSL(SLT_Debug, sock, "Accept failed: %s",
+			VSL(SLT_Debug, ls->sock, "Accept failed: %s",
 			    strerror(errno));
 			vca_pace_bad();
 			break;
 		}
 	}
+	wa->acceptlsock = ls;
+	wa->acceptsock = i;
 	return (i);
 }
 
+/*--------------------------------------------------------------------
+ * Fail a session
+ *
+ * This happens if we accept the socket, but cannot get a session
+ * structure.
+ *
+ * We consider this a DoS situation (false positive:  Extremely popular
+ * busy objects) and silently close the connection with minimum effort
+ * and fuzz, rather than try to send an intelligent message back.
+ */
+
+void
+VCA_FailSess(struct worker *w)
+{
+	struct wrk_accept *wa;
+
+	CHECK_OBJ_NOTNULL(w, WORKER_MAGIC);
+	CAST_OBJ_NOTNULL(wa, (void*)w->ws->f, WRK_ACCEPT_MAGIC);
+	AZ(w->sp);
+	AZ(close(wa->acceptsock));
+	w->stats.sess_drop++;
+	vca_pace_bad();
+}
+
 /*--------------------------------------------------------------------*/
 
 void
 VCA_SetupSess(struct worker *w)
 {
 	struct sess *sp;
+	struct wrk_accept *wa;
 
+	CHECK_OBJ_NOTNULL(w, WORKER_MAGIC);
+	CAST_OBJ_NOTNULL(wa, (void*)w->ws->f, WRK_ACCEPT_MAGIC);
 	sp = w->sp;
-	if (sp == NULL) {
-		AZ(close(w->acceptsock));
-		w->acceptsock = -1;
-		VSC_C_main->client_drop++;
-		/* XXX: 50x Reply ? */
-		vca_pace_bad();
-		INCOMPL();
-	}
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
-	sp->fd = w->acceptsock;
-	sp->id = w->acceptsock;
-	w->acceptsock = -1;
+	sp->fd = wa->acceptsock;
+	sp->id = wa->acceptsock;
+	wa->acceptsock = -1;
 	sp->t_open = TIM_real();
 	sp->t_end = sp->t_end;
-	sp->mylsock = w->acceptlsock;
-	assert(w->acceptaddrlen <= sp->sockaddrlen);
-	memcpy(&sp->sockaddr, &w->acceptaddr, w->acceptaddrlen);
-	sp->sockaddrlen = w->acceptaddrlen;
+	sp->mylsock = wa->acceptlsock;
+	CHECK_OBJ_NOTNULL(sp->mylsock, LISTEN_SOCK_MAGIC);
+	assert(wa->acceptaddrlen <= sp->sockaddrlen);
+	memcpy(&sp->sockaddr, &wa->acceptaddr, wa->acceptaddrlen);
+	sp->sockaddrlen = wa->acceptaddrlen;
 	sp->step = STP_FIRST;
 	vca_pace_good();
-	w->sp = sp;
-	w->stats.client_conn++;
+	w->stats.sess_conn++;
 }
 
 /*--------------------------------------------------------------------*/
diff --git a/bin/varnishd/cache_center.c b/bin/varnishd/cache_center.c
index db51927..122bccf 100644
--- a/bin/varnishd/cache_center.c
+++ b/bin/varnishd/cache_center.c
@@ -1115,7 +1115,7 @@ cnt_lookup(struct sess *sp)
 			WS_ReleaseP(sp->ws, (void*)sp->vary_l);
 		} else {
 			AZ(oc->busyobj->vary);
-			WS_Release(sp->ws, 0); 
+			WS_Release(sp->ws, 0);
 		}
 		sp->vary_b = NULL;
 		sp->vary_l = NULL;
diff --git a/bin/varnishd/cache_pool.c b/bin/varnishd/cache_pool.c
index 1bfc6b4..64aad05 100644
--- a/bin/varnishd/cache_pool.c
+++ b/bin/varnishd/cache_pool.c
@@ -67,7 +67,6 @@ struct poolsock {
 #define POOLSOCK_MAGIC			0x1b0a2d38
 	VTAILQ_ENTRY(poolsock)		list;
 	struct listen_sock		*lsock;
-	int				sock;
 };
 
 /* Number of work requests queued in excess of worker threads available */
@@ -95,121 +94,163 @@ static unsigned			nthr_max;
 static pthread_cond_t		herder_cond;
 static struct lock		herder_mtx;
 
-/*--------------------------------------------------------------------*/
+/*--------------------------------------------------------------------
+ * Nobody is accepting on this socket, so we do.
+ *
+ * As long as we can stick the accepted connection to another thread
+ * we do so, otherwise we return and handle it ourselves.
+ *
+ * Notice calling convention:  Called locked and returns locked, but
+ * works lock in the meantime.
+ *
+ * We store data about the accept in reserved workspace, it is only used
+ * for a brief moment and it takes up around 144 bytes.
+ */
 
 static void
 pool_accept(struct pool *pp, struct worker *w, const struct poolsock *ps)
 {
 	struct worker *w2;
+	struct wrk_accept *wa, *wa2;
 
 	CHECK_OBJ_NOTNULL(pp, POOL_MAGIC);
 	CHECK_OBJ_NOTNULL(w, WORKER_MAGIC);
 	CHECK_OBJ_NOTNULL(ps, POOLSOCK_MAGIC);
 
-	assert(ps->sock >= 0);
+	CHECK_OBJ_NOTNULL(ps->lsock, LISTEN_SOCK_MAGIC);
 	Lck_AssertHeld(&pp->mtx);
 	Lck_Unlock(&pp->mtx);
+	assert(sizeof *wa == WS_Reserve(w->ws, sizeof *wa));
+	wa = (void*)w->ws->f;
 	while (1) {
-		w->acceptsock =
-		    VCA_Accept(ps->sock, &w->acceptaddrlen, &w->acceptaddr);
-		if (w->acceptsock == -1)
+		memset(wa, 0, sizeof *wa);
+		wa->magic = WRK_ACCEPT_MAGIC;
+
+		if (VCA_Accept(ps->lsock, wa) < 0) {
+			w->stats.sess_fail++;
+			/* We're going to pace in vca anyway... */
+			(void)WRK_TrySumStat(w);
 			continue;
-		w->acceptlsock = ps->lsock;
+		}
+
 		Lck_Lock(&pp->mtx);
 		if (VTAILQ_EMPTY(&pp->idle))
 			return;
 		w2 = VTAILQ_FIRST(&pp->idle);
 		VTAILQ_REMOVE(&pp->idle, w2, list);
 		Lck_Unlock(&pp->mtx);
-		w2->acceptaddr = w->acceptaddr;
-		w2->acceptaddrlen = w->acceptaddrlen;
-		w2->acceptsock = w->acceptsock;
-		w2->acceptlsock = w->acceptlsock;
+		assert(sizeof *wa2 == WS_Reserve(w2->ws, sizeof *wa2));
+		wa2 = (void*)w2->ws->f;
+		memcpy(wa2, wa, sizeof *wa);
 		AZ(pthread_cond_signal(&w2->cond));
 	}
 }
 
-/*--------------------------------------------------------------------*/
+/*--------------------------------------------------------------------
+ * This is the work function for worker threads in the pool.
+ */
 
 void
 Pool_Work_Thread(void *priv, struct worker *w)
 {
-	struct pool *qp;
+	struct pool *pp;
 	int stats_clean;
 	struct poolsock *ps;
 
-	CAST_OBJ_NOTNULL(qp, priv, POOL_MAGIC);
-	w->pool = qp;
-	Lck_Lock(&qp->mtx);
-	qp->nthr++;
+	CAST_OBJ_NOTNULL(pp, priv, POOL_MAGIC);
+	w->pool = pp;
+	Lck_Lock(&pp->mtx);
+	pp->nthr++;
 	stats_clean = 1;
 	while (1) {
 
-		Lck_AssertHeld(&qp->mtx);
+		Lck_AssertHeld(&pp->mtx);
 
+		CHECK_OBJ_NOTNULL(w, WORKER_MAGIC);
 		CHECK_OBJ_NOTNULL(w->bereq, HTTP_MAGIC);
 		CHECK_OBJ_NOTNULL(w->beresp, HTTP_MAGIC);
 		CHECK_OBJ_NOTNULL(w->resp, HTTP_MAGIC);
-		CHECK_OBJ_NOTNULL(w, WORKER_MAGIC);
 
-		/* Process queued requests, if any */
-		w->sp = VTAILQ_FIRST(&qp->queue);
+		WS_Reset(w->ws, NULL);
+
+		w->sp = VTAILQ_FIRST(&pp->queue);
 		if (w->sp != NULL) {
-			VTAILQ_REMOVE(&qp->queue, w->sp, poollist);
-			qp->lqueue--;
-		} else if (VTAILQ_EMPTY(&qp->socks)) {
+			/* Process queued requests, if any */
+			assert(pp->lqueue > 0);
+			VTAILQ_REMOVE(&pp->queue, w->sp, poollist);
+			pp->lqueue--;
+		} else if (!VTAILQ_EMPTY(&pp->socks)) {
+			/* Accept on a socket */
+			ps = VTAILQ_FIRST(&pp->socks);
+			VTAILQ_REMOVE(&pp->socks, ps, list);
+			pool_accept(pp, w, ps);
+			Lck_AssertHeld(&pp->mtx);
+			VTAILQ_INSERT_TAIL(&pp->socks, ps, list);
+		} else if (VTAILQ_EMPTY(&pp->socks)) {
+			/* Nothing to do: To sleep, perchance to dream ... */
 			if (isnan(w->lastused))
 				w->lastused = TIM_real();
-			VTAILQ_INSERT_HEAD(&qp->idle, w, list);
+			VTAILQ_INSERT_HEAD(&pp->idle, w, list);
 			if (!stats_clean)
 				WRK_SumStat(w);
-			Lck_CondWait(&w->cond, &qp->mtx);
-		} else {
-			ps = VTAILQ_FIRST(&qp->socks);
-			VTAILQ_REMOVE(&qp->socks, ps, list);
-			pool_accept(qp, w, ps);
-			Lck_AssertHeld(&qp->mtx);
-			VTAILQ_INSERT_TAIL(&qp->socks, ps, list);
+			Lck_CondWait(&w->cond, &pp->mtx);
 		}
-		if (w->sp == NULL && w->acceptsock == -1)
+
+		/*
+		 * If we got neither session or accepted a socket, we were
+		 * woken up to die to cull the herd.
+		 */
+		if (w->sp == NULL && w->ws->r == NULL)
 			break;
-		Lck_Unlock(&qp->mtx);
+
+		Lck_Unlock(&pp->mtx);
+
 		if (w->sp == NULL) {
-			w->sp = SES_New(w, qp->sesspool);
-			VCA_SetupSess(w);
+			/* Turn accepted socket into a session */
+			assert(w->ws->r != NULL);
+			w->sp = SES_New(w, pp->sesspool);
+			if (w->sp == NULL)
+				VCA_FailSess(w);
+			else
+				VCA_SetupSess(w);
+			WS_Release(w->ws, 0);
 		}
-		AN(w->sp);
-		assert(w->acceptsock == -1);
-		stats_clean = 0;
-		w->lastused = NAN;
-		WS_Reset(w->ws, NULL);
-		w->storage_hint = NULL;
-
-		AZ(w->sp->wrk);
-		THR_SetSession(w->sp);
-		w->sp->wrk = w;
-		CHECK_OBJ_ORNULL(w->nobjhead, OBJHEAD_MAGIC);
-		CNT_Session(w->sp);
-		CHECK_OBJ_ORNULL(w->nobjhead, OBJHEAD_MAGIC);
-		THR_SetSession(NULL);
-
-		WS_Assert(w->ws);
-		AZ(w->bereq->ws);
-		AZ(w->beresp->ws);
-		AZ(w->resp->ws);
-		AZ(w->wrw.wfd);
-		AZ(w->storage_hint);
-		assert(w->wlp == w->wlb);
-		w->sp = NULL;
-		if (params->diag_bitmap & 0x00040000) {
-			if (w->vcl != NULL)
-				VCL_Rel(&w->vcl);
+		assert(w->ws->r == NULL);
+
+		if (w->sp != NULL) {
+			CHECK_OBJ_NOTNULL(w->sp, SESS_MAGIC);
+
+			stats_clean = 0;
+			w->lastused = NAN;
+			w->storage_hint = NULL;
+
+			AZ(w->sp->wrk);
+			THR_SetSession(w->sp);
+			w->sp->wrk = w;
+			CHECK_OBJ_ORNULL(w->nobjhead, OBJHEAD_MAGIC);
+			CNT_Session(w->sp);
+			CHECK_OBJ_ORNULL(w->nobjhead, OBJHEAD_MAGIC);
+			THR_SetSession(NULL);
+			w->sp = NULL;
+
+			WS_Assert(w->ws);
+			AZ(w->bereq->ws);
+			AZ(w->beresp->ws);
+			AZ(w->resp->ws);
+			AZ(w->wrw.wfd);
+			AZ(w->storage_hint);
+			assert(w->wlp == w->wlb);
+			if (params->diag_bitmap & 0x00040000) {
+				if (w->vcl != NULL)
+					VCL_Rel(&w->vcl);
+			}
 		}
 		stats_clean = WRK_TrySumStat(w);
-		Lck_Lock(&qp->mtx);
+		Lck_Lock(&pp->mtx);
 	}
-	qp->nthr--;
-	Lck_Unlock(&qp->mtx);
+	assert(pp->nthr > 0);
+	pp->nthr--;
+	Lck_Unlock(&pp->mtx);
 	w->pool = NULL;
 }
 
@@ -320,7 +361,6 @@ pool_mkpool(void)
 			continue;
 		ALLOC_OBJ(ps, POOLSOCK_MAGIC);
 		XXXAN(ps);
-		ps->sock = ls->sock;
 		ps->lsock = ls;
 		VTAILQ_INSERT_TAIL(&pp->socks, ps, list);
 	}
diff --git a/bin/varnishd/cache_vrt_vmod.c b/bin/varnishd/cache_vrt_vmod.c
index bd2e172..557b01b 100644
--- a/bin/varnishd/cache_vrt_vmod.c
+++ b/bin/varnishd/cache_vrt_vmod.c
@@ -81,7 +81,7 @@ VRT_Vmod_Init(void **hdl, void *ptr, int len, const char *nm,
 		if (v->hdl == NULL) {
 			VCLI_Out(cli, "Loading VMOD %s from %s:\n", nm, path);
 			VCLI_Out(cli, "dlopen() failed: %s\n", dlerror());
-			VCLI_Out(cli, "Check child process permissions.\n"); 
+			VCLI_Out(cli, "Check child process permissions.\n");
 			FREE_OBJ(v);
 			return (1);
 		}
diff --git a/bin/varnishd/cache_waiter_kqueue.c b/bin/varnishd/cache_waiter_kqueue.c
index 7decfea..6e2cb05 100644
--- a/bin/varnishd/cache_waiter_kqueue.c
+++ b/bin/varnishd/cache_waiter_kqueue.c
@@ -52,9 +52,9 @@
 struct vwk {
 	unsigned		magic;
 #define VWK_MAGIC		0x1cc2acc2
-	pthread_t 		thread;
+	pthread_t		thread;
 	int			pipes[2];
-	int 			kq;
+	int			kq;
 	struct kevent		ki[NKEV];
 	unsigned		nki;
 	VTAILQ_HEAD(,sess)	sesshead;
diff --git a/bin/varnishd/cache_wrk.c b/bin/varnishd/cache_wrk.c
index 3cfaba0..8e88e1d 100644
--- a/bin/varnishd/cache_wrk.c
+++ b/bin/varnishd/cache_wrk.c
@@ -79,7 +79,7 @@ WRK_SumStat(struct worker *w)
 int
 WRK_TrySumStat(struct worker *w)
 {
-	if (Lck_Trylock(&wstat_mtx)) 
+	if (Lck_Trylock(&wstat_mtx))
 		return (0);
 	wrk_sumstat(w);
 	Lck_Unlock(&wstat_mtx);
@@ -165,7 +165,6 @@ wrk_thread_real(void *priv, unsigned shm_workspace, unsigned sess_workspace,
 	w->bereq = HTTP_create(http0, nhttp);
 	w->beresp = HTTP_create(http1, nhttp);
 	w->resp = HTTP_create(http2, nhttp);
-	w->acceptsock = -1;
 	w->wrw.iov = iov;
 	w->wrw.siov = siov;
 	w->wrw.ciov = siov;
diff --git a/bin/varnishtest/tests/b00000.vtc b/bin/varnishtest/tests/b00000.vtc
index 1f4f766..2018235 100644
--- a/bin/varnishtest/tests/b00000.vtc
+++ b/bin/varnishtest/tests/b00000.vtc
@@ -10,7 +10,7 @@ varnish v1 -storage "-smalloc,1m" -vcl+backend {} -start
 varnish v1 -cliok "param.set diag_bitmap 0x2"
 
 varnish v1 -expect n_object == 0
-varnish v1 -expect client_conn == 0
+varnish v1 -expect sess_conn == 0
 varnish v1 -expect client_req == 0
 varnish v1 -expect cache_miss == 0
 
@@ -21,7 +21,7 @@ client c1 {
 } -run
 
 varnish v1 -expect n_object == 1
-varnish v1 -expect client_conn == 1
+varnish v1 -expect sess_conn == 1
 varnish v1 -expect client_req == 1
 varnish v1 -expect cache_miss == 1
 varnish v1 -expect s_sess == 1
diff --git a/bin/varnishtest/tests/b00001.vtc b/bin/varnishtest/tests/b00001.vtc
index 41dbe63..edec601 100644
--- a/bin/varnishtest/tests/b00001.vtc
+++ b/bin/varnishtest/tests/b00001.vtc
@@ -18,7 +18,7 @@ client c1 {
 } -run
 
 varnish v1 -expect n_object == 0
-varnish v1 -expect client_conn == 1
+varnish v1 -expect sess_conn == 1
 varnish v1 -expect client_req == 1
 varnish v1 -expect s_sess == 1
 varnish v1 -expect s_req == 1
diff --git a/bin/varnishtest/tests/b00002.vtc b/bin/varnishtest/tests/b00002.vtc
index 54a7ab7..8bb7ad3 100644
--- a/bin/varnishtest/tests/b00002.vtc
+++ b/bin/varnishtest/tests/b00002.vtc
@@ -22,7 +22,7 @@ delay .1
 
 varnish v1 -expect n_object == 0
 varnish v1 -expect SMA.Transient.g_alloc == 0
-varnish v1 -expect client_conn == 1
+varnish v1 -expect sess_conn == 1
 varnish v1 -expect client_req == 1
 varnish v1 -expect s_sess == 1
 varnish v1 -expect s_req == 1
diff --git a/bin/varnishtest/tests/b00003.vtc b/bin/varnishtest/tests/b00003.vtc
index 8b9375d..8c2bf00 100644
--- a/bin/varnishtest/tests/b00003.vtc
+++ b/bin/varnishtest/tests/b00003.vtc
@@ -24,7 +24,7 @@ client c2 {
 # Give varnish a chance to update stats
 delay .1
 
-varnish v1 -expect client_conn == 2
+varnish v1 -expect sess_conn == 2
 varnish v1 -expect cache_hit == 1
 varnish v1 -expect cache_miss == 1
 varnish v1 -expect client_req == 2
diff --git a/bin/varnishtest/tests/c00020.vtc b/bin/varnishtest/tests/c00020.vtc
index 31875b2..7fc707c 100644
--- a/bin/varnishtest/tests/c00020.vtc
+++ b/bin/varnishtest/tests/c00020.vtc
@@ -50,7 +50,7 @@ client c2 {
         expect resp.http.X-Varnish == "1006 1003"
 } -run
 
-varnish v1 -expect client_conn == 3
+varnish v1 -expect sess_conn == 3
 varnish v1 -expect cache_hit == 3
 varnish v1 -expect cache_miss == 3
 varnish v1 -expect client_req == 6
diff --git a/bin/varnishtest/tests/c00023.vtc b/bin/varnishtest/tests/c00023.vtc
index 6bed8e3..8c65eaa 100644
--- a/bin/varnishtest/tests/c00023.vtc
+++ b/bin/varnishtest/tests/c00023.vtc
@@ -148,7 +148,7 @@ client c1 {
 
 varnish v1 -cliok "hcb.dump"
 
-varnish v1 -expect client_conn == 2
+varnish v1 -expect sess_conn == 2
 varnish v1 -expect cache_hit == 9
 varnish v1 -expect cache_miss == 9
 varnish v1 -expect client_req == 18
diff --git a/include/vsc_fields.h b/include/vsc_fields.h
index 6f910de..fc81919 100644
--- a/include/vsc_fields.h
+++ b/include/vsc_fields.h
@@ -32,7 +32,7 @@
  *    n - Name:		Field name, in C-source and stats programs
  *    t - Type:		C-type, uint64_t, unless marked in 'f'
  *    l - Local:	Local counter in worker thread.
- *    f - Format: 	Semantics of the value in this field
+ *    f - Format:	Semantics of the value in this field
  *				'a' - Accumulator (deprecated, use 'c')
  *				'b' - Bitmap
  *				'c' - Counter, never decreases.
@@ -40,7 +40,7 @@
  *				'i' - Integer (deprecated, use 'g')
  *    e - Explantion:	Short explanation of field (for screen use)
  *    d - Description:	Long explanation of field (for doc use)
- *	
+ *
  * -----------------------
  * NB: Cleanup in progress
  * -----------------------
@@ -57,9 +57,30 @@
 
 #ifdef VSC_DO_MAIN
 
-VSC_F(client_conn,		uint64_t, 1, 'a', "Client connections accepted", "")
-VSC_F(client_drop,		uint64_t, 0, 'a',
-					"Connection dropped, no sess/wrk", "")
+/*---------------------------------------------------------------------
+ * Sessions
+ *    see: cache_acceptor.c and cache_pool.c
+ */
+
+VSC_F(sess_conn,		uint64_t, 1, 'c',
+    "Sessions accepted",
+	"Count of sessions succesfully accepted"
+)
+VSC_F(sess_drop,		uint64_t, 1, 'c',
+    "Sessions dropped",
+	"Count of sessions silently dropped due to lack of session memory."
+	"  See parameter 'max_sess'."
+)
+
+VSC_F(sess_fail,		uint64_t, 1, 'c',
+    "Session accept failures",
+	"Count of failures to accept TCP connection."
+	"  Either the client changed its mind, or the kernel ran out of"
+	" some resource like filedescriptors."
+)
+
+/*---------------------------------------------------------------------*/
+
 VSC_F(client_req,		uint64_t, 1, 'a', "Client requests received", "")
 
 VSC_F(cache_hit,		uint64_t, 1, 'a', "Cache hits", "")
@@ -195,7 +216,6 @@ VSC_F(hcb_insert,		uint64_t, 0, 'a', "HCB Inserts", "")
 
 VSC_F(esi_errors,		uint64_t, 0, 'a', "ESI parse errors (unlock)", "")
 VSC_F(esi_warnings,		uint64_t, 0, 'a', "ESI parse warnings (unlock)", "")
-VSC_F(accept_fail,		uint64_t, 0, 'a', "Accept failures", "")
 VSC_F(client_drop_late,	uint64_t, 0, 'a', "Connection dropped late", "")
 VSC_F(uptime,		uint64_t, 0, 'a', "Client uptime", "")
 



From geoff at varnish-cache.org  Mon Jan  9 20:51:56 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:51:56 +0100
Subject: [experimental-ims] d885b81 Organize thread pools on a linked list
	instead of in an array.
Message-ID: <E1RkMCC-00071n-SP@project.varnish-software.com>

commit d885b8182a4a4d6dcd57d158bfbefd1bbc251496
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sun Sep 18 20:25:54 2011 +0000

    Organize thread pools on a linked list instead of in an array.

diff --git a/bin/varnishd/cache_pool.c b/bin/varnishd/cache_pool.c
index f8c6871..b593617 100644
--- a/bin/varnishd/cache_pool.c
+++ b/bin/varnishd/cache_pool.c
@@ -74,6 +74,7 @@ struct poolsock {
 struct pool {
 	unsigned		magic;
 #define POOL_MAGIC		0x606658fa
+	VTAILQ_ENTRY(pool)	list;
 	struct lock		mtx;
 	struct workerhead	idle;
 	VTAILQ_HEAD(, sess)	queue;
@@ -86,11 +87,13 @@ struct pool {
 	struct sesspool		*sesspool;
 };
 
-static struct pool		**wq;
-static unsigned			nwq;
+static VTAILQ_HEAD(,pool)	pools = VTAILQ_HEAD_INITIALIZER(pools);
+
 static unsigned			queue_max;
 static unsigned			nthr_max;
 
+static unsigned			nwq;
+
 static pthread_cond_t		herder_cond;
 static struct lock		herder_mtx;
 
@@ -349,28 +352,21 @@ pool_mkpool(void)
 		ps->lsock = ls;
 		VTAILQ_INSERT_TAIL(&pp->socks, ps, list);
 	}
+	VTAILQ_INSERT_TAIL(&pools, pp, list);
 	return (pp);
 }
 
 static void
-wrk_addpools(const unsigned pools)
+wrk_addpools(const unsigned npools)
 {
-	struct pool **pwq, **owq;
+	struct pool *pp;
 	unsigned u;
 
-	pwq = calloc(sizeof *pwq, pools);
-	if (pwq == NULL)
-		return;
-	if (wq != NULL)
-		memcpy(pwq, wq, sizeof *pwq * nwq);
-	owq = wq;
-	wq = pwq;
-	for (u = nwq; u < pools; u++) {
-		wq[u] = pool_mkpool();
-		XXXAN(wq[u]);
+	for (u = nwq; u < npools; u++) {
+		pp = pool_mkpool();
+		XXXAN(pp);
 	}
-	(void)owq;	/* XXX: avoid race, leak it. */
-	nwq = pools;
+	nwq = npools;
 }
 
 /*--------------------------------------------------------------------
@@ -422,6 +418,7 @@ wrk_herdtimer_thread(void *priv)
 	double t_idle;
 	struct VSC_C_main vsm, *vs;
 	int errno_is_multi_threaded;
+	struct pool *pp;
 
 	THR_SetName("wrk_herdtimer");
 
@@ -460,8 +457,8 @@ wrk_herdtimer_thread(void *priv)
 		vs->n_wrk_queued = 0;
 
 		t_idle = TIM_real() - params->wthread_timeout;
-		for (u = 0; u < nwq; u++)
-			wrk_decimate_flock(wq[u], t_idle, vs);
+		VTAILQ_FOREACH(pp, &pools, list)
+			wrk_decimate_flock(pp, t_idle, vs);
 
 		VSC_C_main->n_wrk= vs->n_wrk;
 		VSC_C_main->n_wrk_lqueue = vs->n_wrk_lqueue;
@@ -521,8 +518,8 @@ wrk_breed_flock(struct pool *qp, const pthread_attr_t *tp_attr)
 static void *
 wrk_herder_thread(void *priv)
 {
-	unsigned u, w;
 	pthread_attr_t tp_attr;
+	struct pool *pp, *pp2;
 
 	/* Set the stacksize for worker threads */
 	AZ(pthread_attr_init(&tp_attr));
@@ -530,19 +527,19 @@ wrk_herder_thread(void *priv)
 	THR_SetName("wrk_herder");
 	(void)priv;
 	while (1) {
-		for (u = 0 ; u < nwq; u++) {
+		VTAILQ_FOREACH(pp, &pools, list) {
 			if (params->wthread_stacksize != UINT_MAX)
 				AZ(pthread_attr_setstacksize(&tp_attr,
 				    params->wthread_stacksize));
 
-			wrk_breed_flock(wq[u], &tp_attr);
+			wrk_breed_flock(pp, &tp_attr);
 
 			/*
 			 * Make sure all pools have their minimum complement
 			 */
-			for (w = 0 ; w < nwq; w++)
-				while (wq[w]->nthr < params->wthread_min)
-					wrk_breed_flock(wq[w], &tp_attr);
+			VTAILQ_FOREACH(pp2, &pools, list)
+				while (pp2->nthr < params->wthread_min)
+					wrk_breed_flock(pp2, &tp_attr);
 			/*
 			 * We cannot avoid getting a mutex, so we have a
 			 * bogo mutex just for POSIX_STUPIDITY



From geoff at varnish-cache.org  Mon Jan  9 20:51:57 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:51:57 +0100
Subject: [experimental-ims] 8faf587 Various minor polishing
Message-ID: <E1RkMCD-000728-1x@project.varnish-software.com>

commit 8faf587064362d52586affdc36bfcb27a6c1d947
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Sep 19 08:36:14 2011 +0000

    Various minor polishing

diff --git a/bin/varnishd/cache_pool.c b/bin/varnishd/cache_pool.c
index 2aeb1b9..2d64299 100644
--- a/bin/varnishd/cache_pool.c
+++ b/bin/varnishd/cache_pool.c
@@ -259,7 +259,7 @@ Pool_Work_Thread(void *priv, struct worker *w)
  */
 
 static int
-WRK_Queue(struct pool *pp, struct sess *sp)
+pool_queue(struct pool *pp, struct sess *sp)
 {
 	struct worker *w;
 
@@ -297,7 +297,7 @@ Pool_Schedule(struct pool *pp, struct sess *sp)
 {
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 	AZ(sp->wrk);
-	if (WRK_Queue(pp, sp) == 0)
+	if (pool_queue(pp, sp) == 0)
 		return(0);
 
 	VSC_C_main->client_drop_late++;
@@ -345,7 +345,7 @@ Pool_Wait(struct sess *sp)
  */
 
 static void
-wrk_breed_flock(struct pool *qp, const pthread_attr_t *tp_attr)
+pool_breed(struct pool *qp, const pthread_attr_t *tp_attr)
 {
 	pthread_t tp;
 
@@ -418,7 +418,7 @@ pool_herder(void *priv)
 			AZ(pthread_attr_init(&tp_attr));
 		}
 
-		wrk_breed_flock(pp, &tp_attr);
+		pool_breed(pp, &tp_attr);
 		
 		if (pp->nthr < params->wthread_min)
 			continue;
@@ -466,6 +466,7 @@ pool_herder(void *priv)
 			AZ(pthread_cond_signal(&w->cond));
 		}
 	}
+	NEEDLESS_RETURN(NULL);
 }
 
 /*--------------------------------------------------------------------
@@ -543,6 +544,7 @@ pool_poolherder(void *priv)
 			u += pp->lqueue;
 		VSC_C_main->thread_queue_len = u;
 	}
+	NEEDLESS_RETURN(NULL);
 }
 
 /*--------------------------------------------------------------------*/
@@ -555,5 +557,3 @@ Pool_Init(void)
 	Lck_New(&pool_mtx, lck_wq);
 	AZ(pthread_create(&thr_pool_herder, NULL, pool_poolherder, NULL));
 }
-
-/*--------------------------------------------------------------------*/
diff --git a/bin/varnishd/cache_wrk.c b/bin/varnishd/cache_wrk.c
index 8e88e1d..0c3b18e 100644
--- a/bin/varnishd/cache_wrk.c
+++ b/bin/varnishd/cache_wrk.c
@@ -26,7 +26,7 @@
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
  *
- * Worker thread stuff unrealted to the worker thread pools.
+ * Worker thread stuff unrelated to the worker thread pools.
  */
 
 #include "config.h"



From geoff at varnish-cache.org  Mon Jan  9 20:51:57 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:51:57 +0100
Subject: [experimental-ims] f0746df OS/X does not have CLOCK_MONOTONIC,
	use CLOCK_REALTIME and hope nobody steps the clock...
Message-ID: <E1RkMCD-00072U-7U@project.varnish-software.com>

commit f0746df953ca90e11a48e10d9b6383feb63ac1f6
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Sep 19 10:04:00 2011 +0000

    OS/X does not have CLOCK_MONOTONIC, use CLOCK_REALTIME and hope nobody
    steps the clock...
    
    Fixes	#1022

diff --git a/bin/varnishd/cache_pool.c b/bin/varnishd/cache_pool.c
index 2d64299..675fea6 100644
--- a/bin/varnishd/cache_pool.c
+++ b/bin/varnishd/cache_pool.c
@@ -58,6 +58,10 @@
 #include "stevedore.h"
 #include "hash_slinger.h"
 
+#ifndef CLOCK_MONOTONIC
+#define CLOCK_MONOTONIC CLOCK_REALTIME		/* OS/X is silly */
+#endif
+
 static void *waiter_priv;
 
 VTAILQ_HEAD(workerhead, worker);



From geoff at varnish-cache.org  Mon Jan  9 20:51:57 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:51:57 +0100
Subject: [experimental-ims] e5975a5 Fix OS/X compilation for good.
Message-ID: <E1RkMCD-00072t-Da@project.varnish-software.com>

commit e5975a5f721fa91b954735b54121d54554696cba
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Sep 19 18:26:20 2011 +0000

    Fix OS/X compilation for good.
    
    Fixes	#1022

diff --git a/bin/varnishd/cache_pool.c b/bin/varnishd/cache_pool.c
index 811b233..7b79873 100644
--- a/bin/varnishd/cache_pool.c
+++ b/bin/varnishd/cache_pool.c
@@ -59,9 +59,32 @@
 #include "stevedore.h"
 #include "hash_slinger.h"
 
+/*--------------------------------------------------------------------
+ * MAC OS/X is incredibly moronic when it comes to time and such...
+ */
+
 #ifndef CLOCK_MONOTONIC
-#define CLOCK_MONOTONIC CLOCK_REALTIME		/* OS/X is silly */
-#endif
+#define CLOCK_MONOTONIC	0
+
+static int
+clock_gettime(int foo, struct timespec *ts)
+{
+	struct timeval tv;
+
+	gettimeofday(&tv, NULL);
+	ts->tv_sec = tv.tv_sec;
+	ts->tv_nsec = tv.tv_usec * 1000;
+	return (0);
+}
+
+static int
+pthread_condattr_setclock(pthread_condattr_t *attr, int foo)
+{
+	(void)attr;
+	(void)foo;
+	return (0);
+}
+#endif /* !CLOCK_MONOTONIC */
 
 static void *waiter_priv;
 



From geoff at varnish-cache.org  Mon Jan  9 20:51:57 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:51:57 +0100
Subject: [experimental-ims] 9e39eb2 Increase the servers listen depth to 10.
Message-ID: <E1RkMCD-00073B-JR@project.varnish-software.com>

commit 9e39eb25c239961513f53138d21cc1f275f8f9b3
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Sep 20 07:43:29 2011 +0000

    Increase the servers listen depth to 10.

diff --git a/bin/varnishtest/vtc_server.c b/bin/varnishtest/vtc_server.c
index 061d9cc..542a266 100644
--- a/bin/varnishtest/vtc_server.c
+++ b/bin/varnishtest/vtc_server.c
@@ -132,7 +132,7 @@ server_new(const char *name)
 	bprintf(s->listen, "127.0.0.1:%d", 0);
 	AZ(VSS_parse(s->listen, &s->addr, &s->port));
 	s->repeat = 1;
-	s->depth = 1;
+	s->depth = 10;
 	s->sock = -1;
 	VTAILQ_INSERT_TAIL(&servers, s, list);
 	return (s);



From geoff at varnish-cache.org  Mon Jan  9 20:51:57 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:51:57 +0100
Subject: [experimental-ims] c4a27e7 Work around a bug in OSX
Message-ID: <E1RkMCD-00073K-MR@project.varnish-software.com>

commit c4a27e7243249f33aeaf84ded84b007c145eaee8
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Sep 20 08:03:03 2011 +0000

    Work around a bug in OSX

diff --git a/bin/varnishtest/vtc_varnish.c b/bin/varnishtest/vtc_varnish.c
index c53482e..2ebef93 100644
--- a/bin/varnishtest/vtc_varnish.c
+++ b/bin/varnishtest/vtc_varnish.c
@@ -353,7 +353,15 @@ varnish_launch(struct varnish *v)
 	fd[0].events = POLLIN;
 	fd[1].fd = v->fds[0];
 	fd[1].events = POLLHUP;
+#ifdef __APPLE__
+	/*
+	 * OSX cannot poll a pipe for POLLHUP only, poll just returns
+	 * zero with no revents.
+	 */
+	i = poll(fd, 1, 10000);
+#else
 	i = poll(fd, 2, 10000);
+#endif
 	vtc_log(v->vl, 4, "CLIPOLL %d 0x%x 0x%x",
 	    i, fd[0].revents, fd[1].revents);
 	if (i == 0) {



From geoff at varnish-cache.org  Mon Jan  9 20:51:57 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:51:57 +0100
Subject: [experimental-ims] b99540d Fix a typo.
Message-ID: <E1RkMCD-00073n-To@project.varnish-software.com>

commit b99540df697ff4780cad40384a863f7c428e4b76
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Sep 20 08:57:46 2011 +0000

    Fix a typo.

diff --git a/bin/varnishd/cache_acceptor.c b/bin/varnishd/cache_acceptor.c
index ba4e84a..0808010 100644
--- a/bin/varnishd/cache_acceptor.c
+++ b/bin/varnishd/cache_acceptor.c
@@ -281,7 +281,7 @@ VCA_SetupSess(struct worker *w)
 	sp->id = wa->acceptsock;
 	wa->acceptsock = -1;
 	sp->t_open = TIM_real();
-	sp->t_end = sp->t_end;
+	sp->t_end = sp->t_open;
 	sp->mylsock = wa->acceptlsock;
 	CHECK_OBJ_NOTNULL(sp->mylsock, LISTEN_SOCK_MAGIC);
 	assert(wa->acceptaddrlen <= sp->sockaddrlen);



From geoff at varnish-cache.org  Mon Jan  9 20:51:57 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:51:57 +0100
Subject: [experimental-ims] d5eedbe A missing type-cast
Message-ID: <E1RkMCD-00073T-QZ@project.varnish-software.com>

commit d5eedbe40b0388cf4b8d5001b4a2ee957cfd312b
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Sep 20 08:11:29 2011 +0000

    A missing type-cast

diff --git a/bin/varnishd/storage_persistent_mgt.c b/bin/varnishd/storage_persistent_mgt.c
index c4f2191..3bd5809 100644
--- a/bin/varnishd/storage_persistent_mgt.c
+++ b/bin/varnishd/storage_persistent_mgt.c
@@ -109,7 +109,7 @@ smp_metrics(struct smp_sc *sc)
 	 */
 	sc->free_reserve = sc->aim_segl * 10;
 
-	fprintf(stderr, "free_reserve = %ju\n", sc->free_reserve);
+	fprintf(stderr, "free_reserve = %ju\n", (uintmax_t)sc->free_reserve);
 }
 
 /*--------------------------------------------------------------------



From geoff at varnish-cache.org  Mon Jan  9 20:51:58 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:51:58 +0100
Subject: [experimental-ims] 6734c86 Clear the poll structure beforehand in
	order to make sure there are no lingering bits.
Message-ID: <E1RkMCE-00073y-16@project.varnish-software.com>

commit 6734c8682f4a32c5240ccba10dbc61876a2de89c
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Sep 20 09:05:58 2011 +0000

    Clear the poll structure beforehand in order to make sure there
    are no lingering bits.
    
    Convert an assert to a failure.

diff --git a/bin/varnishtest/vtc_http.c b/bin/varnishtest/vtc_http.c
index 9c0f175..d01240b 100644
--- a/bin/varnishtest/vtc_http.c
+++ b/bin/varnishtest/vtc_http.c
@@ -899,8 +899,9 @@ cmd_http_send(CMD_ARGS)
 	AZ(av[2]);
 	vtc_dump(hp->vl, 4, "send", av[1], -1);
 	i = write(hp->fd, av[1], strlen(av[1]));
-	assert(i == strlen(av[1]));
-
+	if (i != strlen(av[1]))
+		vtc_log(hp->vl, 0, "Write error in http_send(): %s",
+		    strerror(errno));
 }
 
 /**********************************************************************
diff --git a/bin/varnishtest/vtc_varnish.c b/bin/varnishtest/vtc_varnish.c
index 2ebef93..5ef4beb 100644
--- a/bin/varnishtest/vtc_varnish.c
+++ b/bin/varnishtest/vtc_varnish.c
@@ -349,6 +349,7 @@ varnish_launch(struct varnish *v)
 	AZ(pthread_create(&v->tp_vsl, NULL, varnishlog_thread, v));
 
 	/* Wait for the varnish to call home */
+	memset(fd, 0, sizeof fd);
 	fd[0].fd = v->cli_fd;
 	fd[0].events = POLLIN;
 	fd[1].fd = v->fds[0];



From geoff at varnish-cache.org  Mon Jan  9 20:51:58 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:51:58 +0100
Subject: [experimental-ims] 8ff8091 Update zope-plone.vcl to v3 syntax
Message-ID: <E1RkMCE-00074E-4q@project.varnish-software.com>

commit 8ff8091eb4239f2e257288d02a84b67241cb3ad4
Author: Tollef Fog Heen <tfheen at varnish-software.com>
Date:   Tue Sep 20 11:03:08 2011 +0200

    Update zope-plone.vcl to v3 syntax
    
    Thanks to cleberjsantos for initial patch which this is based on.
    
    Fixes: #976

diff --git a/etc/zope-plone.vcl b/etc/zope-plone.vcl
index 8391499..b69f583 100644
--- a/etc/zope-plone.vcl
+++ b/etc/zope-plone.vcl
@@ -14,8 +14,8 @@
 
 # Default backend is the Zope CMS
 backend default {
-	set backend.host = "127.0.0.1";
-	set backend.port = "9673";
+	.host = "127.0.0.1";
+	.port = "9673";
 }
 
 acl purge {
@@ -29,10 +29,10 @@ sub vcl_recv {
 	# requests for unknown hosts
         if (req.http.host ~ "(www.)?example.com") {
                 set req.http.host = "example.com";
-                set req.url = "/VirtualHostBase/http/example.com:80/example.com/VirtualHostRoot" req.url;
+                set req.url = "/VirtualHostBase/http/example.com:80/example.com/VirtualHostRoot" + req.url;
         } elsif (req.http.host ~ "(www.)?example.org") {
                 set req.http.host = "example.org";
-                set req.url = "/VirtualHostBase/http/example.org:80/example.org/VirtualHostRoot" req.url;
+                set req.url = "/VirtualHostBase/http/example.org:80/example.org/VirtualHostRoot" + req.url;
         } else {
                 error 404 "Unknown virtual host.";
         }
@@ -42,7 +42,7 @@ sub vcl_recv {
 
                 # POST - Logins and edits
                 if (req.request == "POST") {
-                        pass;
+                        return(pass);
                 }
                 
                 # PURGE - The CacheFu product can invalidate updated URLs
@@ -50,7 +50,7 @@ sub vcl_recv {
                         if (!client.ip ~ purge) {
                                 error 405 "Not allowed.";
                         }
-                        lookup;
+                        return(lookup);
                 }
         }
 
@@ -60,9 +60,9 @@ sub vcl_recv {
 		# Force lookup of specific urls unlikely to need protection
 		if (req.url ~ "\.(js|css)") {
                         remove req.http.cookie;
-                        lookup;
+                        return(lookup);
                 }
-                pass;
+                return(pass);
         }
 
         # The default vcl_recv is used from here.
@@ -71,13 +71,14 @@ sub vcl_recv {
 # Do the PURGE thing
 sub vcl_hit {
         if (req.request == "PURGE") {
-                set obj.ttl = 0s;
+                purge;
                 error 200 "Purged";
         }
 }
 sub vcl_miss {
         if (req.request == "PURGE") {
-                error 404 "Not in cache";
+                purge;
+                error 200 "Purged";
         }
 }
 
@@ -85,7 +86,7 @@ sub vcl_miss {
 # from Zope by using the CacheFu product
 
 sub vcl_fetch {
-        if (obj.ttl < 3600s) {
-                set obj.ttl = 3600s;
+        if (beresp.ttl < 3600s) {
+                set beresp.ttl = 3600s;
         }
 }



From geoff at varnish-cache.org  Mon Jan  9 20:51:58 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:51:58 +0100
Subject: [experimental-ims] e1fea86 Increase http_req_size,
	http_resp_size to 8192 bytes
Message-ID: <E1RkMCE-00074U-8k@project.varnish-software.com>

commit e1fea86a22813af16533edcab374a1043fdb3593
Author: Tollef Fog Heen <tfheen at varnish-software.com>
Date:   Tue Sep 20 11:07:21 2011 +0200

    Increase http_req_size, http_resp_size to 8192 bytes
    
    Apache (and nginx) uses 8k, so use the same default size to avoid
    people being surprised by more 413 responses than necessary.
    
    Fixes: #1016

diff --git a/bin/varnishd/mgt_param.c b/bin/varnishd/mgt_param.c
index 96a83fc..decd130 100644
--- a/bin/varnishd/mgt_param.c
+++ b/bin/varnishd/mgt_param.c
@@ -525,7 +525,7 @@ static const struct parspec input_parspec[] = {
 		"Maximum length of any HTTP client request header we will "
 		"allow.  The limit is inclusive its continuation lines.\n",
 		0,
-		"4096", "bytes" },
+		"8192", "bytes" },
 	{ "http_req_size", tweak_uint, &master.http_req_size,
 		256, UINT_MAX,
 		"Maximum number of bytes of HTTP client request we will deal "
@@ -541,7 +541,7 @@ static const struct parspec input_parspec[] = {
 		"Maximum length of any HTTP backend response header we will "
 		"allow.  The limit is inclusive its continuation lines.\n",
 		0,
-		"4096", "bytes" },
+		"8192", "bytes" },
 	{ "http_resp_size", tweak_uint, &master.http_resp_size,
 		256, UINT_MAX,
 		"Maximum number of bytes of HTTP backend resonse we will deal "



From geoff at varnish-cache.org  Mon Jan  9 20:51:58 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:51:58 +0100
Subject: [experimental-ims] c94a809 Kqueue is go for Darwin11. Somebody should
	attempt Darwin10 also.
Message-ID: <E1RkMCE-00074p-Cm@project.varnish-software.com>

commit c94a8092e4b3b2935a80e8234d6de73602cbb60c
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Sep 20 09:27:39 2011 +0000

    Kqueue is go for Darwin11.  Somebody should attempt Darwin10 also.

diff --git a/configure.ac b/configure.ac
index 1cdaa0c..c8b61e5 100644
--- a/configure.ac
+++ b/configure.ac
@@ -332,7 +332,7 @@ AC_ARG_ENABLE(kqueue,
 
 if test "$enable_kqueue" = yes; then
 	case $target in
-	*-*-freebsd* | *-*-darwin9* | *-*-netbsd* )
+	*-*-freebsd* | *-*-darwin9* | *-*-darwin11* | *-*-netbsd* )
 		AC_CHECK_FUNCS([kqueue])
 		;;
 	*-*-bsd*)



From geoff at varnish-cache.org  Mon Jan  9 20:51:58 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:51:58 +0100
Subject: [experimental-ims] 0bc47b7 Merge branch 'master' of
	ssh://git.varnish-cache.org/git/varnish-cache
Message-ID: <E1RkMCE-000750-Fw@project.varnish-software.com>

commit 0bc47b73e62173e985390fd53d40af23a1aeabaf
Merge: c94a809 e1fea86
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Sep 20 09:28:22 2011 +0000

    Merge branch 'master' of ssh://git.varnish-cache.org/git/varnish-cache



From geoff at varnish-cache.org  Mon Jan  9 20:51:58 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:51:58 +0100
Subject: [experimental-ims] 2a8a885 More details about failing test
	subprocesses.
Message-ID: <E1RkMCE-00075Q-P5@project.varnish-software.com>

commit 2a8a885a1998cb4a31e45c61b00d8bc5017dc6c6
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Sep 20 09:40:44 2011 +0000

    More details about failing test subprocesses.

diff --git a/bin/varnishtest/vtc_main.c b/bin/varnishtest/vtc_main.c
index b280022..b189d1e 100644
--- a/bin/varnishtest/vtc_main.c
+++ b/bin/varnishtest/vtc_main.c
@@ -224,8 +224,11 @@ tst_cb(const struct vev *ve, int what)
 		free(jp->tmpdir);
 
 		if (stx) {
-			printf("#     top  TEST %s FAILED (%.3f)\n",
+			printf("#     top  TEST %s FAILED (%.3f)",
 			    jp->tst->filename, t);
+			if (WIFSIGNALED(stx))
+				printf(" signal=%d", WTERMSIG(stx));
+			printf(" exit=%d", WEXITSTATUS(stx));
 			if (!vtc_continue) {
 				/* XXX kill -9 other jobs ? */
 				exit(2);



From geoff at varnish-cache.org  Mon Jan  9 20:51:58 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:51:58 +0100
Subject: [experimental-ims] ff3d342 Wait for the health-probe to finish before
	we even start client c2,
	this should solve issues where the c2-v1 connection times out before it
	ever got used.
Message-ID: <E1RkMCE-00075p-UX@project.varnish-software.com>

commit ff3d342e34a8d779f4f8b83540248bde193d1906
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Sep 20 09:48:16 2011 +0000

    Wait for the health-probe to finish before we even start client c2,
    this should solve issues where the c2-v1 connection times out before
    it ever got used.

diff --git a/bin/varnishtest/tests/s00002.vtc b/bin/varnishtest/tests/s00002.vtc
index f7b8820..939d0ee 100644
--- a/bin/varnishtest/tests/s00002.vtc
+++ b/bin/varnishtest/tests/s00002.vtc
@@ -62,8 +62,9 @@ client c1 {
 	expect resp.status == 200
 } -run
 
+sema r1 sync 2
+
 client c2 {
-	sema r1 sync 2
 	txreq -url "/"
 	rxresp
 	expect resp.http.foo == "bar"



From geoff at varnish-cache.org  Mon Jan  9 20:51:59 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:51:59 +0100
Subject: [experimental-ims] e749a3e Make the VSL skeptical of zero-filled VSM.
Message-ID: <E1RkMCF-00076A-3m@project.varnish-software.com>

commit e749a3ee4960ad05c3fc70901a472f2e75379b92
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Sep 20 11:58:37 2011 +0000

    Make the VSL skeptical of zero-filled VSM.

diff --git a/include/vsl.h b/include/vsl.h
index 288c5c7..45406c0 100644
--- a/include/vsl.h
+++ b/include/vsl.h
@@ -66,6 +66,7 @@
  * on this file to extract the table rather than handcode it
  */
 enum VSL_tag_e {
+	SLT_Bogus = 0,
 #define SLTM(foo)	SLT_##foo,
 #include "vsl_tags.h"
 #undef SLTM
diff --git a/lib/libvarnishapi/vsl.c b/lib/libvarnishapi/vsl.c
index 6791888..848fe65 100644
--- a/lib/libvarnishapi/vsl.c
+++ b/lib/libvarnishapi/vsl.c
@@ -173,6 +173,11 @@ vsl_nextlog(struct vsl *vsl, uint32_t **pp)
 	for (w = 0; w < TIMEOUT_USEC;) {
 		t = *vsl->log_ptr;
 
+		if (t == 0) {
+			/* Zero-initialized VSL */
+			VRMB();
+			continue;
+		}
 		if (t == VSL_WRAPMARKER) {
 			/* Wrap around not possible at front */
 			assert(vsl->log_ptr != vsl->log_start + 1);



From geoff at varnish-cache.org  Mon Jan  9 20:51:59 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:51:59 +0100
Subject: [experimental-ims] bc2759d Merge branch 'master' of
	ssh://git.varnish-cache.org/git/varnish-cache
Message-ID: <E1RkMCF-00076a-9t@project.varnish-software.com>

commit bc2759d426afe6423cb94f66c6c5771378a6c177
Merge: d4455ed 8b925e3
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Sep 20 12:11:24 2011 +0000

    Merge branch 'master' of ssh://git.varnish-cache.org/git/varnish-cache



From geoff at varnish-cache.org  Mon Jan  9 20:51:59 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:51:59 +0100
Subject: [experimental-ims] f72c709 Disable this test on OSX,
	the VM mapping is too random to get us the same address again,
	even with hinting.
Message-ID: <E1RkMCF-000773-MH@project.varnish-software.com>

commit f72c7094e338c77650aaf4e00fef9c9344c2c8ae
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Sep 20 12:28:43 2011 +0000

    Disable this test on OSX, the VM mapping is too random to get us
    the same address again, even with hinting.

diff --git a/bin/varnishtest/tests/r00962.vtc b/bin/varnishtest/tests/r00962.vtc
index faf4b4b..aa0fb10 100644
--- a/bin/varnishtest/tests/r00962.vtc
+++ b/bin/varnishtest/tests/r00962.vtc
@@ -1,5 +1,8 @@
 varnishtest "Test address remapping"
 
+# VM-remapping is to random on OSX
+feature	not-OSX
+
 server s1 {
 	rxreq 
 	txresp 
diff --git a/bin/varnishtest/vtc.c b/bin/varnishtest/vtc.c
index a034fef..3bee42a 100644
--- a/bin/varnishtest/vtc.c
+++ b/bin/varnishtest/vtc.c
@@ -492,7 +492,13 @@ cmd_feature(CMD_ARGS)
 		if (sizeof(void*) == 8 && !strcmp(av[i], "64bit"))
 			continue;
 
-		vtc_log(vl, 1, "SKIPPING test, missing feature %s", av[i]);
+		if (!strcmp(av[i], "!OSX")) {
+#if !defined(__APPLE__) || !defined(__MACH__)
+			continue;
+#endif
+		}
+
+		vtc_log(vl, 1, "SKIPPING test, missing feature: %s", av[i]);
 		vtc_stop = 1;
 		return;
 	}



From geoff at varnish-cache.org  Mon Jan  9 20:51:59 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:51:59 +0100
Subject: [experimental-ims] db36b58 This cast is a tad too creative for my
	taste, make it safer.
Message-ID: <E1RkMCF-00077U-Rv@project.varnish-software.com>

commit db36b58c3237c0a63912781595516e3cf9ad36f8
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Sep 20 13:55:51 2011 +0000

    This cast is a tad too creative for my taste, make it safer.

diff --git a/bin/varnishd/cache_vrt_re.c b/bin/varnishd/cache_vrt_re.c
index 9c6cb99..8e1218a 100644
--- a/bin/varnishd/cache_vrt_re.c
+++ b/bin/varnishd/cache_vrt_re.c
@@ -59,8 +59,11 @@ VRT_re_init(void **rep, const char *re)
 void
 VRT_re_fini(void *rep)
 {
+	vre_t *vv;
+
+	vv = rep;
 	if (rep != NULL)
-		VRE_free((vre_t**)&rep);
+		VRE_free(&vv);
 }
 
 int



From geoff at varnish-cache.org  Mon Jan  9 20:52:00 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:00 +0100
Subject: [experimental-ims] 1bb41f9 Add a missing derefence.
Message-ID: <E1RkMCG-00079G-IN@project.varnish-software.com>

commit 1bb41f9b58e51adc4d5cc9f4854d12915bb2ba46
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Sep 20 21:29:03 2011 +0000

    Add a missing derefence.

diff --git a/bin/varnishtest/vtc_http.c b/bin/varnishtest/vtc_http.c
index 32c7e0c..8f96577 100644
--- a/bin/varnishtest/vtc_http.c
+++ b/bin/varnishtest/vtc_http.c
@@ -1078,7 +1078,7 @@ cmd_http_accept(CMD_ARGS)
 	CAST_OBJ_NOTNULL(hp, priv, HTTP_MAGIC);
 	AZ(av[1]);
 	assert(hp->sfd != NULL);
-	assert(hp->sfd >= 0);
+	assert(*hp->sfd >= 0);
 	VTCP_close(&hp->fd);
 	vtc_log(vl, 4, "Accepting");
 	hp->fd = accept(*hp->sfd, NULL, NULL);



From geoff at varnish-cache.org  Mon Jan  9 20:52:00 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:00 +0100
Subject: [experimental-ims] ef4c867 Cleanup sphinx warnings
Message-ID: <E1RkMCG-00079f-MM@project.varnish-software.com>

commit ef4c867ae4635362c6a01596c20113638bc5ea27
Author: Andreas Plesner Jacobsen <apj at mutt.dk>
Date:   Wed Sep 21 10:02:49 2011 +0200

    Cleanup sphinx warnings

diff --git a/doc/sphinx/conf.py.in b/doc/sphinx/conf.py.in
index d6b98ab..0b07a2b 100644
--- a/doc/sphinx/conf.py.in
+++ b/doc/sphinx/conf.py.in
@@ -64,7 +64,7 @@ release = '@VERSION@'
 
 # List of directories, relative to source directory, that shouldn't be searched
 # for source files.
-exclude_trees = ['=build']
+exclude_patterns = ['=build','reference/params.rst']
 
 # The reST default role (used for this markup: `text`) to use for all documents.
 #default_role = None
@@ -147,7 +147,7 @@ html_title = "Varnish version @VERSION@ documentation"
 # Add any paths that contain custom static files (such as style sheets) here,
 # relative to this directory. They are copied after the builtin static files,
 # so a file named "default.css" will overwrite the builtin "default.css".
-html_static_path = ['=static']
+#html_static_path = ['=static']
 
 # If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
 # using the given strftime format.
diff --git a/doc/sphinx/faq/general.rst b/doc/sphinx/faq/general.rst
index d40ef9f..b30f093 100644
--- a/doc/sphinx/faq/general.rst
+++ b/doc/sphinx/faq/general.rst
@@ -337,7 +337,7 @@ Varnish has a feature called **hit for pass**, which is used when Varnish gets a
           * Client 2..N are now given the **hit for pass** object instructing them to go to the backend
 
 The **hit for pass** object will stay cached for the duration of its ttl. This means that subsequent clients requesting /foo will be sent straight to the backend as long as the **hit for pass** object exists.
-The :command:`varnishstat` can tell you how many **hit for pass** objects varnish has served. The default vcl will set ttl for a hit_for_pass object to 120s. But you can override this, using the following logic:
+The :command:`varnishstat` can tell you how many **hit for pass** objects varnish has served. The default vcl will set ttl for a hit_for_pass object to 120s. But you can override this, using the following logic::
 
         sub vcl_fetch {
           if (!obj.cacheable) {
diff --git a/doc/sphinx/installation/upgrade.rst b/doc/sphinx/installation/upgrade.rst
index d00e9b0..eda1515 100644
--- a/doc/sphinx/installation/upgrade.rst
+++ b/doc/sphinx/installation/upgrade.rst
@@ -42,7 +42,7 @@ becomes
 
 	ban("req.url = " + req.url);
 
-``purge`` does not take any arguments anymore, but can be used in vcl_hit or vcl_miss to purge the item from the cache, where you would reduce ttl to 0 in Varnish 2.1.
+``purge`` does not take any arguments anymore, but can be used in vcl_hit or vcl_miss to purge the item from the cache, where you would reduce ttl to 0 in Varnish 2.1::
 
 	sub vcl_hit {
 	  if (req.request == "PURGE") {
@@ -51,7 +51,7 @@ becomes
 	  }
 	}
 
-becomes
+becomes::
 
 	sub vcl_hit {
 	  if (req.request == "PURGE") {
@@ -68,13 +68,13 @@ becomes
 returns are now done with the ``return()`` function
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-``pass``, ``pipe``, ``lookup``, ``deliver``, ``fetch``, ``hash``, ``pipe`` and ``restart`` are no longer keywords, but arguments to ``return()``, so
+``pass``, ``pipe``, ``lookup``, ``deliver``, ``fetch``, ``hash``, ``pipe`` and ``restart`` are no longer keywords, but arguments to ``return()``, so::
 
 	sub vcl_pass {
 	  pass;
 	}
 
-becomes
+becomes::
 
 	sub vcl_pass {
 	  return(pass);
diff --git a/doc/sphinx/reference/params.rst b/doc/sphinx/reference/params.rst
index ce7da4b..f161596 100644
--- a/doc/sphinx/reference/params.rst
+++ b/doc/sphinx/reference/params.rst
@@ -128,6 +128,7 @@ diag_bitmap
 	  0x00020000 - synchronous start of persistence.
 	  0x00040000 - release VCL early.
 	  0x80000000 - do edge-detection on digest.
+
 	Use 0x notation and do the bitor in your head :-)
 
 esi_syntax
@@ -140,6 +141,7 @@ esi_syntax
 	  0x00000002 - Ignore non-esi elements
 	  0x00000004 - Emit parsing debug records
 	  0x00000008 - Force-split parser input (debugging)
+
 	Use 0x notation and do the bitor in your head :-)
 
 expiry_sleep
@@ -203,6 +205,7 @@ gzip_tmp_space
 	  0 - malloc
 	  1 - session workspace
 	  2 - thread workspace
+
 	If you have much gzip/gunzip activity, it may be an advantage to use workspace for these allocations to reduce malloc activity.  Be aware that gzip needs 256+KB and gunzip needs 32+KB of workspace (64+KB if ESI processing).
 
 gzip_window
@@ -216,7 +219,7 @@ http_gzip_support
 	- Default: on
 	- Flags: experimental
 
-	Enable gzip support. When enabled Varnish will compress uncompressed objects before they are stored in the cache. If a client does not support gzip encoding Varnish will uncompress compressed objects on demand. Varnish will also rewrite the Accept-Encoding header of clients indicating support for gzip to::
+	Enable gzip support. When enabled Varnish will compress uncompressed objects before they are stored in the cache. If a client does not support gzip encoding Varnish will uncompress compressed objects on demand. Varnish will also rewrite the Accept-Encoding header of clients indicating support for gzip to:
 
 	Accept-Encoding: gzip
 



From geoff at varnish-cache.org  Mon Jan  9 20:52:00 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:00 +0100
Subject: [experimental-ims] 0d50550 Merge branch 'master' of
	ssh://git.varnish-cache.org/git/varnish-cache
Message-ID: <E1RkMCG-00079y-Pg@project.varnish-software.com>

commit 0d5055017657c821d1a5709176eed4dd922d6aff
Merge: 63af998 ef4c867
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Sep 21 09:55:26 2011 +0000

    Merge branch 'master' of ssh://git.varnish-cache.org/git/varnish-cache



From geoff at varnish-cache.org  Mon Jan  9 20:52:00 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:00 +0100
Subject: [experimental-ims] 5ba20ed Add varnish-counters(7) man page
Message-ID: <E1RkMCG-0007AM-Sz@project.varnish-software.com>

commit 5ba20ed333c5597c46f51c1371ae7c471f1feee3
Author: Tollef Fog Heen <tfheen at varnish-software.com>
Date:   Wed Sep 21 13:03:54 2011 +0200

    Add varnish-counters(7) man page

diff --git a/.gitignore b/.gitignore
index 6cf0950..f35e990 100644
--- a/.gitignore
+++ b/.gitignore
@@ -54,8 +54,10 @@ TAGS
 /lib/libvmod_std/vcc_if.h
 
 # Man-files and binaries
+/man/vsc2rst
 /man/vcl.7
 /man/varnish-cli.7
+/man/varnish-counters.7
 /bin/varnishadm/varnishadm
 /bin/varnishadm/varnishadm.1
 /bin/varnishd/varnishd
diff --git a/man/Makefile.am b/man/Makefile.am
index bc662b6..da95e8c 100644
--- a/man/Makefile.am
+++ b/man/Makefile.am
@@ -1,6 +1,12 @@
 #
 
-dist_man_MANS = vcl.7 varnish-cli.7
+noinst_PROGRAMS = vsc2rst
+vsc2rst_SOURCES = vsc2rst.c \
+	$(top_srcdir)/include/vsc_fields.h
+
+INCLUDES = -I$(top_srcdir)/include
+
+dist_man_MANS = vcl.7 varnish-cli.7 varnish-counters.7
 MAINTAINERCLEANFILES = $(dist_man_MANS)
 
 vcl.7: $(top_srcdir)/doc/sphinx/reference/vcl.rst \
@@ -23,3 +29,13 @@ else
 	@echo "========================================"
 	@false
 endif
+
+varnish-counters.7: vsc2rst
+if HAVE_RST2MAN
+	./vsc2rst | ${RST2MAN} - $@
+else
+	@echo "========================================"
+	@echo "You need rst2man installed to make dist"
+	@echo "========================================"
+	@false
+endif
diff --git a/man/vsc2rst.c b/man/vsc2rst.c
new file mode 100644
index 0000000..9b89cce
--- /dev/null
+++ b/man/vsc2rst.c
@@ -0,0 +1,65 @@
+
+#include <stdio.h>
+
+#define P(x, ...) printf(x "\n", ##__VA_ARGS__)
+#define VSC_F(n, t, l, f, e, d) printf("%s ? %s\n\t%s\n\n", #n, e, d);
+
+int main(int argc, char **argv)
+{
+	P("================");
+	P("varnish-counters");
+	P("================");
+	P("");
+
+	P("---------------------------------");
+	P("Varnish counter field definitions");
+	P("---------------------------------");
+
+	P(":Author: Tollef Fog Heen");
+	P(":Date:   2011-09-20");
+	P(":Version: 1.0");
+	P(":Manual section: 7");
+	P("");
+
+	P("MAIN COUNTERS");
+	P("=============");
+	P("");
+#define VSC_DO_MAIN
+#include "vsc_fields.h"
+#undef VSC_DO_MAIN
+
+	P("");
+	P("LOCK COUNTERS");
+	P("=============");
+	P("");
+#define VSC_DO_LCK
+#include "vsc_fields.h"
+#undef VSC_DO_LCK
+
+	P("");
+	P("PER MALLOC STORAGE COUNTERS");
+	P("===========================");
+	P("");
+#define VSC_DO_SMA
+#include "vsc_fields.h"
+#undef  VSC_DO_SMA
+
+	P("");
+	P("PER FILE STORAGE COUNTERS");
+	P("=========================");
+	P("");
+#define VSC_DO_SMF
+#include "vsc_fields.h"
+#undef VSC_DO_SMF
+
+	P("");
+	P("PER BACKEND COUNTERS");
+	P("====================");
+	P("");
+#define VSC_DO_VBE
+#include "vsc_fields.h"
+#undef VSC_DO_VBE
+
+	return 0;
+}
+



From geoff at varnish-cache.org  Mon Jan  9 20:51:55 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:51:55 +0100
Subject: [experimental-ims] ac1fae5 Screw up the session allocator performance,
	in order to prepare it for per-pool operation.
Message-ID: <E1RkMCB-0006zV-TF@project.varnish-software.com>

commit ac1fae53f4b5c66db6db5d7e3ade181b2ab6dc50
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sat Sep 17 20:06:36 2011 +0000

    Screw up the session allocator performance, in order to prepare it
    for per-pool operation.

diff --git a/bin/varnishd/cache.h b/bin/varnishd/cache.h
index 6c0c525..71cf6b3 100644
--- a/bin/varnishd/cache.h
+++ b/bin/varnishd/cache.h
@@ -108,6 +108,7 @@ struct objhead;
 struct objcore;
 struct busyobj;
 struct storage;
+struct sesspool;
 struct vrt_backend;
 struct cli_proto;
 struct ban;
@@ -849,7 +850,7 @@ void WRW_Sendfile(struct worker *w, int fd, off_t off, unsigned len);
 
 /* cache_session.c [SES] */
 void SES_Init(void);
-struct sess *SES_New(void);
+struct sess *SES_New(struct sesspool *pp);
 struct sess *SES_Alloc(void);
 void SES_Close(struct sess *sp, const char *reason);
 void SES_Delete(struct sess *sp, const char *reason);
diff --git a/bin/varnishd/cache_acceptor.c b/bin/varnishd/cache_acceptor.c
index 0022037..37cc857 100644
--- a/bin/varnishd/cache_acceptor.c
+++ b/bin/varnishd/cache_acceptor.c
@@ -316,7 +316,7 @@ vca_acct(void *arg)
 			i = VCA_Accept(ls->sock, &l, &addr_s);
 			if (i < 0) 
 				continue;
-			sp = SES_New();
+			sp = SES_New(NULL);
 			if (sp == NULL) {
 				AZ(close(i));
 				VSC_C_main->client_drop++;
diff --git a/bin/varnishd/cache_session.c b/bin/varnishd/cache_session.c
index 15373a4..6b34928 100644
--- a/bin/varnishd/cache_session.c
+++ b/bin/varnishd/cache_session.c
@@ -26,16 +26,8 @@
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
  *
- * Session and Client management.
+ * Session management
  *
- * XXX: The two-list session management is actually not a good idea
- * XXX: come to think of it, because we want the sessions reused in
- * XXX: Most Recently Used order.
- * XXX: Another and maybe more interesting option would be to cache
- * XXX: free sessions in the worker threads and postpone session
- * XXX: allocation until then.  This does not quite implment MRU order
- * XXX: but it does save some locking, although not that much because
- * XXX: we still have to do the source-addr lookup.
  */
 
 #include "config.h"
@@ -56,6 +48,7 @@ struct sessmem {
 	unsigned		magic;
 #define SESSMEM_MAGIC		0x555859c5
 
+	struct sesspool		*pool;
 	struct sess		sess;
 	unsigned		workspace;
 	void			*wsp;
@@ -64,13 +57,16 @@ struct sessmem {
 	struct sockaddr_storage	sockaddr[2];
 };
 
-static VTAILQ_HEAD(,sessmem)	ses_free_mem[2] = {
-    VTAILQ_HEAD_INITIALIZER(ses_free_mem[0]),
-    VTAILQ_HEAD_INITIALIZER(ses_free_mem[1]),
+struct sesspool {
+	unsigned		magic;
+#define SESSPOOL_MAGIC		0xd916e202
+	VTAILQ_HEAD(,sessmem)	freelist;
+	struct lock		mtx;
+	unsigned		nsess;
+	unsigned		maxsess;
 };
 
-static unsigned ses_qp;
-static struct lock		ses_mem_mtx;
+static struct sesspool *sesspool;
 
 /*--------------------------------------------------------------------*/
 
@@ -106,8 +102,6 @@ ses_sm_alloc(void)
 	uint16_t nhttp;
 	unsigned l, hl;
 
-	if (VSC_C_main->n_sess_mem >= params->max_sess)
-		return (NULL);
 	/*
 	 * It is not necessary to lock these, but we need to
 	 * cache them locally, to make sure we get a consistent
@@ -115,6 +109,7 @@ ses_sm_alloc(void)
 	 */
 	nws = params->sess_workspace;
 	nhttp = (uint16_t)params->http_max_hdr;
+
 	hl = HTTP_estimate(nhttp);
 	l = sizeof *sm + nws + 2 * hl;
 	p = malloc(l);
@@ -122,6 +117,7 @@ ses_sm_alloc(void)
 		return (NULL);
 	q = p + l;
 
+	/* XXX Stats */
 	Lck_Lock(&stat_mtx);
 	VSC_C_main->n_sess_mem++;
 	Lck_Unlock(&stat_mtx);
@@ -153,7 +149,6 @@ ses_setup(struct sessmem *sm)
 {
 	struct sess *sp;
 
-
 	CHECK_OBJ_NOTNULL(sm, SESSMEM_MAGIC);
 	sp = &sm->sess;
 	memset(sp, 0, sizeof *sp);
@@ -184,39 +179,38 @@ ses_setup(struct sessmem *sm)
  */
 
 struct sess *
-SES_New(void)
+SES_New(struct sesspool *pp)
 {
 	struct sessmem *sm;
 	struct sess *sp;
+	int do_alloc = 0;
 
-	assert(pthread_self() == VCA_thread);
-	assert(ses_qp <= 1);
-	sm = VTAILQ_FIRST(&ses_free_mem[ses_qp]);
-	if (sm == NULL) {
-		/*
-		 * If that queue is empty, flip queues holding the lock
-		 * and try the new unlocked queue.
-		 */
-		Lck_Lock(&ses_mem_mtx);
-		ses_qp = 1 - ses_qp;
-		Lck_Unlock(&ses_mem_mtx);
-		sm = VTAILQ_FIRST(&ses_free_mem[ses_qp]);
-	}
+	if (pp == NULL)
+		pp = sesspool;
+	CHECK_OBJ_NOTNULL(pp, SESSPOOL_MAGIC);
+
+	Lck_Lock(&pp->mtx);
+	sm = VTAILQ_FIRST(&pp->freelist);
 	if (sm != NULL) {
-		VTAILQ_REMOVE(&ses_free_mem[ses_qp], sm, list);
-		sp = &sm->sess;
-		CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
-	} else {
+		VTAILQ_REMOVE(&pp->freelist, sm, list);
+	} else if (pp->nsess < pp->maxsess) {
+		pp->nsess++;
+		do_alloc = 1;
+	}
+	Lck_Unlock(&pp->mtx);
+	if (do_alloc) {
 		sm = ses_sm_alloc();
-		if (sm == NULL)
-			return (NULL);
-		ses_setup(sm);
-		sp = &sm->sess;
-		CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+		if (sm != NULL) {
+			sm->pool = pp;
+			ses_setup(sm);
+		}
 	}
-
+	if (sm == NULL)
+		return (NULL);
+	sp = &sm->sess;
+	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+	/* XXX Stats */
 	VSC_C_main->n_sess++;		/* XXX: locking  ? */
-
 	return (sp);
 }
 
@@ -292,10 +286,14 @@ SES_Delete(struct sess *sp, const char *reason)
 	struct acct *b = &sp->acct_ses;
 	struct sessmem *sm;
 	static char noaddr[] = "-";
+	struct sesspool *pp;
+
 
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 	sm = sp->mem;
 	CHECK_OBJ_NOTNULL(sm, SESSMEM_MAGIC);
+	pp = sm->pool;
+	CHECK_OBJ_NOTNULL(pp, SESSPOOL_MAGIC);
 
 	if (reason != NULL)
 		SES_Close(sp, reason);
@@ -314,28 +312,21 @@ SES_Delete(struct sess *sp, const char *reason)
 	    sp->addr, sp->port, sp->t_end - b->first,
 	    b->sess, b->req, b->pipe, b->pass,
 	    b->fetch, b->hdrbytes, b->bodybytes);
+
 	if (sm->workspace != params->sess_workspace) {
 		Lck_Lock(&stat_mtx);
 		VSC_C_main->n_sess_mem--;
 		Lck_Unlock(&stat_mtx);
 		free(sm);
+		Lck_Lock(&pp->mtx);
+		sesspool->nsess--;
+		Lck_Unlock(&pp->mtx);
 	} else {
 		/* Clean and prepare for reuse */
 		ses_setup(sm);
-		Lck_Lock(&ses_mem_mtx);
-		VTAILQ_INSERT_HEAD(&ses_free_mem[1 - ses_qp], sm, list);
-		Lck_Unlock(&ses_mem_mtx);
-	}
-
-	/* Try to precreate some ses-mem so the acceptor will not have to */
-	if (VSC_C_main->n_sess_mem < VSC_C_main->n_sess + 10) {
-		sm = ses_sm_alloc();
-		if (sm != NULL) {
-			ses_setup(sm);
-			Lck_Lock(&ses_mem_mtx);
-			VTAILQ_INSERT_HEAD(&ses_free_mem[1 - ses_qp], sm, list);
-			Lck_Unlock(&ses_mem_mtx);
-		}
+		Lck_Lock(&pp->mtx);
+		VTAILQ_INSERT_HEAD(&sesspool->freelist, sm, list);
+		Lck_Unlock(&pp->mtx);
 	}
 }
 
@@ -345,6 +336,10 @@ void
 SES_Init()
 {
 
+	ALLOC_OBJ(sesspool, SESSPOOL_MAGIC);
+	VTAILQ_INIT(&sesspool->freelist);
+	Lck_New(&sesspool->mtx, lck_sessmem);
+	sesspool->maxsess = params->max_sess;
+
 	Lck_New(&stat_mtx, lck_stat);
-	Lck_New(&ses_mem_mtx, lck_sessmem);
 }



From geoff at varnish-cache.org  Mon Jan  9 20:52:01 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:01 +0100
Subject: [experimental-ims] 6084802 Mark argc, argv as unused in vsc2rst
Message-ID: <E1RkMCH-0007At-37@project.varnish-software.com>

commit 60848023cb8daed9085fe6a31df045e8bf83410b
Author: Tollef Fog Heen <tfheen at varnish-software.com>
Date:   Wed Sep 21 14:28:11 2011 +0200

    Mark argc, argv as unused in vsc2rst

diff --git a/man/vsc2rst.c b/man/vsc2rst.c
index 9b89cce..e6abcf9 100644
--- a/man/vsc2rst.c
+++ b/man/vsc2rst.c
@@ -6,6 +6,8 @@
 
 int main(int argc, char **argv)
 {
+	(void)argc;
+	(void)argv;
 	P("================");
 	P("varnish-counters");
 	P("================");



From geoff at varnish-cache.org  Mon Jan  9 20:52:01 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:01 +0100
Subject: [experimental-ims] 701e5c3 Format all code examples
Message-ID: <E1RkMCH-0007BK-6y@project.varnish-software.com>

commit 701e5c35e7962be65cca4261b7811b33f0244515
Author: Andreas Plesner Jacobsen <apj at mutt.dk>
Date:   Thu Sep 22 11:10:12 2011 +0200

    Format all code examples

diff --git a/doc/sphinx/installation/upgrade.rst b/doc/sphinx/installation/upgrade.rst
index eda1515..0ed4f9f 100644
--- a/doc/sphinx/installation/upgrade.rst
+++ b/doc/sphinx/installation/upgrade.rst
@@ -20,11 +20,11 @@ To simplify strings, the %-encoding has been removed. If you need non-printable
 ``log`` moved to the std vmod
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-``log`` has moved to the std vmod:
+``log`` has moved to the std vmod::
 
 	log "log something";
 
-becomes
+becomes::
 
 	import std;
 	std.log("log something");
@@ -34,11 +34,11 @@ You only need to import std once.
 purges are now called bans
 ~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-``purge()`` and ``purge_url()`` are now respectively ``ban()`` and ``ban_url()``, so you should replace all occurences:
+``purge()`` and ``purge_url()`` are now respectively ``ban()`` and ``ban_url()``, so you should replace all occurences::
 
 	purge("req.url = " req.url);
 
-becomes
+becomes::
 
 	ban("req.url = " + req.url);
 
@@ -84,22 +84,22 @@ becomes::
 ``req.hash`` is replaced with ``hash_data()``
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-You no longer append to the hash with ``+=``, so
+You no longer append to the hash with ``+=``, so::
 
 	set req.hash += req.url;
 
-becomes
+becomes::
 
 	hash_data(req.url);
 
 ``esi`` is replaced with ``beresp.do_esi``
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-You no longer enable ESI with ``esi``, so
+You no longer enable ESI with ``esi``, so::
 
 	esi;
 
-in ``vcl_fetch`` becomes
+in ``vcl_fetch`` becomes::
 
 	set beresp.do_esi = true;
 



From geoff at varnish-cache.org  Mon Jan  9 20:52:01 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:01 +0100
Subject: [experimental-ims] 6c696ad Add more expressive VSL's for VCL related
	errors.
Message-ID: <E1RkMCH-0007BS-8Y@project.varnish-software.com>

commit 6c696adb19040d271bf783590a52d24e1fc24fc8
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu Sep 22 11:26:34 2011 +0000

    Add more expressive VSL's for VCL related errors.

diff --git a/include/vsl_tags.h b/include/vsl_tags.h
index bae0a4a..21ca1df 100644
--- a/include/vsl_tags.h
+++ b/include/vsl_tags.h
@@ -95,6 +95,9 @@ SLTM(ESI_xmlerror)
 SLTM(Hash)
 
 SLTM(Backend_health)
+
+SLTM(VCL_Debug)
 SLTM(VCL_Log)
+SLTM(VCL_Error)
 
 SLTM(Gzip)



From geoff at varnish-cache.org  Mon Jan  9 20:52:01 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:01 +0100
Subject: [experimental-ims] 74f3c13 A quick style polish
Message-ID: <E1RkMCH-0007Ba-Ai@project.varnish-software.com>

commit 74f3c13ec5b0a451480431cc1a7db58d56bb5b90
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu Sep 22 11:41:39 2011 +0000

    A quick style polish

diff --git a/lib/libvarnish/vre.c b/lib/libvarnish/vre.c
index e97758a..667d177 100644
--- a/lib/libvarnish/vre.c
+++ b/lib/libvarnish/vre.c
@@ -38,41 +38,47 @@ struct vre {
 	pcre *re;
 };
 
-vre_t *VRE_compile(const char *pattern, int options,
-		    const char **errptr, int *erroffset) {
+vre_t *
+VRE_compile(const char *pattern, int options,
+		    const char **errptr, int *erroffset)
+{
 	vre_t *v;
 	*errptr = NULL; *erroffset = 0;
 
 	ALLOC_OBJ(v, VRE_MAGIC);
-	AN(v);
+	if (v == NULL)
+		return (NULL);
 	v->re = pcre_compile(pattern, options, errptr, erroffset, NULL);
 	if (v->re == NULL) {
 		VRE_free(&v);
-		return NULL;
+		return (NULL);
 	}
-	return v;
+	return (v);
 }
 
-int VRE_exec(const vre_t *code, const char *subject, int length,
-	     int startoffset, int options, int *ovector, int ovecsize) {
+int
+VRE_exec(const vre_t *code, const char *subject, int length,
+    int startoffset, int options, int *ovector, int ovecsize)
+{
 	CHECK_OBJ_NOTNULL(code, VRE_MAGIC);
 	int ov[30];
+
 	if (ovector == NULL) {
 		ovector = ov;
 		ovecsize = sizeof(ov)/sizeof(ov[0]);
 	}
 
-	return pcre_exec(code->re, NULL, subject, length,
-			 startoffset, options, ovector, ovecsize);
+	return (pcre_exec(code->re, NULL, subject, length,
+	    startoffset, options, ovector, ovecsize));
 }
 
-void VRE_free(vre_t **vv) {
-
+void
+VRE_free(vre_t **vv)
+{
 	vre_t *v = *vv;
 
 	*vv = NULL;
 	CHECK_OBJ(v, VRE_MAGIC);
 	pcre_free(v->re);
-	v->magic = 0;
 	FREE_OBJ(v);
 }



From geoff at varnish-cache.org  Mon Jan  9 20:52:01 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:01 +0100
Subject: [experimental-ims] 9816a23 Give VRT_re_match a sess* arg and report
	VRE errors using SLT_VCL_Error instead of asserting.
Message-ID: <E1RkMCH-0007Bh-C7@project.varnish-software.com>

commit 9816a237e8c62ed2992fa6b86931dbb0793e2b01
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu Sep 22 11:48:16 2011 +0000

    Give VRT_re_match a sess* arg and report VRE errors using SLT_VCL_Error
    instead of asserting.
    
    Inspired by:	Patch from DocWilco

diff --git a/bin/varnishd/cache_vrt_re.c b/bin/varnishd/cache_vrt_re.c
index 8e1218a..f41b809 100644
--- a/bin/varnishd/cache_vrt_re.c
+++ b/bin/varnishd/cache_vrt_re.c
@@ -67,7 +67,7 @@ VRT_re_fini(void *rep)
 }
 
 int
-VRT_re_match(const char *s, void *re)
+VRT_re_match(const struct sess *sp, const char *s, void *re)
 {
 	vre_t *t;
 	int i;
@@ -79,7 +79,8 @@ VRT_re_match(const char *s, void *re)
 	i = VRE_exec(t, s, strlen(s), 0, 0, NULL, 0);
 	if (i >= 0)
 		return (1);
-	assert(i == VRE_ERROR_NOMATCH);
+	if (i < VRE_ERROR_NOMATCH )
+		WSP(sp, SLT_VCL_Error, "Regexp matching returned %d", i);
 	return (0);
 }
 
@@ -105,6 +106,10 @@ VRT_regsub(const struct sess *sp, int all, const char *str, void *re,
 	/* If it didn't match, we can return the original string */
 	if (i == VRE_ERROR_NOMATCH)
 		return(str);
+	if (i < VRE_ERROR_NOMATCH ) {
+		WSP(sp, SLT_VCL_Error, "Regexp matching returned %d", i);
+		return(str);
+	}
 
 	u = WS_Reserve(sp->http->ws, 0);
 	res.e = res.b = b0 = sp->http->ws->f;
@@ -135,6 +140,11 @@ VRT_regsub(const struct sess *sp, int all, const char *str, void *re,
 			break;
 		memset(&ovector, 0, sizeof(ovector));
 		i = VRE_exec(t, str, strlen(str), 0, 0, ovector, 30);
+		if (i < VRE_ERROR_NOMATCH ) {
+			WSP(sp, SLT_VCL_Error,
+			    "Regexp matching returned %d", i);
+			return(str);
+		}
 	} while (i != VRE_ERROR_NOMATCH);
 
 	/* Copy suffix to match */
diff --git a/include/vrt.h b/include/vrt.h
index 9f0228f..72c4cfe 100644
--- a/include/vrt.h
+++ b/include/vrt.h
@@ -145,7 +145,7 @@ void VRT_acl_log(const struct sess *, const char *msg);
 /* Regexp related */
 void VRT_re_init(void **, const char *);
 void VRT_re_fini(void *);
-int VRT_re_match(const char *, void *re);
+int VRT_re_match(const struct sess *sp, const char *, void *re);
 const char *VRT_regsub(const struct sess *sp, int all, const char *,
     void *, const char *);
 
diff --git a/lib/libvcl/vcc_expr.c b/lib/libvcl/vcc_expr.c
index 3883dd2..1f1c274 100644
--- a/lib/libvcl/vcc_expr.c
+++ b/lib/libvcl/vcc_expr.c
@@ -941,7 +941,7 @@ vcc_expr_cmp(struct vcc *tl, struct expr **e, enum var_type fmt)
 		re = vcc_regexp(tl);
 		ERRCHK(tl);
 		vcc_NextToken(tl);
-		bprintf(buf, "%sVRT_re_match(\v1, %s)", not, re);
+		bprintf(buf, "%sVRT_re_match(sp, \v1, %s)", not, re);
 		*e = vcc_expr_edit(BOOL, buf, *e, NULL);
 		return;
 	}



From geoff at varnish-cache.org  Mon Jan  9 20:52:01 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:01 +0100
Subject: [experimental-ims] cb4bf6d Add an XXX comment
Message-ID: <E1RkMCH-0007Bs-Dw@project.varnish-software.com>

commit cb4bf6db68a974355745a7f73c1b72d2facd6f6c
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu Sep 22 11:53:56 2011 +0000

    Add an XXX comment

diff --git a/bin/varnishd/cache_ban.c b/bin/varnishd/cache_ban.c
index 4c95592..fed958c 100644
--- a/bin/varnishd/cache_ban.c
+++ b/bin/varnishd/cache_ban.c
@@ -286,6 +286,8 @@ ban_parse_http(const struct ban *b, const char *a1)
 
 /*--------------------------------------------------------------------
  * Parse and add a ban test specification
+ *
+ * XXX: This should vector through VRE.
  */
 
 static int



From geoff at varnish-cache.org  Mon Jan  9 20:52:01 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:01 +0100
Subject: [experimental-ims] 8a385bf Make it possible to set limits for VRE
	matching.
Message-ID: <E1RkMCH-0007C3-Go@project.varnish-software.com>

commit 8a385bfc803409428e37e23c4d7f844d03bb74c4
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu Sep 22 12:15:52 2011 +0000

    Make it possible to set limits for VRE matching.

diff --git a/bin/varnishd/cache_vrt_re.c b/bin/varnishd/cache_vrt_re.c
index f41b809..b2afaa5 100644
--- a/bin/varnishd/cache_vrt_re.c
+++ b/bin/varnishd/cache_vrt_re.c
@@ -76,7 +76,7 @@ VRT_re_match(const struct sess *sp, const char *s, void *re)
 		s = "";
 	AN(re);
 	t = re;
-	i = VRE_exec(t, s, strlen(s), 0, 0, NULL, 0);
+	i = VRE_exec(t, s, strlen(s), 0, 0, NULL, 0, &params->vre_limits);
 	if (i >= 0)
 		return (1);
 	if (i < VRE_ERROR_NOMATCH )
@@ -101,7 +101,8 @@ VRT_regsub(const struct sess *sp, int all, const char *str, void *re,
 		str = "";
 	t = re;
 	memset(ovector, 0, sizeof(ovector));
-	i = VRE_exec(t, str, strlen(str), 0, 0, ovector, 30);
+	i = VRE_exec(t, str, strlen(str), 0, 0, ovector, 30,
+	    &params->vre_limits);
 
 	/* If it didn't match, we can return the original string */
 	if (i == VRE_ERROR_NOMATCH)
@@ -139,7 +140,8 @@ VRT_regsub(const struct sess *sp, int all, const char *str, void *re,
 		if (!all)
 			break;
 		memset(&ovector, 0, sizeof(ovector));
-		i = VRE_exec(t, str, strlen(str), 0, 0, ovector, 30);
+		i = VRE_exec(t, str, strlen(str), 0, 0, ovector, 30,
+		    &params->vre_limits);
 		if (i < VRE_ERROR_NOMATCH ) {
 			WSP(sp, SLT_VCL_Error,
 			    "Regexp matching returned %d", i);
diff --git a/bin/varnishd/heritage.h b/bin/varnishd/heritage.h
index 13fce78..7aebf11 100644
--- a/bin/varnishd/heritage.h
+++ b/bin/varnishd/heritage.h
@@ -30,6 +30,7 @@
  */
 
 #include <pthread.h>
+#include "vre.h"
 
 struct listen_sock {
 	unsigned			magic;
@@ -213,6 +214,8 @@ struct params {
 	double			critbit_cooloff;
 
 	double			shortlived;
+
+	struct vre_limits	vre_limits;
 };
 
 /*
diff --git a/bin/varnishd/mgt_param.c b/bin/varnishd/mgt_param.c
index decd130..f247b71 100644
--- a/bin/varnishd/mgt_param.c
+++ b/bin/varnishd/mgt_param.c
@@ -951,6 +951,24 @@ static const struct parspec input_parspec[] = {
 		"Unreferenced VCL objects result in error.\n",
 		0,
 		"on", "bool" },
+
+
+	{ "pcre_match_limit", tweak_uint,
+		&master.vre_limits.match,
+		1, UINT_MAX,
+		"The limit for the  number of internal matching function"
+		" calls in a pcre_exec() execution.",
+		0,
+		"10000", ""},
+
+	{ "pcre_match_limit_recursion", tweak_uint,
+		&master.vre_limits.match_recursion,
+		1, UINT_MAX,
+		"The limit for the  number of internal matching function"
+		" recursions in a pcre_exec() execution.",
+		0,
+		"10000", ""},
+
 	{ NULL, NULL, NULL }
 };
 
diff --git a/include/vre.h b/include/vre.h
index 47dea3f..29c1b08 100644
--- a/include/vre.h
+++ b/include/vre.h
@@ -27,9 +27,21 @@
  *
  * Regular expression support
  *
+ * We wrap PCRE in VRE to make to make it feasible to use something else
+ * without hunting down stuff through out the Varnish source code.
+ *
  */
 
+#ifndef VRE_H_INCLUDED
+#define VRE_H_INCLUDED
+
 struct vre;
+
+struct vre_limits {
+	unsigned	match;
+	unsigned	match_recursion;
+};
+
 typedef struct vre vre_t;
 
 /* This maps to PCRE error codes */
@@ -39,5 +51,9 @@ typedef struct vre vre_t;
 #define VRE_CASELESS           0x00000001
 
 vre_t *VRE_compile(const char *, int, const char **, int *);
-int VRE_exec(const vre_t *, const char *, int, int, int, int *, int);
+int VRE_exec(const vre_t *code, const char *subject, int length,
+    int startoffset, int options, int *ovector, int ovecsize,
+    const volatile struct vre_limits *lim);
 void VRE_free(vre_t **);
+
+#endif /* VRE_H_INCLUDED */
diff --git a/lib/libvarnish/vre.c b/lib/libvarnish/vre.c
index 667d177..68beaf8 100644
--- a/lib/libvarnish/vre.c
+++ b/lib/libvarnish/vre.c
@@ -27,6 +27,7 @@
  */
 
 #include <pcre.h>
+#include <string.h>
 
 #include "libvarnish.h"
 #include "miniobj.h"
@@ -58,17 +59,27 @@ VRE_compile(const char *pattern, int options,
 
 int
 VRE_exec(const vre_t *code, const char *subject, int length,
-    int startoffset, int options, int *ovector, int ovecsize)
+    int startoffset, int options, int *ovector, int ovecsize,
+    const volatile struct vre_limits *lim)
 {
 	CHECK_OBJ_NOTNULL(code, VRE_MAGIC);
 	int ov[30];
+	pcre_extra extra;
 
 	if (ovector == NULL) {
 		ovector = ov;
 		ovecsize = sizeof(ov)/sizeof(ov[0]);
 	}
 
-	return (pcre_exec(code->re, NULL, subject, length,
+	memset(&extra, 0, sizeof extra);
+	if (lim != NULL) {
+		extra.match_limit = lim->match;
+		extra.flags |= PCRE_EXTRA_MATCH_LIMIT;
+		extra.match_limit_recursion = lim->match_recursion;
+		extra.flags |= PCRE_EXTRA_MATCH_LIMIT_RECURSION;
+	}
+
+	return (pcre_exec(code->re, &extra, subject, length,
 	    startoffset, options, ovector, ovecsize));
 }
 
diff --git a/lib/libvarnishapi/vsl.c b/lib/libvarnishapi/vsl.c
index 58a454d..8473d31 100644
--- a/lib/libvarnishapi/vsl.c
+++ b/lib/libvarnishapi/vsl.c
@@ -266,13 +266,13 @@ VSL_NextLog(const struct VSM_data *vd, uint32_t **pp, uint64_t *bits)
 			continue;
 		if (vsl->regincl != NULL) {
 			i = VRE_exec(vsl->regincl, VSL_DATA(p), VSL_LEN(p),
-			    0, 0, NULL, 0);
+			    0, 0, NULL, 0, NULL);
 			if (i == VRE_ERROR_NOMATCH)
 				continue;
 		}
 		if (vsl->regexcl != NULL) {
 			i = VRE_exec(vsl->regexcl, VSL_DATA(p), VSL_LEN(p),
-			    0, 0, NULL, 0);
+			    0, 0, NULL, 0, NULL);
 			if (i != VRE_ERROR_NOMATCH)
 				continue;
 		}
@@ -282,7 +282,7 @@ VSL_NextLog(const struct VSM_data *vd, uint32_t **pp, uint64_t *bits)
 			VTAILQ_FOREACH(vrm, &vsl->matchers, next) {
 				if (vrm->tag == t) {
 					i = VRE_exec(vrm->re, VSL_DATA(p),
-						     VSL_LEN(p), 0, 0, NULL, 0);
+					    VSL_LEN(p), 0, 0, NULL, 0, NULL);
 					if (i >= 0)
 						*bits |= (uintmax_t)1 << j;
 				}



From geoff at varnish-cache.org  Mon Jan  9 20:52:01 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:01 +0100
Subject: [experimental-ims] 5fc5c45 Pass warnings and error flags to distcheck
Message-ID: <E1RkMCH-0007CX-LT@project.varnish-software.com>

commit 5fc5c45ca001714f6f64ebca5a4c720f4dc81bac
Author: Tollef Fog Heen <tfheen at varnish-software.com>
Date:   Wed Sep 21 14:36:55 2011 +0200

    Pass warnings and error flags to distcheck

diff --git a/Makefile.am b/Makefile.am
index bc0bfc2..dad3621 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -11,6 +11,15 @@ pkgconfig_DATA = varnishapi.pc
 
 EXTRA_DIST = LICENSE autogen.sh varnishapi.pc.in
 
+DISTCHECK_CONFIGURE_FLAGS = \
+    --enable-developer-warnings \
+    --enable-debugging-symbols \
+    --enable-dependency-tracking \
+    --enable-diagnostics \
+    --enable-extra-developer-warnings \
+    --enable-tests \
+    --enable-werror
+
 install-data-local:
 	$(install_sh) -d -m 0755 $(DESTDIR)$(localstatedir)/varnish
 



From geoff at varnish-cache.org  Mon Jan  9 20:52:02 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:02 +0100
Subject: [experimental-ims] 6d50fb2 Try harder to fix solaris IOV_MAX related
	issue.
Message-ID: <E1RkMCI-0007FC-Jf@project.varnish-software.com>

commit 6d50fb287fda1826020bef4e0256d83d35c68d83
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sat Oct 1 01:12:19 2011 +0000

    Try harder to fix solaris IOV_MAX related issue.

diff --git a/bin/varnishd/cache_wrw.c b/bin/varnishd/cache_wrw.c
index 946f488..61aa2e5 100644
--- a/bin/varnishd/cache_wrw.c
+++ b/bin/varnishd/cache_wrw.c
@@ -179,14 +179,16 @@ WRW_Write(struct worker *w, const void *ptr, int len)
 		return (0);
 	if (len == -1)
 		len = strlen(ptr);
-	if (wrw->niov == wrw->siov + (wrw->ciov < wrw->siov ? 1 : 0))
+	if (wrw->niov >= wrw->siov - (wrw->ciov < wrw->siov ? 1 : 0))
 		(void)WRW_Flush(w);
 	wrw->iov[wrw->niov].iov_base = TRUST_ME(ptr);
 	wrw->iov[wrw->niov].iov_len = len;
 	wrw->liov += len;
-	if (wrw->ciov < wrw->siov)
-		wrw->cliov += len;
 	wrw->niov++;
+	if (wrw->ciov < wrw->siov) {
+		assert(wrw->niov < wrw->siov);
+		wrw->cliov += len;
+	}
 	return (len);
 }
 
@@ -208,6 +210,7 @@ WRW_Chunked(struct worker *w)
 	wrw->ciov = wrw->niov++;
 	wrw->cliov = 0;
 	assert(wrw->ciov < wrw->siov);
+	assert(wrw->niov < wrw->siov);
 }
 
 /*



From geoff at varnish-cache.org  Mon Jan  9 20:52:02 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:02 +0100
Subject: [experimental-ims] c69d4ca Described some of the counters based on
	notes in the wiki. Please let me know if any of them are way off
Message-ID: <E1RkMCI-0007FV-Qi@project.varnish-software.com>

commit c69d4ca9cf583c8313ae1b92f27624ceccd7eda5
Author: Per Buer <perbu at varnish-software.com>
Date:   Mon Oct 3 09:49:09 2011 +0200

    Described some of the counters based on notes in the wiki.
    Please let me know if any of them are way off

diff --git a/include/vsc_fields.h b/include/vsc_fields.h
index ea668cc..0fd7e46 100644
--- a/include/vsc_fields.h
+++ b/include/vsc_fields.h
@@ -81,19 +81,53 @@ VSC_F(sess_fail,		uint64_t, 1, 'c',
 
 /*---------------------------------------------------------------------*/
 
-VSC_F(client_req,		uint64_t, 1, 'a', "Client requests received", "")
-
-VSC_F(cache_hit,		uint64_t, 1, 'a', "Cache hits", "")
-VSC_F(cache_hitpass,	uint64_t, 1, 'a', "Cache hits for pass", "")
-VSC_F(cache_miss,		uint64_t, 1, 'a', "Cache misses", "")
+VSC_F(client_req,		uint64_t, 1, 'a', 
+      "Client requests received", 
+      "")
+
+VSC_F(cache_hit,		uint64_t, 1, 'a', 
+      "Cache hits", 
+      "Count of cache hits. "
+      "  A cache hit indicates that an object has been delivered to a"
+      "  client without fetching it from a backend server."
+)
 
-VSC_F(backend_conn,	uint64_t, 0, 'a', "Backend conn. success", "")
-VSC_F(backend_unhealthy,	uint64_t, 0, 'a', "Backend conn. not attempted", "")
+VSC_F(cache_hitpass,	uint64_t, 1, 'a', 
+      "Cache hits for pass", 
+      "Count of hits for pass"
+      "  A cache hit for pass indicates that Varnish is going to"
+      "  pass the request to the backend and this decision has been "
+      "  cached in it self. This counts how many times the cached "
+      "  decision is being used."
+)
+VSC_F(cache_miss,		uint64_t, 1, 'a', 
+      "Cache misses", 
+      "Count of misses"
+      "  A cache miss indicates the object was fetched from the"
+      "  backend before delivering it to the backend.")
+
+VSC_F(backend_conn,	uint64_t, 0, 'a', 
+      "Backend conn. success", 
+      "")
+
+VSC_F(backend_unhealthy,	uint64_t, 0, 'a', 
+      "Backend conn. not attempted", 
+      ""
+)
 VSC_F(backend_busy,	uint64_t, 0, 'a', "Backend conn. too many", "")
 VSC_F(backend_fail,	uint64_t, 0, 'a', "Backend conn. failures", "")
-VSC_F(backend_reuse,	uint64_t, 0, 'a', "Backend conn. reuses", "")
+VSC_F(backend_reuse,	uint64_t, 0, 'a', 
+      "Backend conn. reuses", 
+      "Count of backend connection reuses"
+      "  This counter is increased whenever we reuse a recycled connection.")
 VSC_F(backend_toolate,	uint64_t, 0, 'a', "Backend conn. was closed", "")
-VSC_F(backend_recycle,	uint64_t, 0, 'a', "Backend conn. recycles", "")
+VSC_F(backend_recycle,	uint64_t, 0, 'a', 
+      "Backend conn. recycles", 
+      "Count of backend connection recycles"
+      "  This counter is increased whenever we have a keep-alive"
+      "  connection that is put back into the pool of connections."
+      "  It has not yet been used, but it might be, unless the backend"
+      "  closes it.")
 VSC_F(backend_retry,	uint64_t, 0, 'a', "Backend conn. retry", "")
 
 VSC_F(fetch_head,		uint64_t, 1, 'a', "Fetch head", "")
@@ -221,8 +255,13 @@ VSC_F(n_lru_moved,		uint64_t, 0, 'i', "N LRU moved objects", "")
 
 VSC_F(losthdr,		uint64_t, 0, 'a', "HTTP header overflows", "")
 
-VSC_F(n_objsendfile,	uint64_t, 0, 'a', "Objects sent with sendfile", "")
-VSC_F(n_objwrite,		uint64_t, 0, 'a', "Objects sent with write", "")
+VSC_F(n_objsendfile,	uint64_t, 0, 'a', "Objects sent with sendfile", 
+      "The number of objects sent with the sendfile system call. If enabled "
+      "sendfile will be used on object larger than a certain size.")
+VSC_F(n_objwrite,		uint64_t, 0, 'a', "Objects sent with write", 
+      "The number of objects sent with regular write calls."
+      "Writes are used when the objects are too small for sendfile "
+      "or if the sendfile call has been disabled")
 VSC_F(n_objoverflow,	uint64_t, 1, 'a',
 					"Objects overflowing workspace", "")
 



From geoff at varnish-cache.org  Mon Jan  9 20:52:02 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:02 +0100
Subject: [experimental-ims] b4fee20 Bump minimum number of threads for RPMs
Message-ID: <E1RkMCI-0007Fk-TW@project.varnish-software.com>

commit b4fee2036e2befab57e417bb2c832b55fca7207c
Author: Tollef Fog Heen <tfheen at varnish-software.com>
Date:   Mon Oct 3 11:15:19 2011 +0200

    Bump minimum number of threads for RPMs
    
    1 thread is a bit on the low side, make the default on RPM based
    distros 50.

diff --git a/redhat/varnish.sysconfig b/redhat/varnish.sysconfig
index 8ab3c22..d88712f 100644
--- a/redhat/varnish.sysconfig
+++ b/redhat/varnish.sysconfig
@@ -70,7 +70,7 @@ VARNISH_ADMIN_LISTEN_PORT=6082
 VARNISH_SECRET_FILE=/etc/varnish/secret
 #
 # # The minimum number of worker threads to start
-VARNISH_MIN_THREADS=1
+VARNISH_MIN_THREADS=50
 #
 # # The Maximum number of worker threads to start
 VARNISH_MAX_THREADS=1000



From geoff at varnish-cache.org  Mon Jan  9 20:52:02 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:02 +0100
Subject: [experimental-ims] 04b017e Drop debugging message about which
	host/port we have connected to
Message-ID: <E1RkMCI-0007Fy-WD@project.varnish-software.com>

commit 04b017efb938826b06aef08b8c1bd592755806a3
Author: Tollef Fog Heen <tfheen at varnish-software.com>
Date:   Mon Oct 3 14:37:11 2011 +0200

    Drop debugging message about which host/port we have connected to
    
    Fixes: #1024

diff --git a/bin/varnishadm/varnishadm.c b/bin/varnishadm/varnishadm.c
index 151a437..68ebbf7 100644
--- a/bin/varnishadm/varnishadm.c
+++ b/bin/varnishadm/varnishadm.c
@@ -135,7 +135,6 @@ cli_sock(const char *T_arg, const char *S_arg)
 	}
 	free(answer);
 
-	fprintf(stderr, "CLI connected to %s\n", T_arg);
 	return (sock);
 }
 



From geoff at varnish-cache.org  Mon Jan  9 20:52:03 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:03 +0100
Subject: [experimental-ims] e574fbc Fix docs for req.esi
Message-ID: <E1RkMCJ-0007G9-3Y@project.varnish-software.com>

commit e574fbcf96ef5e6668c439fb4527e0752041c088
Author: Andreas Plesner Jacobsen <apj at mutt.dk>
Date:   Wed Oct 5 18:57:49 2011 +0200

    Fix docs for req.esi

diff --git a/doc/sphinx/reference/vcl.rst b/doc/sphinx/reference/vcl.rst
index 9fd526c..aaca549 100644
--- a/doc/sphinx/reference/vcl.rst
+++ b/doc/sphinx/reference/vcl.rst
@@ -740,7 +740,9 @@ req.restarts
   A count of how many times this request has been restarted.
 
 req.esi
-  True if the request is an ESI request.
+  Boolean. Set to false to disable ESI processing regardless of any
+  value in beresp.do_esi. Defaults to true. This variable is subject
+  to change in future versions, you should avoid using it.
 
 req.esi_level
   A count of how many levels of ESI requests we're currently at.
@@ -788,8 +790,9 @@ beresp.do_stream
   as it is delivered so only client can access the object.
 
 beresp.do_esi
-  Boolean. ESI-process the object after fetching it. Defaults to false. Set it
-  to true to parse the object for ESI directives.
+  Boolean. ESI-process the object after fetching it. Defaults to
+  false. Set it to true to parse the object for ESI directives. Will
+  only be honored if req.esi is true.
 
 beresp.do_gzip
   Boolean. Gzip the object before storing it. Defaults to false.



From geoff at varnish-cache.org  Mon Jan  9 20:52:03 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:03 +0100
Subject: [experimental-ims] 6cd21b2 Sense of exit-code test was wrong.
Message-ID: <E1RkMCJ-0007GO-6k@project.varnish-software.com>

commit 6cd21b29d6670d18ff03cb6e377bf9b70bb0929b
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu Oct 6 08:43:22 2011 +0000

    Sense of exit-code test was wrong.

diff --git a/bin/varnishd/mgt_child.c b/bin/varnishd/mgt_child.c
index 4181a43..81e1c8d 100644
--- a/bin/varnishd/mgt_child.c
+++ b/bin/varnishd/mgt_child.c
@@ -478,7 +478,7 @@ mgt_sigchld(const struct vev *e, int what)
 	vsb = VSB_new_auto();
 	XXXAN(vsb);
 	VSB_printf(vsb, "Child (%d) %s", r, status ? "died" : "ended");
-	if (!WIFEXITED(status) && WEXITSTATUS(status)) {
+	if (WIFEXITED(status) && WEXITSTATUS(status)) {
 		VSB_printf(vsb, " status=%d", WEXITSTATUS(status));
 		exit_status |= 0x20;
 	}



From geoff at varnish-cache.org  Mon Jan  9 20:52:03 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:03 +0100
Subject: [experimental-ims] 91280b2 Add a "wait-running" primitive to varnish
	instances,
	so we can avoid fixed sleeps waiting for the child process to start.
Message-ID: <E1RkMCJ-0007GZ-9a@project.varnish-software.com>

commit 91280b2a21406f7553d4d26fdae23ab9e61c1da4
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu Oct 6 09:02:18 2011 +0000

    Add a "wait-running" primitive to varnish instances, so we can
    avoid fixed sleeps waiting for the child process to start.

diff --git a/bin/varnishtest/tests/v00010.vtc b/bin/varnishtest/tests/v00010.vtc
index ec7b947..0421956 100644
--- a/bin/varnishtest/tests/v00010.vtc
+++ b/bin/varnishtest/tests/v00010.vtc
@@ -5,6 +5,8 @@ server s1 {
 	txresp -hdr "Foo: bar" -body "abcdef\n"
 	rxreq
 	txresp -hdr "Panic: please" -body "012345\n"
+	close
+
 	sema r1 sync 2
 	accept
 	rxreq
@@ -33,10 +35,10 @@ client c1 {
 	rxresp
 	txreq -url "/foo"
 	# Don't expect answer, the server crashed.
-	sema r1 sync 2
 } -run
 
-delay 2.5
+varnish v1 -wait-running
+sema r1 sync 2
 
 client c1 {
 	txreq -url "/"
diff --git a/bin/varnishtest/vtc_varnish.c b/bin/varnishtest/vtc_varnish.c
index 5ef4beb..9ba34a7 100644
--- a/bin/varnishtest/vtc_varnish.c
+++ b/bin/varnishtest/vtc_varnish.c
@@ -120,6 +120,30 @@ varnish_ask_cli(const struct varnish *v, const char *cmd, char **repl)
 }
 
 /**********************************************************************
+ *
+ */
+
+static void
+wait_running(const struct varnish *v)
+{
+	char *r;
+	enum VCLI_status_e st;
+
+	while (1) {
+		st = varnish_ask_cli(v, "status", &r);
+		if (st != CLIS_OK)
+			vtc_log(v->vl, 0,
+			    "CLI status command failed: %u %s", st, r);
+		if (!strcmp(r, "Child in state running")) {
+			free(r);
+			break;
+		}
+		free(r);
+		(void)usleep(200000);
+	}
+}
+
+/**********************************************************************
  * Varnishlog gatherer + thread
  */
 
@@ -439,6 +463,7 @@ varnish_start(struct varnish *v)
 		return;
 	if (u != CLIS_OK)
 		vtc_log(v->vl, 0, "CLI start command failed: %u %s", u, resp);
+	wait_running(v);
 	free(resp);
 	u = varnish_ask_cli(v, "debug.xid 1000", &resp);
 	if (vtc_error)
@@ -823,6 +848,10 @@ cmd_varnish(CMD_ARGS)
 			varnish_stop(v);
 			continue;
 		}
+		if (!strcmp(*av, "-wait-running")) {
+			wait_running(v);
+			continue;
+		}
 		if (!strcmp(*av, "-wait")) {
 			varnish_wait(v);
 			continue;



From geoff at varnish-cache.org  Mon Jan  9 20:52:03 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:03 +0100
Subject: [experimental-ims] c3073d0 Update redhat spec file to include release
	candidate logic.
Message-ID: <E1RkMCJ-0007Gz-GF@project.varnish-software.com>

commit c3073d0ab4e4ad53d484a423addedcab93094709
Author: Tollef Fog Heen <tfheen at varnish-software.com>
Date:   Thu Oct 6 10:56:14 2011 +0200

    Update redhat spec file to include release candidate logic.

diff --git a/redhat/varnish.spec b/redhat/varnish.spec
index 0d66765..e8ea8c7 100644
--- a/redhat/varnish.spec
+++ b/redhat/varnish.spec
@@ -1,11 +1,15 @@
+%define v_rc
+%define vd_rc %{?v_rc:-%{?v_rc}}
+
 Summary: High-performance HTTP accelerator
 Name: varnish
-Version: 3.0.0
-Release: 0.20110715%{?dist}
+Version: 3.1.0
+Release: 0.20111006%{?v_rc}%{?dist}
 License: BSD
 Group: System Environment/Daemons
 URL: http://www.varnish-cache.org/
 #Source0: http://repo.varnish-cache.org/source/%{name}-%{version}.tar.gz
+#Source0: %{name}-%{version}%{?vd_rc}.tar.gz
 Source0: %{name}-trunk.tar.gz
 BuildRoot: %{_tmppath}/%{name}-%{version}-%{release}-root-%(%{__id_u} -n)
 # To build from git, start with a make dist, see redhat/README.redhat 
@@ -71,7 +75,7 @@ Documentation files for %name
 #Varnish Cache is a high-performance HTTP accelerator
 
 %prep
-#%setup -q
+#%setup -q -n varnish-%{version}%{?vd_rc}
 %setup -q -n varnish-trunk
 
 mkdir examples



From geoff at varnish-cache.org  Mon Jan  9 20:52:03 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:03 +0100
Subject: [experimental-ims] 0887a4f Try to firm up v00010 even more,
	by explicitly waiting for the crashing child to do so,
	and then explicitly start it again.
Message-ID: <E1RkMCJ-0007HZ-O1@project.varnish-software.com>

commit 0887a4f9891e13c2249558041e743ddc6f6e0e8a
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu Oct 6 10:48:57 2011 +0000

    Try to firm up v00010 even more, by explicitly waiting for the
    crashing child to do so, and then explicitly start it again.

diff --git a/bin/varnishtest/tests/v00010.vtc b/bin/varnishtest/tests/v00010.vtc
index 0421956..45dbc79 100644
--- a/bin/varnishtest/tests/v00010.vtc
+++ b/bin/varnishtest/tests/v00010.vtc
@@ -27,9 +27,6 @@ varnish v1 -cliok "param.set diag_bitmap 0x00001000"
 # Force the (random) port selected to be used again after restart.
 varnish v1 -cliok "param.set listen_address ${v1_addr}:${v1_port}"
 
-# varnishtest defaults to auto_restart off, to avoid masking bugs.
-varnish v1 -cliok "param.set auto_restart on" 
-
 client c1 {
 	txreq -url "/"
 	rxresp
@@ -37,6 +34,11 @@ client c1 {
 	# Don't expect answer, the server crashed.
 } -run
 
+varnish v1 -wait-stopped
+varnish v1 -cliok "panic.show"
+varnish v1 -cliok "panic.clear"
+varnish v1 -clierr 300 "panic.clear"
+varnish v1 -cliok "start"
 varnish v1 -wait-running
 sema r1 sync 2
 
@@ -45,7 +47,3 @@ client c1 {
 	rxresp
 	expect resp.http.foo == "foo"
 } -run
-
-varnish v1 -cliok "panic.show"
-varnish v1 -cliok "panic.clear"
-varnish v1 -clierr 300 "panic.clear"
diff --git a/bin/varnishtest/vtc_varnish.c b/bin/varnishtest/vtc_varnish.c
index 9ba34a7..9690a23 100644
--- a/bin/varnishtest/vtc_varnish.c
+++ b/bin/varnishtest/vtc_varnish.c
@@ -124,12 +124,37 @@ varnish_ask_cli(const struct varnish *v, const char *cmd, char **repl)
  */
 
 static void
+wait_stopped(const struct varnish *v)
+{
+	char *r;
+	enum VCLI_status_e st;
+
+	while (1) {
+		vtc_log(v->vl, 3, "wait-stopped");
+		st = varnish_ask_cli(v, "status", &r);
+		if (st != CLIS_OK)
+			vtc_log(v->vl, 0,
+			    "CLI status command failed: %u %s", st, r);
+		if (!strcmp(r, "Child in state stopped")) {
+			free(r);
+			break;
+		}
+		free(r);
+		(void)usleep(200000);
+	}
+}
+/**********************************************************************
+ *
+ */
+
+static void
 wait_running(const struct varnish *v)
 {
 	char *r;
 	enum VCLI_status_e st;
 
 	while (1) {
+		vtc_log(v->vl, 3, "wait-running");
 		st = varnish_ask_cli(v, "status", &r);
 		if (st != CLIS_OK)
 			vtc_log(v->vl, 0,
@@ -848,6 +873,10 @@ cmd_varnish(CMD_ARGS)
 			varnish_stop(v);
 			continue;
 		}
+		if (!strcmp(*av, "-wait-stopped")) {
+			wait_stopped(v);
+			continue;
+		}
 		if (!strcmp(*av, "-wait-running")) {
 			wait_running(v);
 			continue;



From geoff at varnish-cache.org  Mon Jan  9 20:52:03 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:03 +0100
Subject: [experimental-ims] 1c9bf69 Drop unused variable
Message-ID: <E1RkMCJ-0007I7-US@project.varnish-software.com>

commit 1c9bf690574fe2fdea90107d42bb45b21f773c7d
Author: Tollef Fog Heen <tfheen at varnish-software.com>
Date:   Thu Oct 6 13:45:40 2011 +0200

    Drop unused variable

diff --git a/lib/libvarnishapi/vsl.c b/lib/libvarnishapi/vsl.c
index a34ea1e..8a54c9f 100644
--- a/lib/libvarnishapi/vsl.c
+++ b/lib/libvarnishapi/vsl.c
@@ -215,7 +215,6 @@ VSL_NextLog(const struct VSM_data *vd, uint32_t **pp, uint64_t *bits)
 	struct vsl *vsl;
 	uint32_t *p;
 	unsigned char t;
-	unsigned u;
 	int i;
 
 	CHECK_OBJ_NOTNULL(vd, VSM_MAGIC);
@@ -226,7 +225,6 @@ VSL_NextLog(const struct VSM_data *vd, uint32_t **pp, uint64_t *bits)
 		i = vsl_nextlog(vsl, &p);
 		if (i != 1)
 			return (i);
-		u = VSL_ID(p);
 		t = VSL_TAG(p);
 		if (vsl->skip) {
 			--vsl->skip;



From geoff at varnish-cache.org  Mon Jan  9 20:52:04 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:04 +0100
Subject: [experimental-ims] c327e76 Elminiate pthread poisoning of manager
	process.
Message-ID: <E1RkMCK-0007Ii-5A@project.varnish-software.com>

commit c327e7651c2b4b24730d7b8f8ec2e10115d50455
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Fri Oct 7 10:03:50 2011 +0000

    Elminiate pthread poisoning of manager process.

diff --git a/bin/varnishd/heritage.h b/bin/varnishd/heritage.h
index 7aebf11..bb684b3 100644
--- a/bin/varnishd/heritage.h
+++ b/bin/varnishd/heritage.h
@@ -29,7 +29,6 @@
  * This file contains the heritage passed when mgt forks cache
  */
 
-#include <pthread.h>
 #include "vre.h"
 
 struct listen_sock {
diff --git a/bin/varnishd/mgt.h b/bin/varnishd/mgt.h
index 24293c2..a21ca1c 100644
--- a/bin/varnishd/mgt.h
+++ b/bin/varnishd/mgt.h
@@ -107,3 +107,7 @@ extern unsigned mgt_vcc_err_unref;
 #define VSM_Free(a)		VSM__Free(a)
 #define VSM_Clean()		VSM__Clean()
 
+
+#if defined(PTHREAD_CANCELED) || defined(PTHREAD_MUTEX_DEFAULT)
+#error "Keep pthreads out of in manager process"
+#endif
diff --git a/bin/varnishd/varnishd.c b/bin/varnishd/varnishd.c
index 49e0f81..fd91d08 100644
--- a/bin/varnishd/varnishd.c
+++ b/bin/varnishd/varnishd.c
@@ -653,3 +653,7 @@ main(int argc, char * const *argv)
 		(void)VPF_Remove(pfh);
 	exit(exit_status);
 }
+
+#if defined(PTHREAD_CANCELED) || defined(PTHREAD_MUTEX_DEFAULT)
+#error "Keep pthreads out of in manager process"
+#endif



From geoff at varnish-cache.org  Mon Jan  9 20:52:04 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:04 +0100
Subject: [experimental-ims] 6c17a35 Enforce the convention that varnish
	related files are included with "..." and external files are
	included with <...>
Message-ID: <E1RkMCK-0007J8-BW@project.varnish-software.com>

commit 6c17a356c8b36bf1ae29a5e97473abba5b36a78c
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Fri Oct 7 20:57:13 2011 +0000

    Enforce the convention that varnish related files are included with "..."
    and external files are included with <...>

diff --git a/lib/libvarnish/binary_heap.c b/lib/libvarnish/binary_heap.c
index 01a3c96..f49c736 100644
--- a/lib/libvarnish/binary_heap.c
+++ b/lib/libvarnish/binary_heap.c
@@ -455,7 +455,7 @@ binheap_reorder(const struct binheap *bh, unsigned idx)
 #ifdef TEST_DRIVER
 /* Test driver -------------------------------------------------------*/
 #include <stdio.h>
-#include <miniobj.h>
+#include "miniobj.h"
 
 static void
 vasfail(const char *func, const char *file, int line,
diff --git a/lib/libvarnish/cli_serve.c b/lib/libvarnish/cli_serve.c
index dc4a89b..854c2ff 100644
--- a/lib/libvarnish/cli_serve.c
+++ b/lib/libvarnish/cli_serve.c
@@ -38,15 +38,15 @@
 #include <stdio.h>
 #include <unistd.h>
 
-#include <vqueue.h>
-#include <vsb.h>
-#include <vlu.h>
-#include <vcli.h>
-#include <cli_priv.h>
-#include <cli_common.h>
-#include <cli_serve.h>
-#include <libvarnish.h>
-#include <miniobj.h>
+#include "vqueue.h"
+#include "vsb.h"
+#include "vlu.h"
+#include "vcli.h"
+#include "cli_priv.h"
+#include "cli_common.h"
+#include "cli_serve.h"
+#include "libvarnish.h"
+#include "miniobj.h"
 
 struct VCLS_func {
 	unsigned			magic;
diff --git a/lib/libvarnish/num.c b/lib/libvarnish/num.c
index 5ef2cd7..70255db 100644
--- a/lib/libvarnish/num.c
+++ b/lib/libvarnish/num.c
@@ -35,7 +35,7 @@
 #include <stdlib.h>
 #include <stdint.h>
 
-#include <libvarnish.h>
+#include "libvarnish.h"
 
 static const char err_miss_num[] = "Missing number";
 static const char err_invalid_num[] = "Invalid number";
diff --git a/lib/libvarnish/vct.c b/lib/libvarnish/vct.c
index adcffdb..2740aa1 100644
--- a/lib/libvarnish/vct.c
+++ b/lib/libvarnish/vct.c
@@ -31,7 +31,7 @@
 #include "config.h"
 
 #include <stdint.h>
-#include <vct.h>
+#include "vct.h"
 
 /* NB: VCT always operate in ASCII, don't replace 0x0d with \r etc. */
 



From geoff at varnish-cache.org  Mon Jan  9 20:52:14 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:14 +0100
Subject: [experimental-ims] 7a24aa3 Start #include reorg. The table-building
	#includes will all go into the tbl/ subdirectory
Message-ID: <E1RkMCU-0007Xg-OL@project.varnish-software.com>

commit 7a24aa37d753c951a1df67709e3e3238d6befac7
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sat Oct 8 14:25:44 2011 +0000

    Start #include reorg.  The table-building #includes will all go into
    the tbl/ subdirectory

diff --git a/bin/varnishd/cache.h b/bin/varnishd/cache.h
index 5a91dd8..11e952e 100644
--- a/bin/varnishd/cache.h
+++ b/bin/varnishd/cache.h
@@ -224,7 +224,7 @@ struct acct {
 #define VSC_F(n, t, l, f, e,d)	L##l(t, n)
 #define VSC_DO_MAIN
 struct dstat {
-#include "vsc_fields.h"
+#include "tbl/vsc_fields.h"
 };
 #undef VSC_F
 #undef VSC_DO_MAIN
diff --git a/bin/varnishd/cache_wrk.c b/bin/varnishd/cache_wrk.c
index ba2e41a..bb88664 100644
--- a/bin/varnishd/cache_wrk.c
+++ b/bin/varnishd/cache_wrk.c
@@ -53,7 +53,7 @@ wrk_sumstat(struct worker *w)
 #define L1(n) (VSC_C_main->n += w->stats.n)
 #define VSC_DO_MAIN
 #define VSC_F(n, t, l, f, d, e) L##l(n);
-#include "vsc_fields.h"
+#include "tbl/vsc_fields.h"
 #undef VSC_F
 #undef VSC_DO_MAIN
 #undef L0
diff --git a/include/Makefile.am b/include/Makefile.am
index f0e2daf..3f2994c 100644
--- a/include/Makefile.am
+++ b/include/Makefile.am
@@ -1,14 +1,14 @@
 #
 
 pkginclude_HEADERS = \
-	vsm.h \
-	vsl.h \
+	tbl/vsc_all.h \
+	tbl/vsc_fields.h \
+	tbl/vsl_tags.h \
+	varnishapi.h \
 	vcli.h \
-	vsl_tags.h \
-	vsc_fields.h \
-	vsc_all.h \
 	vsc.h \
-	varnishapi.h
+	vsl.h \
+	vsm.h
 
 nobase_noinst_HEADERS = \
 	ban_vars.h \
diff --git a/include/tbl/README b/include/tbl/README
new file mode 100644
index 0000000..bbb9afc
--- /dev/null
+++ b/include/tbl/README
@@ -0,0 +1,4 @@
+The include files in this directory are special, they are used to
+define sets of objects using macros, such that a list of all such
+objects can be compiled into the code by including these files with
+a properly defined macro.
diff --git a/include/tbl/vsc_all.h b/include/tbl/vsc_all.h
new file mode 100644
index 0000000..e12bd2f
--- /dev/null
+++ b/include/tbl/vsc_all.h
@@ -0,0 +1,60 @@
+/*-
+ * Copyright (c) 2010 Varnish Software AS
+ * All rights reserved.
+ *
+ * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ */
+
+
+
+VSC_DO(LCK, lck, VSC_TYPE_LCK)
+#define VSC_DO_LCK
+#include "tbl/vsc_fields.h"
+#undef VSC_DO_LCK
+VSC_DONE(LCK, lck, VSC_TYPE_LCK)
+
+VSC_DO(MAIN, main, VSC_TYPE_MAIN)
+#define VSC_DO_MAIN
+#include "tbl/vsc_fields.h"
+#undef VSC_DO_MAIN
+VSC_DONE(MAIN, main, VSC_TYPE_MAIN)
+
+VSC_DO(SMA, sma, VSC_TYPE_SMA)
+#define VSC_DO_SMA
+#include "tbl/vsc_fields.h"
+#undef VSC_DO_SMA
+VSC_DONE(SMA, sma, VSC_TYPE_SMA)
+
+VSC_DO(SMF, smf, VSC_TYPE_SMF)
+#define VSC_DO_SMF
+#include "tbl/vsc_fields.h"
+#undef VSC_DO_SMF
+VSC_DONE(SMF, smf, VSC_TYPE_SMF)
+
+VSC_DO(VBE, vbe, VSC_TYPE_VBE)
+#define VSC_DO_VBE
+#include "tbl/vsc_fields.h"
+#undef VSC_DO_VBE
+VSC_DONE(VBE, vbe, VSC_TYPE_VBE)
diff --git a/include/tbl/vsc_fields.h b/include/tbl/vsc_fields.h
new file mode 100644
index 0000000..0fd7e46
--- /dev/null
+++ b/include/tbl/vsc_fields.h
@@ -0,0 +1,376 @@
+/*-
+ * Copyright (c) 2006 Verdens Gang AS
+ * Copyright (c) 2006-2011 Varnish Software AS
+ * All rights reserved.
+ *
+ * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION, "")
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * Definition of all shared memory statistics below.
+ *
+ * Fields (n, t, l, f, e, d):
+ *    n - Name:		Field name, in C-source and stats programs
+ *    t - Type:		C-type, uint64_t, unless marked in 'f'
+ *    l - Local:	Local counter in worker thread.
+ *    f - Format:	Semantics of the value in this field
+ *				'a' - Accumulator (deprecated, use 'c')
+ *				'b' - Bitmap
+ *				'c' - Counter, never decreases.
+ *				'g' - Gauge, goes up and down
+ *				'i' - Integer (deprecated, use 'g')
+ *    e - Explantion:	Short explanation of field (for screen use)
+ *    d - Description:	Long explanation of field (for doc use)
+ *
+ * -----------------------
+ * NB: Cleanup in progress
+ * -----------------------
+ *
+ * Insufficient attention has caused this to become a swamp of conflicting
+ * conventions, shorthands and general mumbo-jumbo.  I'm trying to clean
+ * it up as I go over the code in other business.
+ *
+ * Please see the sessmem section for how it should look.
+ *
+ */
+
+/**********************************************************************/
+
+#ifdef VSC_DO_MAIN
+
+/*---------------------------------------------------------------------
+ * Sessions
+ *    see: cache_acceptor.c and cache_pool.c
+ */
+
+VSC_F(sess_conn,		uint64_t, 1, 'c',
+    "Sessions accepted",
+	"Count of sessions succesfully accepted"
+)
+VSC_F(sess_drop,		uint64_t, 1, 'c',
+    "Sessions dropped",
+	"Count of sessions silently dropped due to lack of session memory."
+	"  See parameter 'max_sess'."
+)
+
+VSC_F(sess_fail,		uint64_t, 1, 'c',
+    "Session accept failures",
+	"Count of failures to accept TCP connection."
+	"  Either the client changed its mind, or the kernel ran out of"
+	" some resource like filedescriptors."
+)
+
+/*---------------------------------------------------------------------*/
+
+VSC_F(client_req,		uint64_t, 1, 'a', 
+      "Client requests received", 
+      "")
+
+VSC_F(cache_hit,		uint64_t, 1, 'a', 
+      "Cache hits", 
+      "Count of cache hits. "
+      "  A cache hit indicates that an object has been delivered to a"
+      "  client without fetching it from a backend server."
+)
+
+VSC_F(cache_hitpass,	uint64_t, 1, 'a', 
+      "Cache hits for pass", 
+      "Count of hits for pass"
+      "  A cache hit for pass indicates that Varnish is going to"
+      "  pass the request to the backend and this decision has been "
+      "  cached in it self. This counts how many times the cached "
+      "  decision is being used."
+)
+VSC_F(cache_miss,		uint64_t, 1, 'a', 
+      "Cache misses", 
+      "Count of misses"
+      "  A cache miss indicates the object was fetched from the"
+      "  backend before delivering it to the backend.")
+
+VSC_F(backend_conn,	uint64_t, 0, 'a', 
+      "Backend conn. success", 
+      "")
+
+VSC_F(backend_unhealthy,	uint64_t, 0, 'a', 
+      "Backend conn. not attempted", 
+      ""
+)
+VSC_F(backend_busy,	uint64_t, 0, 'a', "Backend conn. too many", "")
+VSC_F(backend_fail,	uint64_t, 0, 'a', "Backend conn. failures", "")
+VSC_F(backend_reuse,	uint64_t, 0, 'a', 
+      "Backend conn. reuses", 
+      "Count of backend connection reuses"
+      "  This counter is increased whenever we reuse a recycled connection.")
+VSC_F(backend_toolate,	uint64_t, 0, 'a', "Backend conn. was closed", "")
+VSC_F(backend_recycle,	uint64_t, 0, 'a', 
+      "Backend conn. recycles", 
+      "Count of backend connection recycles"
+      "  This counter is increased whenever we have a keep-alive"
+      "  connection that is put back into the pool of connections."
+      "  It has not yet been used, but it might be, unless the backend"
+      "  closes it.")
+VSC_F(backend_retry,	uint64_t, 0, 'a', "Backend conn. retry", "")
+
+VSC_F(fetch_head,		uint64_t, 1, 'a', "Fetch head", "")
+VSC_F(fetch_length,		uint64_t, 1, 'a', "Fetch with Length", "")
+VSC_F(fetch_chunked,		uint64_t, 1, 'a', "Fetch chunked", "")
+VSC_F(fetch_eof,		uint64_t, 1, 'a', "Fetch EOF", "")
+VSC_F(fetch_bad,		uint64_t, 1, 'a', "Fetch had bad headers", "")
+VSC_F(fetch_close,		uint64_t, 1, 'a', "Fetch wanted close", "")
+VSC_F(fetch_oldhttp,		uint64_t, 1, 'a', "Fetch pre HTTP/1.1 closed", "")
+VSC_F(fetch_zero,		uint64_t, 1, 'a', "Fetch zero len", "")
+VSC_F(fetch_failed,		uint64_t, 1, 'a', "Fetch failed", "")
+VSC_F(fetch_1xx,		uint64_t, 1, 'a', "Fetch no body (1xx)", "")
+VSC_F(fetch_204,		uint64_t, 1, 'a', "Fetch no body (204)", "")
+VSC_F(fetch_304,		uint64_t, 1, 'a', "Fetch no body (304)", "")
+
+/*---------------------------------------------------------------------
+ * Session Memory
+ *    see: cache_session.c
+ */
+
+VSC_F(sessmem_size,		uint64_t, 1, 'g',
+    "Session mem size",
+	"Bytes of memory allocated for last allocated session."
+)
+
+VSC_F(sessmem_alloc,		uint64_t, 1, 'c',
+    "Session mem allocated",
+	"Count of all allocations of session memory."
+)
+
+VSC_F(sessmem_free,		uint64_t, 1, 'c',
+    "Session mem freed",
+	"Count of all frees of session memory."
+)
+
+VSC_F(sessmem_fail,		uint64_t, 1, 'c',
+    "Session mem alloc failed",
+	"Count of session memory allocation failures."
+)
+
+VSC_F(sessmem_limit,		uint64_t, 1, 'c',
+    "Session mem alloc limited",
+	"Count of session memory allocations blocked by limit (max_sess)."
+)
+
+/*---------------------------------------------------------------------
+ * Pools, threads, and sessions
+ *    see: cache_pool.c
+ *
+ */
+
+VSC_F(pools,			uint64_t, 1, 'g',
+    "Number of thread pools",
+	"Number of thread pools.  See also param wthread_pools."
+	"  NB: Presently pools cannot be removed once created."
+)
+
+VSC_F(threads,			uint64_t, 1, 'g',
+    "Total number of threads",
+	"Number of threads in all pools."
+	"  See also params thread_pools, thread_pool_min & thread_pool_max."
+)
+
+VSC_F(threads_limited,		uint64_t, 1, 'c',
+    "Threads hit max",
+	"Number of times more threads were needed, but limit was reached"
+	" in a thread pool."
+	"  See also param thread_pool_max."
+)
+
+VSC_F(threads_created,		uint64_t, 1, 'c',
+    "Threads created",
+	"Total number of threads created in all pools."
+)
+
+VSC_F(threads_destroyed,	uint64_t, 1, 'c',
+    "Threads destoryed",
+	"Total number of threads destroyed in all pools."
+)
+
+VSC_F(threads_failed,		uint64_t, 1, 'c',
+    "Thread creation failed",
+	"Number of times creating a thread failed."
+	"  See VSL::Debug for diagnostics."
+	"  See also param thread_fail_delay."
+)
+
+VSC_F(thread_queue_len,		uint64_t, 1, 'g',
+    "Length of session queue",
+	"Length of session queue waiting for threads."
+	"  NB: Only updates once per second."
+	"  See also param queue_max."
+)
+
+VSC_F(sess_queued,		uint64_t, 1, 'c',
+    "Sessions queued for thread",
+	"Number of times session was queued waiting for a thread."
+	"  See also param queue_max."
+)
+
+VSC_F(sess_dropped,		uint64_t, 1, 'c',
+    "Sessions dropped for thread",
+	"Number of times session was dropped because the queue were too"
+	" long already."
+	"  See also param queue_max."
+)
+
+/*---------------------------------------------------------------------*/
+
+VSC_F(n_sess_mem,		uint64_t, 0, 'i', "N struct sess_mem", "")
+VSC_F(n_sess,			uint64_t, 0, 'i', "N struct sess", "")
+VSC_F(n_object,			uint64_t, 1, 'i', "N struct object", "")
+VSC_F(n_vampireobject,		uint64_t, 1, 'i', "N unresurrected objects", "")
+VSC_F(n_objectcore,		uint64_t, 1, 'i', "N struct objectcore", "")
+VSC_F(n_objecthead,		uint64_t, 1, 'i', "N struct objecthead", "")
+VSC_F(n_waitinglist,		uint64_t, 1, 'i', "N struct waitinglist", "")
+
+VSC_F(n_vbc,		uint64_t, 0, 'i', "N struct vbc", "")
+
+VSC_F(n_backend,		uint64_t, 0, 'i', "N backends", "")
+
+VSC_F(n_expired,		uint64_t, 0, 'i', "N expired objects", "")
+VSC_F(n_lru_nuked,		uint64_t, 0, 'i', "N LRU nuked objects", "")
+VSC_F(n_lru_moved,		uint64_t, 0, 'i', "N LRU moved objects", "")
+
+VSC_F(losthdr,		uint64_t, 0, 'a', "HTTP header overflows", "")
+
+VSC_F(n_objsendfile,	uint64_t, 0, 'a', "Objects sent with sendfile", 
+      "The number of objects sent with the sendfile system call. If enabled "
+      "sendfile will be used on object larger than a certain size.")
+VSC_F(n_objwrite,		uint64_t, 0, 'a', "Objects sent with write", 
+      "The number of objects sent with regular write calls."
+      "Writes are used when the objects are too small for sendfile "
+      "or if the sendfile call has been disabled")
+VSC_F(n_objoverflow,	uint64_t, 1, 'a',
+					"Objects overflowing workspace", "")
+
+VSC_F(s_sess,		uint64_t, 1, 'a', "Total Sessions", "")
+VSC_F(s_req,		uint64_t, 1, 'a', "Total Requests", "")
+VSC_F(s_pipe,		uint64_t, 1, 'a', "Total pipe", "")
+VSC_F(s_pass,		uint64_t, 1, 'a', "Total pass", "")
+VSC_F(s_fetch,		uint64_t, 1, 'a', "Total fetch", "")
+VSC_F(s_hdrbytes,		uint64_t, 1, 'a', "Total header bytes", "")
+VSC_F(s_bodybytes,		uint64_t, 1, 'a', "Total body bytes", "")
+
+VSC_F(sess_closed,		uint64_t, 1, 'a', "Session Closed", "")
+VSC_F(sess_pipeline,	uint64_t, 1, 'a', "Session Pipeline", "")
+VSC_F(sess_readahead,	uint64_t, 1, 'a', "Session Read Ahead", "")
+VSC_F(sess_linger,		uint64_t, 1, 'a', "Session Linger", "")
+VSC_F(sess_herd,		uint64_t, 1, 'a', "Session herd", "")
+
+VSC_F(shm_records,		uint64_t, 0, 'a', "SHM records", "")
+VSC_F(shm_writes,		uint64_t, 0, 'a', "SHM writes", "")
+VSC_F(shm_flushes,		uint64_t, 0, 'a', "SHM flushes due to overflow", "")
+VSC_F(shm_cont,		uint64_t, 0, 'a', "SHM MTX contention", "")
+VSC_F(shm_cycles,		uint64_t, 0, 'a', "SHM cycles through buffer", "")
+
+VSC_F(sms_nreq,		uint64_t, 0, 'a', "SMS allocator requests", "")
+VSC_F(sms_nobj,		uint64_t, 0, 'i', "SMS outstanding allocations", "")
+VSC_F(sms_nbytes,		uint64_t, 0, 'i', "SMS outstanding bytes", "")
+VSC_F(sms_balloc,		uint64_t, 0, 'i', "SMS bytes allocated", "")
+VSC_F(sms_bfree,		uint64_t, 0, 'i', "SMS bytes freed", "")
+
+VSC_F(backend_req,		uint64_t, 0, 'a', "Backend requests made", "")
+
+VSC_F(n_vcl,		uint64_t, 0, 'a', "N vcl total", "")
+VSC_F(n_vcl_avail,		uint64_t, 0, 'a', "N vcl available", "")
+VSC_F(n_vcl_discard,	uint64_t, 0, 'a', "N vcl discarded", "")
+
+VSC_F(n_ban,		uint64_t, 0, 'i', "N total active bans", "")
+VSC_F(n_ban_add,		uint64_t, 0, 'a', "N new bans added", "")
+VSC_F(n_ban_retire,	uint64_t, 0, 'a', "N old bans deleted", "")
+VSC_F(n_ban_obj_test,	uint64_t, 0, 'a', "N objects tested", "")
+VSC_F(n_ban_re_test,	uint64_t, 0, 'a', "N regexps tested against", "")
+VSC_F(n_ban_dups,	uint64_t, 0, 'a', "N duplicate bans removed", "")
+
+VSC_F(hcb_nolock,		uint64_t, 0, 'a', "HCB Lookups without lock", "")
+VSC_F(hcb_lock,		uint64_t, 0, 'a', "HCB Lookups with lock", "")
+VSC_F(hcb_insert,		uint64_t, 0, 'a', "HCB Inserts", "")
+
+VSC_F(esi_errors,		uint64_t, 0, 'a', "ESI parse errors (unlock)", "")
+VSC_F(esi_warnings,		uint64_t, 0, 'a', "ESI parse warnings (unlock)", "")
+VSC_F(client_drop_late,	uint64_t, 0, 'a', "Connection dropped late", "")
+VSC_F(uptime,		uint64_t, 0, 'a', "Client uptime", "")
+
+VSC_F(dir_dns_lookups,	uint64_t, 0, 'a', "DNS director lookups", "")
+VSC_F(dir_dns_failed,	uint64_t, 0, 'a', "DNS director failed lookups", "")
+VSC_F(dir_dns_hit,		uint64_t, 0, 'a', "DNS director cached lookups hit", "")
+VSC_F(dir_dns_cache_full,	uint64_t, 0, 'a', "DNS director full dnscache", "")
+
+VSC_F(vmods,		uint64_t, 0, 'i', "Loaded VMODs", "")
+
+VSC_F(n_gzip,			uint64_t, 0, 'a', "Gzip operations", "")
+VSC_F(n_gunzip,			uint64_t, 0, 'a', "Gunzip operations", "")
+
+#endif
+
+/**********************************************************************/
+
+#ifdef VSC_DO_LCK
+
+VSC_F(creat,		uint64_t, 0, 'a', "Created locks", "")
+VSC_F(destroy,		uint64_t, 0, 'a', "Destroyed locks", "")
+VSC_F(locks,		uint64_t, 0, 'a', "Lock Operations", "")
+VSC_F(colls,		uint64_t, 0, 'a', "Collisions", "")
+
+#endif
+
+/**********************************************************************
+ * All Stevedores support these counters
+ */
+
+#if defined(VSC_DO_SMA) || defined (VSC_DO_SMF)
+VSC_F(c_req,		uint64_t, 0, 'a', "Allocator requests", "")
+VSC_F(c_fail,		uint64_t, 0, 'a', "Allocator failures", "")
+VSC_F(c_bytes,		uint64_t, 0, 'a', "Bytes allocated", "")
+VSC_F(c_freed,		uint64_t, 0, 'a', "Bytes freed", "")
+VSC_F(g_alloc,		uint64_t, 0, 'i', "Allocations outstanding", "")
+VSC_F(g_bytes,		uint64_t, 0, 'i', "Bytes outstanding", "")
+VSC_F(g_space,		uint64_t, 0, 'i', "Bytes available", "")
+#endif
+
+
+/**********************************************************************/
+
+#ifdef VSC_DO_SMA
+/* No SMA specific counters */
+#endif
+
+/**********************************************************************/
+
+#ifdef VSC_DO_SMF
+VSC_F(g_smf,			uint64_t, 0, 'i', "N struct smf", "")
+VSC_F(g_smf_frag,		uint64_t, 0, 'i', "N small free smf", "")
+VSC_F(g_smf_large,		uint64_t, 0, 'i', "N large free smf", "")
+#endif
+
+/**********************************************************************/
+
+#ifdef VSC_DO_VBE
+
+VSC_F(vcls,			uint64_t, 0, 'i', "VCL references", "")
+VSC_F(happy,		uint64_t, 0, 'b', "Happy health probes", "")
+
+#endif
+
diff --git a/include/tbl/vsl_tags.h b/include/tbl/vsl_tags.h
new file mode 100644
index 0000000..21ca1df
--- /dev/null
+++ b/include/tbl/vsl_tags.h
@@ -0,0 +1,103 @@
+/*-
+ * Copyright (c) 2006 Verdens Gang AS
+ * Copyright (c) 2006-2010 Varnish Software AS
+ * All rights reserved.
+ *
+ * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * Define the tags in the shared memory in a reusable format.
+ * Whoever includes this get to define what the SLTM macro does.
+ *
+ * REMEMBER to update the documentation (especially the varnishlog(1) man
+ * page) whenever this list changes.
+ *
+ * XXX: Please add new entries a the end to not break saved log-segments.
+ * XXX: we can resort them when we have a major release.
+ */
+
+SLTM(Debug)
+SLTM(Error)
+SLTM(CLI)
+SLTM(StatSess)
+SLTM(ReqEnd)
+SLTM(SessionOpen)
+SLTM(SessionClose)
+SLTM(BackendOpen)
+SLTM(BackendXID)
+SLTM(BackendReuse)
+SLTM(BackendClose)
+SLTM(HttpGarbage)
+SLTM(Backend)
+SLTM(Length)
+
+SLTM(FetchError)
+
+SLTM(RxRequest)
+SLTM(RxResponse)
+SLTM(RxStatus)
+SLTM(RxURL)
+SLTM(RxProtocol)
+SLTM(RxHeader)
+
+SLTM(TxRequest)
+SLTM(TxResponse)
+SLTM(TxStatus)
+SLTM(TxURL)
+SLTM(TxProtocol)
+SLTM(TxHeader)
+
+SLTM(ObjRequest)
+SLTM(ObjResponse)
+SLTM(ObjStatus)
+SLTM(ObjURL)
+SLTM(ObjProtocol)
+SLTM(ObjHeader)
+
+SLTM(LostHeader)
+
+SLTM(TTL)
+SLTM(Fetch_Body)
+SLTM(VCL_acl)
+SLTM(VCL_call)
+SLTM(VCL_trace)
+SLTM(VCL_return)
+SLTM(VCL_error)
+SLTM(ReqStart)
+SLTM(Hit)
+SLTM(HitPass)
+SLTM(ExpBan)
+SLTM(ExpKill)
+SLTM(WorkThread)
+
+SLTM(ESI_xmlerror)
+
+SLTM(Hash)
+
+SLTM(Backend_health)
+
+SLTM(VCL_Debug)
+SLTM(VCL_Log)
+SLTM(VCL_Error)
+
+SLTM(Gzip)
diff --git a/include/vsc.h b/include/vsc.h
index 7cb3235..0a9cbf9 100644
--- a/include/vsc.h
+++ b/include/vsc.h
@@ -43,7 +43,7 @@
 #define VSC_DO(u,l,t) struct VSC_C_##l {
 #define VSC_DONE(u,l,t) };
 
-#include "vsc_all.h"
+#include "tbl/vsc_all.h"
 
 #undef VSC_DO
 #undef VSC_F
diff --git a/include/vsc_all.h b/include/vsc_all.h
deleted file mode 100644
index 24fa8ec..0000000
--- a/include/vsc_all.h
+++ /dev/null
@@ -1,60 +0,0 @@
-/*-
- * Copyright (c) 2010 Varnish Software AS
- * All rights reserved.
- *
- * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- *
- */
-
-
-
-VSC_DO(LCK, lck, VSC_TYPE_LCK)
-#define VSC_DO_LCK
-#include "vsc_fields.h"
-#undef VSC_DO_LCK
-VSC_DONE(LCK, lck, VSC_TYPE_LCK)
-
-VSC_DO(MAIN, main, VSC_TYPE_MAIN)
-#define VSC_DO_MAIN
-#include "vsc_fields.h"
-#undef VSC_DO_MAIN
-VSC_DONE(MAIN, main, VSC_TYPE_MAIN)
-
-VSC_DO(SMA, sma, VSC_TYPE_SMA)
-#define VSC_DO_SMA
-#include "vsc_fields.h"
-#undef VSC_DO_SMA
-VSC_DONE(SMA, sma, VSC_TYPE_SMA)
-
-VSC_DO(SMF, smf, VSC_TYPE_SMF)
-#define VSC_DO_SMF
-#include "vsc_fields.h"
-#undef VSC_DO_SMF
-VSC_DONE(SMF, smf, VSC_TYPE_SMF)
-
-VSC_DO(VBE, vbe, VSC_TYPE_VBE)
-#define VSC_DO_VBE
-#include "vsc_fields.h"
-#undef VSC_DO_VBE
-VSC_DONE(VBE, vbe, VSC_TYPE_VBE)
diff --git a/include/vsc_fields.h b/include/vsc_fields.h
deleted file mode 100644
index 0fd7e46..0000000
--- a/include/vsc_fields.h
+++ /dev/null
@@ -1,376 +0,0 @@
-/*-
- * Copyright (c) 2006 Verdens Gang AS
- * Copyright (c) 2006-2011 Varnish Software AS
- * All rights reserved.
- *
- * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION, "")
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- *
- * Definition of all shared memory statistics below.
- *
- * Fields (n, t, l, f, e, d):
- *    n - Name:		Field name, in C-source and stats programs
- *    t - Type:		C-type, uint64_t, unless marked in 'f'
- *    l - Local:	Local counter in worker thread.
- *    f - Format:	Semantics of the value in this field
- *				'a' - Accumulator (deprecated, use 'c')
- *				'b' - Bitmap
- *				'c' - Counter, never decreases.
- *				'g' - Gauge, goes up and down
- *				'i' - Integer (deprecated, use 'g')
- *    e - Explantion:	Short explanation of field (for screen use)
- *    d - Description:	Long explanation of field (for doc use)
- *
- * -----------------------
- * NB: Cleanup in progress
- * -----------------------
- *
- * Insufficient attention has caused this to become a swamp of conflicting
- * conventions, shorthands and general mumbo-jumbo.  I'm trying to clean
- * it up as I go over the code in other business.
- *
- * Please see the sessmem section for how it should look.
- *
- */
-
-/**********************************************************************/
-
-#ifdef VSC_DO_MAIN
-
-/*---------------------------------------------------------------------
- * Sessions
- *    see: cache_acceptor.c and cache_pool.c
- */
-
-VSC_F(sess_conn,		uint64_t, 1, 'c',
-    "Sessions accepted",
-	"Count of sessions succesfully accepted"
-)
-VSC_F(sess_drop,		uint64_t, 1, 'c',
-    "Sessions dropped",
-	"Count of sessions silently dropped due to lack of session memory."
-	"  See parameter 'max_sess'."
-)
-
-VSC_F(sess_fail,		uint64_t, 1, 'c',
-    "Session accept failures",
-	"Count of failures to accept TCP connection."
-	"  Either the client changed its mind, or the kernel ran out of"
-	" some resource like filedescriptors."
-)
-
-/*---------------------------------------------------------------------*/
-
-VSC_F(client_req,		uint64_t, 1, 'a', 
-      "Client requests received", 
-      "")
-
-VSC_F(cache_hit,		uint64_t, 1, 'a', 
-      "Cache hits", 
-      "Count of cache hits. "
-      "  A cache hit indicates that an object has been delivered to a"
-      "  client without fetching it from a backend server."
-)
-
-VSC_F(cache_hitpass,	uint64_t, 1, 'a', 
-      "Cache hits for pass", 
-      "Count of hits for pass"
-      "  A cache hit for pass indicates that Varnish is going to"
-      "  pass the request to the backend and this decision has been "
-      "  cached in it self. This counts how many times the cached "
-      "  decision is being used."
-)
-VSC_F(cache_miss,		uint64_t, 1, 'a', 
-      "Cache misses", 
-      "Count of misses"
-      "  A cache miss indicates the object was fetched from the"
-      "  backend before delivering it to the backend.")
-
-VSC_F(backend_conn,	uint64_t, 0, 'a', 
-      "Backend conn. success", 
-      "")
-
-VSC_F(backend_unhealthy,	uint64_t, 0, 'a', 
-      "Backend conn. not attempted", 
-      ""
-)
-VSC_F(backend_busy,	uint64_t, 0, 'a', "Backend conn. too many", "")
-VSC_F(backend_fail,	uint64_t, 0, 'a', "Backend conn. failures", "")
-VSC_F(backend_reuse,	uint64_t, 0, 'a', 
-      "Backend conn. reuses", 
-      "Count of backend connection reuses"
-      "  This counter is increased whenever we reuse a recycled connection.")
-VSC_F(backend_toolate,	uint64_t, 0, 'a', "Backend conn. was closed", "")
-VSC_F(backend_recycle,	uint64_t, 0, 'a', 
-      "Backend conn. recycles", 
-      "Count of backend connection recycles"
-      "  This counter is increased whenever we have a keep-alive"
-      "  connection that is put back into the pool of connections."
-      "  It has not yet been used, but it might be, unless the backend"
-      "  closes it.")
-VSC_F(backend_retry,	uint64_t, 0, 'a', "Backend conn. retry", "")
-
-VSC_F(fetch_head,		uint64_t, 1, 'a', "Fetch head", "")
-VSC_F(fetch_length,		uint64_t, 1, 'a', "Fetch with Length", "")
-VSC_F(fetch_chunked,		uint64_t, 1, 'a', "Fetch chunked", "")
-VSC_F(fetch_eof,		uint64_t, 1, 'a', "Fetch EOF", "")
-VSC_F(fetch_bad,		uint64_t, 1, 'a', "Fetch had bad headers", "")
-VSC_F(fetch_close,		uint64_t, 1, 'a', "Fetch wanted close", "")
-VSC_F(fetch_oldhttp,		uint64_t, 1, 'a', "Fetch pre HTTP/1.1 closed", "")
-VSC_F(fetch_zero,		uint64_t, 1, 'a', "Fetch zero len", "")
-VSC_F(fetch_failed,		uint64_t, 1, 'a', "Fetch failed", "")
-VSC_F(fetch_1xx,		uint64_t, 1, 'a', "Fetch no body (1xx)", "")
-VSC_F(fetch_204,		uint64_t, 1, 'a', "Fetch no body (204)", "")
-VSC_F(fetch_304,		uint64_t, 1, 'a', "Fetch no body (304)", "")
-
-/*---------------------------------------------------------------------
- * Session Memory
- *    see: cache_session.c
- */
-
-VSC_F(sessmem_size,		uint64_t, 1, 'g',
-    "Session mem size",
-	"Bytes of memory allocated for last allocated session."
-)
-
-VSC_F(sessmem_alloc,		uint64_t, 1, 'c',
-    "Session mem allocated",
-	"Count of all allocations of session memory."
-)
-
-VSC_F(sessmem_free,		uint64_t, 1, 'c',
-    "Session mem freed",
-	"Count of all frees of session memory."
-)
-
-VSC_F(sessmem_fail,		uint64_t, 1, 'c',
-    "Session mem alloc failed",
-	"Count of session memory allocation failures."
-)
-
-VSC_F(sessmem_limit,		uint64_t, 1, 'c',
-    "Session mem alloc limited",
-	"Count of session memory allocations blocked by limit (max_sess)."
-)
-
-/*---------------------------------------------------------------------
- * Pools, threads, and sessions
- *    see: cache_pool.c
- *
- */
-
-VSC_F(pools,			uint64_t, 1, 'g',
-    "Number of thread pools",
-	"Number of thread pools.  See also param wthread_pools."
-	"  NB: Presently pools cannot be removed once created."
-)
-
-VSC_F(threads,			uint64_t, 1, 'g',
-    "Total number of threads",
-	"Number of threads in all pools."
-	"  See also params thread_pools, thread_pool_min & thread_pool_max."
-)
-
-VSC_F(threads_limited,		uint64_t, 1, 'c',
-    "Threads hit max",
-	"Number of times more threads were needed, but limit was reached"
-	" in a thread pool."
-	"  See also param thread_pool_max."
-)
-
-VSC_F(threads_created,		uint64_t, 1, 'c',
-    "Threads created",
-	"Total number of threads created in all pools."
-)
-
-VSC_F(threads_destroyed,	uint64_t, 1, 'c',
-    "Threads destoryed",
-	"Total number of threads destroyed in all pools."
-)
-
-VSC_F(threads_failed,		uint64_t, 1, 'c',
-    "Thread creation failed",
-	"Number of times creating a thread failed."
-	"  See VSL::Debug for diagnostics."
-	"  See also param thread_fail_delay."
-)
-
-VSC_F(thread_queue_len,		uint64_t, 1, 'g',
-    "Length of session queue",
-	"Length of session queue waiting for threads."
-	"  NB: Only updates once per second."
-	"  See also param queue_max."
-)
-
-VSC_F(sess_queued,		uint64_t, 1, 'c',
-    "Sessions queued for thread",
-	"Number of times session was queued waiting for a thread."
-	"  See also param queue_max."
-)
-
-VSC_F(sess_dropped,		uint64_t, 1, 'c',
-    "Sessions dropped for thread",
-	"Number of times session was dropped because the queue were too"
-	" long already."
-	"  See also param queue_max."
-)
-
-/*---------------------------------------------------------------------*/
-
-VSC_F(n_sess_mem,		uint64_t, 0, 'i', "N struct sess_mem", "")
-VSC_F(n_sess,			uint64_t, 0, 'i', "N struct sess", "")
-VSC_F(n_object,			uint64_t, 1, 'i', "N struct object", "")
-VSC_F(n_vampireobject,		uint64_t, 1, 'i', "N unresurrected objects", "")
-VSC_F(n_objectcore,		uint64_t, 1, 'i', "N struct objectcore", "")
-VSC_F(n_objecthead,		uint64_t, 1, 'i', "N struct objecthead", "")
-VSC_F(n_waitinglist,		uint64_t, 1, 'i', "N struct waitinglist", "")
-
-VSC_F(n_vbc,		uint64_t, 0, 'i', "N struct vbc", "")
-
-VSC_F(n_backend,		uint64_t, 0, 'i', "N backends", "")
-
-VSC_F(n_expired,		uint64_t, 0, 'i', "N expired objects", "")
-VSC_F(n_lru_nuked,		uint64_t, 0, 'i', "N LRU nuked objects", "")
-VSC_F(n_lru_moved,		uint64_t, 0, 'i', "N LRU moved objects", "")
-
-VSC_F(losthdr,		uint64_t, 0, 'a', "HTTP header overflows", "")
-
-VSC_F(n_objsendfile,	uint64_t, 0, 'a', "Objects sent with sendfile", 
-      "The number of objects sent with the sendfile system call. If enabled "
-      "sendfile will be used on object larger than a certain size.")
-VSC_F(n_objwrite,		uint64_t, 0, 'a', "Objects sent with write", 
-      "The number of objects sent with regular write calls."
-      "Writes are used when the objects are too small for sendfile "
-      "or if the sendfile call has been disabled")
-VSC_F(n_objoverflow,	uint64_t, 1, 'a',
-					"Objects overflowing workspace", "")
-
-VSC_F(s_sess,		uint64_t, 1, 'a', "Total Sessions", "")
-VSC_F(s_req,		uint64_t, 1, 'a', "Total Requests", "")
-VSC_F(s_pipe,		uint64_t, 1, 'a', "Total pipe", "")
-VSC_F(s_pass,		uint64_t, 1, 'a', "Total pass", "")
-VSC_F(s_fetch,		uint64_t, 1, 'a', "Total fetch", "")
-VSC_F(s_hdrbytes,		uint64_t, 1, 'a', "Total header bytes", "")
-VSC_F(s_bodybytes,		uint64_t, 1, 'a', "Total body bytes", "")
-
-VSC_F(sess_closed,		uint64_t, 1, 'a', "Session Closed", "")
-VSC_F(sess_pipeline,	uint64_t, 1, 'a', "Session Pipeline", "")
-VSC_F(sess_readahead,	uint64_t, 1, 'a', "Session Read Ahead", "")
-VSC_F(sess_linger,		uint64_t, 1, 'a', "Session Linger", "")
-VSC_F(sess_herd,		uint64_t, 1, 'a', "Session herd", "")
-
-VSC_F(shm_records,		uint64_t, 0, 'a', "SHM records", "")
-VSC_F(shm_writes,		uint64_t, 0, 'a', "SHM writes", "")
-VSC_F(shm_flushes,		uint64_t, 0, 'a', "SHM flushes due to overflow", "")
-VSC_F(shm_cont,		uint64_t, 0, 'a', "SHM MTX contention", "")
-VSC_F(shm_cycles,		uint64_t, 0, 'a', "SHM cycles through buffer", "")
-
-VSC_F(sms_nreq,		uint64_t, 0, 'a', "SMS allocator requests", "")
-VSC_F(sms_nobj,		uint64_t, 0, 'i', "SMS outstanding allocations", "")
-VSC_F(sms_nbytes,		uint64_t, 0, 'i', "SMS outstanding bytes", "")
-VSC_F(sms_balloc,		uint64_t, 0, 'i', "SMS bytes allocated", "")
-VSC_F(sms_bfree,		uint64_t, 0, 'i', "SMS bytes freed", "")
-
-VSC_F(backend_req,		uint64_t, 0, 'a', "Backend requests made", "")
-
-VSC_F(n_vcl,		uint64_t, 0, 'a', "N vcl total", "")
-VSC_F(n_vcl_avail,		uint64_t, 0, 'a', "N vcl available", "")
-VSC_F(n_vcl_discard,	uint64_t, 0, 'a', "N vcl discarded", "")
-
-VSC_F(n_ban,		uint64_t, 0, 'i', "N total active bans", "")
-VSC_F(n_ban_add,		uint64_t, 0, 'a', "N new bans added", "")
-VSC_F(n_ban_retire,	uint64_t, 0, 'a', "N old bans deleted", "")
-VSC_F(n_ban_obj_test,	uint64_t, 0, 'a', "N objects tested", "")
-VSC_F(n_ban_re_test,	uint64_t, 0, 'a', "N regexps tested against", "")
-VSC_F(n_ban_dups,	uint64_t, 0, 'a', "N duplicate bans removed", "")
-
-VSC_F(hcb_nolock,		uint64_t, 0, 'a', "HCB Lookups without lock", "")
-VSC_F(hcb_lock,		uint64_t, 0, 'a', "HCB Lookups with lock", "")
-VSC_F(hcb_insert,		uint64_t, 0, 'a', "HCB Inserts", "")
-
-VSC_F(esi_errors,		uint64_t, 0, 'a', "ESI parse errors (unlock)", "")
-VSC_F(esi_warnings,		uint64_t, 0, 'a', "ESI parse warnings (unlock)", "")
-VSC_F(client_drop_late,	uint64_t, 0, 'a', "Connection dropped late", "")
-VSC_F(uptime,		uint64_t, 0, 'a', "Client uptime", "")
-
-VSC_F(dir_dns_lookups,	uint64_t, 0, 'a', "DNS director lookups", "")
-VSC_F(dir_dns_failed,	uint64_t, 0, 'a', "DNS director failed lookups", "")
-VSC_F(dir_dns_hit,		uint64_t, 0, 'a', "DNS director cached lookups hit", "")
-VSC_F(dir_dns_cache_full,	uint64_t, 0, 'a', "DNS director full dnscache", "")
-
-VSC_F(vmods,		uint64_t, 0, 'i', "Loaded VMODs", "")
-
-VSC_F(n_gzip,			uint64_t, 0, 'a', "Gzip operations", "")
-VSC_F(n_gunzip,			uint64_t, 0, 'a', "Gunzip operations", "")
-
-#endif
-
-/**********************************************************************/
-
-#ifdef VSC_DO_LCK
-
-VSC_F(creat,		uint64_t, 0, 'a', "Created locks", "")
-VSC_F(destroy,		uint64_t, 0, 'a', "Destroyed locks", "")
-VSC_F(locks,		uint64_t, 0, 'a', "Lock Operations", "")
-VSC_F(colls,		uint64_t, 0, 'a', "Collisions", "")
-
-#endif
-
-/**********************************************************************
- * All Stevedores support these counters
- */
-
-#if defined(VSC_DO_SMA) || defined (VSC_DO_SMF)
-VSC_F(c_req,		uint64_t, 0, 'a', "Allocator requests", "")
-VSC_F(c_fail,		uint64_t, 0, 'a', "Allocator failures", "")
-VSC_F(c_bytes,		uint64_t, 0, 'a', "Bytes allocated", "")
-VSC_F(c_freed,		uint64_t, 0, 'a', "Bytes freed", "")
-VSC_F(g_alloc,		uint64_t, 0, 'i', "Allocations outstanding", "")
-VSC_F(g_bytes,		uint64_t, 0, 'i', "Bytes outstanding", "")
-VSC_F(g_space,		uint64_t, 0, 'i', "Bytes available", "")
-#endif
-
-
-/**********************************************************************/
-
-#ifdef VSC_DO_SMA
-/* No SMA specific counters */
-#endif
-
-/**********************************************************************/
-
-#ifdef VSC_DO_SMF
-VSC_F(g_smf,			uint64_t, 0, 'i', "N struct smf", "")
-VSC_F(g_smf_frag,		uint64_t, 0, 'i', "N small free smf", "")
-VSC_F(g_smf_large,		uint64_t, 0, 'i', "N large free smf", "")
-#endif
-
-/**********************************************************************/
-
-#ifdef VSC_DO_VBE
-
-VSC_F(vcls,			uint64_t, 0, 'i', "VCL references", "")
-VSC_F(happy,		uint64_t, 0, 'b', "Happy health probes", "")
-
-#endif
-
diff --git a/include/vsl.h b/include/vsl.h
index 7f771ce..64b0516 100644
--- a/include/vsl.h
+++ b/include/vsl.h
@@ -74,7 +74,7 @@
 enum VSL_tag_e {
 	SLT_Bogus = 0,
 #define SLTM(foo)	SLT_##foo,
-#include "vsl_tags.h"
+#include "tbl/vsl_tags.h"
 #undef SLTM
 	SLT_Reserved = 255
 };
diff --git a/include/vsl_tags.h b/include/vsl_tags.h
deleted file mode 100644
index 21ca1df..0000000
--- a/include/vsl_tags.h
+++ /dev/null
@@ -1,103 +0,0 @@
-/*-
- * Copyright (c) 2006 Verdens Gang AS
- * Copyright (c) 2006-2010 Varnish Software AS
- * All rights reserved.
- *
- * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- *
- * Define the tags in the shared memory in a reusable format.
- * Whoever includes this get to define what the SLTM macro does.
- *
- * REMEMBER to update the documentation (especially the varnishlog(1) man
- * page) whenever this list changes.
- *
- * XXX: Please add new entries a the end to not break saved log-segments.
- * XXX: we can resort them when we have a major release.
- */
-
-SLTM(Debug)
-SLTM(Error)
-SLTM(CLI)
-SLTM(StatSess)
-SLTM(ReqEnd)
-SLTM(SessionOpen)
-SLTM(SessionClose)
-SLTM(BackendOpen)
-SLTM(BackendXID)
-SLTM(BackendReuse)
-SLTM(BackendClose)
-SLTM(HttpGarbage)
-SLTM(Backend)
-SLTM(Length)
-
-SLTM(FetchError)
-
-SLTM(RxRequest)
-SLTM(RxResponse)
-SLTM(RxStatus)
-SLTM(RxURL)
-SLTM(RxProtocol)
-SLTM(RxHeader)
-
-SLTM(TxRequest)
-SLTM(TxResponse)
-SLTM(TxStatus)
-SLTM(TxURL)
-SLTM(TxProtocol)
-SLTM(TxHeader)
-
-SLTM(ObjRequest)
-SLTM(ObjResponse)
-SLTM(ObjStatus)
-SLTM(ObjURL)
-SLTM(ObjProtocol)
-SLTM(ObjHeader)
-
-SLTM(LostHeader)
-
-SLTM(TTL)
-SLTM(Fetch_Body)
-SLTM(VCL_acl)
-SLTM(VCL_call)
-SLTM(VCL_trace)
-SLTM(VCL_return)
-SLTM(VCL_error)
-SLTM(ReqStart)
-SLTM(Hit)
-SLTM(HitPass)
-SLTM(ExpBan)
-SLTM(ExpKill)
-SLTM(WorkThread)
-
-SLTM(ESI_xmlerror)
-
-SLTM(Hash)
-
-SLTM(Backend_health)
-
-SLTM(VCL_Debug)
-SLTM(VCL_Log)
-SLTM(VCL_Error)
-
-SLTM(Gzip)
diff --git a/lib/libvarnishapi/vsc.c b/lib/libvarnishapi/vsc.c
index 25a1948..18f1fc7 100644
--- a/lib/libvarnishapi/vsc.c
+++ b/lib/libvarnishapi/vsc.c
@@ -309,7 +309,7 @@ iter_call(const struct vsc *vsc, VSC_iter_f *func, void *priv,
 		return (0);						\
 	}
 
-#include "vsc_all.h"
+#include "tbl/vsc_all.h"
 #undef VSC_DO
 #undef VSC_F
 #undef VSC_DONE
@@ -339,7 +339,7 @@ VSC_Iter(struct VSM_data *vd, VSC_iter_f *func, void *priv)
 			if (!i)					\
 				continue;			\
 		}
-#include "vsc_all.h"
+#include "tbl/vsc_all.h"
 #undef VSC_F
 #undef VSC_DO
 #undef VSC_DONE
diff --git a/lib/libvarnishapi/vsl.c b/lib/libvarnishapi/vsl.c
index 8a54c9f..ab538c5 100644
--- a/lib/libvarnishapi/vsl.c
+++ b/lib/libvarnishapi/vsl.c
@@ -53,7 +53,7 @@
 
 const char *VSL_tags[256] = {
 #define SLTM(foo)       [SLT_##foo] = #foo,
-#include "vsl_tags.h"
+#include "tbl/vsl_tags.h"
 #undef SLTM
 };
 
diff --git a/man/vsc2rst.c b/man/vsc2rst.c
index e6abcf9..ad5dec6 100644
--- a/man/vsc2rst.c
+++ b/man/vsc2rst.c
@@ -27,7 +27,7 @@ int main(int argc, char **argv)
 	P("=============");
 	P("");
 #define VSC_DO_MAIN
-#include "vsc_fields.h"
+#include "tbl/vsc_fields.h"
 #undef VSC_DO_MAIN
 
 	P("");
@@ -35,7 +35,7 @@ int main(int argc, char **argv)
 	P("=============");
 	P("");
 #define VSC_DO_LCK
-#include "vsc_fields.h"
+#include "tbl/vsc_fields.h"
 #undef VSC_DO_LCK
 
 	P("");
@@ -43,7 +43,7 @@ int main(int argc, char **argv)
 	P("===========================");
 	P("");
 #define VSC_DO_SMA
-#include "vsc_fields.h"
+#include "tbl/vsc_fields.h"
 #undef  VSC_DO_SMA
 
 	P("");
@@ -51,7 +51,7 @@ int main(int argc, char **argv)
 	P("=========================");
 	P("");
 #define VSC_DO_SMF
-#include "vsc_fields.h"
+#include "tbl/vsc_fields.h"
 #undef VSC_DO_SMF
 
 	P("");
@@ -59,7 +59,7 @@ int main(int argc, char **argv)
 	P("====================");
 	P("");
 #define VSC_DO_VBE
-#include "vsc_fields.h"
+#include "tbl/vsc_fields.h"
 #undef VSC_DO_VBE
 
 	return 0;



From geoff at varnish-cache.org  Mon Jan  9 20:52:14 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:14 +0100
Subject: [experimental-ims] 17937b2 Move the varnishd table building #includes
	under tbl/ as well.
Message-ID: <E1RkMCU-0007Y6-SZ@project.varnish-software.com>

commit 17937b2954fb6b422d8c7b9b0b68a95f46fc825b
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sat Oct 8 14:56:41 2011 +0000

    Move the varnishd table building #includes under tbl/ as well.

diff --git a/bin/varnishd/Makefile.am b/bin/varnishd/Makefile.am
index 0dbc745..b1f2a5c 100644
--- a/bin/varnishd/Makefile.am
+++ b/bin/varnishd/Makefile.am
@@ -78,21 +78,16 @@ varnishd_SOURCES = \
 	vsm.c
 
 noinst_HEADERS = \
-	acct_fields.h \
-	body_status.h \
 	cache.h \
 	cache_backend.h \
-	cache_backend_poll.h \
 	cache_esi.h \
 	cache_waiter.h \
 	common.h \
 	default_vcl.h \
 	hash_slinger.h \
 	heritage.h \
-	locks.h \
 	mgt.h \
 	mgt_cli.h \
-	steps.h \
 	stevedore.h \
 	storage_persistent.h \
 	vparam.h
diff --git a/bin/varnishd/acct_fields.h b/bin/varnishd/acct_fields.h
deleted file mode 100644
index 154f106..0000000
--- a/bin/varnishd/acct_fields.h
+++ /dev/null
@@ -1,40 +0,0 @@
-/*-
- * Copyright (c) 2008 Verdens Gang AS
- * Copyright (c) 2008-2009 Varnish Software AS
- * All rights reserved.
- *
- * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- *
- * These are the stats we keep track of per session.  They will be summed,
- * via the sp->wrk->stats into the s_<name> fields in the SHM file.
- * NB: Remember to mark those in vsc_fields.h to be included in struct dstat.
- */
-
-ACCT(sess)
-ACCT(req)
-ACCT(pipe)
-ACCT(pass)
-ACCT(fetch)
-ACCT(hdrbytes)
-ACCT(bodybytes)
diff --git a/bin/varnishd/body_status.h b/bin/varnishd/body_status.h
deleted file mode 100644
index 33803f9..0000000
--- a/bin/varnishd/body_status.h
+++ /dev/null
@@ -1,38 +0,0 @@
-/*-
- * Copyright (c) 2011 Varnish Software AS
- * All rights reserved.
- *
- * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- *
- * Various ways to handle the body coming from the backend.
- */
-
-/*lint -save -e525 -e539 */
-BODYSTATUS(NONE, none)
-BODYSTATUS(ZERO, zero)
-BODYSTATUS(ERROR, error)
-BODYSTATUS(CHUNKED, chunked)
-BODYSTATUS(LENGTH, length)
-BODYSTATUS(EOF, eof)
-/*lint -restore */
diff --git a/bin/varnishd/cache.h b/bin/varnishd/cache.h
index f46634a..c9b3ac2 100644
--- a/bin/varnishd/cache.h
+++ b/bin/varnishd/cache.h
@@ -67,7 +67,7 @@
 
 enum body_status {
 #define BODYSTATUS(U,l)	BS_##U,
-#include "body_status.h"
+#include "tbl/body_status.h"
 #undef BODYSTATUS
 };
 
@@ -76,7 +76,7 @@ body_status(enum body_status e)
 {
 	switch(e) {
 #define BODYSTATUS(U,l)	case BS_##U: return (#l);
-#include "body_status.h"
+#include "tbl/body_status.h"
 #undef BODYSTATUS
 	default:
 		return ("?");
@@ -143,7 +143,7 @@ typedef struct {
 
 enum step {
 #define STEP(l, u)	STP_##u,
-#include "steps.h"
+#include "tbl/steps.h"
 #undef STEP
 };
 
@@ -213,7 +213,7 @@ struct http_conn {
 struct acct {
 	double			first;
 #define ACCT(foo)	uint64_t	foo;
-#include "acct_fields.h"
+#include "tbl/acct_fields.h"
 #undef ACCT
 };
 
@@ -834,7 +834,7 @@ int Lck_CondWait(pthread_cond_t *cond, struct lock *lck, struct timespec *ts);
 #define Lck_AssertHeld(a) Lck__Assert(a, 1)
 
 #define LOCK(nam) extern struct VSC_C_lck *lck_##nam;
-#include "locks.h"
+#include "tbl/locks.h"
 #undef LOCK
 
 /* cache_panic.c */
diff --git a/bin/varnishd/cache_backend_poll.c b/bin/varnishd/cache_backend_poll.c
index 5fae557..be8ee6a 100644
--- a/bin/varnishd/cache_backend_poll.c
+++ b/bin/varnishd/cache_backend_poll.c
@@ -78,7 +78,7 @@ struct vbp_target {
 
 	/* Collected statistics */
 #define BITMAP(n, c, t, b)	uint64_t	n;
-#include "cache_backend_poll.h"
+#include "tbl/backend_poll.h"
 #undef BITMAP
 
 	double				last;
@@ -236,7 +236,7 @@ vbp_start_poke(struct vbp_target *vt)
 	CHECK_OBJ_NOTNULL(vt, VBP_TARGET_MAGIC);
 
 #define BITMAP(n, c, t, b)	vt->n <<= 1;
-#include "cache_backend_poll.h"
+#include "tbl/backend_poll.h"
 #undef BITMAP
 
 	vt->last = 0;
@@ -262,7 +262,7 @@ vbp_has_poked(struct vbp_target *vt)
 
 	i = 0;
 #define BITMAP(n, c, t, b)	bits[i++] = (vt->n & 1) ? c : '-';
-#include "cache_backend_poll.h"
+#include "tbl/backend_poll.h"
 #undef BITMAP
 	bits[i] = '\0';
 
@@ -396,7 +396,7 @@ vbp_health_one(struct cli *cli, const struct vbp_target *vt)
 #define BITMAP(n, c, t, b)					\
 		if ((vt->n != 0) || (b))			\
 			vbp_bitmap(cli, (c), vt->n, (t));
-#include "cache_backend_poll.h"
+#include "tbl/backend_poll.h"
 #undef BITMAP
 }
 
diff --git a/bin/varnishd/cache_backend_poll.h b/bin/varnishd/cache_backend_poll.h
deleted file mode 100644
index a9ee3a3..0000000
--- a/bin/varnishd/cache_backend_poll.h
+++ /dev/null
@@ -1,36 +0,0 @@
-/*-
- * Copyright (c) 2008-2011 Varnish Software AS
- * All rights reserved.
- *
- * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- *
- */
-
-BITMAP(good_ipv4, '4', "Good IPv4", 0)
-BITMAP(good_ipv6, '6', "Good IPv6", 0)
-BITMAP( err_xmit, 'x', "Error Xmit", 0)
-BITMAP(good_xmit, 'X', "Good Xmit", 0)
-BITMAP( err_recv, 'r', "Error Recv", 0)
-BITMAP(good_recv, 'R', "Good Recv", 0)
-BITMAP(happy,     'H', "Happy", 1)
diff --git a/bin/varnishd/cache_center.c b/bin/varnishd/cache_center.c
index b4a056d..7e91137 100644
--- a/bin/varnishd/cache_center.c
+++ b/bin/varnishd/cache_center.c
@@ -1605,7 +1605,7 @@ CNT_Session(struct sess *sp)
 				cnt_diag(sp, #u); \
 			done = cnt_##l(sp); \
 		        break;
-#include "steps.h"
+#include "tbl/steps.h"
 #undef STEP
 		default:
 			WRONG("State engine misfire");
@@ -1621,7 +1621,7 @@ CNT_Session(struct sess *sp)
 	AZ(w->do_gunzip);
 	AZ(w->do_esi);
 #define ACCT(foo)	AZ(w->acct_tmp.foo);
-#include "acct_fields.h"
+#include "tbl/acct_fields.h"
 #undef ACCT
 	assert(WRW_IsReleased(w));
 }
diff --git a/bin/varnishd/cache_lck.c b/bin/varnishd/cache_lck.c
index b9229b3..18d1b41 100644
--- a/bin/varnishd/cache_lck.c
+++ b/bin/varnishd/cache_lck.c
@@ -196,7 +196,7 @@ Lck_Delete(struct lock *lck)
 }
 
 #define LOCK(nam) struct VSC_C_lck *lck_##nam;
-#include "locks.h"
+#include "tbl/locks.h"
 #undef LOCK
 
 void
@@ -207,6 +207,6 @@ LCK_Init(void)
 #define LOCK(nam)						\
 	lck_##nam = VSM_Alloc(sizeof(struct VSC_C_lck),		\
 	   VSC_CLASS, VSC_TYPE_LCK, #nam);
-#include "locks.h"
+#include "tbl/locks.h"
 #undef LOCK
 }
diff --git a/bin/varnishd/cache_panic.c b/bin/varnishd/cache_panic.c
index 8ca1b4f..a9c5c15 100644
--- a/bin/varnishd/cache_panic.c
+++ b/bin/varnishd/cache_panic.c
@@ -223,7 +223,7 @@ pan_sess(const struct sess *sp)
 	    sp->port ? sp->port : "?");
 	switch (sp->step) {
 #define STEP(l, u) case STP_##u: stp = "STP_" #u; break;
-#include "steps.h"
+#include "tbl/steps.h"
 #undef STEP
 		default: stp = NULL;
 	}
diff --git a/bin/varnishd/cache_session.c b/bin/varnishd/cache_session.c
index 1f500ba..bb96780 100644
--- a/bin/varnishd/cache_session.c
+++ b/bin/varnishd/cache_session.c
@@ -83,7 +83,7 @@ SES_Charge(struct sess *sp)
 	sp->wrk->stats.s_##foo += a->foo;	\
 	sp->acct_ses.foo += a->foo;		\
 	a->foo = 0;
-#include "acct_fields.h"
+#include "tbl/acct_fields.h"
 #undef ACCT
 }
 
diff --git a/bin/varnishd/locks.h b/bin/varnishd/locks.h
deleted file mode 100644
index f1b634b..0000000
--- a/bin/varnishd/locks.h
+++ /dev/null
@@ -1,52 +0,0 @@
-/*-
- * Copyright (c) 2010 Varnish Software AS
- * All rights reserved.
- *
- * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- *
- */
-
-/*lint -save -e525 -e539 */
-LOCK(sms)
-LOCK(smp)
-LOCK(sma)
-LOCK(smf)
-LOCK(hsl)
-LOCK(hcb)
-LOCK(hcl)
-LOCK(vcl)
-LOCK(sessmem)
-LOCK(wstat)
-LOCK(herder)
-LOCK(wq)
-LOCK(objhdr)
-LOCK(exp)
-LOCK(lru)
-LOCK(cli)
-LOCK(ban)
-LOCK(vbp)
-LOCK(vbe)
-LOCK(backend)
-LOCK(vcapace)
-/*lint -restore */
diff --git a/bin/varnishd/steps.h b/bin/varnishd/steps.h
deleted file mode 100644
index c88e434..0000000
--- a/bin/varnishd/steps.h
+++ /dev/null
@@ -1,48 +0,0 @@
-/*-
- * Copyright (c) 2006 Verdens Gang AS
- * Copyright (c) 2006-2011 Varnish Software AS
- * All rights reserved.
- *
- * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- *
- */
-
-/*lint -save -e525 -e539 */
-STEP(wait,	WAIT)
-STEP(first,	FIRST)
-STEP(recv,	RECV)
-STEP(start,	START)
-STEP(pipe,	PIPE)
-STEP(pass,	PASS)
-STEP(lookup,	LOOKUP)
-STEP(miss,	MISS)
-STEP(hit,	HIT)
-STEP(fetch,	FETCH)
-STEP(fetchbody,	FETCHBODY)
-STEP(streambody,STREAMBODY)
-STEP(prepresp,	PREPRESP)
-STEP(deliver,	DELIVER)
-STEP(error,	ERROR)
-STEP(done,	DONE)
-/*lint -restore */
diff --git a/include/Makefile.am b/include/Makefile.am
index de1b19a..652689b 100644
--- a/include/Makefile.am
+++ b/include/Makefile.am
@@ -1,6 +1,11 @@
 #
 
 pkginclude_HEADERS = \
+	tbl/acct_fields.h \
+	tbl/backend_poll.h \
+	tbl/body_status.h \
+	tbl/locks.h \
+	tbl/steps.h \
 	tbl/vsc_all.h \
 	tbl/vsc_fields.h \
 	tbl/vsl_tags.h \
diff --git a/include/tbl/acct_fields.h b/include/tbl/acct_fields.h
new file mode 100644
index 0000000..154f106
--- /dev/null
+++ b/include/tbl/acct_fields.h
@@ -0,0 +1,40 @@
+/*-
+ * Copyright (c) 2008 Verdens Gang AS
+ * Copyright (c) 2008-2009 Varnish Software AS
+ * All rights reserved.
+ *
+ * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * These are the stats we keep track of per session.  They will be summed,
+ * via the sp->wrk->stats into the s_<name> fields in the SHM file.
+ * NB: Remember to mark those in vsc_fields.h to be included in struct dstat.
+ */
+
+ACCT(sess)
+ACCT(req)
+ACCT(pipe)
+ACCT(pass)
+ACCT(fetch)
+ACCT(hdrbytes)
+ACCT(bodybytes)
diff --git a/include/tbl/backend_poll.h b/include/tbl/backend_poll.h
new file mode 100644
index 0000000..a9ee3a3
--- /dev/null
+++ b/include/tbl/backend_poll.h
@@ -0,0 +1,36 @@
+/*-
+ * Copyright (c) 2008-2011 Varnish Software AS
+ * All rights reserved.
+ *
+ * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ */
+
+BITMAP(good_ipv4, '4', "Good IPv4", 0)
+BITMAP(good_ipv6, '6', "Good IPv6", 0)
+BITMAP( err_xmit, 'x', "Error Xmit", 0)
+BITMAP(good_xmit, 'X', "Good Xmit", 0)
+BITMAP( err_recv, 'r', "Error Recv", 0)
+BITMAP(good_recv, 'R', "Good Recv", 0)
+BITMAP(happy,     'H', "Happy", 1)
diff --git a/include/tbl/body_status.h b/include/tbl/body_status.h
new file mode 100644
index 0000000..33803f9
--- /dev/null
+++ b/include/tbl/body_status.h
@@ -0,0 +1,38 @@
+/*-
+ * Copyright (c) 2011 Varnish Software AS
+ * All rights reserved.
+ *
+ * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * Various ways to handle the body coming from the backend.
+ */
+
+/*lint -save -e525 -e539 */
+BODYSTATUS(NONE, none)
+BODYSTATUS(ZERO, zero)
+BODYSTATUS(ERROR, error)
+BODYSTATUS(CHUNKED, chunked)
+BODYSTATUS(LENGTH, length)
+BODYSTATUS(EOF, eof)
+/*lint -restore */
diff --git a/include/tbl/locks.h b/include/tbl/locks.h
new file mode 100644
index 0000000..f1b634b
--- /dev/null
+++ b/include/tbl/locks.h
@@ -0,0 +1,52 @@
+/*-
+ * Copyright (c) 2010 Varnish Software AS
+ * All rights reserved.
+ *
+ * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ */
+
+/*lint -save -e525 -e539 */
+LOCK(sms)
+LOCK(smp)
+LOCK(sma)
+LOCK(smf)
+LOCK(hsl)
+LOCK(hcb)
+LOCK(hcl)
+LOCK(vcl)
+LOCK(sessmem)
+LOCK(wstat)
+LOCK(herder)
+LOCK(wq)
+LOCK(objhdr)
+LOCK(exp)
+LOCK(lru)
+LOCK(cli)
+LOCK(ban)
+LOCK(vbp)
+LOCK(vbe)
+LOCK(backend)
+LOCK(vcapace)
+/*lint -restore */
diff --git a/include/tbl/steps.h b/include/tbl/steps.h
new file mode 100644
index 0000000..c88e434
--- /dev/null
+++ b/include/tbl/steps.h
@@ -0,0 +1,48 @@
+/*-
+ * Copyright (c) 2006 Verdens Gang AS
+ * Copyright (c) 2006-2011 Varnish Software AS
+ * All rights reserved.
+ *
+ * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ */
+
+/*lint -save -e525 -e539 */
+STEP(wait,	WAIT)
+STEP(first,	FIRST)
+STEP(recv,	RECV)
+STEP(start,	START)
+STEP(pipe,	PIPE)
+STEP(pass,	PASS)
+STEP(lookup,	LOOKUP)
+STEP(miss,	MISS)
+STEP(hit,	HIT)
+STEP(fetch,	FETCH)
+STEP(fetchbody,	FETCHBODY)
+STEP(streambody,STREAMBODY)
+STEP(prepresp,	PREPRESP)
+STEP(deliver,	DELIVER)
+STEP(error,	ERROR)
+STEP(done,	DONE)
+/*lint -restore */



From geoff at varnish-cache.org  Mon Jan  9 20:52:15 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:15 +0100
Subject: [experimental-ims] 4a1413b Remove unused #includes
Message-ID: <E1RkMCV-0007YV-2Y@project.varnish-software.com>

commit 4a1413b3b0ac8f80565ba781e2926b8f93167ae2
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sat Oct 8 15:21:27 2011 +0000

    Remove unused #includes

diff --git a/bin/varnishd/cache_acceptor.c b/bin/varnishd/cache_acceptor.c
index c02e170..af5edec 100644
--- a/bin/varnishd/cache_acceptor.c
+++ b/bin/varnishd/cache_acceptor.c
@@ -30,11 +30,10 @@
 
 #include "config.h"
 
-#include <errno.h>
+#include "cache.h"
 
 #include "vcli.h"
 #include "cli_priv.h"
-#include "cache.h"
 
 static pthread_t	VCA_thread;
 static struct timeval	tv_sndtimeo;
diff --git a/bin/varnishd/cache_ban.c b/bin/varnishd/cache_ban.c
index fe1efd0..09e68cc 100644
--- a/bin/varnishd/cache_ban.c
+++ b/bin/varnishd/cache_ban.c
@@ -62,19 +62,16 @@
 
 #include "config.h"
 
-#include <sys/types.h>
-
 #include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
 #include <stdint.h>
 
 #include <pcre.h>
 
+#include "cache.h"
+
 #include "vcli.h"
 #include "vend.h"
 #include "cli_priv.h"
-#include "cache.h"
 #include "hash_slinger.h"
 
 struct ban {
diff --git a/bin/varnishd/cache_center.c b/bin/varnishd/cache_center.c
index 7e91137..0b7f8bf 100644
--- a/bin/varnishd/cache_center.c
+++ b/bin/varnishd/cache_center.c
@@ -62,16 +62,15 @@ DOT acceptor -> start [style=bold,color=green]
 #include <math.h>
 #include <poll.h>
 #include <stdlib.h>
-#include <string.h>
-#include <unistd.h>
 
 #ifndef HAVE_SRANDOMDEV
 #include "compat/srandomdev.h"
 #endif
 
+#include "cache.h"
+
 #include "vcl.h"
 #include "cli_priv.h"
-#include "cache.h"
 #include "hash_slinger.h"
 #include "stevedore.h"
 #include "vsha256.h"
diff --git a/bin/varnishd/cache_cli.c b/bin/varnishd/cache_cli.c
index bfc1cbc..3635aec 100644
--- a/bin/varnishd/cache_cli.c
+++ b/bin/varnishd/cache_cli.c
@@ -36,14 +36,14 @@
 
 #include "config.h"
 
-#include <stdio.h>
 #include <stddef.h>			// offsetof
 
+#include "cache.h"
+
 #include "vcli.h"
 #include "cli_priv.h"
 #include "cli_common.h"
 #include "cli_serve.h"
-#include "cache.h"
 #include "hash_slinger.h"		// objhead
 
 pthread_t		cli_thread;
diff --git a/bin/varnishd/cache_http.c b/bin/varnishd/cache_http.c
index bd0380b..3703b00 100644
--- a/bin/varnishd/cache_http.c
+++ b/bin/varnishd/cache_http.c
@@ -31,15 +31,14 @@
 
 #include "config.h"
 
-#include <unistd.h>
 #include <stdarg.h>
 #include <stdio.h>
-#include <stdlib.h>
 #include <string.h>
 
-#include "vct.h"
 #include "cache.h"
 
+#include "vct.h"
+
 #define HTTPH(a, b, c, d, e, f, g) char b[] = "*" a ":";
 #include "tbl/http_headers.h"
 #undef HTTPH
diff --git a/bin/varnishd/cache_httpconn.c b/bin/varnishd/cache_httpconn.c
index 3ba5d7f..d1ceb6e 100644
--- a/bin/varnishd/cache_httpconn.c
+++ b/bin/varnishd/cache_httpconn.c
@@ -31,14 +31,13 @@
 
 #include "config.h"
 
-#include <stdio.h>
 #include <stdlib.h>
-#include <ctype.h>
 #include <string.h>
-#include <unistd.h>
 
 #include "cache.h"
 
+#include "vct.h"
+
 /*--------------------------------------------------------------------
  * Check if we have a complete HTTP request or response yet
  *
@@ -55,7 +54,7 @@ htc_header_complete(txt *t)
 	Tcheck(*t);
 	assert(*t->e == '\0');
 	/* Skip any leading white space */
-	for (p = t->b ; isspace(*p); p++)
+	for (p = t->b ; vct_issp(*p); p++)
 		continue;
 	if (*p == '\0') {
 		t->e = t->b;
diff --git a/bin/varnishd/cache_pool.c b/bin/varnishd/cache_pool.c
index 1aaa2ac..c5183a4 100644
--- a/bin/varnishd/cache_pool.c
+++ b/bin/varnishd/cache_pool.c
@@ -42,17 +42,13 @@
 
 #include "config.h"
 
-#include <sys/types.h>
-
 #include <errno.h>
-#include <stdio.h>
 #include <math.h>
 #include <stdlib.h>
-#include <string.h>
 
 #include "cache.h"
+
 #include "cache_waiter.h"
-#include "stevedore.h"
 #include "hash_slinger.h"
 
 /*--------------------------------------------------------------------
diff --git a/bin/varnishd/cache_vrt.c b/bin/varnishd/cache_vrt.c
index 86cb10f..02d2a42 100644
--- a/bin/varnishd/cache_vrt.c
+++ b/bin/varnishd/cache_vrt.c
@@ -31,22 +31,19 @@
 
 #include "config.h"
 
-#include <sys/types.h>
-#include <sys/socket.h>
-
 #include <netinet/in.h>
 #include <arpa/inet.h>
 
 #include <stdio.h>
-#include <math.h>
 #include <string.h>
 #include <stdlib.h>
 #include <stdarg.h>
 
+#include "cache.h"
+
 #include "vrt.h"
 #include "vrt_obj.h"
 #include "vcl.h"
-#include "cache.h"
 #include "hash_slinger.h"
 #include "cache_backend.h"
 
diff --git a/bin/varnishd/cache_vrt_re.c b/bin/varnishd/cache_vrt_re.c
index ef2376f..d28510f 100644
--- a/bin/varnishd/cache_vrt_re.c
+++ b/bin/varnishd/cache_vrt_re.c
@@ -34,10 +34,10 @@
 #include <ctype.h>
 #include <stdlib.h>
 
+#include "cache.h"
+
 #include "vrt.h"
 #include "vre.h"
-#include "vcl.h"
-#include "cache.h"
 
 void
 VRT_re_init(void **rep, const char *re)
diff --git a/bin/varnishd/cache_vrt_vmod.c b/bin/varnishd/cache_vrt_vmod.c
index 557b01b..4eb3e84 100644
--- a/bin/varnishd/cache_vrt_vmod.c
+++ b/bin/varnishd/cache_vrt_vmod.c
@@ -31,13 +31,13 @@
 
 #include "config.h"
 
-#include <stdio.h>
 #include <stdlib.h>
 #include <dlfcn.h>
 
+#include "cache.h"
+
 #include "cli_priv.h"
 #include "vrt.h"
-#include "cache.h"
 
 /*--------------------------------------------------------------------
  * Modules stuff
diff --git a/bin/varnishd/cache_waiter_poll.c b/bin/varnishd/cache_waiter_poll.c
index 4678255..df0439b 100644
--- a/bin/varnishd/cache_waiter_poll.c
+++ b/bin/varnishd/cache_waiter_poll.c
@@ -30,14 +30,11 @@
 
 #include "config.h"
 
-#include <stdio.h>
-#include <errno.h>
-#include <string.h>
 #include <stdlib.h>
-#include <unistd.h>
 #include <poll.h>
 
 #include "cache.h"
+
 #include "cache_waiter.h"
 
 #define NEEV	128
diff --git a/bin/varnishd/cache_wrk.c b/bin/varnishd/cache_wrk.c
index bb88664..7e80f04 100644
--- a/bin/varnishd/cache_wrk.c
+++ b/bin/varnishd/cache_wrk.c
@@ -31,11 +31,9 @@
 
 #include "config.h"
 
-#include <stdio.h>
 #include <math.h>
 #include <stdlib.h>
 
-#include "vcl.h"
 #include "cache.h"
 #include "hash_slinger.h"
 #include "vsha256.h"
diff --git a/bin/varnishd/mgt_child.c b/bin/varnishd/mgt_child.c
index 3d6fa28..ba2effe 100644
--- a/bin/varnishd/mgt_child.c
+++ b/bin/varnishd/mgt_child.c
@@ -39,7 +39,6 @@
 #include <signal.h>
 #include <syslog.h>
 #include <errno.h>
-#include <poll.h>
 #include <sys/types.h>
 #include <sys/wait.h>
 
diff --git a/bin/varnishd/mgt_param.c b/bin/varnishd/mgt_param.c
index f247b71..7138369 100644
--- a/bin/varnishd/mgt_param.c
+++ b/bin/varnishd/mgt_param.c
@@ -29,9 +29,6 @@
 
 #include "config.h"
 
-#include <sys/types.h>
-#include <sys/stat.h>
-
 #include <grp.h>
 #include <limits.h>
 #include <pwd.h>
diff --git a/bin/varnishd/rfc2616.c b/bin/varnishd/rfc2616.c
index 51d4c74..36936b6 100644
--- a/bin/varnishd/rfc2616.c
+++ b/bin/varnishd/rfc2616.c
@@ -29,13 +29,10 @@
 
 #include "config.h"
 
-#include <stdio.h>
 #include <stdlib.h>
-#include <limits.h>
 #include <math.h>
 
 #include "cache.h"
-#include "vrt.h"
 
 /*--------------------------------------------------------------------
  * TTL and Age calculation in Varnish
diff --git a/bin/varnishd/storage_file.c b/bin/varnishd/storage_file.c
index 7388878..c363f4e 100644
--- a/bin/varnishd/storage_file.c
+++ b/bin/varnishd/storage_file.c
@@ -31,16 +31,12 @@
 
 #include "config.h"
 
-#include <sys/param.h>
 #include <sys/mman.h>
-#include <sys/stat.h>
-
 #include <stdio.h>
 #include <stdlib.h>
-#include <string.h>
-#include <unistd.h>
 
 #include "cache.h"
+
 #include "stevedore.h"
 
 #ifndef MAP_NOCORE
diff --git a/bin/varnishd/storage_malloc.c b/bin/varnishd/storage_malloc.c
index f72cd48..500e74c 100644
--- a/bin/varnishd/storage_malloc.c
+++ b/bin/varnishd/storage_malloc.c
@@ -31,12 +31,11 @@
 
 #include "config.h"
 
-#include <sys/types.h>
-
 #include <stdio.h>
 #include <stdlib.h>
 
 #include "cache.h"
+
 #include "stevedore.h"
 
 struct sma_sc {
diff --git a/bin/varnishd/storage_synth.c b/bin/varnishd/storage_synth.c
index 1c16b44..0792dbd 100644
--- a/bin/varnishd/storage_synth.c
+++ b/bin/varnishd/storage_synth.c
@@ -30,14 +30,11 @@
 
 #include "config.h"
 
-#include <sys/types.h>
-
-#include <stdio.h>
 #include <stdlib.h>
 
 #include "cache.h"
+
 #include "stevedore.h"
-#include "hash_slinger.h"
 
 static struct lock		sms_mtx;
 
diff --git a/bin/varnishd/varnishd.c b/bin/varnishd/varnishd.c
index 8b35970..0219eb0 100644
--- a/bin/varnishd/varnishd.c
+++ b/bin/varnishd/varnishd.c
@@ -40,7 +40,6 @@
 #include <stdlib.h>
 #include <string.h>
 #include <syslog.h>
-#include <time.h>
 #include <unistd.h>
 
 #include <sys/utsname.h>
@@ -52,7 +51,6 @@
 #include "vsha256.h"
 
 #include "vcli.h"
-#include "cli_priv.h"
 #include "cli_common.h"
 
 #include "vin.h"



From geoff at varnish-cache.org  Mon Jan  9 20:52:15 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:15 +0100
Subject: [experimental-ims] b18e8d0 Make distcheck work again.
Message-ID: <E1RkMCV-0007Yr-7T@project.varnish-software.com>

commit b18e8d0c9309ccdc5fa4c2c4d3ea8fb5bfd617c0
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sat Oct 8 17:38:05 2011 +0000

    Make distcheck work again.

diff --git a/include/Makefile.am b/include/Makefile.am
index 9f317f4..e40ea2a 100644
--- a/include/Makefile.am
+++ b/include/Makefile.am
@@ -55,6 +55,7 @@ nobase_noinst_HEADERS = \
 	vss.h
 
 tbl/vrt_stv_var.h tbl/vcl_returns.h vcl.h vrt_obj.h: $(top_srcdir)/lib/libvcl/generate.py $(top_srcdir)/include/vrt.h
+	mkdir -p tbl
 	@PYTHON@ $(top_srcdir)/lib/libvcl/generate.py $(top_srcdir) $(top_builddir)
 
 BUILT_SOURCES = vcs_version.h vmod_abi.h



From geoff at varnish-cache.org  Mon Jan  9 20:52:15 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:15 +0100
Subject: [experimental-ims] c5c9e33 Eliminate a redundant include file
Message-ID: <E1RkMCV-0007Zc-HB@project.varnish-software.com>

commit c5c9e33ee5f2bf4dd278841a6d82e25a647acef5
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sun Oct 9 08:42:03 2011 +0000

    Eliminate a redundant include file

diff --git a/lib/libvcl/Makefile.am b/lib/libvcl/Makefile.am
index 099ad36..8406ed7 100644
--- a/lib/libvcl/Makefile.am
+++ b/lib/libvcl/Makefile.am
@@ -7,7 +7,6 @@ pkglib_LTLIBRARIES = libvcl.la
 libvcl_la_LDFLAGS = -avoid-version
 
 libvcl_la_SOURCES = \
-	vcc_priv.h \
 	vcc_compile.h \
 	vcc_token_defs.h \
 	symbol_kind.h \
diff --git a/lib/libvcl/generate.py b/lib/libvcl/generate.py
index 335cfd5..9342aaf 100755
--- a/lib/libvcl/generate.py
+++ b/lib/libvcl/generate.py
@@ -866,8 +866,7 @@ fo.write("""
 #include <stdio.h>
 #include <ctype.h>
 #include "config.h"
-#include "vcc_priv.h"
-#include "vsb.h"
+#include "vcc_compile.h"
 """)
 
 emit_vcl_fixed_token(fo, tokens)
diff --git a/lib/libvcl/vcc_acl.c b/lib/libvcl/vcc_acl.c
index bd94a5b..985a18e 100644
--- a/lib/libvcl/vcc_acl.c
+++ b/lib/libvcl/vcc_acl.c
@@ -40,8 +40,6 @@
 
 #include "vcc_compile.h"
 #include "vrt.h"
-#include "vcc_priv.h"
-#include "libvarnish.h"
 
 struct acl_e {
 	VTAILQ_ENTRY(acl_e)	list;
diff --git a/lib/libvcl/vcc_action.c b/lib/libvcl/vcc_action.c
index 1fafe98..61bfb87 100644
--- a/lib/libvcl/vcc_action.c
+++ b/lib/libvcl/vcc_action.c
@@ -34,8 +34,6 @@
 
 
 #include "vcc_compile.h"
-#include "vcc_priv.h"
-#include "libvarnish.h"
 
 /*--------------------------------------------------------------------*/
 
diff --git a/lib/libvcl/vcc_backend.c b/lib/libvcl/vcc_backend.c
index 67fecd6..0c72c1c 100644
--- a/lib/libvcl/vcc_backend.c
+++ b/lib/libvcl/vcc_backend.c
@@ -59,8 +59,6 @@
 #include <string.h>
 
 #include "vcc_compile.h"
-#include "vcc_priv.h"
-#include "libvarnish.h"
 #include "vss.h"
 
 struct host {
diff --git a/lib/libvcl/vcc_backend_util.c b/lib/libvcl/vcc_backend_util.c
index 8d2cc1a..28510f5 100644
--- a/lib/libvcl/vcc_backend_util.c
+++ b/lib/libvcl/vcc_backend_util.c
@@ -34,9 +34,7 @@
 #include <stdarg.h>
 #include <string.h>
 
-#include "vcc_priv.h"
 #include "vcc_compile.h"
-#include "libvarnish.h"
 
 /*--------------------------------------------------------------------
  * Helper functions to complain about duplicate and missing fields
diff --git a/lib/libvcl/vcc_compile.c b/lib/libvcl/vcc_compile.c
index be3ec63..3af30c6 100644
--- a/lib/libvcl/vcc_compile.c
+++ b/lib/libvcl/vcc_compile.c
@@ -60,10 +60,7 @@
 #include <string.h>
 
 #include "vcc_compile.h"
-#include "vcc_priv.h"
-
 #include "libvcl.h"
-#include "libvarnish.h"
 
 struct method method_tab[] = {
 #define VCL_MET_MAC(l,U,m)	{ "vcl_"#l, m, VCL_MET_##U },
diff --git a/lib/libvcl/vcc_compile.h b/lib/libvcl/vcc_compile.h
index 438fe58..141f4f1 100644
--- a/lib/libvcl/vcc_compile.h
+++ b/lib/libvcl/vcc_compile.h
@@ -33,6 +33,20 @@
 #include "miniobj.h"
 #include "vsb.h"
 #include "vcl.h"
+#include "libvarnish.h"
+
+#include "vcc_token_defs.h"
+
+struct vsb;
+
+#define isident1(c) (isalpha(c))
+#define isident(c) (isalpha(c) || isdigit(c) || (c) == '_' || (c) == '-')
+#define isvar(c) (isident(c) || (c) == '.')
+unsigned vcl_fixed_token(const char *p, const char **q);
+extern const char * const vcl_tnames[256];
+void vcl_output_lang_h(struct vsb *sb);
+
+#define PF(t)	(int)((t)->e - (t)->b), (t)->b
 
 #define INDENT		2
 
diff --git a/lib/libvcl/vcc_dir_dns.c b/lib/libvcl/vcc_dir_dns.c
index a5952c8..2e3951e 100644
--- a/lib/libvcl/vcc_dir_dns.c
+++ b/lib/libvcl/vcc_dir_dns.c
@@ -33,8 +33,6 @@
 #include <limits.h>
 
 #include "vcc_compile.h"
-#include "vcc_priv.h"
-#include "libvarnish.h"
 
 /*--------------------------------------------------------------------
  * Parse directors
diff --git a/lib/libvcl/vcc_dir_random.c b/lib/libvcl/vcc_dir_random.c
index 46f74c2..4166918 100644
--- a/lib/libvcl/vcc_dir_random.c
+++ b/lib/libvcl/vcc_dir_random.c
@@ -30,8 +30,6 @@
 #include "config.h"
 
 #include "vcc_compile.h"
-#include "vcc_priv.h"
-#include "libvarnish.h"
 
 /*--------------------------------------------------------------------
  * Parse directors
diff --git a/lib/libvcl/vcc_dir_round_robin.c b/lib/libvcl/vcc_dir_round_robin.c
index b1094fe..835d969 100644
--- a/lib/libvcl/vcc_dir_round_robin.c
+++ b/lib/libvcl/vcc_dir_round_robin.c
@@ -29,8 +29,6 @@
 #include "config.h"
 
 #include "vcc_compile.h"
-#include "vcc_priv.h"
-#include "libvarnish.h"
 
 /*--------------------------------------------------------------------
  * Parse directors
diff --git a/lib/libvcl/vcc_expr.c b/lib/libvcl/vcc_expr.c
index 4fbcd85..dc7d99d 100644
--- a/lib/libvcl/vcc_expr.c
+++ b/lib/libvcl/vcc_expr.c
@@ -37,9 +37,7 @@
 #include <string.h>
 #include <math.h>
 
-#include "vcc_priv.h"
 #include "vcc_compile.h"
-#include "libvarnish.h"
 
 static const char *
 vcc_Type(enum var_type fmt)
diff --git a/lib/libvcl/vcc_parse.c b/lib/libvcl/vcc_parse.c
index 23c9bf7..4023287 100644
--- a/lib/libvcl/vcc_parse.c
+++ b/lib/libvcl/vcc_parse.c
@@ -32,9 +32,7 @@
 #include <stdio.h>
 #include <string.h>
 
-#include "vcc_priv.h"
 #include "vcc_compile.h"
-#include "libvarnish.h"
 
 /*--------------------------------------------------------------------*/
 
diff --git a/lib/libvcl/vcc_priv.h b/lib/libvcl/vcc_priv.h
deleted file mode 100644
index 1e990b2..0000000
--- a/lib/libvcl/vcc_priv.h
+++ /dev/null
@@ -1,43 +0,0 @@
-/*-
- * Copyright (c) 2006 Verdens Gang AS
- * Copyright (c) 2006-2009 Varnish Software AS
- * All rights reserved.
- *
- * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- *
- * Stuff shared between main.c and fixed_token.c
- */
-
-#include "vcc_token_defs.h"
-
-struct vsb;
-
-#define isident1(c) (isalpha(c))
-#define isident(c) (isalpha(c) || isdigit(c) || (c) == '_' || (c) == '-')
-#define isvar(c) (isident(c) || (c) == '.')
-unsigned vcl_fixed_token(const char *p, const char **q);
-extern const char * const vcl_tnames[256];
-void vcl_output_lang_h(struct vsb *sb);
-
-#define PF(t)	(int)((t)->e - (t)->b), (t)->b
diff --git a/lib/libvcl/vcc_storage.c b/lib/libvcl/vcc_storage.c
index 5aff6e7..5f9712a 100644
--- a/lib/libvcl/vcc_storage.c
+++ b/lib/libvcl/vcc_storage.c
@@ -59,9 +59,7 @@
 #include <stdlib.h>
 #include <string.h>
 
-#include "vcc_priv.h"
 #include "vcc_compile.h"
-#include "libvarnish.h"
 
 #define PFX "storage."
 
diff --git a/lib/libvcl/vcc_string.c b/lib/libvcl/vcc_string.c
index c6353d7..dfc82cf 100644
--- a/lib/libvcl/vcc_string.c
+++ b/lib/libvcl/vcc_string.c
@@ -32,9 +32,7 @@
 #include <stdio.h>
 #include <string.h>
 
-#include "vcc_priv.h"
 #include "vcc_compile.h"
-#include "libvarnish.h"
 
 #include "vrt.h"
 #include "vre.h"
diff --git a/lib/libvcl/vcc_symb.c b/lib/libvcl/vcc_symb.c
index 4975c12..0acb0cd 100644
--- a/lib/libvcl/vcc_symb.c
+++ b/lib/libvcl/vcc_symb.c
@@ -32,9 +32,7 @@
 #include <stdlib.h>
 #include <string.h>
 
-#include "vcc_priv.h"
 #include "vcc_compile.h"
-#include "libvarnish.h"
 
 /*--------------------------------------------------------------------*/
 
diff --git a/lib/libvcl/vcc_token.c b/lib/libvcl/vcc_token.c
index 5a79732..63ebbf3 100644
--- a/lib/libvcl/vcc_token.c
+++ b/lib/libvcl/vcc_token.c
@@ -34,8 +34,6 @@
 #include <stdlib.h>
 #include <string.h>
 
-#include "libvarnish.h"
-#include "vcc_priv.h"
 #include "vcc_compile.h"
 
 /*--------------------------------------------------------------------*/
diff --git a/lib/libvcl/vcc_var.c b/lib/libvcl/vcc_var.c
index b6e1ea7..1bbac15 100644
--- a/lib/libvcl/vcc_var.c
+++ b/lib/libvcl/vcc_var.c
@@ -32,9 +32,7 @@
 #include <stdio.h>
 #include <string.h>
 
-#include "vcc_priv.h"
 #include "vcc_compile.h"
-#include "libvarnish.h"
 
 /*--------------------------------------------------------------------*/
 
diff --git a/lib/libvcl/vcc_vmod.c b/lib/libvcl/vcc_vmod.c
index 37a98fd..40c9ed5 100644
--- a/lib/libvcl/vcc_vmod.c
+++ b/lib/libvcl/vcc_vmod.c
@@ -32,9 +32,7 @@
 #include <dlfcn.h>
 #include <string.h>
 
-#include "vcc_priv.h"
 #include "vcc_compile.h"
-#include "libvarnish.h"
 #include "vmod_abi.h"
 
 void
diff --git a/lib/libvcl/vcc_xref.c b/lib/libvcl/vcc_xref.c
index 56a2747..c3ef375 100644
--- a/lib/libvcl/vcc_xref.c
+++ b/lib/libvcl/vcc_xref.c
@@ -41,8 +41,6 @@
 
 #include <stdio.h>
 
-#include "libvarnish.h"
-#include "vcc_priv.h"
 #include "vcc_compile.h"
 
 /*--------------------------------------------------------------------*/



From geoff at varnish-cache.org  Mon Jan  9 20:52:15 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:15 +0100
Subject: [experimental-ims] 2dbbcb4 More #include shufflery
Message-ID: <E1RkMCV-0007Zn-Il@project.varnish-software.com>

commit 2dbbcb4405a4bd25401f40ffcfc27bef2e42fb64
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sun Oct 9 16:12:58 2011 +0000

    More #include shufflery

diff --git a/include/Makefile.am b/include/Makefile.am
index c68e07c..93f61e0 100644
--- a/include/Makefile.am
+++ b/include/Makefile.am
@@ -15,6 +15,7 @@ pkginclude_HEADERS = \
 	tbl/vsc_all.h \
 	tbl/vsc_fields.h \
 	tbl/vsl_tags.h \
+	vapi/vsm.h \
 	varnishapi.h \
 	vcli.h \
 	vsc.h \
diff --git a/include/vapi/vsm.h b/include/vapi/vsm.h
new file mode 100644
index 0000000..8948028
--- /dev/null
+++ b/include/vapi/vsm.h
@@ -0,0 +1,139 @@
+/*-
+ * Copyright (c) 2006 Verdens Gang AS
+ * Copyright (c) 2006-2011 Varnish Software AS
+ * All rights reserved.
+ *
+ * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ */
+
+#ifndef VAPI_VSM_H_INCLUDED
+#define VAPI_VSM_H_INCLUDED
+
+// #include <stdint.h>
+
+/*---------------------------------------------------------------------
+ * VSM level access functions
+ */
+
+struct VSM_data *VSM_New(void);
+	/*
+	 * Allocate and initialize a VSL_data handle structure.
+	 * This is the first thing you will have to do, always.
+	 * You can have multiple active VSL_data handles at the same time
+	 * referencing the same or different shared memory files.
+	 * Returns:
+	 *	Pointer to usable VSL_data handle.
+	 */
+
+typedef void VSM_diag_f(void *priv, const char *fmt, ...);
+
+void VSM_Diag(struct VSM_data *vd, VSM_diag_f *func, void *priv);
+	/*
+	 * Set the diagnostics reporting function.
+	 * Default is fprintf(stderr, ...)
+	 * If func is NULL, diagnostics are disabled.
+	 */
+
+#define VSM_n_USAGE	"[-n varnish_name]"
+
+int VSM_n_Arg(struct VSM_data *vd, const char *n_arg);
+	/*
+	 * Configure which varnishd instance to access.
+	 * Can also be, and normally is done through the VSL_Log_arg()
+	 * and VSC_Arg() functions.
+	 * Returns:
+	 *	 1 on success
+	 *	 -1 on failure, with diagnostic on stderr.
+	 */
+
+const char *VSM_Name(const struct VSM_data *vd);
+	/*
+	 * Return the instance name.
+	 */
+
+void VSM_Delete(struct VSM_data *vd);
+	/*
+	 * Close and deallocate all storage and mappings.
+	 */
+
+/* XXX: extension:  Patience argument for sleeps */
+
+int VSM_Open(struct VSM_data *vd, int diag);
+	/*
+	 * Attempt to open and map the shared memory file.
+	 * If diag is non-zero, diagnostics are emitted.
+	 * Returns:
+	 *	0 on success
+	 *	!= 0 on failure
+	 */
+
+int VSM_ReOpen(struct VSM_data *vd, int diag);
+	/*
+	 * Check if shared memory segment needs to be reopened/remapped
+	 * typically when the varnishd master process restarts.
+	 * diag is passed to VSM_Open()
+	 * Returns:
+	 *	0  No reopen needed.
+	 *	1  shared memory reopened/remapped.
+	 *	-1 failure to reopen.
+	 */
+
+unsigned VSM_Seq(const struct VSM_data *vd);
+	/*
+	 * Return the allocation sequence number
+	 */
+
+struct VSM_head *VSM_Head(const struct VSM_data *vd);
+	/*
+	 * Return the head of the VSM.
+	 */
+
+void *VSM_Find_Chunk(struct VSM_data *vd, const char *class,
+    const char *type, const char *ident, unsigned *lenp);
+	/*
+	 * Find a given chunk in the shared memory.
+	 * Returns pointer or NULL.
+	 * Lenp, if non-NULL, is set to length of chunk.
+	 */
+
+void VSM_Close(struct VSM_data *vd);
+	/*
+	 * Unmap shared memory
+	 * Deallocate all storage (including VSC and VSL allocations)
+	 */
+
+struct VSM_chunk *VSM_iter0(struct VSM_data *vd);
+void VSM_itern(const struct VSM_data *vd, struct VSM_chunk **pp);
+
+#define VSM_FOREACH(var, vd) \
+    for((var) = VSM_iter0((vd)); (var) != NULL; VSM_itern((vd), &(var)))
+
+	/*
+	 * Iterate over all chunks in shared memory
+	 * var = "struct VSM_chunk *"
+	 * vd = "struct VSM_data"
+	 */
+
+#endif /* VAPI_VSM_H_INCLUDED */
diff --git a/include/varnishapi.h b/include/varnishapi.h
index dcdd74f..235308a 100644
--- a/include/varnishapi.h
+++ b/include/varnishapi.h
@@ -31,6 +31,8 @@
 #ifndef VARNISHAPI_H_INCLUDED
 #define VARNISHAPI_H_INCLUDED
 
+#include <vapi/vsm.h>
+
 #include <stdint.h>
 
 #include "vsl.h"
@@ -43,109 +45,6 @@
 
 
 /*---------------------------------------------------------------------
- * VSM level access functions
- */
-
-struct VSM_data *VSM_New(void);
-	/*
-	 * Allocate and initialize a VSL_data handle structure.
-	 * This is the first thing you will have to do, always.
-	 * You can have multiple active VSL_data handles at the same time
-	 * referencing the same or different shared memory files.
-	 * Returns:
-	 *	Pointer to usable VSL_data handle.
-	 */
-
-typedef void VSM_diag_f(void *priv, const char *fmt, ...);
-
-void VSM_Diag(struct VSM_data *vd, VSM_diag_f *func, void *priv);
-	/*
-	 * Set the diagnostics reporting function.
-	 * Default is fprintf(stderr, ...)
-	 * If func is NULL, diagnostics are disabled.
-	 */
-
-#define VSM_n_USAGE	"[-n varnish_name]"
-
-int VSM_n_Arg(struct VSM_data *vd, const char *n_arg);
-	/*
-	 * Configure which varnishd instance to access.
-	 * Can also be, and normally is done through the VSL_Log_arg()
-	 * and VSC_Arg() functions.
-	 * Returns:
-	 *	 1 on success
-	 *	 -1 on failure, with diagnostic on stderr.
-	 */
-
-const char *VSM_Name(const struct VSM_data *vd);
-	/*
-	 * Return the instance name.
-	 */
-
-void VSM_Delete(struct VSM_data *vd);
-	/*
-	 * Close and deallocate all storage and mappings.
-	 */
-
-/* XXX: extension:  Patience argument for sleeps */
-
-int VSM_Open(struct VSM_data *vd, int diag);
-	/*
-	 * Attempt to open and map the shared memory file.
-	 * If diag is non-zero, diagnostics are emitted.
-	 * Returns:
-	 *	0 on success
-	 *	!= 0 on failure
-	 */
-
-int VSM_ReOpen(struct VSM_data *vd, int diag);
-	/*
-	 * Check if shared memory segment needs to be reopened/remapped
-	 * typically when the varnishd master process restarts.
-	 * diag is passed to VSM_Open()
-	 * Returns:
-	 *	0  No reopen needed.
-	 *	1  shared memory reopened/remapped.
-	 *	-1 failure to reopen.
-	 */
-
-unsigned VSM_Seq(const struct VSM_data *vd);
-	/*
-	 * Return the allocation sequence number
-	 */
-
-struct VSM_head *VSM_Head(const struct VSM_data *vd);
-	/*
-	 * Return the head of the VSM.
-	 */
-
-void *VSM_Find_Chunk(struct VSM_data *vd, const char *class,
-    const char *type, const char *ident, unsigned *lenp);
-	/*
-	 * Find a given chunk in the shared memory.
-	 * Returns pointer or NULL.
-	 * Lenp, if non-NULL, is set to length of chunk.
-	 */
-
-void VSM_Close(struct VSM_data *vd);
-	/*
-	 * Unmap shared memory
-	 * Deallocate all storage (including VSC and VSL allocations)
-	 */
-
-struct VSM_chunk *VSM_iter0(struct VSM_data *vd);
-void VSM_itern(const struct VSM_data *vd, struct VSM_chunk **pp);
-
-#define VSM_FOREACH(var, vd) \
-    for((var) = VSM_iter0((vd)); (var) != NULL; VSM_itern((vd), &(var)))
-
-	/*
-	 * Iterate over all chunks in shared memory
-	 * var = "struct VSM_chunk *"
-	 * vd = "struct VSM_data"
-	 */
-
-/*---------------------------------------------------------------------
  * VSC level access functions
  */
 



From geoff at varnish-cache.org  Mon Jan  9 20:52:15 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:15 +0100
Subject: [experimental-ims] 4507c59 Also split out the VSC part of the (v)abi
Message-ID: <E1RkMCV-0007Zw-LI@project.varnish-software.com>

commit 4507c59580f9e5089d9d4e863ca2740daf975fc0
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sun Oct 9 16:20:05 2011 +0000

    Also split out the VSC part of the (v)abi

diff --git a/include/Makefile.am b/include/Makefile.am
index 93f61e0..47fede9 100644
--- a/include/Makefile.am
+++ b/include/Makefile.am
@@ -16,6 +16,7 @@ pkginclude_HEADERS = \
 	tbl/vsc_fields.h \
 	tbl/vsl_tags.h \
 	vapi/vsm.h \
+	vapi/vsc.h \
 	varnishapi.h \
 	vcli.h \
 	vsc.h \
diff --git a/include/vapi/vsc.h b/include/vapi/vsc.h
new file mode 100644
index 0000000..290db9d
--- /dev/null
+++ b/include/vapi/vsc.h
@@ -0,0 +1,85 @@
+/*-
+ * Copyright (c) 2006 Verdens Gang AS
+ * Copyright (c) 2006-2011 Varnish Software AS
+ * All rights reserved.
+ *
+ * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ */
+
+#ifndef VAPI_VSC_H_INCLUDED
+#define VAPI_VSC_H_INCLUDED
+
+/*---------------------------------------------------------------------
+ * VSC level access functions
+ */
+
+void VSC_Setup(struct VSM_data *vd);
+	/*
+	 * Setup vd for use with VSC functions.
+	 */
+
+#define VSC_ARGS	"f:n:"
+#define VSC_n_USAGE	VSM_n_USAGE
+#define VSC_USAGE	VSC_N_USAGE
+
+int VSC_Arg(struct VSM_data *vd, int arg, const char *opt);
+	/*
+	 * Handle standard stat-presenter arguments
+	 * Return:
+	 *	-1 error
+	 *	 0 not handled
+	 *	 1 Handled.
+	 */
+
+int VSC_Open(struct VSM_data *vd, int diag);
+	/*
+	 * Open shared memory for VSC processing.
+	 * args and returns as VSM_Open()
+	 */
+
+struct VSC_C_main *VSC_Main(struct VSM_data *vd);
+	/*
+	 * return Main stats structure
+	 */
+
+struct VSC_point {
+	const char *class;		/* stat struct type		*/
+	const char *ident;		/* stat struct ident		*/
+	const char *name;		/* field name			*/
+	const char *fmt;		/* field format ("uint64_t")	*/
+	int flag;			/* 'a' = counter, 'i' = gauge	*/
+	const char *desc;		/* description			*/
+	const volatile void *ptr;	/* field value			*/
+};
+
+typedef int VSC_iter_f(void *priv, const struct VSC_point *const pt);
+
+int VSC_Iter(struct VSM_data *vd, VSC_iter_f *func, void *priv);
+	/*
+	 * Iterate over all statistics counters, calling "func" for
+	 * each counter not suppressed by any "-f" arguments.
+	 */
+
+#endif /* VAPI_VSC_H_INCLUDED */
diff --git a/include/varnishapi.h b/include/varnishapi.h
index 235308a..6960720 100644
--- a/include/varnishapi.h
+++ b/include/varnishapi.h
@@ -32,6 +32,7 @@
 #define VARNISHAPI_H_INCLUDED
 
 #include <vapi/vsm.h>
+#include <vapi/vsc.h>
 
 #include <stdint.h>
 
@@ -43,58 +44,6 @@
  *	API use failures will trip assert.
  */
 
-
-/*---------------------------------------------------------------------
- * VSC level access functions
- */
-
-void VSC_Setup(struct VSM_data *vd);
-	/*
-	 * Setup vd for use with VSC functions.
-	 */
-
-#define VSC_ARGS	"f:n:"
-#define VSC_n_USAGE	VSM_n_USAGE
-#define VSC_USAGE	VSC_N_USAGE
-
-int VSC_Arg(struct VSM_data *vd, int arg, const char *opt);
-	/*
-	 * Handle standard stat-presenter arguments
-	 * Return:
-	 *	-1 error
-	 *	 0 not handled
-	 *	 1 Handled.
-	 */
-
-int VSC_Open(struct VSM_data *vd, int diag);
-	/*
-	 * Open shared memory for VSC processing.
-	 * args and returns as VSM_Open()
-	 */
-
-struct VSC_C_main *VSC_Main(struct VSM_data *vd);
-	/*
-	 * return Main stats structure
-	 */
-
-struct VSC_point {
-	const char *class;		/* stat struct type		*/
-	const char *ident;		/* stat struct ident		*/
-	const char *name;		/* field name			*/
-	const char *fmt;		/* field format ("uint64_t")	*/
-	int flag;			/* 'a' = counter, 'i' = gauge	*/
-	const char *desc;		/* description			*/
-	const volatile void *ptr;	/* field value			*/
-};
-
-typedef int VSC_iter_f(void *priv, const struct VSC_point *const pt);
-
-int VSC_Iter(struct VSM_data *vd, VSC_iter_f *func, void *priv);
-	/*
-	 * Iterate over all statistics counters, calling "func" for
-	 * each counter not suppressed by any "-f" arguments.
-	 */
-
 /*---------------------------------------------------------------------
  * VSL level access functions
  */



From geoff at varnish-cache.org  Mon Jan  9 20:52:15 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:15 +0100
Subject: [experimental-ims] 6e6edd0 Complete (?) the varnish API #include
	cleanup.
Message-ID: <E1RkMCV-0007a9-ON@project.varnish-software.com>

commit 6e6edd08227b1ced1c0d15117e3ea9f0afde40e7
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sun Oct 9 16:56:04 2011 +0000

    Complete (?) the varnish API #include cleanup.
    
    We now have
    	<vapi/vs[mcl].h> for the programmatic API
    
    	<vapi/vs[cl]_int.h> for the underlying structure,
    			    shared between API and varnishd

diff --git a/bin/varnishadm/varnishadm.c b/bin/varnishadm/varnishadm.c
index 68ebbf7..d371b9f 100644
--- a/bin/varnishadm/varnishadm.c
+++ b/bin/varnishadm/varnishadm.c
@@ -29,6 +29,9 @@
 
 #include "config.h"
 
+#include "vapi/vsm.h"
+#include "vapi/vsl.h"
+
 #include <fcntl.h>
 #include <poll.h>
 #include <stdio.h>
@@ -44,7 +47,6 @@
 #include "vcli.h"
 #include "cli_common.h"
 #include "libvarnish.h"
-#include "varnishapi.h"
 #include "vss.h"
 
 #ifdef HAVE_LIBEDIT
diff --git a/bin/varnishd/cache.h b/bin/varnishd/cache.h
index d3228a5..c5bc614 100644
--- a/bin/varnishd/cache.h
+++ b/bin/varnishd/cache.h
@@ -34,6 +34,9 @@
  */
 #define VARNISH_CACHE_CHILD	1
 
+#include "vapi/vsc_int.h"
+#include "vapi/vsl_int.h"
+
 #include <sys/socket.h>
 
 #include <pthread.h>
@@ -54,9 +57,6 @@
 
 #include "heritage.h"
 
-#include "vsc.h"
-#include "vsl.h"
-
 enum body_status {
 #define BODYSTATUS(U,l)	BS_##U,
 #include "tbl/body_status.h"
diff --git a/bin/varnishd/mgt_shmem.c b/bin/varnishd/mgt_shmem.c
index 17789f8..5b717b0 100644
--- a/bin/varnishd/mgt_shmem.c
+++ b/bin/varnishd/mgt_shmem.c
@@ -86,6 +86,9 @@
 
 #include "config.h"
 
+#include "vapi/vsl_int.h"
+#include "vapi/vsc_int.h"
+
 #include <signal.h>
 #include <stdio.h>
 #include <stdlib.h>
@@ -95,8 +98,6 @@
 #include <sys/mman.h>
 #include <sys/stat.h>
 
-#include "vsc.h"
-#include "vsl.h"
 #include "mgt.h"
 #include "heritage.h"
 #include "vmb.h"
diff --git a/bin/varnishhist/varnishhist.c b/bin/varnishhist/varnishhist.c
index 75f7bd9..9485df7 100644
--- a/bin/varnishhist/varnishhist.c
+++ b/bin/varnishhist/varnishhist.c
@@ -32,6 +32,9 @@
 
 #include "config.h"
 
+#include "vapi/vsm.h"
+#include "vapi/vsl.h"
+
 #include <sys/types.h>
 #include <curses.h>
 #include <errno.h>
@@ -46,8 +49,6 @@
 #include <unistd.h>
 
 #include "libvarnish.h"
-#include "vsl.h"
-#include "varnishapi.h"
 
 #define HIST_N 2000 /* how far back we remember */
 #define HIST_LOW -6 /* low end of log range */
diff --git a/bin/varnishlog/varnishlog.c b/bin/varnishlog/varnishlog.c
index c7749cb..c67432d 100644
--- a/bin/varnishlog/varnishlog.c
+++ b/bin/varnishlog/varnishlog.c
@@ -31,6 +31,9 @@
 
 #include "config.h"
 
+#include "vapi/vsm.h"
+#include "vapi/vsl.h"
+
 #include <errno.h>
 #include <fcntl.h>
 #include <signal.h>
@@ -45,8 +48,6 @@
 #include "vpf.h"
 
 #include "libvarnish.h"
-#include "vsl.h"
-#include "varnishapi.h"
 
 static int	b_flag, c_flag;
 
diff --git a/bin/varnishncsa/base64.c b/bin/varnishncsa/base64.c
index 23dae14..3758604 100644
--- a/bin/varnishncsa/base64.c
+++ b/bin/varnishncsa/base64.c
@@ -6,8 +6,10 @@
 
 #include "config.h"
 
+#include "vapi/vsm.h"
+#include "vapi/vsl.h"
+
 #include <sys/types.h>
-#include "varnishapi.h"
 #include "base64.h"
 
 static const char b64[] =
diff --git a/bin/varnishncsa/varnishncsa.c b/bin/varnishncsa/varnishncsa.c
index 7c35c3e..bbaa366 100644
--- a/bin/varnishncsa/varnishncsa.c
+++ b/bin/varnishncsa/varnishncsa.c
@@ -59,6 +59,9 @@
 
 #include "config.h"
 
+#include "vapi/vsm.h"
+#include "vapi/vsl.h"
+
 #include <ctype.h>
 #include <signal.h>
 #include <stdio.h>
@@ -74,9 +77,7 @@
 #include "vqueue.h"
 
 #include "libvarnish.h"
-#include "vsl.h"
 #include "vre.h"
-#include "varnishapi.h"
 #include "base64.h"
 
 static volatile sig_atomic_t reopen;
diff --git a/bin/varnishreplay/varnishreplay.c b/bin/varnishreplay/varnishreplay.c
index f7355cb..dbb6f51 100644
--- a/bin/varnishreplay/varnishreplay.c
+++ b/bin/varnishreplay/varnishreplay.c
@@ -28,6 +28,9 @@
 
 #include "config.h"
 
+#include "vapi/vsm.h"
+#include "vapi/vsl.h"
+
 #include <sys/types.h>
 #include <sys/signal.h>
 #include <sys/uio.h>
@@ -46,7 +49,6 @@
 #include "vqueue.h"
 
 #include "libvarnish.h"
-#include "varnishapi.h"
 #include "vss.h"
 
 #define freez(x) do { if (x) free(x); x = NULL; } while (0);
diff --git a/bin/varnishsizes/varnishsizes.c b/bin/varnishsizes/varnishsizes.c
index ccff669..ac9dbf8 100644
--- a/bin/varnishsizes/varnishsizes.c
+++ b/bin/varnishsizes/varnishsizes.c
@@ -32,6 +32,9 @@
 
 #include "config.h"
 
+#include "vapi/vsm.h"
+#include "vapi/vsl.h"
+
 #include <sys/types.h>
 #include <curses.h>
 #include <errno.h>
@@ -46,8 +49,6 @@
 #include <unistd.h>
 
 #include "libvarnish.h"
-#include "vsl.h"
-#include "varnishapi.h"
 
 #define HIST_N 2000 /* how far back we remember */
 #define HIST_LOW 1 /* low end of log range */
diff --git a/bin/varnishstat/varnishstat.c b/bin/varnishstat/varnishstat.c
index a8b1d5f..76a2e03 100644
--- a/bin/varnishstat/varnishstat.c
+++ b/bin/varnishstat/varnishstat.c
@@ -32,6 +32,9 @@
 
 #include "config.h"
 
+#include "vapi/vsm.h"
+#include "vapi/vsc.h"
+
 #include <sys/time.h>
 
 #include <errno.h>
@@ -42,8 +45,6 @@
 #include <unistd.h>
 
 #include "libvarnish.h"
-#include "vsc.h"
-#include "varnishapi.h"
 #include "varnishstat.h"
 
 /*--------------------------------------------------------------------*/
diff --git a/bin/varnishstat/varnishstat_curses.c b/bin/varnishstat/varnishstat_curses.c
index 22d21fc..fc948e9 100644
--- a/bin/varnishstat/varnishstat_curses.c
+++ b/bin/varnishstat/varnishstat_curses.c
@@ -32,6 +32,9 @@
 
 #include "config.h"
 
+#include "vapi/vsm.h"
+#include "vapi/vsc.h"
+
 #include <sys/time.h>
 
 #ifdef HAVE_NCURSES_CURSES_H
@@ -48,9 +51,7 @@
 #include <unistd.h>
 
 #include "libvarnish.h"
-#include "vsc.h"
 #include "vqueue.h"
-#include "varnishapi.h"
 #include "varnishstat.h"
 
 #if 0
diff --git a/bin/varnishtest/vtc_varnish.c b/bin/varnishtest/vtc_varnish.c
index aea29a4..7362f1f 100644
--- a/bin/varnishtest/vtc_varnish.c
+++ b/bin/varnishtest/vtc_varnish.c
@@ -28,6 +28,10 @@
 
 #include "config.h"
 
+#include "vapi/vsm.h"
+#include "vapi/vsc.h"
+#include "vapi/vsl.h"
+
 #include <stdio.h>
 
 #include <fcntl.h>
@@ -44,7 +48,6 @@
 #include "vtc.h"
 
 #include "libvarnish.h"
-#include "varnishapi.h"
 
 #include "vcli.h"
 #include "vss.h"
diff --git a/bin/varnishtop/varnishtop.c b/bin/varnishtop/varnishtop.c
index 8459ab5..de3789f 100644
--- a/bin/varnishtop/varnishtop.c
+++ b/bin/varnishtop/varnishtop.c
@@ -32,6 +32,9 @@
 
 #include "config.h"
 
+#include "vapi/vsm.h"
+#include "vapi/vsl.h"
+
 #include <ctype.h>
 #include <curses.h>
 #include <errno.h>
@@ -48,8 +51,6 @@
 #include "vsb.h"
 
 #include "libvarnish.h"
-#include "vsl.h"
-#include "varnishapi.h"
 
 #if 0
 #define AC(x) assert((x) != ERR)
diff --git a/configure.ac b/configure.ac
index ef7d76c..cf1f566 100644
--- a/configure.ac
+++ b/configure.ac
@@ -3,7 +3,7 @@ AC_COPYRIGHT([Copyright (c) 2006 Verdens Gang AS
 Copyright (c) 2006-2011 Varnish Software AS])
 AC_REVISION([$Id$])
 AC_INIT([Varnish], [trunk], [varnish-dev at varnish-cache.org])
-AC_CONFIG_SRCDIR(include/varnishapi.h)
+AC_CONFIG_SRCDIR(include/miniobj.h)
 AM_CONFIG_HEADER(config.h)
 
 # save command line CFLAGS for use in VCC_CC (to pass through things like -m64)
diff --git a/include/Makefile.am b/include/Makefile.am
index 47fede9..0f1782f 100644
--- a/include/Makefile.am
+++ b/include/Makefile.am
@@ -17,10 +17,10 @@ pkginclude_HEADERS = \
 	tbl/vsl_tags.h \
 	vapi/vsm.h \
 	vapi/vsc.h \
-	varnishapi.h \
+	vapi/vsc_int.h \
+	vapi/vsl.h \
+	vapi/vsl_int.h \
 	vcli.h \
-	vsc.h \
-	vsl.h \
 	vsm.h
 
 nobase_noinst_HEADERS = \
diff --git a/include/vapi/vsc.h b/include/vapi/vsc.h
index 290db9d..1101e53 100644
--- a/include/vapi/vsc.h
+++ b/include/vapi/vsc.h
@@ -31,6 +31,8 @@
 #ifndef VAPI_VSC_H_INCLUDED
 #define VAPI_VSC_H_INCLUDED
 
+#include "vapi/vsc_int.h"
+
 /*---------------------------------------------------------------------
  * VSC level access functions
  */
diff --git a/include/vapi/vsc_int.h b/include/vapi/vsc_int.h
new file mode 100644
index 0000000..0a9cbf9
--- /dev/null
+++ b/include/vapi/vsc_int.h
@@ -0,0 +1,50 @@
+/*-
+ * Copyright (c) 2006 Verdens Gang AS
+ * Copyright (c) 2006-2010 Varnish Software AS
+ * All rights reserved.
+ *
+ * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ */
+
+#include <stdint.h>
+
+#define VSC_CLASS          "Stat"
+
+#define VSC_TYPE_MAIN		""
+#define VSC_TYPE_SMA	"SMA"
+#define VSC_TYPE_SMF	"SMF"
+#define VSC_TYPE_VBE	"VBE"
+#define VSC_TYPE_LCK	"LCK"
+
+#define VSC_F(n, t, l, f, e, d)	t n;
+
+#define VSC_DO(u,l,t) struct VSC_C_##l {
+#define VSC_DONE(u,l,t) };
+
+#include "tbl/vsc_all.h"
+
+#undef VSC_DO
+#undef VSC_F
+#undef VSC_DONE
diff --git a/include/vapi/vsl.h b/include/vapi/vsl.h
new file mode 100644
index 0000000..5068f1c
--- /dev/null
+++ b/include/vapi/vsl.h
@@ -0,0 +1,105 @@
+/*-
+ * Copyright (c) 2006 Verdens Gang AS
+ * Copyright (c) 2006-2011 Varnish Software AS
+ * All rights reserved.
+ *
+ * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ */
+
+#ifndef VAPI_VSL_H_INCLUDED
+#define VAPI_VSL_H_INCLUDED
+
+#include <stdint.h>
+
+#include "vapi/vsl_int.h"
+
+/*---------------------------------------------------------------------
+ * VSL level access functions
+ */
+
+void VSL_Setup(struct VSM_data *vd);
+	/*
+	 * Setup vd for use with VSL functions.
+	 */
+
+int VSL_Open(struct VSM_data *vd, int diag);
+	/*
+	 * Attempt to open and map the shared memory file.
+	 * If diag is non-zero, diagnostics are emitted.
+	 * Returns:
+	 *	0 on success
+	 *	!= 0 on failure
+	 */
+
+#define VSL_ARGS	"bCcdI:i:k:n:r:s:X:x:m:"
+#define VSL_b_USAGE	"[-b]"
+#define VSL_c_USAGE	"[-c]"
+#define VSL_C_USAGE	"[-C]"
+#define VSL_d_USAGE	"[-d]"
+#define VSL_i_USAGE	"[-i tag]"
+#define VSL_I_USAGE	"[-I regexp]"
+#define VSL_k_USAGE	"[-k keep]"
+#define VSL_m_USAGE	"[-m tag:regex]"
+#define VSL_n_USAGE	VSM_n_USAGE
+#define VSL_r_USAGE	"[-r file]"
+#define VSL_s_USAGE	"[-s skip]"
+#define VSL_x_USAGE	"[-x tag]"
+#define VSL_X_USAGE	"[-X regexp]"
+#define VSL_USAGE	"[-bCcd] "		\
+			VSL_i_USAGE " "		\
+			VSL_I_USAGE " "		\
+			VSL_k_USAGE " "		\
+			VSL_m_USAGE " "		\
+			VSL_n_USAGE " "		\
+			VSL_r_USAGE " "		\
+			VSL_s_USAGE " "		\
+			VSL_X_USAGE " "		\
+			VSL_x_USAGE
+
+int VSL_Arg(struct VSM_data *vd, int arg, const char *opt);
+	/*
+	 * Handle standard log-presenter arguments
+	 * Return:
+	 *	-1 error
+	 *	 0 not handled
+	 *	 1 Handled.
+	 */
+
+typedef int VSL_handler_f(void *priv, enum VSL_tag_e tag, unsigned fd,
+    unsigned len, unsigned spec, const char *ptr, uint64_t bitmap);
+
+#define VSL_S_CLIENT	(1 << 0)
+#define VSL_S_BACKEND	(1 << 1)
+VSL_handler_f VSL_H_Print;
+struct VSM_data;
+void VSL_Select(const struct VSM_data *vd, unsigned tag);
+void VSL_NonBlocking(const struct VSM_data *vd, int nb);
+int VSL_Dispatch(struct VSM_data *vd, VSL_handler_f *func, void *priv);
+int VSL_NextLog(const struct VSM_data *lh, uint32_t **pp, uint64_t *bitmap);
+int VSL_Matched(const struct VSM_data *vd, uint64_t bitmap);
+int VSL_Name2Tag(const char *name, int l);
+extern const char *VSL_tags[256];
+
+#endif /* VAPI_VSL_H_INCLUDED */
diff --git a/include/vapi/vsl_int.h b/include/vapi/vsl_int.h
new file mode 100644
index 0000000..8176cce
--- /dev/null
+++ b/include/vapi/vsl_int.h
@@ -0,0 +1,82 @@
+/*-
+ * Copyright (c) 2006 Verdens Gang AS
+ * Copyright (c) 2006-2010 Varnish Software AS
+ * All rights reserved.
+ *
+ * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * Define the layout of the shared memory log segment.
+ *
+ * NB: THIS IS NOT A PUBLIC API TO VARNISH!
+ */
+
+#ifndef VAPI_VSL_FMT_H_INCLUDED
+#define VAPI_VSL_FMT_H_INCLUDED
+
+#define VSL_CLASS		"Log"
+
+/*
+ * Shared memory log format
+ *
+ * The log is structured as an array of 32bit unsigned integers.
+ *
+ * The first integer contains a non-zero serial number, which changes
+ * whenever writing the log starts from the front.
+ *
+ * Each logrecord consist of:
+ *	[n]		= ((type & 0xff) << 24) | (length & 0xffff)
+ *	[n + 1]		= identifier
+ *	[n + 2] ... [m]	= content
+ */
+
+#define VSL_CLIENTMARKER	(1U<<30)
+#define VSL_BACKENDMARKER	(1U<<31)
+#define VSL_IDENTMASK		(~(3U<<30))
+
+#define VSL_WORDS(len)		(((len) + 3) / 4)
+#define VSL_END(ptr, len)	((ptr) + 2 + VSL_WORDS(len))
+#define VSL_NEXT(ptr)		VSL_END(ptr, VSL_LEN(ptr))
+#define VSL_LEN(ptr)		((ptr)[0] & 0xffff)
+#define VSL_TAG(ptr)		((ptr)[0] >> 24)
+#define VSL_ID(ptr)		(((ptr)[1]) & VSL_IDENTMASK)
+#define VSL_CLIENT(ptr)		(((ptr)[1]) & VSL_CLIENTMARKER)
+#define VSL_BACKEND(ptr)	(((ptr)[1]) & VSL_BACKENDMARKER)
+#define VSL_DATA(ptr)		((char*)((ptr)+2))
+
+#define VSL_ENDMARKER	(((uint32_t)SLT_Reserved << 24) | 0x454545) /* "EEE" */
+#define VSL_WRAPMARKER	(((uint32_t)SLT_Reserved << 24) | 0x575757) /* "WWW" */
+
+/*
+ * The identifiers in shmlogtag are "SLT_" + XML tag.  A script may be run
+ * on this file to extract the table rather than handcode it
+ */
+enum VSL_tag_e {
+	SLT_Bogus = 0,
+#define SLTM(foo)	SLT_##foo,
+#include "tbl/vsl_tags.h"
+#undef SLTM
+	SLT_Reserved = 255
+};
+
+#endif /* VAPI_VSL_FMT_H_INCLUDED */
diff --git a/include/vapi/vsm.h b/include/vapi/vsm.h
index 8948028..0b7cdd8 100644
--- a/include/vapi/vsm.h
+++ b/include/vapi/vsm.h
@@ -31,8 +31,6 @@
 #ifndef VAPI_VSM_H_INCLUDED
 #define VAPI_VSM_H_INCLUDED
 
-// #include <stdint.h>
-
 /*---------------------------------------------------------------------
  * VSM level access functions
  */
diff --git a/include/varnishapi.h b/include/varnishapi.h
deleted file mode 100644
index 6960720..0000000
--- a/include/varnishapi.h
+++ /dev/null
@@ -1,114 +0,0 @@
-/*-
- * Copyright (c) 2006 Verdens Gang AS
- * Copyright (c) 2006-2011 Varnish Software AS
- * All rights reserved.
- *
- * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- *
- */
-
-#ifndef VARNISHAPI_H_INCLUDED
-#define VARNISHAPI_H_INCLUDED
-
-#include <vapi/vsm.h>
-#include <vapi/vsc.h>
-
-#include <stdint.h>
-
-#include "vsl.h"
-
-/*
- * Various notes:
- *	All malloc failures will result in assert tripping.
- *	API use failures will trip assert.
- */
-
-/*---------------------------------------------------------------------
- * VSL level access functions
- */
-
-void VSL_Setup(struct VSM_data *vd);
-	/*
-	 * Setup vd for use with VSL functions.
-	 */
-
-int VSL_Open(struct VSM_data *vd, int diag);
-	/*
-	 * Attempt to open and map the shared memory file.
-	 * If diag is non-zero, diagnostics are emitted.
-	 * Returns:
-	 *	0 on success
-	 *	!= 0 on failure
-	 */
-
-#define VSL_ARGS	"bCcdI:i:k:n:r:s:X:x:m:"
-#define VSL_b_USAGE	"[-b]"
-#define VSL_c_USAGE	"[-c]"
-#define VSL_C_USAGE	"[-C]"
-#define VSL_d_USAGE	"[-d]"
-#define VSL_i_USAGE	"[-i tag]"
-#define VSL_I_USAGE	"[-I regexp]"
-#define VSL_k_USAGE	"[-k keep]"
-#define VSL_m_USAGE	"[-m tag:regex]"
-#define VSL_n_USAGE	VSM_n_USAGE
-#define VSL_r_USAGE	"[-r file]"
-#define VSL_s_USAGE	"[-s skip]"
-#define VSL_x_USAGE	"[-x tag]"
-#define VSL_X_USAGE	"[-X regexp]"
-#define VSL_USAGE	"[-bCcd] "		\
-			VSL_i_USAGE " "		\
-			VSL_I_USAGE " "		\
-			VSL_k_USAGE " "		\
-			VSL_m_USAGE " "		\
-			VSL_n_USAGE " "		\
-			VSL_r_USAGE " "		\
-			VSL_s_USAGE " "		\
-			VSL_X_USAGE " "		\
-			VSL_x_USAGE
-
-int VSL_Arg(struct VSM_data *vd, int arg, const char *opt);
-	/*
-	 * Handle standard log-presenter arguments
-	 * Return:
-	 *	-1 error
-	 *	 0 not handled
-	 *	 1 Handled.
-	 */
-
-typedef int VSL_handler_f(void *priv, enum VSL_tag_e tag, unsigned fd,
-    unsigned len, unsigned spec, const char *ptr, uint64_t bitmap);
-
-#define VSL_S_CLIENT	(1 << 0)
-#define VSL_S_BACKEND	(1 << 1)
-VSL_handler_f VSL_H_Print;
-struct VSM_data;
-void VSL_Select(const struct VSM_data *vd, unsigned tag);
-void VSL_NonBlocking(const struct VSM_data *vd, int nb);
-int VSL_Dispatch(struct VSM_data *vd, VSL_handler_f *func, void *priv);
-int VSL_NextLog(const struct VSM_data *lh, uint32_t **pp, uint64_t *bitmap);
-int VSL_Matched(const struct VSM_data *vd, uint64_t bitmap);
-int VSL_Name2Tag(const char *name, int l);
-extern const char *VSL_tags[256];
-
-#endif
diff --git a/include/vsc.h b/include/vsc.h
deleted file mode 100644
index 0a9cbf9..0000000
--- a/include/vsc.h
+++ /dev/null
@@ -1,50 +0,0 @@
-/*-
- * Copyright (c) 2006 Verdens Gang AS
- * Copyright (c) 2006-2010 Varnish Software AS
- * All rights reserved.
- *
- * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- *
- */
-
-#include <stdint.h>
-
-#define VSC_CLASS          "Stat"
-
-#define VSC_TYPE_MAIN		""
-#define VSC_TYPE_SMA	"SMA"
-#define VSC_TYPE_SMF	"SMF"
-#define VSC_TYPE_VBE	"VBE"
-#define VSC_TYPE_LCK	"LCK"
-
-#define VSC_F(n, t, l, f, e, d)	t n;
-
-#define VSC_DO(u,l,t) struct VSC_C_##l {
-#define VSC_DONE(u,l,t) };
-
-#include "tbl/vsc_all.h"
-
-#undef VSC_DO
-#undef VSC_F
-#undef VSC_DONE
diff --git a/include/vsl.h b/include/vsl.h
deleted file mode 100644
index 64b0516..0000000
--- a/include/vsl.h
+++ /dev/null
@@ -1,82 +0,0 @@
-/*-
- * Copyright (c) 2006 Verdens Gang AS
- * Copyright (c) 2006-2010 Varnish Software AS
- * All rights reserved.
- *
- * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- *
- * Define the layout of the shared memory log segment.
- *
- * NB: THIS IS NOT A PUBLIC API TO VARNISH!
- */
-
-#ifndef SHMLOG_H_INCLUDED
-#define SHMLOG_H_INCLUDED
-
-#define VSL_CLASS		"Log"
-
-/*
- * Shared memory log format
- *
- * The log is structured as an array of 32bit unsigned integers.
- *
- * The first integer contains a non-zero serial number, which changes
- * whenever writing the log starts from the front.
- *
- * Each logrecord consist of:
- *	[n]		= ((type & 0xff) << 24) | (length & 0xffff)
- *	[n + 1]		= identifier
- *	[n + 2] ... [m]	= content
- */
-
-#define VSL_CLIENTMARKER	(1U<<30)
-#define VSL_BACKENDMARKER	(1U<<31)
-#define VSL_IDENTMASK		(~(3U<<30))
-
-#define VSL_WORDS(len)		(((len) + 3) / 4)
-#define VSL_END(ptr, len)	((ptr) + 2 + VSL_WORDS(len))
-#define VSL_NEXT(ptr)		VSL_END(ptr, VSL_LEN(ptr))
-#define VSL_LEN(ptr)		((ptr)[0] & 0xffff)
-#define VSL_TAG(ptr)		((ptr)[0] >> 24)
-#define VSL_ID(ptr)		(((ptr)[1]) & VSL_IDENTMASK)
-#define VSL_CLIENT(ptr)		(((ptr)[1]) & VSL_CLIENTMARKER)
-#define VSL_BACKEND(ptr)	(((ptr)[1]) & VSL_BACKENDMARKER)
-#define VSL_DATA(ptr)		((char*)((ptr)+2))
-
-#define VSL_ENDMARKER	(((uint32_t)SLT_Reserved << 24) | 0x454545) /* "EEE" */
-#define VSL_WRAPMARKER	(((uint32_t)SLT_Reserved << 24) | 0x575757) /* "WWW" */
-
-/*
- * The identifiers in shmlogtag are "SLT_" + XML tag.  A script may be run
- * on this file to extract the table rather than handcode it
- */
-enum VSL_tag_e {
-	SLT_Bogus = 0,
-#define SLTM(foo)	SLT_##foo,
-#include "tbl/vsl_tags.h"
-#undef SLTM
-	SLT_Reserved = 255
-};
-
-#endif
diff --git a/lib/libvarnishapi/vsc.c b/lib/libvarnishapi/vsc.c
index 18f1fc7..74cf286 100644
--- a/lib/libvarnishapi/vsc.c
+++ b/lib/libvarnishapi/vsc.c
@@ -29,6 +29,9 @@
 
 #include "config.h"
 
+#include "vapi/vsm.h"
+#include "vapi/vsc.h"
+
 #include <sys/types.h>
 #include <sys/stat.h>
 #include <string.h>
@@ -37,10 +40,8 @@
 #include "vas.h"
 #include "vav.h"
 #include "vsm.h"
-#include "vsc.h"
 #include "vqueue.h"
 #include "miniobj.h"
-#include "varnishapi.h"
 
 #include "vsm_api.h"
 
diff --git a/lib/libvarnishapi/vsl.c b/lib/libvarnishapi/vsl.c
index ab538c5..a2efc5d 100644
--- a/lib/libvarnishapi/vsl.c
+++ b/lib/libvarnishapi/vsl.c
@@ -29,6 +29,9 @@
 
 #include "config.h"
 
+#include "vapi/vsm.h"
+#include "vapi/vsl.h"
+
 #include <sys/types.h>
 #include <sys/stat.h>
 
@@ -39,11 +42,9 @@
 
 #include "vas.h"
 #include "vsm.h"
-#include "vsl.h"
 #include "vre.h"
 #include "vbm.h"
 #include "miniobj.h"
-#include "varnishapi.h"
 
 #include "vsm_api.h"
 #include "vsl_api.h"
diff --git a/lib/libvarnishapi/vsl_arg.c b/lib/libvarnishapi/vsl_arg.c
index 21cd8e9..f41129b 100644
--- a/lib/libvarnishapi/vsl_arg.c
+++ b/lib/libvarnishapi/vsl_arg.c
@@ -29,6 +29,9 @@
 
 #include "config.h"
 
+#include "vapi/vsm.h"
+#include "vapi/vsl.h"
+
 #include <sys/types.h>
 #include <sys/stat.h>
 
@@ -44,7 +47,6 @@
 #include "vre.h"
 #include "vbm.h"
 #include "miniobj.h"
-#include "varnishapi.h"
 
 #include "vsm_api.h"
 #include "vsl_api.h"
diff --git a/lib/libvarnishapi/vsm.c b/lib/libvarnishapi/vsm.c
index 888a0d7..422bc74 100644
--- a/lib/libvarnishapi/vsm.c
+++ b/lib/libvarnishapi/vsm.c
@@ -29,6 +29,8 @@
 
 #include "config.h"
 
+#include "vapi/vsm.h"
+
 #include <sys/types.h>
 #include <sys/mman.h>
 #include <sys/stat.h>
@@ -45,7 +47,6 @@
 #include "vsm.h"
 #include "vbm.h"
 #include "miniobj.h"
-#include "varnishapi.h"
 
 #include "vsm_api.h"
 



From geoff at varnish-cache.org  Mon Jan  9 20:52:15 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:15 +0100
Subject: [experimental-ims] 7140886 Add missing <stdint.h> include
Message-ID: <E1RkMCV-0007ad-U0@project.varnish-software.com>

commit 71408867ba8fbc2ef5d51e6353ad9ecee0aee2c6
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sun Oct 9 17:59:45 2011 +0000

    Add missing <stdint.h> include

diff --git a/lib/libvarnishapi/vsm.c b/lib/libvarnishapi/vsm.c
index 422bc74..32d9e3d 100644
--- a/lib/libvarnishapi/vsm.c
+++ b/lib/libvarnishapi/vsm.c
@@ -31,6 +31,7 @@
 
 #include "vapi/vsm.h"
 
+#include <stdint.h>
 #include <sys/types.h>
 #include <sys/mman.h>
 #include <sys/stat.h>



From geoff at varnish-cache.org  Mon Jan  9 20:52:16 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:16 +0100
Subject: [experimental-ims] 6570945 Isolate VAV from libvarnish.h
Message-ID: <E1RkMCW-0007b0-3C@project.varnish-software.com>

commit 6570945671d2ab43b3ced048a31df7133d45c1dc
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sun Oct 9 18:06:29 2011 +0000

    Isolate VAV from libvarnish.h

diff --git a/bin/varnishd/cache_hash.c b/bin/varnishd/cache_hash.c
index 7e45b8a..4001c7c 100644
--- a/bin/varnishd/cache_hash.c
+++ b/bin/varnishd/cache_hash.c
@@ -61,6 +61,7 @@
 #include "stevedore.h"
 #include "hash_slinger.h"
 #include "vsha256.h"
+#include "vav.h"
 #include "cache_backend.h"
 
 static const struct hash_slinger *hash;
diff --git a/bin/varnishd/cache_vrt.c b/bin/varnishd/cache_vrt.c
index f0407cc..a53799e 100644
--- a/bin/varnishd/cache_vrt.c
+++ b/bin/varnishd/cache_vrt.c
@@ -41,6 +41,7 @@
 
 #include "vrt.h"
 #include "vrt_obj.h"
+#include "vav.h"
 #include "vcl.h"
 #include "hash_slinger.h"
 #include "cache_backend.h"
diff --git a/bin/varnishd/mgt_param.c b/bin/varnishd/mgt_param.c
index 7138369..f849fbb 100644
--- a/bin/varnishd/mgt_param.c
+++ b/bin/varnishd/mgt_param.c
@@ -47,6 +47,7 @@
 #include "vparam.h"
 #include "cache_waiter.h"
 
+#include "vav.h"
 #include "vss.h"
 
 #define MAGIC_INIT_STRING	"\001"
diff --git a/bin/varnishd/mgt_shmem.c b/bin/varnishd/mgt_shmem.c
index 5b717b0..38dc59a 100644
--- a/bin/varnishd/mgt_shmem.c
+++ b/bin/varnishd/mgt_shmem.c
@@ -102,6 +102,7 @@
 #include "heritage.h"
 #include "vmb.h"
 #include "vsm.h"
+#include "vav.h"
 #include "flopen.h"
 
 #ifndef MAP_HASSEMAPHORE
diff --git a/bin/varnishd/stevedore.c b/bin/varnishd/stevedore.c
index 143e534..6d71635 100644
--- a/bin/varnishd/stevedore.c
+++ b/bin/varnishd/stevedore.c
@@ -38,6 +38,7 @@
 
 #include "cache.h"
 #include "stevedore.h"
+#include "vav.h"
 #include "cli_priv.h"
 #include "vrt_obj.h"
 
diff --git a/bin/varnishd/varnishd.c b/bin/varnishd/varnishd.c
index 0219eb0..2d7db59 100644
--- a/bin/varnishd/varnishd.c
+++ b/bin/varnishd/varnishd.c
@@ -53,6 +53,7 @@
 #include "vcli.h"
 #include "cli_common.h"
 
+#include "vav.h"
 #include "vin.h"
 #include "heritage.h"
 #include "mgt.h"
diff --git a/bin/varnishtest/vtc.c b/bin/varnishtest/vtc.c
index 506aee1..e31eaca 100644
--- a/bin/varnishtest/vtc.c
+++ b/bin/varnishtest/vtc.c
@@ -39,6 +39,7 @@
 
 #include "vtc.h"
 
+#include "vav.h"
 #include "libvarnish.h"
 
 
diff --git a/include/libvarnish.h b/include/libvarnish.h
index 6008781..a40aa20 100644
--- a/include/libvarnish.h
+++ b/include/libvarnish.h
@@ -38,8 +38,6 @@
 
 struct vsb;
 
-#include "vav.h"
-
 /* from libvarnish/num.c */
 const char *str2bytes(const char *p, uintmax_t *r, uintmax_t rel);
 
diff --git a/lib/libvarnish/argv.c b/lib/libvarnish/argv.c
index ad088cd..a0c57bb 100644
--- a/lib/libvarnish/argv.c
+++ b/lib/libvarnish/argv.c
@@ -43,7 +43,8 @@
 #include <stdio.h>
 #include <string.h>
 
-#include "libvarnish.h"
+#include "vas.h"
+#include "vav.h"
 
 int
 VAV_BackSlash(const char *s, char *res)
diff --git a/lib/libvarnish/cli_serve.c b/lib/libvarnish/cli_serve.c
index b7c52f4..03cf1d4 100644
--- a/lib/libvarnish/cli_serve.c
+++ b/lib/libvarnish/cli_serve.c
@@ -41,6 +41,7 @@
 
 #include "vqueue.h"
 #include "vsb.h"
+#include "vav.h"
 #include "vlu.h"
 #include "vcli.h"
 #include "cli_priv.h"



From geoff at varnish-cache.org  Mon Jan  9 20:52:16 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:16 +0100
Subject: [experimental-ims] c012d86 Cleanup VSUB_ and eliminate from
	libvarnish.h
Message-ID: <E1RkMCW-0007bO-7r@project.varnish-software.com>

commit c012d86e43957404511107fb37247c4249d7c841
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sun Oct 9 18:29:10 2011 +0000

    Cleanup VSUB_ and eliminate from libvarnish.h

diff --git a/bin/varnishd/mgt_vcc.c b/bin/varnishd/mgt_vcc.c
index f8e81ca..bbf813a 100644
--- a/bin/varnishd/mgt_vcc.c
+++ b/bin/varnishd/mgt_vcc.c
@@ -42,6 +42,7 @@
 
 #include "libvcl.h"
 #include "vcli.h"
+#include "vsub.h"
 #include "vcl.h"
 #include "cli_priv.h"
 #include "mgt_cli.h"
@@ -238,7 +239,7 @@ mgt_run_cc(const char *vcl, struct vsb *sb, int C_flag)
 	vp.magic = VCC_PRIV_MAGIC;
 	vp.sf = sf;
 	vp.vcl = vcl;
-	if (SUB_run(sb, run_vcc, &vp, "VCC-compiler", -1)) {
+	if (VSUB_run(sb, run_vcc, &vp, "VCC-compiler", -1)) {
 		(void)unlink(sf);
 		return (NULL);
 	}
@@ -261,13 +262,13 @@ mgt_run_cc(const char *vcl, struct vsb *sb, int C_flag)
 	cmdsb = mgt_make_cc_cmd(sf, of);
 
 	/* Run the C-compiler in a sub-shell */
-	i = SUB_run(sb, run_cc, VSB_data(cmdsb), "C-compiler", 10);
+	i = VSUB_run(sb, run_cc, VSB_data(cmdsb), "C-compiler", 10);
 
 	(void)unlink(sf);
 	VSB_delete(cmdsb);
 
 	if (!i)
-		i = SUB_run(sb, run_dlopen, of, "dlopen", 10);
+		i = VSUB_run(sb, run_dlopen, of, "dlopen", 10);
 
 	if (i) {
 		(void)unlink(of);
diff --git a/include/Makefile.am b/include/Makefile.am
index 66c2ddd..2cbe5d1 100644
--- a/include/Makefile.am
+++ b/include/Makefile.am
@@ -54,6 +54,7 @@ nobase_noinst_HEADERS = \
 	vrt.h \
 	vrt_obj.h \
 	vsb.h \
+	vsub.h \
 	vsha256.h \
 	vss.h \
 	vtcp.h
diff --git a/include/libvarnish.h b/include/libvarnish.h
index a40aa20..5bbe073 100644
--- a/include/libvarnish.h
+++ b/include/libvarnish.h
@@ -41,11 +41,6 @@ struct vsb;
 /* from libvarnish/num.c */
 const char *str2bytes(const char *p, uintmax_t *r, uintmax_t rel);
 
-/* from libvarnish/subproc.c */
-typedef void sub_func_f(void*);
-int SUB_run(struct vsb *sb, sub_func_f *func, void *priv, const char *name,
-    int maxlines);
-
 /* from libvarnish/time.c */
 #define TIM_FORMAT_SIZE 30
 void TIM_format(double t, char *p);
diff --git a/include/vsub.h b/include/vsub.h
new file mode 100644
index 0000000..6705b11
--- /dev/null
+++ b/include/vsub.h
@@ -0,0 +1,35 @@
+/*-
+ * Copyright (c) 2006 Verdens Gang AS
+ * Copyright (c) 2006-2011 Varnish Software AS
+ * All rights reserved.
+ *
+ * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ */
+
+/* from libvarnish/subproc.c */
+typedef void vsub_func_f(void*);
+
+int VSUB_run(struct vsb *sb, vsub_func_f *func, void *priv, const char *name,
+    int maxlines);
diff --git a/lib/libvarnish/Makefile.am b/lib/libvarnish/Makefile.am
index b8651cc..0148460 100644
--- a/lib/libvarnish/Makefile.am
+++ b/lib/libvarnish/Makefile.am
@@ -8,7 +8,7 @@ libvarnish_la_SOURCES = \
 	vav.c \
 	vas.c \
 	binary_heap.c \
-	subproc.c \
+	vsub.c \
 	cli_auth.c \
 	cli_common.c \
 	cli_serve.c \
diff --git a/lib/libvarnish/subproc.c b/lib/libvarnish/subproc.c
deleted file mode 100644
index 99f924e..0000000
--- a/lib/libvarnish/subproc.c
+++ /dev/null
@@ -1,133 +0,0 @@
-/*-
- * Copyright (c) 2006 Verdens Gang AS
- * Copyright (c) 2006-2011 Varnish Software AS
- * All rights reserved.
- *
- * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- *
- * Run stuff in a child process
- */
-
-#include "config.h"
-
-#include <fcntl.h>
-#include <string.h>
-#include <unistd.h>
-#include <stdio.h>
-
-#include <sys/wait.h>
-
-#include "vsb.h"
-#include "vlu.h"
-#include "libvarnish.h"
-
-struct sub_priv {
-	const char	*name;
-	struct vsb	*sb;
-	int		lines;
-	int		maxlines;
-};
-
-static int
-sub_vlu(void *priv, const char *str)
-{
-	struct sub_priv *sp;
-
-	sp = priv;
-	if (!sp->lines++)
-		VSB_printf(sp->sb, "Message from %s:\n", sp->name);
-	if (sp->maxlines < 0 || sp->lines <= sp->maxlines)
-		VSB_printf(sp->sb, "%s\n", str);
-	return (0);
-}
-
-int
-SUB_run(struct vsb *sb, sub_func_f *func, void *priv, const char *name,
-    int maxlines)
-{
-	int rv, p[2], sfd, status;
-	pid_t pid;
-	struct vlu *vlu;
-	struct sub_priv sp;
-
-	sp.sb = sb;
-	sp.name = name;
-	sp.lines = 0;
-	sp.maxlines = maxlines;
-
-	if (pipe(p) < 0) {
-		VSB_printf(sb, "Starting %s: pipe() failed: %s",
-		    name, strerror(errno));
-		return (-1);
-	}
-	assert(p[0] > STDERR_FILENO);
-	assert(p[1] > STDERR_FILENO);
-	if ((pid = fork()) < 0) {
-		VSB_printf(sb, "Starting %s: fork() failed: %s",
-		    name, strerror(errno));
-		AZ(close(p[0]));
-		AZ(close(p[1]));
-		return (-1);
-	}
-	if (pid == 0) {
-		AZ(close(STDIN_FILENO));
-		assert(open("/dev/null", O_RDONLY) == STDIN_FILENO);
-		assert(dup2(p[1], STDOUT_FILENO) == STDOUT_FILENO);
-		assert(dup2(p[1], STDERR_FILENO) == STDERR_FILENO);
-		/* Close all other fds */
-		for (sfd = STDERR_FILENO + 1; sfd < 100; sfd++)
-			(void)close(sfd);
-		func(priv);
-		_exit(1);
-	}
-	AZ(close(p[1]));
-	vlu = VLU_New(&sp, sub_vlu, 0);
-	while (!VLU_Fd(p[0], vlu))
-		continue;
-	AZ(close(p[0]));
-	VLU_Destroy(vlu);
-	if (sp.maxlines >= 0 && sp.lines > sp.maxlines)
-		VSB_printf(sb, "[%d lines truncated]\n",
-		    sp.lines - sp.maxlines);
-	do {
-		rv = waitpid(pid, &status, 0);
-		if (rv < 0 && errno != EINTR) {
-			VSB_printf(sb, "Running %s: waitpid() failed: %s\n",
-			    name, strerror(errno));
-			return (-1);
-		}
-	} while (rv < 0);
-	if (!WIFEXITED(status) || WEXITSTATUS(status) != 0) {
-		VSB_printf(sb, "Running %s failed", name);
-		if (WIFEXITED(status))
-			VSB_printf(sb, ", exit %d", WEXITSTATUS(status));
-		if (WIFSIGNALED(status))
-			VSB_printf(sb, ", signal %d", WTERMSIG(status));
-		if (WCOREDUMP(status))
-			VSB_printf(sb, ", core dumped");
-		VSB_printf(sb, "\n");
-		return (-1);
-	}
-	return (0);
-}
diff --git a/lib/libvarnish/vsub.c b/lib/libvarnish/vsub.c
new file mode 100644
index 0000000..90ceacb
--- /dev/null
+++ b/lib/libvarnish/vsub.c
@@ -0,0 +1,134 @@
+/*-
+ * Copyright (c) 2006 Verdens Gang AS
+ * Copyright (c) 2006-2011 Varnish Software AS
+ * All rights reserved.
+ *
+ * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * Run stuff in a child process
+ */
+
+#include "config.h"
+
+#include <fcntl.h>
+#include <string.h>
+#include <unistd.h>
+#include <stdio.h>
+
+#include <sys/wait.h>
+
+#include "vas.h"
+#include "vlu.h"
+#include "vsb.h"
+#include "vsub.h"
+
+struct vsub_priv {
+	const char	*name;
+	struct vsb	*sb;
+	int		lines;
+	int		maxlines;
+};
+
+static int
+vsub_vlu(void *priv, const char *str)
+{
+	struct vsub_priv *sp;
+
+	sp = priv;
+	if (!sp->lines++)
+		VSB_printf(sp->sb, "Message from %s:\n", sp->name);
+	if (sp->maxlines < 0 || sp->lines <= sp->maxlines)
+		VSB_printf(sp->sb, "%s\n", str);
+	return (0);
+}
+
+int
+VSUB_run(struct vsb *sb, vsub_func_f *func, void *priv, const char *name,
+    int maxlines)
+{
+	int rv, p[2], sfd, status;
+	pid_t pid;
+	struct vlu *vlu;
+	struct vsub_priv sp;
+
+	sp.sb = sb;
+	sp.name = name;
+	sp.lines = 0;
+	sp.maxlines = maxlines;
+
+	if (pipe(p) < 0) {
+		VSB_printf(sb, "Starting %s: pipe() failed: %s",
+		    name, strerror(errno));
+		return (-1);
+	}
+	assert(p[0] > STDERR_FILENO);
+	assert(p[1] > STDERR_FILENO);
+	if ((pid = fork()) < 0) {
+		VSB_printf(sb, "Starting %s: fork() failed: %s",
+		    name, strerror(errno));
+		AZ(close(p[0]));
+		AZ(close(p[1]));
+		return (-1);
+	}
+	if (pid == 0) {
+		AZ(close(STDIN_FILENO));
+		assert(open("/dev/null", O_RDONLY) == STDIN_FILENO);
+		assert(dup2(p[1], STDOUT_FILENO) == STDOUT_FILENO);
+		assert(dup2(p[1], STDERR_FILENO) == STDERR_FILENO);
+		/* Close all other fds */
+		for (sfd = STDERR_FILENO + 1; sfd < 100; sfd++)
+			(void)close(sfd);
+		func(priv);
+		_exit(1);
+	}
+	AZ(close(p[1]));
+	vlu = VLU_New(&sp, vsub_vlu, 0);
+	while (!VLU_Fd(p[0], vlu))
+		continue;
+	AZ(close(p[0]));
+	VLU_Destroy(vlu);
+	if (sp.maxlines >= 0 && sp.lines > sp.maxlines)
+		VSB_printf(sb, "[%d lines truncated]\n",
+		    sp.lines - sp.maxlines);
+	do {
+		rv = waitpid(pid, &status, 0);
+		if (rv < 0 && errno != EINTR) {
+			VSB_printf(sb, "Running %s: waitpid() failed: %s\n",
+			    name, strerror(errno));
+			return (-1);
+		}
+	} while (rv < 0);
+	if (!WIFEXITED(status) || WEXITSTATUS(status) != 0) {
+		VSB_printf(sb, "Running %s failed", name);
+		if (WIFEXITED(status))
+			VSB_printf(sb, ", exit %d", WEXITSTATUS(status));
+		if (WIFSIGNALED(status))
+			VSB_printf(sb, ", signal %d", WTERMSIG(status));
+		if (WCOREDUMP(status))
+			VSB_printf(sb, ", core dumped");
+		VSB_printf(sb, "\n");
+		return (-1);
+	}
+	return (0);
+}



From geoff at varnish-cache.org  Mon Jan  9 20:52:16 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:16 +0100
Subject: [experimental-ims] b37c01d Since we're cleaning up,
	rename TIM_ to VTIM_ and take out of libvarnish.h
Message-ID: <E1RkMCW-0007bZ-AZ@project.varnish-software.com>

commit b37c01dfe71edd703758f555fdc8c51554ab4993
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sun Oct 9 18:51:03 2011 +0000

    Since we're cleaning up, rename TIM_ to VTIM_ and take out of libvarnish.h

diff --git a/bin/varnishd/cache_acceptor.c b/bin/varnishd/cache_acceptor.c
index 87a9180..4feefb6 100644
--- a/bin/varnishd/cache_acceptor.c
+++ b/bin/varnishd/cache_acceptor.c
@@ -34,6 +34,7 @@
 
 #include "vcli.h"
 #include "vtcp.h"
+#include "vtim.h"
 #include "cli_priv.h"
 
 static pthread_t	VCA_thread;
@@ -167,7 +168,7 @@ vca_pace_check(void)
 	p = vca_pace;
 	Lck_Unlock(&pace_mtx);
 	if (p > 0.0)
-		TIM_sleep(p);
+		VTIM_sleep(p);
 }
 
 static void
@@ -273,7 +274,7 @@ VCA_SetupSess(struct worker *w)
 	sp->fd = wa->acceptsock;
 	sp->vsl_id = wa->acceptsock | VSL_CLIENTMARKER ;
 	wa->acceptsock = -1;
-	sp->t_open = TIM_real();
+	sp->t_open = VTIM_real();
 	sp->t_end = sp->t_open;
 	sp->mylsock = wa->acceptlsock;
 	CHECK_OBJ_NOTNULL(sp->mylsock, LISTEN_SOCK_MAGIC);
@@ -313,14 +314,14 @@ vca_acct(void *arg)
 	hack_ready = 1;
 
 	need_test = 1;
-	t0 = TIM_real();
+	t0 = VTIM_real();
 	while (1) {
 		(void)sleep(1);
 #ifdef SO_SNDTIMEO_WORKS
 		if (params->send_timeout != send_timeout) {
 			need_test = 1;
 			send_timeout = params->send_timeout;
-			tv_sndtimeo = TIM_timeval(send_timeout);
+			tv_sndtimeo = VTIM_timeval(send_timeout);
 			VTAILQ_FOREACH(ls, &heritage.socks, list) {
 				if (ls->sock < 0)
 					continue;
@@ -334,7 +335,7 @@ vca_acct(void *arg)
 		if (params->sess_timeout != sess_timeout) {
 			need_test = 1;
 			sess_timeout = params->sess_timeout;
-			tv_rcvtimeo = TIM_timeval(sess_timeout);
+			tv_rcvtimeo = VTIM_timeval(sess_timeout);
 			VTAILQ_FOREACH(ls, &heritage.socks, list) {
 				if (ls->sock < 0)
 					continue;
@@ -344,7 +345,7 @@ vca_acct(void *arg)
 			}
 		}
 #endif
-		now = TIM_real();
+		now = VTIM_real();
 		VSC_C_main->uptime = (uint64_t)(now - t0);
 	}
 	NEEDLESS_RETURN(NULL);
diff --git a/bin/varnishd/cache_backend_poll.c b/bin/varnishd/cache_backend_poll.c
index c90cda9..c9497bb 100644
--- a/bin/varnishd/cache_backend_poll.c
+++ b/bin/varnishd/cache_backend_poll.c
@@ -46,6 +46,7 @@
 #include "cache.h"
 #include "vrt.h"
 #include "vtcp.h"
+#include "vtim.h"
 #include "cache_backend.h"
 
 /* Default averaging rate, we want something pretty responsive */
@@ -131,28 +132,28 @@ vbp_poke(struct vbp_target *vt)
 	bp = vt->backend;
 	CHECK_OBJ_NOTNULL(bp, BACKEND_MAGIC);
 
-	t_start = t_now = TIM_real();
+	t_start = t_now = VTIM_real();
 	t_end = t_start + vt->probe.timeout;
 	tmo = (int)round((t_end - t_now) * 1e3);
 
 	s = -1;
 	if (params->prefer_ipv6 && bp->ipv6 != NULL) {
 		s = vbp_connect(PF_INET6, bp->ipv6, bp->ipv6len, tmo);
-		t_now = TIM_real();
+		t_now = VTIM_real();
 		tmo = (int)round((t_end - t_now) * 1e3);
 		if (s >= 0)
 			vt->good_ipv6 |= 1;
 	}
 	if (tmo > 0 && s < 0 && bp->ipv4 != NULL) {
 		s = vbp_connect(PF_INET, bp->ipv4, bp->ipv4len, tmo);
-		t_now = TIM_real();
+		t_now = VTIM_real();
 		tmo = (int)round((t_end - t_now) * 1e3);
 		if (s >= 0)
 			vt->good_ipv4 |= 1;
 	}
 	if (tmo > 0 && s < 0 && bp->ipv6 != NULL) {
 		s = vbp_connect(PF_INET6, bp->ipv6, bp->ipv6len, tmo);
-		t_now = TIM_real();
+		t_now = VTIM_real();
 		tmo = (int)round((t_end - t_now) * 1e3);
 		if (s >= 0)
 			vt->good_ipv6 |= 1;
@@ -208,7 +209,7 @@ vbp_poke(struct vbp_target *vt)
 		return;
 
 	/* So we have a good receive ... */
-	t_now = TIM_real();
+	t_now = VTIM_real();
 	vt->last = t_now - t_start;
 	vt->good_recv |= 1;
 
@@ -350,7 +351,7 @@ vbp_wrk_poll_backend(void *priv)
 		vbp_has_poked(vt);
 
 		if (!vt->stop)
-			TIM_sleep(vt->probe.interval);
+			VTIM_sleep(vt->probe.interval);
 	}
 	return (NULL);
 }
diff --git a/bin/varnishd/cache_ban.c b/bin/varnishd/cache_ban.c
index f346b4d..8396c7f 100644
--- a/bin/varnishd/cache_ban.c
+++ b/bin/varnishd/cache_ban.c
@@ -70,6 +70,7 @@
 
 #include "vcli.h"
 #include "vend.h"
+#include "vtim.h"
 #include "cli_priv.h"
 #include "hash_slinger.h"
 
@@ -402,7 +403,7 @@ BAN_Insert(struct ban *b)
 	b->spec = malloc(ln + 13L);	/* XXX */
 	XXXAN(b->spec);
 
-	t0 = TIM_real();
+	t0 = VTIM_real();
 	memcpy(b->spec, &t0, sizeof t0);
 	b->spec[12] = (b->flags & BAN_F_REQ) ? 1 : 0;
 	memcpy(b->spec + 13, VSB_data(b->vsb), ln);
@@ -835,10 +836,10 @@ ban_lurker(struct sess *sp, void *priv)
 	while (1) {
 		if (params->ban_lurker_sleep == 0.0) {
 			/* Lurker is disabled.  */
-			TIM_sleep(1.0);
+			VTIM_sleep(1.0);
 			continue;
 		}
-		TIM_sleep(params->ban_lurker_sleep);
+		VTIM_sleep(params->ban_lurker_sleep);
 		ban_lurker_work(sp);
 		WSL_Flush(sp->wrk, 0);
 		WRK_SumStat(sp->wrk);
diff --git a/bin/varnishd/cache_center.c b/bin/varnishd/cache_center.c
index 1415170..6ab8a11 100644
--- a/bin/varnishd/cache_center.c
+++ b/bin/varnishd/cache_center.c
@@ -71,6 +71,7 @@ DOT acceptor -> start [style=bold,color=green]
 
 #include "vcl.h"
 #include "vtcp.h"
+#include "vtim.h"
 #include "cli_priv.h"
 #include "hash_slinger.h"
 #include "stevedore.h"
@@ -211,7 +212,7 @@ cnt_prepresp(struct sess *sp)
 		}
 	}
 
-	sp->t_resp = TIM_real();
+	sp->t_resp = VTIM_real();
 	if (sp->obj->objcore != NULL) {
 		if ((sp->t_resp - sp->obj->last_lru) > params->lru_timeout &&
 		    EXP_Touch(sp->obj->objcore))
@@ -326,7 +327,7 @@ cnt_done(struct sess *sp)
 
 	SES_Charge(sp);
 
-	sp->t_end = TIM_real();
+	sp->t_end = VTIM_real();
 	sp->wrk->lastused = sp->t_end;
 	if (sp->xid == 0) {
 		sp->t_req = sp->t_end;
@@ -466,7 +467,7 @@ cnt_error(struct sess *sp)
 
 	http_PutProtocol(w, sp->vsl_id, h, "HTTP/1.1");
 	http_PutStatus(h, sp->err_code);
-	TIM_format(TIM_real(), date);
+	VTIM_format(VTIM_real(), date);
 	http_PrintfHeader(w, sp->vsl_id, h, "Date: %s", date);
 	http_PrintfHeader(w, sp->vsl_id, h, "Server: Varnish");
 
@@ -581,7 +582,7 @@ cnt_fetch(struct sess *sp)
 		 * What does RFC2616 think about TTL ?
 		 */
 		EXP_Clr(&sp->wrk->exp);
-		sp->wrk->exp.entered = TIM_real();
+		sp->wrk->exp.entered = VTIM_real();
 		RFC2616_Ttl(sp);
 
 		/* pass from vclrecv{} has negative TTL */
@@ -823,7 +824,7 @@ cnt_fetchbody(struct sess *sp)
 	http_CopyHome(sp->wrk, sp->vsl_id, hp2);
 
 	if (http_GetHdr(hp, H_Last_Modified, &b))
-		sp->obj->last_modified = TIM_parse(b);
+		sp->obj->last_modified = VTIM_parse(b);
 	else
 		sp->obj->last_modified = floor(sp->wrk->exp.entered);
 
@@ -1464,7 +1465,7 @@ cnt_start(struct sess *sp)
 
 	/* Update stats of various sorts */
 	sp->wrk->stats.client_req++;
-	sp->t_req = TIM_real();
+	sp->t_req = VTIM_real();
 	sp->wrk->lastused = sp->t_req;
 	sp->wrk->acct_tmp.req++;
 
diff --git a/bin/varnishd/cache_expire.c b/bin/varnishd/cache_expire.c
index 35d0a1e..aed6807 100644
--- a/bin/varnishd/cache_expire.c
+++ b/bin/varnishd/cache_expire.c
@@ -57,6 +57,7 @@
 #include "cache.h"
 #include "hash_slinger.h"
 #include "stevedore.h"
+#include "vtim.h"
 
 static pthread_t exp_thread;
 static struct binheap *exp_heap;
@@ -337,14 +338,14 @@ exp_timer(struct sess *sp, void *priv)
 	struct object *o;
 
 	(void)priv;
-	t = TIM_real();
+	t = VTIM_real();
 	oc = NULL;
 	while (1) {
 		if (oc == NULL) {
 			WSL_Flush(sp->wrk, 0);
 			WRK_SumStat(sp->wrk);
-			TIM_sleep(params->expiry_sleep);
-			t = TIM_real();
+			VTIM_sleep(params->expiry_sleep);
+			t = VTIM_real();
 		}
 
 		Lck_Lock(&exp_mtx);
@@ -360,7 +361,7 @@ exp_timer(struct sess *sp, void *priv)
 		 * got out of date, refresh it and check again.
 		 */
 		if (oc->timer_when > t)
-			t = TIM_real();
+			t = VTIM_real();
 		if (oc->timer_when > t) {
 			Lck_Unlock(&exp_mtx);
 			oc = NULL;
diff --git a/bin/varnishd/cache_pipe.c b/bin/varnishd/cache_pipe.c
index 74343ba..418ded1 100644
--- a/bin/varnishd/cache_pipe.c
+++ b/bin/varnishd/cache_pipe.c
@@ -36,6 +36,7 @@
 
 #include "cache.h"
 #include "vtcp.h"
+#include "vtim.h"
 
 static int
 rdf(int fd0, int fd1)
@@ -90,7 +91,7 @@ PipeSession(struct sess *sp)
 		return;
 	}
 
-	sp->t_resp = TIM_real();
+	sp->t_resp = VTIM_real();
 
 	memset(fds, 0, sizeof fds);
 
diff --git a/bin/varnishd/cache_pool.c b/bin/varnishd/cache_pool.c
index e146043..4421969 100644
--- a/bin/varnishd/cache_pool.c
+++ b/bin/varnishd/cache_pool.c
@@ -51,6 +51,7 @@
 #include "cache_waiter.h"
 #include "hash_slinger.h"
 #include "vtcp.h"
+#include "vtim.h"
 
 /*--------------------------------------------------------------------
  * MAC OS/X is incredibly moronic when it comes to time and such...
@@ -224,7 +225,7 @@ Pool_Work_Thread(void *priv, struct worker *w)
 		} else if (VTAILQ_EMPTY(&pp->socks)) {
 			/* Nothing to do: To sleep, perchance to dream ... */
 			if (isnan(w->lastused))
-				w->lastused = TIM_real();
+				w->lastused = VTIM_real();
 			VTAILQ_INSERT_HEAD(&pp->idle, w, list);
 			if (!stats_clean)
 				WRK_SumStat(w);
@@ -343,7 +344,7 @@ Pool_Schedule(struct pool *pp, struct sess *sp)
 	 * XXX: a notice might be polite, but would potentially
 	 * XXX: sleep whichever thread got us here
 	 */
-	sp->t_end = TIM_real();
+	sp->t_end = VTIM_real();
 	if (sp->vcl != NULL) {
 		/*
 		 * A session parked on a busy object can come here
@@ -401,10 +402,10 @@ pool_breed(struct pool *qp, const pthread_attr_t *tp_attr)
 			Lck_Lock(&pool_mtx);
 			VSC_C_main->threads_limited++;
 			Lck_Unlock(&pool_mtx);
-			TIM_sleep(params->wthread_fail_delay * 1e-3);
+			VTIM_sleep(params->wthread_fail_delay * 1e-3);
 		} else {
 			AZ(pthread_detach(tp));
-			TIM_sleep(params->wthread_add_delay * 1e-3);
+			VTIM_sleep(params->wthread_add_delay * 1e-3);
 			qp->nthr++;
 			Lck_Lock(&pool_mtx);
 			VSC_C_main->threads++;
@@ -476,7 +477,7 @@ pool_herder(void *priv)
 		if (pp->nthr <= params->wthread_min) 
 			continue;
 
-		t_idle = TIM_real() - params->wthread_timeout;
+		t_idle = VTIM_real() - params->wthread_timeout;
 
 		Lck_Lock(&pp->mtx);
 		VSC_C_main->sess_queued += pp->nqueued;
diff --git a/bin/varnishd/cache_response.c b/bin/varnishd/cache_response.c
index 4d7c88b..b4459da 100644
--- a/bin/varnishd/cache_response.c
+++ b/bin/varnishd/cache_response.c
@@ -38,6 +38,7 @@
 #include "cache.h"
 #include "stevedore.h"
 #include "vct.h"
+#include "vtim.h"
 
 /*--------------------------------------------------------------------*/
 
@@ -133,7 +134,7 @@ RES_BuildHttp(const struct sess *sp)
 		http_PrintfHeader(sp->wrk, sp->vsl_id, sp->wrk->resp,
 		    "Transfer-Encoding: chunked");
 
-	TIM_format(TIM_real(), time_str);
+	VTIM_format(VTIM_real(), time_str);
 	http_PrintfHeader(sp->wrk, sp->vsl_id, sp->wrk->resp, "Date: %s", time_str);
 
 	if (sp->xid != sp->obj->xid)
diff --git a/bin/varnishd/cache_shmlog.c b/bin/varnishd/cache_shmlog.c
index 8c06897..75f662d 100644
--- a/bin/varnishd/cache_shmlog.c
+++ b/bin/varnishd/cache_shmlog.c
@@ -34,6 +34,7 @@
 #include "cache.h"
 #include "vmb.h"
 #include "vsm.h"
+#include "vtim.h"
 
 /* These cannot be struct lock, which depends on vsm/vsl working */
 static pthread_mutex_t vsl_mtx;
@@ -285,7 +286,7 @@ VSL_Init(void)
 	vsl_ptr = vsl_start + 1;
 
 	vsl_wrap();
-	VSM_head->starttime = (intmax_t)TIM_real();
+	VSM_head->starttime = (intmax_t)VTIM_real();
 	memset(VSM_head->panicstr, '\0', sizeof *VSM_head->panicstr);
 	memset(VSC_C_main, 0, sizeof *VSC_C_main);
 	VSM_head->child_pid = getpid();
diff --git a/bin/varnishd/cache_vrt.c b/bin/varnishd/cache_vrt.c
index a53799e..5c633e2 100644
--- a/bin/varnishd/cache_vrt.c
+++ b/bin/varnishd/cache_vrt.c
@@ -43,6 +43,7 @@
 #include "vrt_obj.h"
 #include "vav.h"
 #include "vcl.h"
+#include "vtim.h"
 #include "hash_slinger.h"
 #include "cache_backend.h"
 
@@ -276,7 +277,7 @@ VRT_r_now(const struct sess *sp)
 {
 
 	(void)sp;
-	return (TIM_real());
+	return (VTIM_real());
 }
 
 /*--------------------------------------------------------------------*/
@@ -339,8 +340,8 @@ VRT_time_string(const struct sess *sp, double t)
 {
 	char *p;
 
-	AN(p = WS_Alloc(sp->http->ws, TIM_FORMAT_SIZE));
-	TIM_format(t, p);
+	AN(p = WS_Alloc(sp->http->ws, VTIM_FORMAT_SIZE));
+	VTIM_format(t, p);
 	return p;
 }
 
diff --git a/bin/varnishd/cache_vrt_var.c b/bin/varnishd/cache_vrt_var.c
index 1343ae4..ff523df 100644
--- a/bin/varnishd/cache_vrt_var.c
+++ b/bin/varnishd/cache_vrt_var.c
@@ -37,6 +37,7 @@
 
 #include "vrt.h"
 #include "vtcp.h"
+#include "vtim.h"
 #include "vrt_obj.h"
 #include "cache_backend.h"
 #include "hash_slinger.h"
@@ -535,7 +536,7 @@ VRT_r_obj_lastuse(const struct sess *sp)
 
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 	CHECK_OBJ_NOTNULL(sp->obj, OBJECT_MAGIC);	/* XXX */
-	return (TIM_real() - sp->obj->last_use);
+	return (VTIM_real() - sp->obj->last_use);
 }
 
 unsigned
diff --git a/bin/varnishd/cache_waiter_epoll.c b/bin/varnishd/cache_waiter_epoll.c
index d4168db..c66d15a 100644
--- a/bin/varnishd/cache_waiter_epoll.c
+++ b/bin/varnishd/cache_waiter_epoll.c
@@ -42,6 +42,7 @@
 
 #include "cache.h"
 #include "cache_waiter.h"
+#include "vtim.h"
 
 #ifndef EPOLLRDHUP
 #  define EPOLLRDHUP 0
@@ -186,7 +187,7 @@ vwe_thread(void *priv)
 			continue;
 
 		/* check for timeouts */
-		deadline = TIM_real() - params->sess_timeout;
+		deadline = VTIM_real() - params->sess_timeout;
 		for (;;) {
 			sp = VTAILQ_FIRST(&vwe->sesshead);
 			if (sp == NULL)
@@ -215,7 +216,7 @@ vwe_sess_timeout_ticker(void *priv)
 	while (1) {
 		/* ticking */
 		assert(write(vwe->timer_pipes[1], &ticker, 1));
-		TIM_sleep(100 * 1e-3);
+		VTIM_sleep(100 * 1e-3);
 	}
 	return NULL;
 }
diff --git a/bin/varnishd/cache_waiter_kqueue.c b/bin/varnishd/cache_waiter_kqueue.c
index f023c3f..e7cb299 100644
--- a/bin/varnishd/cache_waiter_kqueue.c
+++ b/bin/varnishd/cache_waiter_kqueue.c
@@ -43,6 +43,7 @@
 
 #include "cache.h"
 #include "cache_waiter.h"
+#include "vtim.h"
 
 #define NKEV	100
 
@@ -183,7 +184,7 @@ vwk_thread(void *priv)
 		 * would not know we meant "the old fd of this number".
 		 */
 		vwk_kq_flush(vwk);
-		deadline = TIM_real() - params->sess_timeout;
+		deadline = VTIM_real() - params->sess_timeout;
 		for (;;) {
 			sp = VTAILQ_FIRST(&vwk->sesshead);
 			if (sp == NULL)
diff --git a/bin/varnishd/cache_waiter_poll.c b/bin/varnishd/cache_waiter_poll.c
index df0439b..f3f3289 100644
--- a/bin/varnishd/cache_waiter_poll.c
+++ b/bin/varnishd/cache_waiter_poll.c
@@ -36,6 +36,7 @@
 #include "cache.h"
 
 #include "cache_waiter.h"
+#include "vtim.h"
 
 #define NEEV	128
 
@@ -139,7 +140,7 @@ vwp_main(void *priv)
 		assert(vwp->pollfd[vwp->pipes[1]].fd == -1);
 		v = poll(vwp->pollfd, vwp->hpoll + 1, 100);
 		assert(v >= 0);
-		deadline = TIM_real() - params->sess_timeout;
+		deadline = VTIM_real() - params->sess_timeout;
 		v2 = v;
 		VTAILQ_FOREACH_SAFE(sp, &vwp->sesshead, list, sp2) {
 			if (v != 0 && v2 == 0)
diff --git a/bin/varnishd/cache_waiter_ports.c b/bin/varnishd/cache_waiter_ports.c
index 04c6e92..ca7e229 100644
--- a/bin/varnishd/cache_waiter_ports.c
+++ b/bin/varnishd/cache_waiter_ports.c
@@ -43,6 +43,7 @@
 
 #include "cache.h"
 #include "cache_waiter.h"
+#include "vtim.h"
 
 #define MAX_EVENTS 256
 
@@ -195,7 +196,7 @@ vws_thread(void *priv)
 			vws_port_ev(vws, ev + ei);
 
 		/* check for timeouts */
-		now = TIM_real();
+		now = VTIM_real();
 		deadline = now - params->sess_timeout;
 
 		/*
@@ -234,7 +235,7 @@ vws_thread(void *priv)
 			} else if (tmo > max_t) {
 				timeout = &max_ts;
 			} else {
-				ts = TIM_timespec(tmo);
+				ts = VTIM_timespec(tmo);
 				timeout = &ts;
 			}
 		} else {
diff --git a/bin/varnishd/hash_critbit.c b/bin/varnishd/hash_critbit.c
index 7428097..e5b0746 100644
--- a/bin/varnishd/hash_critbit.c
+++ b/bin/varnishd/hash_critbit.c
@@ -38,6 +38,7 @@
 #include "hash_slinger.h"
 #include "cli_priv.h"
 #include "vmb.h"
+#include "vtim.h"
 
 static struct lock hcb_mtx;
 
@@ -367,7 +368,7 @@ hcb_cleaner(void *priv)
 		VTAILQ_CONCAT(&dead_h, &cool_h, hoh_list);
 		Lck_Unlock(&hcb_mtx);
 		WRK_SumStat(&ww);
-		TIM_sleep(params->critbit_cooloff);
+		VTIM_sleep(params->critbit_cooloff);
 	}
 	NEEDLESS_RETURN(NULL);
 }
diff --git a/bin/varnishd/mgt_child.c b/bin/varnishd/mgt_child.c
index 6c1cf71..e676f48 100644
--- a/bin/varnishd/mgt_child.c
+++ b/bin/varnishd/mgt_child.c
@@ -53,6 +53,7 @@
 #include "vss.h"
 #include "vbm.h"
 #include "vtcp.h"
+#include "vtim.h"
 
 pid_t		child_pid = -1;
 
@@ -439,7 +440,7 @@ mgt_save_panic(void)
 		VSB_delete(child_panic);
 	child_panic = VSB_new_auto();
 	XXXAN(child_panic);
-	TIM_format(TIM_real(), time_str);
+	VTIM_format(VTIM_real(), time_str);
 	VSB_printf(child_panic, "Last panic at: %s\n", time_str);
 	VSB_cat(child_panic, VSM_head->panicstr);
 	AZ(VSB_finish(child_panic));
diff --git a/bin/varnishd/rfc2616.c b/bin/varnishd/rfc2616.c
index 36936b6..6de52e2 100644
--- a/bin/varnishd/rfc2616.c
+++ b/bin/varnishd/rfc2616.c
@@ -33,6 +33,7 @@
 #include <math.h>
 
 #include "cache.h"
+#include "vtim.h"
 
 /*--------------------------------------------------------------------
  * TTL and Age calculation in Varnish
@@ -88,10 +89,10 @@ RFC2616_Ttl(const struct sess *sp)
 		sp->wrk->exp.age = age;
 	}
 	if (http_GetHdr(hp, H_Expires, &p))
-		h_expires = TIM_parse(p);
+		h_expires = VTIM_parse(p);
 
 	if (http_GetHdr(hp, H_Date, &p))
-		h_date = TIM_parse(p);
+		h_date = VTIM_parse(p);
 
 	switch (sp->err_code) {
 	default:
@@ -315,7 +316,7 @@ RFC2616_Do_Cond(const struct sess *sp)
 	if (http_GetHdr(sp->http, H_If_Modified_Since, &p) ) {
 		if (!sp->obj->last_modified)
 			return (0);
-		ims = TIM_parse(p);
+		ims = VTIM_parse(p);
 		if (ims > sp->t_req)	/* [RFC2616 14.25] */
 			return (0);
 		if (sp->obj->last_modified > ims)
diff --git a/bin/varnishd/storage_persistent_silo.c b/bin/varnishd/storage_persistent_silo.c
index ba84b79..2defc0d 100644
--- a/bin/varnishd/storage_persistent_silo.c
+++ b/bin/varnishd/storage_persistent_silo.c
@@ -41,6 +41,7 @@
 #include "stevedore.h"
 #include "hash_slinger.h"
 #include "vsha256.h"
+#include "vtim.h"
 
 #include "persistent.h"
 #include "storage_persistent.h"
@@ -122,7 +123,7 @@ smp_load_seg(const struct sess *sp, const struct smp_sc *sc,
 	struct smp_object *so;
 	struct objcore *oc;
 	uint32_t no;
-	double t_now = TIM_real();
+	double t_now = VTIM_real();
 	struct smp_signctx ctx[1];
 
 	ASSERT_SILO_THREAD(sc);
diff --git a/bin/varnishd/varnishd.c b/bin/varnishd/varnishd.c
index 2d7db59..360cdad 100644
--- a/bin/varnishd/varnishd.c
+++ b/bin/varnishd/varnishd.c
@@ -55,6 +55,7 @@
 
 #include "vav.h"
 #include "vin.h"
+#include "vtim.h"
 #include "heritage.h"
 #include "mgt.h"
 #include "hash_slinger.h"
@@ -376,9 +377,9 @@ main(int argc, char * const *argv)
 	 */
 	AZ(setenv("TZ", "UTC", 1));
 	tzset();
-	assert(TIM_parse("Sun, 06 Nov 1994 08:49:37 GMT") == 784111777);
-	assert(TIM_parse("Sunday, 06-Nov-94 08:49:37 GMT") == 784111777);
-	assert(TIM_parse("Sun Nov  6 08:49:37 1994") == 784111777);
+	assert(VTIM_parse("Sun, 06 Nov 1994 08:49:37 GMT") == 784111777);
+	assert(VTIM_parse("Sunday, 06-Nov-94 08:49:37 GMT") == 784111777);
+	assert(VTIM_parse("Sun Nov  6 08:49:37 1994") == 784111777);
 
 	/*
 	 * Check that our SHA256 works
diff --git a/bin/varnishd/vsm.c b/bin/varnishd/vsm.c
index 28f98e5..dc265ba 100644
--- a/bin/varnishd/vsm.c
+++ b/bin/varnishd/vsm.c
@@ -54,6 +54,7 @@
 #include "common.h"
 #include "vsm.h"
 #include "vmb.h"
+#include "vtim.h"
 
 /* These two come from beyond (mgt_shmem.c actually) */
 struct VSM_head		*VSM_head;
@@ -87,7 +88,7 @@ vsm_release(unsigned seq)
 static void
 vsm_cleanup(void)
 {
-	unsigned now = (unsigned)TIM_mono();
+	unsigned now = (unsigned)VTIM_mono();
 	struct VSM_chunk *sha, *sha2;
 	unsigned seq;
 
@@ -203,7 +204,7 @@ VSM__Free(const void *ptr)
 	AN(sha);
 	seq = vsm_mark();
 	bprintf(sha->class, "%s", VSM_CLASS_COOL);
-	sha->state = (unsigned)TIM_mono();
+	sha->state = (unsigned)VTIM_mono();
 	vsm_release(seq);
 }
 
diff --git a/bin/varnishtest/vtc.c b/bin/varnishtest/vtc.c
index e31eaca..83bbb22 100644
--- a/bin/varnishtest/vtc.c
+++ b/bin/varnishtest/vtc.c
@@ -40,6 +40,7 @@
 #include "vtc.h"
 
 #include "vav.h"
+#include "vtim.h"
 #include "libvarnish.h"
 
 
@@ -129,10 +130,10 @@ macro_get(const char *b, const char *e)
 	l = e - b;
 
 	if (l == 4 && !memcmp(b, "date", l)) {
-		double t = TIM_real();
+		double t = VTIM_real();
 		retval = malloc(64);
 		AN(retval);
-		TIM_format(t, retval);
+		VTIM_format(t, retval);
 		return (retval);
 	}
 
@@ -415,7 +416,7 @@ cmd_delay(CMD_ARGS)
 	AZ(av[2]);
 	f = strtod(av[1], NULL);
 	vtc_log(vl, 3, "delaying %g second(s)", f);
-	TIM_sleep(f);
+	VTIM_sleep(f);
 }
 
 /**********************************************************************
diff --git a/bin/varnishtest/vtc_log.c b/bin/varnishtest/vtc_log.c
index 51c6646..c06f319 100644
--- a/bin/varnishtest/vtc_log.c
+++ b/bin/varnishtest/vtc_log.c
@@ -36,6 +36,7 @@
 #include "vtc.h"
 
 #include "libvarnish.h"
+#include "vtim.h"
 
 static pthread_mutex_t	vtclog_mtx;
 static char		*vtclog_buf;
@@ -58,7 +59,7 @@ void
 vtc_loginit(char *buf, unsigned buflen)
 {
 
-	t0 = TIM_mono();
+	t0 = VTIM_mono();
 	vtclog_buf = buf;
 	vtclog_left = buflen;
 	AZ(pthread_mutex_init(&vtclog_mtx, NULL));
@@ -126,7 +127,7 @@ vtc_log(struct vtclog *vl, unsigned lvl, const char *fmt, ...)
 	double tx;
 
 	CHECK_OBJ_NOTNULL(vl, VTCLOG_MAGIC);
-	tx = TIM_mono() - t0;
+	tx = VTIM_mono() - t0;
 	AZ(pthread_mutex_lock(&vl->mtx));
 	assert(lvl < NLEAD);
 	VSB_clear(vl->vsb);
@@ -162,7 +163,7 @@ vtc_dump(struct vtclog *vl, unsigned lvl, const char *pfx, const char *str, int
 	double tx;
 
 	CHECK_OBJ_NOTNULL(vl, VTCLOG_MAGIC);
-	tx = TIM_mono() - t0;
+	tx = VTIM_mono() - t0;
 	assert(lvl < NLEAD);
 	AZ(pthread_mutex_lock(&vl->mtx));
 	VSB_clear(vl->vsb);
@@ -225,7 +226,7 @@ vtc_hexdump(struct vtclog *vl, unsigned lvl, const char *pfx, const unsigned cha
 	double tx;
 
 	CHECK_OBJ_NOTNULL(vl, VTCLOG_MAGIC);
-	tx = TIM_mono() - t0;
+	tx = VTIM_mono() - t0;
 	assert(len >= 0);
 	assert(lvl < NLEAD);
 	AZ(pthread_mutex_lock(&vl->mtx));
diff --git a/bin/varnishtest/vtc_main.c b/bin/varnishtest/vtc_main.c
index d201946..040afdc 100644
--- a/bin/varnishtest/vtc_main.c
+++ b/bin/varnishtest/vtc_main.c
@@ -42,6 +42,7 @@
 
 #include "libvarnish.h"
 #include "vev.h"
+#include "vtim.h"
 
 #ifndef HAVE_SRANDOMDEV
 #include "compat/srandomdev.h"
@@ -193,7 +194,7 @@ tst_cb(const struct vev *ve, int what)
 		njob--;
 		px = wait4(jp->child, &stx, 0, NULL);
 		assert(px == jp->child);
-		t = TIM_mono() - jp->t0;
+		t = VTIM_mono() - jp->t0;
 		AZ(close(ve->fd));
 
 		if (stx && vtc_verbosity)
@@ -283,7 +284,7 @@ start_test(void)
 	AZ(pipe(p));
 	assert(p[0] > STDERR_FILENO);
 	assert(p[1] > STDERR_FILENO);
-	jp->t0 = TIM_mono();
+	jp->t0 = VTIM_mono();
 	jp->child = fork();
 	assert(jp->child >= 0);
 	if (jp->child == 0) {
diff --git a/include/Makefile.am b/include/Makefile.am
index 2cbe5d1..d62cab3 100644
--- a/include/Makefile.am
+++ b/include/Makefile.am
@@ -57,7 +57,8 @@ nobase_noinst_HEADERS = \
 	vsub.h \
 	vsha256.h \
 	vss.h \
-	vtcp.h
+	vtcp.h \
+	vtim.h
 
 tbl/vrt_stv_var.h tbl/vcl_returns.h vcl.h vrt_obj.h: $(top_srcdir)/lib/libvcl/generate.py $(top_srcdir)/include/vrt.h
 	mkdir -p tbl
diff --git a/include/libvarnish.h b/include/libvarnish.h
index 5bbe073..c0851c8 100644
--- a/include/libvarnish.h
+++ b/include/libvarnish.h
@@ -41,16 +41,6 @@ struct vsb;
 /* from libvarnish/num.c */
 const char *str2bytes(const char *p, uintmax_t *r, uintmax_t rel);
 
-/* from libvarnish/time.c */
-#define TIM_FORMAT_SIZE 30
-void TIM_format(double t, char *p);
-double TIM_parse(const char *p);
-double TIM_mono(void);
-double TIM_real(void);
-void TIM_sleep(double t);
-struct timespec TIM_timespec(double t);
-struct timeval TIM_timeval(double t);
-
 /* from libvarnish/version.c */
 void VCS_Message(const char *);
 
diff --git a/include/vtim.h b/include/vtim.h
new file mode 100644
index 0000000..8db4fbe
--- /dev/null
+++ b/include/vtim.h
@@ -0,0 +1,39 @@
+/*-
+ * Copyright (c) 2006 Verdens Gang AS
+ * Copyright (c) 2006-2011 Varnish Software AS
+ * All rights reserved.
+ *
+ * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ */
+
+/* from libvarnish/vtim.c */
+#define VTIM_FORMAT_SIZE 30
+void VTIM_format(double t, char *p);
+double VTIM_parse(const char *p);
+double VTIM_mono(void);
+double VTIM_real(void);
+void VTIM_sleep(double t);
+struct timespec VTIM_timespec(double t);
+struct timeval VTIM_timeval(double t);
diff --git a/lib/libvarnish/Makefile.am b/lib/libvarnish/Makefile.am
index 0148460..3c045e6 100644
--- a/lib/libvarnish/Makefile.am
+++ b/lib/libvarnish/Makefile.am
@@ -14,7 +14,7 @@ libvarnish_la_SOURCES = \
 	cli_serve.c \
 	flopen.c \
 	num.c \
-	time.c \
+	vtim.c \
 	vtcp.c \
 	vct.c \
 	version.c \
diff --git a/lib/libvarnish/time.c b/lib/libvarnish/time.c
deleted file mode 100644
index 31fc284..0000000
--- a/lib/libvarnish/time.c
+++ /dev/null
@@ -1,293 +0,0 @@
-/*-
- * Copyright (c) 2006 Verdens Gang AS
- * Copyright (c) 2006-2011 Varnish Software AS
- * All rights reserved.
- *
- * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- *
- * Semi-trivial functions to handle HTTP header timestamps according to
- * RFC 2616 section 3.3.
- *
- * In the highly unlikely event of performance trouble, handbuilt versions
- * would likely be faster than relying on the OS time functions.
- *
- * We must parse three different formats:
- *       000000000011111111112222222222
- *       012345678901234567890123456789
- *       ------------------------------
- *	"Sun, 06 Nov 1994 08:49:37 GMT"		RFC822 & RFC1123
- *	"Sunday, 06-Nov-94 08:49:37 GMT"	RFC850
- *	"Sun Nov  6 08:49:37 1994"		ANSI-C asctime()
- *
- * And always output the RFC1123 format.
- *
- */
-
-#include "config.h"
-
-#include <sys/time.h>
-
-#include <stdio.h>
-#include <string.h>
-#include <time.h>
-#include <unistd.h>
-#include <math.h>
-
-#include "libvarnish.h"
-
-/*
- * Note on Solaris: for some reason, clock_gettime(CLOCK_MONOTONIC, &ts) is not
- * implemented in assembly, but falls into a syscall, while gethrtime() doesn't,
- * so we save a syscall by using gethrtime() if it is defined.
- */
-
-double
-TIM_mono(void)
-{
-#ifdef HAVE_GETHRTIME
-	return (gethrtime() * 1e-9);
-#elif  HAVE_CLOCK_GETTIME
-	struct timespec ts;
-
-	assert(clock_gettime(CLOCK_MONOTONIC, &ts) == 0);
-	return (ts.tv_sec + 1e-9 * ts.tv_nsec);
-#else
-	struct timeval tv;
-
-	assert(gettimeofday(&tv, NULL) == 0);
-	return (tv.tv_sec + 1e-6 * tv.tv_usec);
-#endif
-}
-
-double
-TIM_real(void)
-{
-#ifdef HAVE_CLOCK_GETTIME
-	struct timespec ts;
-
-	assert(clock_gettime(CLOCK_REALTIME, &ts) == 0);
-	return (ts.tv_sec + 1e-9 * ts.tv_nsec);
-#else
-	struct timeval tv;
-
-	assert(gettimeofday(&tv, NULL) == 0);
-	return (tv.tv_sec + 1e-6 * tv.tv_usec);
-#endif
-}
-
-void
-TIM_format(double t, char *p)
-{
-	struct tm tm;
-	time_t tt;
-
-	tt = (time_t) t;
-	(void)gmtime_r(&tt, &tm);
-	AN(strftime(p, TIM_FORMAT_SIZE, "%a, %d %b %Y %T GMT", &tm));
-}
-
-/* XXX: add statistics ? */
-static const char *fmts[] = {
-	"%a, %d %b %Y %T GMT",	/* RFC 822 & RFC 1123 */
-	"%A, %d-%b-%y %T GMT",	/* RFC 850 */
-	"%a %b %d %T %Y",	/* ANSI-C asctime() */
-	"%F %T",		/* ISO 8601 */
-	NULL
-};
-
-double
-TIM_parse(const char *p)
-{
-	double t;
-	struct tm tm;
-	const char **r;
-
-	for (r = fmts; *r != NULL; r++) {
-		memset(&tm, 0, sizeof tm);
-		if (strptime(p, *r, &tm) != NULL) {
-			/*
-			 * Make sure this is initialized on the off-chance
-			 * that some raving loonie would apply DST to UTC.
-			 */
-			tm.tm_isdst = -1;
-#if defined(HAVE_TIMEGM)
-			t = timegm(&tm);
-#else
-			/*
-			 * Ahh, another POSIX_STUPIDITY, how unexpected.
-			 * Instead of, as would have been logical, to have
-			 * tm_timezone element, mktime() is standardized as
-			 * always working in localtime.  This brilliant idea
-			 * came from the same people who said "leap-seconds ?
-			 * Naah, screw it!".
-			 *
-			 * On broken systems without a working timegm(),
-			 * it is the responsibility of the calling program
-			 * to set the timezone to UTC.  We check that.
-			 */
-			t = mktime(&tm);
-			assert(!strcmp(tzname[0], "UTC"));
-#endif
-			return (t);
-		}
-	}
-	return (0);
-}
-
-void
-TIM_sleep(double t)
-{
-#ifdef HAVE_NANOSLEEP
-	struct timespec ts;
-
-	ts = TIM_timespec(t);
-
-	(void)nanosleep(&ts, NULL);
-#else
-	if (t >= 1.) {
-		(void)sleep(floor(t));
-		t -= floor(t);
-	}
-	/* XXX: usleep() is not mandated to be thread safe */
-	t *= 1e6;
-	if (t > 0)
-		(void)usleep(floor(t));
-#endif
-}
-
-struct timeval
-TIM_timeval(double t)
-{
-	struct timeval tv;
-
-	tv.tv_sec = (time_t)trunc(t);
-	tv.tv_usec = (int)(1e6 * (t - tv.tv_sec));
-	return (tv);
-}
-
-struct timespec
-TIM_timespec(double t)
-{
-	struct timespec tv;
-
-	tv.tv_sec = (time_t)trunc(t);
-	tv.tv_nsec = (int)(1e9 * (t - tv.tv_sec));
-	return (tv);
-}
-
-
-#ifdef TEST_DRIVER
-
-#include <stdlib.h>
-
-/*
- * Compile with:
- *  cc -o foo -DTEST_DRIVER -I../.. -I../../include time.c assert.c
- * (Solaris)
- *  cc -o foo -DTEST_DRIVER -I../.. -I../../include -lm time.c assert.c
- * Test with:
- *  env TZ=UTC ./foo
- *  env TZ=CET ./foo
- */
-
-static void
-tst(const char *s, time_t good)
-{
-	time_t t;
-	char buf[BUFSIZ];
-
-	t = TIM_parse(s);
-	TIM_format(t, buf);
-	printf("%-30s -> %12jd -> %s\n", s, (intmax_t)t, buf);
-	if (t != good) {
-		printf("Parse error! Got: %jd should have %jd diff %jd\n",
-		    (intmax_t)t, (intmax_t)good, (intmax_t)(t - good));
-		exit (2);
-	}
-}
-
-static int
-tst_delta_check(const char *name, double begin, double end, double ref)
-{
-	const double tol_max = 1.1;
-	const double tol_min = 1;
-
-	printf("%s delta for %fs sleep: %f\n", name, ref, (end - begin));
-
-	if ((end - begin) > tol_max * ref) {
-		printf("%s delta above tolerance: ((%f - %f) = %f) > %f\n",
-		    name, end, begin, (end - begin), tol_max);
-		return (1);
-	} else if ((end - begin) < tol_min * ref) {
-		printf("%s delta below tolerance: ((%f - %f) = %f) < %f\n",
-		    name, end, begin, (end - begin), tol_min);
-		return (1);
-	}
-	return (0);
-}
-
-static void
-tst_delta()
-{
-	double m_begin, m_end;
-	double r_begin, r_end;
-	const double ref = 1;
-	int err = 0;
-
-	r_begin = TIM_real();
-	m_begin = TIM_mono();
-	TIM_sleep(ref);
-	r_end = TIM_real();
-	m_end = TIM_mono();
-
-	err += tst_delta_check("TIM_mono", m_begin, m_end, ref);
-	err += tst_delta_check("TIM_real", r_begin, r_end, ref);
-
-	if (err) {
-		printf("%d time delta test errrors\n", err);
-		exit (2);
-	}
-}
-
-int
-main(int argc, char **argv)
-{
-	time_t t;
-	char buf[BUFSIZ];
-
-	time(&t);
-	memset(buf, 0x55, sizeof buf);
-	TIM_format(t, buf);
-	printf("scan = %d <%s>\n", TIM_parse(buf), buf);
-
-	/* Examples from RFC2616 section 3.3.1 */
-	tst("Sun, 06 Nov 1994 08:49:37 GMT", 784111777);
-	tst("Sunday, 06-Nov-94 08:49:37 GMT", 784111777);
-	tst("Sun Nov  6 08:49:37 1994", 784111777);
-
-	tst_delta();
-
-	return (0);
-}
-#endif
diff --git a/lib/libvarnish/vev.c b/lib/libvarnish/vev.c
index 8c64562..c55ebfc 100644
--- a/lib/libvarnish/vev.c
+++ b/lib/libvarnish/vev.c
@@ -39,6 +39,7 @@
 #include <pthread.h>
 
 #include "libvarnish.h"
+#include "vtim.h"
 #include "vev.h"
 #include "miniobj.h"
 #include "binary_heap.h"
@@ -278,7 +279,7 @@ vev_add(struct vev_base *evb, struct vev *e)
 	e->magic = VEV_MAGIC;	/* before binheap_insert() */
 
 	if (e->timeout != 0.0) {
-		e->__when += TIM_mono() + e->timeout;
+		e->__when += VTIM_mono() + e->timeout;
 		binheap_insert(evb->binheap, e);
 		assert(e->__binheap_idx > 0);
 		DBG(evb, "... bidx = %d\n", e->__binheap_idx);
@@ -455,7 +456,7 @@ vev_schedule_one(struct vev_base *evb)
 	if (e != NULL) {
 		CHECK_OBJ_NOTNULL(e, VEV_MAGIC);
 		assert(e->__binheap_idx == 1);
-		t = TIM_mono();
+		t = VTIM_mono();
 		if (e->__when <= t)
 			return (vev_sched_timeout(evb, e, t));
 		tmo = (int)((e->__when - t) * 1e3);
@@ -478,7 +479,7 @@ vev_schedule_one(struct vev_base *evb)
 		return (vev_sched_signal(evb));
 	if (i == 0) {
 		assert(e != NULL);
-		t = TIM_mono();
+		t = VTIM_mono();
 		if (e->__when <= t)
 			return (vev_sched_timeout(evb, e, t));
 	}
diff --git a/lib/libvarnish/vtim.c b/lib/libvarnish/vtim.c
new file mode 100644
index 0000000..edfb8a5
--- /dev/null
+++ b/lib/libvarnish/vtim.c
@@ -0,0 +1,295 @@
+/*-
+ * Copyright (c) 2006 Verdens Gang AS
+ * Copyright (c) 2006-2011 Varnish Software AS
+ * All rights reserved.
+ *
+ * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * Semi-trivial functions to handle HTTP header timestamps according to
+ * RFC 2616 section 3.3.
+ *
+ * In the highly unlikely event of performance trouble, handbuilt versions
+ * would likely be faster than relying on the OS time functions.
+ *
+ * We must parse three different formats:
+ *       000000000011111111112222222222
+ *       012345678901234567890123456789
+ *       ------------------------------
+ *	"Sun, 06 Nov 1994 08:49:37 GMT"		RFC822 & RFC1123
+ *	"Sunday, 06-Nov-94 08:49:37 GMT"	RFC850
+ *	"Sun Nov  6 08:49:37 1994"		ANSI-C asctime()
+ *
+ * And always output the RFC1123 format.
+ *
+ */
+
+#include "config.h"
+
+#include <sys/time.h>
+
+#include <stdio.h>
+#include <string.h>
+#include <time.h>
+#include <unistd.h>
+#include <math.h>
+
+#include "vas.h"
+#include "vtim.h"
+//#include "libvarnish.h"
+
+/*
+ * Note on Solaris: for some reason, clock_gettime(CLOCK_MONOTONIC, &ts) is not
+ * implemented in assembly, but falls into a syscall, while gethrtime() doesn't,
+ * so we save a syscall by using gethrtime() if it is defined.
+ */
+
+double
+VTIM_mono(void)
+{
+#ifdef HAVE_GETHRTIME
+	return (gethrtime() * 1e-9);
+#elif  HAVE_CLOCK_GETTIME
+	struct timespec ts;
+
+	assert(clock_gettime(CLOCK_MONOTONIC, &ts) == 0);
+	return (ts.tv_sec + 1e-9 * ts.tv_nsec);
+#else
+	struct timeval tv;
+
+	assert(gettimeofday(&tv, NULL) == 0);
+	return (tv.tv_sec + 1e-6 * tv.tv_usec);
+#endif
+}
+
+double
+VTIM_real(void)
+{
+#ifdef HAVE_CLOCK_GETTIME
+	struct timespec ts;
+
+	assert(clock_gettime(CLOCK_REALTIME, &ts) == 0);
+	return (ts.tv_sec + 1e-9 * ts.tv_nsec);
+#else
+	struct timeval tv;
+
+	assert(gettimeofday(&tv, NULL) == 0);
+	return (tv.tv_sec + 1e-6 * tv.tv_usec);
+#endif
+}
+
+void
+VTIM_format(double t, char *p)
+{
+	struct tm tm;
+	time_t tt;
+
+	tt = (time_t) t;
+	(void)gmtime_r(&tt, &tm);
+	AN(strftime(p, VTIM_FORMAT_SIZE, "%a, %d %b %Y %T GMT", &tm));
+}
+
+/* XXX: add statistics ? */
+static const char *fmts[] = {
+	"%a, %d %b %Y %T GMT",	/* RFC 822 & RFC 1123 */
+	"%A, %d-%b-%y %T GMT",	/* RFC 850 */
+	"%a %b %d %T %Y",	/* ANSI-C asctime() */
+	"%F %T",		/* ISO 8601 */
+	NULL
+};
+
+double
+VTIM_parse(const char *p)
+{
+	double t;
+	struct tm tm;
+	const char **r;
+
+	for (r = fmts; *r != NULL; r++) {
+		memset(&tm, 0, sizeof tm);
+		if (strptime(p, *r, &tm) != NULL) {
+			/*
+			 * Make sure this is initialized on the off-chance
+			 * that some raving loonie would apply DST to UTC.
+			 */
+			tm.tm_isdst = -1;
+#if defined(HAVE_TIMEGM)
+			t = timegm(&tm);
+#else
+			/*
+			 * Ahh, another POSIX_STUPIDITY, how unexpected.
+			 * Instead of, as would have been logical, to have
+			 * tm_timezone element, mktime() is standardized as
+			 * always working in localtime.  This brilliant idea
+			 * came from the same people who said "leap-seconds ?
+			 * Naah, screw it!".
+			 *
+			 * On broken systems without a working timegm(),
+			 * it is the responsibility of the calling program
+			 * to set the timezone to UTC.  We check that.
+			 */
+			t = mktime(&tm);
+			assert(!strcmp(tzname[0], "UTC"));
+#endif
+			return (t);
+		}
+	}
+	return (0);
+}
+
+void
+VTIM_sleep(double t)
+{
+#ifdef HAVE_NANOSLEEP
+	struct timespec ts;
+
+	ts = VTIM_timespec(t);
+
+	(void)nanosleep(&ts, NULL);
+#else
+	if (t >= 1.) {
+		(void)sleep(floor(t));
+		t -= floor(t);
+	}
+	/* XXX: usleep() is not mandated to be thread safe */
+	t *= 1e6;
+	if (t > 0)
+		(void)usleep(floor(t));
+#endif
+}
+
+struct timeval
+VTIM_timeval(double t)
+{
+	struct timeval tv;
+
+	tv.tv_sec = (time_t)trunc(t);
+	tv.tv_usec = (int)(1e6 * (t - tv.tv_sec));
+	return (tv);
+}
+
+struct timespec
+VTIM_timespec(double t)
+{
+	struct timespec tv;
+
+	tv.tv_sec = (time_t)trunc(t);
+	tv.tv_nsec = (int)(1e9 * (t - tv.tv_sec));
+	return (tv);
+}
+
+
+#ifdef TEST_DRIVER
+
+#include <stdlib.h>
+
+/*
+ * Compile with:
+ *  cc -o foo -DTEST_DRIVER -I../.. -I../../include time.c assert.c
+ * (Solaris)
+ *  cc -o foo -DTEST_DRIVER -I../.. -I../../include -lm time.c assert.c
+ * Test with:
+ *  env TZ=UTC ./foo
+ *  env TZ=CET ./foo
+ */
+
+static void
+tst(const char *s, time_t good)
+{
+	time_t t;
+	char buf[BUFSIZ];
+
+	t = VTIM_parse(s);
+	VTIM_format(t, buf);
+	printf("%-30s -> %12jd -> %s\n", s, (intmax_t)t, buf);
+	if (t != good) {
+		printf("Parse error! Got: %jd should have %jd diff %jd\n",
+		    (intmax_t)t, (intmax_t)good, (intmax_t)(t - good));
+		exit (2);
+	}
+}
+
+static int
+tst_delta_check(const char *name, double begin, double end, double ref)
+{
+	const double tol_max = 1.1;
+	const double tol_min = 1;
+
+	printf("%s delta for %fs sleep: %f\n", name, ref, (end - begin));
+
+	if ((end - begin) > tol_max * ref) {
+		printf("%s delta above tolerance: ((%f - %f) = %f) > %f\n",
+		    name, end, begin, (end - begin), tol_max);
+		return (1);
+	} else if ((end - begin) < tol_min * ref) {
+		printf("%s delta below tolerance: ((%f - %f) = %f) < %f\n",
+		    name, end, begin, (end - begin), tol_min);
+		return (1);
+	}
+	return (0);
+}
+
+static void
+tst_delta()
+{
+	double m_begin, m_end;
+	double r_begin, r_end;
+	const double ref = 1;
+	int err = 0;
+
+	r_begin = VTIM_real();
+	m_begin = VTIM_mono();
+	VTIM_sleep(ref);
+	r_end = VTIM_real();
+	m_end = VTIM_mono();
+
+	err += tst_delta_check("VTIM_mono", m_begin, m_end, ref);
+	err += tst_delta_check("VTIM_real", r_begin, r_end, ref);
+
+	if (err) {
+		printf("%d time delta test errrors\n", err);
+		exit (2);
+	}
+}
+
+int
+main(int argc, char **argv)
+{
+	time_t t;
+	char buf[BUFSIZ];
+
+	time(&t);
+	memset(buf, 0x55, sizeof buf);
+	VTIM_format(t, buf);
+	printf("scan = %d <%s>\n", VTIM_parse(buf), buf);
+
+	/* Examples from RFC2616 section 3.3.1 */
+	tst("Sun, 06 Nov 1994 08:49:37 GMT", 784111777);
+	tst("Sunday, 06-Nov-94 08:49:37 GMT", 784111777);
+	tst("Sun Nov  6 08:49:37 1994", 784111777);
+
+	tst_delta();
+
+	return (0);
+}
+#endif



From geoff at varnish-cache.org  Mon Jan  9 20:52:16 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:16 +0100
Subject: [experimental-ims] 06b3c20 And rename the various file-accesors to
	VFIL_
Message-ID: <E1RkMCW-0007c2-F4@project.varnish-software.com>

commit 06b3c206e526c919722ff797aa9e7d0add3fd0b0
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sun Oct 9 19:29:19 2011 +0000

    And rename the various file-accesors to VFIL_
    
    Remove srandomdev() look-alike, we already have a compat version

diff --git a/bin/varnishd/mgt_vcc.c b/bin/varnishd/mgt_vcc.c
index bbf813a..df46c41 100644
--- a/bin/varnishd/mgt_vcc.c
+++ b/bin/varnishd/mgt_vcc.c
@@ -44,6 +44,7 @@
 #include "vcli.h"
 #include "vsub.h"
 #include "vcl.h"
+#include "vfil.h"
 #include "cli_priv.h"
 #include "mgt_cli.h"
 
@@ -227,7 +228,7 @@ mgt_run_cc(const char *vcl, struct vsb *sb, int C_flag)
 	struct vcc_priv vp;
 
 	/* Create temporary C source file */
-	sfd = vtmpfile(sf);
+	sfd = VFIL_tmpfile(sf);
 	if (sfd < 0) {
 		VSB_printf(sb, "Failed to create %s: %s", sf, strerror(errno));
 		return (NULL);
@@ -245,7 +246,7 @@ mgt_run_cc(const char *vcl, struct vsb *sb, int C_flag)
 	}
 
 	if (C_flag) {
-		csrc = vreadfile(NULL, sf, NULL);
+		csrc = VFIL_readfile(NULL, sf, NULL);
 		XXXAN(csrc);
 		(void)fputs(csrc, stdout);
 		free(csrc);
@@ -517,7 +518,7 @@ mcf_config_load(struct cli *cli, const char * const *av, void *priv)
 		return;
 	}
 
-	vcl = vreadfile(mgt_vcl_dir, av[3], NULL);
+	vcl = VFIL_readfile(mgt_vcl_dir, av[3], NULL);
 	if (vcl == NULL) {
 		VCLI_Out(cli, "Cannot open '%s'", av[3]);
 		VCLI_SetResult(cli, CLIS_PARAM);
diff --git a/bin/varnishd/varnishd.c b/bin/varnishd/varnishd.c
index 360cdad..bf61ea3 100644
--- a/bin/varnishd/varnishd.c
+++ b/bin/varnishd/varnishd.c
@@ -55,12 +55,17 @@
 
 #include "vav.h"
 #include "vin.h"
+#include "vfil.h"
 #include "vtim.h"
 #include "heritage.h"
 #include "mgt.h"
 #include "hash_slinger.h"
 #include "stevedore.h"
 
+#ifndef HAVE_SRANDOMDEV
+#include "compat/srandomdev.h"
+#endif
+
 struct heritage		heritage;
 volatile struct params	*params;
 unsigned		d_flag = 0;
@@ -355,7 +360,7 @@ main(int argc, char * const *argv)
 	for (o = getdtablesize(); o > STDERR_FILENO; o--)
 		(void)close(o);
 
-	AZ(seed_random());
+	srandomdev();
 
 	mgt_got_fd(STDERR_FILENO);
 
@@ -543,7 +548,7 @@ main(int argc, char * const *argv)
 	}
 
 	if (f_arg != NULL) {
-		vcl = vreadfile(NULL, f_arg, NULL);
+		vcl = VFIL_readfile(NULL, f_arg, NULL);
 		if (vcl == NULL) {
 			fprintf(stderr, "Cannot read '%s': %s\n",
 			    f_arg, strerror(errno));
diff --git a/include/Makefile.am b/include/Makefile.am
index e852c58..70be7f7 100644
--- a/include/Makefile.am
+++ b/include/Makefile.am
@@ -44,6 +44,7 @@ nobase_noinst_HEADERS = \
 	vct.h \
 	vend.h \
 	vev.h \
+	vfil.h \
 	vin.h \
 	vlu.h \
 	vmb.h \
diff --git a/include/libvarnish.h b/include/libvarnish.h
index 1450578..65287b5 100644
--- a/include/libvarnish.h
+++ b/include/libvarnish.h
@@ -41,12 +41,6 @@ struct vsb;
 /* from libvarnish/version.c */
 void VCS_Message(const char *);
 
-/* from libvarnish/vtmpfile.c */
-int seed_random(void);
-int vtmpfile(char *);
-char *vreadfile(const char *pfx, const char *fn, ssize_t *sz);
-char *vreadfd(int fd, ssize_t *sz);
-
 /* Safe printf into a fixed-size buffer */
 #define bprintf(buf, fmt, ...)						\
 	do {								\
diff --git a/include/vfil.h b/include/vfil.h
new file mode 100644
index 0000000..74885ef
--- /dev/null
+++ b/include/vfil.h
@@ -0,0 +1,35 @@
+/*-
+ * Copyright (c) 2006 Verdens Gang AS
+ * Copyright (c) 2006-2011 Varnish Software AS
+ * All rights reserved.
+ *
+ * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ */
+
+/* from libvarnish/vfil.c */
+int seed_random(void);
+int VFIL_tmpfile(char *);
+char *VFIL_readfile(const char *pfx, const char *fn, ssize_t *sz);
+char *VFIL_readfd(int fd, ssize_t *sz);
diff --git a/lib/libvarnish/Makefile.am b/lib/libvarnish/Makefile.am
index e890904..ab2cc94 100644
--- a/lib/libvarnish/Makefile.am
+++ b/lib/libvarnish/Makefile.am
@@ -19,6 +19,7 @@ libvarnish_la_SOURCES = \
 	vct.c \
 	version.c \
 	vev.c \
+	vfil.c \
 	vin.c \
 	vlu.c \
 	vmb.c \
@@ -26,8 +27,7 @@ libvarnish_la_SOURCES = \
 	vre.c \
 	vsb.c \
 	vsha256.c \
-	vss.c \
-	vtmpfile.c
+	vss.c
 
 libvarnish_la_CFLAGS = -DVARNISH_STATE_DIR='"${VARNISH_STATE_DIR}"'
 libvarnish_la_LIBADD = ${RT_LIBS} ${NET_LIBS} ${LIBM} @PCRE_LIBS@
diff --git a/lib/libvarnish/vfil.c b/lib/libvarnish/vfil.c
new file mode 100644
index 0000000..884b106
--- /dev/null
+++ b/lib/libvarnish/vfil.c
@@ -0,0 +1,124 @@
+/*-
+ * Copyright (c) 2006 Verdens Gang AS
+ * Copyright (c) 2006-2011 Varnish Software AS
+ * All rights reserved.
+ *
+ * Author: Dag-Erling Sm?rgrav <des at des.no>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include "config.h"
+
+#include <errno.h>
+#include <fcntl.h>
+#include <limits.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <time.h>
+
+#include <sys/stat.h>
+
+#include "vas.h"
+#include "vfil.h"
+#include "libvarnish.h"
+
+int
+VFIL_tmpfile(char *template)
+{
+	char *b, *e, *p;
+	int fd;
+	char ran;
+
+	for (b = template; *b != '#'; ++b)
+		/* nothing */ ;
+	if (*b == '\0') {
+		errno = EINVAL;
+		return (-1);
+	}
+	for (e = b; *e == '#'; ++e)
+		/* nothing */ ;
+
+	for (;;) {
+		for (p = b; p < e; ++p) {
+			ran = random() % 63;
+			if (ran < 10)
+				*p = '0' + ran;
+			else if (ran < 36)
+				*p = 'A' + ran - 10;
+			else if (ran < 62)
+				*p = 'a' + ran - 36;
+			else
+				*p = '_';
+		}
+		fd = open(template, O_RDWR|O_CREAT|O_EXCL, 0600);
+		if (fd >= 0)
+			return (fd);
+		if (errno != EEXIST)
+			return (-1);
+	}
+	/* not reached */
+}
+
+char *
+VFIL_readfd(int fd, ssize_t *sz)
+{
+	struct stat st;
+	char *f;
+	int i;
+
+	assert(0 == fstat(fd, &st));
+	if (!S_ISREG(st.st_mode))
+		return (NULL);
+	f = malloc(st.st_size + 1);
+	assert(f != NULL);
+	i = read(fd, f, st.st_size);
+	assert(i == st.st_size);
+	f[i] = '\0';
+	if (sz != NULL)
+		*sz = st.st_size;
+	return (f);
+}
+
+char *
+VFIL_readfile(const char *pfx, const char *fn, ssize_t *sz)
+{
+	int fd, err;
+	char *r;
+	char fnb[PATH_MAX + 1];
+
+	if (fn[0] == '/')
+		fd = open(fn, O_RDONLY);
+	else if (pfx != NULL) {
+		bprintf(fnb, "/%s/%s", pfx, fn); /* XXX: graceful length check */
+		fd = open(fnb, O_RDONLY);
+	} else
+		fd = open(fn, O_RDONLY);
+	if (fd < 0)
+		return (NULL);
+	r = VFIL_readfd(fd, sz);
+	err = errno;
+	AZ(close(fd));
+	errno = err;
+	return (r);
+}
diff --git a/lib/libvarnish/vtmpfile.c b/lib/libvarnish/vtmpfile.c
deleted file mode 100644
index 327790a..0000000
--- a/lib/libvarnish/vtmpfile.c
+++ /dev/null
@@ -1,142 +0,0 @@
-/*-
- * Copyright (c) 2006 Verdens Gang AS
- * Copyright (c) 2006-2011 Varnish Software AS
- * All rights reserved.
- *
- * Author: Dag-Erling Sm?rgrav <des at des.no>
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- */
-
-#include "config.h"
-
-#include <errno.h>
-#include <fcntl.h>
-#include <limits.h>
-#include <stdio.h>
-#include <stdlib.h>
-#include <unistd.h>
-#include <time.h>
-
-#include <sys/stat.h>
-
-#include "libvarnish.h"
-
-int
-seed_random(void)
-{
-	int fd;
-	unsigned seed;
-
-	fd = open("/dev/urandom", O_RDONLY);
-	if (fd == -1) {
-		/* urandom not available, fall back to something
-		 * weaker */
-		srandom(time(NULL));
-		return (0);
-	}
-	if (read(fd, &seed, sizeof seed) != sizeof seed)
-		return (1);
-	(void)close(fd);
-	srandom(seed);
-	return (0);
-}
-
-int
-vtmpfile(char *template)
-{
-	char *b, *e, *p;
-	int fd;
-	char ran;
-
-	for (b = template; *b != '#'; ++b)
-		/* nothing */ ;
-	if (*b == '\0') {
-		errno = EINVAL;
-		return (-1);
-	}
-	for (e = b; *e == '#'; ++e)
-		/* nothing */ ;
-
-	for (;;) {
-		for (p = b; p < e; ++p) {
-			ran = random() % 63;
-			if (ran < 10)
-				*p = '0' + ran;
-			else if (ran < 36)
-				*p = 'A' + ran - 10;
-			else if (ran < 62)
-				*p = 'a' + ran - 36;
-			else
-				*p = '_';
-		}
-		fd = open(template, O_RDWR|O_CREAT|O_EXCL, 0600);
-		if (fd >= 0)
-			return (fd);
-		if (errno != EEXIST)
-			return (-1);
-	}
-	/* not reached */
-}
-
-char *
-vreadfd(int fd, ssize_t *sz)
-{
-	struct stat st;
-	char *f;
-	int i;
-
-	assert(0 == fstat(fd, &st));
-	if (!S_ISREG(st.st_mode))
-		return (NULL);
-	f = malloc(st.st_size + 1);
-	assert(f != NULL);
-	i = read(fd, f, st.st_size);
-	assert(i == st.st_size);
-	f[i] = '\0';
-	if (sz != NULL)
-		*sz = st.st_size;
-	return (f);
-}
-
-char *
-vreadfile(const char *pfx, const char *fn, ssize_t *sz)
-{
-	int fd, err;
-	char *r;
-	char fnb[PATH_MAX + 1];
-
-	if (fn[0] == '/')
-		fd = open(fn, O_RDONLY);
-	else if (pfx != NULL) {
-		bprintf(fnb, "/%s/%s", pfx, fn); /* XXX: graceful length check */
-		fd = open(fnb, O_RDONLY);
-	} else
-		fd = open(fn, O_RDONLY);
-	if (fd < 0)
-		return (NULL);
-	r = vreadfd(fd, sz);
-	err = errno;
-	AZ(close(fd));
-	errno = err;
-	return (r);
-}
diff --git a/lib/libvcl/vcc_compile.c b/lib/libvcl/vcc_compile.c
index 3af30c6..dbae2ab 100644
--- a/lib/libvcl/vcc_compile.c
+++ b/lib/libvcl/vcc_compile.c
@@ -61,6 +61,7 @@
 
 #include "vcc_compile.h"
 #include "libvcl.h"
+#include "vfil.h"
 
 struct method method_tab[] = {
 #define VCL_MET_MAC(l,U,m)	{ "vcl_"#l, m, VCL_MET_##U },
@@ -409,7 +410,7 @@ vcc_file_source(const struct vcc *tl, struct vsb *sb, const char *fn)
 	char *f;
 	struct source *sp;
 
-	f = vreadfile(tl->vcl_dir, fn, NULL);
+	f = VFIL_readfile(tl->vcl_dir, fn, NULL);
 	if (f == NULL) {
 		VSB_printf(sb, "Cannot read file '%s': %s\n",
 		    fn, strerror(errno));
diff --git a/lib/libvmod_std/vmod_std_fileread.c b/lib/libvmod_std/vmod_std_fileread.c
index e1be09a..cae57a2 100644
--- a/lib/libvmod_std/vmod_std_fileread.c
+++ b/lib/libvmod_std/vmod_std_fileread.c
@@ -42,6 +42,7 @@
 #include "../../bin/varnishd/cache.h"
 
 #include "vcc_if.h"
+#include "vfil.h"
 
 struct frfile {
 	unsigned			magic;
@@ -102,7 +103,7 @@ vmod_fileread(struct sess *sp, struct vmod_priv *priv, const char *file_name)
 		return (frf->contents);
 	}
 
-	s = vreadfile(NULL, file_name, NULL);
+	s = VFIL_readfile(NULL, file_name, NULL);
 	if (s != NULL) {
 		ALLOC_OBJ(frf, CACHED_FILE_MAGIC);
 		AN(frf);



From geoff at varnish-cache.org  Mon Jan  9 20:52:16 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:16 +0100
Subject: [experimental-ims] a889566 Get rid of the last trace if the
	libvarnish.h kitchen-sink: bprint macros go in vdef.h, VCS_* in vcs.h
Message-ID: <E1RkMCW-0007cJ-J8@project.varnish-software.com>

commit a88956622c34e38826d5f816dad0e0e936813c0c
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sun Oct 9 20:07:25 2011 +0000

    Get rid of the last trace if the libvarnish.h kitchen-sink:
    bprint macros go in vdef.h,  VCS_* in vcs.h

diff --git a/bin/varnishd/common.h b/bin/varnishd/common.h
index 0e6d95d..7fd15b7 100644
--- a/bin/varnishd/common.h
+++ b/bin/varnishd/common.h
@@ -28,11 +28,12 @@
  *
  */
 
-#include "libvarnish.h"
+#include "miniobj.h"
 #include "vas.h"
-#include "vsb.h"
+#include "vcs.h"
+#include "vdef.h"
 #include "vqueue.h"
-#include "miniobj.h"
+#include "vsb.h"
 
 struct cli;
 
diff --git a/bin/varnishhist/varnishhist.c b/bin/varnishhist/varnishhist.c
index 92d17b3..e2fc652 100644
--- a/bin/varnishhist/varnishhist.c
+++ b/bin/varnishhist/varnishhist.c
@@ -49,7 +49,7 @@
 #include <unistd.h>
 
 #include "vas.h"
-#include "libvarnish.h"
+#include "vcs.h"
 
 #define HIST_N 2000 /* how far back we remember */
 #define HIST_LOW -6 /* low end of log range */
diff --git a/bin/varnishlog/varnishlog.c b/bin/varnishlog/varnishlog.c
index 3142d1b..86c8504 100644
--- a/bin/varnishlog/varnishlog.c
+++ b/bin/varnishlog/varnishlog.c
@@ -44,11 +44,10 @@
 
 #include "compat/daemon.h"
 
-#include "vsb.h"
-#include "vpf.h"
-
 #include "vas.h"
-#include "libvarnish.h"
+#include "vcs.h"
+#include "vpf.h"
+#include "vsb.h"
 
 static int	b_flag, c_flag;
 
diff --git a/bin/varnishncsa/varnishncsa.c b/bin/varnishncsa/varnishncsa.c
index 4805d81..05b8dfe 100644
--- a/bin/varnishncsa/varnishncsa.c
+++ b/bin/varnishncsa/varnishncsa.c
@@ -74,12 +74,12 @@
 #include "compat/daemon.h"
 
 #include "vas.h"
-#include "vsb.h"
+#include "vcs.h"
 #include "vpf.h"
 #include "vqueue.h"
-
-#include "libvarnish.h"
 #include "vre.h"
+#include "vsb.h"
+
 #include "base64.h"
 
 static volatile sig_atomic_t reopen;
diff --git a/bin/varnishreplay/varnishreplay.c b/bin/varnishreplay/varnishreplay.c
index ef4118a..ffc403f 100644
--- a/bin/varnishreplay/varnishreplay.c
+++ b/bin/varnishreplay/varnishreplay.c
@@ -46,10 +46,9 @@
 #include <string.h>
 #include <unistd.h>
 
-#include "vqueue.h"
-
 #include "vas.h"
-#include "libvarnish.h"
+#include "vcs.h"
+#include "vqueue.h"
 #include "vss.h"
 
 #define freez(x) do { if (x) free(x); x = NULL; } while (0);
diff --git a/bin/varnishsizes/varnishsizes.c b/bin/varnishsizes/varnishsizes.c
index 70fcad1..d3e3218 100644
--- a/bin/varnishsizes/varnishsizes.c
+++ b/bin/varnishsizes/varnishsizes.c
@@ -49,7 +49,7 @@
 #include <unistd.h>
 
 #include "vas.h"
-#include "libvarnish.h"
+#include "vcs.h"
 
 #define HIST_N 2000 /* how far back we remember */
 #define HIST_LOW 1 /* low end of log range */
diff --git a/bin/varnishstat/varnishstat.h b/bin/varnishstat/varnishstat.h
index 322ebc4..314c7c5 100644
--- a/bin/varnishstat/varnishstat.h
+++ b/bin/varnishstat/varnishstat.h
@@ -28,6 +28,6 @@
  */
 
 #include "vas.h"
-#include "libvarnish.h"
+#include "vcs.h"
 
 void do_curses(struct VSM_data *vd, const struct VSC_C_main *VSC_C_main, int delay);
diff --git a/bin/varnishtest/vtc.c b/bin/varnishtest/vtc.c
index 83bbb22..4404de9 100644
--- a/bin/varnishtest/vtc.c
+++ b/bin/varnishtest/vtc.c
@@ -41,7 +41,6 @@
 
 #include "vav.h"
 #include "vtim.h"
-#include "libvarnish.h"
 
 
 #ifndef HAVE_SRANDOMDEV
diff --git a/bin/varnishtest/vtc.h b/bin/varnishtest/vtc.h
index 3621dc5..1340314 100644
--- a/bin/varnishtest/vtc.h
+++ b/bin/varnishtest/vtc.h
@@ -36,6 +36,7 @@
 
 #include "miniobj.h"
 #include "vas.h"
+#include "vdef.h"
 #include "vqueue.h"
 #include "vsb.h"
 
diff --git a/bin/varnishtest/vtc_client.c b/bin/varnishtest/vtc_client.c
index 55e25fa..2547d80 100644
--- a/bin/varnishtest/vtc_client.c
+++ b/bin/varnishtest/vtc_client.c
@@ -39,7 +39,6 @@
 
 #include "vss.h"
 #include "vtcp.h"
-#include "libvarnish.h"
 
 struct client {
 	unsigned		magic;
diff --git a/bin/varnishtest/vtc_http.c b/bin/varnishtest/vtc_http.c
index 50816bc..889891b 100644
--- a/bin/varnishtest/vtc_http.c
+++ b/bin/varnishtest/vtc_http.c
@@ -37,14 +37,10 @@
 #include <sys/socket.h>
 
 #include "vct.h"
-
-#include "libvarnish.h"
-
+#include "vgz.h"
 #include "vtc.h"
 #include "vtcp.h"
 
-#include "vgz.h"
-
 #define MAX_HDR		50
 
 struct http {
diff --git a/bin/varnishtest/vtc_log.c b/bin/varnishtest/vtc_log.c
index c06f319..753de58 100644
--- a/bin/varnishtest/vtc_log.c
+++ b/bin/varnishtest/vtc_log.c
@@ -34,8 +34,6 @@
 #include <stdarg.h>
 
 #include "vtc.h"
-
-#include "libvarnish.h"
 #include "vtim.h"
 
 static pthread_mutex_t	vtclog_mtx;
diff --git a/bin/varnishtest/vtc_main.c b/bin/varnishtest/vtc_main.c
index 040afdc..3b9d81b 100644
--- a/bin/varnishtest/vtc_main.c
+++ b/bin/varnishtest/vtc_main.c
@@ -38,10 +38,8 @@
 #include <sys/types.h>
 #include <sys/wait.h>
 
-#include "vtc.h"
-
-#include "libvarnish.h"
 #include "vev.h"
+#include "vtc.h"
 #include "vtim.h"
 
 #ifndef HAVE_SRANDOMDEV
diff --git a/bin/varnishtest/vtc_sema.c b/bin/varnishtest/vtc_sema.c
index 8a9228a..c363b4a 100644
--- a/bin/varnishtest/vtc_sema.c
+++ b/bin/varnishtest/vtc_sema.c
@@ -35,8 +35,6 @@
 
 #include "vtc.h"
 
-#include "libvarnish.h"
-
 struct sema {
 	unsigned		magic;
 #define SEMA_MAGIC		0x29b64317
diff --git a/bin/varnishtest/vtc_server.c b/bin/varnishtest/vtc_server.c
index afe62af..5a768a1 100644
--- a/bin/varnishtest/vtc_server.c
+++ b/bin/varnishtest/vtc_server.c
@@ -36,12 +36,10 @@
 #include <sys/types.h>
 #include <sys/socket.h>
 
+#include "vss.h"
 #include "vtc.h"
 #include "vtcp.h"
 
-#include "libvarnish.h"
-#include "vss.h"
-
 struct server {
 	unsigned		magic;
 #define SERVER_MAGIC		0x55286619
diff --git a/bin/varnishtest/vtc_varnish.c b/bin/varnishtest/vtc_varnish.c
index 8b310f6..ba19464 100644
--- a/bin/varnishtest/vtc_varnish.c
+++ b/bin/varnishtest/vtc_varnish.c
@@ -45,13 +45,10 @@
 #include <sys/wait.h>
 #include <sys/socket.h>
 
-#include "vtc.h"
-#include "vtcp.h"
-
-#include "libvarnish.h"
-
 #include "vcli.h"
 #include "vss.h"
+#include "vtc.h"
+#include "vtcp.h"
 
 struct varnish {
 	unsigned		magic;
diff --git a/bin/varnishtop/varnishtop.c b/bin/varnishtop/varnishtop.c
index 1022e32..34278ea 100644
--- a/bin/varnishtop/varnishtop.c
+++ b/bin/varnishtop/varnishtop.c
@@ -46,13 +46,11 @@
 #include <unistd.h>
 #include <limits.h>
 
+#include "vas.h"
+#include "vcs.h"
 #include "vqueue.h"
-
 #include "vsb.h"
 
-#include "vas.h"
-#include "libvarnish.h"
-
 #if 0
 #define AC(x) assert((x) != ERR)
 #else
diff --git a/include/Makefile.am b/include/Makefile.am
index 70be7f7..320f18c 100644
--- a/include/Makefile.am
+++ b/include/Makefile.am
@@ -32,7 +32,6 @@ nobase_noinst_HEADERS = \
 	compat/execinfo.h \
 	compat/srandomdev.h \
 	flopen.h \
-	libvarnish.h \
 	libvcl.h \
 	miniobj.h \
 	persistent.h \
@@ -41,7 +40,9 @@ nobase_noinst_HEADERS = \
 	vbm.h \
 	vcl.h \
 	vcs_version.h \
+	vcs.h \
 	vct.h \
+	vdef.h \
 	vend.h \
 	vev.h \
 	vfil.h \
diff --git a/include/libvarnish.h b/include/libvarnish.h
deleted file mode 100644
index 4cbb18d..0000000
--- a/include/libvarnish.h
+++ /dev/null
@@ -1,47 +0,0 @@
-/*-
- * Copyright (c) 2006 Verdens Gang AS
- * Copyright (c) 2006-2011 Varnish Software AS
- * All rights reserved.
- *
- * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- *
- */
-
-/* from libvarnish/version.c */
-void VCS_Message(const char *);
-
-/* Safe printf into a fixed-size buffer */
-#define bprintf(buf, fmt, ...)						\
-	do {								\
-		assert(snprintf(buf, sizeof buf, fmt, __VA_ARGS__)	\
-		    < sizeof buf);					\
-	} while (0)
-
-/* Safe printf into a fixed-size buffer */
-#define vbprintf(buf, fmt, ap)						\
-	do {								\
-		assert(vsnprintf(buf, sizeof buf, fmt, ap)		\
-		    < sizeof buf);					\
-	} while (0)
-
diff --git a/include/vcs.h b/include/vcs.h
new file mode 100644
index 0000000..f764100
--- /dev/null
+++ b/include/vcs.h
@@ -0,0 +1,32 @@
+/*-
+ * Copyright (c) 2006 Verdens Gang AS
+ * Copyright (c) 2006-2011 Varnish Software AS
+ * All rights reserved.
+ *
+ * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ */
+
+/* from libvarnish/version.c */
+void VCS_Message(const char *);
diff --git a/include/vdef.h b/include/vdef.h
new file mode 100644
index 0000000..c9b24c5
--- /dev/null
+++ b/include/vdef.h
@@ -0,0 +1,44 @@
+/*-
+ * Copyright (c) 2006 Verdens Gang AS
+ * Copyright (c) 2006-2011 Varnish Software AS
+ * All rights reserved.
+ *
+ * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ */
+
+/* Safe printf into a fixed-size buffer */
+#define bprintf(buf, fmt, ...)						\
+	do {								\
+		assert(snprintf(buf, sizeof buf, fmt, __VA_ARGS__)	\
+		    < sizeof buf);					\
+	} while (0)
+
+/* Safe printf into a fixed-size buffer */
+#define vbprintf(buf, fmt, ap)						\
+	do {								\
+		assert(vsnprintf(buf, sizeof buf, fmt, ap)		\
+		    < sizeof buf);					\
+	} while (0)
+
diff --git a/lib/libvarnish/version.c b/lib/libvarnish/version.c
index 014eda4..2220a69 100644
--- a/lib/libvarnish/version.c
+++ b/lib/libvarnish/version.c
@@ -33,7 +33,7 @@
 
 #include <stdio.h>
 
-#include "libvarnish.h"
+#include "vcs.h"
 #include "vcs_version.h"
 
 void
diff --git a/lib/libvarnish/vfil.c b/lib/libvarnish/vfil.c
index 884b106..a4bab9e 100644
--- a/lib/libvarnish/vfil.c
+++ b/lib/libvarnish/vfil.c
@@ -40,8 +40,8 @@
 #include <sys/stat.h>
 
 #include "vas.h"
+#include "vdef.h"
 #include "vfil.h"
-#include "libvarnish.h"
 
 int
 VFIL_tmpfile(char *template)
diff --git a/lib/libvarnish/vin.c b/lib/libvarnish/vin.c
index ccd58a9..65fdb5e 100644
--- a/lib/libvarnish/vin.c
+++ b/lib/libvarnish/vin.c
@@ -37,9 +37,9 @@
 #include <unistd.h>
 
 #include "vas.h"
-#include "vsm.h"
+#include "vdef.h"
 #include "vin.h"
-#include "libvarnish.h"
+#include "vsm.h"
 
 int
 VIN_N_Arg(const char *n_arg, char **name, char **dir, char **vsl)
diff --git a/lib/libvarnish/vlu.c b/lib/libvarnish/vlu.c
index 01d21de..1334a28 100644
--- a/lib/libvarnish/vlu.c
+++ b/lib/libvarnish/vlu.c
@@ -33,7 +33,6 @@
 #include <string.h>
 #include <stdlib.h>
 #include <unistd.h>
-#include "libvarnish.h"
 #include "vas.h"
 #include "vlu.h"
 #include "miniobj.h"
diff --git a/lib/libvarnish/vmb.c b/lib/libvarnish/vmb.c
index 2c004ce..aa069a7 100644
--- a/lib/libvarnish/vmb.c
+++ b/lib/libvarnish/vmb.c
@@ -28,7 +28,6 @@
 
 #include <pthread.h>
 #include <unistd.h>
-#include "libvarnish.h"
 #include "vmb.h"
 
 #ifdef VMB_NEEDS_PTHREAD_WORKAROUND_THIS_IS_BAD_FOR_PERFORMANCE
diff --git a/lib/libvarnish/vpf.c b/lib/libvarnish/vpf.c
index f6543c6..7875c15 100644
--- a/lib/libvarnish/vpf.c
+++ b/lib/libvarnish/vpf.c
@@ -40,7 +40,6 @@
 #include <string.h>
 #include <unistd.h>
 
-#include "libvarnish.h"		/* XXX: for assert() */
 #include "flopen.h"
 #include "vas.h"
 #include "vpf.h"
diff --git a/lib/libvarnish/vre.c b/lib/libvarnish/vre.c
index e330e99..2810bc2 100644
--- a/lib/libvarnish/vre.c
+++ b/lib/libvarnish/vre.c
@@ -29,7 +29,6 @@
 #include <pcre.h>
 #include <string.h>
 
-#include "libvarnish.h"
 #include "miniobj.h"
 #include "vas.h"
 #include "vre.h"
diff --git a/lib/libvarnish/vtim.c b/lib/libvarnish/vtim.c
index edfb8a5..772b559 100644
--- a/lib/libvarnish/vtim.c
+++ b/lib/libvarnish/vtim.c
@@ -56,7 +56,6 @@
 
 #include "vas.h"
 #include "vtim.h"
-//#include "libvarnish.h"
 
 /*
  * Note on Solaris: for some reason, clock_gettime(CLOCK_MONOTONIC, &ts) is not
diff --git a/lib/libvcl/vcc_compile.h b/lib/libvcl/vcc_compile.h
index ebbba86..a245a36 100644
--- a/lib/libvcl/vcc_compile.h
+++ b/lib/libvcl/vcc_compile.h
@@ -30,11 +30,10 @@
 
 #include <stdint.h>
 
-#include "libvarnish.h"
-#include "vas.h"
-
 #include "miniobj.h"
+#include "vas.h"
 #include "vcl.h"
+#include "vdef.h"
 #include "vqueue.h"
 #include "vsb.h"
 



From geoff at varnish-cache.org  Mon Jan  9 20:52:16 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:16 +0100
Subject: [experimental-ims] 874655c Get the rest of the stuff in there too.
Message-ID: <E1RkMCW-0007cm-PZ@project.varnish-software.com>

commit 874655c0bd62f1be7b861b61012dc801cd59ddf8
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sun Oct 9 20:53:57 2011 +0000

    Get the rest of the stuff in there too.

diff --git a/bin/varnishd/cache_panic.c b/bin/varnishd/cache_panic.c
index 464924a..63ab6a3 100644
--- a/bin/varnishd/cache_panic.c
+++ b/bin/varnishd/cache_panic.c
@@ -29,6 +29,8 @@
 
 #include "config.h"
 
+#include "vapi/vsm_int.h"
+
 #include <stdio.h>
 #include <stdlib.h>
 
@@ -40,7 +42,6 @@
 
 #include "cache.h"
 
-#include "vsm.h"
 #include "cache_backend.h"
 #include "cache_waiter.h"
 #include "vcl.h"
diff --git a/bin/varnishd/cache_shmlog.c b/bin/varnishd/cache_shmlog.c
index 75f662d..d16a744 100644
--- a/bin/varnishd/cache_shmlog.c
+++ b/bin/varnishd/cache_shmlog.c
@@ -29,11 +29,13 @@
 
 #include "config.h"
 
+
 #include <stdio.h>
 
 #include "cache.h"
+
+#include "vapi/vsm_int.h"
 #include "vmb.h"
-#include "vsm.h"
 #include "vtim.h"
 
 /* These cannot be struct lock, which depends on vsm/vsl working */
diff --git a/bin/varnishd/mgt_child.c b/bin/varnishd/mgt_child.c
index e676f48..ac76594 100644
--- a/bin/varnishd/mgt_child.c
+++ b/bin/varnishd/mgt_child.c
@@ -43,7 +43,7 @@
 #include <sys/wait.h>
 
 #include "mgt.h"
-#include "vsm.h"
+#include "vapi/vsm_int.h"
 #include "heritage.h"
 #include "vcli.h"
 #include "cli_priv.h"
diff --git a/bin/varnishd/mgt_shmem.c b/bin/varnishd/mgt_shmem.c
index e42487a..8fa5cd9 100644
--- a/bin/varnishd/mgt_shmem.c
+++ b/bin/varnishd/mgt_shmem.c
@@ -101,7 +101,7 @@
 #include "mgt.h"
 #include "heritage.h"
 #include "vmb.h"
-#include "vsm.h"
+#include "vapi/vsm_int.h"
 #include "vav.h"
 #include "vnum.h"
 #include "flopen.h"
diff --git a/bin/varnishd/vsm.c b/bin/varnishd/vsm.c
index d1717fe..bc44c21 100644
--- a/bin/varnishd/vsm.c
+++ b/bin/varnishd/vsm.c
@@ -55,7 +55,7 @@
 #include "common.h"
 
 #include "vmb.h"
-#include "vsm.h"
+#include "vapi/vsm_int.h"
 #include "vtim.h"
 
 /* These two come from beyond (mgt_shmem.c actually) */
diff --git a/include/Makefile.am b/include/Makefile.am
index 320f18c..948cef7 100644
--- a/include/Makefile.am
+++ b/include/Makefile.am
@@ -16,12 +16,12 @@ pkginclude_HEADERS = \
 	tbl/vsc_fields.h \
 	tbl/vsl_tags.h \
 	vapi/vsm.h \
+	vapi/vsm_int.h \
 	vapi/vsc.h \
 	vapi/vsc_int.h \
 	vapi/vsl.h \
 	vapi/vsl_int.h \
-	vcli.h \
-	vsm.h
+	vcli.h
 
 nobase_noinst_HEADERS = \
 	binary_heap.h \
diff --git a/include/vapi/vsm_int.h b/include/vapi/vsm_int.h
index 98e8610..0055b73 100644
--- a/include/vapi/vsm_int.h
+++ b/include/vapi/vsm_int.h
@@ -31,13 +31,12 @@
  * NB: THIS IS NOT A PUBLIC API TO VARNISH!
  */
 
-#ifndef VSM_H_INCLUDED
-#define VSM_H_INCLUDED
+#ifndef VSM_INT_H_INCLUDED
+#define VSM_INT_H_INCLUDED
 
 #define VSM_FILENAME		"_.vsm"
 
-#include <time.h>
-#include <sys/types.h>
+#include <stdint.h>
 
 /*
  * This structure describes each allocation from the shmlog
@@ -62,9 +61,9 @@ struct VSM_head {
 
 	unsigned		hdrsize;
 
-	time_t			starttime;
-	pid_t			master_pid;
-	pid_t			child_pid;
+	uint64_t		starttime;
+	int64_t			master_pid;
+	int64_t			child_pid;
 
 	unsigned		shm_size;
 
@@ -107,6 +106,10 @@ vsm_iter_n(struct VSM_chunk **pp)
 
 #define VSM_ITER(vd) for ((vd) = vsm_iter_0(); (vd) != NULL; vsm_iter_n(&vd))
 
+#else
+
+#define VSM_ITER(vd) while (YOU_NEED_MINIOBJ_TO_USE_VSM_ITER)
+
 #endif /* CHECK_OBJ_NOTNULL */
 
-#endif
+#endif /* VSM_INT_H_INCLUDED */
diff --git a/lib/libvarnish/vin.c b/lib/libvarnish/vin.c
index 65fdb5e..cd3e7d0 100644
--- a/lib/libvarnish/vin.c
+++ b/lib/libvarnish/vin.c
@@ -30,6 +30,8 @@
 
 #include "config.h"
 
+#include "vapi/vsm_int.h"
+
 #include <errno.h>
 #include <limits.h>
 #include <stdio.h>
@@ -39,7 +41,6 @@
 #include "vas.h"
 #include "vdef.h"
 #include "vin.h"
-#include "vsm.h"
 
 int
 VIN_N_Arg(const char *n_arg, char **name, char **dir, char **vsl)
diff --git a/lib/libvarnishapi/vsc.c b/lib/libvarnishapi/vsc.c
index 74cf286..211d284 100644
--- a/lib/libvarnishapi/vsc.c
+++ b/lib/libvarnishapi/vsc.c
@@ -30,6 +30,7 @@
 #include "config.h"
 
 #include "vapi/vsm.h"
+#include "vapi/vsm_int.h"
 #include "vapi/vsc.h"
 
 #include <sys/types.h>
@@ -39,7 +40,6 @@
 
 #include "vas.h"
 #include "vav.h"
-#include "vsm.h"
 #include "vqueue.h"
 #include "miniobj.h"
 
diff --git a/lib/libvarnishapi/vsl.c b/lib/libvarnishapi/vsl.c
index a2efc5d..9fa6a48 100644
--- a/lib/libvarnishapi/vsl.c
+++ b/lib/libvarnishapi/vsl.c
@@ -30,6 +30,7 @@
 #include "config.h"
 
 #include "vapi/vsm.h"
+#include "vapi/vsm_int.h"
 #include "vapi/vsl.h"
 
 #include <sys/types.h>
@@ -41,7 +42,6 @@
 #include <unistd.h>
 
 #include "vas.h"
-#include "vsm.h"
 #include "vre.h"
 #include "vbm.h"
 #include "miniobj.h"
diff --git a/lib/libvarnishapi/vsm.c b/lib/libvarnishapi/vsm.c
index 32d9e3d..eeb2b3c 100644
--- a/lib/libvarnishapi/vsm.c
+++ b/lib/libvarnishapi/vsm.c
@@ -30,6 +30,7 @@
 #include "config.h"
 
 #include "vapi/vsm.h"
+#include "vapi/vsm_int.h"
 
 #include <stdint.h>
 #include <sys/types.h>
@@ -45,7 +46,6 @@
 
 #include "vas.h"
 #include "vin.h"
-#include "vsm.h"
 #include "vbm.h"
 #include "miniobj.h"
 



From geoff at varnish-cache.org  Mon Jan  9 20:52:16 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:16 +0100
Subject: [experimental-ims] 8d05dae Typo
Message-ID: <E1RkMCW-0007d9-TW@project.varnish-software.com>

commit 8d05daeb1a577269db075e7fdc269e43281b9e20
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sun Oct 9 21:20:31 2011 +0000

    Typo

diff --git a/bin/varnishd/mgt_sandbox_solaris.c b/bin/varnishd/mgt_sandbox_solaris.c
index 3b6182f..a90d8b2 100644
--- a/bin/varnishd/mgt_sandbox_solaris.c
+++ b/bin/varnishd/mgt_sandbox_solaris.c
@@ -36,7 +36,7 @@
 
 #include <stdio.h>
 #include <syslog.h>
-#include <strin.h>
+#include <string.h>
 
 #ifdef HAVE_PRIV_H
 #include <priv.h>



From geoff at varnish-cache.org  Mon Jan  9 20:52:17 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:17 +0100
Subject: [experimental-ims] 8aa1d8b Eliminate nested <*.h> includes from
	include/*
Message-ID: <E1RkMCX-0007dO-2q@project.varnish-software.com>

commit 8aa1d8b8b87e009ae06f3cfed452bf8c6594b931
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Oct 10 09:53:50 2011 +0000

    Eliminate nested <*.h> includes from include/*
    
    Sort #includes according to rules which are for me to know and you
    to guess.

diff --git a/bin/varnishadm/varnishadm.c b/bin/varnishadm/varnishadm.c
index b857f8a..e519b01 100644
--- a/bin/varnishadm/varnishadm.c
+++ b/bin/varnishadm/varnishadm.c
@@ -29,23 +29,25 @@
 
 #include "config.h"
 
-#include "vapi/vsm.h"
-#include "vapi/vsl.h"
+#include <sys/socket.h>
+
+#ifdef HAVE_LIBEDIT
+#include <editline/readline.h>
+#endif
 
+#include <errno.h>
 #include <fcntl.h>
 #include <poll.h>
+#include <stdint.h>
 #include <stdio.h>
 #include <stdlib.h>
-#include <unistd.h>
 #include <string.h>
-#include <sys/socket.h>
-
-#ifdef HAVE_LIBEDIT
-#include <editline/readline.h>
-#endif
+#include <unistd.h>
 
 #include "vcli.h"
 #include "cli_common.h"
+#include "vapi/vsl.h"
+#include "vapi/vsm.h"
 #include "vas.h"
 #include "vss.h"
 
diff --git a/bin/varnishd/cache.h b/bin/varnishd/cache.h
index c5bc614..41efdad 100644
--- a/bin/varnishd/cache.h
+++ b/bin/varnishd/cache.h
@@ -34,6 +34,8 @@
  */
 #define VARNISH_CACHE_CHILD	1
 
+#include "common.h"
+
 #include "vapi/vsc_int.h"
 #include "vapi/vsl_int.h"
 
@@ -53,7 +55,6 @@
 #include <sys/epoll.h>
 #endif
 
-#include "common.h"
 
 #include "heritage.h"
 
diff --git a/bin/varnishd/cache_acceptor.c b/bin/varnishd/cache_acceptor.c
index 4feefb6..80d7b81 100644
--- a/bin/varnishd/cache_acceptor.c
+++ b/bin/varnishd/cache_acceptor.c
@@ -32,10 +32,10 @@
 
 #include "cache.h"
 
+#include "cli_priv.h"
 #include "vcli.h"
 #include "vtcp.h"
 #include "vtim.h"
-#include "cli_priv.h"
 
 static pthread_t	VCA_thread;
 static struct timeval	tv_sndtimeo;
diff --git a/bin/varnishd/cache_backend.c b/bin/varnishd/cache_backend.c
index 3ed7fcd..0026259 100644
--- a/bin/varnishd/cache_backend.c
+++ b/bin/varnishd/cache_backend.c
@@ -32,8 +32,8 @@
 
 #include "config.h"
 
-#include <stdlib.h>
 #include <poll.h>
+#include <stdlib.h>
 
 #include "cache.h"
 
diff --git a/bin/varnishd/cache_backend_cfg.c b/bin/varnishd/cache_backend_cfg.c
index e08e09e..2e6d83c 100644
--- a/bin/varnishd/cache_backend_cfg.c
+++ b/bin/varnishd/cache_backend_cfg.c
@@ -36,9 +36,10 @@
 #include <stdlib.h>
 
 #include "cache.h"
-#include "vrt.h"
+
 #include "cache_backend.h"
 #include "cli_priv.h"
+#include "vrt.h"
 
 struct lock VBE_mtx;
 
diff --git a/bin/varnishd/cache_backend_poll.c b/bin/varnishd/cache_backend_poll.c
index c9497bb..a0ad62b 100644
--- a/bin/varnishd/cache_backend_poll.c
+++ b/bin/varnishd/cache_backend_poll.c
@@ -37,17 +37,18 @@
 
 #include "config.h"
 
-#include <stdio.h>
 #include <math.h>
-#include <stdlib.h>
 #include <poll.h>
+#include <stdio.h>
+#include <stdlib.h>
 
-#include "cli_priv.h"
 #include "cache.h"
+
+#include "cache_backend.h"
+#include "cli_priv.h"
 #include "vrt.h"
 #include "vtcp.h"
 #include "vtim.h"
-#include "cache_backend.h"
 
 /* Default averaging rate, we want something pretty responsive */
 #define AVG_RATE			4
diff --git a/bin/varnishd/cache_ban.c b/bin/varnishd/cache_ban.c
index 8396c7f..bec8d85 100644
--- a/bin/varnishd/cache_ban.c
+++ b/bin/varnishd/cache_ban.c
@@ -62,17 +62,16 @@
 
 #include "config.h"
 
-#include <stdio.h>
-
 #include <pcre.h>
+#include <stdio.h>
 
 #include "cache.h"
 
+#include "cli_priv.h"
+#include "hash_slinger.h"
 #include "vcli.h"
 #include "vend.h"
 #include "vtim.h"
-#include "cli_priv.h"
-#include "hash_slinger.h"
 
 struct ban {
 	unsigned		magic;
diff --git a/bin/varnishd/cache_center.c b/bin/varnishd/cache_center.c
index 6ab8a11..ba61e9c 100644
--- a/bin/varnishd/cache_center.c
+++ b/bin/varnishd/cache_center.c
@@ -57,25 +57,25 @@ DOT acceptor -> start [style=bold,color=green]
 
 #include "config.h"
 
-#include <stdio.h>
 #include <errno.h>
 #include <math.h>
 #include <poll.h>
+#include <stdio.h>
 #include <stdlib.h>
 
-#ifndef HAVE_SRANDOMDEV
-#include "compat/srandomdev.h"
-#endif
-
 #include "cache.h"
 
-#include "vcl.h"
-#include "vtcp.h"
-#include "vtim.h"
 #include "cli_priv.h"
 #include "hash_slinger.h"
 #include "stevedore.h"
+#include "vcl.h"
 #include "vsha256.h"
+#include "vtcp.h"
+#include "vtim.h"
+
+#ifndef HAVE_SRANDOMDEV
+#include "compat/srandomdev.h"
+#endif
 
 static unsigned xids;
 
diff --git a/bin/varnishd/cache_cli.c b/bin/varnishd/cache_cli.c
index 3635aec..f66075e 100644
--- a/bin/varnishd/cache_cli.c
+++ b/bin/varnishd/cache_cli.c
@@ -41,8 +41,8 @@
 #include "cache.h"
 
 #include "vcli.h"
-#include "cli_priv.h"
 #include "cli_common.h"
+#include "cli_priv.h"
 #include "cli_serve.h"
 #include "hash_slinger.h"		// objhead
 
diff --git a/bin/varnishd/cache_dir.c b/bin/varnishd/cache_dir.c
index ef882bf..3fd0475 100644
--- a/bin/varnishd/cache_dir.c
+++ b/bin/varnishd/cache_dir.c
@@ -33,6 +33,7 @@
 #include "config.h"
 
 #include "cache.h"
+
 #include "cache_backend.h"
 #include "vtcp.h"
 
diff --git a/bin/varnishd/cache_dir_dns.c b/bin/varnishd/cache_dir_dns.c
index 3fd2aa1..acb3689 100644
--- a/bin/varnishd/cache_dir_dns.c
+++ b/bin/varnishd/cache_dir_dns.c
@@ -29,14 +29,15 @@
 
 #include "config.h"
 
+#include <netinet/in.h>
+
 #include <errno.h>
-#include <stdlib.h>
 #include <netdb.h>
-
 #include <stdio.h>
-#include <netinet/in.h>
+#include <stdlib.h>
 
 #include "cache.h"
+
 #include "cache_backend.h"
 #include "vrt.h"
 
diff --git a/bin/varnishd/cache_dir_random.c b/bin/varnishd/cache_dir_random.c
index f05d8f4..f18c7d7 100644
--- a/bin/varnishd/cache_dir_random.c
+++ b/bin/varnishd/cache_dir_random.c
@@ -50,10 +50,11 @@
 #include <stdlib.h>
 
 #include "cache.h"
+
 #include "cache_backend.h"
+#include "vend.h"
 #include "vrt.h"
 #include "vsha256.h"
-#include "vend.h"
 
 /*--------------------------------------------------------------------*/
 
diff --git a/bin/varnishd/cache_dir_round_robin.c b/bin/varnishd/cache_dir_round_robin.c
index 9d0a257..7d75473 100644
--- a/bin/varnishd/cache_dir_round_robin.c
+++ b/bin/varnishd/cache_dir_round_robin.c
@@ -32,6 +32,7 @@
 #include <stdlib.h>
 
 #include "cache.h"
+
 #include "cache_backend.h"
 #include "vrt.h"
 
diff --git a/bin/varnishd/cache_esi_deliver.c b/bin/varnishd/cache_esi_deliver.c
index 063464a..f7daeeb 100644
--- a/bin/varnishd/cache_esi_deliver.c
+++ b/bin/varnishd/cache_esi_deliver.c
@@ -30,10 +30,11 @@
 
 #include "config.h"
 
-#include <stdlib.h>
 #include <stdio.h>
+#include <stdlib.h>
 
 #include "cache.h"
+
 #include "cache_esi.h"
 #include "vend.h"
 #include "vgz.h"
diff --git a/bin/varnishd/cache_esi_fetch.c b/bin/varnishd/cache_esi_fetch.c
index cac2eb6..b6ef527 100644
--- a/bin/varnishd/cache_esi_fetch.c
+++ b/bin/varnishd/cache_esi_fetch.c
@@ -33,9 +33,10 @@
 #include <stdlib.h>
 
 #include "cache.h"
+
 #include "cache_esi.h"
-#include "vct.h"
 #include "stevedore.h"
+#include "vct.h"
 
 /*---------------------------------------------------------------------
  * Read some bytes.
diff --git a/bin/varnishd/cache_esi_parse.c b/bin/varnishd/cache_esi_parse.c
index c659dd5..37c20e7 100644
--- a/bin/varnishd/cache_esi_parse.c
+++ b/bin/varnishd/cache_esi_parse.c
@@ -34,9 +34,10 @@
 #include <stdlib.h>
 
 #include "cache.h"
+
 #include "cache_esi.h"
-#include "vend.h"
 #include "vct.h"
+#include "vend.h"
 #include "vgz.h"
 
 //#define Debug(fmt, ...) printf(fmt, __VA_ARGS__)
diff --git a/bin/varnishd/cache_expire.c b/bin/varnishd/cache_expire.c
index aed6807..8e18cb5 100644
--- a/bin/varnishd/cache_expire.c
+++ b/bin/varnishd/cache_expire.c
@@ -53,8 +53,9 @@
 
 #include <math.h>
 
-#include "binary_heap.h"
 #include "cache.h"
+
+#include "binary_heap.h"
 #include "hash_slinger.h"
 #include "stevedore.h"
 #include "vtim.h"
diff --git a/bin/varnishd/cache_fetch.c b/bin/varnishd/cache_fetch.c
index d18e726..4bbc272 100644
--- a/bin/varnishd/cache_fetch.c
+++ b/bin/varnishd/cache_fetch.c
@@ -29,14 +29,15 @@
 
 #include "config.h"
 
-#include <stdio.h>
+#include <fcntl.h>
 #include <inttypes.h>
+#include <stdio.h>
 #include <stdlib.h>
-#include <fcntl.h>
 
 #include "cache.h"
-#include "stevedore.h"
+
 #include "cli_priv.h"
+#include "stevedore.h"
 #include "vct.h"
 #include "vtcp.h"
 
diff --git a/bin/varnishd/cache_gzip.c b/bin/varnishd/cache_gzip.c
index 28d61d9..156ac52 100644
--- a/bin/varnishd/cache_gzip.c
+++ b/bin/varnishd/cache_gzip.c
@@ -65,6 +65,7 @@
  */
 
 #include "config.h"
+
 #include <stdio.h>
 #include <stdlib.h>
 
diff --git a/bin/varnishd/cache_hash.c b/bin/varnishd/cache_hash.c
index 4001c7c..67aa3af 100644
--- a/bin/varnishd/cache_hash.c
+++ b/bin/varnishd/cache_hash.c
@@ -52,17 +52,18 @@
 
 #include "config.h"
 
+#include <math.h>
 #include <stdio.h>
 #include <stdlib.h>
-#include <math.h>
 #include <string.h>
 
 #include "cache.h"
-#include "stevedore.h"
+
+#include "cache_backend.h"
 #include "hash_slinger.h"
-#include "vsha256.h"
+#include "stevedore.h"
 #include "vav.h"
-#include "cache_backend.h"
+#include "vsha256.h"
 
 static const struct hash_slinger *hash;
 
diff --git a/bin/varnishd/cache_main.c b/bin/varnishd/cache_main.c
index b4ee185..54d71f8 100644
--- a/bin/varnishd/cache_main.c
+++ b/bin/varnishd/cache_main.c
@@ -34,9 +34,9 @@
 
 #include "cache.h"
 
-#include "stevedore.h"
-#include "hash_slinger.h"
 #include "cache_waiter.h"
+#include "hash_slinger.h"
+#include "stevedore.h"
 
 /*--------------------------------------------------------------------
  * Per thread storage for the session currently being processed by
diff --git a/bin/varnishd/cache_panic.c b/bin/varnishd/cache_panic.c
index 63ab6a3..482708d 100644
--- a/bin/varnishd/cache_panic.c
+++ b/bin/varnishd/cache_panic.c
@@ -29,23 +29,23 @@
 
 #include "config.h"
 
-#include "vapi/vsm_int.h"
-
-#include <stdio.h>
-#include <stdlib.h>
-
 #ifndef HAVE_EXECINFO_H
 #include "compat/execinfo.h"
 #else
 #include <execinfo.h>
 #endif
 
+#include <stdio.h>
+#include <stdlib.h>
+
 #include "cache.h"
 
+#include "vapi/vsm_int.h"
+
 #include "cache_backend.h"
 #include "cache_waiter.h"
-#include "vcl.h"
 #include "libvcl.h"
+#include "vcl.h"
 
 /*
  * The panic string is constructed in memory, then copied to the
diff --git a/bin/varnishd/cache_pipe.c b/bin/varnishd/cache_pipe.c
index 418ded1..c9bc0ac 100644
--- a/bin/varnishd/cache_pipe.c
+++ b/bin/varnishd/cache_pipe.c
@@ -31,10 +31,11 @@
 
 #include "config.h"
 
-#include <stdio.h>
 #include <poll.h>
+#include <stdio.h>
 
 #include "cache.h"
+
 #include "vtcp.h"
 #include "vtim.h"
 
diff --git a/bin/varnishd/cache_response.c b/bin/varnishd/cache_response.c
index b4459da..bacf906 100644
--- a/bin/varnishd/cache_response.c
+++ b/bin/varnishd/cache_response.c
@@ -36,6 +36,7 @@
 #include <stdlib.h>
 
 #include "cache.h"
+
 #include "stevedore.h"
 #include "vct.h"
 #include "vtim.h"
diff --git a/bin/varnishd/cache_session.c b/bin/varnishd/cache_session.c
index bb96780..d55bae0 100644
--- a/bin/varnishd/cache_session.c
+++ b/bin/varnishd/cache_session.c
@@ -35,10 +35,11 @@
 
 #include "config.h"
 
-#include <stdlib.h>
 #include <math.h>
+#include <stdlib.h>
 
 #include "cache.h"
+
 #include "cache_waiter.h"
 
 /*--------------------------------------------------------------------*/
diff --git a/bin/varnishd/cache_shmlog.c b/bin/varnishd/cache_shmlog.c
index d16a744..da39e2d 100644
--- a/bin/varnishd/cache_shmlog.c
+++ b/bin/varnishd/cache_shmlog.c
@@ -29,7 +29,6 @@
 
 #include "config.h"
 
-
 #include <stdio.h>
 
 #include "cache.h"
diff --git a/bin/varnishd/cache_vary.c b/bin/varnishd/cache_vary.c
index 4440d55..c528fb8 100644
--- a/bin/varnishd/cache_vary.c
+++ b/bin/varnishd/cache_vary.c
@@ -56,8 +56,8 @@
 
 #include "cache.h"
 
-#include "vend.h"
 #include "vct.h"
+#include "vend.h"
 
 struct vsb *
 VRY_Create(const struct sess *sp, const struct http *hp)
diff --git a/bin/varnishd/cache_vcl.c b/bin/varnishd/cache_vcl.c
index 99364b5..48568ff 100644
--- a/bin/varnishd/cache_vcl.c
+++ b/bin/varnishd/cache_vcl.c
@@ -33,15 +33,16 @@
 
 #include "config.h"
 
+#include <dlfcn.h>
 #include <stdio.h>
 #include <stdlib.h>
-#include <dlfcn.h>
 
 #include "cache.h"
-#include "vcli.h"
+
 #include "cli_priv.h"
-#include "vcl.h"
 #include "libvcl.h"
+#include "vcl.h"
+#include "vcli.h"
 
 struct vcls {
 	unsigned		magic;
diff --git a/bin/varnishd/cache_vrt.c b/bin/varnishd/cache_vrt.c
index 5c633e2..2302b86 100644
--- a/bin/varnishd/cache_vrt.c
+++ b/bin/varnishd/cache_vrt.c
@@ -39,13 +39,13 @@
 
 #include "cache.h"
 
-#include "vrt.h"
-#include "vrt_obj.h"
+#include "cache_backend.h"
+#include "hash_slinger.h"
 #include "vav.h"
 #include "vcl.h"
+#include "vrt.h"
+#include "vrt_obj.h"
 #include "vtim.h"
-#include "hash_slinger.h"
-#include "cache_backend.h"
 
 const void * const vrt_magic_string_end = &vrt_magic_string_end;
 
diff --git a/bin/varnishd/cache_vrt_re.c b/bin/varnishd/cache_vrt_re.c
index d28510f..d9a7090 100644
--- a/bin/varnishd/cache_vrt_re.c
+++ b/bin/varnishd/cache_vrt_re.c
@@ -36,8 +36,8 @@
 
 #include "cache.h"
 
-#include "vrt.h"
 #include "vre.h"
+#include "vrt.h"
 
 void
 VRT_re_init(void **rep, const char *re)
diff --git a/bin/varnishd/cache_vrt_var.c b/bin/varnishd/cache_vrt_var.c
index ff523df..a00b1b7 100644
--- a/bin/varnishd/cache_vrt_var.c
+++ b/bin/varnishd/cache_vrt_var.c
@@ -35,12 +35,12 @@
 
 #include "cache.h"
 
+#include "cache_backend.h"
+#include "hash_slinger.h"
 #include "vrt.h"
+#include "vrt_obj.h"
 #include "vtcp.h"
 #include "vtim.h"
-#include "vrt_obj.h"
-#include "cache_backend.h"
-#include "hash_slinger.h"
 
 static char vrt_hostname[255] = "";
 
diff --git a/bin/varnishd/cache_vrt_vmod.c b/bin/varnishd/cache_vrt_vmod.c
index 4eb3e84..bb4eeb8 100644
--- a/bin/varnishd/cache_vrt_vmod.c
+++ b/bin/varnishd/cache_vrt_vmod.c
@@ -31,8 +31,8 @@
 
 #include "config.h"
 
-#include <stdlib.h>
 #include <dlfcn.h>
+#include <stdlib.h>
 
 #include "cache.h"
 
diff --git a/bin/varnishd/cache_waiter.c b/bin/varnishd/cache_waiter.c
index cd3e007..08c5809 100644
--- a/bin/varnishd/cache_waiter.c
+++ b/bin/varnishd/cache_waiter.c
@@ -30,10 +30,11 @@
 
 #include "config.h"
 
-#include "vcli.h"
-#include "cli_priv.h"
 #include "cache.h"
+
 #include "cache_waiter.h"
+#include "cli_priv.h"
+#include "vcli.h"
 
 static const struct waiter * const vca_waiters[] = {
     #if defined(HAVE_KQUEUE)
diff --git a/bin/varnishd/cache_waiter_epoll.c b/bin/varnishd/cache_waiter_epoll.c
index c66d15a..3eaa627 100644
--- a/bin/varnishd/cache_waiter_epoll.c
+++ b/bin/varnishd/cache_waiter_epoll.c
@@ -35,12 +35,14 @@
 
 #if defined(HAVE_EPOLL_CTL)
 
+#include <sys/epoll.h>
+
 #include <errno.h>
 #include <fcntl.h>
 #include <stdlib.h>
-#include <sys/epoll.h>
 
 #include "cache.h"
+
 #include "cache_waiter.h"
 #include "vtim.h"
 
diff --git a/bin/varnishd/cache_waiter_kqueue.c b/bin/varnishd/cache_waiter_kqueue.c
index e7cb299..a81f4c2 100644
--- a/bin/varnishd/cache_waiter_kqueue.c
+++ b/bin/varnishd/cache_waiter_kqueue.c
@@ -35,13 +35,16 @@
 
 #if defined(HAVE_KQUEUE)
 
+#include <sys/types.h>
+#include <sys/event.h>
+
 #include <errno.h>
 #include <fcntl.h>
 #include <stdlib.h>
 #include <unistd.h>
-#include <sys/event.h>
 
 #include "cache.h"
+
 #include "cache_waiter.h"
 #include "vtim.h"
 
diff --git a/bin/varnishd/cache_waiter_poll.c b/bin/varnishd/cache_waiter_poll.c
index f3f3289..eb48870 100644
--- a/bin/varnishd/cache_waiter_poll.c
+++ b/bin/varnishd/cache_waiter_poll.c
@@ -30,8 +30,8 @@
 
 #include "config.h"
 
-#include <stdlib.h>
 #include <poll.h>
+#include <stdlib.h>
 
 #include "cache.h"
 
diff --git a/bin/varnishd/cache_waiter_ports.c b/bin/varnishd/cache_waiter_ports.c
index ca7e229..2e02406 100644
--- a/bin/varnishd/cache_waiter_ports.c
+++ b/bin/varnishd/cache_waiter_ports.c
@@ -33,15 +33,16 @@
 
 #if defined(HAVE_PORT_CREATE)
 
-#include <stdio.h>
+#include <sys/time.h>
+
 #include <errno.h>
-#include <stdlib.h>
 #include <math.h>
-
 #include <port.h>
-#include <sys/time.h>
+#include <stdio.h>
+#include <stdlib.h>
 
 #include "cache.h"
+
 #include "cache_waiter.h"
 #include "vtim.h"
 
diff --git a/bin/varnishd/cache_wrk.c b/bin/varnishd/cache_wrk.c
index 7e80f04..1032691 100644
--- a/bin/varnishd/cache_wrk.c
+++ b/bin/varnishd/cache_wrk.c
@@ -35,6 +35,7 @@
 #include <stdlib.h>
 
 #include "cache.h"
+
 #include "hash_slinger.h"
 #include "vsha256.h"
 
diff --git a/bin/varnishd/cache_wrw.c b/bin/varnishd/cache_wrw.c
index 5f9c29b..4fef1aa 100644
--- a/bin/varnishd/cache_wrw.c
+++ b/bin/varnishd/cache_wrw.c
@@ -34,21 +34,21 @@
 
 #include "config.h"
 
-#include <stdio.h>
 #include <sys/types.h>
-#include <sys/uio.h>
-
 #ifdef SENDFILE_WORKS
-#if defined(__FreeBSD__) || defined(__DragonFly__)
-   // We're fine
-#elif defined(__linux__)
-#  include <sys/sendfile.h>
-#elif defined(__sun)
-#  include <sys/sendfile.h>
-#else
-#  error Unknown sendfile() implementation
-#endif
+#  if defined(__FreeBSD__) || defined(__DragonFly__)
+#    include <sys/socket.h>
+#  elif defined(__linux__)
+#    include <sys/sendfile.h>
+#  elif defined(__sun)
+#    include <sys/sendfile.h>
+#  else
+#     error Unknown sendfile() implementation
+#  endif
 #endif /* SENDFILE_WORKS */
+#include <sys/uio.h>
+
+#include <stdio.h>
 
 #include "cache.h"
 
diff --git a/bin/varnishd/common.h b/bin/varnishd/common.h
index 7fd15b7..13249eb 100644
--- a/bin/varnishd/common.h
+++ b/bin/varnishd/common.h
@@ -28,6 +28,11 @@
  *
  */
 
+#include <errno.h>
+#include <stdint.h>
+
+#include <sys/types.h>
+
 #include "miniobj.h"
 #include "vas.h"
 #include "vcs.h"
@@ -73,8 +78,8 @@ const void *pick(const struct choice *cp, const char *which, const char *kind);
 #define NEEDLESS_RETURN(foo)	return (foo)
 
 /* vsm.c */
-extern struct VSM_head		*VSM_head;
-extern const struct VSM_chunk	*vsm_end;
+// extern struct VSM_head		*VSM_head;
+// extern const struct VSM_chunk	*vsm_end;
 
 /*
  * These three should not be called directly, but only through
diff --git a/bin/varnishd/hash_classic.c b/bin/varnishd/hash_classic.c
index 9b8c7d8..1428864 100644
--- a/bin/varnishd/hash_classic.c
+++ b/bin/varnishd/hash_classic.c
@@ -31,12 +31,14 @@
 
 #include "config.h"
 
+#include <sys/types.h>
+
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
-#include <sys/types.h>
 
 #include "cache.h"
+
 #include "hash_slinger.h"
 
 /*--------------------------------------------------------------------*/
diff --git a/bin/varnishd/hash_critbit.c b/bin/varnishd/hash_critbit.c
index e5b0746..add2397 100644
--- a/bin/varnishd/hash_critbit.c
+++ b/bin/varnishd/hash_critbit.c
@@ -28,15 +28,16 @@
  * A Crit Bit tree based hash
  */
 
-#undef PHK
+// #define PHK
 
 #include "config.h"
 
 #include <stdlib.h>
 
 #include "cache.h"
-#include "hash_slinger.h"
+
 #include "cli_priv.h"
+#include "hash_slinger.h"
 #include "vmb.h"
 #include "vtim.h"
 
diff --git a/bin/varnishd/hash_simple_list.c b/bin/varnishd/hash_simple_list.c
index 13cf0ee..c1c17da 100644
--- a/bin/varnishd/hash_simple_list.c
+++ b/bin/varnishd/hash_simple_list.c
@@ -32,6 +32,7 @@
 #include "config.h"
 
 #include "cache.h"
+
 #include "hash_slinger.h"
 
 /*--------------------------------------------------------------------*/
diff --git a/bin/varnishd/mgt_child.c b/bin/varnishd/mgt_child.c
index ac76594..7338e6c 100644
--- a/bin/varnishd/mgt_child.c
+++ b/bin/varnishd/mgt_child.c
@@ -31,27 +31,30 @@
 
 #include "config.h"
 
-#include <unistd.h>
-#include <stdio.h>
-#include <string.h>
+#include <sys/types.h>
+#include <sys/wait.h>
+
+#include <errno.h>
 #include <fcntl.h>
-#include <stdlib.h>
+#include <poll.h>
 #include <signal.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
 #include <syslog.h>
-#include <errno.h>
-#include <sys/types.h>
-#include <sys/wait.h>
+#include <unistd.h>
 
 #include "mgt.h"
-#include "vapi/vsm_int.h"
-#include "heritage.h"
-#include "vcli.h"
+
 #include "cli_priv.h"
+#include "heritage.h"
 #include "mgt_cli.h"
+#include "vapi/vsm_int.h"
+#include "vbm.h"
+#include "vcli.h"
 #include "vev.h"
 #include "vlu.h"
 #include "vss.h"
-#include "vbm.h"
 #include "vtcp.h"
 #include "vtim.h"
 
diff --git a/bin/varnishd/mgt_cli.c b/bin/varnishd/mgt_cli.c
index 4488d51..4a7925a 100644
--- a/bin/varnishd/mgt_cli.c
+++ b/bin/varnishd/mgt_cli.c
@@ -31,32 +31,33 @@
 
 #include "config.h"
 
+#include <sys/socket.h>
+
+#include <fcntl.h>
+#include <poll.h>
 #include <stdarg.h>
 #include <stdio.h>
-#include <fcntl.h>
 #include <stdlib.h>
-#include <syslog.h>
 #include <string.h>
+#include <syslog.h>
 #include <unistd.h>
-#include <sys/socket.h>
 
-#ifndef HAVE_SRANDOMDEV
-#include "compat/srandomdev.h"
-#endif
+#include "mgt.h"
 
-#include "cli_priv.h"
 #include "vcli.h"
 #include "cli_common.h"
+#include "cli_priv.h"
 #include "cli_serve.h"
+#include "heritage.h"
+#include "mgt_cli.h"
 #include "vev.h"
 #include "vlu.h"
 #include "vss.h"
 #include "vtcp.h"
 
-
-#include "mgt.h"
-#include "heritage.h"
-#include "mgt_cli.h"
+#ifndef HAVE_SRANDOMDEV
+#include "compat/srandomdev.h"
+#endif
 
 static int		cli_i = -1, cli_o = -1;
 static struct VCLS	*cls;
diff --git a/bin/varnishd/mgt_param.c b/bin/varnishd/mgt_param.c
index f849fbb..4bb1adf 100644
--- a/bin/varnishd/mgt_param.c
+++ b/bin/varnishd/mgt_param.c
@@ -37,17 +37,16 @@
 #include <string.h>
 #include <unistd.h>
 
-#include "vcli.h"
-#include "cli_priv.h"
-#include "cli_common.h"
 #include "mgt.h"
-#include "mgt_cli.h"
 
-#include "heritage.h"
-#include "vparam.h"
+#include "vcli.h"
 #include "cache_waiter.h"
-
+#include "cli_common.h"
+#include "cli_priv.h"
+#include "heritage.h"
+#include "mgt_cli.h"
 #include "vav.h"
+#include "vparam.h"
 #include "vss.h"
 
 #define MAGIC_INIT_STRING	"\001"
diff --git a/bin/varnishd/mgt_pool.c b/bin/varnishd/mgt_pool.c
index 4b3fe92..489fab8 100644
--- a/bin/varnishd/mgt_pool.c
+++ b/bin/varnishd/mgt_pool.c
@@ -42,15 +42,15 @@
 
 #include "config.h"
 
+#include <limits.h>
 #include <stdio.h>
 #include <string.h>
-#include <limits.h>
 #include <unistd.h>
 
 #include "mgt.h"
 
-#include "vparam.h"
 #include "heritage.h"
+#include "vparam.h"
 
 /*--------------------------------------------------------------------*/
 
diff --git a/bin/varnishd/mgt_sandbox.c b/bin/varnishd/mgt_sandbox.c
index 94f9f4d..3aba7cc 100644
--- a/bin/varnishd/mgt_sandbox.c
+++ b/bin/varnishd/mgt_sandbox.c
@@ -44,15 +44,16 @@
 
 #include "config.h"
 
-#include <stdio.h>
-#include <syslog.h>
-#include <unistd.h>
-
 #ifdef __linux__
 #include <sys/prctl.h>
 #endif
 
+#include <stdio.h>
+#include <syslog.h>
+#include <unistd.h>
+
 #include "mgt.h"
+
 #include "heritage.h"
 
 /*--------------------------------------------------------------------*/
diff --git a/bin/varnishd/mgt_sandbox_solaris.c b/bin/varnishd/mgt_sandbox_solaris.c
index a90d8b2..5c50c83 100644
--- a/bin/varnishd/mgt_sandbox_solaris.c
+++ b/bin/varnishd/mgt_sandbox_solaris.c
@@ -34,15 +34,15 @@
 
 #ifdef HAVE_SETPPRIV
 
-#include <stdio.h>
-#include <syslog.h>
-#include <string.h>
-
 #ifdef HAVE_PRIV_H
 #include <priv.h>
 #endif
+#include <stdio.h>
+#include <string.h>
+#include <syslog.h>
 
 #include "mgt.h"
+
 #include "heritage.h"
 
 /*--------------------------------------------------------------------
diff --git a/bin/varnishd/mgt_shmem.c b/bin/varnishd/mgt_shmem.c
index 8fa5cd9..ae0a50d 100644
--- a/bin/varnishd/mgt_shmem.c
+++ b/bin/varnishd/mgt_shmem.c
@@ -86,25 +86,27 @@
 
 #include "config.h"
 
-#include "vapi/vsl_int.h"
-#include "vapi/vsc_int.h"
+#include <sys/mman.h>
+#include <sys/stat.h>
 
+#include <fcntl.h>
 #include <signal.h>
+#include <stdint.h>
 #include <stdio.h>
 #include <stdlib.h>
-#include <unistd.h>
-#include <fcntl.h>
 #include <string.h>
-#include <sys/mman.h>
-#include <sys/stat.h>
+#include <unistd.h>
 
 #include "mgt.h"
+
+#include "flopen.h"
 #include "heritage.h"
-#include "vmb.h"
+#include "vapi/vsc_int.h"
+#include "vapi/vsl_int.h"
 #include "vapi/vsm_int.h"
 #include "vav.h"
+#include "vmb.h"
 #include "vnum.h"
-#include "flopen.h"
 
 #ifndef MAP_HASSEMAPHORE
 #define MAP_HASSEMAPHORE 0 /* XXX Linux */
diff --git a/bin/varnishd/mgt_vcc.c b/bin/varnishd/mgt_vcc.c
index df46c41..78a29dd 100644
--- a/bin/varnishd/mgt_vcc.c
+++ b/bin/varnishd/mgt_vcc.c
@@ -40,13 +40,13 @@
 
 #include "mgt.h"
 
+#include "cli_priv.h"
 #include "libvcl.h"
-#include "vcli.h"
-#include "vsub.h"
+#include "mgt_cli.h"
 #include "vcl.h"
+#include "vcli.h"
 #include "vfil.h"
-#include "cli_priv.h"
-#include "mgt_cli.h"
+#include "vsub.h"
 
 struct vclprog {
 	VTAILQ_ENTRY(vclprog)	list;
diff --git a/bin/varnishd/rfc2616.c b/bin/varnishd/rfc2616.c
index 6de52e2..d4d5a05 100644
--- a/bin/varnishd/rfc2616.c
+++ b/bin/varnishd/rfc2616.c
@@ -29,10 +29,11 @@
 
 #include "config.h"
 
-#include <stdlib.h>
 #include <math.h>
+#include <stdlib.h>
 
 #include "cache.h"
+
 #include "vtim.h"
 
 /*--------------------------------------------------------------------
diff --git a/bin/varnishd/stevedore.c b/bin/varnishd/stevedore.c
index 6d71635..538ae27 100644
--- a/bin/varnishd/stevedore.c
+++ b/bin/varnishd/stevedore.c
@@ -37,9 +37,11 @@
 #include <stdlib.h>
 
 #include "cache.h"
+
+#include "cli_priv.h"
 #include "stevedore.h"
 #include "vav.h"
-#include "cli_priv.h"
+#include "vrt.h"
 #include "vrt_obj.h"
 
 static VTAILQ_HEAD(, stevedore)	stevedores =
@@ -560,8 +562,6 @@ struct cli_proto cli_stv[] = {
  * VRT functions for stevedores
  */
 
-#include "vrt.h"
-
 static const struct stevedore *
 stv_find(const char *nm)
 {
diff --git a/bin/varnishd/stevedore_utils.c b/bin/varnishd/stevedore_utils.c
index 725a086..42c2b47 100644
--- a/bin/varnishd/stevedore_utils.c
+++ b/bin/varnishd/stevedore_utils.c
@@ -31,27 +31,26 @@
 
 #include "config.h"
 
-#include <fcntl.h>
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#include <unistd.h>
 #include <sys/types.h>
 #include <sys/stat.h>
-
 #ifdef HAVE_SYS_MOUNT_H
-#include <sys/mount.h>
+#  include <sys/mount.h>
 #endif
-
 #ifdef HAVE_SYS_STATVFS_H
-#include <sys/statvfs.h>
+#  include <sys/statvfs.h>
 #endif
-
 #ifdef HAVE_SYS_VFS_H
-#include <sys/vfs.h>
+#  include <sys/vfs.h>
 #endif
 
+#include <fcntl.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+
 #include "mgt.h"
+
 #include "stevedore.h"
 #include "vnum.h"
 
diff --git a/bin/varnishd/storage_file.c b/bin/varnishd/storage_file.c
index 25c9947..171d9d0 100644
--- a/bin/varnishd/storage_file.c
+++ b/bin/varnishd/storage_file.c
@@ -32,6 +32,7 @@
 #include "config.h"
 
 #include <sys/mman.h>
+
 #include <stdio.h>
 #include <stdlib.h>
 
diff --git a/bin/varnishd/storage_persistent.c b/bin/varnishd/storage_persistent.c
index 3ba7061..18b96c3 100644
--- a/bin/varnishd/storage_persistent.c
+++ b/bin/varnishd/storage_persistent.c
@@ -35,22 +35,24 @@
 
 #include "config.h"
 
-#include <stdio.h>
+#include <sys/param.h>
+#include <sys/mman.h>
+
 #include <stdint.h>
+#include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
-#include <sys/param.h>
-#include <sys/mman.h>
 
 #include "cache.h"
-#include "stevedore.h"
+
+#include "cli_priv.h"
 #include "hash_slinger.h"
-#include "vsha256.h"
+#include "persistent.h"
+#include "stevedore.h"
 #include "vcli.h"
-#include "cli_priv.h"
 #include "vend.h"
+#include "vsha256.h"
 
-#include "persistent.h"
 #include "storage_persistent.h"
 
 /*--------------------------------------------------------------------*/
diff --git a/bin/varnishd/storage_persistent_mgt.c b/bin/varnishd/storage_persistent_mgt.c
index c91fb43..7ec9089 100644
--- a/bin/varnishd/storage_persistent_mgt.c
+++ b/bin/varnishd/storage_persistent_mgt.c
@@ -35,12 +35,14 @@
 
 #include "config.h"
 
+#include <sys/mman.h>
+
 #include <math.h>
 #include <stdio.h>
 #include <stdlib.h>
-#include <sys/mman.h>
 
 #include "cache.h"
+
 #include "stevedore.h"
 #include "vsha256.h"
 
diff --git a/bin/varnishd/storage_persistent_silo.c b/bin/varnishd/storage_persistent_silo.c
index 2defc0d..be2e2bf 100644
--- a/bin/varnishd/storage_persistent_silo.c
+++ b/bin/varnishd/storage_persistent_silo.c
@@ -38,8 +38,9 @@
 #include <stdlib.h>
 
 #include "cache.h"
-#include "stevedore.h"
+
 #include "hash_slinger.h"
+#include "stevedore.h"
 #include "vsha256.h"
 #include "vtim.h"
 
diff --git a/bin/varnishd/storage_persistent_subr.c b/bin/varnishd/storage_persistent_subr.c
index 6fad388..e3df368 100644
--- a/bin/varnishd/storage_persistent_subr.c
+++ b/bin/varnishd/storage_persistent_subr.c
@@ -35,12 +35,14 @@
 
 #include "config.h"
 
+#include <sys/mman.h>
+
+#include <stddef.h>
 #include <stdio.h>
 #include <stdlib.h>
-#include <stddef.h>
-#include <sys/mman.h>
 
 #include "cache.h"
+
 #include "vsha256.h"
 
 #include "persistent.h"
diff --git a/bin/varnishd/storage_umem.c b/bin/varnishd/storage_umem.c
index 197d0ac..4459a47 100644
--- a/bin/varnishd/storage_umem.c
+++ b/bin/varnishd/storage_umem.c
@@ -37,11 +37,10 @@
 
 #include <stdio.h>
 #include <stdlib.h>
-
 #include <umem.h>
 
-#include "config.h"
 #include "cache.h"
+
 #include "stevedore.h"
 
 static size_t			smu_max = SIZE_MAX;
diff --git a/bin/varnishd/varnishd.c b/bin/varnishd/varnishd.c
index 9ba9570..7cecfe4 100644
--- a/bin/varnishd/varnishd.c
+++ b/bin/varnishd/varnishd.c
@@ -32,6 +32,7 @@
 #include "config.h"
 
 #include <sys/stat.h>
+#include <sys/utsname.h>
 
 #include <ctype.h>
 #include <errno.h>
@@ -43,25 +44,22 @@
 #include <time.h>
 #include <unistd.h>
 
-#include <sys/utsname.h>
-
-#include "compat/daemon.h"
-
-#include "vev.h"
-#include "vpf.h"
-#include "vsha256.h"
+#include "mgt.h"
 
 #include "vcli.h"
 #include "cli_common.h"
-
+#include "hash_slinger.h"
+#include "heritage.h"
+#include "stevedore.h"
 #include "vav.h"
-#include "vin.h"
+#include "vev.h"
 #include "vfil.h"
+#include "vin.h"
+#include "vpf.h"
+#include "vsha256.h"
 #include "vtim.h"
-#include "heritage.h"
-#include "mgt.h"
-#include "hash_slinger.h"
-#include "stevedore.h"
+
+#include "compat/daemon.h"
 
 #ifndef HAVE_SRANDOMDEV
 #include "compat/srandomdev.h"
diff --git a/bin/varnishd/vsm.c b/bin/varnishd/vsm.c
index bc44c21..5298381 100644
--- a/bin/varnishd/vsm.c
+++ b/bin/varnishd/vsm.c
@@ -54,8 +54,8 @@
 
 #include "common.h"
 
-#include "vmb.h"
 #include "vapi/vsm_int.h"
+#include "vmb.h"
 #include "vtim.h"
 
 /* These two come from beyond (mgt_shmem.c actually) */
diff --git a/bin/varnishhist/varnishhist.c b/bin/varnishhist/varnishhist.c
index e2fc652..6d68a5d 100644
--- a/bin/varnishhist/varnishhist.c
+++ b/bin/varnishhist/varnishhist.c
@@ -32,10 +32,8 @@
 
 #include "config.h"
 
-#include "vapi/vsm.h"
-#include "vapi/vsl.h"
-
 #include <sys/types.h>
+
 #include <curses.h>
 #include <errno.h>
 #include <limits.h>
@@ -48,6 +46,8 @@
 #include <string.h>
 #include <unistd.h>
 
+#include "vapi/vsl.h"
+#include "vapi/vsm.h"
 #include "vas.h"
 #include "vcs.h"
 
diff --git a/bin/varnishlog/varnishlog.c b/bin/varnishlog/varnishlog.c
index 86c8504..e6c4bf8 100644
--- a/bin/varnishlog/varnishlog.c
+++ b/bin/varnishlog/varnishlog.c
@@ -31,9 +31,6 @@
 
 #include "config.h"
 
-#include "vapi/vsm.h"
-#include "vapi/vsl.h"
-
 #include <errno.h>
 #include <fcntl.h>
 #include <signal.h>
@@ -42,13 +39,15 @@
 #include <string.h>
 #include <unistd.h>
 
-#include "compat/daemon.h"
-
+#include "vapi/vsl.h"
+#include "vapi/vsm.h"
 #include "vas.h"
 #include "vcs.h"
 #include "vpf.h"
 #include "vsb.h"
 
+#include "compat/daemon.h"
+
 static int	b_flag, c_flag;
 
 /* Ordering-----------------------------------------------------------*/
diff --git a/bin/varnishncsa/base64.c b/bin/varnishncsa/base64.c
index 3758604..ee1a87b 100644
--- a/bin/varnishncsa/base64.c
+++ b/bin/varnishncsa/base64.c
@@ -6,11 +6,13 @@
 
 #include "config.h"
 
-#include "vapi/vsm.h"
-#include "vapi/vsl.h"
-
 #include <sys/types.h>
+
+#include <stdio.h>		// for test-prog
+
 #include "base64.h"
+#include "vapi/vsl.h"
+#include "vapi/vsm.h"
 
 static const char b64[] =
     "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
@@ -62,7 +64,6 @@ VB64_decode(char *d, unsigned dlen, const char *s)
 }
 
 #ifdef TEST_DRIVER
-#include <stdio.h>
 
 const char *test1 =
 "TWFuIGlzIGRpc3Rpbmd1aXNoZWQsIG5vdCBvbmx5IGJ5IGhpcyByZWFzb24sIGJ1dCBieSB0aGlz"
diff --git a/bin/varnishncsa/varnishncsa.c b/bin/varnishncsa/varnishncsa.c
index 05b8dfe..97a3642 100644
--- a/bin/varnishncsa/varnishncsa.c
+++ b/bin/varnishncsa/varnishncsa.c
@@ -59,20 +59,19 @@
 
 #include "config.h"
 
-#include "vapi/vsm.h"
-#include "vapi/vsl.h"
-
 #include <ctype.h>
+#include <errno.h>
 #include <signal.h>
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
 #include <strings.h>
-#include <unistd.h>
 #include <time.h>
+#include <unistd.h>
 
-#include "compat/daemon.h"
-
+#include "base64.h"
+#include "vapi/vsl.h"
+#include "vapi/vsm.h"
 #include "vas.h"
 #include "vcs.h"
 #include "vpf.h"
@@ -80,7 +79,7 @@
 #include "vre.h"
 #include "vsb.h"
 
-#include "base64.h"
+#include "compat/daemon.h"
 
 static volatile sig_atomic_t reopen;
 
diff --git a/bin/varnishreplay/varnishreplay.c b/bin/varnishreplay/varnishreplay.c
index ffc403f..77758eb 100644
--- a/bin/varnishreplay/varnishreplay.c
+++ b/bin/varnishreplay/varnishreplay.c
@@ -28,9 +28,6 @@
 
 #include "config.h"
 
-#include "vapi/vsm.h"
-#include "vapi/vsl.h"
-
 #include <sys/types.h>
 #include <sys/signal.h>
 #include <sys/uio.h>
@@ -39,13 +36,15 @@
 #include <errno.h>
 #include <fcntl.h>
 #include <pthread.h>
-#include <stdarg.h>
 #include <signal.h>
+#include <stdarg.h>
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
 #include <unistd.h>
 
+#include "vapi/vsl.h"
+#include "vapi/vsm.h"
 #include "vas.h"
 #include "vcs.h"
 #include "vqueue.h"
diff --git a/bin/varnishsizes/varnishsizes.c b/bin/varnishsizes/varnishsizes.c
index d3e3218..a78be9b 100644
--- a/bin/varnishsizes/varnishsizes.c
+++ b/bin/varnishsizes/varnishsizes.c
@@ -32,10 +32,8 @@
 
 #include "config.h"
 
-#include "vapi/vsm.h"
-#include "vapi/vsl.h"
-
 #include <sys/types.h>
+
 #include <curses.h>
 #include <errno.h>
 #include <limits.h>
@@ -48,6 +46,8 @@
 #include <string.h>
 #include <unistd.h>
 
+#include "vapi/vsl.h"
+#include "vapi/vsm.h"
 #include "vas.h"
 #include "vcs.h"
 
diff --git a/bin/varnishstat/varnishstat.c b/bin/varnishstat/varnishstat.c
index b160a26..3c6c4b5 100644
--- a/bin/varnishstat/varnishstat.c
+++ b/bin/varnishstat/varnishstat.c
@@ -32,9 +32,6 @@
 
 #include "config.h"
 
-#include "vapi/vsm.h"
-#include "vapi/vsc.h"
-
 #include <sys/time.h>
 
 #include <errno.h>
@@ -46,6 +43,7 @@
 
 #include "varnishstat.h"
 
+
 /*--------------------------------------------------------------------*/
 
 static int
diff --git a/bin/varnishstat/varnishstat.h b/bin/varnishstat/varnishstat.h
index 314c7c5..0a5d80f 100644
--- a/bin/varnishstat/varnishstat.h
+++ b/bin/varnishstat/varnishstat.h
@@ -27,7 +27,13 @@
  *
  */
 
+#include <stdint.h>
+
+#include "vapi/vsm.h"
+#include "vapi/vsc.h"
+
 #include "vas.h"
 #include "vcs.h"
 
+
 void do_curses(struct VSM_data *vd, const struct VSC_C_main *VSC_C_main, int delay);
diff --git a/bin/varnishstat/varnishstat_curses.c b/bin/varnishstat/varnishstat_curses.c
index 1e93155..9fa6b82 100644
--- a/bin/varnishstat/varnishstat_curses.c
+++ b/bin/varnishstat/varnishstat_curses.c
@@ -32,15 +32,16 @@
 
 #include "config.h"
 
-#include "vapi/vsm.h"
-#include "vapi/vsc.h"
-
 #include <sys/time.h>
 
 #ifdef HAVE_NCURSES_CURSES_H
-#include <ncurses/curses.h>
-#elif HAVE_CURSES_H
-#include <curses.h>
+#  include <ncurses/curses.h>
+#endif
+
+#ifndef HAVE_NCURSES_CURSES_H
+#  ifdef HAVE_CURSES_H
+#    include <curses.h>
+#  endif
 #endif
 #include <errno.h>
 #include <signal.h>
@@ -50,9 +51,12 @@
 #include <time.h>
 #include <unistd.h>
 
-#include "vqueue.h"
 #include "varnishstat.h"
 
+#include "vapi/vsc.h"
+#include "vapi/vsm.h"
+#include "vqueue.h"
+
 #if 0
 #define AC(x) assert((x) != ERR)
 #else
diff --git a/bin/varnishtest/vtc.c b/bin/varnishtest/vtc.c
index 5c6bbfd..b496f9e 100644
--- a/bin/varnishtest/vtc.c
+++ b/bin/varnishtest/vtc.c
@@ -28,14 +28,15 @@
 
 #include "config.h"
 
+#include <sys/types.h>
+#include <sys/wait.h>
+
 #include <ctype.h>
 #include <stdarg.h>
 #include <stdint.h>
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
-#include <sys/types.h>
-#include <sys/wait.h>
 #include <unistd.h>
 
 #include "vtc.h"
diff --git a/bin/varnishtest/vtc.h b/bin/varnishtest/vtc.h
index 1340314..6093656 100644
--- a/bin/varnishtest/vtc.h
+++ b/bin/varnishtest/vtc.h
@@ -27,6 +27,7 @@
  *
  */
 
+#include <errno.h>
 #include <limits.h>
 #include <signal.h>
 #include <pthread.h>
diff --git a/bin/varnishtest/vtc_client.c b/bin/varnishtest/vtc_client.c
index 2547d80..7eb430f 100644
--- a/bin/varnishtest/vtc_client.c
+++ b/bin/varnishtest/vtc_client.c
@@ -28,13 +28,13 @@
 
 #include "config.h"
 
+#include <sys/types.h>
+#include <sys/socket.h>
+
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
 
-#include <sys/types.h>
-#include <sys/socket.h>
-
 #include "vtc.h"
 
 #include "vss.h"
diff --git a/bin/varnishtest/vtc_http.c b/bin/varnishtest/vtc_http.c
index 889891b..698ffb0 100644
--- a/bin/varnishtest/vtc_http.c
+++ b/bin/varnishtest/vtc_http.c
@@ -28,17 +28,18 @@
 
 #include "config.h"
 
-#include <string.h>
-#include <stdio.h>
+#include <sys/types.h>
+#include <sys/socket.h>
+
 #include <poll.h>
+#include <stdio.h>
 #include <stdlib.h>
+#include <string.h>
 
-#include <sys/types.h>
-#include <sys/socket.h>
+#include "vtc.h"
 
 #include "vct.h"
 #include "vgz.h"
-#include "vtc.h"
 #include "vtcp.h"
 
 #define MAX_HDR		50
diff --git a/bin/varnishtest/vtc_log.c b/bin/varnishtest/vtc_log.c
index 753de58..d6d15c1 100644
--- a/bin/varnishtest/vtc_log.c
+++ b/bin/varnishtest/vtc_log.c
@@ -28,12 +28,13 @@
 
 #include "config.h"
 
+#include <stdarg.h>
 #include <stdio.h>
-#include <string.h>
 #include <stdlib.h>
-#include <stdarg.h>
+#include <string.h>
 
 #include "vtc.h"
+
 #include "vtim.h"
 
 static pthread_mutex_t	vtclog_mtx;
diff --git a/bin/varnishtest/vtc_main.c b/bin/varnishtest/vtc_main.c
index 3b9d81b..fb1c42e 100644
--- a/bin/varnishtest/vtc_main.c
+++ b/bin/varnishtest/vtc_main.c
@@ -28,18 +28,22 @@
 
 #include "config.h"
 
-#include <stdio.h>
-#include <string.h>
-#include <fcntl.h>
-#include <stdlib.h>
-#include <unistd.h>
 #include <sys/mman.h>
 #include <sys/stat.h>
 #include <sys/types.h>
 #include <sys/wait.h>
 
-#include "vev.h"
+#include <fcntl.h>
+#include <poll.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+
 #include "vtc.h"
+
+#include "vev.h"
+#include "vqueue.h"
 #include "vtim.h"
 
 #ifndef HAVE_SRANDOMDEV
diff --git a/bin/varnishtest/vtc_sema.c b/bin/varnishtest/vtc_sema.c
index c363b4a..48d4c68 100644
--- a/bin/varnishtest/vtc_sema.c
+++ b/bin/varnishtest/vtc_sema.c
@@ -30,8 +30,8 @@
 
 #include <stdio.h>
 #include <stdlib.h>
-#include <unistd.h>
 #include <string.h>
+#include <unistd.h>
 
 #include "vtc.h"
 
diff --git a/bin/varnishtest/vtc_server.c b/bin/varnishtest/vtc_server.c
index 5a768a1..75c4f30 100644
--- a/bin/varnishtest/vtc_server.c
+++ b/bin/varnishtest/vtc_server.c
@@ -28,16 +28,17 @@
 
 #include "config.h"
 
-#include <stdio.h>
+#include <sys/types.h>
+#include <sys/socket.h>
+
 #include <errno.h>
+#include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
 
-#include <sys/types.h>
-#include <sys/socket.h>
+#include "vtc.h"
 
 #include "vss.h"
-#include "vtc.h"
 #include "vtcp.h"
 
 struct server {
diff --git a/bin/varnishtest/vtc_varnish.c b/bin/varnishtest/vtc_varnish.c
index ba19464..87b9423 100644
--- a/bin/varnishtest/vtc_varnish.c
+++ b/bin/varnishtest/vtc_varnish.c
@@ -28,26 +28,25 @@
 
 #include "config.h"
 
-#include "vapi/vsm.h"
-#include "vapi/vsc.h"
-#include "vapi/vsl.h"
-
-#include <stdio.h>
+#include <sys/types.h>
+#include <sys/wait.h>
+#include <sys/socket.h>
 
 #include <fcntl.h>
+#include <inttypes.h>
+#include <poll.h>
+#include <stdio.h>
 #include <stdlib.h>
-#include <unistd.h>
 #include <string.h>
-#include <poll.h>
-#include <inttypes.h>
+#include <unistd.h>
 
-#include <sys/types.h>
-#include <sys/wait.h>
-#include <sys/socket.h>
+#include "vtc.h"
 
+#include "vapi/vsc.h"
+#include "vapi/vsl.h"
+#include "vapi/vsm.h"
 #include "vcli.h"
 #include "vss.h"
-#include "vtc.h"
 #include "vtcp.h"
 
 struct varnish {
diff --git a/bin/varnishtop/varnishtop.c b/bin/varnishtop/varnishtop.c
index 34278ea..07070c8 100644
--- a/bin/varnishtop/varnishtop.c
+++ b/bin/varnishtop/varnishtop.c
@@ -32,20 +32,19 @@
 
 #include "config.h"
 
-#include "vapi/vsm.h"
-#include "vapi/vsl.h"
-
 #include <ctype.h>
 #include <curses.h>
 #include <errno.h>
+#include <limits.h>
 #include <pthread.h>
 #include <signal.h>
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
 #include <unistd.h>
-#include <limits.h>
 
+#include "vapi/vsl.h"
+#include "vapi/vsm.h"
 #include "vas.h"
 #include "vcs.h"
 #include "vqueue.h"
diff --git a/include/vapi/vsc.h b/include/vapi/vsc.h
index 1101e53..bc52d65 100644
--- a/include/vapi/vsc.h
+++ b/include/vapi/vsc.h
@@ -33,6 +33,8 @@
 
 #include "vapi/vsc_int.h"
 
+struct VSM_data;
+
 /*---------------------------------------------------------------------
  * VSC level access functions
  */
diff --git a/include/vapi/vsc_int.h b/include/vapi/vsc_int.h
index 0a9cbf9..6ceb723 100644
--- a/include/vapi/vsc_int.h
+++ b/include/vapi/vsc_int.h
@@ -28,8 +28,6 @@
  *
  */
 
-#include <stdint.h>
-
 #define VSC_CLASS          "Stat"
 
 #define VSC_TYPE_MAIN		""
diff --git a/include/vapi/vsl.h b/include/vapi/vsl.h
index 5068f1c..9c9a4c9 100644
--- a/include/vapi/vsl.h
+++ b/include/vapi/vsl.h
@@ -31,10 +31,10 @@
 #ifndef VAPI_VSL_H_INCLUDED
 #define VAPI_VSL_H_INCLUDED
 
-#include <stdint.h>
-
 #include "vapi/vsl_int.h"
 
+struct VSM_data;
+
 /*---------------------------------------------------------------------
  * VSL level access functions
  */
diff --git a/include/vapi/vsm.h b/include/vapi/vsm.h
index 0b7cdd8..fd7fbee 100644
--- a/include/vapi/vsm.h
+++ b/include/vapi/vsm.h
@@ -31,6 +31,9 @@
 #ifndef VAPI_VSM_H_INCLUDED
 #define VAPI_VSM_H_INCLUDED
 
+struct VSM_head;
+struct VSM_data;
+
 /*---------------------------------------------------------------------
  * VSM level access functions
  */
diff --git a/include/vapi/vsm_int.h b/include/vapi/vsm_int.h
index 0055b73..3554982 100644
--- a/include/vapi/vsm_int.h
+++ b/include/vapi/vsm_int.h
@@ -36,8 +36,6 @@
 
 #define VSM_FILENAME		"_.vsm"
 
-#include <stdint.h>
-
 /*
  * This structure describes each allocation from the shmlog
  */
@@ -81,6 +79,9 @@ struct VSM_head {
  */
 #ifdef CHECK_OBJ_NOTNULL
 
+extern struct VSM_head *VSM_head;
+extern const struct VSM_chunk *vsm_end;
+
 static inline struct VSM_chunk *
 vsm_iter_0(void)
 {
diff --git a/include/vas.h b/include/vas.h
index a4c9c6f..ff20036 100644
--- a/include/vas.h
+++ b/include/vas.h
@@ -38,8 +38,6 @@
 #ifndef VAS_H_INCLUDED
 #define VAS_H_INCLUDED
 
-#include <errno.h>
-
 typedef void vas_f(const char *, const char *, int, const char *, int, int);
 
 extern vas_f *VAS_Fail;
diff --git a/include/vct.h b/include/vct.h
index fb4c2af..52530a6 100644
--- a/include/vct.h
+++ b/include/vct.h
@@ -28,8 +28,6 @@
  *
  */
 
-#include <stdint.h>
-
 /* from libvarnish/vct.c */
 
 #define VCT_SP			(1<<0)
diff --git a/include/vev.h b/include/vev.h
index 60737bf..37db60f 100644
--- a/include/vev.h
+++ b/include/vev.h
@@ -28,10 +28,6 @@
  *
  */
 
-#include <poll.h>
-
-#include "vqueue.h"
-
 struct vev;
 struct vev_base;
 
diff --git a/include/vnum.h b/include/vnum.h
index a821355..cf55c9a 100644
--- a/include/vnum.h
+++ b/include/vnum.h
@@ -28,7 +28,5 @@
  *
  */
 
-#include <stdint.h>
-
 /* from libvarnish/vnum.c */
 const char *VNUM_2bytes(const char *p, uintmax_t *r, uintmax_t rel);
diff --git a/include/vsb.h b/include/vsb.h
index 5bfbce7..0a8e2bf 100644
--- a/include/vsb.h
+++ b/include/vsb.h
@@ -31,8 +31,6 @@
 #ifndef VSB_H_INCLUDED
 #define VSB_H_INCLUDED
 
-#include <sys/types.h>
-
 /*
  * Structure definition
  */
diff --git a/include/vsha256.h b/include/vsha256.h
index 66811d3..bf42c11 100644
--- a/include/vsha256.h
+++ b/include/vsha256.h
@@ -29,8 +29,6 @@
 #ifndef _SHA256_H_
 #define _SHA256_H_
 
-#include <stdint.h>
-
 #define SHA256_LEN		32
 
 typedef struct SHA256Context {
diff --git a/lib/libvarnish/binary_heap.c b/lib/libvarnish/binary_heap.c
index 4b0edc3..ed99f13 100644
--- a/lib/libvarnish/binary_heap.c
+++ b/lib/libvarnish/binary_heap.c
@@ -35,12 +35,15 @@
 
 #include "config.h"
 
+#include <errno.h>
 #include <limits.h>
 #include <stdint.h>
+#include <stdio.h>		// for testcase
 #include <stdlib.h>
 #include <unistd.h>
 
 #include "binary_heap.h"
+#include "miniobj.h"		// for testcase
 #include "vas.h"
 
 /* Parameters --------------------------------------------------------*/
@@ -455,8 +458,6 @@ binheap_reorder(const struct binheap *bh, unsigned idx)
 
 #ifdef TEST_DRIVER
 /* Test driver -------------------------------------------------------*/
-#include <stdio.h>
-#include "miniobj.h"
 
 static void
 vasfail(const char *func, const char *file, int line,
diff --git a/lib/libvarnish/cli_auth.c b/lib/libvarnish/cli_auth.c
index da0a241..396a127 100644
--- a/lib/libvarnish/cli_auth.c
+++ b/lib/libvarnish/cli_auth.c
@@ -29,6 +29,8 @@
 #include "config.h"
 
 #include <sys/types.h>
+
+#include <errno.h>
 #include <stdio.h>
 #include <unistd.h>
 
diff --git a/lib/libvarnish/cli_common.c b/lib/libvarnish/cli_common.c
index e423da4..cb4e8af 100644
--- a/lib/libvarnish/cli_common.c
+++ b/lib/libvarnish/cli_common.c
@@ -43,13 +43,11 @@
 #include <time.h>
 #include <unistd.h>
 
-#include "vsb.h"
-
-#include "vas.h"
-
 #include "vcli.h"
-#include "cli_priv.h"
 #include "cli_common.h"
+#include "cli_priv.h"
+#include "vas.h"
+#include "vsb.h"
 
 /*lint -e{818} cli could be const */
 void
diff --git a/lib/libvarnish/cli_serve.c b/lib/libvarnish/cli_serve.c
index 13b9663..2c3a09c 100644
--- a/lib/libvarnish/cli_serve.c
+++ b/lib/libvarnish/cli_serve.c
@@ -32,23 +32,25 @@
 #include "config.h"
 
 #include <ctype.h>
+#include <errno.h>
 #include <poll.h>
-#include <string.h>
-#include <stdlib.h>
 #include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
 #include <time.h>
 #include <unistd.h>
 
+#include "miniobj.h"
 #include "vas.h"
-#include "vqueue.h"
-#include "vsb.h"
-#include "vav.h"
-#include "vlu.h"
+
 #include "vcli.h"
-#include "cli_priv.h"
 #include "cli_common.h"
+#include "cli_priv.h"
 #include "cli_serve.h"
-#include "miniobj.h"
+#include "vav.h"
+#include "vlu.h"
+#include "vqueue.h"
+#include "vsb.h"
 
 struct VCLS_func {
 	unsigned			magic;
diff --git a/lib/libvarnish/vav.c b/lib/libvarnish/vav.c
index a0c57bb..24b16b7 100644
--- a/lib/libvarnish/vav.c
+++ b/lib/libvarnish/vav.c
@@ -39,8 +39,9 @@
 #include "config.h"
 
 #include <ctype.h>
-#include <stdlib.h>
+#include <errno.h>
 #include <stdio.h>
+#include <stdlib.h>
 #include <string.h>
 
 #include "vas.h"
diff --git a/lib/libvarnish/vct.c b/lib/libvarnish/vct.c
index 2740aa1..f0e1635 100644
--- a/lib/libvarnish/vct.c
+++ b/lib/libvarnish/vct.c
@@ -31,6 +31,7 @@
 #include "config.h"
 
 #include <stdint.h>
+
 #include "vct.h"
 
 /* NB: VCT always operate in ASCII, don't replace 0x0d with \r etc. */
diff --git a/lib/libvarnish/vev.c b/lib/libvarnish/vev.c
index 004ce08..87d1582 100644
--- a/lib/libvarnish/vev.c
+++ b/lib/libvarnish/vev.c
@@ -29,20 +29,22 @@
 
 #include "config.h"
 
-#include <stdio.h>
 #include <errno.h>
 #include <poll.h>
-#include <time.h>
+#include <pthread.h>
 #include <signal.h>
-#include <string.h>
+#include <stdio.h>
 #include <stdlib.h>
-#include <pthread.h>
+#include <string.h>
+#include <time.h>
 
-#include "vas.h"
-#include "vtim.h"
-#include "vev.h"
 #include "miniobj.h"
+#include "vas.h"
+
 #include "binary_heap.h"
+#include "vqueue.h"
+#include "vev.h"
+#include "vtim.h"
 
 #undef DEBUG_EVENTS
 
diff --git a/lib/libvarnish/vfil.c b/lib/libvarnish/vfil.c
index a4bab9e..ca1d965 100644
--- a/lib/libvarnish/vfil.c
+++ b/lib/libvarnish/vfil.c
@@ -29,15 +29,15 @@
 
 #include "config.h"
 
+#include <sys/stat.h>
+
 #include <errno.h>
 #include <fcntl.h>
 #include <limits.h>
 #include <stdio.h>
 #include <stdlib.h>
-#include <unistd.h>
 #include <time.h>
-
-#include <sys/stat.h>
+#include <unistd.h>
 
 #include "vas.h"
 #include "vdef.h"
diff --git a/lib/libvarnish/vin.c b/lib/libvarnish/vin.c
index cd3e7d0..4dba302 100644
--- a/lib/libvarnish/vin.c
+++ b/lib/libvarnish/vin.c
@@ -30,14 +30,14 @@
 
 #include "config.h"
 
-#include "vapi/vsm_int.h"
-
 #include <errno.h>
 #include <limits.h>
+#include <stdint.h>
 #include <stdio.h>
 #include <string.h>
 #include <unistd.h>
 
+#include "vapi/vsm_int.h"
 #include "vas.h"
 #include "vdef.h"
 #include "vin.h"
diff --git a/lib/libvarnish/vlu.c b/lib/libvarnish/vlu.c
index 1334a28..f149a57 100644
--- a/lib/libvarnish/vlu.c
+++ b/lib/libvarnish/vlu.c
@@ -29,13 +29,16 @@
 
 #include "config.h"
 
+#include <errno.h>
 #include <stdio.h>
-#include <string.h>
 #include <stdlib.h>
+#include <string.h>
 #include <unistd.h>
+
+#include "miniobj.h"
 #include "vas.h"
+
 #include "vlu.h"
-#include "miniobj.h"
 
 struct vlu {
 	unsigned	magic;
diff --git a/lib/libvarnish/vmb.c b/lib/libvarnish/vmb.c
index e6b0f25..3f5c12d 100644
--- a/lib/libvarnish/vmb.c
+++ b/lib/libvarnish/vmb.c
@@ -26,8 +26,11 @@
  * SUCH DAMAGE.
  */
 
+#include "config.h"
+
 #include <pthread.h>
 #include <unistd.h>
+
 #include "vas.h"
 #include "vmb.h"
 
diff --git a/lib/libvarnish/vnum.c b/lib/libvarnish/vnum.c
index 7940e9b..d2f423b 100644
--- a/lib/libvarnish/vnum.c
+++ b/lib/libvarnish/vnum.c
@@ -31,10 +31,11 @@
 #include "config.h"
 
 #include <math.h>
+#include <stdint.h>
 #include <stdio.h>
 #include <stdlib.h>
-#include <stdint.h>
 
+#include "vas.h"
 #include "vnum.h"
 
 static const char err_miss_num[] = "Missing number";
@@ -112,10 +113,6 @@ VNUM_2bytes(const char *p, uintmax_t *r, uintmax_t rel)
 
 #ifdef NUM_C_TEST
 /* Compile with: "cc -o foo -DNUM_C_TEST -I../.. -I../../include num.c -lm" */
-#include "vas.h"
-#include <math.h>
-#include <stdio.h>
-#include <string.h>
 
 struct test_case {
 	const char *str;
diff --git a/lib/libvarnish/vre.c b/lib/libvarnish/vre.c
index 2810bc2..9b1d911 100644
--- a/lib/libvarnish/vre.c
+++ b/lib/libvarnish/vre.c
@@ -26,11 +26,15 @@
  * SUCH DAMAGE.
  */
 
+#include "config.h"
+
+#include <errno.h>
 #include <pcre.h>
 #include <string.h>
 
 #include "miniobj.h"
 #include "vas.h"
+
 #include "vre.h"
 
 struct vre {
diff --git a/lib/libvarnish/vsb.c b/lib/libvarnish/vsb.c
index 4930be2..16f6d4b 100644
--- a/lib/libvarnish/vsb.c
+++ b/lib/libvarnish/vsb.c
@@ -30,6 +30,7 @@ __FBSDID("$FreeBSD: head/sys/kern/subr_vsb.c 222004 2011-05-17 06:36:32Z phk $")
 #include "config.h"
 
 #include <ctype.h>
+#include <errno.h>
 #include <stdarg.h>
 #include <stdio.h>
 #include <stdlib.h>
diff --git a/lib/libvarnish/vsha256.c b/lib/libvarnish/vsha256.c
index 942a186..10b7a6a 100644
--- a/lib/libvarnish/vsha256.c
+++ b/lib/libvarnish/vsha256.c
@@ -28,23 +28,24 @@
 
 #include "config.h"
 
-#include <stdint.h>
-#include <string.h>
+#ifdef HAVE_SYS_ENDIAN_H
+#include <sys/endian.h>
+#define VBYTE_ORDER	_BYTE_ORDER
+#define VBIG_ENDIAN	_BIG_ENDIAN
+#endif
 
 #ifdef HAVE_ENDIAN_H
 #include <endian.h>
 #define VBYTE_ORDER	__BYTE_ORDER
 #define VBIG_ENDIAN	__BIG_ENDIAN
 #endif
-#ifdef HAVE_SYS_ENDIAN_H
-#include <sys/endian.h>
-#define VBYTE_ORDER	_BYTE_ORDER
-#define VBIG_ENDIAN	_BIG_ENDIAN
-#endif
+#include <errno.h>
+#include <stdint.h>
+#include <string.h>
 
 #include "vas.h"
-#include "vsha256.h"
 #include "vend.h"
+#include "vsha256.h"
 
 #if defined(VBYTE_ORDER) && VBYTE_ORDER == VBIG_ENDIAN
 
diff --git a/lib/libvarnish/vss.c b/lib/libvarnish/vss.c
index 297df11..77b8f38 100644
--- a/lib/libvarnish/vss.c
+++ b/lib/libvarnish/vss.c
@@ -44,8 +44,8 @@
 #include <unistd.h>
 
 #include "vas.h"
-#include "vtcp.h"
 #include "vss.h"
+#include "vtcp.h"
 
 /* lightweight addrinfo */
 struct vss_addr {
diff --git a/lib/libvarnish/vsub.c b/lib/libvarnish/vsub.c
index 90ceacb..497119c 100644
--- a/lib/libvarnish/vsub.c
+++ b/lib/libvarnish/vsub.c
@@ -31,12 +31,13 @@
 
 #include "config.h"
 
+#include <sys/wait.h>
+
+#include <errno.h>
 #include <fcntl.h>
+#include <stdio.h>
 #include <string.h>
 #include <unistd.h>
-#include <stdio.h>
-
-#include <sys/wait.h>
 
 #include "vas.h"
 #include "vlu.h"
diff --git a/lib/libvarnish/vtcp.c b/lib/libvarnish/vtcp.c
index c955e65..10c3664 100644
--- a/lib/libvarnish/vtcp.c
+++ b/lib/libvarnish/vtcp.c
@@ -31,25 +31,24 @@
 
 #include <sys/types.h>
 #include <sys/socket.h>
+#include <sys/ioctl.h>
+#ifdef HAVE_SYS_FILIO_H
+#  include <sys/filio.h>
+#endif
 
 #include <netinet/in.h>
-
 #ifdef __linux
-#include <netinet/tcp.h>
+#  include <netinet/tcp.h>
 #endif
 
 #include <errno.h>
-#include <sys/ioctl.h>
-#ifdef HAVE_SYS_FILIO_H
-#include <sys/filio.h>
-#endif
+#include <math.h>
 #include <netdb.h>
 #include <poll.h>
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
 #include <unistd.h>
-#include <math.h>
 
 #include "vas.h"
 #include "vtcp.h"
diff --git a/lib/libvarnish/vtim.c b/lib/libvarnish/vtim.c
index 772b559..42c1558 100644
--- a/lib/libvarnish/vtim.c
+++ b/lib/libvarnish/vtim.c
@@ -48,11 +48,13 @@
 
 #include <sys/time.h>
 
+#include <errno.h>
+#include <math.h>
 #include <stdio.h>
+#include <stdlib.h>
 #include <string.h>
 #include <time.h>
 #include <unistd.h>
-#include <math.h>
 
 #include "vas.h"
 #include "vtim.h"
@@ -200,8 +202,6 @@ VTIM_timespec(double t)
 
 #ifdef TEST_DRIVER
 
-#include <stdlib.h>
-
 /*
  * Compile with:
  *  cc -o foo -DTEST_DRIVER -I../.. -I../../include time.c assert.c
diff --git a/lib/libvarnishapi/vsc.c b/lib/libvarnishapi/vsc.c
index 211d284..eea7f5c 100644
--- a/lib/libvarnishapi/vsc.c
+++ b/lib/libvarnishapi/vsc.c
@@ -29,20 +29,22 @@
 
 #include "config.h"
 
-#include "vapi/vsm.h"
-#include "vapi/vsm_int.h"
-#include "vapi/vsc.h"
-
 #include <sys/types.h>
 #include <sys/stat.h>
-#include <string.h>
+
+#include <errno.h>
+#include <stdint.h>
 #include <stdlib.h>
+#include <string.h>
 
+#include "miniobj.h"
 #include "vas.h"
+
+#include "vapi/vsc.h"
+#include "vapi/vsm.h"
+#include "vapi/vsm_int.h"
 #include "vav.h"
 #include "vqueue.h"
-#include "miniobj.h"
-
 #include "vsm_api.h"
 
 struct vsc_sf {
diff --git a/lib/libvarnishapi/vsl.c b/lib/libvarnishapi/vsl.c
index 9fa6a48..88bd9d6 100644
--- a/lib/libvarnishapi/vsl.c
+++ b/lib/libvarnishapi/vsl.c
@@ -29,26 +29,27 @@
 
 #include "config.h"
 
-#include "vapi/vsm.h"
-#include "vapi/vsm_int.h"
-#include "vapi/vsl.h"
-
-#include <sys/types.h>
 #include <sys/stat.h>
+#include <sys/types.h>
 
+#include <errno.h>
+#include <stdint.h>
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
 #include <unistd.h>
 
-#include "vas.h"
-#include "vre.h"
-#include "vbm.h"
 #include "miniobj.h"
+#include "vas.h"
 
-#include "vsm_api.h"
-#include "vsl_api.h"
+#include "vapi/vsl.h"
+#include "vapi/vsm.h"
+#include "vapi/vsm_int.h"
+#include "vbm.h"
 #include "vmb.h"
+#include "vre.h"
+#include "vsl_api.h"
+#include "vsm_api.h"
 
 /*--------------------------------------------------------------------*/
 
diff --git a/lib/libvarnishapi/vsl_arg.c b/lib/libvarnishapi/vsl_arg.c
index f41129b..56f93e2 100644
--- a/lib/libvarnishapi/vsl_arg.c
+++ b/lib/libvarnishapi/vsl_arg.c
@@ -29,27 +29,27 @@
 
 #include "config.h"
 
-#include "vapi/vsm.h"
-#include "vapi/vsl.h"
-
 #include <sys/types.h>
 #include <sys/stat.h>
 
 #include <ctype.h>
 #include <errno.h>
 #include <fcntl.h>
+#include <stdint.h>
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
 #include <unistd.h>
 
-#include "vas.h"
-#include "vre.h"
-#include "vbm.h"
 #include "miniobj.h"
+#include "vas.h"
 
-#include "vsm_api.h"
+#include "vapi/vsl.h"
+#include "vapi/vsm.h"
+#include "vbm.h"
+#include "vre.h"
 #include "vsl_api.h"
+#include "vsm_api.h"
 
 /*--------------------------------------------------------------------
  * Look up a tag
diff --git a/lib/libvarnishapi/vsm.c b/lib/libvarnishapi/vsm.c
index eeb2b3c..8e8b27b 100644
--- a/lib/libvarnishapi/vsm.c
+++ b/lib/libvarnishapi/vsm.c
@@ -29,26 +29,25 @@
 
 #include "config.h"
 
-#include "vapi/vsm.h"
-#include "vapi/vsm_int.h"
-
-#include <stdint.h>
-#include <sys/types.h>
 #include <sys/mman.h>
 #include <sys/stat.h>
+#include <sys/types.h>
 
 #include <errno.h>
 #include <fcntl.h>
+#include <stdint.h>
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
 #include <unistd.h>
 
-#include "vas.h"
-#include "vin.h"
-#include "vbm.h"
 #include "miniobj.h"
+#include "vas.h"
 
+#include "vapi/vsm.h"
+#include "vapi/vsm_int.h"
+#include "vbm.h"
+#include "vin.h"
 #include "vsm_api.h"
 
 #ifndef MAP_HASSEMAPHORE
diff --git a/lib/libvarnishcompat/daemon.c b/lib/libvarnishcompat/daemon.c
index ef14e23..6b3fbb1 100644
--- a/lib/libvarnishcompat/daemon.c
+++ b/lib/libvarnishcompat/daemon.c
@@ -35,8 +35,8 @@
 
 #include <errno.h>
 #include <fcntl.h>
-#include <stdlib.h>
 #include <signal.h>
+#include <stdlib.h>
 #include <unistd.h>
 
 #include "compat/daemon.h"
diff --git a/lib/libvcl/generate.py b/lib/libvcl/generate.py
index 9342aaf..dde0e89 100755
--- a/lib/libvcl/generate.py
+++ b/lib/libvcl/generate.py
@@ -805,7 +805,9 @@ file_header(fo)
 
 fo.write("""
 #include "config.h"
+
 #include <stdio.h>
+
 #include "vcc_compile.h"
 
 const struct var vcc_vars[] = {
@@ -863,9 +865,10 @@ file_header(fo)
 fo.write("""
 
 #include "config.h"
-#include <stdio.h>
+
 #include <ctype.h>
-#include "config.h"
+#include <stdio.h>
+
 #include "vcc_compile.h"
 """)
 
diff --git a/lib/libvcl/vcc_acl.c b/lib/libvcl/vcc_acl.c
index 985a18e..c962044 100644
--- a/lib/libvcl/vcc_acl.c
+++ b/lib/libvcl/vcc_acl.c
@@ -29,16 +29,18 @@
 
 #include "config.h"
 
-#include <stdio.h>
-#include <stddef.h>
-#include <string.h>
 #include <sys/types.h>
 #include <sys/socket.h>
-#include <netdb.h>
+
 #include <netinet/in.h>
 
+#include <netdb.h>
+#include <stddef.h>
+#include <stdio.h>
+#include <string.h>
 
 #include "vcc_compile.h"
+
 #include "vrt.h"
 
 struct acl_e {
diff --git a/lib/libvcl/vcc_action.c b/lib/libvcl/vcc_action.c
index 61bfb87..9d72b51 100644
--- a/lib/libvcl/vcc_action.c
+++ b/lib/libvcl/vcc_action.c
@@ -32,7 +32,6 @@
 
 #include "config.h"
 
-
 #include "vcc_compile.h"
 
 /*--------------------------------------------------------------------*/
diff --git a/lib/libvcl/vcc_backend.c b/lib/libvcl/vcc_backend.c
index 0c72c1c..03482c4 100644
--- a/lib/libvcl/vcc_backend.c
+++ b/lib/libvcl/vcc_backend.c
@@ -59,6 +59,7 @@
 #include <string.h>
 
 #include "vcc_compile.h"
+
 #include "vss.h"
 
 struct host {
@@ -99,8 +100,6 @@ emit_sockaddr(struct vcc *tl, void *sa, unsigned sal)
  * and put it in an official sockaddr when we load the VCL.
  */
 
-#include <stdio.h>
-
 void
 Emit_Sockaddr(struct vcc *tl, const struct token *t_host, const char *port)
 {
diff --git a/lib/libvcl/vcc_backend_util.c b/lib/libvcl/vcc_backend_util.c
index 28510f5..efba623 100644
--- a/lib/libvcl/vcc_backend_util.c
+++ b/lib/libvcl/vcc_backend_util.c
@@ -30,8 +30,8 @@
 
 #include "config.h"
 
-#include <stdio.h>
 #include <stdarg.h>
+#include <stdio.h>
 #include <string.h>
 
 #include "vcc_compile.h"
diff --git a/lib/libvcl/vcc_compile.c b/lib/libvcl/vcc_compile.c
index dbae2ab..a0c6232 100644
--- a/lib/libvcl/vcc_compile.c
+++ b/lib/libvcl/vcc_compile.c
@@ -54,12 +54,13 @@
 
 #include <ctype.h>
 #include <errno.h>
-#include <stdio.h>
 #include <stdarg.h>
+#include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
 
 #include "vcc_compile.h"
+
 #include "libvcl.h"
 #include "vfil.h"
 
diff --git a/lib/libvcl/vcc_compile.h b/lib/libvcl/vcc_compile.h
index a245a36..c59c632 100644
--- a/lib/libvcl/vcc_compile.h
+++ b/lib/libvcl/vcc_compile.h
@@ -28,6 +28,10 @@
  *
  */
 
+#include <sys/types.h>
+
+#include <errno.h>
+#include <stdio.h>
 #include <stdint.h>
 
 #include "miniobj.h"
@@ -37,6 +41,7 @@
 #include "vqueue.h"
 #include "vsb.h"
 
+
 #include "vcc_token_defs.h"
 
 #ifndef NULL
diff --git a/lib/libvcl/vcc_dir_dns.c b/lib/libvcl/vcc_dir_dns.c
index 2e3951e..a91c021 100644
--- a/lib/libvcl/vcc_dir_dns.c
+++ b/lib/libvcl/vcc_dir_dns.c
@@ -28,9 +28,9 @@
 
 #include "config.h"
 
+#include <limits.h>
 #include <stdio.h>
 #include <string.h>
-#include <limits.h>
 
 #include "vcc_compile.h"
 
diff --git a/lib/libvcl/vcc_expr.c b/lib/libvcl/vcc_expr.c
index dc7d99d..3dfeeaa 100644
--- a/lib/libvcl/vcc_expr.c
+++ b/lib/libvcl/vcc_expr.c
@@ -31,11 +31,11 @@
 
 #include "config.h"
 
+#include <math.h>
 #include <stdarg.h>
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
-#include <math.h>
 
 #include "vcc_compile.h"
 
diff --git a/lib/libvcl/vcc_string.c b/lib/libvcl/vcc_string.c
index dfc82cf..4e4762e 100644
--- a/lib/libvcl/vcc_string.c
+++ b/lib/libvcl/vcc_string.c
@@ -34,8 +34,8 @@
 
 #include "vcc_compile.h"
 
-#include "vrt.h"
 #include "vre.h"
+#include "vrt.h"
 
 /*--------------------------------------------------------------------*/
 
diff --git a/lib/libvcl/vcc_vmod.c b/lib/libvcl/vcc_vmod.c
index 40c9ed5..55b43d7 100644
--- a/lib/libvcl/vcc_vmod.c
+++ b/lib/libvcl/vcc_vmod.c
@@ -28,11 +28,12 @@
 
 #include "config.h"
 
-#include <stdio.h>
 #include <dlfcn.h>
+#include <stdio.h>
 #include <string.h>
 
 #include "vcc_compile.h"
+
 #include "vmod_abi.h"
 
 void
diff --git a/lib/libvmod_std/vmod.py b/lib/libvmod_std/vmod.py
index 1b0f1c0..6a9b4fc 100755
--- a/lib/libvmod_std/vmod.py
+++ b/lib/libvmod_std/vmod.py
@@ -278,6 +278,8 @@ fh.write("\n");
 fh.write(plist)
 
 
+fc.write('#include "config.h"\n')
+fc.write('\n')
 fc.write('#include "vrt.h"\n')
 fc.write('#include "vcc_if.h"\n')
 fc.write('#include "vmod_abi.h"\n')
diff --git a/lib/libvmod_std/vmod_std.c b/lib/libvmod_std/vmod_std.c
index 847c705..79cbf02 100644
--- a/lib/libvmod_std/vmod_std.c
+++ b/lib/libvmod_std/vmod_std.c
@@ -26,13 +26,20 @@
  * SUCH DAMAGE.
  */
 
+#include "config.h"
+
+#include <netinet/in.h>
+
 #include <ctype.h>
+#include <errno.h>
 #include <stdarg.h>
+#include <stdint.h>
 #include <stdlib.h>
 #include <syslog.h>
-#include <netinet/in.h>
+
 #include "vrt.h"
 #include "vtcp.h"
+
 #include "../../bin/varnishd/cache.h"
 
 #include "vcc_if.h"
diff --git a/lib/libvmod_std/vmod_std_conversions.c b/lib/libvmod_std/vmod_std_conversions.c
index 6f06c2c..156cf13 100644
--- a/lib/libvmod_std/vmod_std_conversions.c
+++ b/lib/libvmod_std/vmod_std_conversions.c
@@ -27,8 +27,12 @@
  *
  */
 
+#include "config.h"
+
 #include <ctype.h>
+#include <errno.h>
 #include <math.h>
+#include <stdint.h>
 #include <stdlib.h>
 
 #include "../../bin/varnishd/cache.h"
diff --git a/lib/libvmod_std/vmod_std_fileread.c b/lib/libvmod_std/vmod_std_fileread.c
index cae57a2..8476bd6 100644
--- a/lib/libvmod_std/vmod_std_fileread.c
+++ b/lib/libvmod_std/vmod_std_fileread.c
@@ -37,13 +37,19 @@
  * XXX: underlying file has been updated.
  */
 
+#include "config.h"
+
+#include <errno.h>
+#include <stdint.h>
 #include <stdlib.h>
-#include "vrt.h"
+
 #include "../../bin/varnishd/cache.h"
 
-#include "vcc_if.h"
+#include "vrt.h"
 #include "vfil.h"
 
+#include "vcc_if.h"
+
 struct frfile {
 	unsigned			magic;
 #define CACHED_FILE_MAGIC 0xa8e9d87a
diff --git a/man/vsc2rst.c b/man/vsc2rst.c
index ad5dec6..bf4091f 100644
--- a/man/vsc2rst.c
+++ b/man/vsc2rst.c
@@ -1,3 +1,5 @@
+/* XXX: Copyright ?? */
+#include "config.h"
 
 #include <stdio.h>
 



From geoff at varnish-cache.org  Mon Jan  9 20:52:17 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:17 +0100
Subject: [experimental-ims] 8aba928 Add more missing <stdint.h>'s
Message-ID: <E1RkMCX-0007dk-6o@project.varnish-software.com>

commit 8aba928cfd4291a455f9b501b2871cf62d29ccdf
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Oct 10 10:24:16 2011 +0000

    Add more missing <stdint.h>'s

diff --git a/bin/varnishhist/varnishhist.c b/bin/varnishhist/varnishhist.c
index 6d68a5d..72bc84b 100644
--- a/bin/varnishhist/varnishhist.c
+++ b/bin/varnishhist/varnishhist.c
@@ -41,7 +41,7 @@
 #include <pthread.h>
 #include <regex.h>
 #include <signal.h>
-#include <stdio.h>
+#include <stdint.h>
 #include <stdlib.h>
 #include <string.h>
 #include <unistd.h>
diff --git a/bin/varnishlog/varnishlog.c b/bin/varnishlog/varnishlog.c
index e6c4bf8..e15d102 100644
--- a/bin/varnishlog/varnishlog.c
+++ b/bin/varnishlog/varnishlog.c
@@ -34,6 +34,7 @@
 #include <errno.h>
 #include <fcntl.h>
 #include <signal.h>
+#include <stdint.h>
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
diff --git a/bin/varnishncsa/base64.c b/bin/varnishncsa/base64.c
index ee1a87b..a631098 100644
--- a/bin/varnishncsa/base64.c
+++ b/bin/varnishncsa/base64.c
@@ -11,8 +11,6 @@
 #include <stdio.h>		// for test-prog
 
 #include "base64.h"
-#include "vapi/vsl.h"
-#include "vapi/vsm.h"
 
 static const char b64[] =
     "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
diff --git a/bin/varnishncsa/varnishncsa.c b/bin/varnishncsa/varnishncsa.c
index 97a3642..26ede48 100644
--- a/bin/varnishncsa/varnishncsa.c
+++ b/bin/varnishncsa/varnishncsa.c
@@ -62,6 +62,7 @@
 #include <ctype.h>
 #include <errno.h>
 #include <signal.h>
+#include <stdint.h>
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
diff --git a/bin/varnishreplay/varnishreplay.c b/bin/varnishreplay/varnishreplay.c
index 77758eb..12b6c41 100644
--- a/bin/varnishreplay/varnishreplay.c
+++ b/bin/varnishreplay/varnishreplay.c
@@ -38,6 +38,7 @@
 #include <pthread.h>
 #include <signal.h>
 #include <stdarg.h>
+#include <stdint.h>
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
diff --git a/bin/varnishsizes/varnishsizes.c b/bin/varnishsizes/varnishsizes.c
index a78be9b..ce68a9d 100644
--- a/bin/varnishsizes/varnishsizes.c
+++ b/bin/varnishsizes/varnishsizes.c
@@ -41,6 +41,7 @@
 #include <pthread.h>
 #include <regex.h>
 #include <signal.h>
+#include <stdint.h>
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
diff --git a/bin/varnishtest/vtc.h b/bin/varnishtest/vtc.h
index 6093656..a1b6cbd 100644
--- a/bin/varnishtest/vtc.h
+++ b/bin/varnishtest/vtc.h
@@ -30,6 +30,7 @@
 #include <errno.h>
 #include <limits.h>
 #include <signal.h>
+#include <stdint.h>
 #include <pthread.h>
 #ifdef HAVE_PTHREAD_NP_H
 #include <pthread_np.h>
diff --git a/bin/varnishtop/varnishtop.c b/bin/varnishtop/varnishtop.c
index 07070c8..efe312e 100644
--- a/bin/varnishtop/varnishtop.c
+++ b/bin/varnishtop/varnishtop.c
@@ -38,6 +38,7 @@
 #include <limits.h>
 #include <pthread.h>
 #include <signal.h>
+#include <stdint.h>
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>



From geoff at varnish-cache.org  Mon Jan  9 20:52:17 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:17 +0100
Subject: [experimental-ims] 44a623a Rename cli_serve.h to vcli_serve.h
Message-ID: <E1RkMCX-0007e4-AS@project.varnish-software.com>

commit 44a623a6c77bdbec1a0b6154af3c2665bda6db0a
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Oct 10 11:14:25 2011 +0000

    Rename cli_serve.h to vcli_serve.h

diff --git a/bin/varnishadm/varnishadm.c b/bin/varnishadm/varnishadm.c
index e519b01..818b9ab 100644
--- a/bin/varnishadm/varnishadm.c
+++ b/bin/varnishadm/varnishadm.c
@@ -44,11 +44,10 @@
 #include <string.h>
 #include <unistd.h>
 
-#include "vcli.h"
-#include "cli_common.h"
 #include "vapi/vsl.h"
 #include "vapi/vsm.h"
 #include "vas.h"
+#include "vcli.h"
 #include "vss.h"
 
 #ifdef HAVE_LIBEDIT
diff --git a/bin/varnishd/cache_cli.c b/bin/varnishd/cache_cli.c
index f66075e..a278e84 100644
--- a/bin/varnishd/cache_cli.c
+++ b/bin/varnishd/cache_cli.c
@@ -43,8 +43,8 @@
 #include "vcli.h"
 #include "cli_common.h"
 #include "cli_priv.h"
-#include "cli_serve.h"
 #include "hash_slinger.h"		// objhead
+#include "vcli_serve.h"
 
 pthread_t		cli_thread;
 static struct lock	cli_mtx;
diff --git a/bin/varnishd/mgt_cli.c b/bin/varnishd/mgt_cli.c
index 4a7925a..f1bb2d8 100644
--- a/bin/varnishd/mgt_cli.c
+++ b/bin/varnishd/mgt_cli.c
@@ -47,9 +47,9 @@
 #include "vcli.h"
 #include "cli_common.h"
 #include "cli_priv.h"
-#include "cli_serve.h"
 #include "heritage.h"
 #include "mgt_cli.h"
+#include "vcli_serve.h"
 #include "vev.h"
 #include "vlu.h"
 #include "vss.h"
diff --git a/bin/varnishd/storage_persistent.c b/bin/varnishd/storage_persistent.c
index 18b96c3..d3ab02e 100644
--- a/bin/varnishd/storage_persistent.c
+++ b/bin/varnishd/storage_persistent.c
@@ -47,12 +47,12 @@
 
 #include "cli_priv.h"
 #include "hash_slinger.h"
-#include "persistent.h"
 #include "stevedore.h"
 #include "vcli.h"
 #include "vend.h"
 #include "vsha256.h"
 
+#include "persistent.h"
 #include "storage_persistent.h"
 
 /*--------------------------------------------------------------------*/
diff --git a/include/Makefile.am b/include/Makefile.am
index 935a6f9..9c92b1e 100644
--- a/include/Makefile.am
+++ b/include/Makefile.am
@@ -28,7 +28,6 @@ nobase_noinst_HEADERS = \
 	binary_heap.h \
 	cli_common.h \
 	cli_priv.h \
-	cli_serve.h \
 	compat/daemon.h \
 	compat/execinfo.h \
 	compat/srandomdev.h \
@@ -40,6 +39,7 @@ nobase_noinst_HEADERS = \
 	vav.h \
 	vbm.h \
 	vcl.h \
+	vcli_serve.h \
 	vcs_version.h \
 	vcs.h \
 	vct.h \
diff --git a/include/cli_serve.h b/include/cli_serve.h
deleted file mode 100644
index 0ac29f8..0000000
--- a/include/cli_serve.h
+++ /dev/null
@@ -1,45 +0,0 @@
-/*-
- * Copyright (c) 2010-2011 Varnish Software AS
- * All rights reserved.
- *
- * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- *
- */
-
-struct VCLS;
-typedef void cls_cb_f(void *priv);
-typedef void cls_cbc_f(const struct cli*);
-struct VCLS *VCLS_New(cls_cbc_f *before, cls_cbc_f *after, unsigned maxlen);
-struct cli *VCLS_AddFd(struct VCLS *cs, int fdi, int fdo, cls_cb_f *closefunc,
-    void *priv);
-int VCLS_AddFunc(struct VCLS *cs, unsigned auth, struct cli_proto *clp);
-int VCLS_Poll(struct VCLS *cs, int timeout);
-int VCLS_PollFd(struct VCLS *cs, int fd, int timeout);
-void VCLS_Destroy(struct VCLS **);
-
-/* From libvarnish/cli.c */
-cli_func_t	VCLS_func_close;
-cli_func_t	VCLS_func_help;
-cli_func_t	VCLS_func_ping;
-
diff --git a/include/flopen.h b/include/flopen.h
index e5b6030..9833efa 100644
--- a/include/flopen.h
+++ b/include/flopen.h
@@ -31,8 +31,6 @@
 #ifndef FLOPEN_H_INCLUDED
 #define FLOPEN_H_INCLUDED
 
-#include <sys/types.h>
-
 int flopen(const char *, int, ...);
 int fltest(int fd, pid_t *pid);
 
diff --git a/include/vcli_serve.h b/include/vcli_serve.h
new file mode 100644
index 0000000..0ac29f8
--- /dev/null
+++ b/include/vcli_serve.h
@@ -0,0 +1,45 @@
+/*-
+ * Copyright (c) 2010-2011 Varnish Software AS
+ * All rights reserved.
+ *
+ * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ */
+
+struct VCLS;
+typedef void cls_cb_f(void *priv);
+typedef void cls_cbc_f(const struct cli*);
+struct VCLS *VCLS_New(cls_cbc_f *before, cls_cbc_f *after, unsigned maxlen);
+struct cli *VCLS_AddFd(struct VCLS *cs, int fdi, int fdo, cls_cb_f *closefunc,
+    void *priv);
+int VCLS_AddFunc(struct VCLS *cs, unsigned auth, struct cli_proto *clp);
+int VCLS_Poll(struct VCLS *cs, int timeout);
+int VCLS_PollFd(struct VCLS *cs, int fd, int timeout);
+void VCLS_Destroy(struct VCLS **);
+
+/* From libvarnish/cli.c */
+cli_func_t	VCLS_func_close;
+cli_func_t	VCLS_func_help;
+cli_func_t	VCLS_func_ping;
+
diff --git a/lib/libvarnish/cli_serve.c b/lib/libvarnish/cli_serve.c
index 2c3a09c..9a9daed 100644
--- a/lib/libvarnish/cli_serve.c
+++ b/lib/libvarnish/cli_serve.c
@@ -46,8 +46,8 @@
 #include "vcli.h"
 #include "cli_common.h"
 #include "cli_priv.h"
-#include "cli_serve.h"
 #include "vav.h"
+#include "vcli_serve.h"
 #include "vlu.h"
 #include "vqueue.h"
 #include "vsb.h"



From geoff at varnish-cache.org  Mon Jan  9 20:52:17 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:17 +0100
Subject: [experimental-ims] e94b1e8 Move stevedore.h to storage.h and have it
	contain only the back-side (storage-module facing) bits.
Message-ID: <E1RkMCX-0007eW-NE@project.varnish-software.com>

commit e94b1e8af31ac7ecfdadbcb5ab376955a397f20e
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Oct 10 12:26:31 2011 +0000

    Move stevedore.h to storage.h and have it contain only the back-side
    (storage-module facing) bits.
    
    Move the front-side bits to cache.h

diff --git a/bin/varnishd/Makefile.am b/bin/varnishd/Makefile.am
index b1f2a5c..2b822fa 100644
--- a/bin/varnishd/Makefile.am
+++ b/bin/varnishd/Makefile.am
@@ -88,7 +88,7 @@ noinst_HEADERS = \
 	heritage.h \
 	mgt.h \
 	mgt_cli.h \
-	stevedore.h \
+	storage.h \
 	storage_persistent.h \
 	vparam.h
 
diff --git a/bin/varnishd/cache.h b/bin/varnishd/cache.h
index 41efdad..930f197 100644
--- a/bin/varnishd/cache.h
+++ b/bin/varnishd/cache.h
@@ -46,7 +46,6 @@
 #include <pthread_np.h>
 #endif
 #include <stdarg.h>
-#include <stdint.h>
 #include <string.h>
 #include <limits.h>
 #include <unistd.h>
@@ -94,6 +93,7 @@ enum {
 
 struct iovec;
 struct cli;
+struct exp;
 struct vsb;
 struct sess;
 struct director;
@@ -379,6 +379,15 @@ struct worker {
 	struct acct		acct_tmp;
 };
 
+/* LRU ---------------------------------------------------------------*/
+
+struct lru {
+	unsigned		magic;
+#define LRU_MAGIC		0x3fec7bb0
+	VTAILQ_HEAD(,objcore)	lru_head;
+	struct lock		mtx;
+};
+
 /* Storage -----------------------------------------------------------*/
 
 struct storage {
@@ -961,9 +970,20 @@ enum body_status RFC2616_Body(const struct sess *sp);
 unsigned RFC2616_Req_Gzip(const struct sess *sp);
 int RFC2616_Do_Cond(const struct sess *sp);
 
+/* stevedore.c */
+struct object *STV_NewObject(struct sess *sp, const char *hint, unsigned len,
+    struct exp *, uint16_t nhttp);
+struct storage *STV_alloc(const struct sess *sp, size_t size);
+void STV_trim(struct storage *st, size_t size);
+void STV_free(struct storage *st);
+void STV_open(void);
+void STV_close(void);
+void STV_Freestore(struct object *o);
+
 /* storage_synth.c */
 struct vsb *SMS_Makesynth(struct object *obj);
 void SMS_Finish(struct object *obj);
+void SMS_Init(void);
 
 /* storage_persistent.c */
 void SMP_Init(void);
diff --git a/bin/varnishd/cache_center.c b/bin/varnishd/cache_center.c
index 463a05b..1eb249e 100644
--- a/bin/varnishd/cache_center.c
+++ b/bin/varnishd/cache_center.c
@@ -57,7 +57,6 @@ DOT acceptor -> start [style=bold,color=green]
 
 #include "config.h"
 
-#include <errno.h>
 #include <math.h>
 #include <poll.h>
 #include <stdio.h>
@@ -66,7 +65,6 @@ DOT acceptor -> start [style=bold,color=green]
 #include "cache.h"
 
 #include "hash_slinger.h"
-#include "stevedore.h"
 #include "vcl.h"
 #include "vcli_priv.h"
 #include "vsha256.h"
diff --git a/bin/varnishd/cache_dir_dns.c b/bin/varnishd/cache_dir_dns.c
index acb3689..ae96dbb 100644
--- a/bin/varnishd/cache_dir_dns.c
+++ b/bin/varnishd/cache_dir_dns.c
@@ -31,7 +31,6 @@
 
 #include <netinet/in.h>
 
-#include <errno.h>
 #include <netdb.h>
 #include <stdio.h>
 #include <stdlib.h>
diff --git a/bin/varnishd/cache_esi_fetch.c b/bin/varnishd/cache_esi_fetch.c
index b6ef527..1db5259 100644
--- a/bin/varnishd/cache_esi_fetch.c
+++ b/bin/varnishd/cache_esi_fetch.c
@@ -35,8 +35,6 @@
 #include "cache.h"
 
 #include "cache_esi.h"
-#include "stevedore.h"
-#include "vct.h"
 
 /*---------------------------------------------------------------------
  * Read some bytes.
diff --git a/bin/varnishd/cache_expire.c b/bin/varnishd/cache_expire.c
index 8e18cb5..f9dcdd1 100644
--- a/bin/varnishd/cache_expire.c
+++ b/bin/varnishd/cache_expire.c
@@ -57,7 +57,6 @@
 
 #include "binary_heap.h"
 #include "hash_slinger.h"
-#include "stevedore.h"
 #include "vtim.h"
 
 static pthread_t exp_thread;
diff --git a/bin/varnishd/cache_fetch.c b/bin/varnishd/cache_fetch.c
index 75319d8..4300780 100644
--- a/bin/varnishd/cache_fetch.c
+++ b/bin/varnishd/cache_fetch.c
@@ -36,7 +36,6 @@
 
 #include "cache.h"
 
-#include "stevedore.h"
 #include "vcli_priv.h"
 #include "vct.h"
 #include "vtcp.h"
diff --git a/bin/varnishd/cache_hash.c b/bin/varnishd/cache_hash.c
index 67aa3af..12855e1 100644
--- a/bin/varnishd/cache_hash.c
+++ b/bin/varnishd/cache_hash.c
@@ -61,7 +61,6 @@
 
 #include "cache_backend.h"
 #include "hash_slinger.h"
-#include "stevedore.h"
 #include "vav.h"
 #include "vsha256.h"
 
diff --git a/bin/varnishd/cache_main.c b/bin/varnishd/cache_main.c
index 54d71f8..6e62fd2 100644
--- a/bin/varnishd/cache_main.c
+++ b/bin/varnishd/cache_main.c
@@ -36,7 +36,6 @@
 
 #include "cache_waiter.h"
 #include "hash_slinger.h"
-#include "stevedore.h"
 
 /*--------------------------------------------------------------------
  * Per thread storage for the session currently being processed by
diff --git a/bin/varnishd/cache_pool.c b/bin/varnishd/cache_pool.c
index 4421969..00eaef5 100644
--- a/bin/varnishd/cache_pool.c
+++ b/bin/varnishd/cache_pool.c
@@ -42,7 +42,6 @@
 
 #include "config.h"
 
-#include <errno.h>
 #include <math.h>
 #include <stdlib.h>
 
diff --git a/bin/varnishd/cache_response.c b/bin/varnishd/cache_response.c
index bacf906..9768eeb 100644
--- a/bin/varnishd/cache_response.c
+++ b/bin/varnishd/cache_response.c
@@ -37,7 +37,6 @@
 
 #include "cache.h"
 
-#include "stevedore.h"
 #include "vct.h"
 #include "vtim.h"
 
diff --git a/bin/varnishd/cache_waiter_epoll.c b/bin/varnishd/cache_waiter_epoll.c
index 3eaa627..301fcf2 100644
--- a/bin/varnishd/cache_waiter_epoll.c
+++ b/bin/varnishd/cache_waiter_epoll.c
@@ -37,7 +37,6 @@
 
 #include <sys/epoll.h>
 
-#include <errno.h>
 #include <fcntl.h>
 #include <stdlib.h>
 
diff --git a/bin/varnishd/cache_waiter_kqueue.c b/bin/varnishd/cache_waiter_kqueue.c
index a81f4c2..8bee1c9 100644
--- a/bin/varnishd/cache_waiter_kqueue.c
+++ b/bin/varnishd/cache_waiter_kqueue.c
@@ -38,7 +38,6 @@
 #include <sys/types.h>
 #include <sys/event.h>
 
-#include <errno.h>
 #include <fcntl.h>
 #include <stdlib.h>
 #include <unistd.h>
diff --git a/bin/varnishd/cache_waiter_ports.c b/bin/varnishd/cache_waiter_ports.c
index 2e02406..821ebc3 100644
--- a/bin/varnishd/cache_waiter_ports.c
+++ b/bin/varnishd/cache_waiter_ports.c
@@ -35,7 +35,6 @@
 
 #include <sys/time.h>
 
-#include <errno.h>
 #include <math.h>
 #include <port.h>
 #include <stdio.h>
diff --git a/bin/varnishd/common.h b/bin/varnishd/common.h
index 13249eb..4e56a40 100644
--- a/bin/varnishd/common.h
+++ b/bin/varnishd/common.h
@@ -77,6 +77,10 @@ const void *pick(const struct choice *cp, const char *which, const char *kind);
 
 #define NEEDLESS_RETURN(foo)	return (foo)
 
+/* stevedore.c */
+void STV_Config(const char *spec);
+void STV_Config_Transient(void);
+
 /* vsm.c */
 // extern struct VSM_head		*VSM_head;
 // extern const struct VSM_chunk	*vsm_end;
diff --git a/bin/varnishd/mgt.h b/bin/varnishd/mgt.h
index 9d12752..2b17052 100644
--- a/bin/varnishd/mgt.h
+++ b/bin/varnishd/mgt.h
@@ -86,6 +86,7 @@ extern const char *mgt_vcl_dir;
 extern const char *mgt_vmod_dir;
 extern unsigned mgt_vcc_err_unref;
 
+
 #define REPORT0(pri, fmt)				\
 	do {						\
 		fprintf(stderr, fmt "\n");		\
diff --git a/bin/varnishd/mgt_child.c b/bin/varnishd/mgt_child.c
index 5af26fe..2108230 100644
--- a/bin/varnishd/mgt_child.c
+++ b/bin/varnishd/mgt_child.c
@@ -34,7 +34,6 @@
 #include <sys/types.h>
 #include <sys/wait.h>
 
-#include <errno.h>
 #include <fcntl.h>
 #include <poll.h>
 #include <signal.h>
diff --git a/bin/varnishd/stevedore.c b/bin/varnishd/stevedore.c
index ff1bdf5..6deffb4 100644
--- a/bin/varnishd/stevedore.c
+++ b/bin/varnishd/stevedore.c
@@ -38,7 +38,7 @@
 
 #include "cache.h"
 
-#include "stevedore.h"
+#include "storage.h"
 #include "vav.h"
 #include "vcli_priv.h"
 #include "vrt.h"
diff --git a/bin/varnishd/stevedore.h b/bin/varnishd/stevedore.h
deleted file mode 100644
index 9cd0b7b..0000000
--- a/bin/varnishd/stevedore.h
+++ /dev/null
@@ -1,119 +0,0 @@
-/*-
- * Copyright (c) 2006 Verdens Gang AS
- * Copyright (c) 2006-2011 Varnish Software AS
- * All rights reserved.
- *
- * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- *
- */
-
-struct stevedore;
-struct exp;
-struct sess;
-struct iovec;
-struct object;
-struct objcore;
-struct stv_objsecrets;
-
-typedef void storage_init_f(struct stevedore *, int ac, char * const *av);
-typedef void storage_open_f(const struct stevedore *);
-typedef struct storage *storage_alloc_f(struct stevedore *, size_t size);
-typedef void storage_trim_f(struct storage *, size_t size);
-typedef void storage_free_f(struct storage *);
-typedef struct object *storage_allocobj_f(struct stevedore *, struct sess *sp,
-    unsigned ltot, const struct stv_objsecrets *);
-typedef void storage_close_f(const struct stevedore *);
-
-/* Prototypes for VCL variable responders */
-#define VRTSTVTYPE(ct) typedef ct storage_var_##ct(const struct stevedore *);
-#include "tbl/vrt_stv_var.h"
-#undef VRTSTVTYPE
-
-/*--------------------------------------------------------------------*/
-
-struct lru {
-	unsigned		magic;
-#define LRU_MAGIC		0x3fec7bb0
-	VTAILQ_HEAD(,objcore)	lru_head;
-	struct lock		mtx;
-};
-
-/*--------------------------------------------------------------------*/
-
-struct stevedore {
-	unsigned		magic;
-#define STEVEDORE_MAGIC		0x4baf43db
-	const char		*name;
-	unsigned		transient;
-	storage_init_f		*init;		/* called by mgt process */
-	storage_open_f		*open;		/* called by cache process */
-	storage_alloc_f		*alloc;		/* --//-- */
-	storage_trim_f		*trim;		/* --//-- */
-	storage_free_f		*free;		/* --//-- */
-	storage_close_f		*close;		/* --//-- */
-	storage_allocobj_f	*allocobj;	/* --//-- */
-
-	struct lru		*lru;
-
-#define VRTSTVVAR(nm, vtype, ctype, dval) storage_var_##ctype *var_##nm;
-#include "tbl/vrt_stv_var.h"
-#undef VRTSTVVAR
-
-	/* private fields */
-	void			*priv;
-
-	VTAILQ_ENTRY(stevedore)	list;
-	char			ident[16];	/* XXX: match VSM_chunk.ident */
-};
-
-struct object *STV_MkObject(struct sess *sp, void *ptr, unsigned ltot,
-    const struct stv_objsecrets *soc);
-
-struct object *STV_NewObject(struct sess *sp, const char *hint, unsigned len,
-    struct exp *, uint16_t nhttp);
-struct storage *STV_alloc(const struct sess *sp, size_t size);
-void STV_trim(struct storage *st, size_t size);
-void STV_free(struct storage *st);
-void STV_open(void);
-void STV_close(void);
-void STV_Config(const char *spec);
-void STV_Config_Transient(void);
-void STV_Freestore(struct object *o);
-
-struct lru *LRU_Alloc(void);
-void LRU_Free(struct lru *lru);
-
-int STV_GetFile(const char *fn, int *fdp, const char **fnp, const char *ctx);
-uintmax_t STV_FileSize(int fd, const char *size, unsigned *granularity,
-    const char *ctx);
-
-/* Synthetic Storage */
-void SMS_Init(void);
-
-extern const struct stevedore sma_stevedore;
-extern const struct stevedore smf_stevedore;
-extern const struct stevedore smp_stevedore;
-#ifdef HAVE_LIBUMEM
-extern const struct stevedore smu_stevedore;
-#endif
diff --git a/bin/varnishd/stevedore_utils.c b/bin/varnishd/stevedore_utils.c
index 42c2b47..e428f58 100644
--- a/bin/varnishd/stevedore_utils.c
+++ b/bin/varnishd/stevedore_utils.c
@@ -51,7 +51,7 @@
 
 #include "mgt.h"
 
-#include "stevedore.h"
+#include "storage.h"
 #include "vnum.h"
 
 #ifndef O_LARGEFILE
diff --git a/bin/varnishd/storage.h b/bin/varnishd/storage.h
new file mode 100644
index 0000000..80cad13
--- /dev/null
+++ b/bin/varnishd/storage.h
@@ -0,0 +1,97 @@
+/*-
+ * Copyright (c) 2006 Verdens Gang AS
+ * Copyright (c) 2006-2011 Varnish Software AS
+ * All rights reserved.
+ *
+ * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * This defines the backend interface between the stevedore and the
+ * pluggable storage implementations.
+ *
+ */
+
+struct stv_objsecrets;
+struct stevedore;
+struct sess;
+struct lru;
+
+typedef void storage_init_f(struct stevedore *, int ac, char * const *av);
+typedef void storage_open_f(const struct stevedore *);
+typedef struct storage *storage_alloc_f(struct stevedore *, size_t size);
+typedef void storage_trim_f(struct storage *, size_t size);
+typedef void storage_free_f(struct storage *);
+typedef struct object *storage_allocobj_f(struct stevedore *, struct sess *sp,
+    unsigned ltot, const struct stv_objsecrets *);
+typedef void storage_close_f(const struct stevedore *);
+
+/* Prototypes for VCL variable responders */
+#define VRTSTVTYPE(ct) typedef ct storage_var_##ct(const struct stevedore *);
+#include "tbl/vrt_stv_var.h"
+#undef VRTSTVTYPE
+
+/*--------------------------------------------------------------------*/
+
+struct stevedore {
+	unsigned		magic;
+#define STEVEDORE_MAGIC		0x4baf43db
+	const char		*name;
+	unsigned		transient;
+	storage_init_f		*init;		/* called by mgt process */
+	storage_open_f		*open;		/* called by cache process */
+	storage_alloc_f		*alloc;		/* --//-- */
+	storage_trim_f		*trim;		/* --//-- */
+	storage_free_f		*free;		/* --//-- */
+	storage_close_f		*close;		/* --//-- */
+	storage_allocobj_f	*allocobj;	/* --//-- */
+
+	struct lru		*lru;
+
+#define VRTSTVVAR(nm, vtype, ctype, dval) storage_var_##ctype *var_##nm;
+#include "tbl/vrt_stv_var.h"
+#undef VRTSTVVAR
+
+	/* private fields */
+	void			*priv;
+
+	VTAILQ_ENTRY(stevedore)	list;
+	char			ident[16];	/* XXX: match VSM_chunk.ident */
+};
+
+/*--------------------------------------------------------------------*/
+int STV_GetFile(const char *fn, int *fdp, const char **fnp, const char *ctx);
+uintmax_t STV_FileSize(int fd, const char *size, unsigned *granularity,
+    const char *ctx);
+struct object *STV_MkObject(struct sess *sp, void *ptr, unsigned ltot,
+    const struct stv_objsecrets *soc);
+
+struct lru *LRU_Alloc(void);
+void LRU_Free(struct lru *lru);
+
+/*--------------------------------------------------------------------*/
+extern const struct stevedore sma_stevedore;
+extern const struct stevedore smf_stevedore;
+extern const struct stevedore smp_stevedore;
+#ifdef HAVE_LIBUMEM
+extern const struct stevedore smu_stevedore;
+#endif
diff --git a/bin/varnishd/storage_file.c b/bin/varnishd/storage_file.c
index 171d9d0..ecfc4ff 100644
--- a/bin/varnishd/storage_file.c
+++ b/bin/varnishd/storage_file.c
@@ -37,8 +37,8 @@
 #include <stdlib.h>
 
 #include "cache.h"
+#include "storage.h"
 
-#include "stevedore.h"
 #include "vnum.h"
 
 #ifndef MAP_NOCORE
diff --git a/bin/varnishd/storage_malloc.c b/bin/varnishd/storage_malloc.c
index bc5a565..967137c 100644
--- a/bin/varnishd/storage_malloc.c
+++ b/bin/varnishd/storage_malloc.c
@@ -35,8 +35,8 @@
 #include <stdlib.h>
 
 #include "cache.h"
+#include "storage.h"
 
-#include "stevedore.h"
 #include "vnum.h"
 
 struct sma_sc {
diff --git a/bin/varnishd/storage_persistent.c b/bin/varnishd/storage_persistent.c
index d379c4a..abe25f6 100644
--- a/bin/varnishd/storage_persistent.c
+++ b/bin/varnishd/storage_persistent.c
@@ -44,9 +44,9 @@
 #include <string.h>
 
 #include "cache.h"
+#include "storage.h"
 
 #include "hash_slinger.h"
-#include "stevedore.h"
 #include "vcli.h"
 #include "vcli_priv.h"
 #include "vend.h"
diff --git a/bin/varnishd/storage_persistent_mgt.c b/bin/varnishd/storage_persistent_mgt.c
index 7ec9089..2cdcc6b 100644
--- a/bin/varnishd/storage_persistent_mgt.c
+++ b/bin/varnishd/storage_persistent_mgt.c
@@ -42,8 +42,8 @@
 #include <stdlib.h>
 
 #include "cache.h"
+#include "storage.h"
 
-#include "stevedore.h"
 #include "vsha256.h"
 
 #include "persistent.h"
diff --git a/bin/varnishd/storage_persistent_silo.c b/bin/varnishd/storage_persistent_silo.c
index be2e2bf..8209613 100644
--- a/bin/varnishd/storage_persistent_silo.c
+++ b/bin/varnishd/storage_persistent_silo.c
@@ -38,9 +38,9 @@
 #include <stdlib.h>
 
 #include "cache.h"
+#include "storage.h"
 
 #include "hash_slinger.h"
-#include "stevedore.h"
 #include "vsha256.h"
 #include "vtim.h"
 
diff --git a/bin/varnishd/storage_persistent_subr.c b/bin/varnishd/storage_persistent_subr.c
index e3df368..b4bbb3f 100644
--- a/bin/varnishd/storage_persistent_subr.c
+++ b/bin/varnishd/storage_persistent_subr.c
@@ -42,6 +42,7 @@
 #include <stdlib.h>
 
 #include "cache.h"
+#include "storage.h"
 
 #include "vsha256.h"
 
diff --git a/bin/varnishd/storage_synth.c b/bin/varnishd/storage_synth.c
index 0792dbd..5df2c08 100644
--- a/bin/varnishd/storage_synth.c
+++ b/bin/varnishd/storage_synth.c
@@ -33,8 +33,8 @@
 #include <stdlib.h>
 
 #include "cache.h"
+#include "storage.h"
 
-#include "stevedore.h"
 
 static struct lock		sms_mtx;
 
diff --git a/bin/varnishd/storage_umem.c b/bin/varnishd/storage_umem.c
index 4459a47..9198a99 100644
--- a/bin/varnishd/storage_umem.c
+++ b/bin/varnishd/storage_umem.c
@@ -40,8 +40,7 @@
 #include <umem.h>
 
 #include "cache.h"
-
-#include "stevedore.h"
+#include "storage.h"
 
 static size_t			smu_max = SIZE_MAX;
 static MTX			smu_mtx;
diff --git a/bin/varnishd/varnishd.c b/bin/varnishd/varnishd.c
index 722c10e..7fe72aa 100644
--- a/bin/varnishd/varnishd.c
+++ b/bin/varnishd/varnishd.c
@@ -35,7 +35,6 @@
 #include <sys/utsname.h>
 
 #include <ctype.h>
-#include <errno.h>
 #include <fcntl.h>
 #include <stdio.h>
 #include <stdlib.h>
@@ -48,7 +47,6 @@
 
 #include "hash_slinger.h"
 #include "heritage.h"
-#include "stevedore.h"
 #include "vav.h"
 #include "vcli.h"
 #include "vcli_common.h"



From geoff at varnish-cache.org  Mon Jan  9 20:52:17 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:17 +0100
Subject: [experimental-ims] 3ede515 Smartypants is bad for code. People
	couldn't copy/paste from the reference pages
Message-ID: <E1RkMCX-0007ev-S0@project.varnish-software.com>

commit 3ede515d3401ffca7766abfee549826c92fbcc3d
Author: Andreas Plesner Jacobsen <apj at mutt.dk>
Date:   Mon Oct 10 22:56:34 2011 +0200

    Smartypants is bad for code. People couldn't copy/paste from the reference pages

diff --git a/doc/sphinx/conf.py.in b/doc/sphinx/conf.py.in
index 0b07a2b..5cef666 100644
--- a/doc/sphinx/conf.py.in
+++ b/doc/sphinx/conf.py.in
@@ -155,7 +155,7 @@ html_title = "Varnish version @VERSION@ documentation"
 
 # If true, SmartyPants will be used to convert quotes and dashes to
 # typographically correct entities.
-#html_use_smartypants = True
+html_use_smartypants = False
 
 # Custom sidebar templates, maps document names to template names.
 #html_sidebars = {}



From geoff at varnish-cache.org  Mon Jan  9 20:52:18 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:18 +0100
Subject: [experimental-ims] 3b73cf0 Close a race where varnishtest::client
	would attempt to connect before varnishd::acceptor had called
	listen(2)
Message-ID: <E1RkMCY-0007fH-00@project.varnish-software.com>

commit 3b73cf0e92f432c23387ca498d7783f1d1b18bee
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Oct 12 08:18:43 2011 +0000

    Close a race where varnishtest::client would attempt to connect before
    varnishd::acceptor had called listen(2)
    
    Found and fixed by:	slink aka. Nils Goroll

diff --git a/bin/varnishd/cache_acceptor.c b/bin/varnishd/cache_acceptor.c
index e6ad6e3..f92bace 100644
--- a/bin/varnishd/cache_acceptor.c
+++ b/bin/varnishd/cache_acceptor.c
@@ -376,6 +376,16 @@ ccf_listen_address(struct cli *cli, const char * const *av, void *priv)
 	(void)cli;
 	(void)av;
 	(void)priv;
+
+	/*
+	 * This CLI command is primarily used by varnishtest.  Don't
+	 * respond until liste(2) has been called, in order to avoid
+	 * a race where varnishtest::client would attempt to connect(2)
+	 * before listen(2) has been called.
+	 */
+	while(!hack_ready)
+		(void)usleep(100*1000);
+
 	VTAILQ_FOREACH(ls, &heritage.socks, list) {
 		if (ls->sock < 0)
 			continue;



From geoff at varnish-cache.org  Mon Jan  9 20:52:18 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:18 +0100
Subject: [experimental-ims] e97567d <readline/readline.h> requires <stdio.h>
Message-ID: <E1RkMCY-0007fi-B1@project.varnish-software.com>

commit e97567d452f793d59e93192c55d11be6555b6c03
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Oct 12 08:47:45 2011 +0000

    <readline/readline.h> requires <stdio.h>

diff --git a/bin/varnishadm/varnishadm.c b/bin/varnishadm/varnishadm.c
index 818b9ab..f5d6054 100644
--- a/bin/varnishadm/varnishadm.c
+++ b/bin/varnishadm/varnishadm.c
@@ -32,6 +32,7 @@
 #include <sys/socket.h>
 
 #ifdef HAVE_LIBEDIT
+#include <stdio.h>
 #include <editline/readline.h>
 #endif
 



From geoff at varnish-cache.org  Mon Jan  9 20:52:18 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:18 +0100
Subject: [experimental-ims] aff259b Move struct vbc to cache_backend.h
Message-ID: <E1RkMCY-0007fz-Dp@project.varnish-software.com>

commit aff259bc2e0e19af8a0388851bc60af25511b41d
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Oct 12 11:19:37 2011 +0000

    Move struct vbc to cache_backend.h

diff --git a/bin/varnishd/cache.h b/bin/varnishd/cache.h
index 930f197..7eb3258 100644
--- a/bin/varnishd/cache.h
+++ b/bin/varnishd/cache.h
@@ -91,26 +91,27 @@ enum {
 	HTTP_HDR_FIRST,
 };
 
-struct iovec;
+struct SHA256Context;
+struct VSC_C_lck;
+struct ban;
+struct busyobj;
 struct cli;
-struct exp;
-struct vsb;
-struct sess;
+struct cli_proto;
 struct director;
+struct exp;
+struct iovec;
+struct objcore;
 struct object;
 struct objhead;
-struct objcore;
-struct busyobj;
-struct storage;
+struct pool;
+struct sess;
 struct sesspool;
+struct storage;
+struct vbc;
+struct vef_priv;
 struct vrt_backend;
-struct cli_proto;
-struct ban;
-struct SHA256Context;
-struct VSC_C_lck;
+struct vsb;
 struct waitinglist;
-struct vef_priv;
-struct pool;
 
 #define DIGEST_LEN		32
 
@@ -627,28 +628,6 @@ struct sess {
 #endif
 };
 
-/* -------------------------------------------------------------------*/
-
-/* Backend connection */
-struct vbc {
-	unsigned		magic;
-#define VBC_MAGIC		0x0c5e6592
-	VTAILQ_ENTRY(vbc)	list;
-	struct backend		*backend;
-	struct vdi_simple	*vdis;
-	unsigned		vsl_id;
-	int			fd;
-
-	struct sockaddr_storage	*addr;
-	socklen_t		addrlen;
-
-	uint8_t			recycled;
-
-	/* Timeouts */
-	double			first_byte_timeout;
-	double			between_bytes_timeout;
-};
-
 /* Prototypes etc ----------------------------------------------------*/
 
 /* cache_acceptor.c */
diff --git a/bin/varnishd/cache_backend.h b/bin/varnishd/cache_backend.h
index 21a7061..47e709d 100644
--- a/bin/varnishd/cache_backend.h
+++ b/bin/varnishd/cache_backend.h
@@ -134,6 +134,28 @@ struct backend {
 	struct VSC_C_vbe	*vsc;
 };
 
+/* -------------------------------------------------------------------*/
+
+/* Backend connection */
+struct vbc {
+	unsigned		magic;
+#define VBC_MAGIC		0x0c5e6592
+	VTAILQ_ENTRY(vbc)	list;
+	struct backend		*backend;
+	struct vdi_simple	*vdis;
+	unsigned		vsl_id;
+	int			fd;
+
+	struct sockaddr_storage	*addr;
+	socklen_t		addrlen;
+
+	uint8_t			recycled;
+
+	/* Timeouts */
+	double			first_byte_timeout;
+	double			between_bytes_timeout;
+};
+
 /* cache_backend.c */
 void VBE_ReleaseConn(struct vbc *vc);
 struct backend *vdi_get_backend_if_simple(const struct director *d);
diff --git a/bin/varnishd/cache_cli.c b/bin/varnishd/cache_cli.c
index ea4d367..01da3f2 100644
--- a/bin/varnishd/cache_cli.c
+++ b/bin/varnishd/cache_cli.c
@@ -40,6 +40,7 @@
 
 #include "cache.h"
 
+#include "cache_backend.h"		// struct vbc
 #include "hash_slinger.h"		// objhead
 #include "vcli.h"
 #include "vcli_common.h"
diff --git a/bin/varnishd/cache_fetch.c b/bin/varnishd/cache_fetch.c
index 4300780..26489b6 100644
--- a/bin/varnishd/cache_fetch.c
+++ b/bin/varnishd/cache_fetch.c
@@ -36,6 +36,7 @@
 
 #include "cache.h"
 
+#include "cache_backend.h"
 #include "vcli_priv.h"
 #include "vct.h"
 #include "vtcp.h"
diff --git a/bin/varnishd/cache_pipe.c b/bin/varnishd/cache_pipe.c
index c9bc0ac..8f109a5 100644
--- a/bin/varnishd/cache_pipe.c
+++ b/bin/varnishd/cache_pipe.c
@@ -36,6 +36,7 @@
 
 #include "cache.h"
 
+#include "cache_backend.h"
 #include "vtcp.h"
 #include "vtim.h"
 



From geoff at varnish-cache.org  Mon Jan  9 20:52:18 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:18 +0100
Subject: [experimental-ims] 53a28ab These #includes look surplus to
	requirements.
Message-ID: <E1RkMCY-0007g9-Fa@project.varnish-software.com>

commit 53a28ab7ae637884ffa91ac1bed756de6febc32b
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Oct 12 11:33:23 2011 +0000

    These #includes look surplus to requirements.

diff --git a/bin/varnishd/cache_fetch.c b/bin/varnishd/cache_fetch.c
index 26489b6..71fc8e2 100644
--- a/bin/varnishd/cache_fetch.c
+++ b/bin/varnishd/cache_fetch.c
@@ -29,7 +29,6 @@
 
 #include "config.h"
 
-#include <fcntl.h>
 #include <inttypes.h>
 #include <stdio.h>
 #include <stdlib.h>
diff --git a/bin/varnishd/cache_hash.c b/bin/varnishd/cache_hash.c
index 12855e1..ebbe281 100644
--- a/bin/varnishd/cache_hash.c
+++ b/bin/varnishd/cache_hash.c
@@ -55,7 +55,6 @@
 #include <math.h>
 #include <stdio.h>
 #include <stdlib.h>
-#include <string.h>
 
 #include "cache.h"
 
diff --git a/bin/varnishd/cache_response.c b/bin/varnishd/cache_response.c
index 9768eeb..4f0334b 100644
--- a/bin/varnishd/cache_response.c
+++ b/bin/varnishd/cache_response.c
@@ -29,12 +29,6 @@
 
 #include "config.h"
 
-#include <sys/types.h>
-#include <sys/time.h>
-
-#include <stdio.h>
-#include <stdlib.h>
-
 #include "cache.h"
 
 #include "vct.h"
diff --git a/bin/varnishd/cache_vcl.c b/bin/varnishd/cache_vcl.c
index c77ab08..cc67a7f 100644
--- a/bin/varnishd/cache_vcl.c
+++ b/bin/varnishd/cache_vcl.c
@@ -34,7 +34,6 @@
 #include "config.h"
 
 #include <dlfcn.h>
-#include <stdio.h>
 #include <stdlib.h>
 
 #include "cache.h"
diff --git a/bin/varnishd/cache_vrt_re.c b/bin/varnishd/cache_vrt_re.c
index d9a7090..e6f854b 100644
--- a/bin/varnishd/cache_vrt_re.c
+++ b/bin/varnishd/cache_vrt_re.c
@@ -32,7 +32,6 @@
 #include "config.h"
 
 #include <ctype.h>
-#include <stdlib.h>
 
 #include "cache.h"
 
diff --git a/bin/varnishd/cache_vrt_var.c b/bin/varnishd/cache_vrt_var.c
index a00b1b7..1a16e16 100644
--- a/bin/varnishd/cache_vrt_var.c
+++ b/bin/varnishd/cache_vrt_var.c
@@ -36,8 +36,6 @@
 #include "cache.h"
 
 #include "cache_backend.h"
-#include "hash_slinger.h"
-#include "vrt.h"
 #include "vrt_obj.h"
 #include "vtcp.h"
 #include "vtim.h"
diff --git a/bin/varnishd/hash_classic.c b/bin/varnishd/hash_classic.c
index 1428864..fc88757 100644
--- a/bin/varnishd/hash_classic.c
+++ b/bin/varnishd/hash_classic.c
@@ -31,11 +31,8 @@
 
 #include "config.h"
 
-#include <sys/types.h>
-
 #include <stdio.h>
 #include <stdlib.h>
-#include <string.h>
 
 #include "cache.h"
 



From geoff at varnish-cache.org  Mon Jan  9 20:52:18 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:18 +0100
Subject: [experimental-ims] fe78027 Move everything related to the
	storage-expansion-API into a subdirectory where it will be
	protected from VMODs.
Message-ID: <E1RkMCY-0007gU-JQ@project.varnish-software.com>

commit fe780278743797013f6f26f0701b478a77d6b32a
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Oct 12 15:44:14 2011 +0000

    Move everything related to the storage-expansion-API into a subdirectory
    where it will be protected from VMODs.

diff --git a/bin/varnishd/Makefile.am b/bin/varnishd/Makefile.am
index 2b822fa..5260427 100644
--- a/bin/varnishd/Makefile.am
+++ b/bin/varnishd/Makefile.am
@@ -64,16 +64,16 @@ varnishd_SOURCES = \
 	mgt_shmem.c \
 	mgt_vcc.c \
 	rfc2616.c \
-	stevedore.c \
-	stevedore_utils.c \
-	storage_file.c \
-	storage_malloc.c \
-	storage_persistent.c \
-	storage_persistent_mgt.c \
-	storage_persistent_silo.c \
-	storage_persistent_subr.c \
-	storage_synth.c \
-	storage_umem.c \
+	storage/stevedore.c \
+	storage/stevedore_utils.c \
+	storage/storage_file.c \
+	storage/storage_malloc.c \
+	storage/storage_persistent.c \
+	storage/storage_persistent_mgt.c \
+	storage/storage_persistent_silo.c \
+	storage/storage_persistent_subr.c \
+	storage/storage_synth.c \
+	storage/storage_umem.c \
 	varnishd.c \
 	vsm.c
 
@@ -88,8 +88,8 @@ noinst_HEADERS = \
 	heritage.h \
 	mgt.h \
 	mgt_cli.h \
-	storage.h \
-	storage_persistent.h \
+	storage/storage.h \
+	storage/storage_persistent.h \
 	vparam.h
 
 varnishd_CFLAGS = \
diff --git a/bin/varnishd/cache.h b/bin/varnishd/cache.h
index 7eb3258..33ccc8b 100644
--- a/bin/varnishd/cache.h
+++ b/bin/varnishd/cache.h
@@ -98,7 +98,6 @@ struct busyobj;
 struct cli;
 struct cli_proto;
 struct director;
-struct exp;
 struct iovec;
 struct objcore;
 struct object;
@@ -106,7 +105,6 @@ struct objhead;
 struct pool;
 struct sess;
 struct sesspool;
-struct storage;
 struct vbc;
 struct vef_priv;
 struct vrt_backend;
diff --git a/bin/varnishd/flint.sh b/bin/varnishd/flint.sh
index 690e445..86d4ad0 100755
--- a/bin/varnishd/flint.sh
+++ b/bin/varnishd/flint.sh
@@ -16,6 +16,7 @@ flexelint \
 	-I/usr/local/include \
 	-DVARNISH_STATE_DIR=\"foo\" \
 	*.c \
+	storage/*.c \
 	../../lib/libvarnish/*.c \
 	../../lib/libvarnishcompat/execinfo.c \
 	../../lib/libvcl/*.c \
diff --git a/bin/varnishd/stevedore.c b/bin/varnishd/stevedore.c
deleted file mode 100644
index 6deffb4..0000000
--- a/bin/varnishd/stevedore.c
+++ /dev/null
@@ -1,602 +0,0 @@
-/*-
- * Copyright (c) 2007-2011 Varnish Software AS
- * All rights reserved.
- *
- * Author: Dag-Erling Sm?rgav <des at des.no>
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- *
- * STEVEDORE: one who works at or is responsible for loading and
- * unloading ships in port.  Example: "on the wharves, stevedores were
- * unloading cargo from the far corners of the world." Origin: Spanish
- * estibador, from estibar to pack.  First Known Use: 1788
- */
-
-#include "config.h"
-
-#include <stdio.h>
-#include <stdlib.h>
-
-#include "cache.h"
-
-#include "storage.h"
-#include "vav.h"
-#include "vcli_priv.h"
-#include "vrt.h"
-#include "vrt_obj.h"
-
-static VTAILQ_HEAD(, stevedore)	stevedores =
-    VTAILQ_HEAD_INITIALIZER(stevedores);
-
-static const struct stevedore * volatile stv_next;
-
-static struct stevedore *stv_transient;
-
-/*---------------------------------------------------------------------
- * Default objcore methods
- */
-
-static struct object * __match_proto__(getobj_f)
-default_oc_getobj(struct worker *wrk, struct objcore *oc)
-{
-	struct object *o;
-
-	(void)wrk;
-	if (oc->priv == NULL)
-		return (NULL);
-	CAST_OBJ_NOTNULL(o, oc->priv, OBJECT_MAGIC);
-	return (o);
-}
-
-static void
-default_oc_freeobj(struct objcore *oc)
-{
-	struct object *o;
-
-	CAST_OBJ_NOTNULL(o, oc->priv, OBJECT_MAGIC);
-	oc->priv = NULL;
-	oc->methods = NULL;
-
-	STV_Freestore(o);
-	STV_free(o->objstore);
-}
-
-static struct lru *
-default_oc_getlru(const struct objcore *oc)
-{
-	struct object *o;
-
-	CAST_OBJ_NOTNULL(o, oc->priv, OBJECT_MAGIC);
-	return (o->objstore->stevedore->lru);
-}
-
-static struct objcore_methods default_oc_methods = {
-	.getobj = default_oc_getobj,
-	.freeobj = default_oc_freeobj,
-	.getlru = default_oc_getlru,
-};
-
-
-/*--------------------------------------------------------------------
- */
-
-struct lru *
-LRU_Alloc(void)
-{
-	struct lru *l;
-
-	ALLOC_OBJ(l, LRU_MAGIC);
-	AN(l);
-	VTAILQ_INIT(&l->lru_head);
-	Lck_New(&l->mtx, lck_lru);
-	return (l);
-}
-
-void
-LRU_Free(struct lru *lru)
-{
-	CHECK_OBJ_NOTNULL(lru, LRU_MAGIC);
-	Lck_Delete(&lru->mtx);
-	FREE_OBJ(lru);
-}
-
-/*--------------------------------------------------------------------
- * XXX: trust pointer writes to be atomic
- */
-
-static struct stevedore *
-stv_pick_stevedore(const struct sess *sp, const char **hint)
-{
-	struct stevedore *stv;
-
-	AN(hint);
-	if (*hint != NULL && **hint != '\0') {
-		VTAILQ_FOREACH(stv, &stevedores, list) {
-			if (!strcmp(stv->ident, *hint))
-				return (stv);
-		}
-		if (!strcmp(TRANSIENT_STORAGE, *hint))
-			return (stv_transient);
-
-		/* Hint was not valid, nuke it */
-		WSP(sp, SLT_Debug, "Storage hint not usable");
-		*hint = NULL;
-	}
-	/* pick a stevedore and bump the head along */
-	stv = VTAILQ_NEXT(stv_next, list);
-	if (stv == NULL)
-		stv = VTAILQ_FIRST(&stevedores);
-	AN(stv);
-	AN(stv->name);
-	stv_next = stv;
-	return (stv);
-}
-
-/*-------------------------------------------------------------------*/
-
-static struct storage *
-stv_alloc(const struct sess *sp, size_t size)
-{
-	struct storage *st;
-	struct stevedore *stv;
-	unsigned fail = 0;
-
-	/*
-	 * Always use the stevedore which allocated the object in order to
-	 * keep an object inside the same stevedore.
-	 */
-	CHECK_OBJ_NOTNULL(sp->obj, OBJECT_MAGIC);
-	stv = sp->obj->objstore->stevedore;
-	CHECK_OBJ_NOTNULL(stv, STEVEDORE_MAGIC);
-
-	if (size > (size_t)(params->fetch_maxchunksize) << 10)
-		size = (size_t)(params->fetch_maxchunksize) << 10;
-
-	for (;;) {
-		/* try to allocate from it */
-		AN(stv->alloc);
-		st = stv->alloc(stv, size);
-		if (st != NULL)
-			break;
-
-		if (size > params->fetch_chunksize * 1024LL) {
-			size >>= 1;
-			continue;
-		}
-
-		/* no luck; try to free some space and keep trying */
-		if (EXP_NukeOne(sp, stv->lru) == -1)
-			break;
-
-		/* Enough is enough: try another if we have one */
-		if (++fail >= params->nuke_limit)
-			break;
-	}
-	if (st != NULL)
-		CHECK_OBJ_NOTNULL(st, STORAGE_MAGIC);
-	return (st);
-}
-
-
-/*-------------------------------------------------------------------*
- * Structure used to transport internal knowledge from STV_NewObject()
- * to STV_MkObject().  Nobody else should mess with this struct.
- */
-
-struct stv_objsecrets {
-	unsigned	magic;
-#define STV_OBJ_SECRETES_MAGIC	0x78c87247
-	uint16_t	nhttp;
-	unsigned	lhttp;
-	unsigned	wsl;
-	struct exp	*exp;
-};
-
-/*--------------------------------------------------------------------
- * This function is called by stevedores ->allocobj() method, which
- * very often will be stv_default_allocobj() below, to convert a slab
- * of storage into object which the stevedore can then register in its
- * internal state, before returning it to STV_NewObject().
- * As you probably guessed: All this for persistence.
- */
-
-struct object *
-STV_MkObject(struct sess *sp, void *ptr, unsigned ltot,
-    const struct stv_objsecrets *soc)
-{
-	struct object *o;
-	unsigned l;
-
-	CHECK_OBJ_NOTNULL(soc, STV_OBJ_SECRETES_MAGIC);
-
-	assert(PAOK(ptr));
-	assert(PAOK(soc->wsl));
-	assert(PAOK(soc->lhttp));
-
-	assert(ltot >= sizeof *o + soc->lhttp + soc->wsl);
-
-	o = ptr;
-	memset(o, 0, sizeof *o);
-	o->magic = OBJECT_MAGIC;
-
-	l = PRNDDN(ltot - (sizeof *o + soc->lhttp));
-	assert(l >= soc->wsl);
-
-	o->http = HTTP_create(o + 1, soc->nhttp);
-	WS_Init(o->ws_o, "obj", (char *)(o + 1) + soc->lhttp, soc->wsl);
-	WS_Assert(o->ws_o);
-	assert(o->ws_o->e <= (char*)ptr + ltot);
-
-	http_Setup(o->http, o->ws_o);
-	o->http->magic = HTTP_MAGIC;
-	o->exp = *soc->exp;
-	VTAILQ_INIT(&o->store);
-	sp->wrk->stats.n_object++;
-
-	if (sp->objcore != NULL) {
-		CHECK_OBJ_NOTNULL(sp->objcore, OBJCORE_MAGIC);
-
-		o->objcore = sp->objcore;
-		sp->objcore = NULL;     /* refcnt follows pointer. */
-		BAN_NewObjCore(o->objcore);
-
-		o->objcore->methods = &default_oc_methods;
-		o->objcore->priv = o;
-	}
-	return (o);
-}
-
-/*--------------------------------------------------------------------
- * This is the default ->allocobj() which all stevedores who do not
- * implement persistent storage can rely on.
- */
-
-static struct object *
-stv_default_allocobj(struct stevedore *stv, struct sess *sp, unsigned ltot,
-    const struct stv_objsecrets *soc)
-{
-	struct object *o;
-	struct storage *st;
-
-	CHECK_OBJ_NOTNULL(soc, STV_OBJ_SECRETES_MAGIC);
-	st = stv->alloc(stv, ltot);
-	if (st == NULL)
-		return (NULL);
-	if (st->space < ltot) {
-		stv->free(st);
-		return (NULL);
-	}
-	ltot = st->len = st->space;
-	o = STV_MkObject(sp, st->ptr, ltot, soc);
-	CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC);
-	o->objstore = st;
-	return (o);
-}
-
-/*-------------------------------------------------------------------
- * Allocate storage for an object, based on the header information.
- * XXX: If we know (a hint of) the length, we could allocate space
- * XXX: for the body in the same allocation while we are at it.
- */
-
-struct object *
-STV_NewObject(struct sess *sp, const char *hint, unsigned wsl, struct exp *ep,
-    uint16_t nhttp)
-{
-	struct object *o;
-	struct stevedore *stv, *stv0;
-	unsigned lhttp, ltot;
-	struct stv_objsecrets soc;
-	int i;
-
-	assert(wsl > 0);
-	wsl = PRNDUP(wsl);
-
-	lhttp = HTTP_estimate(nhttp);
-	lhttp = PRNDUP(lhttp);
-
-	memset(&soc, 0, sizeof soc);
-	soc.magic = STV_OBJ_SECRETES_MAGIC;
-	soc.nhttp = nhttp;
-	soc.lhttp = lhttp;
-	soc.wsl = wsl;
-	soc.exp = ep;
-
-	ltot = sizeof *o + wsl + lhttp;
-
-	stv = stv0 = stv_pick_stevedore(sp, &hint);
-	AN(stv->allocobj);
-	o = stv->allocobj(stv, sp, ltot, &soc);
-	if (o == NULL && hint == NULL) {
-		do {
-			stv = stv_pick_stevedore(sp, &hint);
-			AN(stv->allocobj);
-			o = stv->allocobj(stv, sp, ltot, &soc);
-		} while (o == NULL && stv != stv0);
-	}
-	if (o == NULL) {
-		/* no luck; try to free some space and keep trying */
-		for (i = 0; o == NULL && i < params->nuke_limit; i++) {
-			if (EXP_NukeOne(sp, stv->lru) == -1)
-				break;
-			o = stv->allocobj(stv, sp, ltot, &soc);
-		}
-	}
-
-	if (o == NULL)
-		return (NULL);
-	CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC);
-	CHECK_OBJ_NOTNULL(o->objstore, STORAGE_MAGIC);
-	return (o);
-}
-
-/*-------------------------------------------------------------------*/
-
-void
-STV_Freestore(struct object *o)
-{
-	struct storage *st, *stn;
-
-	if (o->esidata != NULL) {
-		STV_free(o->esidata);
-		o->esidata = NULL;
-	}
-	VTAILQ_FOREACH_SAFE(st, &o->store, list, stn) {
-		CHECK_OBJ_NOTNULL(st, STORAGE_MAGIC);
-		VTAILQ_REMOVE(&o->store, st, list);
-		STV_free(st);
-	}
-}
-
-/*-------------------------------------------------------------------*/
-
-struct storage *
-STV_alloc(const struct sess *sp, size_t size)
-{
-
-	return (stv_alloc(sp, size));
-}
-
-void
-STV_trim(struct storage *st, size_t size)
-{
-
-	CHECK_OBJ_NOTNULL(st, STORAGE_MAGIC);
-	AN(st->stevedore);
-	if (st->stevedore->trim)
-		st->stevedore->trim(st, size);
-}
-
-void
-STV_free(struct storage *st)
-{
-
-	CHECK_OBJ_NOTNULL(st, STORAGE_MAGIC);
-	AN(st->stevedore);
-	AN(st->stevedore->free);
-	st->stevedore->free(st);
-}
-
-void
-STV_open(void)
-{
-	struct stevedore *stv;
-
-	VTAILQ_FOREACH(stv, &stevedores, list) {
-		stv->lru = LRU_Alloc();
-		if (stv->open != NULL)
-			stv->open(stv);
-	}
-	stv = stv_transient;
-	if (stv->open != NULL) {
-		stv->lru = LRU_Alloc();
-		stv->open(stv);
-	}
-}
-
-void
-STV_close(void)
-{
-	struct stevedore *stv;
-
-	VTAILQ_FOREACH(stv, &stevedores, list)
-		if (stv->close != NULL)
-			stv->close(stv);
-	stv = stv_transient;
-	if (stv->close != NULL)
-		stv->close(stv);
-}
-
-/*--------------------------------------------------------------------
- * Parse a stevedore argument on the form:
- *	[ name '=' ] strategy [ ',' arg ] *
- */
-
-static const struct choice STV_choice[] = {
-	{ "file",	&smf_stevedore },
-	{ "malloc",	&sma_stevedore },
-	{ "persistent",	&smp_stevedore },
-#ifdef HAVE_LIBUMEM
-	{ "umem",	&smu_stevedore },
-#endif
-	{ NULL,		NULL }
-};
-
-void
-STV_Config(const char *spec)
-{
-	char **av;
-	const char *p, *q;
-	struct stevedore *stv;
-	const struct stevedore *stv2;
-	int ac, l;
-	static unsigned seq = 0;
-
-	ASSERT_MGT();
-	p = strchr(spec, '=');
-	q = strchr(spec, ',');
-	if (p != NULL && (q == NULL || q > p)) {
-		av = VAV_Parse(p + 1, NULL, ARGV_COMMA);
-	} else {
-		av = VAV_Parse(spec, NULL, ARGV_COMMA);
-		p = NULL;
-	}
-	AN(av);
-
-	if (av[0] != NULL)
-		ARGV_ERR("%s\n", av[0]);
-
-	if (av[1] == NULL)
-		ARGV_ERR("-s argument lacks strategy {malloc, file, ...}\n");
-
-	for (ac = 0; av[ac + 2] != NULL; ac++)
-		continue;
-
-	stv2 = pick(STV_choice, av[1], "storage");
-	AN(stv2);
-
-	/* Append strategy to ident string */
-	VSB_printf(vident, ",-s%s", av[1]);
-
-	av += 2;
-
-	CHECK_OBJ_NOTNULL(stv2, STEVEDORE_MAGIC);
-	ALLOC_OBJ(stv, STEVEDORE_MAGIC);
-	AN(stv);
-
-	*stv = *stv2;
-	AN(stv->name);
-	AN(stv->alloc);
-	if (stv->allocobj == NULL)
-		stv->allocobj = stv_default_allocobj;
-
-	if (p == NULL)
-		bprintf(stv->ident, "s%u", seq++);
-	else {
-		l = p - spec;
-		if (l > sizeof stv->ident - 1)
-			l = sizeof stv->ident - 1;
-		bprintf(stv->ident, "%.*s", l, spec);
-	}
-
-	VTAILQ_FOREACH(stv2, &stevedores, list) {
-		if (strcmp(stv2->ident, stv->ident))
-			continue;
-		ARGV_ERR("(-s%s=%s) already defined once\n",
-		    stv->ident, stv->name);
-	}
-
-	if (stv->init != NULL)
-		stv->init(stv, ac, av);
-	else if (ac != 0)
-		ARGV_ERR("(-s%s) too many arguments\n", stv->name);
-
-	if (!strcmp(stv->ident, TRANSIENT_STORAGE)) {
-		stv->transient = 1;
-		AZ(stv_transient);
-		stv_transient = stv;
-	} else {
-		VTAILQ_INSERT_TAIL(&stevedores, stv, list);
-		if (!stv_next)
-			stv_next = VTAILQ_FIRST(&stevedores);
-	}
-}
-
-/*--------------------------------------------------------------------*/
-
-void
-STV_Config_Transient(void)
-{
-
-	ASSERT_MGT();
-
-	if (stv_transient == NULL)
-		STV_Config(TRANSIENT_STORAGE "=malloc");
-}
-
-/*--------------------------------------------------------------------*/
-
-static void
-stv_cli_list(struct cli *cli, const char * const *av, void *priv)
-{
-	struct stevedore *stv;
-
-	ASSERT_MGT();
-	(void)av;
-	(void)priv;
-	VCLI_Out(cli, "Storage devices:\n");
-	stv = stv_transient;
-		VCLI_Out(cli, "\tstorage.%s = %s\n", stv->ident, stv->name);
-	VTAILQ_FOREACH(stv, &stevedores, list)
-		VCLI_Out(cli, "\tstorage.%s = %s\n", stv->ident, stv->name);
-}
-
-/*--------------------------------------------------------------------*/
-
-struct cli_proto cli_stv[] = {
-	{ "storage.list", "storage.list", "List storage devices\n",
-	    0, 0, "", stv_cli_list },
-	{ NULL}
-};
-
-/*--------------------------------------------------------------------
- * VRT functions for stevedores
- */
-
-static const struct stevedore *
-stv_find(const char *nm)
-{
-	const struct stevedore *stv;
-
-	VTAILQ_FOREACH(stv, &stevedores, list)
-		if (!strcmp(stv->ident, nm))
-			return (stv);
-	if (!strcmp(TRANSIENT_STORAGE, nm))
-		return (stv_transient);
-	return (NULL);
-}
-
-int
-VRT_Stv(const char *nm)
-{
-
-	if (stv_find(nm) != NULL)
-		return (1);
-	return (0);
-}
-
-#define VRTSTVVAR(nm, vtype, ctype, dval)	\
-ctype						\
-VRT_Stv_##nm(const char *nm)			\
-{						\
-	const struct stevedore *stv;		\
-						\
-	stv = stv_find(nm);			\
-	if (stv == NULL)			\
-		return (dval);			\
-	if (stv->var_##nm == NULL)		\
-		return (dval);			\
-	return (stv->var_##nm(stv));		\
-}
-
-#include "tbl/vrt_stv_var.h"
-#undef VRTSTVVAR
diff --git a/bin/varnishd/stevedore_utils.c b/bin/varnishd/stevedore_utils.c
deleted file mode 100644
index e428f58..0000000
--- a/bin/varnishd/stevedore_utils.c
+++ /dev/null
@@ -1,244 +0,0 @@
-/*-
- * Copyright (c) 2006 Verdens Gang AS
- * Copyright (c) 2006-2010 Varnish Software AS
- * All rights reserved.
- *
- * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- *
- * Utility functions for stevedores and storage modules
- */
-
-#include "config.h"
-
-#include <sys/types.h>
-#include <sys/stat.h>
-#ifdef HAVE_SYS_MOUNT_H
-#  include <sys/mount.h>
-#endif
-#ifdef HAVE_SYS_STATVFS_H
-#  include <sys/statvfs.h>
-#endif
-#ifdef HAVE_SYS_VFS_H
-#  include <sys/vfs.h>
-#endif
-
-#include <fcntl.h>
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#include <unistd.h>
-
-#include "mgt.h"
-
-#include "storage.h"
-#include "vnum.h"
-
-#ifndef O_LARGEFILE
-#define O_LARGEFILE	0
-#endif
-
-/*--------------------------------------------------------------------
- * Get a storage file.
- *
- * The fn argument can be an existing file, an existing directory or
- * a nonexistent filename in an existing directory.
- *
- * If a directory is specified, the file will be anonymous (unlinked)
- *
- * Return:
- *	 0 if the file was preexisting.
- *	 1 if the file was created.
- *	 2 if the file is anonymous.
- *
- * Uses ARGV_ERR to exit in case of trouble.
- */
-
-int
-STV_GetFile(const char *fn, int *fdp, const char **fnp, const char *ctx)
-{
-	int fd;
-	struct stat st;
-	int retval = 1;
-	char buf[FILENAME_MAX];
-
-	AN(fn);
-	AN(fnp);
-	AN(fdp);
-	*fnp = NULL;
-	*fdp = -1;
-
-	/* try to create a new file of this name */
-	fd = open(fn, O_RDWR | O_CREAT | O_EXCL | O_LARGEFILE, 0600);
-	if (fd >= 0) {
-		*fdp = fd;
-		*fnp = fn;
-		return (retval);
-	}
-
-	if (stat(fn, &st))
-		ARGV_ERR(
-		    "(%s) \"%s\" does not exist and could not be created\n",
-		    ctx, fn);
-
-	if (S_ISDIR(st.st_mode)) {
-		bprintf(buf, "%s/varnish.XXXXXX", fn);
-		fd = mkstemp(buf);
-		if (fd < 0)
-			ARGV_ERR("(%s) \"%s\" mkstemp(%s) failed (%s)\n",
-			    ctx, fn, buf, strerror(errno));
-		AZ(unlink(buf));
-		*fnp = strdup(buf);
-		AN(*fnp);
-		retval = 2;
-	} else if (S_ISREG(st.st_mode)) {
-		fd = open(fn, O_RDWR | O_LARGEFILE);
-		if (fd < 0)
-			ARGV_ERR("(%s) \"%s\" could not open (%s)\n",
-			    ctx, fn, strerror(errno));
-		*fnp = fn;
-		retval = 0;
-	} else
-		ARGV_ERR(
-		    "(%s) \"%s\" is neither file nor directory\n", ctx, fn);
-
-	AZ(fstat(fd, &st));
-	if (!S_ISREG(st.st_mode))
-		ARGV_ERR("(%s) \"%s\" was not a file after opening\n",
-		    ctx, fn);
-
-	*fdp = fd;
-	return (retval);
-}
-
-/*--------------------------------------------------------------------
- * Figure out how much space is in a filesystem
- */
-
-static uintmax_t
-stv_fsspace(int fd, unsigned *bs)
-{
-	uintmax_t bsize, bavail;
-#if defined(HAVE_SYS_STATVFS_H)
-	struct statvfs fsst;
-
-	AZ(fstatvfs(fd, &fsst));
-	bsize = fsst.f_frsize;
-	bavail = fsst.f_bavail;
-#elif defined(HAVE_SYS_MOUNT_H) || defined(HAVE_SYS_VFS_H)
-	struct statfs fsst;
-
-	AZ(fstatfs(sc->fd, &fsst));
-	bsize = fsst.f_bsize;
-	bavail = fsst.f_bavail;
-#else
-#error no struct statfs / struct statvfs
-#endif
-
-	/* We use units of the larger of filesystem blocksize and pagesize */
-	if (*bs < bsize)
-		*bs = bsize;
-	xxxassert(*bs % bsize == 0);
-	return (bsize * bavail);
-}
-
-
-/*--------------------------------------------------------------------
- * Decide file size.
- *
- * If the sizespecification is empty and the file exists with non-zero
- * size, use that, otherwise, interpret the specification.
- *
- * Handle off_t sizes and pointer width limitations.
- */
-
-uintmax_t
-STV_FileSize(int fd, const char *size, unsigned *granularity, const char *ctx)
-{
-	uintmax_t l, fssize;
-	unsigned bs;
-	const char *q;
-	int i;
-	off_t o;
-	struct stat st;
-
-	AZ(fstat(fd, &st));
-	xxxassert(S_ISREG(st.st_mode));
-
-	bs = *granularity;
-	fssize = stv_fsspace(fd, &bs);
-	xxxassert(bs % *granularity == 0);
-
-	if ((size == NULL || *size == '\0') && st.st_size != 0) {
-		/*
-		 * We have no size specification, but an existing file,
-		 * use its existing size.
-		 */
-		l = st.st_size;
-	} else {
-		AN(size);
-		q = VNUM_2bytes(size, &l, fssize);
-
-		if (q != NULL)
-			ARGV_ERR("(%s) size \"%s\": %s\n", size, ctx, q);
-
-		if (l < 1024*1024)
-			ARGV_ERR("(-spersistent) size \"%s\": too small, "
-				 "did you forget to specify M or G?\n", size);
-	}
-
-	/*
-	 * This trickery wouldn't be necessary if X/Open would
-	 * just add OFF_MAX to <limits.h>...
-	 */
-	i = 0;
-	while(1) {
-		o = l;
-		if (o == l && o > 0)
-			break;
-		l >>= 1;
-		i++;
-	}
-	if (i)
-		fprintf(stderr, "WARNING: (%s) file size reduced"
-		    " to %ju due to system \"off_t\" limitations\n", ctx, l);
-	else if (l - st.st_size > fssize) {
-		l = fssize * 80 / 100;
-		fprintf(stderr, "WARNING: (%s) file size reduced"
-		    " to %ju (80%% of available disk space)\n", ctx, l);
-	}
-
-	if (sizeof(void *) == 4 && l > INT32_MAX) { /*lint !e506 !e774 !e845 */
-		fprintf(stderr,
-		    "NB: Storage size limited to 2GB on 32 bit architecture,\n"
-		    "NB: otherwise we could run out of address space.\n"
-		);
-		l = INT32_MAX;
-	}
-
-	/* round down to multiple of filesystem blocksize or pagesize */
-	l -= (l % bs);
-
-	*granularity = bs;
-	return(l);
-}
diff --git a/bin/varnishd/storage.h b/bin/varnishd/storage.h
deleted file mode 100644
index 80cad13..0000000
--- a/bin/varnishd/storage.h
+++ /dev/null
@@ -1,97 +0,0 @@
-/*-
- * Copyright (c) 2006 Verdens Gang AS
- * Copyright (c) 2006-2011 Varnish Software AS
- * All rights reserved.
- *
- * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- *
- * This defines the backend interface between the stevedore and the
- * pluggable storage implementations.
- *
- */
-
-struct stv_objsecrets;
-struct stevedore;
-struct sess;
-struct lru;
-
-typedef void storage_init_f(struct stevedore *, int ac, char * const *av);
-typedef void storage_open_f(const struct stevedore *);
-typedef struct storage *storage_alloc_f(struct stevedore *, size_t size);
-typedef void storage_trim_f(struct storage *, size_t size);
-typedef void storage_free_f(struct storage *);
-typedef struct object *storage_allocobj_f(struct stevedore *, struct sess *sp,
-    unsigned ltot, const struct stv_objsecrets *);
-typedef void storage_close_f(const struct stevedore *);
-
-/* Prototypes for VCL variable responders */
-#define VRTSTVTYPE(ct) typedef ct storage_var_##ct(const struct stevedore *);
-#include "tbl/vrt_stv_var.h"
-#undef VRTSTVTYPE
-
-/*--------------------------------------------------------------------*/
-
-struct stevedore {
-	unsigned		magic;
-#define STEVEDORE_MAGIC		0x4baf43db
-	const char		*name;
-	unsigned		transient;
-	storage_init_f		*init;		/* called by mgt process */
-	storage_open_f		*open;		/* called by cache process */
-	storage_alloc_f		*alloc;		/* --//-- */
-	storage_trim_f		*trim;		/* --//-- */
-	storage_free_f		*free;		/* --//-- */
-	storage_close_f		*close;		/* --//-- */
-	storage_allocobj_f	*allocobj;	/* --//-- */
-
-	struct lru		*lru;
-
-#define VRTSTVVAR(nm, vtype, ctype, dval) storage_var_##ctype *var_##nm;
-#include "tbl/vrt_stv_var.h"
-#undef VRTSTVVAR
-
-	/* private fields */
-	void			*priv;
-
-	VTAILQ_ENTRY(stevedore)	list;
-	char			ident[16];	/* XXX: match VSM_chunk.ident */
-};
-
-/*--------------------------------------------------------------------*/
-int STV_GetFile(const char *fn, int *fdp, const char **fnp, const char *ctx);
-uintmax_t STV_FileSize(int fd, const char *size, unsigned *granularity,
-    const char *ctx);
-struct object *STV_MkObject(struct sess *sp, void *ptr, unsigned ltot,
-    const struct stv_objsecrets *soc);
-
-struct lru *LRU_Alloc(void);
-void LRU_Free(struct lru *lru);
-
-/*--------------------------------------------------------------------*/
-extern const struct stevedore sma_stevedore;
-extern const struct stevedore smf_stevedore;
-extern const struct stevedore smp_stevedore;
-#ifdef HAVE_LIBUMEM
-extern const struct stevedore smu_stevedore;
-#endif
diff --git a/bin/varnishd/storage/stevedore.c b/bin/varnishd/storage/stevedore.c
new file mode 100644
index 0000000..65f9f49
--- /dev/null
+++ b/bin/varnishd/storage/stevedore.c
@@ -0,0 +1,602 @@
+/*-
+ * Copyright (c) 2007-2011 Varnish Software AS
+ * All rights reserved.
+ *
+ * Author: Dag-Erling Sm?rgav <des at des.no>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * STEVEDORE: one who works at or is responsible for loading and
+ * unloading ships in port.  Example: "on the wharves, stevedores were
+ * unloading cargo from the far corners of the world." Origin: Spanish
+ * estibador, from estibar to pack.  First Known Use: 1788
+ */
+
+#include "config.h"
+
+#include <stdio.h>
+#include <stdlib.h>
+
+#include "cache.h"
+
+#include "storage/storage.h"
+#include "vav.h"
+#include "vcli_priv.h"
+#include "vrt.h"
+#include "vrt_obj.h"
+
+static VTAILQ_HEAD(, stevedore)	stevedores =
+    VTAILQ_HEAD_INITIALIZER(stevedores);
+
+static const struct stevedore * volatile stv_next;
+
+static struct stevedore *stv_transient;
+
+/*---------------------------------------------------------------------
+ * Default objcore methods
+ */
+
+static struct object * __match_proto__(getobj_f)
+default_oc_getobj(struct worker *wrk, struct objcore *oc)
+{
+	struct object *o;
+
+	(void)wrk;
+	if (oc->priv == NULL)
+		return (NULL);
+	CAST_OBJ_NOTNULL(o, oc->priv, OBJECT_MAGIC);
+	return (o);
+}
+
+static void
+default_oc_freeobj(struct objcore *oc)
+{
+	struct object *o;
+
+	CAST_OBJ_NOTNULL(o, oc->priv, OBJECT_MAGIC);
+	oc->priv = NULL;
+	oc->methods = NULL;
+
+	STV_Freestore(o);
+	STV_free(o->objstore);
+}
+
+static struct lru *
+default_oc_getlru(const struct objcore *oc)
+{
+	struct object *o;
+
+	CAST_OBJ_NOTNULL(o, oc->priv, OBJECT_MAGIC);
+	return (o->objstore->stevedore->lru);
+}
+
+static struct objcore_methods default_oc_methods = {
+	.getobj = default_oc_getobj,
+	.freeobj = default_oc_freeobj,
+	.getlru = default_oc_getlru,
+};
+
+
+/*--------------------------------------------------------------------
+ */
+
+struct lru *
+LRU_Alloc(void)
+{
+	struct lru *l;
+
+	ALLOC_OBJ(l, LRU_MAGIC);
+	AN(l);
+	VTAILQ_INIT(&l->lru_head);
+	Lck_New(&l->mtx, lck_lru);
+	return (l);
+}
+
+void
+LRU_Free(struct lru *lru)
+{
+	CHECK_OBJ_NOTNULL(lru, LRU_MAGIC);
+	Lck_Delete(&lru->mtx);
+	FREE_OBJ(lru);
+}
+
+/*--------------------------------------------------------------------
+ * XXX: trust pointer writes to be atomic
+ */
+
+static struct stevedore *
+stv_pick_stevedore(const struct sess *sp, const char **hint)
+{
+	struct stevedore *stv;
+
+	AN(hint);
+	if (*hint != NULL && **hint != '\0') {
+		VTAILQ_FOREACH(stv, &stevedores, list) {
+			if (!strcmp(stv->ident, *hint))
+				return (stv);
+		}
+		if (!strcmp(TRANSIENT_STORAGE, *hint))
+			return (stv_transient);
+
+		/* Hint was not valid, nuke it */
+		WSP(sp, SLT_Debug, "Storage hint not usable");
+		*hint = NULL;
+	}
+	/* pick a stevedore and bump the head along */
+	stv = VTAILQ_NEXT(stv_next, list);
+	if (stv == NULL)
+		stv = VTAILQ_FIRST(&stevedores);
+	AN(stv);
+	AN(stv->name);
+	stv_next = stv;
+	return (stv);
+}
+
+/*-------------------------------------------------------------------*/
+
+static struct storage *
+stv_alloc(const struct sess *sp, size_t size)
+{
+	struct storage *st;
+	struct stevedore *stv;
+	unsigned fail = 0;
+
+	/*
+	 * Always use the stevedore which allocated the object in order to
+	 * keep an object inside the same stevedore.
+	 */
+	CHECK_OBJ_NOTNULL(sp->obj, OBJECT_MAGIC);
+	stv = sp->obj->objstore->stevedore;
+	CHECK_OBJ_NOTNULL(stv, STEVEDORE_MAGIC);
+
+	if (size > (size_t)(params->fetch_maxchunksize) << 10)
+		size = (size_t)(params->fetch_maxchunksize) << 10;
+
+	for (;;) {
+		/* try to allocate from it */
+		AN(stv->alloc);
+		st = stv->alloc(stv, size);
+		if (st != NULL)
+			break;
+
+		if (size > params->fetch_chunksize * 1024LL) {
+			size >>= 1;
+			continue;
+		}
+
+		/* no luck; try to free some space and keep trying */
+		if (EXP_NukeOne(sp, stv->lru) == -1)
+			break;
+
+		/* Enough is enough: try another if we have one */
+		if (++fail >= params->nuke_limit)
+			break;
+	}
+	if (st != NULL)
+		CHECK_OBJ_NOTNULL(st, STORAGE_MAGIC);
+	return (st);
+}
+
+
+/*-------------------------------------------------------------------*
+ * Structure used to transport internal knowledge from STV_NewObject()
+ * to STV_MkObject().  Nobody else should mess with this struct.
+ */
+
+struct stv_objsecrets {
+	unsigned	magic;
+#define STV_OBJ_SECRETES_MAGIC	0x78c87247
+	uint16_t	nhttp;
+	unsigned	lhttp;
+	unsigned	wsl;
+	struct exp	*exp;
+};
+
+/*--------------------------------------------------------------------
+ * This function is called by stevedores ->allocobj() method, which
+ * very often will be stv_default_allocobj() below, to convert a slab
+ * of storage into object which the stevedore can then register in its
+ * internal state, before returning it to STV_NewObject().
+ * As you probably guessed: All this for persistence.
+ */
+
+struct object *
+STV_MkObject(struct sess *sp, void *ptr, unsigned ltot,
+    const struct stv_objsecrets *soc)
+{
+	struct object *o;
+	unsigned l;
+
+	CHECK_OBJ_NOTNULL(soc, STV_OBJ_SECRETES_MAGIC);
+
+	assert(PAOK(ptr));
+	assert(PAOK(soc->wsl));
+	assert(PAOK(soc->lhttp));
+
+	assert(ltot >= sizeof *o + soc->lhttp + soc->wsl);
+
+	o = ptr;
+	memset(o, 0, sizeof *o);
+	o->magic = OBJECT_MAGIC;
+
+	l = PRNDDN(ltot - (sizeof *o + soc->lhttp));
+	assert(l >= soc->wsl);
+
+	o->http = HTTP_create(o + 1, soc->nhttp);
+	WS_Init(o->ws_o, "obj", (char *)(o + 1) + soc->lhttp, soc->wsl);
+	WS_Assert(o->ws_o);
+	assert(o->ws_o->e <= (char*)ptr + ltot);
+
+	http_Setup(o->http, o->ws_o);
+	o->http->magic = HTTP_MAGIC;
+	o->exp = *soc->exp;
+	VTAILQ_INIT(&o->store);
+	sp->wrk->stats.n_object++;
+
+	if (sp->objcore != NULL) {
+		CHECK_OBJ_NOTNULL(sp->objcore, OBJCORE_MAGIC);
+
+		o->objcore = sp->objcore;
+		sp->objcore = NULL;     /* refcnt follows pointer. */
+		BAN_NewObjCore(o->objcore);
+
+		o->objcore->methods = &default_oc_methods;
+		o->objcore->priv = o;
+	}
+	return (o);
+}
+
+/*--------------------------------------------------------------------
+ * This is the default ->allocobj() which all stevedores who do not
+ * implement persistent storage can rely on.
+ */
+
+static struct object *
+stv_default_allocobj(struct stevedore *stv, struct sess *sp, unsigned ltot,
+    const struct stv_objsecrets *soc)
+{
+	struct object *o;
+	struct storage *st;
+
+	CHECK_OBJ_NOTNULL(soc, STV_OBJ_SECRETES_MAGIC);
+	st = stv->alloc(stv, ltot);
+	if (st == NULL)
+		return (NULL);
+	if (st->space < ltot) {
+		stv->free(st);
+		return (NULL);
+	}
+	ltot = st->len = st->space;
+	o = STV_MkObject(sp, st->ptr, ltot, soc);
+	CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC);
+	o->objstore = st;
+	return (o);
+}
+
+/*-------------------------------------------------------------------
+ * Allocate storage for an object, based on the header information.
+ * XXX: If we know (a hint of) the length, we could allocate space
+ * XXX: for the body in the same allocation while we are at it.
+ */
+
+struct object *
+STV_NewObject(struct sess *sp, const char *hint, unsigned wsl, struct exp *ep,
+    uint16_t nhttp)
+{
+	struct object *o;
+	struct stevedore *stv, *stv0;
+	unsigned lhttp, ltot;
+	struct stv_objsecrets soc;
+	int i;
+
+	assert(wsl > 0);
+	wsl = PRNDUP(wsl);
+
+	lhttp = HTTP_estimate(nhttp);
+	lhttp = PRNDUP(lhttp);
+
+	memset(&soc, 0, sizeof soc);
+	soc.magic = STV_OBJ_SECRETES_MAGIC;
+	soc.nhttp = nhttp;
+	soc.lhttp = lhttp;
+	soc.wsl = wsl;
+	soc.exp = ep;
+
+	ltot = sizeof *o + wsl + lhttp;
+
+	stv = stv0 = stv_pick_stevedore(sp, &hint);
+	AN(stv->allocobj);
+	o = stv->allocobj(stv, sp, ltot, &soc);
+	if (o == NULL && hint == NULL) {
+		do {
+			stv = stv_pick_stevedore(sp, &hint);
+			AN(stv->allocobj);
+			o = stv->allocobj(stv, sp, ltot, &soc);
+		} while (o == NULL && stv != stv0);
+	}
+	if (o == NULL) {
+		/* no luck; try to free some space and keep trying */
+		for (i = 0; o == NULL && i < params->nuke_limit; i++) {
+			if (EXP_NukeOne(sp, stv->lru) == -1)
+				break;
+			o = stv->allocobj(stv, sp, ltot, &soc);
+		}
+	}
+
+	if (o == NULL)
+		return (NULL);
+	CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC);
+	CHECK_OBJ_NOTNULL(o->objstore, STORAGE_MAGIC);
+	return (o);
+}
+
+/*-------------------------------------------------------------------*/
+
+void
+STV_Freestore(struct object *o)
+{
+	struct storage *st, *stn;
+
+	if (o->esidata != NULL) {
+		STV_free(o->esidata);
+		o->esidata = NULL;
+	}
+	VTAILQ_FOREACH_SAFE(st, &o->store, list, stn) {
+		CHECK_OBJ_NOTNULL(st, STORAGE_MAGIC);
+		VTAILQ_REMOVE(&o->store, st, list);
+		STV_free(st);
+	}
+}
+
+/*-------------------------------------------------------------------*/
+
+struct storage *
+STV_alloc(const struct sess *sp, size_t size)
+{
+
+	return (stv_alloc(sp, size));
+}
+
+void
+STV_trim(struct storage *st, size_t size)
+{
+
+	CHECK_OBJ_NOTNULL(st, STORAGE_MAGIC);
+	AN(st->stevedore);
+	if (st->stevedore->trim)
+		st->stevedore->trim(st, size);
+}
+
+void
+STV_free(struct storage *st)
+{
+
+	CHECK_OBJ_NOTNULL(st, STORAGE_MAGIC);
+	AN(st->stevedore);
+	AN(st->stevedore->free);
+	st->stevedore->free(st);
+}
+
+void
+STV_open(void)
+{
+	struct stevedore *stv;
+
+	VTAILQ_FOREACH(stv, &stevedores, list) {
+		stv->lru = LRU_Alloc();
+		if (stv->open != NULL)
+			stv->open(stv);
+	}
+	stv = stv_transient;
+	if (stv->open != NULL) {
+		stv->lru = LRU_Alloc();
+		stv->open(stv);
+	}
+}
+
+void
+STV_close(void)
+{
+	struct stevedore *stv;
+
+	VTAILQ_FOREACH(stv, &stevedores, list)
+		if (stv->close != NULL)
+			stv->close(stv);
+	stv = stv_transient;
+	if (stv->close != NULL)
+		stv->close(stv);
+}
+
+/*--------------------------------------------------------------------
+ * Parse a stevedore argument on the form:
+ *	[ name '=' ] strategy [ ',' arg ] *
+ */
+
+static const struct choice STV_choice[] = {
+	{ "file",	&smf_stevedore },
+	{ "malloc",	&sma_stevedore },
+	{ "persistent",	&smp_stevedore },
+#ifdef HAVE_LIBUMEM
+	{ "umem",	&smu_stevedore },
+#endif
+	{ NULL,		NULL }
+};
+
+void
+STV_Config(const char *spec)
+{
+	char **av;
+	const char *p, *q;
+	struct stevedore *stv;
+	const struct stevedore *stv2;
+	int ac, l;
+	static unsigned seq = 0;
+
+	ASSERT_MGT();
+	p = strchr(spec, '=');
+	q = strchr(spec, ',');
+	if (p != NULL && (q == NULL || q > p)) {
+		av = VAV_Parse(p + 1, NULL, ARGV_COMMA);
+	} else {
+		av = VAV_Parse(spec, NULL, ARGV_COMMA);
+		p = NULL;
+	}
+	AN(av);
+
+	if (av[0] != NULL)
+		ARGV_ERR("%s\n", av[0]);
+
+	if (av[1] == NULL)
+		ARGV_ERR("-s argument lacks strategy {malloc, file, ...}\n");
+
+	for (ac = 0; av[ac + 2] != NULL; ac++)
+		continue;
+
+	stv2 = pick(STV_choice, av[1], "storage");
+	AN(stv2);
+
+	/* Append strategy to ident string */
+	VSB_printf(vident, ",-s%s", av[1]);
+
+	av += 2;
+
+	CHECK_OBJ_NOTNULL(stv2, STEVEDORE_MAGIC);
+	ALLOC_OBJ(stv, STEVEDORE_MAGIC);
+	AN(stv);
+
+	*stv = *stv2;
+	AN(stv->name);
+	AN(stv->alloc);
+	if (stv->allocobj == NULL)
+		stv->allocobj = stv_default_allocobj;
+
+	if (p == NULL)
+		bprintf(stv->ident, "s%u", seq++);
+	else {
+		l = p - spec;
+		if (l > sizeof stv->ident - 1)
+			l = sizeof stv->ident - 1;
+		bprintf(stv->ident, "%.*s", l, spec);
+	}
+
+	VTAILQ_FOREACH(stv2, &stevedores, list) {
+		if (strcmp(stv2->ident, stv->ident))
+			continue;
+		ARGV_ERR("(-s%s=%s) already defined once\n",
+		    stv->ident, stv->name);
+	}
+
+	if (stv->init != NULL)
+		stv->init(stv, ac, av);
+	else if (ac != 0)
+		ARGV_ERR("(-s%s) too many arguments\n", stv->name);
+
+	if (!strcmp(stv->ident, TRANSIENT_STORAGE)) {
+		stv->transient = 1;
+		AZ(stv_transient);
+		stv_transient = stv;
+	} else {
+		VTAILQ_INSERT_TAIL(&stevedores, stv, list);
+		if (!stv_next)
+			stv_next = VTAILQ_FIRST(&stevedores);
+	}
+}
+
+/*--------------------------------------------------------------------*/
+
+void
+STV_Config_Transient(void)
+{
+
+	ASSERT_MGT();
+
+	if (stv_transient == NULL)
+		STV_Config(TRANSIENT_STORAGE "=malloc");
+}
+
+/*--------------------------------------------------------------------*/
+
+static void
+stv_cli_list(struct cli *cli, const char * const *av, void *priv)
+{
+	struct stevedore *stv;
+
+	ASSERT_MGT();
+	(void)av;
+	(void)priv;
+	VCLI_Out(cli, "Storage devices:\n");
+	stv = stv_transient;
+		VCLI_Out(cli, "\tstorage.%s = %s\n", stv->ident, stv->name);
+	VTAILQ_FOREACH(stv, &stevedores, list)
+		VCLI_Out(cli, "\tstorage.%s = %s\n", stv->ident, stv->name);
+}
+
+/*--------------------------------------------------------------------*/
+
+struct cli_proto cli_stv[] = {
+	{ "storage.list", "storage.list", "List storage devices\n",
+	    0, 0, "", stv_cli_list },
+	{ NULL}
+};
+
+/*--------------------------------------------------------------------
+ * VRT functions for stevedores
+ */
+
+static const struct stevedore *
+stv_find(const char *nm)
+{
+	const struct stevedore *stv;
+
+	VTAILQ_FOREACH(stv, &stevedores, list)
+		if (!strcmp(stv->ident, nm))
+			return (stv);
+	if (!strcmp(TRANSIENT_STORAGE, nm))
+		return (stv_transient);
+	return (NULL);
+}
+
+int
+VRT_Stv(const char *nm)
+{
+
+	if (stv_find(nm) != NULL)
+		return (1);
+	return (0);
+}
+
+#define VRTSTVVAR(nm, vtype, ctype, dval)	\
+ctype						\
+VRT_Stv_##nm(const char *nm)			\
+{						\
+	const struct stevedore *stv;		\
+						\
+	stv = stv_find(nm);			\
+	if (stv == NULL)			\
+		return (dval);			\
+	if (stv->var_##nm == NULL)		\
+		return (dval);			\
+	return (stv->var_##nm(stv));		\
+}
+
+#include "tbl/vrt_stv_var.h"
+#undef VRTSTVVAR
diff --git a/bin/varnishd/storage/stevedore_utils.c b/bin/varnishd/storage/stevedore_utils.c
new file mode 100644
index 0000000..3daebf8
--- /dev/null
+++ b/bin/varnishd/storage/stevedore_utils.c
@@ -0,0 +1,244 @@
+/*-
+ * Copyright (c) 2006 Verdens Gang AS
+ * Copyright (c) 2006-2010 Varnish Software AS
+ * All rights reserved.
+ *
+ * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * Utility functions for stevedores and storage modules
+ */
+
+#include "config.h"
+
+#include <sys/types.h>
+#include <sys/stat.h>
+#ifdef HAVE_SYS_MOUNT_H
+#  include <sys/mount.h>
+#endif
+#ifdef HAVE_SYS_STATVFS_H
+#  include <sys/statvfs.h>
+#endif
+#ifdef HAVE_SYS_VFS_H
+#  include <sys/vfs.h>
+#endif
+
+#include <fcntl.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+
+#include "mgt.h"
+
+#include "storage/storage.h"
+#include "vnum.h"
+
+#ifndef O_LARGEFILE
+#define O_LARGEFILE	0
+#endif
+
+/*--------------------------------------------------------------------
+ * Get a storage file.
+ *
+ * The fn argument can be an existing file, an existing directory or
+ * a nonexistent filename in an existing directory.
+ *
+ * If a directory is specified, the file will be anonymous (unlinked)
+ *
+ * Return:
+ *	 0 if the file was preexisting.
+ *	 1 if the file was created.
+ *	 2 if the file is anonymous.
+ *
+ * Uses ARGV_ERR to exit in case of trouble.
+ */
+
+int
+STV_GetFile(const char *fn, int *fdp, const char **fnp, const char *ctx)
+{
+	int fd;
+	struct stat st;
+	int retval = 1;
+	char buf[FILENAME_MAX];
+
+	AN(fn);
+	AN(fnp);
+	AN(fdp);
+	*fnp = NULL;
+	*fdp = -1;
+
+	/* try to create a new file of this name */
+	fd = open(fn, O_RDWR | O_CREAT | O_EXCL | O_LARGEFILE, 0600);
+	if (fd >= 0) {
+		*fdp = fd;
+		*fnp = fn;
+		return (retval);
+	}
+
+	if (stat(fn, &st))
+		ARGV_ERR(
+		    "(%s) \"%s\" does not exist and could not be created\n",
+		    ctx, fn);
+
+	if (S_ISDIR(st.st_mode)) {
+		bprintf(buf, "%s/varnish.XXXXXX", fn);
+		fd = mkstemp(buf);
+		if (fd < 0)
+			ARGV_ERR("(%s) \"%s\" mkstemp(%s) failed (%s)\n",
+			    ctx, fn, buf, strerror(errno));
+		AZ(unlink(buf));
+		*fnp = strdup(buf);
+		AN(*fnp);
+		retval = 2;
+	} else if (S_ISREG(st.st_mode)) {
+		fd = open(fn, O_RDWR | O_LARGEFILE);
+		if (fd < 0)
+			ARGV_ERR("(%s) \"%s\" could not open (%s)\n",
+			    ctx, fn, strerror(errno));
+		*fnp = fn;
+		retval = 0;
+	} else
+		ARGV_ERR(
+		    "(%s) \"%s\" is neither file nor directory\n", ctx, fn);
+
+	AZ(fstat(fd, &st));
+	if (!S_ISREG(st.st_mode))
+		ARGV_ERR("(%s) \"%s\" was not a file after opening\n",
+		    ctx, fn);
+
+	*fdp = fd;
+	return (retval);
+}
+
+/*--------------------------------------------------------------------
+ * Figure out how much space is in a filesystem
+ */
+
+static uintmax_t
+stv_fsspace(int fd, unsigned *bs)
+{
+	uintmax_t bsize, bavail;
+#if defined(HAVE_SYS_STATVFS_H)
+	struct statvfs fsst;
+
+	AZ(fstatvfs(fd, &fsst));
+	bsize = fsst.f_frsize;
+	bavail = fsst.f_bavail;
+#elif defined(HAVE_SYS_MOUNT_H) || defined(HAVE_SYS_VFS_H)
+	struct statfs fsst;
+
+	AZ(fstatfs(sc->fd, &fsst));
+	bsize = fsst.f_bsize;
+	bavail = fsst.f_bavail;
+#else
+#error no struct statfs / struct statvfs
+#endif
+
+	/* We use units of the larger of filesystem blocksize and pagesize */
+	if (*bs < bsize)
+		*bs = bsize;
+	xxxassert(*bs % bsize == 0);
+	return (bsize * bavail);
+}
+
+
+/*--------------------------------------------------------------------
+ * Decide file size.
+ *
+ * If the sizespecification is empty and the file exists with non-zero
+ * size, use that, otherwise, interpret the specification.
+ *
+ * Handle off_t sizes and pointer width limitations.
+ */
+
+uintmax_t
+STV_FileSize(int fd, const char *size, unsigned *granularity, const char *ctx)
+{
+	uintmax_t l, fssize;
+	unsigned bs;
+	const char *q;
+	int i;
+	off_t o;
+	struct stat st;
+
+	AZ(fstat(fd, &st));
+	xxxassert(S_ISREG(st.st_mode));
+
+	bs = *granularity;
+	fssize = stv_fsspace(fd, &bs);
+	xxxassert(bs % *granularity == 0);
+
+	if ((size == NULL || *size == '\0') && st.st_size != 0) {
+		/*
+		 * We have no size specification, but an existing file,
+		 * use its existing size.
+		 */
+		l = st.st_size;
+	} else {
+		AN(size);
+		q = VNUM_2bytes(size, &l, fssize);
+
+		if (q != NULL)
+			ARGV_ERR("(%s) size \"%s\": %s\n", size, ctx, q);
+
+		if (l < 1024*1024)
+			ARGV_ERR("(-spersistent) size \"%s\": too small, "
+				 "did you forget to specify M or G?\n", size);
+	}
+
+	/*
+	 * This trickery wouldn't be necessary if X/Open would
+	 * just add OFF_MAX to <limits.h>...
+	 */
+	i = 0;
+	while(1) {
+		o = l;
+		if (o == l && o > 0)
+			break;
+		l >>= 1;
+		i++;
+	}
+	if (i)
+		fprintf(stderr, "WARNING: (%s) file size reduced"
+		    " to %ju due to system \"off_t\" limitations\n", ctx, l);
+	else if (l - st.st_size > fssize) {
+		l = fssize * 80 / 100;
+		fprintf(stderr, "WARNING: (%s) file size reduced"
+		    " to %ju (80%% of available disk space)\n", ctx, l);
+	}
+
+	if (sizeof(void *) == 4 && l > INT32_MAX) { /*lint !e506 !e774 !e845 */
+		fprintf(stderr,
+		    "NB: Storage size limited to 2GB on 32 bit architecture,\n"
+		    "NB: otherwise we could run out of address space.\n"
+		);
+		l = INT32_MAX;
+	}
+
+	/* round down to multiple of filesystem blocksize or pagesize */
+	l -= (l % bs);
+
+	*granularity = bs;
+	return(l);
+}
diff --git a/bin/varnishd/storage/storage.h b/bin/varnishd/storage/storage.h
new file mode 100644
index 0000000..80cad13
--- /dev/null
+++ b/bin/varnishd/storage/storage.h
@@ -0,0 +1,97 @@
+/*-
+ * Copyright (c) 2006 Verdens Gang AS
+ * Copyright (c) 2006-2011 Varnish Software AS
+ * All rights reserved.
+ *
+ * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * This defines the backend interface between the stevedore and the
+ * pluggable storage implementations.
+ *
+ */
+
+struct stv_objsecrets;
+struct stevedore;
+struct sess;
+struct lru;
+
+typedef void storage_init_f(struct stevedore *, int ac, char * const *av);
+typedef void storage_open_f(const struct stevedore *);
+typedef struct storage *storage_alloc_f(struct stevedore *, size_t size);
+typedef void storage_trim_f(struct storage *, size_t size);
+typedef void storage_free_f(struct storage *);
+typedef struct object *storage_allocobj_f(struct stevedore *, struct sess *sp,
+    unsigned ltot, const struct stv_objsecrets *);
+typedef void storage_close_f(const struct stevedore *);
+
+/* Prototypes for VCL variable responders */
+#define VRTSTVTYPE(ct) typedef ct storage_var_##ct(const struct stevedore *);
+#include "tbl/vrt_stv_var.h"
+#undef VRTSTVTYPE
+
+/*--------------------------------------------------------------------*/
+
+struct stevedore {
+	unsigned		magic;
+#define STEVEDORE_MAGIC		0x4baf43db
+	const char		*name;
+	unsigned		transient;
+	storage_init_f		*init;		/* called by mgt process */
+	storage_open_f		*open;		/* called by cache process */
+	storage_alloc_f		*alloc;		/* --//-- */
+	storage_trim_f		*trim;		/* --//-- */
+	storage_free_f		*free;		/* --//-- */
+	storage_close_f		*close;		/* --//-- */
+	storage_allocobj_f	*allocobj;	/* --//-- */
+
+	struct lru		*lru;
+
+#define VRTSTVVAR(nm, vtype, ctype, dval) storage_var_##ctype *var_##nm;
+#include "tbl/vrt_stv_var.h"
+#undef VRTSTVVAR
+
+	/* private fields */
+	void			*priv;
+
+	VTAILQ_ENTRY(stevedore)	list;
+	char			ident[16];	/* XXX: match VSM_chunk.ident */
+};
+
+/*--------------------------------------------------------------------*/
+int STV_GetFile(const char *fn, int *fdp, const char **fnp, const char *ctx);
+uintmax_t STV_FileSize(int fd, const char *size, unsigned *granularity,
+    const char *ctx);
+struct object *STV_MkObject(struct sess *sp, void *ptr, unsigned ltot,
+    const struct stv_objsecrets *soc);
+
+struct lru *LRU_Alloc(void);
+void LRU_Free(struct lru *lru);
+
+/*--------------------------------------------------------------------*/
+extern const struct stevedore sma_stevedore;
+extern const struct stevedore smf_stevedore;
+extern const struct stevedore smp_stevedore;
+#ifdef HAVE_LIBUMEM
+extern const struct stevedore smu_stevedore;
+#endif
diff --git a/bin/varnishd/storage/storage_file.c b/bin/varnishd/storage/storage_file.c
new file mode 100644
index 0000000..58e1d3d
--- /dev/null
+++ b/bin/varnishd/storage/storage_file.c
@@ -0,0 +1,615 @@
+/*-
+ * Copyright (c) 2006 Verdens Gang AS
+ * Copyright (c) 2006-2010 Varnish Software AS
+ * All rights reserved.
+ *
+ * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * Storage method based on mmap'ed file
+ */
+
+#include "config.h"
+
+#include <sys/mman.h>
+
+#include <stdio.h>
+#include <stdlib.h>
+
+#include "cache.h"
+#include "storage/storage.h"
+
+#include "vnum.h"
+
+#ifndef MAP_NOCORE
+#define MAP_NOCORE 0 /* XXX Linux */
+#endif
+
+#ifndef MAP_NOSYNC
+#define MAP_NOSYNC 0 /* XXX Linux */
+#endif
+
+#define MINPAGES		128
+
+/*
+ * Number of buckets on free-list.
+ *
+ * Last bucket is "larger than" so choose number so that the second
+ * to last bucket matches the 128k CHUNKSIZE in cache_fetch.c when
+ * using the a 4K minimal page size
+ */
+#define NBUCKET			(128 / 4 + 1)
+
+/*--------------------------------------------------------------------*/
+
+VTAILQ_HEAD(smfhead, smf);
+
+struct smf {
+	unsigned		magic;
+#define SMF_MAGIC		0x0927a8a0
+	struct storage		s;
+	struct smf_sc		*sc;
+
+	int			alloc;
+
+	off_t			size;
+	off_t			offset;
+	unsigned char		*ptr;
+
+	VTAILQ_ENTRY(smf)	order;
+	VTAILQ_ENTRY(smf)	status;
+	struct smfhead		*flist;
+};
+
+struct smf_sc {
+	unsigned		magic;
+#define SMF_SC_MAGIC		0x52962ee7
+	struct lock		mtx;
+	struct VSC_C_smf	*stats;
+
+	const char		*filename;
+	int			fd;
+	unsigned		pagesize;
+	uintmax_t		filesize;
+	struct smfhead		order;
+	struct smfhead		free[NBUCKET];
+	struct smfhead		used;
+};
+
+/*--------------------------------------------------------------------*/
+
+static void
+smf_initfile(struct smf_sc *sc, const char *size)
+{
+	sc->filesize = STV_FileSize(sc->fd, size, &sc->pagesize, "-sfile");
+
+	AZ(ftruncate(sc->fd, (off_t)sc->filesize));
+
+	/* XXX: force block allocation here or in open ? */
+}
+
+static const char default_size[] = "100M";
+static const char default_filename[] = ".";
+
+static void
+smf_init(struct stevedore *parent, int ac, char * const *av)
+{
+	const char *size, *fn, *r;
+	struct smf_sc *sc;
+	unsigned u;
+	uintmax_t page_size;
+
+	AZ(av[ac]);
+
+	fn = default_filename;
+	size = default_size;
+	page_size = getpagesize();
+
+	if (ac > 3)
+		ARGV_ERR("(-sfile) too many arguments\n");
+	if (ac > 0 && *av[0] != '\0')
+		fn = av[0];
+	if (ac > 1 && *av[1] != '\0')
+		size = av[1];
+	if (ac > 2 && *av[2] != '\0') {
+
+		r = VNUM_2bytes(av[2], &page_size, 0);
+		if (r != NULL)
+			ARGV_ERR("(-sfile) granularity \"%s\": %s\n", av[2], r);
+	}
+
+	AN(fn);
+	AN(size);
+
+	ALLOC_OBJ(sc, SMF_SC_MAGIC);
+	XXXAN(sc);
+	VTAILQ_INIT(&sc->order);
+	for (u = 0; u < NBUCKET; u++)
+		VTAILQ_INIT(&sc->free[u]);
+	VTAILQ_INIT(&sc->used);
+	sc->pagesize = page_size;
+
+	parent->priv = sc;
+
+	(void)STV_GetFile(fn, &sc->fd, &sc->filename, "-sfile");
+
+	mgt_child_inherit(sc->fd, "storage_file");
+	smf_initfile(sc, size);
+}
+
+/*--------------------------------------------------------------------
+ * Insert/Remove from correct freelist
+ */
+
+static void
+insfree(struct smf_sc *sc, struct smf *sp)
+{
+	size_t b;
+	struct smf *sp2;
+	size_t ns;
+
+	assert(sp->alloc == 0);
+	assert(sp->flist == NULL);
+	Lck_AssertHeld(&sc->mtx);
+	b = sp->size / sc->pagesize;
+	if (b >= NBUCKET) {
+		b = NBUCKET - 1;
+		sc->stats->g_smf_large++;
+	} else {
+		sc->stats->g_smf_frag++;
+	}
+	sp->flist = &sc->free[b];
+	ns = b * sc->pagesize;
+	VTAILQ_FOREACH(sp2, sp->flist, status) {
+		assert(sp2->size >= ns);
+		assert(sp2->alloc == 0);
+		assert(sp2->flist == sp->flist);
+		if (sp->offset < sp2->offset)
+			break;
+	}
+	if (sp2 == NULL)
+		VTAILQ_INSERT_TAIL(sp->flist, sp, status);
+	else
+		VTAILQ_INSERT_BEFORE(sp2, sp, status);
+}
+
+static void
+remfree(const struct smf_sc *sc, struct smf *sp)
+{
+	size_t b;
+
+	assert(sp->alloc == 0);
+	assert(sp->flist != NULL);
+	Lck_AssertHeld(&sc->mtx);
+	b = sp->size / sc->pagesize;
+	if (b >= NBUCKET) {
+		b = NBUCKET - 1;
+		sc->stats->g_smf_large--;
+	} else {
+		sc->stats->g_smf_frag--;
+	}
+	assert(sp->flist == &sc->free[b]);
+	VTAILQ_REMOVE(sp->flist, sp, status);
+	sp->flist = NULL;
+}
+
+/*--------------------------------------------------------------------
+ * Allocate a range from the first free range that is large enough.
+ */
+
+static struct smf *
+alloc_smf(struct smf_sc *sc, size_t bytes)
+{
+	struct smf *sp, *sp2;
+	size_t b;
+
+	assert(!(bytes % sc->pagesize));
+	b = bytes / sc->pagesize;
+	if (b >= NBUCKET)
+		b = NBUCKET - 1;
+	for (sp = NULL; b < NBUCKET - 1; b++) {
+		sp = VTAILQ_FIRST(&sc->free[b]);
+		if (sp != NULL)
+			break;
+	}
+	if (sp == NULL) {
+		VTAILQ_FOREACH(sp, &sc->free[NBUCKET -1], status)
+			if (sp->size >= bytes)
+				break;
+	}
+	if (sp == NULL)
+		return (sp);
+
+	assert(sp->size >= bytes);
+	remfree(sc, sp);
+
+	if (sp->size == bytes) {
+		sp->alloc = 1;
+		VTAILQ_INSERT_TAIL(&sc->used, sp, status);
+		return (sp);
+	}
+
+	/* Split from front */
+	sp2 = malloc(sizeof *sp2);
+	XXXAN(sp2);
+	sc->stats->g_smf++;
+	*sp2 = *sp;
+
+	sp->offset += bytes;
+	sp->ptr += bytes;
+	sp->size -= bytes;
+
+	sp2->size = bytes;
+	sp2->alloc = 1;
+	VTAILQ_INSERT_BEFORE(sp, sp2, order);
+	VTAILQ_INSERT_TAIL(&sc->used, sp2, status);
+	insfree(sc, sp);
+	return (sp2);
+}
+
+/*--------------------------------------------------------------------
+ * Free a range.  Attempt merge forward and backward, then sort into
+ * free list according to age.
+ */
+
+static void
+free_smf(struct smf *sp)
+{
+	struct smf *sp2;
+	struct smf_sc *sc = sp->sc;
+
+	CHECK_OBJ_NOTNULL(sp, SMF_MAGIC);
+	assert(sp->alloc != 0);
+	assert(sp->size > 0);
+	assert(!(sp->size % sc->pagesize));
+	VTAILQ_REMOVE(&sc->used, sp, status);
+	sp->alloc = 0;
+
+	sp2 = VTAILQ_NEXT(sp, order);
+	if (sp2 != NULL &&
+	    sp2->alloc == 0 &&
+	    (sp2->ptr == sp->ptr + sp->size) &&
+	    (sp2->offset == sp->offset + sp->size)) {
+		sp->size += sp2->size;
+		VTAILQ_REMOVE(&sc->order, sp2, order);
+		remfree(sc, sp2);
+		free(sp2);
+		sc->stats->g_smf--;
+	}
+
+	sp2 = VTAILQ_PREV(sp, smfhead, order);
+	if (sp2 != NULL &&
+	    sp2->alloc == 0 &&
+	    (sp->ptr == sp2->ptr + sp2->size) &&
+	    (sp->offset == sp2->offset + sp2->size)) {
+		remfree(sc, sp2);
+		sp2->size += sp->size;
+		VTAILQ_REMOVE(&sc->order, sp, order);
+		free(sp);
+		sc->stats->g_smf--;
+		sp = sp2;
+	}
+
+	insfree(sc, sp);
+}
+
+/*--------------------------------------------------------------------
+ * Trim the tail of a range.
+ */
+
+static void
+trim_smf(struct smf *sp, size_t bytes)
+{
+	struct smf *sp2;
+	struct smf_sc *sc = sp->sc;
+
+	assert(sp->alloc != 0);
+	assert(bytes > 0);
+	assert(bytes < sp->size);
+	assert(!(bytes % sc->pagesize));
+	assert(!(sp->size % sc->pagesize));
+	CHECK_OBJ_NOTNULL(sp, SMF_MAGIC);
+	sp2 = malloc(sizeof *sp2);
+	XXXAN(sp2);
+	sc->stats->g_smf++;
+	*sp2 = *sp;
+
+	sp2->size -= bytes;
+	sp->size = bytes;
+	sp2->ptr += bytes;
+	sp2->offset += bytes;
+	VTAILQ_INSERT_AFTER(&sc->order, sp, sp2, order);
+	VTAILQ_INSERT_TAIL(&sc->used, sp2, status);
+	free_smf(sp2);
+}
+
+/*--------------------------------------------------------------------
+ * Insert a newly created range as busy, then free it to do any collapses
+ */
+
+static void
+new_smf(struct smf_sc *sc, unsigned char *ptr, off_t off, size_t len)
+{
+	struct smf *sp, *sp2;
+
+	assert(!(len % sc->pagesize));
+	sp = calloc(sizeof *sp, 1);
+	XXXAN(sp);
+	sp->magic = SMF_MAGIC;
+	sp->s.magic = STORAGE_MAGIC;
+	sc->stats->g_smf++;
+
+	sp->sc = sc;
+	sp->size = len;
+	sp->ptr = ptr;
+	sp->offset = off;
+	sp->alloc = 1;
+
+	VTAILQ_FOREACH(sp2, &sc->order, order) {
+		if (sp->ptr < sp2->ptr) {
+			VTAILQ_INSERT_BEFORE(sp2, sp, order);
+			break;
+		}
+	}
+	if (sp2 == NULL)
+		VTAILQ_INSERT_TAIL(&sc->order, sp, order);
+
+	VTAILQ_INSERT_HEAD(&sc->used, sp, status);
+
+	free_smf(sp);
+}
+
+/*--------------------------------------------------------------------*/
+
+/*
+ * XXX: This may be too aggressive and soak up too much address room.
+ * XXX: On the other hand, the user, directly or implicitly asked us to
+ * XXX: use this much storage, so we should make a decent effort.
+ * XXX: worst case (I think), malloc will fail.
+ */
+
+static void
+smf_open_chunk(struct smf_sc *sc, off_t sz, off_t off, off_t *fail, off_t *sum)
+{
+	void *p;
+	off_t h;
+
+	assert(sz != 0);
+	assert(!(sz % sc->pagesize));
+
+	if (*fail < (uintmax_t)sc->pagesize * MINPAGES)
+		return;
+
+	if (sz > 0 && sz < *fail && sz < SSIZE_MAX) {
+		p = mmap(NULL, sz, PROT_READ|PROT_WRITE,
+		    MAP_NOCORE | MAP_NOSYNC | MAP_SHARED, sc->fd, off);
+		if (p != MAP_FAILED) {
+			(void) madvise(p, sz, MADV_RANDOM);
+			(*sum) += sz;
+			new_smf(sc, p, off, sz);
+			return;
+		}
+	}
+
+	if (sz < *fail)
+		*fail = sz;
+
+	h = sz / 2;
+	if (h > SSIZE_MAX)
+		h = SSIZE_MAX;
+	h -= (h % sc->pagesize);
+
+	smf_open_chunk(sc, h, off, fail, sum);
+	smf_open_chunk(sc, sz - h, off + h, fail, sum);
+}
+
+static void
+smf_open(const struct stevedore *st)
+{
+	struct smf_sc *sc;
+	off_t fail = 1 << 30;	/* XXX: where is OFF_T_MAX ? */
+	off_t sum = 0;
+
+	CAST_OBJ_NOTNULL(sc, st->priv, SMF_SC_MAGIC);
+	sc->stats = VSM_Alloc(sizeof *sc->stats,
+	    VSC_CLASS, VSC_TYPE_SMF, st->ident);
+	Lck_New(&sc->mtx, lck_smf);
+	Lck_Lock(&sc->mtx);
+	smf_open_chunk(sc, sc->filesize, 0, &fail, &sum);
+	Lck_Unlock(&sc->mtx);
+	printf("SMF.%s mmap'ed %ju bytes of %ju\n",
+	    st->ident, (uintmax_t)sum, sc->filesize);
+
+	/* XXX */
+	if (sum < MINPAGES * (off_t)getpagesize())
+		exit (2);
+
+	sc->stats->g_space += sc->filesize;
+}
+
+/*--------------------------------------------------------------------*/
+
+static struct storage *
+smf_alloc(struct stevedore *st, size_t size)
+{
+	struct smf *smf;
+	struct smf_sc *sc;
+
+	CAST_OBJ_NOTNULL(sc, st->priv, SMF_SC_MAGIC);
+	assert(size > 0);
+	size += (sc->pagesize - 1);
+	size &= ~(sc->pagesize - 1);
+	Lck_Lock(&sc->mtx);
+	sc->stats->c_req++;
+	smf = alloc_smf(sc, size);
+	if (smf == NULL) {
+		sc->stats->c_fail++;
+		Lck_Unlock(&sc->mtx);
+		return (NULL);
+	}
+	CHECK_OBJ_NOTNULL(smf, SMF_MAGIC);
+	sc->stats->g_alloc++;
+	sc->stats->c_bytes += smf->size;
+	sc->stats->g_bytes += smf->size;
+	sc->stats->g_space -= smf->size;
+	Lck_Unlock(&sc->mtx);
+	CHECK_OBJ_NOTNULL(&smf->s, STORAGE_MAGIC);	/*lint !e774 */
+	XXXAN(smf);
+	assert(smf->size == size);
+	smf->s.space = size;
+	smf->s.priv = smf;
+	smf->s.ptr = smf->ptr;
+	smf->s.len = 0;
+	smf->s.stevedore = st;
+#ifdef SENDFILE_WORKS
+	smf->s.fd = smf->sc->fd;
+	smf->s.where = smf->offset;
+#endif
+	return (&smf->s);
+}
+
+/*--------------------------------------------------------------------*/
+
+static void
+smf_trim(struct storage *s, size_t size)
+{
+	struct smf *smf;
+	struct smf_sc *sc;
+
+	CHECK_OBJ_NOTNULL(s, STORAGE_MAGIC);
+	assert(size > 0);
+	assert(size <= s->space);
+	xxxassert(size > 0);	/* XXX: seen */
+	CAST_OBJ_NOTNULL(smf, s->priv, SMF_MAGIC);
+	assert(size <= smf->size);
+	sc = smf->sc;
+	size += (sc->pagesize - 1);
+	size &= ~(sc->pagesize - 1);
+	if (smf->size > size) {
+		Lck_Lock(&sc->mtx);
+		sc->stats->c_freed += (smf->size - size);
+		sc->stats->g_bytes -= (smf->size - size);
+		sc->stats->g_space += (smf->size - size);
+		trim_smf(smf, size);
+		assert(smf->size == size);
+		Lck_Unlock(&sc->mtx);
+		s->space = size;
+	}
+}
+
+/*--------------------------------------------------------------------*/
+
+static void __match_proto__(storage_free_f)
+smf_free(struct storage *s)
+{
+	struct smf *smf;
+	struct smf_sc *sc;
+
+	CHECK_OBJ_NOTNULL(s, STORAGE_MAGIC);
+	CAST_OBJ_NOTNULL(smf, s->priv, SMF_MAGIC);
+	sc = smf->sc;
+	Lck_Lock(&sc->mtx);
+	sc->stats->g_alloc--;
+	sc->stats->c_freed += smf->size;
+	sc->stats->g_bytes -= smf->size;
+	sc->stats->g_space += smf->size;
+	free_smf(smf);
+	Lck_Unlock(&sc->mtx);
+}
+
+/*--------------------------------------------------------------------*/
+
+const struct stevedore smf_stevedore = {
+	.magic	=	STEVEDORE_MAGIC,
+	.name	=	"file",
+	.init	=	smf_init,
+	.open	=	smf_open,
+	.alloc	=	smf_alloc,
+	.trim	=	smf_trim,
+	.free	=	smf_free,
+};
+
+#ifdef INCLUDE_TEST_DRIVER
+
+void vca_flush(struct sess *sp) {}
+
+#define N	100
+#define M	(128*1024)
+
+struct storage *s[N];
+
+static void
+dumpit(void)
+{
+	struct smf_sc *sc = smf_stevedore.priv;
+	struct smf *s;
+
+	return (0);
+	printf("----------------\n");
+	printf("Order:\n");
+	VTAILQ_FOREACH(s, &sc->order, order) {
+		printf("%10p %12ju %12ju %12ju\n",
+		    s, s->offset, s->size, s->offset + s->size);
+	}
+	printf("Used:\n");
+	VTAILQ_FOREACH(s, &sc->used, status) {
+		printf("%10p %12ju %12ju %12ju\n",
+		    s, s->offset, s->size, s->offset + s->size);
+	}
+	printf("Free:\n");
+	VTAILQ_FOREACH(s, &sc->free, status) {
+		printf("%10p %12ju %12ju %12ju\n",
+		    s, s->offset, s->size, s->offset + s->size);
+	}
+	printf("================\n");
+}
+
+int
+main(int argc, char **argv)
+{
+	int i, j;
+
+	setbuf(stdout, NULL);
+	smf_init(&smf_stevedore, "");
+	smf_open(&smf_stevedore);
+	while (1) {
+		dumpit();
+		i = random() % N;
+		do
+			j = random() % M;
+		while (j == 0);
+		if (s[i] == NULL) {
+			s[i] = smf_alloc(&smf_stevedore, j);
+			printf("A %10p %12d\n", s[i], j);
+		} else if (j < s[i]->space) {
+			smf_trim(s[i], j);
+			printf("T %10p %12d\n", s[i], j);
+		} else {
+			smf_free(s[i]);
+			printf("D %10p\n", s[i]);
+			s[i] = NULL;
+		}
+	}
+}
+
+#endif /* INCLUDE_TEST_DRIVER */
diff --git a/bin/varnishd/storage/storage_malloc.c b/bin/varnishd/storage/storage_malloc.c
new file mode 100644
index 0000000..03d6a55
--- /dev/null
+++ b/bin/varnishd/storage/storage_malloc.c
@@ -0,0 +1,256 @@
+/*-
+ * Copyright (c) 2006 Verdens Gang AS
+ * Copyright (c) 2006-2011 Varnish Software AS
+ * All rights reserved.
+ *
+ * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * Storage method based on malloc(3)
+ */
+
+#include "config.h"
+
+#include <stdio.h>
+#include <stdlib.h>
+
+#include "cache.h"
+#include "storage/storage.h"
+
+#include "vnum.h"
+
+struct sma_sc {
+	unsigned		magic;
+#define SMA_SC_MAGIC		0x1ac8a345
+	struct lock		sma_mtx;
+	size_t			sma_max;
+	size_t			sma_alloc;
+	struct VSC_C_sma	*stats;
+};
+
+struct sma {
+	unsigned		magic;
+#define SMA_MAGIC		0x69ae9bb9
+	struct storage		s;
+	size_t			sz;
+	struct sma_sc		*sc;
+};
+
+static struct storage *
+sma_alloc(struct stevedore *st, size_t size)
+{
+	struct sma_sc *sma_sc;
+	struct sma *sma = NULL;
+	void *p;
+
+	CAST_OBJ_NOTNULL(sma_sc, st->priv, SMA_SC_MAGIC);
+	Lck_Lock(&sma_sc->sma_mtx);
+	sma_sc->stats->c_req++;
+	if (sma_sc->sma_alloc + size > sma_sc->sma_max) {
+		sma_sc->stats->c_fail += size;
+		size = 0;
+	} else {
+		sma_sc->sma_alloc += size;
+		sma_sc->stats->c_bytes += size;
+		sma_sc->stats->g_alloc++;
+		sma_sc->stats->g_bytes += size;
+		if (sma_sc->sma_max != SIZE_MAX)
+			sma_sc->stats->g_space -= size;
+	}
+	Lck_Unlock(&sma_sc->sma_mtx);
+
+	if (size == 0)
+		return (NULL);
+
+	/*
+	 * Do not collaps the sma allocation with sma->s.ptr: it is not
+	 * a good idea.  Not only would it make ->trim impossible,
+	 * performance-wise it would be a catastropy with chunksized
+	 * allocations growing another full page, just to accomodate the sma.
+	 */
+
+	p = malloc(size);
+	if (p != NULL) {
+		ALLOC_OBJ(sma, SMA_MAGIC);
+		if (sma != NULL)
+			sma->s.ptr = p;
+		else
+			free(p);
+	}
+	if (sma == NULL) {
+		Lck_Lock(&sma_sc->sma_mtx);
+		/*
+		 * XXX: Not nice to have counters go backwards, but we do
+		 * XXX: Not want to pick up the lock twice just for stats.
+		 */
+		sma_sc->stats->c_fail++;
+		sma_sc->stats->c_bytes -= size;
+		sma_sc->stats->g_alloc--;
+		sma_sc->stats->g_bytes -= size;
+		if (sma_sc->sma_max != SIZE_MAX)
+			sma_sc->stats->g_space += size;
+		Lck_Unlock(&sma_sc->sma_mtx);
+		return (NULL);
+	}
+	sma->sc = sma_sc;
+	sma->sz = size;
+	sma->s.priv = sma;
+	sma->s.len = 0;
+	sma->s.space = size;
+#ifdef SENDFILE_WORKS
+	sma->s.fd = -1;
+#endif
+	sma->s.stevedore = st;
+	sma->s.magic = STORAGE_MAGIC;
+	return (&sma->s);
+}
+
+static void __match_proto__(storage_free_f)
+sma_free(struct storage *s)
+{
+	struct sma_sc *sma_sc;
+	struct sma *sma;
+
+	CHECK_OBJ_NOTNULL(s, STORAGE_MAGIC);
+	CAST_OBJ_NOTNULL(sma, s->priv, SMA_MAGIC);
+	sma_sc = sma->sc;
+	assert(sma->sz == sma->s.space);
+	Lck_Lock(&sma_sc->sma_mtx);
+	sma_sc->sma_alloc -= sma->sz;
+	sma_sc->stats->g_alloc--;
+	sma_sc->stats->g_bytes -= sma->sz;
+	sma_sc->stats->c_freed += sma->sz;
+	if (sma_sc->sma_max != SIZE_MAX)
+		sma_sc->stats->g_space += sma->sz;
+	Lck_Unlock(&sma_sc->sma_mtx);
+	free(sma->s.ptr);
+	free(sma);
+}
+
+static void
+sma_trim(struct storage *s, size_t size)
+{
+	struct sma_sc *sma_sc;
+	struct sma *sma;
+	void *p;
+	size_t delta;
+
+	CHECK_OBJ_NOTNULL(s, STORAGE_MAGIC);
+	CAST_OBJ_NOTNULL(sma, s->priv, SMA_MAGIC);
+	sma_sc = sma->sc;
+
+	assert(sma->sz == sma->s.space);
+	assert(size < sma->sz);
+	delta = sma->sz - size;
+	if (delta < 256)
+		return;
+	if ((p = realloc(sma->s.ptr, size)) != NULL) {
+		Lck_Lock(&sma_sc->sma_mtx);
+		sma_sc->sma_alloc -= delta;
+		sma_sc->stats->g_bytes -= delta;
+		sma_sc->stats->c_freed += delta;
+		if (sma_sc->sma_max != SIZE_MAX)
+			sma_sc->stats->g_space += delta;
+		sma->sz = size;
+		Lck_Unlock(&sma_sc->sma_mtx);
+		sma->s.ptr = p;
+		s->space = size;
+	}
+}
+
+static double
+sma_used_space(const struct stevedore *st)
+{
+	struct sma_sc *sma_sc;
+
+	CAST_OBJ_NOTNULL(sma_sc, st->priv, SMA_SC_MAGIC);
+	return (sma_sc->sma_alloc);
+}
+
+static double
+sma_free_space(const struct stevedore *st)
+{
+	struct sma_sc *sma_sc;
+
+	CAST_OBJ_NOTNULL(sma_sc, st->priv, SMA_SC_MAGIC);
+	return (sma_sc->sma_max - sma_sc->sma_alloc);
+}
+
+static void
+sma_init(struct stevedore *parent, int ac, char * const *av)
+{
+	const char *e;
+	uintmax_t u;
+	struct sma_sc *sc;
+
+	ASSERT_MGT();
+	ALLOC_OBJ(sc, SMA_SC_MAGIC);
+	AN(sc);
+	sc->sma_max = SIZE_MAX;
+	assert(sc->sma_max == SIZE_MAX);
+	parent->priv = sc;
+
+	AZ(av[ac]);
+	if (ac > 1)
+		ARGV_ERR("(-smalloc) too many arguments\n");
+
+	if (ac == 0 || *av[0] == '\0')
+		 return;
+
+	e = VNUM_2bytes(av[0], &u, 0);
+	if (e != NULL)
+		ARGV_ERR("(-smalloc) size \"%s\": %s\n", av[0], e);
+	if ((u != (uintmax_t)(size_t)u))
+		ARGV_ERR("(-smalloc) size \"%s\": too big\n", av[0]);
+	if (u < 1024*1024)
+		ARGV_ERR("(-smalloc) size \"%s\": too small, "
+			 "did you forget to specify M or G?\n", av[0]);
+
+	sc->sma_max = u;
+}
+
+static void
+sma_open(const struct stevedore *st)
+{
+	struct sma_sc *sma_sc;
+
+	CAST_OBJ_NOTNULL(sma_sc, st->priv, SMA_SC_MAGIC);
+	Lck_New(&sma_sc->sma_mtx, lck_sma);
+	sma_sc->stats = VSM_Alloc(sizeof *sma_sc->stats,
+	    VSC_CLASS, VSC_TYPE_SMA, st->ident);
+	memset(sma_sc->stats, 0, sizeof *sma_sc->stats);
+	if (sma_sc->sma_max != SIZE_MAX)
+		sma_sc->stats->g_space = sma_sc->sma_max;
+}
+
+const struct stevedore sma_stevedore = {
+	.magic	=	STEVEDORE_MAGIC,
+	.name	=	"malloc",
+	.init	=	sma_init,
+	.open	=	sma_open,
+	.alloc	=	sma_alloc,
+	.free	=	sma_free,
+	.trim	=	sma_trim,
+	.var_free_space =	sma_free_space,
+	.var_used_space =	sma_used_space,
+};
diff --git a/bin/varnishd/storage/storage_persistent.c b/bin/varnishd/storage/storage_persistent.c
new file mode 100644
index 0000000..84aef5c
--- /dev/null
+++ b/bin/varnishd/storage/storage_persistent.c
@@ -0,0 +1,678 @@
+/*-
+ * Copyright (c) 2008-2011 Varnish Software AS
+ * All rights reserved.
+ *
+ * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * Persistent storage method
+ *
+ * XXX: Before we start the client or maybe after it stops, we should give the
+ * XXX: stevedores a chance to examine their storage for consistency.
+ *
+ * XXX: Do we ever free the LRU-lists ?
+ */
+
+#include "config.h"
+
+#include <sys/param.h>
+#include <sys/mman.h>
+
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include "cache.h"
+#include "storage/storage.h"
+
+#include "hash_slinger.h"
+#include "vcli.h"
+#include "vcli_priv.h"
+#include "vend.h"
+#include "vsha256.h"
+
+#include "persistent.h"
+#include "storage/storage_persistent.h"
+
+/*--------------------------------------------------------------------*/
+
+/*
+ * silos is unlocked, it only changes during startup when we are
+ * single-threaded
+ */
+static VTAILQ_HEAD(,smp_sc)	silos = VTAILQ_HEAD_INITIALIZER(silos);
+
+/*--------------------------------------------------------------------
+ * Add bans to silos
+ */
+
+static void
+smp_appendban(struct smp_sc *sc, struct smp_signctx *ctx,
+    uint32_t len, const uint8_t *ban)
+{
+	uint8_t *ptr, *ptr2;
+
+	(void)sc;
+	ptr = ptr2 = SIGN_END(ctx);
+
+	memcpy(ptr, "BAN", 4);
+	ptr += 4;
+
+	vbe32enc(ptr, len);
+	ptr += 4;
+
+	memcpy(ptr, ban, len);
+	ptr += len;
+
+	smp_append_sign(ctx, ptr2, ptr - ptr2);
+}
+
+/* Trust that cache_ban.c takes care of locking */
+
+void
+SMP_NewBan(const uint8_t *ban, unsigned ln)
+{
+	struct smp_sc *sc;
+
+	VTAILQ_FOREACH(sc, &silos, list) {
+		smp_appendban(sc, &sc->ban1, ln, ban);
+		smp_appendban(sc, &sc->ban2, ln, ban);
+	}
+}
+
+/*--------------------------------------------------------------------
+ * Attempt to open and read in a ban list
+ */
+
+static int
+smp_open_bans(struct smp_sc *sc, struct smp_signctx *ctx)
+{
+	uint8_t *ptr, *pe;
+	uint32_t length;
+	int i, retval = 0;
+
+	ASSERT_CLI();
+	(void)sc;
+	i = smp_chk_sign(ctx);
+	if (i)
+		return (i);
+	ptr = SIGN_DATA(ctx);
+	pe = ptr + ctx->ss->length;
+
+	while (ptr < pe) {
+		if (memcmp(ptr, "BAN", 4)) {
+			retval = 1001;
+			break;
+		}
+		ptr += 4;
+
+		length = vbe32dec(ptr);
+		ptr += 4;
+
+		if (ptr + length > pe) {
+			retval = 1003;
+			break;
+		}
+
+		BAN_Reload(ptr, length);
+
+		ptr += length;
+	}
+	assert(ptr <= pe);
+	return (retval);
+}
+
+/*--------------------------------------------------------------------
+ * Attempt to open and read in a segment list
+ */
+
+static int
+smp_open_segs(struct smp_sc *sc, struct smp_signctx *ctx)
+{
+	uint64_t length, l;
+	struct smp_segptr *ss, *se;
+	struct smp_seg *sg, *sg1, *sg2;
+	int i, n = 0;
+
+	ASSERT_CLI();
+	i = smp_chk_sign(ctx);
+	if (i)
+		return (i);
+
+	ss = SIGN_DATA(ctx);
+	length = ctx->ss->length;
+
+	if (length == 0) {
+		/* No segments */
+		sc->free_offset = sc->ident->stuff[SMP_SPC_STUFF];
+		return (0);
+	}
+	se = ss + length / sizeof *ss;
+	se--;
+	assert(ss <= se);
+
+	/*
+	 * Locate the free reserve, there are only two basic cases,
+	 * but once we start dropping segments, things gets more complicated.
+	 */
+
+	sc->free_offset = se->offset + se->length;
+	l = sc->mediasize - sc->free_offset;
+	if (se->offset > ss->offset && l >= sc->free_reserve) {
+		/*
+		 * [__xxxxyyyyzzzz___]
+		 * Plenty of space at tail, do nothing.
+		 */
+	} else if (ss->offset > se->offset) {
+		/*
+		 * [zzzz____xxxxyyyy_]
+		 * (make) space between ends
+		 * We might nuke the entire tail end without getting
+		 * enough space, in which case we fall through to the
+		 * last check.
+		 */
+		while (ss < se && ss->offset > se->offset) {
+			l = ss->offset - (se->offset + se->length);
+			if (l > sc->free_reserve)
+				break;
+			ss++;
+			n++;
+		}
+	}
+
+	if (l < sc->free_reserve) {
+		/*
+		 * [__xxxxyyyyzzzz___]
+		 * (make) space at front
+		 */
+		sc->free_offset = sc->ident->stuff[SMP_SPC_STUFF];
+		while (ss < se) {
+			l = ss->offset - sc->free_offset;
+			if (l > sc->free_reserve)
+				break;
+			ss++;
+			n++;
+		}
+	}
+
+	assert (l >= sc->free_reserve);
+
+
+	sg1 = NULL;
+	sg2 = NULL;
+	for(; ss <= se; ss++) {
+		ALLOC_OBJ(sg, SMP_SEG_MAGIC);
+		AN(sg);
+		sg->lru = LRU_Alloc();
+		CHECK_OBJ_NOTNULL(sg->lru, LRU_MAGIC);
+		sg->p = *ss;
+
+		sg->flags |= SMP_SEG_MUSTLOAD;
+
+		/*
+		 * HACK: prevent save_segs from nuking segment until we have
+		 * HACK: loaded it.
+		 */
+		sg->nobj = 1;
+		if (sg1 != NULL) {
+			assert(sg1->p.offset != sg->p.offset);
+			if (sg1->p.offset < sg->p.offset)
+				assert(smp_segend(sg1) <= sg->p.offset);
+			else
+				assert(smp_segend(sg) <= sg1->p.offset);
+		}
+		if (sg2 != NULL) {
+			assert(sg2->p.offset != sg->p.offset);
+			if (sg2->p.offset < sg->p.offset)
+				assert(smp_segend(sg2) <= sg->p.offset);
+			else
+				assert(smp_segend(sg) <= sg2->p.offset);
+		}
+
+		/* XXX: check that they are inside silo */
+		/* XXX: check that they don't overlap */
+		/* XXX: check that they are serial */
+		sg->sc = sc;
+		VTAILQ_INSERT_TAIL(&sc->segments, sg, list);
+		sg2 = sg;
+		if (sg1 == NULL)
+			sg1 = sg;
+	}
+	printf("Dropped %d segments to make free_reserve\n", n);
+	return (0);
+}
+
+/*--------------------------------------------------------------------
+ * Silo worker thread
+ */
+
+static void *
+smp_thread(struct sess *sp, void *priv)
+{
+	struct smp_sc	*sc;
+	struct smp_seg *sg;
+
+	(void)sp;
+	CAST_OBJ_NOTNULL(sc, priv, SMP_SC_MAGIC);
+
+	/* First, load all the objects from all segments */
+	VTAILQ_FOREACH(sg, &sc->segments, list)
+		if (sg->flags & SMP_SEG_MUSTLOAD)
+			smp_load_seg(sp, sc, sg);
+
+	sc->flags |= SMP_SC_LOADED;
+	BAN_TailDeref(&sc->tailban);
+	AZ(sc->tailban);
+	printf("Silo completely loaded\n");
+	while (1) {
+		(void)sleep (1);
+		sg = VTAILQ_FIRST(&sc->segments);
+		if (sg != NULL && sg -> sc->cur_seg &&
+		    sg->nobj == 0) {
+			Lck_Lock(&sc->mtx);
+			smp_save_segs(sc);
+			Lck_Unlock(&sc->mtx);
+		}
+	}
+	NEEDLESS_RETURN(NULL);
+}
+
+/*--------------------------------------------------------------------
+ * Open a silo in the worker process
+ */
+
+static void
+smp_open(const struct stevedore *st)
+{
+	struct smp_sc	*sc;
+
+	ASSERT_CLI();
+
+	CAST_OBJ_NOTNULL(sc, st->priv, SMP_SC_MAGIC);
+
+	Lck_New(&sc->mtx, lck_smp);
+	Lck_Lock(&sc->mtx);
+
+	sc->stevedore = st;
+
+	/* We trust the parent to give us a valid silo, for good measure: */
+	AZ(smp_valid_silo(sc));
+
+	AZ(mprotect(sc->base, 4096, PROT_READ));
+
+	sc->ident = SIGN_DATA(&sc->idn);
+
+	/* We attempt ban1 first, and if that fails, try ban2 */
+	if (smp_open_bans(sc, &sc->ban1))
+		AZ(smp_open_bans(sc, &sc->ban2));
+
+	/* We attempt seg1 first, and if that fails, try seg2 */
+	if (smp_open_segs(sc, &sc->seg1))
+		AZ(smp_open_segs(sc, &sc->seg2));
+
+	/*
+	 * Grap a reference to the tail of the ban list, until the thread
+	 * has loaded all objects, so we can be sure that all of our
+	 * proto-bans survive until then.
+	 */
+	sc->tailban = BAN_TailRef();
+	AN(sc->tailban);
+
+	/* XXX: save segments to ensure consistency between seg1 & seg2 ? */
+
+	/* XXX: abandon early segments to make sure we have free space ? */
+
+	/* Open a new segment, so we are ready to write */
+	smp_new_seg(sc);
+
+	/* Start the worker silo worker thread, it will load the objects */
+	WRK_BgThread(&sc->thread, "persistence", smp_thread, sc);
+
+	VTAILQ_INSERT_TAIL(&silos, sc, list);
+	Lck_Unlock(&sc->mtx);
+}
+
+/*--------------------------------------------------------------------
+ * Close a silo
+ */
+
+static void
+smp_close(const struct stevedore *st)
+{
+	struct smp_sc	*sc;
+
+	ASSERT_CLI();
+
+	CAST_OBJ_NOTNULL(sc, st->priv, SMP_SC_MAGIC);
+	Lck_Lock(&sc->mtx);
+	smp_close_seg(sc, sc->cur_seg);
+	Lck_Unlock(&sc->mtx);
+
+	/* XXX: reap thread */
+}
+
+/*--------------------------------------------------------------------
+ * Allocate a bite.
+ *
+ * Allocate [min_size...max_size] space from the bottom of the segment,
+ * as is convenient.
+ *
+ * If 'so' + 'idx' is given, also allocate a smp_object from the top
+ * of the segment.
+ *
+ * Return the segment in 'ssg' if given.
+ */
+
+static struct storage *
+smp_allocx(struct stevedore *st, size_t min_size, size_t max_size,
+    struct smp_object **so, unsigned *idx, struct smp_seg **ssg)
+{
+	struct smp_sc *sc;
+	struct storage *ss;
+	struct smp_seg *sg;
+	unsigned tries;
+	uint64_t left, extra;
+
+	CAST_OBJ_NOTNULL(sc, st->priv, SMP_SC_MAGIC);
+	assert(min_size <= max_size);
+
+	max_size = IRNUP(sc, max_size);
+	min_size = IRNUP(sc, min_size);
+
+	extra = IRNUP(sc, sizeof(*ss));
+	if (so != NULL) {
+		extra += sizeof(**so);
+		AN(idx);
+	}
+
+	Lck_Lock(&sc->mtx);
+	sg = NULL;
+	ss = NULL;
+	for (tries = 0; tries < 3; tries++) {
+		left = smp_spaceleft(sc, sc->cur_seg);
+		if (left >= extra + min_size)
+			break;
+		smp_close_seg(sc, sc->cur_seg);
+		smp_new_seg(sc);
+	}
+	if (left >= extra + min_size)  {
+		if (left < extra + max_size)
+			max_size = IRNDN(sc, left - extra);
+
+		sg = sc->cur_seg;
+		ss = (void*)(sc->base + sc->next_bot);
+		sc->next_bot += max_size + IRNUP(sc, sizeof(*ss));
+		sg->nalloc++;
+		if (so != NULL) {
+			sc->next_top -= sizeof(**so);
+			*so = (void*)(sc->base + sc->next_top);
+			/* Render this smp_object mostly harmless */
+			(*so)->ttl = 0.;
+			(*so)->ban = 0.;
+			(*so)->ptr = 0;;
+			sg->objs = *so;
+			*idx = ++sg->p.lobjlist;
+		}
+		(void)smp_spaceleft(sc, sg);	/* for the assert */
+	}
+	Lck_Unlock(&sc->mtx);
+
+	if (ss == NULL)
+		return (ss);
+	AN(sg);
+	assert(max_size >= min_size);
+
+	/* Fill the storage structure */
+	memset(ss, 0, sizeof *ss);
+	ss->magic = STORAGE_MAGIC;
+	ss->ptr = PRNUP(sc, ss + 1);
+	ss->space = max_size;
+	ss->priv = sc;
+	ss->stevedore = st;
+#ifdef SENDFILE_WORKS
+	ss->fd = sc->fd;
+#endif
+	if (ssg != NULL)
+		*ssg = sg;
+	return (ss);
+}
+
+/*--------------------------------------------------------------------
+ * Allocate an object
+ */
+
+static struct object *
+smp_allocobj(struct stevedore *stv, struct sess *sp, unsigned ltot,
+    const struct stv_objsecrets *soc)
+{
+	struct object *o;
+	struct storage *st;
+	struct smp_sc	*sc;
+	struct smp_seg *sg;
+	struct smp_object *so;
+	struct objcore *oc;
+	unsigned objidx;
+
+	if (sp->objcore == NULL)
+		return (NULL);		/* from cnt_error */
+	CAST_OBJ_NOTNULL(sc, stv->priv, SMP_SC_MAGIC);
+	AN(sp->objcore);
+	AN(sp->wrk->exp.ttl > 0.);
+
+	ltot = IRNUP(sc, ltot);
+
+	st = smp_allocx(stv, ltot, ltot, &so, &objidx, &sg);
+	if (st == NULL)
+		return (NULL);
+
+	assert(st->space >= ltot);
+	ltot = st->len = st->space;
+
+	o = STV_MkObject(sp, st->ptr, ltot, soc);
+	CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC);
+	o->objstore = st;
+
+	oc = o->objcore;
+	CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
+	oc->flags |= OC_F_LRUDONTMOVE;
+
+	Lck_Lock(&sc->mtx);
+	sg->nfixed++;
+	sg->nobj++;
+
+	/* We have to do this somewhere, might as well be here... */
+	assert(sizeof so->hash == DIGEST_LEN);
+	memcpy(so->hash, oc->objhead->digest, DIGEST_LEN);
+	so->ttl = EXP_Grace(NULL, o);
+	so->ptr = (uint8_t*)o - sc->base;
+	so->ban = BAN_Time(oc->ban);
+
+	smp_init_oc(oc, sg, objidx);
+
+	Lck_Unlock(&sc->mtx);
+	return (o);
+}
+
+/*--------------------------------------------------------------------
+ * Allocate a bite
+ */
+
+static struct storage *
+smp_alloc(struct stevedore *st, size_t size)
+{
+
+	return (smp_allocx(st,
+	    size > 4096 ? 4096 : size, size, NULL, NULL, NULL));
+}
+
+/*--------------------------------------------------------------------
+ * Trim a bite
+ * XXX: We could trim the last allocation.
+ */
+
+static void
+smp_trim(struct storage *ss, size_t size)
+{
+
+	(void)ss;
+	(void)size;
+}
+
+/*--------------------------------------------------------------------
+ * We don't track frees of storage, we track the objects which own the
+ * storage and when there are no more objects in in the first segment,
+ * it can be reclaimed.
+ * XXX: We could free the last allocation, but does that happen ?
+ */
+
+static void __match_proto__(storage_free_f)
+smp_free(struct storage *st)
+{
+
+	/* XXX */
+	(void)st;
+}
+
+
+/*--------------------------------------------------------------------*/
+
+const struct stevedore smp_stevedore = {
+	.magic	=	STEVEDORE_MAGIC,
+	.name	=	"persistent",
+	.init	=	smp_mgt_init,
+	.open	=	smp_open,
+	.close	=	smp_close,
+	.alloc	=	smp_alloc,
+	.allocobj =	smp_allocobj,
+	.free	=	smp_free,
+	.trim	=	smp_trim,
+};
+
+/*--------------------------------------------------------------------
+ * Persistence is a bear to test unadultered, so we cheat by adding
+ * a cli command we can use to make it do tricks for us.
+ */
+
+static void
+debug_report_silo(struct cli *cli, const struct smp_sc *sc, int objs)
+{
+	struct smp_seg *sg;
+	struct objcore *oc;
+
+	VCLI_Out(cli, "Silo: %s (%s)\n",
+	    sc->stevedore->ident, sc->filename);
+	VTAILQ_FOREACH(sg, &sc->segments, list) {
+		VCLI_Out(cli, "  Seg: [0x%jx ... +0x%jx]\n",
+		   (uintmax_t)sg->p.offset, (uintmax_t)sg->p.length);
+		if (sg == sc->cur_seg)
+			VCLI_Out(cli,
+			   "    Alloc: [0x%jx ... 0x%jx] = 0x%jx free\n",
+			   (uintmax_t)(sc->next_bot),
+			   (uintmax_t)(sc->next_top),
+			   (uintmax_t)(sc->next_top - sc->next_bot));
+		VCLI_Out(cli, "    %u nobj, %u alloc, %u lobjlist, %u fixed\n",
+		    sg->nobj, sg->nalloc, sg->p.lobjlist, sg->nfixed);
+		if (objs) {
+			VTAILQ_FOREACH(oc, &sg->lru->lru_head, lru_list)
+				VCLI_Out(cli, "      OC %p\n", oc);
+		}
+	}
+}
+
+static void
+debug_persistent(struct cli *cli, const char * const * av, void *priv)
+{
+	struct smp_sc *sc;
+
+	(void)priv;
+
+	if (av[2] == NULL) {
+		VTAILQ_FOREACH(sc, &silos, list)
+			debug_report_silo(cli, sc, 0);
+		return;
+	}
+	VTAILQ_FOREACH(sc, &silos, list)
+		if (!strcmp(av[2], sc->stevedore->ident))
+			break;
+	if (sc == NULL) {
+		VCLI_Out(cli, "Silo <%s> not found\n", av[2]);
+		VCLI_SetResult(cli, CLIS_PARAM);
+		return;
+	}
+	if (av[3] == NULL) {
+		debug_report_silo(cli, sc, 0);
+		return;
+	}
+	Lck_Lock(&sc->mtx);
+	if (!strcmp(av[3], "sync")) {
+		smp_close_seg(sc, sc->cur_seg);
+		smp_new_seg(sc);
+	} else if (!strcmp(av[3], "dump")) {
+		debug_report_silo(cli, sc, 1);
+	} else {
+		VCLI_Out(cli, "Unknown operation\n");
+		VCLI_SetResult(cli, CLIS_PARAM);
+	}
+	Lck_Unlock(&sc->mtx);
+}
+
+static struct cli_proto debug_cmds[] = {
+        { "debug.persistent", "debug.persistent",
+                "Persistent debugging magic:\n"
+		"\tdebug.persistent [stevedore [cmd]]\n"
+		"With no cmd arg, a summary of the silo is returned.\n"
+		"Possible commands:\n"
+		"\tsync\tClose current segment, open a new one\n"
+		"\tdump\tinclude objcores in silo summary\n"
+		"",
+		0, 2, "d", debug_persistent },
+        { NULL }
+};
+
+/*--------------------------------------------------------------------*/
+
+void
+SMP_Init(void)
+{
+	CLI_AddFuncs(debug_cmds);
+}
+
+/*--------------------------------------------------------------------
+ * Pause until all silos have loaded.
+ */
+
+void
+SMP_Ready(void)
+{
+	struct smp_sc *sc;
+
+	ASSERT_CLI();
+	do {
+		VTAILQ_FOREACH(sc, &silos, list)
+			if (!(sc->flags & SMP_SC_LOADED))
+				break;
+		if (sc != NULL)
+			(void)sleep(1);
+	} while (sc != NULL);
+}
diff --git a/bin/varnishd/storage/storage_persistent.h b/bin/varnishd/storage/storage_persistent.h
new file mode 100644
index 0000000..84f3d21
--- /dev/null
+++ b/bin/varnishd/storage/storage_persistent.h
@@ -0,0 +1,219 @@
+/*-
+ * Copyright (c) 2008-2011 Varnish Software AS
+ * All rights reserved.
+ *
+ * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * Persistent storage method
+ *
+ * XXX: Before we start the client or maybe after it stops, we should give the
+ * XXX: stevedores a chance to examine their storage for consistency.
+ *
+ * XXX: Do we ever free the LRU-lists ?
+ */
+
+#define ASSERT_SILO_THREAD(sc) \
+    do {assert(pthread_self() == (sc)->thread);} while (0)
+
+#define OC_F_NEEDFIXUP OC_F_PRIV
+
+/*
+ * Context for a signature.
+ *
+ * A signature is a sequence of bytes in the silo, signed by a SHA256 hash
+ * which follows the bytes.
+ *
+ * The context structure allows us to append to a signature without
+ * recalculating the entire SHA256 hash.
+ */
+
+struct smp_signctx {
+	struct smp_sign		*ss;
+	struct SHA256Context	ctx;
+	uint32_t		unique;
+	const char		*id;
+};
+
+struct smp_sc;
+
+/* XXX: name confusion with on-media version ? */
+struct smp_seg {
+	unsigned		magic;
+#define SMP_SEG_MAGIC		0x45c61895
+
+	struct smp_sc		*sc;
+	struct lru		*lru;
+
+	VTAILQ_ENTRY(smp_seg)	list;		/* on smp_sc.smp_segments */
+
+	struct smp_segptr	p;
+
+	unsigned		flags;
+#define SMP_SEG_MUSTLOAD	(1 << 0)
+#define SMP_SEG_LOADED		(1 << 1)
+
+	uint32_t		nobj;		/* Number of objects */
+	uint32_t		nalloc;		/* Allocations */
+	uint32_t		nfixed;		/* How many fixed objects */
+
+	/* Only for open segment */
+	struct smp_object	*objs;		/* objdesc array */
+	struct smp_signctx	ctx[1];
+};
+
+VTAILQ_HEAD(smp_seghead, smp_seg);
+
+struct smp_sc {
+	unsigned		magic;
+#define SMP_SC_MAGIC		0x7b73af0a
+	struct stevedore	*parent;
+
+	unsigned		flags;
+#define SMP_SC_LOADED		(1 << 0)
+
+	const struct stevedore	*stevedore;
+	int			fd;
+	const char		*filename;
+	off_t			mediasize;
+	uintptr_t		align;
+	uint32_t		granularity;
+	uint32_t		unique;
+
+	uint8_t			*base;
+
+	struct smp_ident	*ident;
+
+	struct smp_seghead	segments;
+	struct smp_seg		*cur_seg;
+	uint64_t		next_bot;	/* next alloc address bottom */
+	uint64_t		next_top;	/* next alloc address top */
+
+	uint64_t		free_offset;
+
+	pthread_t		thread;
+
+	VTAILQ_ENTRY(smp_sc)	list;
+
+	struct smp_signctx	idn;
+	struct smp_signctx	ban1;
+	struct smp_signctx	ban2;
+	struct smp_signctx	seg1;
+	struct smp_signctx	seg2;
+
+	struct ban		*tailban;
+
+	struct lock		mtx;
+
+	/* Cleaner metrics */
+
+	unsigned		min_nseg;
+	unsigned		aim_nseg;
+	unsigned		max_nseg;
+
+	uint64_t		min_segl;
+	uint64_t		aim_segl;
+	uint64_t		max_segl;
+
+	uint64_t		free_reserve;
+};
+
+/*--------------------------------------------------------------------*/
+
+/* Pointer round up/down & assert */
+#define PRNDN(sc, x)	((void*)RDN2((uintptr_t)(x), sc->align))
+#define PRNUP(sc, x)	((void*)RUP2((uintptr_t)(x), sc->align))
+#define PASSERTALIGN(sc, x)	assert(PRNDN(sc, x) == (x))
+
+/* Integer round up/down & assert */
+#define IRNDN(sc, x)	RDN2(x, sc->align)
+#define IRNUP(sc, x)	RUP2(x, sc->align)
+#define IASSERTALIGN(sc, x)	assert(IRNDN(sc, x) == (x))
+
+/*--------------------------------------------------------------------*/
+
+#define ASSERT_PTR_IN_SILO(sc, ptr) \
+	assert((const void*)(ptr) >= (const void*)((sc)->base) && \
+	    (const void*)(ptr) < (const void *)((sc)->base + (sc)->mediasize))
+
+/*--------------------------------------------------------------------*/
+
+#define SIGN_DATA(ctx)	((void *)((ctx)->ss + 1))
+#define SIGN_END(ctx)	((void *)((int8_t *)SIGN_DATA(ctx) + (ctx)->ss->length))
+
+/* storage_persistent_mgt.c */
+
+void smp_mgt_init(struct stevedore *parent, int ac, char * const *av);
+
+/* storage_persistent_silo.c */
+
+void smp_load_seg(const struct sess *sp, const struct smp_sc *sc,
+    struct smp_seg *sg);
+void smp_new_seg(struct smp_sc *sc);
+void smp_close_seg(struct smp_sc *sc, struct smp_seg *sg);
+void smp_init_oc(struct objcore *oc, struct smp_seg *sg, unsigned objidx);
+void smp_save_segs(struct smp_sc *sc);
+
+/* storage_persistent_subr.c */
+
+void smp_def_sign(const struct smp_sc *sc, struct smp_signctx *ctx,
+    uint64_t off, const char *id);
+int smp_chk_sign(struct smp_signctx *ctx);
+void smp_append_sign(struct smp_signctx *ctx, const void *ptr, uint32_t len);
+void smp_reset_sign(struct smp_signctx *ctx);
+void smp_sync_sign(const struct smp_signctx *ctx);
+void smp_newsilo(struct smp_sc *sc);
+int smp_valid_silo(struct smp_sc *sc);
+
+/*--------------------------------------------------------------------
+ * Caculate payload of some stuff
+ */
+
+static inline uint64_t
+smp_stuff_len(const struct smp_sc *sc, unsigned stuff)
+{
+	uint64_t l;
+
+	assert(stuff < SMP_END_STUFF);
+	l = sc->ident->stuff[stuff + 1] - sc->ident->stuff[stuff];
+	l -= SMP_SIGN_SPACE;
+	return (l);
+}
+
+static inline uint64_t
+smp_segend(const struct smp_seg *sg)
+{
+
+	return (sg->p.offset + sg->p.length);
+}
+
+static inline uint64_t
+smp_spaceleft(const struct smp_sc *sc, const struct smp_seg *sg)
+{
+
+	IASSERTALIGN(sc, sc->next_bot);
+	assert(sc->next_bot <= sc->next_top - IRNUP(sc, SMP_SIGN_SPACE));
+	assert(sc->next_bot >= sg->p.offset);
+	assert(sc->next_top < sg->p.offset + sg->p.length);
+	return ((sc->next_top - sc->next_bot) - IRNUP(sc, SMP_SIGN_SPACE));
+}
diff --git a/bin/varnishd/storage/storage_persistent_mgt.c b/bin/varnishd/storage/storage_persistent_mgt.c
new file mode 100644
index 0000000..1631ea6
--- /dev/null
+++ b/bin/varnishd/storage/storage_persistent_mgt.c
@@ -0,0 +1,205 @@
+/*-
+ * Copyright (c) 2008-2011 Varnish Software AS
+ * All rights reserved.
+ *
+ * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * Persistent storage method
+ *
+ * XXX: Before we start the client or maybe after it stops, we should give the
+ * XXX: stevedores a chance to examine their storage for consistency.
+ *
+ * XXX: Do we ever free the LRU-lists ?
+ */
+
+#include "config.h"
+
+#include <sys/mman.h>
+
+#include <math.h>
+#include <stdio.h>
+#include <stdlib.h>
+
+#include "cache.h"
+#include "storage/storage.h"
+
+#include "vsha256.h"
+
+#include "persistent.h"
+#include "storage/storage_persistent.h"
+
+#ifndef MAP_NOCORE
+#define MAP_NOCORE 0 /* XXX Linux */
+#endif
+
+#ifndef MAP_NOSYNC
+#define MAP_NOSYNC 0 /* XXX Linux */
+#endif
+
+/*--------------------------------------------------------------------
+ * Calculate cleaner metrics from silo dimensions
+ */
+
+static void
+smp_metrics(struct smp_sc *sc)
+{
+
+	/*
+	 * We do not want to loose too big chunks of the silos
+	 * content when we are forced to clean a segment.
+	 *
+	 * For now insist that a segment covers no more than 1% of the silo.
+	 *
+	 * XXX: This should possibly depend on the size of the silo so
+	 * XXX: trivially small silos do not run into trouble along
+	 * XXX: the lines of "one object per segment".
+	 */
+
+	sc->min_nseg = 10;
+	sc->max_segl = smp_stuff_len(sc, SMP_SPC_STUFF) / sc->min_nseg;
+
+	fprintf(stderr, "min_nseg = %u, max_segl = %ju\n",
+	    sc->min_nseg, (uintmax_t)sc->max_segl);
+
+	/*
+	 * The number of segments are limited by the size of the segment
+	 * table(s) and from that follows the minimum size of a segmement.
+	 */
+
+	sc->max_nseg = smp_stuff_len(sc, SMP_SEG1_STUFF) / sc->min_nseg;
+	sc->min_segl = smp_stuff_len(sc, SMP_SPC_STUFF) / sc->max_nseg;
+
+	while (sc->min_segl < sizeof(struct object)) {
+		sc->max_nseg /= 2;
+		sc->min_segl = smp_stuff_len(sc, SMP_SPC_STUFF) / sc->max_nseg;
+	}
+
+	fprintf(stderr, "max_nseg = %u, min_segl = %ju\n",
+	    sc->max_nseg, (uintmax_t)sc->min_segl);
+
+	/*
+	 * Set our initial aim point at the exponential average of the
+	 * two extremes.
+	 *
+	 * XXX: This is a pretty arbitrary choice, but having no idea
+	 * XXX: object count, size distribution or ttl pattern at this
+	 * XXX: point, we have to do something.
+	 */
+
+	sc->aim_nseg =
+	   (unsigned) exp((log(sc->min_nseg) + log(sc->max_nseg))*.5);
+	sc->aim_segl = smp_stuff_len(sc, SMP_SPC_STUFF) / sc->aim_nseg;
+
+	fprintf(stderr, "aim_nseg = %u, aim_segl = %ju\n",
+	    sc->aim_nseg, (uintmax_t)sc->aim_segl);
+
+	/*
+	 * How much space in the free reserve pool ?
+	 */
+	sc->free_reserve = sc->aim_segl * 10;
+
+	fprintf(stderr, "free_reserve = %ju\n", (uintmax_t)sc->free_reserve);
+}
+
+/*--------------------------------------------------------------------
+ * Set up persistent storage silo in the master process.
+ */
+
+void
+smp_mgt_init(struct stevedore *parent, int ac, char * const *av)
+{
+	struct smp_sc		*sc;
+	struct smp_sign		sgn;
+	void *target;
+	int i;
+
+	ASSERT_MGT();
+
+	AZ(av[ac]);
+#define SIZOF(foo)       fprintf(stderr, \
+    "sizeof(%s) = %zu = 0x%zx\n", #foo, sizeof(foo), sizeof(foo));
+	SIZOF(struct smp_ident);
+	SIZOF(struct smp_sign);
+	SIZOF(struct smp_segptr);
+	SIZOF(struct smp_object);
+#undef SIZOF
+
+	/* See comments in persistent.h */
+	assert(sizeof(struct smp_ident) == SMP_IDENT_SIZE);
+
+	/* Allocate softc */
+	ALLOC_OBJ(sc, SMP_SC_MAGIC);
+	XXXAN(sc);
+	sc->parent = parent;
+	sc->fd = -1;
+	VTAILQ_INIT(&sc->segments);
+
+	/* Argument processing */
+	if (ac != 2)
+		ARGV_ERR("(-spersistent) wrong number of arguments\n");
+
+	i = STV_GetFile(av[0], &sc->fd, &sc->filename, "-spersistent");
+	if (i == 2)
+		ARGV_ERR("(-spersistent) need filename (not directory)\n");
+
+	sc->align = sizeof(void*) * 2;
+	sc->granularity = getpagesize();
+	sc->mediasize = STV_FileSize(sc->fd, av[1], &sc->granularity,
+	    "-spersistent");
+
+	AZ(ftruncate(sc->fd, sc->mediasize));
+
+	/* Try to determine correct mmap address */
+	i = read(sc->fd, &sgn, sizeof sgn);
+	assert(i == sizeof sgn);
+	if (!strcmp(sgn.ident, "SILO"))
+		target = (void*)(uintptr_t)sgn.mapped;
+	else
+		target = NULL;
+
+	sc->base = mmap(target, sc->mediasize, PROT_READ|PROT_WRITE,
+	    MAP_NOCORE | MAP_NOSYNC | MAP_SHARED, sc->fd, 0);
+
+	if (sc->base == MAP_FAILED)
+		ARGV_ERR("(-spersistent) failed to mmap (%s)\n",
+		    strerror(errno));
+
+	smp_def_sign(sc, &sc->idn, 0, "SILO");
+	sc->ident = SIGN_DATA(&sc->idn);
+
+	i = smp_valid_silo(sc);
+	if (i) {
+		printf("Warning SILO (%s) not reloaded (reason=%d)\n",
+		    sc->filename, i);
+		smp_newsilo(sc);
+	}
+	AZ(smp_valid_silo(sc));
+
+	smp_metrics(sc);
+
+	parent->priv = sc;
+
+	/* XXX: only for sendfile I guess... */
+	mgt_child_inherit(sc->fd, "storage_persistent");
+}
diff --git a/bin/varnishd/storage/storage_persistent_silo.c b/bin/varnishd/storage/storage_persistent_silo.c
new file mode 100644
index 0000000..5393546
--- /dev/null
+++ b/bin/varnishd/storage/storage_persistent_silo.c
@@ -0,0 +1,524 @@
+/*-
+ * Copyright (c) 2008-2011 Varnish Software AS
+ * All rights reserved.
+ *
+ * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * Persistent storage method
+ *
+ * XXX: Before we start the client or maybe after it stops, we should give the
+ * XXX: stevedores a chance to examine their storage for consistency.
+ *
+ */
+
+#include "config.h"
+
+#include <stdio.h>
+#include <stdlib.h>
+
+#include "cache.h"
+#include "storage/storage.h"
+
+#include "hash_slinger.h"
+#include "vsha256.h"
+#include "vtim.h"
+
+#include "persistent.h"
+#include "storage/storage_persistent.h"
+
+/*--------------------------------------------------------------------
+ * Write the segmentlist back to the silo.
+ *
+ * We write the first copy, sync it synchronously, then write the
+ * second copy and sync it synchronously.
+ *
+ * Provided the kernel doesn't lie, that means we will always have
+ * at least one valid copy on in the silo.
+ */
+
+static void
+smp_save_seg(const struct smp_sc *sc, struct smp_signctx *ctx)
+{
+	struct smp_segptr *ss;
+	struct smp_seg *sg;
+	uint64_t length;
+
+	Lck_AssertHeld(&sc->mtx);
+	smp_reset_sign(ctx);
+	ss = SIGN_DATA(ctx);
+	length = 0;
+	VTAILQ_FOREACH(sg, &sc->segments, list) {
+		assert(sg->p.offset < sc->mediasize);
+		assert(sg->p.offset + sg->p.length <= sc->mediasize);
+		*ss = sg->p;
+		ss++;
+		length += sizeof *ss;
+	}
+	smp_append_sign(ctx, SIGN_DATA(ctx), length);
+	smp_sync_sign(ctx);
+}
+
+void
+smp_save_segs(struct smp_sc *sc)
+{
+	struct smp_seg *sg, *sg2;
+
+	Lck_AssertHeld(&sc->mtx);
+
+	/*
+	 * Remove empty segments from the front of the list
+	 * before we write the segments to disk.
+	 */
+	VTAILQ_FOREACH_SAFE(sg, &sc->segments, list, sg2) {
+		if (sg->nobj > 0)
+			break;
+		if (sg == sc->cur_seg)
+			continue;
+		VTAILQ_REMOVE(&sc->segments, sg, list);
+		LRU_Free(sg->lru);
+		FREE_OBJ(sg);
+	}
+	smp_save_seg(sc, &sc->seg1);
+	smp_save_seg(sc, &sc->seg2);
+}
+
+/*--------------------------------------------------------------------
+ * Load segments
+ *
+ * The overall objective is to register the existence of an object, based
+ * only on the minimally sized struct smp_object, without causing the
+ * main object to be faulted in.
+ *
+ * XXX: We can test this by mprotecting the main body of the segment
+ * XXX: until the first fixup happens, or even just over this loop,
+ * XXX: However: the requires that the smp_objects starter further
+ * XXX: into the segment than a page so that they do not get hit
+ * XXX: by the protection.
+ */
+
+void
+smp_load_seg(const struct sess *sp, const struct smp_sc *sc,
+    struct smp_seg *sg)
+{
+	struct smp_object *so;
+	struct objcore *oc;
+	uint32_t no;
+	double t_now = VTIM_real();
+	struct smp_signctx ctx[1];
+
+	ASSERT_SILO_THREAD(sc);
+	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+	CHECK_OBJ_NOTNULL(sg, SMP_SEG_MAGIC);
+	CHECK_OBJ_NOTNULL(sg->lru, LRU_MAGIC);
+	assert(sg->flags & SMP_SEG_MUSTLOAD);
+	sg->flags &= ~SMP_SEG_MUSTLOAD;
+	AN(sg->p.offset);
+	if (sg->p.objlist == 0)
+		return;
+	smp_def_sign(sc, ctx, sg->p.offset, "SEGHEAD");
+	if (smp_chk_sign(ctx))
+		return;
+
+	/* test SEGTAIL */
+	/* test OBJIDX */
+	so = (void*)(sc->base + sg->p.objlist);
+	sg->objs = so;
+	no = sg->p.lobjlist;
+	/* Clear the bogus "hold" count */
+	sg->nobj = 0;
+	for (;no > 0; so++,no--) {
+		if (so->ttl == 0 || so->ttl < t_now)
+			continue;
+		HSH_Prealloc(sp);
+		oc = sp->wrk->nobjcore;
+		oc->flags |= OC_F_NEEDFIXUP | OC_F_LRUDONTMOVE;
+		oc->flags &= ~OC_F_BUSY;
+		smp_init_oc(oc, sg, no);
+		oc->ban = BAN_RefBan(oc, so->ban, sc->tailban);
+		memcpy(sp->wrk->nobjhead->digest, so->hash, SHA256_LEN);
+		(void)HSH_Insert(sp);
+		AZ(sp->wrk->nobjcore);
+		EXP_Inject(oc, sg->lru, so->ttl);
+		sg->nobj++;
+	}
+	WRK_SumStat(sp->wrk);
+	sg->flags |= SMP_SEG_LOADED;
+}
+
+/*--------------------------------------------------------------------
+ * Create a new segment
+ */
+
+void
+smp_new_seg(struct smp_sc *sc)
+{
+	struct smp_seg *sg, *sg2;
+
+	Lck_AssertHeld(&sc->mtx);
+	ALLOC_OBJ(sg, SMP_SEG_MAGIC);
+	AN(sg);
+	sg->sc = sc;
+	sg->lru = LRU_Alloc();
+	CHECK_OBJ_NOTNULL(sg->lru, LRU_MAGIC);
+
+	/* XXX: find where it goes in silo */
+
+	sg->p.offset = sc->free_offset;
+	// XXX: align */
+	assert(sg->p.offset >= sc->ident->stuff[SMP_SPC_STUFF]);
+	assert(sg->p.offset < sc->mediasize);
+
+	sg->p.length = sc->aim_segl;
+	sg->p.length &= ~7;
+
+	if (smp_segend(sg) > sc->mediasize) {
+		sc->free_offset = sc->ident->stuff[SMP_SPC_STUFF];
+		sg->p.offset = sc->free_offset;
+		sg2 = VTAILQ_FIRST(&sc->segments);
+		if (smp_segend(sg) > sg2->p.offset) {
+			printf("Out of space in persistent silo\n");
+			printf("Committing suicide, restart will make space\n");
+			exit (0);
+		}
+	}
+
+
+	assert(smp_segend(sg) <= sc->mediasize);
+
+	sg2 = VTAILQ_FIRST(&sc->segments);
+	if (sg2 != NULL && sg2->p.offset > sc->free_offset) {
+		if (smp_segend(sg) > sg2->p.offset) {
+			printf("Out of space in persistent silo\n");
+			printf("Committing suicide, restart will make space\n");
+			exit (0);
+		}
+		assert(smp_segend(sg) <= sg2->p.offset);
+	}
+
+	sg->p.offset = IRNUP(sc, sg->p.offset);
+	sg->p.length = IRNDN(sc, sg->p.length);
+	sc->free_offset = sg->p.offset + sg->p.length;
+
+	VTAILQ_INSERT_TAIL(&sc->segments, sg, list);
+
+	/* Neuter the new segment in case there is an old one there */
+	AN(sg->p.offset);
+	smp_def_sign(sc, sg->ctx, sg->p.offset, "SEGHEAD");
+	smp_reset_sign(sg->ctx);
+	smp_sync_sign(sg->ctx);
+
+	/* Set up our allocation points */
+	sc->cur_seg = sg;
+	sc->next_bot = sg->p.offset + IRNUP(sc, SMP_SIGN_SPACE);
+	sc->next_top = smp_segend(sg);
+	sc->next_top -= IRNUP(sc, SMP_SIGN_SPACE);
+	IASSERTALIGN(sc, sc->next_bot);
+	IASSERTALIGN(sc, sc->next_top);
+	sg->objs = (void*)(sc->base + sc->next_top);
+}
+
+/*--------------------------------------------------------------------
+ * Close a segment
+ */
+
+void
+smp_close_seg(struct smp_sc *sc, struct smp_seg *sg)
+{
+	uint64_t left, dst, len;
+	void *dp;
+
+	Lck_AssertHeld(&sc->mtx);
+
+	assert(sg == sc->cur_seg);
+	AN(sg->p.offset);
+	sc->cur_seg = NULL;
+
+	if (sg->nalloc == 0) {
+		/* XXX: if segment is empty, delete instead */
+		VTAILQ_REMOVE(&sc->segments, sg, list);
+		free(sg);
+		return;
+	}
+
+	/*
+	 * If there is enough space left, that we can move the smp_objects
+	 * down without overwriting the present copy, we will do so to
+	 * compact the segment.
+	 */
+	left = smp_spaceleft(sc, sg);
+	len = sizeof(struct smp_object) * sg->p.lobjlist;
+	if (len < left) {
+		dst = sc->next_bot + IRNUP(sc, SMP_SIGN_SPACE);
+		dp = sc->base + dst;
+		assert((uintptr_t)dp + len < (uintptr_t)sg->objs);
+		memcpy(dp, sg->objs, len);
+		sc->next_top = dst;
+		sg->objs = dp;
+		sg->p.length = (sc->next_top - sg->p.offset)
+		     + len + IRNUP(sc, SMP_SIGN_SPACE);
+		(void)smp_spaceleft(sc, sg);	/* for the asserts */
+
+	}
+
+	/* Update the segment header */
+	sg->p.objlist = sc->next_top;
+
+	/* Write the (empty) OBJIDX signature */
+	sc->next_top -= IRNUP(sc, SMP_SIGN_SPACE);
+	assert(sc->next_top >= sc->next_bot);
+	smp_def_sign(sc, sg->ctx, sc->next_top, "OBJIDX");
+	smp_reset_sign(sg->ctx);
+	smp_sync_sign(sg->ctx);
+
+	/* Write the (empty) SEGTAIL signature */
+	smp_def_sign(sc, sg->ctx,
+	    sg->p.offset + sg->p.length - IRNUP(sc, SMP_SIGN_SPACE), "SEGTAIL");
+	smp_reset_sign(sg->ctx);
+	smp_sync_sign(sg->ctx);
+
+	/* Save segment list */
+	smp_save_segs(sc);
+	sc->free_offset = smp_segend(sg);
+}
+
+
+/*---------------------------------------------------------------------
+ */
+
+static struct smp_object *
+smp_find_so(const struct smp_seg *sg, unsigned priv2)
+{
+	struct smp_object *so;
+
+	assert(priv2 > 0);
+	assert(priv2 <= sg->p.lobjlist);
+	so = &sg->objs[sg->p.lobjlist - priv2];
+	return (so);
+}
+
+/*---------------------------------------------------------------------
+ * Check if a given storage structure is valid to use
+ */
+
+static int
+smp_loaded_st(const struct smp_sc *sc, const struct smp_seg *sg,
+    const struct storage *st)
+{
+	struct smp_seg *sg2;
+	const uint8_t *pst;
+	uint64_t o;
+
+	(void)sg;		/* XXX: faster: Start search from here */
+	pst = (const void *)st;
+
+	if (pst < (sc->base + sc->ident->stuff[SMP_SPC_STUFF]))
+		return (0x01);		/* Before silo payload start */
+	if (pst > (sc->base + sc->ident->stuff[SMP_END_STUFF]))
+		return (0x02);		/* After silo end */
+
+	o = pst - sc->base;
+
+	/* Find which segment contains the storage structure */
+	VTAILQ_FOREACH(sg2, &sc->segments, list)
+		if (o > sg2->p.offset && (o + sizeof(*st)) < sg2->p.objlist)
+			break;
+	if (sg2 == NULL)
+		return (0x04);		/* No claiming segment */
+	if (!(sg2->flags & SMP_SEG_LOADED))
+		return (0x08);		/* Claiming segment not loaded */
+
+	/* It is now safe to access the storage structure */
+	if (st->magic != STORAGE_MAGIC)
+		return (0x10);		/* Not enough magic */
+
+	if (o + st->space >= sg2->p.objlist)
+		return (0x20);		/* Allocation not inside segment */
+
+	if (st->len > st->space)
+		return (0x40);		/* Plain bad... */
+
+	/*
+	 * XXX: We could patch up st->stevedore and st->priv here
+	 * XXX: but if things go right, we will never need them.
+	 */
+	return (0);
+}
+
+/*---------------------------------------------------------------------
+ * objcore methods for persistent objects
+ */
+
+static struct object *
+smp_oc_getobj(struct worker *wrk, struct objcore *oc)
+{
+	struct object *o;
+	struct smp_seg *sg;
+	struct smp_object *so;
+	struct storage *st;
+	uint64_t l;
+	int bad;
+
+	/* Some calls are direct, but they should match anyway */
+	assert(oc->methods->getobj == smp_oc_getobj);
+
+	CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
+	if (wrk == NULL)
+		AZ(oc->flags & OC_F_NEEDFIXUP);
+
+	CAST_OBJ_NOTNULL(sg, oc->priv, SMP_SEG_MAGIC);
+	so = smp_find_so(sg, oc->priv2);
+
+	o = (void*)(sg->sc->base + so->ptr);
+	/*
+	 * The object may not be in this segment since we allocate it
+	 * In a separate operation than the smp_object.  We could check
+	 * that it is in a later segment, but that would be complicated.
+	 * XXX: For now, be happy if it is inside th silo
+	 */
+	ASSERT_PTR_IN_SILO(sg->sc, o);
+	CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC);
+
+	/*
+	 * If this flag is not set, it will not be, and the lock is not
+	 * needed to test it.
+	 */
+	if (!(oc->flags & OC_F_NEEDFIXUP))
+		return (o);
+
+	AN(wrk);
+	Lck_Lock(&sg->sc->mtx);
+	/* Check again, we might have raced. */
+	if (oc->flags & OC_F_NEEDFIXUP) {
+		/* We trust caller to have a refcnt for us */
+		o->objcore = oc;
+
+		bad = 0;
+		l = 0;
+		VTAILQ_FOREACH(st, &o->store, list) {
+			bad |= smp_loaded_st(sg->sc, sg, st);
+			if (bad)
+				break;
+			l += st->len;
+		}
+		if (l != o->len)
+			bad |= 0x100;
+
+		if(bad) {
+			EXP_Set_ttl(&o->exp, -1);
+			so->ttl = 0;
+		}
+
+		sg->nfixed++;
+		wrk->stats.n_object++;
+		wrk->stats.n_vampireobject--;
+		oc->flags &= ~OC_F_NEEDFIXUP;
+	}
+	Lck_Unlock(&sg->sc->mtx);
+	EXP_Rearm(o);
+	return (o);
+}
+
+static void
+smp_oc_updatemeta(struct objcore *oc)
+{
+	struct object *o;
+	struct smp_seg *sg;
+	struct smp_object *so;
+	double mttl;
+
+	CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
+	o = smp_oc_getobj(NULL, oc);
+	AN(o);
+
+	CAST_OBJ_NOTNULL(sg, oc->priv, SMP_SEG_MAGIC);
+	CHECK_OBJ_NOTNULL(sg->sc, SMP_SC_MAGIC);
+	so = smp_find_so(sg, oc->priv2);
+
+	mttl = EXP_Grace(NULL, o);
+
+	if (sg == sg->sc->cur_seg) {
+		/* Lock necessary, we might race close_seg */
+		Lck_Lock(&sg->sc->mtx);
+		so->ban = BAN_Time(oc->ban);
+		so->ttl = mttl;
+		Lck_Unlock(&sg->sc->mtx);
+	} else {
+		so->ban = BAN_Time(oc->ban);
+		so->ttl = mttl;
+	}
+}
+
+static void __match_proto__()
+smp_oc_freeobj(struct objcore *oc)
+{
+	struct smp_seg *sg;
+	struct smp_object *so;
+
+	CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
+
+	CAST_OBJ_NOTNULL(sg, oc->priv, SMP_SEG_MAGIC);
+	so = smp_find_so(sg, oc->priv2);
+
+	Lck_Lock(&sg->sc->mtx);
+	so->ttl = 0;
+	so->ptr = 0;
+
+	assert(sg->nobj > 0);
+	assert(sg->nfixed > 0);
+	sg->nobj--;
+	sg->nfixed--;
+
+	Lck_Unlock(&sg->sc->mtx);
+}
+
+/*--------------------------------------------------------------------
+ * Find the per-segment lru list for this object
+ */
+
+static struct lru *
+smp_oc_getlru(const struct objcore *oc)
+{
+	struct smp_seg *sg;
+
+	CAST_OBJ_NOTNULL(sg, oc->priv, SMP_SEG_MAGIC);
+	return (sg->lru);
+}
+
+static struct objcore_methods smp_oc_methods = {
+	.getobj =		smp_oc_getobj,
+	.updatemeta =		smp_oc_updatemeta,
+	.freeobj =		smp_oc_freeobj,
+	.getlru =		smp_oc_getlru,
+};
+
+/*--------------------------------------------------------------------*/
+
+void
+smp_init_oc(struct objcore *oc, struct smp_seg *sg, unsigned objidx)
+{
+
+	oc->priv = sg;
+	oc->priv2 = objidx;
+	oc->methods = &smp_oc_methods;
+}
diff --git a/bin/varnishd/storage/storage_persistent_subr.c b/bin/varnishd/storage/storage_persistent_subr.c
new file mode 100644
index 0000000..2066e72
--- /dev/null
+++ b/bin/varnishd/storage/storage_persistent_subr.c
@@ -0,0 +1,302 @@
+/*-
+ * Copyright (c) 2008-2011 Varnish Software AS
+ * All rights reserved.
+ *
+ * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * Persistent storage method
+ *
+ * XXX: Before we start the client or maybe after it stops, we should give the
+ * XXX: stevedores a chance to examine their storage for consistency.
+ *
+ * XXX: Do we ever free the LRU-lists ?
+ */
+
+#include "config.h"
+
+#include <sys/mman.h>
+
+#include <stddef.h>
+#include <stdio.h>
+#include <stdlib.h>
+
+#include "cache.h"
+#include "storage/storage.h"
+
+#include "vsha256.h"
+
+#include "persistent.h"
+#include "storage/storage_persistent.h"
+
+/*--------------------------------------------------------------------
+ * SIGNATURE functions
+ * The signature is SHA256 over:
+ *    1. The smp_sign struct up to but not including the length field.
+ *    2. smp_sign->length bytes, starting after the smp_sign structure
+ *    3. The smp-sign->length field.
+ * The signature is stored after the byte-range from step 2.
+ */
+
+/*--------------------------------------------------------------------
+ * Define a signature by location and identifier.
+ */
+
+void
+smp_def_sign(const struct smp_sc *sc, struct smp_signctx *ctx,
+    uint64_t off, const char *id)
+{
+
+	AZ(off & 7);			/* Alignment */
+	assert(strlen(id) < sizeof ctx->ss->ident);
+
+	memset(ctx, 0, sizeof *ctx);
+	ctx->ss = (void*)(sc->base + off);
+	ctx->unique = sc->unique;
+	ctx->id = id;
+}
+
+/*--------------------------------------------------------------------
+ * Check that a signature is good, leave state ready for append
+ */
+int
+smp_chk_sign(struct smp_signctx *ctx)
+{
+	struct SHA256Context cx;
+	unsigned char sign[SHA256_LEN];
+	int r = 0;
+
+	if (strncmp(ctx->id, ctx->ss->ident, sizeof ctx->ss->ident))
+		r = 1;
+	else if (ctx->unique != ctx->ss->unique)
+		r = 2;
+	else if ((uintptr_t)ctx->ss != ctx->ss->mapped)
+		r = 3;
+	else {
+		SHA256_Init(&ctx->ctx);
+		SHA256_Update(&ctx->ctx, ctx->ss,
+		    offsetof(struct smp_sign, length));
+		SHA256_Update(&ctx->ctx, SIGN_DATA(ctx), ctx->ss->length);
+		cx = ctx->ctx;
+		SHA256_Update(&cx, &ctx->ss->length, sizeof(ctx->ss->length));
+		SHA256_Final(sign, &cx);
+		if (memcmp(sign, SIGN_END(ctx), sizeof sign))
+			r = 4;
+	}
+	if (r) {
+		fprintf(stderr, "CHK(%p %s %p %s) = %d\n",
+		    ctx, ctx->id, ctx->ss,
+		    r > 1 ? ctx->ss->ident : "<invalid>", r);
+	}
+	return (r);
+}
+
+/*--------------------------------------------------------------------
+ * Append data to a signature
+ */
+void
+smp_append_sign(struct smp_signctx *ctx, const void *ptr, uint32_t len)
+{
+	struct SHA256Context cx;
+	unsigned char sign[SHA256_LEN];
+
+	if (len != 0) {
+		SHA256_Update(&ctx->ctx, ptr, len);
+		ctx->ss->length += len;
+	}
+	cx = ctx->ctx;
+	SHA256_Update(&cx, &ctx->ss->length, sizeof(ctx->ss->length));
+	SHA256_Final(sign, &cx);
+	memcpy(SIGN_END(ctx), sign, sizeof sign);
+XXXAZ(smp_chk_sign(ctx));
+}
+
+/*--------------------------------------------------------------------
+ * Reset a signature to empty, prepare for appending.
+ */
+
+void
+smp_reset_sign(struct smp_signctx *ctx)
+{
+
+	memset(ctx->ss, 0, sizeof *ctx->ss);
+	strcpy(ctx->ss->ident, ctx->id);
+	ctx->ss->unique = ctx->unique;
+	ctx->ss->mapped = (uintptr_t)ctx->ss;
+	SHA256_Init(&ctx->ctx);
+	SHA256_Update(&ctx->ctx, ctx->ss,
+	    offsetof(struct smp_sign, length));
+	smp_append_sign(ctx, NULL, 0);
+}
+
+/*--------------------------------------------------------------------
+ * Force a write of a signature block to the backing store.
+ */
+
+void
+smp_sync_sign(const struct smp_signctx *ctx)
+{
+	int i;
+
+	/* XXX: round to pages */
+	i = msync((void*)ctx->ss, ctx->ss->length + SHA256_LEN, MS_SYNC);
+	if (i && 0)
+		fprintf(stderr, "SyncSign(%p %s) = %d %s\n",
+		    ctx->ss, ctx->id, i, strerror(errno));
+}
+
+/*--------------------------------------------------------------------
+ * Create and force a new signature to backing store
+ */
+
+static void
+smp_new_sign(const struct smp_sc *sc, struct smp_signctx *ctx,
+    uint64_t off, const char *id)
+{
+	smp_def_sign(sc, ctx, off, id);
+	smp_reset_sign(ctx);
+	smp_sync_sign(ctx);
+}
+
+/*--------------------------------------------------------------------
+ * Initialize a Silo with a valid but empty structure.
+ *
+ * XXX: more intelligent sizing of things.
+ */
+
+void
+smp_newsilo(struct smp_sc *sc)
+{
+	struct smp_ident	*si;
+
+	ASSERT_MGT();
+	assert(strlen(SMP_IDENT_STRING) < sizeof si->ident);
+
+	/* Choose a new random number */
+	sc->unique = random();
+
+	smp_reset_sign(&sc->idn);
+	si = sc->ident;
+
+	memset(si, 0, sizeof *si);
+	strcpy(si->ident, SMP_IDENT_STRING);
+	si->byte_order = 0x12345678;
+	si->size = sizeof *si;
+	si->major_version = 2;
+	si->unique = sc->unique;
+	si->mediasize = sc->mediasize;
+	si->granularity = sc->granularity;
+	/*
+	 * Aim for cache-line-width
+	 */
+	si->align = sizeof(void*) * 2;
+	sc->align = si->align;
+
+	si->stuff[SMP_BAN1_STUFF] = sc->granularity;
+	si->stuff[SMP_BAN2_STUFF] = si->stuff[SMP_BAN1_STUFF] + 1024*1024;
+	si->stuff[SMP_SEG1_STUFF] = si->stuff[SMP_BAN2_STUFF] + 1024*1024;
+	si->stuff[SMP_SEG2_STUFF] = si->stuff[SMP_SEG1_STUFF] + 1024*1024;
+	si->stuff[SMP_SPC_STUFF] = si->stuff[SMP_SEG2_STUFF] + 1024*1024;
+	si->stuff[SMP_END_STUFF] = si->mediasize;
+	assert(si->stuff[SMP_SPC_STUFF] < si->stuff[SMP_END_STUFF]);
+
+	smp_new_sign(sc, &sc->ban1, si->stuff[SMP_BAN1_STUFF], "BAN 1");
+	smp_new_sign(sc, &sc->ban2, si->stuff[SMP_BAN2_STUFF], "BAN 2");
+	smp_new_sign(sc, &sc->seg1, si->stuff[SMP_SEG1_STUFF], "SEG 1");
+	smp_new_sign(sc, &sc->seg2, si->stuff[SMP_SEG2_STUFF], "SEG 2");
+
+	smp_append_sign(&sc->idn, si, sizeof *si);
+	smp_sync_sign(&sc->idn);
+}
+
+/*--------------------------------------------------------------------
+ * Check if a silo is valid.
+ */
+
+int
+smp_valid_silo(struct smp_sc *sc)
+{
+	struct smp_ident	*si;
+	int i, j;
+
+	assert(strlen(SMP_IDENT_STRING) < sizeof si->ident);
+
+	i = smp_chk_sign(&sc->idn);
+	if (i)
+		return (i);
+
+	si = sc->ident;
+	if (strcmp(si->ident, SMP_IDENT_STRING))
+		return (12);
+	if (si->byte_order != 0x12345678)
+		return (13);
+	if (si->size != sizeof *si)
+		return (14);
+	if (si->major_version != 2)
+		return (15);
+	if (si->mediasize != sc->mediasize)
+		return (17);
+	if (si->granularity != sc->granularity)
+		return (18);
+	if (si->align < sizeof(void*))
+		return (19);
+	if (!PWR2(si->align))
+		return (20);
+	sc->align = si->align;
+	sc->unique = si->unique;
+
+	/* XXX: Sanity check stuff[6] */
+
+	assert(si->stuff[SMP_BAN1_STUFF] > sizeof *si + SHA256_LEN);
+	assert(si->stuff[SMP_BAN2_STUFF] > si->stuff[SMP_BAN1_STUFF]);
+	assert(si->stuff[SMP_SEG1_STUFF] > si->stuff[SMP_BAN2_STUFF]);
+	assert(si->stuff[SMP_SEG2_STUFF] > si->stuff[SMP_SEG1_STUFF]);
+	assert(si->stuff[SMP_SPC_STUFF] > si->stuff[SMP_SEG2_STUFF]);
+	assert(si->stuff[SMP_END_STUFF] == sc->mediasize);
+
+	assert(smp_stuff_len(sc, SMP_SEG1_STUFF) > 65536);
+	assert(smp_stuff_len(sc, SMP_SEG1_STUFF) ==
+	  smp_stuff_len(sc, SMP_SEG2_STUFF));
+
+	assert(smp_stuff_len(sc, SMP_BAN1_STUFF) > 65536);
+	assert(smp_stuff_len(sc, SMP_BAN1_STUFF) ==
+	  smp_stuff_len(sc, SMP_BAN2_STUFF));
+
+	smp_def_sign(sc, &sc->ban1, si->stuff[SMP_BAN1_STUFF], "BAN 1");
+	smp_def_sign(sc, &sc->ban2, si->stuff[SMP_BAN2_STUFF], "BAN 2");
+	smp_def_sign(sc, &sc->seg1, si->stuff[SMP_SEG1_STUFF], "SEG 1");
+	smp_def_sign(sc, &sc->seg2, si->stuff[SMP_SEG2_STUFF], "SEG 2");
+
+	/* We must have one valid BAN table */
+	i = smp_chk_sign(&sc->ban1);
+	j = smp_chk_sign(&sc->ban2);
+	if (i && j)
+		return (100 + i * 10 + j);
+
+	/* We must have one valid SEG table */
+	i = smp_chk_sign(&sc->seg1);
+	j = smp_chk_sign(&sc->seg2);
+	if (i && j)
+		return (200 + i * 10 + j);
+	return (0);
+}
diff --git a/bin/varnishd/storage/storage_synth.c b/bin/varnishd/storage/storage_synth.c
new file mode 100644
index 0000000..0519738
--- /dev/null
+++ b/bin/varnishd/storage/storage_synth.c
@@ -0,0 +1,120 @@
+/*-
+ * Copyright (c) 2008-2011 Varnish Software AS
+ * All rights reserved.
+ *
+ * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * Storage method for synthetic content, based on vsb.
+ */
+
+#include "config.h"
+
+#include <stdlib.h>
+
+#include "cache.h"
+#include "storage/storage.h"
+
+
+static struct lock		sms_mtx;
+
+static void
+sms_free(struct storage *sto)
+{
+
+	CHECK_OBJ_NOTNULL(sto, STORAGE_MAGIC);
+	Lck_Lock(&sms_mtx);
+	VSC_C_main->sms_nobj--;
+	VSC_C_main->sms_nbytes -= sto->len;
+	VSC_C_main->sms_bfree += sto->len;
+	Lck_Unlock(&sms_mtx);
+	VSB_delete(sto->priv);
+	free(sto);
+}
+
+void
+SMS_Init(void)
+{
+
+	Lck_New(&sms_mtx, lck_sms);
+}
+
+static struct stevedore sms_stevedore = {
+	.magic	=	STEVEDORE_MAGIC,
+	.name	=	"synth",
+	.free	=	sms_free,
+};
+
+struct vsb *
+SMS_Makesynth(struct object *obj)
+{
+	struct storage *sto;
+	struct vsb *vsb;
+
+	CHECK_OBJ_NOTNULL(obj, OBJECT_MAGIC);
+	STV_Freestore(obj);
+	obj->len = 0;
+
+	Lck_Lock(&sms_mtx);
+	VSC_C_main->sms_nreq++;
+	VSC_C_main->sms_nobj++;
+	Lck_Unlock(&sms_mtx);
+
+	sto = calloc(sizeof *sto, 1);
+	XXXAN(sto);
+	vsb = VSB_new_auto();
+	XXXAN(vsb);
+	sto->priv = vsb;
+	sto->len = 0;
+	sto->space = 0;
+#ifdef SENDFILE_WORKS
+	sto->fd = -1;
+#endif
+	sto->stevedore = &sms_stevedore;
+	sto->magic = STORAGE_MAGIC;
+
+	VTAILQ_INSERT_TAIL(&obj->store, sto, list);
+	return (vsb);
+}
+
+void
+SMS_Finish(struct object *obj)
+{
+	struct storage *sto;
+	struct vsb *vsb;
+
+	CHECK_OBJ_NOTNULL(obj, OBJECT_MAGIC);
+	sto = VTAILQ_FIRST(&obj->store);
+	assert(sto->stevedore == &sms_stevedore);
+	vsb = sto->priv;
+	AZ(VSB_finish(vsb));
+
+	sto->ptr = (void*)VSB_data(vsb);
+	sto->len = VSB_len(vsb);
+	sto->space = VSB_len(vsb);
+	obj->len = sto->len;
+	Lck_Lock(&sms_mtx);
+	VSC_C_main->sms_nbytes += sto->len;
+	VSC_C_main->sms_balloc += sto->len;
+	Lck_Unlock(&sms_mtx);
+}
diff --git a/bin/varnishd/storage/storage_umem.c b/bin/varnishd/storage/storage_umem.c
new file mode 100644
index 0000000..78110d4
--- /dev/null
+++ b/bin/varnishd/storage/storage_umem.c
@@ -0,0 +1,166 @@
+/*-
+ * Copyright (c) 2006 Verdens Gang AS
+ * Copyright (c) 2006-2011 Varnish Software AS
+ * All rights reserved.
+ *
+ * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * Storage method based on umem_alloc(3MALLOC)
+ */
+
+#include "config.h"
+
+#ifdef HAVE_LIBUMEM
+
+#include <sys/types.h>
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <umem.h>
+
+#include "cache.h"
+#include "storage/storage.h"
+
+static size_t			smu_max = SIZE_MAX;
+static MTX			smu_mtx;
+
+struct smu {
+	struct storage		s;
+	size_t			sz;
+};
+
+static struct storage *
+smu_alloc(struct stevedore *st, size_t size)
+{
+	struct smu *smu;
+
+	Lck_Lock(&smu_mtx);
+	VSC_C_main->sma_nreq++;
+	if (VSC_C_main->sma_nbytes + size > smu_max)
+		size = 0;
+	else {
+		VSC_C_main->sma_nobj++;
+		VSC_C_main->sma_nbytes += size;
+		VSC_C_main->sma_balloc += size;
+	}
+	Lck_Unlock(&smu_mtx);
+
+	if (size == 0)
+		return (NULL);
+
+	smu = umem_zalloc(sizeof *smu, UMEM_DEFAULT);
+	if (smu == NULL)
+		return (NULL);
+	smu->sz = size;
+	smu->s.priv = smu;
+	smu->s.ptr = umem_alloc(size, UMEM_DEFAULT);
+	XXXAN(smu->s.ptr);
+	smu->s.len = 0;
+	smu->s.space = size;
+	smu->s.fd = -1;
+	smu->s.stevedore = st;
+	smu->s.magic = STORAGE_MAGIC;
+	return (&smu->s);
+}
+
+static void
+smu_free(struct storage *s)
+{
+	struct smu *smu;
+
+	CHECK_OBJ_NOTNULL(s, STORAGE_MAGIC);
+	smu = s->priv;
+	assert(smu->sz == smu->s.space);
+	Lck_Lock(&smu_mtx);
+	VSC_C_main->sma_nobj--;
+	VSC_C_main->sma_nbytes -= smu->sz;
+	VSC_C_main->sma_bfree += smu->sz;
+	Lck_Unlock(&smu_mtx);
+	umem_free(smu->s.ptr, smu->s.space);
+	umem_free(smu, sizeof *smu);
+}
+
+static void
+smu_trim(const struct storage *s, size_t size)
+{
+	struct smu *smu;
+	void *p;
+
+	CHECK_OBJ_NOTNULL(s, STORAGE_MAGIC);
+	smu = s->priv;
+	assert(smu->sz == smu->s.space);
+	if ((p = umem_alloc(size, UMEM_DEFAULT)) != NULL) {
+		memcpy(p, smu->s.ptr, size);
+		umem_free(smu->s.ptr, smu->s.space);
+		Lck_Lock(&smu_mtx);
+		VSC_C_main->sma_nbytes -= (smu->sz - size);
+		VSC_C_main->sma_bfree += smu->sz - size;
+		smu->sz = size;
+		Lck_Unlock(&smu_mtx);
+		smu->s.ptr = p;
+		smu->s.space = size;
+	}
+}
+
+static void
+smu_init(struct stevedore *parent, int ac, char * const *av)
+{
+	const char *e;
+	uintmax_t u;
+
+	(void)parent;
+
+	AZ(av[ac]);
+	if (ac > 1)
+		ARGV_ERR("(-sumem) too many arguments\n");
+
+	if (ac == 0 || *av[0] == '\0')
+		 return;
+
+	e = VNUM_2bytes(av[0], &u, 0);
+	if (e != NULL)
+		ARGV_ERR("(-sumem) size \"%s\": %s\n", av[0], e);
+	if ((u != (uintmax_t)(size_t)u))
+		ARGV_ERR("(-sumem) size \"%s\": too big\n", av[0]);
+	smu_max = u;
+}
+
+static void
+smu_open(const struct stevedore *st)
+{
+	(void)st;
+	AZ(pthread_mutex_init(&smu_mtx, NULL));
+}
+
+const struct stevedore smu_stevedore = {
+	.magic	=	STEVEDORE_MAGIC,
+	.name	=	"umem",
+	.init	=	smu_init,
+	.open	=	smu_open,
+	.alloc	=	smu_alloc,
+	.free	=	smu_free,
+	.trim	=	smu_trim,
+};
+
+#endif /* HAVE_UMEM_H */
diff --git a/bin/varnishd/storage_file.c b/bin/varnishd/storage_file.c
deleted file mode 100644
index ecfc4ff..0000000
--- a/bin/varnishd/storage_file.c
+++ /dev/null
@@ -1,615 +0,0 @@
-/*-
- * Copyright (c) 2006 Verdens Gang AS
- * Copyright (c) 2006-2010 Varnish Software AS
- * All rights reserved.
- *
- * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- *
- * Storage method based on mmap'ed file
- */
-
-#include "config.h"
-
-#include <sys/mman.h>
-
-#include <stdio.h>
-#include <stdlib.h>
-
-#include "cache.h"
-#include "storage.h"
-
-#include "vnum.h"
-
-#ifndef MAP_NOCORE
-#define MAP_NOCORE 0 /* XXX Linux */
-#endif
-
-#ifndef MAP_NOSYNC
-#define MAP_NOSYNC 0 /* XXX Linux */
-#endif
-
-#define MINPAGES		128
-
-/*
- * Number of buckets on free-list.
- *
- * Last bucket is "larger than" so choose number so that the second
- * to last bucket matches the 128k CHUNKSIZE in cache_fetch.c when
- * using the a 4K minimal page size
- */
-#define NBUCKET			(128 / 4 + 1)
-
-/*--------------------------------------------------------------------*/
-
-VTAILQ_HEAD(smfhead, smf);
-
-struct smf {
-	unsigned		magic;
-#define SMF_MAGIC		0x0927a8a0
-	struct storage		s;
-	struct smf_sc		*sc;
-
-	int			alloc;
-
-	off_t			size;
-	off_t			offset;
-	unsigned char		*ptr;
-
-	VTAILQ_ENTRY(smf)	order;
-	VTAILQ_ENTRY(smf)	status;
-	struct smfhead		*flist;
-};
-
-struct smf_sc {
-	unsigned		magic;
-#define SMF_SC_MAGIC		0x52962ee7
-	struct lock		mtx;
-	struct VSC_C_smf	*stats;
-
-	const char		*filename;
-	int			fd;
-	unsigned		pagesize;
-	uintmax_t		filesize;
-	struct smfhead		order;
-	struct smfhead		free[NBUCKET];
-	struct smfhead		used;
-};
-
-/*--------------------------------------------------------------------*/
-
-static void
-smf_initfile(struct smf_sc *sc, const char *size)
-{
-	sc->filesize = STV_FileSize(sc->fd, size, &sc->pagesize, "-sfile");
-
-	AZ(ftruncate(sc->fd, (off_t)sc->filesize));
-
-	/* XXX: force block allocation here or in open ? */
-}
-
-static const char default_size[] = "100M";
-static const char default_filename[] = ".";
-
-static void
-smf_init(struct stevedore *parent, int ac, char * const *av)
-{
-	const char *size, *fn, *r;
-	struct smf_sc *sc;
-	unsigned u;
-	uintmax_t page_size;
-
-	AZ(av[ac]);
-
-	fn = default_filename;
-	size = default_size;
-	page_size = getpagesize();
-
-	if (ac > 3)
-		ARGV_ERR("(-sfile) too many arguments\n");
-	if (ac > 0 && *av[0] != '\0')
-		fn = av[0];
-	if (ac > 1 && *av[1] != '\0')
-		size = av[1];
-	if (ac > 2 && *av[2] != '\0') {
-
-		r = VNUM_2bytes(av[2], &page_size, 0);
-		if (r != NULL)
-			ARGV_ERR("(-sfile) granularity \"%s\": %s\n", av[2], r);
-	}
-
-	AN(fn);
-	AN(size);
-
-	ALLOC_OBJ(sc, SMF_SC_MAGIC);
-	XXXAN(sc);
-	VTAILQ_INIT(&sc->order);
-	for (u = 0; u < NBUCKET; u++)
-		VTAILQ_INIT(&sc->free[u]);
-	VTAILQ_INIT(&sc->used);
-	sc->pagesize = page_size;
-
-	parent->priv = sc;
-
-	(void)STV_GetFile(fn, &sc->fd, &sc->filename, "-sfile");
-
-	mgt_child_inherit(sc->fd, "storage_file");
-	smf_initfile(sc, size);
-}
-
-/*--------------------------------------------------------------------
- * Insert/Remove from correct freelist
- */
-
-static void
-insfree(struct smf_sc *sc, struct smf *sp)
-{
-	size_t b;
-	struct smf *sp2;
-	size_t ns;
-
-	assert(sp->alloc == 0);
-	assert(sp->flist == NULL);
-	Lck_AssertHeld(&sc->mtx);
-	b = sp->size / sc->pagesize;
-	if (b >= NBUCKET) {
-		b = NBUCKET - 1;
-		sc->stats->g_smf_large++;
-	} else {
-		sc->stats->g_smf_frag++;
-	}
-	sp->flist = &sc->free[b];
-	ns = b * sc->pagesize;
-	VTAILQ_FOREACH(sp2, sp->flist, status) {
-		assert(sp2->size >= ns);
-		assert(sp2->alloc == 0);
-		assert(sp2->flist == sp->flist);
-		if (sp->offset < sp2->offset)
-			break;
-	}
-	if (sp2 == NULL)
-		VTAILQ_INSERT_TAIL(sp->flist, sp, status);
-	else
-		VTAILQ_INSERT_BEFORE(sp2, sp, status);
-}
-
-static void
-remfree(const struct smf_sc *sc, struct smf *sp)
-{
-	size_t b;
-
-	assert(sp->alloc == 0);
-	assert(sp->flist != NULL);
-	Lck_AssertHeld(&sc->mtx);
-	b = sp->size / sc->pagesize;
-	if (b >= NBUCKET) {
-		b = NBUCKET - 1;
-		sc->stats->g_smf_large--;
-	} else {
-		sc->stats->g_smf_frag--;
-	}
-	assert(sp->flist == &sc->free[b]);
-	VTAILQ_REMOVE(sp->flist, sp, status);
-	sp->flist = NULL;
-}
-
-/*--------------------------------------------------------------------
- * Allocate a range from the first free range that is large enough.
- */
-
-static struct smf *
-alloc_smf(struct smf_sc *sc, size_t bytes)
-{
-	struct smf *sp, *sp2;
-	size_t b;
-
-	assert(!(bytes % sc->pagesize));
-	b = bytes / sc->pagesize;
-	if (b >= NBUCKET)
-		b = NBUCKET - 1;
-	for (sp = NULL; b < NBUCKET - 1; b++) {
-		sp = VTAILQ_FIRST(&sc->free[b]);
-		if (sp != NULL)
-			break;
-	}
-	if (sp == NULL) {
-		VTAILQ_FOREACH(sp, &sc->free[NBUCKET -1], status)
-			if (sp->size >= bytes)
-				break;
-	}
-	if (sp == NULL)
-		return (sp);
-
-	assert(sp->size >= bytes);
-	remfree(sc, sp);
-
-	if (sp->size == bytes) {
-		sp->alloc = 1;
-		VTAILQ_INSERT_TAIL(&sc->used, sp, status);
-		return (sp);
-	}
-
-	/* Split from front */
-	sp2 = malloc(sizeof *sp2);
-	XXXAN(sp2);
-	sc->stats->g_smf++;
-	*sp2 = *sp;
-
-	sp->offset += bytes;
-	sp->ptr += bytes;
-	sp->size -= bytes;
-
-	sp2->size = bytes;
-	sp2->alloc = 1;
-	VTAILQ_INSERT_BEFORE(sp, sp2, order);
-	VTAILQ_INSERT_TAIL(&sc->used, sp2, status);
-	insfree(sc, sp);
-	return (sp2);
-}
-
-/*--------------------------------------------------------------------
- * Free a range.  Attempt merge forward and backward, then sort into
- * free list according to age.
- */
-
-static void
-free_smf(struct smf *sp)
-{
-	struct smf *sp2;
-	struct smf_sc *sc = sp->sc;
-
-	CHECK_OBJ_NOTNULL(sp, SMF_MAGIC);
-	assert(sp->alloc != 0);
-	assert(sp->size > 0);
-	assert(!(sp->size % sc->pagesize));
-	VTAILQ_REMOVE(&sc->used, sp, status);
-	sp->alloc = 0;
-
-	sp2 = VTAILQ_NEXT(sp, order);
-	if (sp2 != NULL &&
-	    sp2->alloc == 0 &&
-	    (sp2->ptr == sp->ptr + sp->size) &&
-	    (sp2->offset == sp->offset + sp->size)) {
-		sp->size += sp2->size;
-		VTAILQ_REMOVE(&sc->order, sp2, order);
-		remfree(sc, sp2);
-		free(sp2);
-		sc->stats->g_smf--;
-	}
-
-	sp2 = VTAILQ_PREV(sp, smfhead, order);
-	if (sp2 != NULL &&
-	    sp2->alloc == 0 &&
-	    (sp->ptr == sp2->ptr + sp2->size) &&
-	    (sp->offset == sp2->offset + sp2->size)) {
-		remfree(sc, sp2);
-		sp2->size += sp->size;
-		VTAILQ_REMOVE(&sc->order, sp, order);
-		free(sp);
-		sc->stats->g_smf--;
-		sp = sp2;
-	}
-
-	insfree(sc, sp);
-}
-
-/*--------------------------------------------------------------------
- * Trim the tail of a range.
- */
-
-static void
-trim_smf(struct smf *sp, size_t bytes)
-{
-	struct smf *sp2;
-	struct smf_sc *sc = sp->sc;
-
-	assert(sp->alloc != 0);
-	assert(bytes > 0);
-	assert(bytes < sp->size);
-	assert(!(bytes % sc->pagesize));
-	assert(!(sp->size % sc->pagesize));
-	CHECK_OBJ_NOTNULL(sp, SMF_MAGIC);
-	sp2 = malloc(sizeof *sp2);
-	XXXAN(sp2);
-	sc->stats->g_smf++;
-	*sp2 = *sp;
-
-	sp2->size -= bytes;
-	sp->size = bytes;
-	sp2->ptr += bytes;
-	sp2->offset += bytes;
-	VTAILQ_INSERT_AFTER(&sc->order, sp, sp2, order);
-	VTAILQ_INSERT_TAIL(&sc->used, sp2, status);
-	free_smf(sp2);
-}
-
-/*--------------------------------------------------------------------
- * Insert a newly created range as busy, then free it to do any collapses
- */
-
-static void
-new_smf(struct smf_sc *sc, unsigned char *ptr, off_t off, size_t len)
-{
-	struct smf *sp, *sp2;
-
-	assert(!(len % sc->pagesize));
-	sp = calloc(sizeof *sp, 1);
-	XXXAN(sp);
-	sp->magic = SMF_MAGIC;
-	sp->s.magic = STORAGE_MAGIC;
-	sc->stats->g_smf++;
-
-	sp->sc = sc;
-	sp->size = len;
-	sp->ptr = ptr;
-	sp->offset = off;
-	sp->alloc = 1;
-
-	VTAILQ_FOREACH(sp2, &sc->order, order) {
-		if (sp->ptr < sp2->ptr) {
-			VTAILQ_INSERT_BEFORE(sp2, sp, order);
-			break;
-		}
-	}
-	if (sp2 == NULL)
-		VTAILQ_INSERT_TAIL(&sc->order, sp, order);
-
-	VTAILQ_INSERT_HEAD(&sc->used, sp, status);
-
-	free_smf(sp);
-}
-
-/*--------------------------------------------------------------------*/
-
-/*
- * XXX: This may be too aggressive and soak up too much address room.
- * XXX: On the other hand, the user, directly or implicitly asked us to
- * XXX: use this much storage, so we should make a decent effort.
- * XXX: worst case (I think), malloc will fail.
- */
-
-static void
-smf_open_chunk(struct smf_sc *sc, off_t sz, off_t off, off_t *fail, off_t *sum)
-{
-	void *p;
-	off_t h;
-
-	assert(sz != 0);
-	assert(!(sz % sc->pagesize));
-
-	if (*fail < (uintmax_t)sc->pagesize * MINPAGES)
-		return;
-
-	if (sz > 0 && sz < *fail && sz < SSIZE_MAX) {
-		p = mmap(NULL, sz, PROT_READ|PROT_WRITE,
-		    MAP_NOCORE | MAP_NOSYNC | MAP_SHARED, sc->fd, off);
-		if (p != MAP_FAILED) {
-			(void) madvise(p, sz, MADV_RANDOM);
-			(*sum) += sz;
-			new_smf(sc, p, off, sz);
-			return;
-		}
-	}
-
-	if (sz < *fail)
-		*fail = sz;
-
-	h = sz / 2;
-	if (h > SSIZE_MAX)
-		h = SSIZE_MAX;
-	h -= (h % sc->pagesize);
-
-	smf_open_chunk(sc, h, off, fail, sum);
-	smf_open_chunk(sc, sz - h, off + h, fail, sum);
-}
-
-static void
-smf_open(const struct stevedore *st)
-{
-	struct smf_sc *sc;
-	off_t fail = 1 << 30;	/* XXX: where is OFF_T_MAX ? */
-	off_t sum = 0;
-
-	CAST_OBJ_NOTNULL(sc, st->priv, SMF_SC_MAGIC);
-	sc->stats = VSM_Alloc(sizeof *sc->stats,
-	    VSC_CLASS, VSC_TYPE_SMF, st->ident);
-	Lck_New(&sc->mtx, lck_smf);
-	Lck_Lock(&sc->mtx);
-	smf_open_chunk(sc, sc->filesize, 0, &fail, &sum);
-	Lck_Unlock(&sc->mtx);
-	printf("SMF.%s mmap'ed %ju bytes of %ju\n",
-	    st->ident, (uintmax_t)sum, sc->filesize);
-
-	/* XXX */
-	if (sum < MINPAGES * (off_t)getpagesize())
-		exit (2);
-
-	sc->stats->g_space += sc->filesize;
-}
-
-/*--------------------------------------------------------------------*/
-
-static struct storage *
-smf_alloc(struct stevedore *st, size_t size)
-{
-	struct smf *smf;
-	struct smf_sc *sc;
-
-	CAST_OBJ_NOTNULL(sc, st->priv, SMF_SC_MAGIC);
-	assert(size > 0);
-	size += (sc->pagesize - 1);
-	size &= ~(sc->pagesize - 1);
-	Lck_Lock(&sc->mtx);
-	sc->stats->c_req++;
-	smf = alloc_smf(sc, size);
-	if (smf == NULL) {
-		sc->stats->c_fail++;
-		Lck_Unlock(&sc->mtx);
-		return (NULL);
-	}
-	CHECK_OBJ_NOTNULL(smf, SMF_MAGIC);
-	sc->stats->g_alloc++;
-	sc->stats->c_bytes += smf->size;
-	sc->stats->g_bytes += smf->size;
-	sc->stats->g_space -= smf->size;
-	Lck_Unlock(&sc->mtx);
-	CHECK_OBJ_NOTNULL(&smf->s, STORAGE_MAGIC);	/*lint !e774 */
-	XXXAN(smf);
-	assert(smf->size == size);
-	smf->s.space = size;
-	smf->s.priv = smf;
-	smf->s.ptr = smf->ptr;
-	smf->s.len = 0;
-	smf->s.stevedore = st;
-#ifdef SENDFILE_WORKS
-	smf->s.fd = smf->sc->fd;
-	smf->s.where = smf->offset;
-#endif
-	return (&smf->s);
-}
-
-/*--------------------------------------------------------------------*/
-
-static void
-smf_trim(struct storage *s, size_t size)
-{
-	struct smf *smf;
-	struct smf_sc *sc;
-
-	CHECK_OBJ_NOTNULL(s, STORAGE_MAGIC);
-	assert(size > 0);
-	assert(size <= s->space);
-	xxxassert(size > 0);	/* XXX: seen */
-	CAST_OBJ_NOTNULL(smf, s->priv, SMF_MAGIC);
-	assert(size <= smf->size);
-	sc = smf->sc;
-	size += (sc->pagesize - 1);
-	size &= ~(sc->pagesize - 1);
-	if (smf->size > size) {
-		Lck_Lock(&sc->mtx);
-		sc->stats->c_freed += (smf->size - size);
-		sc->stats->g_bytes -= (smf->size - size);
-		sc->stats->g_space += (smf->size - size);
-		trim_smf(smf, size);
-		assert(smf->size == size);
-		Lck_Unlock(&sc->mtx);
-		s->space = size;
-	}
-}
-
-/*--------------------------------------------------------------------*/
-
-static void __match_proto__(storage_free_f)
-smf_free(struct storage *s)
-{
-	struct smf *smf;
-	struct smf_sc *sc;
-
-	CHECK_OBJ_NOTNULL(s, STORAGE_MAGIC);
-	CAST_OBJ_NOTNULL(smf, s->priv, SMF_MAGIC);
-	sc = smf->sc;
-	Lck_Lock(&sc->mtx);
-	sc->stats->g_alloc--;
-	sc->stats->c_freed += smf->size;
-	sc->stats->g_bytes -= smf->size;
-	sc->stats->g_space += smf->size;
-	free_smf(smf);
-	Lck_Unlock(&sc->mtx);
-}
-
-/*--------------------------------------------------------------------*/
-
-const struct stevedore smf_stevedore = {
-	.magic	=	STEVEDORE_MAGIC,
-	.name	=	"file",
-	.init	=	smf_init,
-	.open	=	smf_open,
-	.alloc	=	smf_alloc,
-	.trim	=	smf_trim,
-	.free	=	smf_free,
-};
-
-#ifdef INCLUDE_TEST_DRIVER
-
-void vca_flush(struct sess *sp) {}
-
-#define N	100
-#define M	(128*1024)
-
-struct storage *s[N];
-
-static void
-dumpit(void)
-{
-	struct smf_sc *sc = smf_stevedore.priv;
-	struct smf *s;
-
-	return (0);
-	printf("----------------\n");
-	printf("Order:\n");
-	VTAILQ_FOREACH(s, &sc->order, order) {
-		printf("%10p %12ju %12ju %12ju\n",
-		    s, s->offset, s->size, s->offset + s->size);
-	}
-	printf("Used:\n");
-	VTAILQ_FOREACH(s, &sc->used, status) {
-		printf("%10p %12ju %12ju %12ju\n",
-		    s, s->offset, s->size, s->offset + s->size);
-	}
-	printf("Free:\n");
-	VTAILQ_FOREACH(s, &sc->free, status) {
-		printf("%10p %12ju %12ju %12ju\n",
-		    s, s->offset, s->size, s->offset + s->size);
-	}
-	printf("================\n");
-}
-
-int
-main(int argc, char **argv)
-{
-	int i, j;
-
-	setbuf(stdout, NULL);
-	smf_init(&smf_stevedore, "");
-	smf_open(&smf_stevedore);
-	while (1) {
-		dumpit();
-		i = random() % N;
-		do
-			j = random() % M;
-		while (j == 0);
-		if (s[i] == NULL) {
-			s[i] = smf_alloc(&smf_stevedore, j);
-			printf("A %10p %12d\n", s[i], j);
-		} else if (j < s[i]->space) {
-			smf_trim(s[i], j);
-			printf("T %10p %12d\n", s[i], j);
-		} else {
-			smf_free(s[i]);
-			printf("D %10p\n", s[i]);
-			s[i] = NULL;
-		}
-	}
-}
-
-#endif /* INCLUDE_TEST_DRIVER */
diff --git a/bin/varnishd/storage_malloc.c b/bin/varnishd/storage_malloc.c
deleted file mode 100644
index 967137c..0000000
--- a/bin/varnishd/storage_malloc.c
+++ /dev/null
@@ -1,256 +0,0 @@
-/*-
- * Copyright (c) 2006 Verdens Gang AS
- * Copyright (c) 2006-2011 Varnish Software AS
- * All rights reserved.
- *
- * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- *
- * Storage method based on malloc(3)
- */
-
-#include "config.h"
-
-#include <stdio.h>
-#include <stdlib.h>
-
-#include "cache.h"
-#include "storage.h"
-
-#include "vnum.h"
-
-struct sma_sc {
-	unsigned		magic;
-#define SMA_SC_MAGIC		0x1ac8a345
-	struct lock		sma_mtx;
-	size_t			sma_max;
-	size_t			sma_alloc;
-	struct VSC_C_sma	*stats;
-};
-
-struct sma {
-	unsigned		magic;
-#define SMA_MAGIC		0x69ae9bb9
-	struct storage		s;
-	size_t			sz;
-	struct sma_sc		*sc;
-};
-
-static struct storage *
-sma_alloc(struct stevedore *st, size_t size)
-{
-	struct sma_sc *sma_sc;
-	struct sma *sma = NULL;
-	void *p;
-
-	CAST_OBJ_NOTNULL(sma_sc, st->priv, SMA_SC_MAGIC);
-	Lck_Lock(&sma_sc->sma_mtx);
-	sma_sc->stats->c_req++;
-	if (sma_sc->sma_alloc + size > sma_sc->sma_max) {
-		sma_sc->stats->c_fail += size;
-		size = 0;
-	} else {
-		sma_sc->sma_alloc += size;
-		sma_sc->stats->c_bytes += size;
-		sma_sc->stats->g_alloc++;
-		sma_sc->stats->g_bytes += size;
-		if (sma_sc->sma_max != SIZE_MAX)
-			sma_sc->stats->g_space -= size;
-	}
-	Lck_Unlock(&sma_sc->sma_mtx);
-
-	if (size == 0)
-		return (NULL);
-
-	/*
-	 * Do not collaps the sma allocation with sma->s.ptr: it is not
-	 * a good idea.  Not only would it make ->trim impossible,
-	 * performance-wise it would be a catastropy with chunksized
-	 * allocations growing another full page, just to accomodate the sma.
-	 */
-
-	p = malloc(size);
-	if (p != NULL) {
-		ALLOC_OBJ(sma, SMA_MAGIC);
-		if (sma != NULL)
-			sma->s.ptr = p;
-		else
-			free(p);
-	}
-	if (sma == NULL) {
-		Lck_Lock(&sma_sc->sma_mtx);
-		/*
-		 * XXX: Not nice to have counters go backwards, but we do
-		 * XXX: Not want to pick up the lock twice just for stats.
-		 */
-		sma_sc->stats->c_fail++;
-		sma_sc->stats->c_bytes -= size;
-		sma_sc->stats->g_alloc--;
-		sma_sc->stats->g_bytes -= size;
-		if (sma_sc->sma_max != SIZE_MAX)
-			sma_sc->stats->g_space += size;
-		Lck_Unlock(&sma_sc->sma_mtx);
-		return (NULL);
-	}
-	sma->sc = sma_sc;
-	sma->sz = size;
-	sma->s.priv = sma;
-	sma->s.len = 0;
-	sma->s.space = size;
-#ifdef SENDFILE_WORKS
-	sma->s.fd = -1;
-#endif
-	sma->s.stevedore = st;
-	sma->s.magic = STORAGE_MAGIC;
-	return (&sma->s);
-}
-
-static void __match_proto__(storage_free_f)
-sma_free(struct storage *s)
-{
-	struct sma_sc *sma_sc;
-	struct sma *sma;
-
-	CHECK_OBJ_NOTNULL(s, STORAGE_MAGIC);
-	CAST_OBJ_NOTNULL(sma, s->priv, SMA_MAGIC);
-	sma_sc = sma->sc;
-	assert(sma->sz == sma->s.space);
-	Lck_Lock(&sma_sc->sma_mtx);
-	sma_sc->sma_alloc -= sma->sz;
-	sma_sc->stats->g_alloc--;
-	sma_sc->stats->g_bytes -= sma->sz;
-	sma_sc->stats->c_freed += sma->sz;
-	if (sma_sc->sma_max != SIZE_MAX)
-		sma_sc->stats->g_space += sma->sz;
-	Lck_Unlock(&sma_sc->sma_mtx);
-	free(sma->s.ptr);
-	free(sma);
-}
-
-static void
-sma_trim(struct storage *s, size_t size)
-{
-	struct sma_sc *sma_sc;
-	struct sma *sma;
-	void *p;
-	size_t delta;
-
-	CHECK_OBJ_NOTNULL(s, STORAGE_MAGIC);
-	CAST_OBJ_NOTNULL(sma, s->priv, SMA_MAGIC);
-	sma_sc = sma->sc;
-
-	assert(sma->sz == sma->s.space);
-	assert(size < sma->sz);
-	delta = sma->sz - size;
-	if (delta < 256)
-		return;
-	if ((p = realloc(sma->s.ptr, size)) != NULL) {
-		Lck_Lock(&sma_sc->sma_mtx);
-		sma_sc->sma_alloc -= delta;
-		sma_sc->stats->g_bytes -= delta;
-		sma_sc->stats->c_freed += delta;
-		if (sma_sc->sma_max != SIZE_MAX)
-			sma_sc->stats->g_space += delta;
-		sma->sz = size;
-		Lck_Unlock(&sma_sc->sma_mtx);
-		sma->s.ptr = p;
-		s->space = size;
-	}
-}
-
-static double
-sma_used_space(const struct stevedore *st)
-{
-	struct sma_sc *sma_sc;
-
-	CAST_OBJ_NOTNULL(sma_sc, st->priv, SMA_SC_MAGIC);
-	return (sma_sc->sma_alloc);
-}
-
-static double
-sma_free_space(const struct stevedore *st)
-{
-	struct sma_sc *sma_sc;
-
-	CAST_OBJ_NOTNULL(sma_sc, st->priv, SMA_SC_MAGIC);
-	return (sma_sc->sma_max - sma_sc->sma_alloc);
-}
-
-static void
-sma_init(struct stevedore *parent, int ac, char * const *av)
-{
-	const char *e;
-	uintmax_t u;
-	struct sma_sc *sc;
-
-	ASSERT_MGT();
-	ALLOC_OBJ(sc, SMA_SC_MAGIC);
-	AN(sc);
-	sc->sma_max = SIZE_MAX;
-	assert(sc->sma_max == SIZE_MAX);
-	parent->priv = sc;
-
-	AZ(av[ac]);
-	if (ac > 1)
-		ARGV_ERR("(-smalloc) too many arguments\n");
-
-	if (ac == 0 || *av[0] == '\0')
-		 return;
-
-	e = VNUM_2bytes(av[0], &u, 0);
-	if (e != NULL)
-		ARGV_ERR("(-smalloc) size \"%s\": %s\n", av[0], e);
-	if ((u != (uintmax_t)(size_t)u))
-		ARGV_ERR("(-smalloc) size \"%s\": too big\n", av[0]);
-	if (u < 1024*1024)
-		ARGV_ERR("(-smalloc) size \"%s\": too small, "
-			 "did you forget to specify M or G?\n", av[0]);
-
-	sc->sma_max = u;
-}
-
-static void
-sma_open(const struct stevedore *st)
-{
-	struct sma_sc *sma_sc;
-
-	CAST_OBJ_NOTNULL(sma_sc, st->priv, SMA_SC_MAGIC);
-	Lck_New(&sma_sc->sma_mtx, lck_sma);
-	sma_sc->stats = VSM_Alloc(sizeof *sma_sc->stats,
-	    VSC_CLASS, VSC_TYPE_SMA, st->ident);
-	memset(sma_sc->stats, 0, sizeof *sma_sc->stats);
-	if (sma_sc->sma_max != SIZE_MAX)
-		sma_sc->stats->g_space = sma_sc->sma_max;
-}
-
-const struct stevedore sma_stevedore = {
-	.magic	=	STEVEDORE_MAGIC,
-	.name	=	"malloc",
-	.init	=	sma_init,
-	.open	=	sma_open,
-	.alloc	=	sma_alloc,
-	.free	=	sma_free,
-	.trim	=	sma_trim,
-	.var_free_space =	sma_free_space,
-	.var_used_space =	sma_used_space,
-};
diff --git a/bin/varnishd/storage_persistent.c b/bin/varnishd/storage_persistent.c
deleted file mode 100644
index abe25f6..0000000
--- a/bin/varnishd/storage_persistent.c
+++ /dev/null
@@ -1,678 +0,0 @@
-/*-
- * Copyright (c) 2008-2011 Varnish Software AS
- * All rights reserved.
- *
- * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- *
- * Persistent storage method
- *
- * XXX: Before we start the client or maybe after it stops, we should give the
- * XXX: stevedores a chance to examine their storage for consistency.
- *
- * XXX: Do we ever free the LRU-lists ?
- */
-
-#include "config.h"
-
-#include <sys/param.h>
-#include <sys/mman.h>
-
-#include <stdint.h>
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-
-#include "cache.h"
-#include "storage.h"
-
-#include "hash_slinger.h"
-#include "vcli.h"
-#include "vcli_priv.h"
-#include "vend.h"
-#include "vsha256.h"
-
-#include "persistent.h"
-#include "storage_persistent.h"
-
-/*--------------------------------------------------------------------*/
-
-/*
- * silos is unlocked, it only changes during startup when we are
- * single-threaded
- */
-static VTAILQ_HEAD(,smp_sc)	silos = VTAILQ_HEAD_INITIALIZER(silos);
-
-/*--------------------------------------------------------------------
- * Add bans to silos
- */
-
-static void
-smp_appendban(struct smp_sc *sc, struct smp_signctx *ctx,
-    uint32_t len, const uint8_t *ban)
-{
-	uint8_t *ptr, *ptr2;
-
-	(void)sc;
-	ptr = ptr2 = SIGN_END(ctx);
-
-	memcpy(ptr, "BAN", 4);
-	ptr += 4;
-
-	vbe32enc(ptr, len);
-	ptr += 4;
-
-	memcpy(ptr, ban, len);
-	ptr += len;
-
-	smp_append_sign(ctx, ptr2, ptr - ptr2);
-}
-
-/* Trust that cache_ban.c takes care of locking */
-
-void
-SMP_NewBan(const uint8_t *ban, unsigned ln)
-{
-	struct smp_sc *sc;
-
-	VTAILQ_FOREACH(sc, &silos, list) {
-		smp_appendban(sc, &sc->ban1, ln, ban);
-		smp_appendban(sc, &sc->ban2, ln, ban);
-	}
-}
-
-/*--------------------------------------------------------------------
- * Attempt to open and read in a ban list
- */
-
-static int
-smp_open_bans(struct smp_sc *sc, struct smp_signctx *ctx)
-{
-	uint8_t *ptr, *pe;
-	uint32_t length;
-	int i, retval = 0;
-
-	ASSERT_CLI();
-	(void)sc;
-	i = smp_chk_sign(ctx);
-	if (i)
-		return (i);
-	ptr = SIGN_DATA(ctx);
-	pe = ptr + ctx->ss->length;
-
-	while (ptr < pe) {
-		if (memcmp(ptr, "BAN", 4)) {
-			retval = 1001;
-			break;
-		}
-		ptr += 4;
-
-		length = vbe32dec(ptr);
-		ptr += 4;
-
-		if (ptr + length > pe) {
-			retval = 1003;
-			break;
-		}
-
-		BAN_Reload(ptr, length);
-
-		ptr += length;
-	}
-	assert(ptr <= pe);
-	return (retval);
-}
-
-/*--------------------------------------------------------------------
- * Attempt to open and read in a segment list
- */
-
-static int
-smp_open_segs(struct smp_sc *sc, struct smp_signctx *ctx)
-{
-	uint64_t length, l;
-	struct smp_segptr *ss, *se;
-	struct smp_seg *sg, *sg1, *sg2;
-	int i, n = 0;
-
-	ASSERT_CLI();
-	i = smp_chk_sign(ctx);
-	if (i)
-		return (i);
-
-	ss = SIGN_DATA(ctx);
-	length = ctx->ss->length;
-
-	if (length == 0) {
-		/* No segments */
-		sc->free_offset = sc->ident->stuff[SMP_SPC_STUFF];
-		return (0);
-	}
-	se = ss + length / sizeof *ss;
-	se--;
-	assert(ss <= se);
-
-	/*
-	 * Locate the free reserve, there are only two basic cases,
-	 * but once we start dropping segments, things gets more complicated.
-	 */
-
-	sc->free_offset = se->offset + se->length;
-	l = sc->mediasize - sc->free_offset;
-	if (se->offset > ss->offset && l >= sc->free_reserve) {
-		/*
-		 * [__xxxxyyyyzzzz___]
-		 * Plenty of space at tail, do nothing.
-		 */
-	} else if (ss->offset > se->offset) {
-		/*
-		 * [zzzz____xxxxyyyy_]
-		 * (make) space between ends
-		 * We might nuke the entire tail end without getting
-		 * enough space, in which case we fall through to the
-		 * last check.
-		 */
-		while (ss < se && ss->offset > se->offset) {
-			l = ss->offset - (se->offset + se->length);
-			if (l > sc->free_reserve)
-				break;
-			ss++;
-			n++;
-		}
-	}
-
-	if (l < sc->free_reserve) {
-		/*
-		 * [__xxxxyyyyzzzz___]
-		 * (make) space at front
-		 */
-		sc->free_offset = sc->ident->stuff[SMP_SPC_STUFF];
-		while (ss < se) {
-			l = ss->offset - sc->free_offset;
-			if (l > sc->free_reserve)
-				break;
-			ss++;
-			n++;
-		}
-	}
-
-	assert (l >= sc->free_reserve);
-
-
-	sg1 = NULL;
-	sg2 = NULL;
-	for(; ss <= se; ss++) {
-		ALLOC_OBJ(sg, SMP_SEG_MAGIC);
-		AN(sg);
-		sg->lru = LRU_Alloc();
-		CHECK_OBJ_NOTNULL(sg->lru, LRU_MAGIC);
-		sg->p = *ss;
-
-		sg->flags |= SMP_SEG_MUSTLOAD;
-
-		/*
-		 * HACK: prevent save_segs from nuking segment until we have
-		 * HACK: loaded it.
-		 */
-		sg->nobj = 1;
-		if (sg1 != NULL) {
-			assert(sg1->p.offset != sg->p.offset);
-			if (sg1->p.offset < sg->p.offset)
-				assert(smp_segend(sg1) <= sg->p.offset);
-			else
-				assert(smp_segend(sg) <= sg1->p.offset);
-		}
-		if (sg2 != NULL) {
-			assert(sg2->p.offset != sg->p.offset);
-			if (sg2->p.offset < sg->p.offset)
-				assert(smp_segend(sg2) <= sg->p.offset);
-			else
-				assert(smp_segend(sg) <= sg2->p.offset);
-		}
-
-		/* XXX: check that they are inside silo */
-		/* XXX: check that they don't overlap */
-		/* XXX: check that they are serial */
-		sg->sc = sc;
-		VTAILQ_INSERT_TAIL(&sc->segments, sg, list);
-		sg2 = sg;
-		if (sg1 == NULL)
-			sg1 = sg;
-	}
-	printf("Dropped %d segments to make free_reserve\n", n);
-	return (0);
-}
-
-/*--------------------------------------------------------------------
- * Silo worker thread
- */
-
-static void *
-smp_thread(struct sess *sp, void *priv)
-{
-	struct smp_sc	*sc;
-	struct smp_seg *sg;
-
-	(void)sp;
-	CAST_OBJ_NOTNULL(sc, priv, SMP_SC_MAGIC);
-
-	/* First, load all the objects from all segments */
-	VTAILQ_FOREACH(sg, &sc->segments, list)
-		if (sg->flags & SMP_SEG_MUSTLOAD)
-			smp_load_seg(sp, sc, sg);
-
-	sc->flags |= SMP_SC_LOADED;
-	BAN_TailDeref(&sc->tailban);
-	AZ(sc->tailban);
-	printf("Silo completely loaded\n");
-	while (1) {
-		(void)sleep (1);
-		sg = VTAILQ_FIRST(&sc->segments);
-		if (sg != NULL && sg -> sc->cur_seg &&
-		    sg->nobj == 0) {
-			Lck_Lock(&sc->mtx);
-			smp_save_segs(sc);
-			Lck_Unlock(&sc->mtx);
-		}
-	}
-	NEEDLESS_RETURN(NULL);
-}
-
-/*--------------------------------------------------------------------
- * Open a silo in the worker process
- */
-
-static void
-smp_open(const struct stevedore *st)
-{
-	struct smp_sc	*sc;
-
-	ASSERT_CLI();
-
-	CAST_OBJ_NOTNULL(sc, st->priv, SMP_SC_MAGIC);
-
-	Lck_New(&sc->mtx, lck_smp);
-	Lck_Lock(&sc->mtx);
-
-	sc->stevedore = st;
-
-	/* We trust the parent to give us a valid silo, for good measure: */
-	AZ(smp_valid_silo(sc));
-
-	AZ(mprotect(sc->base, 4096, PROT_READ));
-
-	sc->ident = SIGN_DATA(&sc->idn);
-
-	/* We attempt ban1 first, and if that fails, try ban2 */
-	if (smp_open_bans(sc, &sc->ban1))
-		AZ(smp_open_bans(sc, &sc->ban2));
-
-	/* We attempt seg1 first, and if that fails, try seg2 */
-	if (smp_open_segs(sc, &sc->seg1))
-		AZ(smp_open_segs(sc, &sc->seg2));
-
-	/*
-	 * Grap a reference to the tail of the ban list, until the thread
-	 * has loaded all objects, so we can be sure that all of our
-	 * proto-bans survive until then.
-	 */
-	sc->tailban = BAN_TailRef();
-	AN(sc->tailban);
-
-	/* XXX: save segments to ensure consistency between seg1 & seg2 ? */
-
-	/* XXX: abandon early segments to make sure we have free space ? */
-
-	/* Open a new segment, so we are ready to write */
-	smp_new_seg(sc);
-
-	/* Start the worker silo worker thread, it will load the objects */
-	WRK_BgThread(&sc->thread, "persistence", smp_thread, sc);
-
-	VTAILQ_INSERT_TAIL(&silos, sc, list);
-	Lck_Unlock(&sc->mtx);
-}
-
-/*--------------------------------------------------------------------
- * Close a silo
- */
-
-static void
-smp_close(const struct stevedore *st)
-{
-	struct smp_sc	*sc;
-
-	ASSERT_CLI();
-
-	CAST_OBJ_NOTNULL(sc, st->priv, SMP_SC_MAGIC);
-	Lck_Lock(&sc->mtx);
-	smp_close_seg(sc, sc->cur_seg);
-	Lck_Unlock(&sc->mtx);
-
-	/* XXX: reap thread */
-}
-
-/*--------------------------------------------------------------------
- * Allocate a bite.
- *
- * Allocate [min_size...max_size] space from the bottom of the segment,
- * as is convenient.
- *
- * If 'so' + 'idx' is given, also allocate a smp_object from the top
- * of the segment.
- *
- * Return the segment in 'ssg' if given.
- */
-
-static struct storage *
-smp_allocx(struct stevedore *st, size_t min_size, size_t max_size,
-    struct smp_object **so, unsigned *idx, struct smp_seg **ssg)
-{
-	struct smp_sc *sc;
-	struct storage *ss;
-	struct smp_seg *sg;
-	unsigned tries;
-	uint64_t left, extra;
-
-	CAST_OBJ_NOTNULL(sc, st->priv, SMP_SC_MAGIC);
-	assert(min_size <= max_size);
-
-	max_size = IRNUP(sc, max_size);
-	min_size = IRNUP(sc, min_size);
-
-	extra = IRNUP(sc, sizeof(*ss));
-	if (so != NULL) {
-		extra += sizeof(**so);
-		AN(idx);
-	}
-
-	Lck_Lock(&sc->mtx);
-	sg = NULL;
-	ss = NULL;
-	for (tries = 0; tries < 3; tries++) {
-		left = smp_spaceleft(sc, sc->cur_seg);
-		if (left >= extra + min_size)
-			break;
-		smp_close_seg(sc, sc->cur_seg);
-		smp_new_seg(sc);
-	}
-	if (left >= extra + min_size)  {
-		if (left < extra + max_size)
-			max_size = IRNDN(sc, left - extra);
-
-		sg = sc->cur_seg;
-		ss = (void*)(sc->base + sc->next_bot);
-		sc->next_bot += max_size + IRNUP(sc, sizeof(*ss));
-		sg->nalloc++;
-		if (so != NULL) {
-			sc->next_top -= sizeof(**so);
-			*so = (void*)(sc->base + sc->next_top);
-			/* Render this smp_object mostly harmless */
-			(*so)->ttl = 0.;
-			(*so)->ban = 0.;
-			(*so)->ptr = 0;;
-			sg->objs = *so;
-			*idx = ++sg->p.lobjlist;
-		}
-		(void)smp_spaceleft(sc, sg);	/* for the assert */
-	}
-	Lck_Unlock(&sc->mtx);
-
-	if (ss == NULL)
-		return (ss);
-	AN(sg);
-	assert(max_size >= min_size);
-
-	/* Fill the storage structure */
-	memset(ss, 0, sizeof *ss);
-	ss->magic = STORAGE_MAGIC;
-	ss->ptr = PRNUP(sc, ss + 1);
-	ss->space = max_size;
-	ss->priv = sc;
-	ss->stevedore = st;
-#ifdef SENDFILE_WORKS
-	ss->fd = sc->fd;
-#endif
-	if (ssg != NULL)
-		*ssg = sg;
-	return (ss);
-}
-
-/*--------------------------------------------------------------------
- * Allocate an object
- */
-
-static struct object *
-smp_allocobj(struct stevedore *stv, struct sess *sp, unsigned ltot,
-    const struct stv_objsecrets *soc)
-{
-	struct object *o;
-	struct storage *st;
-	struct smp_sc	*sc;
-	struct smp_seg *sg;
-	struct smp_object *so;
-	struct objcore *oc;
-	unsigned objidx;
-
-	if (sp->objcore == NULL)
-		return (NULL);		/* from cnt_error */
-	CAST_OBJ_NOTNULL(sc, stv->priv, SMP_SC_MAGIC);
-	AN(sp->objcore);
-	AN(sp->wrk->exp.ttl > 0.);
-
-	ltot = IRNUP(sc, ltot);
-
-	st = smp_allocx(stv, ltot, ltot, &so, &objidx, &sg);
-	if (st == NULL)
-		return (NULL);
-
-	assert(st->space >= ltot);
-	ltot = st->len = st->space;
-
-	o = STV_MkObject(sp, st->ptr, ltot, soc);
-	CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC);
-	o->objstore = st;
-
-	oc = o->objcore;
-	CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
-	oc->flags |= OC_F_LRUDONTMOVE;
-
-	Lck_Lock(&sc->mtx);
-	sg->nfixed++;
-	sg->nobj++;
-
-	/* We have to do this somewhere, might as well be here... */
-	assert(sizeof so->hash == DIGEST_LEN);
-	memcpy(so->hash, oc->objhead->digest, DIGEST_LEN);
-	so->ttl = EXP_Grace(NULL, o);
-	so->ptr = (uint8_t*)o - sc->base;
-	so->ban = BAN_Time(oc->ban);
-
-	smp_init_oc(oc, sg, objidx);
-
-	Lck_Unlock(&sc->mtx);
-	return (o);
-}
-
-/*--------------------------------------------------------------------
- * Allocate a bite
- */
-
-static struct storage *
-smp_alloc(struct stevedore *st, size_t size)
-{
-
-	return (smp_allocx(st,
-	    size > 4096 ? 4096 : size, size, NULL, NULL, NULL));
-}
-
-/*--------------------------------------------------------------------
- * Trim a bite
- * XXX: We could trim the last allocation.
- */
-
-static void
-smp_trim(struct storage *ss, size_t size)
-{
-
-	(void)ss;
-	(void)size;
-}
-
-/*--------------------------------------------------------------------
- * We don't track frees of storage, we track the objects which own the
- * storage and when there are no more objects in in the first segment,
- * it can be reclaimed.
- * XXX: We could free the last allocation, but does that happen ?
- */
-
-static void __match_proto__(storage_free_f)
-smp_free(struct storage *st)
-{
-
-	/* XXX */
-	(void)st;
-}
-
-
-/*--------------------------------------------------------------------*/
-
-const struct stevedore smp_stevedore = {
-	.magic	=	STEVEDORE_MAGIC,
-	.name	=	"persistent",
-	.init	=	smp_mgt_init,
-	.open	=	smp_open,
-	.close	=	smp_close,
-	.alloc	=	smp_alloc,
-	.allocobj =	smp_allocobj,
-	.free	=	smp_free,
-	.trim	=	smp_trim,
-};
-
-/*--------------------------------------------------------------------
- * Persistence is a bear to test unadultered, so we cheat by adding
- * a cli command we can use to make it do tricks for us.
- */
-
-static void
-debug_report_silo(struct cli *cli, const struct smp_sc *sc, int objs)
-{
-	struct smp_seg *sg;
-	struct objcore *oc;
-
-	VCLI_Out(cli, "Silo: %s (%s)\n",
-	    sc->stevedore->ident, sc->filename);
-	VTAILQ_FOREACH(sg, &sc->segments, list) {
-		VCLI_Out(cli, "  Seg: [0x%jx ... +0x%jx]\n",
-		   (uintmax_t)sg->p.offset, (uintmax_t)sg->p.length);
-		if (sg == sc->cur_seg)
-			VCLI_Out(cli,
-			   "    Alloc: [0x%jx ... 0x%jx] = 0x%jx free\n",
-			   (uintmax_t)(sc->next_bot),
-			   (uintmax_t)(sc->next_top),
-			   (uintmax_t)(sc->next_top - sc->next_bot));
-		VCLI_Out(cli, "    %u nobj, %u alloc, %u lobjlist, %u fixed\n",
-		    sg->nobj, sg->nalloc, sg->p.lobjlist, sg->nfixed);
-		if (objs) {
-			VTAILQ_FOREACH(oc, &sg->lru->lru_head, lru_list)
-				VCLI_Out(cli, "      OC %p\n", oc);
-		}
-	}
-}
-
-static void
-debug_persistent(struct cli *cli, const char * const * av, void *priv)
-{
-	struct smp_sc *sc;
-
-	(void)priv;
-
-	if (av[2] == NULL) {
-		VTAILQ_FOREACH(sc, &silos, list)
-			debug_report_silo(cli, sc, 0);
-		return;
-	}
-	VTAILQ_FOREACH(sc, &silos, list)
-		if (!strcmp(av[2], sc->stevedore->ident))
-			break;
-	if (sc == NULL) {
-		VCLI_Out(cli, "Silo <%s> not found\n", av[2]);
-		VCLI_SetResult(cli, CLIS_PARAM);
-		return;
-	}
-	if (av[3] == NULL) {
-		debug_report_silo(cli, sc, 0);
-		return;
-	}
-	Lck_Lock(&sc->mtx);
-	if (!strcmp(av[3], "sync")) {
-		smp_close_seg(sc, sc->cur_seg);
-		smp_new_seg(sc);
-	} else if (!strcmp(av[3], "dump")) {
-		debug_report_silo(cli, sc, 1);
-	} else {
-		VCLI_Out(cli, "Unknown operation\n");
-		VCLI_SetResult(cli, CLIS_PARAM);
-	}
-	Lck_Unlock(&sc->mtx);
-}
-
-static struct cli_proto debug_cmds[] = {
-        { "debug.persistent", "debug.persistent",
-                "Persistent debugging magic:\n"
-		"\tdebug.persistent [stevedore [cmd]]\n"
-		"With no cmd arg, a summary of the silo is returned.\n"
-		"Possible commands:\n"
-		"\tsync\tClose current segment, open a new one\n"
-		"\tdump\tinclude objcores in silo summary\n"
-		"",
-		0, 2, "d", debug_persistent },
-        { NULL }
-};
-
-/*--------------------------------------------------------------------*/
-
-void
-SMP_Init(void)
-{
-	CLI_AddFuncs(debug_cmds);
-}
-
-/*--------------------------------------------------------------------
- * Pause until all silos have loaded.
- */
-
-void
-SMP_Ready(void)
-{
-	struct smp_sc *sc;
-
-	ASSERT_CLI();
-	do {
-		VTAILQ_FOREACH(sc, &silos, list)
-			if (!(sc->flags & SMP_SC_LOADED))
-				break;
-		if (sc != NULL)
-			(void)sleep(1);
-	} while (sc != NULL);
-}
diff --git a/bin/varnishd/storage_persistent.h b/bin/varnishd/storage_persistent.h
deleted file mode 100644
index 84f3d21..0000000
--- a/bin/varnishd/storage_persistent.h
+++ /dev/null
@@ -1,219 +0,0 @@
-/*-
- * Copyright (c) 2008-2011 Varnish Software AS
- * All rights reserved.
- *
- * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- *
- * Persistent storage method
- *
- * XXX: Before we start the client or maybe after it stops, we should give the
- * XXX: stevedores a chance to examine their storage for consistency.
- *
- * XXX: Do we ever free the LRU-lists ?
- */
-
-#define ASSERT_SILO_THREAD(sc) \
-    do {assert(pthread_self() == (sc)->thread);} while (0)
-
-#define OC_F_NEEDFIXUP OC_F_PRIV
-
-/*
- * Context for a signature.
- *
- * A signature is a sequence of bytes in the silo, signed by a SHA256 hash
- * which follows the bytes.
- *
- * The context structure allows us to append to a signature without
- * recalculating the entire SHA256 hash.
- */
-
-struct smp_signctx {
-	struct smp_sign		*ss;
-	struct SHA256Context	ctx;
-	uint32_t		unique;
-	const char		*id;
-};
-
-struct smp_sc;
-
-/* XXX: name confusion with on-media version ? */
-struct smp_seg {
-	unsigned		magic;
-#define SMP_SEG_MAGIC		0x45c61895
-
-	struct smp_sc		*sc;
-	struct lru		*lru;
-
-	VTAILQ_ENTRY(smp_seg)	list;		/* on smp_sc.smp_segments */
-
-	struct smp_segptr	p;
-
-	unsigned		flags;
-#define SMP_SEG_MUSTLOAD	(1 << 0)
-#define SMP_SEG_LOADED		(1 << 1)
-
-	uint32_t		nobj;		/* Number of objects */
-	uint32_t		nalloc;		/* Allocations */
-	uint32_t		nfixed;		/* How many fixed objects */
-
-	/* Only for open segment */
-	struct smp_object	*objs;		/* objdesc array */
-	struct smp_signctx	ctx[1];
-};
-
-VTAILQ_HEAD(smp_seghead, smp_seg);
-
-struct smp_sc {
-	unsigned		magic;
-#define SMP_SC_MAGIC		0x7b73af0a
-	struct stevedore	*parent;
-
-	unsigned		flags;
-#define SMP_SC_LOADED		(1 << 0)
-
-	const struct stevedore	*stevedore;
-	int			fd;
-	const char		*filename;
-	off_t			mediasize;
-	uintptr_t		align;
-	uint32_t		granularity;
-	uint32_t		unique;
-
-	uint8_t			*base;
-
-	struct smp_ident	*ident;
-
-	struct smp_seghead	segments;
-	struct smp_seg		*cur_seg;
-	uint64_t		next_bot;	/* next alloc address bottom */
-	uint64_t		next_top;	/* next alloc address top */
-
-	uint64_t		free_offset;
-
-	pthread_t		thread;
-
-	VTAILQ_ENTRY(smp_sc)	list;
-
-	struct smp_signctx	idn;
-	struct smp_signctx	ban1;
-	struct smp_signctx	ban2;
-	struct smp_signctx	seg1;
-	struct smp_signctx	seg2;
-
-	struct ban		*tailban;
-
-	struct lock		mtx;
-
-	/* Cleaner metrics */
-
-	unsigned		min_nseg;
-	unsigned		aim_nseg;
-	unsigned		max_nseg;
-
-	uint64_t		min_segl;
-	uint64_t		aim_segl;
-	uint64_t		max_segl;
-
-	uint64_t		free_reserve;
-};
-
-/*--------------------------------------------------------------------*/
-
-/* Pointer round up/down & assert */
-#define PRNDN(sc, x)	((void*)RDN2((uintptr_t)(x), sc->align))
-#define PRNUP(sc, x)	((void*)RUP2((uintptr_t)(x), sc->align))
-#define PASSERTALIGN(sc, x)	assert(PRNDN(sc, x) == (x))
-
-/* Integer round up/down & assert */
-#define IRNDN(sc, x)	RDN2(x, sc->align)
-#define IRNUP(sc, x)	RUP2(x, sc->align)
-#define IASSERTALIGN(sc, x)	assert(IRNDN(sc, x) == (x))
-
-/*--------------------------------------------------------------------*/
-
-#define ASSERT_PTR_IN_SILO(sc, ptr) \
-	assert((const void*)(ptr) >= (const void*)((sc)->base) && \
-	    (const void*)(ptr) < (const void *)((sc)->base + (sc)->mediasize))
-
-/*--------------------------------------------------------------------*/
-
-#define SIGN_DATA(ctx)	((void *)((ctx)->ss + 1))
-#define SIGN_END(ctx)	((void *)((int8_t *)SIGN_DATA(ctx) + (ctx)->ss->length))
-
-/* storage_persistent_mgt.c */
-
-void smp_mgt_init(struct stevedore *parent, int ac, char * const *av);
-
-/* storage_persistent_silo.c */
-
-void smp_load_seg(const struct sess *sp, const struct smp_sc *sc,
-    struct smp_seg *sg);
-void smp_new_seg(struct smp_sc *sc);
-void smp_close_seg(struct smp_sc *sc, struct smp_seg *sg);
-void smp_init_oc(struct objcore *oc, struct smp_seg *sg, unsigned objidx);
-void smp_save_segs(struct smp_sc *sc);
-
-/* storage_persistent_subr.c */
-
-void smp_def_sign(const struct smp_sc *sc, struct smp_signctx *ctx,
-    uint64_t off, const char *id);
-int smp_chk_sign(struct smp_signctx *ctx);
-void smp_append_sign(struct smp_signctx *ctx, const void *ptr, uint32_t len);
-void smp_reset_sign(struct smp_signctx *ctx);
-void smp_sync_sign(const struct smp_signctx *ctx);
-void smp_newsilo(struct smp_sc *sc);
-int smp_valid_silo(struct smp_sc *sc);
-
-/*--------------------------------------------------------------------
- * Caculate payload of some stuff
- */
-
-static inline uint64_t
-smp_stuff_len(const struct smp_sc *sc, unsigned stuff)
-{
-	uint64_t l;
-
-	assert(stuff < SMP_END_STUFF);
-	l = sc->ident->stuff[stuff + 1] - sc->ident->stuff[stuff];
-	l -= SMP_SIGN_SPACE;
-	return (l);
-}
-
-static inline uint64_t
-smp_segend(const struct smp_seg *sg)
-{
-
-	return (sg->p.offset + sg->p.length);
-}
-
-static inline uint64_t
-smp_spaceleft(const struct smp_sc *sc, const struct smp_seg *sg)
-{
-
-	IASSERTALIGN(sc, sc->next_bot);
-	assert(sc->next_bot <= sc->next_top - IRNUP(sc, SMP_SIGN_SPACE));
-	assert(sc->next_bot >= sg->p.offset);
-	assert(sc->next_top < sg->p.offset + sg->p.length);
-	return ((sc->next_top - sc->next_bot) - IRNUP(sc, SMP_SIGN_SPACE));
-}
diff --git a/bin/varnishd/storage_persistent_mgt.c b/bin/varnishd/storage_persistent_mgt.c
deleted file mode 100644
index 2cdcc6b..0000000
--- a/bin/varnishd/storage_persistent_mgt.c
+++ /dev/null
@@ -1,205 +0,0 @@
-/*-
- * Copyright (c) 2008-2011 Varnish Software AS
- * All rights reserved.
- *
- * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- *
- * Persistent storage method
- *
- * XXX: Before we start the client or maybe after it stops, we should give the
- * XXX: stevedores a chance to examine their storage for consistency.
- *
- * XXX: Do we ever free the LRU-lists ?
- */
-
-#include "config.h"
-
-#include <sys/mman.h>
-
-#include <math.h>
-#include <stdio.h>
-#include <stdlib.h>
-
-#include "cache.h"
-#include "storage.h"
-
-#include "vsha256.h"
-
-#include "persistent.h"
-#include "storage_persistent.h"
-
-#ifndef MAP_NOCORE
-#define MAP_NOCORE 0 /* XXX Linux */
-#endif
-
-#ifndef MAP_NOSYNC
-#define MAP_NOSYNC 0 /* XXX Linux */
-#endif
-
-/*--------------------------------------------------------------------
- * Calculate cleaner metrics from silo dimensions
- */
-
-static void
-smp_metrics(struct smp_sc *sc)
-{
-
-	/*
-	 * We do not want to loose too big chunks of the silos
-	 * content when we are forced to clean a segment.
-	 *
-	 * For now insist that a segment covers no more than 1% of the silo.
-	 *
-	 * XXX: This should possibly depend on the size of the silo so
-	 * XXX: trivially small silos do not run into trouble along
-	 * XXX: the lines of "one object per segment".
-	 */
-
-	sc->min_nseg = 10;
-	sc->max_segl = smp_stuff_len(sc, SMP_SPC_STUFF) / sc->min_nseg;
-
-	fprintf(stderr, "min_nseg = %u, max_segl = %ju\n",
-	    sc->min_nseg, (uintmax_t)sc->max_segl);
-
-	/*
-	 * The number of segments are limited by the size of the segment
-	 * table(s) and from that follows the minimum size of a segmement.
-	 */
-
-	sc->max_nseg = smp_stuff_len(sc, SMP_SEG1_STUFF) / sc->min_nseg;
-	sc->min_segl = smp_stuff_len(sc, SMP_SPC_STUFF) / sc->max_nseg;
-
-	while (sc->min_segl < sizeof(struct object)) {
-		sc->max_nseg /= 2;
-		sc->min_segl = smp_stuff_len(sc, SMP_SPC_STUFF) / sc->max_nseg;
-	}
-
-	fprintf(stderr, "max_nseg = %u, min_segl = %ju\n",
-	    sc->max_nseg, (uintmax_t)sc->min_segl);
-
-	/*
-	 * Set our initial aim point at the exponential average of the
-	 * two extremes.
-	 *
-	 * XXX: This is a pretty arbitrary choice, but having no idea
-	 * XXX: object count, size distribution or ttl pattern at this
-	 * XXX: point, we have to do something.
-	 */
-
-	sc->aim_nseg =
-	   (unsigned) exp((log(sc->min_nseg) + log(sc->max_nseg))*.5);
-	sc->aim_segl = smp_stuff_len(sc, SMP_SPC_STUFF) / sc->aim_nseg;
-
-	fprintf(stderr, "aim_nseg = %u, aim_segl = %ju\n",
-	    sc->aim_nseg, (uintmax_t)sc->aim_segl);
-
-	/*
-	 * How much space in the free reserve pool ?
-	 */
-	sc->free_reserve = sc->aim_segl * 10;
-
-	fprintf(stderr, "free_reserve = %ju\n", (uintmax_t)sc->free_reserve);
-}
-
-/*--------------------------------------------------------------------
- * Set up persistent storage silo in the master process.
- */
-
-void
-smp_mgt_init(struct stevedore *parent, int ac, char * const *av)
-{
-	struct smp_sc		*sc;
-	struct smp_sign		sgn;
-	void *target;
-	int i;
-
-	ASSERT_MGT();
-
-	AZ(av[ac]);
-#define SIZOF(foo)       fprintf(stderr, \
-    "sizeof(%s) = %zu = 0x%zx\n", #foo, sizeof(foo), sizeof(foo));
-	SIZOF(struct smp_ident);
-	SIZOF(struct smp_sign);
-	SIZOF(struct smp_segptr);
-	SIZOF(struct smp_object);
-#undef SIZOF
-
-	/* See comments in persistent.h */
-	assert(sizeof(struct smp_ident) == SMP_IDENT_SIZE);
-
-	/* Allocate softc */
-	ALLOC_OBJ(sc, SMP_SC_MAGIC);
-	XXXAN(sc);
-	sc->parent = parent;
-	sc->fd = -1;
-	VTAILQ_INIT(&sc->segments);
-
-	/* Argument processing */
-	if (ac != 2)
-		ARGV_ERR("(-spersistent) wrong number of arguments\n");
-
-	i = STV_GetFile(av[0], &sc->fd, &sc->filename, "-spersistent");
-	if (i == 2)
-		ARGV_ERR("(-spersistent) need filename (not directory)\n");
-
-	sc->align = sizeof(void*) * 2;
-	sc->granularity = getpagesize();
-	sc->mediasize = STV_FileSize(sc->fd, av[1], &sc->granularity,
-	    "-spersistent");
-
-	AZ(ftruncate(sc->fd, sc->mediasize));
-
-	/* Try to determine correct mmap address */
-	i = read(sc->fd, &sgn, sizeof sgn);
-	assert(i == sizeof sgn);
-	if (!strcmp(sgn.ident, "SILO"))
-		target = (void*)(uintptr_t)sgn.mapped;
-	else
-		target = NULL;
-
-	sc->base = mmap(target, sc->mediasize, PROT_READ|PROT_WRITE,
-	    MAP_NOCORE | MAP_NOSYNC | MAP_SHARED, sc->fd, 0);
-
-	if (sc->base == MAP_FAILED)
-		ARGV_ERR("(-spersistent) failed to mmap (%s)\n",
-		    strerror(errno));
-
-	smp_def_sign(sc, &sc->idn, 0, "SILO");
-	sc->ident = SIGN_DATA(&sc->idn);
-
-	i = smp_valid_silo(sc);
-	if (i) {
-		printf("Warning SILO (%s) not reloaded (reason=%d)\n",
-		    sc->filename, i);
-		smp_newsilo(sc);
-	}
-	AZ(smp_valid_silo(sc));
-
-	smp_metrics(sc);
-
-	parent->priv = sc;
-
-	/* XXX: only for sendfile I guess... */
-	mgt_child_inherit(sc->fd, "storage_persistent");
-}
diff --git a/bin/varnishd/storage_persistent_silo.c b/bin/varnishd/storage_persistent_silo.c
deleted file mode 100644
index 8209613..0000000
--- a/bin/varnishd/storage_persistent_silo.c
+++ /dev/null
@@ -1,524 +0,0 @@
-/*-
- * Copyright (c) 2008-2011 Varnish Software AS
- * All rights reserved.
- *
- * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- *
- * Persistent storage method
- *
- * XXX: Before we start the client or maybe after it stops, we should give the
- * XXX: stevedores a chance to examine their storage for consistency.
- *
- */
-
-#include "config.h"
-
-#include <stdio.h>
-#include <stdlib.h>
-
-#include "cache.h"
-#include "storage.h"
-
-#include "hash_slinger.h"
-#include "vsha256.h"
-#include "vtim.h"
-
-#include "persistent.h"
-#include "storage_persistent.h"
-
-/*--------------------------------------------------------------------
- * Write the segmentlist back to the silo.
- *
- * We write the first copy, sync it synchronously, then write the
- * second copy and sync it synchronously.
- *
- * Provided the kernel doesn't lie, that means we will always have
- * at least one valid copy on in the silo.
- */
-
-static void
-smp_save_seg(const struct smp_sc *sc, struct smp_signctx *ctx)
-{
-	struct smp_segptr *ss;
-	struct smp_seg *sg;
-	uint64_t length;
-
-	Lck_AssertHeld(&sc->mtx);
-	smp_reset_sign(ctx);
-	ss = SIGN_DATA(ctx);
-	length = 0;
-	VTAILQ_FOREACH(sg, &sc->segments, list) {
-		assert(sg->p.offset < sc->mediasize);
-		assert(sg->p.offset + sg->p.length <= sc->mediasize);
-		*ss = sg->p;
-		ss++;
-		length += sizeof *ss;
-	}
-	smp_append_sign(ctx, SIGN_DATA(ctx), length);
-	smp_sync_sign(ctx);
-}
-
-void
-smp_save_segs(struct smp_sc *sc)
-{
-	struct smp_seg *sg, *sg2;
-
-	Lck_AssertHeld(&sc->mtx);
-
-	/*
-	 * Remove empty segments from the front of the list
-	 * before we write the segments to disk.
-	 */
-	VTAILQ_FOREACH_SAFE(sg, &sc->segments, list, sg2) {
-		if (sg->nobj > 0)
-			break;
-		if (sg == sc->cur_seg)
-			continue;
-		VTAILQ_REMOVE(&sc->segments, sg, list);
-		LRU_Free(sg->lru);
-		FREE_OBJ(sg);
-	}
-	smp_save_seg(sc, &sc->seg1);
-	smp_save_seg(sc, &sc->seg2);
-}
-
-/*--------------------------------------------------------------------
- * Load segments
- *
- * The overall objective is to register the existence of an object, based
- * only on the minimally sized struct smp_object, without causing the
- * main object to be faulted in.
- *
- * XXX: We can test this by mprotecting the main body of the segment
- * XXX: until the first fixup happens, or even just over this loop,
- * XXX: However: the requires that the smp_objects starter further
- * XXX: into the segment than a page so that they do not get hit
- * XXX: by the protection.
- */
-
-void
-smp_load_seg(const struct sess *sp, const struct smp_sc *sc,
-    struct smp_seg *sg)
-{
-	struct smp_object *so;
-	struct objcore *oc;
-	uint32_t no;
-	double t_now = VTIM_real();
-	struct smp_signctx ctx[1];
-
-	ASSERT_SILO_THREAD(sc);
-	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
-	CHECK_OBJ_NOTNULL(sg, SMP_SEG_MAGIC);
-	CHECK_OBJ_NOTNULL(sg->lru, LRU_MAGIC);
-	assert(sg->flags & SMP_SEG_MUSTLOAD);
-	sg->flags &= ~SMP_SEG_MUSTLOAD;
-	AN(sg->p.offset);
-	if (sg->p.objlist == 0)
-		return;
-	smp_def_sign(sc, ctx, sg->p.offset, "SEGHEAD");
-	if (smp_chk_sign(ctx))
-		return;
-
-	/* test SEGTAIL */
-	/* test OBJIDX */
-	so = (void*)(sc->base + sg->p.objlist);
-	sg->objs = so;
-	no = sg->p.lobjlist;
-	/* Clear the bogus "hold" count */
-	sg->nobj = 0;
-	for (;no > 0; so++,no--) {
-		if (so->ttl == 0 || so->ttl < t_now)
-			continue;
-		HSH_Prealloc(sp);
-		oc = sp->wrk->nobjcore;
-		oc->flags |= OC_F_NEEDFIXUP | OC_F_LRUDONTMOVE;
-		oc->flags &= ~OC_F_BUSY;
-		smp_init_oc(oc, sg, no);
-		oc->ban = BAN_RefBan(oc, so->ban, sc->tailban);
-		memcpy(sp->wrk->nobjhead->digest, so->hash, SHA256_LEN);
-		(void)HSH_Insert(sp);
-		AZ(sp->wrk->nobjcore);
-		EXP_Inject(oc, sg->lru, so->ttl);
-		sg->nobj++;
-	}
-	WRK_SumStat(sp->wrk);
-	sg->flags |= SMP_SEG_LOADED;
-}
-
-/*--------------------------------------------------------------------
- * Create a new segment
- */
-
-void
-smp_new_seg(struct smp_sc *sc)
-{
-	struct smp_seg *sg, *sg2;
-
-	Lck_AssertHeld(&sc->mtx);
-	ALLOC_OBJ(sg, SMP_SEG_MAGIC);
-	AN(sg);
-	sg->sc = sc;
-	sg->lru = LRU_Alloc();
-	CHECK_OBJ_NOTNULL(sg->lru, LRU_MAGIC);
-
-	/* XXX: find where it goes in silo */
-
-	sg->p.offset = sc->free_offset;
-	// XXX: align */
-	assert(sg->p.offset >= sc->ident->stuff[SMP_SPC_STUFF]);
-	assert(sg->p.offset < sc->mediasize);
-
-	sg->p.length = sc->aim_segl;
-	sg->p.length &= ~7;
-
-	if (smp_segend(sg) > sc->mediasize) {
-		sc->free_offset = sc->ident->stuff[SMP_SPC_STUFF];
-		sg->p.offset = sc->free_offset;
-		sg2 = VTAILQ_FIRST(&sc->segments);
-		if (smp_segend(sg) > sg2->p.offset) {
-			printf("Out of space in persistent silo\n");
-			printf("Committing suicide, restart will make space\n");
-			exit (0);
-		}
-	}
-
-
-	assert(smp_segend(sg) <= sc->mediasize);
-
-	sg2 = VTAILQ_FIRST(&sc->segments);
-	if (sg2 != NULL && sg2->p.offset > sc->free_offset) {
-		if (smp_segend(sg) > sg2->p.offset) {
-			printf("Out of space in persistent silo\n");
-			printf("Committing suicide, restart will make space\n");
-			exit (0);
-		}
-		assert(smp_segend(sg) <= sg2->p.offset);
-	}
-
-	sg->p.offset = IRNUP(sc, sg->p.offset);
-	sg->p.length = IRNDN(sc, sg->p.length);
-	sc->free_offset = sg->p.offset + sg->p.length;
-
-	VTAILQ_INSERT_TAIL(&sc->segments, sg, list);
-
-	/* Neuter the new segment in case there is an old one there */
-	AN(sg->p.offset);
-	smp_def_sign(sc, sg->ctx, sg->p.offset, "SEGHEAD");
-	smp_reset_sign(sg->ctx);
-	smp_sync_sign(sg->ctx);
-
-	/* Set up our allocation points */
-	sc->cur_seg = sg;
-	sc->next_bot = sg->p.offset + IRNUP(sc, SMP_SIGN_SPACE);
-	sc->next_top = smp_segend(sg);
-	sc->next_top -= IRNUP(sc, SMP_SIGN_SPACE);
-	IASSERTALIGN(sc, sc->next_bot);
-	IASSERTALIGN(sc, sc->next_top);
-	sg->objs = (void*)(sc->base + sc->next_top);
-}
-
-/*--------------------------------------------------------------------
- * Close a segment
- */
-
-void
-smp_close_seg(struct smp_sc *sc, struct smp_seg *sg)
-{
-	uint64_t left, dst, len;
-	void *dp;
-
-	Lck_AssertHeld(&sc->mtx);
-
-	assert(sg == sc->cur_seg);
-	AN(sg->p.offset);
-	sc->cur_seg = NULL;
-
-	if (sg->nalloc == 0) {
-		/* XXX: if segment is empty, delete instead */
-		VTAILQ_REMOVE(&sc->segments, sg, list);
-		free(sg);
-		return;
-	}
-
-	/*
-	 * If there is enough space left, that we can move the smp_objects
-	 * down without overwriting the present copy, we will do so to
-	 * compact the segment.
-	 */
-	left = smp_spaceleft(sc, sg);
-	len = sizeof(struct smp_object) * sg->p.lobjlist;
-	if (len < left) {
-		dst = sc->next_bot + IRNUP(sc, SMP_SIGN_SPACE);
-		dp = sc->base + dst;
-		assert((uintptr_t)dp + len < (uintptr_t)sg->objs);
-		memcpy(dp, sg->objs, len);
-		sc->next_top = dst;
-		sg->objs = dp;
-		sg->p.length = (sc->next_top - sg->p.offset)
-		     + len + IRNUP(sc, SMP_SIGN_SPACE);
-		(void)smp_spaceleft(sc, sg);	/* for the asserts */
-
-	}
-
-	/* Update the segment header */
-	sg->p.objlist = sc->next_top;
-
-	/* Write the (empty) OBJIDX signature */
-	sc->next_top -= IRNUP(sc, SMP_SIGN_SPACE);
-	assert(sc->next_top >= sc->next_bot);
-	smp_def_sign(sc, sg->ctx, sc->next_top, "OBJIDX");
-	smp_reset_sign(sg->ctx);
-	smp_sync_sign(sg->ctx);
-
-	/* Write the (empty) SEGTAIL signature */
-	smp_def_sign(sc, sg->ctx,
-	    sg->p.offset + sg->p.length - IRNUP(sc, SMP_SIGN_SPACE), "SEGTAIL");
-	smp_reset_sign(sg->ctx);
-	smp_sync_sign(sg->ctx);
-
-	/* Save segment list */
-	smp_save_segs(sc);
-	sc->free_offset = smp_segend(sg);
-}
-
-
-/*---------------------------------------------------------------------
- */
-
-static struct smp_object *
-smp_find_so(const struct smp_seg *sg, unsigned priv2)
-{
-	struct smp_object *so;
-
-	assert(priv2 > 0);
-	assert(priv2 <= sg->p.lobjlist);
-	so = &sg->objs[sg->p.lobjlist - priv2];
-	return (so);
-}
-
-/*---------------------------------------------------------------------
- * Check if a given storage structure is valid to use
- */
-
-static int
-smp_loaded_st(const struct smp_sc *sc, const struct smp_seg *sg,
-    const struct storage *st)
-{
-	struct smp_seg *sg2;
-	const uint8_t *pst;
-	uint64_t o;
-
-	(void)sg;		/* XXX: faster: Start search from here */
-	pst = (const void *)st;
-
-	if (pst < (sc->base + sc->ident->stuff[SMP_SPC_STUFF]))
-		return (0x01);		/* Before silo payload start */
-	if (pst > (sc->base + sc->ident->stuff[SMP_END_STUFF]))
-		return (0x02);		/* After silo end */
-
-	o = pst - sc->base;
-
-	/* Find which segment contains the storage structure */
-	VTAILQ_FOREACH(sg2, &sc->segments, list)
-		if (o > sg2->p.offset && (o + sizeof(*st)) < sg2->p.objlist)
-			break;
-	if (sg2 == NULL)
-		return (0x04);		/* No claiming segment */
-	if (!(sg2->flags & SMP_SEG_LOADED))
-		return (0x08);		/* Claiming segment not loaded */
-
-	/* It is now safe to access the storage structure */
-	if (st->magic != STORAGE_MAGIC)
-		return (0x10);		/* Not enough magic */
-
-	if (o + st->space >= sg2->p.objlist)
-		return (0x20);		/* Allocation not inside segment */
-
-	if (st->len > st->space)
-		return (0x40);		/* Plain bad... */
-
-	/*
-	 * XXX: We could patch up st->stevedore and st->priv here
-	 * XXX: but if things go right, we will never need them.
-	 */
-	return (0);
-}
-
-/*---------------------------------------------------------------------
- * objcore methods for persistent objects
- */
-
-static struct object *
-smp_oc_getobj(struct worker *wrk, struct objcore *oc)
-{
-	struct object *o;
-	struct smp_seg *sg;
-	struct smp_object *so;
-	struct storage *st;
-	uint64_t l;
-	int bad;
-
-	/* Some calls are direct, but they should match anyway */
-	assert(oc->methods->getobj == smp_oc_getobj);
-
-	CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
-	if (wrk == NULL)
-		AZ(oc->flags & OC_F_NEEDFIXUP);
-
-	CAST_OBJ_NOTNULL(sg, oc->priv, SMP_SEG_MAGIC);
-	so = smp_find_so(sg, oc->priv2);
-
-	o = (void*)(sg->sc->base + so->ptr);
-	/*
-	 * The object may not be in this segment since we allocate it
-	 * In a separate operation than the smp_object.  We could check
-	 * that it is in a later segment, but that would be complicated.
-	 * XXX: For now, be happy if it is inside th silo
-	 */
-	ASSERT_PTR_IN_SILO(sg->sc, o);
-	CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC);
-
-	/*
-	 * If this flag is not set, it will not be, and the lock is not
-	 * needed to test it.
-	 */
-	if (!(oc->flags & OC_F_NEEDFIXUP))
-		return (o);
-
-	AN(wrk);
-	Lck_Lock(&sg->sc->mtx);
-	/* Check again, we might have raced. */
-	if (oc->flags & OC_F_NEEDFIXUP) {
-		/* We trust caller to have a refcnt for us */
-		o->objcore = oc;
-
-		bad = 0;
-		l = 0;
-		VTAILQ_FOREACH(st, &o->store, list) {
-			bad |= smp_loaded_st(sg->sc, sg, st);
-			if (bad)
-				break;
-			l += st->len;
-		}
-		if (l != o->len)
-			bad |= 0x100;
-
-		if(bad) {
-			EXP_Set_ttl(&o->exp, -1);
-			so->ttl = 0;
-		}
-
-		sg->nfixed++;
-		wrk->stats.n_object++;
-		wrk->stats.n_vampireobject--;
-		oc->flags &= ~OC_F_NEEDFIXUP;
-	}
-	Lck_Unlock(&sg->sc->mtx);
-	EXP_Rearm(o);
-	return (o);
-}
-
-static void
-smp_oc_updatemeta(struct objcore *oc)
-{
-	struct object *o;
-	struct smp_seg *sg;
-	struct smp_object *so;
-	double mttl;
-
-	CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
-	o = smp_oc_getobj(NULL, oc);
-	AN(o);
-
-	CAST_OBJ_NOTNULL(sg, oc->priv, SMP_SEG_MAGIC);
-	CHECK_OBJ_NOTNULL(sg->sc, SMP_SC_MAGIC);
-	so = smp_find_so(sg, oc->priv2);
-
-	mttl = EXP_Grace(NULL, o);
-
-	if (sg == sg->sc->cur_seg) {
-		/* Lock necessary, we might race close_seg */
-		Lck_Lock(&sg->sc->mtx);
-		so->ban = BAN_Time(oc->ban);
-		so->ttl = mttl;
-		Lck_Unlock(&sg->sc->mtx);
-	} else {
-		so->ban = BAN_Time(oc->ban);
-		so->ttl = mttl;
-	}
-}
-
-static void __match_proto__()
-smp_oc_freeobj(struct objcore *oc)
-{
-	struct smp_seg *sg;
-	struct smp_object *so;
-
-	CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
-
-	CAST_OBJ_NOTNULL(sg, oc->priv, SMP_SEG_MAGIC);
-	so = smp_find_so(sg, oc->priv2);
-
-	Lck_Lock(&sg->sc->mtx);
-	so->ttl = 0;
-	so->ptr = 0;
-
-	assert(sg->nobj > 0);
-	assert(sg->nfixed > 0);
-	sg->nobj--;
-	sg->nfixed--;
-
-	Lck_Unlock(&sg->sc->mtx);
-}
-
-/*--------------------------------------------------------------------
- * Find the per-segment lru list for this object
- */
-
-static struct lru *
-smp_oc_getlru(const struct objcore *oc)
-{
-	struct smp_seg *sg;
-
-	CAST_OBJ_NOTNULL(sg, oc->priv, SMP_SEG_MAGIC);
-	return (sg->lru);
-}
-
-static struct objcore_methods smp_oc_methods = {
-	.getobj =		smp_oc_getobj,
-	.updatemeta =		smp_oc_updatemeta,
-	.freeobj =		smp_oc_freeobj,
-	.getlru =		smp_oc_getlru,
-};
-
-/*--------------------------------------------------------------------*/
-
-void
-smp_init_oc(struct objcore *oc, struct smp_seg *sg, unsigned objidx)
-{
-
-	oc->priv = sg;
-	oc->priv2 = objidx;
-	oc->methods = &smp_oc_methods;
-}
diff --git a/bin/varnishd/storage_persistent_subr.c b/bin/varnishd/storage_persistent_subr.c
deleted file mode 100644
index b4bbb3f..0000000
--- a/bin/varnishd/storage_persistent_subr.c
+++ /dev/null
@@ -1,302 +0,0 @@
-/*-
- * Copyright (c) 2008-2011 Varnish Software AS
- * All rights reserved.
- *
- * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- *
- * Persistent storage method
- *
- * XXX: Before we start the client or maybe after it stops, we should give the
- * XXX: stevedores a chance to examine their storage for consistency.
- *
- * XXX: Do we ever free the LRU-lists ?
- */
-
-#include "config.h"
-
-#include <sys/mman.h>
-
-#include <stddef.h>
-#include <stdio.h>
-#include <stdlib.h>
-
-#include "cache.h"
-#include "storage.h"
-
-#include "vsha256.h"
-
-#include "persistent.h"
-#include "storage_persistent.h"
-
-/*--------------------------------------------------------------------
- * SIGNATURE functions
- * The signature is SHA256 over:
- *    1. The smp_sign struct up to but not including the length field.
- *    2. smp_sign->length bytes, starting after the smp_sign structure
- *    3. The smp-sign->length field.
- * The signature is stored after the byte-range from step 2.
- */
-
-/*--------------------------------------------------------------------
- * Define a signature by location and identifier.
- */
-
-void
-smp_def_sign(const struct smp_sc *sc, struct smp_signctx *ctx,
-    uint64_t off, const char *id)
-{
-
-	AZ(off & 7);			/* Alignment */
-	assert(strlen(id) < sizeof ctx->ss->ident);
-
-	memset(ctx, 0, sizeof *ctx);
-	ctx->ss = (void*)(sc->base + off);
-	ctx->unique = sc->unique;
-	ctx->id = id;
-}
-
-/*--------------------------------------------------------------------
- * Check that a signature is good, leave state ready for append
- */
-int
-smp_chk_sign(struct smp_signctx *ctx)
-{
-	struct SHA256Context cx;
-	unsigned char sign[SHA256_LEN];
-	int r = 0;
-
-	if (strncmp(ctx->id, ctx->ss->ident, sizeof ctx->ss->ident))
-		r = 1;
-	else if (ctx->unique != ctx->ss->unique)
-		r = 2;
-	else if ((uintptr_t)ctx->ss != ctx->ss->mapped)
-		r = 3;
-	else {
-		SHA256_Init(&ctx->ctx);
-		SHA256_Update(&ctx->ctx, ctx->ss,
-		    offsetof(struct smp_sign, length));
-		SHA256_Update(&ctx->ctx, SIGN_DATA(ctx), ctx->ss->length);
-		cx = ctx->ctx;
-		SHA256_Update(&cx, &ctx->ss->length, sizeof(ctx->ss->length));
-		SHA256_Final(sign, &cx);
-		if (memcmp(sign, SIGN_END(ctx), sizeof sign))
-			r = 4;
-	}
-	if (r) {
-		fprintf(stderr, "CHK(%p %s %p %s) = %d\n",
-		    ctx, ctx->id, ctx->ss,
-		    r > 1 ? ctx->ss->ident : "<invalid>", r);
-	}
-	return (r);
-}
-
-/*--------------------------------------------------------------------
- * Append data to a signature
- */
-void
-smp_append_sign(struct smp_signctx *ctx, const void *ptr, uint32_t len)
-{
-	struct SHA256Context cx;
-	unsigned char sign[SHA256_LEN];
-
-	if (len != 0) {
-		SHA256_Update(&ctx->ctx, ptr, len);
-		ctx->ss->length += len;
-	}
-	cx = ctx->ctx;
-	SHA256_Update(&cx, &ctx->ss->length, sizeof(ctx->ss->length));
-	SHA256_Final(sign, &cx);
-	memcpy(SIGN_END(ctx), sign, sizeof sign);
-XXXAZ(smp_chk_sign(ctx));
-}
-
-/*--------------------------------------------------------------------
- * Reset a signature to empty, prepare for appending.
- */
-
-void
-smp_reset_sign(struct smp_signctx *ctx)
-{
-
-	memset(ctx->ss, 0, sizeof *ctx->ss);
-	strcpy(ctx->ss->ident, ctx->id);
-	ctx->ss->unique = ctx->unique;
-	ctx->ss->mapped = (uintptr_t)ctx->ss;
-	SHA256_Init(&ctx->ctx);
-	SHA256_Update(&ctx->ctx, ctx->ss,
-	    offsetof(struct smp_sign, length));
-	smp_append_sign(ctx, NULL, 0);
-}
-
-/*--------------------------------------------------------------------
- * Force a write of a signature block to the backing store.
- */
-
-void
-smp_sync_sign(const struct smp_signctx *ctx)
-{
-	int i;
-
-	/* XXX: round to pages */
-	i = msync((void*)ctx->ss, ctx->ss->length + SHA256_LEN, MS_SYNC);
-	if (i && 0)
-		fprintf(stderr, "SyncSign(%p %s) = %d %s\n",
-		    ctx->ss, ctx->id, i, strerror(errno));
-}
-
-/*--------------------------------------------------------------------
- * Create and force a new signature to backing store
- */
-
-static void
-smp_new_sign(const struct smp_sc *sc, struct smp_signctx *ctx,
-    uint64_t off, const char *id)
-{
-	smp_def_sign(sc, ctx, off, id);
-	smp_reset_sign(ctx);
-	smp_sync_sign(ctx);
-}
-
-/*--------------------------------------------------------------------
- * Initialize a Silo with a valid but empty structure.
- *
- * XXX: more intelligent sizing of things.
- */
-
-void
-smp_newsilo(struct smp_sc *sc)
-{
-	struct smp_ident	*si;
-
-	ASSERT_MGT();
-	assert(strlen(SMP_IDENT_STRING) < sizeof si->ident);
-
-	/* Choose a new random number */
-	sc->unique = random();
-
-	smp_reset_sign(&sc->idn);
-	si = sc->ident;
-
-	memset(si, 0, sizeof *si);
-	strcpy(si->ident, SMP_IDENT_STRING);
-	si->byte_order = 0x12345678;
-	si->size = sizeof *si;
-	si->major_version = 2;
-	si->unique = sc->unique;
-	si->mediasize = sc->mediasize;
-	si->granularity = sc->granularity;
-	/*
-	 * Aim for cache-line-width
-	 */
-	si->align = sizeof(void*) * 2;
-	sc->align = si->align;
-
-	si->stuff[SMP_BAN1_STUFF] = sc->granularity;
-	si->stuff[SMP_BAN2_STUFF] = si->stuff[SMP_BAN1_STUFF] + 1024*1024;
-	si->stuff[SMP_SEG1_STUFF] = si->stuff[SMP_BAN2_STUFF] + 1024*1024;
-	si->stuff[SMP_SEG2_STUFF] = si->stuff[SMP_SEG1_STUFF] + 1024*1024;
-	si->stuff[SMP_SPC_STUFF] = si->stuff[SMP_SEG2_STUFF] + 1024*1024;
-	si->stuff[SMP_END_STUFF] = si->mediasize;
-	assert(si->stuff[SMP_SPC_STUFF] < si->stuff[SMP_END_STUFF]);
-
-	smp_new_sign(sc, &sc->ban1, si->stuff[SMP_BAN1_STUFF], "BAN 1");
-	smp_new_sign(sc, &sc->ban2, si->stuff[SMP_BAN2_STUFF], "BAN 2");
-	smp_new_sign(sc, &sc->seg1, si->stuff[SMP_SEG1_STUFF], "SEG 1");
-	smp_new_sign(sc, &sc->seg2, si->stuff[SMP_SEG2_STUFF], "SEG 2");
-
-	smp_append_sign(&sc->idn, si, sizeof *si);
-	smp_sync_sign(&sc->idn);
-}
-
-/*--------------------------------------------------------------------
- * Check if a silo is valid.
- */
-
-int
-smp_valid_silo(struct smp_sc *sc)
-{
-	struct smp_ident	*si;
-	int i, j;
-
-	assert(strlen(SMP_IDENT_STRING) < sizeof si->ident);
-
-	i = smp_chk_sign(&sc->idn);
-	if (i)
-		return (i);
-
-	si = sc->ident;
-	if (strcmp(si->ident, SMP_IDENT_STRING))
-		return (12);
-	if (si->byte_order != 0x12345678)
-		return (13);
-	if (si->size != sizeof *si)
-		return (14);
-	if (si->major_version != 2)
-		return (15);
-	if (si->mediasize != sc->mediasize)
-		return (17);
-	if (si->granularity != sc->granularity)
-		return (18);
-	if (si->align < sizeof(void*))
-		return (19);
-	if (!PWR2(si->align))
-		return (20);
-	sc->align = si->align;
-	sc->unique = si->unique;
-
-	/* XXX: Sanity check stuff[6] */
-
-	assert(si->stuff[SMP_BAN1_STUFF] > sizeof *si + SHA256_LEN);
-	assert(si->stuff[SMP_BAN2_STUFF] > si->stuff[SMP_BAN1_STUFF]);
-	assert(si->stuff[SMP_SEG1_STUFF] > si->stuff[SMP_BAN2_STUFF]);
-	assert(si->stuff[SMP_SEG2_STUFF] > si->stuff[SMP_SEG1_STUFF]);
-	assert(si->stuff[SMP_SPC_STUFF] > si->stuff[SMP_SEG2_STUFF]);
-	assert(si->stuff[SMP_END_STUFF] == sc->mediasize);
-
-	assert(smp_stuff_len(sc, SMP_SEG1_STUFF) > 65536);
-	assert(smp_stuff_len(sc, SMP_SEG1_STUFF) ==
-	  smp_stuff_len(sc, SMP_SEG2_STUFF));
-
-	assert(smp_stuff_len(sc, SMP_BAN1_STUFF) > 65536);
-	assert(smp_stuff_len(sc, SMP_BAN1_STUFF) ==
-	  smp_stuff_len(sc, SMP_BAN2_STUFF));
-
-	smp_def_sign(sc, &sc->ban1, si->stuff[SMP_BAN1_STUFF], "BAN 1");
-	smp_def_sign(sc, &sc->ban2, si->stuff[SMP_BAN2_STUFF], "BAN 2");
-	smp_def_sign(sc, &sc->seg1, si->stuff[SMP_SEG1_STUFF], "SEG 1");
-	smp_def_sign(sc, &sc->seg2, si->stuff[SMP_SEG2_STUFF], "SEG 2");
-
-	/* We must have one valid BAN table */
-	i = smp_chk_sign(&sc->ban1);
-	j = smp_chk_sign(&sc->ban2);
-	if (i && j)
-		return (100 + i * 10 + j);
-
-	/* We must have one valid SEG table */
-	i = smp_chk_sign(&sc->seg1);
-	j = smp_chk_sign(&sc->seg2);
-	if (i && j)
-		return (200 + i * 10 + j);
-	return (0);
-}
diff --git a/bin/varnishd/storage_synth.c b/bin/varnishd/storage_synth.c
deleted file mode 100644
index 5df2c08..0000000
--- a/bin/varnishd/storage_synth.c
+++ /dev/null
@@ -1,120 +0,0 @@
-/*-
- * Copyright (c) 2008-2011 Varnish Software AS
- * All rights reserved.
- *
- * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- *
- * Storage method for synthetic content, based on vsb.
- */
-
-#include "config.h"
-
-#include <stdlib.h>
-
-#include "cache.h"
-#include "storage.h"
-
-
-static struct lock		sms_mtx;
-
-static void
-sms_free(struct storage *sto)
-{
-
-	CHECK_OBJ_NOTNULL(sto, STORAGE_MAGIC);
-	Lck_Lock(&sms_mtx);
-	VSC_C_main->sms_nobj--;
-	VSC_C_main->sms_nbytes -= sto->len;
-	VSC_C_main->sms_bfree += sto->len;
-	Lck_Unlock(&sms_mtx);
-	VSB_delete(sto->priv);
-	free(sto);
-}
-
-void
-SMS_Init(void)
-{
-
-	Lck_New(&sms_mtx, lck_sms);
-}
-
-static struct stevedore sms_stevedore = {
-	.magic	=	STEVEDORE_MAGIC,
-	.name	=	"synth",
-	.free	=	sms_free,
-};
-
-struct vsb *
-SMS_Makesynth(struct object *obj)
-{
-	struct storage *sto;
-	struct vsb *vsb;
-
-	CHECK_OBJ_NOTNULL(obj, OBJECT_MAGIC);
-	STV_Freestore(obj);
-	obj->len = 0;
-
-	Lck_Lock(&sms_mtx);
-	VSC_C_main->sms_nreq++;
-	VSC_C_main->sms_nobj++;
-	Lck_Unlock(&sms_mtx);
-
-	sto = calloc(sizeof *sto, 1);
-	XXXAN(sto);
-	vsb = VSB_new_auto();
-	XXXAN(vsb);
-	sto->priv = vsb;
-	sto->len = 0;
-	sto->space = 0;
-#ifdef SENDFILE_WORKS
-	sto->fd = -1;
-#endif
-	sto->stevedore = &sms_stevedore;
-	sto->magic = STORAGE_MAGIC;
-
-	VTAILQ_INSERT_TAIL(&obj->store, sto, list);
-	return (vsb);
-}
-
-void
-SMS_Finish(struct object *obj)
-{
-	struct storage *sto;
-	struct vsb *vsb;
-
-	CHECK_OBJ_NOTNULL(obj, OBJECT_MAGIC);
-	sto = VTAILQ_FIRST(&obj->store);
-	assert(sto->stevedore == &sms_stevedore);
-	vsb = sto->priv;
-	AZ(VSB_finish(vsb));
-
-	sto->ptr = (void*)VSB_data(vsb);
-	sto->len = VSB_len(vsb);
-	sto->space = VSB_len(vsb);
-	obj->len = sto->len;
-	Lck_Lock(&sms_mtx);
-	VSC_C_main->sms_nbytes += sto->len;
-	VSC_C_main->sms_balloc += sto->len;
-	Lck_Unlock(&sms_mtx);
-}
diff --git a/bin/varnishd/storage_umem.c b/bin/varnishd/storage_umem.c
deleted file mode 100644
index 9198a99..0000000
--- a/bin/varnishd/storage_umem.c
+++ /dev/null
@@ -1,166 +0,0 @@
-/*-
- * Copyright (c) 2006 Verdens Gang AS
- * Copyright (c) 2006-2011 Varnish Software AS
- * All rights reserved.
- *
- * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- *
- * Storage method based on umem_alloc(3MALLOC)
- */
-
-#include "config.h"
-
-#ifdef HAVE_LIBUMEM
-
-#include <sys/types.h>
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <umem.h>
-
-#include "cache.h"
-#include "storage.h"
-
-static size_t			smu_max = SIZE_MAX;
-static MTX			smu_mtx;
-
-struct smu {
-	struct storage		s;
-	size_t			sz;
-};
-
-static struct storage *
-smu_alloc(struct stevedore *st, size_t size)
-{
-	struct smu *smu;
-
-	Lck_Lock(&smu_mtx);
-	VSC_C_main->sma_nreq++;
-	if (VSC_C_main->sma_nbytes + size > smu_max)
-		size = 0;
-	else {
-		VSC_C_main->sma_nobj++;
-		VSC_C_main->sma_nbytes += size;
-		VSC_C_main->sma_balloc += size;
-	}
-	Lck_Unlock(&smu_mtx);
-
-	if (size == 0)
-		return (NULL);
-
-	smu = umem_zalloc(sizeof *smu, UMEM_DEFAULT);
-	if (smu == NULL)
-		return (NULL);
-	smu->sz = size;
-	smu->s.priv = smu;
-	smu->s.ptr = umem_alloc(size, UMEM_DEFAULT);
-	XXXAN(smu->s.ptr);
-	smu->s.len = 0;
-	smu->s.space = size;
-	smu->s.fd = -1;
-	smu->s.stevedore = st;
-	smu->s.magic = STORAGE_MAGIC;
-	return (&smu->s);
-}
-
-static void
-smu_free(struct storage *s)
-{
-	struct smu *smu;
-
-	CHECK_OBJ_NOTNULL(s, STORAGE_MAGIC);
-	smu = s->priv;
-	assert(smu->sz == smu->s.space);
-	Lck_Lock(&smu_mtx);
-	VSC_C_main->sma_nobj--;
-	VSC_C_main->sma_nbytes -= smu->sz;
-	VSC_C_main->sma_bfree += smu->sz;
-	Lck_Unlock(&smu_mtx);
-	umem_free(smu->s.ptr, smu->s.space);
-	umem_free(smu, sizeof *smu);
-}
-
-static void
-smu_trim(const struct storage *s, size_t size)
-{
-	struct smu *smu;
-	void *p;
-
-	CHECK_OBJ_NOTNULL(s, STORAGE_MAGIC);
-	smu = s->priv;
-	assert(smu->sz == smu->s.space);
-	if ((p = umem_alloc(size, UMEM_DEFAULT)) != NULL) {
-		memcpy(p, smu->s.ptr, size);
-		umem_free(smu->s.ptr, smu->s.space);
-		Lck_Lock(&smu_mtx);
-		VSC_C_main->sma_nbytes -= (smu->sz - size);
-		VSC_C_main->sma_bfree += smu->sz - size;
-		smu->sz = size;
-		Lck_Unlock(&smu_mtx);
-		smu->s.ptr = p;
-		smu->s.space = size;
-	}
-}
-
-static void
-smu_init(struct stevedore *parent, int ac, char * const *av)
-{
-	const char *e;
-	uintmax_t u;
-
-	(void)parent;
-
-	AZ(av[ac]);
-	if (ac > 1)
-		ARGV_ERR("(-sumem) too many arguments\n");
-
-	if (ac == 0 || *av[0] == '\0')
-		 return;
-
-	e = VNUM_2bytes(av[0], &u, 0);
-	if (e != NULL)
-		ARGV_ERR("(-sumem) size \"%s\": %s\n", av[0], e);
-	if ((u != (uintmax_t)(size_t)u))
-		ARGV_ERR("(-sumem) size \"%s\": too big\n", av[0]);
-	smu_max = u;
-}
-
-static void
-smu_open(const struct stevedore *st)
-{
-	(void)st;
-	AZ(pthread_mutex_init(&smu_mtx, NULL));
-}
-
-const struct stevedore smu_stevedore = {
-	.magic	=	STEVEDORE_MAGIC,
-	.name	=	"umem",
-	.init	=	smu_init,
-	.open	=	smu_open,
-	.alloc	=	smu_alloc,
-	.free	=	smu_free,
-	.trim	=	smu_trim,
-};
-
-#endif /* HAVE_UMEM_H */



From geoff at varnish-cache.org  Mon Jan  9 20:52:18 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:18 +0100
Subject: [experimental-ims] 4620b27 Grammar
Message-ID: <E1RkMCY-0007hA-QG@project.varnish-software.com>

commit 4620b2702c1ee13d51c1a9ec77465426388c7aab
Author: Tollef Fog Heen <tfheen at varnish-software.com>
Date:   Fri Oct 7 08:59:08 2011 +0200

    Grammar

diff --git a/doc/sphinx/index.rst b/doc/sphinx/index.rst
index 019fd3b..2558413 100644
--- a/doc/sphinx/index.rst
+++ b/doc/sphinx/index.rst
@@ -5,8 +5,8 @@ Welcome to Varnish's documentation!
 ===================================
 
 Varnish is a state of the art web accelerator. Its mission is to sit
-in front of a web server and to cache the content. It makes your web site
-go fast.
+in front of a web server and cache content. It makes your web site
+go faster.
 
 We suggest you start by reading the installation guide
 :ref:`install-index`. Once you have Varnish up and running go through



From geoff at varnish-cache.org  Mon Jan  9 20:52:18 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:18 +0100
Subject: [experimental-ims] 91734c4 WRK_Flush was renamed to WRW_Flush a long
	time ago; update some ifdefed code and comments to match
Message-ID: <E1RkMCY-0007hJ-Ru@project.varnish-software.com>

commit 91734c47393d6bba8cca4eed265ee78a1b5f92c8
Author: Tollef Fog Heen <tfheen at varnish-software.com>
Date:   Mon Oct 10 08:14:31 2011 +0200

    WRK_Flush was renamed to WRW_Flush a long time ago; update some ifdefed code and comments to match

diff --git a/bin/varnishd/cache.h b/bin/varnishd/cache.h
index 33ccc8b..22e61ae 100644
--- a/bin/varnishd/cache.h
+++ b/bin/varnishd/cache.h
@@ -255,7 +255,7 @@ struct exp {
 
 struct wrw {
 	int			*wfd;
-	unsigned		werr;	/* valid after WRK_Flush() */
+	unsigned		werr;	/* valid after WRW_Flush() */
 	struct iovec		*iov;
 	unsigned		siov;
 	unsigned		niov;
diff --git a/bin/varnishd/cache_wrw.c b/bin/varnishd/cache_wrw.c
index 4fef1aa..0790ab2 100644
--- a/bin/varnishd/cache_wrw.c
+++ b/bin/varnishd/cache_wrw.c
@@ -264,7 +264,7 @@ WRW_Sendfile(struct worker *w, int fd, off_t off, unsigned len)
 	} while (0);
 #elif defined(__linux__)
 	do {
-		if (WRK_Flush(w) == 0 &&
+		if (WRW_Flush(w) == 0 &&
 		    sendfile(*wrw->wfd, fd, &off, len) != len)
 			wrw->werr++;
 	} while (0);
@@ -293,7 +293,7 @@ WRW_Sendfile(struct worker *w, int fd, off_t off, unsigned len)
 	} while (0);
 #elif defined(__sun) && defined(HAVE_SENDFILE)
 	do {
-		if (WRK_Flush(w) == 0 &&
+		if (WRW_Flush(w) == 0 &&
 		    sendfile(*wrw->wfd, fd, &off, len) != len)
 			wrw->werr++;
 	} while (0);



From geoff at varnish-cache.org  Mon Jan  9 20:52:18 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:18 +0100
Subject: [experimental-ims] 28e866b Update docs for 3.0 Fixes #1032
Message-ID: <E1RkMCY-0007hi-VW@project.varnish-software.com>

commit 28e866b59ba0e9c95bdf314965d5895698616136
Author: Andreas Plesner Jacobsen <apj at mutt.dk>
Date:   Thu Oct 13 09:30:27 2011 +0200

    Update docs for 3.0
    Fixes #1032

diff --git a/doc/sphinx/tutorial/esi.rst b/doc/sphinx/tutorial/esi.rst
index ef79f3e..1d01cb8 100644
--- a/doc/sphinx/tutorial/esi.rst
+++ b/doc/sphinx/tutorial/esi.rst
@@ -48,9 +48,9 @@ For ESI to work you need to activate ESI processing in VCL, like this:::
     sub vcl_fetch {
     	if (req.url == "/test.html") {
            set beresp.do_esi = true; /* Do ESI processing		*/
-           set obj.ttl = 24 h; 	     /* Sets the TTL on the HTML above  */
+           set beresp.ttl = 24 h;    /* Sets the TTL on the HTML above  */
     	} elseif (req.url == "/cgi-bin/date.cgi") {
-           set obj.ttl = 1m;         /* Sets a one minute TTL on	*/
+           set beresp.ttl = 1m;      /* Sets a one minute TTL on	*/
 	       	       	 	     /*  the included object		*/
         }
     }



From geoff at varnish-cache.org  Mon Jan  9 20:52:19 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:19 +0100
Subject: [experimental-ims] 69e59af Merge branch 'master' of
	ssh://git.varnish-cache.org/git/varnish-cache
Message-ID: <E1RkMCZ-0007i4-35@project.varnish-software.com>

commit 69e59af21e03867ca4c92aa8a29bfb41383401b4
Merge: a641a46 28e866b
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu Oct 13 09:56:51 2011 +0000

    Merge branch 'master' of ssh://git.varnish-cache.org/git/varnish-cache



From geoff at varnish-cache.org  Mon Jan  9 20:52:19 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:19 +0100
Subject: [experimental-ims] e600a75 Work around a Solaris bug.
Message-ID: <E1RkMCZ-0007iB-4G@project.varnish-software.com>

commit e600a75ed5d35db87b36bf64d45940e50c0d25c7
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu Oct 13 09:59:03 2011 +0000

    Work around a Solaris bug.
    
    Submitted by:	Nils Goroll

diff --git a/lib/libvarnish/vtcp.c b/lib/libvarnish/vtcp.c
index 10c3664..c0f7a95 100644
--- a/lib/libvarnish/vtcp.c
+++ b/lib/libvarnish/vtcp.c
@@ -265,7 +265,13 @@ VTCP_set_read_timeout(int s, double seconds)
 	timeout.tv_sec = (int)floor(seconds);
 	timeout.tv_usec = (int)(1e6 * (seconds - timeout.tv_sec));
 #ifdef SO_RCVTIMEO_WORKS
-	AZ(setsockopt(s, SOL_SOCKET, SO_RCVTIMEO, &timeout, sizeof timeout));
+	/* 
+	 * Solaris bug (present at least in snv_151 and older): If this fails
+	 * with EINVAL, the socket is half-closed (SS_CANTSENDMORE) and the
+	 * timeout does not get set. Needs to be fixed in Solaris, there is
+	 * nothing we can do about this.
+	 */
+	VTCP_Assert(setsockopt(s, SOL_SOCKET, SO_RCVTIMEO, &timeout, sizeof timeout));
 #else
 	(void)s;
 #endif



From geoff at varnish-cache.org  Mon Jan  9 20:52:19 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:19 +0100
Subject: [experimental-ims] 9988408 Adjustments to solaris sandbox.
Message-ID: <E1RkMCZ-0007iJ-5d@project.varnish-software.com>

commit 99884082bdc3c50e586b2826b9ac7d4e46eb8dcf
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu Oct 13 10:00:56 2011 +0000

    Adjustments to solaris sandbox.
    
    Submitted by:	Nils Goroll

diff --git a/bin/varnishd/mgt/mgt_sandbox_solaris.c b/bin/varnishd/mgt/mgt_sandbox_solaris.c
index 715408e..f243938 100644
--- a/bin/varnishd/mgt/mgt_sandbox_solaris.c
+++ b/bin/varnishd/mgt/mgt_sandbox_solaris.c
@@ -220,10 +220,10 @@ mgt_sandbox_solaris_fini(void)
 		    "Child start warning: Waiving privileges failed on %s: errno=%d (%s)", \
 		    #which, errno, strerror(errno));
 
-	SETPPRIV(PRIV_INHERITABLE, inheritable);
-	SETPPRIV(PRIV_EFFECTIVE, effective);
-	SETPPRIV(PRIV_PERMITTED, permitted);
 	SETPPRIV(PRIV_LIMIT, permitted);
+	SETPPRIV(PRIV_PERMITTED, permitted);
+	SETPPRIV(PRIV_EFFECTIVE, effective);
+	SETPPRIV(PRIV_INHERITABLE, inheritable);
 #undef SETPPRIV
 
 	priv_freeset(inheritable);



From geoff at varnish-cache.org  Mon Jan  9 20:52:19 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:19 +0100
Subject: [experimental-ims] ce54514 Solaris: Test for SO_{RCV,
	SND}TIMEO needs NET_LIBS
Message-ID: <E1RkMCZ-0007ig-A7@project.varnish-software.com>

commit ce5451435bf3ffe64d9113222777e035b3d52bda
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Fri Mar 11 17:36:54 2011 +0100

    Solaris: Test for SO_{RCV,SND}TIMEO needs NET_LIBS

diff --git a/configure.ac b/configure.ac
index cf1f566..8295fbb 100644
--- a/configure.ac
+++ b/configure.ac
@@ -375,11 +375,20 @@ fi
 AM_MISSING_HAS_RUN
 AC_CHECK_PROGS(PYTHON, [python3 python3.1 python3.2 python2.7 python2.6 python2.5 python2 python], [AC_MSG_ERROR([Python is needed to build Varnish, please install python.])])
 
-# Solaris defines SO_{RCV,SND}TIMEO, but does not implement them.
+# Older Solaris versions define SO_{RCV,SND}TIMEO, but do not
+# implement them.
+#
 # Varnish will build and run without these, but connections will not
 # time out, which may leave Varnish vulnerable to denail-of-service
 # attacks which would not be possible on other platforms.
+#
+# Newer Solaris releases with the Volo framework (Solaris 11,
+# Opensolaris starting with onnv_106) do support SO_{RCV,SND}TIMEO
+# (see PSARC 2007/587, initially committed into onnv-gate /
+# OpenSolaris 8348:4137e18bfaf0 Thu Dec 11 20:04:13 2008)
 
+save_LIBS="${LIBS}"
+LIBS="${LIBS} ${NET_LIBS}"
 AC_CACHE_CHECK([whether SO_RCVTIMEO works],
   [ac_cv_so_rcvtimeo_works],
   [AC_RUN_IFELSE(
@@ -404,7 +413,10 @@ return 1;
 if test "$ac_cv_so_rcvtimeo_works" = yes; then
    AC_DEFINE([SO_RCVTIMEO_WORKS], [1], [Define if SO_RCVTIMEO works])
 fi
+LIBS="${save_LIBS}"
 
+save_LIBS="${LIBS}"
+LIBS="${LIBS} ${NET_LIBS}"
 AC_CACHE_CHECK([whether SO_SNDTIMEO works],
   [ac_cv_so_sndtimeo_works],
   [AC_RUN_IFELSE(
@@ -434,6 +446,7 @@ if test "$ac_cv_so_rcvtimeo_works" = no ||
    test "$ac_cv_so_sndtimeo_works" = no; then
 	AC_MSG_WARN([connection timeouts will not work])
 fi
+LIBS="${save_LIBS}"
 
 # Run-time directory
 VARNISH_STATE_DIR='${localstatedir}/varnish'



From geoff at varnish-cache.org  Mon Jan  9 20:52:19 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:19 +0100
Subject: [experimental-ims] 31636e6 Even more strange error returns on Solaris
Message-ID: <E1RkMCZ-0007j4-Ew@project.varnish-software.com>

commit 31636e68029195142aee28b5fefb00841b3990a2
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Fri Oct 14 08:42:04 2011 +0000

    Even more strange error returns on Solaris

diff --git a/include/vtcp.h b/include/vtcp.h
index 022f101..6ad260e 100644
--- a/include/vtcp.h
+++ b/include/vtcp.h
@@ -33,16 +33,24 @@
 #define VTCP_ADDRBUFSIZE		64
 #define VTCP_PORTBUFSIZE		16
 
+static inline int
+VTCP_Check(int a)
+{
+	if (a == 0)
+		return (1);
+	if (errno == ECONNRESET || errno == ENOTCONN)
+		return (1);
 #if (defined (__SVR4) && defined (__sun)) || defined (__NetBSD__)
-/*
- * Solaris returns EINVAL if the other end unexepectedly reset the
- * connection.  This is a bug in Solaris and documented behaviour on NetBSD.
- */
-#define VTCP_Check(a) ((a) == 0 || errno == ECONNRESET || errno == ENOTCONN \
-    || errno == EINVAL)
-#else
-#define VTCP_Check(a) ((a) == 0 || errno == ECONNRESET || errno == ENOTCONN)
+	/*
+	 * Solaris returns EINVAL if the other end unexepectedly reset the
+	 * connection.
+	 * This is a bug in Solaris and documented behaviour on NetBSD.
+	 */
+	if (errno == EINVAL || errno == ETIMEDOUT)
+		return (1);
 #endif
+	return (0);
+}
 
 #define VTCP_Assert(a) assert(VTCP_Check(a))
 



From geoff at varnish-cache.org  Mon Jan  9 20:52:24 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:24 +0100
Subject: [experimental-ims] 601ee74 Ensure obj->response is set sensibly for
	errors
Message-ID: <E1RkMCe-0007jQ-KD@project.varnish-software.com>

commit 601ee74203b8d0088251c43c5e5de380dabdcba0
Author: Kristian Lyngstol <kristian at bohemians.org>
Date:   Mon Oct 17 13:30:29 2011 +0200

    Ensure obj->response is set sensibly for errors
    
    The http_PutProtocol() and http_PutResponse() would in the case of
    workspace overflow leave the headers as NULL and log a
    SLT_LostHeader. This would make Varnish assert correctly later when
    writing to the wire, as these are mandated by HTTP.
    
    This commit changes them to set the fields to static strings instead
    ("HTTP/1.1" and "Lost Response") when failing to write them to the
    workspace. This leaves enough information to complete the protocol in the
    case of overflow.
    
    The patch also increases the synthetic object's workspace from static
    1024 to param->http_resp_size. This leaves more (and configurable)
    room for manipulating the headers of the synthetic object in
    vcl_error.
    
    This whole thing has been a collaboration between Martin and myself. I'll
    leave it a mystery who wrote what line of code, which part of the comment
    and contributed what to the test-case.
    
    In all fairness, it's not a prefect solution, but a far step closer to one.
    So it sort of, kinda, more or less, for now, until we get a better
    solution:
    
    Fixes: #1031

diff --git a/bin/varnishd/cache_center.c b/bin/varnishd/cache_center.c
index 47139c7..6bf0b0c 100644
--- a/bin/varnishd/cache_center.c
+++ b/bin/varnishd/cache_center.c
@@ -435,13 +435,13 @@ cnt_error(struct sess *sp)
 	w = sp->wrk;
 	if (sp->obj == NULL) {
 		HSH_Prealloc(sp);
-		/* XXX: 1024 is a pure guess */
 		EXP_Clr(&w->exp);
-		sp->obj = STV_NewObject(sp, NULL, 1024, &w->exp,
-		     (uint16_t)params->http_max_hdr);
+		sp->obj = STV_NewObject(sp, NULL, params->http_resp_size,
+		     &w->exp, (uint16_t)params->http_max_hdr);
 		if (sp->obj == NULL)
 			sp->obj = STV_NewObject(sp, TRANSIENT_STORAGE,
-			    1024, &w->exp, (uint16_t)params->http_max_hdr);
+			     params->http_resp_size , &w->exp,
+			     (uint16_t)params->http_max_hdr);
 		if (sp->obj == NULL) {
 			sp->doclose = "Out of objects";
 			sp->director = NULL;
diff --git a/bin/varnishd/cache_http.c b/bin/varnishd/cache_http.c
index dd55990..aaa8212 100644
--- a/bin/varnishd/cache_http.c
+++ b/bin/varnishd/cache_http.c
@@ -979,6 +979,9 @@ http_PutProtocol(struct worker *w, unsigned vsl_id, const struct http *to,
 {
 
 	http_PutField(w, vsl_id, to, HTTP_HDR_PROTO, protocol);
+	if (to->hd[HTTP_HDR_PROTO].b == NULL)
+		http_SetH(to, HTTP_HDR_PROTO, "HTTP/1.1");
+	Tcheck(to->hd[HTTP_HDR_PROTO]);
 }
 
 void
@@ -995,6 +998,10 @@ http_PutResponse(struct worker *w, unsigned vsl_id, const struct http *to,
 {
 
 	http_PutField(w, vsl_id, to, HTTP_HDR_RESPONSE, response);
+	if (to->hd[HTTP_HDR_RESPONSE].b == NULL)
+		http_SetH(to, HTTP_HDR_RESPONSE, "Lost Response");
+	Tcheck(to->hd[HTTP_HDR_RESPONSE]);
+
 }
 
 void
diff --git a/bin/varnishtest/tests/r01031.vtc b/bin/varnishtest/tests/r01031.vtc
new file mode 100644
index 0000000..435f973
--- /dev/null
+++ b/bin/varnishtest/tests/r01031.vtc
@@ -0,0 +1,30 @@
+varnishtest "Test overflowing the response through sp->err_reason"
+
+varnish v1 -vcl {
+	backend blatti {
+		.host = "127.0.0.1";
+	}
+
+	sub vcl_recv {
+		error 200 "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa";
+	}
+	sub vcl_error {
+		return(deliver);
+	}
+} -start
+
+client c1 {
+	txreq -req GET
+	rxresp
+	expect resp.status == 200
+	expect resp.msg == "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
+} -run
+
+varnish v1 -cliok "param.set http_resp_size 256"
+
+client c2 {
+	txreq -req GET
+	rxresp
+	expect resp.status == 200
+	expect resp.msg == "Lost Response"
+} -run



From geoff at varnish-cache.org  Mon Jan  9 20:52:24 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:24 +0100
Subject: [experimental-ims] e18a6ab Formally remove error from vcl_deliver VCC
Message-ID: <E1RkMCe-0007so-Qu@project.varnish-software.com>

commit e18a6ab53fbae30b633fbe5f040b5686bec6ea4d
Author: Kristian Lyngstol <kristian at bohemians.org>
Date:   Mon Oct 17 14:46:54 2011 +0200

    Formally remove error from vcl_deliver VCC
    
    Note that error wasn't actually working in vcl_deliver, and this just puts
    VCC in line with the rest of Varnish. Syntax errors are better than assert
    errors.
    
    Re #1027
    
    I'll leave it for later discussion to see if we close #1027, which is
    technically a feature request now, though a request for a feature we used
    to have (not sure how well it worked).

diff --git a/bin/varnishtest/tests/r01027.vtc b/bin/varnishtest/tests/r01027.vtc
new file mode 100644
index 0000000..6c20b00
--- /dev/null
+++ b/bin/varnishtest/tests/r01027.vtc
@@ -0,0 +1,8 @@
+varnishtest "Test if you can error in vcl_deliver"
+
+varnish v1 -badvcl {
+	sub vcl_deliver {
+		error 201 "ok";
+	}
+}
+
diff --git a/lib/libvcl/vcc_action.c b/lib/libvcl/vcc_action.c
index 9d72b51..044d12d 100644
--- a/lib/libvcl/vcc_action.c
+++ b/lib/libvcl/vcc_action.c
@@ -320,7 +320,7 @@ static struct action_table {
 } action_table[] = {
 	{ "error",		parse_error,
 	    VCL_MET_RECV | VCL_MET_PIPE | VCL_MET_PASS | VCL_MET_HASH |
-            VCL_MET_MISS | VCL_MET_HIT | VCL_MET_FETCH | VCL_MET_DELIVER
+            VCL_MET_MISS | VCL_MET_HIT | VCL_MET_FETCH
 	},
 
 #define VCL_RET_MAC(l, U, B)						\



From geoff at varnish-cache.org  Mon Jan  9 20:52:24 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:24 +0100
Subject: [experimental-ims] da68c3a Attempt to document the different options
	for invalidating cached content
Message-ID: <E1RkMCe-0007sv-Sx@project.varnish-software.com>

commit da68c3aa1e84a923a1d95566aa21b86d9ef9cfe0
Author: Andreas Plesner Jacobsen <apj at mutt.dk>
Date:   Wed Oct 19 22:43:25 2011 +0200

    Attempt to document the different options for invalidating cached content

diff --git a/doc/sphinx/reference/index.rst b/doc/sphinx/reference/index.rst
index 84449da..9e41430 100644
--- a/doc/sphinx/reference/index.rst
+++ b/doc/sphinx/reference/index.rst
@@ -21,6 +21,7 @@ The Varnish Reference Manual
 	shmem.rst
 	vmod.rst
 	vmod_std.rst
+	purging_banning.rst
 	vsl.rst
 
 .. todo::
diff --git a/doc/sphinx/reference/purging_banning.rst b/doc/sphinx/reference/purging_banning.rst
new file mode 100644
index 0000000..4d63d8a
--- /dev/null
+++ b/doc/sphinx/reference/purging_banning.rst
@@ -0,0 +1,140 @@
+%%%%%%%%%%%%%%%%%%%%%%%%%%%
+Purging and banning content
+%%%%%%%%%%%%%%%%%%%%%%%%%%%
+
+Varnish has three ways to invalidate content in varnish. You can either purge an object and all the variants of it from the cache, or you can prevent past versions of an object from being served by banning it. Finally, you can force a cache miss to force a backend fetch and override an object in the cache.
+
+Purging
+=======
+
+To purge an object, you need to access that object explicitly, which is why the purge method is only available in VCL in the methods `vcl_hit` and `vcl_miss`. Purging is available in `vcl_miss` to allow for purging of all other variants of this object, even when this is particular request didn't hit a variant. Purging explicitly evicts that object and all variants from the cache immediately. An example implementation of the HTTP PURGE method in VCL::
+
+  acl purgers {
+    "localhost";
+    "192.0.2.1"/24;
+  }
+
+  sub vcl_recv {
+    if (req.request == "PURGE") {
+      if (!client.ip ~ purgers) {
+        error 405 "Not allowed.";
+      }
+      return(lookup);
+    }
+  }
+
+  sub vcl_hit {
+    if (req.request == "PURGE") {
+      purge;
+      error 200 "Purged.";
+    }
+  }
+
+  sub vcl_miss {
+    if (req.request == "PURGE") {
+      purge;
+      error 200 "Purged.";
+    }
+  }
+
+Banning
+=======
+
+Banning prevents varnish from serving all matching objects in the cache at the time of the ban. Banning is not a permanent operation, it is only used to invalidate and evict content already in the cache. It does not immediately evict objects from the cache, but will compare all future hits to this ban, and evict the objects if they match. When a ban has been matched against all objects in the cache, or when all objects in the cache is newer than the ban, it is deleted.
+
+Bans are added to the ban list using the `ban` CLI command or the `ban()` VCL method. They both take a ban expression that matches the objects that should be banned.
+
+Varnish also does duplicate ban detection if `ban_dups` is enabled.
+
+Ban Expressions
+---------------
+
+A ban expression is a list of conditions that needs to be fulfilled to invalidate content. You can match the content of the following variables using equality comparison or regular expressions:
+
+ * req.url
+ * req.http.*
+ * obj.http.*
+
+Conditions are combined using logical and: &&
+
+To ban any content served from an Apache backend, you could use this expression::
+
+ obj.http.Server ~ Apache
+
+To ban a particular URL and hostname::
+
+ req.url == /this/url && req.http.Host == example.com
+
+Banning From CLI
+----------------
+
+To ban from CLI, use the ban or the ban.url commands::
+
+ ban obj.http.Server ~ Apache
+ ban.url /images/.*
+
+ban.url is equivelant to "ban req.url ~"
+
+Banning From VCL
+----------------
+
+To ban from VCL, use the ban() or ban_url() functions. You can use the full arsenal of varnish string manipulation functions to build your ban expression. For example to let users execute requests that purge based on regular expressions::
+
+ sub vcl_recv {
+  if (req.url ~ "^/purgere/") {
+   if (!client.ip ~ purge) {
+    error 405 "Not allowed.";
+   }
+   set req.url = regsub(req.url, "^/purgere/", "/");
+   ban("obj.http.x-url ~ " req.url);
+   error 200 "Banned.";
+  }
+ }
+
+The Ban List
+------------
+
+The ban list can be inspected via the CLI command ban.list.
+
+Example output::
+
+  0xb75096d0 1318329475.377475    10      obj.http.x-url ~ test
+  0xb7509610 1318329470.785875    20G     obj.http.x-url ~ test
+
+The ban list contains the ID of the ban, the timestamp when the ban entered the ban list. A count of the objects that has reached this point in the ban list, optionally postfixed with a 'G' for "Gone", if the ban is no longer valid. Finally, the ban expression is listed. The ban can be marked as Gone if it is a duplicate ban, but is still kept in the list for optimization purposes.
+
+The Ban Lurker
+--------------
+
+Since a ban needs to be be matched against all objects in the cache, one way to speed up the eviction process is to enable the ban lurker. The ban lurker will walk the cache and match all objects to the bans in the ban list, and evict matching objects. The ban lurker is enabled by setting `ban_lurker_sleep` to a value above 0.
+
+Since Varnish 3.0, the ban lurker is enabled by default.
+
+Writing Ban Lurker Friendly Bans
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+To fully utilize the ban lurker, bans need to be written without the use of any req.* parameters, since there is no request to match against when the ban lurker walks the cache.
+
+A simple mode to avoid req.* in bans is to add headers to the cached object containing the parts of the request on which you want to ban, e.g.::
+
+  sub vcl_fetch {
+      set obj.http.x-url = req.url;
+  }
+
+  sub vcl_deliver {
+      unset resp.http.x-url; # Optional
+  }
+
+  sub vcl_recv {
+      if (req.request == "PURGE") {
+            if (client.ip !~ purgers) {
+                error 401 "Not allowed";
+             }
+            purge("obj.http.x-url ~ " req.url); # Assumes req.url is a regex. This might be a bit too simple
+      }
+  }
+
+req.hash_always_miss
+====================
+
+The final way to invalidate an object is a method that allows you to refresh an object by forcing a hash miss for a single request. If you set `req.hash_always_miss` to true, varnish will miss the current object in the cache, thus forcing a fetch from the backend. This can in turn add the freshly fetched object to the cache, thus overriding the current one. The old object will stay in the cache until ttl expires or it is evicted by some other means.



From geoff at varnish-cache.org  Mon Jan  9 20:52:25 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:25 +0100
Subject: [experimental-ims] 30046dc Make EXP_NukeOne() make do with a struct
	worker arg instead of sess.
Message-ID: <E1RkMCf-0007t8-0r@project.varnish-software.com>

commit 30046dce5fda7281b67169a1ad24484c0765c391
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Fri Oct 21 12:19:16 2011 +0000

    Make EXP_NukeOne() make do with a struct worker arg instead of sess.

diff --git a/bin/varnishd/cache.h b/bin/varnishd/cache.h
index 22e61ae..e94180a 100644
--- a/bin/varnishd/cache.h
+++ b/bin/varnishd/cache.h
@@ -697,7 +697,7 @@ void EXP_Inject(struct objcore *oc, struct lru *lru, double when);
 void EXP_Init(void);
 void EXP_Rearm(const struct object *o);
 int EXP_Touch(struct objcore *oc);
-int EXP_NukeOne(const struct sess *sp, struct lru *lru);
+int EXP_NukeOne(struct worker *w, struct lru *lru);
 
 /* cache_fetch.c */
 struct storage *FetchStorage(const struct sess *sp, ssize_t sz);
diff --git a/bin/varnishd/cache_center.c b/bin/varnishd/cache_center.c
index 6bf0b0c..fd3238d 100644
--- a/bin/varnishd/cache_center.c
+++ b/bin/varnishd/cache_center.c
@@ -440,8 +440,8 @@ cnt_error(struct sess *sp)
 		     &w->exp, (uint16_t)params->http_max_hdr);
 		if (sp->obj == NULL)
 			sp->obj = STV_NewObject(sp, TRANSIENT_STORAGE,
-			     params->http_resp_size , &w->exp,
-			     (uint16_t)params->http_max_hdr);
+			    params->http_resp_size, &w->exp,
+			    (uint16_t)params->http_max_hdr);
 		if (sp->obj == NULL) {
 			sp->doclose = "Out of objects";
 			sp->director = NULL;
diff --git a/bin/varnishd/cache_expire.c b/bin/varnishd/cache_expire.c
index 601025c..f7f779d 100644
--- a/bin/varnishd/cache_expire.c
+++ b/bin/varnishd/cache_expire.c
@@ -414,7 +414,7 @@ exp_timer(struct sess *sp, void *priv)
  */
 
 int
-EXP_NukeOne(const struct sess *sp, struct lru *lru)
+EXP_NukeOne(struct worker *w, struct lru *lru)
 {
 	struct objcore *oc;
 	struct object *o;
@@ -446,9 +446,9 @@ EXP_NukeOne(const struct sess *sp, struct lru *lru)
 		return (-1);
 
 	/* XXX: bad idea for -spersistent */
-	o = oc_getobj(sp->wrk, oc);
-	WSL(sp->wrk, SLT_ExpKill, 0, "%u LRU", o->xid);
-	(void)HSH_Deref(sp->wrk, NULL, &o);
+	o = oc_getobj(w, oc);
+	WSL(w, SLT_ExpKill, 0, "%u LRU", o->xid);
+	(void)HSH_Deref(w, NULL, &o);
 	return (1);
 }
 
diff --git a/bin/varnishd/cache_http.c b/bin/varnishd/cache_http.c
index aaa8212..784eb28 100644
--- a/bin/varnishd/cache_http.c
+++ b/bin/varnishd/cache_http.c
@@ -1001,7 +1001,6 @@ http_PutResponse(struct worker *w, unsigned vsl_id, const struct http *to,
 	if (to->hd[HTTP_HDR_RESPONSE].b == NULL)
 		http_SetH(to, HTTP_HDR_RESPONSE, "Lost Response");
 	Tcheck(to->hd[HTTP_HDR_RESPONSE]);
-
 }
 
 void
diff --git a/bin/varnishd/storage/stevedore.c b/bin/varnishd/storage/stevedore.c
index 65f9f49..bd42e54 100644
--- a/bin/varnishd/storage/stevedore.c
+++ b/bin/varnishd/storage/stevedore.c
@@ -184,7 +184,7 @@ stv_alloc(const struct sess *sp, size_t size)
 		}
 
 		/* no luck; try to free some space and keep trying */
-		if (EXP_NukeOne(sp, stv->lru) == -1)
+		if (EXP_NukeOne(sp->wrk, stv->lru) == -1)
 			break;
 
 		/* Enough is enough: try another if we have one */
@@ -336,7 +336,7 @@ STV_NewObject(struct sess *sp, const char *hint, unsigned wsl, struct exp *ep,
 	if (o == NULL) {
 		/* no luck; try to free some space and keep trying */
 		for (i = 0; o == NULL && i < params->nuke_limit; i++) {
-			if (EXP_NukeOne(sp, stv->lru) == -1)
+			if (EXP_NukeOne(sp->wrk, stv->lru) == -1)
 				break;
 			o = stv->allocobj(stv, sp, ltot, &soc);
 		}



From geoff at varnish-cache.org  Mon Jan  9 20:52:25 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:25 +0100
Subject: [experimental-ims] 5e4a502 Merge new material from
	reference/purging_banning, and do a little house keeping
Message-ID: <E1RkMCf-0007tO-B6@project.varnish-software.com>

commit 5e4a502fcc72a0bfef4269d06e08fcd9d8625f0f
Author: Andreas Plesner Jacobsen <apj at mutt.dk>
Date:   Sat Oct 22 10:58:22 2011 +0200

    Merge new material from reference/purging_banning, and do a little house keeping
    
    Remove reference/purging_banning, since all info is now in tutorial/purging

diff --git a/doc/sphinx/reference/index.rst b/doc/sphinx/reference/index.rst
index 9e41430..84449da 100644
--- a/doc/sphinx/reference/index.rst
+++ b/doc/sphinx/reference/index.rst
@@ -21,7 +21,6 @@ The Varnish Reference Manual
 	shmem.rst
 	vmod.rst
 	vmod_std.rst
-	purging_banning.rst
 	vsl.rst
 
 .. todo::
diff --git a/doc/sphinx/reference/purging_banning.rst b/doc/sphinx/reference/purging_banning.rst
deleted file mode 100644
index 4d63d8a..0000000
--- a/doc/sphinx/reference/purging_banning.rst
+++ /dev/null
@@ -1,140 +0,0 @@
-%%%%%%%%%%%%%%%%%%%%%%%%%%%
-Purging and banning content
-%%%%%%%%%%%%%%%%%%%%%%%%%%%
-
-Varnish has three ways to invalidate content in varnish. You can either purge an object and all the variants of it from the cache, or you can prevent past versions of an object from being served by banning it. Finally, you can force a cache miss to force a backend fetch and override an object in the cache.
-
-Purging
-=======
-
-To purge an object, you need to access that object explicitly, which is why the purge method is only available in VCL in the methods `vcl_hit` and `vcl_miss`. Purging is available in `vcl_miss` to allow for purging of all other variants of this object, even when this is particular request didn't hit a variant. Purging explicitly evicts that object and all variants from the cache immediately. An example implementation of the HTTP PURGE method in VCL::
-
-  acl purgers {
-    "localhost";
-    "192.0.2.1"/24;
-  }
-
-  sub vcl_recv {
-    if (req.request == "PURGE") {
-      if (!client.ip ~ purgers) {
-        error 405 "Not allowed.";
-      }
-      return(lookup);
-    }
-  }
-
-  sub vcl_hit {
-    if (req.request == "PURGE") {
-      purge;
-      error 200 "Purged.";
-    }
-  }
-
-  sub vcl_miss {
-    if (req.request == "PURGE") {
-      purge;
-      error 200 "Purged.";
-    }
-  }
-
-Banning
-=======
-
-Banning prevents varnish from serving all matching objects in the cache at the time of the ban. Banning is not a permanent operation, it is only used to invalidate and evict content already in the cache. It does not immediately evict objects from the cache, but will compare all future hits to this ban, and evict the objects if they match. When a ban has been matched against all objects in the cache, or when all objects in the cache is newer than the ban, it is deleted.
-
-Bans are added to the ban list using the `ban` CLI command or the `ban()` VCL method. They both take a ban expression that matches the objects that should be banned.
-
-Varnish also does duplicate ban detection if `ban_dups` is enabled.
-
-Ban Expressions
----------------
-
-A ban expression is a list of conditions that needs to be fulfilled to invalidate content. You can match the content of the following variables using equality comparison or regular expressions:
-
- * req.url
- * req.http.*
- * obj.http.*
-
-Conditions are combined using logical and: &&
-
-To ban any content served from an Apache backend, you could use this expression::
-
- obj.http.Server ~ Apache
-
-To ban a particular URL and hostname::
-
- req.url == /this/url && req.http.Host == example.com
-
-Banning From CLI
-----------------
-
-To ban from CLI, use the ban or the ban.url commands::
-
- ban obj.http.Server ~ Apache
- ban.url /images/.*
-
-ban.url is equivelant to "ban req.url ~"
-
-Banning From VCL
-----------------
-
-To ban from VCL, use the ban() or ban_url() functions. You can use the full arsenal of varnish string manipulation functions to build your ban expression. For example to let users execute requests that purge based on regular expressions::
-
- sub vcl_recv {
-  if (req.url ~ "^/purgere/") {
-   if (!client.ip ~ purge) {
-    error 405 "Not allowed.";
-   }
-   set req.url = regsub(req.url, "^/purgere/", "/");
-   ban("obj.http.x-url ~ " req.url);
-   error 200 "Banned.";
-  }
- }
-
-The Ban List
-------------
-
-The ban list can be inspected via the CLI command ban.list.
-
-Example output::
-
-  0xb75096d0 1318329475.377475    10      obj.http.x-url ~ test
-  0xb7509610 1318329470.785875    20G     obj.http.x-url ~ test
-
-The ban list contains the ID of the ban, the timestamp when the ban entered the ban list. A count of the objects that has reached this point in the ban list, optionally postfixed with a 'G' for "Gone", if the ban is no longer valid. Finally, the ban expression is listed. The ban can be marked as Gone if it is a duplicate ban, but is still kept in the list for optimization purposes.
-
-The Ban Lurker
---------------
-
-Since a ban needs to be be matched against all objects in the cache, one way to speed up the eviction process is to enable the ban lurker. The ban lurker will walk the cache and match all objects to the bans in the ban list, and evict matching objects. The ban lurker is enabled by setting `ban_lurker_sleep` to a value above 0.
-
-Since Varnish 3.0, the ban lurker is enabled by default.
-
-Writing Ban Lurker Friendly Bans
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-To fully utilize the ban lurker, bans need to be written without the use of any req.* parameters, since there is no request to match against when the ban lurker walks the cache.
-
-A simple mode to avoid req.* in bans is to add headers to the cached object containing the parts of the request on which you want to ban, e.g.::
-
-  sub vcl_fetch {
-      set obj.http.x-url = req.url;
-  }
-
-  sub vcl_deliver {
-      unset resp.http.x-url; # Optional
-  }
-
-  sub vcl_recv {
-      if (req.request == "PURGE") {
-            if (client.ip !~ purgers) {
-                error 401 "Not allowed";
-             }
-            purge("obj.http.x-url ~ " req.url); # Assumes req.url is a regex. This might be a bit too simple
-      }
-  }
-
-req.hash_always_miss
-====================
-
-The final way to invalidate an object is a method that allows you to refresh an object by forcing a hash miss for a single request. If you set `req.hash_always_miss` to true, varnish will miss the current object in the cache, thus forcing a fetch from the backend. This can in turn add the freshly fetched object to the cache, thus overriding the current one. The old object will stay in the cache until ttl expires or it is evicted by some other means.
diff --git a/doc/sphinx/tutorial/purging.rst b/doc/sphinx/tutorial/purging.rst
index ce1d945..e5da595 100644
--- a/doc/sphinx/tutorial/purging.rst
+++ b/doc/sphinx/tutorial/purging.rst
@@ -10,8 +10,8 @@ of, in this twitterific day of age serving content that is outdated is
 bad for business.
 
 The solution is to notify Varnish when there is fresh content
-available. This can be done through two mechanisms. HTTP purging and
-bans. First, let me explain the HTTP purges. 
+available. This can be done through three mechanisms. HTTP purging,
+banning and forced cache misses. First, let me explain the HTTP purges. 
 
 
 HTTP Purges
@@ -61,7 +61,9 @@ As you can see we have used to new VCL subroutines, vcl_hit and
 vcl_miss. When we call lookup Varnish will try to lookup the object in
 its cache. It will either hit an object or miss it and so the
 corresponding subroutine is called. In vcl_hit the object that is
-stored in cache is available and we can set the TTL.
+stored in cache is available and we can set the TTL. The purge in
+vcl_miss is necessary to purge all variants in the cases where you hit an
+object, but miss a particular variant.
 
 So for example.com to invalidate their front page they would call out
 to Varnish like this::
@@ -75,14 +77,16 @@ variants as defined by Vary.
 Bans
 ====
 
-There is another way to invalidate content. Bans. You can think of
-bans as a sort of a filter. You *ban* certain content from being
-served from your cache. You can ban content based on any metadata we
-have.
+There is another way to invalidate content: Bans. You can think of
+bans as a sort of a filter on objects already in the cache. You *ban*
+certain content from being served from your cache. You can ban
+content based on any metadata we have.
+A ban will only work on objects already in the cache, it does not
+prevent new content from entering the cache or being served.
 
 Support for bans is built into Varnish and available in the CLI
-interface. For VG to ban every png object belonging on example.com
-they could issue::
+interface. To ban every png object belonging on example.com, issue
+the following command::
 
   ban req.http.host == "example.com" && req.http.url ~ "\.png$"
 
@@ -91,13 +95,14 @@ Quite powerful, really.
 Bans are checked when we hit an object in the cache, but before we
 deliver it. *An object is only checked against newer bans*. 
 
-Bans that only match against beresp.* are also processed by a
-background worker threads called the *ban lurker*. The ban lurker will
-walk the heap and try to match objects and will evict the matching
-objects. How aggressive the ban lurker is can be controlled by the
-parameter ban_lurker_sleep. 
+Bans that only match against obj.* are also processed by a background
+worker threads called the *ban lurker*. The ban lurker will walk the
+heap and try to match objects and will evict the matching objects. How
+aggressive the ban lurker is can be controlled by the parameter
+ban_lurker_sleep. The ban lurker can be disabled by setting
+ban_lurker_sleep to 0.
 
-Bans that are older then the oldest objects in the cache are discarded
+Bans that are older than the oldest objects in the cache are discarded
 without evaluation.  If you have a lot of objects with long TTL, that
 are seldom accessed you might accumulate a lot of bans. This might
 impact CPU usage and thereby performance.
@@ -122,3 +127,49 @@ You can also add bans to Varnish via HTTP. Doing so requires a bit of VCL::
 This VCL sniplet enables Varnish to handle an HTTP BAN method, adding a
 ban on the URL, including the host part.
 
+The ban lurker can help you keep the ban list at a manageable size, so
+we recommend that you avoid using req.* in your bans, as the request
+object is not available in the ban lurker thread.
+
+You can use the following template to write ban lurker friendly bans::
+
+  sub vcl_fetch {
+    set obj.http.x-url = req.url;
+  }
+
+  sub vcl_deliver {
+    unset resp.http.x-url; # Optional
+  }
+
+  sub vcl_recv {
+    if (req.request == "PURGE") {
+      if (client.ip !~ purge) {
+        error 401 "Not allowed";
+      }
+      ban("obj.http.x-url ~ " req.url); # Assumes req.url is a regex. This might be a bit too simple
+    }
+  }
+
+To inspect the current ban list, issue the ban.list command in CLI. This
+will produce a status of all current bans::
+
+  0xb75096d0 1318329475.377475    10      obj.http.x-url ~ test
+  0xb7509610 1318329470.785875    20G     obj.http.x-url ~ test
+
+The ban list contains the ID of the ban, the timestamp when the ban
+entered the ban list. A count of the objects that has reached this point
+in the ban list, optionally postfixed with a 'G' for "Gone", if the ban
+is no longer valid.  Finally, the ban expression is listed. The ban can
+be marked as Gone if it is a duplicate ban, but is still kept in the list
+for optimization purposes.
+
+Forcing a cache miss
+====================
+
+The final way to invalidate an object is a method that allows you to
+refresh an object by forcing a hash miss for a single request. If you set
+req.hash_always_miss to true, varnish will miss the current object in the
+cache, thus forcing a fetch from the backend. This can in turn add the
+freshly fetched object to the cache, thus overriding the current one. The
+old object will stay in the cache until ttl expires or it is evicted by
+some other means.



From geoff at varnish-cache.org  Mon Jan  9 20:52:25 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:25 +0100
Subject: [experimental-ims] 8c80f86 Move the vbc structure from session to
	worker
Message-ID: <E1RkMCf-0007tm-H5@project.varnish-software.com>

commit 8c80f8649476d9226e68a3b4ffa28310bc280c94
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Oct 24 12:08:37 2011 +0000

    Move the vbc structure from session to worker

diff --git a/bin/varnishd/cache.h b/bin/varnishd/cache.h
index 018355e..59d5fa8 100644
--- a/bin/varnishd/cache.h
+++ b/bin/varnishd/cache.h
@@ -338,6 +338,7 @@ struct worker {
 	const char		*storage_hint;
 
 	/* Fetch stuff */
+	struct vbc		*vbc;
 	enum body_status	body_status;
 	struct vfp		*vfp;
 	struct vgz		*vgz_rx;
@@ -606,7 +607,6 @@ struct sess {
 	VTAILQ_ENTRY(sess)	list;
 
 	struct director		*director;
-	struct vbc		*vbc;
 	struct object		*obj;
 	struct objcore		*objcore;
 	struct VCL_conf		*vcl;
@@ -641,8 +641,8 @@ void VBE_UseHealth(const struct director *vdi);
 
 struct vbc *VDI_GetFd(const struct director *, struct sess *sp);
 int VDI_Healthy(const struct director *, const struct sess *sp);
-void VDI_CloseFd(struct sess *sp);
-void VDI_RecycleFd(struct sess *sp);
+void VDI_CloseFd(struct worker *wrk);
+void VDI_RecycleFd(struct worker *wrk);
 void VDI_AddHostHeader(const struct sess *sp);
 void VBE_Poll(void);
 
diff --git a/bin/varnishd/cache_backend.c b/bin/varnishd/cache_backend.c
index 0026259..7bacd26 100644
--- a/bin/varnishd/cache_backend.c
+++ b/bin/varnishd/cache_backend.c
@@ -64,10 +64,10 @@ VDI_AddHostHeader(const struct sess *sp)
 
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 	CHECK_OBJ_NOTNULL(sp->wrk->bereq, HTTP_MAGIC);
-	CHECK_OBJ_NOTNULL(sp->vbc, VBC_MAGIC);
-	CHECK_OBJ_NOTNULL(sp->vbc->vdis, VDI_SIMPLE_MAGIC);
+	CHECK_OBJ_NOTNULL(sp->wrk->vbc, VBC_MAGIC);
+	CHECK_OBJ_NOTNULL(sp->wrk->vbc->vdis, VDI_SIMPLE_MAGIC);
 	http_PrintfHeader(sp->wrk, sp->vsl_id, sp->wrk->bereq,
-	    "Host: %s", sp->vbc->vdis->vrt->hosthdr);
+	    "Host: %s", sp->wrk->vbc->vdis->vrt->hosthdr);
 }
 
 /*--------------------------------------------------------------------*/
diff --git a/bin/varnishd/cache_center.c b/bin/varnishd/cache_center.c
index d60f17e..243123d 100644
--- a/bin/varnishd/cache_center.c
+++ b/bin/varnishd/cache_center.c
@@ -227,7 +227,7 @@ cnt_prepresp(struct sess *sp)
 		if (sp->restarts >= params->max_restarts)
 			break;
 		if (sp->wrk->do_stream) {
-			VDI_CloseFd(sp);
+			VDI_CloseFd(sp->wrk);
 			HSH_Drop(sp);
 		} else {
 			(void)HSH_Deref(sp->wrk, NULL, &sp->obj);
@@ -305,7 +305,7 @@ cnt_done(struct sess *sp)
 	CHECK_OBJ_ORNULL(sp->vcl, VCL_CONF_MAGIC);
 
 	AZ(sp->obj);
-	AZ(sp->vbc);
+	AZ(sp->wrk->vbc);
 	sp->director = NULL;
 	sp->restarts = 0;
 
@@ -537,7 +537,7 @@ cnt_fetch(struct sess *sp)
 	CHECK_OBJ_NOTNULL(sp->vcl, VCL_CONF_MAGIC);
 
 	AN(sp->director);
-	AZ(sp->vbc);
+	AZ(sp->wrk->vbc);
 	AZ(sp->wrk->h_content_length);
 	AZ(sp->wrk->do_close);
 	AZ(sp->wrk->storage_hint);
@@ -606,11 +606,11 @@ cnt_fetch(struct sess *sp)
 		}
 
 		/* We are not going to fetch the body, Close the connection */
-		VDI_CloseFd(sp);
+		VDI_CloseFd(sp->wrk);
 	}
 
 	/* Clean up partial fetch */
-	AZ(sp->vbc);
+	AZ(sp->wrk->vbc);
 
 	if (sp->objcore != NULL) {
 		CHECK_OBJ_NOTNULL(sp->objcore, OBJCORE_MAGIC);
@@ -788,7 +788,7 @@ cnt_fetchbody(struct sess *sp)
 	if (sp->obj == NULL) {
 		sp->err_code = 503;
 		sp->step = STP_ERROR;
-		VDI_CloseFd(sp);
+		VDI_CloseFd(sp->wrk);
 		return (0);
 	}
 	CHECK_OBJ_NOTNULL(sp->obj, OBJECT_MAGIC);
@@ -854,7 +854,7 @@ cnt_fetchbody(struct sess *sp)
 	http_Setup(sp->wrk->beresp, NULL);
 	sp->wrk->vfp = NULL;
 	assert(WRW_IsReleased(sp->wrk));
-	AZ(sp->vbc);
+	AZ(sp->wrk->vbc);
 	AN(sp->director);
 
 	if (i) {
@@ -917,7 +917,7 @@ cnt_streambody(struct sess *sp)
 	http_Setup(sp->wrk->bereq, NULL);
 	http_Setup(sp->wrk->beresp, NULL);
 	sp->wrk->vfp = NULL;
-	AZ(sp->vbc);
+	AZ(sp->wrk->vbc);
 	AN(sp->director);
 
 	if (!i && sp->obj->objcore != NULL) {
diff --git a/bin/varnishd/cache_dir.c b/bin/varnishd/cache_dir.c
index 3fd0475..e95b015 100644
--- a/bin/varnishd/cache_dir.c
+++ b/bin/varnishd/cache_dir.c
@@ -40,54 +40,54 @@
 /* Close a connection ------------------------------------------------*/
 
 void
-VDI_CloseFd(struct sess *sp)
+VDI_CloseFd(struct worker *wrk)
 {
 	struct backend *bp;
 
-	CHECK_OBJ_NOTNULL(sp->vbc, VBC_MAGIC);
-	CHECK_OBJ_NOTNULL(sp->vbc->backend, BACKEND_MAGIC);
-	assert(sp->vbc->fd >= 0);
+	CHECK_OBJ_NOTNULL(wrk->vbc, VBC_MAGIC);
+	CHECK_OBJ_NOTNULL(wrk->vbc->backend, BACKEND_MAGIC);
+	assert(wrk->vbc->fd >= 0);
 
-	bp = sp->vbc->backend;
+	bp = wrk->vbc->backend;
 
-	WSL(sp->wrk, SLT_BackendClose, sp->vbc->vsl_id, "%s", bp->vcl_name);
+	WSL(wrk, SLT_BackendClose, wrk->vbc->vsl_id, "%s", bp->vcl_name);
 
 	/* Checkpoint log to flush all info related to this connection
 	   before the OS reuses the FD */
-	WSL_Flush(sp->wrk, 0);
+	WSL_Flush(wrk, 0);
 
-	VTCP_close(&sp->vbc->fd);
+	VTCP_close(&wrk->vbc->fd);
 	VBE_DropRefConn(bp);
-	sp->vbc->backend = NULL;
-	VBE_ReleaseConn(sp->vbc);
-	sp->vbc = NULL;
-	sp->wrk->do_close = 0;
+	wrk->vbc->backend = NULL;
+	VBE_ReleaseConn(wrk->vbc);
+	wrk->vbc = NULL;
+	wrk->do_close = 0;
 }
 
 /* Recycle a connection ----------------------------------------------*/
 
 void
-VDI_RecycleFd(struct sess *sp)
+VDI_RecycleFd(struct worker *wrk)
 {
 	struct backend *bp;
 
-	CHECK_OBJ_NOTNULL(sp->vbc, VBC_MAGIC);
-	CHECK_OBJ_NOTNULL(sp->vbc->backend, BACKEND_MAGIC);
-	assert(sp->vbc->fd >= 0);
-	AZ(sp->wrk->do_close);
+	CHECK_OBJ_NOTNULL(wrk->vbc, VBC_MAGIC);
+	CHECK_OBJ_NOTNULL(wrk->vbc->backend, BACKEND_MAGIC);
+	assert(wrk->vbc->fd >= 0);
+	AZ(wrk->do_close);
 
-	bp = sp->vbc->backend;
+	bp = wrk->vbc->backend;
 
-	WSL(sp->wrk, SLT_BackendReuse, sp->vbc->vsl_id, "%s", bp->vcl_name);
+	WSL(wrk, SLT_BackendReuse, wrk->vbc->vsl_id, "%s", bp->vcl_name);
 	/*
 	 * Flush the shmlog, so that another session reusing this backend
 	 * will log chronologically later than our use of it.
 	 */
-	WSL_Flush(sp->wrk, 0);
+	WSL_Flush(wrk, 0);
 	Lck_Lock(&bp->mtx);
 	VSC_C_main->backend_recycle++;
-	VTAILQ_INSERT_HEAD(&bp->connlist, sp->vbc, list);
-	sp->vbc = NULL;
+	VTAILQ_INSERT_HEAD(&bp->connlist, wrk->vbc, list);
+	wrk->vbc = NULL;
 	VBE_DropRefLocked(bp);
 }
 
diff --git a/bin/varnishd/cache_fetch.c b/bin/varnishd/cache_fetch.c
index 71fc8e2..ba91720 100644
--- a/bin/varnishd/cache_fetch.c
+++ b/bin/varnishd/cache_fetch.c
@@ -394,12 +394,12 @@ FetchHdr(struct sess *sp)
 
 	hp = w->bereq;
 
-	sp->vbc = VDI_GetFd(NULL, sp);
-	if (sp->vbc == NULL) {
+	sp->wrk->vbc = VDI_GetFd(NULL, sp);
+	if (sp->wrk->vbc == NULL) {
 		WSP(sp, SLT_FetchError, "no backend connection");
 		return (-1);
 	}
-	vc = sp->vbc;
+	vc = sp->wrk->vbc;
 	if (vc->recycled)
 		retry = 1;
 
@@ -420,7 +420,7 @@ FetchHdr(struct sess *sp)
 	if (WRW_FlushRelease(w) || i > 0) {
 		WSP(sp, SLT_FetchError, "backend write error: %d (%s)",
 		    errno, strerror(errno));
-		VDI_CloseFd(sp);
+		VDI_CloseFd(sp->wrk);
 		/* XXX: other cleanup ? */
 		return (retry);
 	}
@@ -443,7 +443,7 @@ FetchHdr(struct sess *sp)
 	if (i < 0) {
 		WSP(sp, SLT_FetchError, "http first read error: %d %d (%s)",
 		    i, errno, w->htc->error);
-		VDI_CloseFd(sp);
+		VDI_CloseFd(sp->wrk);
 		/* XXX: other cleanup ? */
 		/* Retryable if we never received anything */
 		return (i == -1 ? retry : -1);
@@ -457,7 +457,7 @@ FetchHdr(struct sess *sp)
 			WSP(sp, SLT_FetchError,
 			    "http first read error: %d %d (%s)",
 			    i, errno, w->htc->error);
-			VDI_CloseFd(sp);
+			VDI_CloseFd(sp->wrk);
 			/* XXX: other cleanup ? */
 			return (-1);
 		}
@@ -467,7 +467,7 @@ FetchHdr(struct sess *sp)
 
 	if (http_DissectResponse(w, w->htc, hp)) {
 		WSP(sp, SLT_FetchError, "http format error");
-		VDI_CloseFd(sp);
+		VDI_CloseFd(sp->wrk);
 		/* XXX: other cleanup ? */
 		return (-1);
 	}
@@ -541,12 +541,12 @@ FetchBody(struct sess *sp)
 	 */
 	AZ(vfp_nop_end(sp));
 
-	WSL(w, SLT_Fetch_Body, sp->vbc->vsl_id, "%u(%s) cls %d mklen %u",
+	WSL(w, SLT_Fetch_Body, sp->wrk->vbc->vsl_id, "%u(%s) cls %d mklen %u",
 	    w->body_status, body_status(w->body_status),
 	    cls, mklen);
 
 	if (w->body_status == BS_ERROR) {
-		VDI_CloseFd(sp);
+		VDI_CloseFd(sp->wrk);
 		return (__LINE__);
 	}
 
@@ -558,7 +558,7 @@ FetchBody(struct sess *sp)
 			VTAILQ_REMOVE(&sp->obj->store, st, list);
 			STV_free(st);
 		}
-		VDI_CloseFd(sp);
+		VDI_CloseFd(sp->wrk);
 		sp->obj->len = 0;
 		return (__LINE__);
 	}
@@ -566,7 +566,7 @@ FetchBody(struct sess *sp)
 	if (cls == 0 && w->do_close)
 		cls = 1;
 
-	WSL(w, SLT_Length, sp->vbc->vsl_id, "%u", sp->obj->len);
+	WSL(w, SLT_Length, sp->wrk->vbc->vsl_id, "%u", sp->obj->len);
 
 	{
 	/* Sanity check fetch methods accounting */
@@ -589,9 +589,9 @@ FetchBody(struct sess *sp)
 	}
 
 	if (cls)
-		VDI_CloseFd(sp);
+		VDI_CloseFd(sp->wrk);
 	else
-		VDI_RecycleFd(sp);
+		VDI_RecycleFd(sp->wrk);
 
 	return (0);
 }
diff --git a/bin/varnishd/cache_panic.c b/bin/varnishd/cache_panic.c
index 76ea866..231adf1 100644
--- a/bin/varnishd/cache_panic.c
+++ b/bin/varnishd/cache_panic.c
@@ -263,8 +263,8 @@ pan_sess(const struct sess *sp)
 	if (VALID_OBJ(sp->vcl, VCL_CONF_MAGIC))
 		pan_vcl(sp->vcl);
 
-	if (VALID_OBJ(sp->vbc, BACKEND_MAGIC))
-		pan_vbc(sp->vbc);
+	if (VALID_OBJ(sp->wrk->vbc, BACKEND_MAGIC))
+		pan_vbc(sp->wrk->vbc);
 
 	if (VALID_OBJ(sp->obj, OBJECT_MAGIC))
 		pan_object(sp->obj);
diff --git a/bin/varnishd/cache_pipe.c b/bin/varnishd/cache_pipe.c
index 8f109a5..31dcf1a 100644
--- a/bin/varnishd/cache_pipe.c
+++ b/bin/varnishd/cache_pipe.c
@@ -71,10 +71,10 @@ PipeSession(struct sess *sp)
 	CHECK_OBJ_NOTNULL(sp->wrk, WORKER_MAGIC);
 	w = sp->wrk;
 
-	sp->vbc = VDI_GetFd(NULL, sp);
-	if (sp->vbc == NULL)
+	sp->wrk->vbc = VDI_GetFd(NULL, sp);
+	if (sp->wrk->vbc == NULL)
 		return;
-	vc = sp->vbc;
+	vc = sp->wrk->vbc;
 	(void)VTCP_blocking(vc->fd);
 
 	WRW_Reserve(w, &vc->fd);
@@ -89,7 +89,7 @@ PipeSession(struct sess *sp)
 
 	if (i) {
 		SES_Close(sp, "pipe");
-		VDI_CloseFd(sp);
+		VDI_CloseFd(sp->wrk);
 		return;
 	}
 
@@ -129,5 +129,5 @@ PipeSession(struct sess *sp)
 		}
 	}
 	SES_Close(sp, "pipe");
-	VDI_CloseFd(sp);
+	VDI_CloseFd(sp->wrk);
 }
diff --git a/bin/varnishd/cache_vrt_var.c b/bin/varnishd/cache_vrt_var.c
index 1a16e16..92d8451 100644
--- a/bin/varnishd/cache_vrt_var.c
+++ b/bin/varnishd/cache_vrt_var.c
@@ -130,14 +130,16 @@ VRT_l_beresp_saintmode(const struct sess *sp, double a)
 	struct trouble *new;
 	struct trouble *tr;
 	struct trouble *tr2;
+	struct worker *wrk;
 
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
-	if (!sp->vbc)
+	wrk = sp->wrk;
+	if (!wrk->vbc)
 		return;
-	CHECK_OBJ_NOTNULL(sp->vbc, VBC_MAGIC);
-	if (!sp->vbc->backend)
+	CHECK_OBJ_NOTNULL(wrk->vbc, VBC_MAGIC);
+	if (!wrk->vbc->backend)
 		return;
-	CHECK_OBJ_NOTNULL(sp->vbc->backend, BACKEND_MAGIC);
+	CHECK_OBJ_NOTNULL(wrk->vbc->backend, BACKEND_MAGIC);
 	if (!sp->objcore)
 		return;
 	CHECK_OBJ_NOTNULL(sp->objcore, OBJCORE_MAGIC);
@@ -156,8 +158,8 @@ VRT_l_beresp_saintmode(const struct sess *sp, double a)
 	 * timeout at a later date (ie: sort by which entry will time out
 	 * from the list
 	 */
-	Lck_Lock(&sp->vbc->backend->mtx);
-	VTAILQ_FOREACH_SAFE(tr, &sp->vbc->backend->troublelist, list, tr2) {
+	Lck_Lock(&wrk->vbc->backend->mtx);
+	VTAILQ_FOREACH_SAFE(tr, &wrk->vbc->backend->troublelist, list, tr2) {
 		if (tr->timeout < new->timeout) {
 			VTAILQ_INSERT_BEFORE(tr, new, list);
 			new = NULL;
@@ -169,9 +171,9 @@ VRT_l_beresp_saintmode(const struct sess *sp, double a)
 	 * items have a longer timeout.
 	 */
 	if (new)
-		VTAILQ_INSERT_TAIL(&sp->vbc->backend->troublelist, new, list);
+		VTAILQ_INSERT_TAIL(&wrk->vbc->backend->troublelist, new, list);
 
-	Lck_Unlock(&sp->vbc->backend->mtx);
+	Lck_Unlock(&wrk->vbc->backend->mtx);
 }
 
 /*--------------------------------------------------------------------*/
@@ -250,8 +252,8 @@ VRT_r_beresp_backend_name(const struct sess *sp)
 {
 
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
-	CHECK_OBJ_NOTNULL(sp->vbc, VBC_MAGIC);
-	return(sp->vbc->backend->vcl_name);
+	CHECK_OBJ_NOTNULL(sp->wrk->vbc, VBC_MAGIC);
+	return(sp->wrk->vbc->backend->vcl_name);
 }
 
 struct sockaddr_storage *
@@ -259,8 +261,8 @@ VRT_r_beresp_backend_ip(const struct sess *sp)
 {
 
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
-	CHECK_OBJ_NOTNULL(sp->vbc, VBC_MAGIC);
-	return(sp->vbc->addr);
+	CHECK_OBJ_NOTNULL(sp->wrk->vbc, VBC_MAGIC);
+	return(sp->wrk->vbc->addr);
 }
 
 int
@@ -268,8 +270,8 @@ VRT_r_beresp_backend_port(const struct sess *sp)
 {
 
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
-	CHECK_OBJ_NOTNULL(sp->vbc, VBC_MAGIC);
-	return (VTCP_port(sp->vbc->addr));
+	CHECK_OBJ_NOTNULL(sp->wrk->vbc, VBC_MAGIC);
+	return (VTCP_port(sp->wrk->vbc->addr));
 }
 
 const char * __match_proto__()



From geoff at varnish-cache.org  Mon Jan  9 20:52:25 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:25 +0100
Subject: [experimental-ims] 57240f5 Decontaminate STV_alloc() to not need
	session
Message-ID: <E1RkMCf-0007uA-OF@project.varnish-software.com>

commit 57240f5a3e76cbd968518e0325de97bf09d293ea
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Oct 24 12:36:22 2011 +0000

    Decontaminate STV_alloc() to not need session

diff --git a/bin/varnishd/cache.h b/bin/varnishd/cache.h
index 71cc776..a8dff7b 100644
--- a/bin/varnishd/cache.h
+++ b/bin/varnishd/cache.h
@@ -950,7 +950,8 @@ int RFC2616_Do_Cond(const struct sess *sp);
 /* stevedore.c */
 struct object *STV_NewObject(struct sess *sp, const char *hint, unsigned len,
     struct exp *, uint16_t nhttp);
-struct storage *STV_alloc(const struct sess *sp, size_t size);
+struct storage *STV_alloc(struct worker *w, const struct object *obj,
+    size_t size);
 void STV_trim(struct storage *st, size_t size);
 void STV_free(struct storage *st);
 void STV_open(void);
diff --git a/bin/varnishd/cache_esi_fetch.c b/bin/varnishd/cache_esi_fetch.c
index f69e900..5d8f8dc 100644
--- a/bin/varnishd/cache_esi_fetch.c
+++ b/bin/varnishd/cache_esi_fetch.c
@@ -368,7 +368,7 @@ vfp_esi_end(struct sess *sp)
 		l = VSB_len(vsb);
 		assert(l > 0);
 		/* XXX: This is a huge waste of storage... */
-		sp->obj->esidata = STV_alloc(sp, l);
+		sp->obj->esidata = STV_alloc(sp->wrk, sp->obj, l);
 		XXXAN(sp->obj->esidata);
 		memcpy(sp->obj->esidata->ptr, VSB_data(vsb), l);
 		sp->obj->esidata->len = l;
diff --git a/bin/varnishd/cache_fetch.c b/bin/varnishd/cache_fetch.c
index eae583d..ccc2ec3 100644
--- a/bin/varnishd/cache_fetch.c
+++ b/bin/varnishd/cache_fetch.c
@@ -159,7 +159,7 @@ FetchStorage(const struct sess *sp, ssize_t sz)
 		l = sz;
 	if (l == 0)
 		l = params->fetch_chunksize * 1024LL;
-	st = STV_alloc(sp, l);
+	st = STV_alloc(sp->wrk, sp->obj, l);
 	if (st == NULL) {
 		errno = ENOMEM;
 		return (NULL);
diff --git a/bin/varnishd/storage/stevedore.c b/bin/varnishd/storage/stevedore.c
index bd42e54..93e2bf8 100644
--- a/bin/varnishd/storage/stevedore.c
+++ b/bin/varnishd/storage/stevedore.c
@@ -154,7 +154,7 @@ stv_pick_stevedore(const struct sess *sp, const char **hint)
 /*-------------------------------------------------------------------*/
 
 static struct storage *
-stv_alloc(const struct sess *sp, size_t size)
+stv_alloc(struct worker *w, const struct object *obj, size_t size)
 {
 	struct storage *st;
 	struct stevedore *stv;
@@ -164,8 +164,9 @@ stv_alloc(const struct sess *sp, size_t size)
 	 * Always use the stevedore which allocated the object in order to
 	 * keep an object inside the same stevedore.
 	 */
-	CHECK_OBJ_NOTNULL(sp->obj, OBJECT_MAGIC);
-	stv = sp->obj->objstore->stevedore;
+	CHECK_OBJ_NOTNULL(obj, OBJECT_MAGIC);
+	CHECK_OBJ_NOTNULL(w, WORKER_MAGIC);
+	stv = obj->objstore->stevedore;
 	CHECK_OBJ_NOTNULL(stv, STEVEDORE_MAGIC);
 
 	if (size > (size_t)(params->fetch_maxchunksize) << 10)
@@ -184,7 +185,7 @@ stv_alloc(const struct sess *sp, size_t size)
 		}
 
 		/* no luck; try to free some space and keep trying */
-		if (EXP_NukeOne(sp->wrk, stv->lru) == -1)
+		if (EXP_NukeOne(w, stv->lru) == -1)
 			break;
 
 		/* Enough is enough: try another if we have one */
@@ -370,10 +371,10 @@ STV_Freestore(struct object *o)
 /*-------------------------------------------------------------------*/
 
 struct storage *
-STV_alloc(const struct sess *sp, size_t size)
+STV_alloc(struct worker *w, const struct object *obj, size_t size)
 {
 
-	return (stv_alloc(sp, size));
+	return (stv_alloc(w, obj, size));
 }
 
 void



From geoff at varnish-cache.org  Mon Jan  9 20:52:25 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:25 +0100
Subject: [experimental-ims] 9765f1a Downgrade ESI parse from using struct sess
	to struct worker.
Message-ID: <E1RkMCf-0007ud-UP@project.varnish-software.com>

commit 9765f1a846461820e8bcc628c1d1a1f843f4cf39
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Oct 24 13:41:06 2011 +0000

    Downgrade ESI parse from using struct sess to struct worker.

diff --git a/bin/varnishd/cache.h b/bin/varnishd/cache.h
index 46d4067..298f820 100644
--- a/bin/varnishd/cache.h
+++ b/bin/varnishd/cache.h
@@ -717,7 +717,7 @@ void VGZ_Ibuf(struct vgz *, const void *, ssize_t len);
 int VGZ_IbufEmpty(const struct vgz *vg);
 void VGZ_Obuf(struct vgz *, void *, ssize_t len);
 int VGZ_ObufFull(const struct vgz *vg);
-int VGZ_ObufStorage(const struct sess *sp, struct vgz *vg);
+int VGZ_ObufStorage(struct worker *w, struct vgz *vg);
 int VGZ_Gzip(struct vgz *, const void **, size_t *len, enum vgz_flag);
 int VGZ_Gunzip(struct vgz *, const void **, size_t *len);
 void VGZ_Destroy(struct vgz **);
@@ -863,6 +863,13 @@ void VSM_Free(const void *ptr);
 void VSL(enum VSL_tag_e tag, int id, const char *fmt, ...);
 void WSLR(struct worker *w, enum VSL_tag_e tag, int id, txt t);
 void WSL(struct worker *w, enum VSL_tag_e tag, int id, const char *fmt, ...);
+#define WSLB(w, tag, ...) 						\
+	do {								\
+		CHECK_OBJ_NOTNULL(w, WORKER_MAGIC);			\
+		CHECK_OBJ_NOTNULL(w->vbc, VBC_MAGIC);			\
+		WSL(w, tag, (w)->vbc->vsl_id, __VA_ARGS__);	\
+	} while (0)
+
 void WSL_Flush(struct worker *w, int overflow);
 
 #define DSL(flag, tag, id, ...)					\
diff --git a/bin/varnishd/cache_esi.h b/bin/varnishd/cache_esi.h
index 4110687..ff84d41 100644
--- a/bin/varnishd/cache_esi.h
+++ b/bin/varnishd/cache_esi.h
@@ -39,11 +39,10 @@
 #define	VEC_S8	(0x60 + 8)
 #define	VEC_INCL	'I'
 
-typedef ssize_t vep_callback_t(const struct sess *sp,
-    ssize_t l, enum vgz_flag flg);
+typedef ssize_t vep_callback_t(struct worker *w, ssize_t l, enum vgz_flag flg);
 
-void VEP_Init(const struct sess *sp, vep_callback_t *cb);
-void VEP_parse(const struct sess *sp, const char *p, size_t l);
-struct vsb *VEP_Finish(const struct sess *sp);
+void VEP_Init(struct worker *w, vep_callback_t *cb);
+void VEP_Parse(const struct worker *w, const char *p, size_t l);
+struct vsb *VEP_Finish(struct worker *w);
 
 
diff --git a/bin/varnishd/cache_esi_fetch.c b/bin/varnishd/cache_esi_fetch.c
index 00c7432..fab6513 100644
--- a/bin/varnishd/cache_esi_fetch.c
+++ b/bin/varnishd/cache_esi_fetch.c
@@ -78,7 +78,7 @@ vfp_esi_bytes_uu(struct sess *sp, struct http_conn *htc, ssize_t bytes)
 		    st->ptr + st->len, st->space - st->len, bytes);
 		if (w <= 0)
 			return (w);
-		VEP_parse(sp, (const char *)st->ptr + st->len, w);
+		VEP_Parse(sp->wrk, (const char *)st->ptr + st->len, w);
 		st->len += w;
 		sp->obj->len += w;
 		bytes -= w;
@@ -111,11 +111,11 @@ vfp_esi_bytes_gu(struct sess *sp, struct http_conn *htc, ssize_t bytes)
 			VGZ_Ibuf(vg, ibuf, w);
 			bytes -= w;
 		}
-		if (VGZ_ObufStorage(sp, vg))
+		if (VGZ_ObufStorage(sp->wrk, vg))
 			return (-1);
 		i = VGZ_Gunzip(vg, &dp, &dl);
 		xxxassert(i == VGZ_OK || i == VGZ_END);
-		VEP_parse(sp, dp, dl);
+		VEP_Parse(sp->wrk, dp, dl);
 		sp->obj->len += dl;
 	}
 	return (1);
@@ -141,15 +141,15 @@ struct vef_priv {
  */
 
 static ssize_t
-vfp_vep_callback(const struct sess *sp, ssize_t l, enum vgz_flag flg)
+vfp_vep_callback(struct worker *w, ssize_t l, enum vgz_flag flg)
 {
 	struct vef_priv *vef;
 	size_t dl, px;
 	const void *dp;
 	int i;
 
-	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
-	vef = sp->wrk->vef_priv;
+	CHECK_OBJ_NOTNULL(w, WORKER_MAGIC);
+	vef = w->vef_priv;
 	CHECK_OBJ_NOTNULL(vef, VEF_MAGIC);
 	assert(l >= 0);
 
@@ -179,14 +179,14 @@ vfp_vep_callback(const struct sess *sp, ssize_t l, enum vgz_flag flg)
 			l = 0;
 		}
 		do {
-			if (VGZ_ObufStorage(sp, vef->vgz)) {
+			if (VGZ_ObufStorage(w, vef->vgz)) {
 				vef->error = errno;
 				vef->tot += l;
 				return (vef->tot);
 			}
 			i = VGZ_Gzip(vef->vgz, &dp, &dl, flg);
 			vef->tot += dl;
-			sp->obj->len += dl;
+			w->fetch_obj->len += dl;
 		} while (!VGZ_IbufEmpty(vef->vgz) ||
 		    (flg != VGZ_NORMAL && VGZ_ObufFull(vef->vgz)));
 		if (px != 0) {
@@ -219,7 +219,7 @@ vfp_esi_bytes_ug(struct sess *sp, struct http_conn *htc, ssize_t bytes)
 			return (w);
 		bytes -= w;
 		vef->bufp = ibuf;
-		VEP_parse(sp, ibuf, w);
+		VEP_Parse(sp->wrk, ibuf, w);
 		assert(vef->bufp >= ibuf && vef->bufp <= ibuf + w);
 		if (vef->error) {
 			errno = vef->error;
@@ -271,7 +271,7 @@ vfp_esi_bytes_gg(struct sess *sp, struct http_conn *htc, size_t bytes)
 			assert(i >= VGZ_OK);
 			vef->bufp = ibuf2;
 			if (dl > 0)
-				VEP_parse(sp, ibuf2, dl);
+				VEP_Parse(sp->wrk, ibuf2, dl);
 			if (vef->error) {
 				errno = vef->error;
 				return (-1);
@@ -302,7 +302,7 @@ vfp_esi_begin(struct sess *sp, size_t estimate)
 	AZ(sp->wrk->vgz_rx);
 	if (sp->wrk->is_gzip && sp->wrk->do_gunzip) {
 		sp->wrk->vgz_rx = VGZ_NewUngzip(sp->wrk, sp->vsl_id, "U F E");
-		VEP_Init(sp, NULL);
+		VEP_Init(sp->wrk, NULL);
 	} else if (sp->wrk->is_gunzip && sp->wrk->do_gzip) {
 		ALLOC_OBJ(vef, VEF_MAGIC);
 		AN(vef);
@@ -312,7 +312,7 @@ vfp_esi_begin(struct sess *sp, size_t estimate)
 		vef->vgz = VGZ_NewGzip(sp->wrk, sp->vsl_id, "G F E");
 		AZ(sp->wrk->vef_priv);
 		sp->wrk->vef_priv = vef;
-		VEP_Init(sp, vfp_vep_callback);
+		VEP_Init(sp->wrk, vfp_vep_callback);
 	} else if (sp->wrk->is_gzip) {
 		sp->wrk->vgz_rx = VGZ_NewUngzip(sp->wrk, sp->vsl_id, "U F E");
 		ALLOC_OBJ(vef, VEF_MAGIC);
@@ -323,10 +323,10 @@ vfp_esi_begin(struct sess *sp, size_t estimate)
 		vef->vgz = VGZ_NewGzip(sp->wrk, sp->vsl_id, "G F E");
 		AZ(sp->wrk->vef_priv);
 		sp->wrk->vef_priv = vef;
-		VEP_Init(sp, vfp_vep_callback);
+		VEP_Init(sp->wrk, vfp_vep_callback);
 	} else {
 		AZ(sp->wrk->vef_priv);
-		VEP_Init(sp, NULL);
+		VEP_Init(sp->wrk, NULL);
 	}
 
 	(void)estimate;
@@ -362,7 +362,7 @@ vfp_esi_end(struct sess *sp)
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 	AN(sp->wrk->vep);
 
-	vsb = VEP_Finish(sp);
+	vsb = VEP_Finish(sp->wrk);
 
 	if (vsb != NULL) {
 		l = VSB_len(vsb);
diff --git a/bin/varnishd/cache_esi_parse.c b/bin/varnishd/cache_esi_parse.c
index 37c20e7..24fbe8b 100644
--- a/bin/varnishd/cache_esi_parse.c
+++ b/bin/varnishd/cache_esi_parse.c
@@ -35,6 +35,7 @@
 
 #include "cache.h"
 
+#include "cache_backend.h"		// XXX: w->vbc for WSLB
 #include "cache_esi.h"
 #include "vct.h"
 #include "vend.h"
@@ -60,7 +61,7 @@ struct vep_state {
 #define VEP_MAGIC		0x55cb9b82
 	struct vsb		*vsb;
 
-	const struct sess	*sp;
+	struct worker		*wrk;
 	int			dogzip;
 	vep_callback_t		*cb;
 
@@ -186,7 +187,7 @@ vep_error(const struct vep_state *vep, const char *p)
 
 	VSC_C_main->esi_errors++;
 	l = (intmax_t)(vep->ver_p - vep->hack_p);
-	WSP(vep->sp, SLT_ESI_xmlerror, "ERR at %jd %s", l, p);
+	WSLB(vep->wrk, SLT_ESI_xmlerror, "ERR at %jd %s", l, p);
 
 }
 
@@ -202,7 +203,7 @@ vep_warn(const struct vep_state *vep, const char *p)
 	VSC_C_main->esi_warnings++;
 	l = (intmax_t)(vep->ver_p - vep->hack_p);
 	printf("WARNING at %jd %s\n", l, p);
-	WSP(vep->sp, SLT_ESI_xmlerror, "WARN at %jd %s", l, p);
+	WSLB(vep->wrk, SLT_ESI_xmlerror, "WARN at %jd %s", l, p);
 
 }
 
@@ -328,7 +329,7 @@ vep_mark_common(struct vep_state *vep, const char *p, enum vep_mark mark)
 	 */
 
 	if (vep->last_mark != mark && (vep->o_wait > 0 || vep->startup)) {
-		lcb = vep->cb(vep->sp, 0,
+		lcb = vep->cb(vep->wrk, 0,
 		    mark == VERBATIM ? VGZ_RESET : VGZ_ALIGN);
 		if (lcb - vep->o_last > 0)
 			vep_emit_common(vep, lcb - vep->o_last, vep->last_mark);
@@ -338,7 +339,7 @@ vep_mark_common(struct vep_state *vep, const char *p, enum vep_mark mark)
 
 	/* Transfer pending bytes CRC into active mode CRC */
 	if (vep->o_pending) {
-		(void)vep->cb(vep->sp, vep->o_pending, VGZ_NORMAL);
+		(void)vep->cb(vep->wrk, vep->o_pending, VGZ_NORMAL);
 		if (vep->o_crc == 0) {
 			vep->crc = vep->crcp;
 			vep->o_crc = vep->o_pending;
@@ -362,7 +363,7 @@ vep_mark_common(struct vep_state *vep, const char *p, enum vep_mark mark)
 
 	vep->o_wait += l;
 	vep->last_mark = mark;
-	(void)vep->cb(vep->sp, l, VGZ_NORMAL);
+	(void)vep->cb(vep->wrk, l, VGZ_NORMAL);
 }
 
 static void
@@ -499,7 +500,7 @@ vep_do_include(struct vep_state *vep, enum dowhat what)
 		VSB_printf(vep->vsb, "%c", 0);
 	} else {
 		VSB_printf(vep->vsb, "%c", 0);
-		url = vep->sp->wrk->bereq->hd[HTTP_HDR_URL];
+		url = vep->wrk->bereq->hd[HTTP_HDR_URL];
 		/* Look for the last / before a '?' */
 		h = NULL;
 		for (q = url.b; q < url.e && *q != '?'; q++)
@@ -548,15 +549,15 @@ vep_do_include(struct vep_state *vep, enum dowhat what)
  */
 
 void
-VEP_parse(const struct sess *sp, const char *p, size_t l)
+VEP_Parse(const struct worker *w, const char *p, size_t l)
 {
 	struct vep_state *vep;
 	const char *e;
 	struct vep_match *vm;
 	int i;
 
-	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
-	vep = sp->wrk->vep;
+	CHECK_OBJ_NOTNULL(w, WORKER_MAGIC);
+	vep = w->vep;
 	CHECK_OBJ_NOTNULL(vep, VEP_MAGIC);
 	assert(l > 0);
 
@@ -601,7 +602,7 @@ VEP_parse(const struct sess *sp, const char *p, size_t l)
 				p++;
 				vep->state = VEP_STARTTAG;
 			} else if (p < e) {
-				WSP(vep->sp, SLT_ESI_xmlerror,
+				WSLB(vep->wrk, SLT_ESI_xmlerror,
 				    "No ESI processing, first char not '<'");
 				vep->state = VEP_NOTXML;
 			}
@@ -978,33 +979,34 @@ VEP_parse(const struct sess *sp, const char *p, size_t l)
 /*---------------------------------------------------------------------
  */
 
-static ssize_t
-vep_default_cb(const struct sess *sp, ssize_t l, enum vgz_flag flg)
+static ssize_t __match_proto__()
+vep_default_cb(struct worker *w, ssize_t l, enum vgz_flag flg)
 {
 
 	(void)flg;
-	AN(sp->wrk->vep);
-	sp->wrk->vep->cb_x += l;
-Debug("CB(%jd,%d) = %jd\n", (intmax_t)l, flg, (intmax_t)sp->wrk->vep->cb_x);
-	return (sp->wrk->vep->cb_x);
+	AN(w->vep);
+	w->vep->cb_x += l;
+Debug("CB(%jd,%d) = %jd\n", (intmax_t)l, flg, (intmax_t)w->vep->cb_x);
+	return (w->vep->cb_x);
 }
 
 /*---------------------------------------------------------------------
  */
 
 void
-VEP_Init(const struct sess *sp, vep_callback_t *cb)
+VEP_Init(struct worker *w, vep_callback_t *cb)
 {
 	struct vep_state *vep;
 
-	AZ(sp->wrk->vep);
-	sp->wrk->vep = (void*)WS_Alloc(sp->wrk->ws, sizeof *vep);
-	AN(sp->wrk->vep);
+	CHECK_OBJ_NOTNULL(w, WORKER_MAGIC);
+	AZ(w->vep);
+	w->vep = (void*)WS_Alloc(w->ws, sizeof *vep);
+	AN(w->vep);
 
-	vep = sp->wrk->vep;
+	vep = w->vep;
 	memset(vep, 0, sizeof *vep);
 	vep->magic = VEP_MAGIC;
-	vep->sp = sp;
+	vep->wrk = w;
 	vep->vsb = VSB_new_auto();
 	AN(vep->vsb);
 
@@ -1037,24 +1039,24 @@ VEP_Init(const struct sess *sp, vep_callback_t *cb)
  */
 
 struct vsb *
-VEP_Finish(const struct sess *sp)
+VEP_Finish(struct worker *w)
 {
 	struct vep_state *vep;
 	ssize_t l, lcb;
 
-	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
-	vep = sp->wrk->vep;
+	CHECK_OBJ_NOTNULL(w, WORKER_MAGIC);
+	vep = w->vep;
 	CHECK_OBJ_NOTNULL(vep, VEP_MAGIC);
 
 	if (vep->o_pending)
 		vep_mark_common(vep, vep->ver_p, vep->last_mark);
 	if (vep->o_wait > 0) {
-		lcb = vep->cb(vep->sp, 0, VGZ_ALIGN);
+		lcb = vep->cb(vep->wrk, 0, VGZ_ALIGN);
 		vep_emit_common(vep, lcb - vep->o_last, vep->last_mark);
 	}
-	(void)vep->cb(vep->sp, 0, VGZ_FINISH);
+	(void)vep->cb(vep->wrk, 0, VGZ_FINISH);
 
-	sp->wrk->vep = NULL;
+	w->vep = NULL;
 
 	AZ(VSB_finish(vep->vsb));
 	l = VSB_len(vep->vsb);
diff --git a/bin/varnishd/cache_gzip.c b/bin/varnishd/cache_gzip.c
index e40676c..68dc04b 100644
--- a/bin/varnishd/cache_gzip.c
+++ b/bin/varnishd/cache_gzip.c
@@ -255,11 +255,11 @@ VGZ_ObufFull(const struct vgz *vg)
  */
 
 int
-VGZ_ObufStorage(const struct sess *sp, struct vgz *vg)
+VGZ_ObufStorage(struct worker *w, struct vgz *vg)
 {
 	struct storage *st;
 
-	st = FetchStorage(sp->wrk, 0);
+	st = FetchStorage(w, 0);
 	if (st == NULL)
 		return (-1);
 
@@ -466,7 +466,7 @@ vfp_gunzip_bytes(struct sess *sp, struct http_conn *htc, ssize_t bytes)
 			bytes -= w;
 		}
 
-		if (VGZ_ObufStorage(sp, vg)) {
+		if (VGZ_ObufStorage(sp->wrk, vg)) {
 			htc->error = "Could not get storage";
 			return (-1);
 		}
@@ -541,7 +541,7 @@ vfp_gzip_bytes(struct sess *sp, struct http_conn *htc, ssize_t bytes)
 			VGZ_Ibuf(vg, ibuf, w);
 			bytes -= w;
 		}
-		if (VGZ_ObufStorage(sp, vg)) {
+		if (VGZ_ObufStorage(sp->wrk, vg)) {
 			htc->error = "Could not get storage";
 			return (-1);
 		}
@@ -567,7 +567,7 @@ vfp_gzip_end(struct sess *sp)
 	CHECK_OBJ_NOTNULL(vg, VGZ_MAGIC);
 	do {
 		VGZ_Ibuf(vg, "", 0);
-		if (VGZ_ObufStorage(sp, vg))
+		if (VGZ_ObufStorage(sp->wrk, vg))
 			return (-1);
 		i = VGZ_Gzip(vg, &dp, &dl, VGZ_FINISH);
 		sp->obj->len += dl;



From geoff at varnish-cache.org  Mon Jan  9 20:52:26 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:26 +0100
Subject: [experimental-ims] 3e4b829 Push struct session out of all the vfp's
	in one go.
Message-ID: <E1RkMCg-0007v7-4b@project.varnish-software.com>

commit 3e4b82917f6c17672dbfd192dd0bc41c04180b31
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Oct 24 14:25:52 2011 +0000

    Push struct session out of all the vfp's in one go.

diff --git a/bin/varnishd/cache.h b/bin/varnishd/cache.h
index 08ba997..85f5d5f 100644
--- a/bin/varnishd/cache.h
+++ b/bin/varnishd/cache.h
@@ -110,6 +110,7 @@ struct vef_priv;
 struct vrt_backend;
 struct vsb;
 struct waitinglist;
+struct worker;
 
 #define DIGEST_LEN		32
 
@@ -226,9 +227,9 @@ struct dstat {
 
 /* Fetch processors --------------------------------------------------*/
 
-typedef void vfp_begin_f(struct sess *, size_t );
-typedef int vfp_bytes_f(struct sess *, struct http_conn *, ssize_t);
-typedef int vfp_end_f(struct sess *sp);
+typedef void vfp_begin_f(struct worker *, size_t );
+typedef int vfp_bytes_f(struct worker *, struct http_conn *, ssize_t);
+typedef int vfp_end_f(struct worker *);
 
 struct vfp {
 	vfp_begin_f	*begin;
@@ -703,7 +704,7 @@ int EXP_NukeOne(struct worker *w, struct lru *lru);
 /* cache_fetch.c */
 struct storage *FetchStorage(struct worker *w, ssize_t sz);
 int FetchHdr(struct sess *sp);
-int FetchBody(struct sess *sp, struct object *obj);
+int FetchBody(const struct sess *sp, struct object *obj);
 int FetchReqBody(struct sess *sp);
 void Fetch_Init(void);
 
diff --git a/bin/varnishd/cache_esi_fetch.c b/bin/varnishd/cache_esi_fetch.c
index fab6513..1713004 100644
--- a/bin/varnishd/cache_esi_fetch.c
+++ b/bin/varnishd/cache_esi_fetch.c
@@ -34,6 +34,7 @@
 
 #include "cache.h"
 
+#include "cache_backend.h"		// for w->vbc
 #include "cache_esi.h"
 
 /*---------------------------------------------------------------------
@@ -62,26 +63,26 @@ vef_read(struct http_conn *htc, void *buf, ssize_t buflen, ssize_t bytes)
  * We receive a ungzip'ed object, and want to store it ungzip'ed.
  */
 
-static int __match_proto__()
-vfp_esi_bytes_uu(struct sess *sp, struct http_conn *htc, ssize_t bytes)
+static int
+vfp_esi_bytes_uu(struct worker *w, struct http_conn *htc, ssize_t bytes)
 {
-	ssize_t w;
+	ssize_t wl;
 	struct storage *st;
 
-	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+	CHECK_OBJ_NOTNULL(w, WORKER_MAGIC);
 
 	while (bytes > 0) {
-		st = FetchStorage(sp->wrk, 0);
+		st = FetchStorage(w, 0);
 		if (st == NULL)
 			return (-1);
-		w = vef_read(htc,
+		wl = vef_read(htc,
 		    st->ptr + st->len, st->space - st->len, bytes);
-		if (w <= 0)
-			return (w);
-		VEP_Parse(sp->wrk, (const char *)st->ptr + st->len, w);
-		st->len += w;
-		sp->obj->len += w;
-		bytes -= w;
+		if (wl <= 0)
+			return (wl);
+		VEP_Parse(w, (const char *)st->ptr + st->len, wl);
+		st->len += wl;
+		w->fetch_obj->len += wl;
+		bytes -= wl;
 	}
 	return (1);
 }
@@ -90,33 +91,33 @@ vfp_esi_bytes_uu(struct sess *sp, struct http_conn *htc, ssize_t bytes)
  * We receive a gzip'ed object, and want to store it ungzip'ed.
  */
 
-static int __match_proto__()
-vfp_esi_bytes_gu(struct sess *sp, struct http_conn *htc, ssize_t bytes)
+static int
+vfp_esi_bytes_gu(struct worker *w, struct http_conn *htc, ssize_t bytes)
 {
 	struct vgz *vg;
-	ssize_t w;
+	ssize_t wl;
 	uint8_t	ibuf[params->gzip_stack_buffer];
 	int i;
 	size_t dl;
 	const void *dp;
 
-	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
-	vg = sp->wrk->vgz_rx;
+	CHECK_OBJ_NOTNULL(w, WORKER_MAGIC);
+	vg = w->vgz_rx;
 
 	while (bytes > 0) {
 		if (VGZ_IbufEmpty(vg) && bytes > 0) {
-			w = vef_read(htc, ibuf, sizeof ibuf, bytes);
-			if (w <= 0)
-				return (w);
-			VGZ_Ibuf(vg, ibuf, w);
-			bytes -= w;
+			wl = vef_read(htc, ibuf, sizeof ibuf, bytes);
+			if (wl <= 0)
+				return (wl);
+			VGZ_Ibuf(vg, ibuf, wl);
+			bytes -= wl;
 		}
-		if (VGZ_ObufStorage(sp->wrk, vg))
+		if (VGZ_ObufStorage(w, vg))
 			return (-1);
 		i = VGZ_Gunzip(vg, &dp, &dl);
 		xxxassert(i == VGZ_OK || i == VGZ_END);
-		VEP_Parse(sp->wrk, dp, dl);
-		sp->obj->len += dl;
+		VEP_Parse(w, dp, dl);
+		w->fetch_obj->len += dl;
 	}
 	return (1);
 }
@@ -202,34 +203,34 @@ vfp_vep_callback(struct worker *w, ssize_t l, enum vgz_flag flg)
 	return (vef->tot);
 }
 
-static int __match_proto__()
-vfp_esi_bytes_ug(struct sess *sp, struct http_conn *htc, ssize_t bytes)
+static int
+vfp_esi_bytes_ug(const struct worker *w, struct http_conn *htc, ssize_t bytes)
 {
-	ssize_t w;
+	ssize_t wl;
 	char ibuf[params->gzip_stack_buffer];
 	struct vef_priv *vef;
 
-	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
-	vef = sp->wrk->vef_priv;
+	CHECK_OBJ_NOTNULL(w, WORKER_MAGIC);
+	vef = w->vef_priv;
 	CHECK_OBJ_NOTNULL(vef, VEF_MAGIC);
 
 	while (bytes > 0) {
-		w = vef_read(htc, ibuf, sizeof ibuf, bytes);
-		if (w <= 0)
-			return (w);
-		bytes -= w;
+		wl = vef_read(htc, ibuf, sizeof ibuf, bytes);
+		if (wl <= 0)
+			return (wl);
+		bytes -= wl;
 		vef->bufp = ibuf;
-		VEP_Parse(sp->wrk, ibuf, w);
-		assert(vef->bufp >= ibuf && vef->bufp <= ibuf + w);
+		VEP_Parse(w, ibuf, wl);
+		assert(vef->bufp >= ibuf && vef->bufp <= ibuf + wl);
 		if (vef->error) {
 			errno = vef->error;
 			return (-1);
 		}
-		if (vef->bufp < ibuf + w) {
-			w = (ibuf + w) - vef->bufp;
-			assert(w + vef->npend < sizeof vef->pending);
-			memmove(vef->pending + vef->npend, vef->bufp, w);
-			vef->npend += w;
+		if (vef->bufp < ibuf + wl) {
+			wl = (ibuf + wl) - vef->bufp;
+			assert(wl + vef->npend < sizeof vef->pending);
+			memmove(vef->pending + vef->npend, vef->bufp, wl);
+			vef->npend += wl;
 		}
 	}
 	return (1);
@@ -239,10 +240,10 @@ vfp_esi_bytes_ug(struct sess *sp, struct http_conn *htc, ssize_t bytes)
  * We receive a gzip'ed object, and want to store it gzip'ed.
  */
 
-static int __match_proto__()
-vfp_esi_bytes_gg(struct sess *sp, struct http_conn *htc, size_t bytes)
+static int 
+vfp_esi_bytes_gg(const struct worker *w, struct http_conn *htc, size_t bytes)
 {
-	ssize_t w;
+	ssize_t wl;
 	char ibuf[params->gzip_stack_buffer];
 	char ibuf2[params->gzip_stack_buffer];
 	struct vef_priv *vef;
@@ -250,28 +251,28 @@ vfp_esi_bytes_gg(struct sess *sp, struct http_conn *htc, size_t bytes)
 	const void *dp;
 	int i;
 
-	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
-	vef = sp->wrk->vef_priv;
+	CHECK_OBJ_NOTNULL(w, WORKER_MAGIC);
+	vef = w->vef_priv;
 	CHECK_OBJ_NOTNULL(vef, VEF_MAGIC);
 	assert(sizeof ibuf >= 1024);
 	ibuf2[0] = 0; /* For Flexelint */
 
 	while (bytes > 0) {
-		w = vef_read(htc, ibuf, sizeof ibuf, bytes);
-		if (w <= 0)
-			return (w);
-		bytes -= w;
+		wl = vef_read(htc, ibuf, sizeof ibuf, bytes);
+		if (wl <= 0)
+			return (wl);
+		bytes -= wl;
 
 		vef->bufp = ibuf;
-		VGZ_Ibuf(sp->wrk->vgz_rx, ibuf, w);
+		VGZ_Ibuf(w->vgz_rx, ibuf, wl);
 		do {
-			VGZ_Obuf(sp->wrk->vgz_rx, ibuf2, sizeof ibuf2);
-			i = VGZ_Gunzip(sp->wrk->vgz_rx, &dp, &dl);
+			VGZ_Obuf(w->vgz_rx, ibuf2, sizeof ibuf2);
+			i = VGZ_Gunzip(w->vgz_rx, &dp, &dl);
 			/* XXX: check i */
 			assert(i >= VGZ_OK);
 			vef->bufp = ibuf2;
 			if (dl > 0)
-				VEP_Parse(sp->wrk, ibuf2, dl);
+				VEP_Parse(w, ibuf2, dl);
 			if (vef->error) {
 				errno = vef->error;
 				return (-1);
@@ -283,7 +284,7 @@ vfp_esi_bytes_gg(struct sess *sp, struct http_conn *htc, size_t bytes)
 				    vef->bufp, dl);
 				vef->npend += dl;
 			}
-		} while (!VGZ_IbufEmpty(sp->wrk->vgz_rx));
+		} while (!VGZ_IbufEmpty(w->vgz_rx));
 	}
 	return (1);
 }
@@ -292,96 +293,96 @@ vfp_esi_bytes_gg(struct sess *sp, struct http_conn *htc, size_t bytes)
 /*---------------------------------------------------------------------*/
 
 static void __match_proto__()
-vfp_esi_begin(struct sess *sp, size_t estimate)
+vfp_esi_begin(struct worker *w, size_t estimate)
 {
 	struct vef_priv *vef;
 
-	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+	CHECK_OBJ_NOTNULL(w, WORKER_MAGIC);
 	/* XXX: snapshot WS's ? We'll need the space */
 
-	AZ(sp->wrk->vgz_rx);
-	if (sp->wrk->is_gzip && sp->wrk->do_gunzip) {
-		sp->wrk->vgz_rx = VGZ_NewUngzip(sp->wrk, sp->vsl_id, "U F E");
-		VEP_Init(sp->wrk, NULL);
-	} else if (sp->wrk->is_gunzip && sp->wrk->do_gzip) {
+	AZ(w->vgz_rx);
+	if (w->is_gzip && w->do_gunzip) {
+		w->vgz_rx = VGZ_NewUngzip(w, w->vbc->vsl_id, "U F E");
+		VEP_Init(w, NULL);
+	} else if (w->is_gunzip && w->do_gzip) {
 		ALLOC_OBJ(vef, VEF_MAGIC);
 		AN(vef);
 		//vef = (void*)WS_Alloc(sp->ws, sizeof *vef);
 		//memset(vef, 0, sizeof *vef);
 		//vef->magic = VEF_MAGIC;
-		vef->vgz = VGZ_NewGzip(sp->wrk, sp->vsl_id, "G F E");
-		AZ(sp->wrk->vef_priv);
-		sp->wrk->vef_priv = vef;
-		VEP_Init(sp->wrk, vfp_vep_callback);
-	} else if (sp->wrk->is_gzip) {
-		sp->wrk->vgz_rx = VGZ_NewUngzip(sp->wrk, sp->vsl_id, "U F E");
+		vef->vgz = VGZ_NewGzip(w, w->vbc->vsl_id, "G F E");
+		AZ(w->vef_priv);
+		w->vef_priv = vef;
+		VEP_Init(w, vfp_vep_callback);
+	} else if (w->is_gzip) {
+		w->vgz_rx = VGZ_NewUngzip(w, w->vbc->vsl_id, "U F E");
 		ALLOC_OBJ(vef, VEF_MAGIC);
 		AN(vef);
 		//vef = (void*)WS_Alloc(sp->ws, sizeof *vef);
 		//memset(vef, 0, sizeof *vef);
 		//vef->magic = VEF_MAGIC;
-		vef->vgz = VGZ_NewGzip(sp->wrk, sp->vsl_id, "G F E");
-		AZ(sp->wrk->vef_priv);
-		sp->wrk->vef_priv = vef;
-		VEP_Init(sp->wrk, vfp_vep_callback);
+		vef->vgz = VGZ_NewGzip(w, w->vbc->vsl_id, "G F E");
+		AZ(w->vef_priv);
+		w->vef_priv = vef;
+		VEP_Init(w, vfp_vep_callback);
 	} else {
-		AZ(sp->wrk->vef_priv);
-		VEP_Init(sp->wrk, NULL);
+		AZ(w->vef_priv);
+		VEP_Init(w, NULL);
 	}
 
 	(void)estimate;
-	AN(sp->wrk->vep);
+	AN(w->vep);
 }
 
 static int __match_proto__()
-vfp_esi_bytes(struct sess *sp, struct http_conn *htc, ssize_t bytes)
+vfp_esi_bytes(struct worker *w, struct http_conn *htc, ssize_t bytes)
 {
 	int i;
 
-	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
-	AN(sp->wrk->vep);
-	if (sp->wrk->is_gzip && sp->wrk->do_gunzip)
-		i = vfp_esi_bytes_gu(sp, htc, bytes);
-	else if (sp->wrk->is_gunzip && sp->wrk->do_gzip)
-		i = vfp_esi_bytes_ug(sp, htc, bytes);
-	else if (sp->wrk->is_gzip)
-		i = vfp_esi_bytes_gg(sp, htc, bytes);
+	CHECK_OBJ_NOTNULL(w, WORKER_MAGIC);
+	AN(w->vep);
+	if (w->is_gzip && w->do_gunzip)
+		i = vfp_esi_bytes_gu(w, htc, bytes);
+	else if (w->is_gunzip && w->do_gzip)
+		i = vfp_esi_bytes_ug(w, htc, bytes);
+	else if (w->is_gzip)
+		i = vfp_esi_bytes_gg(w, htc, bytes);
 	else
-		i = vfp_esi_bytes_uu(sp, htc, bytes);
-	AN(sp->wrk->vep);
+		i = vfp_esi_bytes_uu(w, htc, bytes);
+	AN(w->vep);
 	return (i);
 }
 
 static int __match_proto__()
-vfp_esi_end(struct sess *sp)
+vfp_esi_end(struct worker *w)
 {
 	struct vsb *vsb;
 	struct vef_priv *vef;
 	ssize_t l;
 
-	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
-	AN(sp->wrk->vep);
+	CHECK_OBJ_NOTNULL(w, WORKER_MAGIC);
+	AN(w->vep);
 
-	vsb = VEP_Finish(sp->wrk);
+	vsb = VEP_Finish(w);
 
 	if (vsb != NULL) {
 		l = VSB_len(vsb);
 		assert(l > 0);
 		/* XXX: This is a huge waste of storage... */
-		sp->obj->esidata = STV_alloc(sp->wrk, l);
-		XXXAN(sp->obj->esidata);
-		memcpy(sp->obj->esidata->ptr, VSB_data(vsb), l);
-		sp->obj->esidata->len = l;
+		w->fetch_obj->esidata = STV_alloc(w, l);
+		XXXAN(w->fetch_obj->esidata);
+		memcpy(w->fetch_obj->esidata->ptr, VSB_data(vsb), l);
+		w->fetch_obj->esidata->len = l;
 		VSB_delete(vsb);
 	}
-	if (sp->wrk->vgz_rx != NULL)
-		VGZ_Destroy(&sp->wrk->vgz_rx);
+	if (w->vgz_rx != NULL)
+		VGZ_Destroy(&w->vgz_rx);
 
-	if (sp->wrk->vef_priv != NULL) {
-		vef = sp->wrk->vef_priv;
+	if (w->vef_priv != NULL) {
+		vef = w->vef_priv;
 		CHECK_OBJ_NOTNULL(vef, VEF_MAGIC);
-		sp->wrk->vef_priv = NULL;
-		VGZ_UpdateObj(vef->vgz, sp->obj);
+		w->vef_priv = NULL;
+		VGZ_UpdateObj(vef->vgz, w->fetch_obj);
 		VGZ_Destroy(&vef->vgz);
 		XXXAZ(vef->error);
 		FREE_OBJ(vef);
diff --git a/bin/varnishd/cache_fetch.c b/bin/varnishd/cache_fetch.c
index 08dd99c..1769d3a 100644
--- a/bin/varnishd/cache_fetch.c
+++ b/bin/varnishd/cache_fetch.c
@@ -58,15 +58,15 @@ static unsigned fetchfrag;
  * as seen on the socket, or zero if unknown.
  */
 static void __match_proto__()
-vfp_nop_begin(struct sess *sp, size_t estimate)
+vfp_nop_begin(struct worker *w, size_t estimate)
 {
 
 	if (fetchfrag > 0) {
 		estimate = fetchfrag;
-		WSP(sp, SLT_Debug, "Fetch %d byte segments:", fetchfrag);
+		WSLB(w, SLT_Debug, "Fetch %d byte segments:", fetchfrag);
 	}
 	if (estimate > 0)
-		(void)FetchStorage(sp->wrk, estimate);
+		(void)FetchStorage(w, estimate);
 }
 
 /*--------------------------------------------------------------------
@@ -80,13 +80,13 @@ vfp_nop_begin(struct sess *sp, size_t estimate)
  */
 
 static int __match_proto__()
-vfp_nop_bytes(struct sess *sp, struct http_conn *htc, ssize_t bytes)
+vfp_nop_bytes(struct worker *w, struct http_conn *htc, ssize_t bytes)
 {
-	ssize_t l, w;
+	ssize_t l, wl;
 	struct storage *st;
 
 	while (bytes > 0) {
-		st = FetchStorage(sp->wrk, 0);
+		st = FetchStorage(w, 0);
 		if (st == NULL) {
 			htc->error = "Could not get storage";
 			return (-1);
@@ -94,14 +94,14 @@ vfp_nop_bytes(struct sess *sp, struct http_conn *htc, ssize_t bytes)
 		l = st->space - st->len;
 		if (l > bytes)
 			l = bytes;
-		w = HTC_Read(htc, st->ptr + st->len, l);
-		if (w <= 0)
-			return (w);
-		st->len += w;
-		sp->obj->len += w;
-		bytes -= w;
-		if (sp->wrk->do_stream)
-			RES_StreamPoll(sp->wrk);
+		wl = HTC_Read(htc, st->ptr + st->len, l);
+		if (wl <= 0)
+			return (wl);
+		st->len += wl;
+		w->fetch_obj->len += wl;
+		bytes -= wl;
+		if (w->do_stream)
+			RES_StreamPoll(w);
 	}
 	return (1);
 }
@@ -116,16 +116,16 @@ vfp_nop_bytes(struct sess *sp, struct http_conn *htc, ssize_t bytes)
  */
 
 static int __match_proto__()
-vfp_nop_end(struct sess *sp)
+vfp_nop_end(struct worker *w)
 {
 	struct storage *st;
 
-	st = VTAILQ_LAST(&sp->obj->store, storagehead);
+	st = VTAILQ_LAST(&w->fetch_obj->store, storagehead);
 	if (st == NULL)
 		return (0);
 
 	if (st->len == 0) {
-		VTAILQ_REMOVE(&sp->obj->store, st, list);
+		VTAILQ_REMOVE(&w->fetch_obj->store, st, list);
 		STV_free(st);
 		return (0);
 	}
@@ -198,7 +198,7 @@ fetch_number(const char *nbr, int radix)
 /*--------------------------------------------------------------------*/
 
 static int
-fetch_straight(struct sess *sp, struct http_conn *htc, const char *b)
+fetch_straight(const struct sess *sp, struct http_conn *htc, const char *b)
 {
 	int i;
 	ssize_t cl;
@@ -206,14 +206,14 @@ fetch_straight(struct sess *sp, struct http_conn *htc, const char *b)
 	assert(sp->wrk->body_status == BS_LENGTH);
 
 	cl = fetch_number(b, 10);
-	sp->wrk->vfp->begin(sp, cl > 0 ? cl : 0);
+	sp->wrk->vfp->begin(sp->wrk, cl > 0 ? cl : 0);
 	if (cl < 0) {
 		WSP(sp, SLT_FetchError, "straight length field bogus");
 		return (-1);
 	} else if (cl == 0)
 		return (0);
 
-	i = sp->wrk->vfp->bytes(sp, htc, cl);
+	i = sp->wrk->vfp->bytes(sp->wrk, htc, cl);
 	if (i <= 0) {
 		WSP(sp, SLT_FetchError, "straight read_error: %d %d (%s)",
 		    i, errno, htc->error);
@@ -235,14 +235,14 @@ fetch_straight(struct sess *sp, struct http_conn *htc, const char *b)
 	} while (0)
 
 static int
-fetch_chunked(struct sess *sp, struct http_conn *htc)
+fetch_chunked(const struct sess *sp, struct http_conn *htc)
 {
 	int i;
 	char buf[20];		/* XXX: 20 is arbitrary */
 	unsigned u;
 	ssize_t cl;
 
-	sp->wrk->vfp->begin(sp, 0);
+	sp->wrk->vfp->begin(sp->wrk, 0);
 	assert(sp->wrk->body_status == BS_CHUNKED);
 	do {
 		/* Skip leading whitespace */
@@ -283,7 +283,7 @@ fetch_chunked(struct sess *sp, struct http_conn *htc)
 			WSP(sp, SLT_FetchError,	"chunked header nbr syntax");
 			return (-1);
 		} else if (cl > 0) {
-			i = sp->wrk->vfp->bytes(sp, htc, cl);
+			i = sp->wrk->vfp->bytes(sp->wrk, htc, cl);
 			CERR();
 		}
 		i = HTC_Read(htc, buf, 1);
@@ -305,13 +305,13 @@ fetch_chunked(struct sess *sp, struct http_conn *htc)
 /*--------------------------------------------------------------------*/
 
 static int
-fetch_eof(struct sess *sp, struct http_conn *htc)
+fetch_eof(const struct sess *sp, struct http_conn *htc)
 {
 	int i;
 
 	assert(sp->wrk->body_status == BS_EOF);
-	sp->wrk->vfp->begin(sp, 0);
-	i = sp->wrk->vfp->bytes(sp, htc, SSIZE_MAX);
+	sp->wrk->vfp->begin(sp->wrk, 0);
+	i = sp->wrk->vfp->bytes(sp->wrk, htc, SSIZE_MAX);
 	if (i < 0) {
 		WSP(sp, SLT_FetchError, "eof read_error: %d (%s)",
 		    errno, htc->error);
@@ -480,7 +480,7 @@ FetchHdr(struct sess *sp)
 /*--------------------------------------------------------------------*/
 
 int
-FetchBody(struct sess *sp, struct object *obj)
+FetchBody(const struct sess *sp, struct object *obj)
 {
 	int cls;
 	struct storage *st;
@@ -518,17 +518,17 @@ FetchBody(struct sess *sp, struct object *obj)
 		cls = fetch_straight(sp, w->htc,
 		    w->h_content_length);
 		mklen = 1;
-		XXXAZ(w->vfp->end(sp));
+		XXXAZ(w->vfp->end(sp->wrk));
 		break;
 	case BS_CHUNKED:
 		cls = fetch_chunked(sp, w->htc);
 		mklen = 1;
-		XXXAZ(w->vfp->end(sp));
+		XXXAZ(w->vfp->end(sp->wrk));
 		break;
 	case BS_EOF:
 		cls = fetch_eof(sp, w->htc);
 		mklen = 1;
-		XXXAZ(w->vfp->end(sp));
+		XXXAZ(w->vfp->end(sp->wrk));
 		break;
 	case BS_ERROR:
 		cls = 1;
@@ -546,7 +546,7 @@ FetchBody(struct sess *sp, struct object *obj)
 	 * sitting on w->storage, we will always call vfp_nop_end()
 	 * to get it trimmed or thrown out if empty.
 	 */
-	AZ(vfp_nop_end(sp));
+	AZ(vfp_nop_end(sp->wrk));
 
 	w->fetch_obj = NULL;
 
diff --git a/bin/varnishd/cache_gzip.c b/bin/varnishd/cache_gzip.c
index 81c2475..541d30e 100644
--- a/bin/varnishd/cache_gzip.c
+++ b/bin/varnishd/cache_gzip.c
@@ -71,6 +71,7 @@
 
 #include "cache.h"
 
+#include "cache_backend.h"	// for w->vbc
 #include "vgz.h"
 
 struct vgz {
@@ -434,24 +435,24 @@ VGZ_Destroy(struct vgz **vgp)
  */
 
 static void __match_proto__()
-vfp_gunzip_begin(struct sess *sp, size_t estimate)
+vfp_gunzip_begin(struct worker *w, size_t estimate)
 {
 	(void)estimate;
-	AZ(sp->wrk->vgz_rx);
-	sp->wrk->vgz_rx = VGZ_NewUngzip(sp->wrk, sp->vsl_id, "U F -");
+	AZ(w->vgz_rx);
+	w->vgz_rx = VGZ_NewUngzip(w, w->vbc->vsl_id, "U F -");
 }
 
 static int __match_proto__()
-vfp_gunzip_bytes(struct sess *sp, struct http_conn *htc, ssize_t bytes)
+vfp_gunzip_bytes(struct worker *w, struct http_conn *htc, ssize_t bytes)
 {
 	struct vgz *vg;
-	ssize_t l, w;
+	ssize_t l, wl;
 	int i = -100;
 	uint8_t	ibuf[params->gzip_stack_buffer];
 	size_t dl;
 	const void *dp;
 
-	vg = sp->wrk->vgz_rx;
+	vg = w->vgz_rx;
 	CHECK_OBJ_NOTNULL(vg, VGZ_MAGIC);
 	AZ(vg->vz.avail_in);
 	while (bytes > 0 || vg->vz.avail_in > 0) {
@@ -459,37 +460,37 @@ vfp_gunzip_bytes(struct sess *sp, struct http_conn *htc, ssize_t bytes)
 			l = sizeof ibuf;
 			if (l > bytes)
 				l = bytes;
-			w = HTC_Read(htc, ibuf, l);
-			if (w <= 0)
-				return (w);
-			VGZ_Ibuf(vg, ibuf, w);
-			bytes -= w;
+			wl = HTC_Read(htc, ibuf, l);
+			if (wl <= 0)
+				return (wl);
+			VGZ_Ibuf(vg, ibuf, wl);
+			bytes -= wl;
 		}
 
-		if (VGZ_ObufStorage(sp->wrk, vg)) {
+		if (VGZ_ObufStorage(w, vg)) {
 			htc->error = "Could not get storage";
 			return (-1);
 		}
 		i = VGZ_Gunzip(vg, &dp, &dl);
 		assert(i == VGZ_OK || i == VGZ_END);
-		sp->obj->len += dl;
-		if (sp->wrk->do_stream)
-			RES_StreamPoll(sp->wrk);
+		w->fetch_obj->len += dl;
+		if (w->do_stream)
+			RES_StreamPoll(w);
 	}
 	if (i == Z_OK || i == Z_STREAM_END)
 		return (1);
 	htc->error = "See other message";
-	WSP(sp, SLT_FetchError, "Gunzip trouble (%d)", i);
+	WSLB(w, SLT_FetchError, "Gunzip trouble (%d)", i);
 	return (-1);
 }
 
 static int __match_proto__()
-vfp_gunzip_end(struct sess *sp)
+vfp_gunzip_end(struct worker *w)
 {
 	struct vgz *vg;
 
-	vg = sp->wrk->vgz_rx;
-	sp->wrk->vgz_rx = NULL;
+	vg = w->vgz_rx;
+	w->vgz_rx = NULL;
 	CHECK_OBJ_NOTNULL(vg, VGZ_MAGIC);
 	VGZ_Destroy(&vg);
 	return (0);
@@ -509,25 +510,25 @@ struct vfp vfp_gunzip = {
  */
 
 static void __match_proto__()
-vfp_gzip_begin(struct sess *sp, size_t estimate)
+vfp_gzip_begin(struct worker *w, size_t estimate)
 {
 	(void)estimate;
 
-	AZ(sp->wrk->vgz_rx);
-	sp->wrk->vgz_rx = VGZ_NewGzip(sp->wrk, sp->vsl_id, "G F -");
+	AZ(w->vgz_rx);
+	w->vgz_rx = VGZ_NewGzip(w, w->vbc->vsl_id, "G F -");
 }
 
 static int __match_proto__()
-vfp_gzip_bytes(struct sess *sp, struct http_conn *htc, ssize_t bytes)
+vfp_gzip_bytes(struct worker *w, struct http_conn *htc, ssize_t bytes)
 {
 	struct vgz *vg;
-	ssize_t l, w;
+	ssize_t l, wl;
 	int i = -100;
 	uint8_t ibuf[params->gzip_stack_buffer];
 	size_t dl;
 	const void *dp;
 
-	vg = sp->wrk->vgz_rx;
+	vg = w->vgz_rx;
 	CHECK_OBJ_NOTNULL(vg, VGZ_MAGIC);
 	AZ(vg->vz.avail_in);
 	while (bytes > 0 || !VGZ_IbufEmpty(vg)) {
@@ -535,46 +536,46 @@ vfp_gzip_bytes(struct sess *sp, struct http_conn *htc, ssize_t bytes)
 			l = sizeof ibuf;
 			if (l > bytes)
 				l = bytes;
-			w = HTC_Read(htc, ibuf, l);
-			if (w <= 0)
-				return (w);
-			VGZ_Ibuf(vg, ibuf, w);
-			bytes -= w;
+			wl = HTC_Read(htc, ibuf, l);
+			if (wl <= 0)
+				return (wl);
+			VGZ_Ibuf(vg, ibuf, wl);
+			bytes -= wl;
 		}
-		if (VGZ_ObufStorage(sp->wrk, vg)) {
+		if (VGZ_ObufStorage(w, vg)) {
 			htc->error = "Could not get storage";
 			return (-1);
 		}
 		i = VGZ_Gzip(vg, &dp, &dl, VGZ_NORMAL);
 		assert(i == Z_OK);
-		sp->obj->len += dl;
-		if (sp->wrk->do_stream)
-			RES_StreamPoll(sp->wrk);
+		w->fetch_obj->len += dl;
+		if (w->do_stream)
+			RES_StreamPoll(w);
 	}
 	return (1);
 }
 
 static int __match_proto__()
-vfp_gzip_end(struct sess *sp)
+vfp_gzip_end(struct worker *w)
 {
 	struct vgz *vg;
 	size_t dl;
 	const void *dp;
 	int i;
 
-	vg = sp->wrk->vgz_rx;
-	sp->wrk->vgz_rx = NULL;
+	vg = w->vgz_rx;
+	w->vgz_rx = NULL;
 	CHECK_OBJ_NOTNULL(vg, VGZ_MAGIC);
 	do {
 		VGZ_Ibuf(vg, "", 0);
-		if (VGZ_ObufStorage(sp->wrk, vg))
+		if (VGZ_ObufStorage(w, vg))
 			return (-1);
 		i = VGZ_Gzip(vg, &dp, &dl, VGZ_FINISH);
-		sp->obj->len += dl;
+		w->fetch_obj->len += dl;
 	} while (i != Z_STREAM_END);
-	if (sp->wrk->do_stream)
-		RES_StreamPoll(sp->wrk);
-	VGZ_UpdateObj(vg, sp->obj);
+	if (w->do_stream)
+		RES_StreamPoll(w);
+	VGZ_UpdateObj(vg, w->fetch_obj);
 	VGZ_Destroy(&vg);
 	return (0);
 }
@@ -593,29 +594,29 @@ struct vfp vfp_gzip = {
  */
 
 static void __match_proto__()
-vfp_testgzip_begin(struct sess *sp, size_t estimate)
+vfp_testgzip_begin(struct worker *w, size_t estimate)
 {
 	(void)estimate;
-	sp->wrk->vgz_rx = VGZ_NewUngzip(sp->wrk, sp->vsl_id, "u F -");
-	CHECK_OBJ_NOTNULL(sp->wrk->vgz_rx, VGZ_MAGIC);
+	w->vgz_rx = VGZ_NewUngzip(w, w->vbc->vsl_id, "u F -");
+	CHECK_OBJ_NOTNULL(w->vgz_rx, VGZ_MAGIC);
 }
 
 static int __match_proto__()
-vfp_testgzip_bytes(struct sess *sp, struct http_conn *htc, ssize_t bytes)
+vfp_testgzip_bytes(struct worker *w, struct http_conn *htc, ssize_t bytes)
 {
 	struct vgz *vg;
-	ssize_t l, w;
+	ssize_t l, wl;
 	int i = -100;
 	uint8_t	obuf[params->gzip_stack_buffer];
 	size_t dl;
 	const void *dp;
 	struct storage *st;
 
-	vg = sp->wrk->vgz_rx;
+	vg = w->vgz_rx;
 	CHECK_OBJ_NOTNULL(vg, VGZ_MAGIC);
 	AZ(vg->vz.avail_in);
 	while (bytes > 0) {
-		st = FetchStorage(sp->wrk, 0);
+		st = FetchStorage(w, 0);
 		if (st == NULL) {
 			htc->error = "Could not get storage";
 			return (-1);
@@ -623,15 +624,15 @@ vfp_testgzip_bytes(struct sess *sp, struct http_conn *htc, ssize_t bytes)
 		l = st->space - st->len;
 		if (l > bytes)
 			l = bytes;
-		w = HTC_Read(htc, st->ptr + st->len, l);
-		if (w <= 0)
-			return (w);
-		bytes -= w;
-		VGZ_Ibuf(vg, st->ptr + st->len, w);
-		st->len += w;
-		sp->obj->len += w;
-		if (sp->wrk->do_stream)
-			RES_StreamPoll(sp->wrk);
+		wl = HTC_Read(htc, st->ptr + st->len, l);
+		if (wl <= 0)
+			return (wl);
+		bytes -= wl;
+		VGZ_Ibuf(vg, st->ptr + st->len, wl);
+		st->len += wl;
+		w->fetch_obj->len += wl;
+		if (w->do_stream)
+			RES_StreamPoll(w);
 
 		while (!VGZ_IbufEmpty(vg)) {
 			VGZ_Obuf(vg, obuf, sizeof obuf);
@@ -642,7 +643,7 @@ vfp_testgzip_bytes(struct sess *sp, struct http_conn *htc, ssize_t bytes)
 			}
 			if (i != VGZ_OK && i != VGZ_END) {
 				htc->error = "See other message";
-				WSP(sp, SLT_FetchError,
+				WSLB(w, SLT_FetchError,
 				    "Invalid Gzip data: %s", vg->vz.msg);
 				return (-1);
 			}
@@ -651,19 +652,19 @@ vfp_testgzip_bytes(struct sess *sp, struct http_conn *htc, ssize_t bytes)
 	if (i == VGZ_OK || i == VGZ_END)
 		return (1);
 	htc->error = "See other message";
-	WSP(sp, SLT_FetchError, "Gunzip trouble (%d)", i);
+	WSLB(w, SLT_FetchError, "Gunzip trouble (%d)", i);
 	return (-1);
 }
 
 static int __match_proto__()
-vfp_testgzip_end(struct sess *sp)
+vfp_testgzip_end(struct worker *w)
 {
 	struct vgz *vg;
 
-	vg = sp->wrk->vgz_rx;
-	sp->wrk->vgz_rx = NULL;
+	vg = w->vgz_rx;
+	w->vgz_rx = NULL;
 	CHECK_OBJ_NOTNULL(vg, VGZ_MAGIC);
-	VGZ_UpdateObj(vg, sp->obj);
+	VGZ_UpdateObj(vg, w->fetch_obj);
 	VGZ_Destroy(&vg);
 	return (0);
 }



From geoff at varnish-cache.org  Mon Jan  9 20:52:26 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:26 +0100
Subject: [experimental-ims] f6baebc Also snapshot the worker thread workspace
	around esi:include processing.
Message-ID: <E1RkMCg-0007vl-DQ@project.varnish-software.com>

commit f6baebcc2931036353356b16662e70de0363a1fe
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Oct 25 07:52:09 2011 +0000

    Also snapshot the worker thread workspace around esi:include
    processing.
    
    Convert a few http_PrintfHeader() to http_SetHeader() for good
    measure: There is no reason to waste workspace on compiled in strings.
    
    Fixes	#1038

diff --git a/bin/varnishd/cache_center.c b/bin/varnishd/cache_center.c
index 418fbe9..91ee652 100644
--- a/bin/varnishd/cache_center.c
+++ b/bin/varnishd/cache_center.c
@@ -467,7 +467,7 @@ cnt_error(struct sess *sp)
 	http_PutStatus(h, sp->err_code);
 	VTIM_format(VTIM_real(), date);
 	http_PrintfHeader(w, sp->vsl_id, h, "Date: %s", date);
-	http_PrintfHeader(w, sp->vsl_id, h, "Server: Varnish");
+	http_SetHeader(w, sp->vsl_id, h, "Server: Varnish");
 
 	if (sp->err_reason != NULL)
 		http_PutResponse(w, sp->vsl_id, h, sp->err_reason);
@@ -727,8 +727,8 @@ cnt_fetchbody(struct sess *sp)
 
 	/* If we do gzip, add the C-E header */
 	if (sp->wrk->do_gzip)
-		http_PrintfHeader(sp->wrk, sp->vsl_id, sp->wrk->beresp,
-		    "Content-Encoding: %s", "gzip");
+		http_SetHeader(sp->wrk, sp->vsl_id, sp->wrk->beresp,
+		    "Content-Encoding: gzip");
 
 	/* But we can't do both at the same time */
 	assert(sp->wrk->do_gzip == 0 || sp->wrk->do_gunzip == 0);
@@ -1193,7 +1193,7 @@ cnt_miss(struct sess *sp)
 		 * the minority of clients which don't.
 		 */
 		http_Unset(sp->wrk->bereq, H_Accept_Encoding);
-		http_PrintfHeader(sp->wrk, sp->vsl_id, sp->wrk->bereq,
+		http_SetHeader(sp->wrk, sp->vsl_id, sp->wrk->bereq,
 		    "Accept-Encoding: gzip");
 	}
 	sp->wrk->connect_timeout = 0;
@@ -1397,7 +1397,7 @@ cnt_recv(struct sess *sp)
 	     (recv_handling != VCL_RET_PASS)) {
 		if (RFC2616_Req_Gzip(sp)) {
 			http_Unset(sp->http, H_Accept_Encoding);
-			http_PrintfHeader(sp->wrk, sp->vsl_id, sp->http,
+			http_SetHeader(sp->wrk, sp->vsl_id, sp->http,
 			    "Accept-Encoding: gzip");
 		} else {
 			http_Unset(sp->http, H_Accept_Encoding);
diff --git a/bin/varnishd/cache_esi_deliver.c b/bin/varnishd/cache_esi_deliver.c
index c79da64..c10452e 100644
--- a/bin/varnishd/cache_esi_deliver.c
+++ b/bin/varnishd/cache_esi_deliver.c
@@ -46,7 +46,8 @@ ved_include(struct sess *sp, const char *src, const char *host)
 {
 	struct object *obj;
 	struct worker *w;
-	char *ws_wm;
+	char *sp_ws_wm;
+	char *wrk_ws_wm;
 	unsigned sxid, res_mode;
 
 	w = sp->wrk;
@@ -65,7 +66,8 @@ ved_include(struct sess *sp, const char *src, const char *host)
 	HTTP_Copy(sp->http, sp->http0);
 
 	/* Take a workspace snapshot */
-	ws_wm = WS_Snapshot(sp->ws);
+	sp_ws_wm = WS_Snapshot(sp->ws);
+	wrk_ws_wm = WS_Snapshot(w->ws);
 
 	http_SetH(sp->http, HTTP_HDR_URL, src);
 	if (host != NULL && *host != '\0')  {
@@ -115,7 +117,8 @@ ved_include(struct sess *sp, const char *src, const char *host)
 	sp->wrk->res_mode = res_mode;
 
 	/* Reset the workspace */
-	WS_Reset(sp->ws, ws_wm);
+	WS_Reset(sp->ws, sp_ws_wm);
+	WS_Reset(w->ws, wrk_ws_wm);
 
 	WRW_Reserve(sp->wrk, &sp->fd);
 	if (sp->wrk->res_mode & RES_CHUNKED)
diff --git a/bin/varnishd/cache_response.c b/bin/varnishd/cache_response.c
index ecc9839..569f24f 100644
--- a/bin/varnishd/cache_response.c
+++ b/bin/varnishd/cache_response.c
@@ -125,7 +125,7 @@ RES_BuildHttp(const struct sess *sp)
 	}
 
 	if (sp->wrk->res_mode & RES_CHUNKED)
-		http_PrintfHeader(sp->wrk, sp->vsl_id, sp->wrk->resp,
+		http_SetHeader(sp->wrk, sp->vsl_id, sp->wrk->resp,
 		    "Transfer-Encoding: chunked");
 
 	VTIM_format(VTIM_real(), time_str);
diff --git a/bin/varnishtest/tests/r01038.vtc b/bin/varnishtest/tests/r01038.vtc
new file mode 100644
index 0000000..a4173b4
--- /dev/null
+++ b/bin/varnishtest/tests/r01038.vtc
@@ -0,0 +1,62 @@
+varnishtest "ticket 1038 regression test"
+
+server s1 {
+	rxreq
+	txresp -nolen -hdr "Transfer-encoding: chunked"
+	chunked {<HTML>}
+
+
+	chunked {<esi:include src="xxx0.htm"/>}
+	chunked {<esi:include src="xxx1.htm"/>}
+	chunked {<esi:include src="xxx2.htm"/>}
+	chunked {<esi:include src="xxx3.htm"/>}
+	chunked {<esi:include src="xxx4.htm"/>}
+	chunked {<esi:include src="xxx5.htm"/>}
+	chunked {<esi:include src="xxx6.htm"/>}
+	chunked {<esi:include src="xxx7.htm"/>}
+	chunked {<esi:include src="xxx8.htm"/>}
+	chunkedlen 0
+	rxreq
+	expect req.url == "/xxx0.htm"
+	txresp -body "foo0"
+	rxreq
+	expect req.url == "/xxx1.htm"
+	txresp -body "foo1"
+	rxreq
+	expect req.url == "/xxx2.htm"
+	txresp -body "foo2"
+	rxreq
+	expect req.url == "/xxx3.htm"
+	txresp -body "foo3"
+	rxreq
+	expect req.url == "/xxx4.htm"
+	txresp -body "foo4"
+	rxreq
+	expect req.url == "/xxx5.htm"
+	txresp -body "foo5"
+	rxreq
+	expect req.url == "/xxx6.htm"
+	txresp -body "foo6"
+	rxreq
+	expect req.url == "/xxx7.htm"
+	txresp -body "foo7"
+	rxreq
+	expect req.url == "/xxx8.htm"
+	txresp -body "foo8"
+} -start
+
+varnish v1 -arg "-p thread_pool_workspace=1024" -vcl+backend {
+	sub vcl_fetch {
+		set beresp.do_esi = true;
+	}
+} -start
+
+client c1 {
+	txreq
+	rxresp
+	txreq
+	rxresp
+} -run
+
+varnish v1 -expect losthdr == 0
+



From geoff at varnish-cache.org  Mon Jan  9 20:52:26 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:26 +0100
Subject: [experimental-ims] 1ff8836 Make it possible to mark http stuff
	"non-fatal".
Message-ID: <E1RkMCg-0007wB-JK@project.varnish-software.com>

commit 1ff88369bf3690020975f2250ebad4ada39d1e0d
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Oct 25 09:33:57 2011 +0000

    Make it possible to mark http stuff "non-fatal".
    
    The thread (s%d/c%d) still dies, but it does not mark this as
    fatal with respect to the test as such.
    
    This is useful where varnishd shuts the connection on a backend
    prematurely, as part of the correct execution of a test.
    
    Fix a undue recursion problem with asserts in the vtc_log code.

diff --git a/bin/varnishtest/vtc.h b/bin/varnishtest/vtc.h
index a1b6cbd..ee5308a 100644
--- a/bin/varnishtest/vtc.h
+++ b/bin/varnishtest/vtc.h
@@ -77,10 +77,10 @@ void cmd_server_genvcl(struct vsb *vsb);
 void vtc_loginit(char *buf, unsigned buflen);
 struct vtclog *vtc_logopen(const char *id);
 void vtc_logclose(struct vtclog *vl);
-void vtc_log(struct vtclog *vl, unsigned lvl, const char *fmt, ...);
-void vtc_dump(struct vtclog *vl, unsigned lvl, const char *pfx,
+void vtc_log(struct vtclog *vl, int lvl, const char *fmt, ...);
+void vtc_dump(struct vtclog *vl, int lvl, const char *pfx,
     const char *str, int len);
-void vtc_hexdump(struct vtclog *vl, unsigned lvl, const char *pfx,
+void vtc_hexdump(struct vtclog *vl, int lvl, const char *pfx,
     const unsigned char *str, int len);
 
 int exec_file(const char *fn, const char *script, const char *tmpdir,
diff --git a/bin/varnishtest/vtc_http.c b/bin/varnishtest/vtc_http.c
index 698ffb0..69485cc 100644
--- a/bin/varnishtest/vtc_http.c
+++ b/bin/varnishtest/vtc_http.c
@@ -67,6 +67,8 @@ struct http {
 
 	int			gziplevel;
 	int			gzipresidual;
+
+	int			fatal;
 };
 
 #define ONLY_CLIENT(hp, av)						\
@@ -137,7 +139,7 @@ http_write(const struct http *hp, int lvl, const char *pfx)
 	vtc_dump(hp->vl, lvl, pfx, VSB_data(hp->vsb), VSB_len(hp->vsb));
 	l = write(hp->fd, VSB_data(hp->vsb), VSB_len(hp->vsb));
 	if (l != VSB_len(hp->vsb))
-		vtc_log(hp->vl, 0, "Write failed: (%d vs %d) %s",
+		vtc_log(hp->vl, hp->fatal, "Write failed: (%d vs %d) %s",
 		    l, VSB_len(hp->vsb), strerror(errno));
 }
 
@@ -339,10 +341,12 @@ http_rxchar(struct http *hp, int n, int eof)
 		pfd[0].revents = 0;
 		i = poll(pfd, 1, hp->timeout);
 		if (i == 0)
-			vtc_log(hp->vl, 0, "HTTP rx timeout (fd:%d %u ms)",
+			vtc_log(hp->vl, hp->fatal,
+			    "HTTP rx timeout (fd:%d %u ms)",
 			    hp->fd, hp->timeout);
 		if (i < 0)
-			vtc_log(hp->vl, 0, "HTTP rx failed (fd:%d poll: %s)",
+			vtc_log(hp->vl, hp->fatal,
+			    "HTTP rx failed (fd:%d poll: %s)",
 			    hp->fd, strerror(errno));
 		assert(i > 0);
 		assert(hp->prxbuf + n < hp->nrxbuf);
@@ -354,10 +358,12 @@ http_rxchar(struct http *hp, int n, int eof)
 		if (i == 0 && eof)
 			return (i);
 		if (i == 0)
-			vtc_log(hp->vl, 0, "HTTP rx EOF (fd:%d read: %s)",
+			vtc_log(hp->vl, hp->fatal,
+			    "HTTP rx EOF (fd:%d read: %s)",
 			    hp->fd, strerror(errno));
 		if (i < 0)
-			vtc_log(hp->vl, 0, "HTTP rx failed (fd:%d read: %s)",
+			vtc_log(hp->vl, hp->fatal,
+			    "HTTP rx failed (fd:%d read: %s)",
 			    hp->fd, strerror(errno));
 		hp->prxbuf += i;
 		hp->rxbuf[hp->prxbuf] = '\0';
@@ -381,7 +387,7 @@ http_rxchunk(struct http *hp)
 	bprintf(hp->chunklen, "%d", i);
 	if ((q == hp->rxbuf + l) ||
 		(*q != '\0' && !vct_islws(*q))) {
-		vtc_log(hp->vl, 0, "chunked fail %02x @ %d",
+		vtc_log(hp->vl, hp->fatal, "chunked fail %02x @ %d",
 		    *q, q - (hp->rxbuf + l));
 	}
 	assert(q != hp->rxbuf + l);
@@ -395,11 +401,11 @@ http_rxchunk(struct http *hp)
 	l = hp->prxbuf;
 	(void)http_rxchar(hp, 2, 0);
 	if(!vct_iscrlf(hp->rxbuf[l]))
-		vtc_log(hp->vl, 0,
+		vtc_log(hp->vl, hp->fatal,
 		    "Wrong chunk tail[0] = %02x",
 		    hp->rxbuf[l] & 0xff);
 	if(!vct_iscrlf(hp->rxbuf[l + 1]))
-		vtc_log(hp->vl, 0,
+		vtc_log(hp->vl, hp->fatal,
 		    "Wrong chunk tail[1] = %02x",
 		    hp->rxbuf[l + 1] & 0xff);
 	hp->prxbuf = l;
@@ -541,7 +547,8 @@ cmd_http_gunzip_body(CMD_ARGS)
 	memset(&vz, 0, sizeof vz);
 
 	if (hp->body[0] != (char)0x1f || hp->body[1] != (char)0x8b)
-		vtc_log(hp->vl, 0, "Gunzip error: Body lacks gzip magics");
+		vtc_log(hp->vl, hp->fatal,
+		    "Gunzip error: Body lacks gzip magics");
 	vz.next_in = TRUST_ME(hp->body);
 	vz.avail_in = hp->bodyl;
 
@@ -570,7 +577,8 @@ cmd_http_gunzip_body(CMD_ARGS)
 	    (uintmax_t)vz.stop_bit,
 	    (uintmax_t)vz.stop_bit >> 3, (uintmax_t)vz.stop_bit & 7);
 	if (i != Z_STREAM_END)
-		vtc_log(hp->vl, 0, "Gunzip error = %d (%s) in:%jd out:%jd",
+		vtc_log(hp->vl, hp->fatal,
+		    "Gunzip error = %d (%s) in:%jd out:%jd",
 		    i, vz.msg, (intmax_t)vz.total_in, (intmax_t)vz.total_out);
 	assert(Z_OK == inflateEnd(&vz));
 }
@@ -602,7 +610,8 @@ gzip_body(const struct http *hp, const char *txt, char **body, int *bodylen)
 	assert(Z_STREAM_END == deflate(&vz, Z_FINISH));
 	i = vz.stop_bit & 7;
 	if (hp->gzipresidual >= 0 && hp->gzipresidual != i)
-		vtc_log(hp->vl, 0, "Wrong gzip residual got %d wanted %d",
+		vtc_log(hp->vl, hp->fatal,
+		    "Wrong gzip residual got %d wanted %d",
 		    i, hp->gzipresidual);
 	*bodylen = vz.total_out;
 	vtc_log(hp->vl, 4, "startbit = %ju %ju/%ju",
@@ -894,7 +903,7 @@ cmd_http_send(CMD_ARGS)
 	vtc_dump(hp->vl, 4, "send", av[1], -1);
 	i = write(hp->fd, av[1], strlen(av[1]));
 	if (i != strlen(av[1]))
-		vtc_log(hp->vl, 0, "Write error in http_send(): %s",
+		vtc_log(hp->vl, hp->fatal, "Write error in http_send(): %s",
 		    strerror(errno));
 }
 
@@ -1036,9 +1045,9 @@ cmd_http_expect_close(CMD_ARGS)
 		fds[0].revents = 0;
 		i = poll(fds, 1, 1000);
 		if (i == 0)
-			vtc_log(vl, 0, "Expected close: timeout");
+			vtc_log(vl, hp->fatal, "Expected close: timeout");
 		if (i != 1 || !(fds[0].revents & POLLIN))
-			vtc_log(vl, 0,
+			vtc_log(vl, hp->fatal,
 			    "Expected close: poll = %d, revents = 0x%x",
 			    i, fds[0].revents);
 		i = read(hp->fd, &c, 1);
@@ -1046,7 +1055,7 @@ cmd_http_expect_close(CMD_ARGS)
 			break;
 		if (i == 1 && vct_islws(c))
 			continue;
-		vtc_log(vl, 0,
+		vtc_log(vl, hp->fatal,
 		    "Expecting close: read = %d, c = 0x%02x", i, c);
 	}
 	vtc_log(vl, 4, "fd=%d EOF, as expected", hp->fd);
@@ -1091,7 +1100,7 @@ cmd_http_accept(CMD_ARGS)
 	vtc_log(vl, 4, "Accepting");
 	hp->fd = accept(*hp->sfd, NULL, NULL);
 	if (hp->fd < 0)
-		vtc_log(vl, 0, "Accepted failed: %s", strerror(errno));
+		vtc_log(vl, hp->fatal, "Accepted failed: %s", strerror(errno));
 	vtc_log(vl, 3, "Accepted socket fd is %d", hp->fd);
 }
 
@@ -1120,6 +1129,26 @@ cmd_http_loop(CMD_ARGS)
 }
 
 /**********************************************************************
+ * Control fatality
+ */
+
+static void
+cmd_http_fatal(CMD_ARGS)
+{
+	struct http *hp;
+	CAST_OBJ_NOTNULL(hp, priv, HTTP_MAGIC);
+
+	AZ(av[1]);
+	if (!strcmp(av[0], "fatal"))
+		hp->fatal = 0;
+	else if (!strcmp(av[0], "non-fatal"))
+		hp->fatal = -1;
+	else {
+		vtc_log(vl, 0, "XXX: fatal %s", cmd->name);
+	}
+}
+
+/**********************************************************************
  * Execute HTTP specifications
  */
 
@@ -1146,6 +1175,8 @@ static const struct cmds http_cmds[] = {
 	{ "close",		cmd_http_close },
 	{ "accept",		cmd_http_accept },
 	{ "loop",		cmd_http_loop },
+	{ "fatal",		cmd_http_fatal },
+	{ "non-fatal",		cmd_http_fatal },
 	{ NULL,			NULL }
 };
 
diff --git a/bin/varnishtest/vtc_log.c b/bin/varnishtest/vtc_log.c
index d6d15c1..313d3ab 100644
--- a/bin/varnishtest/vtc_log.c
+++ b/bin/varnishtest/vtc_log.c
@@ -47,6 +47,7 @@ struct vtclog {
 	const char	*id;
 	struct vsb	*vsb;
 	pthread_mutex_t	mtx;
+	int		act;
 };
 
 static pthread_key_t log_key;
@@ -103,10 +104,12 @@ static const char * const lead[] = {
 #define NLEAD (sizeof(lead)/sizeof(lead[0]))
 
 static void
-vtc_log_emit(const struct vtclog *vl, unsigned lvl)
+vtc_log_emit(const struct vtclog *vl, int lvl)
 {
 	int l;
 
+	if (lvl < 0)
+		lvl = 0;
 	if (vtc_stop && lvl == 0)
 		return;
 	l = VSB_len(vl->vsb);
@@ -121,16 +124,18 @@ vtc_log_emit(const struct vtclog *vl, unsigned lvl)
 
 //lint -e{818}
 void
-vtc_log(struct vtclog *vl, unsigned lvl, const char *fmt, ...)
+vtc_log(struct vtclog *vl, int lvl, const char *fmt, ...)
 {
 	double tx;
 
 	CHECK_OBJ_NOTNULL(vl, VTCLOG_MAGIC);
 	tx = VTIM_mono() - t0;
 	AZ(pthread_mutex_lock(&vl->mtx));
+	vl->act = 1;
 	assert(lvl < NLEAD);
 	VSB_clear(vl->vsb);
-	VSB_printf(vl->vsb, "%s %-4s %4.1f ", lead[lvl], vl->id, tx);
+	VSB_printf(vl->vsb, "%s %-4s %4.1f ",
+	    lead[lvl < 0 ? 1: lvl], vl->id, tx);
 	va_list ap;
 	va_start(ap, fmt);
 	(void)VSB_vprintf(vl->vsb, fmt, ap);
@@ -141,12 +146,14 @@ vtc_log(struct vtclog *vl, unsigned lvl, const char *fmt, ...)
 	vtc_log_emit(vl, lvl);
 
 	VSB_clear(vl->vsb);
+	vl->act = 0;
 	AZ(pthread_mutex_unlock(&vl->mtx));
-	if (lvl == 0) {
+	if (lvl > 0)
+		return;
+	if (lvl == 0) 
 		vtc_error = 1;
-		if (pthread_self() != vtc_thread)
-			pthread_exit(NULL);
-	}
+	if (pthread_self() != vtc_thread)
+		pthread_exit(NULL);
 }
 
 /**********************************************************************
@@ -155,7 +162,7 @@ vtc_log(struct vtclog *vl, unsigned lvl, const char *fmt, ...)
 
 //lint -e{818}
 void
-vtc_dump(struct vtclog *vl, unsigned lvl, const char *pfx, const char *str, int len)
+vtc_dump(struct vtclog *vl, int lvl, const char *pfx, const char *str, int len)
 {
 	int nl = 1;
 	unsigned l;
@@ -164,7 +171,9 @@ vtc_dump(struct vtclog *vl, unsigned lvl, const char *pfx, const char *str, int
 	CHECK_OBJ_NOTNULL(vl, VTCLOG_MAGIC);
 	tx = VTIM_mono() - t0;
 	assert(lvl < NLEAD);
+	assert(lvl >= 0);
 	AZ(pthread_mutex_lock(&vl->mtx));
+	vl->act = 1;
 	VSB_clear(vl->vsb);
 	if (pfx == NULL)
 		pfx = "";
@@ -204,6 +213,7 @@ vtc_dump(struct vtclog *vl, unsigned lvl, const char *pfx, const char *str, int
 	vtc_log_emit(vl, lvl);
 
 	VSB_clear(vl->vsb);
+	vl->act = 0;
 	AZ(pthread_mutex_unlock(&vl->mtx));
 	if (lvl == 0) {
 		vtc_error = 1;
@@ -218,7 +228,7 @@ vtc_dump(struct vtclog *vl, unsigned lvl, const char *pfx, const char *str, int
 
 //lint -e{818}
 void
-vtc_hexdump(struct vtclog *vl, unsigned lvl, const char *pfx, const unsigned char *str, int len)
+vtc_hexdump(struct vtclog *vl, int lvl, const char *pfx, const unsigned char *str, int len)
 {
 	int nl = 1;
 	unsigned l;
@@ -228,7 +238,9 @@ vtc_hexdump(struct vtclog *vl, unsigned lvl, const char *pfx, const unsigned cha
 	tx = VTIM_mono() - t0;
 	assert(len >= 0);
 	assert(lvl < NLEAD);
+	assert(lvl >= 0);
 	AZ(pthread_mutex_lock(&vl->mtx));
+	vl->act = 1;
 	VSB_clear(vl->vsb);
 	if (pfx == NULL)
 		pfx = "";
@@ -260,6 +272,7 @@ vtc_hexdump(struct vtclog *vl, unsigned lvl, const char *pfx, const unsigned cha
 	vtc_log_emit(vl, lvl);
 
 	VSB_clear(vl->vsb);
+	vl->act = 0;
 	AZ(pthread_mutex_unlock(&vl->mtx));
 	if (lvl == 0) {
 		vtc_error = 1;
@@ -279,7 +292,7 @@ vtc_log_VAS_Fail(const char *func, const char *file, int line,
 	(void)err;
 	(void)xxx;
 	vl = pthread_getspecific(log_key);
-	if (vl == NULL) {
+	if (vl == NULL || vl->act) {
 		fprintf(stderr,
 		    "Assert error in %s(), %s line %d:\n"
 		    "  Condition(%s) not true.\n",



From geoff at varnish-cache.org  Mon Jan  9 20:52:26 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:26 +0100
Subject: [experimental-ims] a2f873f Register buffer allocation failuers on
	vgz's and make failure to clean those up non-fatal.
Message-ID: <E1RkMCg-0007wW-NE@project.varnish-software.com>

commit a2f873f20f4ee3b05ada950b168a25523550c99c
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Oct 25 09:44:39 2011 +0000

    Register buffer allocation failuers on vgz's and make failure
    to clean those up non-fatal.
    
    Fixes	#1036

diff --git a/bin/varnishd/cache_gzip.c b/bin/varnishd/cache_gzip.c
index 541d30e..08d298e 100644
--- a/bin/varnishd/cache_gzip.c
+++ b/bin/varnishd/cache_gzip.c
@@ -83,6 +83,7 @@ struct vgz {
 	const char		*id;
 	struct ws		*tmp;
 	char			*tmp_snapshot;
+	const char		*error;
 
 	struct storage		*obuf;
 
@@ -261,8 +262,10 @@ VGZ_ObufStorage(struct worker *w, struct vgz *vg)
 	struct storage *st;
 
 	st = FetchStorage(w, 0);
-	if (st == NULL)
+	if (st == NULL) {
+		vg->error = "Could not get ObufStorage";
 		return (-1);
+	}
 
 	vg->obuf = st;
 	VGZ_Obuf(vg, st->ptr + st->len, st->space - st->len);
@@ -407,6 +410,7 @@ void
 VGZ_Destroy(struct vgz **vgp)
 {
 	struct vgz *vg;
+	const char *err;
 
 	vg = *vgp;
 	CHECK_OBJ_NOTNULL(vg, VGZ_MAGIC);
@@ -419,12 +423,13 @@ VGZ_Destroy(struct vgz **vgp)
 	    (intmax_t)vg->vz.start_bit,
 	    (intmax_t)vg->vz.last_bit,
 	    (intmax_t)vg->vz.stop_bit);
+	err = vg->error;
 	if (vg->tmp != NULL)
 		WS_Reset(vg->tmp, vg->tmp_snapshot);
 	if (vg->dir == VGZ_GZ)
-		AZ(deflateEnd(&vg->vz));
+		assert(deflateEnd(&vg->vz) == 0 || err != NULL);
 	else
-		AZ(inflateEnd(&vg->vz));
+		assert(inflateEnd(&vg->vz) == 0 || err != NULL);
 	FREE_OBJ(vg);
 }
 
@@ -564,18 +569,20 @@ vfp_gzip_end(struct worker *w)
 	int i;
 
 	vg = w->vgz_rx;
-	w->vgz_rx = NULL;
 	CHECK_OBJ_NOTNULL(vg, VGZ_MAGIC);
-	do {
-		VGZ_Ibuf(vg, "", 0);
-		if (VGZ_ObufStorage(w, vg))
-			return (-1);
-		i = VGZ_Gzip(vg, &dp, &dl, VGZ_FINISH);
+	w->vgz_rx = NULL;
+	if (vg->error == NULL) {
+		do {
+			VGZ_Ibuf(vg, "", 0);
+			if (VGZ_ObufStorage(w, vg))
+				return (-1);
+			i = VGZ_Gzip(vg, &dp, &dl, VGZ_FINISH);
 		w->fetch_obj->len += dl;
-	} while (i != Z_STREAM_END);
-	if (w->do_stream)
-		RES_StreamPoll(w);
-	VGZ_UpdateObj(vg, w->fetch_obj);
+		} while (i != Z_STREAM_END);
+		if (w->do_stream)
+			RES_StreamPoll(w);
+		VGZ_UpdateObj(vg, w->fetch_obj);
+	}
 	VGZ_Destroy(&vg);
 	return (0);
 }
@@ -619,6 +626,7 @@ vfp_testgzip_bytes(struct worker *w, struct http_conn *htc, ssize_t bytes)
 		st = FetchStorage(w, 0);
 		if (st == NULL) {
 			htc->error = "Could not get storage";
+			vg->error = htc->error;
 			return (-1);
 		}
 		l = st->space - st->len;
diff --git a/bin/varnishtest/tests/r01036.vtc b/bin/varnishtest/tests/r01036.vtc
new file mode 100644
index 0000000..8fb7600
--- /dev/null
+++ b/bin/varnishtest/tests/r01036.vtc
@@ -0,0 +1,22 @@
+varnishtest "Test case for #1036"
+
+server s1 {
+	rxreq
+	# Send a bodylen of 1,5M, which will exceed cache space of 1M
+	non-fatal
+	txresp -bodylen 1572864
+} -start
+
+varnish v1 -arg "-smalloc,1M" -arg "-pgzip_level=0" -vcl+backend {
+	sub vcl_fetch {
+		set beresp.do_gzip = true;
+	}
+} -start
+
+client c1 {
+	txreq
+	rxresp
+	expect resp.status == "503"
+} -run
+
+



From geoff at varnish-cache.org  Mon Jan  9 20:52:26 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:26 +0100
Subject: [experimental-ims] 15c82a6 Clean up the cache_backend.h pollution
	caused by recent changes to FetchBody.
Message-ID: <E1RkMCg-0007wl-Qj@project.varnish-software.com>

commit 15c82a60ab3962d2b8291173d4b8fb10f3ab3215
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Oct 25 20:04:42 2011 +0000

    Clean up the cache_backend.h pollution caused by recent changes to
    FetchBody.

diff --git a/bin/varnishd/cache.h b/bin/varnishd/cache.h
index b92fd60..bce7377 100644
--- a/bin/varnishd/cache.h
+++ b/bin/varnishd/cache.h
@@ -712,8 +712,8 @@ void Fetch_Init(void);
 struct vgz;
 
 enum vgz_flag { VGZ_NORMAL, VGZ_ALIGN, VGZ_RESET, VGZ_FINISH };
-struct vgz *VGZ_NewUngzip(struct worker *wrk, int vsl_id, const char *id);
-struct vgz *VGZ_NewGzip(struct worker *wrk, int vsl_id, const char *id);
+struct vgz *VGZ_NewUngzip(struct worker *wrk, const char *id);
+struct vgz *VGZ_NewGzip(struct worker *wrk, const char *id);
 void VGZ_Ibuf(struct vgz *, const void *, ssize_t len);
 int VGZ_IbufEmpty(const struct vgz *vg);
 void VGZ_Obuf(struct vgz *, void *, ssize_t len);
@@ -721,7 +721,7 @@ int VGZ_ObufFull(const struct vgz *vg);
 int VGZ_ObufStorage(struct worker *w, struct vgz *vg);
 int VGZ_Gzip(struct vgz *, const void **, size_t *len, enum vgz_flag);
 int VGZ_Gunzip(struct vgz *, const void **, size_t *len);
-void VGZ_Destroy(struct vgz **);
+void VGZ_Destroy(struct vgz **, int vsl_id);
 void VGZ_UpdateObj(const struct vgz*, struct object *);
 int VGZ_WrwGunzip(struct worker *w, struct vgz *, const void *ibuf,
     ssize_t ibufl, char *obuf, ssize_t obufl, ssize_t *obufp);
@@ -864,12 +864,7 @@ void VSM_Free(const void *ptr);
 void VSL(enum VSL_tag_e tag, int id, const char *fmt, ...);
 void WSLR(struct worker *w, enum VSL_tag_e tag, int id, txt t);
 void WSL(struct worker *w, enum VSL_tag_e tag, int id, const char *fmt, ...);
-#define WSLB(w, tag, ...) 						\
-	do {								\
-		CHECK_OBJ_NOTNULL(w, WORKER_MAGIC);			\
-		CHECK_OBJ_NOTNULL(w->vbc, VBC_MAGIC);			\
-		WSL(w, tag, (w)->vbc->vsl_id, __VA_ARGS__);	\
-	} while (0)
+void WSLB(struct worker *w, enum VSL_tag_e tag, const char *fmt, ...);
 
 void WSL_Flush(struct worker *w, int overflow);
 
diff --git a/bin/varnishd/cache_center.c b/bin/varnishd/cache_center.c
index 91ee652..fe7dae6 100644
--- a/bin/varnishd/cache_center.c
+++ b/bin/varnishd/cache_center.c
@@ -901,7 +901,7 @@ cnt_streambody(struct sess *sp)
 	sp->wrk->sctx = &sctx;
 
 	if (sp->wrk->res_mode & RES_GUNZIP) {
-		sctx.vgz = VGZ_NewUngzip(sp->wrk, sp->vsl_id, "U S -");
+		sctx.vgz = VGZ_NewUngzip(sp->wrk, "U S -");
 		sctx.obuf = obuf;
 		sctx.obuf_len = sizeof (obuf);
 	}
@@ -934,7 +934,7 @@ cnt_streambody(struct sess *sp)
 
 	RES_StreamEnd(sp);
 	if (sp->wrk->res_mode & RES_GUNZIP)
-		VGZ_Destroy(&sctx.vgz);
+		VGZ_Destroy(&sctx.vgz, sp->vsl_id);
 
 	sp->wrk->sctx = NULL;
 	assert(WRW_IsReleased(sp->wrk));
diff --git a/bin/varnishd/cache_esi_deliver.c b/bin/varnishd/cache_esi_deliver.c
index c10452e..a06bbb8 100644
--- a/bin/varnishd/cache_esi_deliver.c
+++ b/bin/varnishd/cache_esi_deliver.c
@@ -270,7 +270,7 @@ ESI_Deliver(struct sess *sp)
 	}
 
 	if (isgzip && !sp->wrk->gzip_resp) {
-		vgz = VGZ_NewUngzip(sp->wrk, sp->vsl_id, "U D E");
+		vgz = VGZ_NewUngzip(sp->wrk, "U D E");
 
 		/* Feed a gzip header to gunzip to make it happy */
 		VGZ_Ibuf(vgz, gzip_hdr, sizeof gzip_hdr);
@@ -406,7 +406,7 @@ ESI_Deliver(struct sess *sp)
 	if (vgz != NULL) {
 		if (obufl > 0)
 			(void)WRW_Write(sp->wrk, obuf, obufl);
-		VGZ_Destroy(&vgz);
+		VGZ_Destroy(&vgz, sp->vsl_id);
 	}
 	if (sp->wrk->gzip_resp && sp->esi_level == 0) {
 		/* Emit a gzip literal block with finish bit set */
diff --git a/bin/varnishd/cache_esi_fetch.c b/bin/varnishd/cache_esi_fetch.c
index 1713004..69896aa 100644
--- a/bin/varnishd/cache_esi_fetch.c
+++ b/bin/varnishd/cache_esi_fetch.c
@@ -34,7 +34,6 @@
 
 #include "cache.h"
 
-#include "cache_backend.h"		// for w->vbc
 #include "cache_esi.h"
 
 /*---------------------------------------------------------------------
@@ -302,7 +301,7 @@ vfp_esi_begin(struct worker *w, size_t estimate)
 
 	AZ(w->vgz_rx);
 	if (w->is_gzip && w->do_gunzip) {
-		w->vgz_rx = VGZ_NewUngzip(w, w->vbc->vsl_id, "U F E");
+		w->vgz_rx = VGZ_NewUngzip(w, "U F E");
 		VEP_Init(w, NULL);
 	} else if (w->is_gunzip && w->do_gzip) {
 		ALLOC_OBJ(vef, VEF_MAGIC);
@@ -310,18 +309,18 @@ vfp_esi_begin(struct worker *w, size_t estimate)
 		//vef = (void*)WS_Alloc(sp->ws, sizeof *vef);
 		//memset(vef, 0, sizeof *vef);
 		//vef->magic = VEF_MAGIC;
-		vef->vgz = VGZ_NewGzip(w, w->vbc->vsl_id, "G F E");
+		vef->vgz = VGZ_NewGzip(w, "G F E");
 		AZ(w->vef_priv);
 		w->vef_priv = vef;
 		VEP_Init(w, vfp_vep_callback);
 	} else if (w->is_gzip) {
-		w->vgz_rx = VGZ_NewUngzip(w, w->vbc->vsl_id, "U F E");
+		w->vgz_rx = VGZ_NewUngzip(w, "U F E");
 		ALLOC_OBJ(vef, VEF_MAGIC);
 		AN(vef);
 		//vef = (void*)WS_Alloc(sp->ws, sizeof *vef);
 		//memset(vef, 0, sizeof *vef);
 		//vef->magic = VEF_MAGIC;
-		vef->vgz = VGZ_NewGzip(w, w->vbc->vsl_id, "G F E");
+		vef->vgz = VGZ_NewGzip(w, "G F E");
 		AZ(w->vef_priv);
 		w->vef_priv = vef;
 		VEP_Init(w, vfp_vep_callback);
@@ -376,14 +375,14 @@ vfp_esi_end(struct worker *w)
 		VSB_delete(vsb);
 	}
 	if (w->vgz_rx != NULL)
-		VGZ_Destroy(&w->vgz_rx);
+		VGZ_Destroy(&w->vgz_rx, -1);
 
 	if (w->vef_priv != NULL) {
 		vef = w->vef_priv;
 		CHECK_OBJ_NOTNULL(vef, VEF_MAGIC);
 		w->vef_priv = NULL;
 		VGZ_UpdateObj(vef->vgz, w->fetch_obj);
-		VGZ_Destroy(&vef->vgz);
+		VGZ_Destroy(&vef->vgz,  -1);
 		XXXAZ(vef->error);
 		FREE_OBJ(vef);
 	}
diff --git a/bin/varnishd/cache_esi_parse.c b/bin/varnishd/cache_esi_parse.c
index 24fbe8b..663894a 100644
--- a/bin/varnishd/cache_esi_parse.c
+++ b/bin/varnishd/cache_esi_parse.c
@@ -35,7 +35,6 @@
 
 #include "cache.h"
 
-#include "cache_backend.h"		// XXX: w->vbc for WSLB
 #include "cache_esi.h"
 #include "vct.h"
 #include "vend.h"
diff --git a/bin/varnishd/cache_gzip.c b/bin/varnishd/cache_gzip.c
index 01ccebc..7df69cd 100644
--- a/bin/varnishd/cache_gzip.c
+++ b/bin/varnishd/cache_gzip.c
@@ -71,7 +71,6 @@
 
 #include "cache.h"
 
-#include "cache_backend.h"	// for w->vbc
 #include "vgz.h"
 
 struct vgz {
@@ -79,7 +78,6 @@ struct vgz {
 #define VGZ_MAGIC		0x162df0cb
 	enum {VGZ_GZ,VGZ_UN}	dir;
 	struct worker		*wrk;
-	int			vsl_id;
 	const char		*id;
 	struct ws		*tmp;
 	char			*tmp_snapshot;
@@ -116,7 +114,7 @@ vgz_free(voidpf opaque, voidpf address)
  */
 
 static struct vgz *
-vgz_alloc_vgz(struct worker *wrk, int vsl_id, const char *id)
+vgz_alloc_vgz(struct worker *wrk, const char *id)
 {
 	struct vgz *vg;
 	struct ws *ws;
@@ -131,7 +129,6 @@ vgz_alloc_vgz(struct worker *wrk, int vsl_id, const char *id)
 	memset(vg, 0, sizeof *vg);
 	vg->magic = VGZ_MAGIC;
 	vg->wrk = wrk;
-	vg->vsl_id = vsl_id;
 	vg->id = id;
 
 	switch (params->gzip_tmp_space) {
@@ -153,12 +150,12 @@ vgz_alloc_vgz(struct worker *wrk, int vsl_id, const char *id)
 }
 
 struct vgz *
-VGZ_NewUngzip(struct worker *wrk, int vsl_id, const char *id)
+VGZ_NewUngzip(struct worker *wrk, const char *id)
 {
 	struct vgz *vg;
 
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
-	vg = vgz_alloc_vgz(wrk, vsl_id, id);
+	vg = vgz_alloc_vgz(wrk, id);
 	vg->dir = VGZ_UN;
 	VSC_C_main->n_gunzip++;
 
@@ -173,13 +170,13 @@ VGZ_NewUngzip(struct worker *wrk, int vsl_id, const char *id)
 }
 
 struct vgz *
-VGZ_NewGzip(struct worker *wrk, int vsl_id, const char *id)
+VGZ_NewGzip(struct worker *wrk, const char *id)
 {
 	struct vgz *vg;
 	int i;
 
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
-	vg = vgz_alloc_vgz(wrk, vsl_id, id);
+	vg = vgz_alloc_vgz(wrk, id);
 	vg->dir = VGZ_GZ;
 	VSC_C_main->n_gzip++;
 
@@ -404,10 +401,12 @@ VGZ_UpdateObj(const struct vgz *vg, struct object *obj)
 	obj->gzip_stop	= vg->vz.stop_bit;
 }
 
-/*--------------------------------------------------------------------*/
+/*--------------------------------------------------------------------
+ * Passing a vsl_id of -1 means "use w->vbc->vsl_id"
+ */
 
 void
-VGZ_Destroy(struct vgz **vgp)
+VGZ_Destroy(struct vgz **vgp, int vsl_id)
 {
 	struct vgz *vg;
 	const char *err;
@@ -416,13 +415,22 @@ VGZ_Destroy(struct vgz **vgp)
 	CHECK_OBJ_NOTNULL(vg, VGZ_MAGIC);
 	*vgp = NULL;
 
-	WSL(vg->wrk, SLT_Gzip, vg->vsl_id, "%s %jd %jd %jd %jd %jd",
-	    vg->id,
-	    (intmax_t)vg->vz.total_in,
-	    (intmax_t)vg->vz.total_out,
-	    (intmax_t)vg->vz.start_bit,
-	    (intmax_t)vg->vz.last_bit,
-	    (intmax_t)vg->vz.stop_bit);
+	if (vsl_id < 0)
+		WSLB(vg->wrk, SLT_Gzip, "%s %jd %jd %jd %jd %jd",
+		    vg->id,
+		    (intmax_t)vg->vz.total_in,
+		    (intmax_t)vg->vz.total_out,
+		    (intmax_t)vg->vz.start_bit,
+		    (intmax_t)vg->vz.last_bit,
+		    (intmax_t)vg->vz.stop_bit);
+	else
+		WSL(vg->wrk, SLT_Gzip, vsl_id, "%s %jd %jd %jd %jd %jd",
+		    vg->id,
+		    (intmax_t)vg->vz.total_in,
+		    (intmax_t)vg->vz.total_out,
+		    (intmax_t)vg->vz.start_bit,
+		    (intmax_t)vg->vz.last_bit,
+		    (intmax_t)vg->vz.stop_bit);
 	err = vg->error;
 	if (vg->tmp != NULL)
 		WS_Reset(vg->tmp, vg->tmp_snapshot);
@@ -444,7 +452,7 @@ vfp_gunzip_begin(struct worker *w, size_t estimate)
 {
 	(void)estimate;
 	AZ(w->vgz_rx);
-	w->vgz_rx = VGZ_NewUngzip(w, w->vbc->vsl_id, "U F -");
+	w->vgz_rx = VGZ_NewUngzip(w, "U F -");
 }
 
 static int __match_proto__()
@@ -497,7 +505,7 @@ vfp_gunzip_end(struct worker *w)
 	vg = w->vgz_rx;
 	w->vgz_rx = NULL;
 	CHECK_OBJ_NOTNULL(vg, VGZ_MAGIC);
-	VGZ_Destroy(&vg);
+	VGZ_Destroy(&vg, -1);
 	return (0);
 }
 
@@ -520,7 +528,7 @@ vfp_gzip_begin(struct worker *w, size_t estimate)
 	(void)estimate;
 
 	AZ(w->vgz_rx);
-	w->vgz_rx = VGZ_NewGzip(w, w->vbc->vsl_id, "G F -");
+	w->vgz_rx = VGZ_NewGzip(w, "G F -");
 }
 
 static int __match_proto__()
@@ -583,7 +591,7 @@ vfp_gzip_end(struct worker *w)
 			RES_StreamPoll(w);
 		VGZ_UpdateObj(vg, w->fetch_obj);
 	}
-	VGZ_Destroy(&vg);
+	VGZ_Destroy(&vg, -1);
 	return (0);
 }
 
@@ -604,7 +612,7 @@ static void __match_proto__()
 vfp_testgzip_begin(struct worker *w, size_t estimate)
 {
 	(void)estimate;
-	w->vgz_rx = VGZ_NewUngzip(w, w->vbc->vsl_id, "u F -");
+	w->vgz_rx = VGZ_NewUngzip(w, "u F -");
 	CHECK_OBJ_NOTNULL(w->vgz_rx, VGZ_MAGIC);
 }
 
@@ -673,7 +681,7 @@ vfp_testgzip_end(struct worker *w)
 	w->vgz_rx = NULL;
 	CHECK_OBJ_NOTNULL(vg, VGZ_MAGIC);
 	VGZ_UpdateObj(vg, w->fetch_obj);
-	VGZ_Destroy(&vg);
+	VGZ_Destroy(&vg, -1);
 	return (0);
 }
 
diff --git a/bin/varnishd/cache_response.c b/bin/varnishd/cache_response.c
index 569f24f..84606b6 100644
--- a/bin/varnishd/cache_response.c
+++ b/bin/varnishd/cache_response.c
@@ -162,7 +162,7 @@ res_WriteGunzipObj(const struct sess *sp)
 
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 
-	vg = VGZ_NewUngzip(sp->wrk, sp->vsl_id, "U D -");
+	vg = VGZ_NewUngzip(sp->wrk, "U D -");
 
 	VGZ_Obuf(vg, obuf, sizeof obuf);
 	VTAILQ_FOREACH(st, &sp->obj->store, list) {
@@ -182,7 +182,7 @@ res_WriteGunzipObj(const struct sess *sp)
 		(void)WRW_Write(sp->wrk, obuf, obufl);
 		(void)WRW_Flush(sp->wrk);
 	}
-	VGZ_Destroy(&vg);
+	VGZ_Destroy(&vg, sp->vsl_id);
 	assert(u == sp->obj->len);
 }
 
diff --git a/bin/varnishd/cache_shmlog.c b/bin/varnishd/cache_shmlog.c
index da39e2d..5601463 100644
--- a/bin/varnishd/cache_shmlog.c
+++ b/bin/varnishd/cache_shmlog.c
@@ -33,6 +33,8 @@
 
 #include "cache.h"
 
+#include "cache_backend.h"	// For w->vbc
+
 #include "vapi/vsm_int.h"
 #include "vmb.h"
 #include "vtim.h"
@@ -229,16 +231,14 @@ WSLR(struct worker *w, enum VSL_tag_e tag, int id, txt t)
 
 /*--------------------------------------------------------------------*/
 
-void
-WSL(struct worker *w, enum VSL_tag_e tag, int id, const char *fmt, ...)
+static void
+wsl(struct worker *w, enum VSL_tag_e tag, int id, const char *fmt, va_list ap)
 {
-	va_list ap;
 	char *p;
 	unsigned n, mlen;
 	txt t;
 
 	AN(fmt);
-	va_start(ap, fmt);
 	mlen = params->shm_reclen;
 
 	if (strchr(fmt, '%') == NULL) {
@@ -261,7 +261,6 @@ WSL(struct worker *w, enum VSL_tag_e tag, int id, const char *fmt, ...)
 		assert(w->wlp < w->wle);
 		w->wlr++;
 	}
-	va_end(ap);
 	if (params->diag_bitmap & 0x10000)
 		WSL_Flush(w, 0);
 }
@@ -269,6 +268,36 @@ WSL(struct worker *w, enum VSL_tag_e tag, int id, const char *fmt, ...)
 /*--------------------------------------------------------------------*/
 
 void
+WSL(struct worker *w, enum VSL_tag_e tag, int id, const char *fmt, ...)
+{
+	va_list ap;
+
+	CHECK_OBJ_NOTNULL(w, WORKER_MAGIC);
+	AN(fmt);
+	va_start(ap, fmt);
+	wsl(w, tag, id, fmt, ap);
+	va_end(ap);
+}
+
+
+/*--------------------------------------------------------------------*/
+
+void
+WSLB(struct worker *w, enum VSL_tag_e tag, const char *fmt, ...)
+{
+	va_list ap;
+
+	CHECK_OBJ_NOTNULL(w, WORKER_MAGIC);
+	CHECK_OBJ_NOTNULL(w->vbc, VBC_MAGIC);
+	AN(fmt);
+	va_start(ap, fmt);
+	wsl(w, tag, w->vbc->vsl_id, fmt, ap);
+	va_end(ap);
+}
+
+/*--------------------------------------------------------------------*/
+
+void
 VSL_Init(void)
 {
 	struct VSM_chunk *vsc;



From geoff at varnish-cache.org  Mon Jan  9 20:52:27 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:27 +0100
Subject: [experimental-ims] 10c81c1 Move XML changes files into changes.rst
Message-ID: <E1RkMCh-0007xT-5M@project.varnish-software.com>

commit 10c81c11a60e13dbf6da32e0f7f8ba9c82f94499
Author: Tollef Fog Heen <tfheen at varnish-software.com>
Date:   Mon Oct 31 08:58:24 2011 +0100

    Move XML changes files into changes.rst
    
    Convert all the old changelogs to restructured text, drop xsltproc
    from the build system and adjust documentation appropriately.

diff --git a/configure.ac b/configure.ac
index 4591148..ebea42b 100644
--- a/configure.ac
+++ b/configure.ac
@@ -45,11 +45,6 @@ CC="$PTHREAD_CC"
 AC_PROG_INSTALL
 AC_PROG_LIBTOOL
 AC_PROG_MAKE_SET
-AC_CHECK_PROGS(XSLTPROC, [xsltproc], "no")
-if test "x$XSLTPROC" = "xno"; then
-   AC_MSG_WARN([xsltproc not found ? not building documentation])
-fi
-AM_CONDITIONAL(HAVE_XSLTPROC,[test "x$XSLTPROC" != "xno"])
 AC_ARG_WITH([rst2man],
                AS_HELP_STRING([--with-rst2man=PATH],
                               [Location of rst2man (auto)]),
diff --git a/doc/Makefile.am b/doc/Makefile.am
index 58d9dc7..cfd0fb5 100644
--- a/doc/Makefile.am
+++ b/doc/Makefile.am
@@ -1,64 +1,6 @@
 #
 
-CHANGELOGS = \
-	changes-2.1.5.html \
-	changes-2.1.4.html \
-	changes-2.1.3.html \
-	changes-2.1.2.html \
-	changes-2.1.1.html \
-	changes-2.1.0.html \
-	changes-2.0.6.html \
-	changes-2.0.5.html \
-	changes-2.0.4.html \
-	changes-2.0.3.html \
-	changes-2.0.2.html \
-	changes-2.0.1.html \
-	changes-2.0.html \
-	changes-1.1.2.html \
-	changes-1.1.1.html \
-	changes-1.1.html \
-	changes-1.0.4.html
-
-XML = \
-	changes-2.1.4-2.1.5.xml \
-	changes-2.1.3-2.1.4.xml \
-	changes-2.1.2-2.1.3.xml \
-	changes-2.1.1-2.1.2.xml \
-	changes-2.1.0-2.1.1.xml \
-	changes-2.0.6-2.1.0.xml \
-	changes-2.0.5-2.0.6.xml \
-	changes-2.0.4-2.0.5.xml \
-	changes-2.0.3-2.0.4.xml \
-	changes-2.0.2-2.0.3.xml \
-	changes-2.0.1-2.0.2.xml \
-	changes-2.0-2.0.1.xml \
-	changes-1.1.2-2.0.xml \
-	changes-1.1.1-1.1.2.xml \
-	changes-1.1-1.1.1.xml \
-	changes-1.0.4-1.1.xml \
-	changes-1.0.3-1.0.4.xml \
-	${CHANGELOGS:.html=.xml}
-
-if HAVE_XSLTPROC
-all: ${CHANGELOGS}
-endif
-
-EXTRA_DIST = ${CHANGELOGS} ${XML} \
-	changes.css changes-html.xsl \
-	changes.rst changes.html
-
-CLEANFILES = ${CHANGELOGS}
-SUFFIXES = .xml .html
-
-.xml.html:
-if HAVE_XSLTPROC
-	${XSLTPROC} --xinclude -o $@ $<
-else
-	@echo "========================================"
-	@echo "You need xsltproc installed to make dist"
-	@echo "========================================"
-	@false
-endif
+EXTRA_DIST = changes.rst changes.html
 
 changes.html: changes.rst
 if HAVE_RST2HTML
@@ -70,7 +12,4 @@ else
 	@false
 endif
 
-
-${CHANGELOGS}: changes-html.xsl
-
 SUBDIRS = sphinx
diff --git a/doc/changes-1.0.3-1.0.4.xml b/doc/changes-1.0.3-1.0.4.xml
deleted file mode 100644
index 8cd010e..0000000
--- a/doc/changes-1.0.3-1.0.4.xml
+++ /dev/null
@@ -1,218 +0,0 @@
-<?xml version="1.0" encoding="utf-8"?>
-<!DOCTYPE group [
-<!ENTITY mdash "&#8212;">
-]>
-<!-- $Id$ -->
-<group from="1.0.3" to="1.0.4">
-  <subsystem>
-    <name>varnishd</name>
-
-    <change type="enh,bug" ref="1277">
-      <para>The request workflow has been redesigned to simplify
-      request processing and eliminate code duplication.  All
-      codepaths which need to speak HTTP now share a single
-      implementation of the protocol.  Some new VCL hooks have been
-      added, though they aren't much use yet.  The only real
-      user-visible change should be that Varnish now handles
-      persistent backend connections correctly (see <ticket
-      ref="56"/>).</para>
-    </change>
-
-    <change type="enh" ref="1280">
-      <para>Support for multiple listen addresses has been
-      added.</para>
-    </change>
-
-    <change type="enh" ref="1281">
-      <para>An "include" facility has been added to VCL, allowing
-      VCL code to pull in code fragments from multiple files.</para>
-    </change>
-
-    <change type="enh" ref="1284">
-      <para>Multiple definitions of the same VCL function are now
-      concatenated into one in the order in which they appear in the
-      source.  This simplifies the mechanism for falling back to the
-      built-in default for cases which aren't handled in custom
-      code, and facilitates modularization.</para>
-    </change>
-
-    <change type="bug" ref="1287,1288">
-      <para>The code used to format management command arguments
-      before passing them on to the child process would
-      underestimate the amount of space needed to hold each argument
-      once quotes and special characters were properly escaped,
-      resulting in a buffer overflow.  This has been
-      corrected.</para>
-    </change>
-
-    <change type="enh,bug" ref="1289-1291,1294,1296-1303,1306">
-      <para>The VCL compiler has been overhauled.  Several memory
-      leaks have been plugged, and error detection and reporting has
-      been improved throughout.  Parts of the compiler have been
-      refactored to simplify future extension of the
-      language.</para>
-    </change>
-
-    <change type="bug" ref="1293">
-      <para>A bug in the VCL compiler which resulted in incorrect
-      parsing of the decrement (<code>-=</code>) operator has been
-      fixed.</para>
-    </change>
-
-    <change type="enh" ref="1309,1312,1381">
-      <para>A new <code>-C</code> command-line option has been added
-      which causes <code>varnishd</code> to compile the VCL code
-      (either from a file specified with <code>-f</code> or the
-      built-in default), print the resulting C code and exit.</para>
-    </change>
-
-    <change type="bug" ref="1359">
-      <para>When processing a backend response using chunked
-      encoding, if a chunk header crosses a read buffer boundary,
-      read additional bytes from the backend connection until the
-      chunk header is complete.</para>
-    </change>
-
-    <change type="enh" ref="1373">
-      <para>A new <code>ping_interval</code> run-time parameter
-      controls how often the management process checks that the
-      worker process is alive.</para>
-    </change>
-
-    <change type="bug" ref="1376">
-      <para>A bug which would cause the worker process to
-      dereference a <code>NULL</code> pointer and crash if the
-      backend did not respond has been fixed.</para>
-    </change>
-
-    <change type="bug" ref="1382">
-      <para>In some cases, such as when they are used by AJAX
-      applications to circumvent Internet Explorer's over-eager disk
-      cache, it may be desirable to cache <code>POST</code>
-      requests.  However, the code path responsible for delivering
-      objects from cache would only transmit the response body when
-      replying to a <code>GET</code> request.  This has been
-      extended to also apply to <code>POST</code>.</para>
-
-      <para>This should be revisited at a later date to allow VCL
-      code to control whether the body is delivered.</para>
-    </change>
-
-    <change type="bug" ref="1386">
-      <para>Varnish now respects <code>Cache-control:
-      s-maxage</code>, and prefers it to <code>Cache-control:
-      max-age</code> if both are present.</para>
-
-      <para>This should be revisited at a later date to allow VCL
-      code to control which headers are used and how they are
-      interpreted.</para>
-    </change>
-
-    <change type="enh" ref="1394">
-      <para>When loading a new VCL script, the management process
-      will now load the compiled object to verify that it links
-      correctly before instructing the worker process to load
-      it.</para>
-    </change>
-
-    <change type="enh" ref="1415">
-      <para>A new <code>-P</code> command-line options has been
-      added which causes <code>varnishd</code> to create a PID
-      file.</para>
-    </change>
-
-    <change type="bug" ref="1417">
-      <para>The <code>sendfile_threshold</code> run-time parameter's
-      default value has been set to infinity after a variety of
-      <code>sendfile()</code>-related bugs were discovered on
-      several platforms.</para>
-    </change>
-  </subsystem>
-
-  <subsystem>
-    <name>varnishlog</name>
-
-    <change type="bug" ref="1372,1374">
-      <para>When grouping log entries by request,
-      <code>varnishlog</code> attempts to collapse the log entry for
-      a call to a VCL function with the log entry for the
-      corresponding return from VCL.  When two VCL calls were made
-      in succession, <code>varnishlog</code> would incorrectly omit
-      the newline between the two calls (see <ticket
-      ref="95"/>).</para>
-    </change>
-
-    <change type="enh" ref="1411">
-      <para>New <code>-D</code> and <code>-P</code> command-line
-      options have been added to daemonize and create a pidfile,
-      respectively.</para>
-    </change>
-
-    <change type="bug" ref="1450">
-      <para>The flag that is raised upon reception of a
-      <code>SIGHUP</code> has been marked <code>volatile</code> so it
-      will not be optimized away by the compiler.</para>
-    </change>
-  </subsystem>
-
-  <subsystem>
-    <name>varnishncsa</name>
-
-    <change type="enh,bug" ref="1361-1363">
-      <para>The formatting callback has been largely rewritten for
-      clarity, robustness and efficiency.</para>
-
-      <para>If a request included a <code>Host:</code> header,
-      construct and output an absolute URL.  This makes
-      <code>varnishncsa</code> output from servers which handle
-      multiple virtual hosts far more useful.</para>
-    </change>
-
-    <change type="bug" ref="1450">
-      <para>The flag that is raised upon reception of a
-      <code>SIGHUP</code> has been marked <code>volatile</code> so it
-      will not be optimized away by the compiler.</para>
-    </change>
-  </subsystem>
-
-  <subsystem>
-    <name>Documentation</name>
-
-    <change type="enh">
-      <para>The documentation&mdash;especially the VCL
-      documentation&mdash;has been greatly extended and improved.</para>
-    </change>
-  </subsystem>
-
-  <subsystem>
-    <name>Build system</name>
-
-    <change type="bug" ref="1364">
-      <para>The name and location of the <code>curses</code> or
-      <code>ncurses</code> library is now correctly detected by the
-      <code>configure</code> script instead of being hardcoded into
-      affected Makefiles.  This allows Varnish to build correctly on
-      a wider range of platforms.</para>
-    </change>
-
-    <change type="enh" ref="1368">
-      <para>Compatibility shims for <code>clock_gettime()</code> are
-      now correctly applied where needed, allowing Varnish to build
-      on MacOS X.</para>
-    </change>
-
-    <change type="bug" ref="1399">
-      <para>The <code>autogen.sh</code> script will now correctly
-      detect and warn about <code>automake</code> versions which are
-      known not to work correctly.</para>
-    </change>
-  </subsystem>
-
-  <!--subsystem>
-    <name>Packaging</name>
-
-    <change type="enh">
-      <para>XXX</para>
-    </change>
-  </subsystem-->
-</group>
diff --git a/doc/changes-1.0.4-1.1.xml b/doc/changes-1.0.4-1.1.xml
deleted file mode 100644
index 7bbfc47..0000000
--- a/doc/changes-1.0.4-1.1.xml
+++ /dev/null
@@ -1,267 +0,0 @@
-<?xml version="1.0" encoding="utf-8"?>
-<!DOCTYPE group [
-<!ENTITY mdash "&#8212;">
-]>
-<!-- $Id$ -->
-<group from="1.0.4" to="1.1">
-  <subsystem>
-    <name>varnishd</name>
-
-    <change type="enh" ref="1388">
-      <para>Readability of the C source code generated from VCL code
-      has been improved.</para>
-    </change>
-
-    <change type="enh" ref="1389">
-      <para>Equality (<code>==</code>) and inequality
-      (<code>!=</code>) operators have been implemented for IP
-      addresses (which previously could only be compared using
-      ACLs).</para>
-    </change>
-
-    <change type="enh" ref="1390">
-      <para>The address of the listening socket on which the client
-      connection was received is now available to VCL as the
-      <code>server.ip</code> variable.</para>
-    </change>
-
-    <change type="enh" ref="1395,1397,1398">
-      <para>Each object's hash key is now computed based on a string
-      which is available to VCL as <code>req.hash</code>.  A VCL hook
-      named <code>vcl_hash</code> has been added to allow VCL scripts
-      to control hash generation (for instance, whether or not to
-      include the value of the <code>Host:</code> header in the
-      hash).</para>
-    </change>
-
-    <change type="bug" ref="1423,1559">
-      <para>The setup code for listening sockets has been modified to
-      detect and handle situations where a host name resolves to
-      multiple IP addresses.  It will now attempt to bind to each IP
-      address separately, and report a failure only if none of them
-      worked.</para>
-    </change>
-
-    <change type="enh" ref="1474">
-      <para>Network or protocol errors that occur while retrieving an
-      object from a backend server now result in a synthetic error
-      page being inserted into the cache with a 30-second TTL.  This
-      should help avoid driving an overburdened backend server into
-      the ground by repeatedly requesting the same object.</para>
-    </change>
-
-    <change type="enh" ref="1482">
-      <para>The child process will now drop root privileges
-      immediately upon startup.  The user and group to use are
-      specified with the <code>user</code> and <code>group</code>
-      run-time parameters, which default to <code>nobody</code> and
-      <code>nogroup</code>, respectively.  Other changes have been
-      made in an effort to increase the isolation between parent and
-      child, and reduce the impact of a compromise of the child
-      process.</para>
-    </change>
-
-    <change type="enh" ref="1506">
-      <para>Objects which are received from the backend with a
-      <code>Vary:</code> header are now stored separately according to
-      the values of the headers specified in <code>Vary:</code>.  This
-      allows Varnish to correctly cache e.g. compressed and
-      uncompressed versions of the same object.</para>
-    </change>
-
-    <change type="enh" ref="1521,1524,1525,1526,1528,1532,1533,1572,1573,1581">
-      <para>Each Varnish instance now has a name, which by default is
-      the host name of the machine it runs on, but can be any string
-      that would be valid as a relative or absolute directory name.
-      It is used to construct the name of a directory in which the
-      server state as well as all temporary files are stored.  This
-      makes it possible to run multiple Varnish instances on the same
-      machine without conflict.</para>
-    </change>
-
-    <change type="enh" ref="1549">
-      <para>When invoked with the <code>-C</code> option,
-      <code>varnishd</code> will now not just translate the VCL code
-      to C, but also compile the C code and attempt to load the
-      resulting shared object.</para>
-    </change>
-
-    <change type="enh" ref="1552,1576">
-      <para>Attempts by VCL code to reference a variable outside its
-      scope or to assign a value to a read-only variable will now
-      result in compile-time rather than run-time errors.</para>
-    </change>
-
-    <change type="enh" ref="1557,1558">
-      <para>The new command-line option <code>-F</code> will make
-      <code>varnishd</code> run in the foreground, without enabling
-      debugging.</para>
-    </change>
-
-    <change type="enh" ref="1560">
-      <para>New VCL variables have been introduced to allow inspection
-      and manipulation of the request sent to the backend
-      (<code>bereq.request</code>, <code>bereq.url</code>,
-      <code>bereq.proto</code> and <code>bereq.http</code>) and the
-      response to the client (<code>resp.proto</code>,
-      <code>resp.status</code>, <code>resp.response</code> and
-      <code>resp.http</code>).</para>
-    </change>
-
-    <change type="enh" ref="1564,1565">
-      <para>Statistics from the storage code (including the amount of
-      data and free space in the cache) are now available to
-      <code>varnishstat</code> and other statistics-gathering
-      tools.</para>
-    </change>
-
-    <change type="enh" ref="1548,1561,1567,1568,1570,1586">
-      <para>Objects are now kept on an LRU list which is kept loosely
-      up-to-date (to within a few seconds).  When cache runs out, the
-      objects at the tail end of the LRU list are discarded one by one
-      until there is enough space for the freshly requested object(s).
-      A VCL hook, <code>vcl_discard</code>, is allowed to inspect each
-      object and determine its fate by returning either
-      <code>keep</code> or <code>discard</code>.</para>
-    </change>
-
-    <change type="enh" ref="1612,1640">
-      <para>A new VCL hook, <code>vcl_deliver</code>, provides a
-      chance to adjust the response before it is sent to the
-      client.</para>
-    </change>
-
-    <change type="enh" ref="1626">
-      <para>A new management command, <code>vcl.show</code>, displays
-      the VCL source code of any loaded configuration.</para>
-    </change>
-
-    <change type="enh" ref="1628">
-      <para>A new VCL variable, <code>now</code>, provides VCL scripts
-      with the current time in seconds since the epoch.</para>
-    </change>
-
-    <change type="enh" ref="1628,1629,1630">
-      <para>A new VCL variable, <code>obj.lastuse</code>, reflects the
-      time in seconds since the object in question was last
-      used.</para>
-    </change>
-
-    <change type="enh" ref="1645,1646,1648">
-      <para>VCL scripts can now add an HTTP header (or modify the
-      value of an existing one) by assigning a value to the
-      corresponding variable, and strip an HTTP header by using the
-      <code>remove</code> keyword.</para>
-    </change>
-
-    <change type="enh" ref="1661,1662">
-      <para>VCL scripts can now modify the HTTP status code of cached
-      objects (<code>obj.status</code>) and responses
-      (<code>resp.status</code>)</para>
-    </change>
-
-    <change type="enh" ref="1663">
-      <para>Numeric and other non-textual variables in VCL can now be
-      assigned to textual variables; they will be converted as
-      needed.</para>
-    </change>
-
-    <change type="enh" ref="1666,1667">
-      <para>VCL scripts can now apply regular expression substitutions
-      to textual variables using the <code>regsub</code>
-      function.</para>
-    </change>
-
-    <change type="enh" ref="1674,1675">
-      <para>A new management command, <code>status</code>, returns the
-      state of the child.</para>
-    </change>
-
-    <change type="enh" ref="1719-1721">
-      <para>Varnish will now build and run on Mac OS X.</para>
-    </change>
-  </subsystem>
-
-  <subsystem>
-    <name>varnishadm</name>
-
-    <change type="enh" ref="1502">
-      <para>This is a new utility which sends a single command to a
-      Varnish server's management port and prints the result to
-      <code>stdout</code>, greatly simplifying the use of the
-      management port from scripts.</para>
-    </change>
-  </subsystem>
-
-  <subsystem>
-    <name>varnishhist</name>
-
-    <change type="enh" ref="1593">
-      <para>The user interface has been greatly improved; the
-      histogram will be automatically rescaled and redrawn when the
-      window size changes, and it is updated regularly rather than at
-      a rate dependent on the amount of log data gathered.  In
-      addition, the name of the Varnish instance being watched is
-      displayed in the upper right corner.</para>
-    </change>
-  </subsystem>
-
-  <subsystem>
-    <name>varnishncsa</name>
-
-    <change type="enh" ref="1479">
-      <para>In addition to client traffic, <code>varnishncsa</code>
-      can now also process log data from backend traffic.</para>
-    </change>
-
-    <change type="bug" ref="1531">
-      <para>A bug that would cause <code>varnishncsa</code> to
-      segfault when it encountered an empty HTTP header in the log
-      file has been fixed.</para>
-    </change>
-  </subsystem>
-
-  <subsystem>
-    <name>varnishreplay</name>
-
-    <change type="enh" ref="1574,1594,1595,1597,1598,1620,1623,1624,1625">
-      <para>This new utility will attempt to recreate the HTTP traffic
-      which resulted in the raw Varnish log data which it is
-      fed.</para>
-    </change>
-  </subsystem>
-
-  <subsystem>
-    <name>varnishstat</name>
-
-    <change type="enh" ref="1587">
-      <para>Don't print lifetime averages when it doesn't make any
-      sense&mdash;for instance, there is no point in dividing the
-      amount in bytes of free cache space by the lifetime in seconds
-      of the <code>varnishd</code> process.</para>
-    </change>
-
-    <change type="enh" ref="1589,1590,1591">
-      <para>The user interface has been greatly improved;
-      <code>varnishstat</code> will no longer print more than fits in
-      the terminal, and will respond correctly to window resize
-      events.  The output produced in one-shot mode has been modified
-      to include symbolic names for each entry.  In addition, the name
-      of the Varnish instance being watched is displayed in the upper
-      right corner in curses mode.</para>
-    </change>
-  </subsystem>
-
-  <subsystem>
-    <name>varnishtop</name>
-
-    <change type="enh" ref="1592,1596">
-      <para>The user interface has been greatly improved;
-      <code>varnishtop</code> will now respond correctly to window
-      resize events, and one-shot mode (<code>-1</code>) actually
-      works.  In addition, the name of the Varnish instance being
-      watched is displayed in the upper right corner in curses
-      mode.</para>
-    </change>
-  </subsystem>
-</group>
diff --git a/doc/changes-1.0.4.xml b/doc/changes-1.0.4.xml
deleted file mode 100644
index 739b1bd..0000000
--- a/doc/changes-1.0.4.xml
+++ /dev/null
@@ -1,16 +0,0 @@
-<?xml version="1.0" encoding="utf-8"?>
-<?xml-stylesheet type="text/xml" href="changes-html.xsl"?>
-<!DOCTYPE changelog [
- <!ENTITY mdash "&#8212;">
-]>
-<!-- $Id$ -->
-<changelog xmlns:xi="http://www.w3.org/2001/XInclude">
-  <package>Varnish</package>
-  <version>1.0.4</version>
-
-  <xi:include href="changes-1.0.3-1.0.4.xml"/>
-  <!--xi:include href="changes-1.0.2-1.0.3.xml"/-->
-  <!--xi:include href="changes-1.0.1-1.0.2.xml"/-->
-  <!--xi:include href="changes-1.0-1.0.1.xml"/-->
-  <!--xi:include href="changes-0.9-1.0.xml"/-->
-</changelog>
diff --git a/doc/changes-1.1-1.1.1.xml b/doc/changes-1.1-1.1.1.xml
deleted file mode 100644
index 78d6648..0000000
--- a/doc/changes-1.1-1.1.1.xml
+++ /dev/null
@@ -1,136 +0,0 @@
-<?xml version="1.0" encoding="utf-8"?>
-<!DOCTYPE group [
-<!ENTITY mdash "&#8212;">
-]>
-<!-- $Id$ -->
-<group from="1.1" to="1.1.1">
-  <subsystem>
-    <name>varnishd</name>
-
-    <change type="bug" ref="1745">
-      <para>The code required to allow VCL to read
-      <code>obj.status</code>, which had accidentally been left out,
-      has now been added.</para>
-    </change>
-
-    <change type="bug" ref="1750">
-      <para>Varnish will now always include a <code>Connection:</code>
-      header in its reply to the client, to avoid possible
-      misunderstandings.</para>
-    </change>
-
-    <change type="buf" ref="1751">
-      <para>A bug that triggered an assertion failure when generating
-      synthetic error documents has been corrected.</para>
-    </change>
-
-    <change type="enh" ref="1761,1762,1765,1828">
-      <para>A new VCL function, <code>purge_url</code>, provides the
-      same functionality as the <code>url.purge</code> management
-      command.</para>
-    </change>
-
-    <change type="enh" ref="1780">
-      <para>Previously, Varnish assumed that the response body should
-      be sent only if the request method was <code>GET</code>.  This
-      was a problem for custom request methods (such as
-      <code>PURGE</code>), so the logic has been changed to always
-      send the response body except in the specific case of a
-      <code>HEAD</code> request.</para>
-    </change>
-
-    <change type="bug" ref="1795">
-      <para>Changes to run-time parameters are now correctly
-      propagated to the child process.</para>
-    </change>
-
-    <change type="bug" ref="1797">
-      <para>Due to the way run-time parameters are initialized at
-      startup, <code>varnishd</code> previously required the
-      <code>nobody</code> user and the <code>nogroup</code> group to
-      exist even if a different user and group were specified on the
-      command line.  This has been corrected.</para>
-    </change>
-
-    <change type="bug" ref="1800">
-      <para>Under certain conditions, the VCL compiler would carry on
-      after a syntax error instead of exiting after reporting the
-      error.  This has been corrected.</para>
-    </change>
-
-    <change type="enh" ref="1803-1806">
-      <para>The manner in which the hash string is assembled has been
-      modified to reduce memory usage and memory-to-memory
-      copying.</para>
-    </change>
-
-    <change type="bug" ref="1807">
-      <para>Before calling <code>vcl_miss</code>, Varnish assembles a
-      tentative request object for the backend request which will
-      usually follow.  This object would be leaked if
-      <code>vcl_miss</code> returned anything else than
-      <code>fetch</code>.  This has been corrected.</para>
-    </change>
-
-    <change type="bug" ref="1808">
-      <para>The code necessary to handle an <code>error</code> return
-      from <code>vcl_fetch</code> and <code>vcl_deliver</code> had
-      inadvertantly been left out.  This has been corrected.</para>
-    </change>
-
-    <change type="bug" ref="1810">
-      <para>Varnish no longer prints a spurious "child died" message
-      (the result of reaping the compiler process) after compiling a
-      new VCL configuration.</para>
-    </change>
-
-    <change type="bug" ref="1838">
-      <para>Under some circumstances, due to an error in the workspace
-      management code, Varnish would lose the "tail" of a request,
-      i.e. the part of the request that has been received from the
-      client but not yet processed.  The most obvious symptom of this
-      was that POST requests would work with some browsers but not
-      others, depending on details of the browser's HTTP
-      implementation.  This has been corrected.</para>
-    </change>
-
-    <change type="bug" ref="1855,1859">
-      <para>On some platforms, due to incorrect assumptions in the CLI
-      code, the management process would crash while processing
-      commands received over the management port.  This has been
-      corrected.</para>
-    </change>
-  </subsystem>
-
-  <subsystem>
-    <name>Build system</name>
-
-    <change type="bug" ref="1752">
-      <para>The top-level Makefile will now honor
-      <code>$DESTDIR</code> when creating the state directory.</para>
-    </change>
-
-    <change type="enh" ref="1743,1846">
-      <para>The Debian and RedHat packages are now split into three
-      (main / lib / devel) as is customary.</para>
-    </change>
-
-    <change type="enh" ref="1753,1754,1783-1791,1798,1823,1794">
-      <para>A number of compile-time and run-time portability issues
-      have been addressed.</para>
-    </change>
-
-    <change type="enh" ref="1811">
-      <para>The <code>autogen.sh</code> script had workarounds for
-      problems with the GNU autotools on FreeBSD; these are no longer
-      needed and have been removed.</para>
-    </change>
-
-    <change type="enh" ref="1817">
-      <para>The <code>libcompat</code> library has been renamed to
-      <code>libvarnishcompat</code> and is now dynamic rather than
-      static.  This simplifies the build process and resolves an issue
-      with the Mac OS X linker.</para>
-    </change>
-  </subsystem>
-</group>
diff --git a/doc/changes-1.1.1-1.1.2.xml b/doc/changes-1.1.1-1.1.2.xml
deleted file mode 100644
index 5f59902..0000000
--- a/doc/changes-1.1.1-1.1.2.xml
+++ /dev/null
@@ -1,157 +0,0 @@
-<?xml version="1.0" encoding="utf-8"?>
-<!DOCTYPE group [
-<!ENTITY mdash "&#8212;">
-]>
-<!-- $Id$ -->
-<group from="1.1.1" to="1.1.2">
-  <subsystem>
-    <name>varnishd</name>
-
-    <change type="bug" ref="1809,1913">
-      <para>When switching to a new VCL configuration, a race
-      condition exists which may cause Varnish to reference a backend
-      which no longer exists (see <ticket ref="144"/>).  This race
-      condition has not been entirely eliminated, but it should occur
-      less frequently.</para>
-    </change>
-
-    <change type="bug" ref="1942">
-      <para>When dropping a TCP session before any requests were
-      processed, an assertion would be triggered due to an
-      uninitialized timestamp (see <ticket ref="132"/>).  The
-      timestamp is now correctly initialized.</para>
-    </change>
-
-    <change type="bug" ref="1955,1976,1977">
-      <para>Varnish will now correctly generate a <code>Date:</code>
-      header for every response instead of copying the one it got from
-      the backend (see <ticket ref="157"/>).</para>
-    </change>
-
-    <change type="bug" ref="1971">
-      <para>Comparisons in VCL which involve a non-existent string
-      (usually a header which is not present in the request or object
-      being processed) would cause a NULL pointer dereference; now the
-      comparison will simply fail.</para>
-    </change>
-
-    <change type="bug" ref="1972">
-      <para>A bug in the VCL compiler which would cause a double-free
-      when processing <code>include</code> directives has been
-      fixed.</para>
-    </change>
-
-    <change type="bug" ref="1991">
-      <para>A resource leak in the worker thread management code has
-      been fixed.</para>
-    </change>
-
-    <change type="bug" ref="1809">
-      <para>When connecting to a backend, Varnish will usually get the
-      address from a cache.  When the cache is refreshed, existing
-      connections may end up with a reference to an address structure
-      which no longer exists, resulting in a crash.  This race
-      condition has been somewhat mitigated, but not entirely
-      eliminated (see <ticket ref="144"/>.)</para>
-    </change>
-
-    <change type="bug" ref="1888">
-      <para>Varnish will now pass the correct protocol version in pipe
-      mode: the backend will get what the client sent, and vice
-      versa.</para>
-    </change>
-
-    <change type="bug" ref="2057,2077,2080,2086">
-      <para>The core of the pipe mode code has been rewritten to
-      increase robustness and eliminate spurious error messages when
-      either end closes the connection in a manner Varnish did not
-      anticipate.</para>
-    </change>
-
-    <change type="bug" ref="2181">
-      <para>A memory leak in the backend code has been plugged.</para>
-    </change>
-
-    <change type="bug" ref="2232">
-      <para>When using the <code>kqueue</code> acceptor, if a client
-      shuts down the request side of the connection (as many clients
-      do after sending their final request), it was possible for the
-      acceptor code to receive the <code>EOF</code> event and recycle
-      the session while the last request was still being serviced,
-      resulting in a assertion failure and a crash when the worker
-      thread later tried to delete the session.  This should no longer
-      happen (see <ticket ref="162"/>.)</para>
-    </change>
-
-    <change type="bug" ref="2275">
-      <para>A mismatch between the recorded length of a cached object
-      and the amount of data actually present in cache for that object
-      can occasionally occur (see <ticket ref="167"/>.)  This has been
-      partially fixed, but may still occur for error pages generated
-      by Varnish when a problem arises while retrieving an object from
-      the backend.</para>
-    </change>
-
-    <change type="bug" ref="2285,2286">
-      <para>Some socket-related system calls may return unexpected
-      error codes when operating on a TCP connection that has been
-      shut down at the other end.  These error codes would previously
-      cause assertion failures, but are now recognized as harmless
-      conditions.</para>
-    </change>
-  </subsystem>
-
-  <subsystem>
-    <name>varnishhist</name>
-
-    <change type="enh">
-      <para>Pressing <code>0</code> though <code>9</code> while
-      <code>varnishhist</code> is running will change the refresh
-      interval to the corresponding power of two, in seconds.</para>
-    </change>
-  </subsystem>
-
-  <subsystem>
-    <name>varnishncsa</name>
-
-    <change type="enh">
-      <para>The <code>varnishncsa</code> tool can now daemonize and
-      write a PID file like <code>varnishlog</code>, using the same
-      command-line options.  It will also reopen its output upon receipt
-      of a <code>SIGHUP</code> if invoked with <code>-w</code>.</para>
-    </change>
-  </subsystem>
-
-  <subsystem>
-    <name>varnishstat</name>
-
-    <change type="enh">
-      <para>Pressing <code>0</code> though <code>9</code> while
-      <code>varnishstat</code> is running will change the refresh
-      interval to the corresponding power of two, in seconds.</para>
-    </change>
-  </subsystem>
-
-  <subsystem>
-    <name>Build system</name>
-
-    <change type="enh" ref="2033">
-      <para>Varnish's <code>&lt;queue.h&gt;</code> has been modified
-      to avoid conflicts with <code>&lt;sys/queue.h&gt;</code> on
-      platforms where the latter is included indirectly through system
-      headers.</para>
-    </change>
-
-    <change type="enh" ref="2032,2133,2097,2106,2222-2228">
-      <para>Several steps have been taken towards Solaris
-      support, but this is not yet complete.</para>
-    </change>
-
-    <change type="bug" ref="2116,2154">
-      <para>When <code>configure</code> was run without an explicit
-      prefix, Varnish's idea of the default state directory would be
-      garbage and a state directory would have to be specified
-      manually with <code>-n</code>.  This has been corrected.</para>
-    </change>
-  </subsystem>
-</group>
diff --git a/doc/changes-1.1.1.xml b/doc/changes-1.1.1.xml
deleted file mode 100644
index 784249e..0000000
--- a/doc/changes-1.1.1.xml
+++ /dev/null
@@ -1,12 +0,0 @@
-<?xml version="1.0" encoding="utf-8"?>
-<?xml-stylesheet type="text/xml" href="changes-html.xsl"?>
-<!DOCTYPE changelog [
- <!ENTITY mdash "&#8212;">
-]>
-<!-- $Id$ -->
-<changelog xmlns:xi="http://www.w3.org/2001/XInclude">
-  <package>Varnish</package>
-  <version>1.1.1</version>
-
-  <xi:include href="changes-1.1-1.1.1.xml"/>
-</changelog>
diff --git a/doc/changes-1.1.2-2.0.xml b/doc/changes-1.1.2-2.0.xml
deleted file mode 100644
index 1b56acb..0000000
--- a/doc/changes-1.1.2-2.0.xml
+++ /dev/null
@@ -1,199 +0,0 @@
-<?xml version="1.0" encoding="utf-8"?>
-<!DOCTYPE group [
-<!ENTITY mdash "&#8212;">
-]>
-<!-- $Id$ -->
-<group from="1.1.2" to="2.0">
-  <subsystem>
-    <name>varnishd</name>
-
-    <change type="bug">
-      <para>Only look for sendfile on platforms where we know how to
-      use it, which is FreeBSD for now.</para>
-    </change>
-
-    <change type="bug">
-      <para>Make it possible to adjust the shared memory log size and
-      bump the size from 8MB to 80MB.</para>
-    </change>
-
-    <change type="enh">
-      <para>Fix up the handling of request bodies to better match
-      what RFC2616 mandates.  This makes PUT, DELETE, OPTIONS and
-      TRACE work in addition to POST. </para>
-    </change>
-
-    <change type="enh">
-      <para>Change how backends are defined, to a constant structural
-      defintion style.  See
-      http://varnish.projects.linpro.no/wiki/VclSyntaxChanges for the
-      details.</para>
-    </change>
-
-    <change type="enh">
-      <para>Add directors, which wrap backends.  Currently, there's a
-	random director and a round-robin director.</para>
-    </change>
-
-    <change type="enh">
-      <para>Add "grace", which is for how long and object will be
-	served, even after it has expired.  To use this, both the
-	object's and the request's <code>grace</code> parameter need
-	to be set.</para>
-    </change>
-
-    <change type="enh">
-      <para>Manual pages have been updated for new VCL syntax and
-      varnishd options.</para>
-    </change>
-
-    <change type="enh">
-      <para>Man pages and other docs have been updated.</para>
-    </change>
-
-    <change type="enh">
-      <para>The shared memory log file is now locked in memory, so it
-      should not be paged out to disk.</para>
-    </change>
-
-    <change type="enh">
-      <para>We now handle Vary correctly, as well as Expect.</para>
-    </change>
-
-    <change type="enh">
-      <para>ESI include support is implemented.</para>
-    </change>
-
-    <change type="enh">
-      <para>Make it possible to limit how much memory the malloc uses.</para>
-    </change>
-
-    <change type="enh">
-      <para>Solaris is now supported.</para>
-    </change>
-
-    <change type="enh">
-      <para>There is now a <code>regsuball</code> function, which
-      works like <code>regsub</code> except it replaces all
-      occurences of the regex, not just the first.</para>
-    </change>
-
-    <change type="enh">
-      <para>Backend and director declarations can have
-      a <code>.connect_timeout</code> parameter, which tells us how
-      long to wait for a successful connection.</para>
-    </change>
-
-    <change type="enh">
-      <para>It is now possible to select the acceptor to use by
-      changing the <code>acceptor</code> parameter.</para>
-    </change>
-
-    <change type="enh">
-      <para>Backends can have probes associated with them, which can
-      be checked with <code>req.backend.health</code> in VCL as well as
-      being handled by directors which do load-balancing.</para>
-    </change>
-
-    <change type="enh">
-      <para>Support larger-than-2GB files also on 32 bit hosts.
-      Please note that this does not mean we can support caches
-      bigger than 2GB, it just means logfiles and similar can be
-      bigger.</para>
-    </change>
-
-    <change type="bug">
-      <para>In some cases, we would remove the wrong header when we
-      were stripping Content-Transfer-Encoding headers from a
-      request.  This has been fixed.</para>
-    </change>
-
-    <change type="enh">
-      <para>Backends can have a <code>.max_connections</code>
-      associated with them.</para>
-    </change>
-
-    <change type="bug">
-      <para>On Linux, we need to set the dumpable bit on the child if
-      we want core dumps.  Make sure it's set.</para>
-    </change>
-
-    <change type="bug">
-      <para>Doing <code>purge.hash()</code> with an empty string
-      would cause us to dump core.  Fixed so we don't do that any
-      more.</para>
-    </change>
-
-    <change type="bug">
-      <para>We ran into a problem with glibc's malloc on Linux where
-      it seemed like it failed to ever give memory back to the OS,
-      causing the system to swap.  We have now switched to jemalloc
-      which appears not to have this problem.</para>
-    </change>
-
-    <change type="bug">
-      <para><code>max_restarts</code> was never checked, so we always
-      ended up running out of workspace.  Now, <code>vcl_error</code>
-      is called when we reach <code>max_restarts</code>.</para>
-    </change>
-  </subsystem>
-  <subsystem>
-    <name>varnishtest</name>
-
-    <change type="enh">
-      <para><code>varnishtest</code> is a tool to do correctness tests
-      of varnishd.  The test suite is run by using <code>make
-      check</code>.</para>
-    </change>
-  </subsystem>
-
-  <subsystem>
-    <name>varnishtop</name>
-
-    <change type="enh">
-      <para>We now set the field widths dynamically based on the size
-      of the terminal and the name of the longest field.</para>
-    </change>
-  </subsystem>
-
-  <subsystem>
-    <name>varnishstat</name>
-
-    <change type="enh">
-      <para><code>varnishstat -1</code> now displays the uptime too.</para>
-    </change>
-  </subsystem>
-
-  <subsystem>
-    <name>varnishncsa</name>
-
-    <change type="bug">
-      <para><code>varnishncsa</code> now does fflush after each
-      write.  This makes <code>tail -f</code> work correctly, as well
-      as avoiding broken lines in the log file.</para>
-    </change>
-
-    <change type="enh">
-      <para>It is possible to get <code>varnishncsa</code> to output
-      the <code>X-Forwarded-For</code> instead of the client IP by
-      passing <code>-f</code> to it.</para>
-    </change>
-  </subsystem>
-
-  <subsystem>
-    <name>Build system</name>
-
-    <change type="bug">
-      <para>Various sanity checks have been added
-      to <code>configure</code>, it now complains about no ncurses or
-      if SO_RCVTIMEO or SO_SNDTIMEO are non-functional.  It also
-      aborts if there's no working acceptor mechanism </para>
-    </change>
-
-    <change type="enh">
-      <para>The C compiler invocation is decided by the configure
-      script and can now be overridden by passing <code>VCC_CC</code>
-      when running configure.</para>
-    </change>
-  </subsystem>
-</group>
diff --git a/doc/changes-1.1.2.xml b/doc/changes-1.1.2.xml
deleted file mode 100644
index e629003..0000000
--- a/doc/changes-1.1.2.xml
+++ /dev/null
@@ -1,12 +0,0 @@
-<?xml version="1.0" encoding="utf-8"?>
-<?xml-stylesheet type="text/xml" href="changes-html.xsl"?>
-<!DOCTYPE changelog [
- <!ENTITY mdash "&#8212;">
-]>
-<!-- $Id$ -->
-<changelog xmlns:xi="http://www.w3.org/2001/XInclude">
-  <package>Varnish</package>
-  <version>1.1.2</version>
-
-  <xi:include href="changes-1.1.1-1.1.2.xml"/>
-</changelog>
diff --git a/doc/changes-1.1.xml b/doc/changes-1.1.xml
deleted file mode 100644
index a1b1c38..0000000
--- a/doc/changes-1.1.xml
+++ /dev/null
@@ -1,12 +0,0 @@
-<?xml version="1.0" encoding="utf-8"?>
-<?xml-stylesheet type="text/xml" href="changes-html.xsl"?>
-<!DOCTYPE changelog [
- <!ENTITY mdash "&#8212;">
-]>
-<!-- $Id$ -->
-<changelog xmlns:xi="http://www.w3.org/2001/XInclude">
-  <package>Varnish</package>
-  <version>1.1</version>
-
-  <xi:include href="changes-1.0.4-1.1.xml"/>
-</changelog>
diff --git a/doc/changes-2.0-2.0.1.xml b/doc/changes-2.0-2.0.1.xml
deleted file mode 100644
index 39422ab..0000000
--- a/doc/changes-2.0-2.0.1.xml
+++ /dev/null
@@ -1,29 +0,0 @@
-<?xml version="1.0" encoding="utf-8"?>
-<!DOCTYPE group [
-<!ENTITY mdash "&#8212;">
-]>
-<!-- $Id$ -->
-<group from="2.0" to="2.0.1">
-  <subsystem>
-    <name>varnishd</name>
-
-    <change type="bug" ref="354">
-      <para>When receiving a garbled HTTP
-      request, <code>varnishd</code> would sometimes crash.  This has
-      been fixed.</para>
-    </change>
-
-    <change type="bug">
-      <para>There was an off-by-one error in the ACL compilation.
-      Now fixed.</para>
-    </change>
-  </subsystem>
-
-  <subsystem>
-    <name>Red Hat spec file</name>
-
-    <change type="bug">
-      <para>A typo in the spec file made the .rpm file names wrong.</para>
-    </change>
-  </subsystem>
-</group>
diff --git a/doc/changes-2.0.1-2.0.2.xml b/doc/changes-2.0.1-2.0.2.xml
deleted file mode 100644
index 51e5627..0000000
--- a/doc/changes-2.0.1-2.0.2.xml
+++ /dev/null
@@ -1,25 +0,0 @@
-<?xml version="1.0" encoding="utf-8"?>
-<!DOCTYPE group [
-<!ENTITY mdash "&#8212;">
-]>
-<!-- $Id$ -->
-<group from="2.0.1" to="2.0.2">
-  <subsystem>
-    <name>varnishd</name>
-
-    <change type="bug" ref="345">
-      <para>In high-load situations, when using
-      ESI, <code>varnishd</code> would sometimes mishandle objects and
-      crash.  This has been worked around. </para>
-    </change>
-  </subsystem>
-
-  <subsystem>
-    <name>varnishreplay</name>
-
-    <change type="bug">
-      <para><code>varnishreplay</code> did not work correctly on
-      Linux, due to a too small stack.  This has now been fixed.</para>
-    </change>
-  </subsystem>
-</group>
diff --git a/doc/changes-2.0.1.xml b/doc/changes-2.0.1.xml
deleted file mode 100644
index 9d84398..0000000
--- a/doc/changes-2.0.1.xml
+++ /dev/null
@@ -1,12 +0,0 @@
-<?xml version="1.0" encoding="utf-8"?>
-<?xml-stylesheet type="text/xml" href="changes-html.xsl"?>
-<!DOCTYPE changelog [
- <!ENTITY mdash "&#8212;">
-]>
-<!-- $Id$ -->
-<changelog xmlns:xi="http://www.w3.org/2001/XInclude">
-  <package>Varnish</package>
-  <version>2.0.1</version>
-
-  <xi:include href="changes-2.0-2.0.1.xml"/>
-</changelog>
diff --git a/doc/changes-2.0.2-2.0.3.xml b/doc/changes-2.0.2-2.0.3.xml
deleted file mode 100644
index 1c0737a..0000000
--- a/doc/changes-2.0.2-2.0.3.xml
+++ /dev/null
@@ -1,225 +0,0 @@
-<?xml version="1.0" encoding="utf-8"?>
-<!DOCTYPE group [
-<!ENTITY mdash "&#8212;">
-]>
-<!-- $Id$ -->
-<group from="2.0.2" to="2.0.3">
-  <subsystem>
-    <name>varnishd</name>
-
-    <change type="bug" ref="386">
-      <para>Handle If-Modified-Since and ESI sub-objects better,
-      fixing a problem where we sometimes neglected to insert
-      included objects.</para>
-    </change>
-
-    <change type="enh" ref="365">
-      <para><code>restart</code> in <code>vcl_hit</code> is now supported.</para>
-    </change>
-
-    <change type="bug">
-      <para>Setting the TTL of an object to 0 seconds would sometimes
-      cause it to be delivered for up to one second - epsilon.  This
-      has been corrected and we should now never deliver those
-      objects to other clients.</para>
-    </change>
-
-    <change type="enh">
-      <para>The malloc storage backend now prints the maximum storage
-      size, just like the file backend.</para>
-    </change>
-
-    <change type="bug" ref="362">
-      <para>Various small documentation bugs have been fixed.</para>
-    </change>
-
-    <change type="bug" >
-      <para>Varnish did not set a default interval for backend
-      probes, causing it to poll the backend continuously.  This has
-      been corrected.</para>
-    </change>
-
-    <change type="enh">
-      <para>Allow "true" and "false" when setting boolean parameters,
-      in addition to on/off, enable/disable and yes/no.</para>
-    </change>
-
-    <change type="enh">
-      <para>Default to always talking HTTP 1.1 with the backend.</para>
-    </change>
-
-    <change type="bug" ref="368">
-      <para>Varnish did not make sure the file it was loading was a
-      regular file.   This could cause Varnish to crash if it was
-      asked to load a directory or other non-regular file.  We now
-      check that the file is a regular file before loading it.</para>
-    </change>
-
-    <change type="enh" ref="210">
-      <para>The binary heap used for expiry processing had
-      scalability problems.  Work around this by using stripes of a
-      fixed size, which should make this scale better, particularly
-      when starting up and having lots of objects.</para>
-    </change>
-
-    <change type="bug">
-      <para>When we imported the <code>jemalloc</code> library into
-      the Varnish tree, it did not compile without warnings.  This
-      has now been fixed.</para>
-    </change>
-
-    <change type="bug" ref="278">
-      <para>Varnish took a very long time to detect that the backend
-      did not respond.  To remedy this, we now have read timeouts in
-      addition to the connect timeout.  Both
-      the <code>first_byte_timeout</code> and
-      the <code>between_bytes_timeout</code> defaults to 60 seconds.
-      The connect timeout is no longer in milliseconds, but rather in
-      seconds.</para>
-    </change>
-
-    <change type="enh">
-      <para>Previously, the VCL to C conversion as well as the
-      invocation of the C compiler was done in the management
-      process.  This is now done in a separate sub-process.  This
-      prevents any bugs in the VCL compiler from affecting the
-      management process.</para>
-    </change>
-
-    <change type="bug">
-      <para>Chunked encoding headers were counted in the statistics
-      for header bytes.  They no longer are.</para>
-    </change>
-
-    <change type="bug">
-      <para>ESI processed objects were not counted in the statistics
-      for body bytes.  They now are.</para>
-    </change>
-
-    <change type="enh">
-      <para>It is now possible to adjust the maximum record length of
-      log entries in the shmlog by tuning the <code>shm_reclen</code>
-      parameter.</para>
-    </change>
-
-    <change type="enh">
-      <para>The management parameters listed in the CLI were not
-      sorted, which made it hard to find the parameter you were
-      looking for.  They are now sorted, which should make this
-      easier.</para>
-    </change>
-
-    <change type="enh">
-      <para>Add a new hashing type, "critbit", which uses a lock-less
-      tree based lookup algorithm.  This is experimental and should
-      not be enabled in production environments without proper
-      testing.</para>
-    </change>
-
-    <change type="enh">
-      <para>The session workspace had a default size of 8k.  It is
-      now 16k, which should make VCLs where many headers are
-      processed less prone to panics.</para>
-    </change>
-
-    <change type="enh">
-      <para>We have seen that people seem to be confused as to which
-      actions in the different VCL functions return and which ones
-      don't.  Add a new syntax <code>return(action)</code> to make
-      this more explicit.  The old syntax is still supported.</para>
-    </change>
-
-    <change type="bug" ref="97">
-      <para>Varnish would return an error if any of the management
-      IPs listed in the <code>-T</code> parameter could not be
-      listened to.  We now only return an error if none of them can
-      be listened to.</para>
-    </change>
-
-    <change type="bug" ref="416">
-      <para>In the case of the backend or client giving us too many
-      parameters, we used to just ignore the overflowing headers.
-      This is problematic if you end up ignoreing Content-Length,
-      Transfer-Encoding and similar headers.  We now give out a 400
-      error to the client if it sends us too many and 503 if we get
-      too many from the backend.</para>
-    </change>
-
-    <change type="bug" ref="387">
-      <para>We used panic if we got a too large chunked header.
-      This behaviour has been changed into just failing the
-      transaction.</para>
-    </change>
-
-    <change type="enh">
-      <para>Varnish now supports an extended purge method where it is
-      possible to do <code>purge req.http.host ~ "web1.com" &amp;&amp; req.url ~ "\.png"
-      </code> and similar.  See the documentation for details.</para>
-    </change>
-
-    <change type="bug" ref="418">
-      <para>Under heavy load, Varnish would sometimes crash when
-      trying to update the per-request statistics.  This has now been
-      fixed.</para>
-    </change>
-
-    <change type="enh">
-      <para>It is now possible to not save the hash string in the
-      session and object workspace.  This will save a lot of memory
-      on sites with many small objects.  Disabling
-      the <code>purge_hash</code> parameter also disables
-      the <code>purge.hash</code> facility.</para>
-    </change>
-
-    <change type="enh">
-      <para>Varnish now supports <code>!~</code> as a "no match"
-      regular expression matcher.</para>
-    </change>
-
-    <change type="bug" ref="425">
-      <para>In some cases, you could get serialised access to "pass"
-      objects.  We now make it default to the default_ttl value; this
-      can be overridden in vcl_fetch.</para>
-    </change>
-
-    <change type="bug" ref="417">
-      <para>Varnish did not check the syntax of <code>regsub</code>
-      calls properly.  More checking has been added.</para>
-    </change>
-
-    <change type="bug" ref="427">
-      <para>If the client closed the connection while Varnish was
-      processing ESI elements, Varnish would crash while trying to
-      write the object to the client.  We now check if the client has
-      closed the connection.</para>
-    </change>
-
-    <change type="bug" ref="433">
-      <para>The ESI parser had a bug where it would crash if an XML
-      comment would span storage segments.  This has been
-      fixed.</para>
-    </change>
-  </subsystem>
-
-  <subsystem>
-    <name>VCL Manual page</name>
-
-    <change type="bug">
-      <para>The documentation on how capturing parentheses work was
-      wrong.  This has been corrected.</para>
-    </change>
-
-    <change type="enh">
-      <para>Grace has now been documented.</para>
-    </change>
-  </subsystem>
-
-  <subsystem>
-    <name>varnishreplay</name>
-
-    <change type="bug">
-      <para><code>varnishreplay</code> did not work correctly on
-      Linux, due to a too small stack.  This has now been fixed.</para>
-    </change>
-  </subsystem>
-</group>
diff --git a/doc/changes-2.0.2.xml b/doc/changes-2.0.2.xml
deleted file mode 100644
index c56ca46..0000000
--- a/doc/changes-2.0.2.xml
+++ /dev/null
@@ -1,12 +0,0 @@
-<?xml version="1.0" encoding="utf-8"?>
-<?xml-stylesheet type="text/xml" href="changes-html.xsl"?>
-<!DOCTYPE changelog [
- <!ENTITY mdash "&#8212;">
-]>
-<!-- $Id$ -->
-<changelog xmlns:xi="http://www.w3.org/2001/XInclude">
-  <package>Varnish</package>
-  <version>2.0.2</version>
-
-  <xi:include href="changes-2.0.1-2.0.2.xml"/>
-</changelog>
diff --git a/doc/changes-2.0.3-2.0.4.xml b/doc/changes-2.0.3-2.0.4.xml
deleted file mode 100644
index d7bb0b8..0000000
--- a/doc/changes-2.0.3-2.0.4.xml
+++ /dev/null
@@ -1,98 +0,0 @@
-<?xml version="1.0" encoding="utf-8"?>
-<!DOCTYPE group [
-<!ENTITY mdash "&#8212;">
-]>
-<!-- $Id$ -->
-<group from="2.0.3" to="2.0.4">
-  <subsystem>
-    <name>varnishd</name>
-
-    <change type="enh">
-      <para>Make Varnish more portable by pulling in fixes for
-      Solaris and NetBSD.</para>
-    </change>
-
-    <change type="bug">
-      <para>Correct description of -a in the manual page.</para>
-    </change>
-
-    <change type="bug">
-      <para>Ensure we are compiling in C99 mode.</para>
-    </change>
-
-    <change type="bug" ref="458">
-      <para>If error was called with a null reason, we would crash on
-      Solaris.  Make sure this no longer happens.</para>
-    </change>
-
-    <change type="bug" ref="428">
-      <para>Varnish used to crash if you asked it to use a
-      non-existent waiter.  This has now been fixed.</para>
-    </change>
-
-    <change type="enh">
-      <para>Add documentation to the default VCL explaining that
-      using <code>Connection: close</code> in <code>vcl_close</code>
-      is generally a good idea.</para>
-    </change>
-
-    <change type="enh">
-      <para>Add minimal facility for dealing with TELNET option
-	negotiation by returning WONT to DO and DONT requests.</para>
-    </change>
-
-    <change type="enh">
-      <para>If the backend is unhealthy, use a graced object if one is
-      available.</para>
-    </change>
-
-    <change type="enh">
-      <para>Make <code>server.hostname</code>
-      and <code>server.identity</code> available to VCL.  The latter
-      can be set with the <code>-i</code> parameter
-      to <code>varnishd</code>.</para>
-    </change>
-
-    <change type="enh">
-      <para>Make <code>restart</code> available
-      from <code>vcl_error</code>.</para>
-    </change>
-
-    <change type="enh">
-      <para>Previously, only the TTL of an object was considered in
-      whether it would be marked as cacheable.  This has been changed
-      to take the grace into consideration as well.</para>
-    </change>
-
-    <change type="bug" ref="445">
-      <para>Previously, if an included ESI fragment had a zero size,
-      we would send out a zero-sized chunk which signifies
-      end-of-transmission.  We now ignore zero-sized chunks.</para>
-    </change>
-
-    <change type="bug">
-      <para>We accidentially slept for far too long when we reached
-      the maximum number of open file descriptors.  This has been
-      corrected and <code>accept_fd_holdoff</code> now works
-      correctly.</para>
-    </change>
-    <change type="bug" ref="476">
-      <para>Previously, when ESI processing, we did not look at the
-      full length, but stopped at the first NULL byte.  We no longer
-      do that, enabling ESI processing of binary data.</para>
-    </change>
-  </subsystem>
-
-  <subsystem>
-    <name>varnishtest</name>
-    <change type="bug">
-      <para>Make sure system "..." returns successfully to ensure
-      test failures do not go unnoticed.</para>
-    </change>
-
-    <change type="enh">
-      <para>Make it possible to send NULL bytes through the testing
-      framework.</para>
-    </change>
-  </subsystem>
-</group>
diff --git a/doc/changes-2.0.3.xml b/doc/changes-2.0.3.xml
deleted file mode 100644
index 69dd9fd..0000000
--- a/doc/changes-2.0.3.xml
+++ /dev/null
@@ -1,12 +0,0 @@
-<?xml version="1.0" encoding="utf-8"?>
-<?xml-stylesheet type="text/xml" href="changes-html.xsl"?>
-<!DOCTYPE changelog [
- <!ENTITY mdash "&#8212;">
-]>
-<!-- $Id$ -->
-<changelog xmlns:xi="http://www.w3.org/2001/XInclude">
-  <package>Varnish</package>
-  <version>2.0.3</version>
-
-  <xi:include href="changes-2.0.2-2.0.3.xml"/>
-</changelog>
diff --git a/doc/changes-2.0.4-2.0.5.xml b/doc/changes-2.0.4-2.0.5.xml
deleted file mode 100644
index ff5e15d..0000000
--- a/doc/changes-2.0.4-2.0.5.xml
+++ /dev/null
@@ -1,161 +0,0 @@
-<?xml version="1.0" encoding="utf-8"?>
-<!DOCTYPE group [
-<!ENTITY mdash "&#8212;">
-]>
-<!-- $Id$ -->
-<group from="2.0.4" to="2.0.5">
-  <subsystem>
-    <name>varnishd</name>
-
-    <change type="bug" ref="498">
-      <para>Handle object workspace overruns better.</para>
-    </change>
-
-    <change type="enh">
-      <para>Allow turning off ESI processing per request by using
-      <code>set req.esi = off</code>.</para>
-    </change>
-
-    <change type="enh">
-      <para>Tell the kernel that we expect to use the mmap-ed file in
-      a random fashion.  On Linux, this turns off/down readahead and
-      increases performance.</para>
-    </change>
-
-    <change type="enh">
-      <para>Make it possible to change the maximum number of HTTP
-      headers we allow by
-      passing <code>--with-max-header-fields=NUM</code> rather than
-      changing the code.</para>
-    </change>
-
-    <change type="enh" ref="428">
-      <para>Implement support for HTTP continuation lines.</para>
-    </change>
-
-    <change type="enh">
-      <para>Change how connections are closed and only use SO_LINGER
-      for orderly connection closure.  This should hopefully make
-      worker threads less prone to hangups on network problems.</para>
-    </change>
-
-    <change type="bug" ref="502">
-      <para>Handle multi-element purges correctly.  Previously we
-      ended up with parse errors when this was done from VCL.</para>
-    </change>
-
-    <change type="bug" ref="506, 549">
-      <para>Handle illegal responses from the backend better by
-      serving a 503 page rather than panic-ing.</para>
-    </change>
-
-    <change type="enh">
-      <para>When we run into an assertion that is not true, Varnish
-      would previously dump a little bit of information about itself.
-      Extend that information with a backtrace.  Note that this relies
-      on the varnish binary being unstripped.</para>
-    </change>
-
-    <change type="enh">
-      <para>Add a session_max parameter that limits the maximum
-      number of sessions we keep open before we start dropping new
-      connections summarily.</para>
-    </change>
-
-    <change type="bug">
-      <para>Try to consume less memory when doing ESI processing by
-      properly rolling back used workspace after processing an
-      object.  This should make it possible to
-      turn <code>sess_workspace</code> quite a bit for users with
-      ESI-heavy pages.</para>
-    </change>
-
-    <change type="enh">
-      <para>Turn on <code>session_linger</code> by default.  Tests
-      have shown that <code>session_linger</code> helps a fair bit
-      with performance.</para>
-    </change>
-
-    <change type="enh">
-      <para>Rewrite the epoll acceptor for better performance.  This
-      should lead to both higher processing rates and maximum number
-      of connections on Linux.</para>
-    </change>
-
-    <change type="enh">
-      <para>Add If-None-Match support, this gives significant
-      bandwidth savings for users with compliant browsers.</para>
-    </change>
-
-    <change type="bug" ref="529">
-      <para>RFC2616 specifies
-      that <code>ETag</code>, <code>Content-Location</code>, <code>Expires</code>, <code>Cache-Control</code>
-      and <code>Vary</code> should be emitted when delivering a
-      response with the 304 response code.</para>
-    </change>
-
-    <change type="bug" ref="531">
-      <para>Various fixes which makes Varnish compile and work on AIX.</para>
-    </change>
-
-    <change type="enh">
-      <para>Turn on TCP_DEFER_ACCEPT on Linux.  This should make us
-      less suspecible to denial of service attacks as well as give us
-      slightly better performance.</para>
-    </change>
-
-    <change type="enh">
-      <para>Add an <code>.initial</code> property to the backend
-      probe specification.  This is the number of good probes we
-      pretend to have seen.  The default is one less than .threshold,
-      which means the first probe will decide if we consider the
-      backend healthy.</para>
-    </change>
-
-    <change type="enh">
-      <para>Make it possible to compare strings against other
-      string-like objects, not just plain strings.  This allows you to
-      compare two headers, for instance.</para>
-    </change>
-
-    <change type="bug">
-      <para>When support for <code>restart</code>
-      in <code>vcl_error</code> was added, there was no check to
-      prevent infinte recursion.  This has now been fixed.
-     </para>
-    </change>
-
-    <change type="enh">
-      <para>Turn on purge_dups by default.  This should make us
-      consume less memory when there are many bans for the same
-      pattern added.</para>
-    </change>
-
-    <change type="enh">
-      <para>Add a new log tag called <code>FetchError</code> which
-      tries to explain why we could not fetch an object from the
-      backend.</para>
-    </change>
-
-    <change type="enh">
-      <para>Change the default <code>srcaddr_ttl</code> to 0.  It is
-      not used by anything and has been removed in the development
-      version.  This will increase performance somewhat.</para>
-    </change>
-
-  </subsystem>
-
-  <subsystem>
-    <name>varnishtop</name>
-    <change type="bug" ref="514">
-      <para>varnishtop did not handle variable-length log fields
-      correctly. This is now fixed.</para>
-    </change>
-
-    <change type="enh">
-      <para>varnishtop previously did not print the name of the tag,
-      which made it very hard to understand.  We now print out the
-      tag name.</para>
-    </change>
-  </subsystem>
-</group>
diff --git a/doc/changes-2.0.4.xml b/doc/changes-2.0.4.xml
deleted file mode 100644
index bf08056..0000000
--- a/doc/changes-2.0.4.xml
+++ /dev/null
@@ -1,12 +0,0 @@
-<?xml version="1.0" encoding="utf-8"?>
-<?xml-stylesheet type="text/xml" href="changes-html.xsl"?>
-<!DOCTYPE changelog [
- <!ENTITY mdash "&#8212;">
-]>
-<!-- $Id$ -->
-<changelog xmlns:xi="http://www.w3.org/2001/XInclude">
-  <package>Varnish</package>
-  <version>2.0.4</version>
-
-  <xi:include href="changes-2.0.3-2.0.4.xml"/>
-</changelog>
diff --git a/doc/changes-2.0.5-2.0.6.xml b/doc/changes-2.0.5-2.0.6.xml
deleted file mode 100644
index 257bd0e..0000000
--- a/doc/changes-2.0.5-2.0.6.xml
+++ /dev/null
@@ -1,53 +0,0 @@
-<?xml version="1.0" encoding="utf-8"?>
-<!DOCTYPE group [
-<!ENTITY mdash "&#8212;">
-]>
-<!-- $Id$ -->
-<group from="2.0.5" to="2.0.6">
-  <subsystem>
-    <name>varnishd</name>
-
-    <change type="bug" ref="585">
-      <para>2.0.5 had an off-by-one error in the ESI handling causing
-      includes to fail a large part of the time. This has now been
-      fixed.</para>
-    </change>
-
-    <change type="bug" ref="584">
-      <para>Try harder to not confuse backends when sending them
-      backend probes.  We half-closed the connection, something some
-      backends thought meant we had dropped the connection.  Stop
-      doing so, and add the capability for specifying the expected
-      response code.</para>
-    </change>
-
-    <change type="bug">
-      <para>In 2.0.5, session lingering was turned on.  This caused
-      statistics to not be counted often enough in some cases.  This
-      has now been fixed.</para>
-    </change>
-
-    <change type="bug" ref="547">
-      <para>Avoid triggering an assert if the other end closes the
-      connection while we are lingering and waiting for another
-      request from them.</para>
-    </change>
-
-    <change type="bug" ref="577">
-      <para>When generating backtraces, prefer the built-in backtrace
-      function if such exists.  This fixes a problem compiling 2.0.5
-      on Solaris.</para>
-    </change>
-
-    <change type="enh" ref="572">
-      <para>Make it possible to specify the per-thread stack size.
-      This might be useful on 32 bit systems with their limited
-      address space.</para>
-    </change>
-
-    <change type="bug" ref="559">
-      <para>Document the <code>-C</code> option
-      to <code>varnishd</code>.</para>
-    </change>
-  </subsystem>
-</group>
diff --git a/doc/changes-2.0.5.xml b/doc/changes-2.0.5.xml
deleted file mode 100644
index 5c6c47a..0000000
--- a/doc/changes-2.0.5.xml
+++ /dev/null
@@ -1,12 +0,0 @@
-<?xml version="1.0" encoding="utf-8"?>
-<?xml-stylesheet type="text/xml" href="changes-html.xsl"?>
-<!DOCTYPE changelog [
- <!ENTITY mdash "&#8212;">
-]>
-<!-- $Id$ -->
-<changelog xmlns:xi="http://www.w3.org/2001/XInclude">
-  <package>Varnish</package>
-  <version>2.0.5</version>
-
-  <xi:include href="changes-2.0.4-2.0.5.xml"/>
-</changelog>
diff --git a/doc/changes-2.0.6-2.1.0.xml b/doc/changes-2.0.6-2.1.0.xml
deleted file mode 100644
index 7ecbd59..0000000
--- a/doc/changes-2.0.6-2.1.0.xml
+++ /dev/null
@@ -1,316 +0,0 @@
-<?xml version="1.0" encoding="utf-8"?>
-<!DOCTYPE group [
-<!ENTITY mdash "&#8212;">
-]>
-<!-- $Id$ -->
-<group from="2.0.6" to="2.1.0">
-  <subsystem>
-    <name>varnishd</name>
-
-    <change type="enh">
-      <para>Persistent storage is now experimentally supported using
-      the <code>persistent</code> stevedore.  It has the same command
-      line arguments as the file stevedore.</para>
-    </change>
-
-    <change type="enh">
-      <para><code>obj.*</code> is now called <code>beresp.*</code>
-      in <code>vcl_fetch</code>, and <code>obj.*</code> is now
-      read-only.</para>
-    </change>
-
-    <change type="enh">
-      <para>The regular expression engine is now PCRE instead of POSIX
-      regular expressions.</para>
-    </change>
-
-    <change type="enh">
-      <para><code>req.*</code> is now available
-      in <code>vcl_deliver</code>.</para>
-    </change>
-
-    <change type="enh">
-      <para>Add saint mode where we can attempt to grace an object if
-	we don't like the backend response for some reason.</para>
-
-      <para>Related, add <code>saintmode_threshold</code> which is the
-	threshold for the number of objects to be added to the trouble
-	list before the backend is considered sick.</para>
-    </change>
-
-    <change type="enh">
-      <para>Add a new hashing method called critbit.  This autoscales
-      and should work better on large object workloads than the
-      classic hash.  Critbit has been made the default hash algorithm.</para>
-    </change>
-
-    <change type="enh">
-      <para>When closing connections, we experimented with sending RST
-      to free up load balancers and free up threads more quickly.
-      This caused some problems with NAT routers and so has been
-      reverted for now.</para>
-    </change>
-
-    <change type="enh">
-      <para>Add thread that checks objects against ban list in order
-	to prevent ban list from growing forever.  Note that this
-	needs purges to be written so they don't depend
-	on <code>req.*</code>.  Enabled by
-	setting <code>ban_lurker_sleep</code> to a nonzero
-	value.</para>
-    </change>
-
-    <change type="enh">
-      <para>The shared memory log file format was limited to maximum
-      64k simultaneous connections.  This is now a 32 bit field which
-      removes this limitation.</para>
-    </change>
-
-    <change type="enh">
-      <para>Remove obj_workspace, this is now sized automatically.
-	</para>
-    </change>
-
-    <change type="enh">
-      <para>Rename acceptors to waiters</para>
-    </change>
-
-    <change type="enh">
-      <para><code>vcl_prefetch</code> has been removed.  It was never
-      fully implemented.</para>
-    </change>
-
-    <change type="enh">
-      <para>Add support for authenticating CLI connections.</para>
-    </change>
-
-    <change type="enh">
-      <para>Add hash director that chooses which backend to use
-      depending on <code>req.hash</code>.</para>
-    </change>
-
-    <change type="enh">
-      <para>Add client director that chooses which backend to use
-      depending on the client's IP address.  Note that this ignores
-      the <code>X-Forwarded-For</code> header.</para>
-    </change>
-
-    <change type="enh">
-      <para><code>varnishd</code> now displays a banner by default
-      when you connect to the CLI.</para>
-    </change>
-
-    <change type="enh">
-      <para>Increase performance somewhat by moving statistics
-	gathering into a per-worker structure that is regularly
-	flushed to the global stats.</para>
-    </change>
-
-    <change type="enh">
-      <para>Make sure we store the header and body of object together.
-	This may in some cases improve performance and is needed for
-	persistence.</para>
-    </change>
-
-    <change type="enh">
-      <para>Remove client-side address accounting.  It was never used
-      for anything and presented a performance problem.</para>
-    </change>
-
-    <change type="enh">
-      <para>Add a timestamp to bans, so you can know how old they are.</para>
-    </change>
-
-    <change type="enh">
-      <para>Quite a few people got confused over the warning about not
-	being able to lock the shared memory log into RAM, so stop
-	warning about that.</para>
-    </change>
-
-    <change type="enh" ref="329">
-      <para>Change the default CLI timeout to 10 seconds.</para>
-    </change>
-
-    <change type="bug" ref="444">
-      <para>We previously forced all inserts into the cache to be GET
-	requests.  This has been changed to allow POST as well in
-	order to be able to implement purge-on-POST semantics.</para>
-    </change>
-
-    <change type="enh">
-      <para>The CLI command <code>stats</code> now only lists non-zero
-	values.</para>
-    </change>
-
-    <change type="enh">
-      <para>The CLI command <code>stats</code> now only lists non-zero
-	values.</para>
-    </change>
-
-    <change type="enh">
-      <para>Use <code>daemon(3)</code> from <code>libcompat</code> on
-	Darwin.</para>
-    </change>
-
-    <change type="enh">
-      <para>Remove <code>vcl_discard</code> as it causes too much
-	complexity and never actually worked particularly well.</para>
-    </change>
-
-    <change type="enh">
-      <para>Remove <code>vcl_timeout</code> as it causes too much
-	complexity and never actually worked particularly well.</para>
-    </change>
-
-    <change type="enh">
-      <para>Update the documentation so it refers
-      to <code>sess_workspace</code>, not <code>http_workspace</code>.</para>
-    </change>
-
-    <change type="enh">
-      <para>Document the <code>-i</code> switch
-      to <code>varnishd</code> as well as
-      the <code>server.identity</code>
-      and <code>server.hostname</code> VCL variables.</para>
-    </change>
-
-    <change type="enh">
-      <para><code>purge.hash</code> is now deprecated and no longer
-      shown in help listings.</para>
-    </change>
-
-    <change type="enh" ref="607">
-      <para>When processing ESI, replace the five mandatory XML
-      entities when we encounter them.</para>
-    </change>
-
-    <change type="enh">
-      <para>Add string representations of time and relative
-	time.</para>
-    </change>
-
-    <change type="bug" ref="604">
-      <para>Add locking for <code>n_vbe_conn</code> to make it stop
-      underflowing.</para>
-    </change>
-
-    <change type="bug" ref="207">
-      <para>When ESI-processing content, check for illegal XML
-      character entities.</para>
-    </change>
-
-    <change type="bug" ref="207">
-      <para>Varnish can now connect its CLI to a remote instance when
-      starting up, rather than just being connected to.</para>
-    </change>
-
-    <change type="enh">
-      <para>It is no longer needed to specify the maximum number of
-	HTTP headers to allow from backends.  This is now a run-time
-	parameter.</para>
-    </change>
-
-    <change type="enh" ref="601, 540">
-      <para>The <code>X-Forwarded-For</code> header is now generated
-      by <code>vcl_recv</code> rather than the C code.</para>
-    </change>
-
-    <change type="enh">
-      <para>It is now possible to not send all CLI traffic to
-	syslog.</para>
-    </change>
-
-    <change type="enh">
-      <para>It is now possible to not send all CLI traffic to
-	syslog.</para>
-    </change>
-
-    <change type="enh">
-      <para>In the case of varnish crashing, it now outputs a
-      identifying string with the OS, OS revision, architecture and
-      storage parameters together with the backtrace.</para>
-    </change>
-
-    <change type="enh">
-      <para>Use exponential backoff when we run out of file
-	descriptors or sessions.</para>
-    </change>
-
-    <change type="enh" ref="635">
-      <para>Allow setting backend timeouts to zero.</para>
-    </change>
-
-    <change type="enh">
-      <para>Count uptime in the shared memory log.</para>
-    </change>
-
-    <change type="enh" ref="620">
-      <para>Try to detect the case of two running varnishes with the
-      same shmlog and storage by writing the master and child process
-      ids to the shmlog and refusing to start if they are still
-      running.</para>
-    </change>
-
-    <change type="bug" ref="524">
-      <para>Make sure to use EOF mode when serving ESI content to
-	HTTP/1.0 clients.</para>
-    </change>
-
-    <change type="bug" ref="495">
-      <para>Make sure we close the connection if it either
-	sends <code>Connection: close</code> or it is a HTTP/1.0
-	backend that does not send <code>Connection:
-	keep-alive</code>.</para>
-    </change>
-
-    <change type="enh">
-      <para>Increase the default session workspace to 64k on 64-bit
-	systems.</para>
-    </change>
-
-    <change type="bug" ref="644">
-      <para>Make the <code>epoll</code> waiter use level triggering,
-      not edge triggering as edge triggering caused problems on very
-      busy servers.</para>
-    </change>
-
-    <change type="bug" ref="626">
-      <para>Handle unforeseen client disconnections better on Solaris.</para>
-    </change>
-
-    <change type="enh">
-      <para>Make session lingering apply to new sessions, not just
-      reused sessions.</para>
-    </change>
-  </subsystem>
-  <subsystem>
-    <name>varnishstat</name>
-
-    <change type="enh">
-      <para>Make use of the new uptime field in the shared memory log
-      rather than synthesizing it from the start time.</para>
-    </change>
-  </subsystem>
-  <subsystem>
-    <name>varnishlog</name>
-
-    <change type="enh">
-      <para>Exit at the end of the file when started
-	with <code>-d</code>.</para>
-    </change>
-  </subsystem>
-
-  <subsystem>
-    <name>varnishadm</name>
-
-    <change type="enh">
-      <para><code>varnishadm</code> can now have a timeout when trying
-      to connect to the running <code>varnishd</code>.</para>
-    </change>
-
-    <change type="enh">
-      <para><code>varnishadm</code> now knows how to respond to the
-      secret from a secured <code>varnishd</code></para>
-    </change>
-  </subsystem>
-</group>
diff --git a/doc/changes-2.0.6.xml b/doc/changes-2.0.6.xml
deleted file mode 100644
index bf742f0..0000000
--- a/doc/changes-2.0.6.xml
+++ /dev/null
@@ -1,12 +0,0 @@
-<?xml version="1.0" encoding="utf-8"?>
-<?xml-stylesheet type="text/xml" href="changes-html.xsl"?>
-<!DOCTYPE changelog [
- <!ENTITY mdash "&#8212;">
-]>
-<!-- $Id$ -->
-<changelog xmlns:xi="http://www.w3.org/2001/XInclude">
-  <package>Varnish</package>
-  <version>2.0.6</version>
-
-  <xi:include href="changes-2.0.5-2.0.6.xml"/>
-</changelog>
diff --git a/doc/changes-2.0.xml b/doc/changes-2.0.xml
deleted file mode 100644
index a510d76..0000000
--- a/doc/changes-2.0.xml
+++ /dev/null
@@ -1,12 +0,0 @@
-<?xml version="1.0" encoding="utf-8"?>
-<?xml-stylesheet type="text/xml" href="changes-html.xsl"?>
-<!DOCTYPE changelog [
- <!ENTITY mdash "&#8212;">
-]>
-<!-- $Id$ -->
-<changelog xmlns:xi="http://www.w3.org/2001/XInclude">
-  <package>Varnish</package>
-  <version>2.0</version>
-
-  <xi:include href="changes-1.1.2-2.0.xml"/>
-</changelog>
diff --git a/doc/changes-2.1.0-2.1.1.xml b/doc/changes-2.1.0-2.1.1.xml
deleted file mode 100644
index 3c3a7c9..0000000
--- a/doc/changes-2.1.0-2.1.1.xml
+++ /dev/null
@@ -1,93 +0,0 @@
-<?xml version="1.0" encoding="utf-8"?>
-<!DOCTYPE group [
-<!ENTITY mdash "&#8212;">
-]>
-<!-- $Id: changes-2.0.6-2.1.0.xml 4672 2010-04-19 10:26:23Z tfheen $ -->
-<group from="2.0.6" to="2.1.0">
-  <subsystem>
-    <name>varnishd</name>
-
-    <change type="bug">
-      <para>The changelog in 2.1.0 included syntax errors, causing
-      the generated changelog to be empty.</para>
-    </change>
-
-    <change type="bug">
-      <para>The help text for <code>default_grace</code> was wrongly
-      formatted and included a syntax error.  This has now been fixed.</para>
-    </change>
-
-    <change type="bug">
-      <para><code>varnishd</code> now closes the file descriptor used
-      to read the management secret file (from the <code>-S</code>
-      parameter).</para>
-    </change>
-
-    <change type="bug" ref="669">
-      <para>The child would previously try to close every valid file
-	descriptor, something which could cause problems if the file
-	descriptor ulimit was set too high.  We now keep track of all
-	the file descriptors we open and only close up to that number.
-	</para>
-    </change>
-
-    <change type="bug">
-      <para>ESI was partially broken in 2.1.0 due to a bug in the
-      rollback of session workspace.  This has been fixed.</para>
-    </change>
-
-    <change type="bug">
-      <para>Reject the <code>auth</code>command rather than crash if
-      there is no <code>-S</code> parameter given.</para>
-    </change>
-
-    <change type="enh">
-      <para>Align pointers in allocated objects.  This will in theory
-      make Varnish a tiny bit faster at the expense of slightly more
-      memory usage.</para>
-    </change>
-
-    <change type="bug">
-      <para>Ensure the master process process id is updated in the
-      shared memory log file after we go into the background.</para>
-    </change>
-
-    <change type="bug" id="679">
-      <para><code>HEAD</code> requests would be converted to GET
-      requests too early, which affected <code>pass</code>
-      and <code>pipe</code>.  This has been fixed.</para>
-    </change>
-
-    <change type="bug">
-      <para>Update the documentation to point out that the TTL is no
-      longer taken into account to decide whether an object is
-      cacheable or not.</para>
-    </change>
-
-    <change type="enh">
-      <para>Add support for completely obliterating an object and all
-      variants of it.  Currently, this has to be done using inline C.</para>
-    </change>
-
-    <change type="enh">
-      <para>Add experimental support for the <code>Range</code>
-      header.  This has to be enabled using the parameter
-      http_range_support.</para>
-    </change>
-
-    <change type="bug" id="684">
-      <para>The <code>critbit</code> hasher could get into a deadlock
-      and had a race condition.  Both those have now been fixed.</para>
-    </change>
-
-  </subsystem>
-  <subsystem>
-    <name>varnishsizes</name>
-
-    <change type="enh">
-      <para><code>varnishsizes</code>, which is
-      like <code>varnishhost</code>, but for the length of objects,
-      has been added..</para>
-    </change>
-  </subsystem>
-</group>
diff --git a/doc/changes-2.1.0.xml b/doc/changes-2.1.0.xml
deleted file mode 100644
index 73305c5..0000000
--- a/doc/changes-2.1.0.xml
+++ /dev/null
@@ -1,12 +0,0 @@
-<?xml version="1.0" encoding="utf-8"?>
-<?xml-stylesheet type="text/xml" href="changes-html.xsl"?>
-<!DOCTYPE changelog [
- <!ENTITY mdash "&#8212;">
-]>
-<!-- $Id$ -->
-<changelog xmlns:xi="http://www.w3.org/2001/XInclude">
-  <package>Varnish</package>
-  <version>2.1</version>
-
-  <xi:include href="changes-2.0.6-2.1.0.xml"/>
-</changelog>
diff --git a/doc/changes-2.1.1-2.1.2.xml b/doc/changes-2.1.1-2.1.2.xml
deleted file mode 100644
index 486d530..0000000
--- a/doc/changes-2.1.1-2.1.2.xml
+++ /dev/null
@@ -1,19 +0,0 @@
-<?xml version="1.0" encoding="utf-8"?>
-<!DOCTYPE group [
-<!ENTITY mdash "&#8212;">
-]>
-<!-- $Id: changes-2.0.6-2.1.0.xml 4672 2010-04-19 10:26:23Z tfheen $ -->
-<group from="2.1.1" to="2.1.2">
-  <subsystem>
-    <name>varnishd</name>
-
-    <change type="bug" ref="694">
-      <para>When adding <code>Range</code> support for 2.1.1, we
-      accidentially introduced a bug which would append garbage to
-      objects larger than the chunk size, by default 128k. Browsers
-      would do the right thing due to Content-Length, but some load
-      balancers would get very confused.</para>
-    </change>
-
-  </subsystem>
-</group>
diff --git a/doc/changes-2.1.1.xml b/doc/changes-2.1.1.xml
deleted file mode 100644
index f5a91ba..0000000
--- a/doc/changes-2.1.1.xml
+++ /dev/null
@@ -1,12 +0,0 @@
-<?xml version="1.0" encoding="utf-8"?>
-<?xml-stylesheet type="text/xml" href="changes-html.xsl"?>
-<!DOCTYPE changelog [
- <!ENTITY mdash "&#8212;">
-]>
-<!-- $Id: changes-2.1.1.xml 4709 2010-04-21 10:40:27Z tfheen $ -->
-<changelog xmlns:xi="http://www.w3.org/2001/XInclude">
-  <package>Varnish</package>
-  <version>2.1</version>
-
-  <xi:include href="changes-2.1.0-2.1.1.xml"/>
-</changelog>
diff --git a/doc/changes-2.1.2-2.1.3.xml b/doc/changes-2.1.2-2.1.3.xml
deleted file mode 100644
index 638bbf4..0000000
--- a/doc/changes-2.1.2-2.1.3.xml
+++ /dev/null
@@ -1,75 +0,0 @@
-<?xml version="1.0" encoding="utf-8"?>
-<!DOCTYPE group [
-<!ENTITY mdash "&#8212;">
-]>
-<!-- $Id: changes-2.1.2-2.1.3.xml 4672 2010-04-19 10:26:23Z tfheen $ -->
-<group from="2.1.2" to="2.1.3">
-  <subsystem>
-    <name>varnishd</name>
-
-    <change type="enh">
-      <para>Improve scalability of critbit.</para>
-    </change>
-
-    <change type="bug">
-      <para>The critbit hash algorithm has now been tightened to make
-      sure the tree is in a consistent state at all points, and the
-      time we wait for an object to cool off after it is eligible for
-      garbage collection has been tweaked.</para>
-    </change>
-
-    <change type="enh">
-      <para>Add <code>log</code> command to VCL.  This emits
-      a <code>VCL_log</code> entry into the shared memory log.</para>
-    </change>
-
-    <change type="bug">
-      <para>Only emit Length and ReqEnd log entries if we actually
-      have an XID.  This should get rid of some empty log lines in
-      varnishncsa.</para>
-    </change>
-
-    <change type="bug" ref="722">
-      <para>Destroy directors in a predictable fashion, namely reverse
-      of creation order.</para>
-    </change>
-
-    <change type="bug" ref="719">
-      <para>Fix bug when ESI elements spanned storage elements causing
-      a panic.</para>
-    </change>
-
-    <change type="enh">
-      <para>In some cases, the VCL compiler would panic instead of
-      giving sensible messages.  This has now been fixed.</para>
-    </change>
-
-    <change type="bug">
-      <para>Correct an off-by-one error when the requested range
-      exceeds the size of an object.</para>
-    </change>
-
-    <change type="bug">
-      <para>Handle requests for the end of an object correctly.</para>
-    </change>
-
-    <change type="bug" ref="700">
-      <para>Allow tabulator characters in the third field of the
-      first line of HTTP requests</para>
-    </change>
-
-    <change type="bug">
-      <para>On Solaris, if the remote end sends us an RST, all system
-      calls related to that socket will return EINVAL.  We now handle
-      this better.</para>
-    </change>
-  </subsystem>
-  <subsystem>
-    <name>libvarnishapi</name>
-    <change type="bug">
-      <para>The <code>-X</code> parameter didn't work correctly.  This
-      has been fixed.</para>
-    </change>
-  </subsystem>
-
-</group>
diff --git a/doc/changes-2.1.2.xml b/doc/changes-2.1.2.xml
deleted file mode 100644
index c34aae6..0000000
--- a/doc/changes-2.1.2.xml
+++ /dev/null
@@ -1,12 +0,0 @@
-<?xml version="1.0" encoding="utf-8"?>
-<?xml-stylesheet type="text/xml" href="changes-html.xsl"?>
-<!DOCTYPE changelog [
- <!ENTITY mdash "&#8212;">
-]>
-<!-- $Id: changes-2.1.1.xml 4709 2010-04-21 10:40:27Z tfheen $ -->
-<changelog xmlns:xi="http://www.w3.org/2001/XInclude">
-  <package>Varnish</package>
-  <version>2.1.2</version>
-
-  <xi:include href="changes-2.1.1-2.1.2.xml"/>
-</changelog>
diff --git a/doc/changes-2.1.3-2.1.4.xml b/doc/changes-2.1.3-2.1.4.xml
deleted file mode 100644
index 775b6bd..0000000
--- a/doc/changes-2.1.3-2.1.4.xml
+++ /dev/null
@@ -1,170 +0,0 @@
-<?xml version="1.0" encoding="utf-8"?>
-<!DOCTYPE group [
-<!ENTITY mdash "&#8212;">
-]>
-<group from="2.1.3" to="2.1.4">
-  <subsystem>
-    <name>varnishd</name>
-    <change type="bug">
-      <para>An embarrasing typo in the new binary heap layout caused
-      inflated obj/objcore/objhdr counts and could cause odd problems
-      when the LRU expunge mechanism was invoked.  This has been
-      fixed.</para>
-    </change>
-
-    <change type="enh">
-      <para>We now have updated documentation in the reStructuredText
-      format.  Manual pages and reference documentation are both built
-      from this.</para>
-    </change>
-
-    <change type="enh">
-      <para>We now include a DNS director which uses DNS for choosing
-      which backend to route requests to.  Please see the
-      documentation for more details.</para>
-    </change>
-
-    <change type="bug" ref="758">
-      <para>If you restarted a request, the HTTP header
-      <code>X-Forwarded-For</code> would be updated multiple times.
-      This has been fixed.</para>
-    </change>
-
-    <change type="bug" ref="742">
-      <para>If a VCL contained a % sign, and the <code>vcl.show</code>
-      CLI command was used, varnishd would crash.  This has been
-      fixed.</para>
-    </change>
-
-    <change type="bug" ref="730">
-      <para>When doing a pass operation, we would remove the
-      <code>Content-Length</code>, <code>Age</code> and
-      <code>Proxy-Auth</code> headers.  We are no longer doing
-      this.</para>
-    </change>
-
-    <change type="enh">
-      <para><code>now</code> has a string representation, making it
-      easier to construct <code>Expires</code> headers in VCL.</para>
-    </change>
-
-    <change type="bug" ref="739">
-      <para>In a high traffic environment, we would sometimes reuse a
-      file descriptor before flushing the logs from a worker thread to
-      the shared log buffer.  This would cause confusion in some of
-      the tools.  This has been fixed by explicitly flushing the log
-      when a backend connection is closed.</para>
-    </change>
-
-    <change type="bug" ref="744">
-      <para>If the communication between the management and the child
-      process gets out of sync, we have no way to recover.
-      Previously, <code>varnishd</code> would be confused, but we now
-      just kill the child and restart it.</para>
-    </change>
-
-    <change type="bug" ref="479">
-      <para>If the backend closes the connection on us just as we sent
-      a request to it, we retry the request.  This should solve some
-      interoperability problems with Apache and the mpm-itk multi
-      processing module.</para>
-    </change>
-
-    <change type="enh">
-      <para><code>varnishd</code> now only provides help output the
-      current CLI session is authenticated for.</para>
-    </change>
-
-    <change type="bug" ref="733">
-      <para>If the backend does not tell us which length indication it
-      is using, we now assume the resource ends EOF at.</para>
-    </change>
-
-    <change type="enh">
-      <para>The client director now has a variable
-      <code>client.identity</code> which is used to choose which
-      backend should receive a given request.</para>
-    </change>
-
-    <change type="enh">
-      <para>The Solaris <code>port</code> waiter has been updated, and
-      other portability fixes for Solaris.</para>
-    </change>
-
-    <change type="bug" ref="746">
-      <para>There was a corner case in the close-down processing of pipes, this has now been fixed.</para>
-    </change>
-
-    <change type="bug">
-      <para>Previously, if we stopped polling a backend which was
-      sick, it never got marked as healthy.  This has now been
-      changed.</para>
-    </change>
-
-    <change type="enh">
-      <para>It is now possible to specify ports as part of the <code>.host</code> field in VCL.</para>
-    </change>
-
-    <change type="bug" ref="768">
-      <para>The synthetic counters were not locked properly, and so
-      the <code>sms_</code> counters could underflow.  This has now
-      been fixed.</para>
-    </change>
-
-    <change type="bug" ref="769">
-      <para>The value of <code>obj.status</code> as a string in vcl_error would not be correct in all cases.  This has been fixed.</para>
-    </change>
-
-    <change type="bug" ref="776">
-      <para>Varnish would try to trim storage segments completely
-      filled when using the malloc stevedore and the object was
-      received chunked encoding.  This has been fixed.</para>
-    </change>
-
-    <change type="bug" ref="763">
-      <para>If a buggy backend sends us a <code>Vary</code> header
-      with two colons, we would previously abort.  We now rather fix
-      this up and ignore the extra colon.</para>
-    </change>
-
-    <change type="enh">
-      <para><code>req.hash_always_miss</code> and
-      <code>req.hash_ignore_busy</code> has been added, to make
-      preloading or periodically refreshing content work better.</para>
-    </change>
-
-  </subsystem>
-  <subsystem>
-    <name>varnishncsa</name>
-    <change type="bug">
-      <para><code>varnishncsa</code> would in some cases be confused
-      by ESI requests and output invalid lines.  This has now been
-      fixed.</para>
-    </change>
-  </subsystem>
-
-  <subsystem>
-    <name>varnishlog</name>
-    <change type="enh">
-      <para><code>varnishlog</code> now allows <code>-o</code> and <code>-u</code> together.</para>
-    </change>
-  </subsystem>
-
-  <subsystem>
-    <name>varnishtop</name>
-    <change type="bug">
-      <para><code>varnishtop</code> would crash on 32 bit
-      architectures.  This has been fixed.</para>
-    </change>
-  </subsystem>
-
-  <subsystem>
-    <name>libvarnishapi</name>
-    <change type="bug">
-      <para>Regex inclusion and exclusion had problems with matching
-      particular parts of the string being matched.  This has been
-      fixed.</para>
-    </change>
-  </subsystem>
-
-</group>
diff --git a/doc/changes-2.1.3.xml b/doc/changes-2.1.3.xml
deleted file mode 100644
index 6cfe24d..0000000
--- a/doc/changes-2.1.3.xml
+++ /dev/null
@@ -1,12 +0,0 @@
-<?xml version="1.0" encoding="utf-8"?>
-<?xml-stylesheet type="text/xml" href="changes-html.xsl"?>
-<!DOCTYPE changelog [
- <!ENTITY mdash "&#8212;">
-]>
-<!-- $Id: changes-2.1.3.xml 4709 2010-04-21 10:40:27Z tfheen $ -->
-<changelog xmlns:xi="http://www.w3.org/2001/XInclude">
-  <package>Varnish</package>
-  <version>2.1.3</version>
-
-  <xi:include href="changes-2.1.2-2.1.3.xml"/>
-</changelog>
diff --git a/doc/changes-2.1.4-2.1.5.xml b/doc/changes-2.1.4-2.1.5.xml
deleted file mode 100644
index 9b7e8ae..0000000
--- a/doc/changes-2.1.4-2.1.5.xml
+++ /dev/null
@@ -1,132 +0,0 @@
-<?xml version="1.0" encoding="utf-8"?>
-<!DOCTYPE group [
-<!ENTITY mdash "&#8212;">
-]>
-<group from="2.1.4" to="2.1.5">
-  <subsystem>
-    <name>varnishd</name>
-    <change type="bug" ref="801">
-      <para>On pass from vcl_recv, we did not remove the backends Content-Length
-      header before adding our own. This could cause confusion for
-      browsers and has been fixed.</para>
-    </change>
-   
-    <change type="bug" ref="806">
-      <para>Make pass with content-length work again. An issue with regards
-      to 304, Content-Length and pass has been resolved.</para>
-    </change>
-
-    <change type="bug" ref="803">
-      <para>An issue relating to passed requests with If-Modified-Since
-      headers has been fixed. Varnish did not recognize that the
-      304-response did not have a body.</para>
-    </change>
-   
-    <change type="bug">
-      <para>A potential lock-inversion with the ban lurker thread has been
-      resolved.</para>
-    </change>
-
-    <change type="bug">
-      <para>Several build-dependency issues relating to rst2man have been
-      fixed. Varnish should now build from source without rst2man if you
-      are using tar-balls.</para>
-    </change>
-
-    <change type="bug" ref="780">
-      <para>Ensure Varnish reads the expected last CRLF after chunked data
-      from the backend. This allows re-use of the connection.</para>
-    </change>
-
-    <change type="bug">
-      <para>Remove a GNU Make-ism during <code>make dist</code> to make BSD
-      happier.</para>
-    </change>
-
-    <change type="enh">
-      <para>Document the log, set, unset, return and restart statements in
-      the VCL documentation.</para>
-    </change>
-
-    <change type="bug" ref="808">
-      <para>Fix an embarrassingly old bug where Varnish would run out of
-      workspace when requests come in fast over a single connection,
-      typically during synthetic benchmarks.</para>
-    </change>
-
-    <change type="bug" ref="795">
-      <para>Varnish will now allow <code>If-Modified-Since</code> requests
-      to objects without a <code>Last-Modified</code>-header, and instead
-      use the time the object was cached instead.</para>
-    </change>
-
-    <change type="bug" ref="789">
-      <para>Do not filter out <code>Content-Range</code> headers in
-      pass.</para>
-    </change>
-
-    <change type="enh">
-      <para>Require -d, -b, -f, -S or -T when starting Varnishd. In human
-      terms, this means that it is legal to start varnishd without a Vcl or
-      backend, but only if you have a CLI channel of some kind.</para>
-    </change>
-
-    <change type="bug">
-      <para>Don't suppress <code>Cache-Control</code> headers in pass
-      responses.</para>
-    </change>
-
-    <change type="enh">
-      <para>Merge multi-line Cache-Control and Vary header fields. Until
-      now, no browsers have needed this, but Chromium seems to find it
-      necessary to spread its Cache-Control across two lines, and we get to
-      deal with it.</para>
-    </change>
-
-    <change type="bug" ref="812">
-      <para>Make new-purge not touch busy objects. This fixes a potential
-      crash when calling <code>VRT_purge</code>.</para>
-    </change>
-    
-    <change type="enh">
-      <para>If there are everal grace-able objects, pick the least expired
-      one.</para>
-    </change>
-
-    <change type="bug" ref="817">
-      <para>Fix an issue with <code>varnishadm -T :6082</code>
-      shorthand.</para>
-    </change>
-
-    <change type="enh">
-      <para>Add bourn-shell like "here" documents on the CLI. Typical
-      usage:
-      <code>
-      vcl.inline vcl_new &lt;&lt; 42
-      backend foo {...}
-      sub vcl_recv {...}
-      42
-      </code></para>
-    </change>
-
-    <change type="enh">
-      <para>Add CLI version to the CLI-banner, starting with version 1.0 to
-      mark here-documents.</para>
-    </change>
-
-    <change type="bug">
-      <para>Fix a problem with the expiry thread slacking off during high
-      load.</para>
-    </change>
-
-  </subsystem>
-
-  <subsystem>
-    <name>varnishtest</name>
-    <change type="bug" ref="804">
-      <para>Remove no longer existing <code>-L</code> option.</para>
-    </change>
-  </subsystem>
-
-
-</group>
diff --git a/doc/changes-2.1.4.xml b/doc/changes-2.1.4.xml
deleted file mode 100644
index c9211d8..0000000
--- a/doc/changes-2.1.4.xml
+++ /dev/null
@@ -1,12 +0,0 @@
-<?xml version="1.0" encoding="utf-8"?>
-<?xml-stylesheet type="text/xml" href="changes-html.xsl"?>
-<!DOCTYPE changelog [
- <!ENTITY mdash "&#8212;">
-]>
-<!-- $Id: changes-2.1.4.xml 4709 2010-04-21 10:40:27Z tfheen $ -->
-<changelog xmlns:xi="http://www.w3.org/2001/XInclude">
-  <package>Varnish</package>
-  <version>2.1.4</version>
-
-  <xi:include href="changes-2.1.3-2.1.4.xml"/>
-</changelog>
diff --git a/doc/changes-2.1.5.xml b/doc/changes-2.1.5.xml
deleted file mode 100644
index 7e37731..0000000
--- a/doc/changes-2.1.5.xml
+++ /dev/null
@@ -1,11 +0,0 @@
-<?xml version="1.0" encoding="utf-8"?>
-<?xml-stylesheet type="text/xml" href="changes-html.xsl"?>
-<!DOCTYPE changelog [
- <!ENTITY mdash "&#8212;">
-]>
-<changelog xmlns:xi="http://www.w3.org/2001/XInclude">
-  <package>Varnish</package>
-  <version>2.1.5</version>
-
-  <xi:include href="changes-2.1.4-2.1.5.xml"/>
-</changelog>
diff --git a/doc/changes-html.xsl b/doc/changes-html.xsl
deleted file mode 100644
index eeaa1a4..0000000
--- a/doc/changes-html.xsl
+++ /dev/null
@@ -1,92 +0,0 @@
-<?xml version="1.0" encoding="utf-8"?>
-<!DOCTYPE stylesheet [
- <!ENTITY space "&#32;">
- <!ENTITY nbsp "&#160;">
-]>
-<!-- $Id$ -->
-<xsl:stylesheet version="1.0"
- xmlns:xsl="http://www.w3.org/1999/XSL/Transform"
- xmlns="http://www.w3.org/1999/xhtml">
-  <xsl:output method="xml" encoding="utf-8" media-type="text/html" indent="yes"/>
-
-  <xsl:strip-space elements="*"/>
-
-  <xsl:template match="/changelog">
-    <html>
-      <head>
-	<title><xsl:call-template name="title"/></title>
-	<link rel="stylesheet" type="text/css" href="changes.css"/>
-      </head>
-      <body>
-	<h1><xsl:call-template name="title"/></h1>
-	<xsl:apply-templates select="group"/>
-      </body>
-    </html>
-  </xsl:template>
-
-  <xsl:template name="title">
-    <xsl:text>Change log for&space;</xsl:text>
-    <xsl:value-of select="package"/>
-    <xsl:text>&space;</xsl:text>
-    <xsl:value-of select="version"/>
-  </xsl:template>
-
-  <xsl:template match="group">
-    <h2>
-      <xsl:text>Changes between&space;</xsl:text>
-      <xsl:value-of select="@from"/>
-      <xsl:text>&space;and&space;</xsl:text>
-      <xsl:value-of select="@to"/>
-    </h2>
-    <xsl:apply-templates select="subsystem"/>
-  </xsl:template>
-
-  <xsl:template match="subsystem">
-    <h3>
-      <xsl:value-of select="name"/>
-    </h3>
-    <ul>
-      <xsl:apply-templates select="change"/>
-    </ul>
-  </xsl:template>
-
-  <xsl:template match="change">
-    <li>
-      <xsl:apply-templates/>
-    </li>
-  </xsl:template>
-
-  <xsl:template match="para">
-    <p>
-      <xsl:apply-templates/>
-    </p>
-  </xsl:template>
-
-  <xsl:template match="ticket">
-    <a>
-      <xsl:attribute name="href">
-	<xsl:text>http://varnish.projects.linpro.no/ticket/</xsl:text>
-	<xsl:value-of select="@ref"/>
-      </xsl:attribute>
-      <xsl:text>ticket #</xsl:text>
-      <xsl:value-of select="@ref"/>
-    </a>
-  </xsl:template>
-
-  <xsl:template match="code">
-    <span>
-      <xsl:attribute name="class">
-	<xsl:value-of select="name()"/>
-      </xsl:attribute>
-      <xsl:apply-templates/>
-    </span>
-  </xsl:template>
-
-  <xsl:template match="*" priority="-1">
-    <xsl:message>Warning: no template for element <xsl:value-of select="name(
-)"/></xsl:message>
-    <xsl:value-of select="concat('&lt;', name(), '&gt;')"/>
-    <xsl:apply-templates/>
-    <xsl:value-of select="concat('&lt;/', name(), '&gt;')"/>
-  </xsl:template>
-</xsl:stylesheet>
diff --git a/doc/changes-wiki.xsl b/doc/changes-wiki.xsl
deleted file mode 100644
index 9ff0d11..0000000
--- a/doc/changes-wiki.xsl
+++ /dev/null
@@ -1,76 +0,0 @@
-<?xml version="1.0" encoding="utf-8"?>
-<!DOCTYPE stylesheet [
- <!ENTITY lf "&#10;">
-]>
-<!-- $Id$ -->
-<xsl:stylesheet version="1.0"
- xmlns:xsl="http://www.w3.org/1999/XSL/Transform"
- xmlns="http://www.w3.org/1999/xhtml">
-  <xsl:output method="text" encoding="utf-8"/>
-
-  <xsl:strip-space elements="*"/>
-
-  <xsl:template match="/changelog">
-    <xsl:text>== </xsl:text>
-    <xsl:call-template name="title"/>
-    <xsl:text> ==&lf;</xsl:text>
-    <xsl:apply-templates select="group"/>
-  </xsl:template>
-
-  <xsl:template name="title">
-    <xsl:text>Change log for </xsl:text>
-    <xsl:value-of select="package"/>
-    <xsl:text> </xsl:text>
-    <xsl:value-of select="version"/>
-  </xsl:template>
-
-  <xsl:template match="group">
-    <xsl:text>=== </xsl:text>
-    <xsl:text>Changes between </xsl:text>
-    <xsl:value-of select="@from"/>
-    <xsl:text> and </xsl:text>
-    <xsl:value-of select="@to"/>
-    <xsl:text> ===&lf;</xsl:text>
-    <xsl:apply-templates select="subsystem"/>
-  </xsl:template>
-
-  <xsl:template match="subsystem">
-    <xsl:text>==== </xsl:text>
-    <xsl:value-of select="name"/>
-    <xsl:text> ====&lf;</xsl:text>
-    <xsl:apply-templates select="change"/>
-  </xsl:template>
-
-  <xsl:template match="change">
-    <xsl:text> * </xsl:text>
-    <xsl:apply-templates/>
-  </xsl:template>
-
-  <xsl:template match="para">
-    <xsl:apply-templates/>
-    <xsl:text>&lf;</xsl:text>
-  </xsl:template>
-
-  <xsl:template match="ticket">
-    <xsl:text>#</xsl:text>
-    <xsl:value-of select="@ref"/>
-  </xsl:template>
-
-  <xsl:template match="code">
-    <xsl:text> {{{</xsl:text>
-    <xsl:apply-templates/>
-    <xsl:text>}}} </xsl:text>
-  </xsl:template>
-
-  <xsl:template match="text()">
-    <xsl:value-of select="normalize-space()"/>
-  </xsl:template>
-
-  <xsl:template match="*" priority="-1">
-    <xsl:message>Warning: no template for element <xsl:value-of select="name(
-)"/></xsl:message>
-    <xsl:value-of select="concat('&lt;', name(), '&gt;')"/>
-    <xsl:apply-templates/>
-    <xsl:value-of select="concat('&lt;/', name(), '&gt;')"/>
-  </xsl:template>
-</xsl:stylesheet>
diff --git a/doc/changes.css b/doc/changes.css
deleted file mode 100644
index f3d5c71..0000000
--- a/doc/changes.css
+++ /dev/null
@@ -1,31 +0,0 @@
-/* $Id$ */
-
-body {
-	background-color: white;
-	color: black;
-	font-family: sans-serif;
-	max-width: 40em;
-	margin: 1in auto 1in auto;
-}
-
-h1 {
-	font-size: 200%;
-	font-weight: bold;
-	color: maroon;
-}
-
-h2 {
-	font-size: 160%;
-	font-weight: bold;
-	color: maroon;
-}
-
-h3 {
-	font-size: 120%;
-	font-weight: bold;
-	color: maroon;
-}
-
-.code {
-	font-family: monospace;
-}
diff --git a/doc/changes.rst b/doc/changes.rst
index 06540bf..cbf4834 100644
--- a/doc/changes.rst
+++ b/doc/changes.rst
@@ -425,3 +425,1416 @@ Other
   version.
 - The documentation has been improved all over and should now be in
   much better shape than before
+
+===========================
+Changes from 2.1.4 to 2.1.5
+===========================
+
+varnishd
+--------
+
+-  On pass from vcl\_recv, we did not remove the backends Content-Length
+   header before adding our own. This could cause confusion for browsers
+   and has been fixed.
+
+-  Make pass with content-length work again. An issue with regards to
+   304, Content-Length and pass has been resolved.
+
+-  An issue relating to passed requests with If-Modified-Since headers
+   has been fixed. Varnish did not recognize that the 304-response did
+   not have a body.
+
+-  A potential lock-inversion with the ban lurker thread has been
+   resolved.
+
+-  Several build-dependency issues relating to rst2man have been fixed.
+   Varnish should now build from source without rst2man if you are using
+   tar-balls.
+
+-  Ensure Varnish reads the expected last CRLF after chunked data from
+   the backend. This allows re-use of the connection.
+
+-  Remove a GNU Make-ism during make dist to make BSD happier.
+
+-  Document the log, set, unset, return and restart statements in the
+   VCL documentation.
+
+-  Fix an embarrassingly old bug where Varnish would run out of
+   workspace when requests come in fast over a single connection,
+   typically during synthetic benchmarks.
+
+-  Varnish will now allow If-Modified-Since requests to objects without
+   a Last-Modified-header, and instead use the time the object was
+   cached instead.
+
+-  Do not filter out Content-Range headers in pass.
+
+-  Require -d, -b, -f, -S or -T when starting Varnishd. In human terms,
+   this means that it is legal to start varnishd without a Vcl or
+   backend, but only if you have a CLI channel of some kind.
+
+-  Don't suppress Cache-Control headers in pass responses.
+
+-  Merge multi-line Cache-Control and Vary header fields. Until now, no
+   browsers have needed this, but Chromium seems to find it necessary to
+   spread its Cache-Control across two lines, and we get to deal with
+   it.
+
+-  Make new-purge not touch busy objects. This fixes a potential crash
+   when calling VRT\_purge.
+
+-  If there are everal grace-able objects, pick the least expired one.
+
+-  Fix an issue with varnishadm -T :6082 shorthand.
+
+-  Add bourn-shell like "here" documents on the CLI. Typical usage:
+   vcl.inline vcl\_new << 42 backend foo {...} sub vcl\_recv {...} 42
+
+-  Add CLI version to the CLI-banner, starting with version 1.0 to mark
+   here-documents.
+
+-  Fix a problem with the expiry thread slacking off during high load.
+
+varnishtest
+-----------
+
+-  Remove no longer existing -L option.
+
+===========================
+Changes from 2.1.3 to 2.1.4
+===========================
+
+varnishd
+--------
+
+-  An embarrasing typo in the new binary heap layout caused inflated
+   obj/objcore/objhdr counts and could cause odd problems when the LRU
+   expunge mechanism was invoked. This has been fixed.
+
+-  We now have updated documentation in the reStructuredText format.
+   Manual pages and reference documentation are both built from this.
+
+-  We now include a DNS director which uses DNS for choosing which
+   backend to route requests to. Please see the documentation for more
+   details.
+
+-  If you restarted a request, the HTTP header X-Forwarded-For would be
+   updated multiple times. This has been fixed.
+
+-  If a VCL contained a % sign, and the vcl.show CLI command was used,
+   varnishd would crash. This has been fixed.
+
+-  When doing a pass operation, we would remove the Content-Length, Age
+   and Proxy-Auth headers. We are no longer doing this.
+
+-  now has a string representation, making it easier to construct
+   Expires headers in VCL.
+
+-  In a high traffic environment, we would sometimes reuse a file
+   descriptor before flushing the logs from a worker thread to the
+   shared log buffer. This would cause confusion in some of the tools.
+   This has been fixed by explicitly flushing the log when a backend
+   connection is closed.
+
+-  If the communication between the management and the child process
+   gets out of sync, we have no way to recover. Previously, varnishd
+   would be confused, but we now just kill the child and restart it.
+
+-  If the backend closes the connection on us just as we sent a request
+   to it, we retry the request. This should solve some interoperability
+   problems with Apache and the mpm-itk multi processing module.
+
+-  varnishd now only provides help output the current CLI session is
+   authenticated for.
+
+-  If the backend does not tell us which length indication it is using,
+   we now assume the resource ends EOF at.
+
+-  The client director now has a variable client.identity which is used
+   to choose which backend should receive a given request.
+
+-  The Solaris port waiter has been updated, and other portability fixes
+   for Solaris.
+
+-  There was a corner case in the close-down processing of pipes, this
+   has now been fixed.
+
+-  Previously, if we stopped polling a backend which was sick, it never
+   got marked as healthy. This has now been changed.
+
+-  It is now possible to specify ports as part of the .host field in
+   VCL.
+
+-  The synthetic counters were not locked properly, and so the sms\_
+   counters could underflow. This has now been fixed.
+
+-  The value of obj.status as a string in vcl\_error would not be
+   correct in all cases. This has been fixed.
+
+-  Varnish would try to trim storage segments completely filled when
+   using the malloc stevedore and the object was received chunked
+   encoding. This has been fixed.
+
+-  If a buggy backend sends us a Vary header with two colons, we would
+   previously abort. We now rather fix this up and ignore the extra
+   colon.
+
+-  req.hash\_always\_miss and req.hash\_ignore\_busy has been added, to
+   make preloading or periodically refreshing content work better.
+
+varnishncsa
+-----------
+
+-  varnishncsa would in some cases be confused by ESI requests and
+   output invalid lines. This has now been fixed.
+
+varnishlog
+----------
+
+-  varnishlog now allows -o and -u together.
+
+varnishtop
+----------
+
+-  varnishtop would crash on 32 bit architectures. This has been fixed.
+
+libvarnishapi
+-------------
+
+-  Regex inclusion and exclusion had problems with matching particular
+   parts of the string being matched. This has been fixed.
+
+
+===========================
+Changes from 2.1.2 to 2.1.3
+===========================
+
+varnishd
+--------
+
+-  Improve scalability of critbit.
+
+-  The critbit hash algorithm has now been tightened to make sure the
+   tree is in a consistent state at all points, and the time we wait for
+   an object to cool off after it is eligible for garbage collection has
+   been tweaked.
+
+-  Add log command to VCL. This emits a VCL\_log entry into the shared
+   memory log.
+
+-  Only emit Length and ReqEnd log entries if we actually have an XID.
+   This should get rid of some empty log lines in varnishncsa.
+
+-  Destroy directors in a predictable fashion, namely reverse of
+   creation order.
+
+-  Fix bug when ESI elements spanned storage elements causing a panic.
+
+-  In some cases, the VCL compiler would panic instead of giving
+   sensible messages. This has now been fixed.
+
+-  Correct an off-by-one error when the requested range exceeds the size
+   of an object.
+
+-  Handle requests for the end of an object correctly.
+
+-  Allow tabulator characters in the third field of the first line of
+   HTTP requests
+
+-  On Solaris, if the remote end sends us an RST, all system calls
+   related to that socket will return EINVAL. We now handle this better.
+
+libvarnishapi
+-------------
+
+-  The -X parameter didn't work correctly. This has been fixed.
+
+===========================
+Changes from 2.1.1 to 2.1.2
+===========================
+
+varnishd
+--------
+
+-  When adding Range support for 2.1.1, we accidentially introduced a
+   bug which would append garbage to objects larger than the chunk size,
+   by default 128k. Browsers would do the right thing due to
+   Content-Length, but some load balancers would get very confused.
+
+===========================
+Changes from 2.1.1 to 2.1.1
+===========================
+
+varnishd
+--------
+
+-  The changelog in 2.1.0 included syntax errors, causing the generated
+   changelog to be empty.
+
+-  The help text for default\_grace was wrongly formatted and included a
+   syntax error. This has now been fixed.
+
+-  varnishd now closes the file descriptor used to read the management
+   secret file (from the -S parameter).
+
+-  The child would previously try to close every valid file descriptor,
+   something which could cause problems if the file descriptor ulimit
+   was set too high. We now keep track of all the file descriptors we
+   open and only close up to that number.
+
+-  ESI was partially broken in 2.1.0 due to a bug in the rollback of
+   session workspace. This has been fixed.
+
+-  Reject the authcommand rather than crash if there is no -S parameter
+   given.
+
+-  Align pointers in allocated objects. This will in theory make Varnish
+   a tiny bit faster at the expense of slightly more memory usage.
+
+-  Ensure the master process process id is updated in the shared memory
+   log file after we go into the background.
+
+-  HEAD requests would be converted to GET requests too early, which
+   affected pass and pipe. This has been fixed.
+
+-  Update the documentation to point out that the TTL is no longer taken
+   into account to decide whether an object is cacheable or not.
+
+-  Add support for completely obliterating an object and all variants of
+   it. Currently, this has to be done using inline C.
+
+-  Add experimental support for the Range header. This has to be enabled
+   using the parameter http\_range\_support.
+
+-  The critbit hasher could get into a deadlock and had a race
+   condition. Both those have now been fixed.
+
+varnishsizes
+-----------~
+
+-  varnishsizes, which is like varnishhost, but for the length of
+   objects, has been added..
+
+
+===========================
+Changes from 2.0.6 to 2.1.0
+===========================
+
+varnishd
+--------
+
+-  Persistent storage is now experimentally supported using the
+   persistent stevedore. It has the same command line arguments as the
+   file stevedore.
+
+-  obj.\* is now called beresp.\* in vcl\_fetch, and obj.\* is now
+   read-only.
+
+-  The regular expression engine is now PCRE instead of POSIX regular
+   expressions.
+
+-  req.\* is now available in vcl\_deliver.
+
+-  Add saint mode where we can attempt to grace an object if we don't
+   like the backend response for some reason.
+
+   Related, add saintmode\_threshold which is the threshold for the
+   number of objects to be added to the trouble list before the backend
+   is considered sick.
+
+-  Add a new hashing method called critbit. This autoscales and should
+   work better on large object workloads than the classic hash. Critbit
+   has been made the default hash algorithm.
+
+-  When closing connections, we experimented with sending RST to free up
+   load balancers and free up threads more quickly. This caused some
+   problems with NAT routers and so has been reverted for now.
+
+-  Add thread that checks objects against ban list in order to prevent
+   ban list from growing forever. Note that this needs purges to be
+   written so they don't depend on req.\*. Enabled by setting
+   ban\_lurker\_sleep to a nonzero value.
+
+-  The shared memory log file format was limited to maximum 64k
+   simultaneous connections. This is now a 32 bit field which removes
+   this limitation.
+
+-  Remove obj\_workspace, this is now sized automatically.
+
+-  Rename acceptors to waiters
+
+-  vcl\_prefetch has been removed. It was never fully implemented.
+
+-  Add support for authenticating CLI connections.
+
+-  Add hash director that chooses which backend to use depending on
+   req.hash.
+
+-  Add client director that chooses which backend to use depending on
+   the client's IP address. Note that this ignores the X-Forwarded-For
+   header.
+
+-  varnishd now displays a banner by default when you connect to the
+   CLI.
+
+-  Increase performance somewhat by moving statistics gathering into a
+   per-worker structure that is regularly flushed to the global stats.
+
+-  Make sure we store the header and body of object together. This may
+   in some cases improve performance and is needed for persistence.
+
+-  Remove client-side address accounting. It was never used for anything
+   and presented a performance problem.
+
+-  Add a timestamp to bans, so you can know how old they are.
+
+-  Quite a few people got confused over the warning about not being able
+   to lock the shared memory log into RAM, so stop warning about that.
+
+-  Change the default CLI timeout to 10 seconds.
+
+-  We previously forced all inserts into the cache to be GET requests.
+   This has been changed to allow POST as well in order to be able to
+   implement purge-on-POST semantics.
+
+-  The CLI command stats now only lists non-zero values.
+
+-  The CLI command stats now only lists non-zero values.
+
+-  Use daemon(3) from libcompat on Darwin.
+
+-  Remove vcl\_discard as it causes too much complexity and never
+   actually worked particularly well.
+
+-  Remove vcl\_timeout as it causes too much complexity and never
+   actually worked particularly well.
+
+-  Update the documentation so it refers to sess\_workspace, not
+   http\_workspace.
+
+-  Document the -i switch to varnishd as well as the server.identity and
+   server.hostname VCL variables.
+
+-  purge.hash is now deprecated and no longer shown in help listings.
+
+-  When processing ESI, replace the five mandatory XML entities when we
+   encounter them.
+
+-  Add string representations of time and relative time.
+
+-  Add locking for n\_vbe\_conn to make it stop underflowing.
+
+-  When ESI-processing content, check for illegal XML character
+   entities.
+
+-  Varnish can now connect its CLI to a remote instance when starting
+   up, rather than just being connected to.
+
+-  It is no longer needed to specify the maximum number of HTTP headers
+   to allow from backends. This is now a run-time parameter.
+
+-  The X-Forwarded-For header is now generated by vcl\_recv rather than
+   the C code.
+
+-  It is now possible to not send all CLI traffic to syslog.
+
+-  It is now possible to not send all CLI traffic to syslog.
+
+-  In the case of varnish crashing, it now outputs a identifying string
+   with the OS, OS revision, architecture and storage parameters
+   together with the backtrace.
+
+-  Use exponential backoff when we run out of file descriptors or
+   sessions.
+
+-  Allow setting backend timeouts to zero.
+
+-  Count uptime in the shared memory log.
+
+-  Try to detect the case of two running varnishes with the same shmlog
+   and storage by writing the master and child process ids to the shmlog
+   and refusing to start if they are still running.
+
+-  Make sure to use EOF mode when serving ESI content to HTTP/1.0
+   clients.
+
+-  Make sure we close the connection if it either sends Connection:
+   close or it is a HTTP/1.0 backend that does not send Connection:
+   keep-alive.
+
+-  Increase the default session workspace to 64k on 64-bit systems.
+
+-  Make the epoll waiter use level triggering, not edge triggering as
+   edge triggering caused problems on very busy servers.
+
+-  Handle unforeseen client disconnections better on Solaris.
+
+-  Make session lingering apply to new sessions, not just reused
+   sessions.
+
+varnishstat
+-----------
+
+-  Make use of the new uptime field in the shared memory log rather than
+   synthesizing it from the start time.
+
+varnishlog
+----------
+
+-  Exit at the end of the file when started with -d.
+
+varnishadm
+----------
+
+-  varnishadm can now have a timeout when trying to connect to the
+   running varnishd.
+
+-  varnishadm now knows how to respond to the secret from a secured
+   varnishd
+
+===========================
+Changes from 2.0.5 to 2.0.6
+===========================
+
+varnishd
+--------
+
+-  2.0.5 had an off-by-one error in the ESI handling causing includes to
+   fail a large part of the time. This has now been fixed.
+
+-  Try harder to not confuse backends when sending them backend probes.
+   We half-closed the connection, something some backends thought meant
+   we had dropped the connection. Stop doing so, and add the capability
+   for specifying the expected response code.
+
+-  In 2.0.5, session lingering was turned on. This caused statistics to
+   not be counted often enough in some cases. This has now been fixed.
+
+-  Avoid triggering an assert if the other end closes the connection
+   while we are lingering and waiting for another request from them.
+
+-  When generating backtraces, prefer the built-in backtrace function if
+   such exists. This fixes a problem compiling 2.0.5 on Solaris.
+
+-  Make it possible to specify the per-thread stack size. This might be
+   useful on 32 bit systems with their limited address space.
+
+-  Document the -C option to varnishd.
+
+===========================
+Changes from 2.0.4 to 2.0.5
+===========================
+
+varnishd
+--------
+
+-  Handle object workspace overruns better.
+
+-  Allow turning off ESI processing per request by using set req.esi =
+   off.
+
+-  Tell the kernel that we expect to use the mmap-ed file in a random
+   fashion. On Linux, this turns off/down readahead and increases
+   performance.
+
+-  Make it possible to change the maximum number of HTTP headers we
+   allow by passing --with-max-header-fields=NUM rather than changing
+   the code.
+
+-  Implement support for HTTP continuation lines.
+
+-  Change how connections are closed and only use SO\_LINGER for orderly
+   connection closure. This should hopefully make worker threads less
+   prone to hangups on network problems.
+
+-  Handle multi-element purges correctly. Previously we ended up with
+   parse errors when this was done from VCL.
+
+-  Handle illegal responses from the backend better by serving a 503
+   page rather than panic-ing.
+
+-  When we run into an assertion that is not true, Varnish would
+   previously dump a little bit of information about itself. Extend that
+   information with a backtrace. Note that this relies on the varnish
+   binary being unstripped.
+
+-  Add a session\_max parameter that limits the maximum number of
+   sessions we keep open before we start dropping new connections
+   summarily.
+
+-  Try to consume less memory when doing ESI processing by properly
+   rolling back used workspace after processing an object. This should
+   make it possible to turn sess\_workspace quite a bit for users with
+   ESI-heavy pages.
+
+-  Turn on session\_linger by default. Tests have shown that
+   session\_linger helps a fair bit with performance.
+
+-  Rewrite the epoll acceptor for better performance. This should lead
+   to both higher processing rates and maximum number of connections on
+   Linux.
+
+-  Add If-None-Match support, this gives significant bandwidth savings
+   for users with compliant browsers.
+
+-  RFC2616 specifies that ETag, Content-Location, Expires, Cache-Control
+   and Vary should be emitted when delivering a response with the 304
+   response code.
+
+-  Various fixes which makes Varnish compile and work on AIX.
+
+-  Turn on TCP\_DEFER\_ACCEPT on Linux. This should make us less
+   suspecible to denial of service attacks as well as give us slightly
+   better performance.
+
+-  Add an .initial property to the backend probe specification. This is
+   the number of good probes we pretend to have seen. The default is one
+   less than .threshold, which means the first probe will decide if we
+   consider the backend healthy.
+
+-  Make it possible to compare strings against other string-like
+   objects, not just plain strings. This allows you to compare two
+   headers, for instance.
+
+-  When support for restart in vcl\_error was added, there was no check
+   to prevent infinte recursion. This has now been fixed.
+
+-  Turn on purge\_dups by default. This should make us consume less
+   memory when there are many bans for the same pattern added.
+
+-  Add a new log tag called FetchError which tries to explain why we
+   could not fetch an object from the backend.
+
+-  Change the default srcaddr\_ttl to 0. It is not used by anything and
+   has been removed in the development version. This will increase
+   performance somewhat.
+
+varnishtop
+----------
+
+-  varnishtop did not handle variable-length log fields correctly. This
+   is now fixed.
+
+-  varnishtop previously did not print the name of the tag, which made
+   it very hard to understand. We now print out the tag name.
+
+===========================
+Changes from 2.0.3 to 2.0.4
+===========================
+
+varnishd
+--------
+
+-  Make Varnish more portable by pulling in fixes for Solaris and
+   NetBSD.
+
+-  Correct description of -a in the manual page.
+
+-  Ensure we are compiling in C99 mode.
+
+-  If error was called with a null reason, we would crash on Solaris.
+   Make sure this no longer happens.
+
+-  Varnish used to crash if you asked it to use a non-existent waiter.
+   This has now been fixed.
+
+-  Add documentation to the default VCL explaining that using
+   Connection: close in vcl\_close is generally a good idea.
+
+-  Add minimal facility for dealing with TELNET option negotiation by
+   returning WONT to DO and DONT requests.
+
+-  If the backend is unhealthy, use a graced object if one is available.
+
+-  Make server.hostname and server.identity available to VCL. The latter
+   can be set with the -i parameter to varnishd.
+
+-  Make restart available from vcl\_error.
+
+-  Previously, only the TTL of an object was considered in whether it
+   would be marked as cacheable. This has been changed to take the grace
+   into consideration as well.
+
+-  Previously, if an included ESI fragment had a zero size, we would
+   send out a zero-sized chunk which signifies end-of-transmission. We
+   now ignore zero-sized chunks.
+
+-  We accidentially slept for far too long when we reached the maximum
+   number of open file descriptors. This has been corrected and
+   accept\_fd\_holdoff now works correctly.
+
+-  Previously, when ESI processing, we did not look at the full length,
+   but stopped at the first NULL byte. We no longer do that, enabling
+   ESI processing of binary data.
+
+varnishtest
+-----------
+
+-  Make sure system "..." returns successfully to ensure test failures
+   do not go unnoticed.
+
+-  Make it possible to send NULL bytes through the testing framework.
+
+===========================
+Changes from 2.0.2 to 2.0.3
+===========================
+
+varnishd
+--------
+
+-  Handle If-Modified-Since and ESI sub-objects better, fixing a problem
+   where we sometimes neglected to insert included objects.
+
+-  restart in vcl\_hit is now supported.
+
+-  Setting the TTL of an object to 0 seconds would sometimes cause it to
+   be delivered for up to one second - epsilon. This has been corrected
+   and we should now never deliver those objects to other clients.
+
+-  The malloc storage backend now prints the maximum storage size, just
+   like the file backend.
+
+-  Various small documentation bugs have been fixed.
+
+-  Varnish did not set a default interval for backend probes, causing it
+   to poll the backend continuously. This has been corrected.
+
+-  Allow "true" and "false" when setting boolean parameters, in addition
+   to on/off, enable/disable and yes/no.
+
+-  Default to always talking HTTP 1.1 with the backend.
+
+-  Varnish did not make sure the file it was loading was a regular file.
+   This could cause Varnish to crash if it was asked to load a directory
+   or other non-regular file. We now check that the file is a regular
+   file before loading it.
+
+-  The binary heap used for expiry processing had scalability problems.
+   Work around this by using stripes of a fixed size, which should make
+   this scale better, particularly when starting up and having lots of
+   objects.
+
+-  When we imported the jemalloc library into the Varnish tree, it did
+   not compile without warnings. This has now been fixed.
+
+-  Varnish took a very long time to detect that the backend did not
+   respond. To remedy this, we now have read timeouts in addition to the
+   connect timeout. Both the first\_byte\_timeout and the
+   between\_bytes\_timeout defaults to 60 seconds. The connect timeout
+   is no longer in milliseconds, but rather in seconds.
+
+-  Previously, the VCL to C conversion as well as the invocation of the
+   C compiler was done in the management process. This is now done in a
+   separate sub-process. This prevents any bugs in the VCL compiler from
+   affecting the management process.
+
+-  Chunked encoding headers were counted in the statistics for header
+   bytes. They no longer are.
+
+-  ESI processed objects were not counted in the statistics for body
+   bytes. They now are.
+
+-  It is now possible to adjust the maximum record length of log entries
+   in the shmlog by tuning the shm\_reclen parameter.
+
+-  The management parameters listed in the CLI were not sorted, which
+   made it hard to find the parameter you were looking for. They are now
+   sorted, which should make this easier.
+
+-  Add a new hashing type, "critbit", which uses a lock-less tree based
+   lookup algorithm. This is experimental and should not be enabled in
+   production environments without proper testing.
+
+-  The session workspace had a default size of 8k. It is now 16k, which
+   should make VCLs where many headers are processed less prone to
+   panics.
+
+-  We have seen that people seem to be confused as to which actions in
+   the different VCL functions return and which ones don't. Add a new
+   syntax return(action) to make this more explicit. The old syntax is
+   still supported.
+
+-  Varnish would return an error if any of the management IPs listed in
+   the -T parameter could not be listened to. We now only return an
+   error if none of them can be listened to.
+
+-  In the case of the backend or client giving us too many parameters,
+   we used to just ignore the overflowing headers. This is problematic
+   if you end up ignoreing Content-Length, Transfer-Encoding and similar
+   headers. We now give out a 400 error to the client if it sends us too
+   many and 503 if we get too many from the backend.
+
+-  We used panic if we got a too large chunked header. This behaviour
+   has been changed into just failing the transaction.
+
+-  Varnish now supports an extended purge method where it is possible to
+   do purge req.http.host ~ "web1.com" && req.url ~ "\\.png" and
+   similar. See the documentation for details.
+
+-  Under heavy load, Varnish would sometimes crash when trying to update
+   the per-request statistics. This has now been fixed.
+
+-  It is now possible to not save the hash string in the session and
+   object workspace. This will save a lot of memory on sites with many
+   small objects. Disabling the purge\_hash parameter also disables the
+   purge.hash facility.
+
+-  Varnish now supports !~ as a "no match" regular expression matcher.
+
+-  In some cases, you could get serialised access to "pass" objects. We
+   now make it default to the default\_ttl value; this can be overridden
+   in vcl\_fetch.
+
+-  Varnish did not check the syntax of regsub calls properly. More
+   checking has been added.
+
+-  If the client closed the connection while Varnish was processing ESI
+   elements, Varnish would crash while trying to write the object to the
+   client. We now check if the client has closed the connection.
+
+-  The ESI parser had a bug where it would crash if an XML comment would
+   span storage segments. This has been fixed.
+
+VCL Manual page
+--------------~
+
+-  The documentation on how capturing parentheses work was wrong. This
+   has been corrected.
+
+-  Grace has now been documented.
+
+varnishreplay
+-------------
+
+-  varnishreplay did not work correctly on Linux, due to a too small
+   stack. This has now been fixed.
+
+===========================
+Changes from 2.0.1 to 2.0.2
+===========================
+
+varnishd
+--------
+
+-  In high-load situations, when using ESI, varnishd would sometimes
+   mishandle objects and crash. This has been worked around.
+
+varnishreplay
+-------------
+
+-  varnishreplay did not work correctly on Linux, due to a too small
+   stack. This has now been fixed.
+
+
+=========================
+Changes from 2.0 to 2.0.1
+=========================
+
+varnishd
+--------
+
+-  When receiving a garbled HTTP request, varnishd would sometimes
+   crash. This has been fixed.
+
+-  There was an off-by-one error in the ACL compilation. Now fixed.
+
+Red Hat spec file
+----------------~
+
+-  A typo in the spec file made the .rpm file names wrong.
+
+=========================
+Changes from 1.1.2 to 2.0
+=========================
+
+varnishd
+--------
+
+-  Only look for sendfile on platforms where we know how to use it,
+   which is FreeBSD for now.
+
+-  Make it possible to adjust the shared memory log size and bump the
+   size from 8MB to 80MB.
+
+-  Fix up the handling of request bodies to better match what RFC2616
+   mandates. This makes PUT, DELETE, OPTIONS and TRACE work in addition
+   to POST.
+
+-  Change how backends are defined, to a constant structural defintion
+   style. See http://varnish.projects.linpro.no/wiki/VclSyntaxChanges
+   for the details.
+
+-  Add directors, which wrap backends. Currently, there's a random
+   director and a round-robin director.
+
+-  Add "grace", which is for how long and object will be served, even
+   after it has expired. To use this, both the object's and the
+   request's grace parameter need to be set.
+
+-  Manual pages have been updated for new VCL syntax and varnishd
+   options.
+
+-  Man pages and other docs have been updated.
+
+-  The shared memory log file is now locked in memory, so it should not
+   be paged out to disk.
+
+-  We now handle Vary correctly, as well as Expect.
+
+-  ESI include support is implemented.
+
+-  Make it possible to limit how much memory the malloc uses.
+
+-  Solaris is now supported.
+
+-  There is now a regsuball function, which works like regsub except it
+   replaces all occurences of the regex, not just the first.
+
+-  Backend and director declarations can have a .connect\_timeout
+   parameter, which tells us how long to wait for a successful
+   connection.
+
+-  It is now possible to select the acceptor to use by changing the
+   acceptor parameter.
+
+-  Backends can have probes associated with them, which can be checked
+   with req.backend.health in VCL as well as being handled by directors
+   which do load-balancing.
+
+-  Support larger-than-2GB files also on 32 bit hosts. Please note that
+   this does not mean we can support caches bigger than 2GB, it just
+   means logfiles and similar can be bigger.
+
+-  In some cases, we would remove the wrong header when we were
+   stripping Content-Transfer-Encoding headers from a request. This has
+   been fixed.
+
+-  Backends can have a .max\_connections associated with them.
+
+-  On Linux, we need to set the dumpable bit on the child if we want
+   core dumps. Make sure it's set.
+
+-  Doing purge.hash() with an empty string would cause us to dump core.
+   Fixed so we don't do that any more.
+
+-  We ran into a problem with glibc's malloc on Linux where it seemed
+   like it failed to ever give memory back to the OS, causing the system
+   to swap. We have now switched to jemalloc which appears not to have
+   this problem.
+
+-  max\_restarts was never checked, so we always ended up running out of
+   workspace. Now, vcl\_error is called when we reach max\_restarts.
+
+varnishtest
+-----------
+
+-  varnishtest is a tool to do correctness tests of varnishd. The test
+   suite is run by using make check.
+
+varnishtop
+----------
+
+-  We now set the field widths dynamically based on the size of the
+   terminal and the name of the longest field.
+
+varnishstat
+-----------
+
+-  varnishstat -1 now displays the uptime too.
+
+varnishncsa
+-----------
+
+-  varnishncsa now does fflush after each write. This makes tail -f work
+   correctly, as well as avoiding broken lines in the log file.
+
+-  It is possible to get varnishncsa to output the X-Forwarded-For
+   instead of the client IP by passing -f to it.
+
+Build system
+-----------~
+
+-  Various sanity checks have been added to configure, it now complains
+   about no ncurses or if SO\_RCVTIMEO or SO\_SNDTIMEO are
+   non-functional. It also aborts if there's no working acceptor
+   mechanism
+
+-  The C compiler invocation is decided by the configure script and can
+   now be overridden by passing VCC\_CC when running configure.
+
+===========================
+Changes from 1.1.1 to 1.1.2
+===========================
+
+varnishd
+--------
+
+-  When switching to a new VCL configuration, a race condition exists
+   which may cause Varnish to reference a backend which no longer exists
+   (see `ticket #144 <http://varnish.projects.linpro.no/ticket/144>`_).
+   This race condition has not been entirely eliminated, but it should
+   occur less frequently.
+
+-  When dropping a TCP session before any requests were processed, an
+   assertion would be triggered due to an uninitialized timestamp (see
+   `ticket #132 <http://varnish.projects.linpro.no/ticket/132>`_). The
+   timestamp is now correctly initialized.
+
+-  Varnish will now correctly generate a Date: header for every response
+   instead of copying the one it got from the backend (see `ticket
+   #157 <http://varnish.projects.linpro.no/ticket/157>`_).
+
+-  Comparisons in VCL which involve a non-existent string (usually a
+   header which is not present in the request or object being processed)
+   would cause a NULL pointer dereference; now the comparison will
+   simply fail.
+
+-  A bug in the VCL compiler which would cause a double-free when
+   processing include directives has been fixed.
+
+-  A resource leak in the worker thread management code has been fixed.
+
+-  When connecting to a backend, Varnish will usually get the address
+   from a cache. When the cache is refreshed, existing connections may
+   end up with a reference to an address structure which no longer
+   exists, resulting in a crash. This race condition has been somewhat
+   mitigated, but not entirely eliminated (see `ticket
+   #144 <http://varnish.projects.linpro.no/ticket/144>`_.)
+
+-  Varnish will now pass the correct protocol version in pipe mode: the
+   backend will get what the client sent, and vice versa.
+
+-  The core of the pipe mode code has been rewritten to increase
+   robustness and eliminate spurious error messages when either end
+   closes the connection in a manner Varnish did not anticipate.
+
+-  A memory leak in the backend code has been plugged.
+
+-  When using the kqueue acceptor, if a client shuts down the request
+   side of the connection (as many clients do after sending their final
+   request), it was possible for the acceptor code to receive the EOF
+   event and recycle the session while the last request was still being
+   serviced, resulting in a assertion failure and a crash when the
+   worker thread later tried to delete the session. This should no
+   longer happen (see `ticket
+   #162 <http://varnish.projects.linpro.no/ticket/162>`_.)
+
+-  A mismatch between the recorded length of a cached object and the
+   amount of data actually present in cache for that object can
+   occasionally occur (see `ticket
+   #167 <http://varnish.projects.linpro.no/ticket/167>`_.) This has been
+   partially fixed, but may still occur for error pages generated by
+   Varnish when a problem arises while retrieving an object from the
+   backend.
+
+-  Some socket-related system calls may return unexpected error codes
+   when operating on a TCP connection that has been shut down at the
+   other end. These error codes would previously cause assertion
+   failures, but are now recognized as harmless conditions.
+
+varnishhist
+-----------
+
+-  Pressing 0 though 9 while varnishhist is running will change the
+   refresh interval to the corresponding power of two, in seconds.
+
+varnishncsa
+-----------
+
+-  The varnishncsa tool can now daemonize and write a PID file like
+   varnishlog, using the same command-line options. It will also reopen
+   its output upon receipt of a SIGHUP if invoked with -w.
+
+varnishstat
+-----------
+
+-  Pressing 0 though 9 while varnishstat is running will change the
+   refresh interval to the corresponding power of two, in seconds.
+
+Build system
+-----------~
+
+-  Varnish's <queue.h> has been modified to avoid conflicts with
+   <sys/queue.h> on platforms where the latter is included indirectly
+   through system headers.
+
+-  Several steps have been taken towards Solaris support, but this is
+   not yet complete.
+
+-  When configure was run without an explicit prefix, Varnish's idea of
+   the default state directory would be garbage and a state directory
+   would have to be specified manually with -n. This has been corrected.
+
+=========================
+Changes from 1.1 to 1.1.1
+=========================
+
+varnishd
+--------
+
+-  The code required to allow VCL to read obj.status, which had
+   accidentally been left out, has now been added.
+
+-  Varnish will now always include a Connection: header in its reply to
+   the client, to avoid possible misunderstandings.
+
+-  A bug that triggered an assertion failure when generating synthetic
+   error documents has been corrected.
+
+-  A new VCL function, purge\_url, provides the same functionality as
+   the url.purge management command.
+
+-  Previously, Varnish assumed that the response body should be sent
+   only if the request method was GET. This was a problem for custom
+   request methods (such as PURGE), so the logic has been changed to
+   always send the response body except in the specific case of a HEAD
+   request.
+
+-  Changes to run-time parameters are now correctly propagated to the
+   child process.
+
+-  Due to the way run-time parameters are initialized at startup,
+   varnishd previously required the nobody user and the nogroup group to
+   exist even if a different user and group were specified on the
+   command line. This has been corrected.
+
+-  Under certain conditions, the VCL compiler would carry on after a
+   syntax error instead of exiting after reporting the error. This has
+   been corrected.
+
+-  The manner in which the hash string is assembled has been modified to
+   reduce memory usage and memory-to-memory copying.
+
+-  Before calling vcl\_miss, Varnish assembles a tentative request
+   object for the backend request which will usually follow. This object
+   would be leaked if vcl\_miss returned anything else than fetch. This
+   has been corrected.
+
+-  The code necessary to handle an error return from vcl\_fetch and
+   vcl\_deliver had inadvertantly been left out. This has been
+   corrected.
+
+-  Varnish no longer prints a spurious "child died" message (the result
+   of reaping the compiler process) after compiling a new VCL
+   configuration.
+
+-  Under some circumstances, due to an error in the workspace management
+   code, Varnish would lose the "tail" of a request, i.e. the part of
+   the request that has been received from the client but not yet
+   processed. The most obvious symptom of this was that POST requests
+   would work with some browsers but not others, depending on details of
+   the browser's HTTP implementation. This has been corrected.
+
+-  On some platforms, due to incorrect assumptions in the CLI code, the
+   management process would crash while processing commands received
+   over the management port. This has been corrected.
+
+Build system
+-----------~
+
+-  The top-level Makefile will now honor $DESTDIR when creating the
+   state directory.
+
+-  The Debian and RedHat packages are now split into three (main / lib /
+   devel) as is customary.
+
+-  A number of compile-time and run-time portability issues have been
+   addressed.
+
+-  The autogen.sh script had workarounds for problems with the GNU
+   autotools on FreeBSD; these are no longer needed and have been
+   removed.
+
+-  The libcompat library has been renamed to libvarnishcompat and is now
+   dynamic rather than static. This simplifies the build process and
+   resolves an issue with the Mac OS X linker.
+
+=========================
+Changes from 1.0.4 to 1.1
+=========================
+
+varnishd
+--------
+
+-  Readability of the C source code generated from VCL code has been
+   improved.
+
+-  Equality (==) and inequality (!=) operators have been implemented for
+   IP addresses (which previously could only be compared using ACLs).
+
+-  The address of the listening socket on which the client connection
+   was received is now available to VCL as the server.ip variable.
+
+-  Each object's hash key is now computed based on a string which is
+   available to VCL as req.hash. A VCL hook named vcl\_hash has been
+   added to allow VCL scripts to control hash generation (for instance,
+   whether or not to include the value of the Host: header in the hash).
+
+-  The setup code for listening sockets has been modified to detect and
+   handle situations where a host name resolves to multiple IP
+   addresses. It will now attempt to bind to each IP address separately,
+   and report a failure only if none of them worked.
+
+-  Network or protocol errors that occur while retrieving an object from
+   a backend server now result in a synthetic error page being inserted
+   into the cache with a 30-second TTL. This should help avoid driving
+   an overburdened backend server into the ground by repeatedly
+   requesting the same object.
+
+-  The child process will now drop root privileges immediately upon
+   startup. The user and group to use are specified with the user and
+   group run-time parameters, which default to nobody and nogroup,
+   respectively. Other changes have been made in an effort to increase
+   the isolation between parent and child, and reduce the impact of a
+   compromise of the child process.
+
+-  Objects which are received from the backend with a Vary: header are
+   now stored separately according to the values of the headers
+   specified in Vary:. This allows Varnish to correctly cache e.g.
+   compressed and uncompressed versions of the same object.
+
+-  Each Varnish instance now has a name, which by default is the host
+   name of the machine it runs on, but can be any string that would be
+   valid as a relative or absolute directory name. It is used to
+   construct the name of a directory in which the server state as well
+   as all temporary files are stored. This makes it possible to run
+   multiple Varnish instances on the same machine without conflict.
+
+-  When invoked with the -C option, varnishd will now not just translate
+   the VCL code to C, but also compile the C code and attempt to load
+   the resulting shared object.
+
+-  Attempts by VCL code to reference a variable outside its scope or to
+   assign a value to a read-only variable will now result in
+   compile-time rather than run-time errors.
+
+-  The new command-line option -F will make varnishd run in the
+   foreground, without enabling debugging.
+
+-  New VCL variables have been introduced to allow inspection and
+   manipulation of the request sent to the backend (bereq.request,
+   bereq.url, bereq.proto and bereq.http) and the response to the client
+   (resp.proto, resp.status, resp.response and resp.http).
+
+-  Statistics from the storage code (including the amount of data and
+   free space in the cache) are now available to varnishstat and other
+   statistics-gathering tools.
+
+-  Objects are now kept on an LRU list which is kept loosely up-to-date
+   (to within a few seconds). When cache runs out, the objects at the
+   tail end of the LRU list are discarded one by one until there is
+   enough space for the freshly requested object(s). A VCL hook,
+   vcl\_discard, is allowed to inspect each object and determine its
+   fate by returning either keep or discard.
+
+-  A new VCL hook, vcl\_deliver, provides a chance to adjust the
+   response before it is sent to the client.
+
+-  A new management command, vcl.show, displays the VCL source code of
+   any loaded configuration.
+
+-  A new VCL variable, now, provides VCL scripts with the current time
+   in seconds since the epoch.
+
+-  A new VCL variable, obj.lastuse, reflects the time in seconds since
+   the object in question was last used.
+
+-  VCL scripts can now add an HTTP header (or modify the value of an
+   existing one) by assigning a value to the corresponding variable, and
+   strip an HTTP header by using the remove keyword.
+
+-  VCL scripts can now modify the HTTP status code of cached objects
+   (obj.status) and responses (resp.status)
+
+-  Numeric and other non-textual variables in VCL can now be assigned to
+   textual variables; they will be converted as needed.
+
+-  VCL scripts can now apply regular expression substitutions to textual
+   variables using the regsub function.
+
+-  A new management command, status, returns the state of the child.
+
+-  Varnish will now build and run on Mac OS X.
+
+varnishadm
+----------
+
+-  This is a new utility which sends a single command to a Varnish
+   server's management port and prints the result to stdout, greatly
+   simplifying the use of the management port from scripts.
+
+varnishhist
+-----------
+
+-  The user interface has been greatly improved; the histogram will be
+   automatically rescaled and redrawn when the window size changes, and
+   it is updated regularly rather than at a rate dependent on the amount
+   of log data gathered. In addition, the name of the Varnish instance
+   being watched is displayed in the upper right corner.
+
+varnishncsa
+-----------
+
+-  In addition to client traffic, varnishncsa can now also process log
+   data from backend traffic.
+
+-  A bug that would cause varnishncsa to segfault when it encountered an
+   empty HTTP header in the log file has been fixed.
+
+varnishreplay
+-------------
+
+-  This new utility will attempt to recreate the HTTP traffic which
+   resulted in the raw Varnish log data which it is fed.
+
+varnishstat
+-----------
+
+-  Don't print lifetime averages when it doesn't make any sense?for
+   instance, there is no point in dividing the amount in bytes of free
+   cache space by the lifetime in seconds of the varnishd process.
+
+-  The user interface has been greatly improved; varnishstat will no
+   longer print more than fits in the terminal, and will respond
+   correctly to window resize events. The output produced in one-shot
+   mode has been modified to include symbolic names for each entry. In
+   addition, the name of the Varnish instance being watched is displayed
+   in the upper right corner in curses mode.
+
+varnishtop
+----------
+
+-  The user interface has been greatly improved; varnishtop will now
+   respond correctly to window resize events, and one-shot mode (-1)
+   actually works. In addition, the name of the Varnish instance being
+   watched is displayed in the upper right corner in curses mode.
+
+===========================
+Changes from 1.0.3 to 1.0.4
+===========================
+
+varnishd
+--------
+
+-  The request workflow has been redesigned to simplify request
+   processing and eliminate code duplication. All codepaths which need
+   to speak HTTP now share a single implementation of the protocol. Some
+   new VCL hooks have been added, though they aren't much use yet. The
+   only real user-visible change should be that Varnish now handles
+   persistent backend connections correctly (see `ticket
+   #56 <http://varnish.projects.linpro.no/ticket/56>`_).
+
+-  Support for multiple listen addresses has been added.
+
+-  An "include" facility has been added to VCL, allowing VCL code to
+   pull in code fragments from multiple files.
+
+-  Multiple definitions of the same VCL function are now concatenated
+   into one in the order in which they appear in the source. This
+   simplifies the mechanism for falling back to the built-in default for
+   cases which aren't handled in custom code, and facilitates
+   modularization.
+
+-  The code used to format management command arguments before passing
+   them on to the child process would underestimate the amount of space
+   needed to hold each argument once quotes and special characters were
+   properly escaped, resulting in a buffer overflow. This has been
+   corrected.
+
+-  The VCL compiler has been overhauled. Several memory leaks have been
+   plugged, and error detection and reporting has been improved
+   throughout. Parts of the compiler have been refactored to simplify
+   future extension of the language.
+
+-  A bug in the VCL compiler which resulted in incorrect parsing of the
+   decrement (-=) operator has been fixed.
+
+-  A new -C command-line option has been added which causes varnishd to
+   compile the VCL code (either from a file specified with -f or the
+   built-in default), print the resulting C code and exit.
+
+-  When processing a backend response using chunked encoding, if a chunk
+   header crosses a read buffer boundary, read additional bytes from the
+   backend connection until the chunk header is complete.
+
+-  A new ping\_interval run-time parameter controls how often the
+   management process checks that the worker process is alive.
+
+-  A bug which would cause the worker process to dereference a NULL
+   pointer and crash if the backend did not respond has been fixed.
+
+-  In some cases, such as when they are used by AJAX applications to
+   circumvent Internet Explorer's over-eager disk cache, it may be
+   desirable to cache POST requests. However, the code path responsible
+   for delivering objects from cache would only transmit the response
+   body when replying to a GET request. This has been extended to also
+   apply to POST.
+
+   This should be revisited at a later date to allow VCL code to control
+   whether the body is delivered.
+
+-  Varnish now respects Cache-control: s-maxage, and prefers it to
+   Cache-control: max-age if both are present.
+
+   This should be revisited at a later date to allow VCL code to control
+   which headers are used and how they are interpreted.
+
+-  When loading a new VCL script, the management process will now load
+   the compiled object to verify that it links correctly before
+   instructing the worker process to load it.
+
+-  A new -P command-line options has been added which causes varnishd to
+   create a PID file.
+
+-  The sendfile\_threshold run-time parameter's default value has been
+   set to infinity after a variety of sendfile()-related bugs were
+   discovered on several platforms.
+
+varnishlog
+----------
+
+-  When grouping log entries by request, varnishlog attempts to collapse
+   the log entry for a call to a VCL function with the log entry for the
+   corresponding return from VCL. When two VCL calls were made in
+   succession, varnishlog would incorrectly omit the newline between the
+   two calls (see `ticket
+   #95 <http://varnish.projects.linpro.no/ticket/95>`_).
+
+-  New -D and -P command-line options have been added to daemonize and
+   create a pidfile, respectively.
+
+-  The flag that is raised upon reception of a SIGHUP has been marked
+   volatile so it will not be optimized away by the compiler.
+
+varnishncsa
+-----------
+
+-  The formatting callback has been largely rewritten for clarity,
+   robustness and efficiency.
+
+   If a request included a Host: header, construct and output an
+   absolute URL. This makes varnishncsa output from servers which handle
+   multiple virtual hosts far more useful.
+
+-  The flag that is raised upon reception of a SIGHUP has been marked
+   volatile so it will not be optimized away by the compiler.
+
+Documentation
+-------------
+
+-  The documentation?especially the VCL documentation?has been greatly
+   extended and improved.
+
+Build system
+------------
+
+-  The name and location of the curses or ncurses library is now
+   correctly detected by the configure script instead of being hardcoded
+   into affected Makefiles. This allows Varnish to build correctly on a
+   wider range of platforms.
+
+-  Compatibility shims for clock\_gettime() are now correctly applied
+   where needed, allowing Varnish to build on MacOS X.
+
+-  The autogen.sh script will now correctly detect and warn about
+   automake versions which are known not to work correctly.
diff --git a/doc/sphinx/installation/install.rst b/doc/sphinx/installation/install.rst
index d725f81..fe8496d 100644
--- a/doc/sphinx/installation/install.rst
+++ b/doc/sphinx/installation/install.rst
@@ -92,7 +92,6 @@ installed. On a Debian or Ubuntu system these are:
 * libtool 
 * autoconf
 * libncurses-dev
-* xsltproc
 * groff-base
 * libpcre3-dev
 * pkg-config
@@ -107,7 +106,6 @@ packages installed:
 * autoconf 
 * libtool
 * ncurses-devel
-* libxslt
 * groff
 * pcre-devel
 * pkgconfig
diff --git a/redhat/varnish.spec b/redhat/varnish.spec
index 5cf9c97..ead5745 100644
--- a/redhat/varnish.spec
+++ b/redhat/varnish.spec
@@ -15,7 +15,7 @@ BuildRoot: %{_tmppath}/%{name}-%{version}-%{release}-root-%(%{__id_u} -n)
 # To build from git, start with a make dist, see redhat/README.redhat 
 # You will need at least automake autoconf libtool python-docutils
 #BuildRequires: automake autoconf libtool python-docutils
-BuildRequires: ncurses-devel libxslt groff pcre-devel pkgconfig
+BuildRequires: ncurses-devel groff pcre-devel pkgconfig
 Requires: varnish-libs = %{version}-%{release}
 Requires: logrotate
 Requires: ncurses



From geoff at varnish-cache.org  Mon Jan  9 20:52:27 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:27 +0100
Subject: [experimental-ims] 6e4e013 Overhaul the detection and reporting of
	fetch errors,
	to properly catch trouble that materializes only when we destroy the
	VGZ instance.
Message-ID: <E1RkMCh-0007yn-Ng@project.varnish-software.com>

commit 6e4e013f407c6685241c7a4c88a72dbd671102ba
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Oct 31 14:20:34 2011 +0000

    Overhaul the detection and reporting of fetch errors, to properly
    catch trouble that materializes only when we destroy the VGZ instance.
    
    Fixes	#1037
    Fixes	#1043
    Fixes	#1044

diff --git a/bin/varnishd/cache.h b/bin/varnishd/cache.h
index bce7377..53e8ae8 100644
--- a/bin/varnishd/cache.h
+++ b/bin/varnishd/cache.h
@@ -345,6 +345,7 @@ struct worker {
 	struct vfp		*vfp;
 	struct vgz		*vgz_rx;
 	struct vef_priv		*vef_priv;
+	unsigned		fetch_failed;
 	unsigned		do_stream;
 	unsigned		do_esi;
 	unsigned		do_gzip;
@@ -703,6 +704,8 @@ int EXP_NukeOne(struct worker *w, struct lru *lru);
 
 /* cache_fetch.c */
 struct storage *FetchStorage(struct worker *w, ssize_t sz);
+int FetchError(struct worker *w, const char *error);
+int FetchError2(struct worker *w, const char *error, const char *more);
 int FetchHdr(struct sess *sp);
 int FetchBody(struct worker *w, struct object *obj);
 int FetchReqBody(struct sess *sp);
@@ -721,7 +724,7 @@ int VGZ_ObufFull(const struct vgz *vg);
 int VGZ_ObufStorage(struct worker *w, struct vgz *vg);
 int VGZ_Gzip(struct vgz *, const void **, size_t *len, enum vgz_flag);
 int VGZ_Gunzip(struct vgz *, const void **, size_t *len);
-void VGZ_Destroy(struct vgz **, int vsl_id);
+int VGZ_Destroy(struct vgz **, int vsl_id);
 void VGZ_UpdateObj(const struct vgz*, struct object *);
 int VGZ_WrwGunzip(struct worker *w, struct vgz *, const void *ibuf,
     ssize_t ibufl, char *obuf, ssize_t obufl, ssize_t *obufp);
diff --git a/bin/varnishd/cache_center.c b/bin/varnishd/cache_center.c
index fe7dae6..37c4b7b 100644
--- a/bin/varnishd/cache_center.c
+++ b/bin/varnishd/cache_center.c
@@ -934,7 +934,7 @@ cnt_streambody(struct sess *sp)
 
 	RES_StreamEnd(sp);
 	if (sp->wrk->res_mode & RES_GUNZIP)
-		VGZ_Destroy(&sctx.vgz, sp->vsl_id);
+		(void)VGZ_Destroy(&sctx.vgz, sp->vsl_id);
 
 	sp->wrk->sctx = NULL;
 	assert(WRW_IsReleased(sp->wrk));
diff --git a/bin/varnishd/cache_esi_deliver.c b/bin/varnishd/cache_esi_deliver.c
index a06bbb8..2c62bb7 100644
--- a/bin/varnishd/cache_esi_deliver.c
+++ b/bin/varnishd/cache_esi_deliver.c
@@ -406,7 +406,7 @@ ESI_Deliver(struct sess *sp)
 	if (vgz != NULL) {
 		if (obufl > 0)
 			(void)WRW_Write(sp->wrk, obuf, obufl);
-		VGZ_Destroy(&vgz, sp->vsl_id);
+		(void)VGZ_Destroy(&vgz, sp->vsl_id);
 	}
 	if (sp->wrk->gzip_resp && sp->esi_level == 0) {
 		/* Emit a gzip literal block with finish bit set */
diff --git a/bin/varnishd/cache_esi_fetch.c b/bin/varnishd/cache_esi_fetch.c
index 69896aa..1c75054 100644
--- a/bin/varnishd/cache_esi_fetch.c
+++ b/bin/varnishd/cache_esi_fetch.c
@@ -112,7 +112,7 @@ vfp_esi_bytes_gu(struct worker *w, struct http_conn *htc, ssize_t bytes)
 			bytes -= wl;
 		}
 		if (VGZ_ObufStorage(w, vg))
-			return (-1);
+			return(FetchError(w, "Could not get storage"));
 		i = VGZ_Gunzip(vg, &dp, &dl);
 		xxxassert(i == VGZ_OK || i == VGZ_END);
 		VEP_Parse(w, dp, dl);
@@ -297,7 +297,6 @@ vfp_esi_begin(struct worker *w, size_t estimate)
 	struct vef_priv *vef;
 
 	CHECK_OBJ_NOTNULL(w, WORKER_MAGIC);
-	/* XXX: snapshot WS's ? We'll need the space */
 
 	AZ(w->vgz_rx);
 	if (w->is_gzip && w->do_gunzip) {
@@ -306,9 +305,6 @@ vfp_esi_begin(struct worker *w, size_t estimate)
 	} else if (w->is_gunzip && w->do_gzip) {
 		ALLOC_OBJ(vef, VEF_MAGIC);
 		AN(vef);
-		//vef = (void*)WS_Alloc(sp->ws, sizeof *vef);
-		//memset(vef, 0, sizeof *vef);
-		//vef->magic = VEF_MAGIC;
 		vef->vgz = VGZ_NewGzip(w, "G F E");
 		AZ(w->vef_priv);
 		w->vef_priv = vef;
@@ -317,9 +313,6 @@ vfp_esi_begin(struct worker *w, size_t estimate)
 		w->vgz_rx = VGZ_NewUngzip(w, "U F E");
 		ALLOC_OBJ(vef, VEF_MAGIC);
 		AN(vef);
-		//vef = (void*)WS_Alloc(sp->ws, sizeof *vef);
-		//memset(vef, 0, sizeof *vef);
-		//vef->magic = VEF_MAGIC;
 		vef->vgz = VGZ_NewGzip(w, "G F E");
 		AZ(w->vef_priv);
 		w->vef_priv = vef;
@@ -339,6 +332,7 @@ vfp_esi_bytes(struct worker *w, struct http_conn *htc, ssize_t bytes)
 	int i;
 
 	CHECK_OBJ_NOTNULL(w, WORKER_MAGIC);
+	AZ(w->fetch_failed);
 	AN(w->vep);
 	if (w->is_gzip && w->do_gunzip)
 		i = vfp_esi_bytes_gu(w, htc, bytes);
@@ -358,35 +352,48 @@ vfp_esi_end(struct worker *w)
 	struct vsb *vsb;
 	struct vef_priv *vef;
 	ssize_t l;
+	int retval;
 
 	CHECK_OBJ_NOTNULL(w, WORKER_MAGIC);
 	AN(w->vep);
 
+	retval = w->fetch_failed;
+
+	if (w->vgz_rx != NULL && VGZ_Destroy(&w->vgz_rx, -1) != VGZ_END)
+		retval = FetchError(w,
+		    "Gunzip+ESI Failed at the very end");
+
 	vsb = VEP_Finish(w);
 
 	if (vsb != NULL) {
-		l = VSB_len(vsb);
-		assert(l > 0);
-		/* XXX: This is a huge waste of storage... */
-		w->fetch_obj->esidata = STV_alloc(w, l);
-		XXXAN(w->fetch_obj->esidata);
-		memcpy(w->fetch_obj->esidata->ptr, VSB_data(vsb), l);
-		w->fetch_obj->esidata->len = l;
+		if (!retval) {
+			l = VSB_len(vsb);
+			assert(l > 0);
+			/* XXX: This is a huge waste of storage... */
+			w->fetch_obj->esidata = STV_alloc(w, l);
+			if (w->fetch_obj->esidata != NULL) {
+				memcpy(w->fetch_obj->esidata->ptr,
+				    VSB_data(vsb), l);
+				w->fetch_obj->esidata->len = l;
+			} else {
+				retval = FetchError(w,
+				    "Could not allocate storage for esidata");
+			}
+		}
 		VSB_delete(vsb);
 	}
-	if (w->vgz_rx != NULL)
-		VGZ_Destroy(&w->vgz_rx, -1);
 
 	if (w->vef_priv != NULL) {
 		vef = w->vef_priv;
 		CHECK_OBJ_NOTNULL(vef, VEF_MAGIC);
 		w->vef_priv = NULL;
 		VGZ_UpdateObj(vef->vgz, w->fetch_obj);
-		VGZ_Destroy(&vef->vgz,  -1);
-		XXXAZ(vef->error);
+		if (VGZ_Destroy(&vef->vgz,  -1) != VGZ_END)
+			retval = FetchError(w, 
+			    "ESI+Gzip Failed at the very end");
 		FREE_OBJ(vef);
 	}
-	return (0);
+	return (retval);
 }
 
 struct vfp vfp_esi = {
diff --git a/bin/varnishd/cache_fetch.c b/bin/varnishd/cache_fetch.c
index 6e92963..dd3a1c7 100644
--- a/bin/varnishd/cache_fetch.c
+++ b/bin/varnishd/cache_fetch.c
@@ -43,6 +43,36 @@
 static unsigned fetchfrag;
 
 /*--------------------------------------------------------------------
+ * We want to issue the first error we encounter on fetching and
+ * supress the rest.  This function does that.
+ *
+ * Other code is allowed to look at w->fetch_failed to bail out
+ *
+ * For convenience, always return -1
+ */
+
+int
+FetchError2(struct worker *w, const char *error, const char *more)
+{
+
+	CHECK_OBJ_NOTNULL(w, WORKER_MAGIC);
+	if (!w->fetch_failed) {
+		if (more == NULL)
+			WSLB(w, SLT_FetchError, "%s", error);
+		else
+			WSLB(w, SLT_FetchError, "%s: %s", error, more);
+	}
+	w->fetch_failed = 1;
+	return (-1);
+}
+
+int
+FetchError(struct worker *w, const char *error)
+{
+	return(FetchError2(w, error, NULL));
+}
+
+/*--------------------------------------------------------------------
  * VFP_NOP
  *
  * This fetch-processor does nothing but store the object.
@@ -74,7 +104,8 @@ vfp_nop_begin(struct worker *w, size_t estimate)
  *
  * Process (up to) 'bytes' from the socket.
  *
- * Return -1 on error
+ * Return -1 on error, issue FetchError()
+ *	will not be called again, once error happens.
  * Return 0 on EOF on socket even if bytes not reached.
  * Return 1 when 'bytes' have been processed.
  */
@@ -85,17 +116,18 @@ vfp_nop_bytes(struct worker *w, struct http_conn *htc, ssize_t bytes)
 	ssize_t l, wl;
 	struct storage *st;
 
+	AZ(w->fetch_failed);
 	while (bytes > 0) {
 		st = FetchStorage(w, 0);
-		if (st == NULL) {
-			htc->error = "Could not get storage";
-			return (-1);
-		}
+		if (st == NULL)
+			return(FetchError(w, "Could not get storage"));
 		l = st->space - st->len;
 		if (l > bytes)
 			l = bytes;
 		wl = HTC_Read(htc, st->ptr + st->len, l);
-		if (wl <= 0)
+		if (wl < 0)
+			return(FetchError(w, htc->error));
+		if (wl == 0)
 			return (wl);
 		st->len += wl;
 		w->fetch_obj->len += wl;
@@ -205,17 +237,13 @@ fetch_straight(struct worker *w, struct http_conn *htc, ssize_t cl)
 	assert(w->body_status == BS_LENGTH);
 
 	if (cl < 0) {
-		WSLB(w, SLT_FetchError, "straight length field bogus");
-		return (-1);
+		return (FetchError(w, "straight length field bogus"));
 	} else if (cl == 0)
 		return (0);
 
 	i = w->vfp->bytes(w, htc, cl);
-	if (i <= 0) {
-		WSLB(w, SLT_FetchError, "straight read_error: %d %d (%s)",
-		    i, errno, htc->error);
-		return (-1);
-	}
+	if (i <= 0)
+		return (FetchError(w, "straight insufficient bytes"));
 	return (0);
 }
 
@@ -257,10 +285,8 @@ fetch_chunked(struct worker *w, struct http_conn *htc)
 				break;
 		}
 
-		if (u >= sizeof buf) {
-			WSLB(w, SLT_FetchError,	"chunked header too long");
-			return (-1);
-		}
+		if (u >= sizeof buf) 
+			return (FetchError(w,"chunked header too long"));
 
 		/* Skip trailing white space */
 		while(vct_islws(buf[u]) && buf[u] != '\n') {
@@ -268,19 +294,17 @@ fetch_chunked(struct worker *w, struct http_conn *htc)
 			CERR();
 		}
 
-		if (buf[u] != '\n') {
-			WSLB(w, SLT_FetchError,	"chunked header char syntax");
-			return (-1);
-		}
+		if (buf[u] != '\n') 
+			return (FetchError(w,"chunked header no NL"));
 		buf[u] = '\0';
 
 		cl = fetch_number(buf, 16);
 		if (cl < 0) {
-			WSLB(w, SLT_FetchError,	"chunked header nbr syntax");
-			return (-1);
+			return (FetchError(w,"chunked header number syntax"));
 		} else if (cl > 0) {
 			i = w->vfp->bytes(w, htc, cl);
-			CERR();
+			if (i <= 0)
+				return (-1);
 		}
 		i = HTC_Read(htc, buf, 1);
 		CERR();
@@ -288,10 +312,8 @@ fetch_chunked(struct worker *w, struct http_conn *htc)
 			i = HTC_Read(htc, buf, 1);
 			CERR();
 		}
-		if (buf[0] != '\n') {
-			WSLB(w, SLT_FetchError,	"chunked tail syntax");
-			return (-1);
-		}
+		if (buf[0] != '\n')
+			return (FetchError(w,"chunked tail no NL"));
 	} while (cl > 0);
 	return (0);
 }
@@ -307,11 +329,8 @@ fetch_eof(struct worker *w, struct http_conn *htc)
 
 	assert(w->body_status == BS_EOF);
 	i = w->vfp->bytes(w, htc, SSIZE_MAX);
-	if (i < 0) {
-		WSLB(w, SLT_FetchError, "eof read_error: %d (%s)",
-		    errno, htc->error);
+	if (i < 0) 
 		return (-1);
-	}
 	return (0);
 }
 
@@ -497,6 +516,7 @@ FetchBody(struct worker *w, struct object *obj)
 	AZ(VTAILQ_FIRST(&obj->store));
 
 	w->fetch_obj = obj;
+	w->fetch_failed = 0;
 
 	/* XXX: pick up estimate from objdr ? */
 	cl = 0;
@@ -572,6 +592,7 @@ FetchBody(struct worker *w, struct object *obj)
 		obj->len = 0;
 		return (__LINE__);
 	}
+	AZ(w->fetch_failed);
 
 	if (cls == 0 && w->do_close)
 		cls = 1;
@@ -588,6 +609,7 @@ FetchBody(struct worker *w, struct object *obj)
 		if (w->do_stream)
 			/* Streaming might have started freeing stuff */
 			assert (uu <= obj->len);
+
 		else
 			assert(uu == obj->len);
 	}
diff --git a/bin/varnishd/cache_gzip.c b/bin/varnishd/cache_gzip.c
index 7df69cd..73f4bb9 100644
--- a/bin/varnishd/cache_gzip.c
+++ b/bin/varnishd/cache_gzip.c
@@ -81,7 +81,7 @@ struct vgz {
 	const char		*id;
 	struct ws		*tmp;
 	char			*tmp_snapshot;
-	const char		*error;
+	int			last_i;
 
 	struct storage		*obuf;
 
@@ -201,8 +201,6 @@ VGZ_NewGzip(struct worker *wrk, const char *id)
 	    16 + params->gzip_window,	/* Window bits (16=gzip + 15) */
 	    params->gzip_memlevel,	/* memLevel */
 	    Z_DEFAULT_STRATEGY);
-	if (i != Z_OK)
-		printf("deflateInit2() = %d\n", i);
 	assert(Z_OK == i);
 	return (vg);
 }
@@ -259,10 +257,8 @@ VGZ_ObufStorage(struct worker *w, struct vgz *vg)
 	struct storage *st;
 
 	st = FetchStorage(w, 0);
-	if (st == NULL) {
-		vg->error = "Could not get ObufStorage";
+	if (st == NULL) 
 		return (-1);
-	}
 
 	vg->obuf = st;
 	VGZ_Obuf(vg, st->ptr + st->len, st->space - st->len);
@@ -294,6 +290,7 @@ VGZ_Gunzip(struct vgz *vg, const void **pptr, size_t *plen)
 		if (vg->obuf != NULL)
 			vg->obuf->len += l;
 	}
+	vg->last_i = i;
 	if (i == Z_OK)
 		return (VGZ_OK);
 	if (i == Z_STREAM_END)
@@ -336,6 +333,7 @@ VGZ_Gzip(struct vgz *vg, const void **pptr, size_t *plen, enum vgz_flag flags)
 		if (vg->obuf != NULL)
 			vg->obuf->len += l;
 	}
+	vg->last_i = i;
 	if (i == Z_OK)
 		return (0);
 	if (i == Z_STREAM_END)
@@ -405,11 +403,11 @@ VGZ_UpdateObj(const struct vgz *vg, struct object *obj)
  * Passing a vsl_id of -1 means "use w->vbc->vsl_id"
  */
 
-void
+int
 VGZ_Destroy(struct vgz **vgp, int vsl_id)
 {
 	struct vgz *vg;
-	const char *err;
+	int i;
 
 	vg = *vgp;
 	CHECK_OBJ_NOTNULL(vg, VGZ_MAGIC);
@@ -431,14 +429,22 @@ VGZ_Destroy(struct vgz **vgp, int vsl_id)
 		    (intmax_t)vg->vz.start_bit,
 		    (intmax_t)vg->vz.last_bit,
 		    (intmax_t)vg->vz.stop_bit);
-	err = vg->error;
 	if (vg->tmp != NULL)
 		WS_Reset(vg->tmp, vg->tmp_snapshot);
 	if (vg->dir == VGZ_GZ)
-		assert(deflateEnd(&vg->vz) == 0 || err != NULL);
+		i = deflateEnd(&vg->vz);
 	else
-		assert(inflateEnd(&vg->vz) == 0 || err != NULL);
+		i = inflateEnd(&vg->vz);
+	if (vg->last_i == Z_STREAM_END && i == Z_OK)
+		i = Z_STREAM_END;
 	FREE_OBJ(vg);
+	if (i == Z_OK)
+		return (VGZ_OK);
+	if (i == Z_STREAM_END)
+		return (VGZ_END);
+	if (i == Z_BUF_ERROR)
+		return (VGZ_STUCK);
+	return (VGZ_ERROR);
 }
 
 /*--------------------------------------------------------------------
@@ -465,6 +471,7 @@ vfp_gunzip_bytes(struct worker *w, struct http_conn *htc, ssize_t bytes)
 	size_t dl;
 	const void *dp;
 
+	AZ(w->fetch_failed);
 	vg = w->vgz_rx;
 	CHECK_OBJ_NOTNULL(vg, VGZ_MAGIC);
 	AZ(vg->vz.avail_in);
@@ -474,27 +481,25 @@ vfp_gunzip_bytes(struct worker *w, struct http_conn *htc, ssize_t bytes)
 			if (l > bytes)
 				l = bytes;
 			wl = HTC_Read(htc, ibuf, l);
-			if (wl <= 0)
+			if (wl < 0)
+				return(FetchError(w, htc->error));
+			if (wl == 0)
 				return (wl);
 			VGZ_Ibuf(vg, ibuf, wl);
 			bytes -= wl;
 		}
 
-		if (VGZ_ObufStorage(w, vg)) {
-			htc->error = "Could not get storage";
-			return (-1);
-		}
+		if (VGZ_ObufStorage(w, vg)) 
+			return(FetchError(w, "Could not get storage"));
 		i = VGZ_Gunzip(vg, &dp, &dl);
-		assert(i == VGZ_OK || i == VGZ_END);
+		if (i != VGZ_OK && i != VGZ_END) 
+			return(FetchError(w, "Gunzip data error"));
 		w->fetch_obj->len += dl;
 		if (w->do_stream)
 			RES_StreamPoll(w);
 	}
-	if (i == Z_OK || i == Z_STREAM_END)
-		return (1);
-	htc->error = "See other message";
-	WSLB(w, SLT_FetchError, "Gunzip trouble (%d)", i);
-	return (-1);
+	assert(i == Z_OK || i == Z_STREAM_END);
+	return (1);
 }
 
 static int __match_proto__()
@@ -505,7 +510,12 @@ vfp_gunzip_end(struct worker *w)
 	vg = w->vgz_rx;
 	w->vgz_rx = NULL;
 	CHECK_OBJ_NOTNULL(vg, VGZ_MAGIC);
-	VGZ_Destroy(&vg, -1);
+	if (w->fetch_failed) {
+		(void)VGZ_Destroy(&vg, -1);
+		return(0);
+	}
+	if (VGZ_Destroy(&vg, -1) != VGZ_END)
+		return(FetchError(w, "Gunzip error at the very end"));
 	return (0);
 }
 
@@ -541,6 +551,7 @@ vfp_gzip_bytes(struct worker *w, struct http_conn *htc, ssize_t bytes)
 	size_t dl;
 	const void *dp;
 
+	AZ(w->fetch_failed);
 	vg = w->vgz_rx;
 	CHECK_OBJ_NOTNULL(vg, VGZ_MAGIC);
 	AZ(vg->vz.avail_in);
@@ -550,15 +561,15 @@ vfp_gzip_bytes(struct worker *w, struct http_conn *htc, ssize_t bytes)
 			if (l > bytes)
 				l = bytes;
 			wl = HTC_Read(htc, ibuf, l);
-			if (wl <= 0)
+			if (wl < 0)
+				return(FetchError(w, htc->error));
+			if (wl == 0)
 				return (wl);
 			VGZ_Ibuf(vg, ibuf, wl);
 			bytes -= wl;
 		}
-		if (VGZ_ObufStorage(w, vg)) {
-			htc->error = "Could not get storage";
-			return (-1);
-		}
+		if (VGZ_ObufStorage(w, vg)) 
+			return(FetchError(w, "Could not get storage"));
 		i = VGZ_Gzip(vg, &dp, &dl, VGZ_NORMAL);
 		assert(i == Z_OK);
 		w->fetch_obj->len += dl;
@@ -579,19 +590,22 @@ vfp_gzip_end(struct worker *w)
 	vg = w->vgz_rx;
 	CHECK_OBJ_NOTNULL(vg, VGZ_MAGIC);
 	w->vgz_rx = NULL;
-	if (vg->error == NULL) {
-		do {
-			VGZ_Ibuf(vg, "", 0);
-			if (VGZ_ObufStorage(w, vg))
-				return (-1);
-			i = VGZ_Gzip(vg, &dp, &dl, VGZ_FINISH);
-			w->fetch_obj->len += dl;
-		} while (i != Z_STREAM_END);
-		if (w->do_stream)
-			RES_StreamPoll(w);
-		VGZ_UpdateObj(vg, w->fetch_obj);
+	if (w->fetch_failed) {
+		(void)VGZ_Destroy(&vg, -1);
+		return(0);
 	}
-	VGZ_Destroy(&vg, -1);
+	do {
+		VGZ_Ibuf(vg, "", 0);
+		if (VGZ_ObufStorage(w, vg))
+			return(FetchError(w, "Could not get storage"));
+		i = VGZ_Gzip(vg, &dp, &dl, VGZ_FINISH);
+		w->fetch_obj->len += dl;
+	} while (i != Z_STREAM_END);
+	if (w->do_stream)
+		RES_StreamPoll(w);
+	VGZ_UpdateObj(vg, w->fetch_obj);
+	if (VGZ_Destroy(&vg, -1) != VGZ_END)
+		return(FetchError(w, "Gzip error at the very end"));
 	return (0);
 }
 
@@ -627,21 +641,21 @@ vfp_testgzip_bytes(struct worker *w, struct http_conn *htc, ssize_t bytes)
 	const void *dp;
 	struct storage *st;
 
+	AZ(w->fetch_failed);
 	vg = w->vgz_rx;
 	CHECK_OBJ_NOTNULL(vg, VGZ_MAGIC);
 	AZ(vg->vz.avail_in);
 	while (bytes > 0) {
 		st = FetchStorage(w, 0);
-		if (st == NULL) {
-			htc->error = "Could not get storage";
-			vg->error = htc->error;
-			return (-1);
-		}
+		if (st == NULL)
+			return(FetchError(w, "Could not get storage"));
 		l = st->space - st->len;
 		if (l > bytes)
 			l = bytes;
 		wl = HTC_Read(htc, st->ptr + st->len, l);
-		if (wl <= 0)
+		if (wl < 0)
+			return(FetchError(w, htc->error));
+		if (wl == 0)
 			return (wl);
 		bytes -= wl;
 		VGZ_Ibuf(vg, st->ptr + st->len, wl);
@@ -653,23 +667,15 @@ vfp_testgzip_bytes(struct worker *w, struct http_conn *htc, ssize_t bytes)
 		while (!VGZ_IbufEmpty(vg)) {
 			VGZ_Obuf(vg, obuf, sizeof obuf);
 			i = VGZ_Gunzip(vg, &dp, &dl);
-			if (i == VGZ_END && !VGZ_IbufEmpty(vg)) {
-				htc->error = "Junk after gzip data";
-				return (-1);
-			}
-			if (i != VGZ_OK && i != VGZ_END) {
-				htc->error = "See other message";
-				WSLB(w, SLT_FetchError,
-				    "Invalid Gzip data: %s", vg->vz.msg);
-				return (-1);
-			}
+			if (i == VGZ_END && !VGZ_IbufEmpty(vg)) 
+				return(FetchError(w, "Junk after gzip data"));
+			if (i != VGZ_OK && i != VGZ_END)
+				return(FetchError2(w,
+				    "Invalid Gzip data", vg->vz.msg));
 		}
 	}
-	if (i == VGZ_OK || i == VGZ_END)
-		return (1);
-	htc->error = "See other message";
-	WSLB(w, SLT_FetchError, "Gunzip trouble (%d)", i);
-	return (-1);
+	assert(i == VGZ_OK || i == VGZ_END);
+	return (1);
 }
 
 static int __match_proto__()
@@ -680,8 +686,13 @@ vfp_testgzip_end(struct worker *w)
 	vg = w->vgz_rx;
 	w->vgz_rx = NULL;
 	CHECK_OBJ_NOTNULL(vg, VGZ_MAGIC);
+	if (w->fetch_failed) {
+		(void)VGZ_Destroy(&vg, -1);
+		return(0);
+	}
 	VGZ_UpdateObj(vg, w->fetch_obj);
-	VGZ_Destroy(&vg, -1);
+	if (VGZ_Destroy(&vg, -1) != VGZ_END)
+		return(FetchError(w, "TestGunzip error at the very end"));
 	return (0);
 }
 
diff --git a/bin/varnishd/cache_response.c b/bin/varnishd/cache_response.c
index 84606b6..04d2da5 100644
--- a/bin/varnishd/cache_response.c
+++ b/bin/varnishd/cache_response.c
@@ -182,7 +182,7 @@ res_WriteGunzipObj(const struct sess *sp)
 		(void)WRW_Write(sp->wrk, obuf, obufl);
 		(void)WRW_Flush(sp->wrk);
 	}
-	VGZ_Destroy(&vg, sp->vsl_id);
+	(void)VGZ_Destroy(&vg, sp->vsl_id);
 	assert(u == sp->obj->len);
 }
 
diff --git a/bin/varnishtest/tests/g00004.vtc b/bin/varnishtest/tests/g00004.vtc
new file mode 100644
index 0000000..5d518e0
--- /dev/null
+++ b/bin/varnishtest/tests/g00004.vtc
@@ -0,0 +1,43 @@
+varnishtest "truncated gzip from backend"
+
+server s1 -repeat 2 {
+	rxreq
+	txresp -nolen \
+		-hdr "Content-Encoding: gzip" \
+		-hdr "Transfer-Encoding: Chunked"
+	send "18\r\n"
+		# A truncate gzip file
+		sendhex "1f8b"
+		sendhex "08"
+		sendhex "00"
+		sendhex "f5 64 ae 4e  02 03 f3 cd cf 53 f0 4f"
+		sendhex "2e 51 30 36 54 30 b0 b4"
+	send "\r\n"
+	chunkedlen 0
+
+} -start
+
+varnish v1 \
+	-arg {-p diag_bitmap=0x00010000} \
+	-vcl+backend {
+	sub vcl_fetch {
+		if (req.url == "/gunzip") {
+			set beresp.do_gunzip = true;
+		} 
+	}
+} 
+
+varnish v1 -start
+
+client c1 {
+	txreq
+	rxresp
+	expect resp.status == 503
+} -run
+
+client c1 {
+	txreq -url /gunzip
+	rxresp
+	expect resp.status == 503
+} -run
+
diff --git a/bin/varnishtest/tests/r01037.vtc b/bin/varnishtest/tests/r01037.vtc
new file mode 100644
index 0000000..acb77d3
--- /dev/null
+++ b/bin/varnishtest/tests/r01037.vtc
@@ -0,0 +1,29 @@
+varnishtest "Test case for #1037"
+
+server s1 {
+	rxreq
+	# Send a bodylen of 1,5M, which will exceed cache space of 1M
+	non-fatal
+	txresp -nolen -hdr "Transfer-encoding: chunked"
+	chunked {<HTML>}
+	chunkedlen 500000
+	chunked {<esi:remove>}
+	chunkedlen 500000
+	chunked {</esi:remove>}
+	chunkedlen 500000
+	chunkedlen 0
+} -start
+
+varnish v1 -arg "-smalloc,1M" -arg "-pgzip_level=0" -vcl+backend {
+	sub vcl_fetch {
+		set beresp.do_esi = true;
+		set beresp.do_gzip = true;
+	}
+} -start
+
+client c1 {
+	txreq
+	rxresp
+	expect resp.status == "503"
+} -run
+



From geoff at varnish-cache.org  Mon Jan  9 20:52:27 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:27 +0100
Subject: [experimental-ims] 2b47445 Merge branch 'master' of
	ssh://git.varnish-cache.org/git/varnish-cache
Message-ID: <E1RkMCh-0007z4-QM@project.varnish-software.com>

commit 2b4744555f48c7314f4aaaf42a1749c09e9acdef
Merge: 6e4e013 e9e02a4
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Oct 31 14:22:10 2011 +0000

    Merge branch 'master' of ssh://git.varnish-cache.org/git/varnish-cache



From geoff at varnish-cache.org  Mon Jan  9 20:52:27 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:27 +0100
Subject: [experimental-ims] 01dfd01 More cleanup and simplification of
	FetchError reporting.
Message-ID: <E1RkMCh-0007zL-VR@project.varnish-software.com>

commit 01dfd019aafca831af9adabc0bcc17a03097a40d
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Oct 31 21:37:47 2011 +0000

    More cleanup and simplification of FetchError reporting.

diff --git a/bin/varnishd/cache.h b/bin/varnishd/cache.h
index 53e8ae8..1875ece 100644
--- a/bin/varnishd/cache.h
+++ b/bin/varnishd/cache.h
@@ -199,7 +199,6 @@ struct http_conn {
 	struct ws		*ws;
 	txt			rxbuf;
 	txt			pipeline;
-	const char		*error;
 };
 
 /*--------------------------------------------------------------------*/
@@ -785,7 +784,7 @@ void HTC_Init(struct http_conn *htc, struct ws *ws, int fd, unsigned vsl_id,
     unsigned maxbytes, unsigned maxhdr);
 int HTC_Reinit(struct http_conn *htc);
 int HTC_Rx(struct http_conn *htc);
-ssize_t HTC_Read(struct http_conn *htc, void *d, size_t len);
+ssize_t HTC_Read(struct worker *w, struct http_conn *htc, void *d, size_t len);
 int HTC_Complete(struct http_conn *htc);
 
 #define HTTPH(a, b, c, d, e, f, g) extern char b[];
diff --git a/bin/varnishd/cache_esi_fetch.c b/bin/varnishd/cache_esi_fetch.c
index 1c75054..728436c 100644
--- a/bin/varnishd/cache_esi_fetch.c
+++ b/bin/varnishd/cache_esi_fetch.c
@@ -44,7 +44,8 @@
  */
 
 static ssize_t
-vef_read(struct http_conn *htc, void *buf, ssize_t buflen, ssize_t bytes)
+vef_read(struct worker *w, struct http_conn *htc, void *buf, ssize_t buflen,
+    ssize_t bytes)
 {
 	ssize_t d;
 
@@ -55,7 +56,7 @@ vef_read(struct http_conn *htc, void *buf, ssize_t buflen, ssize_t bytes)
 		if (d < bytes)
 			bytes = d;
 	}
-	return (HTC_Read(htc, buf, bytes));
+	return (HTC_Read(w, htc, buf, bytes));
 }
 
 /*---------------------------------------------------------------------
@@ -74,7 +75,7 @@ vfp_esi_bytes_uu(struct worker *w, struct http_conn *htc, ssize_t bytes)
 		st = FetchStorage(w, 0);
 		if (st == NULL)
 			return (-1);
-		wl = vef_read(htc,
+		wl = vef_read(w, htc,
 		    st->ptr + st->len, st->space - st->len, bytes);
 		if (wl <= 0)
 			return (wl);
@@ -105,14 +106,14 @@ vfp_esi_bytes_gu(struct worker *w, struct http_conn *htc, ssize_t bytes)
 
 	while (bytes > 0) {
 		if (VGZ_IbufEmpty(vg) && bytes > 0) {
-			wl = vef_read(htc, ibuf, sizeof ibuf, bytes);
+			wl = vef_read(w, htc, ibuf, sizeof ibuf, bytes);
 			if (wl <= 0)
 				return (wl);
 			VGZ_Ibuf(vg, ibuf, wl);
 			bytes -= wl;
 		}
 		if (VGZ_ObufStorage(w, vg))
-			return(FetchError(w, "Could not get storage"));
+			return(-1);
 		i = VGZ_Gunzip(vg, &dp, &dl);
 		xxxassert(i == VGZ_OK || i == VGZ_END);
 		VEP_Parse(w, dp, dl);
@@ -180,7 +181,7 @@ vfp_vep_callback(struct worker *w, ssize_t l, enum vgz_flag flg)
 		}
 		do {
 			if (VGZ_ObufStorage(w, vef->vgz)) {
-				vef->error = errno;
+				vef->error = ENOMEM;
 				vef->tot += l;
 				return (vef->tot);
 			}
@@ -203,7 +204,7 @@ vfp_vep_callback(struct worker *w, ssize_t l, enum vgz_flag flg)
 }
 
 static int
-vfp_esi_bytes_ug(const struct worker *w, struct http_conn *htc, ssize_t bytes)
+vfp_esi_bytes_ug(struct worker *w, struct http_conn *htc, ssize_t bytes)
 {
 	ssize_t wl;
 	char ibuf[params->gzip_stack_buffer];
@@ -214,7 +215,7 @@ vfp_esi_bytes_ug(const struct worker *w, struct http_conn *htc, ssize_t bytes)
 	CHECK_OBJ_NOTNULL(vef, VEF_MAGIC);
 
 	while (bytes > 0) {
-		wl = vef_read(htc, ibuf, sizeof ibuf, bytes);
+		wl = vef_read(w, htc, ibuf, sizeof ibuf, bytes);
 		if (wl <= 0)
 			return (wl);
 		bytes -= wl;
@@ -240,7 +241,7 @@ vfp_esi_bytes_ug(const struct worker *w, struct http_conn *htc, ssize_t bytes)
  */
 
 static int 
-vfp_esi_bytes_gg(const struct worker *w, struct http_conn *htc, size_t bytes)
+vfp_esi_bytes_gg(struct worker *w, struct http_conn *htc, size_t bytes)
 {
 	ssize_t wl;
 	char ibuf[params->gzip_stack_buffer];
@@ -257,7 +258,7 @@ vfp_esi_bytes_gg(const struct worker *w, struct http_conn *htc, size_t bytes)
 	ibuf2[0] = 0; /* For Flexelint */
 
 	while (bytes > 0) {
-		wl = vef_read(htc, ibuf, sizeof ibuf, bytes);
+		wl = vef_read(w, htc, ibuf, sizeof ibuf, bytes);
 		if (wl <= 0)
 			return (wl);
 		bytes -= wl;
@@ -334,6 +335,7 @@ vfp_esi_bytes(struct worker *w, struct http_conn *htc, ssize_t bytes)
 	CHECK_OBJ_NOTNULL(w, WORKER_MAGIC);
 	AZ(w->fetch_failed);
 	AN(w->vep);
+	assert(w->htc == htc);
 	if (w->is_gzip && w->do_gunzip)
 		i = vfp_esi_bytes_gu(w, htc, bytes);
 	else if (w->is_gunzip && w->do_gzip)
diff --git a/bin/varnishd/cache_fetch.c b/bin/varnishd/cache_fetch.c
index dd3a1c7..fb2ce41 100644
--- a/bin/varnishd/cache_fetch.c
+++ b/bin/varnishd/cache_fetch.c
@@ -120,14 +120,12 @@ vfp_nop_bytes(struct worker *w, struct http_conn *htc, ssize_t bytes)
 	while (bytes > 0) {
 		st = FetchStorage(w, 0);
 		if (st == NULL)
-			return(FetchError(w, "Could not get storage"));
+			return(-1);
 		l = st->space - st->len;
 		if (l > bytes)
 			l = bytes;
-		wl = HTC_Read(htc, st->ptr + st->len, l);
-		if (wl < 0)
-			return(FetchError(w, htc->error));
-		if (wl == 0)
+		wl = HTC_Read(w, htc, st->ptr + st->len, l);
+		if (wl <= 0)
 			return (wl);
 		st->len += wl;
 		w->fetch_obj->len += wl;
@@ -173,7 +171,8 @@ static struct vfp vfp_nop = {
 };
 
 /*--------------------------------------------------------------------
- * Fetch Storage
+ * Fetch Storage to put object into.
+ *
  */
 
 struct storage *
@@ -196,7 +195,7 @@ FetchStorage(struct worker *w, ssize_t sz)
 		l = params->fetch_chunksize * 1024LL;
 	st = STV_alloc(w, l);
 	if (st == NULL) {
-		errno = ENOMEM;
+		(void)FetchError(w, "Could not get storage");
 		return (NULL);
 	}
 	AZ(st->len);
@@ -247,17 +246,11 @@ fetch_straight(struct worker *w, struct http_conn *htc, ssize_t cl)
 	return (0);
 }
 
-/*--------------------------------------------------------------------*/
-/* XXX: Cleanup.  It must be possible somehow :-( */
-
-#define CERR() do {						\
-		if (i != 1) {					\
-			WSLB(w, SLT_FetchError,			\
-			    "chunked read_error: %d (%s)",	\
-			    errno, htc->error);			\
-			return (-1);				\
-		}						\
-	} while (0)
+/*--------------------------------------------------------------------
+ * Read a chunked HTTP object.
+ * XXX: Reading one byte at a time is pretty pessimal.
+ */
+ 
 
 static int
 fetch_chunked(struct worker *w, struct http_conn *htc)
@@ -271,15 +264,17 @@ fetch_chunked(struct worker *w, struct http_conn *htc)
 	do {
 		/* Skip leading whitespace */
 		do {
-			i = HTC_Read(htc, buf, 1);
-			CERR();
+			i = HTC_Read(w, htc, buf, 1);
+			if (i <= 0)
+				return (i);
 		} while (vct_islws(buf[0]));
 
 		/* Collect hex digits, skipping leading zeros */
 		for (u = 1; u < sizeof buf; u++) {
 			do {
-				i = HTC_Read(htc, buf + u, 1);
-				CERR();
+				i = HTC_Read(w, htc, buf + u, 1);
+				if (i <= 0)
+					return (i);
 			} while (u == 1 && buf[0] == '0' && buf[u] == '0');
 			if (!vct_ishex(buf[u]))
 				break;
@@ -290,8 +285,9 @@ fetch_chunked(struct worker *w, struct http_conn *htc)
 
 		/* Skip trailing white space */
 		while(vct_islws(buf[u]) && buf[u] != '\n') {
-			i = HTC_Read(htc, buf + u, 1);
-			CERR();
+			i = HTC_Read(w, htc, buf + u, 1);
+			if (i <= 0)
+				return (i);
 		}
 
 		if (buf[u] != '\n') 
@@ -306,11 +302,13 @@ fetch_chunked(struct worker *w, struct http_conn *htc)
 			if (i <= 0)
 				return (-1);
 		}
-		i = HTC_Read(htc, buf, 1);
-		CERR();
+		i = HTC_Read(w, htc, buf, 1);
+		if (i <= 0)
+			return (-1);
 		if (buf[0] == '\r') {
-			i = HTC_Read(htc, buf, 1);
-			CERR();
+			i = HTC_Read(w, htc, buf, 1);
+			if (i <= 0)
+				return (-1);
 		}
 		if (buf[0] != '\n')
 			return (FetchError(w,"chunked tail no NL"));
@@ -358,7 +356,7 @@ FetchReqBody(struct sess *sp)
 				rdcnt = sizeof buf;
 			else
 				rdcnt = content_length;
-			rdcnt = HTC_Read(sp->htc, buf, rdcnt);
+			rdcnt = HTC_Read(sp->wrk, sp->htc, buf, rdcnt);
 			if (rdcnt <= 0)
 				return (1);
 			content_length -= rdcnt;
@@ -459,7 +457,7 @@ FetchHdr(struct sess *sp)
 
 	if (i < 0) {
 		WSP(sp, SLT_FetchError, "http first read error: %d %d (%s)",
-		    i, errno, w->htc->error);
+		    i, errno, strerror(errno));
 		VDI_CloseFd(sp->wrk);
 		/* XXX: other cleanup ? */
 		/* Retryable if we never received anything */
@@ -473,7 +471,7 @@ FetchHdr(struct sess *sp)
 		if (i < 0) {
 			WSP(sp, SLT_FetchError,
 			    "http first read error: %d %d (%s)",
-			    i, errno, w->htc->error);
+			    i, errno, strerror(errno));
 			VDI_CloseFd(sp->wrk);
 			/* XXX: other cleanup ? */
 			return (-1);
diff --git a/bin/varnishd/cache_gzip.c b/bin/varnishd/cache_gzip.c
index 73f4bb9..1820ffc 100644
--- a/bin/varnishd/cache_gzip.c
+++ b/bin/varnishd/cache_gzip.c
@@ -480,17 +480,15 @@ vfp_gunzip_bytes(struct worker *w, struct http_conn *htc, ssize_t bytes)
 			l = sizeof ibuf;
 			if (l > bytes)
 				l = bytes;
-			wl = HTC_Read(htc, ibuf, l);
-			if (wl < 0)
-				return(FetchError(w, htc->error));
-			if (wl == 0)
+			wl = HTC_Read(w, htc, ibuf, l);
+			if (wl <= 0)
 				return (wl);
 			VGZ_Ibuf(vg, ibuf, wl);
 			bytes -= wl;
 		}
 
 		if (VGZ_ObufStorage(w, vg)) 
-			return(FetchError(w, "Could not get storage"));
+			return(-1);
 		i = VGZ_Gunzip(vg, &dp, &dl);
 		if (i != VGZ_OK && i != VGZ_END) 
 			return(FetchError(w, "Gunzip data error"));
@@ -560,16 +558,14 @@ vfp_gzip_bytes(struct worker *w, struct http_conn *htc, ssize_t bytes)
 			l = sizeof ibuf;
 			if (l > bytes)
 				l = bytes;
-			wl = HTC_Read(htc, ibuf, l);
-			if (wl < 0)
-				return(FetchError(w, htc->error));
-			if (wl == 0)
+			wl = HTC_Read(w, htc, ibuf, l);
+			if (wl <= 0)
 				return (wl);
 			VGZ_Ibuf(vg, ibuf, wl);
 			bytes -= wl;
 		}
 		if (VGZ_ObufStorage(w, vg)) 
-			return(FetchError(w, "Could not get storage"));
+			return(-1);
 		i = VGZ_Gzip(vg, &dp, &dl, VGZ_NORMAL);
 		assert(i == Z_OK);
 		w->fetch_obj->len += dl;
@@ -597,7 +593,7 @@ vfp_gzip_end(struct worker *w)
 	do {
 		VGZ_Ibuf(vg, "", 0);
 		if (VGZ_ObufStorage(w, vg))
-			return(FetchError(w, "Could not get storage"));
+			return(-1);
 		i = VGZ_Gzip(vg, &dp, &dl, VGZ_FINISH);
 		w->fetch_obj->len += dl;
 	} while (i != Z_STREAM_END);
@@ -648,14 +644,12 @@ vfp_testgzip_bytes(struct worker *w, struct http_conn *htc, ssize_t bytes)
 	while (bytes > 0) {
 		st = FetchStorage(w, 0);
 		if (st == NULL)
-			return(FetchError(w, "Could not get storage"));
+			return(-1);
 		l = st->space - st->len;
 		if (l > bytes)
 			l = bytes;
-		wl = HTC_Read(htc, st->ptr + st->len, l);
-		if (wl < 0)
-			return(FetchError(w, htc->error));
-		if (wl == 0)
+		wl = HTC_Read(w, htc, st->ptr + st->len, l);
+		if (wl <= 0)
 			return (wl);
 		bytes -= wl;
 		VGZ_Ibuf(vg, st->ptr + st->len, wl);
diff --git a/bin/varnishd/cache_httpconn.c b/bin/varnishd/cache_httpconn.c
index ba881c9..1b718e5 100644
--- a/bin/varnishd/cache_httpconn.c
+++ b/bin/varnishd/cache_httpconn.c
@@ -27,6 +27,17 @@
  * SUCH DAMAGE.
  *
  * HTTP protocol requests
+ *
+ * The trouble with the "until magic sequence" design of HTTP protocol messages
+ * is that either you have to read a single character at a time, which is
+ * inefficient, or you risk reading too much, and pre-read some of the object,
+ * or even the next pipelined request, which follows the one you want.
+ *
+ * HTC reads a HTTP protocol header into a workspace, subject to limits,
+ * and stops when we see the magic marker (double [CR]NL), and if we overshoot,
+ * it keeps track of the "pipelined" data.
+ *
+ * We use this both for client and backend connections.
  */
 
 #include "config.h"
@@ -53,7 +64,8 @@ htc_header_complete(txt *t)
 	/* Skip any leading white space */
 	for (p = t->b ; vct_issp(*p); p++)
 		continue;
-	if (*p == '\0') {
+	if (p == t->e) {
+		/* All white space */
 		t->e = t->b;
 		*t->e = '\0';
 		return (0);
@@ -85,7 +97,6 @@ HTC_Init(struct http_conn *htc, struct ws *ws, int fd, unsigned vsl_id,
 	htc->vsl_id = vsl_id;
 	htc->maxbytes = maxbytes;
 	htc->maxhdr = maxhdr;
-	htc->error = "No error recorded";
 
 	(void)WS_Reserve(htc->ws, htc->maxbytes);
 	htc->rxbuf.b = ws->f;
@@ -107,7 +118,6 @@ HTC_Reinit(struct http_conn *htc)
 	unsigned l;
 
 	CHECK_OBJ_NOTNULL(htc, HTTP_CONN_MAGIC);
-	htc->error = "No error recorded";
 	(void)WS_Reserve(htc->ws, htc->maxbytes);
 	htc->rxbuf.b = htc->ws->f;
 	htc->rxbuf.e = htc->ws->f;
@@ -123,7 +133,7 @@ HTC_Reinit(struct http_conn *htc)
 }
 
 /*--------------------------------------------------------------------
- *
+ * Return 1 if we have a complete HTTP procol header
  */
 
 int
@@ -137,6 +147,8 @@ HTC_Complete(struct http_conn *htc)
 	if (i == 0)
 		return (0);
 	WS_ReleaseP(htc->ws, htc->rxbuf.e);
+	AZ(htc->pipeline.b);
+	AZ(htc->pipeline.e);
 	if (htc->rxbuf.b + i < htc->rxbuf.e) {
 		htc->pipeline.b = htc->rxbuf.b + i;
 		htc->pipeline.e = htc->rxbuf.e;
@@ -168,6 +180,10 @@ HTC_Rx(struct http_conn *htc)
 	}
 	i = read(htc->fd, htc->rxbuf.e, i);
 	if (i <= 0) {
+		/*
+		 * We wouldn't come here if we had a complete HTTP header
+		 * so consequently an EOF can not be OK
+		 */
 		WS_ReleaseP(htc->ws, htc->rxbuf.b);
 		return (-1);
 	}
@@ -176,16 +192,21 @@ HTC_Rx(struct http_conn *htc)
 	return (HTC_Complete(htc));
 }
 
+/*--------------------------------------------------------------------
+ * Read up to len bytes, returning pipelined data first.
+ */
+
 ssize_t
-HTC_Read(struct http_conn *htc, void *d, size_t len)
+HTC_Read(struct worker *w, struct http_conn *htc, void *d, size_t len)
 {
 	size_t l;
 	unsigned char *p;
 	ssize_t i;
 
+	CHECK_OBJ_NOTNULL(w, WORKER_MAGIC);
+	CHECK_OBJ_NOTNULL(htc, HTTP_CONN_MAGIC);
 	l = 0;
 	p = d;
-	CHECK_OBJ_NOTNULL(htc, HTTP_CONN_MAGIC);
 	if (htc->pipeline.b) {
 		l = Tlen(htc->pipeline);
 		if (l > len)
@@ -201,9 +222,8 @@ HTC_Read(struct http_conn *htc, void *d, size_t len)
 		return (l);
 	i = read(htc->fd, p, len);
 	if (i < 0) {
-		htc->error = strerror(errno);
+		WSL(w, SLT_FetchError, htc->vsl_id, "%s", strerror(errno));
 		return (i);
-	} else if (i == 0)
-		htc->error = "Remote closed connection";
+	}
 	return (i + l);
 }



From geoff at varnish-cache.org  Mon Jan  9 20:52:28 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:28 +0100
Subject: [experimental-ims] fe425ce Avoid infinite loop in regsuball
Message-ID: <E1RkMCi-00080B-8e@project.varnish-software.com>

commit fe425ce63a03b0de1a9c257c4e27cf1defe6a862
Author: Tollef Fog Heen <tfheen at varnish-software.com>
Date:   Tue Nov 1 09:25:22 2011 +0100

    Avoid infinite loop in regsuball
    
    Also optimise out a few strlen calls in favour of just tracking the
    lengths.
    
    Fixes: #1047

diff --git a/bin/varnishd/cache_vrt_re.c b/bin/varnishd/cache_vrt_re.c
index e6f854b..955b7bf 100644
--- a/bin/varnishd/cache_vrt_re.c
+++ b/bin/varnishd/cache_vrt_re.c
@@ -90,13 +90,16 @@ VRT_regsub(const struct sess *sp, int all, const char *str, void *re,
 	char *b0;
 	const char *s;
 	unsigned u, x;
+	int options = 0;
+	size_t len;
 
 	AN(re);
 	if (str == NULL)
 		str = "";
 	t = re;
 	memset(ovector, 0, sizeof(ovector));
-	i = VRE_exec(t, str, strlen(str), 0, 0, ovector, 30,
+	len = strlen(str);
+	i = VRE_exec(t, str, len, 0, options, ovector, 30,
 	    &params->vre_limits);
 
 	/* If it didn't match, we can return the original string */
@@ -132,10 +135,12 @@ VRT_regsub(const struct sess *sp, int all, const char *str, void *re,
 			}
 		}
 		str += ovector[1];
+		len -= ovector[1];
 		if (!all)
 			break;
 		memset(&ovector, 0, sizeof(ovector));
-		i = VRE_exec(t, str, strlen(str), 0, 0, ovector, 30,
+		options |= VRE_NOTEMPTY_ATSTART;
+		i = VRE_exec(t, str, len, 0, options, ovector, 30,
 		    &params->vre_limits);
 		if (i < VRE_ERROR_NOMATCH ) {
 			WSP(sp, SLT_VCL_Error,
@@ -145,8 +150,7 @@ VRT_regsub(const struct sess *sp, int all, const char *str, void *re,
 	} while (i != VRE_ERROR_NOMATCH);
 
 	/* Copy suffix to match */
-	l = strlen(str) + 1;
-	Tadd(&res, str, l);
+	Tadd(&res, str, len+1);
 	if (res.b >= res.e) {
 		WS_Release(sp->http->ws, 0);
 		return (str);
diff --git a/bin/varnishtest/tests/c00047.vtc b/bin/varnishtest/tests/c00047.vtc
new file mode 100644
index 0000000..4b23194
--- /dev/null
+++ b/bin/varnishtest/tests/c00047.vtc
@@ -0,0 +1,33 @@
+varnishtest "Test VCL regsuball()"
+
+server s1 {
+	rxreq 
+	txresp \
+		-hdr "foo: barbar" \
+		-hdr "bar: bbbar"
+} -start
+
+varnish v1 -vcl+backend { 
+	sub vcl_fetch {
+		set beresp.http.baz1 = regsuball(beresp.http.foo, "barb", "zz");
+		set beresp.http.baz2 = regsuball(beresp.http.foo, "ar", "zz");
+		set beresp.http.baz3 = regsuball(beresp.http.foo, "^", "baz");
+		set beresp.http.baz4 = regsuball(beresp.http.foo, "^[;]*", "baz");
+		set beresp.http.baz5 = regsuball(beresp.http.bar, "^b*", "b");
+		set beresp.http.baz6 = regsuball(beresp.http.foo, "^b*", "z");
+		set beresp.http.baz7 = regsuball(beresp.http.foo, "ping", "pong");
+	}
+} -start 
+
+client c1 {
+	txreq -url "/"
+	rxresp
+	expect resp.status == 200
+	expect resp.http.baz1 == "zzar"
+	expect resp.http.baz2 == "bzzbzz"
+	expect resp.http.baz3 == "bazbarbar"
+	expect resp.http.baz4 == "bazbarbar"
+	expect resp.http.baz5 == "bar"
+	expect resp.http.baz6 == "zarbar"
+	expect resp.http.baz7 == "barbar"
+} -run
diff --git a/include/vre.h b/include/vre.h
index 29c1b08..a1206e5 100644
--- a/include/vre.h
+++ b/include/vre.h
@@ -49,6 +49,7 @@ typedef struct vre vre_t;
 
 /* And those to PCRE options */
 #define VRE_CASELESS           0x00000001
+#define VRE_NOTEMPTY_ATSTART   0x10000000
 
 vre_t *VRE_compile(const char *, int, const char **, int *);
 int VRE_exec(const vre_t *code, const char *subject, int length,



From geoff at varnish-cache.org  Mon Jan  9 20:52:28 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:28 +0100
Subject: [experimental-ims] 41c39c8 Add health control of backends from CLI
Message-ID: <E1RkMCi-00081w-VV@project.varnish-software.com>

commit 41c39c857907f3b26f8ce540863121b27288649c
Author: Tollef Fog Heen <tfheen at varnish-software.com>
Date:   Thu Nov 3 13:03:51 2011 +0100

    Add health control of backends from CLI
    
    Make it possible to mark backends as sick or healthy by way of
    
      backend.set_health $backendname (sick|healthy|auto)

diff --git a/bin/varnishd/cache_backend.c b/bin/varnishd/cache_backend.c
index 1a3ae5b..cad637b 100644
--- a/bin/varnishd/cache_backend.c
+++ b/bin/varnishd/cache_backend.c
@@ -256,7 +256,10 @@ vbe_Healthy(const struct vdi_simple *vs, const struct sess *sp)
 	backend = vs->backend;
 	CHECK_OBJ_NOTNULL(backend, BACKEND_MAGIC);
 
-	if (!backend->healthy)
+	if (backend->admin_health == from_probe && !backend->healthy)
+		return (0);
+
+	if (backend->admin_health == sick)
 		return (0);
 
 	/* VRT/VCC sets threshold to UINT_MAX to mark that it's not
@@ -267,6 +270,9 @@ vbe_Healthy(const struct vdi_simple *vs, const struct sess *sp)
 	else
 		threshold = vs->vrt->saintmode_threshold;
 
+	if (backend->admin_health == healthy)
+		threshold = UINT_MAX;
+
 	/* Saintmode is disabled */
 	if (threshold == 0)
 		return (1);
diff --git a/bin/varnishd/cache_backend.h b/bin/varnishd/cache_backend.h
index 8937673..880c429 100644
--- a/bin/varnishd/cache_backend.h
+++ b/bin/varnishd/cache_backend.h
@@ -106,6 +106,12 @@ struct trouble {
  * An instance of a backend from a VCL program.
  */
 
+enum health_status {
+	healthy,
+	sick,
+	from_probe
+};
+
 struct backend {
 	unsigned		magic;
 #define BACKEND_MAGIC		0x64c4c7c6
@@ -130,6 +136,7 @@ struct backend {
 
 	struct vbp_target	*probe;
 	unsigned		healthy;
+	enum health_status	admin_health;
 	VTAILQ_HEAD(, trouble)	troublelist;
 
 	struct VSC_C_vbe	*vsc;
diff --git a/bin/varnishd/cache_backend_cfg.c b/bin/varnishd/cache_backend_cfg.c
index a4f3285..8168be0 100644
--- a/bin/varnishd/cache_backend_cfg.c
+++ b/bin/varnishd/cache_backend_cfg.c
@@ -38,6 +38,7 @@
 #include "cache.h"
 
 #include "cache_backend.h"
+#include "vcli.h"
 #include "vcli_priv.h"
 #include "vrt.h"
 
@@ -229,6 +230,7 @@ VBE_AddBackend(struct cli *cli, const struct vrt_backend *vb)
 	assert(b->ipv4 != NULL || b->ipv6 != NULL);
 
 	b->healthy = 1;
+	b->admin_health = from_probe;
 
 	VTAILQ_INSERT_TAIL(&backends, b, list);
 	VSC_C_main->n_backend++;
@@ -274,25 +276,151 @@ VRT_fini_dir(struct cli *cli, struct director *b)
 
 /*--------------------------------------------------------------------*/
 
+static int
+backend_find(const char *matcher, struct backend **r, int n)
+{
+	struct backend *b;
+	char *vcl_name;
+	char *s;
+	char *match_ip = NULL;
+	char *match_port = NULL;
+	int found = 0;
+
+	s = strchr(matcher, '(');
+
+	if (s == NULL) {
+		/* Simple match, max one hit */
+		VTAILQ_FOREACH(b, &backends, list) {
+			CHECK_OBJ_NOTNULL(b, BACKEND_MAGIC);
+			if (strcmp(b->vcl_name, matcher) == 0) {
+				if (r && found < n)
+					r[found] = b;
+				found++;
+			}
+		}
+		return found;
+	}
+
+	vcl_name = strndup(matcher, s - matcher);
+	AN(vcl_name);
+	s++;
+	while (*s != ')') {
+		if (*s == ':') {
+			/* Port */
+			s++;
+			match_port = s;
+			if (!(s = strchr(match_port, ','))) {
+				s = strchr(match_port, ')');
+			}
+			XXXAN(s);
+			match_port = strndup(match_port, s - match_port);
+			AN(match_port);
+			if (*s == ',')
+				s++;
+		} else {
+			/* IP */
+			match_ip = s;
+			if (!(s = strchr(match_ip, ','))) {
+				s = strchr(match_ip, ')');
+			}
+			XXXAN(s);
+			match_ip = strndup(match_ip, s - match_ip);
+			AN(match_ip);
+			if (*s == ',')
+				s++;
+		}
+	}
+	VTAILQ_FOREACH(b, &backends, list) {
+		CHECK_OBJ_NOTNULL(b, BACKEND_MAGIC);
+		if (match_port && strcmp(b->port, match_port) != 0)
+			continue;
+		if (match_ip &&
+		    (strcmp(b->ipv4_addr, match_ip) != 0) &&
+		    (strcmp(b->ipv6_addr, match_ip) != 0))
+			continue;
+		if (strcmp(b->vcl_name, vcl_name) == 0) {
+			if (r && found < n)
+				r[found] = b;
+			found++;
+		}
+	}
+	return found;
+}
+
 static void
-cli_debug_backend(struct cli *cli, const char * const *av, void *priv)
+cli_backend_list(struct cli *cli, const char * const *av, void *priv)
 {
 	struct backend *b;
+	const char *ah;
 
 	(void)av;
 	(void)priv;
 	ASSERT_CLI();
+	VCLI_Out(cli, "%-30s %10s %15s %15s", "Backend name",
+		 "Conns", "Probed healthy", "Admin health");
 	VTAILQ_FOREACH(b, &backends, list) {
 		CHECK_OBJ_NOTNULL(b, BACKEND_MAGIC);
-		VCLI_Out(cli, "%p %s %d %d\n",
-			 b, b->display_name,
-			 b->refcount, b->n_conn);
+		if (b->admin_health == from_probe) {
+			ah = "Auto";
+		} else if (b->admin_health == sick) {
+			ah = "Sick";
+		} else {
+			ah = "Healthy";
+		}
+		VCLI_Out(cli, "\n%-30s %10d %15s %15s",
+			 b->display_name,
+			 b->refcount,
+			 (b ->healthy ? "Yes" : "No"),
+			 ah);
+	}
+}
+
+static void
+cli_backend_set_health(struct cli *cli, const char * const *av, void *priv)
+{
+	struct backend **b;
+	enum health_status state;
+	int n;
+	const char *wstate;
+
+	(void)av;
+	(void)priv;
+	ASSERT_CLI();
+	wstate = av[3];
+	if (strcmp(wstate, "healthy") == 0) {
+		state = healthy;
+	} else if (strcmp(wstate, "sick") == 0) {
+		state = sick;
+	} else if (strcmp(wstate, "auto") == 0) {
+		state = from_probe;
+	} else {
+		VCLI_Out(cli, "Invalid state %s", wstate);
+		VCLI_SetResult(cli, CLIS_CANT);
+		return;
+	}
+	n = backend_find(av[2], NULL, 0);
+	if (n == 0) {
+		VCLI_Out(cli, "No matching backends");
+		VCLI_SetResult(cli, CLIS_CANT);
+		return;
+	}
+
+	b = calloc(n, sizeof(struct backend *));
+	AN(b);
+	n = backend_find(av[2], b, n);
+
+	VCLI_Out(cli, "Set state to %s for the following backends:", wstate);
+	for (int i = 0; i < n; i++) {
+		b[i]->admin_health = state;
+		VCLI_Out(cli, "\n\t%s", b[i]->display_name);
 	}
 }
 
-static struct cli_proto debug_cmds[] = {
-	{ "debug.backend", "debug.backend",
-	    "\tExamine Backend internals\n", 0, 0, "d", cli_debug_backend },
+static struct cli_proto backend_cmds[] = {
+	{ "backend.list", "backend.list",
+	    "\tList all backends\n", 0, 0, "d", cli_backend_list },
+	{ "backend.set_health", "backend.set_health matcher state",
+	    "\tShow a backend\n", 2, 2, "d", cli_backend_set_health },
 	{ NULL }
 };
 
@@ -303,5 +431,5 @@ VBE_Init(void)
 {
 
 	Lck_New(&VBE_mtx, lck_vbe);
-	CLI_AddFuncs(debug_cmds);
+	CLI_AddFuncs(backend_cmds);
 }
diff --git a/bin/varnishtest/tests/c00048.vtc b/bin/varnishtest/tests/c00048.vtc
new file mode 100644
index 0000000..ed8671f
--- /dev/null
+++ b/bin/varnishtest/tests/c00048.vtc
@@ -0,0 +1,55 @@
+varnishtest "Forcing health of backends"
+
+server s1 -repeat 3 {
+	rxreq
+	txresp
+} -start
+
+varnish v1 -vcl {
+	backend s1 {
+		.host = "${s1_addr}";
+		.port = "${s1_port}";
+		.probe = {
+			.window = 8;
+			.initial = 7;
+			.threshold = 8;
+			.interval = 10s;
+		}
+	}
+
+	sub vcl_recv {
+		return(pass);
+	}
+
+} -start
+
+delay 1
+
+varnish v1 -cliok "backend.set_health s1 auto"
+
+client c1 {
+	txreq
+	rxresp
+	expect resp.status == 200
+} -run
+
+varnish v1 -cliok "backend.set_health s1 sick"
+
+client c1 {
+	txreq
+	rxresp
+	expect resp.status == 503
+} -run
+
+varnish v1 -cliok "backend.set_health s1 healthy"
+
+client c1 {
+	txreq
+	rxresp
+	expect resp.status == 200
+} -run
+
+varnish v1 -clierr 300 "backend.set_health s1 foo"
+varnish v1 -clierr 300 "backend.set_health s2 foo"
+varnish v1 -clierr 300 "backend.set_health s2 auto"
+



From geoff at varnish-cache.org  Mon Jan  9 20:52:29 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:29 +0100
Subject: [experimental-ims] 878036c Revert "Add support for JSON output
	format". Work in progress, pushed in wrong direction.
Message-ID: <E1RkMCj-00082L-5Y@project.varnish-software.com>

commit 878036c77425393f58a53d05b5de97d063a7cddf
Author: Lasse Karstensen <lasse at varnish-software.com>
Date:   Thu Nov 3 19:24:37 2011 +0100

    Revert "Add support for JSON output format". Work in progress, pushed in wrong direction.
    
    This reverts commit 5f45e4ca4996027638a0b28949aed68ffe7eaf8c.

diff --git a/bin/varnishstat/varnishstat.c b/bin/varnishstat/varnishstat.c
index 00ddda8..3c6c4b5 100644
--- a/bin/varnishstat/varnishstat.c
+++ b/bin/varnishstat/varnishstat.c
@@ -82,60 +82,6 @@ do_xml(struct VSM_data *vd)
 	printf("</varnishstat>\n");
 }
 
-
-/*--------------------------------------------------------------------*/
-
-struct json_priv {
-	int first;
-};
-
-static int
-do_json_cb(void *priv, const struct VSC_point * const pt)
-{
-	uint64_t val;
-	struct json_priv *jp;
-	jp = priv;
-
-	assert(!strcmp(pt->fmt, "uint64_t"));
-	val = *(const volatile uint64_t*)pt->ptr;
-
-	if (jp->first ) jp->first = 0; else printf(",\n");
-	printf("\t\"%s\": {", pt->name);
-	if (strcmp(pt->class, ""))
-		printf("\t\t\"type\": \"%s\",\n", pt->class);
-	if (strcmp(pt->ident, ""))
-		printf("\t\t\"ident\": \"%s\",\n", pt->ident);
-	printf("\"value\": %ju, ", val);
-
-	printf("\"flag\": \"%c\",", pt->flag);
-	printf("\"description\": \"%s\"", pt->desc);
-	printf("}"); // closes name section.
-	if (jp->first) printf("\n");
-	return (0);
-}
-
-static void
-do_json(struct VSM_data *vd)
-{
-	/// TODO: support jsonp?
-	char time_stamp[20];
-	time_t now;
-
-	struct json_priv jp;
-
-	memset(&jp, 0, sizeof jp);
-	jp.first = 1;
-
-	printf("{\n"); 
-	now = time(NULL);
-
-	(void)strftime(time_stamp, 20, "%Y-%m-%dT%H:%M:%S", localtime(&now));
-	printf("\t\"timestamp\": \"%s\",\n", time_stamp);
-	(void)VSC_Iter(vd, do_json_cb, &jp);
-	printf("\n}");
-}
-
-
 /*--------------------------------------------------------------------*/
 
 struct once_priv {
@@ -143,7 +89,6 @@ struct once_priv {
 	int pad;
 };
 
-
 static int
 do_once_cb(void *priv, const struct VSC_point * const pt)
 {
@@ -246,12 +191,12 @@ main(int argc, char * const *argv)
 	int c;
 	struct VSM_data *vd;
 	const struct VSC_C_main *VSC_C_main;
-	int delay = 1, once = 0, xml = 0, json = 0;
+	int delay = 1, once = 0, xml = 0;
 
 	vd = VSM_New();
 	VSC_Setup(vd);
 
-	while ((c = getopt(argc, argv, VSC_ARGS "1f:lVw:xj")) != -1) {
+	while ((c = getopt(argc, argv, VSC_ARGS "1f:lVw:x")) != -1) {
 		switch (c) {
 		case '1':
 			once = 1;
@@ -270,9 +215,6 @@ main(int argc, char * const *argv)
 		case 'x':
 			xml = 1;
 			break;
-		case 'j':
-			json = 1;
-			break;
 		default:
 			if (VSC_Arg(vd, c, optarg) > 0)
 				break;
@@ -287,8 +229,6 @@ main(int argc, char * const *argv)
 
 	if (xml)
 		do_xml(vd);
-	else if (json)
-		do_json(vd);
 	else if (once)
 		do_once(vd, VSC_C_main);
 	else



From geoff at varnish-cache.org  Mon Jan  9 20:52:29 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:29 +0100
Subject: [experimental-ims] fa2af30 FlexeLint spotted a memory leak in Tollefs
	"admin-health" code, so I started fixing that,
	and then I fixed another couple of nits and polished the naming a bit
	and ...
Message-ID: <E1RkMCj-000834-DN@project.varnish-software.com>

commit fa2af30fc399792a5a92676db75241964d9ac154
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Nov 7 11:06:42 2011 +0000

    FlexeLint spotted a memory leak in Tollefs "admin-health" code, so I
    started fixing that, and then I fixed another couple of nits and polished
    the naming a bit and ...
    
    Well, you know how it is.
    
    Changed the backend matching function to take a call-back function
    to eliminate the need for the memory which got leaked.
    
    This meant that backend.list could also use it, if only the parsing
    code accepted a blank input.
    
    So I rewrote the parsing code to also allow empty names, so you can:
    	backend.list (:80)
    	backend.list (192.168)
    etc.
    
    And then I added a summary report of the probing, if it is happening,
    and explicitly say that no probing is happening if it isn't.

diff --git a/bin/varnishd/cache_backend.c b/bin/varnishd/cache_backend.c
index cad637b..c94d819 100644
--- a/bin/varnishd/cache_backend.c
+++ b/bin/varnishd/cache_backend.c
@@ -256,10 +256,10 @@ vbe_Healthy(const struct vdi_simple *vs, const struct sess *sp)
 	backend = vs->backend;
 	CHECK_OBJ_NOTNULL(backend, BACKEND_MAGIC);
 
-	if (backend->admin_health == from_probe && !backend->healthy)
+	if (backend->admin_health == ah_probe && !backend->healthy)
 		return (0);
 
-	if (backend->admin_health == sick)
+	if (backend->admin_health == ah_sick)
 		return (0);
 
 	/* VRT/VCC sets threshold to UINT_MAX to mark that it's not
@@ -270,7 +270,7 @@ vbe_Healthy(const struct vdi_simple *vs, const struct sess *sp)
 	else
 		threshold = vs->vrt->saintmode_threshold;
 
-	if (backend->admin_health == healthy)
+	if (backend->admin_health == ah_healthy)
 		threshold = UINT_MAX;
 
 	/* Saintmode is disabled */
diff --git a/bin/varnishd/cache_backend.h b/bin/varnishd/cache_backend.h
index 880c429..91bff1e 100644
--- a/bin/varnishd/cache_backend.h
+++ b/bin/varnishd/cache_backend.h
@@ -106,10 +106,11 @@ struct trouble {
  * An instance of a backend from a VCL program.
  */
 
-enum health_status {
-	healthy,
-	sick,
-	from_probe
+enum admin_health {
+	ah_invalid = 0,
+	ah_healthy,
+	ah_sick,
+	ah_probe
 };
 
 struct backend {
@@ -136,7 +137,7 @@ struct backend {
 
 	struct vbp_target	*probe;
 	unsigned		healthy;
-	enum health_status	admin_health;
+	enum admin_health	admin_health;
 	VTAILQ_HEAD(, trouble)	troublelist;
 
 	struct VSC_C_vbe	*vsc;
@@ -178,6 +179,7 @@ void VBE_DropRefLocked(struct backend *b);
 void VBP_Insert(struct backend *b, struct vrt_backend_probe const *p, const char *hosthdr);
 void VBP_Remove(struct backend *b, struct vrt_backend_probe const *p);
 void VBP_Use(const struct backend *b, const struct vrt_backend_probe const *p);
+void VBP_Summary(struct cli *cli, const struct vbp_target *vt);
 
 /* Init functions for directors */
 typedef void dir_init_f(struct cli *, struct director **, int , const void*);
diff --git a/bin/varnishd/cache_backend_cfg.c b/bin/varnishd/cache_backend_cfg.c
index 8168be0..e87584a 100644
--- a/bin/varnishd/cache_backend_cfg.c
+++ b/bin/varnishd/cache_backend_cfg.c
@@ -32,6 +32,7 @@
 
 #include "config.h"
 
+#include <ctype.h>
 #include <stdio.h>
 #include <stdlib.h>
 
@@ -44,6 +45,7 @@
 
 struct lock VBE_mtx;
 
+
 /*
  * The list of backends is not locked, it is only ever accessed from
  * the CLI thread, so there is no need.
@@ -79,6 +81,11 @@ VBE_Poll(void)
 
 	ASSERT_CLI();
 	VTAILQ_FOREACH_SAFE(b, &backends, list, b2) {
+		assert(
+			b->admin_health == ah_healthy ||
+			b->admin_health == ah_sick ||
+			b->admin_health == ah_probe
+		);
 		if (b->refcount == 0 && b->probe == NULL)
 			VBE_Nuke(b);
 	}
@@ -230,7 +237,7 @@ VBE_AddBackend(struct cli *cli, const struct vrt_backend *vb)
 	assert(b->ipv4 != NULL || b->ipv6 != NULL);
 
 	b->healthy = 1;
-	b->admin_health = from_probe;
+	b->admin_health = ah_probe;
 
 	VTAILQ_INSERT_TAIL(&backends, b, list);
 	VSC_C_main->n_backend++;
@@ -274,157 +281,222 @@ VRT_fini_dir(struct cli *cli, struct director *b)
 	b->priv = NULL;
 }
 
-/*--------------------------------------------------------------------*/
+/*---------------------------------------------------------------------
+ * String to admin_health
+ */
+
+static enum admin_health
+vbe_str2adminhealth(const char *wstate)
+{
+
+	if (strcasecmp(wstate, "healthy") == 0)
+		return(ah_healthy);
+	if (strcasecmp(wstate, "sick") == 0)
+		return(ah_sick);
+	if (strcmp(wstate, "auto") == 0)
+		return(ah_probe);
+	return (ah_invalid);
+}
+
+/*---------------------------------------------------------------------
+ * A general function for finding backends and doing things with them.
+ *
+ * Return -1 on match-argument parse errors.
+ *
+ * If the call-back function returns non-zero, the search is terminated
+ * and we relay that return value.
+ *
+ * Otherwise we return the number of matches.
+ */
+
+typedef int bf_func(struct cli *cli, struct backend *b, void *priv);
 
 static int
-backend_find(const char *matcher, struct backend **r, int n)
+backend_find(struct cli *cli, const char *matcher, bf_func *func, void *priv)
 {
 	struct backend *b;
-	char *vcl_name;
-	char *s;
-	char *match_ip = NULL;
-	char *match_port = NULL;
+	const char *s;
+	const char *name_b;
+	ssize_t name_l = 0;
+	const char *ip_b = NULL;
+	ssize_t ip_l = 0;
+	const char *port_b = NULL;
+	ssize_t port_l = 0;
 	int found = 0;
+	int i;
 
-	s = strchr(matcher, '(');
+	name_b = matcher;
+	if (matcher != NULL) {
+		s = strchr(matcher,'(');
 
-	if (s == NULL) {
-		/* Simple match, max one hit */
-		VTAILQ_FOREACH(b, &backends, list) {
-			CHECK_OBJ_NOTNULL(b, BACKEND_MAGIC);
-			if (strcmp(b->vcl_name, matcher) == 0) {
-				if (r && found < n)
-					r[found] = b;
-				found++;
-			}
-		}
-		return found;
-	}
+		if (s != NULL)
+			name_l = s - name_b;
+		else
+			name_l = strlen(name_b);
 
-	vcl_name = strndup(matcher, s - matcher);
-	AN(vcl_name);
-	s++;
-	while (*s != ')') {
-		if (*s == ':') {
-			/* Port */
+		if (s != NULL) {
 			s++;
-			match_port = s;
-			if (!(s = strchr(match_port, ','))) {
-				s = strchr(match_port, ')');
+			while (isspace(*s))
+				s++;
+			ip_b = s;
+			while (*s != '\0' &&
+			    *s != ')' &&
+			    *s != ':' &&
+			    !isspace(*s))
+				s++;
+			ip_l = s - ip_b;
+			while (isspace(*s))
+				s++;
+			if (*s == ':') {
+				s++;
+				while (isspace(*s))
+					s++;
+				port_b = s;
+				while (*s != '\0' && *s != ')' && !isspace(*s))
+					s++;
+				port_l = s - port_b;
 			}
-			XXXAN(s);
-			match_port = strndup(match_port, s - match_port);
-			AN(match_port);
-			if (*s == ',')
+			while (isspace(*s))
 				s++;
-		} else {
-			/* IP */
-			match_ip = s;
-			if (!(s = strchr(match_ip, ','))) {
-				s = strchr(match_ip, ')');
+			if (*s != ')') {
+				VCLI_Out(cli,
+				    "Match string syntax error:"
+				    " ')' not found.");
+				VCLI_SetResult(cli, CLIS_CANT);
+				return (-1);
 			}
-			XXXAN(s);
-			match_ip = strndup(match_ip, s - match_ip);
-			AN(match_ip);
-			if (*s == ',')
+			s++;
+			while (isspace(*s))
 				s++;
+			if (*s != '\0') {
+				VCLI_Out(cli,
+				    "Match string syntax error:"
+				    " junk after ')'");
+				VCLI_SetResult(cli, CLIS_CANT);
+				return (-1);
+			}
 		}
 	}
 	VTAILQ_FOREACH(b, &backends, list) {
 		CHECK_OBJ_NOTNULL(b, BACKEND_MAGIC);
-		if (match_port && strcmp(b->port, match_port) != 0)
+		if (port_b != NULL && strncmp(b->port, port_b, port_l) != 0)
 			continue;
-		if (match_ip &&
-		    (strcmp(b->ipv4_addr, match_ip) != 0) &&
-		    (strcmp(b->ipv6_addr, match_ip) != 0))
+		if (name_b != NULL && strncmp(b->vcl_name, name_b, name_l) != 0)
 			continue;
-		if (strcmp(b->vcl_name, vcl_name) == 0) {
-			if (r && found < n)
-				r[found] = b;
-			found++;
-		}
+		if (ip_b != NULL &&
+		    (b->ipv4_addr == NULL ||
+		      strncmp(b->ipv4_addr, ip_b, ip_l)) &&
+		    (b->ipv6_addr == NULL ||
+		      strncmp(b->ipv6_addr, ip_b, ip_l)))
+			continue;
+		found++;
+		i = func(cli, b, priv);
+		if (i)
+			return(i);
+	}
+	return (found);
+}
+
+/*---------------------------------------------------------------------*/
+
+static int __match_proto__()
+do_list(struct cli *cli, struct backend *b, void *priv)
+{
+	int *hdr;
+
+	AN(priv);
+	hdr = priv;
+	if (!*hdr) {
+		VCLI_Out(cli, "%-30s %-6s %-10s %s",
+		    "Backend name", "Refs", "Admin", "Probe");
+		*hdr = 1;
+	}
+	CHECK_OBJ_NOTNULL(b, BACKEND_MAGIC);
+
+	VCLI_Out(cli, "\n%-30s %-6d", b->display_name, b->refcount);
+
+	if (b->admin_health == ah_probe)
+		VCLI_Out(cli, " %-10s", "probe");
+	else if (b->admin_health == ah_sick)
+		VCLI_Out(cli, " %-10s", "sick");
+	else if (b->admin_health == ah_healthy)
+		VCLI_Out(cli, " %-10s", "healthy");
+	else
+		VCLI_Out(cli, " %-10s", "invalid");
+
+	if (b->probe == NULL)
+		VCLI_Out(cli, " %s", "Healthy (no probe)");
+	else {
+		if (b->healthy)
+			VCLI_Out(cli, " %s", "Healthy ");
+		else
+			VCLI_Out(cli, " %s", "Sick ");
+		VBP_Summary(cli, b->probe);
 	}
-	return found;
+
+	return (0);
 }
 
 static void
 cli_backend_list(struct cli *cli, const char * const *av, void *priv)
 {
-	struct backend *b;
-	const char *ah;
+	int hdr = 0;
 
 	(void)av;
 	(void)priv;
 	ASSERT_CLI();
-	VCLI_Out(cli, "%-30s %10s %15s %15s", "Backend name",
-		 "Conns", "Probed healthy", "Admin health");
-	VTAILQ_FOREACH(b, &backends, list) {
-		CHECK_OBJ_NOTNULL(b, BACKEND_MAGIC);
-		if (b->admin_health == from_probe) {
-			ah = "Auto";
-		} else if (b->admin_health == sick) {
-			ah = "Sick";
-		} else {
-			ah = "Healthy";
-		}
-		VCLI_Out(cli, "\n%-30s %10d %15s %15s",
-			 b->display_name,
-			 b->refcount,
-			 (b ->healthy ? "Yes" : "No"),
-			 ah);
-	}
+	(void)backend_find(cli, av[2], do_list, &hdr);
+}
+
+/*---------------------------------------------------------------------*/
+
+static int __match_proto__()
+do_set_health(struct cli *cli, struct backend *b, void *priv)
+{
+	enum admin_health state;
+
+	(void)cli;
+	state = *(enum admin_health*)priv;
+	CHECK_OBJ_NOTNULL(b, BACKEND_MAGIC);
+	b->admin_health = state;
+	return (0);
 }
 
 static void
 cli_backend_set_health(struct cli *cli, const char * const *av, void *priv)
 {
-	struct backend **b;
-	enum health_status state;
+	enum admin_health state;
 	int n;
-	const char *wstate;
 
 	(void)av;
 	(void)priv;
 	ASSERT_CLI();
-	wstate = av[3];
-	if (strcmp(wstate, "healthy") == 0) {
-		state = healthy;
-	} else if (strcmp(wstate, "sick") == 0) {
-		state = sick;
-	} else if (strcmp(wstate, "auto") == 0) {
-		state = from_probe;
-	} else {
-		VCLI_Out(cli, "Invalid state %s", wstate);
-		VCLI_SetResult(cli, CLIS_CANT);
+	AN(av[2]);
+	AN(av[3]);
+	state = vbe_str2adminhealth(av[3]);
+	if (state == ah_invalid) {
+		VCLI_Out(cli, "Invalid state %s", av[3]);
+		VCLI_SetResult(cli, CLIS_PARAM);
 		return;
 	}
-	n = backend_find(av[2], NULL, 0);
+	n = backend_find(cli, av[2], do_set_health, &state);
 	if (n == 0) {
-		VCLI_Out(cli, "No matching backends");
-		VCLI_SetResult(cli, CLIS_CANT);
-		return;
-	}
-
-	b = calloc(n, sizeof(struct backend *));
-	AN(b);
-	n = backend_find(av[2], b, n);
-
-	VCLI_Out(cli, "Set state to %s for the following backends:", wstate);
-	for (int i = 0; i < n; i++) {
-		b[i]->admin_health = state;
-		VCLI_Out(cli, "\n\t%s", b[i]->display_name);
+		VCLI_Out(cli, "No Backends matches");
+		VCLI_SetResult(cli, CLIS_PARAM);
 	}
 }
 
+/*---------------------------------------------------------------------*/
+
 static struct cli_proto backend_cmds[] = {
 	{ "backend.list", "backend.list",
-	    "\tList all backends\n", 0, 0, "d", cli_backend_list },
+	    "\tList all backends\n", 0, 1, "d", cli_backend_list },
 	{ "backend.set_health", "backend.set_health matcher state",
 	    "\tShow a backend\n", 2, 2, "d", cli_backend_set_health },
 	{ NULL }
 };
 
-/*--------------------------------------------------------------------*/
+/*---------------------------------------------------------------------*/
 
 void
 VBE_Init(void)
diff --git a/bin/varnishd/cache_backend_poll.c b/bin/varnishd/cache_backend_poll.c
index d79a806..eb6cc45 100644
--- a/bin/varnishd/cache_backend_poll.c
+++ b/bin/varnishd/cache_backend_poll.c
@@ -361,6 +361,14 @@ vbp_wrk_poll_backend(void *priv)
  * Cli functions
  */
 
+void
+VBP_Summary(struct cli *cli, const struct vbp_target *vt)
+{
+
+	CHECK_OBJ_NOTNULL(vt, VBP_TARGET_MAGIC);
+	VCLI_Out(cli, "%d/%d", vt->good, vt->probe.window);
+}
+
 static void
 vbp_bitmap(struct cli *cli, char c, uint64_t map, const char *lbl)
 {
diff --git a/bin/varnishtest/tests/c00048.vtc b/bin/varnishtest/tests/c00048.vtc
index ed8671f..a4d0847 100644
--- a/bin/varnishtest/tests/c00048.vtc
+++ b/bin/varnishtest/tests/c00048.vtc
@@ -49,7 +49,7 @@ client c1 {
 	expect resp.status == 200
 } -run
 
-varnish v1 -clierr 300 "backend.set_health s1 foo"
-varnish v1 -clierr 300 "backend.set_health s2 foo"
-varnish v1 -clierr 300 "backend.set_health s2 auto"
+varnish v1 -clierr 106 "backend.set_health s1 foo"
+varnish v1 -clierr 106 "backend.set_health s2 foo"
+varnish v1 -clierr 106 "backend.set_health s2 auto"
 



From geoff at varnish-cache.org  Mon Jan  9 20:52:29 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:29 +0100
Subject: [experimental-ims] ae14008 Simplify json callback
Message-ID: <E1RkMCj-00084I-QR@project.varnish-software.com>

commit ae14008200db8e4a5574af34a07d0ea517b12183
Author: Tollef Fog Heen <tfheen at varnish-software.com>
Date:   Mon Nov 7 13:26:51 2011 +0100

    Simplify json callback

diff --git a/bin/varnishstat/varnishstat.c b/bin/varnishstat/varnishstat.c
index c20b87f..b5d6df3 100644
--- a/bin/varnishstat/varnishstat.c
+++ b/bin/varnishstat/varnishstat.c
@@ -90,8 +90,6 @@ do_json_cb(void *priv, const struct VSC_point * const pt)
 {
 	uint64_t val;
 	int *jp;
-	char jsonkey[255];
-	char jsontmp[255];
 
 	jp = priv;
 
@@ -99,18 +97,14 @@ do_json_cb(void *priv, const struct VSC_point * const pt)
 	val = *(const volatile uint64_t*)pt->ptr;
 
 	if (*jp) *jp = 0; else printf(",\n");
-	jsonkey[0] = '\0';
 
+	printf("\t\"");
 	/* build the JSON key name.  */
-	if (strcmp(pt->ident, "")  && strcmp(pt->class, ""))  sprintf(jsonkey, "%s.%s", pt->class, pt->ident);
-	if (strcmp(pt->ident, "")  && !strcmp(pt->class, "")) sprintf(jsonkey, "%s", pt->ident);
-	if (!strcmp(pt->ident, "") && strcmp(pt->class, ""))  sprintf(jsonkey, "%s", pt->class);
-
-	strcpy(jsontmp, jsonkey);
-	if (strcmp(jsonkey, "")) sprintf(jsonkey, "%s.%s", jsontmp, pt->name);
-	else strcpy(jsonkey, pt->name);
-
-	printf("\t\"%s\": {", jsonkey);
+	if (pt->class[0])
+		printf("%s.", pt->class);
+	if (pt->ident[0])
+		printf("%s.", pt->ident);
+	printf("%s\": {", pt->name);
 
 	if (strcmp(pt->class, "")) printf("\"type\": \"%s\", ",  pt->class);
 	if (strcmp(pt->ident, "")) printf("\"ident\": \"%s\", ", pt->ident);



From geoff at varnish-cache.org  Mon Jan  9 20:52:30 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:30 +0100
Subject: [experimental-ims] 35f8ce5 Add proper handling of TCP timeouts on
	client side
Message-ID: <E1RkMCk-00084l-0O@project.varnish-software.com>

commit 35f8ce5db8b3cf1025369bf4dbf6cd71a7556748
Author: Tollef Fog Heen <tfheen at varnish-software.com>
Date:   Mon Nov 7 14:05:36 2011 +0100

    Add proper handling of TCP timeouts on client side
    
    Retry if we hit a TCP timeout if some data was sent and only error out
    when we get an error or no data was sent in the time period.

diff --git a/bin/varnishd/cache_wrw.c b/bin/varnishd/cache_wrw.c
index 0790ab2..5673b54 100644
--- a/bin/varnishd/cache_wrw.c
+++ b/bin/varnishd/cache_wrw.c
@@ -123,7 +123,37 @@ WRW_Flush(struct worker *w)
 			wrw->iov[wrw->ciov].iov_len = 0;
 		}
 		i = writev(*wrw->wfd, wrw->iov, wrw->niov);
-		if (i != wrw->liov) {
+		while (i != wrw->liov && i > 0) {
+			/* Remove sent data from start of I/O vector,
+			 * then retry; we hit a timeout, but some data
+			 * was sent.
+
+			 XXX: Add a "minimum sent data per timeout
+			 counter to prevent slowlaris attacks
+			*/
+			size_t used = 0;
+
+			WSL(w, SLT_Debug, *wrw->wfd,
+			    "Hit send timeout, wrote = %ld/%ld; retrying",
+			    i, wrw->liov);
+
+			for (int j = 0; j < wrw->niov; j++) {
+				if (used + wrw->iov[j].iov_len > i) {
+					/* Cutoff is in this iov */
+					int used_here = i - used;
+					wrw->iov[j].iov_len -= used_here;
+					wrw->iov[j].iov_base = (char*)wrw->iov[j].iov_base + used_here;
+					memmove(wrw->iov, &wrw->iov[j],
+						(wrw->niov - j) * sizeof(struct iovec));
+					wrw->niov -= j;
+					wrw->liov -= i;
+					break;
+				}
+				used += wrw->iov[j].iov_len;
+			}
+			i = writev(*wrw->wfd, wrw->iov, wrw->niov);
+		}
+		if (i <= 0) {
 			wrw->werr++;
 			WSL(w, SLT_Debug, *wrw->wfd,
 			    "Write error, retval = %d, len = %d, errno = %s",



From geoff at varnish-cache.org  Mon Jan  9 20:52:30 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:30 +0100
Subject: [experimental-ims] 0dead8f Rewrite the ban-lurker.
Message-ID: <E1RkMCk-00085E-82@project.varnish-software.com>

commit 0dead8f0bd7b69d6d1530f824406c31f9df7b3ee
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Nov 8 13:07:52 2011 +0000

    Rewrite the ban-lurker.
    
    Use a mark&mark strategy to test all obj.* bans even if there are
    req.* bans in the way.

diff --git a/bin/varnishd/cache_ban.c b/bin/varnishd/cache_ban.c
index e7478a0..8dc7cb4 100644
--- a/bin/varnishd/cache_ban.c
+++ b/bin/varnishd/cache_ban.c
@@ -81,12 +81,14 @@ struct ban {
 	unsigned		flags;
 #define BAN_F_GONE		(1 << 0)
 #define BAN_F_REQ		(1 << 2)
-#define BAN_F_LURK		(3 << 3)	/* ban-lurker-color */
+#define BAN_F_LURK		(3 << 6)	/* ban-lurker-color */
 	VTAILQ_HEAD(,objcore)	objcore;
 	struct vsb		*vsb;
 	uint8_t			*spec;
 };
 
+#define LURK_SHIFT 6
+
 struct ban_test {
 	uint8_t			arg1;
 	const char		*arg1_spec;
@@ -417,6 +419,7 @@ BAN_Insert(struct ban *b)
 	/* Hunt down duplicates, and mark them as gone */
 	bi = b;
 	pcount = 0;
+	Lck_Lock(&ban_mtx);
 	while(bi != be) {
 		bi = VTAILQ_NEXT(bi, list);
 		if (bi->flags & BAN_F_GONE)
@@ -425,9 +428,9 @@ BAN_Insert(struct ban *b)
 		if (memcmp(b->spec + 8, bi->spec + 8, ln - 8))
 			continue;
 		bi->flags |= BAN_F_GONE;
+		VSC_C_main->n_ban_gone++;
 		pcount++;
 	}
-	Lck_Lock(&ban_mtx);
 	be->refcount--;
 	VSC_C_main->n_ban_dups += pcount;
 	Lck_Unlock(&ban_mtx);
@@ -643,7 +646,13 @@ ban_evaluate(const uint8_t *bs, const struct http *objhttp,
 }
 
 /*--------------------------------------------------------------------
- * Check an object any fresh bans
+ * Check an object against all applicable bans
+ *
+ * Return:
+ *	-1 not all bans checked, but none of the checked matched
+ *		Only if !has_req
+ *	0 No bans matched, object moved to ban_start.
+ *	1 Ban matched, object removed from ban list.
  */
 
 static int
@@ -677,6 +686,12 @@ ban_check_object(struct object *o, const struct sess *sp, int has_req)
 		CHECK_OBJ_NOTNULL(b, BAN_MAGIC);
 		if (b->flags & BAN_F_GONE)
 			continue;
+		if ((b->flags & BAN_F_LURK) &&
+		    (b->flags & BAN_F_LURK) == (oc->flags & OC_F_LURK)) {
+			AZ(b->flags & BAN_F_REQ);
+			/* Lurker already tested this */
+			continue;
+		}
 		if (!has_req && (b->flags & BAN_F_REQ)) {
 			/*
 			 * We cannot test this one, but there might
@@ -687,23 +702,27 @@ ban_check_object(struct object *o, const struct sess *sp, int has_req)
 			break;
 	}
 
+	Lck_Lock(&ban_mtx);
+	VSC_C_main->n_ban_obj_test++;
+	VSC_C_main->n_ban_re_test += tests;
+
 	if (b == oc->ban && skipped > 0) {
+		AZ(has_req);
+		Lck_Unlock(&ban_mtx);
 		/*
 		 * Not banned, but some tests were skipped, so we cannot know
 		 * for certain that it cannot be, so we just have to give up.
 		 */
-		return (0);
+		return (-1);
 	}
 
-	Lck_Lock(&ban_mtx);
 	oc->ban->refcount--;
 	VTAILQ_REMOVE(&oc->ban->objcore, oc, ban_list);
 	if (b == oc->ban) {	/* not banned */
+		b->flags &= ~BAN_F_LURK;
 		VTAILQ_INSERT_TAIL(&b0->objcore, oc, ban_list);
 		b0->refcount++;
 	}
-	VSC_C_main->n_ban_obj_test++;
-	VSC_C_main->n_ban_re_test += tests;
 	Lck_Unlock(&ban_mtx);
 
 	if (b == oc->ban) {	/* not banned */
@@ -725,7 +744,7 @@ int
 BAN_CheckObject(struct object *o, const struct sess *sp)
 {
 
-	return (ban_check_object(o, sp, 1));
+	return (ban_check_object(o, sp, 1) > 0);
 }
 
 static struct ban *
@@ -736,6 +755,8 @@ ban_CheckLast(void)
 	Lck_AssertHeld(&ban_mtx);
 	b = VTAILQ_LAST(&ban_head, banhead_s);
 	if (b != VTAILQ_FIRST(&ban_head) && b->refcount == 0) {
+		if (b->flags & BAN_F_GONE)
+			VSC_C_main->n_ban_gone--;
 		VSC_C_main->n_ban--;
 		VSC_C_main->n_ban_retire++;
 		VTAILQ_REMOVE(&ban_head, b, list);
@@ -746,84 +767,144 @@ ban_CheckLast(void)
 }
 
 /*--------------------------------------------------------------------
- * Ban tail lurker thread
+ * Ban lurker thread
  */
 
 static int
-ban_lurker_work(const struct sess *sp)
+ban_lurker_work(const struct sess *sp, unsigned pass)
 {
-	struct ban *b, *bf;
+	struct ban *b, *b0, *b2;
 	struct objhead *oh;
 	struct objcore *oc, *oc2;
 	struct object *o;
 	int i;
 
-	Lck_Lock(&ban_mtx);
+	AN(pass & BAN_F_LURK);
+	AZ(pass & ~BAN_F_LURK);
 
-	/* First try to route the last ban */
-	bf = ban_CheckLast();
-	if (bf != NULL) {
+	/* First route the last ban(s) */
+	do {
+		Lck_Lock(&ban_mtx);
+		b2 = ban_CheckLast();
 		Lck_Unlock(&ban_mtx);
-		BAN_Free(bf);
-		return (0);
-	}
+		if (b2 != NULL)
+			BAN_Free(b2);
+	} while (b2 != NULL);
 
-	/* Find the last ban give up, if we have only one */
-	b = VTAILQ_LAST(&ban_head, banhead_s);
-	if (b == ban_start) {
-		Lck_Unlock(&ban_mtx);
-		return (0);
+	/*
+	 * Find out if we have any bans we can do something about
+	 * If we find any, tag them with our pass number.
+	 */
+	i = 0;
+	b0 = NULL;
+	VTAILQ_FOREACH(b, &ban_head, list) {
+		if (b->flags & BAN_F_GONE)
+			continue;
+		if (b->flags & BAN_F_REQ)
+			continue;
+		if (b == VTAILQ_LAST(&ban_head, banhead_s))
+			continue;
+		if (b0 == NULL)
+			b0 = b;
+		i++;
+		b->flags &= ~BAN_F_LURK;
+		b->flags |= pass;
 	}
-
-	/* Find the first object on it, if any */
-	oc = VTAILQ_FIRST(&b->objcore);
-	if (oc == NULL) {
-		Lck_Unlock(&ban_mtx);
+	if (params->diag_bitmap & 0x80000)
+		VSL(SLT_Debug, 0, "lurker: %d actionable bans", i);
+	if (i == 0)
 		return (0);
-	}
 
-	/* Try to lock the objhead */
-	oh = oc->objhead;
-	CHECK_OBJ_NOTNULL(oh, OBJHEAD_MAGIC);
-	if (Lck_Trylock(&oh->mtx)) {
+	VTAILQ_FOREACH_REVERSE(b, &ban_head, banhead_s, list) {
+		if (params->diag_bitmap & 0x80000)
+			VSL(SLT_Debug, 0, "lurker doing %f %d",
+			    ban_time(b->spec), b->refcount);
+		while (1) {
+			Lck_Lock(&ban_mtx);
+			oc = VTAILQ_FIRST(&b->objcore);
+			if (oc == NULL)
+				break;
+			CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
+			if (params->diag_bitmap & 0x80000)
+				VSL(SLT_Debug, 0, "test: %p %d %d",
+				    oc, oc->flags & OC_F_LURK, pass);
+			if ((oc->flags & OC_F_LURK) == pass)
+				break;
+			oh = oc->objhead;
+			CHECK_OBJ_NOTNULL(oh, OBJHEAD_MAGIC);
+			if (Lck_Trylock(&oh->mtx)) {
+				Lck_Unlock(&ban_mtx);
+				VTIM_sleep(params->ban_lurker_sleep);
+				continue;
+			}
+			/*
+			 * See if the objcore is still on the objhead since
+			 * we race against HSH_Deref() which comes in the
+			 * opposite locking order.
+			 */
+			VTAILQ_FOREACH(oc2, &oh->objcs, list)
+				if (oc == oc2)
+					break;
+			if (oc2 == NULL) {
+				Lck_Unlock(&oh->mtx);
+				Lck_Unlock(&ban_mtx);
+				VTIM_sleep(params->ban_lurker_sleep);
+				continue;
+			}
+			/*
+			 * Grab a reference to the OC and we can let go of
+			 * the BAN mutex
+			 */
+			AN(oc->refcnt);
+			oc->refcnt++;
+			oc->flags &= ~OC_F_LURK;
+			Lck_Unlock(&ban_mtx);
+			/*
+			 * Get the object and check it against all relevant bans
+			 */
+			o = oc_getobj(sp->wrk, oc);
+			i = ban_check_object(o, sp, 0);
+			if (params->diag_bitmap & 0x80000)
+				VSL(SLT_Debug, 0, "lurker got: %p %d",
+				    oc, i);
+			if (i == -1) {
+				/* Not banned, not moved */
+				oc->flags |= pass;
+				Lck_Lock(&ban_mtx);
+				VTAILQ_REMOVE(&b->objcore, oc, ban_list);
+				VTAILQ_INSERT_TAIL(&b->objcore, oc, ban_list);
+				Lck_Unlock(&ban_mtx);
+			}
+			Lck_Unlock(&oh->mtx);
+			if (params->diag_bitmap & 0x80000)
+				VSL(SLT_Debug, 0, "lurker done: %p %d %d",
+				    oc, oc->flags & OC_F_LURK, pass);
+			(void)HSH_Deref(sp->wrk, NULL, &o);
+			VTIM_sleep(params->ban_lurker_sleep);
+		}
+		Lck_AssertHeld(&ban_mtx);
+		if (!(b->flags & BAN_F_REQ)) {
+			if (!(b->flags & BAN_F_GONE)) {
+				b->flags |= BAN_F_GONE;
+				VSC_C_main->n_ban_gone++;
+			}
+			if (params->diag_bitmap & 0x80000)
+				VSL(SLT_Debug, 0, "lurker BAN %f now gone",
+				    ban_time(b->spec));
+		}
 		Lck_Unlock(&ban_mtx);
-		return (0);
-	}
-
-	/*
-	 * See if the objcore is still on the objhead since we race against
-	 * HSH_Deref() which comes in the opposite locking order.
-	 */
-	VTAILQ_FOREACH(oc2, &oh->objcs, list)
-		if (oc == oc2)
+		VTIM_sleep(params->ban_lurker_sleep);
+		if (b == b0)
 			break;
-	if (oc2 == NULL) {
-		Lck_Unlock(&oh->mtx);
-		Lck_Unlock(&ban_mtx);
-		return (0);
 	}
-	/*
-	 * Grab a reference to the OC and we can let go of the BAN mutex
-	 */
-	AN(oc->refcnt);
-	oc->refcnt++;
-	Lck_Unlock(&ban_mtx);
-
-	/*
-	 * Get the object and check it against all relevant bans
-	 */
-	o = oc_getobj(sp->wrk, oc);
-	i = ban_check_object(o, sp, 0);
-	Lck_Unlock(&oh->mtx);
-	WSP(sp, SLT_Debug, "lurker: %p %g %d", oc, o->exp.ttl, i);
-	(void)HSH_Deref(sp->wrk, NULL, &o);
-	return (i);
+	return (1);
 }
 
 static void * __match_proto__(bgthread_t)
 ban_lurker(struct sess *sp, void *priv)
 {
 	struct ban *bf;
+	unsigned pass = (1 << LURK_SHIFT);
 
 	int i = 0;
 	(void)priv;
@@ -843,14 +924,18 @@ ban_lurker(struct sess *sp, void *priv)
 				VTIM_sleep(1.0);
 		}
 
-		i = ban_lurker_work(sp);
+		i = ban_lurker_work(sp, pass);
 		WSL_Flush(sp->wrk, 0);
 		WRK_SumStat(sp->wrk);
-
-		if (i != 0)
+		if (i) {
+			pass += (1 << LURK_SHIFT);
+			pass &= BAN_F_LURK;
+			if (pass == 0)
+				pass += (1 << LURK_SHIFT);
 			VTIM_sleep(params->ban_lurker_sleep);
-		else
+		} else {
 			VTIM_sleep(1.0);
+		}
 	}
 	NEEDLESS_RETURN(NULL);
 }
@@ -964,13 +1049,20 @@ ccf_ban_list(struct cli *cli, const char * const *av, void *priv)
 
 	VCLI_Out(cli, "Present bans:\n");
 	VTAILQ_FOREACH(b, &ban_head, list) {
-		if (b == bl)
+		if (b == bl && !(params->diag_bitmap & 0x80000))
 			break;
 		VCLI_Out(cli, "%10.6f %5u%s\t", ban_time(b->spec),
 		    bl == b ? b->refcount - 1 : b->refcount,
 		    b->flags & BAN_F_GONE ? "G" : " ");
 		ban_render(cli, b->spec);
 		VCLI_Out(cli, "\n");
+		if (params->diag_bitmap & 0x80000) {
+			Lck_Lock(&ban_mtx);
+			struct objcore *oc;
+			VTAILQ_FOREACH(oc, &b->objcore, ban_list)
+				VCLI_Out(cli, "     %p\n", oc);
+			Lck_Unlock(&ban_mtx);
+		}
 	}
 
 	BAN_TailDeref(&bl);
@@ -989,10 +1081,14 @@ BAN_Init(void)
 
 	Lck_New(&ban_mtx, lck_ban);
 	CLI_AddFuncs(ban_cmds);
+	assert(BAN_F_LURK == OC_F_LURK);
+	AN((1 << LURK_SHIFT) & BAN_F_LURK);
+	AN((2 << LURK_SHIFT) & BAN_F_LURK);
 
 	ban_magic = BAN_New();
 	AN(ban_magic);
 	ban_magic->flags |= BAN_F_GONE;
+	VSC_C_main->n_ban_gone++;
 	BAN_Insert(ban_magic);
 	WRK_BgThread(&ban_thread, "ban-lurker", ban_lurker, NULL);
 }
diff --git a/bin/varnishd/mgt/mgt_param.c b/bin/varnishd/mgt/mgt_param.c
index 3169914..e2eb16b 100644
--- a/bin/varnishd/mgt/mgt_param.c
+++ b/bin/varnishd/mgt/mgt_param.c
@@ -841,6 +841,7 @@ static const struct parspec input_parspec[] = {
 		"  0x00010000 - synchronize shmlog.\n"
 		"  0x00020000 - synchronous start of persistence.\n"
 		"  0x00040000 - release VCL early.\n"
+		"  0x00080000 - ban-lurker debugging.\n"
 		"  0x80000000 - do edge-detection on digest.\n"
 		"Use 0x notation and do the bitor in your head :-)\n",
 		0,
diff --git a/bin/varnishtest/tests/c00049.vtc b/bin/varnishtest/tests/c00049.vtc
new file mode 100644
index 0000000..facd979
--- /dev/null
+++ b/bin/varnishtest/tests/c00049.vtc
@@ -0,0 +1,131 @@
+varnishtest "ban lurker test"
+
+
+server s1 {
+	rxreq
+	expect req.url == "/alpha"
+	txresp -hdr "Foo: /alpha"
+
+	rxreq
+	expect req.url == "/beta"
+	txresp -hdr "Foo: /beta"
+
+	rxreq
+	expect req.url == "/gamma"
+	txresp -hdr "Foo: /gamma"
+
+	rxreq
+	expect req.url == "/delta"
+	txresp -hdr "Foo: /delta"
+
+	rxreq
+	expect req.url == "/alpha"
+	txresp -hdr "Foo: /alpha2"
+
+	rxreq
+	expect req.url == "/beta"
+	txresp -hdr "Foo: /beta2"
+
+	rxreq
+	expect req.url == "/delta"
+	txresp -hdr "Foo: /delta2"
+
+} -start
+
+varnish v1 -vcl+backend {
+} -start
+
+varnish v1 -cliok "param.set ban_lurker_sleep 0"
+varnish v1 -cliok "param.set diag_bitmap 0x80000"
+
+varnish v1 -cliok "ban.list"
+
+client c1 {
+	txreq -url "/alpha"
+	rxresp
+	expect resp.http.foo == /alpha
+} -run
+
+delay 0.1
+varnish v1 -cliok "ban req.url == /alpha"
+varnish v1 -cliok "ban.list"
+varnish v1 -expect n_ban == 2
+varnish v1 -expect n_ban_gone == 1
+
+client c1 {
+	txreq -url "/beta"
+	rxresp
+	expect resp.http.foo == /beta
+} -run
+
+delay 0.1
+varnish v1 -cliok "ban obj.http.foo == /beta"
+varnish v1 -cliok "ban.list"
+varnish v1 -expect n_ban == 3
+
+client c1 {
+	txreq -url "/gamma"
+	rxresp
+	expect resp.http.foo == /gamma
+} -run
+
+delay 0.1
+varnish v1 -cliok "ban obj.http.foo == /gamma"
+varnish v1 -cliok "ban.list"
+varnish v1 -expect n_ban == 4
+
+client c1 {
+	txreq -url "/delta"
+	rxresp
+	expect resp.http.foo == /delta
+} -run
+
+delay 0.1
+varnish v1 -cliok "ban req.url == /delta"
+
+varnish v1 -expect n_ban_gone == 1
+varnish v1 -cliok "ban obj.http.foo == /gamma"
+# Dup-check should have added one
+varnish v1 -expect n_ban_gone == 2
+
+varnish v1 -cliok "ban req.url == /epsilon"
+varnish v1 -cliok "ban.list"
+varnish v1 -expect n_ban == 7
+varnish v1 -expect n_ban_gone == 2
+
+varnish v1 -cliok "param.set ban_lurker_sleep .01"
+delay 1
+varnish v1 -cliok "param.set ban_lurker_sleep .00"
+varnish v1 -cliok "ban.list"
+varnish v1 -expect n_ban == 7
+varnish v1 -expect n_ban_gone == 4
+
+client c1 {
+	txreq -url "/alpha"
+	rxresp
+	expect resp.http.foo == /alpha2
+} -run
+
+delay 1
+varnish v1 -cliok "ban.list"
+varnish v1 -expect n_ban == 4
+
+client c1 {
+	txreq -url "/beta"
+	rxresp
+	expect resp.http.foo == /beta2
+} -run
+
+varnish v1 -cliok "ban.list"
+
+
+client c1 {
+	txreq -url "/delta"
+	rxresp
+	expect resp.http.foo == /delta2
+} -run
+
+delay 1
+varnish v1 -cliok "ban.list"
+varnish v1 -expect n_ban == 1
+varnish v1 -expect n_ban_gone == 0



From geoff at varnish-cache.org  Mon Jan  9 20:52:30 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:30 +0100
Subject: [experimental-ims] 94f4464 Don't start the ban-lurker until
	-spersistent is done loading.
Message-ID: <E1RkMCk-00085f-E6@project.varnish-software.com>

commit 94f4464792d5b57041bd8e9b8d089f2bd8bd9bb7
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Nov 8 13:44:29 2011 +0000

    Don't start the ban-lurker until -spersistent is done loading.

diff --git a/bin/varnishd/cache_ban.c b/bin/varnishd/cache_ban.c
index 8dc7cb4..c01a3f6 100644
--- a/bin/varnishd/cache_ban.c
+++ b/bin/varnishd/cache_ban.c
@@ -102,6 +102,7 @@ static struct lock ban_mtx;
 static struct ban *ban_magic;
 static pthread_t ban_thread;
 static struct ban * volatile ban_start;
+static bgthread_t ban_lurker;
 
 /*--------------------------------------------------------------------
  * BAN string magic markers
@@ -583,6 +584,7 @@ BAN_Compile(void)
 
 	SMP_NewBan(ban_magic->spec, ban_len(ban_magic->spec));
 	ban_start = VTAILQ_FIRST(&ban_head);
+	WRK_BgThread(&ban_thread, "ban-lurker", ban_lurker, NULL);
 }
 
 /*--------------------------------------------------------------------
@@ -1090,5 +1092,4 @@ BAN_Init(void)
 	ban_magic->flags |= BAN_F_GONE;
 	VSC_C_main->n_ban_gone++;
 	BAN_Insert(ban_magic);
-	WRK_BgThread(&ban_thread, "ban-lurker", ban_lurker, NULL);
 }



From geoff at varnish-cache.org  Mon Jan  9 20:52:30 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:30 +0100
Subject: [experimental-ims] 992ebc9 Implement VRE options with hard linkage to
	PCRE options instead of maintaining magic hex-bit values that
	must match.
Message-ID: <E1RkMCk-000861-Jr@project.varnish-software.com>

commit 992ebc91708a50ebf383f8ec58ac7b402c092617
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Nov 8 15:04:04 2011 +0000

    Implement VRE options with hard linkage to PCRE options instead
    of maintaining magic hex-bit values that must match.

diff --git a/include/vre.h b/include/vre.h
index a1206e5..59ffeb0 100644
--- a/include/vre.h
+++ b/include/vre.h
@@ -48,8 +48,8 @@ typedef struct vre vre_t;
 #define VRE_ERROR_NOMATCH         (-1)
 
 /* And those to PCRE options */
-#define VRE_CASELESS           0x00000001
-#define VRE_NOTEMPTY_ATSTART   0x10000000
+extern const unsigned VRE_CASELESS;
+extern const unsigned VRE_NOTEMPTY_ATSTART;
 
 vre_t *VRE_compile(const char *, int, const char **, int *);
 int VRE_exec(const vre_t *code, const char *subject, int length,
diff --git a/lib/libvarnish/vre.c b/lib/libvarnish/vre.c
index 9b1d911..cffe396 100644
--- a/lib/libvarnish/vre.c
+++ b/lib/libvarnish/vre.c
@@ -43,6 +43,19 @@ struct vre {
 	pcre *re;
 };
 
+/*
+ * We don't want to spread or even expose the majority of PCRE options
+ * so we establish our own options and implement hard linkage to PCRE
+ * here.
+ */
+const unsigned VRE_CASELESS = PCRE_CASELESS;
+const unsigned VRE_NOTEMPTY_ATSTART =
+#ifdef PCRE_NOTEMPTY_ATSTART
+	PCRE_NOTEMPTY_ATSTART;
+#else
+	0;
+#endif
+
 vre_t *
 VRE_compile(const char *pattern, int options,
 		    const char **errptr, int *erroffset)



From geoff at varnish-cache.org  Mon Jan  9 20:52:30 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:30 +0100
Subject: [experimental-ims] 8241149 Draw the line between the managers and the
	childs copy of the struct params much clearer. Manager should
	never read the childs copy.
Message-ID: <E1RkMCk-00086i-VY@project.varnish-software.com>

commit 8241149492e5649c8a3126ee363f089f33491655
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu Nov 10 08:57:41 2011 +0000

    Draw the line between the managers and the childs copy of the struct
    params much clearer.  Manager should never read the childs copy.

diff --git a/bin/varnishd/cache_main.c b/bin/varnishd/cache_main.c
index 8293d38..c610bc3 100644
--- a/bin/varnishd/cache_main.c
+++ b/bin/varnishd/cache_main.c
@@ -37,6 +37,8 @@
 #include "waiter/cache_waiter.h"
 #include "hash/hash_slinger.h"
 
+volatile struct params	*params;
+
 /*--------------------------------------------------------------------
  * Per thread storage for the session currently being processed by
  * the thread.  This is used for panic messages.
diff --git a/bin/varnishd/mgt/mgt.h b/bin/varnishd/mgt/mgt.h
index 1656af8..61493b3 100644
--- a/bin/varnishd/mgt/mgt.h
+++ b/bin/varnishd/mgt/mgt.h
@@ -58,10 +58,11 @@ void mgt_cli_secret(const char *S_arg);
 void mgt_cli_close_all(void);
 
 /* mgt_param.c */
-void MCF_ParamSync(void);
 void MCF_ParamInit(struct cli *);
 void MCF_ParamSet(struct cli *, const char *param, const char *val);
 void MCF_DumpRst(void);
+extern struct params mgt_param;
+
 
 /* mgt_sandbox.c */
 void mgt_sandbox(void);
diff --git a/bin/varnishd/mgt/mgt_child.c b/bin/varnishd/mgt/mgt_child.c
index ff8a776..9c6d37d 100644
--- a/bin/varnishd/mgt/mgt_child.c
+++ b/bin/varnishd/mgt/mgt_child.c
@@ -214,7 +214,7 @@ MGT_Child_Cli_Fail(void)
 		return;
 	REPORT(LOG_ERR, "Child (%jd) not responding to CLI, killing it.",
 	    (intmax_t)child_pid);
-	if (params->diag_bitmap & 0x1000)
+	if (mgt_param.diag_bitmap & 0x1000)
 		(void)kill(child_pid, SIGKILL);
 	else
 		(void)kill(child_pid, SIGQUIT);
@@ -315,7 +315,7 @@ start_child(struct cli *cli)
 	heritage.std_fd = cp[1];
 	child_output = cp[0];
 
-	MCF_ParamSync();
+	AN(params);
 	if ((pid = fork()) < 0) {
 		perror("Could not fork child");
 		exit(1);
@@ -374,10 +374,10 @@ start_child(struct cli *cli)
 	AZ(vev_add(mgt_evb, e));
 	ev_listen = e;
 	AZ(ev_poker);
-	if (params->ping_interval > 0) {
+	if (mgt_param.ping_interval > 0) {
 		e = vev_new();
 		XXXAN(e);
-		e->timeout = params->ping_interval;
+		e->timeout = mgt_param.ping_interval;
 		e->callback = child_poker;
 		e->name = "child poker";
 		AZ(vev_add(mgt_evb, e));
@@ -522,7 +522,7 @@ mgt_sigchld(const struct vev *e, int what)
 
 	REPORT0(LOG_DEBUG, "Child cleanup complete");
 
-	if (child_state == CH_DIED && params->auto_restart)
+	if (child_state == CH_DIED && mgt_param.auto_restart)
 		start_child(NULL);
 	else if (child_state == CH_DIED) {
 		child_state = CH_STOPPED;
diff --git a/bin/varnishd/mgt/mgt_cli.c b/bin/varnishd/mgt/mgt_cli.c
index c8f22c3..ec8fd4d 100644
--- a/bin/varnishd/mgt/mgt_cli.c
+++ b/bin/varnishd/mgt/mgt_cli.c
@@ -170,7 +170,7 @@ mcf_askchild(struct cli *cli, const char * const *av, void *priv)
 		return;
 	}
 	VSB_delete(vsb);
-	(void)VCLI_ReadResult(cli_i, &u, &q, params->cli_timeout);
+	(void)VCLI_ReadResult(cli_i, &u, &q, mgt_param.cli_timeout);
 	VCLI_SetResult(cli, u);
 	VCLI_Out(cli, "%s", q);
 	free(q);
@@ -192,7 +192,7 @@ mgt_cli_askchild(unsigned *status, char **resp, const char *fmt, ...) {
 	int i, j;
 	va_list ap;
 	unsigned u;
-	char buf[params->cli_buffer], *p;
+	char buf[mgt_param.cli_buffer], *p;
 
 	if (resp != NULL)
 		*resp = NULL;
@@ -219,7 +219,7 @@ mgt_cli_askchild(unsigned *status, char **resp, const char *fmt, ...) {
 		return (CLIS_COMMS);
 	}
 
-	(void)VCLI_ReadResult(cli_i, &u, resp, params->cli_timeout);
+	(void)VCLI_ReadResult(cli_i, &u, resp, mgt_param.cli_timeout);
 	if (status != NULL)
 		*status = u;
 	if (u == CLIS_COMMS)
@@ -316,7 +316,7 @@ static void
 mgt_cli_cb_before(const struct cli *cli)
 {
 
-	if (params->syslog_cli_traffic)
+	if (mgt_param.syslog_cli_traffic)
 		syslog(LOG_NOTICE, "CLI %s Rd %s", cli->ident, cli->cmd);
 }
 
@@ -324,7 +324,7 @@ static void
 mgt_cli_cb_after(const struct cli *cli)
 {
 
-	if (params->syslog_cli_traffic)
+	if (mgt_param.syslog_cli_traffic)
 		syslog(LOG_NOTICE, "CLI %s Wr %03u %s",
 		    cli->ident, cli->result, VSB_data(cli->sb));
 }
@@ -335,7 +335,7 @@ static void
 mgt_cli_init_cls(void)
 {
 
-	cls = VCLS_New(mgt_cli_cb_before, mgt_cli_cb_after, params->cli_buffer);
+	cls = VCLS_New(mgt_cli_cb_before, mgt_cli_cb_after, mgt_param.cli_buffer);
 	AN(cls);
 	AZ(VCLS_AddFunc(cls, MCF_NOAUTH, cli_auth));
 	AZ(VCLS_AddFunc(cls, MCF_AUTH, cli_proto));
diff --git a/bin/varnishd/mgt/mgt_param.c b/bin/varnishd/mgt/mgt_param.c
index e2eb16b..20456d7 100644
--- a/bin/varnishd/mgt/mgt_param.c
+++ b/bin/varnishd/mgt/mgt_param.c
@@ -51,7 +51,7 @@
 #include "mgt_cli.h"
 
 #define MAGIC_INIT_STRING	"\001"
-struct params master;
+struct params mgt_param;
 static int nparspec;
 static struct parspec const ** parspec;
 static int margin;
@@ -262,7 +262,7 @@ tweak_user(struct cli *cli, const struct parspec *par, const char *arg)
 		if (!strcmp(arg, MAGIC_INIT_STRING)) {
 			pw = getpwnam("nobody");
 			if (pw == NULL) {
-				master.uid = getuid();
+				mgt_param.uid = getuid();
 				return;
 			}
 		} else
@@ -272,19 +272,19 @@ tweak_user(struct cli *cli, const struct parspec *par, const char *arg)
 			VCLI_SetResult(cli, CLIS_PARAM);
 			return;
 		}
-		REPLACE(master.user, pw->pw_name);
-		master.uid = pw->pw_uid;
-		master.gid = pw->pw_gid;
+		REPLACE(mgt_param.user, pw->pw_name);
+		mgt_param.uid = pw->pw_uid;
+		mgt_param.gid = pw->pw_gid;
 
 		/* set group to user's primary group */
 		if ((gr = getgrgid(pw->pw_gid)) != NULL &&
 		    (gr = getgrnam(gr->gr_name)) != NULL &&
 		    gr->gr_gid == pw->pw_gid)
-			REPLACE(master.group, gr->gr_name);
-	} else if (master.user) {
-		VCLI_Out(cli, "%s (%d)", master.user, (int)master.uid);
+			REPLACE(mgt_param.group, gr->gr_name);
+	} else if (mgt_param.user) {
+		VCLI_Out(cli, "%s (%d)", mgt_param.user, (int)mgt_param.uid);
 	} else {
-		VCLI_Out(cli, "%d", (int)master.uid);
+		VCLI_Out(cli, "%d", (int)mgt_param.uid);
 	}
 }
 
@@ -303,8 +303,8 @@ tweak_group(struct cli *cli, const struct parspec *par, const char *arg)
 			gr = getgrnam("nogroup");
 			if (gr == NULL) {
 				/* Only replace if tweak_user didn't */
-				if (master.gid == 0)
-					master.gid = getgid();
+				if (mgt_param.gid == 0)
+					mgt_param.gid = getgid();
 				return;
 			}
 		} else
@@ -314,12 +314,12 @@ tweak_group(struct cli *cli, const struct parspec *par, const char *arg)
 			VCLI_SetResult(cli, CLIS_PARAM);
 			return;
 		}
-		REPLACE(master.group, gr->gr_name);
-		master.gid = gr->gr_gid;
-	} else if (master.group) {
-		VCLI_Out(cli, "%s (%d)", master.group, (int)master.gid);
+		REPLACE(mgt_param.group, gr->gr_name);
+		mgt_param.gid = gr->gr_gid;
+	} else if (mgt_param.group) {
+		VCLI_Out(cli, "%s (%d)", mgt_param.group, (int)mgt_param.gid);
 	} else {
-		VCLI_Out(cli, "%d", (int)master.gid);
+		VCLI_Out(cli, "%d", (int)mgt_param.gid);
 	}
 }
 
@@ -350,7 +350,7 @@ tweak_listen_address(struct cli *cli, const struct parspec *par,
 
 	(void)par;
 	if (arg == NULL) {
-		VCLI_Quote(cli, master.listen_address);
+		VCLI_Quote(cli, mgt_param.listen_address);
 		return;
 	}
 
@@ -401,7 +401,7 @@ tweak_listen_address(struct cli *cli, const struct parspec *par,
 		return;
 	}
 
-	REPLACE(master.listen_address, arg);
+	REPLACE(mgt_param.listen_address, arg);
 
 	clean_listen_sock_head(&heritage.socks);
 	heritage.nsocks = 0;
@@ -452,9 +452,9 @@ tweak_diag_bitmap(struct cli *cli, const struct parspec *par, const char *arg)
 	(void)par;
 	if (arg != NULL) {
 		u = strtoul(arg, NULL, 0);
-		master.diag_bitmap = u;
+		mgt_param.diag_bitmap = u;
 	} else {
-		VCLI_Out(cli, "0x%x", master.diag_bitmap);
+		VCLI_Out(cli, "0x%x", mgt_param.diag_bitmap);
 	}
 }
 
@@ -500,7 +500,7 @@ static const struct parspec input_parspec[] = {
 		"The unprivileged group to run as.",
 		MUST_RESTART,
 		MAGIC_INIT_STRING },
-	{ "default_ttl", tweak_timeout_double, &master.default_ttl,
+	{ "default_ttl", tweak_timeout_double, &mgt_param.default_ttl,
 		0, UINT_MAX,
 		"The TTL assigned to objects if neither the backend nor "
 		"the VCL code assigns one.\n"
@@ -510,7 +510,7 @@ static const struct parspec input_parspec[] = {
 		"flush of the cache use \"ban.url .\"",
 		0,
 		"120", "seconds" },
-	{ "sess_workspace", tweak_uint, &master.sess_workspace, 1024, UINT_MAX,
+	{ "sess_workspace", tweak_uint, &mgt_param.sess_workspace, 1024, UINT_MAX,
 		"Bytes of HTTP protocol workspace allocated for sessions. "
 		"This space must be big enough for the entire HTTP protocol "
 		"header and any edits done to it in the VCL code.\n"
@@ -518,13 +518,13 @@ static const struct parspec input_parspec[] = {
 		DELAYED_EFFECT,
 		"65536",
 		"bytes" },
-	{ "http_req_hdr_len", tweak_uint, &master.http_req_hdr_len,
+	{ "http_req_hdr_len", tweak_uint, &mgt_param.http_req_hdr_len,
 		40, UINT_MAX,
 		"Maximum length of any HTTP client request header we will "
 		"allow.  The limit is inclusive its continuation lines.\n",
 		0,
 		"8192", "bytes" },
-	{ "http_req_size", tweak_uint, &master.http_req_size,
+	{ "http_req_size", tweak_uint, &mgt_param.http_req_size,
 		256, UINT_MAX,
 		"Maximum number of bytes of HTTP client request we will deal "
 		"with.  This is a limit on all bytes up to the double blank "
@@ -534,13 +534,13 @@ static const struct parspec input_parspec[] = {
 		"how much of that the request is allowed to take up.",
 		0,
 		"32768", "bytes" },
-	{ "http_resp_hdr_len", tweak_uint, &master.http_resp_hdr_len,
+	{ "http_resp_hdr_len", tweak_uint, &mgt_param.http_resp_hdr_len,
 		40, UINT_MAX,
 		"Maximum length of any HTTP backend response header we will "
 		"allow.  The limit is inclusive its continuation lines.\n",
 		0,
 		"8192", "bytes" },
-	{ "http_resp_size", tweak_uint, &master.http_resp_size,
+	{ "http_resp_size", tweak_uint, &mgt_param.http_resp_size,
 		256, UINT_MAX,
 		"Maximum number of bytes of HTTP backend resonse we will deal "
 		"with.  This is a limit on all bytes up to the double blank "
@@ -550,7 +550,7 @@ static const struct parspec input_parspec[] = {
 		"how much of that the request is allowed to take up.",
 		0,
 		"32768", "bytes" },
-	{ "http_max_hdr", tweak_uint, &master.http_max_hdr, 32, 65535,
+	{ "http_max_hdr", tweak_uint, &mgt_param.http_max_hdr, 32, 65535,
 		"Maximum number of HTTP headers we will deal with in "
 		"client request or backend reponses.  "
 		"Note that the first line occupies five header fields.\n"
@@ -558,7 +558,7 @@ static const struct parspec input_parspec[] = {
 		"objects allocate exact space for the headers they store.\n",
 		0,
 		"64", "header lines" },
-	{ "shm_workspace", tweak_uint, &master.shm_workspace, 4096, UINT_MAX,
+	{ "shm_workspace", tweak_uint, &mgt_param.shm_workspace, 4096, UINT_MAX,
 		"Bytes of shmlog workspace allocated for worker threads. "
 		"If too big, it wastes some ram, if too small it causes "
 		"needless flushes of the SHM workspace.\n"
@@ -567,12 +567,12 @@ static const struct parspec input_parspec[] = {
 		"Minimum is 4096 bytes.",
 		DELAYED_EFFECT,
 		"8192", "bytes" },
-	{ "shm_reclen", tweak_uint, &master.shm_reclen, 16, 65535,
+	{ "shm_reclen", tweak_uint, &mgt_param.shm_reclen, 16, 65535,
 		"Maximum number of bytes in SHM log record.\n"
 		"Maximum is 65535 bytes.",
 		0,
 		"255", "bytes" },
-	{ "default_grace", tweak_timeout_double, &master.default_grace,
+	{ "default_grace", tweak_timeout_double, &mgt_param.default_grace,
 		0, UINT_MAX,
 		"Default grace period.  We will deliver an object "
 		"this long after it has expired, provided another thread "
@@ -581,7 +581,7 @@ static const struct parspec input_parspec[] = {
 		"made until they are fetched from the backend again.\n",
 		DELAYED_EFFECT,
 		"10", "seconds" },
-	{ "default_keep", tweak_timeout_double, &master.default_keep,
+	{ "default_keep", tweak_timeout_double, &mgt_param.default_keep,
 		0, UINT_MAX,
 		"Default keep period.  We will keep a useless object "
 		"around this long, making it available for conditional "
@@ -590,49 +590,49 @@ static const struct parspec input_parspec[] = {
 		"cache at the end of ttl+grace+keep.",
 		DELAYED_EFFECT,
 		"0", "seconds" },
-	{ "sess_timeout", tweak_timeout, &master.sess_timeout, 0, 0,
+	{ "sess_timeout", tweak_timeout, &mgt_param.sess_timeout, 0, 0,
 		"Idle timeout for persistent sessions. "
 		"If a HTTP request has not been received in this many "
 		"seconds, the session is closed.",
 		0,
 		"5", "seconds" },
-	{ "expiry_sleep", tweak_timeout_double, &master.expiry_sleep, 0, 60,
+	{ "expiry_sleep", tweak_timeout_double, &mgt_param.expiry_sleep, 0, 60,
 		"How long the expiry thread sleeps when there is nothing "
 		"for it to do.\n",
 		0,
 		"1", "seconds" },
-	{ "pipe_timeout", tweak_timeout, &master.pipe_timeout, 0, 0,
+	{ "pipe_timeout", tweak_timeout, &mgt_param.pipe_timeout, 0, 0,
 		"Idle timeout for PIPE sessions. "
 		"If nothing have been received in either direction for "
 		"this many seconds, the session is closed.\n",
 		0,
 		"60", "seconds" },
-	{ "send_timeout", tweak_timeout, &master.send_timeout, 0, 0,
+	{ "send_timeout", tweak_timeout, &mgt_param.send_timeout, 0, 0,
 		"Send timeout for client connections. "
 		"If the HTTP response hasn't been transmitted in this many\n"
                 "seconds the session is closed. \n"
 		"See setsockopt(2) under SO_SNDTIMEO for more information.",
 		DELAYED_EFFECT,
 		"600", "seconds" },
-	{ "idle_send_timeout", tweak_timeout, &master.idle_send_timeout, 0, 0,
+	{ "idle_send_timeout", tweak_timeout, &mgt_param.idle_send_timeout, 0, 0,
 		"Time to wait with no data sent. "
 		"If no data has been transmitted in this many\n"
                 "seconds the session is closed. \n"
 		"See setsockopt(2) under SO_SNDTIMEO for more information.",
 		DELAYED_EFFECT,
 		"60", "seconds" },
-	{ "auto_restart", tweak_bool, &master.auto_restart, 0, 0,
+	{ "auto_restart", tweak_bool, &mgt_param.auto_restart, 0, 0,
 		"Restart child process automatically if it dies.\n",
 		0,
 		"on", "bool" },
 	{ "nuke_limit",
-		tweak_uint, &master.nuke_limit, 0, UINT_MAX,
+		tweak_uint, &mgt_param.nuke_limit, 0, UINT_MAX,
 		"Maximum number of objects we attempt to nuke in order"
 		"to make space for a object body.",
 		EXPERIMENTAL,
 		"50", "allocations" },
 	{ "fetch_chunksize",
-		tweak_uint, &master.fetch_chunksize, 4, UINT_MAX / 1024.,
+		tweak_uint, &mgt_param.fetch_chunksize, 4, UINT_MAX / 1024.,
 		"The default chunksize used by fetcher. "
 		"This should be bigger than the majority of objects with "
 		"short TTLs.\n"
@@ -641,7 +641,7 @@ static const struct parspec input_parspec[] = {
 		EXPERIMENTAL,
 		"128", "kilobytes" },
 	{ "fetch_maxchunksize",
-		tweak_uint, &master.fetch_maxchunksize, 64, UINT_MAX / 1024.,
+		tweak_uint, &mgt_param.fetch_maxchunksize, 64, UINT_MAX / 1024.,
 		"The maximum chunksize we attempt to allocate from storage. "
 		"Making this too large may cause delays and storage "
 		"fragmentation.\n",
@@ -649,12 +649,12 @@ static const struct parspec input_parspec[] = {
 		"262144", "kilobytes" },
 #ifdef SENDFILE_WORKS
 	{ "sendfile_threshold",
-		tweak_uint, &master.sendfile_threshold, 0, UINT_MAX,
+		tweak_uint, &mgt_param.sendfile_threshold, 0, UINT_MAX,
 		"The minimum size of objects transmitted with sendfile.",
 		EXPERIMENTAL,
 		"-1", "bytes" },
 #endif /* SENDFILE_WORKS */
-	{ "vcl_trace", tweak_bool,  &master.vcl_trace, 0, 0,
+	{ "vcl_trace", tweak_bool,  &mgt_param.vcl_trace, 0, 0,
 		"Trace VCL execution in the shmlog.\n"
 		"Enabling this will allow you to see the path each "
 		"request has taken through the VCL program.\n"
@@ -668,22 +668,22 @@ static const struct parspec input_parspec[] = {
 		"Possible formats: host, host:port, :port",
 		MUST_RESTART,
 		":80" },
-	{ "listen_depth", tweak_uint, &master.listen_depth, 0, UINT_MAX,
+	{ "listen_depth", tweak_uint, &mgt_param.listen_depth, 0, UINT_MAX,
 		"Listen queue depth.",
 		MUST_RESTART,
 		"1024", "connections" },
-	{ "cli_timeout", tweak_timeout, &master.cli_timeout, 0, 0,
+	{ "cli_timeout", tweak_timeout, &mgt_param.cli_timeout, 0, 0,
 		"Timeout for the childs replies to CLI requests from "
-		"the master.",
+		"the mgt_param.",
 		0,
 		"10", "seconds" },
-	{ "ping_interval", tweak_uint, &master.ping_interval, 0, UINT_MAX,
+	{ "ping_interval", tweak_uint, &mgt_param.ping_interval, 0, UINT_MAX,
 		"Interval between pings from parent to child.\n"
 		"Zero will disable pinging entirely, which makes "
 		"it possible to attach a debugger to the child.",
 		MUST_RESTART,
 		"3", "seconds" },
-	{ "lru_interval", tweak_timeout, &master.lru_timeout, 0, 0,
+	{ "lru_interval", tweak_timeout, &mgt_param.lru_timeout, 0, 0,
 		"Grace period before object moves on LRU list.\n"
 		"Objects are only moved to the front of the LRU "
 		"list if they have not been moved there already inside "
@@ -698,14 +698,14 @@ static const struct parspec input_parspec[] = {
 		"and %o will be replaced with the output file name.",
 		MUST_RELOAD,
 		VCC_CC , NULL },
-	{ "max_restarts", tweak_uint, &master.max_restarts, 0, UINT_MAX,
+	{ "max_restarts", tweak_uint, &mgt_param.max_restarts, 0, UINT_MAX,
 		"Upper limit on how many times a request can restart."
 		"\nBe aware that restarts are likely to cause a hit against "
 		"the backend, so don't increase thoughtlessly.\n",
 		0,
 		"4", "restarts" },
 	{ "esi_syntax",
-		tweak_uint, &master.esi_syntax, 0, UINT_MAX,
+		tweak_uint, &mgt_param.esi_syntax, 0, UINT_MAX,
 		"Bitmap controlling ESI parsing code:\n"
 		"  0x00000001 - Don't check if it looks like XML\n"
 		"  0x00000002 - Ignore non-esi elements\n"
@@ -715,12 +715,12 @@ static const struct parspec input_parspec[] = {
 		0,
 		"0", "bitmap" },
 	{ "max_esi_depth",
-		tweak_uint, &master.max_esi_depth, 0, UINT_MAX,
+		tweak_uint, &mgt_param.max_esi_depth, 0, UINT_MAX,
 		"Maximum depth of esi:include processing.\n",
 		0,
 		"5", "levels" },
 	{ "connect_timeout", tweak_timeout_double,
-		&master.connect_timeout,0, UINT_MAX,
+		&mgt_param.connect_timeout,0, UINT_MAX,
 		"Default connection timeout for backend connections. "
 		"We only try to connect to the backend for this many "
 		"seconds before giving up. "
@@ -729,7 +729,7 @@ static const struct parspec input_parspec[] = {
 		0,
 		"0.7", "s" },
 	{ "first_byte_timeout", tweak_timeout_double,
-		&master.first_byte_timeout,0, UINT_MAX,
+		&mgt_param.first_byte_timeout,0, UINT_MAX,
 		"Default timeout for receiving first byte from backend. "
 		"We only wait for this many seconds for the first "
 		"byte before giving up. A value of 0 means it will never time "
@@ -739,7 +739,7 @@ static const struct parspec input_parspec[] = {
 		0,
 		"60", "s" },
 	{ "between_bytes_timeout", tweak_timeout_double,
-		&master.between_bytes_timeout,0, UINT_MAX,
+		&mgt_param.between_bytes_timeout,0, UINT_MAX,
 		"Default timeout between bytes when receiving data from "
 		"backend. "
 		"We only wait for this many seconds between bytes "
@@ -749,7 +749,7 @@ static const struct parspec input_parspec[] = {
 		0,
 		"60", "s" },
 	{ "acceptor_sleep_max", tweak_timeout_double,
-		&master.acceptor_sleep_max, 0,  10,
+		&mgt_param.acceptor_sleep_max, 0,  10,
 		"If we run out of resources, such as file descriptors or "
 		"worker threads, the acceptor will sleep between accepts.\n"
 		"This parameter limits how long it can sleep between "
@@ -757,7 +757,7 @@ static const struct parspec input_parspec[] = {
 		EXPERIMENTAL,
 		"0.050", "s" },
 	{ "acceptor_sleep_incr", tweak_timeout_double,
-		&master.acceptor_sleep_incr, 0,  1,
+		&mgt_param.acceptor_sleep_incr, 0,  1,
 		"If we run out of resources, such as file descriptors or "
 		"worker threads, the acceptor will sleep between accepts.\n"
 		"This parameter control how much longer we sleep, each time "
@@ -765,25 +765,25 @@ static const struct parspec input_parspec[] = {
 		EXPERIMENTAL,
 		"0.001", "s" },
 	{ "acceptor_sleep_decay", tweak_generic_double,
-		&master.acceptor_sleep_decay, 0,  1,
+		&mgt_param.acceptor_sleep_decay, 0,  1,
 		"If we run out of resources, such as file descriptors or "
 		"worker threads, the acceptor will sleep between accepts.\n"
 		"This parameter (multiplicatively) reduce the sleep duration "
 		"for each succesfull accept. (ie: 0.9 = reduce by 10%)",
 		EXPERIMENTAL,
 		"0.900", "" },
-	{ "clock_skew", tweak_uint, &master.clock_skew, 0, UINT_MAX,
+	{ "clock_skew", tweak_uint, &mgt_param.clock_skew, 0, UINT_MAX,
 		"How much clockskew we are willing to accept between the "
 		"backend and our own clock.",
 		0,
 		"10", "s" },
-	{ "prefer_ipv6", tweak_bool, &master.prefer_ipv6, 0, 0,
+	{ "prefer_ipv6", tweak_bool, &mgt_param.prefer_ipv6, 0, 0,
 		"Prefer IPv6 address when connecting to backends which "
 		"have both IPv4 and IPv6 addresses.",
 		0,
 		"off", "bool" },
 	{ "session_max", tweak_uint,
-		&master.max_sess, 1000, UINT_MAX,
+		&mgt_param.max_sess, 1000, UINT_MAX,
 		"Maximum number of sessions we will allocate from one pool "
 		"before just dropping connections.\n"
 		"This is mostly an anti-DoS measure, and setting it plenty "
@@ -792,7 +792,7 @@ static const struct parspec input_parspec[] = {
 		0,
 		"100000", "sessions" },
 	{ "session_linger", tweak_uint,
-		&master.session_linger,0, UINT_MAX,
+		&mgt_param.session_linger,0, UINT_MAX,
 		"How long time the workerthread lingers on the session "
 		"to see if a new request appears right away.\n"
 		"If sessions are reused, as much as half of all reuses "
@@ -803,18 +803,18 @@ static const struct parspec input_parspec[] = {
 		"more sessions take a detour around the waiter.",
 		EXPERIMENTAL,
 		"50", "ms" },
-	{ "cli_buffer", tweak_uint, &master.cli_buffer, 4096, UINT_MAX,
+	{ "cli_buffer", tweak_uint, &mgt_param.cli_buffer, 4096, UINT_MAX,
 		"Size of buffer for CLI input."
 		"\nYou may need to increase this if you have big VCL files "
 		"and use the vcl.inline CLI command.\n"
 		"NB: Must be specified with -p to have effect.\n",
 		0,
 		"8192", "bytes" },
-	{ "log_hashstring", tweak_bool, &master.log_hash, 0, 0,
+	{ "log_hashstring", tweak_bool, &mgt_param.log_hash, 0, 0,
 		"Log the hash string components to shared memory log.\n",
 		0,
 		"on", "bool" },
-	{ "log_local_address", tweak_bool, &master.log_local_addr, 0, 0,
+	{ "log_local_address", tweak_bool, &mgt_param.log_local_addr, 0, 0,
 		"Log the local address on the TCP connection in the "
 		"SessionOpen shared memory record.\n",
 		0,
@@ -846,16 +846,16 @@ static const struct parspec input_parspec[] = {
 		"Use 0x notation and do the bitor in your head :-)\n",
 		0,
 		"0", "bitmap" },
-	{ "ban_dups", tweak_bool, &master.ban_dups, 0, 0,
+	{ "ban_dups", tweak_bool, &mgt_param.ban_dups, 0, 0,
 		"Detect and eliminate duplicate bans.\n",
 		0,
 		"on", "bool" },
-	{ "syslog_cli_traffic", tweak_bool, &master.syslog_cli_traffic, 0, 0,
+	{ "syslog_cli_traffic", tweak_bool, &mgt_param.syslog_cli_traffic, 0, 0,
 		"Log all CLI traffic to syslog(LOG_INFO).\n",
 		0,
 		"on", "bool" },
 	{ "ban_lurker_sleep", tweak_timeout_double,
-		&master.ban_lurker_sleep, 0, UINT_MAX,
+		&mgt_param.ban_lurker_sleep, 0, UINT_MAX,
 		"How long time does the ban lurker thread sleeps between "
 		"successful attempts to push the last item up the ban "
 		" list.  It always sleeps a second when nothing can be done.\n"
@@ -863,17 +863,17 @@ static const struct parspec input_parspec[] = {
 		0,
 		"0.01", "s" },
 	{ "saintmode_threshold", tweak_uint,
-		&master.saintmode_threshold, 0, UINT_MAX,
+		&mgt_param.saintmode_threshold, 0, UINT_MAX,
 		"The maximum number of objects held off by saint mode before "
 		"no further will be made to the backend until one times out.  "
 		"A value of 0 disables saintmode.",
 		EXPERIMENTAL,
 		"10", "objects" },
-	{ "http_range_support", tweak_bool, &master.http_range_support, 0, 0,
+	{ "http_range_support", tweak_bool, &mgt_param.http_range_support, 0, 0,
 		"Enable support for HTTP Range headers.\n",
 		EXPERIMENTAL,
 		"on", "bool" },
-	{ "http_gzip_support", tweak_bool, &master.http_gzip_support, 0, 0,
+	{ "http_gzip_support", tweak_bool, &mgt_param.http_gzip_support, 0, 0,
 		"Enable gzip support. When enabled Varnish will compress "
 		"uncompressed objects before they are stored in the cache. "
 		"If a client does not support gzip encoding Varnish will "
@@ -887,7 +887,7 @@ static const struct parspec input_parspec[] = {
 		"Varnish reference.",
 		EXPERIMENTAL,
 		"on", "bool" },
-	{ "gzip_tmp_space", tweak_uint, &master.gzip_tmp_space, 0, 2,
+	{ "gzip_tmp_space", tweak_uint, &mgt_param.gzip_tmp_space, 0, 2,
 		"Where temporary space for gzip/gunzip is allocated:\n"
 		"  0 - malloc\n"
 		"  2 - thread workspace\n"
@@ -897,21 +897,21 @@ static const struct parspec input_parspec[] = {
 		" needs 32+KB of workspace (64+KB if ESI processing).",
 		EXPERIMENTAL,
 		"0", "" },
-	{ "gzip_level", tweak_uint, &master.gzip_level, 0, 9,
+	{ "gzip_level", tweak_uint, &mgt_param.gzip_level, 0, 9,
 		"Gzip compression level: 0=debug, 1=fast, 9=best",
 		0,
 		"6", ""},
-	{ "gzip_window", tweak_uint, &master.gzip_window, 8, 15,
+	{ "gzip_window", tweak_uint, &mgt_param.gzip_window, 8, 15,
 		"Gzip window size 8=least, 15=most compression.\n"
 		"Memory impact is 8=1k, 9=2k, ... 15=128k.",
 		0,
 		"15", ""},
-	{ "gzip_memlevel", tweak_uint, &master.gzip_memlevel, 1, 9,
+	{ "gzip_memlevel", tweak_uint, &mgt_param.gzip_memlevel, 1, 9,
 		"Gzip memory level 1=slow/least, 9=fast/most compression.\n"
 		"Memory impact is 1=1k, 2=2k, ... 9=256k.",
 		0,
 		"8", ""},
-	{ "gzip_stack_buffer", tweak_uint, &master.gzip_stack_buffer,
+	{ "gzip_stack_buffer", tweak_uint, &mgt_param.gzip_stack_buffer,
 	        2048, UINT_MAX,
 		"Size of stack buffer used for gzip processing.\n"
 		"The stack buffers are used for in-transit data,"
@@ -922,13 +922,13 @@ static const struct parspec input_parspec[] = {
 		EXPERIMENTAL,
 		"32768", "Bytes" },
 	{ "shortlived", tweak_timeout_double,
-		&master.shortlived, 0, UINT_MAX,
+		&mgt_param.shortlived, 0, UINT_MAX,
 		"Objects created with TTL shorter than this are always "
 		"put in transient storage.\n",
 		0,
 		"10.0", "s" },
 	{ "critbit_cooloff", tweak_timeout_double,
-		&master.critbit_cooloff, 60, 254,
+		&mgt_param.critbit_cooloff, 60, 254,
 		"How long time the critbit hasher keeps deleted objheads "
 		"on the cooloff list.\n",
 		WIZARD,
@@ -959,7 +959,7 @@ static const struct parspec input_parspec[] = {
 
 
 	{ "pcre_match_limit", tweak_uint,
-		&master.vre_limits.match,
+		&mgt_param.vre_limits.match,
 		1, UINT_MAX,
 		"The limit for the  number of internal matching function"
 		" calls in a pcre_exec() execution.",
@@ -967,7 +967,7 @@ static const struct parspec input_parspec[] = {
 		"10000", ""},
 
 	{ "pcre_match_limit_recursion", tweak_uint,
-		&master.vre_limits.match_recursion,
+		&mgt_param.vre_limits.match_recursion,
 		1, UINT_MAX,
 		"The limit for the  number of internal matching function"
 		" recursions in a pcre_exec() execution.",
@@ -1062,37 +1062,31 @@ mcf_param_show(struct cli *cli, const char * const *av, void *priv)
 /*--------------------------------------------------------------------*/
 
 void
-MCF_ParamSync(void)
-{
-	if (params != &master)
-		*params = master;
-}
-
-/*--------------------------------------------------------------------*/
-
-void
 MCF_ParamSet(struct cli *cli, const char *param, const char *val)
 {
 	const struct parspec *pp;
 
 	pp = mcf_findpar(param);
-	if (pp != NULL) {
-		pp->func(cli, pp, val);
-		if (cli->result != CLIS_OK) {
-			VCLI_Out(cli, "(attempting to set param %s to %s)\n",
-			    pp->name, val);
-		} else if (child_pid >= 0 && pp->flags & MUST_RESTART) {
-			VCLI_Out(cli, "Change will take effect"
-			    " when child is restarted");
-		} else if (pp->flags & MUST_RELOAD) {
-			VCLI_Out(cli, "Change will take effect"
-			    " when VCL script is reloaded");
-		}
-		MCF_ParamSync();
+	if (pp == NULL) {
+		VCLI_SetResult(cli, CLIS_PARAM);
+		VCLI_Out(cli, "Unknown parameter \"%s\".", param);
 		return;
 	}
-	VCLI_SetResult(cli, CLIS_PARAM);
-	VCLI_Out(cli, "Unknown parameter \"%s\".", param);
+	pp->func(cli, pp, val);
+
+	if (cli->result == CLIS_OK && params != NULL)
+		*params = mgt_param;
+
+	if (cli->result != CLIS_OK) {
+		VCLI_Out(cli, "(attempting to set param %s to %s)\n",
+		    pp->name, val);
+	} else if (child_pid >= 0 && pp->flags & MUST_RESTART) {
+		VCLI_Out(cli, "Change will take effect"
+		    " when child is restarted");
+	} else if (pp->flags & MUST_RELOAD) {
+		VCLI_Out(cli, "Change will take effect"
+		    " when VCL script is reloaded");
+	}
 }
 
 
@@ -1173,8 +1167,6 @@ MCF_ParamInit(struct cli *cli)
 	/* XXX: We do this twice, to get past any interdependencies */
 	MCF_SetDefaults(NULL);
 	MCF_SetDefaults(cli);
-
-	params = &master;
 }
 
 /*--------------------------------------------------------------------*/
diff --git a/bin/varnishd/mgt/mgt_pool.c b/bin/varnishd/mgt/mgt_pool.c
index 978e60a..94459b4 100644
--- a/bin/varnishd/mgt/mgt_pool.c
+++ b/bin/varnishd/mgt/mgt_pool.c
@@ -59,8 +59,8 @@ tweak_thread_pool_min(struct cli *cli, const struct parspec *par,
     const char *arg)
 {
 
-	tweak_generic_uint(cli, &master.wthread_min, arg,
-	    (unsigned)par->min, master.wthread_max);
+	tweak_generic_uint(cli, &mgt_param.wthread_min, arg,
+	    (unsigned)par->min, mgt_param.wthread_max);
 }
 
 /*--------------------------------------------------------------------
@@ -86,7 +86,7 @@ tweak_stack_size(struct cli *cli, const struct parspec *par,
 		arg = buf;
 	}
 
-	tweak_generic_uint(cli, &master.wthread_stacksize, arg,
+	tweak_generic_uint(cli, &mgt_param.wthread_stacksize, arg,
 	    low, (uint)par->max);
 }
 
@@ -98,14 +98,14 @@ tweak_thread_pool_max(struct cli *cli, const struct parspec *par,
 {
 
 	(void)par;
-	tweak_generic_uint(cli, &master.wthread_max, arg,
-	    master.wthread_min, UINT_MAX);
+	tweak_generic_uint(cli, &mgt_param.wthread_max, arg,
+	    mgt_param.wthread_min, UINT_MAX);
 }
 
 /*--------------------------------------------------------------------*/
 
 const struct parspec WRK_parspec[] = {
-	{ "thread_pools", tweak_uint, &master.wthread_pools, 1, UINT_MAX,
+	{ "thread_pools", tweak_uint, &mgt_param.wthread_pools, 1, UINT_MAX,
 		"Number of worker thread pools.\n"
 		"\n"
 		"Increasing number of worker pools decreases lock "
@@ -135,7 +135,7 @@ const struct parspec WRK_parspec[] = {
 		"Minimum is 2 threads.",
 		EXPERIMENTAL | DELAYED_EFFECT,
 		"5", "threads" },
-	{ "thread_pool_timeout", tweak_timeout, &master.wthread_timeout, 1, 0,
+	{ "thread_pool_timeout", tweak_timeout, &mgt_param.wthread_timeout, 1, 0,
 		"Thread idle threshold.\n"
 		"\n"
 		"Threads in excess of thread_pool_min, which have been idle "
@@ -145,7 +145,7 @@ const struct parspec WRK_parspec[] = {
 		EXPERIMENTAL | DELAYED_EFFECT,
 		"300", "seconds" },
 	{ "thread_pool_purge_delay",
-		tweak_timeout, &master.wthread_purge_delay, 100, 0,
+		tweak_timeout, &mgt_param.wthread_purge_delay, 100, 0,
 		"Wait this long between purging threads.\n"
 		"\n"
 		"This controls the decay of thread pools when idle(-ish).\n"
@@ -154,7 +154,7 @@ const struct parspec WRK_parspec[] = {
 		EXPERIMENTAL | DELAYED_EFFECT,
 		"1000", "milliseconds" },
 	{ "thread_pool_add_threshold",
-		tweak_uint, &master.wthread_add_threshold, 0, UINT_MAX,
+		tweak_uint, &mgt_param.wthread_add_threshold, 0, UINT_MAX,
 		"Overflow threshold for worker thread creation.\n"
 		"\n"
 		"Setting this too low, will result in excess worker threads, "
@@ -164,7 +164,7 @@ const struct parspec WRK_parspec[] = {
 		EXPERIMENTAL,
 		"2", "requests" },
 	{ "thread_pool_add_delay",
-		tweak_timeout, &master.wthread_add_delay, 0, UINT_MAX,
+		tweak_timeout, &mgt_param.wthread_add_delay, 0, UINT_MAX,
 		"Wait at least this long between creating threads.\n"
 		"\n"
 		"Setting this too long results in insuffient worker threads.\n"
@@ -174,7 +174,7 @@ const struct parspec WRK_parspec[] = {
 		0,
 		"2", "milliseconds" },
 	{ "thread_pool_fail_delay",
-		tweak_timeout, &master.wthread_fail_delay, 100, UINT_MAX,
+		tweak_timeout, &mgt_param.wthread_fail_delay, 100, UINT_MAX,
 		"Wait at least this long after a failed thread creation "
 		"before trying to create another thread.\n"
 		"\n"
@@ -192,7 +192,7 @@ const struct parspec WRK_parspec[] = {
 		EXPERIMENTAL,
 		"200", "milliseconds" },
 	{ "thread_stats_rate",
-		tweak_uint, &master.wthread_stats_rate, 0, UINT_MAX,
+		tweak_uint, &mgt_param.wthread_stats_rate, 0, UINT_MAX,
 		"Worker threads accumulate statistics, and dump these into "
 		"the global stats counters if the lock is free when they "
 		"finish a request.\n"
@@ -201,14 +201,14 @@ const struct parspec WRK_parspec[] = {
 		"its accumulated stats into the global counters.\n",
 		EXPERIMENTAL,
 		"10", "requests" },
-	{ "queue_max", tweak_uint, &master.queue_max, 0, UINT_MAX,
+	{ "queue_max", tweak_uint, &mgt_param.queue_max, 0, UINT_MAX,
 		"Percentage permitted queue length.\n"
 		"\n"
 		"This sets the ratio of queued requests to worker threads, "
 		"above which sessions will be dropped instead of queued.\n",
 		EXPERIMENTAL,
 		"100", "%" },
-	{ "rush_exponent", tweak_uint, &master.rush_exponent, 2, UINT_MAX,
+	{ "rush_exponent", tweak_uint, &mgt_param.rush_exponent, 2, UINT_MAX,
 		"How many parked request we start for each completed "
 		"request on the object.\n"
 		"NB: Even with the implict delay of delivery, "
@@ -217,13 +217,13 @@ const struct parspec WRK_parspec[] = {
 		EXPERIMENTAL,
 		"3", "requests per request" },
 	{ "thread_pool_stack",
-		tweak_stack_size, &master.wthread_stacksize, 0, UINT_MAX,
+		tweak_stack_size, &mgt_param.wthread_stacksize, 0, UINT_MAX,
 		"Worker thread stack size.\n"
 		"On 32bit systems you may need to tweak this down to fit "
 		"many threads into the limited address space.\n",
 		EXPERIMENTAL,
 		"-1", "bytes" },
-	{ "thread_pool_workspace", tweak_uint, &master.wthread_workspace,
+	{ "thread_pool_workspace", tweak_uint, &mgt_param.wthread_workspace,
 		1024, UINT_MAX,
 		"Bytes of HTTP protocol workspace allocated for worker "
 		"threads. "
diff --git a/bin/varnishd/mgt/mgt_sandbox.c b/bin/varnishd/mgt/mgt_sandbox.c
index b01d243..e5fe99c 100644
--- a/bin/varnishd/mgt/mgt_sandbox.c
+++ b/bin/varnishd/mgt/mgt_sandbox.c
@@ -68,8 +68,8 @@ mgt_sandbox(void)
 	mgt_sandbox_solaris_privsep();
 #else
 	if (geteuid() == 0) {
-		XXXAZ(setgid(params->gid));
-		XXXAZ(setuid(params->uid));
+		XXXAZ(setgid(mgt_param.gid));
+		XXXAZ(setuid(mgt_param.uid));
 	} else {
 		REPORT0(LOG_INFO, "Not running as root, no priv-sep");
 	}
diff --git a/bin/varnishd/mgt/mgt_sandbox_solaris.c b/bin/varnishd/mgt/mgt_sandbox_solaris.c
index becaa2e..8c26d69 100644
--- a/bin/varnishd/mgt/mgt_sandbox_solaris.c
+++ b/bin/varnishd/mgt/mgt_sandbox_solaris.c
@@ -158,10 +158,10 @@ void
 mgt_sandbox_solaris_privsep(void)
 {
 	if (priv_ineffect(PRIV_PROC_SETID)) {
-                if (getgid() != params->gid)
-                        XXXAZ(setgid(params->gid));
-                if (getuid() != params->uid)
-                        XXXAZ(setuid(params->uid));
+                if (getgid() != mgt_param.gid)
+                        XXXAZ(setgid(mgt_param.gid));
+                if (getuid() != mgt_param.uid)
+                        XXXAZ(setuid(mgt_param.uid));
         } else {
                 REPORT(LOG_INFO, "Privilege %s missing, will not change uid/gid",
 		    PRIV_PROC_SETID);
diff --git a/bin/varnishd/mgt/mgt_shmem.c b/bin/varnishd/mgt/mgt_shmem.c
index 630865d..37083df 100644
--- a/bin/varnishd/mgt/mgt_shmem.c
+++ b/bin/varnishd/mgt/mgt_shmem.c
@@ -229,7 +229,6 @@ void
 mgt_SHM_Init(const char *l_arg)
 {
 	int i, fill;
-	struct params *pp;
 	const char *q;
 	uintmax_t size, s1, s2, ps;
 	char **av, **ap;
@@ -321,10 +320,10 @@ mgt_SHM_Init(const char *l_arg)
 	    VSC_CLASS, VSC_TYPE_MAIN, "");
 	AN(VSC_C_main);
 
-	pp = VSM_Alloc(sizeof *pp, VSM_CLASS_PARAM, "", "");
-	AN(pp);
-	*pp = *params;
-	params = pp;
+	/* XXX: We need to zero params if we dealloc/clean/wash */
+	params = VSM_Alloc(sizeof *params, VSM_CLASS_PARAM, "", "");
+	AN(params);
+	*params = mgt_param;
 
 	vsl_log_start = VSM_Alloc(s1, VSL_CLASS, "", "");
 	AN(vsl_log_start);
diff --git a/bin/varnishd/varnishd.c b/bin/varnishd/varnishd.c
index e0ba759..6ae1842 100644
--- a/bin/varnishd/varnishd.c
+++ b/bin/varnishd/varnishd.c
@@ -64,7 +64,6 @@
 #endif
 
 struct heritage		heritage;
-volatile struct params	*params;
 unsigned		d_flag = 0;
 pid_t			mgt_pid;
 struct vev_base		*mgt_evb;
@@ -190,17 +189,17 @@ tackle_warg(const char *argv)
 	u = arg_ul(av[1]);
 	if (u < 1)
 		usage();
-	params->wthread_max = params->wthread_min = u;
+	mgt_param.wthread_max = mgt_param.wthread_min = u;
 
 	if (av[2] != NULL) {
 		u = arg_ul(av[2]);
-		if (u < params->wthread_min)
+		if (u < mgt_param.wthread_min)
 			usage();
-		params->wthread_max = u;
+		mgt_param.wthread_max = u;
 
 		if (av[3] != NULL) {
 			u = arg_ul(av[3]);
-			params->wthread_timeout = u;
+			mgt_param.wthread_timeout = u;
 		}
 	}
 	VAV_Free(av);
diff --git a/bin/varnishd/vparam.h b/bin/varnishd/vparam.h
index 2a83de1..2d4a97f 100644
--- a/bin/varnishd/vparam.h
+++ b/bin/varnishd/vparam.h
@@ -55,7 +55,5 @@ void tweak_uint(struct cli *cli, const struct parspec *par, const char *arg);
 void tweak_timeout(struct cli *cli,
     const struct parspec *par, const char *arg);
 
-extern struct params master;
-
 /* mgt_pool.c */
 extern const struct parspec WRK_parspec[];



From geoff at varnish-cache.org  Mon Jan  9 20:52:31 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:31 +0100
Subject: [experimental-ims] a37d3cc Fix a struct member hit by a global
	name-space edit
Message-ID: <E1RkMCl-00087A-AH@project.varnish-software.com>

commit a37d3cc15b5fa9b5f381481a8c95581c3b29bf2c
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu Nov 10 09:12:04 2011 +0000

    Fix a struct member hit by a global name-space edit

diff --git a/bin/varnishd/cache_cli.c b/bin/varnishd/cache_cli.c
index ac2afc1..1e09d23 100644
--- a/bin/varnishd/cache_cli.c
+++ b/bin/varnishd/cache_cli.c
@@ -102,7 +102,7 @@ CLI_Run(void)
 
 	add_check = 1;
 
-	AN(VCLS_AddFd(cls, heritage.cli_in, heritage.VCLI_Out, NULL, NULL));
+	AN(VCLS_AddFd(cls, heritage.cli_in, heritage.cli_out, NULL, NULL));
 
 	do {
 		i = VCLS_Poll(cls, -1);
diff --git a/bin/varnishd/heritage.h b/bin/varnishd/heritage.h
index 39fc03e..e6e6558 100644
--- a/bin/varnishd/heritage.h
+++ b/bin/varnishd/heritage.h
@@ -46,7 +46,7 @@ struct heritage {
 
 	/* Two pipe(2)'s for CLI connection between cache and mgt.  */
 	int				cli_in;
-	int				VCLI_Out;
+	int				cli_out;
 
 	/* File descriptor for stdout/stderr */
 	int				std_fd;
diff --git a/bin/varnishd/mgt/mgt_child.c b/bin/varnishd/mgt/mgt_child.c
index 9503e5b..cfe0b2f 100644
--- a/bin/varnishd/mgt/mgt_child.c
+++ b/bin/varnishd/mgt/mgt_child.c
@@ -303,8 +303,8 @@ start_child(struct cli *cli)
 
 	/* Open pipe for child->mgr CLI */
 	AZ(pipe(cp));
-	heritage.VCLI_Out = cp[1];
-	mgt_child_inherit(heritage.VCLI_Out, "VCLI_Out");
+	heritage.cli_out = cp[1];
+	mgt_child_inherit(heritage.cli_out, "cli_out");
 	child_cli_in = cp[0];
 
 	/*
@@ -356,8 +356,8 @@ start_child(struct cli *cli)
 	mgt_child_inherit(heritage.cli_in, NULL);
 	closex(&heritage.cli_in);
 
-	mgt_child_inherit(heritage.VCLI_Out, NULL);
-	closex(&heritage.VCLI_Out);
+	mgt_child_inherit(heritage.cli_out, NULL);
+	closex(&heritage.cli_out);
 
 	close_sockets();
 



From geoff at varnish-cache.org  Mon Jan  9 20:52:31 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:31 +0100
Subject: [experimental-ims] 3bf99f6 Firmly split manager and child views of
	hashing.
Message-ID: <E1RkMCl-00087I-Dp@project.varnish-software.com>

commit 3bf99f6c093d87b49cdec143138c98c3cb4e14f6
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu Nov 10 09:33:39 2011 +0000

    Firmly split manager and child views of hashing.

diff --git a/bin/varnishd/Makefile.am b/bin/varnishd/Makefile.am
index c95b09a..f7c7620 100644
--- a/bin/varnishd/Makefile.am
+++ b/bin/varnishd/Makefile.am
@@ -54,6 +54,7 @@ varnishd_SOURCES = \
 	cache_ws.c \
 	hash/hash_classic.c \
 	hash/hash_critbit.c \
+	hash/hash_mgt.c \
 	hash/hash_simple_list.c \
 	mgt/mgt_child.c \
 	mgt/mgt_cli.c \
diff --git a/bin/varnishd/cache_hash.c b/bin/varnishd/cache_hash.c
index 669fd40..db865de 100644
--- a/bin/varnishd/cache_hash.c
+++ b/bin/varnishd/cache_hash.c
@@ -59,7 +59,6 @@
 #include "cache.h"
 
 #include "hash/hash_slinger.h"
-#include "vav.h"
 #include "vsha256.h"
 
 static const struct hash_slinger *hash;
@@ -743,53 +742,11 @@ HSH_Deref(struct worker *w, struct objcore *oc, struct object **oo)
 }
 
 void
-HSH_Init(void)
+HSH_Init(const struct hash_slinger *slinger)
 {
 
 	assert(DIGEST_LEN == SHA256_LEN);	/* avoid #include pollution */
-	hash = heritage.hash;
+	hash = slinger;
 	if (hash->start != NULL)
 		hash->start();
 }
-
-static const struct choice hsh_choice[] = {
-	{ "classic",		&hcl_slinger },
-	{ "simple",		&hsl_slinger },
-	{ "simple_list",	&hsl_slinger },	/* backwards compat */
-	{ "critbit",		&hcb_slinger },
-	{ NULL,			NULL }
-};
-
-/*--------------------------------------------------------------------*/
-
-void
-HSH_config(const char *h_arg)
-{
-	char **av;
-	int ac;
-	const struct hash_slinger *hp;
-
-	ASSERT_MGT();
-	av = VAV_Parse(h_arg, NULL, ARGV_COMMA);
-	AN(av);
-
-	if (av[0] != NULL)
-		ARGV_ERR("%s\n", av[0]);
-
-	if (av[1] == NULL)
-		ARGV_ERR("-h argument is empty\n");
-
-	for (ac = 0; av[ac + 2] != NULL; ac++)
-		continue;
-
-	hp = pick(hsh_choice, av[1], "hash");
-	CHECK_OBJ_NOTNULL(hp, SLINGER_MAGIC);
-	VSB_printf(vident, ",-h%s", av[1]);
-	heritage.hash = hp;
-	if (hp->init != NULL)
-		hp->init(ac, av + 2);
-	else if (ac > 0)
-		ARGV_ERR("Hash method \"%s\" takes no arguments\n",
-		    hp->name);
-}
-
diff --git a/bin/varnishd/cache_main.c b/bin/varnishd/cache_main.c
index 5dc2d00..f8f06e1 100644
--- a/bin/varnishd/cache_main.c
+++ b/bin/varnishd/cache_main.c
@@ -120,7 +120,7 @@ child_main(void)
 	Pool_Init();
 
 	EXP_Init();
-	HSH_Init();
+	HSH_Init(heritage.hash);
 	BAN_Init();
 
 	VCA_Init();
diff --git a/bin/varnishd/hash/hash_mgt.c b/bin/varnishd/hash/hash_mgt.c
new file mode 100644
index 0000000..6226749
--- /dev/null
+++ b/bin/varnishd/hash/hash_mgt.c
@@ -0,0 +1,84 @@
+/*-
+ * Copyright (c) 2006 Verdens Gang AS
+ * Copyright (c) 2006-2011 Varnish Software AS
+ * All rights reserved.
+ *
+ * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ */
+
+#include "config.h"
+
+#include <math.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <unistd.h>
+
+#include "mgt/mgt.h"
+#include "heritage.h"
+
+#include "hash/hash_slinger.h"
+#include "vav.h"
+
+static const struct choice hsh_choice[] = {
+	{ "classic",		&hcl_slinger },
+	{ "simple",		&hsl_slinger },
+	{ "simple_list",	&hsl_slinger },	/* backwards compat */
+	{ "critbit",		&hcb_slinger },
+	{ NULL,			NULL }
+};
+
+/*--------------------------------------------------------------------*/
+
+void
+HSH_config(const char *h_arg)
+{
+	char **av;
+	int ac;
+	const struct hash_slinger *hp;
+
+	ASSERT_MGT();
+	av = VAV_Parse(h_arg, NULL, ARGV_COMMA);
+	AN(av);
+
+	if (av[0] != NULL)
+		ARGV_ERR("%s\n", av[0]);
+
+	if (av[1] == NULL)
+		ARGV_ERR("-h argument is empty\n");
+
+	for (ac = 0; av[ac + 2] != NULL; ac++)
+		continue;
+
+	hp = pick(hsh_choice, av[1], "hash");
+	CHECK_OBJ_NOTNULL(hp, SLINGER_MAGIC);
+	VSB_printf(vident, ",-h%s", av[1]);
+	heritage.hash = hp;
+	if (hp->init != NULL)
+		hp->init(ac, av + 2);
+	else if (ac > 0)
+		ARGV_ERR("Hash method \"%s\" takes no arguments\n",
+		    hp->name);
+}
+
diff --git a/bin/varnishd/hash/hash_slinger.h b/bin/varnishd/hash/hash_slinger.h
index 2c142cf..23b892d 100644
--- a/bin/varnishd/hash/hash_slinger.h
+++ b/bin/varnishd/hash/hash_slinger.h
@@ -57,7 +57,7 @@ struct objcore *HSH_Lookup(struct sess *sp, struct objhead **poh);
 void HSH_Unbusy(const struct sess *sp);
 void HSH_Ref(struct objcore *o);
 void HSH_Drop(struct sess *sp);
-void HSH_Init(void);
+void HSH_Init(const struct hash_slinger *slinger);
 void HSH_AddString(const struct sess *sp, const char *str);
 struct objcore *HSH_Insert(const struct sess *sp);
 void HSH_Purge(const struct sess *, struct objhead *, double ttl, double grace);



From geoff at varnish-cache.org  Mon Jan  9 20:52:33 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:33 +0100
Subject: [experimental-ims] 0af2c24 Make sure the CLI headers we produce tell
	the truth about the length
Message-ID: <E1RkMCn-0008Ap-LF@project.varnish-software.com>

commit 0af2c245047f904d91d800b659f62c3af2908413
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sun Nov 13 22:08:33 2011 +0000

    Make sure the CLI headers we produce tell the truth about the length

diff --git a/lib/libvarnish/cli_common.c b/lib/libvarnish/cli_common.c
index 3a44e9f..4a17de5 100644
--- a/lib/libvarnish/cli_common.c
+++ b/lib/libvarnish/cli_common.c
@@ -87,6 +87,7 @@ VCLI_WriteResult(int fd, unsigned status, const char *result)
 	int i, l;
 	struct iovec iov[3];
 	char nl[2] = "\n";
+	size_t len;
 	char res[CLI_LINE0_LEN + 2];	/*
 					 * NUL + one more so we can catch
 					 * any misformats by snprintf
@@ -95,15 +96,18 @@ VCLI_WriteResult(int fd, unsigned status, const char *result)
 	assert(status >= 100);
 	assert(status <= 999);		/*lint !e650 const out of range */
 
+	len = strlen(result);
+
 	i = snprintf(res, sizeof res,
-	    "%-3d %-8jd\n", status, (intmax_t)strlen(result));
+	    "%-3d %-8zd\n", status, len);
 	assert(i == CLI_LINE0_LEN);
+	assert(strtoul(res + 3, NULL, 10) == len);
 
 	iov[0].iov_base = res;
 	iov[0].iov_len = CLI_LINE0_LEN;
 
 	iov[1].iov_base = (void*)(uintptr_t)result;	/* TRUST ME */
-	iov[1].iov_len = strlen(result);
+	iov[1].iov_len = len;
 
 	iov[2].iov_base = nl;
 	iov[2].iov_len = 1;



From geoff at varnish-cache.org  Mon Jan  9 20:52:30 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:30 +0100
Subject: [experimental-ims] 0fe28b3 Polish whitespace and update copyrights.
Message-ID: <E1RkMCk-00086D-NA@project.varnish-software.com>

commit 0fe28b30be96ca4232b6d037598202ba1272290e
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Nov 9 09:47:15 2011 +0000

    Polish whitespace and update copyrights.

diff --git a/bin/varnishd/cache_backend.h b/bin/varnishd/cache_backend.h
index 91bff1e..72a1283 100644
--- a/bin/varnishd/cache_backend.h
+++ b/bin/varnishd/cache_backend.h
@@ -1,6 +1,6 @@
 /*-
  * Copyright (c) 2006 Verdens Gang AS
- * Copyright (c) 2006-2010 Varnish Software AS
+ * Copyright (c) 2006-2011 Varnish Software AS
  * All rights reserved.
  *
  * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
diff --git a/bin/varnishd/cache_backend_cfg.c b/bin/varnishd/cache_backend_cfg.c
index e87584a..cbb8c85 100644
--- a/bin/varnishd/cache_backend_cfg.c
+++ b/bin/varnishd/cache_backend_cfg.c
@@ -1,6 +1,6 @@
 /*-
  * Copyright (c) 2006 Verdens Gang AS
- * Copyright (c) 2006-2010 Varnish Software AS
+ * Copyright (c) 2006-2011 Varnish Software AS
  * All rights reserved.
  *
  * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
diff --git a/bin/varnishd/cache_ban.c b/bin/varnishd/cache_ban.c
index c01a3f6..648997c 100644
--- a/bin/varnishd/cache_ban.c
+++ b/bin/varnishd/cache_ban.c
@@ -51,7 +51,7 @@
  *	(lump) - compiled regexp
  * A lump is:
  *	4 bytes - be32: length
- * 	n bytes - content
+ *	n bytes - content
  *
  * In a perfect world, we should vector through VRE to get to PCRE,
  * but since we rely on PCRE's ability to encode the regexp into a
diff --git a/bin/varnishd/cache_dir.c b/bin/varnishd/cache_dir.c
index 59ef5fc..d4794f9 100644
--- a/bin/varnishd/cache_dir.c
+++ b/bin/varnishd/cache_dir.c
@@ -1,6 +1,6 @@
 /*-
  * Copyright (c) 2006 Verdens Gang AS
- * Copyright (c) 2006-2010 Varnish Software AS
+ * Copyright (c) 2006-2011 Varnish Software AS
  * All rights reserved.
  *
  * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
diff --git a/bin/varnishd/cache_esi_fetch.c b/bin/varnishd/cache_esi_fetch.c
index 728436c..f414e7b 100644
--- a/bin/varnishd/cache_esi_fetch.c
+++ b/bin/varnishd/cache_esi_fetch.c
@@ -240,7 +240,7 @@ vfp_esi_bytes_ug(struct worker *w, struct http_conn *htc, ssize_t bytes)
  * We receive a gzip'ed object, and want to store it gzip'ed.
  */
 
-static int 
+static int
 vfp_esi_bytes_gg(struct worker *w, struct http_conn *htc, size_t bytes)
 {
 	ssize_t wl;
@@ -391,7 +391,7 @@ vfp_esi_end(struct worker *w)
 		w->vef_priv = NULL;
 		VGZ_UpdateObj(vef->vgz, w->fetch_obj);
 		if (VGZ_Destroy(&vef->vgz,  -1) != VGZ_END)
-			retval = FetchError(w, 
+			retval = FetchError(w,
 			    "ESI+Gzip Failed at the very end");
 		FREE_OBJ(vef);
 	}
diff --git a/bin/varnishd/cache_fetch.c b/bin/varnishd/cache_fetch.c
index ed1e5c8..67bf970 100644
--- a/bin/varnishd/cache_fetch.c
+++ b/bin/varnishd/cache_fetch.c
@@ -277,7 +277,7 @@ fetch_chunked(struct worker *w, struct http_conn *htc)
 				break;
 		}
 
-		if (u >= sizeof buf) 
+		if (u >= sizeof buf)
 			return (FetchError(w,"chunked header too long"));
 
 		/* Skip trailing white space */
@@ -285,7 +285,7 @@ fetch_chunked(struct worker *w, struct http_conn *htc)
 			if (HTC_Read(w, htc, buf + u, 1) <= 0)
 				return (-1);
 
-		if (buf[u] != '\n') 
+		if (buf[u] != '\n')
 			return (FetchError(w,"chunked header no NL"));
 
 		buf[u] = '\0';
@@ -316,7 +316,7 @@ fetch_eof(struct worker *w, struct http_conn *htc)
 
 	assert(w->body_status == BS_EOF);
 	i = w->vfp->bytes(w, htc, SSIZE_MAX);
-	if (i < 0) 
+	if (i < 0)
 		return (-1);
 	return (0);
 }
diff --git a/bin/varnishd/cache_gzip.c b/bin/varnishd/cache_gzip.c
index 1820ffc..6da1595 100644
--- a/bin/varnishd/cache_gzip.c
+++ b/bin/varnishd/cache_gzip.c
@@ -257,7 +257,7 @@ VGZ_ObufStorage(struct worker *w, struct vgz *vg)
 	struct storage *st;
 
 	st = FetchStorage(w, 0);
-	if (st == NULL) 
+	if (st == NULL)
 		return (-1);
 
 	vg->obuf = st;
@@ -487,10 +487,10 @@ vfp_gunzip_bytes(struct worker *w, struct http_conn *htc, ssize_t bytes)
 			bytes -= wl;
 		}
 
-		if (VGZ_ObufStorage(w, vg)) 
+		if (VGZ_ObufStorage(w, vg))
 			return(-1);
 		i = VGZ_Gunzip(vg, &dp, &dl);
-		if (i != VGZ_OK && i != VGZ_END) 
+		if (i != VGZ_OK && i != VGZ_END)
 			return(FetchError(w, "Gunzip data error"));
 		w->fetch_obj->len += dl;
 		if (w->do_stream)
@@ -564,7 +564,7 @@ vfp_gzip_bytes(struct worker *w, struct http_conn *htc, ssize_t bytes)
 			VGZ_Ibuf(vg, ibuf, wl);
 			bytes -= wl;
 		}
-		if (VGZ_ObufStorage(w, vg)) 
+		if (VGZ_ObufStorage(w, vg))
 			return(-1);
 		i = VGZ_Gzip(vg, &dp, &dl, VGZ_NORMAL);
 		assert(i == Z_OK);
@@ -661,7 +661,7 @@ vfp_testgzip_bytes(struct worker *w, struct http_conn *htc, ssize_t bytes)
 		while (!VGZ_IbufEmpty(vg)) {
 			VGZ_Obuf(vg, obuf, sizeof obuf);
 			i = VGZ_Gunzip(vg, &dp, &dl);
-			if (i == VGZ_END && !VGZ_IbufEmpty(vg)) 
+			if (i == VGZ_END && !VGZ_IbufEmpty(vg))
 				return(FetchError(w, "Junk after gzip data"));
 			if (i != VGZ_OK && i != VGZ_END)
 				return(FetchError2(w,
diff --git a/bin/varnishd/cache_httpconn.c b/bin/varnishd/cache_httpconn.c
index 1b718e5..9e0a052 100644
--- a/bin/varnishd/cache_httpconn.c
+++ b/bin/varnishd/cache_httpconn.c
@@ -1,6 +1,6 @@
 /*-
  * Copyright (c) 2006 Verdens Gang AS
- * Copyright (c) 2006-2009 Varnish Software AS
+ * Copyright (c) 2006-2011 Varnish Software AS
  * All rights reserved.
  *
  * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
diff --git a/bin/varnishd/cache_pipe.c b/bin/varnishd/cache_pipe.c
index 31dcf1a..be17ac9 100644
--- a/bin/varnishd/cache_pipe.c
+++ b/bin/varnishd/cache_pipe.c
@@ -1,6 +1,6 @@
 /*-
  * Copyright (c) 2006 Verdens Gang AS
- * Copyright (c) 2006-2010 Varnish Software AS
+ * Copyright (c) 2006-2011 Varnish Software AS
  * All rights reserved.
  *
  * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
diff --git a/bin/varnishd/cache_pool.c b/bin/varnishd/cache_pool.c
index 141a0c7..aed596e 100644
--- a/bin/varnishd/cache_pool.c
+++ b/bin/varnishd/cache_pool.c
@@ -451,13 +451,13 @@ pool_herder(void *priv)
 		}
 
 		pool_breed(pp, &tp_attr);
-		
+
 		if (pp->nthr < params->wthread_min)
 			continue;
 
 		AZ(clock_gettime(CLOCK_MONOTONIC, &ts));
 		ts.tv_sec += params->wthread_purge_delay / 1000;
-		ts.tv_nsec += 
+		ts.tv_nsec +=
 		    (params->wthread_purge_delay % 1000) * 1000000;
 		if (ts.tv_nsec >= 1000000000) {
 			ts.tv_sec++;
@@ -470,7 +470,7 @@ pool_herder(void *priv)
 		if (!i)
 			continue;
 
-		if (pp->nthr <= params->wthread_min) 
+		if (pp->nthr <= params->wthread_min)
 			continue;
 
 		t_idle = VTIM_real() - params->wthread_timeout;
@@ -480,7 +480,7 @@ pool_herder(void *priv)
 		VSC_C_main->sess_dropped += pp->ndropped;
 		pp->nqueued = pp->ndropped = 0;
 		w = VTAILQ_LAST(&pp->idle, workerhead);
-		if (w != NULL &&  
+		if (w != NULL &&
 		    (w->lastused < t_idle || pp->nthr > params->wthread_max)) {
 			VTAILQ_REMOVE(&pp->idle, w, list);
 		} else
@@ -567,12 +567,12 @@ pool_poolherder(void *priv)
 				VSC_C_main->pools++;
 				nwq++;
 				continue;
-			} 
+			}
 		}
 		/* XXX: remove pools */
 		(void)sleep(1);
 		u = 0;
-		VTAILQ_FOREACH(pp, &pools, list) 
+		VTAILQ_FOREACH(pp, &pools, list)
 			u += pp->lqueue;
 		VSC_C_main->thread_queue_len = u;
 	}
diff --git a/bin/varnishd/mgt/mgt_sandbox_solaris.c b/bin/varnishd/mgt/mgt_sandbox_solaris.c
index f243938..becaa2e 100644
--- a/bin/varnishd/mgt/mgt_sandbox_solaris.c
+++ b/bin/varnishd/mgt/mgt_sandbox_solaris.c
@@ -48,7 +48,7 @@
 
 /*--------------------------------------------------------------------
  * SOLARIS PRIVILEGES: Note on use of symbolic PRIV_* constants
- * 
+ *
  * For privileges which existed in Solaris 10 FCS, we may use the constants from
  * sys/priv_names.h
  *
@@ -140,7 +140,7 @@ mgt_sandbox_solaris_init(void)
 		    errno, strerror(errno));
 		return;
 	}
-	
+
 	priv_emptyset(priv_all);
 
 	mgt_sandbox_solaris_add_effective(priv_all);
@@ -168,7 +168,7 @@ mgt_sandbox_solaris_privsep(void)
         }
 }
 
-/* 
+/*
  * Waive most privileges in the child
  *
  * as of onnv_151a, we should end up with:
@@ -206,7 +206,7 @@ mgt_sandbox_solaris_fini(void)
 	priv_copyset(effective, permitted);
 	mgt_sandbox_solaris_add_permitted(permitted);
 
-	/* 
+	/*
 	 * invert the sets and clear privileges such that setppriv will always
 	 * succeed
 	 */
diff --git a/bin/varnishhist/varnishhist.c b/bin/varnishhist/varnishhist.c
index 692eb81..a4bc6b7 100644
--- a/bin/varnishhist/varnishhist.c
+++ b/bin/varnishhist/varnishhist.c
@@ -1,6 +1,6 @@
 /*-
  * Copyright (c) 2006 Verdens Gang AS
- * Copyright (c) 2006-2010 Varnish Software AS
+ * Copyright (c) 2006-2011 Varnish Software AS
  * All rights reserved.
  *
  * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
diff --git a/bin/varnishstat/varnishstat.c b/bin/varnishstat/varnishstat.c
index b5d6df3..d0ef409 100644
--- a/bin/varnishstat/varnishstat.c
+++ b/bin/varnishstat/varnishstat.c
@@ -311,6 +311,6 @@ main(int argc, char * const *argv)
 		printf("\n");
 
 		sleep(delay);
-	} 
+	}
 	exit(0);
 }
diff --git a/bin/varnishtest/vtc_log.c b/bin/varnishtest/vtc_log.c
index 57fa27a..890a98b 100644
--- a/bin/varnishtest/vtc_log.c
+++ b/bin/varnishtest/vtc_log.c
@@ -150,7 +150,7 @@ vtc_log(struct vtclog *vl, int lvl, const char *fmt, ...)
 	AZ(pthread_mutex_unlock(&vl->mtx));
 	if (lvl > 0)
 		return;
-	if (lvl == 0) 
+	if (lvl == 0)
 		vtc_error = 1;
 	if (pthread_self() != vtc_thread)
 		pthread_exit(NULL);
diff --git a/include/tbl/vsc_fields.h b/include/tbl/vsc_fields.h
index 904a9e2..abc5d18 100644
--- a/include/tbl/vsc_fields.h
+++ b/include/tbl/vsc_fields.h
@@ -81,48 +81,48 @@ VSC_F(sess_fail,		uint64_t, 1, 'c',
 
 /*---------------------------------------------------------------------*/
 
-VSC_F(client_req,		uint64_t, 1, 'a', 
-      "Client requests received", 
+VSC_F(client_req,		uint64_t, 1, 'a',
+      "Client requests received",
       "")
 
-VSC_F(cache_hit,		uint64_t, 1, 'a', 
-      "Cache hits", 
+VSC_F(cache_hit,		uint64_t, 1, 'a',
+      "Cache hits",
       "Count of cache hits. "
       "  A cache hit indicates that an object has been delivered to a"
       "  client without fetching it from a backend server."
 )
 
-VSC_F(cache_hitpass,	uint64_t, 1, 'a', 
-      "Cache hits for pass", 
+VSC_F(cache_hitpass,	uint64_t, 1, 'a',
+      "Cache hits for pass",
       "Count of hits for pass"
       "  A cache hit for pass indicates that Varnish is going to"
       "  pass the request to the backend and this decision has been "
       "  cached in it self. This counts how many times the cached "
       "  decision is being used."
 )
-VSC_F(cache_miss,		uint64_t, 1, 'a', 
-      "Cache misses", 
+VSC_F(cache_miss,		uint64_t, 1, 'a',
+      "Cache misses",
       "Count of misses"
       "  A cache miss indicates the object was fetched from the"
       "  backend before delivering it to the backend.")
 
-VSC_F(backend_conn,	uint64_t, 0, 'a', 
-      "Backend conn. success", 
+VSC_F(backend_conn,	uint64_t, 0, 'a',
+      "Backend conn. success",
       "")
 
-VSC_F(backend_unhealthy,	uint64_t, 0, 'a', 
-      "Backend conn. not attempted", 
+VSC_F(backend_unhealthy,	uint64_t, 0, 'a',
+      "Backend conn. not attempted",
       ""
 )
 VSC_F(backend_busy,	uint64_t, 0, 'a', "Backend conn. too many", "")
 VSC_F(backend_fail,	uint64_t, 0, 'a', "Backend conn. failures", "")
-VSC_F(backend_reuse,	uint64_t, 0, 'a', 
-      "Backend conn. reuses", 
+VSC_F(backend_reuse,	uint64_t, 0, 'a',
+      "Backend conn. reuses",
       "Count of backend connection reuses"
       "  This counter is increased whenever we reuse a recycled connection.")
 VSC_F(backend_toolate,	uint64_t, 0, 'a', "Backend conn. was closed", "")
-VSC_F(backend_recycle,	uint64_t, 0, 'a', 
-      "Backend conn. recycles", 
+VSC_F(backend_recycle,	uint64_t, 0, 'a',
+      "Backend conn. recycles",
       "Count of backend connection recycles"
       "  This counter is increased whenever we have a keep-alive"
       "  connection that is put back into the pool of connections."
@@ -255,10 +255,10 @@ VSC_F(n_lru_moved,		uint64_t, 0, 'i', "N LRU moved objects", "")
 
 VSC_F(losthdr,		uint64_t, 0, 'a', "HTTP header overflows", "")
 
-VSC_F(n_objsendfile,	uint64_t, 0, 'a', "Objects sent with sendfile", 
+VSC_F(n_objsendfile,	uint64_t, 0, 'a', "Objects sent with sendfile",
       "The number of objects sent with the sendfile system call. If enabled "
       "sendfile will be used on object larger than a certain size.")
-VSC_F(n_objwrite,		uint64_t, 0, 'a', "Objects sent with write", 
+VSC_F(n_objwrite,		uint64_t, 0, 'a', "Objects sent with write",
       "The number of objects sent with regular write calls."
       "Writes are used when the objects are too small for sendfile "
       "or if the sendfile call has been disabled")
diff --git a/lib/libvarnish/vre.c b/lib/libvarnish/vre.c
index cffe396..eac8121 100644
--- a/lib/libvarnish/vre.c
+++ b/lib/libvarnish/vre.c
@@ -1,5 +1,5 @@
 /*-
- * Copyright (c) 2006-2009 Varnish Software AS
+ * Copyright (c) 2006-2011 Varnish Software AS
  * All rights reserved.
  *
  * Author: Tollef Fog Heen <tfheen at redpill-linpro.com>
diff --git a/lib/libvarnish/vtcp.c b/lib/libvarnish/vtcp.c
index c0f7a95..21de7f2 100644
--- a/lib/libvarnish/vtcp.c
+++ b/lib/libvarnish/vtcp.c
@@ -265,7 +265,7 @@ VTCP_set_read_timeout(int s, double seconds)
 	timeout.tv_sec = (int)floor(seconds);
 	timeout.tv_usec = (int)(1e6 * (seconds - timeout.tv_sec));
 #ifdef SO_RCVTIMEO_WORKS
-	/* 
+	/*
 	 * Solaris bug (present at least in snv_151 and older): If this fails
 	 * with EINVAL, the socket is half-closed (SS_CANTSENDMORE) and the
 	 * timeout does not get set. Needs to be fixed in Solaris, there is



From geoff at varnish-cache.org  Mon Jan  9 20:52:33 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:33 +0100
Subject: [experimental-ims] f3a414f Stuff all of the worker process into a
	cache/ subdirectory.
Message-ID: <E1RkMCn-0008A9-0H@project.varnish-software.com>

commit f3a414fbf75eb4e4927aa7d46bd7e92a8abf5fd4
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sun Nov 13 10:55:14 2011 +0000

    Stuff all of the worker process into a cache/ subdirectory.
    
    Yes, I should have done this five years ago...

diff --git a/bin/varnishd/Makefile.am b/bin/varnishd/Makefile.am
index 245bc6b..850a18a 100644
--- a/bin/varnishd/Makefile.am
+++ b/bin/varnishd/Makefile.am
@@ -3,6 +3,7 @@
 INCLUDES = \
 	-I$(top_srcdir)/include \
 	-I$(top_srcdir)/lib/libvgz \
+	-I$(top_builddir)/bin/varnishd \
 	-I$(top_builddir)/include
 
 sbin_PROGRAMS = varnishd
@@ -10,44 +11,44 @@ sbin_PROGRAMS = varnishd
 dist_man_MANS = varnishd.1
 
 varnishd_SOURCES = \
-	cache_acceptor.c \
-	cache_backend.c \
-	cache_backend_cfg.c \
-	cache_backend_poll.c \
-	cache_ban.c \
-	cache_center.c \
-	cache_cli.c \
-	cache_dir.c \
-	cache_dir_dns.c \
-	cache_dir_random.c \
-	cache_dir_round_robin.c \
-	cache_esi_deliver.c \
-	cache_esi_fetch.c \
-	cache_esi_parse.c \
-	cache_expire.c \
-	cache_fetch.c \
-	cache_gzip.c \
-	cache_hash.c \
-	cache_http.c \
-	cache_httpconn.c \
-	cache_lck.c \
-	cache_main.c \
-	cache_panic.c \
-	cache_pipe.c \
-	cache_pool.c \
-	cache_response.c \
-	cache_rfc2616.c \
-	cache_session.c \
-	cache_shmlog.c \
-	cache_vary.c \
-	cache_vcl.c \
-	cache_vrt.c \
-	cache_vrt_re.c \
-	cache_vrt_var.c \
-	cache_vrt_vmod.c \
-	cache_wrk.c \
-	cache_wrw.c \
-	cache_ws.c \
+	cache/cache_acceptor.c \
+	cache/cache_backend.c \
+	cache/cache_backend_cfg.c \
+	cache/cache_backend_poll.c \
+	cache/cache_ban.c \
+	cache/cache_center.c \
+	cache/cache_cli.c \
+	cache/cache_dir.c \
+	cache/cache_dir_dns.c \
+	cache/cache_dir_random.c \
+	cache/cache_dir_round_robin.c \
+	cache/cache_esi_deliver.c \
+	cache/cache_esi_fetch.c \
+	cache/cache_esi_parse.c \
+	cache/cache_expire.c \
+	cache/cache_fetch.c \
+	cache/cache_gzip.c \
+	cache/cache_hash.c \
+	cache/cache_http.c \
+	cache/cache_httpconn.c \
+	cache/cache_lck.c \
+	cache/cache_main.c \
+	cache/cache_panic.c \
+	cache/cache_pipe.c \
+	cache/cache_pool.c \
+	cache/cache_response.c \
+	cache/cache_rfc2616.c \
+	cache/cache_session.c \
+	cache/cache_shmlog.c \
+	cache/cache_vary.c \
+	cache/cache_vcl.c \
+	cache/cache_vrt.c \
+	cache/cache_vrt_re.c \
+	cache/cache_vrt_var.c \
+	cache/cache_vrt_vmod.c \
+	cache/cache_wrk.c \
+	cache/cache_wrw.c \
+	cache/cache_ws.c \
 	common/common_vsm.c \
 	hash/hash_classic.c \
 	hash/hash_critbit.c \
@@ -79,9 +80,9 @@ varnishd_SOURCES = \
 	waiter/cache_waiter_ports.c 
 
 noinst_HEADERS = \
-	cache.h \
-	cache_backend.h \
-	cache_esi.h \
+	cache/cache.h \
+	cache/cache_backend.h \
+	cache/cache_esi.h \
 	common/common.h \
 	common/heritage.h \
 	common/params.h \
diff --git a/bin/varnishd/cache.h b/bin/varnishd/cache.h
deleted file mode 100644
index 653fe77..0000000
--- a/bin/varnishd/cache.h
+++ /dev/null
@@ -1,1039 +0,0 @@
-/*-
- * Copyright (c) 2006 Verdens Gang AS
- * Copyright (c) 2006-2011 Varnish Software AS
- * All rights reserved.
- *
- * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- *
- */
-
-/*
- * This macro can be used in .h files to isolate bits that the manager
- * should not (need to) see, such as pthread mutexes etc.
- */
-#define VARNISH_CACHE_CHILD	1
-
-#include "common/common.h"
-
-#include "vapi/vsc_int.h"
-#include "vapi/vsl_int.h"
-
-#include <sys/socket.h>
-
-#include <pthread.h>
-#ifdef HAVE_PTHREAD_NP_H
-#include <pthread_np.h>
-#endif
-#include <stdarg.h>
-#include <string.h>
-#include <limits.h>
-#include <unistd.h>
-
-#if defined(HAVE_EPOLL_CTL)
-#include <sys/epoll.h>
-#endif
-
-
-#include "common/params.h"
-
-enum body_status {
-#define BODYSTATUS(U,l)	BS_##U,
-#include "tbl/body_status.h"
-#undef BODYSTATUS
-};
-
-static inline const char *
-body_status(enum body_status e)
-{
-	switch(e) {
-#define BODYSTATUS(U,l)	case BS_##U: return (#l);
-#include "tbl/body_status.h"
-#undef BODYSTATUS
-	default:
-		return ("?");
-	}
-}
-
-/*
- * NB: HDR_STATUS is only used in cache_http.c, everybody else uses the
- * http->status integer field.
- */
-
-enum {
-	/* Fields from the first line of HTTP proto */
-	HTTP_HDR_REQ,
-	HTTP_HDR_URL,
-	HTTP_HDR_PROTO,
-	HTTP_HDR_STATUS,
-	HTTP_HDR_RESPONSE,
-	/* HTTP header lines */
-	HTTP_HDR_FIRST,
-};
-
-struct SHA256Context;
-struct VSC_C_lck;
-struct ban;
-struct busyobj;
-struct cli;
-struct cli_proto;
-struct director;
-struct iovec;
-struct objcore;
-struct object;
-struct objhead;
-struct pool;
-struct sess;
-struct sesspool;
-struct vbc;
-struct vef_priv;
-struct vrt_backend;
-struct vsb;
-struct waitinglist;
-struct worker;
-
-#define DIGEST_LEN		32
-
-/* Name of transient storage */
-#define TRANSIENT_STORAGE	"Transient"
-
-/*--------------------------------------------------------------------
- * Pointer aligment magic
- */
-
-#define PALGN		(sizeof(void *) - 1)
-#define PAOK(p)		(((uintptr_t)(p) & PALGN) == 0)
-#define PRNDDN(p)	((uintptr_t)(p) & ~PALGN)
-#define PRNDUP(p)	(((uintptr_t)(p) + PALGN) & ~PALGN)
-
-/*--------------------------------------------------------------------*/
-
-typedef struct {
-	char			*b;
-	char			*e;
-} txt;
-
-/*--------------------------------------------------------------------*/
-
-enum step {
-#define STEP(l, u)	STP_##u,
-#include "tbl/steps.h"
-#undef STEP
-};
-
-/*--------------------------------------------------------------------
- * Workspace structure for quick memory allocation.
- */
-
-struct ws {
-	unsigned		magic;
-#define WS_MAGIC		0x35fac554
-	unsigned		overflow;	/* workspace overflowed */
-	const char		*id;		/* identity */
-	char			*s;		/* (S)tart of buffer */
-	char			*f;		/* (F)ree pointer */
-	char			*r;		/* (R)eserved length */
-	char			*e;		/* (E)nd of buffer */
-};
-
-/*--------------------------------------------------------------------
- * HTTP Request/Response/Header handling structure.
- */
-
-enum httpwhence {
-	HTTP_Rx	 = 1,
-	HTTP_Tx  = 2,
-	HTTP_Obj = 3
-};
-
-/* NB: remember to update http_Copy() if you add fields */
-struct http {
-	unsigned		magic;
-#define HTTP_MAGIC		0x6428b5c9
-
-	enum httpwhence		logtag;
-
-	struct ws		*ws;
-	txt			*hd;
-	unsigned char		*hdf;
-#define HDF_FILTER		(1 << 0)	/* Filtered by Connection */
-	uint16_t		shd;		/* Size of hd space */
-	uint16_t		nhd;		/* Next free hd */
-	uint16_t		status;
-	uint8_t			protover;
-	uint8_t			conds;		/* If-* headers present */
-};
-
-/*--------------------------------------------------------------------
- * HTTP Protocol connection structure
- */
-
-struct http_conn {
-	unsigned		magic;
-#define HTTP_CONN_MAGIC		0x3e19edd1
-
-	int			fd;
-	unsigned		vsl_id;
-	unsigned		maxbytes;
-	unsigned		maxhdr;
-	struct ws		*ws;
-	txt			rxbuf;
-	txt			pipeline;
-};
-
-/*--------------------------------------------------------------------*/
-
-struct acct {
-	double			first;
-#define ACCT(foo)	uint64_t	foo;
-#include "tbl/acct_fields.h"
-#undef ACCT
-};
-
-/*--------------------------------------------------------------------*/
-
-#define L0(t, n)
-#define L1(t, n)		t n;
-#define VSC_F(n, t, l, f, e,d)	L##l(t, n)
-#define VSC_DO_MAIN
-struct dstat {
-#include "tbl/vsc_fields.h"
-};
-#undef VSC_F
-#undef VSC_DO_MAIN
-#undef L0
-#undef L1
-
-/* Fetch processors --------------------------------------------------*/
-
-typedef void vfp_begin_f(struct worker *, size_t );
-typedef int vfp_bytes_f(struct worker *, struct http_conn *, ssize_t);
-typedef int vfp_end_f(struct worker *);
-
-struct vfp {
-	vfp_begin_f	*begin;
-	vfp_bytes_f	*bytes;
-	vfp_end_f	*end;
-};
-
-extern struct vfp vfp_gunzip;
-extern struct vfp vfp_gzip;
-extern struct vfp vfp_testgzip;
-extern struct vfp vfp_esi;
-
-/*--------------------------------------------------------------------*/
-
-struct exp {
-	double			ttl;
-	double			grace;
-	double			keep;
-	double			age;
-	double			entered;
-};
-
-/*--------------------------------------------------------------------*/
-
-struct wrw {
-	int			*wfd;
-	unsigned		werr;	/* valid after WRW_Flush() */
-	struct iovec		*iov;
-	unsigned		siov;
-	unsigned		niov;
-	ssize_t			liov;
-	ssize_t			cliov;
-	unsigned		ciov;	/* Chunked header marker */
-};
-
-/*--------------------------------------------------------------------*/
-
-struct stream_ctx {
-	unsigned		magic;
-#define STREAM_CTX_MAGIC	0x8213728b
-
-	struct vgz		*vgz;
-	void			*obuf;
-	ssize_t			obuf_len;
-	ssize_t			obuf_ptr;
-
-	/* Next byte we will take from storage */
-	ssize_t			stream_next;
-
-	/* First byte of storage if we free it as we go (pass) */
-	ssize_t			stream_front;
-};
-
-/*--------------------------------------------------------------------*/
-
-struct wrk_accept {
-	unsigned		magic;
-#define WRK_ACCEPT_MAGIC	0x8c4b4d59
-
-	/* Accept stuff */
-	struct sockaddr_storage	acceptaddr;
-	socklen_t		acceptaddrlen;
-	int			acceptsock;
-	struct listen_sock	*acceptlsock;
-};
-
-/*--------------------------------------------------------------------*/
-
-struct worker {
-	unsigned		magic;
-#define WORKER_MAGIC		0x6391adcf
-	struct pool		*pool;
-	struct objhead		*nobjhead;
-	struct objcore		*nobjcore;
-	struct waitinglist	*nwaitinglist;
-	struct busyobj		*nbusyobj;
-	void			*nhashpriv;
-	struct dstat		stats;
-
-	/* Pool stuff */
-	double			lastused;
-
-	struct wrw		wrw;
-
-	pthread_cond_t		cond;
-
-	VTAILQ_ENTRY(worker)	list;
-	struct sess		*sp;
-
-	struct VCL_conf		*vcl;
-
-	uint32_t		*wlb, *wlp, *wle;
-	unsigned		wlr;
-
-	/* Lookup stuff */
-	struct SHA256Context	*sha256ctx;
-
-	struct http_conn	htc[1];
-	struct ws		ws[1];
-	struct http		*bereq;
-	struct http		*beresp;
-	struct http		*resp;
-
-	struct exp		exp;
-
-	/* This is only here so VRT can find it */
-	const char		*storage_hint;
-
-	/* Fetch stuff */
-	struct vbc		*vbc;
-	struct object		*fetch_obj;
-	enum body_status	body_status;
-	struct vfp		*vfp;
-	struct vgz		*vgz_rx;
-	struct vef_priv		*vef_priv;
-	unsigned		fetch_failed;
-	unsigned		do_stream;
-	unsigned		do_esi;
-	unsigned		do_gzip;
-	unsigned		is_gzip;
-	unsigned		do_gunzip;
-	unsigned		is_gunzip;
-	unsigned		do_close;
-	char			*h_content_length;
-
-	/* Stream state */
-	struct stream_ctx	*sctx;
-
-	/* ESI stuff */
-	struct vep_state	*vep;
-	int			gzip_resp;
-	ssize_t			l_crc;
-	uint32_t		crc;
-
-	/* Timeouts */
-	double			connect_timeout;
-	double			first_byte_timeout;
-	double			between_bytes_timeout;
-
-	/* Delivery mode */
-	unsigned		res_mode;
-#define RES_LEN			(1<<1)
-#define RES_EOF			(1<<2)
-#define RES_CHUNKED		(1<<3)
-#define RES_ESI			(1<<4)
-#define RES_ESI_CHILD		(1<<5)
-#define RES_GUNZIP		(1<<6)
-
-	/* Temporary accounting */
-	struct acct		acct_tmp;
-};
-
-/* LRU ---------------------------------------------------------------*/
-
-struct lru {
-	unsigned		magic;
-#define LRU_MAGIC		0x3fec7bb0
-	VTAILQ_HEAD(,objcore)	lru_head;
-	struct lock		mtx;
-};
-
-/* Storage -----------------------------------------------------------*/
-
-struct storage {
-	unsigned		magic;
-#define STORAGE_MAGIC		0x1a4e51c0
-
-#ifdef SENDFILE_WORKS
-	int			fd;
-	off_t			where;
-#endif
-
-	VTAILQ_ENTRY(storage)	list;
-	struct stevedore	*stevedore;
-	void			*priv;
-
-	unsigned char		*ptr;
-	unsigned		len;
-	unsigned		space;
-};
-
-/* Object core structure ---------------------------------------------
- * Objects have sideways references in the binary heap and the LRU list
- * and we want to avoid paging in a lot of objects just to move them up
- * or down the binheap or to move a unrelated object on the LRU list.
- * To avoid this we use a proxy object, objcore, to hold the relevant
- * housekeeping fields parts of an object.
- */
-
-typedef struct object *getobj_f(struct worker *wrk, struct objcore *oc);
-typedef void updatemeta_f(struct objcore *oc);
-typedef void freeobj_f(struct objcore *oc);
-typedef struct lru *getlru_f(const struct objcore *oc);
-
-struct objcore_methods {
-	getobj_f	*getobj;
-	updatemeta_f	*updatemeta;
-	freeobj_f	*freeobj;
-	getlru_f	*getlru;
-};
-
-struct objcore {
-	unsigned		magic;
-#define OBJCORE_MAGIC		0x4d301302
-	unsigned		refcnt;
-	struct objcore_methods	*methods;
-	void			*priv;
-	unsigned		priv2;
-	struct objhead		*objhead;
-	struct busyobj		*busyobj;
-	double			timer_when;
-	unsigned		flags;
-#define OC_F_BUSY		(1<<1)
-#define OC_F_PASS		(1<<2)
-#define OC_F_LRUDONTMOVE	(1<<4)
-#define OC_F_PRIV		(1<<5)		/* Stevedore private flag */
-#define OC_F_LURK		(3<<6)		/* Ban-lurker-color */
-	unsigned		timer_idx;
-	VTAILQ_ENTRY(objcore)	list;
-	VTAILQ_ENTRY(objcore)	lru_list;
-	VTAILQ_ENTRY(objcore)	ban_list;
-	struct ban		*ban;
-};
-
-static inline struct object *
-oc_getobj(struct worker *wrk, struct objcore *oc)
-{
-
-	CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
-	AZ(oc->flags & OC_F_BUSY);
-	AN(oc->methods);
-	AN(oc->methods->getobj);
-	return (oc->methods->getobj(wrk, oc));
-}
-
-static inline void
-oc_updatemeta(struct objcore *oc)
-{
-
-	CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
-	AN(oc->methods);
-	if (oc->methods->updatemeta != NULL)
-		oc->methods->updatemeta(oc);
-}
-
-static inline void
-oc_freeobj(struct objcore *oc)
-{
-
-	CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
-	AN(oc->methods);
-	AN(oc->methods->freeobj);
-	oc->methods->freeobj(oc);
-}
-
-static inline struct lru *
-oc_getlru(const struct objcore *oc)
-{
-
-	CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
-	AN(oc->methods);
-	AN(oc->methods->getlru);
-	return (oc->methods->getlru(oc));
-}
-
-/* Busy Object structure ---------------------------------------------*/
-
-struct busyobj {
-	unsigned		magic;
-#define BUSYOBJ_MAGIC		0x23b95567
-	uint8_t			*vary;
-};
-
-/* Object structure --------------------------------------------------*/
-
-VTAILQ_HEAD(storagehead, storage);
-
-struct object {
-	unsigned		magic;
-#define OBJECT_MAGIC		0x32851d42
-	unsigned		xid;
-	struct storage		*objstore;
-	struct objcore		*objcore;
-
-	struct ws		ws_o[1];
-
-	uint8_t			*vary;
-	unsigned		hits;
-	uint16_t		response;
-
-	/* XXX: make bitmap */
-	uint8_t			gziped;
-	/* Bit positions in the gzip stream */
-	ssize_t			gzip_start;
-	ssize_t			gzip_last;
-	ssize_t			gzip_stop;
-
-	ssize_t			len;
-
-	struct exp		exp;
-
-	double			last_modified;
-	double			last_lru;
-
-	struct http		*http;
-
-	struct storagehead	store;
-
-	struct storage		*esidata;
-
-	double			last_use;
-
-};
-
-/* -------------------------------------------------------------------*/
-
-struct sess {
-	unsigned		magic;
-#define SESS_MAGIC		0x2c2f9c5a
-	int			fd;
-	unsigned		vsl_id;
-	unsigned		xid;
-
-	int			restarts;
-	int			esi_level;
-	int			disable_esi;
-
-	uint8_t			hash_ignore_busy;
-	uint8_t			hash_always_miss;
-
-	struct worker		*wrk;
-
-	socklen_t		sockaddrlen;
-	socklen_t		mysockaddrlen;
-	struct sockaddr_storage	sockaddr;
-	struct sockaddr_storage	mysockaddr;
-	struct listen_sock	*mylsock;
-
-	/* formatted ascii client address */
-	char			*addr;
-	char			*port;
-	char			*client_identity;
-
-	/* HTTP request */
-	const char		*doclose;
-	struct http		*http;
-	struct http		*http0;
-
-	struct ws		ws[1];
-	char			*ws_ses;	/* WS above session data */
-	char			*ws_req;	/* WS above request data */
-
-	unsigned char		digest[DIGEST_LEN];
-
-	/* Built Vary string */
-	uint8_t			*vary_b;
-	uint8_t			*vary_l;
-	uint8_t			*vary_e;
-
-	struct http_conn	htc[1];
-
-	/* Timestamps, all on TIM_real() timescale */
-	double			t_open;
-	double			t_req;
-	double			t_resp;
-	double			t_end;
-
-	/* Acceptable grace period */
-	struct exp		exp;
-
-	enum step		step;
-	unsigned		cur_method;
-	unsigned		handling;
-	unsigned char		sendbody;
-	unsigned char		wantbody;
-	uint16_t		err_code;
-	const char		*err_reason;
-
-	VTAILQ_ENTRY(sess)	list;
-
-	struct director		*director;
-	struct object		*obj;
-	struct objcore		*objcore;
-	struct VCL_conf		*vcl;
-
-	/* The busy objhead we sleep on */
-	struct objhead		*hash_objhead;
-
-	/* Various internal stuff */
-	struct sessmem		*mem;
-
-	VTAILQ_ENTRY(sess)	poollist;
-	uint64_t		req_bodybytes;
-	struct acct		acct_ses;
-
-#if defined(HAVE_EPOLL_CTL)
-	struct epoll_event ev;
-#endif
-};
-
-/* Prototypes etc ----------------------------------------------------*/
-
-/* cache_acceptor.c */
-void VCA_Prep(struct sess *sp);
-void VCA_Init(void);
-void VCA_Shutdown(void);
-int VCA_Accept(struct listen_sock *ls, struct wrk_accept *wa);
-void VCA_SetupSess(struct worker *w);
-void VCA_FailSess(struct worker *w);
-
-/* cache_backend.c */
-void VBE_UseHealth(const struct director *vdi);
-
-struct vbc *VDI_GetFd(const struct director *, struct sess *sp);
-int VDI_Healthy(const struct director *, const struct sess *sp);
-void VDI_CloseFd(struct worker *wrk);
-void VDI_RecycleFd(struct worker *wrk);
-void VDI_AddHostHeader(const struct sess *sp);
-void VBE_Poll(void);
-
-/* cache_backend_cfg.c */
-void VBE_Init(void);
-struct backend *VBE_AddBackend(struct cli *cli, const struct vrt_backend *vb);
-
-/* cache_backend_poll.c */
-void VBP_Init(void);
-
-/* cache_ban.c */
-struct ban *BAN_New(void);
-int BAN_AddTest(struct cli *, struct ban *, const char *, const char *,
-    const char *);
-void BAN_Free(struct ban *b);
-void BAN_Insert(struct ban *b);
-void BAN_Init(void);
-void BAN_NewObjCore(struct objcore *oc);
-void BAN_DestroyObj(struct objcore *oc);
-int BAN_CheckObject(struct object *o, const struct sess *sp);
-void BAN_Reload(const uint8_t *ban, unsigned len);
-struct ban *BAN_TailRef(void);
-void BAN_Compile(void);
-struct ban *BAN_RefBan(struct objcore *oc, double t0, const struct ban *tail);
-void BAN_TailDeref(struct ban **ban);
-double BAN_Time(const struct ban *ban);
-
-/* cache_center.c [CNT] */
-void CNT_Session(struct sess *sp);
-void CNT_Init(void);
-
-/* cache_cli.c [CLI] */
-void CLI_Init(void);
-void CLI_Run(void);
-void CLI_AddFuncs(struct cli_proto *p);
-extern pthread_t cli_thread;
-#define ASSERT_CLI() do {assert(pthread_self() == cli_thread);} while (0)
-
-/* cache_expiry.c */
-void EXP_Clr(struct exp *e);
-double EXP_Get_ttl(const struct exp *e);
-double EXP_Get_grace(const struct exp *e);
-double EXP_Get_keep(const struct exp *e);
-void EXP_Set_ttl(struct exp *e, double v);
-void EXP_Set_grace(struct exp *e, double v);
-void EXP_Set_keep(struct exp *e, double v);
-
-double EXP_Ttl(const struct sess *, const struct object*);
-double EXP_Grace(const struct sess *, const struct object*);
-void EXP_Insert(struct object *o);
-void EXP_Inject(struct objcore *oc, struct lru *lru, double when);
-void EXP_Init(void);
-void EXP_Rearm(const struct object *o);
-int EXP_Touch(struct objcore *oc);
-int EXP_NukeOne(struct worker *w, struct lru *lru);
-
-/* cache_fetch.c */
-struct storage *FetchStorage(struct worker *w, ssize_t sz);
-int FetchError(struct worker *w, const char *error);
-int FetchError2(struct worker *w, const char *error, const char *more);
-int FetchHdr(struct sess *sp);
-int FetchBody(struct worker *w, struct object *obj);
-int FetchReqBody(struct sess *sp);
-void Fetch_Init(void);
-
-/* cache_gzip.c */
-struct vgz;
-
-enum vgz_flag { VGZ_NORMAL, VGZ_ALIGN, VGZ_RESET, VGZ_FINISH };
-struct vgz *VGZ_NewUngzip(struct worker *wrk, const char *id);
-struct vgz *VGZ_NewGzip(struct worker *wrk, const char *id);
-void VGZ_Ibuf(struct vgz *, const void *, ssize_t len);
-int VGZ_IbufEmpty(const struct vgz *vg);
-void VGZ_Obuf(struct vgz *, void *, ssize_t len);
-int VGZ_ObufFull(const struct vgz *vg);
-int VGZ_ObufStorage(struct worker *w, struct vgz *vg);
-int VGZ_Gzip(struct vgz *, const void **, size_t *len, enum vgz_flag);
-int VGZ_Gunzip(struct vgz *, const void **, size_t *len);
-int VGZ_Destroy(struct vgz **, int vsl_id);
-void VGZ_UpdateObj(const struct vgz*, struct object *);
-int VGZ_WrwGunzip(struct worker *w, struct vgz *, const void *ibuf,
-    ssize_t ibufl, char *obuf, ssize_t obufl, ssize_t *obufp);
-
-/* Return values */
-#define VGZ_ERROR	-1
-#define VGZ_OK		0
-#define VGZ_END		1
-#define VGZ_STUCK	2
-
-/* cache_http.c */
-unsigned HTTP_estimate(unsigned nhttp);
-void HTTP_Copy(struct http *to, const struct http * const fm);
-struct http *HTTP_create(void *p, uint16_t nhttp);
-const char *http_StatusMessage(unsigned);
-unsigned http_EstimateWS(const struct http *fm, unsigned how, uint16_t *nhd);
-void HTTP_Init(void);
-void http_ClrHeader(struct http *to);
-unsigned http_Write(struct worker *w, unsigned vsl_id, const struct http *hp,
-    int resp);
-void http_CopyResp(struct http *to, const struct http *fm);
-void http_SetResp(struct http *to, const char *proto, uint16_t status,
-    const char *response);
-void http_FilterFields(struct worker *w, unsigned vsl_id, struct http *to,
-    const struct http *fm, unsigned how);
-void http_FilterHeader(const struct sess *sp, unsigned how);
-void http_PutProtocol(struct worker *w, unsigned vsl_id, const struct http *to,
-    const char *protocol);
-void http_PutStatus(struct http *to, uint16_t status);
-void http_PutResponse(struct worker *w, unsigned vsl_id, const struct http *to,
-    const char *response);
-void http_PrintfHeader(struct worker *w, unsigned vsl_id, struct http *to,
-    const char *fmt, ...);
-void http_SetHeader(struct worker *w, unsigned vsl_id, struct http *to,
-    const char *hdr);
-void http_SetH(const struct http *to, unsigned n, const char *fm);
-void http_ForceGet(const struct http *to);
-void http_Setup(struct http *ht, struct ws *ws);
-int http_GetHdr(const struct http *hp, const char *hdr, char **ptr);
-int http_GetHdrData(const struct http *hp, const char *hdr,
-    const char *field, char **ptr);
-int http_GetHdrField(const struct http *hp, const char *hdr,
-    const char *field, char **ptr);
-double http_GetHdrQ(const struct http *hp, const char *hdr, const char *field);
-uint16_t http_GetStatus(const struct http *hp);
-const char *http_GetReq(const struct http *hp);
-int http_HdrIs(const struct http *hp, const char *hdr, const char *val);
-uint16_t http_DissectRequest(struct sess *sp);
-uint16_t http_DissectResponse(struct worker *w, const struct http_conn *htc,
-    struct http *sp);
-const char *http_DoConnection(const struct http *hp);
-void http_CopyHome(struct worker *w, unsigned vsl_id, const struct http *hp);
-void http_Unset(struct http *hp, const char *hdr);
-void http_CollectHdr(struct http *hp, const char *hdr);
-
-/* cache_httpconn.c */
-void HTC_Init(struct http_conn *htc, struct ws *ws, int fd, unsigned vsl_id,
-    unsigned maxbytes, unsigned maxhdr);
-int HTC_Reinit(struct http_conn *htc);
-int HTC_Rx(struct http_conn *htc);
-ssize_t HTC_Read(struct worker *w, struct http_conn *htc, void *d, size_t len);
-int HTC_Complete(struct http_conn *htc);
-
-#define HTTPH(a, b, c, d, e, f, g) extern char b[];
-#include "tbl/http_headers.h"
-#undef HTTPH
-
-/* cache_main.c */
-void THR_SetName(const char *name);
-const char* THR_GetName(void);
-void THR_SetSession(const struct sess *sp);
-const struct sess * THR_GetSession(void);
-
-/* cache_lck.c */
-
-/* Internal functions, call only through macros below */
-void Lck__Lock(struct lock *lck, const char *p, const char *f, int l);
-void Lck__Unlock(struct lock *lck, const char *p, const char *f, int l);
-int Lck__Trylock(struct lock *lck, const char *p, const char *f, int l);
-void Lck__New(struct lock *lck, struct VSC_C_lck *, const char *);
-void Lck__Assert(const struct lock *lck, int held);
-
-/* public interface: */
-void LCK_Init(void);
-void Lck_Delete(struct lock *lck);
-int Lck_CondWait(pthread_cond_t *cond, struct lock *lck, struct timespec *ts);
-
-#define Lck_New(a, b) Lck__New(a, b, #b)
-#define Lck_Lock(a) Lck__Lock(a, __func__, __FILE__, __LINE__)
-#define Lck_Unlock(a) Lck__Unlock(a, __func__, __FILE__, __LINE__)
-#define Lck_Trylock(a) Lck__Trylock(a, __func__, __FILE__, __LINE__)
-#define Lck_AssertHeld(a) Lck__Assert(a, 1)
-
-#define LOCK(nam) extern struct VSC_C_lck *lck_##nam;
-#include "tbl/locks.h"
-#undef LOCK
-
-/* cache_panic.c */
-void PAN_Init(void);
-
-/* cache_pipe.c */
-void PipeSession(struct sess *sp);
-
-/* cache_pool.c */
-void Pool_Init(void);
-void Pool_Work_Thread(void *priv, struct worker *w);
-void Pool_Wait(struct sess *sp);
-int Pool_Schedule(struct pool *pp, struct sess *sp);
-
-#define WRW_IsReleased(w)	((w)->wrw.wfd == NULL)
-int WRW_Error(const struct worker *w);
-void WRW_Chunked(struct worker *w);
-void WRW_EndChunk(struct worker *w);
-void WRW_Reserve(struct worker *w, int *fd);
-unsigned WRW_Flush(struct worker *w);
-unsigned WRW_FlushRelease(struct worker *w);
-unsigned WRW_Write(struct worker *w, const void *ptr, int len);
-unsigned WRW_WriteH(struct worker *w, const txt *hh, const char *suf);
-#ifdef SENDFILE_WORKS
-void WRW_Sendfile(struct worker *w, int fd, off_t off, unsigned len);
-#endif  /* SENDFILE_WORKS */
-
-/* cache_session.c [SES] */
-struct sess *SES_New(struct worker *wrk, struct sesspool *pp);
-struct sess *SES_Alloc(void);
-void SES_Close(struct sess *sp, const char *reason);
-void SES_Delete(struct sess *sp, const char *reason);
-void SES_Charge(struct sess *sp);
-struct sesspool *SES_NewPool(struct pool *pp);
-void SES_DeletePool(struct sesspool *sp, struct worker *wrk);
-int SES_Schedule(struct sess *sp);
-
-
-/* cache_shmlog.c */
-void VSL_Init(void);
-void *VSM_Alloc(unsigned size, const char *class, const char *type,
-    const char *ident);
-void VSM_Free(const void *ptr);
-#ifdef VSL_ENDMARKER
-void VSL(enum VSL_tag_e tag, int id, const char *fmt, ...);
-void WSLR(struct worker *w, enum VSL_tag_e tag, int id, txt t);
-void WSL(struct worker *w, enum VSL_tag_e tag, int id, const char *fmt, ...);
-void WSLB(struct worker *w, enum VSL_tag_e tag, const char *fmt, ...);
-
-void WSL_Flush(struct worker *w, int overflow);
-
-#define DSL(flag, tag, id, ...)					\
-	do {							\
-		if (cache_param->diag_bitmap & (flag))		\
-			VSL((tag), (id), __VA_ARGS__);		\
-	} while (0)
-
-#define WSP(sess, tag, ...)					\
-	WSL((sess)->wrk, tag, (sess)->vsl_id, __VA_ARGS__)
-
-#define WSPR(sess, tag, txt)					\
-	WSLR((sess)->wrk, tag, (sess)->vsl_id, txt)
-
-#define INCOMPL() do {							\
-	VSL(SLT_Debug, 0, "INCOMPLETE AT: %s(%d)", __func__, __LINE__); \
-	fprintf(stderr,							\
-	    "INCOMPLETE AT: %s(%d)\n",					\
-	    (const char *)__func__, __LINE__);				\
-	abort();							\
-	} while (0)
-#endif
-
-/* cache_response.c */
-void RES_BuildHttp(const struct sess *sp);
-void RES_WriteObj(struct sess *sp);
-void RES_StreamStart(struct sess *sp);
-void RES_StreamEnd(struct sess *sp);
-void RES_StreamPoll(struct worker *);
-
-/* cache_vary.c */
-struct vsb *VRY_Create(const struct sess *sp, const struct http *hp);
-int VRY_Match(struct sess *sp, const uint8_t *vary);
-void VRY_Validate(const uint8_t *vary);
-
-/* cache_vcl.c */
-void VCL_Init(void);
-void VCL_Refresh(struct VCL_conf **vcc);
-void VCL_Rel(struct VCL_conf **vcc);
-void VCL_Poll(void);
-const char *VCL_Return_Name(unsigned method);
-
-#define VCL_MET_MAC(l,u,b) void VCL_##l##_method(struct sess *);
-#include "tbl/vcl_returns.h"
-#undef VCL_MET_MAC
-
-/* cache_vrt.c */
-
-char *VRT_String(struct ws *ws, const char *h, const char *p, va_list ap);
-char *VRT_StringList(char *d, unsigned dl, const char *p, va_list ap);
-
-void ESI_Deliver(struct sess *);
-void ESI_DeliverChild(const struct sess *);
-
-/* cache_vrt_vmod.c */
-void VMOD_Init(void);
-
-/* cache_wrk.c */
-
-void WRK_Init(void);
-int WRK_TrySumStat(struct worker *w);
-void WRK_SumStat(struct worker *w);
-void *WRK_thread(void *priv);
-typedef void *bgthread_t(struct sess *, void *priv);
-void WRK_BgThread(pthread_t *thr, const char *name, bgthread_t *func,
-    void *priv);
-
-/* cache_ws.c */
-
-void WS_Init(struct ws *ws, const char *id, void *space, unsigned len);
-unsigned WS_Reserve(struct ws *ws, unsigned bytes);
-void WS_Release(struct ws *ws, unsigned bytes);
-void WS_ReleaseP(struct ws *ws, char *ptr);
-void WS_Assert(const struct ws *ws);
-void WS_Reset(struct ws *ws, char *p);
-char *WS_Alloc(struct ws *ws, unsigned bytes);
-char *WS_Dup(struct ws *ws, const char *);
-char *WS_Snapshot(struct ws *ws);
-unsigned WS_Free(const struct ws *ws);
-
-/* rfc2616.c */
-void RFC2616_Ttl(const struct sess *sp);
-enum body_status RFC2616_Body(const struct sess *sp);
-unsigned RFC2616_Req_Gzip(const struct sess *sp);
-int RFC2616_Do_Cond(const struct sess *sp);
-
-/* stevedore.c */
-struct object *STV_NewObject(struct sess *sp, const char *hint, unsigned len,
-    struct exp *, uint16_t nhttp);
-struct storage *STV_alloc(struct worker *w, size_t size);
-void STV_trim(struct storage *st, size_t size);
-void STV_free(struct storage *st);
-void STV_open(void);
-void STV_close(void);
-void STV_Freestore(struct object *o);
-
-/* storage_synth.c */
-struct vsb *SMS_Makesynth(struct object *obj);
-void SMS_Finish(struct object *obj);
-void SMS_Init(void);
-
-/* storage_persistent.c */
-void SMP_Init(void);
-void SMP_Ready(void);
-void SMP_NewBan(const uint8_t *ban, unsigned len);
-
-/*
- * A normal pointer difference is signed, but we never want a negative value
- * so this little tool will make sure we don't get that.
- */
-
-static inline unsigned
-pdiff(const void *b, const void *e)
-{
-
-	assert(b <= e);
-	return
-	    ((unsigned)((const unsigned char *)e - (const unsigned char *)b));
-}
-
-static inline void
-Tcheck(const txt t)
-{
-
-	AN(t.b);
-	AN(t.e);
-	assert(t.b <= t.e);
-}
-
-/*
- * unsigned length of a txt
- */
-
-static inline unsigned
-Tlen(const txt t)
-{
-
-	Tcheck(t);
-	return ((unsigned)(t.e - t.b));
-}
-
-static inline void
-Tadd(txt *t, const char *p, int l)
-{
-	Tcheck(*t);
-
-	if (l <= 0) {
-	} if (t->b + l < t->e) {
-		memcpy(t->b, p, l);
-		t->b += l;
-	} else {
-		t->b = t->e;
-	}
-}
-
-static inline void
-AssertObjBusy(const struct object *o)
-{
-	AN(o->objcore);
-	AN (o->objcore->flags & OC_F_BUSY);
-}
-
-static inline void
-AssertObjCorePassOrBusy(const struct objcore *oc)
-{
-	if (oc != NULL)
-		AN (oc->flags & OC_F_BUSY);
-}
diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
new file mode 100644
index 0000000..653fe77
--- /dev/null
+++ b/bin/varnishd/cache/cache.h
@@ -0,0 +1,1039 @@
+/*-
+ * Copyright (c) 2006 Verdens Gang AS
+ * Copyright (c) 2006-2011 Varnish Software AS
+ * All rights reserved.
+ *
+ * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ */
+
+/*
+ * This macro can be used in .h files to isolate bits that the manager
+ * should not (need to) see, such as pthread mutexes etc.
+ */
+#define VARNISH_CACHE_CHILD	1
+
+#include "common/common.h"
+
+#include "vapi/vsc_int.h"
+#include "vapi/vsl_int.h"
+
+#include <sys/socket.h>
+
+#include <pthread.h>
+#ifdef HAVE_PTHREAD_NP_H
+#include <pthread_np.h>
+#endif
+#include <stdarg.h>
+#include <string.h>
+#include <limits.h>
+#include <unistd.h>
+
+#if defined(HAVE_EPOLL_CTL)
+#include <sys/epoll.h>
+#endif
+
+
+#include "common/params.h"
+
+enum body_status {
+#define BODYSTATUS(U,l)	BS_##U,
+#include "tbl/body_status.h"
+#undef BODYSTATUS
+};
+
+static inline const char *
+body_status(enum body_status e)
+{
+	switch(e) {
+#define BODYSTATUS(U,l)	case BS_##U: return (#l);
+#include "tbl/body_status.h"
+#undef BODYSTATUS
+	default:
+		return ("?");
+	}
+}
+
+/*
+ * NB: HDR_STATUS is only used in cache_http.c, everybody else uses the
+ * http->status integer field.
+ */
+
+enum {
+	/* Fields from the first line of HTTP proto */
+	HTTP_HDR_REQ,
+	HTTP_HDR_URL,
+	HTTP_HDR_PROTO,
+	HTTP_HDR_STATUS,
+	HTTP_HDR_RESPONSE,
+	/* HTTP header lines */
+	HTTP_HDR_FIRST,
+};
+
+struct SHA256Context;
+struct VSC_C_lck;
+struct ban;
+struct busyobj;
+struct cli;
+struct cli_proto;
+struct director;
+struct iovec;
+struct objcore;
+struct object;
+struct objhead;
+struct pool;
+struct sess;
+struct sesspool;
+struct vbc;
+struct vef_priv;
+struct vrt_backend;
+struct vsb;
+struct waitinglist;
+struct worker;
+
+#define DIGEST_LEN		32
+
+/* Name of transient storage */
+#define TRANSIENT_STORAGE	"Transient"
+
+/*--------------------------------------------------------------------
+ * Pointer aligment magic
+ */
+
+#define PALGN		(sizeof(void *) - 1)
+#define PAOK(p)		(((uintptr_t)(p) & PALGN) == 0)
+#define PRNDDN(p)	((uintptr_t)(p) & ~PALGN)
+#define PRNDUP(p)	(((uintptr_t)(p) + PALGN) & ~PALGN)
+
+/*--------------------------------------------------------------------*/
+
+typedef struct {
+	char			*b;
+	char			*e;
+} txt;
+
+/*--------------------------------------------------------------------*/
+
+enum step {
+#define STEP(l, u)	STP_##u,
+#include "tbl/steps.h"
+#undef STEP
+};
+
+/*--------------------------------------------------------------------
+ * Workspace structure for quick memory allocation.
+ */
+
+struct ws {
+	unsigned		magic;
+#define WS_MAGIC		0x35fac554
+	unsigned		overflow;	/* workspace overflowed */
+	const char		*id;		/* identity */
+	char			*s;		/* (S)tart of buffer */
+	char			*f;		/* (F)ree pointer */
+	char			*r;		/* (R)eserved length */
+	char			*e;		/* (E)nd of buffer */
+};
+
+/*--------------------------------------------------------------------
+ * HTTP Request/Response/Header handling structure.
+ */
+
+enum httpwhence {
+	HTTP_Rx	 = 1,
+	HTTP_Tx  = 2,
+	HTTP_Obj = 3
+};
+
+/* NB: remember to update http_Copy() if you add fields */
+struct http {
+	unsigned		magic;
+#define HTTP_MAGIC		0x6428b5c9
+
+	enum httpwhence		logtag;
+
+	struct ws		*ws;
+	txt			*hd;
+	unsigned char		*hdf;
+#define HDF_FILTER		(1 << 0)	/* Filtered by Connection */
+	uint16_t		shd;		/* Size of hd space */
+	uint16_t		nhd;		/* Next free hd */
+	uint16_t		status;
+	uint8_t			protover;
+	uint8_t			conds;		/* If-* headers present */
+};
+
+/*--------------------------------------------------------------------
+ * HTTP Protocol connection structure
+ */
+
+struct http_conn {
+	unsigned		magic;
+#define HTTP_CONN_MAGIC		0x3e19edd1
+
+	int			fd;
+	unsigned		vsl_id;
+	unsigned		maxbytes;
+	unsigned		maxhdr;
+	struct ws		*ws;
+	txt			rxbuf;
+	txt			pipeline;
+};
+
+/*--------------------------------------------------------------------*/
+
+struct acct {
+	double			first;
+#define ACCT(foo)	uint64_t	foo;
+#include "tbl/acct_fields.h"
+#undef ACCT
+};
+
+/*--------------------------------------------------------------------*/
+
+#define L0(t, n)
+#define L1(t, n)		t n;
+#define VSC_F(n, t, l, f, e,d)	L##l(t, n)
+#define VSC_DO_MAIN
+struct dstat {
+#include "tbl/vsc_fields.h"
+};
+#undef VSC_F
+#undef VSC_DO_MAIN
+#undef L0
+#undef L1
+
+/* Fetch processors --------------------------------------------------*/
+
+typedef void vfp_begin_f(struct worker *, size_t );
+typedef int vfp_bytes_f(struct worker *, struct http_conn *, ssize_t);
+typedef int vfp_end_f(struct worker *);
+
+struct vfp {
+	vfp_begin_f	*begin;
+	vfp_bytes_f	*bytes;
+	vfp_end_f	*end;
+};
+
+extern struct vfp vfp_gunzip;
+extern struct vfp vfp_gzip;
+extern struct vfp vfp_testgzip;
+extern struct vfp vfp_esi;
+
+/*--------------------------------------------------------------------*/
+
+struct exp {
+	double			ttl;
+	double			grace;
+	double			keep;
+	double			age;
+	double			entered;
+};
+
+/*--------------------------------------------------------------------*/
+
+struct wrw {
+	int			*wfd;
+	unsigned		werr;	/* valid after WRW_Flush() */
+	struct iovec		*iov;
+	unsigned		siov;
+	unsigned		niov;
+	ssize_t			liov;
+	ssize_t			cliov;
+	unsigned		ciov;	/* Chunked header marker */
+};
+
+/*--------------------------------------------------------------------*/
+
+struct stream_ctx {
+	unsigned		magic;
+#define STREAM_CTX_MAGIC	0x8213728b
+
+	struct vgz		*vgz;
+	void			*obuf;
+	ssize_t			obuf_len;
+	ssize_t			obuf_ptr;
+
+	/* Next byte we will take from storage */
+	ssize_t			stream_next;
+
+	/* First byte of storage if we free it as we go (pass) */
+	ssize_t			stream_front;
+};
+
+/*--------------------------------------------------------------------*/
+
+struct wrk_accept {
+	unsigned		magic;
+#define WRK_ACCEPT_MAGIC	0x8c4b4d59
+
+	/* Accept stuff */
+	struct sockaddr_storage	acceptaddr;
+	socklen_t		acceptaddrlen;
+	int			acceptsock;
+	struct listen_sock	*acceptlsock;
+};
+
+/*--------------------------------------------------------------------*/
+
+struct worker {
+	unsigned		magic;
+#define WORKER_MAGIC		0x6391adcf
+	struct pool		*pool;
+	struct objhead		*nobjhead;
+	struct objcore		*nobjcore;
+	struct waitinglist	*nwaitinglist;
+	struct busyobj		*nbusyobj;
+	void			*nhashpriv;
+	struct dstat		stats;
+
+	/* Pool stuff */
+	double			lastused;
+
+	struct wrw		wrw;
+
+	pthread_cond_t		cond;
+
+	VTAILQ_ENTRY(worker)	list;
+	struct sess		*sp;
+
+	struct VCL_conf		*vcl;
+
+	uint32_t		*wlb, *wlp, *wle;
+	unsigned		wlr;
+
+	/* Lookup stuff */
+	struct SHA256Context	*sha256ctx;
+
+	struct http_conn	htc[1];
+	struct ws		ws[1];
+	struct http		*bereq;
+	struct http		*beresp;
+	struct http		*resp;
+
+	struct exp		exp;
+
+	/* This is only here so VRT can find it */
+	const char		*storage_hint;
+
+	/* Fetch stuff */
+	struct vbc		*vbc;
+	struct object		*fetch_obj;
+	enum body_status	body_status;
+	struct vfp		*vfp;
+	struct vgz		*vgz_rx;
+	struct vef_priv		*vef_priv;
+	unsigned		fetch_failed;
+	unsigned		do_stream;
+	unsigned		do_esi;
+	unsigned		do_gzip;
+	unsigned		is_gzip;
+	unsigned		do_gunzip;
+	unsigned		is_gunzip;
+	unsigned		do_close;
+	char			*h_content_length;
+
+	/* Stream state */
+	struct stream_ctx	*sctx;
+
+	/* ESI stuff */
+	struct vep_state	*vep;
+	int			gzip_resp;
+	ssize_t			l_crc;
+	uint32_t		crc;
+
+	/* Timeouts */
+	double			connect_timeout;
+	double			first_byte_timeout;
+	double			between_bytes_timeout;
+
+	/* Delivery mode */
+	unsigned		res_mode;
+#define RES_LEN			(1<<1)
+#define RES_EOF			(1<<2)
+#define RES_CHUNKED		(1<<3)
+#define RES_ESI			(1<<4)
+#define RES_ESI_CHILD		(1<<5)
+#define RES_GUNZIP		(1<<6)
+
+	/* Temporary accounting */
+	struct acct		acct_tmp;
+};
+
+/* LRU ---------------------------------------------------------------*/
+
+struct lru {
+	unsigned		magic;
+#define LRU_MAGIC		0x3fec7bb0
+	VTAILQ_HEAD(,objcore)	lru_head;
+	struct lock		mtx;
+};
+
+/* Storage -----------------------------------------------------------*/
+
+struct storage {
+	unsigned		magic;
+#define STORAGE_MAGIC		0x1a4e51c0
+
+#ifdef SENDFILE_WORKS
+	int			fd;
+	off_t			where;
+#endif
+
+	VTAILQ_ENTRY(storage)	list;
+	struct stevedore	*stevedore;
+	void			*priv;
+
+	unsigned char		*ptr;
+	unsigned		len;
+	unsigned		space;
+};
+
+/* Object core structure ---------------------------------------------
+ * Objects have sideways references in the binary heap and the LRU list
+ * and we want to avoid paging in a lot of objects just to move them up
+ * or down the binheap or to move a unrelated object on the LRU list.
+ * To avoid this we use a proxy object, objcore, to hold the relevant
+ * housekeeping fields parts of an object.
+ */
+
+typedef struct object *getobj_f(struct worker *wrk, struct objcore *oc);
+typedef void updatemeta_f(struct objcore *oc);
+typedef void freeobj_f(struct objcore *oc);
+typedef struct lru *getlru_f(const struct objcore *oc);
+
+struct objcore_methods {
+	getobj_f	*getobj;
+	updatemeta_f	*updatemeta;
+	freeobj_f	*freeobj;
+	getlru_f	*getlru;
+};
+
+struct objcore {
+	unsigned		magic;
+#define OBJCORE_MAGIC		0x4d301302
+	unsigned		refcnt;
+	struct objcore_methods	*methods;
+	void			*priv;
+	unsigned		priv2;
+	struct objhead		*objhead;
+	struct busyobj		*busyobj;
+	double			timer_when;
+	unsigned		flags;
+#define OC_F_BUSY		(1<<1)
+#define OC_F_PASS		(1<<2)
+#define OC_F_LRUDONTMOVE	(1<<4)
+#define OC_F_PRIV		(1<<5)		/* Stevedore private flag */
+#define OC_F_LURK		(3<<6)		/* Ban-lurker-color */
+	unsigned		timer_idx;
+	VTAILQ_ENTRY(objcore)	list;
+	VTAILQ_ENTRY(objcore)	lru_list;
+	VTAILQ_ENTRY(objcore)	ban_list;
+	struct ban		*ban;
+};
+
+static inline struct object *
+oc_getobj(struct worker *wrk, struct objcore *oc)
+{
+
+	CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
+	AZ(oc->flags & OC_F_BUSY);
+	AN(oc->methods);
+	AN(oc->methods->getobj);
+	return (oc->methods->getobj(wrk, oc));
+}
+
+static inline void
+oc_updatemeta(struct objcore *oc)
+{
+
+	CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
+	AN(oc->methods);
+	if (oc->methods->updatemeta != NULL)
+		oc->methods->updatemeta(oc);
+}
+
+static inline void
+oc_freeobj(struct objcore *oc)
+{
+
+	CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
+	AN(oc->methods);
+	AN(oc->methods->freeobj);
+	oc->methods->freeobj(oc);
+}
+
+static inline struct lru *
+oc_getlru(const struct objcore *oc)
+{
+
+	CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
+	AN(oc->methods);
+	AN(oc->methods->getlru);
+	return (oc->methods->getlru(oc));
+}
+
+/* Busy Object structure ---------------------------------------------*/
+
+struct busyobj {
+	unsigned		magic;
+#define BUSYOBJ_MAGIC		0x23b95567
+	uint8_t			*vary;
+};
+
+/* Object structure --------------------------------------------------*/
+
+VTAILQ_HEAD(storagehead, storage);
+
+struct object {
+	unsigned		magic;
+#define OBJECT_MAGIC		0x32851d42
+	unsigned		xid;
+	struct storage		*objstore;
+	struct objcore		*objcore;
+
+	struct ws		ws_o[1];
+
+	uint8_t			*vary;
+	unsigned		hits;
+	uint16_t		response;
+
+	/* XXX: make bitmap */
+	uint8_t			gziped;
+	/* Bit positions in the gzip stream */
+	ssize_t			gzip_start;
+	ssize_t			gzip_last;
+	ssize_t			gzip_stop;
+
+	ssize_t			len;
+
+	struct exp		exp;
+
+	double			last_modified;
+	double			last_lru;
+
+	struct http		*http;
+
+	struct storagehead	store;
+
+	struct storage		*esidata;
+
+	double			last_use;
+
+};
+
+/* -------------------------------------------------------------------*/
+
+struct sess {
+	unsigned		magic;
+#define SESS_MAGIC		0x2c2f9c5a
+	int			fd;
+	unsigned		vsl_id;
+	unsigned		xid;
+
+	int			restarts;
+	int			esi_level;
+	int			disable_esi;
+
+	uint8_t			hash_ignore_busy;
+	uint8_t			hash_always_miss;
+
+	struct worker		*wrk;
+
+	socklen_t		sockaddrlen;
+	socklen_t		mysockaddrlen;
+	struct sockaddr_storage	sockaddr;
+	struct sockaddr_storage	mysockaddr;
+	struct listen_sock	*mylsock;
+
+	/* formatted ascii client address */
+	char			*addr;
+	char			*port;
+	char			*client_identity;
+
+	/* HTTP request */
+	const char		*doclose;
+	struct http		*http;
+	struct http		*http0;
+
+	struct ws		ws[1];
+	char			*ws_ses;	/* WS above session data */
+	char			*ws_req;	/* WS above request data */
+
+	unsigned char		digest[DIGEST_LEN];
+
+	/* Built Vary string */
+	uint8_t			*vary_b;
+	uint8_t			*vary_l;
+	uint8_t			*vary_e;
+
+	struct http_conn	htc[1];
+
+	/* Timestamps, all on TIM_real() timescale */
+	double			t_open;
+	double			t_req;
+	double			t_resp;
+	double			t_end;
+
+	/* Acceptable grace period */
+	struct exp		exp;
+
+	enum step		step;
+	unsigned		cur_method;
+	unsigned		handling;
+	unsigned char		sendbody;
+	unsigned char		wantbody;
+	uint16_t		err_code;
+	const char		*err_reason;
+
+	VTAILQ_ENTRY(sess)	list;
+
+	struct director		*director;
+	struct object		*obj;
+	struct objcore		*objcore;
+	struct VCL_conf		*vcl;
+
+	/* The busy objhead we sleep on */
+	struct objhead		*hash_objhead;
+
+	/* Various internal stuff */
+	struct sessmem		*mem;
+
+	VTAILQ_ENTRY(sess)	poollist;
+	uint64_t		req_bodybytes;
+	struct acct		acct_ses;
+
+#if defined(HAVE_EPOLL_CTL)
+	struct epoll_event ev;
+#endif
+};
+
+/* Prototypes etc ----------------------------------------------------*/
+
+/* cache_acceptor.c */
+void VCA_Prep(struct sess *sp);
+void VCA_Init(void);
+void VCA_Shutdown(void);
+int VCA_Accept(struct listen_sock *ls, struct wrk_accept *wa);
+void VCA_SetupSess(struct worker *w);
+void VCA_FailSess(struct worker *w);
+
+/* cache_backend.c */
+void VBE_UseHealth(const struct director *vdi);
+
+struct vbc *VDI_GetFd(const struct director *, struct sess *sp);
+int VDI_Healthy(const struct director *, const struct sess *sp);
+void VDI_CloseFd(struct worker *wrk);
+void VDI_RecycleFd(struct worker *wrk);
+void VDI_AddHostHeader(const struct sess *sp);
+void VBE_Poll(void);
+
+/* cache_backend_cfg.c */
+void VBE_Init(void);
+struct backend *VBE_AddBackend(struct cli *cli, const struct vrt_backend *vb);
+
+/* cache_backend_poll.c */
+void VBP_Init(void);
+
+/* cache_ban.c */
+struct ban *BAN_New(void);
+int BAN_AddTest(struct cli *, struct ban *, const char *, const char *,
+    const char *);
+void BAN_Free(struct ban *b);
+void BAN_Insert(struct ban *b);
+void BAN_Init(void);
+void BAN_NewObjCore(struct objcore *oc);
+void BAN_DestroyObj(struct objcore *oc);
+int BAN_CheckObject(struct object *o, const struct sess *sp);
+void BAN_Reload(const uint8_t *ban, unsigned len);
+struct ban *BAN_TailRef(void);
+void BAN_Compile(void);
+struct ban *BAN_RefBan(struct objcore *oc, double t0, const struct ban *tail);
+void BAN_TailDeref(struct ban **ban);
+double BAN_Time(const struct ban *ban);
+
+/* cache_center.c [CNT] */
+void CNT_Session(struct sess *sp);
+void CNT_Init(void);
+
+/* cache_cli.c [CLI] */
+void CLI_Init(void);
+void CLI_Run(void);
+void CLI_AddFuncs(struct cli_proto *p);
+extern pthread_t cli_thread;
+#define ASSERT_CLI() do {assert(pthread_self() == cli_thread);} while (0)
+
+/* cache_expiry.c */
+void EXP_Clr(struct exp *e);
+double EXP_Get_ttl(const struct exp *e);
+double EXP_Get_grace(const struct exp *e);
+double EXP_Get_keep(const struct exp *e);
+void EXP_Set_ttl(struct exp *e, double v);
+void EXP_Set_grace(struct exp *e, double v);
+void EXP_Set_keep(struct exp *e, double v);
+
+double EXP_Ttl(const struct sess *, const struct object*);
+double EXP_Grace(const struct sess *, const struct object*);
+void EXP_Insert(struct object *o);
+void EXP_Inject(struct objcore *oc, struct lru *lru, double when);
+void EXP_Init(void);
+void EXP_Rearm(const struct object *o);
+int EXP_Touch(struct objcore *oc);
+int EXP_NukeOne(struct worker *w, struct lru *lru);
+
+/* cache_fetch.c */
+struct storage *FetchStorage(struct worker *w, ssize_t sz);
+int FetchError(struct worker *w, const char *error);
+int FetchError2(struct worker *w, const char *error, const char *more);
+int FetchHdr(struct sess *sp);
+int FetchBody(struct worker *w, struct object *obj);
+int FetchReqBody(struct sess *sp);
+void Fetch_Init(void);
+
+/* cache_gzip.c */
+struct vgz;
+
+enum vgz_flag { VGZ_NORMAL, VGZ_ALIGN, VGZ_RESET, VGZ_FINISH };
+struct vgz *VGZ_NewUngzip(struct worker *wrk, const char *id);
+struct vgz *VGZ_NewGzip(struct worker *wrk, const char *id);
+void VGZ_Ibuf(struct vgz *, const void *, ssize_t len);
+int VGZ_IbufEmpty(const struct vgz *vg);
+void VGZ_Obuf(struct vgz *, void *, ssize_t len);
+int VGZ_ObufFull(const struct vgz *vg);
+int VGZ_ObufStorage(struct worker *w, struct vgz *vg);
+int VGZ_Gzip(struct vgz *, const void **, size_t *len, enum vgz_flag);
+int VGZ_Gunzip(struct vgz *, const void **, size_t *len);
+int VGZ_Destroy(struct vgz **, int vsl_id);
+void VGZ_UpdateObj(const struct vgz*, struct object *);
+int VGZ_WrwGunzip(struct worker *w, struct vgz *, const void *ibuf,
+    ssize_t ibufl, char *obuf, ssize_t obufl, ssize_t *obufp);
+
+/* Return values */
+#define VGZ_ERROR	-1
+#define VGZ_OK		0
+#define VGZ_END		1
+#define VGZ_STUCK	2
+
+/* cache_http.c */
+unsigned HTTP_estimate(unsigned nhttp);
+void HTTP_Copy(struct http *to, const struct http * const fm);
+struct http *HTTP_create(void *p, uint16_t nhttp);
+const char *http_StatusMessage(unsigned);
+unsigned http_EstimateWS(const struct http *fm, unsigned how, uint16_t *nhd);
+void HTTP_Init(void);
+void http_ClrHeader(struct http *to);
+unsigned http_Write(struct worker *w, unsigned vsl_id, const struct http *hp,
+    int resp);
+void http_CopyResp(struct http *to, const struct http *fm);
+void http_SetResp(struct http *to, const char *proto, uint16_t status,
+    const char *response);
+void http_FilterFields(struct worker *w, unsigned vsl_id, struct http *to,
+    const struct http *fm, unsigned how);
+void http_FilterHeader(const struct sess *sp, unsigned how);
+void http_PutProtocol(struct worker *w, unsigned vsl_id, const struct http *to,
+    const char *protocol);
+void http_PutStatus(struct http *to, uint16_t status);
+void http_PutResponse(struct worker *w, unsigned vsl_id, const struct http *to,
+    const char *response);
+void http_PrintfHeader(struct worker *w, unsigned vsl_id, struct http *to,
+    const char *fmt, ...);
+void http_SetHeader(struct worker *w, unsigned vsl_id, struct http *to,
+    const char *hdr);
+void http_SetH(const struct http *to, unsigned n, const char *fm);
+void http_ForceGet(const struct http *to);
+void http_Setup(struct http *ht, struct ws *ws);
+int http_GetHdr(const struct http *hp, const char *hdr, char **ptr);
+int http_GetHdrData(const struct http *hp, const char *hdr,
+    const char *field, char **ptr);
+int http_GetHdrField(const struct http *hp, const char *hdr,
+    const char *field, char **ptr);
+double http_GetHdrQ(const struct http *hp, const char *hdr, const char *field);
+uint16_t http_GetStatus(const struct http *hp);
+const char *http_GetReq(const struct http *hp);
+int http_HdrIs(const struct http *hp, const char *hdr, const char *val);
+uint16_t http_DissectRequest(struct sess *sp);
+uint16_t http_DissectResponse(struct worker *w, const struct http_conn *htc,
+    struct http *sp);
+const char *http_DoConnection(const struct http *hp);
+void http_CopyHome(struct worker *w, unsigned vsl_id, const struct http *hp);
+void http_Unset(struct http *hp, const char *hdr);
+void http_CollectHdr(struct http *hp, const char *hdr);
+
+/* cache_httpconn.c */
+void HTC_Init(struct http_conn *htc, struct ws *ws, int fd, unsigned vsl_id,
+    unsigned maxbytes, unsigned maxhdr);
+int HTC_Reinit(struct http_conn *htc);
+int HTC_Rx(struct http_conn *htc);
+ssize_t HTC_Read(struct worker *w, struct http_conn *htc, void *d, size_t len);
+int HTC_Complete(struct http_conn *htc);
+
+#define HTTPH(a, b, c, d, e, f, g) extern char b[];
+#include "tbl/http_headers.h"
+#undef HTTPH
+
+/* cache_main.c */
+void THR_SetName(const char *name);
+const char* THR_GetName(void);
+void THR_SetSession(const struct sess *sp);
+const struct sess * THR_GetSession(void);
+
+/* cache_lck.c */
+
+/* Internal functions, call only through macros below */
+void Lck__Lock(struct lock *lck, const char *p, const char *f, int l);
+void Lck__Unlock(struct lock *lck, const char *p, const char *f, int l);
+int Lck__Trylock(struct lock *lck, const char *p, const char *f, int l);
+void Lck__New(struct lock *lck, struct VSC_C_lck *, const char *);
+void Lck__Assert(const struct lock *lck, int held);
+
+/* public interface: */
+void LCK_Init(void);
+void Lck_Delete(struct lock *lck);
+int Lck_CondWait(pthread_cond_t *cond, struct lock *lck, struct timespec *ts);
+
+#define Lck_New(a, b) Lck__New(a, b, #b)
+#define Lck_Lock(a) Lck__Lock(a, __func__, __FILE__, __LINE__)
+#define Lck_Unlock(a) Lck__Unlock(a, __func__, __FILE__, __LINE__)
+#define Lck_Trylock(a) Lck__Trylock(a, __func__, __FILE__, __LINE__)
+#define Lck_AssertHeld(a) Lck__Assert(a, 1)
+
+#define LOCK(nam) extern struct VSC_C_lck *lck_##nam;
+#include "tbl/locks.h"
+#undef LOCK
+
+/* cache_panic.c */
+void PAN_Init(void);
+
+/* cache_pipe.c */
+void PipeSession(struct sess *sp);
+
+/* cache_pool.c */
+void Pool_Init(void);
+void Pool_Work_Thread(void *priv, struct worker *w);
+void Pool_Wait(struct sess *sp);
+int Pool_Schedule(struct pool *pp, struct sess *sp);
+
+#define WRW_IsReleased(w)	((w)->wrw.wfd == NULL)
+int WRW_Error(const struct worker *w);
+void WRW_Chunked(struct worker *w);
+void WRW_EndChunk(struct worker *w);
+void WRW_Reserve(struct worker *w, int *fd);
+unsigned WRW_Flush(struct worker *w);
+unsigned WRW_FlushRelease(struct worker *w);
+unsigned WRW_Write(struct worker *w, const void *ptr, int len);
+unsigned WRW_WriteH(struct worker *w, const txt *hh, const char *suf);
+#ifdef SENDFILE_WORKS
+void WRW_Sendfile(struct worker *w, int fd, off_t off, unsigned len);
+#endif  /* SENDFILE_WORKS */
+
+/* cache_session.c [SES] */
+struct sess *SES_New(struct worker *wrk, struct sesspool *pp);
+struct sess *SES_Alloc(void);
+void SES_Close(struct sess *sp, const char *reason);
+void SES_Delete(struct sess *sp, const char *reason);
+void SES_Charge(struct sess *sp);
+struct sesspool *SES_NewPool(struct pool *pp);
+void SES_DeletePool(struct sesspool *sp, struct worker *wrk);
+int SES_Schedule(struct sess *sp);
+
+
+/* cache_shmlog.c */
+void VSL_Init(void);
+void *VSM_Alloc(unsigned size, const char *class, const char *type,
+    const char *ident);
+void VSM_Free(const void *ptr);
+#ifdef VSL_ENDMARKER
+void VSL(enum VSL_tag_e tag, int id, const char *fmt, ...);
+void WSLR(struct worker *w, enum VSL_tag_e tag, int id, txt t);
+void WSL(struct worker *w, enum VSL_tag_e tag, int id, const char *fmt, ...);
+void WSLB(struct worker *w, enum VSL_tag_e tag, const char *fmt, ...);
+
+void WSL_Flush(struct worker *w, int overflow);
+
+#define DSL(flag, tag, id, ...)					\
+	do {							\
+		if (cache_param->diag_bitmap & (flag))		\
+			VSL((tag), (id), __VA_ARGS__);		\
+	} while (0)
+
+#define WSP(sess, tag, ...)					\
+	WSL((sess)->wrk, tag, (sess)->vsl_id, __VA_ARGS__)
+
+#define WSPR(sess, tag, txt)					\
+	WSLR((sess)->wrk, tag, (sess)->vsl_id, txt)
+
+#define INCOMPL() do {							\
+	VSL(SLT_Debug, 0, "INCOMPLETE AT: %s(%d)", __func__, __LINE__); \
+	fprintf(stderr,							\
+	    "INCOMPLETE AT: %s(%d)\n",					\
+	    (const char *)__func__, __LINE__);				\
+	abort();							\
+	} while (0)
+#endif
+
+/* cache_response.c */
+void RES_BuildHttp(const struct sess *sp);
+void RES_WriteObj(struct sess *sp);
+void RES_StreamStart(struct sess *sp);
+void RES_StreamEnd(struct sess *sp);
+void RES_StreamPoll(struct worker *);
+
+/* cache_vary.c */
+struct vsb *VRY_Create(const struct sess *sp, const struct http *hp);
+int VRY_Match(struct sess *sp, const uint8_t *vary);
+void VRY_Validate(const uint8_t *vary);
+
+/* cache_vcl.c */
+void VCL_Init(void);
+void VCL_Refresh(struct VCL_conf **vcc);
+void VCL_Rel(struct VCL_conf **vcc);
+void VCL_Poll(void);
+const char *VCL_Return_Name(unsigned method);
+
+#define VCL_MET_MAC(l,u,b) void VCL_##l##_method(struct sess *);
+#include "tbl/vcl_returns.h"
+#undef VCL_MET_MAC
+
+/* cache_vrt.c */
+
+char *VRT_String(struct ws *ws, const char *h, const char *p, va_list ap);
+char *VRT_StringList(char *d, unsigned dl, const char *p, va_list ap);
+
+void ESI_Deliver(struct sess *);
+void ESI_DeliverChild(const struct sess *);
+
+/* cache_vrt_vmod.c */
+void VMOD_Init(void);
+
+/* cache_wrk.c */
+
+void WRK_Init(void);
+int WRK_TrySumStat(struct worker *w);
+void WRK_SumStat(struct worker *w);
+void *WRK_thread(void *priv);
+typedef void *bgthread_t(struct sess *, void *priv);
+void WRK_BgThread(pthread_t *thr, const char *name, bgthread_t *func,
+    void *priv);
+
+/* cache_ws.c */
+
+void WS_Init(struct ws *ws, const char *id, void *space, unsigned len);
+unsigned WS_Reserve(struct ws *ws, unsigned bytes);
+void WS_Release(struct ws *ws, unsigned bytes);
+void WS_ReleaseP(struct ws *ws, char *ptr);
+void WS_Assert(const struct ws *ws);
+void WS_Reset(struct ws *ws, char *p);
+char *WS_Alloc(struct ws *ws, unsigned bytes);
+char *WS_Dup(struct ws *ws, const char *);
+char *WS_Snapshot(struct ws *ws);
+unsigned WS_Free(const struct ws *ws);
+
+/* rfc2616.c */
+void RFC2616_Ttl(const struct sess *sp);
+enum body_status RFC2616_Body(const struct sess *sp);
+unsigned RFC2616_Req_Gzip(const struct sess *sp);
+int RFC2616_Do_Cond(const struct sess *sp);
+
+/* stevedore.c */
+struct object *STV_NewObject(struct sess *sp, const char *hint, unsigned len,
+    struct exp *, uint16_t nhttp);
+struct storage *STV_alloc(struct worker *w, size_t size);
+void STV_trim(struct storage *st, size_t size);
+void STV_free(struct storage *st);
+void STV_open(void);
+void STV_close(void);
+void STV_Freestore(struct object *o);
+
+/* storage_synth.c */
+struct vsb *SMS_Makesynth(struct object *obj);
+void SMS_Finish(struct object *obj);
+void SMS_Init(void);
+
+/* storage_persistent.c */
+void SMP_Init(void);
+void SMP_Ready(void);
+void SMP_NewBan(const uint8_t *ban, unsigned len);
+
+/*
+ * A normal pointer difference is signed, but we never want a negative value
+ * so this little tool will make sure we don't get that.
+ */
+
+static inline unsigned
+pdiff(const void *b, const void *e)
+{
+
+	assert(b <= e);
+	return
+	    ((unsigned)((const unsigned char *)e - (const unsigned char *)b));
+}
+
+static inline void
+Tcheck(const txt t)
+{
+
+	AN(t.b);
+	AN(t.e);
+	assert(t.b <= t.e);
+}
+
+/*
+ * unsigned length of a txt
+ */
+
+static inline unsigned
+Tlen(const txt t)
+{
+
+	Tcheck(t);
+	return ((unsigned)(t.e - t.b));
+}
+
+static inline void
+Tadd(txt *t, const char *p, int l)
+{
+	Tcheck(*t);
+
+	if (l <= 0) {
+	} if (t->b + l < t->e) {
+		memcpy(t->b, p, l);
+		t->b += l;
+	} else {
+		t->b = t->e;
+	}
+}
+
+static inline void
+AssertObjBusy(const struct object *o)
+{
+	AN(o->objcore);
+	AN (o->objcore->flags & OC_F_BUSY);
+}
+
+static inline void
+AssertObjCorePassOrBusy(const struct objcore *oc)
+{
+	if (oc != NULL)
+		AN (oc->flags & OC_F_BUSY);
+}
diff --git a/bin/varnishd/cache/cache_acceptor.c b/bin/varnishd/cache/cache_acceptor.c
new file mode 100644
index 0000000..8c83121
--- /dev/null
+++ b/bin/varnishd/cache/cache_acceptor.c
@@ -0,0 +1,430 @@
+/*-
+ * Copyright (c) 2006 Verdens Gang AS
+ * Copyright (c) 2006-2011 Varnish Software AS
+ * All rights reserved.
+ *
+ * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ */
+
+#include "config.h"
+
+#include "cache.h"
+#include "common/heritage.h"
+
+#include "vcli.h"
+#include "vcli_priv.h"
+#include "vtcp.h"
+#include "vtim.h"
+
+static pthread_t	VCA_thread;
+static struct timeval	tv_sndtimeo;
+static struct timeval	tv_rcvtimeo;
+
+/*--------------------------------------------------------------------
+ * We want to get out of any kind of trouble-hit TCP connections as fast
+ * as absolutely possible, so we set them LINGER enabled with zero timeout,
+ * so that even if there are outstanding write data on the socket, a close(2)
+ * will return immediately.
+ */
+static const struct linger linger = {
+	.l_onoff	=	0,
+};
+
+static unsigned char	need_sndtimeo, need_rcvtimeo, need_linger, need_test;
+
+static void
+sock_test(int fd)
+{
+	struct linger lin;
+	struct timeval tv;
+	socklen_t l;
+	int i;
+
+	l = sizeof lin;
+	i = getsockopt(fd, SOL_SOCKET, SO_LINGER, &lin, &l);
+	if (i) {
+		VTCP_Assert(i);
+		return;
+	}
+	assert(l == sizeof lin);
+	if (memcmp(&lin, &linger, l))
+		need_linger = 1;
+
+#ifdef SO_SNDTIMEO_WORKS
+	l = sizeof tv;
+	i = getsockopt(fd, SOL_SOCKET, SO_SNDTIMEO, &tv, &l);
+	if (i) {
+		VTCP_Assert(i);
+		return;
+	}
+	assert(l == sizeof tv);
+	if (memcmp(&tv, &tv_sndtimeo, l))
+		need_sndtimeo = 1;
+#else
+	(void)tv;
+	(void)tv_sndtimeo;
+	(void)need_sndtimeo;
+#endif
+
+#ifdef SO_RCVTIMEO_WORKS
+	l = sizeof tv;
+	i = getsockopt(fd, SOL_SOCKET, SO_RCVTIMEO, &tv, &l);
+	if (i) {
+		VTCP_Assert(i);
+		return;
+	}
+	assert(l == sizeof tv);
+	if (memcmp(&tv, &tv_rcvtimeo, l))
+		need_rcvtimeo = 1;
+#else
+	(void)tv;
+	(void)tv_rcvtimeo;
+	(void)need_rcvtimeo;
+#endif
+
+	need_test = 0;
+}
+
+/*--------------------------------------------------------------------
+ * Called once the workerthread gets hold of the session, to do setup
+ * setup overhead, we don't want to bother the acceptor thread with.
+ */
+
+void
+VCA_Prep(struct sess *sp)
+{
+	char addr[VTCP_ADDRBUFSIZE];
+	char port[VTCP_PORTBUFSIZE];
+
+	VTCP_name(&sp->sockaddr, sp->sockaddrlen,
+	    addr, sizeof addr, port, sizeof port);
+	sp->addr = WS_Dup(sp->ws, addr);
+	sp->port = WS_Dup(sp->ws, port);
+	if (cache_param->log_local_addr) {
+		AZ(getsockname(sp->fd, (void*)&sp->mysockaddr, &sp->mysockaddrlen));
+		VTCP_name(&sp->mysockaddr, sp->mysockaddrlen,
+		    addr, sizeof addr, port, sizeof port);
+		WSP(sp, SLT_SessionOpen, "%s %s %s %s",
+		    sp->addr, sp->port, addr, port);
+	} else {
+		WSP(sp, SLT_SessionOpen, "%s %s %s",
+		    sp->addr, sp->port, sp->mylsock->name);
+	}
+	sp->acct_ses.first = sp->t_open;
+	if (need_test)
+		sock_test(sp->fd);
+	if (need_linger)
+		VTCP_Assert(setsockopt(sp->fd, SOL_SOCKET, SO_LINGER,
+		    &linger, sizeof linger));
+#ifdef SO_SNDTIMEO_WORKS
+	if (need_sndtimeo)
+		VTCP_Assert(setsockopt(sp->fd, SOL_SOCKET, SO_SNDTIMEO,
+		    &tv_sndtimeo, sizeof tv_sndtimeo));
+#endif
+#ifdef SO_RCVTIMEO_WORKS
+	if (need_rcvtimeo)
+		VTCP_Assert(setsockopt(sp->fd, SOL_SOCKET, SO_RCVTIMEO,
+		    &tv_rcvtimeo, sizeof tv_rcvtimeo));
+#endif
+}
+
+/*--------------------------------------------------------------------
+ * If accept(2)'ing fails, we pace ourselves to relive any resource
+ * shortage if possible.
+ */
+
+static double vca_pace = 0.0;
+static struct lock pace_mtx;
+
+static void
+vca_pace_check(void)
+{
+	double p;
+
+	if (vca_pace == 0.0)
+		return;
+	Lck_Lock(&pace_mtx);
+	p = vca_pace;
+	Lck_Unlock(&pace_mtx);
+	if (p > 0.0)
+		VTIM_sleep(p);
+}
+
+static void
+vca_pace_bad(void)
+{
+
+	Lck_Lock(&pace_mtx);
+	vca_pace += cache_param->acceptor_sleep_incr;
+	if (vca_pace > cache_param->acceptor_sleep_max)
+		vca_pace = cache_param->acceptor_sleep_max;
+	Lck_Unlock(&pace_mtx);
+}
+
+static void
+vca_pace_good(void)
+{
+
+	if (vca_pace == 0.0)
+		return;
+	Lck_Lock(&pace_mtx);
+	vca_pace *= cache_param->acceptor_sleep_decay;
+	if (vca_pace < cache_param->acceptor_sleep_incr)
+		vca_pace = 0.0;
+	Lck_Unlock(&pace_mtx);
+}
+
+/*--------------------------------------------------------------------
+ * Accept on a listen socket, and handle error returns.
+ */
+
+static int hack_ready;
+
+int
+VCA_Accept(struct listen_sock *ls, struct wrk_accept *wa)
+{
+	int i;
+
+	CHECK_OBJ_NOTNULL(ls, LISTEN_SOCK_MAGIC);
+	vca_pace_check();
+
+	while(!hack_ready)
+		(void)usleep(100*1000);
+
+	wa->acceptaddrlen = sizeof wa->acceptaddr;
+	i = accept(ls->sock, (void*)&wa->acceptaddr, &wa->acceptaddrlen);
+
+	if (i < 0) {
+		switch (errno) {
+		case ECONNABORTED:
+			break;
+		case EMFILE:
+			VSL(SLT_Debug, ls->sock, "Too many open files");
+			vca_pace_bad();
+			break;
+		default:
+			VSL(SLT_Debug, ls->sock, "Accept failed: %s",
+			    strerror(errno));
+			vca_pace_bad();
+			break;
+		}
+	}
+	wa->acceptlsock = ls;
+	wa->acceptsock = i;
+	return (i);
+}
+
+/*--------------------------------------------------------------------
+ * Fail a session
+ *
+ * This happens if we accept the socket, but cannot get a session
+ * structure.
+ *
+ * We consider this a DoS situation (false positive:  Extremely popular
+ * busy objects) and silently close the connection with minimum effort
+ * and fuzz, rather than try to send an intelligent message back.
+ */
+
+void
+VCA_FailSess(struct worker *w)
+{
+	struct wrk_accept *wa;
+
+	CHECK_OBJ_NOTNULL(w, WORKER_MAGIC);
+	CAST_OBJ_NOTNULL(wa, (void*)w->ws->f, WRK_ACCEPT_MAGIC);
+	AZ(w->sp);
+	AZ(close(wa->acceptsock));
+	w->stats.sess_drop++;
+	vca_pace_bad();
+}
+
+/*--------------------------------------------------------------------*/
+
+void
+VCA_SetupSess(struct worker *w)
+{
+	struct sess *sp;
+	struct wrk_accept *wa;
+
+	CHECK_OBJ_NOTNULL(w, WORKER_MAGIC);
+	CAST_OBJ_NOTNULL(wa, (void*)w->ws->f, WRK_ACCEPT_MAGIC);
+	sp = w->sp;
+	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+	sp->fd = wa->acceptsock;
+	sp->vsl_id = wa->acceptsock | VSL_CLIENTMARKER ;
+	wa->acceptsock = -1;
+	sp->t_open = VTIM_real();
+	sp->t_end = sp->t_open;
+	sp->mylsock = wa->acceptlsock;
+	CHECK_OBJ_NOTNULL(sp->mylsock, LISTEN_SOCK_MAGIC);
+	assert(wa->acceptaddrlen <= sp->sockaddrlen);
+	memcpy(&sp->sockaddr, &wa->acceptaddr, wa->acceptaddrlen);
+	sp->sockaddrlen = wa->acceptaddrlen;
+	sp->step = STP_FIRST;
+	vca_pace_good();
+	w->stats.sess_conn++;
+}
+
+/*--------------------------------------------------------------------*/
+
+static void *
+vca_acct(void *arg)
+{
+#ifdef SO_RCVTIMEO_WORKS
+	double sess_timeout = 0;
+#endif
+#ifdef SO_SNDTIMEO_WORKS
+	double send_timeout = 0;
+#endif
+	struct listen_sock *ls;
+	double t0, now;
+
+	THR_SetName("cache-acceptor");
+	(void)arg;
+
+	VTAILQ_FOREACH(ls, &heritage.socks, list) {
+		if (ls->sock < 0)
+			continue;
+		AZ(listen(ls->sock, cache_param->listen_depth));
+		AZ(setsockopt(ls->sock, SOL_SOCKET, SO_LINGER,
+		    &linger, sizeof linger));
+	}
+
+	hack_ready = 1;
+
+	need_test = 1;
+	t0 = VTIM_real();
+	while (1) {
+		(void)sleep(1);
+#ifdef SO_SNDTIMEO_WORKS
+		if (cache_param->idle_send_timeout != send_timeout) {
+			need_test = 1;
+			send_timeout = cache_param->idle_send_timeout;
+			tv_sndtimeo = VTIM_timeval(send_timeout);
+			VTAILQ_FOREACH(ls, &heritage.socks, list) {
+				if (ls->sock < 0)
+					continue;
+				AZ(setsockopt(ls->sock, SOL_SOCKET,
+				    SO_SNDTIMEO,
+				    &tv_sndtimeo, sizeof tv_sndtimeo));
+			}
+		}
+#endif
+#ifdef SO_RCVTIMEO_WORKS
+		if (cache_param->sess_timeout != sess_timeout) {
+			need_test = 1;
+			sess_timeout = cache_param->sess_timeout;
+			tv_rcvtimeo = VTIM_timeval(sess_timeout);
+			VTAILQ_FOREACH(ls, &heritage.socks, list) {
+				if (ls->sock < 0)
+					continue;
+				AZ(setsockopt(ls->sock, SOL_SOCKET,
+				    SO_RCVTIMEO,
+				    &tv_rcvtimeo, sizeof tv_rcvtimeo));
+			}
+		}
+#endif
+		now = VTIM_real();
+		VSC_C_main->uptime = (uint64_t)(now - t0);
+	}
+	NEEDLESS_RETURN(NULL);
+}
+
+
+/*--------------------------------------------------------------------*/
+
+static void
+ccf_start(struct cli *cli, const char * const *av, void *priv)
+{
+
+	(void)cli;
+	(void)av;
+	(void)priv;
+
+	AZ(pthread_create(&VCA_thread, NULL, vca_acct, NULL));
+}
+
+/*--------------------------------------------------------------------*/
+
+static void
+ccf_listen_address(struct cli *cli, const char * const *av, void *priv)
+{
+	struct listen_sock *ls;
+	char h[32], p[32];
+
+	(void)cli;
+	(void)av;
+	(void)priv;
+
+	/*
+	 * This CLI command is primarily used by varnishtest.  Don't
+	 * respond until liste(2) has been called, in order to avoid
+	 * a race where varnishtest::client would attempt to connect(2)
+	 * before listen(2) has been called.
+	 */
+	while(!hack_ready)
+		(void)usleep(100*1000);
+
+	VTAILQ_FOREACH(ls, &heritage.socks, list) {
+		if (ls->sock < 0)
+			continue;
+		VTCP_myname(ls->sock, h, sizeof h, p, sizeof p);
+		VCLI_Out(cli, "%s %s\n", h, p);
+	}
+}
+
+/*--------------------------------------------------------------------*/
+
+static struct cli_proto vca_cmds[] = {
+	{ CLI_SERVER_START,	"i", ccf_start },
+	{ "debug.listen_address",
+	    "debug.listen_address",
+	    "Report the actual listen address\n", 0, 0,
+	    "d", ccf_listen_address, NULL },
+	{ NULL }
+};
+
+void
+VCA_Init(void)
+{
+
+	CLI_AddFuncs(vca_cmds);
+	Lck_New(&pace_mtx, lck_vcapace);
+}
+
+void
+VCA_Shutdown(void)
+{
+	struct listen_sock *ls;
+	int i;
+
+	VTAILQ_FOREACH(ls, &heritage.socks, list) {
+		if (ls->sock < 0)
+			continue;
+		i = ls->sock;
+		ls->sock = -1;
+		(void)close(i);
+	}
+}
diff --git a/bin/varnishd/cache/cache_backend.c b/bin/varnishd/cache/cache_backend.c
new file mode 100644
index 0000000..1f290c3
--- /dev/null
+++ b/bin/varnishd/cache/cache_backend.c
@@ -0,0 +1,517 @@
+/*-
+ * Copyright (c) 2006 Verdens Gang AS
+ * Copyright (c) 2006-2011 Varnish Software AS
+ * All rights reserved.
+ *
+ * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * Handle backend connections and backend request structures.
+ *
+ */
+
+#include "config.h"
+
+#include <poll.h>
+#include <stdlib.h>
+
+#include "cache.h"
+
+#include "cache_backend.h"
+#include "vrt.h"
+#include "vtcp.h"
+
+/*--------------------------------------------------------------------
+ * The "simple" director really isn't, since thats where all the actual
+ * connections happen.  Nontheless, pretend it is simple by sequestering
+ * the directoricity of it under this line.
+ */
+
+struct vdi_simple {
+	unsigned		magic;
+#define VDI_SIMPLE_MAGIC	0x476d25b7
+	struct director		dir;
+	struct backend		*backend;
+	const struct vrt_backend *vrt;
+};
+
+/*--------------------------------------------------------------------
+ * Create default Host: header for backend request
+ */
+void
+VDI_AddHostHeader(const struct sess *sp)
+{
+
+	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+	CHECK_OBJ_NOTNULL(sp->wrk->bereq, HTTP_MAGIC);
+	CHECK_OBJ_NOTNULL(sp->wrk->vbc, VBC_MAGIC);
+	CHECK_OBJ_NOTNULL(sp->wrk->vbc->vdis, VDI_SIMPLE_MAGIC);
+	http_PrintfHeader(sp->wrk, sp->vsl_id, sp->wrk->bereq,
+	    "Host: %s", sp->wrk->vbc->vdis->vrt->hosthdr);
+}
+
+/*--------------------------------------------------------------------*/
+
+/* Private interface from backend_cfg.c */
+void
+VBE_ReleaseConn(struct vbc *vc)
+{
+
+	CHECK_OBJ_NOTNULL(vc, VBC_MAGIC);
+	assert(vc->backend == NULL);
+	assert(vc->fd < 0);
+
+	vc->addr = NULL;
+	vc->addrlen = 0;
+	vc->recycled = 0;
+	Lck_Lock(&VBE_mtx);
+	VSC_C_main->n_vbc--;
+	Lck_Unlock(&VBE_mtx);
+	FREE_OBJ(vc);
+}
+
+#define FIND_TMO(tmx, dst, sp, be)		\
+	do {					\
+		dst = sp->wrk->tmx;		\
+		if (dst == 0.0)			\
+			dst = be->tmx;		\
+		if (dst == 0.0)			\
+			dst = cache_param->tmx;	\
+	} while (0)
+
+/*--------------------------------------------------------------------
+ * Attempt to connect to a given addrinfo entry.
+ *
+ * Must be called with locked backend, but will release the backend
+ * lock during the slow/sleeping stuff, so that other worker threads
+ * can have a go, while we ponder.
+ *
+ */
+
+static int
+vbe_TryConnect(const struct sess *sp, int pf, const struct sockaddr_storage *sa,
+    socklen_t salen, const struct vdi_simple *vs)
+{
+	int s, i, tmo;
+	double tmod;
+
+	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+	CHECK_OBJ_NOTNULL(vs, VDI_SIMPLE_MAGIC);
+
+	s = socket(pf, SOCK_STREAM, 0);
+	if (s < 0)
+		return (s);
+
+	FIND_TMO(connect_timeout, tmod, sp, vs->vrt);
+
+	tmo = (int)(tmod * 1000.0);
+
+	i = VTCP_connect(s, sa, salen, tmo);
+
+	if (i != 0) {
+		AZ(close(s));
+		return (-1);
+	}
+
+	return (s);
+}
+
+/*--------------------------------------------------------------------*/
+
+static void
+bes_conn_try(const struct sess *sp, struct vbc *vc, const struct vdi_simple *vs)
+{
+	int s;
+	struct backend *bp = vs->backend;
+	char abuf1[VTCP_ADDRBUFSIZE];
+	char pbuf1[VTCP_PORTBUFSIZE];
+
+	CHECK_OBJ_NOTNULL(vs, VDI_SIMPLE_MAGIC);
+
+	Lck_Lock(&bp->mtx);
+	bp->refcount++;
+	bp->n_conn++;		/* It mostly works */
+	Lck_Unlock(&bp->mtx);
+
+	s = -1;
+	assert(bp->ipv6 != NULL || bp->ipv4 != NULL);
+
+	/* release lock during stuff that can take a long time */
+
+	if (cache_param->prefer_ipv6 && bp->ipv6 != NULL) {
+		s = vbe_TryConnect(sp, PF_INET6, bp->ipv6, bp->ipv6len, vs);
+		vc->addr = bp->ipv6;
+		vc->addrlen = bp->ipv6len;
+	}
+	if (s == -1 && bp->ipv4 != NULL) {
+		s = vbe_TryConnect(sp, PF_INET, bp->ipv4, bp->ipv4len, vs);
+		vc->addr = bp->ipv4;
+		vc->addrlen = bp->ipv4len;
+	}
+	if (s == -1 && !cache_param->prefer_ipv6 && bp->ipv6 != NULL) {
+		s = vbe_TryConnect(sp, PF_INET6, bp->ipv6, bp->ipv6len, vs);
+		vc->addr = bp->ipv6;
+		vc->addrlen = bp->ipv6len;
+	}
+
+	vc->fd = s;
+	if (s < 0) {
+		Lck_Lock(&bp->mtx);
+		bp->n_conn--;
+		bp->refcount--;		/* Only keep ref on success */
+		Lck_Unlock(&bp->mtx);
+		vc->addr = NULL;
+		vc->addrlen = 0;
+	} else {
+		vc->vsl_id = s | VSL_BACKENDMARKER;
+		VTCP_myname(s, abuf1, sizeof abuf1, pbuf1, sizeof pbuf1);
+		WSL(sp->wrk, SLT_BackendOpen, vc->vsl_id, "%s %s %s ",
+		    vs->backend->display_name, abuf1, pbuf1);
+	}
+
+}
+
+/*--------------------------------------------------------------------
+ * Check that there is still something at the far end of a given socket.
+ * We poll the fd with instant timeout, if there are any events we can't
+ * use it (backends are not allowed to pipeline).
+ */
+
+static int
+vbe_CheckFd(int fd)
+{
+	struct pollfd pfd;
+
+	pfd.fd = fd;
+	pfd.events = POLLIN;
+	pfd.revents = 0;
+	return(poll(&pfd, 1, 0) == 0);
+}
+
+/*--------------------------------------------------------------------
+ * Manage a pool of vbc structures.
+ * XXX: as an experiment, make this caching controled by a parameter
+ * XXX: so we can see if it has any effect.
+ */
+
+static struct vbc *
+vbe_NewConn(void)
+{
+	struct vbc *vc;
+
+	ALLOC_OBJ(vc, VBC_MAGIC);
+	XXXAN(vc);
+	vc->fd = -1;
+	Lck_Lock(&VBE_mtx);
+	VSC_C_main->n_vbc++;
+	Lck_Unlock(&VBE_mtx);
+	return (vc);
+}
+
+/*--------------------------------------------------------------------
+ * It evaluates if a backend is healthy _for_a_specific_object_.
+ * That means that it relies on sp->objcore->objhead. This is mainly for
+ * saint-mode, but also takes backend->healthy into account. If
+ * cache_param->saintmode_threshold is 0, this is basically just a test of
+ * backend->healthy.
+ *
+ * The threshold has to be evaluated _after_ the timeout check, otherwise
+ * items would never time out once the threshold is reached.
+ */
+
+static unsigned int
+vbe_Healthy(const struct vdi_simple *vs, const struct sess *sp)
+{
+	struct trouble *tr;
+	struct trouble *tr2;
+	struct trouble *old;
+	unsigned i = 0, retval;
+	unsigned int threshold;
+	struct backend *backend;
+	uintptr_t target;
+	double now;
+
+	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+	CHECK_OBJ_NOTNULL(vs, VDI_SIMPLE_MAGIC);
+	backend = vs->backend;
+	CHECK_OBJ_NOTNULL(backend, BACKEND_MAGIC);
+
+	if (backend->admin_health == ah_probe && !backend->healthy)
+		return (0);
+
+	if (backend->admin_health == ah_sick)
+		return (0);
+
+	/* VRT/VCC sets threshold to UINT_MAX to mark that it's not
+	 * specified by VCL (thus use param).
+	 */
+	if (vs->vrt->saintmode_threshold == UINT_MAX)
+		threshold = cache_param->saintmode_threshold;
+	else
+		threshold = vs->vrt->saintmode_threshold;
+
+	if (backend->admin_health == ah_healthy)
+		threshold = UINT_MAX;
+
+	/* Saintmode is disabled */
+	if (threshold == 0)
+		return (1);
+
+	if (sp->objcore == NULL)
+		return (1);
+
+	now = sp->t_req;
+	target = (uintptr_t)(sp->objcore->objhead);
+
+	old = NULL;
+	retval = 1;
+	Lck_Lock(&backend->mtx);
+	VTAILQ_FOREACH_SAFE(tr, &backend->troublelist, list, tr2) {
+		CHECK_OBJ_NOTNULL(tr, TROUBLE_MAGIC);
+
+		if (tr->timeout < now) {
+			VTAILQ_REMOVE(&backend->troublelist, tr, list);
+			old = tr;
+			retval = 1;
+			break;
+		}
+
+		if (tr->target == target) {
+			retval = 0;
+			break;
+		}
+
+		/* If the threshold is at 1, a single entry on the list
+		 * will disable the backend. Since 0 is disable, ++i
+		 * instead of i++ to allow this behavior.
+		 */
+		if (++i >= threshold) {
+			retval = 0;
+			break;
+		}
+	}
+	Lck_Unlock(&backend->mtx);
+
+	if (old != NULL)
+		FREE_OBJ(old);
+
+	return (retval);
+}
+
+/*--------------------------------------------------------------------
+ * Get a connection to a particular backend.
+ */
+
+static struct vbc *
+vbe_GetVbe(const struct sess *sp, struct vdi_simple *vs)
+{
+	struct vbc *vc;
+	struct backend *bp;
+
+	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+	CHECK_OBJ_NOTNULL(vs, VDI_SIMPLE_MAGIC);
+	bp = vs->backend;
+	CHECK_OBJ_NOTNULL(bp, BACKEND_MAGIC);
+
+	/* first look for vbc's we can recycle */
+	while (1) {
+		Lck_Lock(&bp->mtx);
+		vc = VTAILQ_FIRST(&bp->connlist);
+		if (vc != NULL) {
+			bp->refcount++;
+			assert(vc->backend == bp);
+			assert(vc->fd >= 0);
+			AN(vc->addr);
+			VTAILQ_REMOVE(&bp->connlist, vc, list);
+		}
+		Lck_Unlock(&bp->mtx);
+		if (vc == NULL)
+			break;
+		if (vbe_CheckFd(vc->fd)) {
+			/* XXX locking of stats */
+			VSC_C_main->backend_reuse += 1;
+			WSP(sp, SLT_Backend, "%d %s %s",
+			    vc->fd, sp->director->vcl_name, bp->display_name);
+			vc->vdis = vs;
+			vc->recycled = 1;
+			return (vc);
+		}
+		VSC_C_main->backend_toolate++;
+		WSL(sp->wrk, SLT_BackendClose, vc->vsl_id, "%s", bp->display_name);
+
+		/* Checkpoint log to flush all info related to this connection
+		   before the OS reuses the FD */
+		WSL_Flush(sp->wrk, 0);
+
+		VTCP_close(&vc->fd);
+		VBE_DropRefConn(bp);
+		vc->backend = NULL;
+		VBE_ReleaseConn(vc);
+	}
+
+	if (!vbe_Healthy(vs, sp)) {
+		VSC_C_main->backend_unhealthy++;
+		return (NULL);
+	}
+
+	if (vs->vrt->max_connections > 0 &&
+	    bp->n_conn >= vs->vrt->max_connections) {
+		VSC_C_main->backend_busy++;
+		return (NULL);
+	}
+
+	vc = vbe_NewConn();
+	assert(vc->fd == -1);
+	AZ(vc->backend);
+	bes_conn_try(sp, vc, vs);
+	if (vc->fd < 0) {
+		VBE_ReleaseConn(vc);
+		VSC_C_main->backend_fail++;
+		return (NULL);
+	}
+	vc->backend = bp;
+	VSC_C_main->backend_conn++;
+	WSP(sp, SLT_Backend, "%d %s %s",
+	    vc->fd, sp->director->vcl_name, bp->display_name);
+	vc->vdis = vs;
+	return (vc);
+}
+
+/*--------------------------------------------------------------------
+ * Returns the backend if and only if the this is a simple director.
+ * XXX: Needs a better name and possibly needs a better general approach.
+ * XXX: This is mainly used by the DNS director to fetch the actual backend
+ * XXX: so it can compare DNS lookups with the actual IP.
+ */
+
+struct backend *
+vdi_get_backend_if_simple(const struct director *d)
+{
+	CHECK_OBJ_NOTNULL(d, DIRECTOR_MAGIC);
+	struct vdi_simple *vs, *vs2;
+
+	vs2 = d->priv;
+	if (vs2->magic != VDI_SIMPLE_MAGIC)
+		return (NULL);
+	CAST_OBJ_NOTNULL(vs, d->priv, VDI_SIMPLE_MAGIC);
+	return (vs->backend);
+}
+
+/*--------------------------------------------------------------------
+ *
+ */
+
+void
+VBE_UseHealth(const struct director *vdi)
+{
+	struct vdi_simple *vs;
+
+	ASSERT_CLI();
+
+	if (strcmp(vdi->name, "simple"))
+		return;
+	CAST_OBJ_NOTNULL(vs, vdi->priv, VDI_SIMPLE_MAGIC);
+	if (vs->vrt->probe == NULL)
+		return;
+	VBP_Use(vs->backend, vs->vrt->probe);
+}
+
+/*--------------------------------------------------------------------
+ *
+ */
+
+static struct vbc * __match_proto__(vdi_getfd_f)
+vdi_simple_getfd(const struct director *d, struct sess *sp)
+{
+	struct vdi_simple *vs;
+	struct vbc *vc;
+
+	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+	CHECK_OBJ_NOTNULL(d, DIRECTOR_MAGIC);
+	CAST_OBJ_NOTNULL(vs, d->priv, VDI_SIMPLE_MAGIC);
+	vc = vbe_GetVbe(sp, vs);
+	if (vc != NULL) {
+		FIND_TMO(first_byte_timeout,
+		    vc->first_byte_timeout, sp, vs->vrt);
+		FIND_TMO(between_bytes_timeout,
+		    vc->between_bytes_timeout, sp, vs->vrt);
+	}
+	return (vc);
+}
+
+static unsigned
+vdi_simple_healthy(const struct director *d, const struct sess *sp)
+{
+	struct vdi_simple *vs;
+
+	CHECK_OBJ_NOTNULL(d, DIRECTOR_MAGIC);
+	CAST_OBJ_NOTNULL(vs, d->priv, VDI_SIMPLE_MAGIC);
+	return (vbe_Healthy(vs, sp));
+}
+
+static void
+vdi_simple_fini(const struct director *d)
+{
+	struct vdi_simple *vs;
+
+	ASSERT_CLI();
+	CHECK_OBJ_NOTNULL(d, DIRECTOR_MAGIC);
+	CAST_OBJ_NOTNULL(vs, d->priv, VDI_SIMPLE_MAGIC);
+
+	if (vs->vrt->probe != NULL)
+		VBP_Remove(vs->backend, vs->vrt->probe);
+	VBE_DropRefVcl(vs->backend);
+	free(vs->dir.vcl_name);
+	vs->dir.magic = 0;
+	FREE_OBJ(vs);
+}
+
+void
+VRT_init_dir_simple(struct cli *cli, struct director **bp, int idx,
+    const void *priv)
+{
+	const struct vrt_backend *t;
+	struct vdi_simple *vs;
+
+	ASSERT_CLI();
+	(void)cli;
+	t = priv;
+
+	ALLOC_OBJ(vs, VDI_SIMPLE_MAGIC);
+	XXXAN(vs);
+	vs->dir.magic = DIRECTOR_MAGIC;
+	vs->dir.priv = vs;
+	vs->dir.name = "simple";
+	REPLACE(vs->dir.vcl_name, t->vcl_name);
+	vs->dir.getfd = vdi_simple_getfd;
+	vs->dir.fini = vdi_simple_fini;
+	vs->dir.healthy = vdi_simple_healthy;
+
+	vs->vrt = t;
+
+	vs->backend = VBE_AddBackend(cli, t);
+	if (vs->vrt->probe != NULL)
+		VBP_Insert(vs->backend, vs->vrt->probe, vs->vrt->hosthdr);
+
+	bp[idx] = &vs->dir;
+}
diff --git a/bin/varnishd/cache/cache_backend.h b/bin/varnishd/cache/cache_backend.h
new file mode 100644
index 0000000..72a1283
--- /dev/null
+++ b/bin/varnishd/cache/cache_backend.h
@@ -0,0 +1,192 @@
+/*-
+ * Copyright (c) 2006 Verdens Gang AS
+ * Copyright (c) 2006-2011 Varnish Software AS
+ * All rights reserved.
+ *
+ * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * This is the central switch-board for backend connections and it is
+ * slightly complicated by a number of optimizations.
+ *
+ * The data structures:
+ *
+ *    A vrt_backend is a definition of a backend in a VCL program.
+ *
+ *    A backend is a TCP destination, possibly multi-homed and it has a
+ *    number of associated properties and statistics.
+ *
+ *    A vbc is an open TCP connection to a backend.
+ *
+ *    A bereq is a memory carrier for handling a HTTP transaction with
+ *    a backend over a vbc.
+ *
+ *    A director is a piece of code that selects which backend to use,
+ *    by whatever method or metric it chooses.
+ *
+ * The relationships:
+ *
+ *    Backends and directors get instantiated when VCL's are loaded,
+ *    and this always happen in the CLI thread.
+ *
+ *    When a VCL tries to instantiate a backend, any existing backend
+ *    with the same identity (== definition in VCL) will be used instead
+ *    so that vbc's can be reused across VCL changes.
+ *
+ *    Directors disapper with the VCL that created them.
+ *
+ *    Backends disappear when their reference count drop to zero.
+ *
+ *    Backends have their host/port name looked up to addrinfo structures
+ *    when they are instantiated, and we just cache that result and cycle
+ *    through the entries (for multihomed backends) on failure only.
+ *    XXX: add cli command to redo lookup.
+ *
+ *    bereq is sort of a step-child here, we just manage the pool of them.
+ *
+ */
+
+struct vbp_target;
+struct vbc;
+struct vrt_backend_probe;
+
+/*--------------------------------------------------------------------
+ * A director is a piece of code which selects one of possibly multiple
+ * backends to use.
+ */
+
+typedef struct vbc *vdi_getfd_f(const struct director *, struct sess *sp);
+typedef void vdi_fini_f(const struct director *);
+typedef unsigned vdi_healthy(const struct director *, const struct sess *sp);
+
+struct director {
+	unsigned		magic;
+#define DIRECTOR_MAGIC		0x3336351d
+	const char		*name;
+	char			*vcl_name;
+	vdi_getfd_f		*getfd;
+	vdi_fini_f		*fini;
+	vdi_healthy		*healthy;
+	void			*priv;
+};
+
+/*--------------------------------------------------------------------
+ * List of objectheads that have recently been rejected by VCL.
+ */
+
+struct trouble {
+	unsigned		magic;
+#define TROUBLE_MAGIC		0x4211ab21
+	uintptr_t		target;
+	double			timeout;
+	VTAILQ_ENTRY(trouble)	list;
+};
+
+/*--------------------------------------------------------------------
+ * An instance of a backend from a VCL program.
+ */
+
+enum admin_health {
+	ah_invalid = 0,
+	ah_healthy,
+	ah_sick,
+	ah_probe
+};
+
+struct backend {
+	unsigned		magic;
+#define BACKEND_MAGIC		0x64c4c7c6
+
+	VTAILQ_ENTRY(backend)	list;
+	int			refcount;
+	struct lock		mtx;
+
+	char			*vcl_name;
+	char			*display_name;
+	char			*ipv4_addr;
+	char			*ipv6_addr;
+	char			*port;
+
+	struct sockaddr_storage	*ipv4;
+	socklen_t		ipv4len;
+	struct sockaddr_storage	*ipv6;
+	socklen_t		ipv6len;
+
+	unsigned		n_conn;
+	VTAILQ_HEAD(, vbc)	connlist;
+
+	struct vbp_target	*probe;
+	unsigned		healthy;
+	enum admin_health	admin_health;
+	VTAILQ_HEAD(, trouble)	troublelist;
+
+	struct VSC_C_vbe	*vsc;
+};
+
+/* -------------------------------------------------------------------*/
+
+/* Backend connection */
+struct vbc {
+	unsigned		magic;
+#define VBC_MAGIC		0x0c5e6592
+	VTAILQ_ENTRY(vbc)	list;
+	struct backend		*backend;
+	struct vdi_simple	*vdis;
+	unsigned		vsl_id;
+	int			fd;
+
+	struct sockaddr_storage	*addr;
+	socklen_t		addrlen;
+
+	uint8_t			recycled;
+
+	/* Timeouts */
+	double			first_byte_timeout;
+	double			between_bytes_timeout;
+};
+
+/* cache_backend.c */
+void VBE_ReleaseConn(struct vbc *vc);
+struct backend *vdi_get_backend_if_simple(const struct director *d);
+
+/* cache_backend_cfg.c */
+extern struct lock VBE_mtx;
+void VBE_DropRefConn(struct backend *);
+void VBE_DropRefVcl(struct backend *);
+void VBE_DropRefLocked(struct backend *b);
+
+/* cache_backend_poll.c */
+void VBP_Insert(struct backend *b, struct vrt_backend_probe const *p, const char *hosthdr);
+void VBP_Remove(struct backend *b, struct vrt_backend_probe const *p);
+void VBP_Use(const struct backend *b, const struct vrt_backend_probe const *p);
+void VBP_Summary(struct cli *cli, const struct vbp_target *vt);
+
+/* Init functions for directors */
+typedef void dir_init_f(struct cli *, struct director **, int , const void*);
+dir_init_f VRT_init_dir_simple;
+dir_init_f VRT_init_dir_dns;
+dir_init_f VRT_init_dir_hash;
+dir_init_f VRT_init_dir_random;
+dir_init_f VRT_init_dir_round_robin;
+dir_init_f VRT_init_dir_fallback;
+dir_init_f VRT_init_dir_client;
diff --git a/bin/varnishd/cache/cache_backend_cfg.c b/bin/varnishd/cache/cache_backend_cfg.c
new file mode 100644
index 0000000..cbb8c85
--- /dev/null
+++ b/bin/varnishd/cache/cache_backend_cfg.c
@@ -0,0 +1,507 @@
+/*-
+ * Copyright (c) 2006 Verdens Gang AS
+ * Copyright (c) 2006-2011 Varnish Software AS
+ * All rights reserved.
+ *
+ * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * Handle configuration of backends from VCL programs.
+ *
+ */
+
+#include "config.h"
+
+#include <ctype.h>
+#include <stdio.h>
+#include <stdlib.h>
+
+#include "cache.h"
+
+#include "cache_backend.h"
+#include "vcli.h"
+#include "vcli_priv.h"
+#include "vrt.h"
+
+struct lock VBE_mtx;
+
+
+/*
+ * The list of backends is not locked, it is only ever accessed from
+ * the CLI thread, so there is no need.
+ */
+static VTAILQ_HEAD(, backend) backends = VTAILQ_HEAD_INITIALIZER(backends);
+
+/*--------------------------------------------------------------------
+ */
+
+static void
+VBE_Nuke(struct backend *b)
+{
+
+	ASSERT_CLI();
+	VTAILQ_REMOVE(&backends, b, list);
+	free(b->ipv4);
+	free(b->ipv4_addr);
+	free(b->ipv6);
+	free(b->ipv6_addr);
+	free(b->port);
+	VSM_Free(b->vsc);
+	FREE_OBJ(b);
+	VSC_C_main->n_backend--;
+}
+
+/*--------------------------------------------------------------------
+ */
+
+void
+VBE_Poll(void)
+{
+	struct backend *b, *b2;
+
+	ASSERT_CLI();
+	VTAILQ_FOREACH_SAFE(b, &backends, list, b2) {
+		assert(
+			b->admin_health == ah_healthy ||
+			b->admin_health == ah_sick ||
+			b->admin_health == ah_probe
+		);
+		if (b->refcount == 0 && b->probe == NULL)
+			VBE_Nuke(b);
+	}
+}
+
+/*--------------------------------------------------------------------
+ * Drop a reference to a backend.
+ * The last reference must come from the watcher in the CLI thread,
+ * as only that thread is allowed to clean up the backend list.
+ */
+
+void
+VBE_DropRefLocked(struct backend *b)
+{
+	int i;
+	struct vbc *vbe, *vbe2;
+
+	CHECK_OBJ_NOTNULL(b, BACKEND_MAGIC);
+	assert(b->refcount > 0);
+
+	i = --b->refcount;
+	Lck_Unlock(&b->mtx);
+	if (i > 0)
+		return;
+
+	ASSERT_CLI();
+	VTAILQ_FOREACH_SAFE(vbe, &b->connlist, list, vbe2) {
+		VTAILQ_REMOVE(&b->connlist, vbe, list);
+		if (vbe->fd >= 0) {
+			AZ(close(vbe->fd));
+			vbe->fd = -1;
+		}
+		vbe->backend = NULL;
+		VBE_ReleaseConn(vbe);
+	}
+	VBE_Nuke(b);
+}
+
+void
+VBE_DropRefVcl(struct backend *b)
+{
+
+	CHECK_OBJ_NOTNULL(b, BACKEND_MAGIC);
+
+	Lck_Lock(&b->mtx);
+	b->vsc->vcls--;
+	VBE_DropRefLocked(b);
+}
+
+void
+VBE_DropRefConn(struct backend *b)
+{
+
+	CHECK_OBJ_NOTNULL(b, BACKEND_MAGIC);
+
+	Lck_Lock(&b->mtx);
+	assert(b->n_conn > 0);
+	b->n_conn--;
+	VBE_DropRefLocked(b);
+}
+
+/*--------------------------------------------------------------------
+ * See lib/libvcl/vcc_backend.c::emit_sockaddr()
+ */
+
+static void
+copy_sockaddr(struct sockaddr_storage **sa, socklen_t *len,
+    const unsigned char *src)
+{
+
+	assert(*src > 0);
+	*sa = calloc(sizeof **sa, 1);
+	XXXAN(*sa);
+	memcpy(*sa, src + 1, *src);
+	*len = *src;
+}
+
+/*--------------------------------------------------------------------
+ * Add a backend/director instance when loading a VCL.
+ * If an existing backend is matched, grab a refcount and return.
+ * Else create a new backend structure with reference initialized to one.
+ */
+
+struct backend *
+VBE_AddBackend(struct cli *cli, const struct vrt_backend *vb)
+{
+	struct backend *b;
+	char buf[128];
+
+	AN(vb->vcl_name);
+	assert(vb->ipv4_sockaddr != NULL || vb->ipv6_sockaddr != NULL);
+	(void)cli;
+	ASSERT_CLI();
+
+	/* Run through the list and see if we already have this backend */
+	VTAILQ_FOREACH(b, &backends, list) {
+		CHECK_OBJ_NOTNULL(b, BACKEND_MAGIC);
+		if (strcmp(b->vcl_name, vb->vcl_name))
+			continue;
+		if (vb->ipv4_sockaddr != NULL && (
+		    b->ipv4len != vb->ipv4_sockaddr[0] ||
+		    memcmp(b->ipv4, vb->ipv4_sockaddr + 1, b->ipv4len)))
+			continue;
+		if (vb->ipv6_sockaddr != NULL && (
+		    b->ipv6len != vb->ipv6_sockaddr[0] ||
+		    memcmp(b->ipv6, vb->ipv6_sockaddr + 1, b->ipv6len)))
+			continue;
+		b->refcount++;
+		b->vsc->vcls++;
+		return (b);
+	}
+
+	/* Create new backend */
+	ALLOC_OBJ(b, BACKEND_MAGIC);
+	XXXAN(b);
+	Lck_New(&b->mtx, lck_backend);
+	b->refcount = 1;
+
+	bprintf(buf, "%s(%s,%s,%s)",
+	    vb->vcl_name,
+	    vb->ipv4_addr == NULL ? "" : vb->ipv4_addr,
+	    vb->ipv6_addr == NULL ? "" : vb->ipv6_addr, vb->port);
+
+	b->vsc = VSM_Alloc(sizeof *b->vsc, VSC_CLASS, VSC_TYPE_VBE, buf);
+	b->vsc->vcls++;
+
+	VTAILQ_INIT(&b->connlist);
+
+	VTAILQ_INIT(&b->troublelist);
+
+	/*
+	 * This backend may live longer than the VCL that instantiated it
+	 * so we cannot simply reference the VCL's copy of things.
+	 */
+	REPLACE(b->vcl_name, vb->vcl_name);
+	REPLACE(b->display_name, buf);
+	REPLACE(b->ipv4_addr, vb->ipv4_addr);
+	REPLACE(b->ipv6_addr, vb->ipv6_addr);
+	REPLACE(b->port, vb->port);
+
+	/*
+	 * Copy over the sockaddrs
+	 */
+	if (vb->ipv4_sockaddr != NULL)
+		copy_sockaddr(&b->ipv4, &b->ipv4len, vb->ipv4_sockaddr);
+	if (vb->ipv6_sockaddr != NULL)
+		copy_sockaddr(&b->ipv6, &b->ipv6len, vb->ipv6_sockaddr);
+
+	assert(b->ipv4 != NULL || b->ipv6 != NULL);
+
+	b->healthy = 1;
+	b->admin_health = ah_probe;
+
+	VTAILQ_INSERT_TAIL(&backends, b, list);
+	VSC_C_main->n_backend++;
+	return (b);
+}
+
+/*--------------------------------------------------------------------*/
+
+void
+VRT_init_dir(struct cli *cli, struct director **dir, const char *name,
+    int idx, const void *priv)
+{
+
+	ASSERT_CLI();
+	if (!strcmp(name, "simple"))
+		VRT_init_dir_simple(cli, dir, idx, priv);
+	else if (!strcmp(name, "hash"))
+		VRT_init_dir_hash(cli, dir, idx, priv);
+	else if (!strcmp(name, "random"))
+		VRT_init_dir_random(cli, dir, idx, priv);
+	else if (!strcmp(name, "dns"))
+		VRT_init_dir_dns(cli, dir, idx, priv);
+	else if (!strcmp(name, "round-robin"))
+		VRT_init_dir_round_robin(cli, dir, idx, priv);
+	else if (!strcmp(name, "fallback"))
+		VRT_init_dir_fallback(cli, dir, idx, priv);
+	else if (!strcmp(name, "client"))
+		VRT_init_dir_client(cli, dir, idx, priv);
+	else
+		INCOMPL();
+}
+
+void
+VRT_fini_dir(struct cli *cli, struct director *b)
+{
+
+	(void)cli;
+	ASSERT_CLI();
+	CHECK_OBJ_NOTNULL(b, DIRECTOR_MAGIC);
+	b->fini(b);
+	b->priv = NULL;
+}
+
+/*---------------------------------------------------------------------
+ * String to admin_health
+ */
+
+static enum admin_health
+vbe_str2adminhealth(const char *wstate)
+{
+
+	if (strcasecmp(wstate, "healthy") == 0)
+		return(ah_healthy);
+	if (strcasecmp(wstate, "sick") == 0)
+		return(ah_sick);
+	if (strcmp(wstate, "auto") == 0)
+		return(ah_probe);
+	return (ah_invalid);
+}
+
+/*---------------------------------------------------------------------
+ * A general function for finding backends and doing things with them.
+ *
+ * Return -1 on match-argument parse errors.
+ *
+ * If the call-back function returns non-zero, the search is terminated
+ * and we relay that return value.
+ *
+ * Otherwise we return the number of matches.
+ */
+
+typedef int bf_func(struct cli *cli, struct backend *b, void *priv);
+
+static int
+backend_find(struct cli *cli, const char *matcher, bf_func *func, void *priv)
+{
+	struct backend *b;
+	const char *s;
+	const char *name_b;
+	ssize_t name_l = 0;
+	const char *ip_b = NULL;
+	ssize_t ip_l = 0;
+	const char *port_b = NULL;
+	ssize_t port_l = 0;
+	int found = 0;
+	int i;
+
+	name_b = matcher;
+	if (matcher != NULL) {
+		s = strchr(matcher,'(');
+
+		if (s != NULL)
+			name_l = s - name_b;
+		else
+			name_l = strlen(name_b);
+
+		if (s != NULL) {
+			s++;
+			while (isspace(*s))
+				s++;
+			ip_b = s;
+			while (*s != '\0' &&
+			    *s != ')' &&
+			    *s != ':' &&
+			    !isspace(*s))
+				s++;
+			ip_l = s - ip_b;
+			while (isspace(*s))
+				s++;
+			if (*s == ':') {
+				s++;
+				while (isspace(*s))
+					s++;
+				port_b = s;
+				while (*s != '\0' && *s != ')' && !isspace(*s))
+					s++;
+				port_l = s - port_b;
+			}
+			while (isspace(*s))
+				s++;
+			if (*s != ')') {
+				VCLI_Out(cli,
+				    "Match string syntax error:"
+				    " ')' not found.");
+				VCLI_SetResult(cli, CLIS_CANT);
+				return (-1);
+			}
+			s++;
+			while (isspace(*s))
+				s++;
+			if (*s != '\0') {
+				VCLI_Out(cli,
+				    "Match string syntax error:"
+				    " junk after ')'");
+				VCLI_SetResult(cli, CLIS_CANT);
+				return (-1);
+			}
+		}
+	}
+	VTAILQ_FOREACH(b, &backends, list) {
+		CHECK_OBJ_NOTNULL(b, BACKEND_MAGIC);
+		if (port_b != NULL && strncmp(b->port, port_b, port_l) != 0)
+			continue;
+		if (name_b != NULL && strncmp(b->vcl_name, name_b, name_l) != 0)
+			continue;
+		if (ip_b != NULL &&
+		    (b->ipv4_addr == NULL ||
+		      strncmp(b->ipv4_addr, ip_b, ip_l)) &&
+		    (b->ipv6_addr == NULL ||
+		      strncmp(b->ipv6_addr, ip_b, ip_l)))
+			continue;
+		found++;
+		i = func(cli, b, priv);
+		if (i)
+			return(i);
+	}
+	return (found);
+}
+
+/*---------------------------------------------------------------------*/
+
+static int __match_proto__()
+do_list(struct cli *cli, struct backend *b, void *priv)
+{
+	int *hdr;
+
+	AN(priv);
+	hdr = priv;
+	if (!*hdr) {
+		VCLI_Out(cli, "%-30s %-6s %-10s %s",
+		    "Backend name", "Refs", "Admin", "Probe");
+		*hdr = 1;
+	}
+	CHECK_OBJ_NOTNULL(b, BACKEND_MAGIC);
+
+	VCLI_Out(cli, "\n%-30s %-6d", b->display_name, b->refcount);
+
+	if (b->admin_health == ah_probe)
+		VCLI_Out(cli, " %-10s", "probe");
+	else if (b->admin_health == ah_sick)
+		VCLI_Out(cli, " %-10s", "sick");
+	else if (b->admin_health == ah_healthy)
+		VCLI_Out(cli, " %-10s", "healthy");
+	else
+		VCLI_Out(cli, " %-10s", "invalid");
+
+	if (b->probe == NULL)
+		VCLI_Out(cli, " %s", "Healthy (no probe)");
+	else {
+		if (b->healthy)
+			VCLI_Out(cli, " %s", "Healthy ");
+		else
+			VCLI_Out(cli, " %s", "Sick ");
+		VBP_Summary(cli, b->probe);
+	}
+
+	return (0);
+}
+
+static void
+cli_backend_list(struct cli *cli, const char * const *av, void *priv)
+{
+	int hdr = 0;
+
+	(void)av;
+	(void)priv;
+	ASSERT_CLI();
+	(void)backend_find(cli, av[2], do_list, &hdr);
+}
+
+/*---------------------------------------------------------------------*/
+
+static int __match_proto__()
+do_set_health(struct cli *cli, struct backend *b, void *priv)
+{
+	enum admin_health state;
+
+	(void)cli;
+	state = *(enum admin_health*)priv;
+	CHECK_OBJ_NOTNULL(b, BACKEND_MAGIC);
+	b->admin_health = state;
+	return (0);
+}
+
+static void
+cli_backend_set_health(struct cli *cli, const char * const *av, void *priv)
+{
+	enum admin_health state;
+	int n;
+
+	(void)av;
+	(void)priv;
+	ASSERT_CLI();
+	AN(av[2]);
+	AN(av[3]);
+	state = vbe_str2adminhealth(av[3]);
+	if (state == ah_invalid) {
+		VCLI_Out(cli, "Invalid state %s", av[3]);
+		VCLI_SetResult(cli, CLIS_PARAM);
+		return;
+	}
+	n = backend_find(cli, av[2], do_set_health, &state);
+	if (n == 0) {
+		VCLI_Out(cli, "No Backends matches");
+		VCLI_SetResult(cli, CLIS_PARAM);
+	}
+}
+
+/*---------------------------------------------------------------------*/
+
+static struct cli_proto backend_cmds[] = {
+	{ "backend.list", "backend.list",
+	    "\tList all backends\n", 0, 1, "d", cli_backend_list },
+	{ "backend.set_health", "backend.set_health matcher state",
+	    "\tShow a backend\n", 2, 2, "d", cli_backend_set_health },
+	{ NULL }
+};
+
+/*---------------------------------------------------------------------*/
+
+void
+VBE_Init(void)
+{
+
+	Lck_New(&VBE_mtx, lck_vbe);
+	CLI_AddFuncs(backend_cmds);
+}
diff --git a/bin/varnishd/cache/cache_backend_poll.c b/bin/varnishd/cache/cache_backend_poll.c
new file mode 100644
index 0000000..efd64cb
--- /dev/null
+++ b/bin/varnishd/cache/cache_backend_poll.c
@@ -0,0 +1,597 @@
+/*-
+ * Copyright (c) 2006 Verdens Gang AS
+ * Copyright (c) 2006-2011 Varnish Software AS
+ * All rights reserved.
+ *
+ * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * Poll backends for collection of health statistics
+ *
+ * We co-opt threads from the worker pool for probing the backends,
+ * but we want to avoid a potentially messy cleanup operation when we
+ * retire the backend, so the thread owns the health information, which
+ * the backend references, rather than the other way around.
+ *
+ */
+
+#include "config.h"
+
+#include <math.h>
+#include <poll.h>
+#include <stdio.h>
+#include <stdlib.h>
+
+#include "cache.h"
+
+#include "cache_backend.h"
+#include "vcli_priv.h"
+#include "vrt.h"
+#include "vtcp.h"
+#include "vtim.h"
+
+/* Default averaging rate, we want something pretty responsive */
+#define AVG_RATE			4
+
+struct vbp_vcl {
+	unsigned			magic;
+#define VBP_VCL_MAGIC			0x70829764
+
+	VTAILQ_ENTRY(vbp_vcl)		list;
+	const struct vrt_backend_probe	*probep;
+	struct vrt_backend_probe	probe;
+	const char			*hosthdr;
+};
+
+struct vbp_target {
+	unsigned			magic;
+#define VBP_TARGET_MAGIC		0x6b7cb656
+
+	struct backend			*backend;
+	VTAILQ_HEAD( ,vbp_vcl)		vcls;
+
+	struct vrt_backend_probe	probe;
+	int				stop;
+	struct vsb			*vsb;
+	char				*req;
+	int				req_len;
+
+	char				resp_buf[128];
+	unsigned			good;
+
+	/* Collected statistics */
+#define BITMAP(n, c, t, b)	uint64_t	n;
+#include "tbl/backend_poll.h"
+#undef BITMAP
+
+	double				last;
+	double				avg;
+	double				rate;
+
+	VTAILQ_ENTRY(vbp_target)	list;
+	pthread_t			thread;
+};
+
+static VTAILQ_HEAD(, vbp_target)	vbp_list =
+    VTAILQ_HEAD_INITIALIZER(vbp_list);
+
+static struct lock			vbp_mtx;
+
+/*--------------------------------------------------------------------
+ * Poke one backend, once, but possibly at both IPv4 and IPv6 addresses.
+ *
+ * We do deliberately not use the stuff in cache_backend.c, because we
+ * want to measure the backends response without local distractions.
+ */
+
+static int
+vbp_connect(int pf, const struct sockaddr_storage *sa, socklen_t salen, int tmo)
+{
+	int s, i;
+
+	s = socket(pf, SOCK_STREAM, 0);
+	if (s < 0)
+		return (s);
+
+	i = VTCP_connect(s, sa, salen, tmo);
+	if (i == 0)
+		return (s);
+	VTCP_close(&s);
+	return (-1);
+}
+
+static void
+vbp_poke(struct vbp_target *vt)
+{
+	int s, tmo, i;
+	double t_start, t_now, t_end;
+	unsigned rlen, resp;
+	struct backend *bp;
+	char buf[8192], *p;
+	struct pollfd pfda[1], *pfd = pfda;
+
+	bp = vt->backend;
+	CHECK_OBJ_NOTNULL(bp, BACKEND_MAGIC);
+
+	t_start = t_now = VTIM_real();
+	t_end = t_start + vt->probe.timeout;
+	tmo = (int)round((t_end - t_now) * 1e3);
+
+	s = -1;
+	if (cache_param->prefer_ipv6 && bp->ipv6 != NULL) {
+		s = vbp_connect(PF_INET6, bp->ipv6, bp->ipv6len, tmo);
+		t_now = VTIM_real();
+		tmo = (int)round((t_end - t_now) * 1e3);
+		if (s >= 0)
+			vt->good_ipv6 |= 1;
+	}
+	if (tmo > 0 && s < 0 && bp->ipv4 != NULL) {
+		s = vbp_connect(PF_INET, bp->ipv4, bp->ipv4len, tmo);
+		t_now = VTIM_real();
+		tmo = (int)round((t_end - t_now) * 1e3);
+		if (s >= 0)
+			vt->good_ipv4 |= 1;
+	}
+	if (tmo > 0 && s < 0 && bp->ipv6 != NULL) {
+		s = vbp_connect(PF_INET6, bp->ipv6, bp->ipv6len, tmo);
+		t_now = VTIM_real();
+		tmo = (int)round((t_end - t_now) * 1e3);
+		if (s >= 0)
+			vt->good_ipv6 |= 1;
+	}
+	if (s < 0) {
+		/* Got no connection: failed */
+		return;
+	}
+	if (tmo <= 0) {
+		/* Spent too long time getting it */
+		VTCP_close(&s);
+		return;
+	}
+
+	/* Send the request */
+	i = write(s, vt->req, vt->req_len);
+	if (i != vt->req_len) {
+		if (i < 0)
+			vt->err_xmit |= 1;
+		VTCP_close(&s);
+		return;
+	}
+	vt->good_xmit |= 1;
+
+	pfd->fd = s;
+	rlen = 0;
+	do {
+		pfd->events = POLLIN;
+		pfd->revents = 0;
+		tmo = (int)round((t_end - t_now) * 1e3);
+		if (tmo > 0)
+			i = poll(pfd, 1, tmo);
+		if (i == 0 || tmo <= 0) {
+			VTCP_close(&s);
+			return;
+		}
+		if (rlen < sizeof vt->resp_buf)
+			i = read(s, vt->resp_buf + rlen,
+			    sizeof vt->resp_buf - rlen);
+		else
+			i = read(s, buf, sizeof buf);
+		rlen += i;
+	} while (i > 0);
+
+	VTCP_close(&s);
+
+	if (i < 0) {
+		vt->err_recv |= 1;
+		return;
+	}
+
+	if (rlen == 0)
+		return;
+
+	/* So we have a good receive ... */
+	t_now = VTIM_real();
+	vt->last = t_now - t_start;
+	vt->good_recv |= 1;
+
+	/* Now find out if we like the response */
+	vt->resp_buf[sizeof vt->resp_buf - 1] = '\0';
+	p = strchr(vt->resp_buf, '\r');
+	if (p != NULL)
+		*p = '\0';
+	p = strchr(vt->resp_buf, '\n');
+	if (p != NULL)
+		*p = '\0';
+
+	i = sscanf(vt->resp_buf, "HTTP/%*f %u %s", &resp, buf);
+
+	if (i == 2 && resp == vt->probe.exp_status)
+		vt->happy |= 1;
+}
+
+/*--------------------------------------------------------------------
+ * Record pokings...
+ */
+
+static void
+vbp_start_poke(struct vbp_target *vt)
+{
+	CHECK_OBJ_NOTNULL(vt, VBP_TARGET_MAGIC);
+
+#define BITMAP(n, c, t, b)	vt->n <<= 1;
+#include "tbl/backend_poll.h"
+#undef BITMAP
+
+	vt->last = 0;
+	vt->resp_buf[0] = '\0';
+}
+
+static void
+vbp_has_poked(struct vbp_target *vt)
+{
+	unsigned i, j;
+	uint64_t u;
+	const char *logmsg;
+	char bits[10];
+
+	CHECK_OBJ_NOTNULL(vt, VBP_TARGET_MAGIC);
+
+	/* Calculate exponential average */
+	if (vt->happy & 1) {
+		if (vt->rate < AVG_RATE)
+			vt->rate += 1.0;
+		vt->avg += (vt->last - vt->avg) / vt->rate;
+	}
+
+	i = 0;
+#define BITMAP(n, c, t, b)	bits[i++] = (vt->n & 1) ? c : '-';
+#include "tbl/backend_poll.h"
+#undef BITMAP
+	bits[i] = '\0';
+
+	u = vt->happy;
+	for (i = j = 0; i < vt->probe.window; i++) {
+		if (u & 1)
+			j++;
+		u >>= 1;
+	}
+	vt->good = j;
+
+	if (vt->good >= vt->probe.threshold) {
+		if (vt->backend->healthy)
+			logmsg = "Still healthy";
+		else
+			logmsg = "Back healthy";
+		vt->backend->healthy = 1;
+	} else {
+		if (vt->backend->healthy)
+			logmsg = "Went sick";
+		else
+			logmsg = "Still sick";
+		vt->backend->healthy = 0;
+	}
+	VSL(SLT_Backend_health, 0, "%s %s %s %u %u %u %.6f %.6f %s",
+	    vt->backend->vcl_name, logmsg, bits,
+	    vt->good, vt->probe.threshold, vt->probe.window,
+	    vt->last, vt->avg, vt->resp_buf);
+	vt->backend->vsc->happy = vt->happy;
+}
+
+/*--------------------------------------------------------------------
+ * Build request from probe spec
+ */
+
+static void
+vbp_build_req(struct vsb *vsb, const struct vbp_vcl *vcl)
+{
+
+	XXXAN(vsb);
+	XXXAN(vcl);
+	VSB_clear(vsb);
+	if(vcl->probe.request != NULL) {
+		VSB_cat(vsb, vcl->probe.request);
+	} else {
+		VSB_printf(vsb, "GET %s HTTP/1.1\r\n",
+		    vcl->probe.url != NULL ?  vcl->probe.url : "/");
+		if (vcl->hosthdr != NULL)
+			VSB_printf(vsb, "Host: %s\r\n", vcl->hosthdr);
+		VSB_printf(vsb, "Connection: close\r\n");
+		VSB_printf(vsb, "\r\n");
+	}
+	AZ(VSB_finish(vsb));
+}
+
+/*--------------------------------------------------------------------
+ * One thread per backend to be poked.
+ */
+
+static void *
+vbp_wrk_poll_backend(void *priv)
+{
+	struct vbp_target *vt;
+	struct vbp_vcl *vcl = NULL;
+
+	THR_SetName("backend poll");
+
+	CAST_OBJ_NOTNULL(vt, priv, VBP_TARGET_MAGIC);
+
+	while (!vt->stop) {
+		Lck_Lock(&vbp_mtx);
+		if (VTAILQ_FIRST(&vt->vcls) != vcl) {
+			vcl = VTAILQ_FIRST(&vt->vcls);
+			vbp_build_req(vt->vsb, vcl);
+			vt->probe = vcl->probe;
+		}
+		Lck_Unlock(&vbp_mtx);
+
+		vt->req = VSB_data(vt->vsb);
+		vt->req_len = VSB_len(vt->vsb);
+
+		vbp_start_poke(vt);
+		vbp_poke(vt);
+		vbp_has_poked(vt);
+
+		if (!vt->stop)
+			VTIM_sleep(vt->probe.interval);
+	}
+	return (NULL);
+}
+
+/*--------------------------------------------------------------------
+ * Cli functions
+ */
+
+void
+VBP_Summary(struct cli *cli, const struct vbp_target *vt)
+{
+
+	CHECK_OBJ_NOTNULL(vt, VBP_TARGET_MAGIC);
+	VCLI_Out(cli, "%d/%d", vt->good, vt->probe.window);
+}
+
+static void
+vbp_bitmap(struct cli *cli, char c, uint64_t map, const char *lbl)
+{
+	int i;
+	uint64_t u = (1ULL << 63);
+
+	for (i = 0; i < 64; i++) {
+		if (map & u)
+			VCLI_Out(cli, "%c", c);
+		else
+			VCLI_Out(cli, "-");
+		map <<= 1;
+	}
+	VCLI_Out(cli, " %s\n", lbl);
+}
+
+/*lint -e{506} constant value boolean */
+/*lint -e{774} constant value boolean */
+static void
+vbp_health_one(struct cli *cli, const struct vbp_target *vt)
+{
+
+	VCLI_Out(cli, "Backend %s is %s\n",
+	    vt->backend->vcl_name,
+	    vt->backend->healthy ? "Healthy" : "Sick");
+	VCLI_Out(cli, "Current states  good: %2u threshold: %2u window: %2u\n",
+	    vt->good, vt->probe.threshold, vt->probe.window);
+	VCLI_Out(cli, "Average responsetime of good probes: %.6f\n", vt->avg);
+	VCLI_Out(cli,
+	    "Oldest                       "
+	    "                             Newest\n");
+	VCLI_Out(cli,
+	    "============================="
+	    "===================================\n");
+
+#define BITMAP(n, c, t, b)					\
+		if ((vt->n != 0) || (b))			\
+			vbp_bitmap(cli, (c), vt->n, (t));
+#include "tbl/backend_poll.h"
+#undef BITMAP
+}
+
+static void
+vbp_health(struct cli *cli, const char * const *av, void *priv)
+{
+	struct vbp_target *vt;
+
+	ASSERT_CLI();
+	(void)av;
+	(void)priv;
+
+	VTAILQ_FOREACH(vt, &vbp_list, list)
+		vbp_health_one(cli, vt);
+}
+
+static struct cli_proto debug_cmds[] = {
+	{ "debug.health", "debug.health",
+		"\tDump backend health stuff\n",
+		0, 0, "d", vbp_health },
+	{ NULL }
+};
+
+/*--------------------------------------------------------------------
+ * A new VCL wants to probe this backend,
+ */
+
+static struct vbp_vcl *
+vbp_new_vcl(const struct vrt_backend_probe *p, const char *hosthdr)
+{
+	struct vbp_vcl *vcl;
+
+	ALLOC_OBJ(vcl, VBP_VCL_MAGIC);
+	XXXAN(vcl);
+	vcl->probep = p;
+	vcl->probe = *p;
+	vcl->hosthdr = hosthdr;
+
+	/*
+	 * Sanitize and insert defaults
+	 * XXX: we could make these defaults parameters
+	 */
+	if (vcl->probe.timeout == 0.0)
+		vcl->probe.timeout = 2.0;
+	if (vcl->probe.interval == 0.0)
+		vcl->probe.interval = 5.0;
+	if (vcl->probe.window == 0)
+		vcl->probe.window = 8;
+	if (vcl->probe.threshold == 0)
+		vcl->probe.threshold = 3;
+	if (vcl->probe.exp_status == 0)
+		vcl->probe.exp_status = 200;
+
+	if (vcl->probe.threshold == ~0U)
+		vcl->probe.initial = vcl->probe.threshold - 1;
+
+	if (vcl->probe.initial > vcl->probe.threshold)
+		vcl->probe.initial = vcl->probe.threshold;
+	return (vcl);
+}
+
+/*--------------------------------------------------------------------
+ * Insert/Remove/Use called from cache_backend.c
+ */
+
+void
+VBP_Insert(struct backend *b, const struct vrt_backend_probe *p, const char *hosthdr)
+{
+	struct vbp_target *vt;
+	struct vbp_vcl *vcl;
+	int startthread = 0;
+	unsigned u;
+
+	ASSERT_CLI();
+	AN(p);
+
+	if (b->probe == NULL) {
+		ALLOC_OBJ(vt, VBP_TARGET_MAGIC);
+		XXXAN(vt);
+		VTAILQ_INIT(&vt->vcls);
+		vt->backend = b;
+		vt->vsb = VSB_new_auto();
+		XXXAN(vt->vsb);
+		b->probe = vt;
+		startthread = 1;
+		VTAILQ_INSERT_TAIL(&vbp_list, vt, list);
+	} else {
+		vt = b->probe;
+	}
+
+	VTAILQ_FOREACH(vcl, &vt->vcls, list)
+		assert (vcl->probep != p);
+
+	vcl = vbp_new_vcl(p, hosthdr);
+	Lck_Lock(&vbp_mtx);
+	VTAILQ_INSERT_TAIL(&vt->vcls, vcl, list);
+	Lck_Unlock(&vbp_mtx);
+
+	if (startthread) {
+		for (u = 0; u < vcl->probe.initial; u++) {
+			vbp_start_poke(vt);
+			vt->happy |= 1;
+			vbp_has_poked(vt);
+		}
+		AZ(pthread_create(&vt->thread, NULL, vbp_wrk_poll_backend, vt));
+	}
+}
+
+void
+VBP_Use(const struct backend *b, const struct vrt_backend_probe *p)
+{
+	struct vbp_target *vt;
+	struct vbp_vcl *vcl;
+
+	ASSERT_CLI();
+	AN(p);
+	CHECK_OBJ_NOTNULL(b, BACKEND_MAGIC);
+	AN(b->probe);
+	vt = b->probe;
+
+	VTAILQ_FOREACH(vcl, &vt->vcls, list) {
+		if (vcl->probep != p)
+			continue;
+
+		Lck_Lock(&vbp_mtx);
+		VTAILQ_REMOVE(&vt->vcls, vcl, list);
+		VTAILQ_INSERT_HEAD(&vt->vcls, vcl, list);
+		Lck_Unlock(&vbp_mtx);
+		return;
+	}
+}
+
+void
+VBP_Remove(struct backend *b, struct vrt_backend_probe const *p)
+{
+	struct vbp_target *vt;
+	struct vbp_vcl *vcl;
+	void *ret;
+
+	ASSERT_CLI();
+	AN(p);
+	CHECK_OBJ_NOTNULL(b, BACKEND_MAGIC);
+	AN(b->probe);
+	vt = b->probe;
+
+	VTAILQ_FOREACH(vcl, &vt->vcls, list)
+		if (vcl->probep == p)
+			break;
+
+	AN(vcl);
+
+	Lck_Lock(&vbp_mtx);
+	VTAILQ_REMOVE(&vt->vcls, vcl, list);
+	Lck_Unlock(&vbp_mtx);
+
+	FREE_OBJ(vcl);
+
+	if (!VTAILQ_EMPTY(&vt->vcls))
+		return;
+
+	/* No more polling for this backend */
+
+	b->healthy = 1;
+
+	vt->stop = 1;
+	AZ(pthread_cancel(vt->thread));
+	AZ(pthread_join(vt->thread, &ret));
+
+	b->healthy = 1;
+
+	VTAILQ_REMOVE(&vbp_list, vt, list);
+	b->probe = NULL;
+	VSB_delete(vt->vsb);
+	FREE_OBJ(vt);
+}
+
+/*--------------------------------------------------------------------
+ * Initialize the backend probe subsystem
+ */
+
+void
+VBP_Init(void)
+{
+
+	Lck_New(&vbp_mtx, lck_vbp);
+	CLI_AddFuncs(debug_cmds);
+}
diff --git a/bin/varnishd/cache/cache_ban.c b/bin/varnishd/cache/cache_ban.c
new file mode 100644
index 0000000..768e631
--- /dev/null
+++ b/bin/varnishd/cache/cache_ban.c
@@ -0,0 +1,1108 @@
+/*-
+ * Copyright (c) 2006 Verdens Gang AS
+ * Copyright (c) 2006-2011 Varnish Software AS
+ * All rights reserved.
+ *
+ * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * Ban processing
+ *
+ * A ban consists of a number of conditions (or tests), all of which must be
+ * satisfied.  Here are some potential bans we could support:
+ *
+ *	req.url == "/foo"
+ *	req.url ~ ".iso" && obj.size > 10MB
+ *	req.http.host ~ "web1.com" && obj.set-cookie ~ "USER=29293"
+ *
+ * We make the "&&" mandatory from the start, leaving the syntax space
+ * for latter handling of "||" as well.
+ *
+ * Bans are compiled into bytestrings as follows:
+ *	8 bytes	- double: timestamp		XXX: Byteorder ?
+ *	4 bytes - be32: length
+ *	1 byte - flags: 0x01: BAN_F_REQ
+ *	N tests
+ * A test have this form:
+ *	1 byte - arg (see ban_vars.h col 3 "BAN_ARG_XXX")
+ *	(n bytes) - http header name, canonical encoding
+ *	lump - comparison arg
+ *	1 byte - operation (BAN_OPER_)
+ *	(lump) - compiled regexp
+ * A lump is:
+ *	4 bytes - be32: length
+ *	n bytes - content
+ *
+ * In a perfect world, we should vector through VRE to get to PCRE,
+ * but since we rely on PCRE's ability to encode the regexp into a
+ * byte string, that would be a little bit artificial, so this is
+ * the exception that confirmes the rule.
+ *
+ */
+
+#include "config.h"
+
+#include <pcre.h>
+#include <stdio.h>
+
+#include "cache.h"
+
+#include "hash/hash_slinger.h"
+#include "vcli.h"
+#include "vcli_priv.h"
+#include "vend.h"
+#include "vtim.h"
+
+struct ban {
+	unsigned		magic;
+#define BAN_MAGIC		0x700b08ea
+	VTAILQ_ENTRY(ban)	list;
+	unsigned		refcount;
+	unsigned		flags;
+#define BAN_F_GONE		(1 << 0)
+#define BAN_F_REQ		(1 << 2)
+#define BAN_F_LURK		(3 << 6)	/* ban-lurker-color */
+	VTAILQ_HEAD(,objcore)	objcore;
+	struct vsb		*vsb;
+	uint8_t			*spec;
+};
+
+#define LURK_SHIFT 6
+
+struct ban_test {
+	uint8_t			arg1;
+	const char		*arg1_spec;
+	uint8_t			oper;
+	const char		*arg2;
+	const void		*arg2_spec;
+};
+
+static VTAILQ_HEAD(banhead_s,ban) ban_head = VTAILQ_HEAD_INITIALIZER(ban_head);
+static struct lock ban_mtx;
+static struct ban *ban_magic;
+static pthread_t ban_thread;
+static struct ban * volatile ban_start;
+static bgthread_t ban_lurker;
+
+/*--------------------------------------------------------------------
+ * BAN string magic markers
+ */
+
+#define	BAN_OPER_EQ	0x10
+#define	BAN_OPER_NEQ	0x11
+#define	BAN_OPER_MATCH	0x12
+#define	BAN_OPER_NMATCH	0x13
+
+#define BAN_ARG_URL	0x18
+#define BAN_ARG_REQHTTP	0x19
+#define BAN_ARG_OBJHTTP	0x1a
+
+/*--------------------------------------------------------------------
+ * Variables we can purge on
+ */
+
+static const struct pvar {
+	const char		*name;
+	unsigned		flag;
+	uint8_t			tag;
+} pvars[] = {
+#define PVAR(a, b, c)	{ (a), (b), (c) },
+#include "tbl/ban_vars.h"
+#undef PVAR
+	{ 0, 0, 0}
+};
+
+/*--------------------------------------------------------------------
+ * Storage handling of bans
+ */
+
+struct ban *
+BAN_New(void)
+{
+	struct ban *b;
+
+	ALLOC_OBJ(b, BAN_MAGIC);
+	if (b == NULL)
+		return (b);
+	b->vsb = VSB_new_auto();
+	if (b->vsb == NULL) {
+		FREE_OBJ(b);
+		return (NULL);
+	}
+	VTAILQ_INIT(&b->objcore);
+	return (b);
+}
+
+void
+BAN_Free(struct ban *b)
+{
+
+	CHECK_OBJ_NOTNULL(b, BAN_MAGIC);
+	AZ(b->refcount);
+	assert(VTAILQ_EMPTY(&b->objcore));
+
+	if (b->vsb != NULL)
+		VSB_delete(b->vsb);
+	if (b->spec != NULL)
+		free(b->spec);
+	FREE_OBJ(b);
+}
+
+/*--------------------------------------------------------------------
+ * Get & Release a tail reference, used to hold the list stable for
+ * traversals etc.
+ */
+
+struct ban *
+BAN_TailRef(void)
+{
+	struct ban *b;
+
+	ASSERT_CLI();
+	Lck_Lock(&ban_mtx);
+	b = VTAILQ_LAST(&ban_head, banhead_s);
+	AN(b);
+	b->refcount++;
+	Lck_Unlock(&ban_mtx);
+	return (b);
+}
+
+void
+BAN_TailDeref(struct ban **bb)
+{
+	struct ban *b;
+
+	b = *bb;
+	*bb = NULL;
+	Lck_Lock(&ban_mtx);
+	b->refcount--;
+	Lck_Unlock(&ban_mtx);
+}
+
+/*--------------------------------------------------------------------
+ * Extract time and length from ban-spec
+ */
+
+static double
+ban_time(const uint8_t *banspec)
+{
+	double t;
+
+	assert(sizeof t == 8);
+	memcpy(&t, banspec, sizeof t);
+	return (t);
+}
+
+static unsigned
+ban_len(const uint8_t *banspec)
+{
+	unsigned u;
+
+	u = vbe32dec(banspec + 8);
+	return (u);
+}
+
+/*--------------------------------------------------------------------
+ * Access a lump of bytes in a ban test spec
+ */
+
+static void
+ban_add_lump(const struct ban *b, const void *p, uint32_t len)
+{
+	uint8_t buf[sizeof len];
+
+	vbe32enc(buf, len);
+	VSB_bcat(b->vsb, buf, sizeof buf);
+	VSB_bcat(b->vsb, p, len);
+}
+
+static const void *
+ban_get_lump(const uint8_t **bs)
+{
+	const void *r;
+	unsigned ln;
+
+	ln = vbe32dec(*bs);
+	*bs += 4;
+	r = (const void*)*bs;
+	*bs += ln;
+	return (r);
+}
+
+/*--------------------------------------------------------------------
+ * Pick a test apart from a spec string
+ */
+
+static void
+ban_iter(const uint8_t **bs, struct ban_test *bt)
+{
+
+	memset(bt, 0, sizeof *bt);
+	bt->arg1 = *(*bs)++;
+	if (bt->arg1 == BAN_ARG_REQHTTP || bt->arg1 == BAN_ARG_OBJHTTP) {
+		bt->arg1_spec = (const char *)*bs;
+		(*bs) += (*bs)[0] + 2;
+	}
+	bt->arg2 = ban_get_lump(bs);
+	bt->oper = *(*bs)++;
+	if (bt->oper == BAN_OPER_MATCH || bt->oper == BAN_OPER_NMATCH)
+		bt->arg2_spec = ban_get_lump(bs);
+}
+
+/*--------------------------------------------------------------------
+ * Parse and add a http argument specification
+ * Output something which HTTP_GetHdr understands
+ */
+
+static void
+ban_parse_http(const struct ban *b, const char *a1)
+{
+	int l;
+
+	l = strlen(a1) + 1;
+	assert(l <= 127);
+	VSB_putc(b->vsb, (char)l);
+	VSB_cat(b->vsb, a1);
+	VSB_putc(b->vsb, ':');
+	VSB_putc(b->vsb, '\0');
+}
+
+/*--------------------------------------------------------------------
+ * Parse and add a ban test specification
+ */
+
+static int
+ban_parse_regexp(struct cli *cli, const struct ban *b, const char *a3)
+{
+	const char *error;
+	int erroroffset, rc;
+	size_t sz;
+	pcre *re;
+
+	re = pcre_compile(a3, 0, &error, &erroroffset, NULL);
+	if (re == NULL) {
+		VSL(SLT_Debug, 0, "REGEX: <%s>", error);
+		VCLI_Out(cli, "%s", error);
+		VCLI_SetResult(cli, CLIS_PARAM);
+		return (-1);
+	}
+	rc = pcre_fullinfo(re, NULL, PCRE_INFO_SIZE, &sz);
+	AZ(rc);
+	ban_add_lump(b, re, sz);
+	return (0);
+}
+
+/*--------------------------------------------------------------------
+ * Add a (and'ed) test-condition to a ban
+ */
+
+int
+BAN_AddTest(struct cli *cli, struct ban *b, const char *a1, const char *a2,
+    const char *a3)
+{
+	const struct pvar *pv;
+	int i;
+
+	CHECK_OBJ_NOTNULL(b, BAN_MAGIC);
+
+	for (pv = pvars; pv->name != NULL; pv++)
+		if (!strncmp(a1, pv->name, strlen(pv->name)))
+			break;
+	if (pv->name == NULL) {
+		VCLI_Out(cli, "unknown or unsupported field \"%s\"", a1);
+		VCLI_SetResult(cli, CLIS_PARAM);
+		return (-1);
+	}
+
+	if (pv->flag & PVAR_REQ)
+		b->flags |= BAN_F_REQ;
+
+	VSB_putc(b->vsb, pv->tag);
+	if (pv->flag & PVAR_HTTP)
+		ban_parse_http(b, a1 + strlen(pv->name));
+
+	ban_add_lump(b, a3, strlen(a3) + 1);
+	if (!strcmp(a2, "~")) {
+		VSB_putc(b->vsb, BAN_OPER_MATCH);
+		i = ban_parse_regexp(cli, b, a3);
+		if (i)
+			return (i);
+	} else if (!strcmp(a2, "!~")) {
+		VSB_putc(b->vsb, BAN_OPER_NMATCH);
+		i = ban_parse_regexp(cli, b, a3);
+		if (i)
+			return (i);
+	} else if (!strcmp(a2, "==")) {
+		VSB_putc(b->vsb, BAN_OPER_EQ);
+	} else if (!strcmp(a2, "!=")) {
+		VSB_putc(b->vsb, BAN_OPER_NEQ);
+	} else {
+		VCLI_Out(cli,
+		    "expected conditional (~, !~, == or !=) got \"%s\"", a2);
+		VCLI_SetResult(cli, CLIS_PARAM);
+		return (-1);
+	}
+	return (0);
+}
+
+/*--------------------------------------------------------------------
+ * We maintain ban_start as a pointer to the first element of the list
+ * as a separate variable from the VTAILQ, to avoid depending on the
+ * internals of the VTAILQ macros.  We tacitly assume that a pointer
+ * write is always atomic in doing so.
+ */
+
+void
+BAN_Insert(struct ban *b)
+{
+	struct ban  *bi, *be;
+	ssize_t ln;
+	double t0;
+
+	CHECK_OBJ_NOTNULL(b, BAN_MAGIC);
+
+	AZ(VSB_finish(b->vsb));
+	ln = VSB_len(b->vsb);
+	assert(ln >= 0);
+
+	b->spec = malloc(ln + 13L);	/* XXX */
+	XXXAN(b->spec);
+
+	t0 = VTIM_real();
+	memcpy(b->spec, &t0, sizeof t0);
+	b->spec[12] = (b->flags & BAN_F_REQ) ? 1 : 0;
+	memcpy(b->spec + 13, VSB_data(b->vsb), ln);
+	ln += 13;
+	vbe32enc(b->spec + 8, ln);
+
+	VSB_delete(b->vsb);
+	b->vsb = NULL;
+
+	Lck_Lock(&ban_mtx);
+	VTAILQ_INSERT_HEAD(&ban_head, b, list);
+	ban_start = b;
+	VSC_C_main->bans++;
+	VSC_C_main->bans_added++;
+	if (b->flags & BAN_F_REQ)
+		VSC_C_main->bans_req++;
+
+	be = VTAILQ_LAST(&ban_head, banhead_s);
+	if (cache_param->ban_dups && be != b)
+		be->refcount++;
+	else
+		be = NULL;
+
+	SMP_NewBan(b->spec, ln);
+	Lck_Unlock(&ban_mtx);
+
+	if (be == NULL)
+		return;
+
+	/* Hunt down duplicates, and mark them as gone */
+	bi = b;
+	Lck_Lock(&ban_mtx);
+	while(bi != be) {
+		bi = VTAILQ_NEXT(bi, list);
+		if (bi->flags & BAN_F_GONE)
+			continue;
+		/* Safe because the length is part of the fixed size hdr */
+		if (memcmp(b->spec + 8, bi->spec + 8, ln - 8))
+			continue;
+		bi->flags |= BAN_F_GONE;
+		VSC_C_main->bans_gone++;
+		VSC_C_main->bans_dups++;
+	}
+	be->refcount--;
+	Lck_Unlock(&ban_mtx);
+}
+
+/*--------------------------------------------------------------------
+ * A new object is created, grab a reference to the newest ban
+ */
+
+void
+BAN_NewObjCore(struct objcore *oc)
+{
+
+	CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
+	AZ(oc->ban);
+	Lck_Lock(&ban_mtx);
+	oc->ban = ban_start;
+	ban_start->refcount++;
+	VTAILQ_INSERT_TAIL(&ban_start->objcore, oc, ban_list);
+	Lck_Unlock(&ban_mtx);
+}
+
+/*--------------------------------------------------------------------
+ * An object is destroyed, release its ban reference
+ */
+
+void
+BAN_DestroyObj(struct objcore *oc)
+{
+
+	CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
+	if (oc->ban == NULL)
+		return;
+	CHECK_OBJ_NOTNULL(oc->ban, BAN_MAGIC);
+	Lck_Lock(&ban_mtx);
+	assert(oc->ban->refcount > 0);
+	oc->ban->refcount--;
+	VTAILQ_REMOVE(&oc->ban->objcore, oc, ban_list);
+	oc->ban = NULL;
+	Lck_Unlock(&ban_mtx);
+}
+
+/*--------------------------------------------------------------------
+ * Find and/or Grab a reference to an objects ban based on timestamp
+ * Assume we hold a TailRef, so list traversal is safe.
+ */
+
+struct ban *
+BAN_RefBan(struct objcore *oc, double t0, const struct ban *tail)
+{
+	struct ban *b;
+	double t1 = 0;
+
+	VTAILQ_FOREACH(b, &ban_head, list) {
+		t1 = ban_time(b->spec);
+		if (t1 <= t0)
+			break;
+		if (b == tail)
+			break;
+	}
+	AN(b);
+	assert(t1 == t0);
+	Lck_Lock(&ban_mtx);
+	b->refcount++;
+	VTAILQ_INSERT_TAIL(&b->objcore, oc, ban_list);
+	Lck_Unlock(&ban_mtx);
+	return (b);
+}
+
+/*--------------------------------------------------------------------
+ * Put a skeleton ban in the list, unless there is an identical,
+ * time & condition, ban already in place.
+ *
+ * If a newer ban has same condition, mark the new ban GONE.
+ * mark any older bans, with the same condition, GONE as well.
+ */
+
+void
+BAN_Reload(const uint8_t *ban, unsigned len)
+{
+	struct ban *b, *b2;
+	int gone = 0;
+	double t0, t1, t2 = 9e99;
+
+	ASSERT_CLI();
+
+	t0 = ban_time(ban);
+	assert(len == ban_len(ban));
+
+	Lck_Lock(&ban_mtx);
+
+	VTAILQ_FOREACH(b, &ban_head, list) {
+		t1 = ban_time(b->spec);
+		assert (t1 < t2);
+		t2 = t1;
+		if (t1 == t0) {
+			Lck_Unlock(&ban_mtx);
+			return;
+		}
+		if (t1 < t0)
+			break;
+		if (!memcmp(b->spec + 8, ban + 8, len - 8)) {
+			gone |= BAN_F_GONE;
+			VSC_C_main->bans_dups++;
+			VSC_C_main->bans_gone++;
+		}
+	}
+
+	VSC_C_main->bans++;
+	VSC_C_main->bans_added++;
+
+	b2 = BAN_New();
+	AN(b2);
+	b2->spec = malloc(len);
+	AN(b2->spec);
+	memcpy(b2->spec, ban, len);
+	b2->flags |= gone;
+	if (ban[12])
+		b2->flags |= BAN_F_REQ;
+	if (b == NULL)
+		VTAILQ_INSERT_TAIL(&ban_head, b2, list);
+	else
+		VTAILQ_INSERT_BEFORE(b, b2, list);
+
+	/* Hunt down older duplicates */
+	for (b = VTAILQ_NEXT(b2, list); b != NULL; b = VTAILQ_NEXT(b, list)) {
+		if (b->flags & BAN_F_GONE)
+			continue;
+		if (!memcmp(b->spec + 8, ban + 8, len - 8)) {
+			b->flags |= BAN_F_GONE;
+			VSC_C_main->bans_dups++;
+			VSC_C_main->bans_gone++;
+		}
+	}
+	Lck_Unlock(&ban_mtx);
+}
+
+/*--------------------------------------------------------------------
+ * Get a bans timestamp
+ */
+
+double
+BAN_Time(const struct ban *b)
+{
+
+	if (b == NULL)
+		return (0.0);
+
+	CHECK_OBJ_NOTNULL(b, BAN_MAGIC);
+	return (ban_time(b->spec));
+}
+
+/*--------------------------------------------------------------------
+ * All silos have read their bans, ready for action
+ */
+
+void
+BAN_Compile(void)
+{
+
+	ASSERT_CLI();
+
+	SMP_NewBan(ban_magic->spec, ban_len(ban_magic->spec));
+	ban_start = VTAILQ_FIRST(&ban_head);
+	WRK_BgThread(&ban_thread, "ban-lurker", ban_lurker, NULL);
+}
+
+/*--------------------------------------------------------------------
+ * Evaluate ban-spec
+ */
+
+static int
+ban_evaluate(const uint8_t *bs, const struct http *objhttp,
+    const struct http *reqhttp, unsigned *tests)
+{
+	struct ban_test bt;
+	const uint8_t *be;
+	char *arg1;
+
+	be = bs + ban_len(bs);
+	bs += 13;
+	while (bs < be) {
+		(*tests)++;
+		ban_iter(&bs, &bt);
+		arg1 = NULL;
+		switch (bt.arg1) {
+		case BAN_ARG_URL:
+			arg1 = reqhttp->hd[HTTP_HDR_URL].b;
+			break;
+		case BAN_ARG_REQHTTP:
+			(void)http_GetHdr(reqhttp, bt.arg1_spec, &arg1);
+			break;
+		case BAN_ARG_OBJHTTP:
+			(void)http_GetHdr(objhttp, bt.arg1_spec, &arg1);
+			break;
+		default:
+			INCOMPL();
+		}
+
+		switch (bt.oper) {
+		case BAN_OPER_EQ:
+			if (arg1 == NULL || strcmp(arg1, bt.arg2))
+				return (0);
+			break;
+		case BAN_OPER_NEQ:
+			if (arg1 != NULL && !strcmp(arg1, bt.arg2))
+				return (0);
+			break;
+		case BAN_OPER_MATCH:
+			if (arg1 == NULL ||
+			    pcre_exec(bt.arg2_spec, NULL, arg1, strlen(arg1),
+			    0, 0, NULL, 0) < 0)
+				return (0);
+			break;
+		case BAN_OPER_NMATCH:
+			if (arg1 != NULL &&
+			    pcre_exec(bt.arg2_spec, NULL, arg1, strlen(arg1),
+			    0, 0, NULL, 0) >= 0)
+				return (0);
+			break;
+		default:
+			INCOMPL();
+		}
+	}
+	return (1);
+}
+
+/*--------------------------------------------------------------------
+ * Check an object against all applicable bans
+ *
+ * Return:
+ *	-1 not all bans checked, but none of the checked matched
+ *		Only if !has_req
+ *	0 No bans matched, object moved to ban_start.
+ *	1 Ban matched, object removed from ban list.
+ */
+
+static int
+ban_check_object(struct object *o, const struct sess *sp, int has_req)
+{
+	struct ban *b;
+	struct objcore *oc;
+	struct ban * volatile b0;
+	unsigned tests, skipped;
+
+	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+	CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC);
+	oc = o->objcore;
+	CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
+	CHECK_OBJ_NOTNULL(oc->ban, BAN_MAGIC);
+
+	b0 = ban_start;
+	CHECK_OBJ_NOTNULL(b0, BAN_MAGIC);
+
+	if (b0 == oc->ban)
+		return (0);
+
+	/*
+	 * This loop is safe without locks, because we know we hold
+	 * a refcount on a ban somewhere in the list and we do not
+	 * inspect the list past that ban.
+	 */
+	tests = 0;
+	skipped = 0;
+	for (b = b0; b != oc->ban; b = VTAILQ_NEXT(b, list)) {
+		CHECK_OBJ_NOTNULL(b, BAN_MAGIC);
+		if (b->flags & BAN_F_GONE)
+			continue;
+		if ((b->flags & BAN_F_LURK) &&
+		    (b->flags & BAN_F_LURK) == (oc->flags & OC_F_LURK)) {
+			AZ(b->flags & BAN_F_REQ);
+			/* Lurker already tested this */
+			continue;
+		}
+		if (!has_req && (b->flags & BAN_F_REQ)) {
+			/*
+			 * We cannot test this one, but there might
+			 * be other bans that match, so we soldier on
+			 */
+			skipped++;
+		} else if (ban_evaluate(b->spec, o->http, sp->http, &tests))
+			break;
+	}
+
+	Lck_Lock(&ban_mtx);
+	VSC_C_main->bans_tested++;
+	VSC_C_main->bans_tests_tested += tests;
+
+	if (b == oc->ban && skipped > 0) {
+		AZ(has_req);
+		Lck_Unlock(&ban_mtx);
+		/*
+		 * Not banned, but some tests were skipped, so we cannot know
+		 * for certain that it cannot be, so we just have to give up.
+		 */
+		return (-1);
+	}
+
+	oc->ban->refcount--;
+	VTAILQ_REMOVE(&oc->ban->objcore, oc, ban_list);
+	if (b == oc->ban) {	/* not banned */
+		b->flags &= ~BAN_F_LURK;
+		VTAILQ_INSERT_TAIL(&b0->objcore, oc, ban_list);
+		b0->refcount++;
+	}
+	Lck_Unlock(&ban_mtx);
+
+	if (b == oc->ban) {	/* not banned */
+		oc->ban = b0;
+		oc_updatemeta(oc);
+		return (0);
+	} else {
+		EXP_Clr(&o->exp);
+		oc->ban = NULL;
+		oc_updatemeta(oc);
+		/* BAN also changed, but that is not important any more */
+		WSP(sp, SLT_ExpBan, "%u was banned", o->xid);
+		EXP_Rearm(o);
+		return (1);
+	}
+}
+
+int
+BAN_CheckObject(struct object *o, const struct sess *sp)
+{
+
+	return (ban_check_object(o, sp, 1) > 0);
+}
+
+static struct ban *
+ban_CheckLast(void)
+{
+	struct ban *b;
+
+	Lck_AssertHeld(&ban_mtx);
+	b = VTAILQ_LAST(&ban_head, banhead_s);
+	if (b != VTAILQ_FIRST(&ban_head) && b->refcount == 0) {
+		if (b->flags & BAN_F_GONE)
+			VSC_C_main->bans_gone--;
+		if (b->flags & BAN_F_REQ)
+			VSC_C_main->bans_req--;
+		VSC_C_main->bans--;
+		VSC_C_main->bans_deleted++;
+		VTAILQ_REMOVE(&ban_head, b, list);
+	} else {
+		b = NULL;
+	}
+	return (b);
+}
+
+/*--------------------------------------------------------------------
+ * Ban lurker thread
+ */
+
+static int
+ban_lurker_work(const struct sess *sp, unsigned pass)
+{
+	struct ban *b, *b0, *b2;
+	struct objhead *oh;
+	struct objcore *oc, *oc2;
+	struct object *o;
+	int i;
+
+	AN(pass & BAN_F_LURK);
+	AZ(pass & ~BAN_F_LURK);
+
+	/* First route the last ban(s) */
+	do {
+		Lck_Lock(&ban_mtx);
+		b2 = ban_CheckLast();
+		Lck_Unlock(&ban_mtx);
+		if (b2 != NULL)
+			BAN_Free(b2);
+	} while (b2 != NULL);
+
+	/*
+	 * Find out if we have any bans we can do something about
+	 * If we find any, tag them with our pass number.
+	 */
+	i = 0;
+	b0 = NULL;
+	VTAILQ_FOREACH(b, &ban_head, list) {
+		if (b->flags & BAN_F_GONE)
+			continue;
+		if (b->flags & BAN_F_REQ)
+			continue;
+		if (b == VTAILQ_LAST(&ban_head, banhead_s))
+			continue;
+		if (b0 == NULL)
+			b0 = b;
+		i++;
+		b->flags &= ~BAN_F_LURK;
+		b->flags |= pass;
+	}
+	if (cache_param->diag_bitmap & 0x80000)
+		VSL(SLT_Debug, 0, "lurker: %d actionable bans", i);
+	if (i == 0)
+		return (0);
+
+	VTAILQ_FOREACH_REVERSE(b, &ban_head, banhead_s, list) {
+		if (cache_param->diag_bitmap & 0x80000)
+			VSL(SLT_Debug, 0, "lurker doing %f %d",
+			    ban_time(b->spec), b->refcount);
+		while (1) {
+			Lck_Lock(&ban_mtx);
+			oc = VTAILQ_FIRST(&b->objcore);
+			if (oc == NULL)
+				break;
+			CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
+			if (cache_param->diag_bitmap & 0x80000)
+				VSL(SLT_Debug, 0, "test: %p %d %d",
+				    oc, oc->flags & OC_F_LURK, pass);
+			if ((oc->flags & OC_F_LURK) == pass)
+				break;
+			oh = oc->objhead;
+			CHECK_OBJ_NOTNULL(oh, OBJHEAD_MAGIC);
+			if (Lck_Trylock(&oh->mtx)) {
+				Lck_Unlock(&ban_mtx);
+				VTIM_sleep(cache_param->ban_lurker_sleep);
+				continue;
+			}
+			/*
+			 * See if the objcore is still on the objhead since
+			 * we race against HSH_Deref() which comes in the
+			 * opposite locking order.
+			 */
+			VTAILQ_FOREACH(oc2, &oh->objcs, list)
+				if (oc == oc2)
+					break;
+			if (oc2 == NULL) {
+				Lck_Unlock(&oh->mtx);
+				Lck_Unlock(&ban_mtx);
+				VTIM_sleep(cache_param->ban_lurker_sleep);
+				continue;
+			}
+			/*
+			 * Grab a reference to the OC and we can let go of
+			 * the BAN mutex
+			 */
+			AN(oc->refcnt);
+			oc->refcnt++;
+			oc->flags &= ~OC_F_LURK;
+			Lck_Unlock(&ban_mtx);
+			/*
+			 * Get the object and check it against all relevant bans
+			 */
+			o = oc_getobj(sp->wrk, oc);
+			i = ban_check_object(o, sp, 0);
+			if (cache_param->diag_bitmap & 0x80000)
+				VSL(SLT_Debug, 0, "lurker got: %p %d",
+				    oc, i);
+			if (i == -1) {
+				/* Not banned, not moved */
+				oc->flags |= pass;
+				Lck_Lock(&ban_mtx);
+				VTAILQ_REMOVE(&b->objcore, oc, ban_list);
+				VTAILQ_INSERT_TAIL(&b->objcore, oc, ban_list);
+				Lck_Unlock(&ban_mtx);
+			}
+			Lck_Unlock(&oh->mtx);
+			if (cache_param->diag_bitmap & 0x80000)
+				VSL(SLT_Debug, 0, "lurker done: %p %d %d",
+				    oc, oc->flags & OC_F_LURK, pass);
+			(void)HSH_Deref(sp->wrk, NULL, &o);
+			VTIM_sleep(cache_param->ban_lurker_sleep);
+		}
+		Lck_AssertHeld(&ban_mtx);
+		if (!(b->flags & BAN_F_REQ)) {
+			if (!(b->flags & BAN_F_GONE)) {
+				b->flags |= BAN_F_GONE;
+				VSC_C_main->bans_gone++;
+			}
+			if (cache_param->diag_bitmap & 0x80000)
+				VSL(SLT_Debug, 0, "lurker BAN %f now gone",
+				    ban_time(b->spec));
+		}
+		Lck_Unlock(&ban_mtx);
+		VTIM_sleep(cache_param->ban_lurker_sleep);
+		if (b == b0)
+			break;
+	}
+	return (1);
+}
+
+static void * __match_proto__(bgthread_t)
+ban_lurker(struct sess *sp, void *priv)
+{
+	struct ban *bf;
+	unsigned pass = (1 << LURK_SHIFT);
+
+	int i = 0;
+	(void)priv;
+	while (1) {
+
+		while (cache_param->ban_lurker_sleep == 0.0) {
+			/*
+			 * Ban-lurker is disabled:
+			 * Clean the last ban, if possible, and sleep
+			 */
+			Lck_Lock(&ban_mtx);
+			bf = ban_CheckLast();
+			Lck_Unlock(&ban_mtx);
+			if (bf != NULL)
+				BAN_Free(bf);
+			else
+				VTIM_sleep(1.0);
+		}
+
+		i = ban_lurker_work(sp, pass);
+		WSL_Flush(sp->wrk, 0);
+		WRK_SumStat(sp->wrk);
+		if (i) {
+			pass += (1 << LURK_SHIFT);
+			pass &= BAN_F_LURK;
+			if (pass == 0)
+				pass += (1 << LURK_SHIFT);
+			VTIM_sleep(cache_param->ban_lurker_sleep);
+		} else {
+			VTIM_sleep(1.0);
+		}
+	}
+	NEEDLESS_RETURN(NULL);
+}
+
+/*--------------------------------------------------------------------
+ * CLI functions to add bans
+ */
+
+static void
+ccf_ban(struct cli *cli, const char * const *av, void *priv)
+{
+	int narg, i;
+	struct ban *b;
+
+	(void)priv;
+
+	/* First do some cheap checks on the arguments */
+	for (narg = 0; av[narg + 2] != NULL; narg++)
+		continue;
+	if ((narg % 4) != 3) {
+		VCLI_Out(cli, "Wrong number of arguments");
+		VCLI_SetResult(cli, CLIS_PARAM);
+		return;
+	}
+	for (i = 3; i < narg; i += 4) {
+		if (strcmp(av[i + 2], "&&")) {
+			VCLI_Out(cli, "Found \"%s\" expected &&", av[i + 2]);
+			VCLI_SetResult(cli, CLIS_PARAM);
+			return;
+		}
+	}
+
+	b = BAN_New();
+	if (b == NULL) {
+		VCLI_Out(cli, "Out of Memory");
+		VCLI_SetResult(cli, CLIS_CANT);
+		return;
+	}
+	for (i = 0; i < narg; i += 4)
+		if (BAN_AddTest(cli, b, av[i + 2], av[i + 3], av[i + 4])) {
+			BAN_Free(b);
+			return;
+		}
+	BAN_Insert(b);
+}
+
+static void
+ccf_ban_url(struct cli *cli, const char * const *av, void *priv)
+{
+	const char *aav[6];
+
+	(void)priv;
+	aav[0] = NULL;
+	aav[1] = "ban";
+	aav[2] = "req.url";
+	aav[3] = "~";
+	aav[4] = av[2];
+	aav[5] = NULL;
+	ccf_ban(cli, aav, priv);
+}
+
+static void
+ban_render(struct cli *cli, const uint8_t *bs)
+{
+	struct ban_test bt;
+	const uint8_t *be;
+
+	be = bs + ban_len(bs);
+	bs += 13;
+	while (bs < be) {
+		ban_iter(&bs, &bt);
+		switch (bt.arg1) {
+		case BAN_ARG_URL:
+			VCLI_Out(cli, "req.url");
+			break;
+		case BAN_ARG_REQHTTP:
+			VCLI_Out(cli, "req.http.%.*s",
+			    bt.arg1_spec[0] - 1, bt.arg1_spec + 1);
+			break;
+		case BAN_ARG_OBJHTTP:
+			VCLI_Out(cli, "obj.http.%.*s",
+			    bt.arg1_spec[0] - 1, bt.arg1_spec + 1);
+			break;
+		default:
+			INCOMPL();
+		}
+		switch (bt.oper) {
+		case BAN_OPER_EQ:	VCLI_Out(cli, " == "); break;
+		case BAN_OPER_NEQ:	VCLI_Out(cli, " != "); break;
+		case BAN_OPER_MATCH:	VCLI_Out(cli, " ~ "); break;
+		case BAN_OPER_NMATCH:	VCLI_Out(cli, " !~ "); break;
+		default:
+			INCOMPL();
+		}
+		VCLI_Out(cli, "%s", bt.arg2);
+		if (bs < be)
+			VCLI_Out(cli, " && ");
+	}
+}
+
+static void
+ccf_ban_list(struct cli *cli, const char * const *av, void *priv)
+{
+	struct ban *b, *bl;
+
+	(void)av;
+	(void)priv;
+
+	/* Get a reference so we are safe to traverse the list */
+	bl = BAN_TailRef();
+
+	VCLI_Out(cli, "Present bans:\n");
+	VTAILQ_FOREACH(b, &ban_head, list) {
+		if (b == bl && !(cache_param->diag_bitmap & 0x80000))
+			break;
+		VCLI_Out(cli, "%10.6f %5u%s\t", ban_time(b->spec),
+		    bl == b ? b->refcount - 1 : b->refcount,
+		    b->flags & BAN_F_GONE ? "G" : " ");
+		ban_render(cli, b->spec);
+		VCLI_Out(cli, "\n");
+		if (cache_param->diag_bitmap & 0x80000) {
+			Lck_Lock(&ban_mtx);
+			struct objcore *oc;
+			VTAILQ_FOREACH(oc, &b->objcore, ban_list)
+				VCLI_Out(cli, "     %p\n", oc);
+			Lck_Unlock(&ban_mtx);
+		}
+	}
+
+	BAN_TailDeref(&bl);
+}
+
+static struct cli_proto ban_cmds[] = {
+	{ CLI_BAN_URL,				"", ccf_ban_url },
+	{ CLI_BAN,				"", ccf_ban },
+	{ CLI_BAN_LIST,				"", ccf_ban_list },
+	{ NULL }
+};
+
+void
+BAN_Init(void)
+{
+
+	Lck_New(&ban_mtx, lck_ban);
+	CLI_AddFuncs(ban_cmds);
+	assert(BAN_F_LURK == OC_F_LURK);
+	AN((1 << LURK_SHIFT) & BAN_F_LURK);
+	AN((2 << LURK_SHIFT) & BAN_F_LURK);
+
+	ban_magic = BAN_New();
+	AN(ban_magic);
+	ban_magic->flags |= BAN_F_GONE;
+	VSC_C_main->bans_gone++;
+	BAN_Insert(ban_magic);
+}
diff --git a/bin/varnishd/cache/cache_center.c b/bin/varnishd/cache/cache_center.c
new file mode 100644
index 0000000..e42fac8
--- /dev/null
+++ b/bin/varnishd/cache/cache_center.c
@@ -0,0 +1,1691 @@
+/*-
+ * Copyright (c) 2006 Verdens Gang AS
+ * Copyright (c) 2006-2011 Varnish Software AS
+ * All rights reserved.
+ *
+ * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * This file contains the central state machine for pushing requests.
+ *
+ * We cannot just use direct calls because it is possible to kick a
+ * request back to the lookup stage (usually after a rewrite).  The
+ * state engine also allows us to break the processing up into some
+ * logical chunks which improves readability a little bit.
+ *
+ * Since the states are rather nasty in detail, I have decided to embedd
+ * a dot(1) graph in the source code comments.  So to see the big picture,
+ * extract the DOT lines and run though dot(1), for instance with the
+ * command:
+ *	sed -n '/^DOT/s///p' cache_center.c | dot -Tps > /tmp/_.ps
+ */
+
+/*
+DOT digraph vcl_center {
+xDOT	page="8.2,11.5"
+DOT	size="7.2,10.5"
+DOT	margin="0.5"
+DOT	center="1"
+DOT acceptor [
+DOT	shape=hexagon
+DOT	label="Request received"
+DOT ]
+DOT ERROR [shape=plaintext]
+DOT RESTART [shape=plaintext]
+DOT acceptor -> start [style=bold,color=green]
+ */
+
+#include "config.h"
+
+#include <math.h>
+#include <poll.h>
+#include <stdio.h>
+#include <stdlib.h>
+
+#include "cache.h"
+
+#include "hash/hash_slinger.h"
+#include "vcl.h"
+#include "vcli_priv.h"
+#include "vsha256.h"
+#include "vtcp.h"
+#include "vtim.h"
+
+#ifndef HAVE_SRANDOMDEV
+#include "compat/srandomdev.h"
+#endif
+
+static unsigned xids;
+
+/*--------------------------------------------------------------------
+ * WAIT
+ * Wait (briefly) until we have a full request in our htc.
+ */
+
+static int
+cnt_wait(struct sess *sp)
+{
+	int i;
+	struct pollfd pfd[1];
+
+	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+	AZ(sp->vcl);
+	AZ(sp->obj);
+	assert(sp->xid == 0);
+
+	i = HTC_Complete(sp->htc);
+	if (i == 0 && cache_param->session_linger > 0) {
+		pfd[0].fd = sp->fd;
+		pfd[0].events = POLLIN;
+		pfd[0].revents = 0;
+		i = poll(pfd, 1, cache_param->session_linger);
+		if (i)
+			i = HTC_Rx(sp->htc);
+	}
+	if (i == 0) {
+		WSP(sp, SLT_Debug, "herding");
+		sp->wrk->stats.sess_herd++;
+		SES_Charge(sp);
+		sp->wrk = NULL;
+		Pool_Wait(sp);
+		return (1);
+	}
+	if (i == 1) {
+		sp->step = STP_START;
+		return (0);
+	}
+	if (i == -2) {
+		SES_Close(sp, "overflow");
+		return (0);
+	}
+	if (i == -1 && Tlen(sp->htc->rxbuf) == 0 &&
+	    (errno == 0 || errno == ECONNRESET))
+		SES_Close(sp, "EOF");
+	else
+		SES_Close(sp, "error");
+	sp->step = STP_DONE;
+	return (0);
+}
+
+/*--------------------------------------------------------------------
+ * We have a refcounted object on the session, now deliver it.
+ *
+DOT subgraph xcluster_prepresp {
+DOT	prepresp [
+DOT		shape=ellipse
+DOT		label="Filter obj.->resp."
+DOT	]
+DOT	vcl_deliver [
+DOT		shape=record
+DOT		label="vcl_deliver()|resp."
+DOT	]
+DOT	prepresp -> vcl_deliver [style=bold,color=green]
+DOT	prepresp -> vcl_deliver [style=bold,color=cyan]
+DOT	prepresp -> vcl_deliver [style=bold,color=red]
+DOT	prepresp -> vcl_deliver [style=bold,color=blue,]
+DOT	vcl_deliver -> deliver [style=bold,color=green,label=deliver]
+DOT	vcl_deliver -> deliver [style=bold,color=red]
+DOT	vcl_deliver -> deliver [style=bold,color=blue]
+DOT     vcl_deliver -> errdeliver [label="error"]
+DOT     errdeliver [label="ERROR",shape=plaintext]
+DOT     vcl_deliver -> rstdeliver [label="restart",color=purple]
+DOT     rstdeliver [label="RESTART",shape=plaintext]
+DOT     vcl_deliver -> streambody [style=bold,color=cyan,label="deliver"]
+DOT }
+ *
+ */
+
+static int
+cnt_prepresp(struct sess *sp)
+{
+
+	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+	CHECK_OBJ_NOTNULL(sp->obj, OBJECT_MAGIC);
+	CHECK_OBJ_NOTNULL(sp->vcl, VCL_CONF_MAGIC);
+
+	if (sp->wrk->do_stream)
+		AssertObjCorePassOrBusy(sp->obj->objcore);
+
+	sp->wrk->res_mode = 0;
+
+	if ((sp->wrk->h_content_length != NULL || !sp->wrk->do_stream) &&
+	    !sp->wrk->do_gzip && !sp->wrk->do_gunzip)
+		sp->wrk->res_mode |= RES_LEN;
+
+	if (!sp->disable_esi && sp->obj->esidata != NULL) {
+		/* In ESI mode, we don't know the aggregate length */
+		sp->wrk->res_mode &= ~RES_LEN;
+		sp->wrk->res_mode |= RES_ESI;
+	}
+
+	if (sp->esi_level > 0) {
+		sp->wrk->res_mode &= ~RES_LEN;
+		sp->wrk->res_mode |= RES_ESI_CHILD;
+	}
+
+	if (cache_param->http_gzip_support && sp->obj->gziped &&
+	    !RFC2616_Req_Gzip(sp)) {
+		/*
+		 * We don't know what it uncompresses to
+		 * XXX: we could cache that
+		 */
+		sp->wrk->res_mode &= ~RES_LEN;
+		sp->wrk->res_mode |= RES_GUNZIP;
+	}
+
+	if (!(sp->wrk->res_mode & (RES_LEN|RES_CHUNKED|RES_EOF))) {
+		if (sp->obj->len == 0 && !sp->wrk->do_stream)
+			/*
+			 * If the object is empty, neither ESI nor GUNZIP
+			 * can make it any different size
+			 */
+			sp->wrk->res_mode |= RES_LEN;
+		else if (!sp->wantbody) {
+			/* Nothing */
+		} else if (sp->http->protover >= 11) {
+			sp->wrk->res_mode |= RES_CHUNKED;
+		} else {
+			sp->wrk->res_mode |= RES_EOF;
+			sp->doclose = "EOF mode";
+		}
+	}
+
+	sp->t_resp = VTIM_real();
+	if (sp->obj->objcore != NULL) {
+		if ((sp->t_resp - sp->obj->last_lru) > cache_param->lru_timeout &&
+		    EXP_Touch(sp->obj->objcore))
+			sp->obj->last_lru = sp->t_resp;
+		sp->obj->last_use = sp->t_resp;	/* XXX: locking ? */
+	}
+	http_Setup(sp->wrk->resp, sp->wrk->ws);
+	RES_BuildHttp(sp);
+	VCL_deliver_method(sp);
+	switch (sp->handling) {
+	case VCL_RET_DELIVER:
+		break;
+	case VCL_RET_RESTART:
+		if (sp->restarts >= cache_param->max_restarts)
+			break;
+		if (sp->wrk->do_stream) {
+			VDI_CloseFd(sp->wrk);
+			HSH_Drop(sp);
+		} else {
+			(void)HSH_Deref(sp->wrk, NULL, &sp->obj);
+		}
+		AZ(sp->obj);
+		sp->restarts++;
+		sp->director = NULL;
+		sp->wrk->h_content_length = NULL;
+		http_Setup(sp->wrk->bereq, NULL);
+		http_Setup(sp->wrk->beresp, NULL);
+		http_Setup(sp->wrk->resp, NULL);
+		sp->step = STP_RECV;
+		return (0);
+	default:
+		WRONG("Illegal action in vcl_deliver{}");
+	}
+	if (sp->wrk->do_stream) {
+		AssertObjCorePassOrBusy(sp->obj->objcore);
+		sp->step = STP_STREAMBODY;
+	} else {
+		sp->step = STP_DELIVER;
+	}
+	return (0);
+}
+
+/*--------------------------------------------------------------------
+ * Deliver an already stored object
+ *
+DOT subgraph xcluster_deliver {
+DOT	deliver [
+DOT		shape=ellipse
+DOT		label="Send body"
+DOT	]
+DOT }
+DOT deliver -> DONE [style=bold,color=green]
+DOT deliver -> DONE [style=bold,color=red]
+DOT deliver -> DONE [style=bold,color=blue]
+ *
+ */
+
+static int
+cnt_deliver(struct sess *sp)
+{
+
+	sp->director = NULL;
+	sp->restarts = 0;
+
+	RES_WriteObj(sp);
+
+	assert(WRW_IsReleased(sp->wrk));
+	assert(sp->wrk->wrw.ciov == sp->wrk->wrw.siov);
+	(void)HSH_Deref(sp->wrk, NULL, &sp->obj);
+	http_Setup(sp->wrk->resp, NULL);
+	sp->step = STP_DONE;
+	return (0);
+}
+
+/*--------------------------------------------------------------------
+ * This is the final state, figure out if we should close or recycle
+ * the client connection
+ *
+DOT	DONE [
+DOT		shape=hexagon
+DOT		label="Request completed"
+DOT	]
+ */
+
+static int
+cnt_done(struct sess *sp)
+{
+	double dh, dp, da;
+	int i;
+
+	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+	CHECK_OBJ_ORNULL(sp->vcl, VCL_CONF_MAGIC);
+
+	AZ(sp->obj);
+	AZ(sp->wrk->vbc);
+	sp->director = NULL;
+	sp->restarts = 0;
+
+	sp->wrk->do_esi = 0;
+	sp->wrk->do_gunzip = 0;
+	sp->wrk->do_gzip = 0;
+	sp->wrk->do_stream = 0;
+	sp->wrk->is_gunzip = 0;
+	sp->wrk->is_gzip = 0;
+
+	if (sp->vcl != NULL && sp->esi_level == 0) {
+		if (sp->wrk->vcl != NULL)
+			VCL_Rel(&sp->wrk->vcl);
+		sp->wrk->vcl = sp->vcl;
+		sp->vcl = NULL;
+	}
+
+	SES_Charge(sp);
+
+	sp->t_end = VTIM_real();
+	sp->wrk->lastused = sp->t_end;
+	if (sp->xid == 0) {
+		sp->t_req = sp->t_end;
+		sp->t_resp = sp->t_end;
+	} else if (sp->esi_level == 0) {
+		dp = sp->t_resp - sp->t_req;
+		da = sp->t_end - sp->t_resp;
+		dh = sp->t_req - sp->t_open;
+		/* XXX: Add StatReq == StatSess */
+		/* XXX: Workaround for pipe */
+		if (sp->fd >= 0) {
+			WSP(sp, SLT_Length, "%ju",
+			    (uintmax_t)sp->req_bodybytes);
+		}
+		WSP(sp, SLT_ReqEnd, "%u %.9f %.9f %.9f %.9f %.9f",
+		    sp->xid, sp->t_req, sp->t_end, dh, dp, da);
+	}
+	sp->xid = 0;
+	sp->t_open = sp->t_end;
+	sp->t_resp = NAN;
+	WSL_Flush(sp->wrk, 0);
+
+	/* If we did an ESI include, don't mess up our state */
+	if (sp->esi_level > 0)
+		return (1);
+
+	sp->req_bodybytes = 0;
+
+	sp->t_req = NAN;
+	sp->hash_always_miss = 0;
+	sp->hash_ignore_busy = 0;
+
+	if (sp->fd >= 0 && sp->doclose != NULL) {
+		/*
+		 * This is an orderly close of the connection; ditch nolinger
+		 * before we close, to get queued data transmitted.
+		 */
+		// XXX: not yet (void)VTCP_linger(sp->fd, 0);
+		SES_Close(sp, sp->doclose);
+	}
+
+	if (sp->fd < 0) {
+		sp->wrk->stats.sess_closed++;
+		SES_Delete(sp, NULL);
+		return (1);
+	}
+
+	if (sp->wrk->stats.client_req >= cache_param->wthread_stats_rate)
+		WRK_SumStat(sp->wrk);
+	/* Reset the workspace to the session-watermark */
+	WS_Reset(sp->ws, sp->ws_ses);
+	WS_Reset(sp->wrk->ws, NULL);
+
+	i = HTC_Reinit(sp->htc);
+	if (i == 1) {
+		sp->wrk->stats.sess_pipeline++;
+		sp->step = STP_START;
+		return (0);
+	}
+	if (Tlen(sp->htc->rxbuf)) {
+		sp->wrk->stats.sess_readahead++;
+		sp->step = STP_WAIT;
+		return (0);
+	}
+	if (cache_param->session_linger > 0) {
+		sp->wrk->stats.sess_linger++;
+		sp->step = STP_WAIT;
+		return (0);
+	}
+	sp->wrk->stats.sess_herd++;
+	sp->wrk = NULL;
+	Pool_Wait(sp);
+	return (1);
+}
+
+/*--------------------------------------------------------------------
+ * Emit an error
+ *
+DOT subgraph xcluster_error {
+DOT	vcl_error [
+DOT		shape=record
+DOT		label="vcl_error()|resp."
+DOT	]
+DOT	ERROR -> vcl_error
+DOT	vcl_error-> prepresp [label=deliver]
+DOT }
+DOT vcl_error-> rsterr [label="restart",color=purple]
+DOT rsterr [label="RESTART",shape=plaintext]
+ */
+
+static int
+cnt_error(struct sess *sp)
+{
+	struct worker *w;
+	struct http *h;
+	char date[40];
+
+	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+
+	sp->wrk->do_esi = 0;
+	sp->wrk->is_gzip = 0;
+	sp->wrk->is_gunzip = 0;
+	sp->wrk->do_gzip = 0;
+	sp->wrk->do_gunzip = 0;
+	sp->wrk->do_stream = 0;
+
+	w = sp->wrk;
+	if (sp->obj == NULL) {
+		HSH_Prealloc(sp);
+		EXP_Clr(&w->exp);
+		sp->obj = STV_NewObject(sp, NULL, cache_param->http_resp_size,
+		     &w->exp, (uint16_t)cache_param->http_max_hdr);
+		if (sp->obj == NULL)
+			sp->obj = STV_NewObject(sp, TRANSIENT_STORAGE,
+			    cache_param->http_resp_size, &w->exp,
+			    (uint16_t)cache_param->http_max_hdr);
+		if (sp->obj == NULL) {
+			sp->doclose = "Out of objects";
+			sp->director = NULL;
+			sp->wrk->h_content_length = NULL;
+			http_Setup(sp->wrk->beresp, NULL);
+			http_Setup(sp->wrk->bereq, NULL);
+			sp->step = STP_DONE;
+			return(0);
+		}
+		AN(sp->obj);
+		sp->obj->xid = sp->xid;
+		sp->obj->exp.entered = sp->t_req;
+	} else {
+		/* XXX: Null the headers ? */
+	}
+	CHECK_OBJ_NOTNULL(sp->obj, OBJECT_MAGIC);
+	h = sp->obj->http;
+
+	if (sp->err_code < 100 || sp->err_code > 999)
+		sp->err_code = 501;
+
+	http_PutProtocol(w, sp->vsl_id, h, "HTTP/1.1");
+	http_PutStatus(h, sp->err_code);
+	VTIM_format(VTIM_real(), date);
+	http_PrintfHeader(w, sp->vsl_id, h, "Date: %s", date);
+	http_SetHeader(w, sp->vsl_id, h, "Server: Varnish");
+
+	if (sp->err_reason != NULL)
+		http_PutResponse(w, sp->vsl_id, h, sp->err_reason);
+	else
+		http_PutResponse(w, sp->vsl_id, h,
+		    http_StatusMessage(sp->err_code));
+	VCL_error_method(sp);
+
+	if (sp->handling == VCL_RET_RESTART &&
+	    sp->restarts <  cache_param->max_restarts) {
+		HSH_Drop(sp);
+		sp->director = NULL;
+		sp->restarts++;
+		sp->step = STP_RECV;
+		return (0);
+	} else if (sp->handling == VCL_RET_RESTART)
+		sp->handling = VCL_RET_DELIVER;
+
+
+	/* We always close when we take this path */
+	sp->doclose = "error";
+	sp->wantbody = 1;
+
+	assert(sp->handling == VCL_RET_DELIVER);
+	sp->err_code = 0;
+	sp->err_reason = NULL;
+	http_Setup(sp->wrk->bereq, NULL);
+	sp->step = STP_PREPRESP;
+	return (0);
+}
+
+/*--------------------------------------------------------------------
+ * Fetch response headers from the backend
+ *
+DOT subgraph xcluster_fetch {
+DOT	fetch [
+DOT		shape=ellipse
+DOT		label="fetch hdr\nfrom backend\n(find obj.ttl)"
+DOT	]
+DOT	vcl_fetch [
+DOT		shape=record
+DOT		label="vcl_fetch()|req.\nbereq.\nberesp."
+DOT	]
+DOT	fetch -> vcl_fetch [style=bold,color=blue]
+DOT	fetch -> vcl_fetch [style=bold,color=red]
+DOT	fetch_pass [
+DOT		shape=ellipse
+DOT		label="obj.f.pass=true"
+DOT	]
+DOT	vcl_fetch -> fetch_pass [label="hit_for_pass",style=bold,color=red]
+DOT }
+DOT fetch_pass -> fetchbody [style=bold,color=red]
+DOT vcl_fetch -> fetchbody [label="deliver",style=bold,color=blue]
+DOT vcl_fetch -> rstfetch [label="restart",color=purple]
+DOT rstfetch [label="RESTART",shape=plaintext]
+DOT fetch -> errfetch
+DOT vcl_fetch -> errfetch [label="error"]
+DOT errfetch [label="ERROR",shape=plaintext]
+ */
+
+static int
+cnt_fetch(struct sess *sp)
+{
+	int i;
+
+	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+	CHECK_OBJ_NOTNULL(sp->vcl, VCL_CONF_MAGIC);
+
+	AN(sp->director);
+	AZ(sp->wrk->vbc);
+	AZ(sp->wrk->h_content_length);
+	AZ(sp->wrk->do_close);
+	AZ(sp->wrk->storage_hint);
+
+	http_Setup(sp->wrk->beresp, sp->wrk->ws);
+
+	i = FetchHdr(sp);
+	/*
+	 * If we recycle a backend connection, there is a finite chance
+	 * that the backend closed it before we get a request to it.
+	 * Do a single retry in that case.
+	 */
+	if (i == 1) {
+		VSC_C_main->backend_retry++;
+		i = FetchHdr(sp);
+	}
+
+	if (i) {
+		sp->handling = VCL_RET_ERROR;
+		sp->err_code = 503;
+	} else {
+		/*
+		 * These two headers can be spread over multiple actual headers
+		 * and we rely on their content outside of VCL, so collect them
+		 * into one line here.
+		 */
+		http_CollectHdr(sp->wrk->beresp, H_Cache_Control);
+		http_CollectHdr(sp->wrk->beresp, H_Vary);
+
+		/*
+		 * Figure out how the fetch is supposed to happen, before the
+		 * headers are adultered by VCL
+		 * NB: Also sets other sp->wrk variables
+		 */
+		sp->wrk->body_status = RFC2616_Body(sp);
+
+		sp->err_code = http_GetStatus(sp->wrk->beresp);
+
+		/*
+		 * What does RFC2616 think about TTL ?
+		 */
+		EXP_Clr(&sp->wrk->exp);
+		sp->wrk->exp.entered = VTIM_real();
+		RFC2616_Ttl(sp);
+
+		/* pass from vclrecv{} has negative TTL */
+		if (sp->objcore == NULL)
+			sp->wrk->exp.ttl = -1.;
+
+		AZ(sp->wrk->do_esi);
+
+		VCL_fetch_method(sp);
+
+		switch (sp->handling) {
+		case VCL_RET_HIT_FOR_PASS:
+			if (sp->objcore != NULL)
+				sp->objcore->flags |= OC_F_PASS;
+			sp->step = STP_FETCHBODY;
+			return (0);
+		case VCL_RET_DELIVER:
+			AssertObjCorePassOrBusy(sp->objcore);
+			sp->step = STP_FETCHBODY;
+			return (0);
+		default:
+			break;
+		}
+
+		/* We are not going to fetch the body, Close the connection */
+		VDI_CloseFd(sp->wrk);
+	}
+
+	/* Clean up partial fetch */
+	AZ(sp->wrk->vbc);
+
+	if (sp->objcore != NULL) {
+		CHECK_OBJ_NOTNULL(sp->objcore, OBJCORE_MAGIC);
+		AZ(HSH_Deref(sp->wrk, sp->objcore, NULL));
+		sp->objcore = NULL;
+	}
+	http_Setup(sp->wrk->bereq, NULL);
+	http_Setup(sp->wrk->beresp, NULL);
+	sp->wrk->h_content_length = NULL;
+	sp->director = NULL;
+	sp->wrk->storage_hint = NULL;
+
+	switch (sp->handling) {
+	case VCL_RET_RESTART:
+		sp->restarts++;
+		sp->step = STP_RECV;
+		return (0);
+	case VCL_RET_ERROR:
+		sp->step = STP_ERROR;
+		return (0);
+	default:
+		WRONG("Illegal action in vcl_fetch{}");
+	}
+}
+
+/*--------------------------------------------------------------------
+ * Fetch response body from the backend
+ *
+DOT subgraph xcluster_body {
+DOT	fetchbody [
+DOT		shape=diamond
+DOT		label="stream ?"
+DOT	]
+DOT	fetchbody2 [
+DOT		shape=ellipse
+DOT		label="fetch body\nfrom backend\n"
+DOT	]
+DOT }
+DOT fetchbody -> fetchbody2 [label=no,style=bold,color=red]
+DOT fetchbody -> fetchbody2 [style=bold,color=blue]
+DOT fetchbody -> prepresp [label=yes,style=bold,color=cyan]
+DOT fetchbody2 -> prepresp [style=bold,color=red]
+DOT fetchbody2 -> prepresp [style=bold,color=blue]
+ */
+
+
+static int
+cnt_fetchbody(struct sess *sp)
+{
+	int i;
+	struct http *hp, *hp2;
+	char *b;
+	uint16_t nhttp;
+	unsigned l;
+	struct vsb *vary = NULL;
+	int varyl = 0, pass;
+
+	assert(sp->handling == VCL_RET_HIT_FOR_PASS ||
+	    sp->handling == VCL_RET_DELIVER);
+
+	if (sp->objcore == NULL) {
+		/* This is a pass from vcl_recv */
+		pass = 1;
+		/* VCL may have fiddled this, but that doesn't help */
+		sp->wrk->exp.ttl = -1.;
+	} else if (sp->handling == VCL_RET_HIT_FOR_PASS) {
+		/* pass from vcl_fetch{} -> hit-for-pass */
+		/* XXX: the bereq was not filtered pass... */
+		pass = 1;
+	} else {
+		/* regular object */
+		pass = 0;
+	}
+
+	/*
+	 * The VCL variables beresp.do_g[un]zip tells us how we want the
+	 * object processed before it is stored.
+	 *
+	 * The backend Content-Encoding header tells us what we are going
+	 * to receive, which we classify in the following three classes:
+	 *
+	 *	"Content-Encoding: gzip"	--> object is gzip'ed.
+	 *	no Content-Encoding		--> object is not gzip'ed.
+	 *	anything else			--> do nothing wrt gzip
+	 *
+	 */
+
+	AZ(sp->wrk->vfp);
+
+	/* We do nothing unless the param is set */
+	if (!cache_param->http_gzip_support)
+		sp->wrk->do_gzip = sp->wrk->do_gunzip = 0;
+
+	sp->wrk->is_gzip =
+	    http_HdrIs(sp->wrk->beresp, H_Content_Encoding, "gzip");
+
+	sp->wrk->is_gunzip =
+	    !http_GetHdr(sp->wrk->beresp, H_Content_Encoding, NULL);
+
+	/* It can't be both */
+	assert(sp->wrk->is_gzip == 0 || sp->wrk->is_gunzip == 0);
+
+	/* We won't gunzip unless it is gzip'ed */
+	if (sp->wrk->do_gunzip && !sp->wrk->is_gzip)
+		sp->wrk->do_gunzip = 0;
+
+	/* If we do gunzip, remove the C-E header */
+	if (sp->wrk->do_gunzip)
+		http_Unset(sp->wrk->beresp, H_Content_Encoding);
+
+	/* We wont gzip unless it is ungziped */
+	if (sp->wrk->do_gzip && !sp->wrk->is_gunzip)
+		sp->wrk->do_gzip = 0;
+
+	/* If we do gzip, add the C-E header */
+	if (sp->wrk->do_gzip)
+		http_SetHeader(sp->wrk, sp->vsl_id, sp->wrk->beresp,
+		    "Content-Encoding: gzip");
+
+	/* But we can't do both at the same time */
+	assert(sp->wrk->do_gzip == 0 || sp->wrk->do_gunzip == 0);
+
+	/* ESI takes precedence and handles gzip/gunzip itself */
+	if (sp->wrk->do_esi)
+		sp->wrk->vfp = &vfp_esi;
+	else if (sp->wrk->do_gunzip)
+		sp->wrk->vfp = &vfp_gunzip;
+	else if (sp->wrk->do_gzip)
+		sp->wrk->vfp = &vfp_gzip;
+	else if (sp->wrk->is_gzip)
+		sp->wrk->vfp = &vfp_testgzip;
+
+	if (sp->wrk->do_esi || sp->esi_level > 0)
+		sp->wrk->do_stream = 0;
+	if (!sp->wantbody)
+		sp->wrk->do_stream = 0;
+
+	l = http_EstimateWS(sp->wrk->beresp,
+	    pass ? HTTPH_R_PASS : HTTPH_A_INS, &nhttp);
+
+	/* Create Vary instructions */
+	if (sp->objcore != NULL) {
+		CHECK_OBJ_NOTNULL(sp->objcore, OBJCORE_MAGIC);
+		vary = VRY_Create(sp, sp->wrk->beresp);
+		if (vary != NULL) {
+			varyl = VSB_len(vary);
+			assert(varyl > 0);
+			l += varyl;
+		}
+	}
+
+	/*
+	 * Space for producing a Content-Length: header including padding
+	 * A billion gigabytes is enough for anybody.
+	 */
+	l += strlen("Content-Length: XxxXxxXxxXxxXxxXxx") + sizeof(void *);
+
+	if (sp->wrk->exp.ttl < cache_param->shortlived || sp->objcore == NULL)
+		sp->wrk->storage_hint = TRANSIENT_STORAGE;
+
+	sp->obj = STV_NewObject(sp, sp->wrk->storage_hint, l,
+	    &sp->wrk->exp, nhttp);
+	if (sp->obj == NULL) {
+		/*
+		 * Try to salvage the transaction by allocating a
+		 * shortlived object on Transient storage.
+		 */
+		sp->obj = STV_NewObject(sp, TRANSIENT_STORAGE, l,
+		    &sp->wrk->exp, nhttp);
+		if (sp->wrk->exp.ttl > cache_param->shortlived)
+			sp->wrk->exp.ttl = cache_param->shortlived;
+		sp->wrk->exp.grace = 0.0;
+		sp->wrk->exp.keep = 0.0;
+	}
+	if (sp->obj == NULL) {
+		sp->err_code = 503;
+		sp->step = STP_ERROR;
+		VDI_CloseFd(sp->wrk);
+		return (0);
+	}
+	CHECK_OBJ_NOTNULL(sp->obj, OBJECT_MAGIC);
+
+	sp->wrk->storage_hint = NULL;
+
+	if (sp->wrk->do_gzip || (sp->wrk->is_gzip && !sp->wrk->do_gunzip))
+		sp->obj->gziped = 1;
+
+	if (vary != NULL) {
+		sp->obj->vary =
+		    (void *)WS_Alloc(sp->obj->http->ws, varyl);
+		AN(sp->obj->vary);
+		memcpy(sp->obj->vary, VSB_data(vary), varyl);
+		VRY_Validate(sp->obj->vary);
+		VSB_delete(vary);
+	}
+
+	sp->obj->xid = sp->xid;
+	sp->obj->response = sp->err_code;
+	WS_Assert(sp->obj->ws_o);
+
+	/* Filter into object */
+	hp = sp->wrk->beresp;
+	hp2 = sp->obj->http;
+
+	hp2->logtag = HTTP_Obj;
+	http_CopyResp(hp2, hp);
+	http_FilterFields(sp->wrk, sp->vsl_id, hp2, hp,
+	    pass ? HTTPH_R_PASS : HTTPH_A_INS);
+	http_CopyHome(sp->wrk, sp->vsl_id, hp2);
+
+	if (http_GetHdr(hp, H_Last_Modified, &b))
+		sp->obj->last_modified = VTIM_parse(b);
+	else
+		sp->obj->last_modified = floor(sp->wrk->exp.entered);
+
+	assert(WRW_IsReleased(sp->wrk));
+
+	/*
+	 * If we can deliver a 304 reply, we don't bother streaming.
+	 * Notice that vcl_deliver{} could still nuke the headers
+	 * that allow the 304, in which case we return 200 non-stream.
+	 */
+	if (sp->obj->response == 200 &&
+	    sp->http->conds &&
+	    RFC2616_Do_Cond(sp))
+		sp->wrk->do_stream = 0;
+
+	AssertObjCorePassOrBusy(sp->obj->objcore);
+
+	if (sp->wrk->do_stream) {
+		sp->step = STP_PREPRESP;
+		return (0);
+	}
+
+	/* Use unmodified headers*/
+	i = FetchBody(sp->wrk, sp->obj);
+
+	sp->wrk->h_content_length = NULL;
+
+	http_Setup(sp->wrk->bereq, NULL);
+	http_Setup(sp->wrk->beresp, NULL);
+	sp->wrk->vfp = NULL;
+	assert(WRW_IsReleased(sp->wrk));
+	AZ(sp->wrk->vbc);
+	AN(sp->director);
+
+	if (i) {
+		HSH_Drop(sp);
+		AZ(sp->obj);
+		sp->err_code = 503;
+		sp->step = STP_ERROR;
+		return (0);
+	}
+
+	if (sp->obj->objcore != NULL) {
+		EXP_Insert(sp->obj);
+		AN(sp->obj->objcore);
+		AN(sp->obj->objcore->ban);
+		HSH_Unbusy(sp);
+	}
+	sp->wrk->acct_tmp.fetch++;
+	sp->step = STP_PREPRESP;
+	return (0);
+}
+
+/*--------------------------------------------------------------------
+ * Stream the body as we fetch it
+DOT subgraph xstreambody {
+DOT	streambody [
+DOT		shape=ellipse
+DOT		label="streaming\nfetch/deliver"
+DOT	]
+DOT }
+DOT streambody -> DONE [style=bold,color=cyan]
+ */
+
+static int
+cnt_streambody(struct sess *sp)
+{
+	int i;
+	struct stream_ctx sctx;
+	uint8_t obuf[sp->wrk->res_mode & RES_GUNZIP ?
+	    cache_param->gzip_stack_buffer : 1];
+
+	memset(&sctx, 0, sizeof sctx);
+	sctx.magic = STREAM_CTX_MAGIC;
+	AZ(sp->wrk->sctx);
+	sp->wrk->sctx = &sctx;
+
+	if (sp->wrk->res_mode & RES_GUNZIP) {
+		sctx.vgz = VGZ_NewUngzip(sp->wrk, "U S -");
+		sctx.obuf = obuf;
+		sctx.obuf_len = sizeof (obuf);
+	}
+
+	RES_StreamStart(sp);
+
+	AssertObjCorePassOrBusy(sp->obj->objcore);
+
+	i = FetchBody(sp->wrk, sp->obj);
+
+	sp->wrk->h_content_length = NULL;
+
+	http_Setup(sp->wrk->bereq, NULL);
+	http_Setup(sp->wrk->beresp, NULL);
+	sp->wrk->vfp = NULL;
+	AZ(sp->wrk->vbc);
+	AN(sp->director);
+
+	if (!i && sp->obj->objcore != NULL) {
+		EXP_Insert(sp->obj);
+		AN(sp->obj->objcore);
+		AN(sp->obj->objcore->ban);
+		HSH_Unbusy(sp);
+	} else {
+		sp->doclose = "Stream error";
+	}
+	sp->wrk->acct_tmp.fetch++;
+	sp->director = NULL;
+	sp->restarts = 0;
+
+	RES_StreamEnd(sp);
+	if (sp->wrk->res_mode & RES_GUNZIP)
+		(void)VGZ_Destroy(&sctx.vgz, sp->vsl_id);
+
+	sp->wrk->sctx = NULL;
+	assert(WRW_IsReleased(sp->wrk));
+	assert(sp->wrk->wrw.ciov == sp->wrk->wrw.siov);
+	(void)HSH_Deref(sp->wrk, NULL, &sp->obj);
+	http_Setup(sp->wrk->resp, NULL);
+	sp->step = STP_DONE;
+	return (0);
+}
+
+/*--------------------------------------------------------------------
+ * The very first request
+ */
+static int
+cnt_first(struct sess *sp)
+{
+
+	/*
+	 * XXX: If we don't have acceptfilters we are somewhat subject
+	 * XXX: to DoS'ing here.  One remedy would be to set a shorter
+	 * XXX: SO_RCVTIMEO and once we have received something here
+	 * XXX: increase it to the normal value.
+	 */
+
+	assert(sp->xid == 0);
+	assert(sp->restarts == 0);
+	VCA_Prep(sp);
+
+	/* Record the session watermark */
+	sp->ws_ses = WS_Snapshot(sp->ws);
+
+	/* Receive a HTTP protocol request */
+	HTC_Init(sp->htc, sp->ws, sp->fd, sp->vsl_id, cache_param->http_req_size,
+	    cache_param->http_req_hdr_len);
+	sp->wrk->lastused = sp->t_open;
+	sp->wrk->acct_tmp.sess++;
+
+	sp->step = STP_WAIT;
+	return (0);
+}
+
+/*--------------------------------------------------------------------
+ * HIT
+ * We had a cache hit.  Ask VCL, then march off as instructed.
+ *
+DOT subgraph xcluster_hit {
+DOT	hit [
+DOT		shape=record
+DOT		label="vcl_hit()|req.\nobj."
+DOT	]
+DOT }
+DOT hit -> err_hit [label="error"]
+DOT err_hit [label="ERROR",shape=plaintext]
+DOT hit -> rst_hit [label="restart",color=purple]
+DOT rst_hit [label="RESTART",shape=plaintext]
+DOT hit -> pass [label=pass,style=bold,color=red]
+DOT hit -> prepresp [label="deliver",style=bold,color=green]
+ */
+
+static int
+cnt_hit(struct sess *sp)
+{
+
+	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+	CHECK_OBJ_NOTNULL(sp->obj, OBJECT_MAGIC);
+	CHECK_OBJ_NOTNULL(sp->vcl, VCL_CONF_MAGIC);
+
+	assert(!(sp->obj->objcore->flags & OC_F_PASS));
+
+	AZ(sp->wrk->do_stream);
+
+	VCL_hit_method(sp);
+
+	if (sp->handling == VCL_RET_DELIVER) {
+		/* Dispose of any body part of the request */
+		(void)FetchReqBody(sp);
+		AZ(sp->wrk->bereq->ws);
+		AZ(sp->wrk->beresp->ws);
+		sp->step = STP_PREPRESP;
+		return (0);
+	}
+
+	/* Drop our object, we won't need it */
+	(void)HSH_Deref(sp->wrk, NULL, &sp->obj);
+	sp->objcore = NULL;
+
+	switch(sp->handling) {
+	case VCL_RET_PASS:
+		sp->step = STP_PASS;
+		return (0);
+	case VCL_RET_ERROR:
+		sp->step = STP_ERROR;
+		return (0);
+	case VCL_RET_RESTART:
+		sp->director = NULL;
+		sp->restarts++;
+		sp->step = STP_RECV;
+		return (0);
+	default:
+		WRONG("Illegal action in vcl_hit{}");
+	}
+}
+
+/*--------------------------------------------------------------------
+ * LOOKUP
+ * Hash things together and look object up in hash-table.
+ *
+ * LOOKUP consists of two substates so that we can reenter if we
+ * encounter a busy object.
+ *
+DOT subgraph xcluster_lookup {
+DOT	hash [
+DOT		shape=record
+DOT		label="vcl_hash()|req."
+DOT	]
+DOT	lookup [
+DOT		shape=diamond
+DOT		label="obj in cache ?\ncreate if not"
+DOT	]
+DOT	lookup2 [
+DOT		shape=diamond
+DOT		label="obj.f.pass ?"
+DOT	]
+DOT	hash -> lookup [label="hash",style=bold,color=green]
+DOT	lookup -> lookup2 [label="yes",style=bold,color=green]
+DOT }
+DOT lookup2 -> hit [label="no", style=bold,color=green]
+DOT lookup2 -> pass [label="yes",style=bold,color=red]
+DOT lookup -> miss [label="no",style=bold,color=blue]
+ */
+
+static int
+cnt_lookup(struct sess *sp)
+{
+	struct objcore *oc;
+	struct object *o;
+	struct objhead *oh;
+
+	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+	CHECK_OBJ_NOTNULL(sp->vcl, VCL_CONF_MAGIC);
+
+	if (sp->hash_objhead == NULL) {
+		/* Not a waiting list return */
+		AZ(sp->vary_b);
+		AZ(sp->vary_l);
+		AZ(sp->vary_e);
+		(void)WS_Reserve(sp->ws, 0);
+	} else {
+		AN(sp->ws->r);
+	}
+	sp->vary_b = (void*)sp->ws->f;
+	sp->vary_e = (void*)sp->ws->r;
+	sp->vary_b[2] = '\0';
+
+	oc = HSH_Lookup(sp, &oh);
+
+	if (oc == NULL) {
+		/*
+		 * We lost the session to a busy object, disembark the
+		 * worker thread.   The hash code to restart the session,
+		 * still in STP_LOOKUP, later when the busy object isn't.
+		 * NB:  Do not access sp any more !
+		 */
+		return (1);
+	}
+
+
+	CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
+	CHECK_OBJ_NOTNULL(oh, OBJHEAD_MAGIC);
+
+	/* If we inserted a new object it's a miss */
+	if (oc->flags & OC_F_BUSY) {
+		sp->wrk->stats.cache_miss++;
+
+		if (sp->vary_l != NULL) {
+			assert(oc->busyobj->vary == sp->vary_b);
+			VRY_Validate(oc->busyobj->vary);
+			WS_ReleaseP(sp->ws, (void*)sp->vary_l);
+		} else {
+			AZ(oc->busyobj->vary);
+			WS_Release(sp->ws, 0);
+		}
+		sp->vary_b = NULL;
+		sp->vary_l = NULL;
+		sp->vary_e = NULL;
+
+		sp->objcore = oc;
+		sp->step = STP_MISS;
+		return (0);
+	}
+
+	o = oc_getobj(sp->wrk, oc);
+	CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC);
+	sp->obj = o;
+
+	WS_Release(sp->ws, 0);
+	sp->vary_b = NULL;
+	sp->vary_l = NULL;
+	sp->vary_e = NULL;
+
+	if (oc->flags & OC_F_PASS) {
+		sp->wrk->stats.cache_hitpass++;
+		WSP(sp, SLT_HitPass, "%u", sp->obj->xid);
+		(void)HSH_Deref(sp->wrk, NULL, &sp->obj);
+		sp->objcore = NULL;
+		sp->step = STP_PASS;
+		return (0);
+	}
+
+	sp->wrk->stats.cache_hit++;
+	WSP(sp, SLT_Hit, "%u", sp->obj->xid);
+	sp->step = STP_HIT;
+	return (0);
+}
+
+/*--------------------------------------------------------------------
+ * We had a miss, ask VCL, proceed as instructed
+ *
+DOT subgraph xcluster_miss {
+DOT	miss [
+DOT		shape=ellipse
+DOT		label="filter req.->bereq."
+DOT	]
+DOT	vcl_miss [
+DOT		shape=record
+DOT		label="vcl_miss()|req.\nbereq."
+DOT	]
+DOT	miss -> vcl_miss [style=bold,color=blue]
+DOT }
+DOT vcl_miss -> rst_miss [label="restart",color=purple]
+DOT rst_miss [label="RESTART",shape=plaintext]
+DOT vcl_miss -> err_miss [label="error"]
+DOT err_miss [label="ERROR",shape=plaintext]
+DOT vcl_miss -> fetch [label="fetch",style=bold,color=blue]
+DOT vcl_miss -> pass [label="pass",style=bold,color=red]
+DOT
+ */
+
+static int
+cnt_miss(struct sess *sp)
+{
+
+	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+	CHECK_OBJ_NOTNULL(sp->vcl, VCL_CONF_MAGIC);
+
+	AZ(sp->obj);
+	AN(sp->objcore);
+	WS_Reset(sp->wrk->ws, NULL);
+	http_Setup(sp->wrk->bereq, sp->wrk->ws);
+	http_FilterHeader(sp, HTTPH_R_FETCH);
+	http_ForceGet(sp->wrk->bereq);
+	if (cache_param->http_gzip_support) {
+		/*
+		 * We always ask the backend for gzip, even if the
+		 * client doesn't grok it.  We will uncompress for
+		 * the minority of clients which don't.
+		 */
+		http_Unset(sp->wrk->bereq, H_Accept_Encoding);
+		http_SetHeader(sp->wrk, sp->vsl_id, sp->wrk->bereq,
+		    "Accept-Encoding: gzip");
+	}
+	sp->wrk->connect_timeout = 0;
+	sp->wrk->first_byte_timeout = 0;
+	sp->wrk->between_bytes_timeout = 0;
+	VCL_miss_method(sp);
+	switch(sp->handling) {
+	case VCL_RET_ERROR:
+		AZ(HSH_Deref(sp->wrk, sp->objcore, NULL));
+		sp->objcore = NULL;
+		http_Setup(sp->wrk->bereq, NULL);
+		sp->step = STP_ERROR;
+		return (0);
+	case VCL_RET_PASS:
+		AZ(HSH_Deref(sp->wrk, sp->objcore, NULL));
+		sp->objcore = NULL;
+		sp->step = STP_PASS;
+		return (0);
+	case VCL_RET_FETCH:
+		sp->step = STP_FETCH;
+		return (0);
+	case VCL_RET_RESTART:
+		AZ(HSH_Deref(sp->wrk, sp->objcore, NULL));
+		sp->objcore = NULL;
+		INCOMPL();
+	default:
+		WRONG("Illegal action in vcl_miss{}");
+	}
+}
+
+/*--------------------------------------------------------------------
+ * Start pass processing by getting headers from backend, then
+ * continue in passbody.
+ *
+DOT subgraph xcluster_pass {
+DOT	pass [
+DOT		shape=ellipse
+DOT		label="deref obj."
+DOT	]
+DOT	pass2 [
+DOT		shape=ellipse
+DOT		label="filter req.->bereq."
+DOT	]
+DOT	vcl_pass [
+DOT		shape=record
+DOT		label="vcl_pass()|req.\nbereq."
+DOT	]
+DOT	pass_do [
+DOT		shape=ellipse
+DOT		label="create anon object\n"
+DOT	]
+DOT	pass -> pass2 [style=bold, color=red]
+DOT	pass2 -> vcl_pass [style=bold, color=red]
+DOT	vcl_pass -> pass_do [label="pass"] [style=bold, color=red]
+DOT }
+DOT pass_do -> fetch [style=bold, color=red]
+DOT vcl_pass -> rst_pass [label="restart",color=purple]
+DOT rst_pass [label="RESTART",shape=plaintext]
+DOT vcl_pass -> err_pass [label="error"]
+DOT err_pass [label="ERROR",shape=plaintext]
+ */
+
+static int
+cnt_pass(struct sess *sp)
+{
+
+	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+	CHECK_OBJ_NOTNULL(sp->vcl, VCL_CONF_MAGIC);
+	AZ(sp->obj);
+
+	WS_Reset(sp->wrk->ws, NULL);
+	http_Setup(sp->wrk->bereq, sp->wrk->ws);
+	http_FilterHeader(sp, HTTPH_R_PASS);
+
+	sp->wrk->connect_timeout = 0;
+	sp->wrk->first_byte_timeout = 0;
+	sp->wrk->between_bytes_timeout = 0;
+	VCL_pass_method(sp);
+	if (sp->handling == VCL_RET_ERROR) {
+		http_Setup(sp->wrk->bereq, NULL);
+		sp->step = STP_ERROR;
+		return (0);
+	}
+	assert(sp->handling == VCL_RET_PASS);
+	sp->wrk->acct_tmp.pass++;
+	sp->sendbody = 1;
+	sp->step = STP_FETCH;
+	return (0);
+}
+
+/*--------------------------------------------------------------------
+ * Ship the request header to the backend unchanged, then pipe
+ * until one of the ends close the connection.
+ *
+DOT subgraph xcluster_pipe {
+DOT	pipe [
+DOT		shape=ellipse
+DOT		label="Filter req.->bereq."
+DOT	]
+DOT	vcl_pipe [
+DOT		shape=record
+DOT		label="vcl_pipe()|req.\nbereq\."
+DOT	]
+DOT	pipe_do [
+DOT		shape=ellipse
+DOT		label="send bereq.\npipe until close"
+DOT	]
+DOT	vcl_pipe -> pipe_do [label="pipe",style=bold,color=orange]
+DOT	pipe -> vcl_pipe [style=bold,color=orange]
+DOT }
+DOT pipe_do -> DONE [style=bold,color=orange]
+DOT vcl_pipe -> err_pipe [label="error"]
+DOT err_pipe [label="ERROR",shape=plaintext]
+ */
+
+static int
+cnt_pipe(struct sess *sp)
+{
+
+	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+	CHECK_OBJ_NOTNULL(sp->vcl, VCL_CONF_MAGIC);
+
+	sp->wrk->acct_tmp.pipe++;
+	WS_Reset(sp->wrk->ws, NULL);
+	http_Setup(sp->wrk->bereq, sp->wrk->ws);
+	http_FilterHeader(sp, HTTPH_R_PIPE);
+
+	VCL_pipe_method(sp);
+
+	if (sp->handling == VCL_RET_ERROR)
+		INCOMPL();
+	assert(sp->handling == VCL_RET_PIPE);
+
+	PipeSession(sp);
+	assert(WRW_IsReleased(sp->wrk));
+	http_Setup(sp->wrk->bereq, NULL);
+	sp->step = STP_DONE;
+	return (0);
+}
+
+/*--------------------------------------------------------------------
+ * RECV
+ * We have a complete request, set everything up and start it.
+ *
+DOT subgraph xcluster_recv {
+DOT	recv [
+DOT		shape=record
+DOT		label="vcl_recv()|req."
+DOT	]
+DOT }
+DOT RESTART -> recv
+DOT recv -> pipe [label="pipe",style=bold,color=orange]
+DOT recv -> pass2 [label="pass",style=bold,color=red]
+DOT recv -> err_recv [label="error"]
+DOT err_recv [label="ERROR",shape=plaintext]
+DOT recv -> hash [label="lookup",style=bold,color=green]
+ */
+
+static int
+cnt_recv(struct sess *sp)
+{
+	unsigned recv_handling;
+
+	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+	CHECK_OBJ_NOTNULL(sp->vcl, VCL_CONF_MAGIC);
+	AZ(sp->obj);
+	assert(sp->wrk->wrw.ciov == sp->wrk->wrw.siov);
+
+	/* By default we use the first backend */
+	AZ(sp->director);
+	sp->director = sp->vcl->director[0];
+	AN(sp->director);
+
+	sp->disable_esi = 0;
+	sp->hash_always_miss = 0;
+	sp->hash_ignore_busy = 0;
+	sp->client_identity = NULL;
+
+	http_CollectHdr(sp->http, H_Cache_Control);
+
+	VCL_recv_method(sp);
+	recv_handling = sp->handling;
+
+	if (sp->restarts >= cache_param->max_restarts) {
+		if (sp->err_code == 0)
+			sp->err_code = 503;
+		sp->step = STP_ERROR;
+		return (0);
+	}
+
+	/* Zap these, in case we came here through restart */
+	sp->wrk->do_esi = 0;
+	sp->wrk->is_gzip = 0;
+	sp->wrk->is_gunzip = 0;
+	sp->wrk->do_gzip = 0;
+	sp->wrk->do_gunzip = 0;
+	sp->wrk->do_stream = 0;
+
+	if (cache_param->http_gzip_support &&
+	     (recv_handling != VCL_RET_PIPE) &&
+	     (recv_handling != VCL_RET_PASS)) {
+		if (RFC2616_Req_Gzip(sp)) {
+			http_Unset(sp->http, H_Accept_Encoding);
+			http_SetHeader(sp->wrk, sp->vsl_id, sp->http,
+			    "Accept-Encoding: gzip");
+		} else {
+			http_Unset(sp->http, H_Accept_Encoding);
+		}
+	}
+
+	SHA256_Init(sp->wrk->sha256ctx);
+	VCL_hash_method(sp);
+	assert(sp->handling == VCL_RET_HASH);
+	SHA256_Final(sp->digest, sp->wrk->sha256ctx);
+
+	if (!strcmp(sp->http->hd[HTTP_HDR_REQ].b, "HEAD"))
+		sp->wantbody = 0;
+	else
+		sp->wantbody = 1;
+
+	sp->sendbody = 0;
+	switch(recv_handling) {
+	case VCL_RET_LOOKUP:
+		/* XXX: discard req body, if any */
+		sp->step = STP_LOOKUP;
+		return (0);
+	case VCL_RET_PIPE:
+		if (sp->esi_level > 0) {
+			/* XXX: VSL something */
+			INCOMPL();
+			/* sp->step = STP_DONE; */
+			return (1);
+		}
+		sp->step = STP_PIPE;
+		return (0);
+	case VCL_RET_PASS:
+		sp->step = STP_PASS;
+		return (0);
+	case VCL_RET_ERROR:
+		/* XXX: discard req body, if any */
+		sp->step = STP_ERROR;
+		return (0);
+	default:
+		WRONG("Illegal action in vcl_recv{}");
+	}
+}
+
+/*--------------------------------------------------------------------
+ * START
+ * Handle a request, wherever it came from recv/restart.
+ *
+DOT start [shape=box,label="Dissect request"]
+DOT start -> recv [style=bold,color=green]
+ */
+
+static int
+cnt_start(struct sess *sp)
+{
+	uint16_t done;
+	char *p;
+	const char *r = "HTTP/1.1 100 Continue\r\n\r\n";
+
+	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+	AZ(sp->restarts);
+	AZ(sp->obj);
+	AZ(sp->vcl);
+
+	/* Update stats of various sorts */
+	sp->wrk->stats.client_req++;
+	sp->t_req = VTIM_real();
+	sp->wrk->lastused = sp->t_req;
+	sp->wrk->acct_tmp.req++;
+
+	/* Assign XID and log */
+	sp->xid = ++xids;				/* XXX not locked */
+	WSP(sp, SLT_ReqStart, "%s %s %u", sp->addr, sp->port,  sp->xid);
+
+	/* Borrow VCL reference from worker thread */
+	VCL_Refresh(&sp->wrk->vcl);
+	sp->vcl = sp->wrk->vcl;
+	sp->wrk->vcl = NULL;
+
+	http_Setup(sp->http, sp->ws);
+	done = http_DissectRequest(sp);
+
+	/* If we could not even parse the request, just close */
+	if (done == 400) {
+		sp->step = STP_DONE;
+		SES_Close(sp, "junk");
+		return (0);
+	}
+
+	/* Catch request snapshot */
+	sp->ws_req = WS_Snapshot(sp->ws);
+
+	/* Catch original request, before modification */
+	HTTP_Copy(sp->http0, sp->http);
+
+	if (done != 0) {
+		sp->err_code = done;
+		sp->step = STP_ERROR;
+		return (0);
+	}
+
+	sp->doclose = http_DoConnection(sp->http);
+
+	/* XXX: Handle TRACE & OPTIONS of Max-Forwards = 0 */
+
+	/*
+	 * Handle Expect headers
+	 */
+	if (http_GetHdr(sp->http, H_Expect, &p)) {
+		if (strcasecmp(p, "100-continue")) {
+			sp->err_code = 417;
+			sp->step = STP_ERROR;
+			return (0);
+		}
+
+		/* XXX: Don't bother with write failures for now */
+		(void)write(sp->fd, r, strlen(r));
+		/* XXX: When we do ESI includes, this is not removed
+		 * XXX: because we use http0 as our basis.  Believed
+		 * XXX: safe, but potentially confusing.
+		 */
+		http_Unset(sp->http, H_Expect);
+	}
+
+	sp->step = STP_RECV;
+	return (0);
+}
+
+/*--------------------------------------------------------------------
+ * Central state engine dispatcher.
+ *
+ * Kick the session around until it has had enough.
+ *
+ */
+
+static void
+cnt_diag(struct sess *sp, const char *state)
+{
+	if (sp->wrk != NULL) {
+		WSP(sp, SLT_Debug, "thr %p STP_%s sp %p obj %p vcl %p",
+		    pthread_self(), state, sp, sp->obj, sp->vcl);
+		WSL_Flush(sp->wrk, 0);
+	} else {
+		VSL(SLT_Debug, sp->vsl_id,
+		    "thr %p STP_%s sp %p obj %p vcl %p",
+		    pthread_self(), state, sp, sp->obj, sp->vcl);
+	}
+}
+
+void
+CNT_Session(struct sess *sp)
+{
+	int done;
+	struct worker *w;
+
+	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+	w = sp->wrk;
+	CHECK_OBJ_NOTNULL(w, WORKER_MAGIC);
+
+	/*
+	 * Possible entrance states
+	 */
+	assert(
+	    sp->step == STP_FIRST ||
+	    sp->step == STP_START ||
+	    sp->step == STP_LOOKUP ||
+	    sp->step == STP_RECV);
+
+	AZ(w->do_stream);
+	AZ(w->is_gzip);
+	AZ(w->do_gzip);
+	AZ(w->is_gunzip);
+	AZ(w->do_gunzip);
+	AZ(w->do_esi);
+
+	/*
+	 * Whenever we come in from the acceptor or waiter, we need to set
+	 * blocking mode, but there is no point in setting it when we come from
+	 * ESI or when a parked sessions returns.
+	 * It would be simpler to do this in the acceptor or waiter, but we'd
+	 * rather do the syscall in the worker thread.
+	 * On systems which return errors for ioctl, we close early
+	 */
+	if ((sp->step == STP_FIRST || sp->step == STP_START) &&
+	    VTCP_blocking(sp->fd)) {
+		if (errno == ECONNRESET)
+			SES_Close(sp, "remote closed");
+		else
+			SES_Close(sp, "error");
+		sp->step = STP_DONE;
+	}
+
+	/*
+	 * NB: Once done is set, we can no longer touch sp!
+	 */
+	for (done = 0; !done; ) {
+		assert(sp->wrk == w);
+		/*
+		 * This is a good place to be paranoid about the various
+		 * pointers still pointing to the things we expect.
+		 */
+		CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+		CHECK_OBJ_ORNULL(sp->obj, OBJECT_MAGIC);
+		CHECK_OBJ_NOTNULL(sp->wrk, WORKER_MAGIC);
+		CHECK_OBJ_ORNULL(w->nobjhead, OBJHEAD_MAGIC);
+		WS_Assert(w->ws);
+
+		switch (sp->step) {
+#define STEP(l,u) \
+		    case STP_##u: \
+			if (cache_param->diag_bitmap & 0x01) \
+				cnt_diag(sp, #u); \
+			done = cnt_##l(sp); \
+		        break;
+#include "tbl/steps.h"
+#undef STEP
+		default:
+			WRONG("State engine misfire");
+		}
+		WS_Assert(w->ws);
+		CHECK_OBJ_ORNULL(w->nobjhead, OBJHEAD_MAGIC);
+	}
+	WSL_Flush(w, 0);
+	AZ(w->do_stream);
+	AZ(w->is_gzip);
+	AZ(w->do_gzip);
+	AZ(w->is_gunzip);
+	AZ(w->do_gunzip);
+	AZ(w->do_esi);
+#define ACCT(foo)	AZ(w->acct_tmp.foo);
+#include "tbl/acct_fields.h"
+#undef ACCT
+	assert(WRW_IsReleased(w));
+}
+
+/*
+DOT }
+*/
+
+/*--------------------------------------------------------------------
+ * Debugging aids
+ */
+
+static void
+cli_debug_xid(struct cli *cli, const char * const *av, void *priv)
+{
+	(void)priv;
+	if (av[2] != NULL)
+		xids = strtoul(av[2], NULL, 0);
+	VCLI_Out(cli, "XID is %u", xids);
+}
+
+/*
+ * Default to seed=1, this is the only seed value POSIXl guarantees will
+ * result in a reproducible random number sequence.
+ */
+static void
+cli_debug_srandom(struct cli *cli, const char * const *av, void *priv)
+{
+	(void)priv;
+	unsigned seed = 1;
+
+	if (av[2] != NULL)
+		seed = strtoul(av[2], NULL, 0);
+	srandom(seed);
+	srand48(random());
+	VCLI_Out(cli, "Random(3) seeded with %lu", seed);
+}
+
+static struct cli_proto debug_cmds[] = {
+	{ "debug.xid", "debug.xid",
+		"\tExamine or set XID\n", 0, 1, "d", cli_debug_xid },
+	{ "debug.srandom", "debug.srandom",
+		"\tSeed the random(3) function\n", 0, 1, "d", cli_debug_srandom },
+	{ NULL }
+};
+
+/*--------------------------------------------------------------------
+ *
+ */
+
+void
+CNT_Init(void)
+{
+
+	srandomdev();
+	srand48(random());
+	xids = random();
+	CLI_AddFuncs(debug_cmds);
+}
+
+
diff --git a/bin/varnishd/cache/cache_cli.c b/bin/varnishd/cache/cache_cli.c
new file mode 100644
index 0000000..f29f86a
--- /dev/null
+++ b/bin/varnishd/cache/cache_cli.c
@@ -0,0 +1,243 @@
+/*-
+ * Copyright (c) 2006 Verdens Gang AS
+ * Copyright (c) 2006-2011 Varnish Software AS
+ * All rights reserved.
+ *
+ * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * Caching process CLI handling.
+ *
+ * We only have one CLI source, the stdin/stdout pipes from the manager
+ * process, but we complicate things by having undocumented commands that
+ * we do not want to show in a plain help, and by having commands that the
+ * manager has already shown in help before asking us.
+ */
+
+#include "config.h"
+
+#include <stddef.h>			// offsetof
+
+#include "cache.h"
+#include "common/heritage.h"
+
+#include "cache_backend.h"		// struct vbc
+#include "hash/hash_slinger.h"		// struct objhead
+#include "vcli.h"
+#include "vcli_common.h"
+#include "vcli_priv.h"
+#include "vcli_serve.h"
+
+pthread_t		cli_thread;
+static struct lock	cli_mtx;
+static int		add_check;
+static struct VCLS	*cls;
+
+/*
+ * The CLI commandlist is split in three:
+ *  - Commands we get from/share with the manager, we don't show these
+ *	in help, as the manager already did that.
+ *  - Cache process commands, show in help
+ *  - Undocumented debug commands, show in undocumented "help -d"
+ */
+
+/*--------------------------------------------------------------------
+ * Add CLI functions to the appropriate command set
+ */
+
+void
+CLI_AddFuncs(struct cli_proto *p)
+{
+
+	AZ(add_check);
+	Lck_Lock(&cli_mtx);
+	AZ(VCLS_AddFunc(cls, 0, p));
+	Lck_Unlock(&cli_mtx);
+}
+
+static void
+cli_cb_before(const struct cli *cli)
+{
+
+	ASSERT_CLI();
+	VSL(SLT_CLI, 0, "Rd %s", cli->cmd);
+	VCL_Poll();
+	VBE_Poll();
+	Lck_Lock(&cli_mtx);
+}
+
+static void
+cli_cb_after(const struct cli *cli)
+{
+
+	ASSERT_CLI();
+	Lck_Unlock(&cli_mtx);
+	VSL(SLT_CLI, 0, "Wr %03u %u %s",
+	    cli->result, VSB_len(cli->sb), VSB_data(cli->sb));
+}
+
+void
+CLI_Run(void)
+{
+	int i;
+
+	add_check = 1;
+
+	AN(VCLS_AddFd(cls, heritage.cli_in, heritage.cli_out, NULL, NULL));
+
+	do {
+		i = VCLS_Poll(cls, -1);
+	} while(i > 0);
+	VSL(SLT_CLI, 0, "EOF on CLI connection, worker stops");
+	VCA_Shutdown();
+}
+
+/*--------------------------------------------------------------------*/
+
+static void
+cli_debug_sizeof(struct cli *cli, const char * const *av, void *priv)
+{
+	(void)av;
+	(void)priv;
+
+#define SZOF(foo)       VCLI_Out(cli, \
+    "sizeof(%s) = %zd = 0x%zx\n", #foo, sizeof(foo), sizeof(foo))
+	SZOF(struct ws);
+	SZOF(struct http);
+	SZOF(struct http_conn);
+	SZOF(struct acct);
+	SZOF(struct worker);
+	SZOF(struct storage);
+	SZOF(struct object);
+	SZOF(struct objcore);
+	SZOF(struct objhead);
+	SZOF(struct sess);
+	SZOF(struct vbc);
+	SZOF(struct VSC_C_main);
+	SZOF(struct lock);
+#if 0
+#define OFOF(foo, bar)	{ foo __foo; VCLI_Out(cli, \
+    "%-30s = 0x%4zx @ 0x%4zx\n", \
+	#foo "." #bar, sizeof(__foo.bar), offsetof(foo, bar)); }
+#if 0
+	OFOF(struct objhead, magic);
+	OFOF(struct objhead, refcnt);
+	OFOF(struct objhead, mtx);
+	OFOF(struct objhead, objcs);
+	OFOF(struct objhead, digest);
+	OFOF(struct objhead, waitinglist);
+	OFOF(struct objhead, _u);
+#endif
+#if 0
+	OFOF(struct http, magic);
+	OFOF(struct http, logtag);
+	OFOF(struct http, ws);
+	OFOF(struct http, hd);
+	OFOF(struct http, hdf);
+	OFOF(struct http, shd);
+	OFOF(struct http, nhd);
+	OFOF(struct http, status);
+	OFOF(struct http, protover);
+	OFOF(struct http, conds);
+#endif
+#if 0
+	OFOF(struct storage, magic);
+	OFOF(struct storage, fd);
+	OFOF(struct storage, where);
+	OFOF(struct storage, list);
+	OFOF(struct storage, stevedore);
+	OFOF(struct storage, priv);
+	OFOF(struct storage, ptr);
+	OFOF(struct storage, len);
+	OFOF(struct storage, space);
+#endif
+#if 0
+	OFOF(struct object, magic);
+	OFOF(struct object, xid);
+	OFOF(struct object, objstore);
+	OFOF(struct object, objcore);
+	OFOF(struct object, ws_o);
+	OFOF(struct object, vary);
+	OFOF(struct object, hits);
+	OFOF(struct object, response);
+	OFOF(struct object, gziped);
+	OFOF(struct object, gzip_start);
+	OFOF(struct object, gzip_last);
+	OFOF(struct object, gzip_stop);
+	OFOF(struct object, len);
+	OFOF(struct object, age);
+	OFOF(struct object, entered);
+	OFOF(struct object, exp);
+	OFOF(struct object, last_modified);
+	OFOF(struct object, last_lru);
+	OFOF(struct object, http);
+	OFOF(struct object, store);
+	OFOF(struct object, esidata);
+	OFOF(struct object, last_use);
+#endif
+#undef OFOF
+#endif
+}
+
+/*--------------------------------------------------------------------*/
+
+static void
+ccf_panic(struct cli *cli, const char * const *av, void *priv)
+{
+
+	(void)cli;
+	(void)av;
+	(void)priv;
+	assert(!strcmp("", "You asked for it"));
+}
+
+/*--------------------------------------------------------------------*/
+
+static struct cli_proto master_cmds[] = {
+	{ CLI_PING,		"i", VCLS_func_ping },
+	{ CLI_HELP,             "i", VCLS_func_help },
+	{ "debug.sizeof", "debug.sizeof",
+		"\tDump sizeof various data structures\n",
+		0, 0, "d", cli_debug_sizeof },
+	{ "debug.panic.worker", "debug.panic.worker",
+		"\tPanic the worker process.\n",
+		0, 0, "d", ccf_panic },
+	{ NULL }
+};
+
+/*--------------------------------------------------------------------
+ * Initialize the CLI subsystem
+ */
+
+void
+CLI_Init(void)
+{
+
+	Lck_New(&cli_mtx, lck_cli);
+	cli_thread = pthread_self();
+
+	cls = VCLS_New(cli_cb_before, cli_cb_after, cache_param->cli_buffer);
+	AN(cls);
+
+	CLI_AddFuncs(master_cmds);
+}
diff --git a/bin/varnishd/cache/cache_dir.c b/bin/varnishd/cache/cache_dir.c
new file mode 100644
index 0000000..d4794f9
--- /dev/null
+++ b/bin/varnishd/cache/cache_dir.c
@@ -0,0 +1,121 @@
+/*-
+ * Copyright (c) 2006 Verdens Gang AS
+ * Copyright (c) 2006-2011 Varnish Software AS
+ * All rights reserved.
+ *
+ * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * Handle backend connections and backend request structures.
+ *
+ */
+
+#include "config.h"
+
+#include "cache.h"
+
+#include "cache_backend.h"
+#include "vtcp.h"
+
+/* Close a connection ------------------------------------------------*/
+
+void
+VDI_CloseFd(struct worker *wrk)
+{
+	struct backend *bp;
+
+	CHECK_OBJ_NOTNULL(wrk->vbc, VBC_MAGIC);
+	CHECK_OBJ_NOTNULL(wrk->vbc->backend, BACKEND_MAGIC);
+	assert(wrk->vbc->fd >= 0);
+
+	bp = wrk->vbc->backend;
+
+	WSL(wrk, SLT_BackendClose, wrk->vbc->vsl_id, "%s", bp->display_name);
+
+	/* Checkpoint log to flush all info related to this connection
+	   before the OS reuses the FD */
+	WSL_Flush(wrk, 0);
+
+	VTCP_close(&wrk->vbc->fd);
+	VBE_DropRefConn(bp);
+	wrk->vbc->backend = NULL;
+	VBE_ReleaseConn(wrk->vbc);
+	wrk->vbc = NULL;
+	wrk->do_close = 0;
+}
+
+/* Recycle a connection ----------------------------------------------*/
+
+void
+VDI_RecycleFd(struct worker *wrk)
+{
+	struct backend *bp;
+
+	CHECK_OBJ_NOTNULL(wrk->vbc, VBC_MAGIC);
+	CHECK_OBJ_NOTNULL(wrk->vbc->backend, BACKEND_MAGIC);
+	assert(wrk->vbc->fd >= 0);
+	AZ(wrk->do_close);
+
+	bp = wrk->vbc->backend;
+
+	WSL(wrk, SLT_BackendReuse, wrk->vbc->vsl_id, "%s", bp->display_name);
+	/*
+	 * Flush the shmlog, so that another session reusing this backend
+	 * will log chronologically later than our use of it.
+	 */
+	WSL_Flush(wrk, 0);
+	Lck_Lock(&bp->mtx);
+	VSC_C_main->backend_recycle++;
+	VTAILQ_INSERT_HEAD(&bp->connlist, wrk->vbc, list);
+	wrk->vbc = NULL;
+	VBE_DropRefLocked(bp);
+}
+
+/* Get a connection --------------------------------------------------*/
+
+struct vbc *
+VDI_GetFd(const struct director *d, struct sess *sp)
+{
+
+	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+	if (d == NULL)
+		d = sp->director;
+	CHECK_OBJ_NOTNULL(d, DIRECTOR_MAGIC);
+	return (d->getfd(d, sp));
+}
+
+/* Check health ------------------------------------------------------
+ *
+ * The target is really an objhead pointer, but since it can not be
+ * dereferenced during health-checks, we pass it as uintptr_t, which
+ * hopefully will make people investigate, before mucking about with it.
+ */
+
+int
+VDI_Healthy(const struct director *d, const struct sess *sp)
+{
+
+	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+	CHECK_OBJ_NOTNULL(d, DIRECTOR_MAGIC);
+	return (d->healthy(d, sp));
+}
diff --git a/bin/varnishd/cache/cache_dir_dns.c b/bin/varnishd/cache/cache_dir_dns.c
new file mode 100644
index 0000000..ae96dbb
--- /dev/null
+++ b/bin/varnishd/cache/cache_dir_dns.c
@@ -0,0 +1,469 @@
+/*-
+ * Copyright (c) 2009-2010 Varnish Software AS
+ * All rights reserved.
+ *
+ * Author: Kristian Lyngstol <kristian at redpill-linpro.com>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ */
+
+#include "config.h"
+
+#include <netinet/in.h>
+
+#include <netdb.h>
+#include <stdio.h>
+#include <stdlib.h>
+
+#include "cache.h"
+
+#include "cache_backend.h"
+#include "vrt.h"
+
+/*--------------------------------------------------------------------*/
+
+/* FIXME: Should eventually be a configurable variable. */
+#define VDI_DNS_MAX_CACHE		1024
+#define VDI_DNS_GROUP_MAX_BACKENDS	1024
+
+/* DNS Cache entry
+ */
+struct vdi_dns_hostgroup {
+	unsigned			magic;
+#define VDI_DNSDIR_MAGIC		0x1bacab21
+	char				*hostname;
+	struct director			*hosts[VDI_DNS_GROUP_MAX_BACKENDS];
+	unsigned			nhosts;
+	unsigned			next_host; /* Next to use...*/
+	double				ttl;
+	VTAILQ_ENTRY(vdi_dns_hostgroup)	list;
+};
+
+struct vdi_dns {
+	unsigned			magic;
+#define VDI_DNS_MAGIC			0x1337a178
+	struct director			dir;
+	struct director			**hosts;
+	unsigned			nhosts;
+	VTAILQ_HEAD(_cachelist,vdi_dns_hostgroup)	cachelist;
+	unsigned			ncachelist;
+	pthread_rwlock_t		rwlock;
+	const char			*suffix;
+	double			ttl;
+};
+
+/* Compare an IPv4 backend to a IPv4 addr/len */
+static int
+vdi_dns_comp_addrinfo4(const struct backend *bp,
+		       const struct sockaddr_storage *addr,
+		       const socklen_t len)
+{
+	uint32_t u, p;
+	const struct sockaddr_in *bps = (const void *)bp->ipv4;
+	const struct sockaddr_in *bpd = (const void *)addr;
+
+	if (bp->ipv4len != len || len <= 0)
+		return (0);
+
+	u = bpd->sin_addr.s_addr;
+	p = bps->sin_addr.s_addr;
+
+	return (u == p);
+}
+
+/* Compare an IPv6 backend to a IPv6 addr/len */
+static int
+vdi_dns_comp_addrinfo6(const struct backend *bp,
+		       const struct sockaddr_storage *addr,
+		       const socklen_t len)
+{
+	const uint8_t *u, *p;
+	const struct sockaddr_in6 *bps = (const void *)bp->ipv6;
+	const struct sockaddr_in6 *bpd = (const void *)addr;
+
+	if (bp->ipv6len != len || len <= 0)
+		return (0);
+
+	u = bpd->sin6_addr.s6_addr;
+	p = bps->sin6_addr.s6_addr;
+
+	return (!memcmp(u, p, 16));
+}
+
+/* Check if a backends socket is the same as addr */
+static int
+vdi_dns_comp_addrinfo(const struct director *dir,
+		      const struct sockaddr_storage *addr,
+		      const socklen_t len)
+{
+	struct backend *bp;
+
+	bp = vdi_get_backend_if_simple(dir);
+	AN(bp);
+	if (addr->ss_family == PF_INET && bp->ipv4) {
+		return (vdi_dns_comp_addrinfo4(bp, addr, len));
+	} else if (addr->ss_family == PF_INET6 && bp->ipv6) {
+		return (vdi_dns_comp_addrinfo6(bp, addr, len));
+	}
+	return (0);
+}
+
+/* Pick a host from an existing hostgroup.
+ * Balance on round-robin if multiple backends are available and only pick
+ * healthy ones.
+ */
+static struct director *
+vdi_dns_pick_host(const struct sess *sp, struct vdi_dns_hostgroup *group) {
+	int initial, i, nhosts, current;
+	if (group->nhosts == 0)
+		return (NULL); // In case of error.
+	if (group->next_host >= group->nhosts)
+		group->next_host = 0;
+
+	/* Pick a healthy backend */
+	initial = group->next_host;
+	nhosts = group->nhosts;
+	for (i=0; i < nhosts; i++) {
+		if (i + initial >= nhosts)
+			current = i + initial - nhosts;
+		else
+			current = i + initial;
+		if (VDI_Healthy(group->hosts[current], sp)) {
+			group->next_host = current+1;
+			return (group->hosts[current]);
+		}
+	}
+
+	return (NULL);
+}
+
+/* Remove an item from the dns cache.
+ * If *group is NULL, the head is popped.
+ * Remember locking.
+ */
+static void
+vdi_dns_pop_cache(struct vdi_dns *vs,
+		  struct vdi_dns_hostgroup *group)
+{
+	if (group == NULL)
+		group = VTAILQ_LAST( &vs->cachelist, _cachelist );
+	assert(group != NULL);
+	free(group->hostname);
+	VTAILQ_REMOVE(&vs->cachelist, group, list);
+	FREE_OBJ(group);
+	vs->ncachelist--;
+}
+
+/* Dummy in case someone feels like optimizing it? meh...
+ */
+static inline int
+vdi_dns_groupmatch(const struct vdi_dns_hostgroup *group, const char *hostname)
+{
+	return (!strcmp(group->hostname, hostname));
+}
+
+/* Search the cache for 'hostname' and put a backend-pointer as necessary,
+ * return true for cache hit. This could still be a NULL backend if we did
+ * a lookup earlier and didn't find a host (ie: cache failed too)
+ *
+ * if rwlock is true, the first timed out object found (if any) is popped
+ * and freed.
+ */
+static int
+vdi_dns_cache_has(const struct sess *sp,
+		  struct vdi_dns *vs,
+		  const char *hostname,
+		  struct director **backend,
+		  int rwlock)
+{
+	struct director *ret;
+	struct vdi_dns_hostgroup *hostgr;
+	struct vdi_dns_hostgroup *hostgr2;
+	VTAILQ_FOREACH_SAFE(hostgr, &vs->cachelist, list, hostgr2) {
+		CHECK_OBJ_NOTNULL(hostgr, VDI_DNSDIR_MAGIC);
+		if (hostgr->ttl <= sp->t_req) {
+			if (rwlock)
+				vdi_dns_pop_cache(vs, hostgr);
+			return (0);
+		}
+		if (vdi_dns_groupmatch(hostgr, hostname)) {
+			ret = (vdi_dns_pick_host(sp, hostgr));
+			*backend = ret;
+			if (*backend != NULL)
+				CHECK_OBJ_NOTNULL(*backend, DIRECTOR_MAGIC);
+			return (1);
+		}
+	}
+	return (0);
+}
+
+/* Add a newly cached item to the dns cache list.
+ * (Sorry for the list_add/_add confusion...)
+ */
+static void
+vdi_dns_cache_list_add(const struct sess *sp,
+		       struct vdi_dns *vs,
+		       struct vdi_dns_hostgroup *new)
+{
+	if (vs->ncachelist >= VDI_DNS_MAX_CACHE) {
+		VSC_C_main->dir_dns_cache_full++;
+		vdi_dns_pop_cache(vs, NULL);
+	}
+	CHECK_OBJ_NOTNULL(new, VDI_DNSDIR_MAGIC);
+	assert(new->hostname != 0);
+	new->ttl = sp->t_req + vs->ttl;
+	VTAILQ_INSERT_HEAD(&vs->cachelist, new, list);
+	vs->ncachelist++;
+}
+
+/* Add an item to the dns cache.
+ * XXX: Might want to factor the getaddrinfo() out of the lock and do the
+ * cache_has() afterwards to do multiple dns lookups in parallel...
+ */
+static int
+vdi_dns_cache_add(const struct sess *sp,
+		  struct vdi_dns *vs,
+		  const char *hostname,
+		  struct director **backend)
+{
+	int error, i, host = 0;
+	struct addrinfo *res0, *res, hint;
+	struct vdi_dns_hostgroup *new;
+
+	/* Due to possible race while upgrading the lock, we have to
+	 * recheck if the result is already looked up. The overhead for
+	 * this is insignificant unless dns isn't cached properly (all
+	 * unique names or something equally troublesome).
+	 */
+
+	if (vdi_dns_cache_has(sp, vs, hostname, backend, 1))
+		return (1);
+
+	memset(&hint, 0, sizeof hint);
+	hint.ai_family = PF_UNSPEC;
+	hint.ai_socktype = SOCK_STREAM;
+
+	ALLOC_OBJ(new, VDI_DNSDIR_MAGIC);
+	XXXAN(new);
+
+	REPLACE(new->hostname, hostname);
+
+	error = getaddrinfo(hostname, "80", &hint, &res0);
+	VSC_C_main->dir_dns_lookups++;
+	if (error) {
+		vdi_dns_cache_list_add(sp, vs, new);
+		VSC_C_main->dir_dns_failed++;
+		return (0);
+	}
+
+	for (res = res0; res; res = res->ai_next) {
+		if (res->ai_family != PF_INET && res->ai_family != PF_INET6)
+			continue;
+
+		for (i = 0; i < vs->nhosts; i++) {
+			struct sockaddr_storage ss_hack;
+			memcpy(&ss_hack, res->ai_addr, res->ai_addrlen);
+			if (vdi_dns_comp_addrinfo(vs->hosts[i],
+			    &ss_hack, res->ai_addrlen)) {
+				new->hosts[host] = vs->hosts[i];
+				CHECK_OBJ_NOTNULL(new->hosts[host],
+				    DIRECTOR_MAGIC);
+				host++;
+			}
+		}
+	}
+	freeaddrinfo(res0);
+
+	new->nhosts = host;
+	vdi_dns_cache_list_add(sp, vs, new);
+	*backend = vdi_dns_pick_host(sp, new);
+	return (1);
+}
+
+/* Walk through the cached lookups looking for the relevant host, add one
+ * if it isn't already cached.
+ *
+ * Returns a backend or NULL.
+ */
+static struct director *
+vdi_dns_walk_cache(const struct sess *sp,
+		   struct vdi_dns *vs,
+		   const char *hostname)
+{
+	struct director *backend = NULL;
+	int ret;
+
+	AZ(pthread_rwlock_rdlock(&vs->rwlock));
+	ret = vdi_dns_cache_has(sp, vs, hostname, &backend, 0);
+	AZ(pthread_rwlock_unlock(&vs->rwlock));
+	if (!ret) {
+		/*
+		 * XXX: Isn't there a race here where another thread
+		 * XXX: could grab the lock and add it before we do ?
+		 * XXX: Should 'ret' be checked for that ?
+		 */
+		AZ(pthread_rwlock_wrlock(&vs->rwlock));
+		ret = vdi_dns_cache_add(sp, vs, hostname, &backend);
+		AZ(pthread_rwlock_unlock(&vs->rwlock));
+	} else
+		VSC_C_main->dir_dns_hit++;
+
+	/* Bank backend == cached a failure, so to speak */
+	if (backend != NULL)
+		CHECK_OBJ_NOTNULL(backend, DIRECTOR_MAGIC);
+	return (backend);
+}
+
+/* Parses the Host:-header and heads out to find a backend.
+ */
+static struct director *
+vdi_dns_find_backend(const struct sess *sp, struct vdi_dns *vs)
+{
+	struct director *ret;
+	struct http *hp;
+	char *p, *q;
+	char hostname[NI_MAXHOST];
+
+	/* bereq is only present after recv et. al, otherwise use req (ie:
+	 * use req for health checks in vcl_recv and such).
+	 */
+	if (sp->wrk->bereq)
+		hp = sp->wrk->bereq;
+	else
+		hp = sp->http;
+
+
+	CHECK_OBJ_NOTNULL(hp, HTTP_MAGIC);
+	if (http_GetHdr(hp, H_Host, &p) == 0)
+		return (NULL);
+
+	q = strchr(p, ':');
+	if (q == NULL)
+		q = strchr(p, '\0');
+	AN(q);
+
+	bprintf(hostname, "%.*s%s", (int)(q - p), p,
+	    vs->suffix ? vs->suffix : "");
+
+	ret = vdi_dns_walk_cache(sp, vs, hostname);
+	return (ret);
+}
+
+static struct vbc *
+vdi_dns_getfd(const struct director *director, struct sess *sp)
+{
+	struct vdi_dns *vs;
+	struct director *dir;
+	struct vbc *vbe;
+
+	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+	CHECK_OBJ_NOTNULL(director, DIRECTOR_MAGIC);
+	CAST_OBJ_NOTNULL(vs, director->priv, VDI_DNS_MAGIC);
+
+	dir = vdi_dns_find_backend(sp, vs);
+	if (!dir || !VDI_Healthy(dir, sp))
+		return (NULL);
+
+	vbe = VDI_GetFd(dir, sp);
+	return (vbe);
+}
+
+static unsigned
+vdi_dns_healthy(const struct director *dir, const struct sess *sp)
+{
+	/* XXX: Fooling -Werror for a bit until it's actually implemented.
+	 */
+	(void)dir;
+	(void)sp;
+	return (1);
+
+	/*
+	struct vdi_dns *vs;
+	struct director *dir;
+	int i;
+
+	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+	CHECK_OBJ_NOTNULL(sp->director, DIRECTOR_MAGIC);
+	CAST_OBJ_NOTNULL(vs, sp->director->priv, VDI_DNS_MAGIC);
+
+	dir = vdi_dns_find_backend(sp, vs);
+
+	if (dir)
+		return (1);
+	return (0);
+	*/
+}
+
+static void
+vdi_dns_fini(const struct director *d)
+{
+	struct vdi_dns *vs;
+
+	CHECK_OBJ_NOTNULL(d, DIRECTOR_MAGIC);
+	CAST_OBJ_NOTNULL(vs, d->priv, VDI_DNS_MAGIC);
+
+	free(vs->hosts);
+	free(vs->dir.vcl_name);
+	vs->dir.magic = 0;
+	/* FIXME: Free the cache */
+	AZ(pthread_rwlock_destroy(&vs->rwlock));
+	FREE_OBJ(vs);
+}
+
+void
+VRT_init_dir_dns(struct cli *cli, struct director **bp, int idx,
+    const void *priv)
+{
+	const struct vrt_dir_dns *t;
+	struct vdi_dns *vs;
+	const struct vrt_dir_dns_entry *te;
+	int i;
+
+	ASSERT_CLI();
+	(void)cli;
+	t = priv;
+	ALLOC_OBJ(vs, VDI_DNS_MAGIC);
+	XXXAN(vs);
+	vs->hosts = calloc(sizeof(struct director *), t->nmember);
+	XXXAN(vs->hosts);
+
+	vs->dir.magic = DIRECTOR_MAGIC;
+	vs->dir.priv = vs;
+	vs->dir.name = "dns";
+	REPLACE(vs->dir.vcl_name, t->name);
+	vs->dir.getfd = vdi_dns_getfd;
+	vs->dir.fini = vdi_dns_fini;
+	vs->dir.healthy = vdi_dns_healthy;
+
+	vs->suffix = t->suffix;
+	vs->ttl = t->ttl;
+
+	te = t->members;
+	for (i = 0; i < t->nmember; i++, te++)
+		vs->hosts[i] = bp[te->host];
+	vs->nhosts = t->nmember;
+	vs->ttl = t->ttl;
+	VTAILQ_INIT(&vs->cachelist);
+	AZ(pthread_rwlock_init(&vs->rwlock, NULL));
+	bp[idx] = &vs->dir;
+}
diff --git a/bin/varnishd/cache/cache_dir_random.c b/bin/varnishd/cache/cache_dir_random.c
new file mode 100644
index 0000000..d6570ed
--- /dev/null
+++ b/bin/varnishd/cache/cache_dir_random.c
@@ -0,0 +1,285 @@
+/*-
+ * Copyright (c) 2006 Verdens Gang AS
+ * Copyright (c) 2006-2011 Varnish Software AS
+ * All rights reserved.
+ *
+ * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * This code is shared between the random, client and hash directors, because
+ * they share the same properties and most of the same selection logic.
+ *
+ * The random director picks a backend on random.
+ *
+ * The hash director picks based on the hash from vcl_hash{}
+ *
+ * The client director picks based on client identity or IP-address
+ *
+ * In all cases, the choice is by weight of the healthy subset of
+ * configured backends.
+ *
+ * Failures to get a connection are retried, here all three policies
+ * fall back to a deterministically random choice, by weight in the
+ * healthy subset.
+ *
+ */
+
+#include "config.h"
+
+#include <math.h>
+#include <stdlib.h>
+
+#include "cache.h"
+
+#include "cache_backend.h"
+#include "vend.h"
+#include "vrt.h"
+#include "vsha256.h"
+
+/*--------------------------------------------------------------------*/
+
+struct vdi_random_host {
+	struct director		*backend;
+	double			weight;
+};
+
+enum crit_e {c_random, c_hash, c_client};
+
+struct vdi_random {
+	unsigned		magic;
+#define VDI_RANDOM_MAGIC	0x3771ae23
+	struct director		dir;
+
+	enum crit_e		criteria;
+	unsigned		retries;
+	double			tot_weight;
+	struct vdi_random_host	*hosts;
+	unsigned		nhosts;
+};
+
+/*
+ * Applies sha256 using the given context and input/length, and returns
+ * a double in the range [0...1[ based on the hash.
+ */
+static double
+vdi_random_sha(const char *input, ssize_t len)
+{
+	struct SHA256Context ctx;
+	uint8_t sign[SHA256_LEN];
+
+	AN(input);
+	SHA256_Init(&ctx);
+	SHA256_Update(&ctx, input, len);
+	SHA256_Final(sign, &ctx);
+	return (scalbn(vle32dec(sign), -32));
+}
+
+/*
+ * Sets up the initial seed for picking a backend according to policy.
+ */
+static double
+vdi_random_init_seed(const struct vdi_random *vs, const struct sess *sp)
+{
+	const char *p;
+	double retval;
+
+	switch (vs->criteria) {
+	case c_client:
+		if (sp->client_identity != NULL)
+			p = sp->client_identity;
+		else
+			p = sp->addr;
+		retval = vdi_random_sha(p, strlen(p));
+		break;
+	case c_hash:
+		AN(sp->digest);
+		retval = scalbn(vle32dec(sp->digest), -32);
+		break;
+	case c_random:
+	default:
+		retval = scalbn(random(), -31);
+		break;
+	}
+	return (retval);
+}
+
+/*
+ * Find the healthy backend corresponding to the weight r [0...1[
+ */
+static struct vbc *
+vdi_random_pick_one(struct sess *sp, const struct vdi_random *vs, double r)
+{
+	double w[vs->nhosts];
+	int i;
+	double s1;
+
+	assert(r >= 0.0 && r < 1.0);
+
+	memset(w, 0, sizeof w);
+	/* Sum up the weights of healty backends */
+	s1 = 0.0;
+	for (i = 0; i < vs->nhosts; i++) {
+		if (VDI_Healthy(vs->hosts[i].backend, sp))
+			w[i] = vs->hosts[i].weight;
+		s1 += w[i];
+	}
+
+	if (s1 == 0.0)
+		return (NULL);
+
+	r *= s1;
+	s1 = 0.0;
+	for (i = 0; i < vs->nhosts; i++)  {
+		s1 += w[i];
+		if (r < s1)
+			return(VDI_GetFd(vs->hosts[i].backend, sp));
+	}
+	return (NULL);
+}
+
+/*
+ * Try the specified number of times to get a backend.
+ * First one according to policy, after that, deterministically
+ * random by rehashing the key.
+ */
+static struct vbc *
+vdi_random_getfd(const struct director *d, struct sess *sp)
+{
+	int k;
+	struct vdi_random *vs;
+	double r;
+	struct vbc *vbe;
+
+	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+	CHECK_OBJ_NOTNULL(d, DIRECTOR_MAGIC);
+	CAST_OBJ_NOTNULL(vs, d->priv, VDI_RANDOM_MAGIC);
+
+	r = vdi_random_init_seed(vs, sp);
+
+	for (k = 0; k < vs->retries; k++) {
+		vbe = vdi_random_pick_one(sp, vs, r);
+		if (vbe != NULL)
+			return (vbe);
+		r = vdi_random_sha((void *)&r, sizeof(r));
+	}
+	return (NULL);
+}
+
+/*
+ * Healthy if just a single backend is...
+ */
+static unsigned
+vdi_random_healthy(const struct director *d, const struct sess *sp)
+{
+	struct vdi_random *vs;
+	int i;
+
+	CHECK_OBJ_NOTNULL(d, DIRECTOR_MAGIC);
+	CAST_OBJ_NOTNULL(vs, d->priv, VDI_RANDOM_MAGIC);
+
+	for (i = 0; i < vs->nhosts; i++) {
+		if (VDI_Healthy(vs->hosts[i].backend, sp))
+			return (1);
+	}
+	return (0);
+}
+
+static void
+vdi_random_fini(const struct director *d)
+{
+	struct vdi_random *vs;
+
+	CHECK_OBJ_NOTNULL(d, DIRECTOR_MAGIC);
+	CAST_OBJ_NOTNULL(vs, d->priv, VDI_RANDOM_MAGIC);
+
+	free(vs->hosts);
+	free(vs->dir.vcl_name);
+	vs->dir.magic = 0;
+	FREE_OBJ(vs);
+}
+
+static void
+vrt_init(struct cli *cli, struct director **bp, int idx,
+    const void *priv, enum crit_e criteria)
+{
+	const struct vrt_dir_random *t;
+	struct vdi_random *vs;
+	const struct vrt_dir_random_entry *te;
+	struct vdi_random_host *vh;
+	int i;
+
+	ASSERT_CLI();
+	(void)cli;
+	t = priv;
+
+	ALLOC_OBJ(vs, VDI_RANDOM_MAGIC);
+	XXXAN(vs);
+	vs->hosts = calloc(sizeof *vh, t->nmember);
+	XXXAN(vs->hosts);
+
+	vs->dir.magic = DIRECTOR_MAGIC;
+	vs->dir.priv = vs;
+	vs->dir.name = "random";
+	REPLACE(vs->dir.vcl_name, t->name);
+	vs->dir.getfd = vdi_random_getfd;
+	vs->dir.fini = vdi_random_fini;
+	vs->dir.healthy = vdi_random_healthy;
+
+	vs->criteria = criteria;
+	vs->retries = t->retries;
+	if (vs->retries == 0)
+		vs->retries = t->nmember;
+	vh = vs->hosts;
+	te = t->members;
+	vs->tot_weight = 0.;
+	for (i = 0; i < t->nmember; i++, vh++, te++) {
+		assert(te->weight > 0.0);
+		vh->weight = te->weight;
+		vs->tot_weight += vh->weight;
+		vh->backend = bp[te->host];
+		AN(vh->backend);
+	}
+	vs->nhosts = t->nmember;
+	bp[idx] = &vs->dir;
+}
+
+void
+VRT_init_dir_random(struct cli *cli, struct director **bp, int idx,
+    const void *priv)
+{
+	vrt_init(cli, bp, idx, priv, c_random);
+}
+
+void
+VRT_init_dir_hash(struct cli *cli, struct director **bp, int idx,
+    const void *priv)
+{
+	vrt_init(cli, bp, idx, priv, c_hash);
+}
+
+void
+VRT_init_dir_client(struct cli *cli, struct director **bp, int idx,
+    const void *priv)
+{
+	vrt_init(cli, bp, idx, priv, c_client);
+}
diff --git a/bin/varnishd/cache/cache_dir_round_robin.c b/bin/varnishd/cache/cache_dir_round_robin.c
new file mode 100644
index 0000000..7d75473
--- /dev/null
+++ b/bin/varnishd/cache/cache_dir_round_robin.c
@@ -0,0 +1,176 @@
+/*-
+ * Copyright (c) 2008-2011 Varnish Software AS
+ * All rights reserved.
+ *
+ * Author: Petter Knudsen <petter at linpro.no>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ */
+
+#include "config.h"
+
+#include <stdlib.h>
+
+#include "cache.h"
+
+#include "cache_backend.h"
+#include "vrt.h"
+
+/*--------------------------------------------------------------------*/
+
+struct vdi_round_robin_host {
+	struct director			*backend;
+};
+
+enum mode_e { m_round_robin, m_fallback };
+
+struct vdi_round_robin {
+	unsigned			magic;
+#define VDI_ROUND_ROBIN_MAGIC		0x2114a178
+	struct director			dir;
+	enum mode_e			mode;
+	struct vdi_round_robin_host	*hosts;
+	unsigned			nhosts;
+	unsigned			next_host;
+};
+
+static struct vbc *
+vdi_round_robin_getfd(const struct director *d, struct sess *sp)
+{
+	int i;
+	struct vdi_round_robin *vs;
+	struct director *backend;
+	struct vbc *vbe;
+
+	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+	CHECK_OBJ_NOTNULL(d, DIRECTOR_MAGIC);
+	CAST_OBJ_NOTNULL(vs, d->priv, VDI_ROUND_ROBIN_MAGIC);
+
+	/*
+	 * In fallback mode we ignore the next_host and always grab the
+	 * first healthy backend we can find.
+	 */
+	for (i = 0; i < vs->nhosts; i++) {
+		if (vs->mode == m_round_robin) {
+			backend = vs->hosts[vs->next_host].backend;
+			vs->next_host = (vs->next_host + 1) % vs->nhosts;
+		} else /* m_fallback */ {
+			backend = vs->hosts[i].backend;
+		}
+		if (!VDI_Healthy(backend, sp))
+			continue;
+		vbe = VDI_GetFd(backend, sp);
+		if (vbe != NULL)
+			return (vbe);
+	}
+
+	return (NULL);
+}
+
+static unsigned
+vdi_round_robin_healthy(const struct director *d, const struct sess *sp)
+{
+	struct vdi_round_robin *vs;
+	struct director *backend;
+	int i;
+
+	CHECK_OBJ_NOTNULL(d, DIRECTOR_MAGIC);
+	CAST_OBJ_NOTNULL(vs, d->priv, VDI_ROUND_ROBIN_MAGIC);
+
+	for (i = 0; i < vs->nhosts; i++) {
+		backend = vs->hosts[i].backend;
+		if (VDI_Healthy(backend, sp))
+			return (1);
+	}
+	return (0);
+}
+
+static void
+vdi_round_robin_fini(const struct director *d)
+{
+	struct vdi_round_robin *vs;
+
+	CHECK_OBJ_NOTNULL(d, DIRECTOR_MAGIC);
+	CAST_OBJ_NOTNULL(vs, d->priv, VDI_ROUND_ROBIN_MAGIC);
+
+	free(vs->hosts);
+	free(vs->dir.vcl_name);
+	vs->dir.magic = 0;
+	vs->next_host = 0;
+	FREE_OBJ(vs);
+}
+
+static void
+vrt_init_dir(struct cli *cli, struct director **bp, int idx,
+    const void *priv, enum mode_e mode)
+{
+	const struct vrt_dir_round_robin *t;
+	struct vdi_round_robin *vs;
+	const struct vrt_dir_round_robin_entry *te;
+	struct vdi_round_robin_host *vh;
+	int i;
+
+	ASSERT_CLI();
+	(void)cli;
+	t = priv;
+
+	ALLOC_OBJ(vs, VDI_ROUND_ROBIN_MAGIC);
+	XXXAN(vs);
+	vs->hosts = calloc(sizeof *vh, t->nmember);
+	XXXAN(vs->hosts);
+
+	vs->dir.magic = DIRECTOR_MAGIC;
+	vs->dir.priv = vs;
+	vs->dir.name = "round_robin";
+	REPLACE(vs->dir.vcl_name, t->name);
+	vs->dir.getfd = vdi_round_robin_getfd;
+	vs->dir.fini = vdi_round_robin_fini;
+	vs->dir.healthy = vdi_round_robin_healthy;
+
+	vs->mode = mode;
+	vh = vs->hosts;
+	te = t->members;
+	for (i = 0; i < t->nmember; i++, vh++, te++) {
+		vh->backend = bp[te->host];
+		AN (vh->backend);
+	}
+	vs->nhosts = t->nmember;
+	vs->next_host = 0;
+
+	bp[idx] = &vs->dir;
+}
+
+void
+VRT_init_dir_round_robin(struct cli *cli, struct director **bp, int idx,
+    const void *priv)
+{
+	vrt_init_dir(cli, bp, idx, priv, m_round_robin);
+}
+
+void
+VRT_init_dir_fallback(struct cli *cli, struct director **bp, int idx,
+    const void *priv)
+{
+	vrt_init_dir(cli, bp, idx, priv, m_fallback);
+}
+
diff --git a/bin/varnishd/cache/cache_esi.h b/bin/varnishd/cache/cache_esi.h
new file mode 100644
index 0000000..ff84d41
--- /dev/null
+++ b/bin/varnishd/cache/cache_esi.h
@@ -0,0 +1,48 @@
+/*-
+ * Copyright (c) 2011 Varnish Software AS
+ * All rights reserved.
+ *
+ * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ */
+
+#define	VEC_GZ	(0x21)
+#define	VEC_V1	(0x40 + 1)
+#define	VEC_V2	(0x40 + 2)
+#define	VEC_V8	(0x40 + 8)
+#define	VEC_C1	(0x50 + 1)
+#define	VEC_C2	(0x50 + 2)
+#define	VEC_C8	(0x50 + 8)
+#define	VEC_S1	(0x60 + 1)
+#define	VEC_S2	(0x60 + 2)
+#define	VEC_S8	(0x60 + 8)
+#define	VEC_INCL	'I'
+
+typedef ssize_t vep_callback_t(struct worker *w, ssize_t l, enum vgz_flag flg);
+
+void VEP_Init(struct worker *w, vep_callback_t *cb);
+void VEP_Parse(const struct worker *w, const char *p, size_t l);
+struct vsb *VEP_Finish(struct worker *w);
+
+
diff --git a/bin/varnishd/cache/cache_esi_deliver.c b/bin/varnishd/cache/cache_esi_deliver.c
new file mode 100644
index 0000000..4051027
--- /dev/null
+++ b/bin/varnishd/cache/cache_esi_deliver.c
@@ -0,0 +1,570 @@
+/*-
+ * Copyright (c) 2011 Varnish Software AS
+ * All rights reserved.
+ *
+ * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * VED - Varnish Esi Delivery
+ */
+
+#include "config.h"
+
+#include <stdio.h>
+#include <stdlib.h>
+
+#include "cache.h"
+
+#include "cache_esi.h"
+#include "vend.h"
+#include "vgz.h"
+
+/*--------------------------------------------------------------------*/
+
+static void
+ved_include(struct sess *sp, const char *src, const char *host)
+{
+	struct object *obj;
+	struct worker *w;
+	char *sp_ws_wm;
+	char *wrk_ws_wm;
+	unsigned sxid, res_mode;
+
+	w = sp->wrk;
+
+	if (sp->esi_level >= cache_param->max_esi_depth)
+		return;
+	sp->esi_level++;
+
+	(void)WRW_FlushRelease(w);
+
+	obj = sp->obj;
+	sp->obj = NULL;
+	res_mode = sp->wrk->res_mode;
+
+	/* Reset request to status before we started messing with it */
+	HTTP_Copy(sp->http, sp->http0);
+
+	/* Take a workspace snapshot */
+	sp_ws_wm = WS_Snapshot(sp->ws);
+	wrk_ws_wm = WS_Snapshot(w->ws);
+
+	http_SetH(sp->http, HTTP_HDR_URL, src);
+	if (host != NULL && *host != '\0')  {
+		http_Unset(sp->http, H_Host);
+		http_Unset(sp->http, H_If_Modified_Since);
+		http_SetHeader(w, sp->vsl_id, sp->http, host);
+	}
+	/*
+	 * XXX: We should decide if we should cache the director
+	 * XXX: or not (for session/backend coupling).  Until then
+	 * XXX: make sure we don't trip up the check in vcl_recv.
+	 */
+	sp->director = NULL;
+	sp->step = STP_RECV;
+	http_ForceGet(sp->http);
+
+	/* Don't do conditionals */
+	sp->http->conds = 0;
+	http_Unset(sp->http, H_If_Modified_Since);
+
+	/* Client content already taken care of */
+	http_Unset(sp->http, H_Content_Length);
+
+	sp->wrk->do_esi = 0;
+	sp->wrk->is_gzip = 0;
+	sp->wrk->is_gunzip = 0;
+	sp->wrk->do_gzip = 0;
+	sp->wrk->do_gunzip = 0;
+	sp->wrk->do_stream = 0;
+
+	sxid = sp->xid;
+	while (1) {
+		sp->wrk = w;
+		CNT_Session(sp);
+		if (sp->step == STP_DONE)
+			break;
+		AZ(sp->wrk);
+		WSL_Flush(w, 0);
+		DSL(0x20, SLT_Debug, sp->vsl_id, "loop waiting for ESI");
+		(void)usleep(10000);
+	}
+	sp->xid = sxid;
+	AN(sp->wrk);
+	assert(sp->step == STP_DONE);
+	sp->esi_level--;
+	sp->obj = obj;
+	sp->wrk->res_mode = res_mode;
+
+	/* Reset the workspace */
+	WS_Reset(sp->ws, sp_ws_wm);
+	WS_Reset(w->ws, wrk_ws_wm);
+
+	WRW_Reserve(sp->wrk, &sp->fd);
+	if (sp->wrk->res_mode & RES_CHUNKED)
+		WRW_Chunked(sp->wrk);
+}
+
+/*--------------------------------------------------------------------*/
+
+
+//#define Debug(fmt, ...) printf(fmt, __VA_ARGS__)
+#define Debug(fmt, ...) /**/
+
+static ssize_t
+ved_decode_len(uint8_t **pp)
+{
+	uint8_t *p;
+	ssize_t l;
+
+	p = *pp;
+	switch (*p & 15) {
+	case 1:
+		l = p[1];
+		p += 2;
+		break;
+	case 2:
+		l = vbe16dec(p + 1);
+		p += 3;
+		break;
+	case 8:
+		l = vbe64dec(p + 1);
+		p += 9;
+		break;
+	default:
+		printf("Illegal Length %d %d\n", *p, (*p & 15));
+		INCOMPL();
+	}
+	*pp = p;
+	assert(l > 0);
+	return (l);
+}
+
+/*---------------------------------------------------------------------
+ * If a gzip'ed ESI object includes a ungzip'ed object, we need to make
+ * it looked like a gzip'ed data stream.  The official way to do so would
+ * be to fire up libvgz and gzip it, but we don't, we fake it.
+ *
+ * First, we cannot know if it is ungzip'ed on purpose, the admin may
+ * know something we don't.
+ *
+ * What do you mean "BS ?"
+ *
+ * All right then...
+ *
+ * The matter of the fact is that we simply will not fire up a gzip in
+ * the output path because it costs too much memory and CPU, so we simply
+ * wrap the data in very convenient "gzip copy-blocks" and send it down
+ * the stream with a bit more overhead.
+ */
+
+static void
+ved_pretend_gzip(const struct sess *sp, const uint8_t *p, ssize_t l)
+{
+	uint8_t buf1[5], buf2[5];
+	uint16_t lx;
+
+	lx = 65535;
+	buf1[0] = 0;
+	vle16enc(buf1 + 1, lx);
+	vle16enc(buf1 + 3, ~lx);
+
+	while (l > 0) {
+		if (l >= 65535) {
+			lx = 65535;
+			(void)WRW_Write(sp->wrk, buf1, sizeof buf1);
+		} else {
+			lx = (uint16_t)l;
+			buf2[0] = 0;
+			vle16enc(buf2 + 1, lx);
+			vle16enc(buf2 + 3, ~lx);
+			(void)WRW_Write(sp->wrk, buf2, sizeof buf2);
+		}
+		(void)WRW_Write(sp->wrk, p, lx);
+		sp->wrk->crc = crc32(sp->wrk->crc, p, lx);
+		sp->wrk->l_crc += lx;
+		l -= lx;
+		p += lx;
+	}
+	/* buf2 is local, have to flush */
+	(void)WRW_Flush(sp->wrk);
+}
+
+/*---------------------------------------------------------------------
+ */
+
+static const uint8_t gzip_hdr[] = {
+	0x1f, 0x8b, 0x08,
+	0x00, 0x00, 0x00, 0x00,
+	0x00,
+	0x02, 0x03
+};
+
+void
+ESI_Deliver(struct sess *sp)
+{
+	struct storage *st;
+	uint8_t *p, *e, *q, *r;
+	unsigned off;
+	ssize_t l, l2, l_icrc = 0;
+	uint32_t icrc = 0;
+	uint8_t tailbuf[8 + 5];
+	int isgzip;
+	struct vgz *vgz = NULL;
+	char obuf[cache_param->gzip_stack_buffer];
+	ssize_t obufl = 0;
+	size_t dl;
+	const void *dp;
+	int i;
+
+	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+	st = sp->obj->esidata;
+	AN(st);
+	assert(sizeof obuf >= 1024);
+
+	obuf[0] = 0;	/* For flexelint */
+
+	p = st->ptr;
+	e = st->ptr + st->len;
+
+	if (*p == VEC_GZ) {
+		isgzip = 1;
+		p++;
+	} else {
+		isgzip = 0;
+	}
+
+	if (sp->esi_level == 0) {
+		/*
+		 * Only the top level document gets to decide this.
+		 */
+		sp->wrk->gzip_resp = 0;
+		if (isgzip && !(sp->wrk->res_mode & RES_GUNZIP)) {
+			assert(sizeof gzip_hdr == 10);
+			/* Send out the gzip header */
+			(void)WRW_Write(sp->wrk, gzip_hdr, 10);
+			sp->wrk->l_crc = 0;
+			sp->wrk->gzip_resp = 1;
+			sp->wrk->crc = crc32(0L, Z_NULL, 0);
+		}
+	}
+
+	if (isgzip && !sp->wrk->gzip_resp) {
+		vgz = VGZ_NewUngzip(sp->wrk, "U D E");
+
+		/* Feed a gzip header to gunzip to make it happy */
+		VGZ_Ibuf(vgz, gzip_hdr, sizeof gzip_hdr);
+		VGZ_Obuf(vgz, obuf, sizeof obuf);
+		i = VGZ_Gunzip(vgz, &dp, &dl);
+		assert(i == VGZ_OK);
+		assert(VGZ_IbufEmpty(vgz));
+		assert(dl == 0);
+
+		obufl = 0;
+	}
+
+	st = VTAILQ_FIRST(&sp->obj->store);
+	off = 0;
+
+	while (p < e) {
+		switch (*p) {
+		case VEC_V1:
+		case VEC_V2:
+		case VEC_V8:
+			l = ved_decode_len(&p);
+			if (isgzip) {
+				assert(*p == VEC_C1 || *p == VEC_C2 ||
+				    *p == VEC_C8);
+				l_icrc = ved_decode_len(&p);
+				icrc = vbe32dec(p);
+				p += 4;
+				if (sp->wrk->gzip_resp) {
+					sp->wrk->crc = crc32_combine(
+					    sp->wrk->crc, icrc, l_icrc);
+					sp->wrk->l_crc += l_icrc;
+				}
+			}
+			/*
+			 * There is no guarantee that the 'l' bytes are all
+			 * in the same storage segment, so loop over storage
+			 * until we have processed them all.
+			 */
+			while (l > 0) {
+				l2 = l;
+				if (l2 > st->len - off)
+					l2 = st->len - off;
+				l -= l2;
+
+				if (sp->wrk->gzip_resp && isgzip) {
+					/*
+					 * We have a gzip'ed VEC and delivers
+					 * a gzip'ed ESI response.
+					 */
+					(void)WRW_Write(sp->wrk, st->ptr + off, l2);
+				} else if (sp->wrk->gzip_resp) {
+					/*
+					 * A gzip'ed ESI response, but the VEC
+					 * was not gzip'ed.
+					 */
+					ved_pretend_gzip(sp, st->ptr + off, l2);
+				} else if (isgzip) {
+					/*
+					 * A gzip'ed VEC, but ungzip'ed ESI
+					 * response
+					 */
+					AN(vgz);
+					i = VGZ_WrwGunzip(sp->wrk, vgz,
+						st->ptr + off, l2,
+						obuf, sizeof obuf, &obufl);
+					if (WRW_Error(sp->wrk)) {
+						SES_Close(sp, "remote closed");
+						p = e;
+						break;
+					}
+					assert (i == VGZ_OK || i == VGZ_END);
+				} else {
+					/*
+					 * Ungzip'ed VEC, ungzip'ed ESI response
+					 */
+					(void)WRW_Write(sp->wrk, st->ptr + off, l2);
+				}
+				off += l2;
+				if (off == st->len) {
+					st = VTAILQ_NEXT(st, list);
+					off = 0;
+				}
+			}
+			break;
+		case VEC_S1:
+		case VEC_S2:
+		case VEC_S8:
+			l = ved_decode_len(&p);
+			Debug("SKIP1(%d)\n", (int)l);
+			/*
+			 * There is no guarantee that the 'l' bytes are all
+			 * in the same storage segment, so loop over storage
+			 * until we have processed them all.
+			 */
+			while (l > 0) {
+				l2 = l;
+				if (l2 > st->len - off)
+					l2 = st->len - off;
+				l -= l2;
+				off += l2;
+				if (off == st->len) {
+					st = VTAILQ_NEXT(st, list);
+					off = 0;
+				}
+			}
+			break;
+		case VEC_INCL:
+			p++;
+			q = (void*)strchr((const char*)p, '\0');
+			AN(q);
+			q++;
+			r = (void*)strchr((const char*)q, '\0');
+			AN(r);
+			if (obufl > 0) {
+				(void)WRW_Write(sp->wrk, obuf, obufl);
+				obufl = 0;
+			}
+			if (WRW_Flush(sp->wrk)) {
+				SES_Close(sp, "remote closed");
+				p = e;
+				break;
+			}
+			Debug("INCL [%s][%s] BEGIN\n", q, p);
+			ved_include(sp, (const char*)q, (const char*)p);
+			Debug("INCL [%s][%s] END\n", q, p);
+			p = r + 1;
+			break;
+		default:
+			printf("XXXX 0x%02x [%s]\n", *p, p);
+			INCOMPL();
+		}
+	}
+	if (vgz != NULL) {
+		if (obufl > 0)
+			(void)WRW_Write(sp->wrk, obuf, obufl);
+		(void)VGZ_Destroy(&vgz, sp->vsl_id);
+	}
+	if (sp->wrk->gzip_resp && sp->esi_level == 0) {
+		/* Emit a gzip literal block with finish bit set */
+		tailbuf[0] = 0x01;
+		tailbuf[1] = 0x00;
+		tailbuf[2] = 0x00;
+		tailbuf[3] = 0xff;
+		tailbuf[4] = 0xff;
+
+		/* Emit CRC32 */
+		vle32enc(tailbuf + 5, sp->wrk->crc);
+
+		/* MOD(2^32) length */
+		vle32enc(tailbuf + 9, sp->wrk->l_crc);
+
+		(void)WRW_Write(sp->wrk, tailbuf, 13);
+	}
+	(void)WRW_Flush(sp->wrk);
+}
+
+/*---------------------------------------------------------------------
+ * Include an object in a gzip'ed ESI object delivery
+ */
+
+static uint8_t
+ved_deliver_byterange(const struct sess *sp, ssize_t low, ssize_t high)
+{
+	struct storage *st;
+	ssize_t l, lx;
+	u_char *p;
+
+//printf("BR %jd %jd\n", low, high);
+	lx = 0;
+	VTAILQ_FOREACH(st, &sp->obj->store, list) {
+		p = st->ptr;
+		l = st->len;
+//printf("[0-] %jd %jd\n", lx, lx + l);
+		if (lx + l < low) {
+			lx += l;
+			continue;
+		}
+		if (lx == high)
+			return (p[0]);
+		assert(lx < high);
+		if (lx < low) {
+			p += (low - lx);
+			l -= (low - lx);
+			lx = low;
+		}
+//printf("[1-] %jd %jd\n", lx, lx + l);
+		if (lx + l >= high)
+			l = high - lx;
+//printf("[2-] %jd %jd\n", lx, lx + l);
+		assert(lx >= low && lx + l <= high);
+		if (l != 0)
+			(void)WRW_Write(sp->wrk, p, l);
+		if (lx + st->len > high)
+			return(p[l]);
+		lx += st->len;
+	}
+	INCOMPL();
+}
+
+void
+ESI_DeliverChild(const struct sess *sp)
+{
+	struct storage *st;
+	struct object *obj;
+	ssize_t start, last, stop, lpad;
+	u_char *p, cc;
+	uint32_t icrc;
+	uint32_t ilen;
+	uint8_t *dbits;
+
+	if (!sp->obj->gziped) {
+		VTAILQ_FOREACH(st, &sp->obj->store, list)
+			ved_pretend_gzip(sp, st->ptr, st->len);
+		return;
+	}
+	/*
+	 * This is the interesting case: Deliver all the deflate
+	 * blocks, stripping the "LAST" bit of the last one and
+	 * padding it, as necessary, to a byte boundary.
+	 */
+
+	dbits = (void*)WS_Alloc(sp->wrk->ws, 8);
+	AN(dbits);
+	obj = sp->obj;
+	CHECK_OBJ_NOTNULL(obj, OBJECT_MAGIC);
+	start = obj->gzip_start;
+	last = obj->gzip_last;
+	stop = obj->gzip_stop;
+	assert(start > 0 && start < obj->len * 8);
+	assert(last > 0 && last < obj->len * 8);
+	assert(stop > 0 && stop < obj->len * 8);
+	assert(last >= start);
+	assert(last < stop);
+
+	/* The start bit must be byte aligned. */
+	AZ(start & 7);
+
+	/*
+	 * XXX: optimize for the case where the 'last'
+	 * XXX: bit is in a empty copy block
+	 */
+	*dbits = ved_deliver_byterange(sp, start/8, last/8);
+	*dbits &= ~(1U << (last & 7));
+	(void)WRW_Write(sp->wrk, dbits, 1);
+	cc = ved_deliver_byterange(sp, 1 + last/8, stop/8);
+	switch((int)(stop & 7)) {
+	case 0: /* xxxxxxxx */
+		/* I think we have an off by one here, but that's OK */
+		lpad = 0;
+		break;
+	case 1: /* x000.... 00000000 00000000 11111111 11111111 */
+	case 3: /* xxx000.. 00000000 00000000 11111111 11111111 */
+	case 5: /* xxxxx000 00000000 00000000 11111111 11111111 */
+		dbits[1] = cc | 0x00;
+		dbits[2] = 0x00; dbits[3] = 0x00;
+	        dbits[4] = 0xff; dbits[5] = 0xff;
+		lpad = 5;
+		break;
+	case 2: /* xx010000 00000100 00000001 00000000 */
+		dbits[1] = cc | 0x08;
+		dbits[2] = 0x20;
+		dbits[3] = 0x80;
+		dbits[4] = 0x00;
+		lpad = 4;
+		break;
+	case 4: /* xxxx0100 00000001 00000000 */
+		dbits[1] = cc | 0x20;
+		dbits[2] = 0x80;
+		dbits[3] = 0x00;
+		lpad = 3;
+		break;
+	case 6: /* xxxxxx01 00000000 */
+		dbits[1] = cc | 0x80;
+		dbits[2] = 0x00;
+		lpad = 2;
+		break;
+	case 7:	/* xxxxxxx0 00...... 00000000 00000000 11111111 11111111 */
+		dbits[1] = cc | 0x00;
+		dbits[2] = 0x00;
+		dbits[3] = 0x00; dbits[4] = 0x00;
+	        dbits[5] = 0xff; dbits[6] = 0xff;
+		lpad = 6;
+		break;
+	default:
+		INCOMPL();
+	}
+	if (lpad > 0)
+		(void)WRW_Write(sp->wrk, dbits + 1, lpad);
+	st = VTAILQ_LAST(&sp->obj->store, storagehead);
+	assert(st->len > 8);
+
+	p = st->ptr + st->len - 8;
+	icrc = vle32dec(p);
+	ilen = vle32dec(p + 4);
+	sp->wrk->crc = crc32_combine(sp->wrk->crc, icrc, ilen);
+	sp->wrk->l_crc += ilen;
+}
diff --git a/bin/varnishd/cache/cache_esi_fetch.c b/bin/varnishd/cache/cache_esi_fetch.c
new file mode 100644
index 0000000..5ec8f6b
--- /dev/null
+++ b/bin/varnishd/cache/cache_esi_fetch.c
@@ -0,0 +1,405 @@
+/*-
+ * Copyright (c) 2011 Varnish Software AS
+ * All rights reserved.
+ *
+ * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * VEF Varnish Esi Fetching
+ */
+
+#include "config.h"
+
+#include <stdlib.h>
+
+#include "cache.h"
+
+#include "cache_esi.h"
+
+/*---------------------------------------------------------------------
+ * Read some bytes.
+ *
+ * If the esi_syntax&8 bit is set, we read only a couple of bytes at
+ * a time, in order to stress the parse/pending/callback code.
+ */
+
+static ssize_t
+vef_read(struct worker *w, struct http_conn *htc, void *buf, ssize_t buflen,
+    ssize_t bytes)
+{
+	ssize_t d;
+
+	if (buflen < bytes)
+		bytes = buflen;
+	if (cache_param->esi_syntax & 0x8) {
+		d = (random() & 3) + 1;
+		if (d < bytes)
+			bytes = d;
+	}
+	return (HTC_Read(w, htc, buf, bytes));
+}
+
+/*---------------------------------------------------------------------
+ * We receive a ungzip'ed object, and want to store it ungzip'ed.
+ */
+
+static int
+vfp_esi_bytes_uu(struct worker *w, struct http_conn *htc, ssize_t bytes)
+{
+	ssize_t wl;
+	struct storage *st;
+
+	CHECK_OBJ_NOTNULL(w, WORKER_MAGIC);
+
+	while (bytes > 0) {
+		st = FetchStorage(w, 0);
+		if (st == NULL)
+			return (-1);
+		wl = vef_read(w, htc,
+		    st->ptr + st->len, st->space - st->len, bytes);
+		if (wl <= 0)
+			return (wl);
+		VEP_Parse(w, (const char *)st->ptr + st->len, wl);
+		st->len += wl;
+		w->fetch_obj->len += wl;
+		bytes -= wl;
+	}
+	return (1);
+}
+
+/*---------------------------------------------------------------------
+ * We receive a gzip'ed object, and want to store it ungzip'ed.
+ */
+
+static int
+vfp_esi_bytes_gu(struct worker *w, struct http_conn *htc, ssize_t bytes)
+{
+	struct vgz *vg;
+	ssize_t wl;
+	uint8_t	ibuf[cache_param->gzip_stack_buffer];
+	int i;
+	size_t dl;
+	const void *dp;
+
+	CHECK_OBJ_NOTNULL(w, WORKER_MAGIC);
+	vg = w->vgz_rx;
+
+	while (bytes > 0) {
+		if (VGZ_IbufEmpty(vg) && bytes > 0) {
+			wl = vef_read(w, htc, ibuf, sizeof ibuf, bytes);
+			if (wl <= 0)
+				return (wl);
+			VGZ_Ibuf(vg, ibuf, wl);
+			bytes -= wl;
+		}
+		if (VGZ_ObufStorage(w, vg))
+			return(-1);
+		i = VGZ_Gunzip(vg, &dp, &dl);
+		xxxassert(i == VGZ_OK || i == VGZ_END);
+		VEP_Parse(w, dp, dl);
+		w->fetch_obj->len += dl;
+	}
+	return (1);
+}
+
+/*---------------------------------------------------------------------
+ */
+
+struct vef_priv {
+	unsigned		magic;
+#define VEF_MAGIC		0xf104b51f
+	struct vgz		*vgz;
+
+	char			*bufp;
+	ssize_t			tot;
+	int			error;
+	char			pending[20];
+	ssize_t			npend;
+};
+
+/*---------------------------------------------------------------------
+ * We receive a [un]gzip'ed object, and want to store it gzip'ed.
+ */
+
+static ssize_t
+vfp_vep_callback(struct worker *w, ssize_t l, enum vgz_flag flg)
+{
+	struct vef_priv *vef;
+	size_t dl, px;
+	const void *dp;
+	int i;
+
+	CHECK_OBJ_NOTNULL(w, WORKER_MAGIC);
+	vef = w->vef_priv;
+	CHECK_OBJ_NOTNULL(vef, VEF_MAGIC);
+	assert(l >= 0);
+
+	if (vef->error) {
+		vef->tot += l;
+		return (vef->tot);
+	}
+
+	/*
+	 * l == 0 is valid when 'flg' calls for action, but in the
+	 * normal case we can just ignore a l==0 request.
+	 * (It would cause Z_BUF_ERROR anyway)
+	 */
+	if (l == 0 && flg == VGZ_NORMAL)
+		return (vef->tot);
+
+	do {
+		px = vef->npend;
+		if (l < px)
+			px = l;
+		if (px != 0) {
+			VGZ_Ibuf(vef->vgz, vef->pending, px);
+			l -= px;
+		} else {
+			VGZ_Ibuf(vef->vgz, vef->bufp, l);
+			vef->bufp += l;
+			l = 0;
+		}
+		do {
+			if (VGZ_ObufStorage(w, vef->vgz)) {
+				vef->error = ENOMEM;
+				vef->tot += l;
+				return (vef->tot);
+			}
+			i = VGZ_Gzip(vef->vgz, &dp, &dl, flg);
+			vef->tot += dl;
+			w->fetch_obj->len += dl;
+		} while (!VGZ_IbufEmpty(vef->vgz) ||
+		    (flg != VGZ_NORMAL && VGZ_ObufFull(vef->vgz)));
+		if (px != 0) {
+			memmove(vef->pending, vef->pending + px,
+			    vef->npend - px);
+			vef->npend -= px;
+		}
+	} while (l > 0);
+	if (flg == VGZ_FINISH)
+		assert(i == 1);			/* XXX */
+	else
+		assert(i == 0);			/* XXX */
+	return (vef->tot);
+}
+
+static int
+vfp_esi_bytes_ug(struct worker *w, struct http_conn *htc, ssize_t bytes)
+{
+	ssize_t wl;
+	char ibuf[cache_param->gzip_stack_buffer];
+	struct vef_priv *vef;
+
+	CHECK_OBJ_NOTNULL(w, WORKER_MAGIC);
+	vef = w->vef_priv;
+	CHECK_OBJ_NOTNULL(vef, VEF_MAGIC);
+
+	while (bytes > 0) {
+		wl = vef_read(w, htc, ibuf, sizeof ibuf, bytes);
+		if (wl <= 0)
+			return (wl);
+		bytes -= wl;
+		vef->bufp = ibuf;
+		VEP_Parse(w, ibuf, wl);
+		assert(vef->bufp >= ibuf && vef->bufp <= ibuf + wl);
+		if (vef->error) {
+			errno = vef->error;
+			return (-1);
+		}
+		if (vef->bufp < ibuf + wl) {
+			wl = (ibuf + wl) - vef->bufp;
+			assert(wl + vef->npend < sizeof vef->pending);
+			memmove(vef->pending + vef->npend, vef->bufp, wl);
+			vef->npend += wl;
+		}
+	}
+	return (1);
+}
+
+/*---------------------------------------------------------------------
+ * We receive a gzip'ed object, and want to store it gzip'ed.
+ */
+
+static int
+vfp_esi_bytes_gg(struct worker *w, struct http_conn *htc, size_t bytes)
+{
+	ssize_t wl;
+	char ibuf[cache_param->gzip_stack_buffer];
+	char ibuf2[cache_param->gzip_stack_buffer];
+	struct vef_priv *vef;
+	size_t dl;
+	const void *dp;
+	int i;
+
+	CHECK_OBJ_NOTNULL(w, WORKER_MAGIC);
+	vef = w->vef_priv;
+	CHECK_OBJ_NOTNULL(vef, VEF_MAGIC);
+	assert(sizeof ibuf >= 1024);
+	ibuf2[0] = 0; /* For Flexelint */
+
+	while (bytes > 0) {
+		wl = vef_read(w, htc, ibuf, sizeof ibuf, bytes);
+		if (wl <= 0)
+			return (wl);
+		bytes -= wl;
+
+		vef->bufp = ibuf;
+		VGZ_Ibuf(w->vgz_rx, ibuf, wl);
+		do {
+			VGZ_Obuf(w->vgz_rx, ibuf2, sizeof ibuf2);
+			i = VGZ_Gunzip(w->vgz_rx, &dp, &dl);
+			/* XXX: check i */
+			assert(i >= VGZ_OK);
+			vef->bufp = ibuf2;
+			if (dl > 0)
+				VEP_Parse(w, ibuf2, dl);
+			if (vef->error) {
+				errno = vef->error;
+				return (-1);
+			}
+			if (vef->bufp < ibuf2 + dl) {
+				dl = (ibuf2 + dl) - vef->bufp;
+				assert(dl + vef->npend < sizeof vef->pending);
+				memmove(vef->pending + vef->npend,
+				    vef->bufp, dl);
+				vef->npend += dl;
+			}
+		} while (!VGZ_IbufEmpty(w->vgz_rx));
+	}
+	return (1);
+}
+
+
+/*---------------------------------------------------------------------*/
+
+static void __match_proto__()
+vfp_esi_begin(struct worker *w, size_t estimate)
+{
+	struct vef_priv *vef;
+
+	CHECK_OBJ_NOTNULL(w, WORKER_MAGIC);
+
+	AZ(w->vgz_rx);
+	if (w->is_gzip && w->do_gunzip) {
+		w->vgz_rx = VGZ_NewUngzip(w, "U F E");
+		VEP_Init(w, NULL);
+	} else if (w->is_gunzip && w->do_gzip) {
+		ALLOC_OBJ(vef, VEF_MAGIC);
+		AN(vef);
+		vef->vgz = VGZ_NewGzip(w, "G F E");
+		AZ(w->vef_priv);
+		w->vef_priv = vef;
+		VEP_Init(w, vfp_vep_callback);
+	} else if (w->is_gzip) {
+		w->vgz_rx = VGZ_NewUngzip(w, "U F E");
+		ALLOC_OBJ(vef, VEF_MAGIC);
+		AN(vef);
+		vef->vgz = VGZ_NewGzip(w, "G F E");
+		AZ(w->vef_priv);
+		w->vef_priv = vef;
+		VEP_Init(w, vfp_vep_callback);
+	} else {
+		AZ(w->vef_priv);
+		VEP_Init(w, NULL);
+	}
+
+	(void)estimate;
+	AN(w->vep);
+}
+
+static int __match_proto__()
+vfp_esi_bytes(struct worker *w, struct http_conn *htc, ssize_t bytes)
+{
+	int i;
+
+	CHECK_OBJ_NOTNULL(w, WORKER_MAGIC);
+	AZ(w->fetch_failed);
+	AN(w->vep);
+	assert(w->htc == htc);
+	if (w->is_gzip && w->do_gunzip)
+		i = vfp_esi_bytes_gu(w, htc, bytes);
+	else if (w->is_gunzip && w->do_gzip)
+		i = vfp_esi_bytes_ug(w, htc, bytes);
+	else if (w->is_gzip)
+		i = vfp_esi_bytes_gg(w, htc, bytes);
+	else
+		i = vfp_esi_bytes_uu(w, htc, bytes);
+	AN(w->vep);
+	return (i);
+}
+
+static int __match_proto__()
+vfp_esi_end(struct worker *w)
+{
+	struct vsb *vsb;
+	struct vef_priv *vef;
+	ssize_t l;
+	int retval;
+
+	CHECK_OBJ_NOTNULL(w, WORKER_MAGIC);
+	AN(w->vep);
+
+	retval = w->fetch_failed;
+
+	if (w->vgz_rx != NULL && VGZ_Destroy(&w->vgz_rx, -1) != VGZ_END)
+		retval = FetchError(w,
+		    "Gunzip+ESI Failed at the very end");
+
+	vsb = VEP_Finish(w);
+
+	if (vsb != NULL) {
+		if (!retval) {
+			l = VSB_len(vsb);
+			assert(l > 0);
+			/* XXX: This is a huge waste of storage... */
+			w->fetch_obj->esidata = STV_alloc(w, l);
+			if (w->fetch_obj->esidata != NULL) {
+				memcpy(w->fetch_obj->esidata->ptr,
+				    VSB_data(vsb), l);
+				w->fetch_obj->esidata->len = l;
+			} else {
+				retval = FetchError(w,
+				    "Could not allocate storage for esidata");
+			}
+		}
+		VSB_delete(vsb);
+	}
+
+	if (w->vef_priv != NULL) {
+		vef = w->vef_priv;
+		CHECK_OBJ_NOTNULL(vef, VEF_MAGIC);
+		w->vef_priv = NULL;
+		VGZ_UpdateObj(vef->vgz, w->fetch_obj);
+		if (VGZ_Destroy(&vef->vgz,  -1) != VGZ_END)
+			retval = FetchError(w,
+			    "ESI+Gzip Failed at the very end");
+		FREE_OBJ(vef);
+	}
+	return (retval);
+}
+
+struct vfp vfp_esi = {
+        .begin  =       vfp_esi_begin,
+        .bytes  =       vfp_esi_bytes,
+        .end    =       vfp_esi_end,
+};
diff --git a/bin/varnishd/cache/cache_esi_parse.c b/bin/varnishd/cache/cache_esi_parse.c
new file mode 100644
index 0000000..9e2b4f6
--- /dev/null
+++ b/bin/varnishd/cache/cache_esi_parse.c
@@ -0,0 +1,1189 @@
+/*-
+ * Copyright (c) 2011 Varnish Software AS
+ * All rights reserved.
+ *
+ * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * VEP Varnish Esi Parsing
+ */
+
+#include "config.h"
+
+#include <stdio.h>
+#include <stdlib.h>
+
+#include "cache.h"
+
+#include "cache_esi.h"
+#include "vct.h"
+#include "vend.h"
+#include "vgz.h"
+
+//#define Debug(fmt, ...) printf(fmt, __VA_ARGS__)
+#define Debug(fmt, ...) /**/
+
+struct vep_state;
+
+enum dowhat {DO_ATTR, DO_TAG};
+typedef void dostuff_f(struct vep_state *, enum dowhat);
+
+struct vep_match {
+	const char	*match;
+	const char	* const *state;
+};
+
+enum vep_mark { VERBATIM = 0, SKIP };
+
+struct vep_state {
+	unsigned		magic;
+#define VEP_MAGIC		0x55cb9b82
+	struct vsb		*vsb;
+
+	struct worker		*wrk;
+	int			dogzip;
+	vep_callback_t		*cb;
+
+	/* Internal Counter for default call-back function */
+	ssize_t			cb_x;
+
+	/* parser state */
+	const char		*state;
+	unsigned		startup;
+	unsigned		esi_found;
+
+	unsigned		endtag;
+	unsigned		emptytag;
+	unsigned		canattr;
+
+	unsigned		remove;
+
+	ssize_t			o_wait;
+	ssize_t			o_pending;
+	ssize_t			o_total;
+	uint32_t		crc;
+	ssize_t			o_crc;
+	uint32_t		crcp;
+	ssize_t			o_last;
+
+const char		*hack_p;
+	const char		*ver_p;
+
+	const char		*until;
+	const char		*until_p;
+	const char		*until_s;
+
+	int			in_esi_tag;
+
+	const char		*esicmt;
+	const char		*esicmt_p;
+
+	struct vep_match	*attr;
+	struct vsb		*attr_vsb;
+	int			attr_delim;
+
+	struct vep_match	*match;
+	struct vep_match	*match_hit;
+
+	char			tag[10];
+	int			tag_i;
+
+	dostuff_f		*dostuff;
+
+	struct vsb		*include_src;
+
+	unsigned		nm_skip;
+	unsigned		nm_verbatim;
+	unsigned		nm_pending;
+	enum vep_mark		last_mark;
+};
+
+/*---------------------------------------------------------------------*/
+
+static const char * const VEP_START =		"[Start]";
+static const char * const VEP_TESTXML =		"[TestXml]";
+static const char * const VEP_NOTXML =		"[NotXml]";
+
+static const char * const VEP_NEXTTAG =		"[NxtTag]";
+static const char * const VEP_NOTMYTAG =	"[NotMyTag]";
+
+static const char * const VEP_STARTTAG =	"[StartTag]";
+static const char * const VEP_COMMENT =		"[Comment]";
+static const char * const VEP_CDATA =		"[CDATA]";
+static const char * const VEP_ESITAG =		"[ESITag]";
+
+static const char * const VEP_ESIREMOVE =	"[ESI:Remove]";
+static const char * const VEP_ESIINCLUDE =	"[ESI:Include]";
+static const char * const VEP_ESICOMMENT =	"[ESI:Comment]";
+static const char * const VEP_ESIBOGON =	"[ESI:Bogon]";
+
+static const char * const VEP_INTAG =		"[InTag]";
+static const char * const VEP_TAGERROR =	"[TagError]";
+
+static const char * const VEP_ATTR =		"[Attribute]";
+static const char * const VEP_SKIPATTR =	"[SkipAttribute]";
+static const char * const VEP_ATTRDELIM =	"[AttrDelim]";
+static const char * const VEP_ATTRGETVAL =	"[AttrGetValue]";
+static const char * const VEP_ATTRVAL =		"[AttrValue]";
+
+static const char * const VEP_UNTIL =		"[Until]";
+static const char * const VEP_MATCHBUF =	"[MatchBuf]";
+static const char * const VEP_MATCH =		"[Match]";
+
+/*---------------------------------------------------------------------*/
+
+static struct vep_match vep_match_starttag[] = {
+	{ "!--",	&VEP_COMMENT },
+	{ "esi:",	&VEP_ESITAG },
+	{ "![CDATA[",	&VEP_CDATA },
+	{ NULL,		&VEP_NOTMYTAG }
+};
+
+/*---------------------------------------------------------------------*/
+
+static struct vep_match vep_match_esi[] = {
+	{ "include",	&VEP_ESIINCLUDE },
+	{ "remove",	&VEP_ESIREMOVE },
+	{ "comment",	&VEP_ESICOMMENT },
+	{ NULL,		&VEP_ESIBOGON }
+};
+
+/*---------------------------------------------------------------------*/
+
+static struct vep_match vep_match_attr_include[] = {
+	{ "src=",	&VEP_ATTRGETVAL },
+	{ NULL,		&VEP_SKIPATTR }
+};
+
+/*--------------------------------------------------------------------
+ * Report a parsing error
+ */
+
+static void
+vep_error(const struct vep_state *vep, const char *p)
+{
+	intmax_t l;
+
+	VSC_C_main->esi_errors++;
+	l = (intmax_t)(vep->ver_p - vep->hack_p);
+	WSLB(vep->wrk, SLT_ESI_xmlerror, "ERR at %jd %s", l, p);
+
+}
+
+/*--------------------------------------------------------------------
+ * Report a parsing warning
+ */
+
+static void
+vep_warn(const struct vep_state *vep, const char *p)
+{
+	intmax_t l;
+
+	VSC_C_main->esi_warnings++;
+	l = (intmax_t)(vep->ver_p - vep->hack_p);
+	printf("WARNING at %jd %s\n", l, p);
+	WSLB(vep->wrk, SLT_ESI_xmlerror, "WARN at %jd %s", l, p);
+
+}
+
+/*---------------------------------------------------------------------
+ * return match or NULL if more input needed.
+ */
+
+static struct vep_match *
+vep_match(struct vep_state *vep, const char *b, const char *e)
+{
+	struct vep_match *vm;
+	const char *q, *r;
+	ssize_t l;
+
+	for (vm = vep->match; vm->match; vm++) {
+		r = b;
+		for (q = vm->match; *q && r < e; q++, r++)
+			if (*q != *r)
+				break;
+		if (*q != '\0' && r == e) {
+			if (b != vep->tag) {
+				l = e - b;
+				assert(l < sizeof vep->tag);
+				memmove(vep->tag, b, l);
+				vep->tag_i = l;
+			}
+			return (NULL);
+		}
+		if (*q == '\0')
+			return (vm);
+	}
+	return (vm);
+}
+
+/*---------------------------------------------------------------------
+ *
+ */
+
+static void
+vep_emit_len(const struct vep_state *vep, ssize_t l, int m8, int m16, int m64)
+{
+	uint8_t buf[9];
+
+	assert(l > 0);
+	if (l < 256) {
+		buf[0] = (uint8_t)m8;
+		buf[1] = (uint8_t)l;
+		assert((ssize_t)buf[1] == l);
+		VSB_bcat(vep->vsb, buf, 2);
+	} else if (l < 65536) {
+		buf[0] = (uint8_t)m16;
+		vbe16enc(buf + 1, (uint16_t)l);
+		assert((ssize_t)vbe16dec(buf + 1) == l);
+		VSB_bcat(vep->vsb, buf, 3);
+	} else {
+		buf[0] = (uint8_t)m64;
+		vbe64enc(buf + 1, l);
+		assert((ssize_t)vbe64dec(buf + 1) == l);
+		VSB_bcat(vep->vsb, buf, 9);
+	}
+}
+
+static void
+vep_emit_skip(const struct vep_state *vep, ssize_t l)
+{
+
+	if (cache_param->esi_syntax & 0x20) {
+		Debug("---> SKIP(%jd)\n", (intmax_t)l);
+	}
+	vep_emit_len(vep, l, VEC_S1, VEC_S2, VEC_S8);
+}
+
+static void
+vep_emit_verbatim(const struct vep_state *vep, ssize_t l, ssize_t l_crc)
+{
+	uint8_t buf[4];
+
+	if (cache_param->esi_syntax & 0x20) {
+		Debug("---> VERBATIM(%jd)\n", (intmax_t)l);
+	}
+	vep_emit_len(vep, l, VEC_V1, VEC_V2, VEC_V8);
+	if (vep->dogzip) {
+		vep_emit_len(vep, l_crc, VEC_C1, VEC_C2, VEC_C8);
+		vbe32enc(buf, vep->crc);
+		VSB_bcat(vep->vsb, buf, sizeof buf);
+	}
+}
+
+static void
+vep_emit_common(struct vep_state *vep, ssize_t l, enum vep_mark mark)
+{
+
+	assert(l > 0);
+	assert(mark == SKIP || mark == VERBATIM);
+	if (mark == SKIP)
+		vep_emit_skip(vep, l);
+	else
+		vep_emit_verbatim(vep, l, vep->o_crc);
+
+	vep->crc = crc32(0L, Z_NULL, 0);
+	vep->o_crc = 0;
+	vep->o_total += l;
+}
+
+/*---------------------------------------------------------------------
+ *
+ */
+
+static void
+vep_mark_common(struct vep_state *vep, const char *p, enum vep_mark mark)
+{
+	ssize_t l, lcb;
+
+	assert(mark == SKIP || mark == VERBATIM);
+
+	/* The NO-OP case, no data, no pending data & no change of mode */
+	if (vep->last_mark == mark && p == vep->ver_p && vep->o_pending == 0)
+		return;
+
+	/*
+	 * If we changed mode, emit whatever the opposite mode
+	 * assembled before the pending bytes.
+	 */
+
+	if (vep->last_mark != mark && (vep->o_wait > 0 || vep->startup)) {
+		lcb = vep->cb(vep->wrk, 0,
+		    mark == VERBATIM ? VGZ_RESET : VGZ_ALIGN);
+		if (lcb - vep->o_last > 0)
+			vep_emit_common(vep, lcb - vep->o_last, vep->last_mark);
+		vep->o_last = lcb;
+		vep->o_wait = 0;
+	}
+
+	/* Transfer pending bytes CRC into active mode CRC */
+	if (vep->o_pending) {
+		(void)vep->cb(vep->wrk, vep->o_pending, VGZ_NORMAL);
+		if (vep->o_crc == 0) {
+			vep->crc = vep->crcp;
+			vep->o_crc = vep->o_pending;
+		} else {
+			vep->crc = crc32_combine(vep->crc,
+			    vep->crcp, vep->o_pending);
+			vep->o_crc += vep->o_pending;
+		}
+		vep->crcp = crc32(0L, Z_NULL, 0);
+		vep->o_wait += vep->o_pending;
+		vep->o_pending = 0;
+	}
+
+	/* * Process this bit of input */
+	AN(vep->ver_p);
+	l = p - vep->ver_p;
+	assert(l >= 0);
+	vep->crc = crc32(vep->crc, (const void*)vep->ver_p, l);
+	vep->o_crc += l;
+	vep->ver_p = p;
+
+	vep->o_wait += l;
+	vep->last_mark = mark;
+	(void)vep->cb(vep->wrk, l, VGZ_NORMAL);
+}
+
+static void
+vep_mark_verbatim(struct vep_state *vep, const char *p)
+{
+
+	vep_mark_common(vep, p, VERBATIM);
+	vep->nm_verbatim++;
+}
+
+static void
+vep_mark_skip(struct vep_state *vep, const char *p)
+{
+
+	vep_mark_common(vep, p, SKIP);
+	vep->nm_skip++;
+}
+
+static void
+vep_mark_pending(struct vep_state *vep, const char *p)
+{
+	ssize_t l;
+
+	AN(vep->ver_p);
+	l = p - vep->ver_p;
+	assert(l > 0);
+	assert(l >= 0);
+	vep->crcp = crc32(vep->crcp, (const void *)vep->ver_p, l);
+	vep->ver_p = p;
+
+	vep->o_pending += l;
+	vep->nm_pending++;
+}
+
+/*---------------------------------------------------------------------
+ */
+
+static void __match_proto__()
+vep_do_comment(struct vep_state *vep, enum dowhat what)
+{
+	Debug("DO_COMMENT(%d)\n", what);
+	assert(what == DO_TAG);
+	if (!vep->emptytag)
+		vep_error(vep, "ESI 1.0 <esi:comment> needs final '/'");
+}
+
+/*---------------------------------------------------------------------
+ */
+
+static void __match_proto__()
+vep_do_remove(struct vep_state *vep, enum dowhat what)
+{
+	Debug("DO_REMOVE(%d, end %d empty %d remove %d)\n",
+	    what, vep->endtag, vep->emptytag, vep->remove);
+	assert(what == DO_TAG);
+	if (vep->emptytag) {
+		vep_error(vep,
+		    "ESI 1.0 <esi:remove/> not legal");
+	} else {
+		if (vep->remove && !vep->endtag)
+			vep_error(vep,
+			    "ESI 1.0 <esi:remove> already open");
+		else if (!vep->remove && vep->endtag)
+			vep_error(vep,
+			    "ESI 1.0 <esi:remove> not open");
+		else
+			vep->remove = !vep->endtag;
+	}
+}
+
+/*---------------------------------------------------------------------
+ */
+
+static void __match_proto__()
+vep_do_include(struct vep_state *vep, enum dowhat what)
+{
+	char *p, *q, *h;
+	ssize_t l;
+	txt url;
+
+	Debug("DO_INCLUDE(%d)\n", what);
+	if (what == DO_ATTR) {
+		Debug("ATTR (%s) (%s)\n", vep->match_hit->match,
+			VSB_data(vep->attr_vsb));
+		if (vep->include_src != NULL) {
+			vep_error(vep,
+			    "ESI 1.0 <esi:include> "
+			    "has multiple src= attributes");
+			vep->state = VEP_TAGERROR;
+			VSB_delete(vep->attr_vsb);
+			VSB_delete(vep->include_src);
+			vep->attr_vsb = NULL;
+			vep->include_src = NULL;
+			return;
+		}
+		XXXAZ(vep->include_src);	/* multiple src= */
+		vep->include_src = vep->attr_vsb;
+		return;
+	}
+	assert(what == DO_TAG);
+	if (!vep->emptytag)
+		vep_warn(vep,
+		    "ESI 1.0 <esi:include> lacks final '/'");
+	if (vep->include_src == NULL) {
+		vep_error(vep,
+		    "ESI 1.0 <esi:include> lacks src attr");
+		return;
+	}
+
+	/*
+	 * Strictly speaking, we ought to spit out any piled up skip before
+	 * emitting the VEC for the include, but objectively that makes no
+	 * difference and robs us of a chance to collapse another skip into
+	 * this on so we don't do that.
+	 * However, we cannot tolerate any verbatim stuff piling up.
+	 * The mark_skip() before calling dostuff should have taken
+	 * care of that.  Make sure.
+	 */
+	assert(vep->o_wait == 0 || vep->last_mark == SKIP);
+	/* XXX: what if it contains NUL bytes ?? */
+	p = VSB_data(vep->include_src);
+	l = VSB_len(vep->include_src);
+	h = 0;
+
+	VSB_printf(vep->vsb, "%c", VEC_INCL);
+	if (l > 7 && !memcmp(p, "http://", 7)) {
+		h = p + 7;
+		p = strchr(h, '/');
+		AN(p);
+		Debug("HOST <%.*s> PATH <%s>\n", (int)(p-h),h, p);
+		VSB_printf(vep->vsb, "Host: %.*s%c",
+		    (int)(p-h), h, 0);
+	} else if (*p == '/') {
+		VSB_printf(vep->vsb, "%c", 0);
+	} else {
+		VSB_printf(vep->vsb, "%c", 0);
+		url = vep->wrk->bereq->hd[HTTP_HDR_URL];
+		/* Look for the last / before a '?' */
+		h = NULL;
+		for (q = url.b; q < url.e && *q != '?'; q++)
+			if (*q == '/')
+				h = q;
+		if (h == NULL)
+			h = q + 1;
+
+		Debug("INCL:: [%.*s]/[%s]\n",
+		    (int)(h - url.b), url.b, p);
+		VSB_printf(vep->vsb, "%.*s/", (int)(h - url.b), url.b);
+	}
+	l -= (p - VSB_data(vep->include_src));
+	for (q = p; *q != '\0'; ) {
+		if (*q == '&') {
+#define R(w,f,r)							\
+			if (q + w <= p + l && !memcmp(q, f, w)) { \
+				VSB_printf(vep->vsb, "%c", r);	\
+				q += w;				\
+				continue;			\
+			}
+			R(6, "&apos;", '\'');
+			R(6, "&quot;", '"');
+			R(4, "&lt;", '<');
+			R(4, "&gt;", '>');
+			R(5, "&amp;", '&');
+		}
+		VSB_printf(vep->vsb, "%c", *q++);
+	}
+#undef R
+	VSB_printf(vep->vsb, "%c", 0);
+
+	VSB_delete(vep->include_src);
+	vep->include_src = NULL;
+}
+
+/*---------------------------------------------------------------------
+ * Lex/Parse object for ESI instructions
+ *
+ * This function is called with the input object piecemal so do not
+ * assume that we have more than one char available at at time, but
+ * optimize for getting huge chunks.
+ *
+ * NB: At the bottom of this source-file, there is a dot-diagram matching
+ * NB: the state-machine.  Please maintain it along with the code.
+ */
+
+void
+VEP_Parse(const struct worker *w, const char *p, size_t l)
+{
+	struct vep_state *vep;
+	const char *e;
+	struct vep_match *vm;
+	int i;
+
+	CHECK_OBJ_NOTNULL(w, WORKER_MAGIC);
+	vep = w->vep;
+	CHECK_OBJ_NOTNULL(vep, VEP_MAGIC);
+	assert(l > 0);
+
+	/* XXX: Really need to fix this */
+	if (vep->hack_p == NULL)
+		vep->hack_p = p;
+
+	vep->ver_p = p;
+
+	e = p + l;
+
+	while (p < e) {
+		AN(vep->state);
+		i = e - p;
+		if (i > 10)
+			i = 10;
+		Debug("EP %s %d (%.*s) [%.*s]\n",
+		    vep->state,
+		    vep->remove,
+		    vep->tag_i, vep->tag,
+		    i, p);
+		assert(p >= vep->ver_p);
+
+		/******************************************************
+		 * SECTION A
+		 */
+
+		if (vep->state == VEP_START) {
+			if (cache_param->esi_syntax & 0x1)
+				vep->state = VEP_NEXTTAG;
+			else
+				vep->state = VEP_TESTXML;
+		} else if (vep->state == VEP_TESTXML) {
+			/*
+			 * If the first non-whitespace char is different
+			 * from '<' we assume this is not XML.
+			 */
+			while (p < e && vct_islws(*p))
+				p++;
+			vep_mark_verbatim(vep, p);
+			if (p < e && *p == '<') {
+				p++;
+				vep->state = VEP_STARTTAG;
+			} else if (p < e) {
+				WSLB(vep->wrk, SLT_ESI_xmlerror,
+				    "No ESI processing, first char not '<'");
+				vep->state = VEP_NOTXML;
+			}
+		} else if (vep->state == VEP_NOTXML) {
+			/*
+			 * This is not recognized as XML, just skip thru
+			 * vfp_esi_end() will handle the rest
+			 */
+			p = e;
+			vep_mark_verbatim(vep, p);
+
+		/******************************************************
+		 * SECTION B
+		 */
+
+		} else if (vep->state == VEP_NOTMYTAG) {
+			if (cache_param->esi_syntax & 0x2) {
+				p++;
+				vep->state = VEP_NEXTTAG;
+			} else {
+				vep->tag_i = 0;
+				while (p < e) {
+					if (*p++ == '>') {
+						vep->state = VEP_NEXTTAG;
+						break;
+					}
+				}
+			}
+			if (p == e && !vep->remove)
+				vep_mark_verbatim(vep, p);
+		} else if (vep->state == VEP_NEXTTAG) {
+			/*
+			 * Hunt for start of next tag and keep an eye
+			 * out for end of EsiCmt if armed.
+			 */
+			vep->emptytag = 0;
+			vep->endtag = 0;
+			vep->attr = NULL;
+			vep->dostuff = NULL;
+			while (p < e && *p != '<') {
+				if (vep->esicmt_p == NULL) {
+					p++;
+					continue;
+				}
+				if (*p != *vep->esicmt_p) {
+					p++;
+					vep->esicmt_p = vep->esicmt;
+					continue;
+				}
+				if (!vep->remove &&
+				    vep->esicmt_p == vep->esicmt)
+					vep_mark_verbatim(vep, p);
+				p++;
+				if (*++vep->esicmt_p == '\0') {
+					vep->esi_found = 1;
+					vep->esicmt = NULL;
+					vep->esicmt_p = NULL;
+					/*
+					 * The end of the esicmt
+					 * should not be emitted.
+					 * But the stuff before should
+					 */
+					vep_mark_skip(vep, p);
+				}
+			}
+			if (p < e) {
+				if (!vep->remove)
+					vep_mark_verbatim(vep, p);
+				assert(*p == '<');
+				p++;
+				vep->state = VEP_STARTTAG;
+			} else if (vep->esicmt_p == vep->esicmt && !vep->remove)
+				vep_mark_verbatim(vep, p);
+
+		/******************************************************
+		 * SECTION C
+		 */
+
+		} else if (vep->state == VEP_STARTTAG) {
+			/*
+			 * Start of tag, set up match table
+			 */
+			if (p < e) {
+				if (*p == '/') {
+					vep->endtag = 1;
+					p++;
+				}
+				vep->match = vep_match_starttag;
+				vep->state = VEP_MATCH;
+			}
+		} else if (vep->state == VEP_COMMENT) {
+			/*
+			 * We are in a comment, find out if it is an
+			 * ESI comment or a regular comment
+			 */
+			if (vep->esicmt == NULL)
+				vep->esicmt_p = vep->esicmt = "esi";
+			while (p < e) {
+				if (*p != *vep->esicmt_p) {
+					vep->esicmt_p = vep->esicmt = NULL;
+					vep->until_p = vep->until = "-->";
+					vep->until_s = VEP_NEXTTAG;
+					vep->state = VEP_UNTIL;
+					vep_mark_verbatim(vep, p);
+					break;
+				}
+				p++;
+				if (*++vep->esicmt_p != '\0')
+					continue;
+				if (vep->remove)
+					vep_error(vep,
+					    "ESI 1.0 Nested <!--esi"
+					    " element in <esi:remove>");
+				vep->esicmt_p = vep->esicmt = "-->";
+				vep->state = VEP_NEXTTAG;
+				vep_mark_skip(vep, p);
+				break;
+			}
+		} else if (vep->state == VEP_CDATA) {
+			/*
+			 * Easy: just look for the end of CDATA
+			 */
+			vep->until_p = vep->until = "]]>";
+			vep->until_s = VEP_NEXTTAG;
+			vep->state = VEP_UNTIL;
+		} else if (vep->state == VEP_ESITAG) {
+			vep->in_esi_tag = 1;
+			vep->esi_found = 1;
+			vep_mark_skip(vep, p);
+			vep->match = vep_match_esi;
+			vep->state = VEP_MATCH;
+		} else if (vep->state == VEP_ESIINCLUDE) {
+			if (vep->remove) {
+				vep_error(vep,
+				    "ESI 1.0 <esi:include> element"
+				    " nested in <esi:remove>");
+				vep->state = VEP_TAGERROR;
+			} else if (vep->endtag) {
+				vep_error(vep,
+				    "ESI 1.0 </esi:include> illegal end-tag");
+				vep->state = VEP_TAGERROR;
+			} else {
+				vep->dostuff = vep_do_include;
+				vep->state = VEP_INTAG;
+				vep->attr = vep_match_attr_include;
+			}
+		} else if (vep->state == VEP_ESIREMOVE) {
+			vep->dostuff = vep_do_remove;
+			vep->state = VEP_INTAG;
+		} else if (vep->state == VEP_ESICOMMENT) {
+			if (vep->remove) {
+				vep_error(vep,
+				    "ESI 1.0 <esi:comment> element"
+				    " nested in <esi:remove>");
+				vep->state = VEP_TAGERROR;
+			} else if (vep->endtag) {
+				vep_error(vep,
+				    "ESI 1.0 </esi:comment> illegal end-tag");
+				vep->state = VEP_TAGERROR;
+			} else {
+				vep->dostuff = vep_do_comment;
+				vep->state = VEP_INTAG;
+			}
+		} else if (vep->state == VEP_ESIBOGON) {
+			vep_error(vep,
+			    "ESI 1.0 <esi:bogus> element");
+			vep->state = VEP_TAGERROR;
+
+		/******************************************************
+		 * SECTION D
+		 */
+
+		} else if (vep->state == VEP_INTAG) {
+			vep->tag_i = 0;
+			while (p < e && vct_islws(*p) && !vep->emptytag) {
+				p++;
+				vep->canattr = 1;
+			}
+			if (p < e && *p == '/' && !vep->emptytag) {
+				p++;
+				vep->emptytag = 1;
+				vep->canattr = 0;
+			}
+			if (p < e && *p == '>') {
+				p++;
+				AN(vep->dostuff);
+				vep_mark_skip(vep, p);
+				vep->dostuff(vep, DO_TAG);
+				vep->in_esi_tag = 0;
+				vep->state = VEP_NEXTTAG;
+			} else if (p < e && vep->emptytag) {
+				vep_error(vep,
+				    "XML 1.0 '>' does not follow '/' in tag");
+				vep->state = VEP_TAGERROR;
+			} else if (p < e && vep->canattr &&
+			    vct_isxmlnamestart(*p)) {
+				vep->state = VEP_ATTR;
+			} else if (p < e) {
+				vep_error(vep,
+				    "XML 1.0 Illegal attribute start char");
+				vep->state = VEP_TAGERROR;
+			}
+		} else if (vep->state == VEP_TAGERROR) {
+			while (p < e && *p != '>')
+				p++;
+			if (p < e) {
+				p++;
+				vep_mark_skip(vep, p);
+				vep->in_esi_tag = 0;
+				vep->state = VEP_NEXTTAG;
+			}
+
+		/******************************************************
+		 * SECTION E
+		 */
+
+		} else if (vep->state == VEP_ATTR) {
+			AZ(vep->attr_delim);
+			if (vep->attr == NULL) {
+				p++;
+				AZ(vep->attr_vsb);
+				vep->state = VEP_SKIPATTR;
+			} else {
+				vep->match = vep->attr;
+				vep->state = VEP_MATCH;
+			}
+		} else if (vep->state == VEP_SKIPATTR) {
+			while (p < e && vct_isxmlname(*p))
+				p++;
+			if (p < e && *p == '=') {
+				p++;
+				vep->state = VEP_ATTRDELIM;
+			} else if (p < e && *p == '>') {
+				vep->state = VEP_INTAG;
+			} else if (p < e && *p == '/') {
+				vep->state = VEP_INTAG;
+			} else if (p < e && vct_issp(*p)) {
+				vep->state = VEP_INTAG;
+			} else if (p < e) {
+				vep_error(vep,
+				    "XML 1.0 Illegal attr char");
+				vep->state = VEP_TAGERROR;
+			}
+		} else if (vep->state == VEP_ATTRGETVAL) {
+			vep->attr_vsb = VSB_new_auto();
+			vep->state = VEP_ATTRDELIM;
+		} else if (vep->state == VEP_ATTRDELIM) {
+			AZ(vep->attr_delim);
+			if (*p == '"' || *p == '\'') {
+				vep->attr_delim = *p++;
+				vep->state = VEP_ATTRVAL;
+			} else if (!vct_issp(*p)) {
+				vep->attr_delim = ' ';
+				vep->state = VEP_ATTRVAL;
+			} else {
+				vep_error(vep,
+				    "XML 1.0 Illegal attribute delimiter");
+				vep->state = VEP_TAGERROR;
+			}
+
+		} else if (vep->state == VEP_ATTRVAL) {
+			while (p < e && *p != '>' && *p != vep->attr_delim &&
+			   (vep->attr_delim != ' ' || !vct_issp(*p))) {
+				if (vep->attr_vsb != NULL)
+					VSB_bcat(vep->attr_vsb, p, 1);
+				p++;
+			}
+			if (p < e && *p == '>') {
+				vep_error(vep,
+				    "XML 1.0 Missing end attribute delimiter");
+				vep->state = VEP_TAGERROR;
+				vep->attr_delim = 0;
+				if (vep->attr_vsb != NULL) {
+					AZ(VSB_finish(vep->attr_vsb));
+					VSB_delete(vep->attr_vsb);
+					vep->attr_vsb = NULL;
+				}
+			} else if (p < e) {
+				vep->attr_delim = 0;
+				p++;
+				vep->state = VEP_INTAG;
+				if (vep->attr_vsb != NULL) {
+					AZ(VSB_finish(vep->attr_vsb));
+					AN(vep->dostuff);
+					vep->dostuff(vep, DO_ATTR);
+					vep->attr_vsb = NULL;
+				}
+			}
+
+		/******************************************************
+		 * Utility Section
+		 */
+
+		} else if (vep->state == VEP_MATCH) {
+			/*
+			 * Match against a table
+			 */
+			vm = vep_match(vep, p, e);
+			vep->match_hit = vm;
+			if (vm != NULL) {
+				if (vm->match != NULL)
+					p += strlen(vm->match);
+				vep->state = *vm->state;
+				vep->match = NULL;
+				vep->tag_i = 0;
+			} else {
+				memcpy(vep->tag, p, e - p);
+				vep->tag_i = e - p;
+				vep->state = VEP_MATCHBUF;
+				p = e;
+			}
+		} else if (vep->state == VEP_MATCHBUF) {
+			/*
+			 * Match against a table while split over input
+			 * sections.
+			 */
+			do {
+				if (*p == '>') {
+					for (vm = vep->match;
+					    vm->match != NULL; vm++)
+						continue;
+					AZ(vm->match);
+				} else {
+					vep->tag[vep->tag_i++] = *p++;
+					vm = vep_match(vep,
+					    vep->tag, vep->tag + vep->tag_i);
+					if (vm && vm->match == NULL) {
+						vep->tag_i--;
+						p--;
+					}
+				}
+			} while (vm == NULL && p < e);
+			vep->match_hit = vm;
+			if (vm == NULL) {
+				assert(p == e);
+			} else {
+				vep->state = *vm->state;
+				vep->match = NULL;
+			}
+		} else if (vep->state == VEP_UNTIL) {
+			/*
+			 * Skip until we see magic string
+			 */
+			while (p < e) {
+				if (*p++ != *vep->until_p++) {
+					vep->until_p = vep->until;
+				} else if (*vep->until_p == '\0') {
+					vep->state = vep->until_s;
+					break;
+				}
+			}
+			if (p == e && !vep->remove)
+				vep_mark_verbatim(vep, p);
+		} else {
+			Debug("*** Unknown state %s\n", vep->state);
+			INCOMPL();
+		}
+	}
+	/*
+	 * We must always mark up the storage we got, try to do so
+	 * in the most efficient way, in particular with respect to
+	 * minimizing and limiting use of pending.
+	 */
+	if (p == vep->ver_p)
+		;
+	else if (vep->in_esi_tag)
+		vep_mark_skip(vep, p);
+	else if (vep->remove)
+		vep_mark_skip(vep, p);
+	else
+		vep_mark_pending(vep, p);
+}
+
+/*---------------------------------------------------------------------
+ */
+
+static ssize_t __match_proto__()
+vep_default_cb(struct worker *w, ssize_t l, enum vgz_flag flg)
+{
+
+	(void)flg;
+	AN(w->vep);
+	w->vep->cb_x += l;
+Debug("CB(%jd,%d) = %jd\n", (intmax_t)l, flg, (intmax_t)w->vep->cb_x);
+	return (w->vep->cb_x);
+}
+
+/*---------------------------------------------------------------------
+ */
+
+void
+VEP_Init(struct worker *w, vep_callback_t *cb)
+{
+	struct vep_state *vep;
+
+	CHECK_OBJ_NOTNULL(w, WORKER_MAGIC);
+	AZ(w->vep);
+	w->vep = (void*)WS_Alloc(w->ws, sizeof *vep);
+	AN(w->vep);
+
+	vep = w->vep;
+	memset(vep, 0, sizeof *vep);
+	vep->magic = VEP_MAGIC;
+	vep->wrk = w;
+	vep->vsb = VSB_new_auto();
+	AN(vep->vsb);
+
+	if (cb != NULL) {
+		vep->dogzip = 1;
+		/* XXX */
+		VSB_printf(vep->vsb, "%c", VEC_GZ);
+		vep->cb = cb;
+	} else {
+		vep->cb = vep_default_cb;
+	}
+
+	vep->state = VEP_START;
+	vep->crc = crc32(0L, Z_NULL, 0);
+	vep->crcp = crc32(0L, Z_NULL, 0);
+
+	/*
+	 * We must force the GZIP header out as a SKIP string, otherwise
+	 * an object starting with <esi:include would have its GZIP header
+	 * appear after the included object (e000026.vtc)
+	 */
+	vep->startup = 1;
+	vep->ver_p = "";
+	vep->last_mark = SKIP;
+	vep_mark_common(vep, vep->ver_p, VERBATIM);
+	vep->startup = 0;
+}
+
+/*---------------------------------------------------------------------
+ */
+
+struct vsb *
+VEP_Finish(struct worker *w)
+{
+	struct vep_state *vep;
+	ssize_t l, lcb;
+
+	CHECK_OBJ_NOTNULL(w, WORKER_MAGIC);
+	vep = w->vep;
+	CHECK_OBJ_NOTNULL(vep, VEP_MAGIC);
+
+	if (vep->o_pending)
+		vep_mark_common(vep, vep->ver_p, vep->last_mark);
+	if (vep->o_wait > 0) {
+		lcb = vep->cb(vep->wrk, 0, VGZ_ALIGN);
+		vep_emit_common(vep, lcb - vep->o_last, vep->last_mark);
+	}
+	(void)vep->cb(vep->wrk, 0, VGZ_FINISH);
+
+	w->vep = NULL;
+
+	AZ(VSB_finish(vep->vsb));
+	l = VSB_len(vep->vsb);
+	if (vep->esi_found && l > 0)
+		return (vep->vsb);
+	VSB_delete(vep->vsb);
+	return (NULL);
+}
+
+#if 0
+
+digraph xml {
+	rankdir="LR"
+	size="7,10"
+#################################################################
+# SECTION A
+#
+
+START		[shape=ellipse]
+TESTXML		[shape=ellipse]
+NOTXML		[shape=ellipse]
+NEXTTAGa	[shape=hexagon, label="NEXTTAG"]
+STARTTAGa	[shape=hexagon, label="STARTTAG"]
+START		-> TESTXML
+START		-> NEXTTAGa	[style=dotted, label="syntax:1"]
+TESTXML		-> TESTXML	[label="lws"]
+TESTXML		-> NOTXML
+TESTXML		-> STARTTAGa	[label="'<'"]
+
+#################################################################
+# SECTION B
+
+NOTMYTAG	[shape=ellipse]
+NEXTTAG		[shape=ellipse]
+NOTMYTAG	-> NEXTTAG	[style=dotted, label="syntax:2"]
+STARTTAGb	[shape=hexagon, label="STARTTAG"]
+NOTMYTAG	-> NEXTTAG	[label="'>'"]
+NOTMYTAG	-> NOTMYTAG	[label="*"]
+NEXTTAG		-> NEXTTAG	[label="'-->'"]
+NEXTTAG		-> NEXTTAG	[label="*"]
+NEXTTAG		-> STARTTAGb	[label="'<'"]
+
+#################################################################
+# SECTION C
+
+STARTTAG	[shape=ellipse]
+COMMENT		[shape=ellipse]
+CDATA		[shape=ellipse]
+ESITAG		[shape=ellipse]
+ESIETAG		[shape=ellipse]
+ESIINCLUDE	[shape=ellipse]
+ESIREMOVE	[shape=ellipse]
+ESICOMMENT	[shape=ellipse]
+ESIBOGON	[shape=ellipse]
+INTAGc		[shape=hexagon, label="INTAG"]
+NOTMYTAGc	[shape=hexagon, label="NOTMYTAG"]
+NEXTTAGc	[shape=hexagon, label="NEXTTAG"]
+TAGERRORc	[shape=hexagon, label="TAGERROR"]
+C1		[shape=circle,label=""]
+STARTTAG	-> COMMENT	[label="'<!--'"]
+STARTTAG	-> ESITAG	[label="'<esi'"]
+STARTTAG	-> CDATA	[label="'<![CDATA['"]
+STARTTAG	-> NOTMYTAGc	[label="'*'"]
+COMMENT		-> NEXTTAGc	[label="'esi'"]
+COMMENT		-> C1		[label="*"]
+C1		-> C1		[label="*"]
+C1		-> NEXTTAGc	[label="-->"]
+CDATA		-> CDATA	[label="*"]
+CDATA		-> NEXTTAGc	[label="]]>"]
+ESITAG		-> ESIINCLUDE	[label="'include'"]
+ESITAG		-> ESIREMOVE	[label="'remove'"]
+ESITAG		-> ESICOMMENT	[label="'comment'"]
+ESITAG		-> ESIBOGON	[label="*"]
+ESICOMMENT	-> INTAGc
+ESICOMMENT	-> TAGERRORc
+ESICOMMENT	-> TAGERRORc	[style=dotted, label="nested\nin\nremove"]
+ESIREMOVE	-> INTAGc
+ESIREMOVE	-> TAGERRORc
+ESIINCLUDE	-> INTAGc
+ESIINCLUDE	-> TAGERRORc
+ESIINCLUDE	-> TAGERRORc	[style=dotted, label="nested\nin\nremove"]
+ESIBOGON	-> TAGERRORc
+
+#################################################################
+# SECTION D
+
+INTAG		[shape=ellipse]
+TAGERROR	[shape=ellipse]
+NEXTTAGd	[shape=hexagon, label="NEXTTAG"]
+ATTRd		[shape=hexagon, label="ATTR"]
+D1		[shape=circle, label=""]
+D2		[shape=circle, label=""]
+INTAG		-> D1		[label="lws"]
+D1		-> D2		[label="/"]
+INTAG		-> D2		[label="/"]
+INTAG		-> NEXTTAGd	[label=">"]
+D1		-> NEXTTAGd	[label=">"]
+D2		-> NEXTTAGd	[label=">"]
+D1		-> ATTRd	[label="XMLstartchar"]
+D1		-> TAGERROR	[label="*"]
+D2		-> TAGERROR	[label="*"]
+TAGERROR	-> TAGERROR	[label="*"]
+TAGERROR	-> NEXTTAGd	[label="'>'"]
+
+#################################################################
+# SECTION E
+
+ATTR		[shape=ellipse]
+SKIPATTR	[shape=ellipse]
+ATTRGETVAL	[shape=ellipse]
+ATTRDELIM	[shape=ellipse]
+ATTRVAL		[shape=ellipse]
+TAGERRORe	[shape=hexagon, label="TAGERROR"]
+INTAGe		[shape=hexagon, label="INTAG"]
+ATTR		-> SKIPATTR	[label="*"]
+ATTR		-> ATTRGETVAL	[label="wanted attr"]
+SKIPATTR	-> SKIPATTR	[label="XMLname"]
+SKIPATTR	-> ATTRDELIM	[label="'='"]
+SKIPATTR	-> TAGERRORe	[label="*"]
+ATTRGETVAL	-> ATTRDELIM
+ATTRDELIM	-> ATTRVAL	[label="\""]
+ATTRDELIM	-> ATTRVAL	[label="\'"]
+ATTRDELIM	-> ATTRVAL	[label="*"]
+ATTRDELIM	-> TAGERRORe	[label="lws"]
+ATTRVAL		-> TAGERRORe	[label="'>'"]
+ATTRVAL		-> INTAGe	[label="delim"]
+ATTRVAL		-> ATTRVAL	[label="*"]
+
+}
+
+#endif
diff --git a/bin/varnishd/cache/cache_expire.c b/bin/varnishd/cache/cache_expire.c
new file mode 100644
index 0000000..23e3fc6
--- /dev/null
+++ b/bin/varnishd/cache/cache_expire.c
@@ -0,0 +1,490 @@
+/*-
+ * Copyright (c) 2006 Verdens Gang AS
+ * Copyright (c) 2006-2011 Varnish Software AS
+ * All rights reserved.
+ *
+ * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * LRU and object timer handling.
+ *
+ * We have two data structures, a LRU-list and a binary heap for the timers
+ * and two ways to kill objects: TTL-timeouts and LRU cleanups.
+ *
+ * Any object on the LRU is also on the binheap and vice versa.
+ *
+ * We hold a single object reference for both data structures.
+ *
+ * An attempted overview:
+ *
+ *	                        EXP_Ttl()      EXP_Grace()   EXP_Keep()
+ *				   |                |            |
+ *      entered                    v                v            |
+ *         |                       +--------------->+            |
+ *         v                       |      grace                  |
+ *         +---------------------->+                             |
+ *                  ttl            |                             v
+ *                                 +---------------------------->+
+ *                                     keep
+ *
+ */
+
+#include "config.h"
+
+#include <math.h>
+
+#include "cache.h"
+
+#include "binary_heap.h"
+#include "hash/hash_slinger.h"
+#include "vtim.h"
+
+static pthread_t exp_thread;
+static struct binheap *exp_heap;
+static struct lock exp_mtx;
+
+/*--------------------------------------------------------------------
+ * struct exp manipulations
+ *
+ * The Get/Set functions encapsulate the mutual magic between the
+ * fields in one single place.
+ */
+
+void
+EXP_Clr(struct exp *e)
+{
+
+	e->ttl = -1;
+	e->grace = -1;
+	e->keep = -1;
+	e->age = 0;
+	e->entered = 0;
+}
+
+#define EXP_ACCESS(fld, low_val, extra)				\
+	double							\
+	EXP_Get_##fld(const struct exp *e)			\
+	{							\
+		return (e->fld > 0. ? e->fld : low_val);	\
+	}							\
+								\
+	void							\
+	EXP_Set_##fld(struct exp *e, double v)			\
+	{							\
+		if (v > 0.)					\
+			e->fld = v;				\
+		else {						\
+			e->fld = -1.;				\
+			extra;					\
+		}						\
+	}							\
+
+EXP_ACCESS(ttl, -1., (e->grace = e->keep = -1.))
+EXP_ACCESS(grace, 0., )
+EXP_ACCESS(keep, 0.,)
+
+/*--------------------------------------------------------------------
+ * Calculate an objects effective keep, grace or ttl time, suitably
+ * adjusted for defaults and by per-session limits.
+ */
+
+static double
+EXP_Keep(const struct sess *sp, const struct object *o)
+{
+	double r;
+
+	r = (double)cache_param->default_keep;
+	if (o->exp.keep > 0.)
+		r = o->exp.keep;
+	if (sp != NULL && sp->exp.keep > 0. && sp->exp.keep < r)
+		r = sp->exp.keep;
+	return (EXP_Ttl(sp, o) + r);
+}
+
+double
+EXP_Grace(const struct sess *sp, const struct object *o)
+{
+	double r;
+
+	r = (double)cache_param->default_grace;
+	if (o->exp.grace >= 0.)
+		r = o->exp.grace;
+	if (sp != NULL && sp->exp.grace > 0. && sp->exp.grace < r)
+		r = sp->exp.grace;
+	return (EXP_Ttl(sp, o) + r);
+}
+
+double
+EXP_Ttl(const struct sess *sp, const struct object *o)
+{
+	double r;
+
+	r = o->exp.ttl;
+	if (sp != NULL && sp->exp.ttl > 0. && sp->exp.ttl < r)
+		r = sp->exp.ttl;
+	return (o->exp.entered + r);
+}
+
+/*--------------------------------------------------------------------
+ * When & why does the timer fire for this object ?
+ */
+
+static int
+update_object_when(const struct object *o)
+{
+	struct objcore *oc;
+	double when, w2;
+
+	CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC);
+	oc = o->objcore;
+	CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
+	Lck_AssertHeld(&exp_mtx);
+
+	when = EXP_Keep(NULL, o);
+	w2 = EXP_Grace(NULL, o);
+	if (w2 > when)
+		when = w2;
+	assert(!isnan(when));
+	if (when == oc->timer_when)
+		return (0);
+	oc->timer_when = when;
+	return (1);
+}
+
+/*--------------------------------------------------------------------*/
+
+static void
+exp_insert(struct objcore *oc, struct lru *lru)
+{
+	CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
+	CHECK_OBJ_NOTNULL(lru, LRU_MAGIC);
+
+	Lck_AssertHeld(&lru->mtx);
+	Lck_AssertHeld(&exp_mtx);
+	assert(oc->timer_idx == BINHEAP_NOIDX);
+	binheap_insert(exp_heap, oc);
+	assert(oc->timer_idx != BINHEAP_NOIDX);
+	VTAILQ_INSERT_TAIL(&lru->lru_head, oc, lru_list);
+}
+
+/*--------------------------------------------------------------------
+ * Object has been added to cache, record in lru & binheap.
+ *
+ * The objcore comes with a reference, which we inherit.
+ */
+
+void
+EXP_Inject(struct objcore *oc, struct lru *lru, double when)
+{
+
+	CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
+	CHECK_OBJ_NOTNULL(lru, LRU_MAGIC);
+
+	Lck_Lock(&lru->mtx);
+	Lck_Lock(&exp_mtx);
+	oc->timer_when = when;
+	exp_insert(oc, lru);
+	Lck_Unlock(&exp_mtx);
+	Lck_Unlock(&lru->mtx);
+}
+
+/*--------------------------------------------------------------------
+ * Object has been added to cache, record in lru & binheap.
+ *
+ * We grab a reference to the object, which will keep it around until
+ * we decide its time to let it go.
+ */
+
+void
+EXP_Insert(struct object *o)
+{
+	struct objcore *oc;
+	struct lru *lru;
+
+	CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC);
+	oc = o->objcore;
+	CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
+	AssertObjBusy(o);
+	HSH_Ref(oc);
+
+	assert(o->exp.entered != 0 && !isnan(o->exp.entered));
+	o->last_lru = o->exp.entered;
+
+	lru = oc_getlru(oc);
+	CHECK_OBJ_NOTNULL(lru, LRU_MAGIC);
+	Lck_Lock(&lru->mtx);
+	Lck_Lock(&exp_mtx);
+	(void)update_object_when(o);
+	exp_insert(oc, lru);
+	Lck_Unlock(&exp_mtx);
+	Lck_Unlock(&lru->mtx);
+	oc_updatemeta(oc);
+}
+
+/*--------------------------------------------------------------------
+ * Object was used, move to tail of LRU list.
+ *
+ * To avoid the exp_mtx becoming a hotspot, we only attempt to move
+ * objects if they have not been moved recently and if the lock is available.
+ * This optimization obviously leaves the LRU list imperfectly sorted.
+ */
+
+int
+EXP_Touch(struct objcore *oc)
+{
+	struct lru *lru;
+
+	CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
+
+	/*
+	 * For -spersistent we don't move objects on the lru list.  Each
+	 * segment has its own LRU list, and the order on it is not material
+	 * for anything.  The code below would move the objects to the
+	 * LRU list of the currently open segment, which would prevent
+	 * the cleaner from doing its job.
+	 */
+	if (oc->flags & OC_F_LRUDONTMOVE)
+		return (0);
+
+	lru = oc_getlru(oc);
+	CHECK_OBJ_NOTNULL(lru, LRU_MAGIC);
+
+	/*
+	 * We only need the LRU lock here.  The locking order is LRU->EXP
+	 * so we can trust the content of the oc->timer_idx without the
+	 * EXP lock.   Since each lru list has its own lock, this should
+	 * reduce contention a fair bit
+	 */
+	if (Lck_Trylock(&lru->mtx))
+		return (0);
+
+	if (oc->timer_idx != BINHEAP_NOIDX) {
+		VTAILQ_REMOVE(&lru->lru_head, oc, lru_list);
+		VTAILQ_INSERT_TAIL(&lru->lru_head, oc, lru_list);
+		VSC_C_main->n_lru_moved++;
+	}
+	Lck_Unlock(&lru->mtx);
+	return (1);
+}
+
+/*--------------------------------------------------------------------
+ * We have changed one or more of the object timers, shuffle it
+ * accordingly in the binheap
+ *
+ * The VCL code can send us here on a non-cached object, just return.
+ *
+ * XXX: special case check for ttl = 0 ?
+ */
+
+void
+EXP_Rearm(const struct object *o)
+{
+	struct objcore *oc;
+	struct lru *lru;
+
+	CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC);
+	oc = o->objcore;
+	if (oc == NULL)
+		return;
+	CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
+	lru = oc_getlru(oc);
+	Lck_Lock(&lru->mtx);
+	Lck_Lock(&exp_mtx);
+	/*
+	 * The hang-man might have this object of the binheap while
+	 * tending to a timer.  If so, we do not muck with it here.
+	 */
+	if (oc->timer_idx != BINHEAP_NOIDX && update_object_when(o)) {
+		assert(oc->timer_idx != BINHEAP_NOIDX);
+		binheap_reorder(exp_heap, oc->timer_idx);
+		assert(oc->timer_idx != BINHEAP_NOIDX);
+	}
+	Lck_Unlock(&exp_mtx);
+	Lck_Unlock(&lru->mtx);
+	oc_updatemeta(oc);
+}
+
+/*--------------------------------------------------------------------
+ * This thread monitors the root of the binary heap and whenever an
+ * object expires, accounting also for graceability, it is killed.
+ */
+
+static void * __match_proto__(void *start_routine(void *))
+exp_timer(struct sess *sp, void *priv)
+{
+	struct objcore *oc;
+	struct lru *lru;
+	double t;
+	struct object *o;
+
+	(void)priv;
+	t = VTIM_real();
+	oc = NULL;
+	while (1) {
+		if (oc == NULL) {
+			WSL_Flush(sp->wrk, 0);
+			WRK_SumStat(sp->wrk);
+			VTIM_sleep(cache_param->expiry_sleep);
+			t = VTIM_real();
+		}
+
+		Lck_Lock(&exp_mtx);
+		oc = binheap_root(exp_heap);
+		if (oc == NULL) {
+			Lck_Unlock(&exp_mtx);
+			continue;
+		}
+		CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
+
+		/*
+		 * We may have expired so many objects that our timestamp
+		 * got out of date, refresh it and check again.
+		 */
+		if (oc->timer_when > t)
+			t = VTIM_real();
+		if (oc->timer_when > t) {
+			Lck_Unlock(&exp_mtx);
+			oc = NULL;
+			continue;
+		}
+
+		/*
+		 * It's time...
+		 * Technically we should drop the exp_mtx, get the lru->mtx
+		 * get the exp_mtx again and then check that the oc is still
+		 * on the binheap.  We take the shorter route and try to
+		 * get the lru->mtx and punt if we fail.
+		 */
+
+		lru = oc_getlru(oc);
+		CHECK_OBJ_NOTNULL(lru, LRU_MAGIC);
+		if (Lck_Trylock(&lru->mtx)) {
+			Lck_Unlock(&exp_mtx);
+			oc = NULL;
+			continue;
+		}
+
+		/* Remove from binheap */
+		assert(oc->timer_idx != BINHEAP_NOIDX);
+		binheap_delete(exp_heap, oc->timer_idx);
+		assert(oc->timer_idx == BINHEAP_NOIDX);
+
+		/* And from LRU */
+		lru = oc_getlru(oc);
+		VTAILQ_REMOVE(&lru->lru_head, oc, lru_list);
+
+		Lck_Unlock(&exp_mtx);
+		Lck_Unlock(&lru->mtx);
+
+		VSC_C_main->n_expired++;
+
+		CHECK_OBJ_NOTNULL(oc->objhead, OBJHEAD_MAGIC);
+		o = oc_getobj(sp->wrk, oc);
+		WSL(sp->wrk, SLT_ExpKill, 0, "%u %.0f",
+		    o->xid, EXP_Ttl(NULL, o) - t);
+		(void)HSH_Deref(sp->wrk, oc, NULL);
+	}
+	NEEDLESS_RETURN(NULL);
+}
+
+/*--------------------------------------------------------------------
+ * Attempt to make space by nuking the oldest object on the LRU list
+ * which isn't in use.
+ * Returns: 1: did, 0: didn't, -1: can't
+ */
+
+int
+EXP_NukeOne(struct worker *w, struct lru *lru)
+{
+	struct objcore *oc;
+	struct object *o;
+
+	/* Find the first currently unused object on the LRU.  */
+	Lck_Lock(&lru->mtx);
+	Lck_Lock(&exp_mtx);
+	VTAILQ_FOREACH(oc, &lru->lru_head, lru_list) {
+		CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
+		assert (oc->timer_idx != BINHEAP_NOIDX);
+		/*
+		 * It wont release any space if we cannot release the last
+		 * reference, besides, if somebody else has a reference,
+		 * it's a bad idea to nuke this object anyway.
+		 */
+		if (oc->refcnt == 1)
+			break;
+	}
+	if (oc != NULL) {
+		VTAILQ_REMOVE(&lru->lru_head, oc, lru_list);
+		binheap_delete(exp_heap, oc->timer_idx);
+		assert(oc->timer_idx == BINHEAP_NOIDX);
+		VSC_C_main->n_lru_nuked++;
+	}
+	Lck_Unlock(&exp_mtx);
+	Lck_Unlock(&lru->mtx);
+
+	if (oc == NULL)
+		return (-1);
+
+	/* XXX: bad idea for -spersistent */
+	o = oc_getobj(w, oc);
+	WSL(w, SLT_ExpKill, 0, "%u LRU", o->xid);
+	(void)HSH_Deref(w, NULL, &o);
+	return (1);
+}
+
+/*--------------------------------------------------------------------
+ * BinHeap helper functions for objcore.
+ */
+
+static int
+object_cmp(void *priv, void *a, void *b)
+{
+	struct objcore *aa, *bb;
+
+	(void)priv;
+	CAST_OBJ_NOTNULL(aa, a, OBJCORE_MAGIC);
+	CAST_OBJ_NOTNULL(bb, b, OBJCORE_MAGIC);
+	return (aa->timer_when < bb->timer_when);
+}
+
+static void
+object_update(void *priv, void *p, unsigned u)
+{
+	struct objcore *oc;
+
+	(void)priv;
+	CAST_OBJ_NOTNULL(oc, p, OBJCORE_MAGIC);
+	oc->timer_idx = u;
+}
+
+/*--------------------------------------------------------------------*/
+
+void
+EXP_Init(void)
+{
+
+	Lck_New(&exp_mtx, lck_exp);
+	exp_heap = binheap_new(NULL, object_cmp, object_update);
+	XXXAN(exp_heap);
+	WRK_BgThread(&exp_thread, "cache-timeout", exp_timer, NULL);
+}
diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c
new file mode 100644
index 0000000..a678dcc
--- /dev/null
+++ b/bin/varnishd/cache/cache_fetch.c
@@ -0,0 +1,645 @@
+/*-
+ * Copyright (c) 2006 Verdens Gang AS
+ * Copyright (c) 2006-2011 Varnish Software AS
+ * All rights reserved.
+ *
+ * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include "config.h"
+
+#include <inttypes.h>
+#include <stdio.h>
+#include <stdlib.h>
+
+#include "cache.h"
+
+#include "cache_backend.h"
+#include "vcli_priv.h"
+#include "vct.h"
+#include "vtcp.h"
+
+static unsigned fetchfrag;
+
+/*--------------------------------------------------------------------
+ * We want to issue the first error we encounter on fetching and
+ * supress the rest.  This function does that.
+ *
+ * Other code is allowed to look at w->fetch_failed to bail out
+ *
+ * For convenience, always return -1
+ */
+
+int
+FetchError2(struct worker *w, const char *error, const char *more)
+{
+
+	CHECK_OBJ_NOTNULL(w, WORKER_MAGIC);
+	if (!w->fetch_failed) {
+		if (more == NULL)
+			WSLB(w, SLT_FetchError, "%s", error);
+		else
+			WSLB(w, SLT_FetchError, "%s: %s", error, more);
+	}
+	w->fetch_failed = 1;
+	return (-1);
+}
+
+int
+FetchError(struct worker *w, const char *error)
+{
+	return(FetchError2(w, error, NULL));
+}
+
+/*--------------------------------------------------------------------
+ * VFP_NOP
+ *
+ * This fetch-processor does nothing but store the object.
+ * It also documents the API
+ */
+
+/*--------------------------------------------------------------------
+ * VFP_BEGIN
+ *
+ * Called to set up stuff.
+ *
+ * 'estimate' is the estimate of the number of bytes we expect to receive,
+ * as seen on the socket, or zero if unknown.
+ */
+static void __match_proto__()
+vfp_nop_begin(struct worker *w, size_t estimate)
+{
+
+	if (estimate > 0)
+		(void)FetchStorage(w, estimate);
+}
+
+/*--------------------------------------------------------------------
+ * VFP_BYTES
+ *
+ * Process (up to) 'bytes' from the socket.
+ *
+ * Return -1 on error, issue FetchError()
+ *	will not be called again, once error happens.
+ * Return 0 on EOF on socket even if bytes not reached.
+ * Return 1 when 'bytes' have been processed.
+ */
+
+static int __match_proto__()
+vfp_nop_bytes(struct worker *w, struct http_conn *htc, ssize_t bytes)
+{
+	ssize_t l, wl;
+	struct storage *st;
+
+	AZ(w->fetch_failed);
+	while (bytes > 0) {
+		st = FetchStorage(w, 0);
+		if (st == NULL)
+			return(-1);
+		l = st->space - st->len;
+		if (l > bytes)
+			l = bytes;
+		wl = HTC_Read(w, htc, st->ptr + st->len, l);
+		if (wl <= 0)
+			return (wl);
+		st->len += wl;
+		w->fetch_obj->len += wl;
+		bytes -= wl;
+		if (w->do_stream)
+			RES_StreamPoll(w);
+	}
+	return (1);
+}
+
+/*--------------------------------------------------------------------
+ * VFP_END
+ *
+ * Finish & cleanup
+ *
+ * Return -1 for error
+ * Return 0 for OK
+ */
+
+static int __match_proto__()
+vfp_nop_end(struct worker *w)
+{
+	struct storage *st;
+
+	st = VTAILQ_LAST(&w->fetch_obj->store, storagehead);
+	if (st == NULL)
+		return (0);
+
+	if (st->len == 0) {
+		VTAILQ_REMOVE(&w->fetch_obj->store, st, list);
+		STV_free(st);
+		return (0);
+	}
+	if (st->len < st->space)
+		STV_trim(st, st->len);
+	return (0);
+}
+
+static struct vfp vfp_nop = {
+	.begin	=	vfp_nop_begin,
+	.bytes	=	vfp_nop_bytes,
+	.end	=	vfp_nop_end,
+};
+
+/*--------------------------------------------------------------------
+ * Fetch Storage to put object into.
+ *
+ */
+
+struct storage *
+FetchStorage(struct worker *w, ssize_t sz)
+{
+	ssize_t l;
+	struct storage *st;
+	struct object *obj;
+
+	obj = w->fetch_obj;
+	CHECK_OBJ_NOTNULL(obj, OBJECT_MAGIC);
+	st = VTAILQ_LAST(&obj->store, storagehead);
+	if (st != NULL && st->len < st->space)
+		return (st);
+
+	l = fetchfrag;
+	if (l == 0)
+		l = sz;
+	if (l == 0)
+		l = cache_param->fetch_chunksize * 1024LL;
+	st = STV_alloc(w, l);
+	if (st == NULL) {
+		(void)FetchError(w, "Could not get storage");
+		return (NULL);
+	}
+	AZ(st->len);
+	VTAILQ_INSERT_TAIL(&obj->store, st, list);
+	return (st);
+}
+
+/*--------------------------------------------------------------------
+ * Convert a string to a size_t safely
+ */
+
+static ssize_t
+fetch_number(const char *nbr, int radix)
+{
+	uintmax_t cll;
+	ssize_t cl;
+	char *q;
+
+	if (*nbr == '\0')
+		return (-1);
+	cll = strtoumax(nbr, &q, radix);
+	if (q == NULL || *q != '\0')
+		return (-1);
+
+	cl = (ssize_t)cll;
+	if((uintmax_t)cl != cll) /* Protect against bogusly large values */
+		return (-1);
+	return (cl);
+}
+
+/*--------------------------------------------------------------------*/
+
+static int
+fetch_straight(struct worker *w, struct http_conn *htc, ssize_t cl)
+{
+	int i;
+
+	assert(w->body_status == BS_LENGTH);
+
+	if (cl < 0) {
+		return (FetchError(w, "straight length field bogus"));
+	} else if (cl == 0)
+		return (0);
+
+	i = w->vfp->bytes(w, htc, cl);
+	if (i <= 0)
+		return (FetchError(w, "straight insufficient bytes"));
+	return (0);
+}
+
+/*--------------------------------------------------------------------
+ * Read a chunked HTTP object.
+ *
+ * XXX: Reading one byte at a time is pretty pessimal.
+ */
+
+static int
+fetch_chunked(struct worker *w, struct http_conn *htc)
+{
+	int i;
+	char buf[20];		/* XXX: 20 is arbitrary */
+	unsigned u;
+	ssize_t cl;
+
+	assert(w->body_status == BS_CHUNKED);
+	do {
+		/* Skip leading whitespace */
+		do {
+			if (HTC_Read(w, htc, buf, 1) <= 0)
+				return (-1);
+		} while (vct_islws(buf[0]));
+
+		if (!vct_ishex(buf[0]))
+			return (FetchError(w,"chunked header non-hex"));
+
+		/* Collect hex digits, skipping leading zeros */
+		for (u = 1; u < sizeof buf; u++) {
+			do {
+				if (HTC_Read(w, htc, buf + u, 1) <= 0)
+					return (-1);
+			} while (u == 1 && buf[0] == '0' && buf[u] == '0');
+			if (!vct_ishex(buf[u]))
+				break;
+		}
+
+		if (u >= sizeof buf)
+			return (FetchError(w,"chunked header too long"));
+
+		/* Skip trailing white space */
+		while(vct_islws(buf[u]) && buf[u] != '\n')
+			if (HTC_Read(w, htc, buf + u, 1) <= 0)
+				return (-1);
+
+		if (buf[u] != '\n')
+			return (FetchError(w,"chunked header no NL"));
+
+		buf[u] = '\0';
+		cl = fetch_number(buf, 16);
+		if (cl < 0)
+			return (FetchError(w,"chunked header number syntax"));
+
+		if (cl > 0 && w->vfp->bytes(w, htc, cl) <= 0)
+			return (-1);
+
+		i = HTC_Read(w, htc, buf, 1);
+		if (i <= 0)
+			return (-1);
+		if (buf[0] == '\r' && HTC_Read(w, htc, buf, 1) <= 0)
+			return (-1);
+		if (buf[0] != '\n')
+			return (FetchError(w,"chunked tail no NL"));
+	} while (cl > 0);
+	return (0);
+}
+
+/*--------------------------------------------------------------------*/
+
+static int
+fetch_eof(struct worker *w, struct http_conn *htc)
+{
+	int i;
+
+	assert(w->body_status == BS_EOF);
+	i = w->vfp->bytes(w, htc, SSIZE_MAX);
+	if (i < 0)
+		return (-1);
+	return (0);
+}
+
+/*--------------------------------------------------------------------
+ * Fetch any body attached to the incoming request, and either write it
+ * to the backend (if we pass) or discard it (anything else).
+ * This is mainly a separate function to isolate the stack buffer and
+ * to contain the complexity when we start handling chunked encoding.
+ */
+
+int
+FetchReqBody(struct sess *sp)
+{
+	unsigned long content_length;
+	char buf[8192];
+	char *ptr, *endp;
+	int rdcnt;
+
+	if (http_GetHdr(sp->http, H_Content_Length, &ptr)) {
+
+		content_length = strtoul(ptr, &endp, 10);
+		/* XXX should check result of conversion */
+		while (content_length) {
+			if (content_length > sizeof buf)
+				rdcnt = sizeof buf;
+			else
+				rdcnt = content_length;
+			rdcnt = HTC_Read(sp->wrk, sp->htc, buf, rdcnt);
+			if (rdcnt <= 0)
+				return (1);
+			content_length -= rdcnt;
+			if (!sp->sendbody)
+				continue;
+			(void)WRW_Write(sp->wrk, buf, rdcnt); /* XXX: stats ? */
+			if (WRW_Flush(sp->wrk))
+				return (2);
+		}
+	}
+	if (http_GetHdr(sp->http, H_Transfer_Encoding, NULL)) {
+		/* XXX: Handle chunked encoding. */
+		WSP(sp, SLT_Debug, "Transfer-Encoding in request");
+		return (1);
+	}
+	return (0);
+}
+
+/*--------------------------------------------------------------------
+ * Send request, and receive the HTTP protocol response, but not the
+ * response body.
+ *
+ * Return value:
+ *	-1 failure, not retryable
+ *	 0 success
+ *	 1 failure which can be retried.
+ */
+
+int
+FetchHdr(struct sess *sp)
+{
+	struct vbc *vc;
+	struct worker *w;
+	char *b;
+	struct http *hp;
+	int retry = -1;
+	int i;
+
+	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+	CHECK_OBJ_NOTNULL(sp->wrk, WORKER_MAGIC);
+	w = sp->wrk;
+
+	AN(sp->director);
+	AZ(sp->obj);
+
+	if (sp->objcore != NULL) {		/* pass has no objcore */
+		CHECK_OBJ_NOTNULL(sp->objcore, OBJCORE_MAGIC);
+		AN(sp->objcore->flags & OC_F_BUSY);
+	}
+
+	hp = w->bereq;
+
+	sp->wrk->vbc = VDI_GetFd(NULL, sp);
+	if (sp->wrk->vbc == NULL) {
+		WSP(sp, SLT_FetchError, "no backend connection");
+		return (-1);
+	}
+	vc = sp->wrk->vbc;
+	if (vc->recycled)
+		retry = 1;
+
+	/*
+	 * Now that we know our backend, we can set a default Host:
+	 * header if one is necessary.  This cannot be done in the VCL
+	 * because the backend may be chosen by a director.
+	 */
+	if (!http_GetHdr(hp, H_Host, &b))
+		VDI_AddHostHeader(sp);
+
+	(void)VTCP_blocking(vc->fd);	/* XXX: we should timeout instead */
+	WRW_Reserve(w, &vc->fd);
+	(void)http_Write(w, vc->vsl_id, hp, 0);	/* XXX: stats ? */
+
+	/* Deal with any message-body the request might have */
+	i = FetchReqBody(sp);
+	if (WRW_FlushRelease(w) || i > 0) {
+		WSP(sp, SLT_FetchError, "backend write error: %d (%s)",
+		    errno, strerror(errno));
+		VDI_CloseFd(sp->wrk);
+		/* XXX: other cleanup ? */
+		return (retry);
+	}
+
+	/* Checkpoint the vsl.here */
+	WSL_Flush(w, 0);
+
+	/* XXX is this the right place? */
+	VSC_C_main->backend_req++;
+
+	/* Receive response */
+
+	HTC_Init(w->htc, w->ws, vc->fd, vc->vsl_id, cache_param->http_resp_size,
+	    cache_param->http_resp_hdr_len);
+
+	VTCP_set_read_timeout(vc->fd, vc->first_byte_timeout);
+
+	i = HTC_Rx(w->htc);
+
+	if (i < 0) {
+		WSP(sp, SLT_FetchError, "http first read error: %d %d (%s)",
+		    i, errno, strerror(errno));
+		VDI_CloseFd(sp->wrk);
+		/* XXX: other cleanup ? */
+		/* Retryable if we never received anything */
+		return (i == -1 ? retry : -1);
+	}
+
+	VTCP_set_read_timeout(vc->fd, vc->between_bytes_timeout);
+
+	while (i == 0) {
+		i = HTC_Rx(w->htc);
+		if (i < 0) {
+			WSP(sp, SLT_FetchError,
+			    "http first read error: %d %d (%s)",
+			    i, errno, strerror(errno));
+			VDI_CloseFd(sp->wrk);
+			/* XXX: other cleanup ? */
+			return (-1);
+		}
+	}
+
+	hp = w->beresp;
+
+	if (http_DissectResponse(w, w->htc, hp)) {
+		WSP(sp, SLT_FetchError, "http format error");
+		VDI_CloseFd(sp->wrk);
+		/* XXX: other cleanup ? */
+		return (-1);
+	}
+	return (0);
+}
+
+/*--------------------------------------------------------------------*/
+
+int
+FetchBody(struct worker *w, struct object *obj)
+{
+	int cls;
+	struct storage *st;
+	int mklen;
+	ssize_t cl;
+
+	CHECK_OBJ_NOTNULL(w, WORKER_MAGIC);
+	AZ(w->fetch_obj);
+	CHECK_OBJ_NOTNULL(w->vbc, VBC_MAGIC);
+	CHECK_OBJ_NOTNULL(obj, OBJECT_MAGIC);
+	CHECK_OBJ_NOTNULL(obj->http, HTTP_MAGIC);
+
+	if (w->vfp == NULL)
+		w->vfp = &vfp_nop;
+
+	AssertObjCorePassOrBusy(obj->objcore);
+
+	AZ(w->vgz_rx);
+	AZ(VTAILQ_FIRST(&obj->store));
+
+	w->fetch_obj = obj;
+	w->fetch_failed = 0;
+
+	/* XXX: pick up estimate from objdr ? */
+	cl = 0;
+	switch (w->body_status) {
+	case BS_NONE:
+		cls = 0;
+		mklen = 0;
+		break;
+	case BS_ZERO:
+		cls = 0;
+		mklen = 1;
+		break;
+	case BS_LENGTH:
+		cl = fetch_number( w->h_content_length, 10);
+		w->vfp->begin(w, cl > 0 ? cl : 0);
+		cls = fetch_straight(w, w->htc, cl);
+		mklen = 1;
+		if (w->vfp->end(w))
+			cls = -1;
+		break;
+	case BS_CHUNKED:
+		w->vfp->begin(w, cl);
+		cls = fetch_chunked(w, w->htc);
+		mklen = 1;
+		if (w->vfp->end(w))
+			cls = -1;
+		break;
+	case BS_EOF:
+		w->vfp->begin(w, cl);
+		cls = fetch_eof(w, w->htc);
+		mklen = 1;
+		if (w->vfp->end(w))
+			cls = -1;
+		break;
+	case BS_ERROR:
+		cls = 1;
+		mklen = 0;
+		break;
+	default:
+		cls = 0;
+		mklen = 0;
+		INCOMPL();
+	}
+	AZ(w->vgz_rx);
+
+	/*
+	 * It is OK for ->end to just leave the last storage segment
+	 * sitting on w->storage, we will always call vfp_nop_end()
+	 * to get it trimmed or thrown out if empty.
+	 */
+	AZ(vfp_nop_end(w));
+
+	w->fetch_obj = NULL;
+
+	WSLB(w, SLT_Fetch_Body, "%u(%s) cls %d mklen %u",
+	    w->body_status, body_status(w->body_status),
+	    cls, mklen);
+
+	if (w->body_status == BS_ERROR) {
+		VDI_CloseFd(w);
+		return (__LINE__);
+	}
+
+	if (cls < 0) {
+		w->stats.fetch_failed++;
+		/* XXX: Wouldn't this store automatically be released ? */
+		while (!VTAILQ_EMPTY(&obj->store)) {
+			st = VTAILQ_FIRST(&obj->store);
+			VTAILQ_REMOVE(&obj->store, st, list);
+			STV_free(st);
+		}
+		VDI_CloseFd(w);
+		obj->len = 0;
+		return (__LINE__);
+	}
+	AZ(w->fetch_failed);
+
+	if (cls == 0 && w->do_close)
+		cls = 1;
+
+	WSLB(w, SLT_Length, "%u", obj->len);
+
+	{
+	/* Sanity check fetch methods accounting */
+		ssize_t uu;
+
+		uu = 0;
+		VTAILQ_FOREACH(st, &obj->store, list)
+			uu += st->len;
+		if (w->do_stream)
+			/* Streaming might have started freeing stuff */
+			assert (uu <= obj->len);
+
+		else
+			assert(uu == obj->len);
+	}
+
+	if (mklen > 0) {
+		http_Unset(obj->http, H_Content_Length);
+		http_PrintfHeader(w, w->vbc->vsl_id, obj->http,
+		    "Content-Length: %jd", (intmax_t)obj->len);
+	}
+
+	if (cls)
+		VDI_CloseFd(w);
+	else
+		VDI_RecycleFd(w);
+
+	return (0);
+}
+
+/*--------------------------------------------------------------------
+ * Debugging aids
+ */
+
+static void
+debug_fragfetch(struct cli *cli, const char * const *av, void *priv)
+{
+	(void)priv;
+	(void)cli;
+	fetchfrag = strtoul(av[2], NULL, 0);
+}
+
+static struct cli_proto debug_cmds[] = {
+	{ "debug.fragfetch", "debug.fragfetch",
+		"\tEnable fetch fragmentation\n", 1, 1, "d", debug_fragfetch },
+	{ NULL }
+};
+
+/*--------------------------------------------------------------------
+ *
+ */
+
+void
+Fetch_Init(void)
+{
+
+	CLI_AddFuncs(debug_cmds);
+}
diff --git a/bin/varnishd/cache/cache_gzip.c b/bin/varnishd/cache/cache_gzip.c
new file mode 100644
index 0000000..32a7413
--- /dev/null
+++ b/bin/varnishd/cache/cache_gzip.c
@@ -0,0 +1,697 @@
+/*-
+ * Copyright (c) 2011 Varnish Software AS
+ * All rights reserved.
+ *
+ * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * Interaction with the linvgz (zlib) library.
+ *
+ * The zlib library pollutes namespace a LOT when you include the "vgz.h"
+ * (aka (zlib.h") file so we contain the damage by vectoring all access
+ * to libz through this source file.
+ *
+ * The API defined by this file, will also insulate the rest of the code,
+ * should we find a better gzip library at a later date.
+ *
+ * The absolutely worst case gzip processing path, once we have pipe-lining,
+ * will be the following, so we need to be a bit careful with the scratch
+ * space we use:
+ *
+ *	Backend		Tmp	Input	Output
+ *         |		----------------------
+ *	   v
+ *	 gunzip		wrk	stack	?
+ *         |
+ *	   v
+ *	  esi
+ *         |
+ *	   v
+ *	  gzip		wrk	?	storage
+ *         |
+ *	   v
+ *	  cache
+ *         |
+ *	   v
+ *	 gunzip		wrk	storage	stack
+ *         |
+ *	   v
+ *	 client
+ *
+ * XXXX: The two '?' are obviously the same memory, but I have yet to decide
+ * where it goes.   As usual we try to avoid the session->ws if we can but
+ * I may have to use that.
+ *
+ */
+
+#include "config.h"
+
+#include <stdio.h>
+#include <stdlib.h>
+
+#include "cache.h"
+
+#include "vgz.h"
+
+struct vgz {
+	unsigned		magic;
+#define VGZ_MAGIC		0x162df0cb
+	enum {VGZ_GZ,VGZ_UN}	dir;
+	struct worker		*wrk;
+	const char		*id;
+	struct ws		*tmp;
+	char			*tmp_snapshot;
+	int			last_i;
+
+	struct storage		*obuf;
+
+	z_stream		vz;
+};
+
+/*--------------------------------------------------------------------*/
+
+static voidpf
+vgz_alloc(voidpf opaque, uInt items, uInt size)
+{
+	struct vgz *vg;
+
+	CAST_OBJ_NOTNULL(vg, opaque, VGZ_MAGIC);
+
+	return (WS_Alloc(vg->tmp, items * size));
+}
+
+static void
+vgz_free(voidpf opaque, voidpf address)
+{
+	struct vgz *vg;
+
+	CAST_OBJ_NOTNULL(vg, opaque, VGZ_MAGIC);
+	(void)address;
+}
+
+/*--------------------------------------------------------------------
+ * Set up a gunzip instance
+ */
+
+static struct vgz *
+vgz_alloc_vgz(struct worker *wrk, const char *id)
+{
+	struct vgz *vg;
+	struct ws *ws;
+
+	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
+	ws = wrk->ws;
+	WS_Assert(ws);
+	// XXX: we restore workspace in esi:include
+	// vg = (void*)WS_Alloc(ws, sizeof *vg);
+	ALLOC_OBJ(vg, VGZ_MAGIC);
+	AN(vg);
+	memset(vg, 0, sizeof *vg);
+	vg->magic = VGZ_MAGIC;
+	vg->wrk = wrk;
+	vg->id = id;
+
+	switch (cache_param->gzip_tmp_space) {
+	case 0:
+	case 1:
+		/* malloc, the default */
+		break;
+	case 2:
+		vg->tmp = wrk->ws;
+		vg->tmp_snapshot = WS_Snapshot(vg->tmp);
+		vg->vz.zalloc = vgz_alloc;
+		vg->vz.zfree = vgz_free;
+		vg->vz.opaque = vg;
+		break;
+	default:
+		assert(0 == __LINE__);
+	}
+	return (vg);
+}
+
+struct vgz *
+VGZ_NewUngzip(struct worker *wrk, const char *id)
+{
+	struct vgz *vg;
+
+	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
+	vg = vgz_alloc_vgz(wrk, id);
+	vg->dir = VGZ_UN;
+	VSC_C_main->n_gunzip++;
+
+	/*
+	 * Max memory usage according to zonf.h:
+	 *	mem_needed = "a few kb" + (1 << (windowBits))
+	 * Since we don't control windowBits, we have to assume
+	 * it is 15, so 34-35KB or so.
+	 */
+	assert(Z_OK == inflateInit2(&vg->vz, 31));
+	return (vg);
+}
+
+struct vgz *
+VGZ_NewGzip(struct worker *wrk, const char *id)
+{
+	struct vgz *vg;
+	int i;
+
+	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
+	vg = vgz_alloc_vgz(wrk, id);
+	vg->dir = VGZ_GZ;
+	VSC_C_main->n_gzip++;
+
+	/*
+	 * From zconf.h:
+	 *
+	 *	mem_needed = "a few kb"
+	 *		+ (1 << (windowBits+2))
+	 *		+  (1 << (memLevel+9))
+	 *
+	 * windowBits [8..15] (-> 1K..128K)
+	 * memLevel [1..9] (-> 1K->256K)
+	 *
+	 * XXX: They probably needs to be params...
+	 *
+	 * XXX: It may be more efficent to malloc them, rather than have
+	 * XXX: too many worker threads grow the stacks.
+	 */
+	i = deflateInit2(&vg->vz,
+	    cache_param->gzip_level,		/* Level */
+	    Z_DEFLATED,			/* Method */
+	    16 + cache_param->gzip_window,	/* Window bits (16=gzip + 15) */
+	    cache_param->gzip_memlevel,	/* memLevel */
+	    Z_DEFAULT_STRATEGY);
+	assert(Z_OK == i);
+	return (vg);
+}
+
+/*--------------------------------------------------------------------*/
+
+void
+VGZ_Ibuf(struct vgz *vg, const void *ptr, ssize_t len)
+{
+
+	CHECK_OBJ_NOTNULL(vg, VGZ_MAGIC);
+
+	AZ(vg->vz.avail_in);
+	vg->vz.next_in = TRUST_ME(ptr);
+	vg->vz.avail_in = len;
+}
+
+int
+VGZ_IbufEmpty(const struct vgz *vg)
+{
+
+	CHECK_OBJ_NOTNULL(vg, VGZ_MAGIC);
+	return (vg->vz.avail_in == 0);
+}
+
+/*--------------------------------------------------------------------*/
+
+void
+VGZ_Obuf(struct vgz *vg, void *ptr, ssize_t len)
+{
+
+	CHECK_OBJ_NOTNULL(vg, VGZ_MAGIC);
+
+	vg->vz.next_out = TRUST_ME(ptr);
+	vg->vz.avail_out = len;
+}
+
+int
+VGZ_ObufFull(const struct vgz *vg)
+{
+
+	CHECK_OBJ_NOTNULL(vg, VGZ_MAGIC);
+	return (vg->vz.avail_out == 0);
+}
+
+/*--------------------------------------------------------------------
+ * Keep the outbuffer supplied with storage and file it under the
+ * sp->obj as it fills.
+ */
+
+int
+VGZ_ObufStorage(struct worker *w, struct vgz *vg)
+{
+	struct storage *st;
+
+	st = FetchStorage(w, 0);
+	if (st == NULL)
+		return (-1);
+
+	vg->obuf = st;
+	VGZ_Obuf(vg, st->ptr + st->len, st->space - st->len);
+
+	return (0);
+}
+
+/*--------------------------------------------------------------------*/
+
+int
+VGZ_Gunzip(struct vgz *vg, const void **pptr, size_t *plen)
+{
+	int i;
+	ssize_t l;
+	const uint8_t *before;
+
+	CHECK_OBJ_NOTNULL(vg, VGZ_MAGIC);
+
+	*pptr = NULL;
+	*plen = 0;
+	AN(vg->vz.next_out);
+	AN(vg->vz.avail_out);
+	before = vg->vz.next_out;
+	i = inflate(&vg->vz, 0);
+	if (i == Z_OK || i == Z_STREAM_END) {
+		*pptr = before;
+		l = (const uint8_t *)vg->vz.next_out - before;
+		*plen = l;
+		if (vg->obuf != NULL)
+			vg->obuf->len += l;
+	}
+	vg->last_i = i;
+	if (i == Z_OK)
+		return (VGZ_OK);
+	if (i == Z_STREAM_END)
+		return (VGZ_END);
+	if (i == Z_BUF_ERROR)
+		return (VGZ_STUCK);
+	VSL(SLT_Debug, 0, "Unknown INFLATE=%d (%s)\n", i, vg->vz.msg);
+	return (VGZ_ERROR);
+}
+
+/*--------------------------------------------------------------------*/
+
+int
+VGZ_Gzip(struct vgz *vg, const void **pptr, size_t *plen, enum vgz_flag flags)
+{
+	int i;
+	int zflg;
+	ssize_t l;
+	const uint8_t *before;
+
+	CHECK_OBJ_NOTNULL(vg, VGZ_MAGIC);
+
+	*pptr = NULL;
+	*plen = 0;
+	AN(vg->vz.next_out);
+	AN(vg->vz.avail_out);
+	before = vg->vz.next_out;
+	switch(flags) {
+	case VGZ_NORMAL:	zflg = Z_NO_FLUSH; break;
+	case VGZ_ALIGN:		zflg = Z_SYNC_FLUSH; break;
+	case VGZ_RESET:		zflg = Z_FULL_FLUSH; break;
+	case VGZ_FINISH:	zflg = Z_FINISH; break;
+	default:		INCOMPL();
+	}
+	i = deflate(&vg->vz, zflg);
+	if (i == Z_OK || i == Z_STREAM_END) {
+		*pptr = before;
+		l = (const uint8_t *)vg->vz.next_out - before;
+		*plen = l;
+		if (vg->obuf != NULL)
+			vg->obuf->len += l;
+	}
+	vg->last_i = i;
+	if (i == Z_OK)
+		return (0);
+	if (i == Z_STREAM_END)
+		return (1);
+	if (i == Z_BUF_ERROR)
+		return (2);
+	return (-1);
+}
+
+/*--------------------------------------------------------------------
+ * Gunzip ibuf into outb, if it runs full, emit it with WRW.
+ * Leave flushing to caller, more data may be coming.
+ */
+
+int
+VGZ_WrwGunzip(struct worker *w, struct vgz *vg, const void *ibuf,
+    ssize_t ibufl, char *obuf, ssize_t obufl, ssize_t *obufp)
+{
+	int i;
+	size_t dl;
+	const void *dp;
+
+	CHECK_OBJ_NOTNULL(vg, VGZ_MAGIC);
+	assert(obufl > 16);
+	VGZ_Ibuf(vg, ibuf, ibufl);
+	if (ibufl == 0)
+		return (VGZ_OK);
+	VGZ_Obuf(vg, obuf + *obufp, obufl - *obufp);
+	do {
+		if (obufl == *obufp)
+			i = VGZ_STUCK;
+		else {
+			i = VGZ_Gunzip(vg, &dp, &dl);
+			*obufp += dl;
+		}
+		if (i < VGZ_OK) {
+			/* XXX: VSL ? */
+			return (-1);
+		}
+		if (obufl == *obufp || i == VGZ_STUCK) {
+			w->acct_tmp.bodybytes += *obufp;
+			(void)WRW_Write(w, obuf, *obufp);
+			(void)WRW_Flush(w);
+			*obufp = 0;
+			VGZ_Obuf(vg, obuf + *obufp, obufl - *obufp);
+		}
+	} while (!VGZ_IbufEmpty(vg));
+	if (i == VGZ_STUCK)
+		i = VGZ_OK;
+	return (i);
+}
+
+/*--------------------------------------------------------------------*/
+
+void
+VGZ_UpdateObj(const struct vgz *vg, struct object *obj)
+{
+
+	CHECK_OBJ_NOTNULL(vg, VGZ_MAGIC);
+	CHECK_OBJ_NOTNULL(obj, OBJECT_MAGIC);
+	obj->gzip_start	= vg->vz.start_bit;
+	obj->gzip_last	= vg->vz.last_bit;
+	obj->gzip_stop	= vg->vz.stop_bit;
+}
+
+/*--------------------------------------------------------------------
+ * Passing a vsl_id of -1 means "use w->vbc->vsl_id"
+ */
+
+int
+VGZ_Destroy(struct vgz **vgp, int vsl_id)
+{
+	struct vgz *vg;
+	int i;
+
+	vg = *vgp;
+	CHECK_OBJ_NOTNULL(vg, VGZ_MAGIC);
+	*vgp = NULL;
+
+	if (vsl_id < 0)
+		WSLB(vg->wrk, SLT_Gzip, "%s %jd %jd %jd %jd %jd",
+		    vg->id,
+		    (intmax_t)vg->vz.total_in,
+		    (intmax_t)vg->vz.total_out,
+		    (intmax_t)vg->vz.start_bit,
+		    (intmax_t)vg->vz.last_bit,
+		    (intmax_t)vg->vz.stop_bit);
+	else
+		WSL(vg->wrk, SLT_Gzip, vsl_id, "%s %jd %jd %jd %jd %jd",
+		    vg->id,
+		    (intmax_t)vg->vz.total_in,
+		    (intmax_t)vg->vz.total_out,
+		    (intmax_t)vg->vz.start_bit,
+		    (intmax_t)vg->vz.last_bit,
+		    (intmax_t)vg->vz.stop_bit);
+	if (vg->tmp != NULL)
+		WS_Reset(vg->tmp, vg->tmp_snapshot);
+	if (vg->dir == VGZ_GZ)
+		i = deflateEnd(&vg->vz);
+	else
+		i = inflateEnd(&vg->vz);
+	if (vg->last_i == Z_STREAM_END && i == Z_OK)
+		i = Z_STREAM_END;
+	FREE_OBJ(vg);
+	if (i == Z_OK)
+		return (VGZ_OK);
+	if (i == Z_STREAM_END)
+		return (VGZ_END);
+	if (i == Z_BUF_ERROR)
+		return (VGZ_STUCK);
+	return (VGZ_ERROR);
+}
+
+/*--------------------------------------------------------------------
+ * VFP_GUNZIP
+ *
+ * A VFP for gunzip'ing an object as we receive it from the backend
+ */
+
+static void __match_proto__()
+vfp_gunzip_begin(struct worker *w, size_t estimate)
+{
+	(void)estimate;
+	AZ(w->vgz_rx);
+	w->vgz_rx = VGZ_NewUngzip(w, "U F -");
+}
+
+static int __match_proto__()
+vfp_gunzip_bytes(struct worker *w, struct http_conn *htc, ssize_t bytes)
+{
+	struct vgz *vg;
+	ssize_t l, wl;
+	int i = -100;
+	uint8_t	ibuf[cache_param->gzip_stack_buffer];
+	size_t dl;
+	const void *dp;
+
+	AZ(w->fetch_failed);
+	vg = w->vgz_rx;
+	CHECK_OBJ_NOTNULL(vg, VGZ_MAGIC);
+	AZ(vg->vz.avail_in);
+	while (bytes > 0 || vg->vz.avail_in > 0) {
+		if (vg->vz.avail_in == 0 && bytes > 0) {
+			l = sizeof ibuf;
+			if (l > bytes)
+				l = bytes;
+			wl = HTC_Read(w, htc, ibuf, l);
+			if (wl <= 0)
+				return (wl);
+			VGZ_Ibuf(vg, ibuf, wl);
+			bytes -= wl;
+		}
+
+		if (VGZ_ObufStorage(w, vg))
+			return(-1);
+		i = VGZ_Gunzip(vg, &dp, &dl);
+		if (i != VGZ_OK && i != VGZ_END)
+			return(FetchError(w, "Gunzip data error"));
+		w->fetch_obj->len += dl;
+		if (w->do_stream)
+			RES_StreamPoll(w);
+	}
+	assert(i == Z_OK || i == Z_STREAM_END);
+	return (1);
+}
+
+static int __match_proto__()
+vfp_gunzip_end(struct worker *w)
+{
+	struct vgz *vg;
+
+	vg = w->vgz_rx;
+	w->vgz_rx = NULL;
+	CHECK_OBJ_NOTNULL(vg, VGZ_MAGIC);
+	if (w->fetch_failed) {
+		(void)VGZ_Destroy(&vg, -1);
+		return(0);
+	}
+	if (VGZ_Destroy(&vg, -1) != VGZ_END)
+		return(FetchError(w, "Gunzip error at the very end"));
+	return (0);
+}
+
+struct vfp vfp_gunzip = {
+        .begin  =       vfp_gunzip_begin,
+        .bytes  =       vfp_gunzip_bytes,
+        .end    =       vfp_gunzip_end,
+};
+
+
+/*--------------------------------------------------------------------
+ * VFP_GZIP
+ *
+ * A VFP for gzip'ing an object as we receive it from the backend
+ */
+
+static void __match_proto__()
+vfp_gzip_begin(struct worker *w, size_t estimate)
+{
+	(void)estimate;
+
+	AZ(w->vgz_rx);
+	w->vgz_rx = VGZ_NewGzip(w, "G F -");
+}
+
+static int __match_proto__()
+vfp_gzip_bytes(struct worker *w, struct http_conn *htc, ssize_t bytes)
+{
+	struct vgz *vg;
+	ssize_t l, wl;
+	int i = -100;
+	uint8_t ibuf[cache_param->gzip_stack_buffer];
+	size_t dl;
+	const void *dp;
+
+	AZ(w->fetch_failed);
+	vg = w->vgz_rx;
+	CHECK_OBJ_NOTNULL(vg, VGZ_MAGIC);
+	AZ(vg->vz.avail_in);
+	while (bytes > 0 || !VGZ_IbufEmpty(vg)) {
+		if (VGZ_IbufEmpty(vg) && bytes > 0) {
+			l = sizeof ibuf;
+			if (l > bytes)
+				l = bytes;
+			wl = HTC_Read(w, htc, ibuf, l);
+			if (wl <= 0)
+				return (wl);
+			VGZ_Ibuf(vg, ibuf, wl);
+			bytes -= wl;
+		}
+		if (VGZ_ObufStorage(w, vg))
+			return(-1);
+		i = VGZ_Gzip(vg, &dp, &dl, VGZ_NORMAL);
+		assert(i == Z_OK);
+		w->fetch_obj->len += dl;
+		if (w->do_stream)
+			RES_StreamPoll(w);
+	}
+	return (1);
+}
+
+static int __match_proto__()
+vfp_gzip_end(struct worker *w)
+{
+	struct vgz *vg;
+	size_t dl;
+	const void *dp;
+	int i;
+
+	vg = w->vgz_rx;
+	CHECK_OBJ_NOTNULL(vg, VGZ_MAGIC);
+	w->vgz_rx = NULL;
+	if (w->fetch_failed) {
+		(void)VGZ_Destroy(&vg, -1);
+		return(0);
+	}
+	do {
+		VGZ_Ibuf(vg, "", 0);
+		if (VGZ_ObufStorage(w, vg))
+			return(-1);
+		i = VGZ_Gzip(vg, &dp, &dl, VGZ_FINISH);
+		w->fetch_obj->len += dl;
+	} while (i != Z_STREAM_END);
+	if (w->do_stream)
+		RES_StreamPoll(w);
+	VGZ_UpdateObj(vg, w->fetch_obj);
+	if (VGZ_Destroy(&vg, -1) != VGZ_END)
+		return(FetchError(w, "Gzip error at the very end"));
+	return (0);
+}
+
+struct vfp vfp_gzip = {
+        .begin  =       vfp_gzip_begin,
+        .bytes  =       vfp_gzip_bytes,
+        .end    =       vfp_gzip_end,
+};
+
+/*--------------------------------------------------------------------
+ * VFP_TESTGZIP
+ *
+ * A VFP for testing that received gzip data is valid, and for
+ * collecting the magic bits while we're at it.
+ */
+
+static void __match_proto__()
+vfp_testgzip_begin(struct worker *w, size_t estimate)
+{
+	(void)estimate;
+	w->vgz_rx = VGZ_NewUngzip(w, "u F -");
+	CHECK_OBJ_NOTNULL(w->vgz_rx, VGZ_MAGIC);
+}
+
+static int __match_proto__()
+vfp_testgzip_bytes(struct worker *w, struct http_conn *htc, ssize_t bytes)
+{
+	struct vgz *vg;
+	ssize_t l, wl;
+	int i = -100;
+	uint8_t	obuf[cache_param->gzip_stack_buffer];
+	size_t dl;
+	const void *dp;
+	struct storage *st;
+
+	AZ(w->fetch_failed);
+	vg = w->vgz_rx;
+	CHECK_OBJ_NOTNULL(vg, VGZ_MAGIC);
+	AZ(vg->vz.avail_in);
+	while (bytes > 0) {
+		st = FetchStorage(w, 0);
+		if (st == NULL)
+			return(-1);
+		l = st->space - st->len;
+		if (l > bytes)
+			l = bytes;
+		wl = HTC_Read(w, htc, st->ptr + st->len, l);
+		if (wl <= 0)
+			return (wl);
+		bytes -= wl;
+		VGZ_Ibuf(vg, st->ptr + st->len, wl);
+		st->len += wl;
+		w->fetch_obj->len += wl;
+		if (w->do_stream)
+			RES_StreamPoll(w);
+
+		while (!VGZ_IbufEmpty(vg)) {
+			VGZ_Obuf(vg, obuf, sizeof obuf);
+			i = VGZ_Gunzip(vg, &dp, &dl);
+			if (i == VGZ_END && !VGZ_IbufEmpty(vg))
+				return(FetchError(w, "Junk after gzip data"));
+			if (i != VGZ_OK && i != VGZ_END)
+				return(FetchError2(w,
+				    "Invalid Gzip data", vg->vz.msg));
+		}
+	}
+	assert(i == VGZ_OK || i == VGZ_END);
+	return (1);
+}
+
+static int __match_proto__()
+vfp_testgzip_end(struct worker *w)
+{
+	struct vgz *vg;
+
+	vg = w->vgz_rx;
+	w->vgz_rx = NULL;
+	CHECK_OBJ_NOTNULL(vg, VGZ_MAGIC);
+	if (w->fetch_failed) {
+		(void)VGZ_Destroy(&vg, -1);
+		return(0);
+	}
+	VGZ_UpdateObj(vg, w->fetch_obj);
+	if (VGZ_Destroy(&vg, -1) != VGZ_END)
+		return(FetchError(w, "TestGunzip error at the very end"));
+	return (0);
+}
+
+struct vfp vfp_testgzip = {
+        .begin  =       vfp_testgzip_begin,
+        .bytes  =       vfp_testgzip_bytes,
+        .end    =       vfp_testgzip_end,
+};
diff --git a/bin/varnishd/cache/cache_hash.c b/bin/varnishd/cache/cache_hash.c
new file mode 100644
index 0000000..db865de
--- /dev/null
+++ b/bin/varnishd/cache/cache_hash.c
@@ -0,0 +1,752 @@
+/*-
+ * Copyright (c) 2006 Verdens Gang AS
+ * Copyright (c) 2006-2011 Varnish Software AS
+ * All rights reserved.
+ *
+ * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * This is the central hash-table code, it relies on a chosen hash
+ * implementation only for the actual hashing, all the housekeeping
+ * happens here.
+ *
+ * We have two kinds of structures, objecthead and object.  An objecthead
+ * corresponds to a given (Host:, URL) tupple, and the objects hung from
+ * the objecthead may represent various variations (ie: Vary: header,
+ * different TTL etc) instances of that web-entity.
+ *
+ * Each objecthead has a mutex which locks both its own fields, the
+ * list of objects and fields in the objects.
+ *
+ * The hash implementation must supply a reference count facility on
+ * the objecthead, and return with a reference held after a lookup.
+ *
+ * Lookups in the hash implementation returns with a ref held and each
+ * object hung from the objhead holds a ref as well.
+ *
+ * Objects have refcounts which are locked by the objecthead mutex.
+ *
+ * New objects are always marked busy, and they can go from busy to
+ * not busy only once.
+ */
+
+#include "config.h"
+
+#include <math.h>
+#include <stdio.h>
+#include <stdlib.h>
+
+#include "cache.h"
+
+#include "hash/hash_slinger.h"
+#include "vsha256.h"
+
+static const struct hash_slinger *hash;
+
+/*---------------------------------------------------------------------*/
+/* Precreate an objhead and object for later use */
+void
+HSH_Prealloc(const struct sess *sp)
+{
+	struct worker *w;
+	struct objhead *oh;
+	struct objcore *oc;
+	struct waitinglist *wl;
+
+	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+	CHECK_OBJ_NOTNULL(sp->wrk, WORKER_MAGIC);
+	w = sp->wrk;
+
+	if (w->nobjcore == NULL) {
+		ALLOC_OBJ(oc, OBJCORE_MAGIC);
+		XXXAN(oc);
+		w->nobjcore = oc;
+		w->stats.n_objectcore++;
+		oc->flags |= OC_F_BUSY;
+	}
+	CHECK_OBJ_NOTNULL(w->nobjcore, OBJCORE_MAGIC);
+
+	if (w->nobjhead == NULL) {
+		ALLOC_OBJ(oh, OBJHEAD_MAGIC);
+		XXXAN(oh);
+		oh->refcnt = 1;
+		VTAILQ_INIT(&oh->objcs);
+		Lck_New(&oh->mtx, lck_objhdr);
+		w->nobjhead = oh;
+		w->stats.n_objecthead++;
+	}
+	CHECK_OBJ_NOTNULL(w->nobjhead, OBJHEAD_MAGIC);
+
+	if (w->nwaitinglist == NULL) {
+		ALLOC_OBJ(wl, WAITINGLIST_MAGIC);
+		XXXAN(wl);
+		VTAILQ_INIT(&wl->list);
+		w->nwaitinglist = wl;
+		w->stats.n_waitinglist++;
+	}
+	CHECK_OBJ_NOTNULL(w->nwaitinglist, WAITINGLIST_MAGIC);
+
+	if (w->nbusyobj == NULL) {
+		ALLOC_OBJ(w->nbusyobj, BUSYOBJ_MAGIC);
+		XXXAN(w->nbusyobj);
+	}
+
+	if (hash->prep != NULL)
+		hash->prep(sp);
+}
+
+void
+HSH_Cleanup(struct worker *w)
+{
+
+	if (w->nobjcore != NULL) {
+		FREE_OBJ(w->nobjcore);
+		w->stats.n_objectcore--;
+		w->nobjcore = NULL;
+	}
+	if (w->nobjhead != NULL) {
+		Lck_Delete(&w->nobjhead->mtx);
+		FREE_OBJ(w->nobjhead);
+		w->nobjhead = NULL;
+		w->stats.n_objecthead--;
+	}
+	if (w->nwaitinglist != NULL) {
+		FREE_OBJ(w->nwaitinglist);
+		w->nwaitinglist = NULL;
+	}
+	if (w->nhashpriv != NULL) {
+		/* XXX: If needed, add slinger method for this */
+		free(w->nhashpriv);
+		w->nhashpriv = NULL;
+	}
+	if (w->nbusyobj != NULL) {
+		FREE_OBJ(w->nbusyobj);
+		w->nbusyobj = NULL;
+	}
+}
+
+void
+HSH_DeleteObjHead(struct worker *w, struct objhead *oh)
+{
+
+	AZ(oh->refcnt);
+	assert(VTAILQ_EMPTY(&oh->objcs));
+	Lck_Delete(&oh->mtx);
+	w->stats.n_objecthead--;
+	FREE_OBJ(oh);
+}
+
+void
+HSH_AddString(const struct sess *sp, const char *str)
+{
+	int l;
+
+	if (str == NULL)
+		str = "";
+	l = strlen(str);
+
+	SHA256_Update(sp->wrk->sha256ctx, str, l);
+	SHA256_Update(sp->wrk->sha256ctx, "#", 1);
+
+	if (cache_param->log_hash)
+		WSP(sp, SLT_Hash, "%s", str);
+}
+
+/*---------------------------------------------------------------------
+ * This is a debugging hack to enable testing of boundary conditions
+ * in the hash algorithm.
+ * We trap the first 9 different digests and translate them to different
+ * digests with edge bit conditions
+ */
+
+static struct hsh_magiclist {
+	unsigned char was[SHA256_LEN];
+	unsigned char now[SHA256_LEN];
+} hsh_magiclist[] = {
+	{ .now = {	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 } },
+	{ .now = {	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01 } },
+	{ .now = {	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02 } },
+	{ .now = {	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40 } },
+	{ .now = {	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80 } },
+	{ .now = {	0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 } },
+	{ .now = {	0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 } },
+	{ .now = {	0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 } },
+	{ .now = {	0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 } },
+};
+
+#define HSH_NMAGIC (sizeof hsh_magiclist / sizeof hsh_magiclist[0])
+
+static void
+hsh_testmagic(void *result)
+{
+	int i, j;
+	static int nused = 0;
+
+	for (i = 0; i < nused; i++)
+		if (!memcmp(hsh_magiclist[i].was, result, SHA256_LEN))
+			break;
+	if (i == nused && i < HSH_NMAGIC)
+		memcpy(hsh_magiclist[nused++].was, result, SHA256_LEN);
+	if (i == nused)
+		return;
+	assert(i < HSH_NMAGIC);
+	fprintf(stderr, "HASHMAGIC: <");
+	for (j = 0; j < SHA256_LEN; j++)
+		fprintf(stderr, "%02x", ((unsigned char*)result)[j]);
+	fprintf(stderr, "> -> <");
+	memcpy(result, hsh_magiclist[i].now, SHA256_LEN);
+	for (j = 0; j < SHA256_LEN; j++)
+		fprintf(stderr, "%02x", ((unsigned char*)result)[j]);
+	fprintf(stderr, ">\n");
+}
+
+/*---------------------------------------------------------------------
+ * Insert an object which magically appears out of nowhere or, more likely,
+ * comes off some persistent storage device.
+ * Return it with a reference held.
+ */
+
+struct objcore *
+HSH_Insert(const struct sess *sp)
+{
+	struct worker *w;
+	struct objhead *oh;
+	struct objcore *oc;
+
+	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+	CHECK_OBJ_NOTNULL(sp->wrk, WORKER_MAGIC);
+	AN(hash);
+	w = sp->wrk;
+
+	HSH_Prealloc(sp);
+	if (cache_param->diag_bitmap & 0x80000000)
+		hsh_testmagic(sp->wrk->nobjhead->digest);
+
+	AZ(sp->hash_objhead);
+	AN(w->nobjhead);
+	oh = hash->lookup(sp, w->nobjhead);
+	CHECK_OBJ_NOTNULL(oh, OBJHEAD_MAGIC);
+	if (oh == w->nobjhead)
+		w->nobjhead = NULL;
+	Lck_Lock(&oh->mtx);
+	assert(oh->refcnt > 0);
+
+	/* Insert (precreated) objcore in objecthead */
+	oc = w->nobjcore;
+	w->nobjcore = NULL;
+	oc->refcnt = 1;
+	CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
+	AZ(oc->flags & OC_F_BUSY);
+
+	VTAILQ_INSERT_HEAD(&oh->objcs, oc, list);
+	/* NB: do not deref objhead the new object inherits our reference */
+	oc->objhead = oh;
+	Lck_Unlock(&oh->mtx);
+	sp->wrk->stats.n_vampireobject++;
+	return (oc);
+}
+
+/*---------------------------------------------------------------------
+ */
+
+struct objcore *
+HSH_Lookup(struct sess *sp, struct objhead **poh)
+{
+	struct worker *w;
+	struct objhead *oh;
+	struct objcore *oc;
+	struct objcore *busy_oc, *grace_oc;
+	struct object *o;
+	double grace_ttl;
+
+	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+	CHECK_OBJ_NOTNULL(sp->wrk, WORKER_MAGIC);
+	CHECK_OBJ_NOTNULL(sp->http, HTTP_MAGIC);
+	AN(sp->director);
+	AN(hash);
+	w = sp->wrk;
+
+	HSH_Prealloc(sp);
+	memcpy(sp->wrk->nobjhead->digest, sp->digest, sizeof sp->digest);
+	if (cache_param->diag_bitmap & 0x80000000)
+		hsh_testmagic(sp->wrk->nobjhead->digest);
+
+	if (sp->hash_objhead != NULL) {
+		/*
+		 * This sess came off the waiting list, and brings a
+		 * oh refcnt with it.
+		 */
+		CHECK_OBJ_NOTNULL(sp->hash_objhead, OBJHEAD_MAGIC);
+		oh = sp->hash_objhead;
+		sp->hash_objhead = NULL;
+	} else {
+		AN(w->nobjhead);
+		oh = hash->lookup(sp, w->nobjhead);
+		if (oh == w->nobjhead)
+			w->nobjhead = NULL;
+	}
+
+	CHECK_OBJ_NOTNULL(oh, OBJHEAD_MAGIC);
+	Lck_Lock(&oh->mtx);
+	assert(oh->refcnt > 0);
+	busy_oc = NULL;
+	grace_oc = NULL;
+	grace_ttl = NAN;
+	VTAILQ_FOREACH(oc, &oh->objcs, list) {
+		/* Must be at least our own ref + the objcore we examine */
+		assert(oh->refcnt > 1);
+		CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
+		assert(oc->objhead == oh);
+
+		if (oc->flags & OC_F_BUSY) {
+			CHECK_OBJ_NOTNULL(oc->busyobj, BUSYOBJ_MAGIC);
+			if (sp->hash_ignore_busy)
+				continue;
+
+			if (oc->busyobj->vary != NULL &&
+			    !VRY_Match(sp, oc->busyobj->vary))
+				continue;
+
+			busy_oc = oc;
+			continue;
+		}
+
+		o = oc_getobj(sp->wrk, oc);
+		CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC);
+
+		if (o->exp.ttl <= 0.)
+			continue;
+		if (BAN_CheckObject(o, sp))
+			continue;
+		if (o->vary != NULL && !VRY_Match(sp, o->vary))
+			continue;
+
+		/* If still valid, use it */
+		if (EXP_Ttl(sp, o) >= sp->t_req)
+			break;
+
+		/*
+		 * Remember any matching objects inside their grace period
+		 * and if there are several, use the least expired one.
+		 */
+		if (EXP_Grace(sp, o) >= sp->t_req) {
+			if (grace_oc == NULL ||
+			    grace_ttl < o->exp.entered + o->exp.ttl) {
+				grace_oc = oc;
+				grace_ttl = o->exp.entered + o->exp.ttl;
+			}
+		}
+	}
+
+	/*
+	 * If we have seen a busy object or the backend is unhealthy, and
+	 * we have an object in grace, use it, if req.grace is also
+	 * satisified.
+	 * XXX: Interesting footnote:  The busy object might be for a
+	 * XXX: different "Vary:" than we sought.  We have no way of knowing
+	 * XXX: this until the object is unbusy'ed, so in practice we
+	 * XXX: serialize fetch of all Vary's if grace is possible.
+	 */
+
+	AZ(sp->objcore);
+	sp->objcore = grace_oc;		/* XXX: Hack-ish */
+	if (oc == NULL			/* We found no live object */
+	    && grace_oc != NULL		/* There is a grace candidate */
+	    && (busy_oc != NULL		/* Somebody else is already busy */
+	    || !VDI_Healthy(sp->director, sp))) {
+					/* Or it is impossible to fetch */
+		o = oc_getobj(sp->wrk, grace_oc);
+		CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC);
+		oc = grace_oc;
+	}
+	sp->objcore = NULL;
+
+	if (oc != NULL && !sp->hash_always_miss) {
+		o = oc_getobj(sp->wrk, oc);
+		CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC);
+		assert(oc->objhead == oh);
+
+		/* We found an object we like */
+		oc->refcnt++;
+		if (o->hits < INT_MAX)
+			o->hits++;
+		assert(oh->refcnt > 1);
+		Lck_Unlock(&oh->mtx);
+		assert(hash->deref(oh));
+		*poh = oh;
+		return (oc);
+	}
+
+	if (busy_oc != NULL) {
+		/* There are one or more busy objects, wait for them */
+		if (sp->esi_level == 0) {
+			CHECK_OBJ_NOTNULL(sp->wrk->nwaitinglist,
+			    WAITINGLIST_MAGIC);
+			if (oh->waitinglist == NULL) {
+				oh->waitinglist = sp->wrk->nwaitinglist;
+				sp->wrk->nwaitinglist = NULL;
+			}
+			VTAILQ_INSERT_TAIL(&oh->waitinglist->list, sp, list);
+		}
+		if (cache_param->diag_bitmap & 0x20)
+			WSP(sp, SLT_Debug,
+				"on waiting list <%p>", oh);
+		SES_Charge(sp);
+		/*
+		 * The objhead reference transfers to the sess, we get it
+		 * back when the sess comes off the waiting list and
+		 * calls us again
+		 */
+		sp->hash_objhead = oh;
+		sp->wrk = NULL;
+		Lck_Unlock(&oh->mtx);
+		return (NULL);
+	}
+
+	/* Insert (precreated) objcore in objecthead */
+	oc = w->nobjcore;
+	w->nobjcore = NULL;
+	AN(oc->flags & OC_F_BUSY);
+	oc->refcnt = 1;
+
+	/* XXX: clear w->nbusyobj before use */
+	VRY_Validate(sp->vary_b);
+	if (sp->vary_l != NULL)
+		w->nbusyobj->vary = sp->vary_b;
+	else
+		w->nbusyobj->vary = NULL;
+	oc->busyobj = w->nbusyobj;
+	w->nbusyobj = NULL;
+
+	/*
+	 * Busy objects go on the tail, so they will not trip up searches.
+	 * HSH_Unbusy() will move them to the front.
+	 */
+	VTAILQ_INSERT_TAIL(&oh->objcs, oc, list);
+	oc->objhead = oh;
+	/* NB: do not deref objhead the new object inherits our reference */
+	Lck_Unlock(&oh->mtx);
+	*poh = oh;
+	return (oc);
+}
+
+/*---------------------------------------------------------------------
+ */
+
+static void
+hsh_rush(struct objhead *oh)
+{
+	unsigned u;
+	struct sess *sp;
+	struct waitinglist *wl;
+
+	CHECK_OBJ_NOTNULL(oh, OBJHEAD_MAGIC);
+	Lck_AssertHeld(&oh->mtx);
+	wl = oh->waitinglist;
+	CHECK_OBJ_NOTNULL(wl, WAITINGLIST_MAGIC);
+	for (u = 0; u < cache_param->rush_exponent; u++) {
+		sp = VTAILQ_FIRST(&wl->list);
+		if (sp == NULL)
+			break;
+		CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+		AZ(sp->wrk);
+		VTAILQ_REMOVE(&wl->list, sp, list);
+		DSL(0x20, SLT_Debug, sp->vsl_id, "off waiting list");
+		if (SES_Schedule(sp)) {
+			/*
+			 * We could not schedule the session, leave the
+			 * rest on the busy list.
+			 */
+			break;
+		}
+	}
+	if (VTAILQ_EMPTY(&wl->list)) {
+		oh->waitinglist = NULL;
+		FREE_OBJ(wl);
+	}
+}
+
+/*---------------------------------------------------------------------
+ * Purge an entire objhead
+ */
+
+void
+HSH_Purge(const struct sess *sp, struct objhead *oh, double ttl, double grace)
+{
+	struct objcore *oc, **ocp;
+	unsigned spc, nobj, n;
+	struct object *o;
+
+	CHECK_OBJ_NOTNULL(oh, OBJHEAD_MAGIC);
+	spc = WS_Reserve(sp->wrk->ws, 0);
+	ocp = (void*)sp->wrk->ws->f;
+	Lck_Lock(&oh->mtx);
+	assert(oh->refcnt > 0);
+	nobj = 0;
+	VTAILQ_FOREACH(oc, &oh->objcs, list) {
+		CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
+		assert(oc->objhead == oh);
+		if (oc->flags & OC_F_BUSY) {
+			/*
+			 * We cannot purge busy objects here, because their
+			 * owners have special rights to them, and may nuke
+			 * them without concern for the refcount, which by
+			 * definition always must be one, so they don't check.
+			 */
+			continue;
+		}
+
+		(void)oc_getobj(sp->wrk, oc); /* XXX: still needed ? */
+
+		xxxassert(spc >= sizeof *ocp);
+		oc->refcnt++;
+		spc -= sizeof *ocp;
+		ocp[nobj++] = oc;
+	}
+	Lck_Unlock(&oh->mtx);
+
+	/* NB: inverse test to catch NAN also */
+	if (!(ttl > 0.))
+		ttl = -1.;
+	if (!(grace > 0.))
+		grace = -1.;
+	for (n = 0; n < nobj; n++) {
+		oc = ocp[n];
+		CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
+		o = oc_getobj(sp->wrk, oc);
+		if (o == NULL)
+			continue;
+		CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC);
+		o->exp.ttl = ttl;
+		o->exp.grace = grace;
+		EXP_Rearm(o);
+		(void)HSH_Deref(sp->wrk, NULL, &o);
+	}
+	WS_Release(sp->wrk->ws, 0);
+}
+
+
+/*---------------------------------------------------------------------
+ * Kill a busy object we don't need anyway.
+ * There may be sessions on the waiting list, so we cannot just blow
+ * it out of the water.
+ */
+
+void
+HSH_Drop(struct sess *sp)
+{
+	struct object *o;
+
+	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+	o = sp->obj;
+	CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC);
+	AssertObjCorePassOrBusy(o->objcore);
+	o->exp.ttl = -1.;
+	if (o->objcore != NULL)		/* Pass has no objcore */
+		HSH_Unbusy(sp);
+	(void)HSH_Deref(sp->wrk, NULL, &sp->obj);
+}
+
+void
+HSH_Unbusy(const struct sess *sp)
+{
+	struct object *o;
+	struct objhead *oh;
+	struct objcore *oc;
+
+	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+	o = sp->obj;
+	CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC);
+	oc = o->objcore;
+	CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
+	oh = oc->objhead;
+	CHECK_OBJ(oh, OBJHEAD_MAGIC);
+
+	AssertObjBusy(o);
+	AN(oc->ban);
+	assert(oc->refcnt > 0);
+	assert(oh->refcnt > 0);
+	if (o->ws_o->overflow)
+		sp->wrk->stats.n_objoverflow++;
+	if (cache_param->diag_bitmap & 0x40)
+		WSP(sp, SLT_Debug,
+		    "Object %u workspace free %u", o->xid, WS_Free(o->ws_o));
+
+	/* XXX: pretouch neighbors on oh->objcs to prevent page-on under mtx */
+	Lck_Lock(&oh->mtx);
+	assert(oh->refcnt > 0);
+	/* XXX: strictly speaking, we should sort in Date: order. */
+	VTAILQ_REMOVE(&oh->objcs, oc, list);
+	VTAILQ_INSERT_HEAD(&oh->objcs, oc, list);
+	oc->flags &= ~OC_F_BUSY;
+	AZ(sp->wrk->nbusyobj);
+	sp->wrk->nbusyobj = oc->busyobj;
+	oc->busyobj = NULL;
+	if (oh->waitinglist != NULL)
+		hsh_rush(oh);
+	AN(oc->ban);
+	Lck_Unlock(&oh->mtx);
+	assert(oc_getobj(sp->wrk, oc) == o);
+}
+
+void
+HSH_Ref(struct objcore *oc)
+{
+	struct objhead *oh;
+
+	CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
+	oh = oc->objhead;
+	CHECK_OBJ_NOTNULL(oh, OBJHEAD_MAGIC);
+	Lck_Lock(&oh->mtx);
+	assert(oc->refcnt > 0);
+	oc->refcnt++;
+	Lck_Unlock(&oh->mtx);
+}
+
+/*--------------------------------------------------------------------
+ * Dereference objcore and or object
+ *
+ * Can deal with:
+ *	bare objcore (incomplete fetch)
+ *	bare object (pass)
+ *	object with objcore
+ *	XXX later:  objcore with object (?)
+ *
+ * But you can only supply one of the two arguments at a time.
+ *
+ * Returns zero if target was destroyed.
+ */
+
+int
+HSH_Deref(struct worker *w, struct objcore *oc, struct object **oo)
+{
+	struct object *o = NULL;
+	struct objhead *oh;
+	unsigned r;
+
+	/* Only one arg at a time */
+	assert(oc == NULL || oo == NULL);
+
+	if (oo != NULL) {
+		o = *oo;
+		*oo = NULL;
+		CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC);
+		oc = o->objcore;
+	}
+
+	if (o != NULL && oc == NULL) {
+		/*
+		 * A pass object with neither objcore nor objhdr reference.
+		 * -> simply free the (Transient) storage
+		 */
+		STV_Freestore(o);
+		STV_free(o->objstore);
+		w->stats.n_object--;
+		return (0);
+	}
+
+	CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
+
+	oh = oc->objhead;
+	CHECK_OBJ_NOTNULL(oh, OBJHEAD_MAGIC);
+
+	Lck_Lock(&oh->mtx);
+	assert(oh->refcnt > 0);
+	assert(oc->refcnt > 0);
+	r = --oc->refcnt;
+	if (!r)
+		VTAILQ_REMOVE(&oh->objcs, oc, list);
+	else {
+		/* Must have an object */
+		AN(oc->methods);
+	}
+	if (oh->waitinglist != NULL)
+		hsh_rush(oh);
+	Lck_Unlock(&oh->mtx);
+	if (r != 0)
+		return (r);
+
+	BAN_DestroyObj(oc);
+	AZ(oc->ban);
+
+	if (oc->flags & OC_F_BUSY) {
+		CHECK_OBJ_NOTNULL(oc->busyobj, BUSYOBJ_MAGIC);
+		if (w->nbusyobj == NULL)
+			w->nbusyobj = oc->busyobj;
+		else
+			FREE_OBJ(oc->busyobj);
+		oc->busyobj = NULL;
+	}
+	AZ(oc->busyobj);
+
+	if (oc->methods != NULL) {
+		oc_freeobj(oc);
+		w->stats.n_object--;
+	}
+	FREE_OBJ(oc);
+
+	w->stats.n_objectcore--;
+	/* Drop our ref on the objhead */
+	assert(oh->refcnt > 0);
+	if (hash->deref(oh))
+		return (0);
+	HSH_DeleteObjHead(w, oh);
+	return (0);
+}
+
+void
+HSH_Init(const struct hash_slinger *slinger)
+{
+
+	assert(DIGEST_LEN == SHA256_LEN);	/* avoid #include pollution */
+	hash = slinger;
+	if (hash->start != NULL)
+		hash->start();
+}
diff --git a/bin/varnishd/cache/cache_http.c b/bin/varnishd/cache/cache_http.c
new file mode 100644
index 0000000..784eb28
--- /dev/null
+++ b/bin/varnishd/cache/cache_http.c
@@ -0,0 +1,1119 @@
+/*-
+ * Copyright (c) 2006 Verdens Gang AS
+ * Copyright (c) 2006-2011 Varnish Software AS
+ * All rights reserved.
+ *
+ * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * HTTP request storage and manipulation
+ */
+
+#include "config.h"
+
+#include <stdio.h>
+
+#include "cache.h"
+
+#include "vct.h"
+
+#define HTTPH(a, b, c, d, e, f, g) char b[] = "*" a ":";
+#include "tbl/http_headers.h"
+#undef HTTPH
+
+/*lint -save -e773 not () */
+#define LOGMTX2(ax, bx, cx)	[bx] = SLT_##ax##cx
+
+#define LOGMTX1(ax) {					\
+	LOGMTX2(ax, HTTP_HDR_REQ,	Request),	\
+	LOGMTX2(ax, HTTP_HDR_RESPONSE,	Response),	\
+	LOGMTX2(ax, HTTP_HDR_STATUS,	Status),	\
+	LOGMTX2(ax, HTTP_HDR_URL,	URL),		\
+	LOGMTX2(ax, HTTP_HDR_PROTO,	Protocol),	\
+	LOGMTX2(ax, HTTP_HDR_FIRST,	Header),	\
+	}
+
+static const enum VSL_tag_e logmtx[][HTTP_HDR_FIRST + 1] = {
+	[HTTP_Rx] = LOGMTX1(Rx),
+	[HTTP_Tx] = LOGMTX1(Tx),
+	[HTTP_Obj] = LOGMTX1(Obj)
+};
+/*lint -restore */
+
+static enum VSL_tag_e
+http2shmlog(const struct http *hp, int t)
+{
+
+	CHECK_OBJ_NOTNULL(hp, HTTP_MAGIC);
+	if (t > HTTP_HDR_FIRST)
+		t = HTTP_HDR_FIRST;
+	assert(hp->logtag >= HTTP_Rx && hp->logtag <= HTTP_Obj); /*lint !e685*/
+	assert(t >= HTTP_HDR_REQ && t <= HTTP_HDR_FIRST);
+	return (logmtx[hp->logtag][t]);
+}
+
+static void
+WSLH(struct worker *w, unsigned vsl_id, const struct http *hp, unsigned hdr)
+{
+
+	AN(vsl_id & (VSL_CLIENTMARKER|VSL_BACKENDMARKER));
+	WSLR(w, http2shmlog(hp, hdr), vsl_id, hp->hd[hdr]);
+}
+
+/*--------------------------------------------------------------------*/
+/* List of canonical HTTP response code names from RFC2616 */
+
+static struct http_msg {
+	unsigned	nbr;
+	const char	*txt;
+} http_msg[] = {
+#define HTTP_RESP(n, t)	{ n, t},
+#include "tbl/http_response.h"
+	{ 0, NULL }
+};
+
+const char *
+http_StatusMessage(unsigned status)
+{
+	struct http_msg *mp;
+
+	assert(status >= 100 && status <= 999);
+	for (mp = http_msg; mp->nbr != 0 && mp->nbr <= status; mp++)
+		if (mp->nbr == status)
+			return (mp->txt);
+	return ("Unknown Error");
+}
+
+/*--------------------------------------------------------------------*/
+
+unsigned
+HTTP_estimate(unsigned nhttp)
+{
+
+	/* XXX: We trust the structs to size-aligned as necessary */
+	return (sizeof (struct http) + (sizeof (txt) + 1) * nhttp);
+}
+
+struct http *
+HTTP_create(void *p, uint16_t nhttp)
+{
+	struct http *hp;
+
+	hp = p;
+	hp->magic = HTTP_MAGIC;
+	hp->hd = (void*)(hp + 1);
+	hp->shd = nhttp;
+	hp->hdf = (void*)(hp->hd + nhttp);
+	return (hp);
+}
+
+/*--------------------------------------------------------------------*/
+
+void
+http_Setup(struct http *hp, struct ws *ws)
+{
+	uint16_t shd;
+	txt *hd;
+	unsigned char *hdf;
+
+	/* XXX: This is not elegant, is it efficient ? */
+	shd = hp->shd;
+	hd = hp->hd;
+	hdf = hp->hdf;
+	memset(hp, 0, sizeof *hp);
+	memset(hd, 0, sizeof *hd * shd);
+	memset(hdf, 0, sizeof *hdf * shd);
+	hp->magic = HTTP_MAGIC;
+	hp->ws = ws;
+	hp->nhd = HTTP_HDR_FIRST;
+	hp->shd = shd;
+	hp->hd = hd;
+	hp->hdf = hdf;
+}
+
+/*--------------------------------------------------------------------*/
+
+static int
+http_IsHdr(const txt *hh, const char *hdr)
+{
+	unsigned l;
+
+	Tcheck(*hh);
+	AN(hdr);
+	l = hdr[0];
+	assert(l == strlen(hdr + 1));
+	assert(hdr[l] == ':');
+	hdr++;
+	return (!strncasecmp(hdr, hh->b, l));
+}
+
+/*--------------------------------------------------------------------
+ * This function collapses multiple headerlines of the same name.
+ * The lines are joined with a comma, according to [rfc2616, 4.2bot, p32]
+ */
+
+void
+http_CollectHdr(struct http *hp, const char *hdr)
+{
+	unsigned u, v, ml, f = 0, x;
+	char *b = NULL, *e = NULL;
+
+	for (u = HTTP_HDR_FIRST; u < hp->nhd; u++) {
+		while (u < hp->nhd && http_IsHdr(&hp->hd[u], hdr)) {
+			Tcheck(hp->hd[u]);
+			if (f == 0) {
+				/* Found first header, just record the fact */
+				f = u;
+				break;
+			}
+			if (b == NULL) {
+				/* Found second header, start our collection */
+				ml = WS_Reserve(hp->ws, 0);
+				b = hp->ws->f;
+				e = b + ml;
+				x = Tlen(hp->hd[f]);
+				if (b + x < e) {
+					memcpy(b, hp->hd[f].b, x);
+					b += x;
+				} else
+					b = e;
+			}
+
+			AN(b);
+			AN(e);
+
+			/* Append the Nth header we found */
+			if (b < e)
+				*b++ = ',';
+			x = Tlen(hp->hd[u]) - *hdr;
+			if (b + x < e) {
+				memcpy(b, hp->hd[u].b + *hdr, x);
+				b += x;
+			} else
+				b = e;
+
+			/* Shift remaining headers up one slot */
+			for (v = u; v < hp->nhd - 1; v++)
+				hp->hd[v] = hp->hd[v + 1];
+			hp->nhd--;
+		}
+
+	}
+	if (b == NULL)
+		return;
+	AN(e);
+	if (b >= e) {
+		WS_Release(hp->ws, 0);
+		return;
+	}
+	*b = '\0';
+	hp->hd[f].b = hp->ws->f;
+	hp->hd[f].e = b;
+	WS_ReleaseP(hp->ws, b + 1);
+}
+
+
+/*--------------------------------------------------------------------*/
+
+static unsigned
+http_findhdr(const struct http *hp, unsigned l, const char *hdr)
+{
+	unsigned u;
+
+	for (u = HTTP_HDR_FIRST; u < hp->nhd; u++) {
+		Tcheck(hp->hd[u]);
+		if (hp->hd[u].e < hp->hd[u].b + l + 1)
+			continue;
+		if (hp->hd[u].b[l] != ':')
+			continue;
+		if (strncasecmp(hdr, hp->hd[u].b, l))
+			continue;
+		return (u);
+	}
+	return (0);
+}
+
+int
+http_GetHdr(const struct http *hp, const char *hdr, char **ptr)
+{
+	unsigned u, l;
+	char *p;
+
+	l = hdr[0];
+	diagnostic(l == strlen(hdr + 1));
+	assert(hdr[l] == ':');
+	hdr++;
+	u = http_findhdr(hp, l - 1, hdr);
+	if (u == 0) {
+		if (ptr != NULL)
+			*ptr = NULL;
+		return (0);
+	}
+	if (ptr != NULL) {
+		p = hp->hd[u].b + l;
+		while (vct_issp(*p))
+			p++;
+		*ptr = p;
+	}
+	return (1);
+}
+
+
+/*--------------------------------------------------------------------
+ * Find a given data element in a header according to RFC2616's #rule
+ * (section 2.1, p15)
+ */
+
+int
+http_GetHdrData(const struct http *hp, const char *hdr,
+    const char *field, char **ptr)
+{
+	char *h, *e;
+	unsigned fl;
+
+	if (ptr != NULL)
+		*ptr = NULL;
+	if (!http_GetHdr(hp, hdr, &h))
+		return (0);
+	AN(h);
+	e = strchr(h, '\0');
+	fl = strlen(field);
+	while (h + fl <= e) {
+		/* Skip leading whitespace and commas */
+		if (vct_islws(*h) || *h == ',') {
+			h++;
+			continue;
+		}
+		/* Check for substrings before memcmp() */
+		if ((h + fl == e || vct_issepctl(h[fl])) &&
+		    !memcmp(h, field, fl)) {
+			if (ptr != NULL) {
+				h += fl;
+				while (vct_islws(*h))
+					h++;
+				*ptr = h;
+			}
+			return (1);
+		}
+		/* Skip until end of header or comma */
+		while (*h && *h != ',')
+			h++;
+	}
+	return (0);
+}
+
+/*--------------------------------------------------------------------
+ * Find a given headerfields Q value.
+ */
+
+double
+http_GetHdrQ(const struct http *hp, const char *hdr, const char *field)
+{
+	char *h;
+	int i;
+	double a, b;
+
+	h = NULL;
+	i = http_GetHdrData(hp, hdr, field, &h);
+	if (!i)
+		return (0.);
+
+	if (h == NULL)
+		return (1.);
+	/* Skip whitespace, looking for '=' */
+	while (*h && vct_issp(*h))
+		h++;
+	if (*h++ != ';')
+		return (1.);
+	while (*h && vct_issp(*h))
+		h++;
+	if (*h++ != 'q')
+		return (1.);
+	while (*h && vct_issp(*h))
+		h++;
+	if (*h++ != '=')
+		return (1.);
+	while (*h && vct_issp(*h))
+		h++;
+	a = 0.;
+	while (vct_isdigit(*h)) {
+		a *= 10.;
+		a += *h - '0';
+		h++;
+	}
+	if (*h++ != '.')
+		return (a);
+	b = .1;
+	while (vct_isdigit(*h)) {
+		a += b * (*h - '0');
+		b *= .1;
+		h++;
+	}
+	return (a);
+}
+
+/*--------------------------------------------------------------------
+ * Find a given headerfields value.
+ */
+
+int
+http_GetHdrField(const struct http *hp, const char *hdr,
+    const char *field, char **ptr)
+{
+	char *h;
+	int i;
+
+	if (ptr != NULL)
+		*ptr = NULL;
+
+	h = NULL;
+	i = http_GetHdrData(hp, hdr, field, &h);
+	if (!i)
+		return (i);
+
+	if (ptr != NULL && h != NULL) {
+		/* Skip whitespace, looking for '=' */
+		while (*h && vct_issp(*h))
+			h++;
+		if (*h == '=') {
+			h++;
+			while (*h && vct_issp(*h))
+				h++;
+			*ptr = h;
+		}
+	}
+	return (i);
+}
+
+/*--------------------------------------------------------------------
+ * XXX: redo with http_GetHdrField() ?
+ */
+
+const char *
+http_DoConnection(const struct http *hp)
+{
+	char *p, *q;
+	const char *ret;
+	unsigned u;
+
+	if (!http_GetHdr(hp, H_Connection, &p)) {
+		if (hp->protover < 11)
+			return ("not HTTP/1.1");
+		return (NULL);
+	}
+	ret = NULL;
+	AN(p);
+	for (; *p; p++) {
+		if (vct_issp(*p))
+			continue;
+		if (*p == ',')
+			continue;
+		for (q = p + 1; *q; q++)
+			if (*q == ',' || vct_issp(*q))
+				break;
+		u = pdiff(p, q);
+		if (u == 5 && !strncasecmp(p, "close", u))
+			ret = "Connection: close";
+		u = http_findhdr(hp, u, p);
+		if (u != 0)
+			hp->hdf[u] |= HDF_FILTER;
+		if (!*q)
+			break;
+		p = q;
+	}
+	return (ret);
+}
+
+/*--------------------------------------------------------------------*/
+
+int
+http_HdrIs(const struct http *hp, const char *hdr, const char *val)
+{
+	char *p;
+
+	if (!http_GetHdr(hp, hdr, &p))
+		return (0);
+	AN(p);
+	if (!strcasecmp(p, val))
+		return (1);
+	return (0);
+}
+
+/*--------------------------------------------------------------------*/
+
+uint16_t
+http_GetStatus(const struct http *hp)
+{
+
+	return (hp->status);
+}
+
+const char *
+http_GetReq(const struct http *hp)
+{
+
+	Tcheck(hp->hd[HTTP_HDR_REQ]);
+	return (hp->hd[HTTP_HDR_REQ].b);
+}
+
+/*--------------------------------------------------------------------
+ * Dissect the headers of the HTTP protocol message.
+ * Detect conditionals (headers which start with '^[Ii][Ff]-')
+ */
+
+static uint16_t
+http_dissect_hdrs(struct worker *w, struct http *hp, unsigned vsl_id, char *p,
+    const struct http_conn *htc)
+{
+	char *q, *r;
+	txt t = htc->rxbuf;
+
+	if (*p == '\r')
+		p++;
+
+	hp->nhd = HTTP_HDR_FIRST;
+	hp->conds = 0;
+	r = NULL;		/* For FlexeLint */
+	for (; p < t.e; p = r) {
+
+		/* Find end of next header */
+		q = r = p;
+		while (r < t.e) {
+			if (!vct_iscrlf(*r)) {
+				r++;
+				continue;
+			}
+			q = r;
+			assert(r < t.e);
+			r += vct_skipcrlf(r);
+			if (r >= t.e)
+				break;
+			/* If line does not continue: got it. */
+			if (!vct_issp(*r))
+				break;
+
+			/* Clear line continuation LWS to spaces */
+			while (vct_islws(*q))
+				*q++ = ' ';
+		}
+
+		if (q - p > htc->maxhdr) {
+			VSC_C_main->losthdr++;
+			WSL(w, SLT_LostHeader, vsl_id, "%.*s",
+			    q - p > 20 ? 20 : q - p, p);
+			return (413);
+		}
+
+		/* Empty header = end of headers */
+		if (p == q)
+			break;
+
+		if ((p[0] == 'i' || p[0] == 'I') &&
+		    (p[1] == 'f' || p[1] == 'F') &&
+		    p[2] == '-')
+			hp->conds = 1;
+
+		while (q > p && vct_issp(q[-1]))
+			q--;
+		*q = '\0';
+
+		if (hp->nhd < hp->shd) {
+			hp->hdf[hp->nhd] = 0;
+			hp->hd[hp->nhd].b = p;
+			hp->hd[hp->nhd].e = q;
+			WSLH(w, vsl_id, hp, hp->nhd);
+			hp->nhd++;
+		} else {
+			VSC_C_main->losthdr++;
+			WSL(w, SLT_LostHeader, vsl_id, "%.*s",
+			    q - p > 20 ? 20 : q - p, p);
+			return (413);
+		}
+	}
+	return (0);
+}
+
+/*--------------------------------------------------------------------
+ * Deal with first line of HTTP protocol message.
+ */
+
+static uint16_t
+http_splitline(struct worker *w, unsigned vsl_id, struct http *hp,
+    const struct http_conn *htc, int h1, int h2, int h3)
+{
+	char *p, *q;
+
+	CHECK_OBJ_NOTNULL(htc, HTTP_CONN_MAGIC);
+	CHECK_OBJ_NOTNULL(hp, HTTP_MAGIC);
+
+	/* XXX: Assert a NUL at rx.e ? */
+	Tcheck(htc->rxbuf);
+
+	/* Skip leading LWS */
+	for (p = htc->rxbuf.b ; vct_islws(*p); p++)
+		continue;
+
+	/* First field cannot contain SP, CRLF or CTL */
+	q = p;
+	for (; !vct_issp(*p); p++) {
+		if (vct_isctl(*p))
+			return (400);
+	}
+	hp->hd[h1].b = q;
+	hp->hd[h1].e = p;
+
+	/* Skip SP */
+	for (; vct_issp(*p); p++) {
+		if (vct_isctl(*p))
+			return (400);
+	}
+
+	/* Second field cannot contain LWS or CTL */
+	q = p;
+	for (; !vct_islws(*p); p++) {
+		if (vct_isctl(*p))
+			return (400);
+	}
+	hp->hd[h2].b = q;
+	hp->hd[h2].e = p;
+
+	if (!Tlen(hp->hd[h2]))
+		return (413);
+
+	/* Skip SP */
+	for (; vct_issp(*p); p++) {
+		if (vct_isctl(*p))
+			return (400);
+	}
+
+	/* Third field is optional and cannot contain CTL */
+	q = p;
+	if (!vct_iscrlf(*p)) {
+		for (; !vct_iscrlf(*p); p++)
+			if (!vct_issep(*p) && vct_isctl(*p))
+				return (400);
+	}
+	hp->hd[h3].b = q;
+	hp->hd[h3].e = p;
+
+	/* Skip CRLF */
+	p += vct_skipcrlf(p);
+
+	*hp->hd[h1].e = '\0';
+	WSLH(w, vsl_id, hp, h1);
+
+	*hp->hd[h2].e = '\0';
+	WSLH(w, vsl_id, hp, h2);
+
+	if (hp->hd[h3].e != NULL) {
+		*hp->hd[h3].e = '\0';
+		WSLH(w, vsl_id, hp, h3);
+	}
+
+	return (http_dissect_hdrs(w, hp, vsl_id, p, htc));
+}
+
+/*--------------------------------------------------------------------*/
+
+static void
+http_ProtoVer(struct http *hp)
+{
+
+	if (!strcasecmp(hp->hd[HTTP_HDR_PROTO].b, "HTTP/1.0"))
+		hp->protover = 10;
+	else if (!strcasecmp(hp->hd[HTTP_HDR_PROTO].b, "HTTP/1.1"))
+		hp->protover = 11;
+	else
+		hp->protover = 9;
+}
+
+
+/*--------------------------------------------------------------------*/
+
+uint16_t
+http_DissectRequest(struct sess *sp)
+{
+	struct http_conn *htc;
+	struct http *hp;
+	uint16_t retval;
+
+	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+	htc = sp->htc;
+	CHECK_OBJ_NOTNULL(htc, HTTP_CONN_MAGIC);
+	hp = sp->http;
+	CHECK_OBJ_NOTNULL(hp, HTTP_MAGIC);
+
+	hp->logtag = HTTP_Rx;
+
+	retval = http_splitline(sp->wrk, sp->vsl_id, hp, htc,
+	    HTTP_HDR_REQ, HTTP_HDR_URL, HTTP_HDR_PROTO);
+	if (retval != 0) {
+		WSPR(sp, SLT_HttpGarbage, htc->rxbuf);
+		return (retval);
+	}
+	http_ProtoVer(hp);
+	return (retval);
+}
+
+/*--------------------------------------------------------------------*/
+
+uint16_t
+http_DissectResponse(struct worker *w, const struct http_conn *htc,
+    struct http *hp)
+{
+	int j;
+	uint16_t retval = 0;
+	char *p;
+
+
+	CHECK_OBJ_NOTNULL(htc, HTTP_CONN_MAGIC);
+	CHECK_OBJ_NOTNULL(hp, HTTP_MAGIC);
+	hp->logtag = HTTP_Rx;
+
+	if (http_splitline(w, htc->vsl_id, hp, htc,
+	    HTTP_HDR_PROTO, HTTP_HDR_STATUS, HTTP_HDR_RESPONSE))
+		retval = 503;
+
+	if (retval == 0 && memcmp(hp->hd[HTTP_HDR_PROTO].b, "HTTP/1.", 7))
+		retval = 503;
+
+	if (retval == 0 && Tlen(hp->hd[HTTP_HDR_STATUS]) != 3)
+		retval = 503;
+
+	if (retval == 0) {
+		hp->status = 0;
+		p = hp->hd[HTTP_HDR_STATUS].b;
+		for (j = 100; j != 0; j /= 10) {
+			if (!vct_isdigit(*p)) {
+				retval = 503;
+				break;
+			}
+			hp->status += (uint16_t)(j * (*p - '0'));
+			p++;
+		}
+		if (*p != '\0')
+			retval = 503;
+	}
+
+	if (retval != 0) {
+		WSLR(w, SLT_HttpGarbage, htc->vsl_id, htc->rxbuf);
+		assert(retval >= 100 && retval <= 999);
+		hp->status = retval;
+	} else {
+		http_ProtoVer(hp);
+	}
+
+	if (hp->hd[HTTP_HDR_RESPONSE].b == NULL ||
+	    !Tlen(hp->hd[HTTP_HDR_RESPONSE])) {
+		/* Backend didn't send a response string, use the standard */
+		hp->hd[HTTP_HDR_RESPONSE].b =
+		    TRUST_ME(http_StatusMessage(hp->status));
+		hp->hd[HTTP_HDR_RESPONSE].e =
+		    strchr(hp->hd[HTTP_HDR_RESPONSE].b, '\0');
+	}
+	return (retval);
+}
+
+/*--------------------------------------------------------------------*/
+
+void
+http_SetH(const struct http *to, unsigned n, const char *fm)
+{
+
+	assert(n < to->shd);
+	AN(fm);
+	to->hd[n].b = TRUST_ME(fm);
+	to->hd[n].e = strchr(to->hd[n].b, '\0');
+	to->hdf[n] = 0;
+}
+
+static void
+http_copyh(const struct http *to, const struct http *fm, unsigned n)
+{
+
+	assert(n < HTTP_HDR_FIRST);
+	Tcheck(fm->hd[n]);
+	to->hd[n] = fm->hd[n];
+	to->hdf[n] = fm->hdf[n];
+}
+
+void
+http_ForceGet(const struct http *to)
+{
+	if (strcmp(http_GetReq(to), "GET"))
+		http_SetH(to, HTTP_HDR_REQ, "GET");
+}
+
+void
+http_CopyResp(struct http *to, const struct http *fm)
+{
+
+	CHECK_OBJ_NOTNULL(fm, HTTP_MAGIC);
+	CHECK_OBJ_NOTNULL(to, HTTP_MAGIC);
+	http_SetH(to, HTTP_HDR_PROTO, "HTTP/1.1");
+	to->status = fm->status;
+	http_copyh(to, fm, HTTP_HDR_RESPONSE);
+}
+
+void
+http_SetResp(struct http *to, const char *proto, uint16_t status,
+    const char *response)
+{
+
+	CHECK_OBJ_NOTNULL(to, HTTP_MAGIC);
+	http_SetH(to, HTTP_HDR_PROTO, proto);
+	assert(status >= 100 && status <= 999);
+	to->status = status;
+	http_SetH(to, HTTP_HDR_RESPONSE, response);
+}
+
+static void
+http_copyheader(struct worker *w, unsigned vsl_id, struct http *to,
+    const struct http *fm, unsigned n)
+{
+
+	CHECK_OBJ_NOTNULL(fm, HTTP_MAGIC);
+	CHECK_OBJ_NOTNULL(to, HTTP_MAGIC);
+	assert(n < fm->shd);
+	Tcheck(fm->hd[n]);
+	if (to->nhd < to->shd) {
+		to->hd[to->nhd] = fm->hd[n];
+		to->hdf[to->nhd] = 0;
+		to->nhd++;
+	} else  {
+		VSC_C_main->losthdr++;
+		WSLR(w, SLT_LostHeader, vsl_id, fm->hd[n]);
+	}
+}
+
+/*--------------------------------------------------------------------
+ * Estimate how much workspace we need to Filter this header according
+ * to 'how'.
+ */
+
+unsigned
+http_EstimateWS(const struct http *fm, unsigned how, uint16_t *nhd)
+{
+	unsigned u, l;
+
+	l = 0;
+	*nhd = HTTP_HDR_FIRST;
+	CHECK_OBJ_NOTNULL(fm, HTTP_MAGIC);
+	for (u = 0; u < fm->nhd; u++) {
+		if (fm->hd[u].b == NULL)
+			continue;
+		if (fm->hdf[u] & HDF_FILTER)
+			continue;
+#define HTTPH(a, b, c, d, e, f, g) \
+		if (((e) & how) && http_IsHdr(&fm->hd[u], (b))) \
+			continue;
+#include "tbl/http_headers.h"
+#undef HTTPH
+		l += PRNDUP(Tlen(fm->hd[u]) + 1);
+		(*nhd)++;
+		// fm->hdf[u] |= HDF_COPY;
+	}
+	return (l);
+}
+
+/*--------------------------------------------------------------------*/
+
+void
+http_FilterFields(struct worker *w, unsigned vsl_id, struct http *to,
+    const struct http *fm, unsigned how)
+{
+	unsigned u;
+
+	CHECK_OBJ_NOTNULL(fm, HTTP_MAGIC);
+	CHECK_OBJ_NOTNULL(to, HTTP_MAGIC);
+	to->nhd = HTTP_HDR_FIRST;
+	to->status = fm->status;
+	for (u = HTTP_HDR_FIRST; u < fm->nhd; u++) {
+		if (fm->hd[u].b == NULL)
+			continue;
+		if (fm->hdf[u] & HDF_FILTER)
+			continue;
+#define HTTPH(a, b, c, d, e, f, g) \
+		if (((e) & how) && http_IsHdr(&fm->hd[u], (b))) \
+			continue;
+#include "tbl/http_headers.h"
+#undef HTTPH
+		http_copyheader(w, vsl_id, to, fm, u);
+	}
+}
+
+/*--------------------------------------------------------------------*/
+
+void
+http_FilterHeader(const struct sess *sp, unsigned how)
+{
+	struct http *hp;
+
+	hp = sp->wrk->bereq;
+	CHECK_OBJ_NOTNULL(hp, HTTP_MAGIC);
+	hp->logtag = HTTP_Tx;
+
+	http_copyh(hp, sp->http, HTTP_HDR_REQ);
+	http_copyh(hp, sp->http, HTTP_HDR_URL);
+	if (how == HTTPH_R_FETCH)
+		http_SetH(hp, HTTP_HDR_PROTO, "HTTP/1.1");
+	else
+		http_copyh(hp, sp->http, HTTP_HDR_PROTO);
+	http_FilterFields(sp->wrk, sp->vsl_id, hp, sp->http, how);
+	http_PrintfHeader(sp->wrk, sp->vsl_id, hp, "X-Varnish: %u", sp->xid);
+}
+
+/*--------------------------------------------------------------------
+ * This function copies any header fields which reference foreign
+ * storage into our own WS.
+ */
+
+void
+http_CopyHome(struct worker *w, unsigned vsl_id, const struct http *hp)
+{
+	unsigned u, l;
+	char *p;
+
+	for (u = 0; u < hp->nhd; u++) {
+		if (hp->hd[u].b == NULL)
+			continue;
+		if (hp->hd[u].b >= hp->ws->s && hp->hd[u].e <= hp->ws->e) {
+			WSLH(w, vsl_id, hp, u);
+			continue;
+		}
+		l = Tlen(hp->hd[u]);
+		p = WS_Alloc(hp->ws, l + 1);
+		if (p != NULL) {
+			WSLH(w, vsl_id, hp, u);
+			memcpy(p, hp->hd[u].b, l + 1L);
+			hp->hd[u].b = p;
+			hp->hd[u].e = p + l;
+		} else {
+			/* XXX This leaves a slot empty */
+			VSC_C_main->losthdr++;
+			WSLR(w, SLT_LostHeader, vsl_id, hp->hd[u]);
+			hp->hd[u].b = NULL;
+			hp->hd[u].e = NULL;
+		}
+	}
+}
+
+/*--------------------------------------------------------------------*/
+
+void
+http_ClrHeader(struct http *to)
+{
+
+	CHECK_OBJ_NOTNULL(to, HTTP_MAGIC);
+	to->nhd = HTTP_HDR_FIRST;
+	to->status = 0;
+	to->protover = 0;
+	to->conds = 0;
+	memset(to->hd, 0, sizeof *to->hd * to->shd);
+}
+
+/*--------------------------------------------------------------------*/
+
+void
+http_SetHeader(struct worker *w, unsigned vsl_id, struct http *to,
+    const char *hdr)
+{
+
+	CHECK_OBJ_NOTNULL(to, HTTP_MAGIC);
+	if (to->nhd >= to->shd) {
+		VSC_C_main->losthdr++;
+		WSL(w, SLT_LostHeader, vsl_id, "%s", hdr);
+		return;
+	}
+	http_SetH(to, to->nhd++, hdr);
+}
+
+/*--------------------------------------------------------------------*/
+
+static void
+http_PutField(struct worker *w, unsigned vsl_id, const struct http *to,
+    int field, const char *string)
+{
+	char *p;
+	unsigned l;
+
+	CHECK_OBJ_NOTNULL(to, HTTP_MAGIC);
+	l = strlen(string);
+	p = WS_Alloc(to->ws, l + 1);
+	if (p == NULL) {
+		WSL(w, SLT_LostHeader, vsl_id, "%s", string);
+		to->hd[field].b = NULL;
+		to->hd[field].e = NULL;
+		to->hdf[field] = 0;
+	} else {
+		memcpy(p, string, l + 1L);
+		to->hd[field].b = p;
+		to->hd[field].e = p + l;
+		to->hdf[field] = 0;
+	}
+}
+
+void
+http_PutProtocol(struct worker *w, unsigned vsl_id, const struct http *to,
+    const char *protocol)
+{
+
+	http_PutField(w, vsl_id, to, HTTP_HDR_PROTO, protocol);
+	if (to->hd[HTTP_HDR_PROTO].b == NULL)
+		http_SetH(to, HTTP_HDR_PROTO, "HTTP/1.1");
+	Tcheck(to->hd[HTTP_HDR_PROTO]);
+}
+
+void
+http_PutStatus(struct http *to, uint16_t status)
+{
+
+	assert(status >= 100 && status <= 999);
+	to->status = status;
+}
+
+void
+http_PutResponse(struct worker *w, unsigned vsl_id, const struct http *to,
+    const char *response)
+{
+
+	http_PutField(w, vsl_id, to, HTTP_HDR_RESPONSE, response);
+	if (to->hd[HTTP_HDR_RESPONSE].b == NULL)
+		http_SetH(to, HTTP_HDR_RESPONSE, "Lost Response");
+	Tcheck(to->hd[HTTP_HDR_RESPONSE]);
+}
+
+void
+http_PrintfHeader(struct worker *w, unsigned vsl_id, struct http *to,
+    const char *fmt, ...)
+{
+	va_list ap;
+	unsigned l, n;
+
+	CHECK_OBJ_NOTNULL(to, HTTP_MAGIC);
+	l = WS_Reserve(to->ws, 0);
+	va_start(ap, fmt);
+	n = vsnprintf(to->ws->f, l, fmt, ap);
+	va_end(ap);
+	if (n + 1 >= l || to->nhd >= to->shd) {
+		VSC_C_main->losthdr++;
+		WSL(w, SLT_LostHeader, vsl_id, "%s", to->ws->f);
+		WS_Release(to->ws, 0);
+	} else {
+		to->hd[to->nhd].b = to->ws->f;
+		to->hd[to->nhd].e = to->ws->f + n;
+		to->hdf[to->nhd] = 0;
+		WS_Release(to->ws, n + 1);
+		to->nhd++;
+	}
+}
+/*--------------------------------------------------------------------*/
+
+void
+http_Unset(struct http *hp, const char *hdr)
+{
+	uint16_t u, v;
+
+	for (v = u = HTTP_HDR_FIRST; u < hp->nhd; u++) {
+		if (hp->hd[u].b == NULL)
+			continue;
+		if (http_IsHdr(&hp->hd[u], hdr))
+			continue;
+		if (v != u) {
+			memcpy(&hp->hd[v], &hp->hd[u], sizeof *hp->hd);
+			memcpy(&hp->hdf[v], &hp->hdf[u], sizeof *hp->hdf);
+		}
+		v++;
+	}
+	hp->nhd = v;
+}
+
+/*--------------------------------------------------------------------*/
+
+void
+HTTP_Copy(struct http *to, const struct http * const fm)
+{
+
+	to->conds = fm->conds;
+	to->logtag = fm->logtag;
+	to->status = fm->status;
+	to->protover = fm->protover;
+	to->nhd = fm->nhd;
+	assert(fm->nhd <= to->shd);
+	memcpy(to->hd, fm->hd, fm->nhd * sizeof *to->hd);
+	memcpy(to->hdf, fm->hdf, fm->nhd * sizeof *to->hdf);
+}
+
+/*--------------------------------------------------------------------*/
+
+unsigned
+http_Write(struct worker *w, unsigned vsl_id, const struct http *hp, int resp)
+{
+	unsigned u, l;
+
+	if (resp) {
+		l = WRW_WriteH(w, &hp->hd[HTTP_HDR_PROTO], " ");
+		WSLH(w, vsl_id, hp, HTTP_HDR_PROTO);
+
+		hp->hd[HTTP_HDR_STATUS].b = WS_Alloc(w->ws, 4);
+		AN(hp->hd[HTTP_HDR_STATUS].b);
+
+		sprintf(hp->hd[HTTP_HDR_STATUS].b, "%3d", hp->status);
+		hp->hd[HTTP_HDR_STATUS].e = hp->hd[HTTP_HDR_STATUS].b + 3;
+
+		l += WRW_WriteH(w, &hp->hd[HTTP_HDR_STATUS], " ");
+		WSLH(w, vsl_id, hp, HTTP_HDR_STATUS);
+
+		l += WRW_WriteH(w, &hp->hd[HTTP_HDR_RESPONSE], "\r\n");
+		WSLH(w, vsl_id, hp, HTTP_HDR_RESPONSE);
+	} else {
+		AN(hp->hd[HTTP_HDR_URL].b);
+		l = WRW_WriteH(w, &hp->hd[HTTP_HDR_REQ], " ");
+		WSLH(w, vsl_id, hp, HTTP_HDR_REQ);
+		l += WRW_WriteH(w, &hp->hd[HTTP_HDR_URL], " ");
+		WSLH(w, vsl_id, hp, HTTP_HDR_URL);
+		l += WRW_WriteH(w, &hp->hd[HTTP_HDR_PROTO], "\r\n");
+		WSLH(w, vsl_id, hp, HTTP_HDR_PROTO);
+	}
+	for (u = HTTP_HDR_FIRST; u < hp->nhd; u++) {
+		if (hp->hd[u].b == NULL)
+			continue;
+		AN(hp->hd[u].b);
+		AN(hp->hd[u].e);
+		l += WRW_WriteH(w, &hp->hd[u], "\r\n");
+		WSLH(w, vsl_id, hp, u);
+	}
+	l += WRW_Write(w, "\r\n", -1);
+	return (l);
+}
+
+/*--------------------------------------------------------------------*/
+
+void
+HTTP_Init(void)
+{
+
+#define HTTPH(a, b, c, d, e, f, g) b[0] = (char)strlen(b + 1);
+#include "tbl/http_headers.h"
+#undef HTTPH
+}
diff --git a/bin/varnishd/cache/cache_httpconn.c b/bin/varnishd/cache/cache_httpconn.c
new file mode 100644
index 0000000..9e0a052
--- /dev/null
+++ b/bin/varnishd/cache/cache_httpconn.c
@@ -0,0 +1,229 @@
+/*-
+ * Copyright (c) 2006 Verdens Gang AS
+ * Copyright (c) 2006-2011 Varnish Software AS
+ * All rights reserved.
+ *
+ * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * HTTP protocol requests
+ *
+ * The trouble with the "until magic sequence" design of HTTP protocol messages
+ * is that either you have to read a single character at a time, which is
+ * inefficient, or you risk reading too much, and pre-read some of the object,
+ * or even the next pipelined request, which follows the one you want.
+ *
+ * HTC reads a HTTP protocol header into a workspace, subject to limits,
+ * and stops when we see the magic marker (double [CR]NL), and if we overshoot,
+ * it keeps track of the "pipelined" data.
+ *
+ * We use this both for client and backend connections.
+ */
+
+#include "config.h"
+
+#include "cache.h"
+
+#include "vct.h"
+
+/*--------------------------------------------------------------------
+ * Check if we have a complete HTTP request or response yet
+ *
+ * Return values:
+ *	 0  No, keep trying
+ *	>0  Yes, it is this many bytes long.
+ */
+
+static int
+htc_header_complete(txt *t)
+{
+	const char *p;
+
+	Tcheck(*t);
+	assert(*t->e == '\0');
+	/* Skip any leading white space */
+	for (p = t->b ; vct_issp(*p); p++)
+		continue;
+	if (p == t->e) {
+		/* All white space */
+		t->e = t->b;
+		*t->e = '\0';
+		return (0);
+	}
+	while (1) {
+		p = strchr(p, '\n');
+		if (p == NULL)
+			return (0);
+		p++;
+		if (*p == '\r')
+			p++;
+		if (*p == '\n')
+			break;
+	}
+	p++;
+	return (p - t->b);
+}
+
+/*--------------------------------------------------------------------*/
+
+void
+HTC_Init(struct http_conn *htc, struct ws *ws, int fd, unsigned vsl_id,
+    unsigned maxbytes, unsigned maxhdr)
+{
+
+	htc->magic = HTTP_CONN_MAGIC;
+	htc->ws = ws;
+	htc->fd = fd;
+	htc->vsl_id = vsl_id;
+	htc->maxbytes = maxbytes;
+	htc->maxhdr = maxhdr;
+
+	(void)WS_Reserve(htc->ws, htc->maxbytes);
+	htc->rxbuf.b = ws->f;
+	htc->rxbuf.e = ws->f;
+	*htc->rxbuf.e = '\0';
+	htc->pipeline.b = NULL;
+	htc->pipeline.e = NULL;
+}
+
+/*--------------------------------------------------------------------
+ * Start over, and recycle any pipelined input.
+ * The WS_Reset is safe, even though the pipelined input is stored in
+ * the ws somewhere, because WS_Reset only fiddles pointers.
+ */
+
+int
+HTC_Reinit(struct http_conn *htc)
+{
+	unsigned l;
+
+	CHECK_OBJ_NOTNULL(htc, HTTP_CONN_MAGIC);
+	(void)WS_Reserve(htc->ws, htc->maxbytes);
+	htc->rxbuf.b = htc->ws->f;
+	htc->rxbuf.e = htc->ws->f;
+	if (htc->pipeline.b != NULL) {
+		l = Tlen(htc->pipeline);
+		memmove(htc->rxbuf.b, htc->pipeline.b, l);
+		htc->rxbuf.e += l;
+		htc->pipeline.b = NULL;
+		htc->pipeline.e = NULL;
+	}
+	*htc->rxbuf.e = '\0';
+	return (HTC_Complete(htc));
+}
+
+/*--------------------------------------------------------------------
+ * Return 1 if we have a complete HTTP procol header
+ */
+
+int
+HTC_Complete(struct http_conn *htc)
+{
+	int i;
+
+	CHECK_OBJ_NOTNULL(htc, HTTP_CONN_MAGIC);
+	i = htc_header_complete(&htc->rxbuf);
+	assert(i >= 0);
+	if (i == 0)
+		return (0);
+	WS_ReleaseP(htc->ws, htc->rxbuf.e);
+	AZ(htc->pipeline.b);
+	AZ(htc->pipeline.e);
+	if (htc->rxbuf.b + i < htc->rxbuf.e) {
+		htc->pipeline.b = htc->rxbuf.b + i;
+		htc->pipeline.e = htc->rxbuf.e;
+		htc->rxbuf.e = htc->pipeline.b;
+	}
+	return (1);
+}
+
+/*--------------------------------------------------------------------
+ * Receive more HTTP protocol bytes
+ * Returns:
+ *	-2 overflow
+ *	-1 error
+ *	 0 more needed
+ *	 1 got complete HTTP header
+ */
+
+int
+HTC_Rx(struct http_conn *htc)
+{
+	int i;
+
+	CHECK_OBJ_NOTNULL(htc, HTTP_CONN_MAGIC);
+	AN(htc->ws->r);
+	i = (htc->ws->r - htc->rxbuf.e) - 1;	/* space for NUL */
+	if (i <= 0) {
+		WS_ReleaseP(htc->ws, htc->rxbuf.b);
+		return (-2);
+	}
+	i = read(htc->fd, htc->rxbuf.e, i);
+	if (i <= 0) {
+		/*
+		 * We wouldn't come here if we had a complete HTTP header
+		 * so consequently an EOF can not be OK
+		 */
+		WS_ReleaseP(htc->ws, htc->rxbuf.b);
+		return (-1);
+	}
+	htc->rxbuf.e += i;
+	*htc->rxbuf.e = '\0';
+	return (HTC_Complete(htc));
+}
+
+/*--------------------------------------------------------------------
+ * Read up to len bytes, returning pipelined data first.
+ */
+
+ssize_t
+HTC_Read(struct worker *w, struct http_conn *htc, void *d, size_t len)
+{
+	size_t l;
+	unsigned char *p;
+	ssize_t i;
+
+	CHECK_OBJ_NOTNULL(w, WORKER_MAGIC);
+	CHECK_OBJ_NOTNULL(htc, HTTP_CONN_MAGIC);
+	l = 0;
+	p = d;
+	if (htc->pipeline.b) {
+		l = Tlen(htc->pipeline);
+		if (l > len)
+			l = len;
+		memcpy(p, htc->pipeline.b, l);
+		p += l;
+		len -= l;
+		htc->pipeline.b += l;
+		if (htc->pipeline.b == htc->pipeline.e)
+			htc->pipeline.b = htc->pipeline.e = NULL;
+	}
+	if (len == 0)
+		return (l);
+	i = read(htc->fd, p, len);
+	if (i < 0) {
+		WSL(w, SLT_FetchError, htc->vsl_id, "%s", strerror(errno));
+		return (i);
+	}
+	return (i + l);
+}
diff --git a/bin/varnishd/cache/cache_lck.c b/bin/varnishd/cache/cache_lck.c
new file mode 100644
index 0000000..2aef6dc
--- /dev/null
+++ b/bin/varnishd/cache/cache_lck.c
@@ -0,0 +1,210 @@
+/*-
+ * Copyright (c) 2008-2010 Varnish Software AS
+ * All rights reserved.
+ *
+ * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * The geniuses who came up with pthreads did not think operations like
+ * pthread_assert_mutex_held() were important enough to include them in
+ * the API.
+ *
+ * Build our own locks on top of pthread mutexes and hope that the next
+ * civilization is better at such crucial details than this one.
+ */
+
+#include "config.h"
+
+#include <stdlib.h>
+
+#include "cache.h"
+
+/*The constability of lck depends on platform pthreads implementation */
+
+struct ilck {
+	unsigned		magic;
+#define ILCK_MAGIC		0x7b86c8a5
+	pthread_mutex_t		mtx;
+	int			held;
+	pthread_t		owner;
+	VTAILQ_ENTRY(ilck)	list;
+	const char		*w;
+	struct VSC_C_lck	*stat;
+};
+
+static VTAILQ_HEAD(, ilck)	ilck_head =
+    VTAILQ_HEAD_INITIALIZER(ilck_head);
+
+static pthread_mutex_t		lck_mtx;
+
+void __match_proto__()
+Lck__Lock(struct lock *lck, const char *p, const char *f, int l)
+{
+	struct ilck *ilck;
+	int r;
+
+	CAST_OBJ_NOTNULL(ilck, lck->priv, ILCK_MAGIC);
+	if (!(cache_param->diag_bitmap & 0x18)) {
+		AZ(pthread_mutex_lock(&ilck->mtx));
+		AZ(ilck->held);
+		ilck->stat->locks++;
+		ilck->owner = pthread_self();
+		ilck->held = 1;
+		return;
+	}
+	r = pthread_mutex_trylock(&ilck->mtx);
+	assert(r == 0 || r == EBUSY);
+	if (r) {
+		ilck->stat->colls++;
+		if (cache_param->diag_bitmap & 0x8)
+			VSL(SLT_Debug, 0, "MTX_CONTEST(%s,%s,%d,%s)",
+			    p, f, l, ilck->w);
+		AZ(pthread_mutex_lock(&ilck->mtx));
+	} else if (cache_param->diag_bitmap & 0x8) {
+		VSL(SLT_Debug, 0, "MTX_LOCK(%s,%s,%d,%s)", p, f, l, ilck->w);
+	}
+	AZ(ilck->held);
+	ilck->stat->locks++;
+	ilck->owner = pthread_self();
+	ilck->held = 1;
+}
+
+void __match_proto__()
+Lck__Unlock(struct lock *lck, const char *p, const char *f, int l)
+{
+	struct ilck *ilck;
+
+	CAST_OBJ_NOTNULL(ilck, lck->priv, ILCK_MAGIC);
+	assert(pthread_equal(ilck->owner, pthread_self()));
+	AN(ilck->held);
+	ilck->held = 0;
+	AZ(pthread_mutex_unlock(&ilck->mtx));
+	if (cache_param->diag_bitmap & 0x8)
+		VSL(SLT_Debug, 0, "MTX_UNLOCK(%s,%s,%d,%s)", p, f, l, ilck->w);
+}
+
+int __match_proto__()
+Lck__Trylock(struct lock *lck, const char *p, const char *f, int l)
+{
+	struct ilck *ilck;
+	int r;
+
+	CAST_OBJ_NOTNULL(ilck, lck->priv, ILCK_MAGIC);
+	r = pthread_mutex_trylock(&ilck->mtx);
+	assert(r == 0 || r == EBUSY);
+	if (cache_param->diag_bitmap & 0x8)
+		VSL(SLT_Debug, 0,
+		    "MTX_TRYLOCK(%s,%s,%d,%s) = %d", p, f, l, ilck->w);
+	if (r == 0) {
+		AZ(ilck->held);
+		ilck->held = 1;
+		ilck->owner = pthread_self();
+	}
+	return (r);
+}
+
+void
+Lck__Assert(const struct lock *lck, int held)
+{
+	struct ilck *ilck;
+
+	CAST_OBJ_NOTNULL(ilck, lck->priv, ILCK_MAGIC);
+	if (held)
+		assert(ilck->held &&
+		    pthread_equal(ilck->owner, pthread_self()));
+	else
+		assert(!ilck->held ||
+		    !pthread_equal(ilck->owner, pthread_self()));
+}
+
+int __match_proto__()
+Lck_CondWait(pthread_cond_t *cond, struct lock *lck, struct timespec *ts)
+{
+	struct ilck *ilck;
+	int retval = 0;
+
+	CAST_OBJ_NOTNULL(ilck, lck->priv, ILCK_MAGIC);
+	AN(ilck->held);
+	assert(pthread_equal(ilck->owner, pthread_self()));
+	ilck->held = 0;
+	if (ts == NULL) {
+		AZ(pthread_cond_wait(cond, &ilck->mtx));
+	} else {
+		retval = pthread_cond_timedwait(cond, &ilck->mtx, ts);
+		assert(retval == 0 || retval == ETIMEDOUT);
+	}
+	AZ(ilck->held);
+	ilck->held = 1;
+	ilck->owner = pthread_self();
+	return (retval);
+}
+
+void
+Lck__New(struct lock *lck, struct VSC_C_lck *st, const char *w)
+{
+	struct ilck *ilck;
+
+	AN(st);
+	AZ(lck->priv);
+	ALLOC_OBJ(ilck, ILCK_MAGIC);
+	AN(ilck);
+	ilck->w = w;
+	ilck->stat = st;
+	ilck->stat->creat++;
+	AZ(pthread_mutex_init(&ilck->mtx, NULL));
+	AZ(pthread_mutex_lock(&lck_mtx));
+	VTAILQ_INSERT_TAIL(&ilck_head, ilck, list);
+	AZ(pthread_mutex_unlock(&lck_mtx));
+	lck->priv = ilck;
+}
+
+void
+Lck_Delete(struct lock *lck)
+{
+	struct ilck *ilck;
+
+	CAST_OBJ_NOTNULL(ilck, lck->priv, ILCK_MAGIC);
+	ilck->stat->destroy++;
+	lck->priv = NULL;
+	AZ(pthread_mutex_lock(&lck_mtx));
+	VTAILQ_REMOVE(&ilck_head, ilck, list);
+	AZ(pthread_mutex_unlock(&lck_mtx));
+	AZ(pthread_mutex_destroy(&ilck->mtx));
+	FREE_OBJ(ilck);
+}
+
+#define LOCK(nam) struct VSC_C_lck *lck_##nam;
+#include "tbl/locks.h"
+#undef LOCK
+
+void
+LCK_Init(void)
+{
+
+	AZ(pthread_mutex_init(&lck_mtx, NULL));
+#define LOCK(nam)						\
+	lck_##nam = VSM_Alloc(sizeof(struct VSC_C_lck),		\
+	   VSC_CLASS, VSC_TYPE_LCK, #nam);
+#include "tbl/locks.h"
+#undef LOCK
+}
diff --git a/bin/varnishd/cache/cache_main.c b/bin/varnishd/cache/cache_main.c
new file mode 100644
index 0000000..eb3fa1d
--- /dev/null
+++ b/bin/varnishd/cache/cache_main.c
@@ -0,0 +1,146 @@
+/*-
+ * Copyright (c) 2006 Verdens Gang AS
+ * Copyright (c) 2006-2009 Varnish Software AS
+ * All rights reserved.
+ *
+ * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include "config.h"
+
+#include <stdio.h>
+#include <stdlib.h>
+
+#include "cache.h"
+#include "common/heritage.h"
+
+#include "waiter/cache_waiter.h"
+#include "hash/hash_slinger.h"
+
+volatile struct params	*cache_param;
+
+/*--------------------------------------------------------------------
+ * Per thread storage for the session currently being processed by
+ * the thread.  This is used for panic messages.
+ */
+
+static pthread_key_t sp_key;
+
+void
+THR_SetSession(const struct sess *sp)
+{
+
+	AZ(pthread_setspecific(sp_key, sp));
+}
+
+const struct sess *
+THR_GetSession(void)
+{
+
+	return (pthread_getspecific(sp_key));
+}
+
+/*--------------------------------------------------------------------
+ * Name threads if our pthreads implementation supports it.
+ */
+
+static pthread_key_t name_key;
+
+void
+THR_SetName(const char *name)
+{
+
+	AZ(pthread_setspecific(name_key, name));
+#ifdef HAVE_PTHREAD_SET_NAME_NP
+	pthread_set_name_np(pthread_self(), name);
+#endif
+}
+
+const char *
+THR_GetName(void)
+{
+
+	return (pthread_getspecific(name_key));
+}
+
+/*--------------------------------------------------------------------
+ * XXX: Think more about which order we start things
+ */
+
+void
+child_main(void)
+{
+
+	setbuf(stdout, NULL);
+	setbuf(stderr, NULL);
+	printf("Child starts\n");
+
+	AZ(pthread_key_create(&sp_key, NULL));
+	AZ(pthread_key_create(&name_key, NULL));
+
+	THR_SetName("cache-main");
+
+	VSL_Init();	/* First, LCK needs it. */
+
+	LCK_Init();	/* Second, locking */
+
+	WAIT_Init();
+	PAN_Init();
+	CLI_Init();
+	Fetch_Init();
+
+	CNT_Init();
+	VCL_Init();
+
+	HTTP_Init();
+
+	VBE_Init();
+	VBP_Init();
+	WRK_Init();
+	Pool_Init();
+
+	EXP_Init();
+	HSH_Init(heritage.hash);
+	BAN_Init();
+
+	VCA_Init();
+
+	SMS_Init();
+	SMP_Init();
+	STV_open();
+
+	VMOD_Init();
+
+	BAN_Compile();
+
+	/* Wait for persistent storage to load if asked to */
+	if (cache_param->diag_bitmap & 0x00020000)
+		SMP_Ready();
+
+	CLI_Run();
+
+	STV_close();
+
+	printf("Child dies\n");
+}
diff --git a/bin/varnishd/cache/cache_panic.c b/bin/varnishd/cache/cache_panic.c
new file mode 100644
index 0000000..c626ae3
--- /dev/null
+++ b/bin/varnishd/cache/cache_panic.c
@@ -0,0 +1,387 @@
+/*-
+ * Copyright (c) 2006 Verdens Gang AS
+ * Copyright (c) 2006-2011 Varnish Software AS
+ * All rights reserved.
+ *
+ * Author: Dag-Erling Sm?rgrav <des at des.no>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include "config.h"
+
+#ifndef HAVE_EXECINFO_H
+#include "compat/execinfo.h"
+#else
+#include <execinfo.h>
+#endif
+
+#include <stdio.h>
+#include <stdlib.h>
+
+#include "cache.h"
+
+#include "vapi/vsm_int.h"
+
+#include "cache_backend.h"
+#include "waiter/cache_waiter.h"
+#include "vcl.h"
+
+/*
+ * The panic string is constructed in memory, then copied to the
+ * shared memory.
+ *
+ * It can be extracted post-mortem from a core dump using gdb:
+ *
+ * (gdb) printf "%s", panicstr
+ */
+
+static struct vsb vsps, *vsp;
+static pthread_mutex_t panicstr_mtx = PTHREAD_MUTEX_INITIALIZER;
+
+/*--------------------------------------------------------------------*/
+
+static void
+pan_ws(const struct ws *ws, int indent)
+{
+
+	VSB_printf(vsp, "%*sws = %p { %s\n", indent, "",
+	    ws, ws->overflow ? "overflow" : "");
+	VSB_printf(vsp, "%*sid = \"%s\",\n", indent + 2, "", ws->id);
+	VSB_printf(vsp, "%*s{s,f,r,e} = {%p", indent + 2, "", ws->s);
+	if (ws->f > ws->s)
+		VSB_printf(vsp, ",+%ld", (long) (ws->f - ws->s));
+	else
+		VSB_printf(vsp, ",%p", ws->f);
+	if (ws->r > ws->s)
+		VSB_printf(vsp, ",+%ld", (long) (ws->r - ws->s));
+	else
+		VSB_printf(vsp, ",%p", ws->r);
+	if (ws->e > ws->s)
+		VSB_printf(vsp, ",+%ld", (long) (ws->e - ws->s));
+	else
+		VSB_printf(vsp, ",%p", ws->e);
+	VSB_printf(vsp, "},\n");
+	VSB_printf(vsp, "%*s},\n", indent, "" );
+}
+
+/*--------------------------------------------------------------------*/
+
+static void
+pan_vbc(const struct vbc *vbc)
+{
+
+	struct backend *be;
+
+	be = vbc->backend;
+
+	VSB_printf(vsp, "  backend = %p fd = %d {\n", be, vbc->fd);
+	VSB_printf(vsp, "    display_name = \"%s\",\n", be->display_name);
+	VSB_printf(vsp, "  },\n");
+}
+
+/*--------------------------------------------------------------------*/
+
+static void
+pan_storage(const struct storage *st)
+{
+	int i, j;
+
+#define MAX_BYTES (4*16)
+#define show(ch) (((ch) > 31 && (ch) < 127) ? (ch) : '.')
+
+	VSB_printf(vsp, "      %u {\n", st->len);
+	for (i = 0; i < MAX_BYTES && i < st->len; i += 16) {
+		VSB_printf(vsp, "        ");
+		for (j = 0; j < 16; ++j) {
+			if (i + j < st->len)
+				VSB_printf(vsp, "%02x ", st->ptr[i + j]);
+			else
+				VSB_printf(vsp, "   ");
+		}
+		VSB_printf(vsp, "|");
+		for (j = 0; j < 16; ++j)
+			if (i + j < st->len)
+				VSB_printf(vsp, "%c", show(st->ptr[i + j]));
+		VSB_printf(vsp, "|\n");
+	}
+	if (st->len > MAX_BYTES)
+		VSB_printf(vsp, "        [%u more]\n", st->len - MAX_BYTES);
+	VSB_printf(vsp, "      },\n");
+
+#undef show
+#undef MAX_BYTES
+}
+
+/*--------------------------------------------------------------------*/
+
+static void
+pan_http(const char *id, const struct http *h, int indent)
+{
+	int i;
+
+	VSB_printf(vsp, "%*shttp[%s] = {\n", indent, "", id);
+	VSB_printf(vsp, "%*sws = %p[%s]\n", indent + 2, "",
+	    h->ws, h->ws ? h->ws->id : "");
+	for (i = 0; i < h->nhd; ++i) {
+		if (h->hd[i].b == NULL && h->hd[i].e == NULL)
+			continue;
+		VSB_printf(vsp, "%*s\"%.*s\",\n", indent + 4, "",
+		    (int)(h->hd[i].e - h->hd[i].b),
+		    h->hd[i].b);
+	}
+	VSB_printf(vsp, "%*s},\n", indent, "");
+}
+
+
+/*--------------------------------------------------------------------*/
+
+static void
+pan_object(const struct object *o)
+{
+	const struct storage *st;
+
+	VSB_printf(vsp, "  obj = %p {\n", o);
+	VSB_printf(vsp, "    xid = %u,\n", o->xid);
+	pan_ws(o->ws_o, 4);
+	pan_http("obj", o->http, 4);
+	VSB_printf(vsp, "    len = %jd,\n", (intmax_t)o->len);
+	VSB_printf(vsp, "    store = {\n");
+	VTAILQ_FOREACH(st, &o->store, list)
+		pan_storage(st);
+	VSB_printf(vsp, "    },\n");
+	VSB_printf(vsp, "  },\n");
+}
+
+/*--------------------------------------------------------------------*/
+
+static void
+pan_vcl(const struct VCL_conf *vcl)
+{
+	int i;
+
+	VSB_printf(vsp, "    vcl = {\n");
+	VSB_printf(vsp, "      srcname = {\n");
+	for (i = 0; i < vcl->nsrc; ++i)
+		VSB_printf(vsp, "        \"%s\",\n", vcl->srcname[i]);
+	VSB_printf(vsp, "      },\n");
+	VSB_printf(vsp, "    },\n");
+}
+
+
+/*--------------------------------------------------------------------*/
+
+static void
+pan_wrk(const struct worker *wrk)
+{
+
+	VSB_printf(vsp, "  worker = %p {\n", wrk);
+	pan_ws(wrk->ws, 4);
+	if (wrk->bereq->ws != NULL)
+		pan_http("bereq", wrk->bereq, 4);
+	if (wrk->beresp->ws != NULL)
+		pan_http("beresp", wrk->beresp, 4);
+	if (wrk->resp->ws != NULL)
+		pan_http("resp", wrk->resp, 4);
+	VSB_printf(vsp, "    },\n");
+}
+
+/*--------------------------------------------------------------------*/
+
+static void
+pan_sess(const struct sess *sp)
+{
+	const char *stp, *hand;
+
+	VSB_printf(vsp, "sp = %p {\n", sp);
+	VSB_printf(vsp,
+	    "  fd = %d, id = %d, xid = %u,\n",
+	    sp->fd, sp->vsl_id & VSL_IDENTMASK, sp->xid);
+	VSB_printf(vsp, "  client = %s %s,\n",
+	    sp->addr ? sp->addr : "?.?.?.?",
+	    sp->port ? sp->port : "?");
+	switch (sp->step) {
+#define STEP(l, u) case STP_##u: stp = "STP_" #u; break;
+#include "tbl/steps.h"
+#undef STEP
+		default: stp = NULL;
+	}
+	hand = VCL_Return_Name(sp->handling);
+	if (stp != NULL)
+		VSB_printf(vsp, "  step = %s,\n", stp);
+	else
+		VSB_printf(vsp, "  step = 0x%x,\n", sp->step);
+	if (hand != NULL)
+		VSB_printf(vsp, "  handling = %s,\n", hand);
+	else
+		VSB_printf(vsp, "  handling = 0x%x,\n", sp->handling);
+	if (sp->err_code)
+		VSB_printf(vsp,
+		    "  err_code = %d, err_reason = %s,\n", sp->err_code,
+		    sp->err_reason ? sp->err_reason : "(null)");
+
+	VSB_printf(vsp, "  restarts = %d, esi_level = %d\n",
+	    sp->restarts, sp->esi_level);
+
+	VSB_printf(vsp, "  flags = ");
+	if (sp->wrk->do_stream)	VSB_printf(vsp, " do_stream");
+	if (sp->wrk->do_gzip)	VSB_printf(vsp, " do_gzip");
+	if (sp->wrk->do_gunzip)	VSB_printf(vsp, " do_gunzip");
+	if (sp->wrk->do_esi)	VSB_printf(vsp, " do_esi");
+	if (sp->wrk->do_close)	VSB_printf(vsp, " do_close");
+	if (sp->wrk->is_gzip)	VSB_printf(vsp, " is_gzip");
+	if (sp->wrk->is_gunzip)	VSB_printf(vsp, " is_gunzip");
+	VSB_printf(vsp, "\n");
+	VSB_printf(vsp, "  bodystatus = %d\n", sp->wrk->body_status);
+
+	pan_ws(sp->ws, 2);
+	pan_http("req", sp->http, 2);
+
+	if (sp->wrk != NULL)
+		pan_wrk(sp->wrk);
+
+	if (VALID_OBJ(sp->vcl, VCL_CONF_MAGIC))
+		pan_vcl(sp->vcl);
+
+	if (VALID_OBJ(sp->wrk->vbc, BACKEND_MAGIC))
+		pan_vbc(sp->wrk->vbc);
+
+	if (VALID_OBJ(sp->obj, OBJECT_MAGIC))
+		pan_object(sp->obj);
+
+	VSB_printf(vsp, "},\n");
+}
+
+/*--------------------------------------------------------------------*/
+
+static void
+pan_backtrace(void)
+{
+	void *array[10];
+	size_t size;
+	size_t i;
+
+	size = backtrace (array, 10);
+	if (size == 0)
+		return;
+	VSB_printf(vsp, "Backtrace:\n");
+	for (i = 0; i < size; i++) {
+		VSB_printf (vsp, "  ");
+		if (Symbol_Lookup(vsp, array[i]) < 0) {
+			char **strings;
+			strings = backtrace_symbols(&array[i], 1);
+			if (strings != NULL && strings[0] != NULL)
+				VSB_printf(vsp, "%p: %s", array[i], strings[0]);
+			else
+				VSB_printf(vsp, "%p: (?)", array[i]);
+		}
+		VSB_printf (vsp, "\n");
+	}
+}
+
+/*--------------------------------------------------------------------*/
+
+static void
+pan_ic(const char *func, const char *file, int line, const char *cond,
+    int err, int xxx)
+{
+	const char *q;
+	const struct sess *sp;
+
+	AZ(pthread_mutex_lock(&panicstr_mtx)); /* Won't be released,
+						  we're going to die
+						  anyway */
+	switch(xxx) {
+	case 3:
+		VSB_printf(vsp,
+		    "Wrong turn at %s:%d:\n%s\n", file, line, cond);
+		break;
+	case 2:
+		VSB_printf(vsp,
+		    "Panic from VCL:\n  %s\n", cond);
+		break;
+	case 1:
+		VSB_printf(vsp,
+		    "Missing errorhandling code in %s(), %s line %d:\n"
+		    "  Condition(%s) not true.",
+		    func, file, line, cond);
+		break;
+	default:
+	case 0:
+		VSB_printf(vsp,
+		    "Assert error in %s(), %s line %d:\n"
+		    "  Condition(%s) not true.\n",
+		    func, file, line, cond);
+		break;
+	}
+	if (err)
+		VSB_printf(vsp, "errno = %d (%s)\n", err, strerror(err));
+
+	q = THR_GetName();
+	if (q != NULL)
+		VSB_printf(vsp, "thread = (%s)\n", q);
+
+	VSB_printf(vsp, "ident = %s,%s\n",
+	    VSB_data(vident) + 1, WAIT_GetName());
+
+	pan_backtrace();
+
+	if (!(cache_param->diag_bitmap & 0x2000)) {
+		sp = THR_GetSession();
+		if (sp != NULL)
+			pan_sess(sp);
+	}
+	VSB_printf(vsp, "\n");
+	VSB_bcat(vsp, "", 1);	/* NUL termination */
+
+	if (cache_param->diag_bitmap & 0x4000)
+		(void)fputs(VSM_head->panicstr, stderr);
+
+#ifdef HAVE_ABORT2
+	if (cache_param->diag_bitmap & 0x8000) {
+		void *arg[1];
+		char *p;
+
+		for (p = VSM_head->panicstr; *p; p++)
+			if (*p == '\n')
+				*p = ' ';
+		arg[0] = VSM_head->panicstr;
+		abort2(VSM_head->panicstr, 1, arg);
+	}
+#endif
+	if (cache_param->diag_bitmap & 0x1000)
+		exit(4);
+	else
+		abort();
+}
+
+/*--------------------------------------------------------------------*/
+
+void
+PAN_Init(void)
+{
+
+	VAS_Fail = pan_ic;
+	vsp = &vsps;
+	AN(VSB_new(vsp, VSM_head->panicstr, sizeof VSM_head->panicstr,
+	    VSB_FIXEDLEN));
+}
diff --git a/bin/varnishd/cache/cache_pipe.c b/bin/varnishd/cache/cache_pipe.c
new file mode 100644
index 0000000..4180d39
--- /dev/null
+++ b/bin/varnishd/cache/cache_pipe.c
@@ -0,0 +1,133 @@
+/*-
+ * Copyright (c) 2006 Verdens Gang AS
+ * Copyright (c) 2006-2011 Varnish Software AS
+ * All rights reserved.
+ *
+ * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * XXX: charge bytes to srcaddr
+ */
+
+#include "config.h"
+
+#include <poll.h>
+#include <stdio.h>
+
+#include "cache.h"
+
+#include "cache_backend.h"
+#include "vtcp.h"
+#include "vtim.h"
+
+static int
+rdf(int fd0, int fd1)
+{
+	int i, j;
+	char buf[BUFSIZ], *p;
+
+	i = read(fd0, buf, sizeof buf);
+	if (i <= 0)
+		return (1);
+	for (p = buf; i > 0; i -= j, p += j) {
+		j = write(fd1, p, i);
+		if (j <= 0)
+			return (1);
+		if (i != j)
+			(void)usleep(100000);		/* XXX hack */
+	}
+	return (0);
+}
+
+void
+PipeSession(struct sess *sp)
+{
+	struct vbc *vc;
+	struct worker *w;
+	struct pollfd fds[2];
+	int i;
+
+	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+	CHECK_OBJ_NOTNULL(sp->wrk, WORKER_MAGIC);
+	w = sp->wrk;
+
+	sp->wrk->vbc = VDI_GetFd(NULL, sp);
+	if (sp->wrk->vbc == NULL)
+		return;
+	vc = sp->wrk->vbc;
+	(void)VTCP_blocking(vc->fd);
+
+	WRW_Reserve(w, &vc->fd);
+	sp->wrk->acct_tmp.hdrbytes +=
+	    http_Write(w, sp->vsl_id, sp->wrk->bereq, 0);
+
+	if (sp->htc->pipeline.b != NULL)
+		sp->wrk->acct_tmp.bodybytes +=
+		    WRW_Write(w, sp->htc->pipeline.b, Tlen(sp->htc->pipeline));
+
+	i = WRW_FlushRelease(w);
+
+	if (i) {
+		SES_Close(sp, "pipe");
+		VDI_CloseFd(sp->wrk);
+		return;
+	}
+
+	sp->t_resp = VTIM_real();
+
+	memset(fds, 0, sizeof fds);
+
+	// XXX: not yet (void)VTCP_linger(vc->fd, 0);
+	fds[0].fd = vc->fd;
+	fds[0].events = POLLIN | POLLERR;
+
+	// XXX: not yet (void)VTCP_linger(sp->fd, 0);
+	fds[1].fd = sp->fd;
+	fds[1].events = POLLIN | POLLERR;
+
+	while (fds[0].fd > -1 || fds[1].fd > -1) {
+		fds[0].revents = 0;
+		fds[1].revents = 0;
+		i = poll(fds, 2, cache_param->pipe_timeout * 1000);
+		if (i < 1)
+			break;
+		if (fds[0].revents && rdf(vc->fd, sp->fd)) {
+			if (fds[1].fd == -1)
+				break;
+			(void)shutdown(vc->fd, SHUT_RD);
+			(void)shutdown(sp->fd, SHUT_WR);
+			fds[0].events = 0;
+			fds[0].fd = -1;
+		}
+		if (fds[1].revents && rdf(sp->fd, vc->fd)) {
+			if (fds[0].fd == -1)
+				break;
+			(void)shutdown(sp->fd, SHUT_RD);
+			(void)shutdown(vc->fd, SHUT_WR);
+			fds[1].events = 0;
+			fds[1].fd = -1;
+		}
+	}
+	SES_Close(sp, "pipe");
+	VDI_CloseFd(sp->wrk);
+}
diff --git a/bin/varnishd/cache/cache_pool.c b/bin/varnishd/cache/cache_pool.c
new file mode 100644
index 0000000..79a5fcd
--- /dev/null
+++ b/bin/varnishd/cache/cache_pool.c
@@ -0,0 +1,594 @@
+/*-
+ * Copyright (c) 2006 Verdens Gang AS
+ * Copyright (c) 2006-2011 Varnish Software AS
+ * All rights reserved.
+ *
+ * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * We maintain a number of worker thread pools, to spread lock contention.
+ *
+ * Pools can be added on the fly, as a means to mitigate lock contention,
+ * but can only be removed again by a restart. (XXX: we could fix that)
+ *
+ * Two threads herd the pools, one eliminates idle threads and aggregates
+ * statistics for all the pools, the other thread creates new threads
+ * on demand, subject to various numerical constraints.
+ *
+ * The algorithm for when to create threads needs to be reactive enough
+ * to handle startup spikes, but sufficiently attenuated to not cause
+ * thread pileups.  This remains subject for improvement.
+ */
+
+#include "config.h"
+
+#include <math.h>
+#include <stdlib.h>
+
+#include "cache.h"
+#include "common/heritage.h"
+
+#include "waiter/cache_waiter.h"
+#include "vtcp.h"
+#include "vtim.h"
+
+/*--------------------------------------------------------------------
+ * MAC OS/X is incredibly moronic when it comes to time and such...
+ */
+
+#ifndef CLOCK_MONOTONIC
+#define CLOCK_MONOTONIC	0
+
+#include <sys/time.h>
+
+static int
+clock_gettime(int foo, struct timespec *ts)
+{
+	struct timeval tv;
+
+	(void)foo;
+	gettimeofday(&tv, NULL);
+	ts->tv_sec = tv.tv_sec;
+	ts->tv_nsec = tv.tv_usec * 1000;
+	return (0);
+}
+
+static int
+pthread_condattr_setclock(pthread_condattr_t *attr, int foo)
+{
+	(void)attr;
+	(void)foo;
+	return (0);
+}
+#endif /* !CLOCK_MONOTONIC */
+
+static void *waiter_priv;
+
+VTAILQ_HEAD(workerhead, worker);
+
+struct poolsock {
+	unsigned			magic;
+#define POOLSOCK_MAGIC			0x1b0a2d38
+	VTAILQ_ENTRY(poolsock)		list;
+	struct listen_sock		*lsock;
+};
+
+/* Number of work requests queued in excess of worker threads available */
+
+struct pool {
+	unsigned		magic;
+#define POOL_MAGIC		0x606658fa
+	VTAILQ_ENTRY(pool)	list;
+
+	pthread_cond_t		herder_cond;
+	struct lock		herder_mtx;
+	pthread_t		herder_thr;
+
+	struct lock		mtx;
+	struct workerhead	idle;
+	VTAILQ_HEAD(, sess)	queue;
+	VTAILQ_HEAD(, poolsock)	socks;
+	unsigned		nthr;
+	unsigned		lqueue;
+	unsigned		last_lqueue;
+	uintmax_t		ndropped;
+	uintmax_t		nqueued;
+	struct sesspool		*sesspool;
+};
+
+static struct lock		pool_mtx;
+static pthread_t		thr_pool_herder;
+
+/*--------------------------------------------------------------------
+ * Nobody is accepting on this socket, so we do.
+ *
+ * As long as we can stick the accepted connection to another thread
+ * we do so, otherwise we return and handle it ourselves.
+ *
+ * Notice calling convention:  Called locked and returns locked, but
+ * works lock in the meantime.
+ *
+ * We store data about the accept in reserved workspace, it is only used
+ * for a brief moment and it takes up around 144 bytes.
+ */
+
+static int
+pool_accept(struct pool *pp, struct worker *w, const struct poolsock *ps)
+{
+	struct worker *w2;
+	struct wrk_accept *wa, *wa2;
+
+	CHECK_OBJ_NOTNULL(pp, POOL_MAGIC);
+	CHECK_OBJ_NOTNULL(w, WORKER_MAGIC);
+	CHECK_OBJ_NOTNULL(ps, POOLSOCK_MAGIC);
+
+	CHECK_OBJ_NOTNULL(ps->lsock, LISTEN_SOCK_MAGIC);
+	Lck_AssertHeld(&pp->mtx);
+	Lck_Unlock(&pp->mtx);
+	assert(sizeof *wa == WS_Reserve(w->ws, sizeof *wa));
+	wa = (void*)w->ws->f;
+	while (1) {
+		memset(wa, 0, sizeof *wa);
+		wa->magic = WRK_ACCEPT_MAGIC;
+
+		if (ps->lsock->sock < 0) {
+			/* Socket Shutdown */
+			Lck_Lock(&pp->mtx);
+			return (-1);
+		}
+		if (VCA_Accept(ps->lsock, wa) < 0) {
+			w->stats.sess_fail++;
+			/* We're going to pace in vca anyway... */
+			(void)WRK_TrySumStat(w);
+			continue;
+		}
+
+		Lck_Lock(&pp->mtx);
+		if (VTAILQ_EMPTY(&pp->idle))
+			return (0);
+		w2 = VTAILQ_FIRST(&pp->idle);
+		VTAILQ_REMOVE(&pp->idle, w2, list);
+		Lck_Unlock(&pp->mtx);
+		assert(sizeof *wa2 == WS_Reserve(w2->ws, sizeof *wa2));
+		wa2 = (void*)w2->ws->f;
+		memcpy(wa2, wa, sizeof *wa);
+		AZ(pthread_cond_signal(&w2->cond));
+	}
+}
+
+/*--------------------------------------------------------------------
+ * This is the work function for worker threads in the pool.
+ */
+
+void
+Pool_Work_Thread(void *priv, struct worker *w)
+{
+	struct pool *pp;
+	int stats_clean, i;
+	struct poolsock *ps;
+
+	CAST_OBJ_NOTNULL(pp, priv, POOL_MAGIC);
+	w->pool = pp;
+	Lck_Lock(&pp->mtx);
+	stats_clean = 1;
+	while (1) {
+
+		Lck_AssertHeld(&pp->mtx);
+
+		CHECK_OBJ_NOTNULL(w, WORKER_MAGIC);
+		CHECK_OBJ_NOTNULL(w->bereq, HTTP_MAGIC);
+		CHECK_OBJ_NOTNULL(w->beresp, HTTP_MAGIC);
+		CHECK_OBJ_NOTNULL(w->resp, HTTP_MAGIC);
+
+		WS_Reset(w->ws, NULL);
+
+		w->sp = VTAILQ_FIRST(&pp->queue);
+		if (w->sp != NULL) {
+			/* Process queued requests, if any */
+			assert(pp->lqueue > 0);
+			VTAILQ_REMOVE(&pp->queue, w->sp, poollist);
+			pp->lqueue--;
+		} else if (!VTAILQ_EMPTY(&pp->socks)) {
+			/* Accept on a socket */
+			ps = VTAILQ_FIRST(&pp->socks);
+			VTAILQ_REMOVE(&pp->socks, ps, list);
+			i = pool_accept(pp, w, ps);
+			Lck_AssertHeld(&pp->mtx);
+			if (i < 0) {
+				/* Socket Shutdown */
+				FREE_OBJ(ps);
+				WS_Release(w->ws, 0);
+				continue;
+			}
+			VTAILQ_INSERT_TAIL(&pp->socks, ps, list);
+		} else if (VTAILQ_EMPTY(&pp->socks)) {
+			/* Nothing to do: To sleep, perchance to dream ... */
+			if (isnan(w->lastused))
+				w->lastused = VTIM_real();
+			VTAILQ_INSERT_HEAD(&pp->idle, w, list);
+			if (!stats_clean)
+				WRK_SumStat(w);
+			(void)Lck_CondWait(&w->cond, &pp->mtx, NULL);
+		}
+
+		/*
+		 * If we got neither session or accepted a socket, we were
+		 * woken up to die to cull the herd.
+		 */
+		if (w->sp == NULL && w->ws->r == NULL)
+			break;
+
+		Lck_Unlock(&pp->mtx);
+
+		if (w->sp == NULL) {
+			/* Turn accepted socket into a session */
+			assert(w->ws->r != NULL);
+			w->sp = SES_New(w, pp->sesspool);
+			if (w->sp == NULL)
+				VCA_FailSess(w);
+			else
+				VCA_SetupSess(w);
+			WS_Release(w->ws, 0);
+		}
+		assert(w->ws->r == NULL);
+
+		if (w->sp != NULL) {
+			CHECK_OBJ_NOTNULL(w->sp, SESS_MAGIC);
+
+			stats_clean = 0;
+			w->lastused = NAN;
+			w->storage_hint = NULL;
+
+			AZ(w->sp->wrk);
+			THR_SetSession(w->sp);
+			w->sp->wrk = w;
+			CNT_Session(w->sp);
+			THR_SetSession(NULL);
+			w->sp = NULL;
+
+			WS_Assert(w->ws);
+			AZ(w->bereq->ws);
+			AZ(w->beresp->ws);
+			AZ(w->resp->ws);
+			AZ(w->wrw.wfd);
+			AZ(w->storage_hint);
+			assert(w->wlp == w->wlb);
+			if (cache_param->diag_bitmap & 0x00040000) {
+				if (w->vcl != NULL)
+					VCL_Rel(&w->vcl);
+			}
+		}
+		stats_clean = WRK_TrySumStat(w);
+		Lck_Lock(&pp->mtx);
+	}
+	Lck_Unlock(&pp->mtx);
+	w->pool = NULL;
+}
+
+/*--------------------------------------------------------------------
+ * Queue a workrequest if possible.
+ *
+ * Return zero if the request was queued, negative if it wasn't.
+ */
+
+static int
+pool_queue(struct pool *pp, struct sess *sp)
+{
+	struct worker *w;
+
+	Lck_Lock(&pp->mtx);
+
+	/* If there are idle threads, we tickle the first one into action */
+	w = VTAILQ_FIRST(&pp->idle);
+	if (w != NULL) {
+		VTAILQ_REMOVE(&pp->idle, w, list);
+		Lck_Unlock(&pp->mtx);
+		w->sp = sp;
+		AZ(pthread_cond_signal(&w->cond));
+		return (0);
+	}
+
+	/* If we have too much in the queue already, refuse. */
+	if (pp->lqueue > (cache_param->queue_max * pp->nthr) / 100) {
+		pp->ndropped++;
+		Lck_Unlock(&pp->mtx);
+		return (-1);
+	}
+
+	VTAILQ_INSERT_TAIL(&pp->queue, sp, poollist);
+	pp->nqueued++;
+	pp->lqueue++;
+	Lck_Unlock(&pp->mtx);
+	AZ(pthread_cond_signal(&pp->herder_cond));
+	return (0);
+}
+
+/*--------------------------------------------------------------------*/
+
+int
+Pool_Schedule(struct pool *pp, struct sess *sp)
+{
+	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+	AZ(sp->wrk);
+	if (pool_queue(pp, sp) == 0)
+		return(0);
+
+	VSC_C_main->client_drop_late++;
+
+	/*
+	 * Couldn't queue it -- kill it.
+	 *
+	 * XXX: a notice might be polite, but would potentially
+	 * XXX: sleep whichever thread got us here
+	 */
+	sp->t_end = VTIM_real();
+	if (sp->vcl != NULL) {
+		/*
+		 * A session parked on a busy object can come here
+		 * after it wakes up.  Loose the VCL reference.
+		 */
+		VCL_Rel(&sp->vcl);
+	}
+	return (1);
+}
+
+/*--------------------------------------------------------------------
+ * Wait for another request
+ */
+
+void
+Pool_Wait(struct sess *sp)
+{
+
+	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+	AZ(sp->obj);
+	AZ(sp->vcl);
+	assert(sp->fd >= 0);
+	/*
+	 * Set nonblocking in the worker-thread, before passing to the
+	 * acceptor thread, to reduce syscall density of the latter.
+	 */
+	if (VTCP_nonblocking(sp->fd))
+		SES_Close(sp, "remote closed");
+	waiter->pass(waiter_priv, sp);
+}
+
+/*--------------------------------------------------------------------
+ * Create another thread, if necessary & possible
+ */
+
+static void
+pool_breed(struct pool *qp, const pthread_attr_t *tp_attr)
+{
+	pthread_t tp;
+
+	/*
+	 * If we need more threads, and have space, create
+	 * one more thread.
+	 */
+	if (qp->nthr < cache_param->wthread_min ||	/* Not enough threads yet */
+	    (qp->lqueue > cache_param->wthread_add_threshold && /* more needed */
+	    qp->lqueue > qp->last_lqueue)) {	/* not getting better since last */
+		if (qp->nthr > cache_param->wthread_max) {
+			Lck_Lock(&pool_mtx);
+			VSC_C_main->threads_limited++;
+			Lck_Unlock(&pool_mtx);
+		} else if (pthread_create(&tp, tp_attr, WRK_thread, qp)) {
+			VSL(SLT_Debug, 0, "Create worker thread failed %d %s",
+			    errno, strerror(errno));
+			Lck_Lock(&pool_mtx);
+			VSC_C_main->threads_limited++;
+			Lck_Unlock(&pool_mtx);
+			VTIM_sleep(cache_param->wthread_fail_delay * 1e-3);
+		} else {
+			AZ(pthread_detach(tp));
+			VTIM_sleep(cache_param->wthread_add_delay * 1e-3);
+			qp->nthr++;
+			Lck_Lock(&pool_mtx);
+			VSC_C_main->threads++;
+			VSC_C_main->threads_created++;
+			Lck_Unlock(&pool_mtx);
+		}
+	}
+	qp->last_lqueue = qp->lqueue;
+}
+
+/*--------------------------------------------------------------------
+ * Herd a single pool
+ *
+ * This thread wakes up whenever a pool queues.
+ *
+ * The trick here is to not be too aggressive about creating threads.
+ * We do this by only examining one pool at a time, and by sleeping
+ * a short while whenever we create a thread and a little while longer
+ * whenever we fail to, hopefully missing a lot of cond_signals in
+ * the meantime.
+ *
+ * XXX: probably need a lot more work.
+ *
+ */
+
+static void*
+pool_herder(void *priv)
+{
+	struct pool *pp;
+	pthread_attr_t tp_attr;
+	struct timespec ts;
+	double t_idle;
+	struct worker *w;
+	int i;
+
+	CAST_OBJ_NOTNULL(pp, priv, POOL_MAGIC);
+	AZ(pthread_attr_init(&tp_attr));
+
+	while (1) {
+		/* Set the stacksize for worker threads we create */
+		if (cache_param->wthread_stacksize != UINT_MAX)
+			AZ(pthread_attr_setstacksize(&tp_attr,
+			    cache_param->wthread_stacksize));
+		else {
+			AZ(pthread_attr_destroy(&tp_attr));
+			AZ(pthread_attr_init(&tp_attr));
+		}
+
+		pool_breed(pp, &tp_attr);
+
+		if (pp->nthr < cache_param->wthread_min)
+			continue;
+
+		AZ(clock_gettime(CLOCK_MONOTONIC, &ts));
+		ts.tv_sec += cache_param->wthread_purge_delay / 1000;
+		ts.tv_nsec +=
+		    (cache_param->wthread_purge_delay % 1000) * 1000000;
+		if (ts.tv_nsec >= 1000000000) {
+			ts.tv_sec++;
+			ts.tv_nsec -= 1000000000;
+		}
+
+		Lck_Lock(&pp->herder_mtx);
+		i = Lck_CondWait(&pp->herder_cond, &pp->herder_mtx, &ts);
+		Lck_Unlock(&pp->herder_mtx);
+		if (!i)
+			continue;
+
+		if (pp->nthr <= cache_param->wthread_min)
+			continue;
+
+		t_idle = VTIM_real() - cache_param->wthread_timeout;
+
+		Lck_Lock(&pp->mtx);
+		VSC_C_main->sess_queued += pp->nqueued;
+		VSC_C_main->sess_dropped += pp->ndropped;
+		pp->nqueued = pp->ndropped = 0;
+		w = VTAILQ_LAST(&pp->idle, workerhead);
+		if (w != NULL &&
+		    (w->lastused < t_idle || pp->nthr > cache_param->wthread_max)) {
+			VTAILQ_REMOVE(&pp->idle, w, list);
+		} else
+			w = NULL;
+		Lck_Unlock(&pp->mtx);
+
+		/* And give it a kiss on the cheek... */
+		if (w != NULL) {
+			pp->nthr--;
+			Lck_Lock(&pool_mtx);
+			VSC_C_main->threads--;
+			VSC_C_main->threads_destroyed++;
+			Lck_Unlock(&pool_mtx);
+			AZ(w->sp);
+			AZ(pthread_cond_signal(&w->cond));
+		}
+	}
+	NEEDLESS_RETURN(NULL);
+}
+
+/*--------------------------------------------------------------------
+ * Add a thread pool
+ */
+
+static struct pool *
+pool_mkpool(void)
+{
+	struct pool *pp;
+	struct listen_sock *ls;
+	struct poolsock *ps;
+	pthread_condattr_t cv_attr;
+
+	ALLOC_OBJ(pp, POOL_MAGIC);
+	XXXAN(pp);
+	Lck_New(&pp->mtx, lck_wq);
+
+	VTAILQ_INIT(&pp->queue);
+	VTAILQ_INIT(&pp->idle);
+	VTAILQ_INIT(&pp->socks);
+	pp->sesspool = SES_NewPool(pp);
+	AN(pp->sesspool);
+
+	VTAILQ_FOREACH(ls, &heritage.socks, list) {
+		if (ls->sock < 0)
+			continue;
+		ALLOC_OBJ(ps, POOLSOCK_MAGIC);
+		XXXAN(ps);
+		ps->lsock = ls;
+		VTAILQ_INSERT_TAIL(&pp->socks, ps, list);
+	}
+
+	AZ(pthread_condattr_init(&cv_attr));
+	AZ(pthread_condattr_setclock(&cv_attr, CLOCK_MONOTONIC));
+	AZ(pthread_cond_init(&pp->herder_cond, &cv_attr));
+	AZ(pthread_condattr_destroy(&cv_attr));
+	Lck_New(&pp->herder_mtx, lck_herder);
+	AZ(pthread_create(&pp->herder_thr, NULL, pool_herder, pp));
+
+	return (pp);
+}
+
+/*--------------------------------------------------------------------
+ * This thread adjusts the number of pools to match the parameter.
+ *
+ */
+
+static void *
+pool_poolherder(void *priv)
+{
+	unsigned nwq;
+	VTAILQ_HEAD(,pool)	pools = VTAILQ_HEAD_INITIALIZER(pools);
+	struct pool *pp;
+	uint64_t u;
+
+	THR_SetName("pool_herder");
+	(void)priv;
+
+	nwq = 0;
+	while (1) {
+		if (nwq < cache_param->wthread_pools) {
+			pp = pool_mkpool();
+			if (pp != NULL) {
+				VTAILQ_INSERT_TAIL(&pools, pp, list);
+				VSC_C_main->pools++;
+				nwq++;
+				continue;
+			}
+		}
+		/* XXX: remove pools */
+		if (0)
+			SES_DeletePool(NULL, NULL);
+		(void)sleep(1);
+		u = 0;
+		VTAILQ_FOREACH(pp, &pools, list)
+			u += pp->lqueue;
+		VSC_C_main->thread_queue_len = u;
+	}
+	NEEDLESS_RETURN(NULL);
+}
+
+/*--------------------------------------------------------------------*/
+
+void
+Pool_Init(void)
+{
+
+	waiter_priv = waiter->init();
+	Lck_New(&pool_mtx, lck_wq);
+	AZ(pthread_create(&thr_pool_herder, NULL, pool_poolherder, NULL));
+}
diff --git a/bin/varnishd/cache/cache_response.c b/bin/varnishd/cache/cache_response.c
new file mode 100644
index 0000000..487a514
--- /dev/null
+++ b/bin/varnishd/cache/cache_response.c
@@ -0,0 +1,427 @@
+/*-
+ * Copyright (c) 2006 Verdens Gang AS
+ * Copyright (c) 2006-2011 Varnish Software AS
+ * All rights reserved.
+ *
+ * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include "config.h"
+
+#include "cache.h"
+
+#include "vct.h"
+#include "vtim.h"
+
+/*--------------------------------------------------------------------*/
+
+static void
+res_dorange(const struct sess *sp, const char *r, ssize_t *plow, ssize_t *phigh)
+{
+	ssize_t low, high, has_low;
+
+	assert(sp->obj->response == 200);
+	if (strncmp(r, "bytes=", 6))
+		return;
+	r += 6;
+
+	/* The low end of range */
+	has_low = low = 0;
+	if (!vct_isdigit(*r) && *r != '-')
+		return;
+	while (vct_isdigit(*r)) {
+		has_low = 1;
+		low *= 10;
+		low += *r - '0';
+		r++;
+	}
+
+	if (low >= sp->obj->len)
+		return;
+
+	if (*r != '-')
+		return;
+	r++;
+
+	/* The high end of range */
+	if (vct_isdigit(*r)) {
+		high = 0;
+		while (vct_isdigit(*r)) {
+			high *= 10;
+			high += *r - '0';
+			r++;
+		}
+		if (!has_low) {
+			low = sp->obj->len - high;
+			high = sp->obj->len - 1;
+		}
+	} else
+		high = sp->obj->len - 1;
+	if (*r != '\0')
+		return;
+
+	if (high >= sp->obj->len)
+		high = sp->obj->len - 1;
+
+	if (low > high)
+		return;
+
+	http_PrintfHeader(sp->wrk, sp->vsl_id, sp->wrk->resp,
+	    "Content-Range: bytes %jd-%jd/%jd",
+	    (intmax_t)low, (intmax_t)high, (intmax_t)sp->obj->len);
+	http_Unset(sp->wrk->resp, H_Content_Length);
+	assert(sp->wrk->res_mode & RES_LEN);
+	http_PrintfHeader(sp->wrk, sp->vsl_id, sp->wrk->resp,
+	    "Content-Length: %jd", (intmax_t)(1 + high - low));
+	http_SetResp(sp->wrk->resp, "HTTP/1.1", 206, "Partial Content");
+
+	*plow = low;
+	*phigh = high;
+}
+
+/*--------------------------------------------------------------------*/
+
+void
+RES_BuildHttp(const struct sess *sp)
+{
+	char time_str[30];
+
+	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+
+
+	http_ClrHeader(sp->wrk->resp);
+	sp->wrk->resp->logtag = HTTP_Tx;
+	http_CopyResp(sp->wrk->resp, sp->obj->http);
+	http_FilterFields(sp->wrk, sp->vsl_id, sp->wrk->resp, sp->obj->http,
+	    HTTPH_A_DELIVER);
+
+	if (!(sp->wrk->res_mode & RES_LEN)) {
+		http_Unset(sp->wrk->resp, H_Content_Length);
+	} else if (cache_param->http_range_support) {
+		/* We only accept ranges if we know the length */
+		http_SetHeader(sp->wrk, sp->vsl_id, sp->wrk->resp,
+		    "Accept-Ranges: bytes");
+	}
+
+	if (sp->wrk->res_mode & RES_CHUNKED)
+		http_SetHeader(sp->wrk, sp->vsl_id, sp->wrk->resp,
+		    "Transfer-Encoding: chunked");
+
+	VTIM_format(VTIM_real(), time_str);
+	http_PrintfHeader(sp->wrk, sp->vsl_id, sp->wrk->resp, "Date: %s", time_str);
+
+	if (sp->xid != sp->obj->xid)
+		http_PrintfHeader(sp->wrk, sp->vsl_id, sp->wrk->resp,
+		    "X-Varnish: %u %u", sp->xid, sp->obj->xid);
+	else
+		http_PrintfHeader(sp->wrk, sp->vsl_id, sp->wrk->resp,
+		    "X-Varnish: %u", sp->xid);
+	http_PrintfHeader(sp->wrk, sp->vsl_id, sp->wrk->resp, "Age: %.0f",
+	    sp->obj->exp.age + sp->t_resp - sp->obj->exp.entered);
+	http_SetHeader(sp->wrk, sp->vsl_id, sp->wrk->resp, "Via: 1.1 varnish");
+	http_PrintfHeader(sp->wrk, sp->vsl_id, sp->wrk->resp, "Connection: %s",
+	    sp->doclose ? "close" : "keep-alive");
+}
+
+/*--------------------------------------------------------------------
+ * We have a gzip'ed object and need to ungzip it for a client which
+ * does not understand gzip.
+ * XXX: handle invalid gzip data better (how ?)
+ */
+
+static void
+res_WriteGunzipObj(const struct sess *sp)
+{
+	struct storage *st;
+	unsigned u = 0;
+	struct vgz *vg;
+	char obuf[cache_param->gzip_stack_buffer];
+	ssize_t obufl = 0;
+	int i;
+
+	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+
+	vg = VGZ_NewUngzip(sp->wrk, "U D -");
+
+	VGZ_Obuf(vg, obuf, sizeof obuf);
+	VTAILQ_FOREACH(st, &sp->obj->store, list) {
+		CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+		CHECK_OBJ_NOTNULL(st, STORAGE_MAGIC);
+		u += st->len;
+
+		VSC_C_main->n_objwrite++;
+
+		i = VGZ_WrwGunzip(sp->wrk, vg,
+		    st->ptr, st->len,
+		    obuf, sizeof obuf, &obufl);
+		/* XXX: error check */
+		(void)i;
+	}
+	if (obufl) {
+		(void)WRW_Write(sp->wrk, obuf, obufl);
+		(void)WRW_Flush(sp->wrk);
+	}
+	(void)VGZ_Destroy(&vg, sp->vsl_id);
+	assert(u == sp->obj->len);
+}
+
+/*--------------------------------------------------------------------*/
+
+static void
+res_WriteDirObj(const struct sess *sp, ssize_t low, ssize_t high)
+{
+	ssize_t u = 0;
+	size_t ptr, off, len;
+	struct storage *st;
+
+	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+
+	ptr = 0;
+	VTAILQ_FOREACH(st, &sp->obj->store, list) {
+		CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+		CHECK_OBJ_NOTNULL(st, STORAGE_MAGIC);
+		u += st->len;
+		len = st->len;
+		off = 0;
+		if (ptr + len <= low) {
+			/* This segment is too early */
+			ptr += len;
+			continue;
+		}
+		if (ptr < low) {
+			/* Chop front of segment off */
+			off += (low - ptr);
+			len -= (low - ptr);
+			ptr += (low - ptr);
+		}
+		if (ptr + len > high)
+			/* Chop tail of segment off */
+			len = 1 + high - ptr;
+
+		ptr += len;
+
+		sp->wrk->acct_tmp.bodybytes += len;
+#ifdef SENDFILE_WORKS
+		/*
+		 * XXX: the overhead of setting up sendfile is not
+		 * XXX: epsilon and maybe not even delta, so avoid
+		 * XXX: engaging sendfile for small objects.
+		 * XXX: Should use getpagesize() ?
+		 */
+		if (st->fd >= 0 &&
+		    st->len >= cache_param->sendfile_threshold) {
+			VSC_C_main->n_objsendfile++;
+			WRW_Sendfile(sp->wrk, st->fd, st->where + off, len);
+			continue;
+		}
+#endif /* SENDFILE_WORKS */
+		VSC_C_main->n_objwrite++;
+		(void)WRW_Write(sp->wrk, st->ptr + off, len);
+	}
+	assert(u == sp->obj->len);
+}
+
+/*--------------------------------------------------------------------
+ * Deliver an object.
+ * Attempt optimizations like 304 and 206 here.
+ */
+
+void
+RES_WriteObj(struct sess *sp)
+{
+	char *r;
+	ssize_t low, high;
+
+	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+
+	WRW_Reserve(sp->wrk, &sp->fd);
+
+	if (sp->obj->response == 200 &&
+	    sp->http->conds &&
+	    RFC2616_Do_Cond(sp)) {
+		sp->wantbody = 0;
+		http_SetResp(sp->wrk->resp, "HTTP/1.1", 304, "Not Modified");
+		http_Unset(sp->wrk->resp, H_Content_Length);
+		http_Unset(sp->wrk->resp, H_Transfer_Encoding);
+	}
+
+	/*
+	 * If nothing special planned, we can attempt Range support
+	 */
+	low = 0;
+	high = sp->obj->len - 1;
+	if (
+	    sp->wantbody &&
+	    (sp->wrk->res_mode & RES_LEN) &&
+	    !(sp->wrk->res_mode & (RES_ESI|RES_ESI_CHILD|RES_GUNZIP)) &&
+	    cache_param->http_range_support &&
+	    sp->obj->response == 200 &&
+	    http_GetHdr(sp->http, H_Range, &r))
+		res_dorange(sp, r, &low, &high);
+
+	/*
+	 * Always remove C-E if client don't grok it
+	 */
+	if (sp->wrk->res_mode & RES_GUNZIP)
+		http_Unset(sp->wrk->resp, H_Content_Encoding);
+
+	/*
+	 * Send HTTP protocol header, unless interior ESI object
+	 */
+	if (!(sp->wrk->res_mode & RES_ESI_CHILD))
+		sp->wrk->acct_tmp.hdrbytes +=
+		    http_Write(sp->wrk, sp->vsl_id, sp->wrk->resp, 1);
+
+	if (!sp->wantbody)
+		sp->wrk->res_mode &= ~RES_CHUNKED;
+
+	if (sp->wrk->res_mode & RES_CHUNKED)
+		WRW_Chunked(sp->wrk);
+
+	if (!sp->wantbody) {
+		/* This was a HEAD or conditional request */
+	} else if (sp->obj->len == 0) {
+		/* Nothing to do here */
+	} else if (sp->wrk->res_mode & RES_ESI) {
+		ESI_Deliver(sp);
+	} else if (sp->wrk->res_mode & RES_ESI_CHILD && sp->wrk->gzip_resp) {
+		ESI_DeliverChild(sp);
+	} else if (sp->wrk->res_mode & RES_ESI_CHILD &&
+	    !sp->wrk->gzip_resp && sp->obj->gziped) {
+		res_WriteGunzipObj(sp);
+	} else if (sp->wrk->res_mode & RES_GUNZIP) {
+		res_WriteGunzipObj(sp);
+	} else {
+		res_WriteDirObj(sp, low, high);
+	}
+
+	if (sp->wrk->res_mode & RES_CHUNKED &&
+	    !(sp->wrk->res_mode & RES_ESI_CHILD))
+		WRW_EndChunk(sp->wrk);
+
+	if (WRW_FlushRelease(sp->wrk) && sp->fd >= 0)
+		SES_Close(sp, "remote closed");
+}
+
+/*--------------------------------------------------------------------*/
+
+void
+RES_StreamStart(struct sess *sp)
+{
+	struct stream_ctx *sctx;
+
+	sctx = sp->wrk->sctx;
+	CHECK_OBJ_NOTNULL(sctx, STREAM_CTX_MAGIC);
+
+	AZ(sp->wrk->res_mode & RES_ESI_CHILD);
+	AN(sp->wantbody);
+
+	WRW_Reserve(sp->wrk, &sp->fd);
+	/*
+	 * Always remove C-E if client don't grok it
+	 */
+	if (sp->wrk->res_mode & RES_GUNZIP)
+		http_Unset(sp->wrk->resp, H_Content_Encoding);
+
+	if (!(sp->wrk->res_mode & RES_CHUNKED) &&
+	    sp->wrk->h_content_length != NULL)
+		http_PrintfHeader(sp->wrk, sp->vsl_id, sp->wrk->resp,
+		    "Content-Length: %s", sp->wrk->h_content_length);
+
+	sp->wrk->acct_tmp.hdrbytes +=
+	    http_Write(sp->wrk, sp->vsl_id, sp->wrk->resp, 1);
+
+	if (sp->wrk->res_mode & RES_CHUNKED)
+		WRW_Chunked(sp->wrk);
+}
+
+void
+RES_StreamPoll(struct worker *w)
+{
+	struct stream_ctx *sctx;
+	struct storage *st;
+	ssize_t l, l2;
+	void *ptr;
+
+	CHECK_OBJ_NOTNULL(w, WORKER_MAGIC);
+	CHECK_OBJ_NOTNULL(w->fetch_obj, OBJECT_MAGIC);
+	sctx = w->sctx;
+	CHECK_OBJ_NOTNULL(sctx, STREAM_CTX_MAGIC);
+	if (w->fetch_obj->len == sctx->stream_next)
+		return;
+	assert(w->fetch_obj->len > sctx->stream_next);
+	l = sctx->stream_front;
+	VTAILQ_FOREACH(st, &w->fetch_obj->store, list) {
+		if (st->len + l <= sctx->stream_next) {
+			l += st->len;
+			continue;
+		}
+		l2 = st->len + l - sctx->stream_next;
+		ptr = st->ptr + (sctx->stream_next - l);
+		if (w->res_mode & RES_GUNZIP) {
+			(void)VGZ_WrwGunzip(w, sctx->vgz, ptr, l2,
+			    sctx->obuf, sctx->obuf_len, &sctx->obuf_ptr);
+		} else {
+			(void)WRW_Write(w, ptr, l2);
+		}
+		l += st->len;
+		sctx->stream_next += l2;
+	}
+	if (!(w->res_mode & RES_GUNZIP))
+		(void)WRW_Flush(w);
+
+	if (w->fetch_obj->objcore == NULL ||
+	    (w->fetch_obj->objcore->flags & OC_F_PASS)) {
+		/*
+		 * This is a pass object, release storage as soon as we
+		 * have delivered it.
+		 */
+		while (1) {
+			st = VTAILQ_FIRST(&w->fetch_obj->store);
+			if (st == NULL ||
+			    sctx->stream_front + st->len > sctx->stream_next)
+				break;
+			VTAILQ_REMOVE(&w->fetch_obj->store, st, list);
+			sctx->stream_front += st->len;
+			STV_free(st);
+		}
+	}
+}
+
+void
+RES_StreamEnd(struct sess *sp)
+{
+	struct stream_ctx *sctx;
+
+	sctx = sp->wrk->sctx;
+	CHECK_OBJ_NOTNULL(sctx, STREAM_CTX_MAGIC);
+
+	if (sp->wrk->res_mode & RES_GUNZIP && sctx->obuf_ptr > 0)
+		(void)WRW_Write(sp->wrk, sctx->obuf, sctx->obuf_ptr);
+	if (sp->wrk->res_mode & RES_CHUNKED &&
+	    !(sp->wrk->res_mode & RES_ESI_CHILD))
+		WRW_EndChunk(sp->wrk);
+	if (WRW_FlushRelease(sp->wrk))
+		SES_Close(sp, "remote closed");
+}
diff --git a/bin/varnishd/cache/cache_rfc2616.c b/bin/varnishd/cache/cache_rfc2616.c
new file mode 100644
index 0000000..4041f45
--- /dev/null
+++ b/bin/varnishd/cache/cache_rfc2616.c
@@ -0,0 +1,336 @@
+/*-
+ * Copyright (c) 2006 Verdens Gang AS
+ * Copyright (c) 2006-2011 Varnish Software AS
+ * All rights reserved.
+ *
+ * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include "config.h"
+
+#include <math.h>
+#include <stdlib.h>
+
+#include "cache.h"
+
+#include "vtim.h"
+
+/*--------------------------------------------------------------------
+ * TTL and Age calculation in Varnish
+ *
+ * RFC2616 has a lot to say about how caches should calculate the TTL
+ * and expiry times of objects, but it sort of misses the case that
+ * applies to Varnish:  the server-side cache.
+ *
+ * A normal cache, shared or single-client, has no symbiotic relationship
+ * with the server, and therefore must take a very defensive attitude
+ * if the Data/Expiry/Age/max-age data does not make sense.  Overall
+ * the policy described in section 13 of RFC 2616 results in no caching
+ * happening on the first little sign of trouble.
+ *
+ * Varnish on the other hand tries to offload as many transactions from
+ * the backend as possible, and therefore just passing through everything
+ * if there is a clock-skew between backend and Varnish is not a workable
+ * choice.
+ *
+ * Varnish implements a policy which is RFC2616 compliant when there
+ * is no clockskew, and falls as gracefully as possible otherwise.
+ * Our "clockless cache" model is syntehsized from the bits of RFC2616
+ * that talks about how a cache should react to a clockless origin server,
+ * and more or less uses the inverse logic for the opposite relationship.
+ *
+ */
+
+void
+RFC2616_Ttl(const struct sess *sp)
+{
+	unsigned max_age, age;
+	double h_date, h_expires;
+	char *p;
+	const struct http *hp;
+
+	hp = sp->wrk->beresp;
+
+	assert(sp->wrk->exp.entered != 0.0 && !isnan(sp->wrk->exp.entered));
+	/* If all else fails, cache using default ttl */
+	sp->wrk->exp.ttl = cache_param->default_ttl;
+
+	max_age = age = 0;
+	h_expires = 0;
+	h_date = 0;
+
+	/*
+	 * Initial cacheability determination per [RFC2616, 13.4]
+	 * We do not support ranges yet, so 206 is out.
+	 */
+
+	if (http_GetHdr(hp, H_Age, &p)) {
+		age = strtoul(p, NULL, 0);
+		sp->wrk->exp.age = age;
+	}
+	if (http_GetHdr(hp, H_Expires, &p))
+		h_expires = VTIM_parse(p);
+
+	if (http_GetHdr(hp, H_Date, &p))
+		h_date = VTIM_parse(p);
+
+	switch (sp->err_code) {
+	default:
+		sp->wrk->exp.ttl = -1.;
+		break;
+	case 200: /* OK */
+	case 203: /* Non-Authoritative Information */
+	case 300: /* Multiple Choices */
+	case 301: /* Moved Permanently */
+	case 302: /* Moved Temporarily */
+	case 307: /* Temporary Redirect */
+	case 410: /* Gone */
+	case 404: /* Not Found */
+		/*
+		 * First find any relative specification from the backend
+		 * These take precedence according to RFC2616, 13.2.4
+		 */
+
+		if ((http_GetHdrField(hp, H_Cache_Control, "s-maxage", &p) ||
+		    http_GetHdrField(hp, H_Cache_Control, "max-age", &p)) &&
+		    p != NULL) {
+
+			if (*p == '-')
+				max_age = 0;
+			else
+				max_age = strtoul(p, NULL, 0);
+
+			if (age > max_age)
+				sp->wrk->exp.ttl = 0;
+			else
+				sp->wrk->exp.ttl = max_age - age;
+			break;
+		}
+
+		/* No expire header, fall back to default */
+		if (h_expires == 0)
+			break;
+
+
+		/* If backend told us it is expired already, don't cache. */
+		if (h_expires < h_date) {
+			sp->wrk->exp.ttl = 0;
+			break;
+		}
+
+		if (h_date == 0 ||
+		    fabs(h_date - sp->wrk->exp.entered) < cache_param->clock_skew) {
+			/*
+			 * If we have no Date: header or if it is
+			 * sufficiently close to our clock we will
+			 * trust Expires: relative to our own clock.
+			 */
+			if (h_expires < sp->wrk->exp.entered)
+				sp->wrk->exp.ttl = 0;
+			else
+				sp->wrk->exp.ttl = h_expires -
+				    sp->wrk->exp.entered;
+			break;
+		} else {
+			/*
+			 * But even if the clocks are out of whack we can still
+			 * derive a relative time from the two headers.
+			 * (the negative ttl case is caught above)
+			 */
+			sp->wrk->exp.ttl = (int)(h_expires - h_date);
+		}
+
+	}
+
+	/* calculated TTL, Our time, Date, Expires, max-age, age */
+	WSP(sp, SLT_TTL,
+	    "%u RFC %.0f %.0f %.0f %.0f %.0f %.0f %.0f %u",
+	    sp->xid, sp->wrk->exp.ttl, -1., -1., sp->wrk->exp.entered,
+	    sp->wrk->exp.age, h_date, h_expires, max_age);
+}
+
+/*--------------------------------------------------------------------
+ * Body existence, fetch method and close policy.
+ */
+
+enum body_status
+RFC2616_Body(const struct sess *sp)
+{
+	struct http *hp;
+	char *b;
+
+	hp = sp->wrk->beresp;
+
+	if (hp->protover < 11 && !http_HdrIs(hp, H_Connection, "keep-alive"))
+		sp->wrk->do_close = 1;
+	else if (http_HdrIs(hp, H_Connection, "close"))
+		sp->wrk->do_close = 1;
+	else
+		sp->wrk->do_close = 0;
+
+	if (!strcasecmp(http_GetReq(sp->wrk->bereq), "head")) {
+		/*
+		 * A HEAD request can never have a body in the reply,
+		 * no matter what the headers might say.
+		 * [RFC2516 4.3 p33]
+		 */
+		sp->wrk->stats.fetch_head++;
+		return (BS_NONE);
+	}
+
+	if (hp->status <= 199) {
+		/*
+		 * 1xx responses never have a body.
+		 * [RFC2616 4.3 p33]
+		 */
+		sp->wrk->stats.fetch_1xx++;
+		return (BS_NONE);
+	}
+
+	if (hp->status == 204) {
+		/*
+		 * 204 is "No Content", obviously don't expect a body.
+		 * [RFC2616 10.2.5 p60]
+		 */
+		sp->wrk->stats.fetch_204++;
+		return (BS_NONE);
+	}
+
+	if (hp->status == 304) {
+		/*
+		 * 304 is "Not Modified" it has no body.
+		 * [RFC2616 10.3.5 p63]
+		 */
+		sp->wrk->stats.fetch_304++;
+		return (BS_NONE);
+	}
+
+	if (http_HdrIs(hp, H_Transfer_Encoding, "chunked")) {
+		 sp->wrk->stats.fetch_chunked++;
+		return (BS_CHUNKED);
+	}
+
+	if (http_GetHdr(hp, H_Transfer_Encoding, &b)) {
+		sp->wrk->stats.fetch_bad++;
+		return (BS_ERROR);
+	}
+
+	if (http_GetHdr(hp, H_Content_Length, &sp->wrk->h_content_length)) {
+		sp->wrk->stats.fetch_length++;
+		return (BS_LENGTH);
+	}
+
+	if (http_HdrIs(hp, H_Connection, "keep-alive")) {
+		/*
+		 * Keep alive with neither TE=Chunked or C-Len is impossible.
+		 * We assume a zero length body.
+		 */
+		sp->wrk->stats.fetch_zero++;
+		return (BS_ZERO);
+	}
+
+	if (http_HdrIs(hp, H_Connection, "close")) {
+		/*
+		 * In this case, it is safe to just read what comes.
+		 */
+		sp->wrk->stats.fetch_close++;
+		return (BS_EOF);
+	}
+
+	if (hp->protover < 11) {
+		/*
+		 * With no Connection header, assume EOF.
+		 */
+		sp->wrk->stats.fetch_oldhttp++;
+		return (BS_EOF);
+	}
+
+	/*
+	 * Fall back to EOF transfer.
+	 */
+	sp->wrk->stats.fetch_eof++;
+	return (BS_EOF);
+}
+
+/*--------------------------------------------------------------------
+ * Find out if the request can receive a gzip'ed response
+ */
+
+unsigned
+RFC2616_Req_Gzip(const struct sess *sp)
+{
+
+
+	/*
+	 * "x-gzip" is for http/1.0 backwards compat, final note in 14.3
+	 * p104 says to not do q values for x-gzip, so we just test
+	 * for its existence.
+	 */
+	if (http_GetHdrData(sp->http, H_Accept_Encoding, "x-gzip", NULL))
+		return (1);
+
+	/*
+	 * "gzip" is the real thing, but the 'q' value must be nonzero.
+	 * We do not care a hoot if the client prefers some other
+	 * compression more than gzip: Varnish only does gzip.
+	 */
+	if (http_GetHdrQ(sp->http, H_Accept_Encoding, "gzip") > 0.)
+		return (1);
+
+	/* Bad client, no gzip. */
+	return (0);
+}
+
+/*--------------------------------------------------------------------*/
+
+int
+RFC2616_Do_Cond(const struct sess *sp)
+{
+	char *p, *e;
+	double ims;
+	int do_cond = 0;
+
+	/* RFC 2616 13.3.4 states we need to match both ETag
+	   and If-Modified-Since if present*/
+
+	if (http_GetHdr(sp->http, H_If_Modified_Since, &p) ) {
+		if (!sp->obj->last_modified)
+			return (0);
+		ims = VTIM_parse(p);
+		if (ims > sp->t_req)	/* [RFC2616 14.25] */
+			return (0);
+		if (sp->obj->last_modified > ims)
+			return (0);
+		do_cond = 1;
+	}
+
+	if (http_GetHdr(sp->http, H_If_None_Match, &p) &&
+	    http_GetHdr(sp->obj->http, H_ETag, &e)) {
+		if (strcmp(p,e) != 0)
+			return (0);
+		do_cond = 1;
+	}
+
+	return (do_cond);
+}
diff --git a/bin/varnishd/cache/cache_session.c b/bin/varnishd/cache/cache_session.c
new file mode 100644
index 0000000..7befbcc
--- /dev/null
+++ b/bin/varnishd/cache/cache_session.c
@@ -0,0 +1,419 @@
+/*-
+ * Copyright (c) 2006 Verdens Gang AS
+ * Copyright (c) 2006-2010 Varnish Software AS
+ * All rights reserved.
+ *
+ * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * Session management
+ *
+ * This is a little bit of a mixed back, containing both memory management
+ * and various state-change functions.
+ *
+ */
+
+#include "config.h"
+
+#include <math.h>
+#include <stdlib.h>
+
+#include "cache.h"
+
+#include "waiter/cache_waiter.h"
+
+/*--------------------------------------------------------------------*/
+
+struct sessmem {
+	unsigned		magic;
+#define SESSMEM_MAGIC		0x555859c5
+
+	struct sesspool		*pool;
+
+	unsigned		workspace;
+	uint16_t		nhttp;
+	void			*wsp;
+	struct http		*http[2];
+	VTAILQ_ENTRY(sessmem)	list;
+
+	struct sess		sess;
+};
+
+struct sesspool {
+	unsigned		magic;
+#define SESSPOOL_MAGIC		0xd916e202
+	struct pool		*pool;
+	VTAILQ_HEAD(,sessmem)	freelist;
+	struct lock		mtx;
+	unsigned		nsess;
+	unsigned		dly_free_cnt;
+};
+
+/*--------------------------------------------------------------------
+ * Charge statistics from worker to request and session.
+ */
+
+void
+SES_Charge(struct sess *sp)
+{
+	struct acct *a = &sp->wrk->acct_tmp;
+
+	sp->req_bodybytes += a->bodybytes;
+
+#define ACCT(foo)				\
+	sp->wrk->stats.s_##foo += a->foo;	\
+	sp->acct_ses.foo += a->foo;		\
+	a->foo = 0;
+#include "tbl/acct_fields.h"
+#undef ACCT
+}
+
+/*--------------------------------------------------------------------
+ * This function allocates a session + assorted peripheral data
+ * structures in one single malloc operation.
+ */
+
+static struct sessmem *
+ses_sm_alloc(void)
+{
+	struct sessmem *sm;
+	unsigned char *p, *q;
+	unsigned nws;
+	uint16_t nhttp;
+	unsigned l, hl;
+
+	/*
+	 * It is not necessary to lock these, but we need to
+	 * cache them locally, to make sure we get a consistent
+	 * view of the value.
+	 */
+	nws = cache_param->sess_workspace;
+	nhttp = (uint16_t)cache_param->http_max_hdr;
+
+	hl = HTTP_estimate(nhttp);
+	l = sizeof *sm + nws + 2 * hl;
+	VSC_C_main->sessmem_size = l;
+	p = malloc(l);
+	if (p == NULL)
+		return (NULL);
+	q = p + l;
+
+	/* Don't waste time zeroing the workspace */
+	memset(p, 0, l - nws);
+
+	sm = (void*)p;
+	p += sizeof *sm;
+
+	sm->magic = SESSMEM_MAGIC;
+	sm->workspace = nws;
+	sm->nhttp = nhttp;
+
+	sm->http[0] = HTTP_create(p, nhttp);
+	p += hl;
+
+	sm->http[1] = HTTP_create(p, nhttp);
+	p += hl;
+
+	sm->wsp = p;
+	p += nws;
+
+	assert(p == q);
+
+	return (sm);
+}
+
+/*--------------------------------------------------------------------
+ * This prepares a session for use, based on its sessmem structure.
+ */
+
+static void
+ses_setup(struct sessmem *sm)
+{
+	struct sess *sp;
+
+	CHECK_OBJ_NOTNULL(sm, SESSMEM_MAGIC);
+	sp = &sm->sess;
+	memset(sp, 0, sizeof *sp);
+
+	/* We assume that the sess has been zeroed by the time we get here */
+	AZ(sp->magic);
+
+	sp->magic = SESS_MAGIC;
+	sp->mem = sm;
+	sp->sockaddrlen = sizeof(sp->sockaddr);
+	sp->mysockaddrlen = sizeof(sp->mysockaddr);
+	sp->sockaddr.ss_family = sp->mysockaddr.ss_family = PF_UNSPEC;
+	sp->t_open = NAN;
+	sp->t_req = NAN;
+	sp->t_resp = NAN;
+	sp->t_end = NAN;
+	EXP_Clr(&sp->exp);
+
+	WS_Init(sp->ws, "sess", sm->wsp, sm->workspace);
+	sp->http = sm->http[0];
+	sp->http0 = sm->http[1];
+}
+
+/*--------------------------------------------------------------------
+ * Get a new session, preferably by recycling an already ready one
+ */
+
+struct sess *
+SES_New(struct worker *wrk, struct sesspool *pp)
+{
+	struct sessmem *sm;
+	struct sess *sp;
+	int do_alloc;
+
+	CHECK_OBJ_NOTNULL(pp, SESSPOOL_MAGIC);
+
+	do_alloc = 0;
+	Lck_Lock(&pp->mtx);
+	sm = VTAILQ_FIRST(&pp->freelist);
+	if (sm != NULL) {
+		VTAILQ_REMOVE(&pp->freelist, sm, list);
+	} else if (pp->nsess < cache_param->max_sess) {
+		pp->nsess++;
+		do_alloc = 1;
+	}
+	wrk->stats.sessmem_free += pp->dly_free_cnt;
+	pp->dly_free_cnt = 0;
+	Lck_Unlock(&pp->mtx);
+	if (do_alloc) {
+		sm = ses_sm_alloc();
+		if (sm != NULL) {
+			wrk->stats.sessmem_alloc++;
+			sm->pool = pp;
+			ses_setup(sm);
+		} else {
+			wrk->stats.sessmem_fail++;
+		}
+	} else if (sm == NULL) {
+		wrk->stats.sessmem_limit++;
+	}
+	if (sm == NULL)
+		return (NULL);
+	sp = &sm->sess;
+	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+	return (sp);
+}
+
+/*--------------------------------------------------------------------
+ * Allocate a session for use by background threads.
+ */
+
+struct sess *
+SES_Alloc(void)
+{
+	struct sess *sp;
+	struct sessmem *sm;
+
+	sm = ses_sm_alloc();
+	AN(sm);
+	ses_setup(sm);
+	sp = &sm->sess;
+	sp->sockaddrlen = 0;
+	return (sp);
+}
+
+/*--------------------------------------------------------------------
+ * Schedule a session back on a work-thread from its pool
+ */
+
+int
+SES_Schedule(struct sess *sp)
+{
+	struct sessmem *sm;
+	struct sesspool *pp;
+
+	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+
+	AZ(sp->wrk);
+
+	sm = sp->mem;
+	CHECK_OBJ_NOTNULL(sm, SESSMEM_MAGIC);
+
+	pp = sm->pool;
+	CHECK_OBJ_NOTNULL(pp, SESSPOOL_MAGIC);
+
+	AN(pp->pool);
+
+	if (Pool_Schedule(pp->pool, sp)) {
+		SES_Delete(sp, "dropped");
+		return (1);
+	}
+	return (0);
+}
+
+/*--------------------------------------------------------------------
+ * Handle a session (from waiter)
+ *
+ * Status: see HTC_Rx()
+ */
+
+void
+SES_Handle(struct sess *sp, int status)
+{
+
+	switch (status) {
+	case -2:
+		SES_Delete(sp, "blast");
+		break;
+	case -1:
+		SES_Delete(sp, "no request");
+		break;
+	case 1:
+		sp->step = STP_START;
+		(void)SES_Schedule(sp);
+		break;
+	default:
+		WRONG("Unexpected return from HTC_Rx()");
+	}
+}
+
+/*--------------------------------------------------------------------
+ * Close a sessions connection.
+ */
+
+void
+SES_Close(struct sess *sp, const char *reason)
+{
+	int i;
+
+	assert(sp->fd >= 0);
+	VSL(SLT_SessionClose, sp->vsl_id, "%s", reason);
+	i = close(sp->fd);
+	assert(i == 0 || errno != EBADF); /* XXX EINVAL seen */
+	sp->fd = -1;
+}
+
+/*--------------------------------------------------------------------
+ * (Close &) Free or Recycle a session.
+ *
+ * If the workspace has changed, deleted it, otherwise wash it, and put
+ * it up for adoption.
+ *
+ * XXX: We should also check nhttp
+ */
+
+void
+SES_Delete(struct sess *sp, const char *reason)
+{
+	struct acct *b;
+	struct sessmem *sm;
+	static char noaddr[] = "-";
+	struct worker *wrk;
+	struct sesspool *pp;
+
+
+	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+	sm = sp->mem;
+	CHECK_OBJ_NOTNULL(sm, SESSMEM_MAGIC);
+	pp = sm->pool;
+	CHECK_OBJ_NOTNULL(pp, SESSPOOL_MAGIC);
+	wrk = sp->wrk;
+	CHECK_OBJ_ORNULL(wrk, WORKER_MAGIC);
+
+
+	if (reason != NULL)
+		SES_Close(sp, reason);
+	assert(sp->fd < 0);
+
+	AZ(sp->obj);
+	AZ(sp->vcl);
+	if (sp->addr == NULL)
+		sp->addr = noaddr;
+	if (sp->port == NULL)
+		sp->port = noaddr;
+
+	b = &sp->acct_ses;
+	assert(!isnan(b->first));
+	assert(!isnan(sp->t_end));
+
+	VSL(SLT_StatSess, sp->vsl_id, "%s %s %.0f %ju %ju %ju %ju %ju %ju %ju",
+	    sp->addr, sp->port, sp->t_end - b->first,
+	    b->sess, b->req, b->pipe, b->pass,
+	    b->fetch, b->hdrbytes, b->bodybytes);
+
+	if (sm->workspace != cache_param->sess_workspace ||
+	    sm->nhttp != (uint16_t)cache_param->http_max_hdr ||
+	    pp->nsess > cache_param->max_sess) {
+		free(sm);
+		Lck_Lock(&pp->mtx);
+		if (wrk != NULL)
+			wrk->stats.sessmem_free++;
+		else
+			pp->dly_free_cnt++;
+		pp->nsess--;
+		Lck_Unlock(&pp->mtx);
+	} else {
+		/* Clean and prepare for reuse */
+		ses_setup(sm);
+		Lck_Lock(&pp->mtx);
+		if (wrk != NULL) {
+			wrk->stats.sessmem_free += pp->dly_free_cnt;
+			pp->dly_free_cnt = 0;
+		}
+		VTAILQ_INSERT_HEAD(&pp->freelist, sm, list);
+		Lck_Unlock(&pp->mtx);
+	}
+}
+
+/*--------------------------------------------------------------------
+ * Create and delete pools
+ */
+
+struct sesspool *
+SES_NewPool(struct pool *pp)
+{
+	struct sesspool *sp;
+
+	ALLOC_OBJ(sp, SESSPOOL_MAGIC);
+	AN(sp);
+	sp->pool = pp;
+	VTAILQ_INIT(&sp->freelist);
+	Lck_New(&sp->mtx, lck_sessmem);
+	return (sp);
+}
+
+void
+SES_DeletePool(struct sesspool *sp, struct worker *wrk)
+{
+	struct sessmem *sm;
+
+	CHECK_OBJ_NOTNULL(sp, SESSPOOL_MAGIC);
+	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
+	Lck_Lock(&sp->mtx);
+	while (!VTAILQ_EMPTY(&sp->freelist)) {
+		sm = VTAILQ_FIRST(&sp->freelist);
+		CHECK_OBJ_NOTNULL(sm, SESSMEM_MAGIC);
+		VTAILQ_REMOVE(&sp->freelist, sm, list);
+		FREE_OBJ(sm);
+		wrk->stats.sessmem_free++;
+		sp->nsess--;
+	}
+	AZ(sp->nsess);
+	Lck_Unlock(&sp->mtx);
+	Lck_Delete(&sp->mtx);
+	FREE_OBJ(sp);
+}
diff --git a/bin/varnishd/cache/cache_shmlog.c b/bin/varnishd/cache/cache_shmlog.c
new file mode 100644
index 0000000..1252fa3
--- /dev/null
+++ b/bin/varnishd/cache/cache_shmlog.c
@@ -0,0 +1,346 @@
+/*-
+ * Copyright (c) 2006 Verdens Gang AS
+ * Copyright (c) 2006-2011 Varnish Software AS
+ * All rights reserved.
+ *
+ * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include "config.h"
+
+#include <stdio.h>
+
+#include "cache.h"
+
+#include "cache_backend.h"	// For w->vbc
+
+#include "vapi/vsm_int.h"
+#include "vmb.h"
+#include "vtim.h"
+
+/* These cannot be struct lock, which depends on vsm/vsl working */
+static pthread_mutex_t vsl_mtx;
+static pthread_mutex_t vsm_mtx;
+
+static uint32_t			*vsl_start;
+static const uint32_t		*vsl_end;
+static uint32_t			*vsl_ptr;
+
+static inline uint32_t
+vsl_w0(uint32_t type, uint32_t length)
+{
+
+	assert(length < 0x10000);
+        return (((type & 0xff) << 24) | length);
+}
+
+/*--------------------------------------------------------------------*/
+
+static inline void
+vsl_hdr(enum VSL_tag_e tag, uint32_t *p, unsigned len, unsigned id)
+{
+
+	assert(((uintptr_t)p & 0x3) == 0);
+
+	p[1] = id;
+	VMB();
+	p[0] = vsl_w0(tag, len);
+}
+
+/*--------------------------------------------------------------------*/
+
+static void
+vsl_wrap(void)
+{
+
+	assert(vsl_ptr >= vsl_start + 1);
+	assert(vsl_ptr < vsl_end);
+	vsl_start[1] = VSL_ENDMARKER;
+	do
+		vsl_start[0]++;
+	while (vsl_start[0] == 0);
+	VWMB();
+	if (vsl_ptr != vsl_start + 1) {
+		*vsl_ptr = VSL_WRAPMARKER;
+		vsl_ptr = vsl_start + 1;
+	}
+	VSC_C_main->shm_cycles++;
+}
+
+/*--------------------------------------------------------------------
+ * Reserve bytes for a record, wrap if necessary
+ */
+
+static uint32_t *
+vsl_get(unsigned len, unsigned records, unsigned flushes)
+{
+	uint32_t *p;
+
+	if (pthread_mutex_trylock(&vsl_mtx)) {
+		AZ(pthread_mutex_lock(&vsl_mtx));
+		VSC_C_main->shm_cont++;
+	}
+	assert(vsl_ptr < vsl_end);
+	assert(((uintptr_t)vsl_ptr & 0x3) == 0);
+
+	VSC_C_main->shm_writes++;
+	VSC_C_main->shm_flushes += flushes;
+	VSC_C_main->shm_records += records;
+
+	/* Wrap if necessary */
+	if (VSL_END(vsl_ptr, len) >= vsl_end)
+		vsl_wrap();
+
+	p = vsl_ptr;
+	vsl_ptr = VSL_END(vsl_ptr, len);
+
+	*vsl_ptr = VSL_ENDMARKER;
+
+	assert(vsl_ptr < vsl_end);
+	assert(((uintptr_t)vsl_ptr & 0x3) == 0);
+	AZ(pthread_mutex_unlock(&vsl_mtx));
+
+	return (p);
+}
+
+/*--------------------------------------------------------------------
+ * This variant copies a byte-range directly to the log, without
+ * taking the detour over sprintf()
+ */
+
+static void
+VSLR(enum VSL_tag_e tag, int id, const char *b, unsigned len)
+{
+	uint32_t *p;
+	unsigned mlen;
+
+	mlen = cache_param->shm_reclen;
+
+	/* Truncate */
+	if (len > mlen)
+		len = mlen;
+
+	p = vsl_get(len, 1, 0);
+
+	memcpy(p + 2, b, len);
+	vsl_hdr(tag, p, len, id);
+}
+
+/*--------------------------------------------------------------------*/
+
+void
+VSL(enum VSL_tag_e tag, int id, const char *fmt, ...)
+{
+	va_list ap;
+	unsigned n, mlen = cache_param->shm_reclen;
+	char buf[mlen];
+
+	/*
+	 * XXX: consider formatting into a stack buffer then move into
+	 * XXX: shmlog with VSLR().
+	 */
+	AN(fmt);
+	va_start(ap, fmt);
+
+	if (strchr(fmt, '%') == NULL) {
+		VSLR(tag, id, fmt, strlen(fmt));
+	} else {
+		n = vsnprintf(buf, mlen, fmt, ap);
+		if (n > mlen)
+			n = mlen;
+		VSLR(tag, id, buf, n);
+	}
+	va_end(ap);
+}
+
+/*--------------------------------------------------------------------*/
+
+void
+WSL_Flush(struct worker *w, int overflow)
+{
+	uint32_t *p;
+	unsigned l;
+
+	l = pdiff(w->wlb, w->wlp);
+	if (l == 0)
+		return;
+
+	assert(l >= 8);
+
+	p = vsl_get(l - 8, w->wlr, overflow);
+
+	memcpy(p + 1, w->wlb + 1, l - 4);
+	VWMB();
+	p[0] = w->wlb[0];
+	w->wlp = w->wlb;
+	w->wlr = 0;
+}
+
+/*--------------------------------------------------------------------*/
+
+void
+WSLR(struct worker *w, enum VSL_tag_e tag, int id, txt t)
+{
+	unsigned l, mlen;
+
+	Tcheck(t);
+	mlen = cache_param->shm_reclen;
+
+	/* Truncate */
+	l = Tlen(t);
+	if (l > mlen) {
+		l = mlen;
+		t.e = t.b + l;
+	}
+
+	assert(w->wlp < w->wle);
+
+	/* Wrap if necessary */
+	if (VSL_END(w->wlp, l) >= w->wle)
+		WSL_Flush(w, 1);
+	assert (VSL_END(w->wlp, l) < w->wle);
+	memcpy(VSL_DATA(w->wlp), t.b, l);
+	vsl_hdr(tag, w->wlp, l, id);
+	w->wlp = VSL_END(w->wlp, l);
+	assert(w->wlp < w->wle);
+	w->wlr++;
+	if (cache_param->diag_bitmap & 0x10000)
+		WSL_Flush(w, 0);
+}
+
+/*--------------------------------------------------------------------*/
+
+static void
+wsl(struct worker *w, enum VSL_tag_e tag, int id, const char *fmt, va_list ap)
+{
+	char *p;
+	unsigned n, mlen;
+	txt t;
+
+	AN(fmt);
+	mlen = cache_param->shm_reclen;
+
+	if (strchr(fmt, '%') == NULL) {
+		t.b = TRUST_ME(fmt);
+		t.e = strchr(t.b, '\0');
+		WSLR(w, tag, id, t);
+	} else {
+		assert(w->wlp < w->wle);
+
+		/* Wrap if we cannot fit a full size record */
+		if (VSL_END(w->wlp, mlen) >= w->wle)
+			WSL_Flush(w, 1);
+
+		p = VSL_DATA(w->wlp);
+		n = vsnprintf(p, mlen, fmt, ap);
+		if (n > mlen)
+			n = mlen;	/* we truncate long fields */
+		vsl_hdr(tag, w->wlp, n, id);
+		w->wlp = VSL_END(w->wlp, n);
+		assert(w->wlp < w->wle);
+		w->wlr++;
+	}
+	if (cache_param->diag_bitmap & 0x10000)
+		WSL_Flush(w, 0);
+}
+
+/*--------------------------------------------------------------------*/
+
+void
+WSL(struct worker *w, enum VSL_tag_e tag, int id, const char *fmt, ...)
+{
+	va_list ap;
+
+	CHECK_OBJ_NOTNULL(w, WORKER_MAGIC);
+	AN(fmt);
+	va_start(ap, fmt);
+	wsl(w, tag, id, fmt, ap);
+	va_end(ap);
+}
+
+
+/*--------------------------------------------------------------------*/
+
+void
+WSLB(struct worker *w, enum VSL_tag_e tag, const char *fmt, ...)
+{
+	va_list ap;
+
+	CHECK_OBJ_NOTNULL(w, WORKER_MAGIC);
+	CHECK_OBJ_NOTNULL(w->vbc, VBC_MAGIC);
+	AN(fmt);
+	va_start(ap, fmt);
+	wsl(w, tag, w->vbc->vsl_id, fmt, ap);
+	va_end(ap);
+}
+
+/*--------------------------------------------------------------------*/
+
+void
+VSL_Init(void)
+{
+	struct VSM_chunk *vsc;
+
+	AZ(pthread_mutex_init(&vsl_mtx, NULL));
+	AZ(pthread_mutex_init(&vsm_mtx, NULL));
+
+	VSM__Clean();
+
+	VSM_ITER(vsc)
+		if (!strcmp(vsc->class, VSL_CLASS))
+			break;
+	AN(vsc);
+	vsl_start = VSM_PTR(vsc);
+	vsl_end = VSM_NEXT(vsc);
+	vsl_ptr = vsl_start + 1;
+
+	vsl_wrap();
+	VSM_head->starttime = (intmax_t)VTIM_real();
+	memset(VSM_head->panicstr, '\0', sizeof *VSM_head->panicstr);
+	memset(VSC_C_main, 0, sizeof *VSC_C_main);
+	VSM_head->child_pid = getpid();
+}
+
+/*--------------------------------------------------------------------*/
+
+void *
+VSM_Alloc(unsigned size, const char *class, const char *type,
+    const char *ident)
+{
+	void *p;
+
+	AZ(pthread_mutex_lock(&vsm_mtx));
+	p = VSM__Alloc(size, class, type, ident);
+	AZ(pthread_mutex_unlock(&vsm_mtx));
+	return (p);
+}
+
+void
+VSM_Free(const void *ptr)
+{
+
+	AZ(pthread_mutex_lock(&vsm_mtx));
+	VSM__Free(ptr);
+	AZ(pthread_mutex_unlock(&vsm_mtx));
+}
diff --git a/bin/varnishd/cache/cache_vary.c b/bin/varnishd/cache/cache_vary.c
new file mode 100644
index 0000000..026f937
--- /dev/null
+++ b/bin/varnishd/cache/cache_vary.c
@@ -0,0 +1,257 @@
+/*-
+ * Copyright (c) 2006-2011 Varnish Software AS
+ * All rights reserved.
+ *
+ * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * Do Vary processing.
+ *
+ * When we insert an object into the cache which has a Vary: header,
+ * we encode a vary matching string containing the headers mentioned
+ * and their value.
+ *
+ * When we match an object in the cache, we check the present request
+ * against the vary matching string.
+ *
+ * The only kind of header-munging we do is leading & trailing space
+ * removal.  All the potential "q=foo" gymnastics is not worth the
+ * effort.
+ *
+ * The vary matching string has the following format:
+ *
+ * Sequence of: {
+ *	<msb>			\   Length of header contents.
+ *	<lsb>			/
+ *	<length of header + 1>	\
+ *	<header>		 \  Same format as argument to http_GetHdr()
+ *	':'			 /
+ *	'\0'			/
+ *      <header>		>   Only present if length != 0xffff
+ * }
+ *      '\0'
+ */
+
+#include "config.h"
+
+#include "cache.h"
+
+#include "vct.h"
+#include "vend.h"
+
+struct vsb *
+VRY_Create(const struct sess *sp, const struct http *hp)
+{
+	char *v, *p, *q, *h, *e;
+	struct vsb *sb, *sbh;
+	int l;
+
+	/* No Vary: header, no worries */
+	if (!http_GetHdr(hp, H_Vary, &v))
+		return (NULL);
+
+	/* For vary matching string */
+	sb = VSB_new_auto();
+	AN(sb);
+
+	/* For header matching strings */
+	sbh = VSB_new_auto();
+	AN(sbh);
+
+	if (*v == ':') {
+		WSP(sp, SLT_Error, "Vary header had extra ':', fix backend");
+		v++;
+	}
+	for (p = v; *p; p++) {
+
+		/* Find next header-name */
+		if (vct_issp(*p))
+			continue;
+		for (q = p; *q && !vct_issp(*q) && *q != ','; q++)
+			continue;
+
+		/* Build a header-matching string out of it */
+		VSB_clear(sbh);
+		VSB_printf(sbh, "%c%.*s:%c",
+		    (char)(1 + (q - p)), (int)(q - p), p, 0);
+		AZ(VSB_finish(sbh));
+
+		if (http_GetHdr(sp->http, VSB_data(sbh), &h)) {
+			AZ(vct_issp(*h));
+			/* Trim trailing space */
+			e = strchr(h, '\0');
+			while (e > h && vct_issp(e[-1]))
+				e--;
+			/* Encode two byte length and contents */
+			l = e - h;
+			assert(!(l & ~0xffff));
+		} else {
+			e = h;
+			l = 0xffff;
+		}
+		VSB_printf(sb, "%c%c", (unsigned)l >> 8, l & 0xff);
+		/* Append to vary matching string */
+		VSB_bcat(sb, VSB_data(sbh), VSB_len(sbh));
+		if (e != h)
+			VSB_bcat(sb, h, e - h);
+
+		while (vct_issp(*q))
+			q++;
+		if (*q == '\0')
+			break;
+		xxxassert(*q == ',');
+		p = q;
+	}
+	/* Terminate vary matching string */
+	VSB_printf(sb, "%c%c%c", 0xff, 0xff, 0);
+
+	VSB_delete(sbh);
+	AZ(VSB_finish(sb));
+	return(sb);
+}
+
+/*
+ * Find length of a vary entry
+ */
+static unsigned
+vry_len(const uint8_t *p)
+{
+	unsigned l = vbe16dec(p);
+
+	return (2 + p[2] + 2 + (l == 0xffff ? 0 : l));
+}
+
+/*
+ * Compare two vary entries
+ */
+static int
+vry_cmp(const uint8_t * const *v1, uint8_t * const *v2)
+{
+	unsigned retval = 0;
+
+	if (!memcmp(*v1, *v2, vry_len(*v1))) {
+		/* Same same */
+		retval = 0;
+	} else if (memcmp((*v1) + 2, (*v2) + 2, (*v1)[2] + 2)) {
+		/* Different header */
+		retval = 1;
+	} else if (cache_param->http_gzip_support &&
+	    !strcasecmp(H_Accept_Encoding, (const char*)((*v1)+2))) {
+		/*
+		 * If we do gzip processing, we do not vary on Accept-Encoding,
+		 * because we want everybody to get the gzip'ed object, and
+		 * varnish will gunzip as necessary.  We implement the skip at
+		 * check time, rather than create time, so that object in
+		 * persistent storage can be used with either setting of
+		 * http_gzip_support.
+		 */
+		retval = 0;
+	} else {
+		/* Same header, different content */
+		retval = 2;
+	}
+	return (retval);
+}
+
+int
+VRY_Match(struct sess *sp, const uint8_t *vary)
+{
+	uint8_t *vsp = sp->vary_b;
+	char *h, *e;
+	unsigned lh, ln;
+	int i, retval = 1, oflo = 0;
+
+	AN(vsp);
+	while (vary[2]) {
+		i = vry_cmp(&vary, &vsp);
+		if (i == 1) {
+			/* Build a new entry */
+
+			i = http_GetHdr(sp->http, (const char*)(vary+2), &h);
+			if (i) {
+				/* Trim trailing space */
+				e = strchr(h, '\0');
+				while (e > h && vct_issp(e[-1]))
+					e--;
+				lh = e - h;
+				assert(lh < 0xffff);
+			} else {
+				e = h = NULL;
+				lh = 0xffff;
+			}
+
+			/* Length of the entire new vary entry */
+			ln = 2 + vary[2] + 2 + (lh == 0xffff ? 0 : lh);
+			if (vsp + ln >= sp->vary_e) {
+				vsp = sp->vary_b;
+				oflo = 1;
+			}
+
+			/*
+			 * We MUST have space for one entry and the end marker
+			 * after it, which prevents old junk from confusing us
+			 */
+			assert(vsp + ln + 2 < sp->vary_e);
+
+			vbe16enc(vsp, (uint16_t)lh);
+			memcpy(vsp + 2, vary + 2, vary[2] + 2);
+			if (h != NULL && e != NULL) {
+				memcpy(vsp + 2 + vsp[2] + 2, h, e - h);
+				vsp[2 + vary[2] + 2 + (e - h) + 2] = '\0';
+			} else
+				vsp[2 + vary[2] + 2 + 2] = '\0';
+
+			i = vry_cmp(&vary, &vsp);
+			assert(i != 1);	/* hdr must be the same now */
+		}
+		if (i != 0)
+			retval = 0;
+		vsp += vry_len(vsp);
+		vary += vry_len(vary);
+	}
+	if (vsp + 3 > sp->vary_e)
+		oflo = 1;
+
+	if (oflo) {
+		/* XXX: Should log this */
+		vsp = sp->vary_b;
+	}
+	vsp[0] = 0xff;
+	vsp[1] = 0xff;
+	vsp[2] = 0;
+	if (oflo)
+		sp->vary_l = NULL;
+	else
+		sp->vary_l = vsp + 3;
+	return (retval);
+}
+
+void
+VRY_Validate(const uint8_t *vary)
+{
+
+	while (vary[2] != 0) {
+		assert(strlen((const char*)vary+3) == vary[2]);
+		vary += vry_len(vary);
+	}
+}
diff --git a/bin/varnishd/cache/cache_vcl.c b/bin/varnishd/cache/cache_vcl.c
new file mode 100644
index 0000000..068b482
--- /dev/null
+++ b/bin/varnishd/cache/cache_vcl.c
@@ -0,0 +1,365 @@
+/*-
+ * Copyright (c) 2006 Verdens Gang AS
+ * Copyright (c) 2006-2011 Varnish Software AS
+ * All rights reserved.
+ *
+ * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * Interface *to* compiled VCL code:  Loading, unloading, calling into etc.
+ *
+ * The interface *from* the compiled VCL code is in cache_vrt.c.
+ */
+
+#include "config.h"
+
+#include <dlfcn.h>
+#include <stdlib.h>
+
+#include "cache.h"
+
+#include "vcl.h"
+#include "vcli.h"
+#include "vcli_priv.h"
+
+struct vcls {
+	unsigned		magic;
+#define VVCLS_MAGIC		0x214188f2
+	VTAILQ_ENTRY(vcls)	list;
+	char			*name;
+	void			*dlh;
+	struct VCL_conf		conf[1];
+};
+
+/*
+ * XXX: Presently all modifications to this list happen from the
+ * CLI event-engine, so no locking is necessary
+ */
+static VTAILQ_HEAD(, vcls)	vcl_head =
+    VTAILQ_HEAD_INITIALIZER(vcl_head);
+
+
+static struct lock		vcl_mtx;
+static struct vcls		*vcl_active; /* protected by vcl_mtx */
+
+/*--------------------------------------------------------------------*/
+
+const char *
+VCL_Return_Name(unsigned method)
+{
+
+	switch (method) {
+#define VCL_RET_MAC(l, U, B) case VCL_RET_##U: return(#l);
+#include "tbl/vcl_returns.h"
+#undef VCL_RET_MAC
+	default:
+		return (NULL);
+	}
+}
+
+/*--------------------------------------------------------------------*/
+
+static void
+VCL_Get(struct VCL_conf **vcc)
+{
+	static int once = 0;
+
+	while (!once && vcl_active == NULL) {
+		(void)sleep(1);
+	}
+	once = 1;
+
+	Lck_Lock(&vcl_mtx);
+	AN(vcl_active);
+	*vcc = vcl_active->conf;
+	AN(*vcc);
+	AZ((*vcc)->discard);
+	(*vcc)->busy++;
+	Lck_Unlock(&vcl_mtx);
+}
+
+void
+VCL_Refresh(struct VCL_conf **vcc)
+{
+	if (*vcc == vcl_active->conf)
+		return;
+	if (*vcc != NULL)
+		VCL_Rel(vcc);	/* XXX: optimize locking */
+	VCL_Get(vcc);
+}
+
+void
+VCL_Rel(struct VCL_conf **vcc)
+{
+	struct VCL_conf *vc;
+
+	AN(*vcc);
+	vc = *vcc;
+	*vcc = NULL;
+
+	Lck_Lock(&vcl_mtx);
+	assert(vc->busy > 0);
+	vc->busy--;
+	/*
+	 * We do not garbage collect discarded VCL's here, that happens
+	 * in VCL_Poll() which is called from the CLI thread.
+	 */
+	Lck_Unlock(&vcl_mtx);
+}
+
+/*--------------------------------------------------------------------*/
+
+static struct vcls *
+vcl_find(const char *name)
+{
+	struct vcls *vcl;
+
+	ASSERT_CLI();
+	VTAILQ_FOREACH(vcl, &vcl_head, list) {
+		if (vcl->conf->discard)
+			continue;
+		if (!strcmp(vcl->name, name))
+			return (vcl);
+	}
+	return (NULL);
+}
+
+static int
+VCL_Load(const char *fn, const char *name, struct cli *cli)
+{
+	struct vcls *vcl;
+	struct VCL_conf const *cnf;
+
+	ASSERT_CLI();
+	vcl = vcl_find(name);
+	if (vcl != NULL) {
+		VCLI_Out(cli, "Config '%s' already loaded", name);
+		return (1);
+	}
+
+	ALLOC_OBJ(vcl, VVCLS_MAGIC);
+	XXXAN(vcl);
+
+	vcl->dlh = dlopen(fn, RTLD_NOW | RTLD_LOCAL);
+
+	if (vcl->dlh == NULL) {
+		VCLI_Out(cli, "dlopen(%s): %s\n", fn, dlerror());
+		FREE_OBJ(vcl);
+		return (1);
+	}
+	cnf = dlsym(vcl->dlh, "VCL_conf");
+	if (cnf == NULL) {
+		VCLI_Out(cli, "Internal error: No VCL_conf symbol\n");
+		(void)dlclose(vcl->dlh);
+		FREE_OBJ(vcl);
+		return (1);
+	}
+	memcpy(vcl->conf, cnf, sizeof *cnf);
+
+	if (vcl->conf->magic != VCL_CONF_MAGIC) {
+		VCLI_Out(cli, "Wrong VCL_CONF_MAGIC\n");
+		(void)dlclose(vcl->dlh);
+		FREE_OBJ(vcl);
+		return (1);
+	}
+	if (vcl->conf->init_vcl(cli)) {
+		VCLI_Out(cli, "VCL \"%s\" Failed to initialize", name);
+		(void)dlclose(vcl->dlh);
+		FREE_OBJ(vcl);
+		return (1);
+	}
+	REPLACE(vcl->name, name);
+	VCLI_Out(cli, "Loaded \"%s\" as \"%s\"", fn , name);
+	VTAILQ_INSERT_TAIL(&vcl_head, vcl, list);
+	(void)vcl->conf->init_func(NULL);
+	Lck_Lock(&vcl_mtx);
+	if (vcl_active == NULL)
+		vcl_active = vcl;
+	Lck_Unlock(&vcl_mtx);
+	VSC_C_main->n_vcl++;
+	VSC_C_main->n_vcl_avail++;
+	return (0);
+}
+
+/*--------------------------------------------------------------------
+ * This function is polled from the CLI thread to dispose of any non-busy
+ * VCLs which have been discarded.
+ */
+
+static void
+VCL_Nuke(struct vcls *vcl)
+{
+
+	ASSERT_CLI();
+	assert(vcl != vcl_active);
+	assert(vcl->conf->discard);
+	assert(vcl->conf->busy == 0);
+	VTAILQ_REMOVE(&vcl_head, vcl, list);
+	(void)vcl->conf->fini_func(NULL);
+	vcl->conf->fini_vcl(NULL);
+	free(vcl->name);
+	(void)dlclose(vcl->dlh);
+	FREE_OBJ(vcl);
+	VSC_C_main->n_vcl--;
+	VSC_C_main->n_vcl_discard--;
+}
+
+/*--------------------------------------------------------------------*/
+
+void
+VCL_Poll(void)
+{
+	struct vcls *vcl, *vcl2;
+
+	ASSERT_CLI();
+	VTAILQ_FOREACH_SAFE(vcl, &vcl_head, list, vcl2)
+		if (vcl->conf->discard && vcl->conf->busy == 0)
+			VCL_Nuke(vcl);
+}
+
+/*--------------------------------------------------------------------*/
+
+static void
+ccf_config_list(struct cli *cli, const char * const *av, void *priv)
+{
+	struct vcls *vcl;
+	const char *flg;
+
+	(void)av;
+	(void)priv;
+	ASSERT_CLI();
+	VTAILQ_FOREACH(vcl, &vcl_head, list) {
+		if (vcl == vcl_active) {
+			flg = "active";
+		} else if (vcl->conf->discard) {
+			flg = "discarded";
+		} else
+			flg = "available";
+		VCLI_Out(cli, "%-10s %6u %s\n",
+		    flg,
+		    vcl->conf->busy,
+		    vcl->name);
+	}
+}
+
+static void
+ccf_config_load(struct cli *cli, const char * const *av, void *priv)
+{
+
+	(void)av;
+	(void)priv;
+	ASSERT_CLI();
+	if (VCL_Load(av[3], av[2], cli))
+		VCLI_SetResult(cli, CLIS_PARAM);
+	return;
+}
+
+static void
+ccf_config_discard(struct cli *cli, const char * const *av, void *priv)
+{
+	struct vcls *vcl;
+
+	ASSERT_CLI();
+	(void)av;
+	(void)priv;
+	vcl = vcl_find(av[2]);
+	if (vcl == NULL) {
+		VCLI_SetResult(cli, CLIS_PARAM);
+		VCLI_Out(cli, "VCL '%s' unknown", av[2]);
+		return;
+	}
+	Lck_Lock(&vcl_mtx);
+	if (vcl == vcl_active) {
+		Lck_Unlock(&vcl_mtx);
+		VCLI_SetResult(cli, CLIS_PARAM);
+		VCLI_Out(cli, "VCL %s is the active VCL", av[2]);
+		return;
+	}
+	VSC_C_main->n_vcl_discard++;
+	VSC_C_main->n_vcl_avail--;
+	vcl->conf->discard = 1;
+	Lck_Unlock(&vcl_mtx);
+	if (vcl->conf->busy == 0)
+		VCL_Nuke(vcl);
+}
+
+static void
+ccf_config_use(struct cli *cli, const char * const *av, void *priv)
+{
+	struct vcls *vcl;
+	int i;
+
+	(void)av;
+	(void)priv;
+	vcl = vcl_find(av[2]);
+	if (vcl == NULL) {
+		VCLI_Out(cli, "No VCL named '%s'", av[2]);
+		VCLI_SetResult(cli, CLIS_PARAM);
+		return;
+	}
+	Lck_Lock(&vcl_mtx);
+	vcl_active = vcl;
+	Lck_Unlock(&vcl_mtx);
+
+	/* Tickle this VCL's backends to take over health polling */
+	for(i = 1; i < vcl->conf->ndirector; i++)
+		VBE_UseHealth(vcl->conf->director[i]);
+}
+
+/*--------------------------------------------------------------------*/
+
+#define VCL_MET_MAC(func, upper, bitmap)				\
+void									\
+VCL_##func##_method(struct sess *sp)					\
+{									\
+									\
+	sp->handling = 0;						\
+	sp->cur_method = VCL_MET_ ## upper;				\
+	WSP(sp, SLT_VCL_call, "%s", #func);				\
+	(void)sp->vcl->func##_func(sp);					\
+	WSP(sp, SLT_VCL_return, "%s", VCL_Return_Name(sp->handling));	\
+	sp->cur_method = 0;						\
+	assert((1U << sp->handling) & bitmap);				\
+	assert(!((1U << sp->handling) & ~bitmap));			\
+}
+
+#include "tbl/vcl_returns.h"
+#undef VCL_MET_MAC
+
+/*--------------------------------------------------------------------*/
+
+static struct cli_proto vcl_cmds[] = {
+	{ CLI_VCL_LOAD,         "i", ccf_config_load },
+	{ CLI_VCL_LIST,         "i", ccf_config_list },
+	{ CLI_VCL_DISCARD,      "i", ccf_config_discard },
+	{ CLI_VCL_USE,          "i", ccf_config_use },
+	{ NULL }
+};
+
+void
+VCL_Init()
+{
+
+	CLI_AddFuncs(vcl_cmds);
+	Lck_New(&vcl_mtx, lck_vcl);
+}
diff --git a/bin/varnishd/cache/cache_vrt.c b/bin/varnishd/cache/cache_vrt.c
new file mode 100644
index 0000000..c20b552
--- /dev/null
+++ b/bin/varnishd/cache/cache_vrt.c
@@ -0,0 +1,535 @@
+/*-
+ * Copyright (c) 2006 Verdens Gang AS
+ * Copyright (c) 2006-2010 Varnish Software AS
+ * All rights reserved.
+ *
+ * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * Runtime support for compiled VCL programs
+ */
+
+#include "config.h"
+
+#include <netinet/in.h>
+#include <arpa/inet.h>
+
+#include <stdio.h>
+#include <stdlib.h>
+
+#include "cache.h"
+
+#include "cache_backend.h"
+#include "hash/hash_slinger.h"
+#include "vav.h"
+#include "vcl.h"
+#include "vrt.h"
+#include "vrt_obj.h"
+#include "vtim.h"
+
+const void * const vrt_magic_string_end = &vrt_magic_string_end;
+
+/*--------------------------------------------------------------------*/
+
+void
+VRT_error(struct sess *sp, unsigned code, const char *reason)
+{
+
+	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+	WSL(sp->wrk, SLT_Debug, 0, "VCL_error(%u, %s)", code, reason ?
+	    reason : "(null)");
+	if (code < 100 || code > 999)
+		code = 503;
+	sp->err_code = (uint16_t)code;
+	sp->err_reason = reason ? reason : http_StatusMessage(sp->err_code);
+}
+
+/*--------------------------------------------------------------------*/
+
+void
+VRT_count(const struct sess *sp, unsigned u)
+{
+
+	if (sp == NULL)
+		return;
+	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+	if (cache_param->vcl_trace)
+		WSP(sp, SLT_VCL_trace, "%u %d.%d", u,
+		    sp->vcl->ref[u].line, sp->vcl->ref[u].pos);
+}
+
+/*--------------------------------------------------------------------*/
+
+void
+VRT_acl_log(const struct sess *sp, const char *msg)
+{
+	WSP(sp, SLT_VCL_acl, msg);
+}
+
+/*--------------------------------------------------------------------*/
+
+static struct http *
+vrt_selecthttp(const struct sess *sp, enum gethdr_e where)
+{
+	struct http *hp;
+
+	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+	switch (where) {
+	case HDR_REQ:
+		hp = sp->http;
+		break;
+	case HDR_BEREQ:
+		hp = sp->wrk->bereq;
+		break;
+	case HDR_BERESP:
+		hp = sp->wrk->beresp;
+		break;
+	case HDR_RESP:
+		hp = sp->wrk->resp;
+		break;
+	case HDR_OBJ:
+		CHECK_OBJ_NOTNULL(sp->obj, OBJECT_MAGIC);
+		hp = sp->obj->http;
+		break;
+	default:
+		INCOMPL();
+	}
+	CHECK_OBJ_NOTNULL(hp, HTTP_MAGIC);
+	return (hp);
+}
+
+char *
+VRT_GetHdr(const struct sess *sp, enum gethdr_e where, const char *n)
+{
+	char *p;
+	struct http *hp;
+
+	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+	hp = vrt_selecthttp(sp, where);
+	if (!http_GetHdr(hp, n, &p))
+		return (NULL);
+	return (p);
+}
+
+/*--------------------------------------------------------------------
+ * XXX: Optimize the single element case ?
+ */
+
+char *
+VRT_StringList(char *d, unsigned dl, const char *p, va_list ap)
+{
+	char *b, *e;
+	unsigned x;
+
+	b = d;
+	e = b + dl;
+	while (p != vrt_magic_string_end && b < e) {
+		if (p != NULL) {
+			x = strlen(p);
+			if (b + x < e)
+				memcpy(b, p, x);
+			b += x;
+		}
+		p = va_arg(ap, const char *);
+	}
+	if (b >= e)
+		return (NULL);
+	*b++ = '\0';
+	return (b);
+}
+
+/*--------------------------------------------------------------------
+ * XXX: Optimize the single element case ?
+ */
+
+char *
+VRT_String(struct ws *ws, const char *h, const char *p, va_list ap)
+{
+	char *b, *e;
+	unsigned u, x;
+
+	u = WS_Reserve(ws, 0);
+	e = b = ws->f;
+	e += u;
+	if (h != NULL) {
+		x = strlen(h);
+		if (b + x < e)
+			memcpy(b, h, x);
+		b += x;
+		if (b < e)
+			*b = ' ';
+		b++;
+	}
+	b = VRT_StringList(b, e > b ? e - b : 0, p, ap);
+	if (b == NULL || b == e) {
+		WS_Release(ws, 0);
+		return (NULL);
+	}
+	e = b;
+	b = ws->f;
+	WS_Release(ws, e - b);
+	return (b);
+}
+
+/*--------------------------------------------------------------------
+ * Build a string on the worker threads workspace
+ */
+
+const char *
+VRT_WrkString(const struct sess *sp, const char *p, ...)
+{
+	va_list ap;
+	char *b;
+
+	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+	va_start(ap, p);
+	b = VRT_String(sp->wrk->ws, NULL, p, ap);
+	va_end(ap);
+	return (b);
+}
+
+/*--------------------------------------------------------------------*/
+
+void
+VRT_SetHdr(const struct sess *sp , enum gethdr_e where, const char *hdr,
+    const char *p, ...)
+{
+	struct http *hp;
+	va_list ap;
+	char *b;
+
+	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+	hp = vrt_selecthttp(sp, where);
+	va_start(ap, p);
+	if (p == NULL) {
+		http_Unset(hp, hdr);
+	} else {
+		b = VRT_String(hp->ws, hdr + 1, p, ap);
+		if (b == NULL) {
+			WSP(sp, SLT_LostHeader, "%s", hdr + 1);
+		} else {
+			http_Unset(hp, hdr);
+			http_SetHeader(sp->wrk, sp->vsl_id, hp, b);
+		}
+	}
+	va_end(ap);
+}
+
+/*--------------------------------------------------------------------*/
+
+void
+VRT_handling(struct sess *sp, unsigned hand)
+{
+
+	if (sp == NULL) {
+		assert(hand == VCL_RET_OK);
+		return;
+	}
+	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+	assert(hand < VCL_RET_MAX);
+	sp->handling = hand;
+}
+
+/*--------------------------------------------------------------------
+ * Add an element to the array/list of hash bits.
+ */
+
+void
+VRT_hashdata(const struct sess *sp, const char *str, ...)
+{
+	va_list ap;
+	const char *p;
+
+	HSH_AddString(sp, str);
+	va_start(ap, str);
+	while (1) {
+		p = va_arg(ap, const char *);
+		if (p == vrt_magic_string_end)
+			break;
+		HSH_AddString(sp, p);
+	}
+}
+
+/*--------------------------------------------------------------------*/
+
+double
+VRT_r_now(const struct sess *sp)
+{
+
+	(void)sp;
+	return (VTIM_real());
+}
+
+/*--------------------------------------------------------------------*/
+
+char *
+VRT_IP_string(const struct sess *sp, const struct sockaddr_storage *sa)
+{
+	char *p;
+	const struct sockaddr_in *si4;
+	const struct sockaddr_in6 *si6;
+	const void *addr;
+	int len;
+
+	switch (sa->ss_family) {
+	case AF_INET:
+		len = INET_ADDRSTRLEN;
+		si4 = (const void *)sa;
+		addr = &(si4->sin_addr);
+		break;
+	case AF_INET6:
+		len = INET6_ADDRSTRLEN;
+		si6 = (const void *)sa;
+		addr = &(si6->sin6_addr);
+		break;
+	default:
+		INCOMPL();
+	}
+	XXXAN(len);
+	AN(p = WS_Alloc(sp->http->ws, len));
+	AN(inet_ntop(sa->ss_family, addr, p, len));
+	return (p);
+}
+
+char *
+VRT_int_string(const struct sess *sp, int num)
+{
+	char *p;
+	int size;
+
+	size = snprintf(NULL, 0, "%d", num) + 1;
+	AN(p = WS_Alloc(sp->http->ws, size));
+	assert(snprintf(p, size, "%d", num) < size);
+	return (p);
+}
+
+char *
+VRT_double_string(const struct sess *sp, double num)
+{
+	char *p;
+	int size;
+
+	size = snprintf(NULL, 0, "%.3f", num) + 1;
+	AN(p = WS_Alloc(sp->http->ws, size));
+	assert(snprintf(p, size, "%.3f", num) < size);
+	return (p);
+}
+
+char *
+VRT_time_string(const struct sess *sp, double t)
+{
+	char *p;
+
+	AN(p = WS_Alloc(sp->http->ws, VTIM_FORMAT_SIZE));
+	VTIM_format(t, p);
+	return (p);
+}
+
+const char *
+VRT_backend_string(const struct sess *sp, const struct director *d)
+{
+	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+	if (d == NULL)
+		d = sp->director;
+	if (d == NULL)
+		return (NULL);
+	return (d->vcl_name);
+}
+
+const char *
+VRT_bool_string(const struct sess *sp, unsigned val)
+{
+
+	(void)sp;
+	return (val ? "true" : "false");
+}
+
+/*--------------------------------------------------------------------*/
+
+void
+VRT_Rollback(struct sess *sp)
+{
+
+	HTTP_Copy(sp->http, sp->http0);
+	WS_Reset(sp->ws, sp->ws_req);
+}
+
+/*--------------------------------------------------------------------*/
+
+void
+VRT_panic(const struct sess *sp, const char *str, ...)
+{
+	va_list ap;
+	char *b;
+
+	va_start(ap, str);
+	b = VRT_String(sp->http->ws, "PANIC: ", str, ap);
+	va_end(ap);
+	VAS_Fail("VCL", "", 0, b, 0, 2);
+}
+
+/*--------------------------------------------------------------------*/
+
+void
+VRT_synth_page(const struct sess *sp, unsigned flags, const char *str, ...)
+{
+	va_list ap;
+	const char *p;
+	struct vsb *vsb;
+
+	(void)flags;
+	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+	CHECK_OBJ_NOTNULL(sp->obj, OBJECT_MAGIC);
+	vsb = SMS_Makesynth(sp->obj);
+	AN(vsb);
+
+	VSB_cat(vsb, str);
+	va_start(ap, str);
+	p = va_arg(ap, const char *);
+	while (p != vrt_magic_string_end) {
+		if (p == NULL)
+			p = "(null)";
+		VSB_cat(vsb, p);
+		p = va_arg(ap, const char *);
+	}
+	va_end(ap);
+	SMS_Finish(sp->obj);
+	http_Unset(sp->obj->http, H_Content_Length);
+	http_PrintfHeader(sp->wrk, sp->vsl_id, sp->obj->http,
+	    "Content-Length: %d", sp->obj->len);
+}
+
+/*--------------------------------------------------------------------*/
+
+void
+VRT_ban(struct sess *sp, char *cmds, ...)
+{
+	char *a1, *a2, *a3;
+	va_list ap;
+	struct ban *b;
+	int good;
+
+	(void)sp;
+	b = BAN_New();
+	va_start(ap, cmds);
+	a1 = cmds;
+	good = 0;
+	while (a1 != NULL) {
+		good = 0;
+		a2 = va_arg(ap, char *);
+		if (a2 == NULL)
+			break;
+		a3 = va_arg(ap, char *);
+		if (a3 == NULL)
+			break;
+		if (BAN_AddTest(NULL, b, a1, a2, a3))
+			break;
+		a1 = va_arg(ap, char *);
+		good = 1;
+	}
+	if (!good)
+		/* XXX: report error how ? */
+		BAN_Free(b);
+	else
+		BAN_Insert(b);
+}
+
+/*--------------------------------------------------------------------*/
+
+void
+VRT_ban_string(struct sess *sp, const char *str)
+{
+	char *a1, *a2, *a3;
+	char **av;
+	struct ban *b;
+	int good;
+	int i;
+
+	(void)sp;
+	av = VAV_Parse(str, NULL, ARGV_NOESC);
+	if (av[0] != NULL) {
+		/* XXX: report error how ? */
+		VAV_Free(av);
+		return;
+	}
+	b = BAN_New();
+	good = 0;
+	for (i = 1; ;) {
+		a1 = av[i++];
+		if (a1 == NULL)
+			break;
+		good = 0;
+		a2 = av[i++];
+		if (a2 == NULL)
+			break;
+		a3 = av[i++];
+		if (a3 == NULL)
+			break;
+		if (BAN_AddTest(NULL, b, a1, a2, a3))
+			break;
+		good = 1;
+		if (av[i] == NULL)
+			break;
+		good = 0;
+		if (strcmp(av[i++], "&&"))
+			break;
+	}
+	if (!good)
+		/* XXX: report error how ? */
+		BAN_Free(b);
+	else
+		BAN_Insert(b);
+	VAV_Free(av);
+}
+
+/*--------------------------------------------------------------------
+ * "real" purges
+ */
+
+void
+VRT_purge(const struct sess *sp, double ttl, double grace)
+{
+	if (sp->cur_method == VCL_MET_HIT)
+		HSH_Purge(sp, sp->obj->objcore->objhead, ttl, grace);
+	else if (sp->cur_method == VCL_MET_MISS)
+		HSH_Purge(sp, sp->objcore->objhead, ttl, grace);
+}
+
+/*--------------------------------------------------------------------
+ * Simple stuff
+ */
+
+int
+VRT_strcmp(const char *s1, const char *s2)
+{
+	if (s1 == NULL || s2 == NULL)
+		return(1);
+	return (strcmp(s1, s2));
+}
+
+void
+VRT_memmove(void *dst, const void *src, unsigned len)
+{
+
+	(void)memmove(dst, src, len);
+}
diff --git a/bin/varnishd/cache/cache_vrt_re.c b/bin/varnishd/cache/cache_vrt_re.c
new file mode 100644
index 0000000..7759d0a
--- /dev/null
+++ b/bin/varnishd/cache/cache_vrt_re.c
@@ -0,0 +1,162 @@
+/*-
+ * Copyright (c) 2006 Verdens Gang AS
+ * Copyright (c) 2006-2009 Varnish Software AS
+ * All rights reserved.
+ *
+ * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * Runtime support for compiled VCL programs, regexps
+ */
+
+#include "config.h"
+
+#include <ctype.h>
+
+#include "cache.h"
+
+#include "vre.h"
+#include "vrt.h"
+
+void
+VRT_re_init(void **rep, const char *re)
+{
+	vre_t *t;
+	const char *error;
+	int erroroffset;
+
+	/* This was already check-compiled by the VCL compiler */
+	t = VRE_compile(re, 0, &error, &erroroffset);
+	AN(t);
+	*rep = t;
+}
+
+void
+VRT_re_fini(void *rep)
+{
+	vre_t *vv;
+
+	vv = rep;
+	if (rep != NULL)
+		VRE_free(&vv);
+}
+
+int
+VRT_re_match(const struct sess *sp, const char *s, void *re)
+{
+	vre_t *t;
+	int i;
+
+	if (s == NULL)
+		s = "";
+	AN(re);
+	t = re;
+	i = VRE_exec(t, s, strlen(s), 0, 0, NULL, 0, &cache_param->vre_limits);
+	if (i >= 0)
+		return (1);
+	if (i < VRE_ERROR_NOMATCH )
+		WSP(sp, SLT_VCL_Error, "Regexp matching returned %d", i);
+	return (0);
+}
+
+const char *
+VRT_regsub(const struct sess *sp, int all, const char *str, void *re,
+    const char *sub)
+{
+	int ovector[30];
+	vre_t *t;
+	int i, l;
+	txt res;
+	char *b0;
+	const char *s;
+	unsigned u, x;
+	int options = 0;
+	size_t len;
+
+	AN(re);
+	if (str == NULL)
+		str = "";
+	t = re;
+	memset(ovector, 0, sizeof(ovector));
+	len = strlen(str);
+	i = VRE_exec(t, str, len, 0, options, ovector, 30,
+	    &cache_param->vre_limits);
+
+	/* If it didn't match, we can return the original string */
+	if (i == VRE_ERROR_NOMATCH)
+		return(str);
+	if (i < VRE_ERROR_NOMATCH ) {
+		WSP(sp, SLT_VCL_Error, "Regexp matching returned %d", i);
+		return(str);
+	}
+
+	u = WS_Reserve(sp->http->ws, 0);
+	res.e = res.b = b0 = sp->http->ws->f;
+	res.e += u;
+
+	do {
+		/* Copy prefix to match */
+		Tadd(&res, str, ovector[0]);
+		for (s = sub ; *s != '\0'; s++ ) {
+			if (*s != '\\' || s[1] == '\0') {
+				if (res.b < res.e)
+					*res.b++ = *s;
+				continue;
+			}
+			s++;
+			if (isdigit(*s)) {
+				x = *s - '0';
+				l = ovector[2*x+1] - ovector[2*x];
+				Tadd(&res, str + ovector[2*x], l);
+				continue;
+			} else {
+				if (res.b < res.e)
+					*res.b++ = *s;
+			}
+		}
+		str += ovector[1];
+		len -= ovector[1];
+		if (!all)
+			break;
+		memset(&ovector, 0, sizeof(ovector));
+		options |= VRE_NOTEMPTY_ATSTART;
+		i = VRE_exec(t, str, len, 0, options, ovector, 30,
+		    &cache_param->vre_limits);
+		if (i < VRE_ERROR_NOMATCH ) {
+			WS_Release(sp->http->ws, 0);
+			WSP(sp, SLT_VCL_Error,
+			    "Regexp matching returned %d", i);
+			return(str);
+		}
+	} while (i != VRE_ERROR_NOMATCH);
+
+	/* Copy suffix to match */
+	Tadd(&res, str, len+1);
+	if (res.b >= res.e) {
+		WS_Release(sp->http->ws, 0);
+		return (str);
+	}
+	Tcheck(res);
+	WS_ReleaseP(sp->http->ws, res.b);
+	return (b0);
+}
diff --git a/bin/varnishd/cache/cache_vrt_var.c b/bin/varnishd/cache/cache_vrt_var.c
new file mode 100644
index 0000000..860c7aa
--- /dev/null
+++ b/bin/varnishd/cache/cache_vrt_var.c
@@ -0,0 +1,550 @@
+/*-
+ * Copyright (c) 2006 Verdens Gang AS
+ * Copyright (c) 2006-2011 Varnish Software AS
+ * All rights reserved.
+ *
+ * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * Runtime support for compiled VCL programs
+ */
+#include "config.h"
+
+#include <stdio.h>
+#include <stdlib.h>
+
+#include "cache.h"
+#include "common/heritage.h"
+
+#include "cache_backend.h"
+#include "vrt_obj.h"
+#include "vtcp.h"
+#include "vtim.h"
+
+static char vrt_hostname[255] = "";
+
+/*--------------------------------------------------------------------*/
+
+static void
+vrt_do_string(struct worker *w, int fd, const struct http *hp, int fld,
+    const char *err, const char *p, va_list ap)
+{
+	char *b;
+
+	// AN(p);
+	AN(hp);
+	b = VRT_String(hp->ws, NULL, p, ap);
+	if (b == NULL || *b == '\0') {
+		WSL(w, SLT_LostHeader, fd, err);
+	} else {
+		http_SetH(hp, fld, b);
+	}
+	va_end(ap);
+}
+
+#define VRT_DO_HDR(obj, hdr, http, fld)				\
+void								\
+VRT_l_##obj##_##hdr(const struct sess *sp, const char *p, ...)	\
+{								\
+	va_list ap;						\
+								\
+	va_start(ap, p);					\
+	vrt_do_string(sp->wrk, sp->fd,				\
+	    http, fld, #obj "." #hdr, p, ap);			\
+	va_end(ap);						\
+}								\
+								\
+const char *							\
+VRT_r_##obj##_##hdr(const struct sess *sp)			\
+{								\
+	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);			\
+	CHECK_OBJ_NOTNULL(http, HTTP_MAGIC);			\
+	return (http->hd[fld].b);				\
+}
+
+VRT_DO_HDR(req,   request,	sp->http,		HTTP_HDR_REQ)
+VRT_DO_HDR(req,   url,		sp->http,		HTTP_HDR_URL)
+VRT_DO_HDR(req,   proto,	sp->http,		HTTP_HDR_PROTO)
+VRT_DO_HDR(bereq, request,	sp->wrk->bereq,		HTTP_HDR_REQ)
+VRT_DO_HDR(bereq, url,		sp->wrk->bereq,		HTTP_HDR_URL)
+VRT_DO_HDR(bereq, proto,	sp->wrk->bereq,		HTTP_HDR_PROTO)
+VRT_DO_HDR(obj,   proto,	sp->obj->http,		HTTP_HDR_PROTO)
+VRT_DO_HDR(obj,   response,	sp->obj->http,		HTTP_HDR_RESPONSE)
+VRT_DO_HDR(resp,  proto,	sp->wrk->resp,		HTTP_HDR_PROTO)
+VRT_DO_HDR(resp,  response,	sp->wrk->resp,		HTTP_HDR_RESPONSE)
+VRT_DO_HDR(beresp,  proto,	sp->wrk->beresp,	HTTP_HDR_PROTO)
+VRT_DO_HDR(beresp,  response,	sp->wrk->beresp,	HTTP_HDR_RESPONSE)
+
+/*--------------------------------------------------------------------*/
+
+#define VRT_DO_STATUS(obj, http)				\
+void								\
+VRT_l_##obj##_status(const struct sess *sp, int num)		\
+{								\
+								\
+	assert(num >= 100 && num <= 999);			\
+	http->status = (uint16_t)num;				\
+}								\
+								\
+int								\
+VRT_r_##obj##_status(const struct sess *sp)			\
+{								\
+								\
+	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);			\
+	return(http->status);					\
+}
+
+VRT_DO_STATUS(obj, sp->obj->http)
+VRT_DO_STATUS(beresp, sp->wrk->beresp)
+VRT_DO_STATUS(resp, sp->wrk->resp)
+
+/*--------------------------------------------------------------------*/
+
+/* XXX: review this */
+/* Add an objecthead to the saintmode list for the (hopefully) relevant
+ * backend. Some double-up asserting here to avoid assert-errors when there
+ * is no object.
+ */
+void
+VRT_l_beresp_saintmode(const struct sess *sp, double a)
+{
+	struct trouble *new;
+	struct trouble *tr;
+	struct trouble *tr2;
+	struct worker *wrk;
+
+	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+	wrk = sp->wrk;
+	if (!wrk->vbc)
+		return;
+	CHECK_OBJ_NOTNULL(wrk->vbc, VBC_MAGIC);
+	if (!wrk->vbc->backend)
+		return;
+	CHECK_OBJ_NOTNULL(wrk->vbc->backend, BACKEND_MAGIC);
+	if (!sp->objcore)
+		return;
+	CHECK_OBJ_NOTNULL(sp->objcore, OBJCORE_MAGIC);
+
+	/* Setting a negative holdoff period is a mistake. Detecting this
+	 * when compiling the VCL would be better.
+	 */
+	assert(a > 0);
+
+	ALLOC_OBJ(new, TROUBLE_MAGIC);
+	AN(new);
+	new->target = (uintptr_t)(sp->objcore->objhead);
+	new->timeout = sp->t_req + a;
+
+	/* Insert the new item on the list before the first item with a
+	 * timeout at a later date (ie: sort by which entry will time out
+	 * from the list
+	 */
+	Lck_Lock(&wrk->vbc->backend->mtx);
+	VTAILQ_FOREACH_SAFE(tr, &wrk->vbc->backend->troublelist, list, tr2) {
+		if (tr->timeout < new->timeout) {
+			VTAILQ_INSERT_BEFORE(tr, new, list);
+			new = NULL;
+			break;
+		}
+	}
+
+	/* Insert the item at the end if the list is empty or all other
+	 * items have a longer timeout.
+	 */
+	if (new)
+		VTAILQ_INSERT_TAIL(&wrk->vbc->backend->troublelist, new, list);
+
+	Lck_Unlock(&wrk->vbc->backend->mtx);
+}
+
+/*--------------------------------------------------------------------*/
+
+#define VBERESP(dir, type, onm, field)					\
+void									\
+VRT_l_##dir##_##onm(const struct sess *sp, type a)			\
+{									\
+	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);				\
+	sp->wrk->field = a;						\
+}									\
+									\
+type									\
+VRT_r_##dir##_##onm(const struct sess *sp)				\
+{									\
+	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);				\
+	return (sp->wrk->field);					\
+}
+
+VBERESP(beresp, unsigned, do_esi, do_esi)
+VBERESP(beresp, unsigned, do_gzip, do_gzip)
+VBERESP(beresp, unsigned, do_gunzip, do_gunzip)
+VBERESP(beresp, unsigned, do_stream, do_stream)
+
+/*--------------------------------------------------------------------*/
+
+const char * __match_proto__()
+VRT_r_client_identity(struct sess *sp)
+{
+	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+	if (sp->client_identity != NULL)
+		return (sp->client_identity);
+	else
+		return (sp->addr);
+}
+
+void
+VRT_l_client_identity(struct sess *sp, const char *str, ...)
+{
+	va_list ap;
+	char *b;
+
+	va_start(ap, str);
+	b = VRT_String(sp->http->ws, NULL, str, ap);
+	va_end(ap);
+	sp->client_identity = b;
+}
+
+/*--------------------------------------------------------------------*/
+
+#define BEREQ_TIMEOUT(which)					\
+void __match_proto__()						\
+VRT_l_bereq_##which(struct sess *sp, double num)		\
+{								\
+								\
+	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);			\
+	sp->wrk->which = (num > 0.0 ? num : 0.0);		\
+}								\
+								\
+double __match_proto__()					\
+VRT_r_bereq_##which(struct sess *sp)				\
+{								\
+								\
+	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);			\
+	return(sp->wrk->which);					\
+}
+
+BEREQ_TIMEOUT(connect_timeout)
+BEREQ_TIMEOUT(first_byte_timeout)
+BEREQ_TIMEOUT(between_bytes_timeout)
+
+/*--------------------------------------------------------------------*/
+
+const char *
+VRT_r_beresp_backend_name(const struct sess *sp)
+{
+
+	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+	CHECK_OBJ_NOTNULL(sp->wrk->vbc, VBC_MAGIC);
+	return(sp->wrk->vbc->backend->vcl_name);
+}
+
+struct sockaddr_storage *
+VRT_r_beresp_backend_ip(const struct sess *sp)
+{
+
+	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+	CHECK_OBJ_NOTNULL(sp->wrk->vbc, VBC_MAGIC);
+	return(sp->wrk->vbc->addr);
+}
+
+int
+VRT_r_beresp_backend_port(const struct sess *sp)
+{
+
+	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+	CHECK_OBJ_NOTNULL(sp->wrk->vbc, VBC_MAGIC);
+	return (VTCP_port(sp->wrk->vbc->addr));
+}
+
+const char * __match_proto__()
+VRT_r_beresp_storage(struct sess *sp)
+{
+	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+	if (sp->wrk->storage_hint != NULL)
+		return (sp->wrk->storage_hint);
+	else
+		return (NULL);
+}
+
+void __match_proto__()
+VRT_l_beresp_storage(struct sess *sp, const char *str, ...)
+{
+	va_list ap;
+	char *b;
+
+	va_start(ap, str);
+	b = VRT_String(sp->wrk->ws, NULL, str, ap);
+	va_end(ap);
+	sp->wrk->storage_hint = b;
+}
+
+/*--------------------------------------------------------------------*/
+
+void
+VRT_l_req_backend(struct sess *sp, struct director *be)
+{
+	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+	sp->director = be;
+}
+
+struct director * __match_proto__()
+VRT_r_req_backend(struct sess *sp)
+{
+	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+	return (sp->director);
+}
+
+/*--------------------------------------------------------------------*/
+
+void
+VRT_l_req_esi(struct sess *sp, unsigned process_esi)
+{
+	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+	/*
+	 * Only allow you to turn of esi in the main request
+	 * else everything gets confused
+	 */
+	if(sp->esi_level == 0)
+		sp->disable_esi = !process_esi;
+}
+
+unsigned __match_proto__()
+VRT_r_req_esi(struct sess *sp)
+{
+	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+	return (!sp->disable_esi);
+}
+
+int
+VRT_r_req_esi_level(const struct sess *sp)
+{
+
+	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+	return(sp->esi_level);
+}
+
+/*--------------------------------------------------------------------*/
+
+unsigned __match_proto__()
+VRT_r_req_can_gzip(struct sess *sp)
+{
+
+	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+	return (RFC2616_Req_Gzip(sp));
+}
+
+
+/*--------------------------------------------------------------------*/
+
+int
+VRT_r_req_restarts(const struct sess *sp)
+{
+
+	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+	return (sp->restarts);
+}
+
+/*--------------------------------------------------------------------
+ * NB: TTL is relative to when object was created, whereas grace and
+ * keep are relative to ttl.
+ */
+
+#define VRT_DO_EXP(which, exp, fld, offset, extra)		\
+								\
+void __match_proto__()						\
+VRT_l_##which##_##fld(struct sess *sp, double a)		\
+{								\
+								\
+	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);			\
+	if (a > 0.)						\
+		a += offset;					\
+	EXP_Set_##fld(&exp, a);					\
+	extra;							\
+}								\
+								\
+double __match_proto__()					\
+VRT_r_##which##_##fld(struct sess *sp)				\
+{								\
+								\
+	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);			\
+	return(EXP_Get_##fld(&exp) - offset);			\
+}
+
+static void
+vrt_wsp_exp(const struct sess *sp, unsigned xid, const struct exp *e)
+{
+	WSP(sp, SLT_TTL, "%u VCL %.0f %.0f %.0f %.0f %.0f",
+	    xid, e->ttl - (sp->t_req - e->entered), e->grace, e->keep,
+	    sp->t_req, e->age + (sp->t_req - e->entered));
+}
+
+VRT_DO_EXP(req, sp->exp, ttl, 0, )
+VRT_DO_EXP(req, sp->exp, grace, 0, )
+VRT_DO_EXP(req, sp->exp, keep, 0, )
+
+VRT_DO_EXP(obj, sp->obj->exp, grace, 0,
+   EXP_Rearm(sp->obj);
+   vrt_wsp_exp(sp, sp->obj->xid, &sp->obj->exp);)
+VRT_DO_EXP(obj, sp->obj->exp, ttl, (sp->t_req - sp->obj->exp.entered),
+   EXP_Rearm(sp->obj);
+   vrt_wsp_exp(sp, sp->obj->xid, &sp->obj->exp);)
+VRT_DO_EXP(obj, sp->obj->exp, keep, 0,
+   EXP_Rearm(sp->obj);
+   vrt_wsp_exp(sp, sp->obj->xid, &sp->obj->exp);)
+
+VRT_DO_EXP(beresp, sp->wrk->exp, grace, 0,
+   vrt_wsp_exp(sp, sp->xid, &sp->wrk->exp);)
+VRT_DO_EXP(beresp, sp->wrk->exp, ttl, 0,
+   vrt_wsp_exp(sp, sp->xid, &sp->wrk->exp);)
+VRT_DO_EXP(beresp, sp->wrk->exp, keep, 0,
+   vrt_wsp_exp(sp, sp->xid, &sp->wrk->exp);)
+
+/*--------------------------------------------------------------------
+ * req.xid
+ */
+
+const char * __match_proto__()
+VRT_r_req_xid(struct sess *sp)
+{
+	char *p;
+	int size;
+	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+
+	size = snprintf(NULL, 0, "%u", sp->xid) + 1;
+	AN(p = WS_Alloc(sp->http->ws, size));
+	assert(snprintf(p, size, "%u", sp->xid) < size);
+	return (p);
+}
+
+/*--------------------------------------------------------------------*/
+
+#define REQ_BOOL(which)						\
+void __match_proto__()						\
+VRT_l_req_##which(struct sess *sp, unsigned val)		\
+{								\
+								\
+	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);			\
+	sp->which = val ? 1 : 0;				\
+}								\
+								\
+unsigned __match_proto__()					\
+VRT_r_req_##which(struct sess *sp)				\
+{								\
+								\
+	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);			\
+	return(sp->which);					\
+}
+
+REQ_BOOL(hash_ignore_busy)
+REQ_BOOL(hash_always_miss)
+
+/*--------------------------------------------------------------------*/
+
+struct sockaddr_storage *
+VRT_r_client_ip(struct sess *sp)
+{
+
+	return (&sp->sockaddr);
+}
+
+struct sockaddr_storage *
+VRT_r_server_ip(struct sess *sp)
+{
+	int i;
+
+	if (sp->mysockaddr.ss_family == AF_UNSPEC) {
+		i = getsockname(sp->fd,
+		    (void*)&sp->mysockaddr, &sp->mysockaddrlen);
+		assert(VTCP_Check(i));
+	}
+
+	return (&sp->mysockaddr);
+}
+
+const char*
+VRT_r_server_identity(struct sess *sp)
+{
+	(void)sp;
+
+	if (heritage.identity[0] != '\0')
+		return (heritage.identity);
+	else
+		return (heritage.name);
+}
+
+
+const char*
+VRT_r_server_hostname(struct sess *sp)
+{
+	(void)sp;
+
+	if (vrt_hostname[0] == '\0')
+		AZ(gethostname(vrt_hostname, sizeof(vrt_hostname)));
+
+	return (vrt_hostname);
+}
+
+/*--------------------------------------------------------------------
+ * XXX: This is pessimistically silly
+ */
+
+int
+VRT_r_server_port(struct sess *sp)
+{
+	int i;
+
+	if (sp->mysockaddr.ss_family == AF_UNSPEC) {
+		i = getsockname(sp->fd,
+		    (void*)&sp->mysockaddr, &sp->mysockaddrlen);
+		assert(VTCP_Check(i));
+	}
+	return (VTCP_port(&sp->mysockaddr));
+}
+
+/*--------------------------------------------------------------------*/
+
+int
+VRT_r_obj_hits(const struct sess *sp)
+{
+
+	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+	CHECK_OBJ_NOTNULL(sp->obj, OBJECT_MAGIC);	/* XXX */
+	return (sp->obj->hits);
+}
+
+double
+VRT_r_obj_lastuse(const struct sess *sp)
+{
+
+	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+	CHECK_OBJ_NOTNULL(sp->obj, OBJECT_MAGIC);	/* XXX */
+	return (VTIM_real() - sp->obj->last_use);
+}
+
+unsigned
+VRT_r_req_backend_healthy(const struct sess *sp)
+{
+	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+	CHECK_OBJ_NOTNULL(sp->director, DIRECTOR_MAGIC);
+	return (VDI_Healthy(sp->director, sp));
+}
+
diff --git a/bin/varnishd/cache/cache_vrt_vmod.c b/bin/varnishd/cache/cache_vrt_vmod.c
new file mode 100644
index 0000000..6b3b846
--- /dev/null
+++ b/bin/varnishd/cache/cache_vrt_vmod.c
@@ -0,0 +1,180 @@
+/*-
+ * Copyright (c) 2006 Verdens Gang AS
+ * Copyright (c) 2006-2011 Varnish Software AS
+ * All rights reserved.
+ *
+ * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * Runtime support for compiled VCL programs
+ */
+
+#include "config.h"
+
+#include <dlfcn.h>
+#include <stdlib.h>
+
+#include "cache.h"
+
+#include "vcli_priv.h"
+#include "vrt.h"
+
+/*--------------------------------------------------------------------
+ * Modules stuff
+ */
+
+struct vmod {
+	unsigned		magic;
+#define VMOD_MAGIC		0xb750219c
+
+	VTAILQ_ENTRY(vmod)	list;
+
+	int			ref;
+
+	char			*nm;
+	char			*path;
+	void			*hdl;
+	const void		*funcs;
+	int			funclen;
+	const void		*idptr;
+};
+
+static VTAILQ_HEAD(,vmod)	vmods = VTAILQ_HEAD_INITIALIZER(vmods);
+
+int
+VRT_Vmod_Init(void **hdl, void *ptr, int len, const char *nm,
+    const char *path, struct cli *cli)
+{
+	struct vmod *v;
+	void *x, *y, *z, *w;
+
+	ASSERT_CLI();
+
+	VTAILQ_FOREACH(v, &vmods, list)
+		if (!strcmp(v->nm, nm))	// Also path, len ?
+			break;
+	if (v == NULL) {
+		ALLOC_OBJ(v, VMOD_MAGIC);
+		AN(v);
+
+		v->hdl = dlopen(path, RTLD_NOW | RTLD_LOCAL);
+		if (v->hdl == NULL) {
+			VCLI_Out(cli, "Loading VMOD %s from %s:\n", nm, path);
+			VCLI_Out(cli, "dlopen() failed: %s\n", dlerror());
+			VCLI_Out(cli, "Check child process permissions.\n");
+			FREE_OBJ(v);
+			return (1);
+		}
+
+		x = dlsym(v->hdl, "Vmod_Name");
+		y = dlsym(v->hdl, "Vmod_Len");
+		z = dlsym(v->hdl, "Vmod_Func");
+		w = dlsym(v->hdl, "Vmod_Id");
+		if (x == NULL || y == NULL || z == NULL || w == NULL) {
+			VCLI_Out(cli, "Loading VMOD %s from %s:\n", nm, path);
+			VCLI_Out(cli, "VMOD symbols not found\n");
+			VCLI_Out(cli, "Check relative pathnames.\n");
+			(void)dlclose(v->hdl);
+			FREE_OBJ(v);
+			return (1);
+		}
+		AN(x);
+		AN(y);
+		AN(z);
+		AN(w);
+		if (strcmp(x, nm)) {
+			VCLI_Out(cli, "Loading VMOD %s from %s:\n", nm, path);
+			VCLI_Out(cli, "File contain wrong VMOD (\"%s\")\n", x);
+			VCLI_Out(cli, "Check relative pathnames ?.\n");
+			(void)dlclose(v->hdl);
+			FREE_OBJ(v);
+			return (1);
+		}
+
+		v->funclen = *(const int *)y;
+		v->funcs = z;
+
+		REPLACE(v->nm, nm);
+		REPLACE(v->path, path);
+
+		VSC_C_main->vmods++;
+		VTAILQ_INSERT_TAIL(&vmods, v, list);
+		v->idptr = w;
+	}
+
+	assert(len == v->funclen);
+	memcpy(ptr, v->funcs, v->funclen);
+	v->ref++;
+
+	*hdl = v;
+	return (0);
+}
+
+void
+VRT_Vmod_Fini(void **hdl)
+{
+	struct vmod *v;
+
+	ASSERT_CLI();
+
+	AN(*hdl);
+	CAST_OBJ_NOTNULL(v, *hdl, VMOD_MAGIC);
+	*hdl = NULL;
+	if (--v->ref != 0)
+		return;
+#ifndef DONT_DLCLOSE_VMODS
+	AZ(dlclose(v->hdl));
+#endif
+	free(v->nm);
+	free(v->path);
+	VTAILQ_REMOVE(&vmods, v, list);
+	VSC_C_main->vmods--;
+	FREE_OBJ(v);
+}
+
+/*---------------------------------------------------------------------*/
+
+static void
+ccf_debug_vmod(struct cli *cli, const char * const *av, void *priv)
+{
+	struct vmod *v;
+
+	(void)av;
+	(void)priv;
+	ASSERT_CLI();
+	VTAILQ_FOREACH(v, &vmods, list)
+		VCLI_Out(cli, "%5d %s (%s)\n", v->ref, v->nm, v->path);
+}
+
+static struct cli_proto vcl_cmds[] = {
+	{ "debug.vmod", "debug.vmod", "show loaded vmods", 0, 0,
+		"d", ccf_debug_vmod },
+	{ NULL }
+};
+
+void
+VMOD_Init(void)
+{
+
+	CLI_AddFuncs(vcl_cmds);
+}
diff --git a/bin/varnishd/cache/cache_wrk.c b/bin/varnishd/cache/cache_wrk.c
new file mode 100644
index 0000000..bfee84c
--- /dev/null
+++ b/bin/varnishd/cache/cache_wrk.c
@@ -0,0 +1,204 @@
+/*-
+ * Copyright (c) 2006 Verdens Gang AS
+ * Copyright (c) 2006-2011 Varnish Software AS
+ * All rights reserved.
+ *
+ * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * Worker thread stuff unrelated to the worker thread pools.
+ */
+
+#include "config.h"
+
+#include <math.h>
+#include <stdlib.h>
+
+#include "cache.h"
+
+#include "hash/hash_slinger.h"
+#include "vsha256.h"
+
+static struct lock		wstat_mtx;
+
+/*--------------------------------------------------------------------*/
+
+static void
+wrk_sumstat(struct worker *w)
+{
+
+	Lck_AssertHeld(&wstat_mtx);
+#define L0(n)
+#define L1(n) (VSC_C_main->n += w->stats.n)
+#define VSC_DO_MAIN
+#define VSC_F(n, t, l, f, d, e) L##l(n);
+#include "tbl/vsc_fields.h"
+#undef VSC_F
+#undef VSC_DO_MAIN
+#undef L0
+#undef L1
+	memset(&w->stats, 0, sizeof w->stats);
+}
+
+void
+WRK_SumStat(struct worker *w)
+{
+
+	Lck_Lock(&wstat_mtx);
+	wrk_sumstat(w);
+	Lck_Unlock(&wstat_mtx);
+}
+
+int
+WRK_TrySumStat(struct worker *w)
+{
+	if (Lck_Trylock(&wstat_mtx))
+		return (0);
+	wrk_sumstat(w);
+	Lck_Unlock(&wstat_mtx);
+	return (1);
+}
+
+/*--------------------------------------------------------------------
+ * Create and starte a back-ground thread which as its own worker and
+ * session data structures;
+ */
+
+struct bgthread {
+	unsigned	magic;
+#define BGTHREAD_MAGIC	0x23b5152b
+	const char	*name;
+	bgthread_t	*func;
+	void		*priv;
+};
+
+static void *
+wrk_bgthread(void *arg)
+{
+	struct bgthread *bt;
+	struct worker ww;
+	struct sess *sp;
+	uint32_t logbuf[1024];	/* XXX:  size ? */
+
+	CAST_OBJ_NOTNULL(bt, arg, BGTHREAD_MAGIC);
+	THR_SetName(bt->name);
+	sp = SES_Alloc();
+	XXXAN(sp);
+	memset(&ww, 0, sizeof ww);
+	sp->wrk = &ww;
+	ww.magic = WORKER_MAGIC;
+	ww.wlp = ww.wlb = logbuf;
+	ww.wle = logbuf + (sizeof logbuf) / 4;
+
+	(void)bt->func(sp, bt->priv);
+
+	WRONG("BgThread terminated");
+
+	NEEDLESS_RETURN(NULL);
+}
+
+void
+WRK_BgThread(pthread_t *thr, const char *name, bgthread_t *func, void *priv)
+{
+	struct bgthread *bt;
+
+	ALLOC_OBJ(bt, BGTHREAD_MAGIC);
+	AN(bt);
+
+	bt->name = name;
+	bt->func = func;
+	bt->priv = priv;
+	AZ(pthread_create(thr, NULL, wrk_bgthread, bt));
+}
+
+/*--------------------------------------------------------------------*/
+
+static void *
+wrk_thread_real(void *priv, unsigned shm_workspace, unsigned sess_workspace,
+    uint16_t nhttp, unsigned http_space, unsigned siov)
+{
+	struct worker *w, ww;
+	uint32_t wlog[shm_workspace / 4];
+	/* XXX: can we trust these to be properly aligned ? */
+	unsigned char ws[sess_workspace];
+	unsigned char http0[http_space];
+	unsigned char http1[http_space];
+	unsigned char http2[http_space];
+	struct iovec iov[siov];
+	struct SHA256Context sha256;
+
+	THR_SetName("cache-worker");
+	w = &ww;
+	memset(w, 0, sizeof *w);
+	w->magic = WORKER_MAGIC;
+	w->lastused = NAN;
+	w->wlb = w->wlp = wlog;
+	w->wle = wlog + (sizeof wlog) / 4;
+	w->sha256ctx = &sha256;
+	w->bereq = HTTP_create(http0, nhttp);
+	w->beresp = HTTP_create(http1, nhttp);
+	w->resp = HTTP_create(http2, nhttp);
+	w->wrw.iov = iov;
+	w->wrw.siov = siov;
+	w->wrw.ciov = siov;
+	AZ(pthread_cond_init(&w->cond, NULL));
+
+	WS_Init(w->ws, "wrk", ws, sess_workspace);
+
+	VSL(SLT_WorkThread, 0, "%p start", w);
+
+	Pool_Work_Thread(priv, w);
+	AZ(w->pool);
+
+	VSL(SLT_WorkThread, 0, "%p end", w);
+	if (w->vcl != NULL)
+		VCL_Rel(&w->vcl);
+	AZ(pthread_cond_destroy(&w->cond));
+	HSH_Cleanup(w);
+	WRK_SumStat(w);
+	return (NULL);
+}
+
+void *
+WRK_thread(void *priv)
+{
+	uint16_t nhttp;
+	unsigned siov;
+
+	assert(cache_param->http_max_hdr <= 65535);
+	/* We need to snapshot these two for consistency */
+	nhttp = (uint16_t)cache_param->http_max_hdr;
+	siov = nhttp * 2;
+	if (siov > IOV_MAX)
+		siov = IOV_MAX;
+	return (wrk_thread_real(priv,
+	    cache_param->shm_workspace,
+	    cache_param->wthread_workspace,
+	    nhttp, HTTP_estimate(nhttp), siov));
+}
+
+void
+WRK_Init(void)
+{
+	Lck_New(&wstat_mtx, lck_wstat);
+}
diff --git a/bin/varnishd/cache/cache_wrw.c b/bin/varnishd/cache/cache_wrw.c
new file mode 100644
index 0000000..2160f69
--- /dev/null
+++ b/bin/varnishd/cache/cache_wrw.c
@@ -0,0 +1,344 @@
+/*-
+ * Copyright (c) 2006 Verdens Gang AS
+ * Copyright (c) 2006-2011 Varnish Software AS
+ * All rights reserved.
+ *
+ * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * Write data to fd
+ * We try to use writev() if possible in order to minimize number of
+ * syscalls made and packets sent.  It also just might allow the worker
+ * thread to complete the request without holding stuff locked.
+ */
+
+#include "config.h"
+
+#include <sys/types.h>
+#ifdef SENDFILE_WORKS
+#  if defined(__FreeBSD__) || defined(__DragonFly__)
+#    include <sys/socket.h>
+#  elif defined(__linux__)
+#    include <sys/sendfile.h>
+#  elif defined(__sun)
+#    include <sys/sendfile.h>
+#  else
+#     error Unknown sendfile() implementation
+#  endif
+#endif /* SENDFILE_WORKS */
+#include <sys/uio.h>
+
+#include <stdio.h>
+
+#include "cache.h"
+#include "vtim.h"
+
+/*--------------------------------------------------------------------
+ */
+
+int
+WRW_Error(const struct worker *w)
+{
+
+	return (w->wrw.werr);
+}
+
+void
+WRW_Reserve(struct worker *w, int *fd)
+{
+	struct wrw *wrw;
+
+	CHECK_OBJ_NOTNULL(w, WORKER_MAGIC);
+	wrw = &w->wrw;
+	AZ(wrw->wfd);
+	wrw->werr = 0;
+	wrw->liov = 0;
+	wrw->niov = 0;
+	wrw->ciov = wrw->siov;
+	wrw->wfd = fd;
+}
+
+static void
+WRW_Release(struct worker *w)
+{
+	struct wrw *wrw;
+
+	CHECK_OBJ_NOTNULL(w, WORKER_MAGIC);
+	wrw = &w->wrw;
+	AN(wrw->wfd);
+	wrw->werr = 0;
+	wrw->liov = 0;
+	wrw->niov = 0;
+	wrw->ciov = wrw->siov;
+	wrw->wfd = NULL;
+}
+
+unsigned
+WRW_Flush(struct worker *w)
+{
+	ssize_t i;
+	struct wrw *wrw;
+	char cbuf[32];
+
+	CHECK_OBJ_NOTNULL(w, WORKER_MAGIC);
+	wrw = &w->wrw;
+	AN(wrw->wfd);
+
+	/* For chunked, there must be one slot reserved for the chunked tail */
+	if (wrw->ciov < wrw->siov)
+		assert(wrw->niov < wrw->siov);
+
+	if (*wrw->wfd >= 0 && wrw->liov > 0 && wrw->werr == 0) {
+		if (wrw->ciov < wrw->siov && wrw->cliov > 0) {
+			bprintf(cbuf, "00%jx\r\n", (intmax_t)wrw->cliov);
+			i = strlen(cbuf);
+			wrw->iov[wrw->ciov].iov_base = cbuf;
+			wrw->iov[wrw->ciov].iov_len = i;
+			wrw->liov += i;
+
+			wrw->iov[wrw->niov].iov_base = cbuf + i - 2;
+			wrw->iov[wrw->niov++].iov_len = 2;
+			wrw->liov += 2;
+		} else if (wrw->ciov < wrw->siov) {
+			wrw->iov[wrw->ciov].iov_base = cbuf;
+			wrw->iov[wrw->ciov].iov_len = 0;
+		}
+		i = writev(*wrw->wfd, wrw->iov, wrw->niov);
+		while (i != wrw->liov && i > 0) {
+			/* Remove sent data from start of I/O vector,
+			 * then retry; we hit a timeout, but some data
+			 * was sent.
+
+			 XXX: Add a "minimum sent data per timeout
+			 counter to prevent slowlaris attacks
+			*/
+			size_t used = 0;
+
+			if (VTIM_real() - w->sp->t_resp > cache_param->send_timeout) {
+				WSL(w, SLT_Debug, *wrw->wfd,
+				    "Hit total send timeout, wrote = %ld/%ld; not retrying",
+				    i, wrw->liov);
+				i = -1;
+				break;
+			}
+
+			WSL(w, SLT_Debug, *wrw->wfd,
+			    "Hit send timeout, wrote = %ld/%ld; retrying",
+			    i, wrw->liov);
+
+			for (int j = 0; j < wrw->niov; j++) {
+				if (used + wrw->iov[j].iov_len > i) {
+					/* Cutoff is in this iov */
+					int used_here = i - used;
+					wrw->iov[j].iov_len -= used_here;
+					wrw->iov[j].iov_base = (char*)wrw->iov[j].iov_base + used_here;
+					memmove(wrw->iov, &wrw->iov[j],
+						(wrw->niov - j) * sizeof(struct iovec));
+					wrw->niov -= j;
+					wrw->liov -= i;
+					break;
+				}
+				used += wrw->iov[j].iov_len;
+			}
+			i = writev(*wrw->wfd, wrw->iov, wrw->niov);
+		}
+		if (i <= 0) {
+			wrw->werr++;
+			WSL(w, SLT_Debug, *wrw->wfd,
+			    "Write error, retval = %d, len = %d, errno = %s",
+			    i, wrw->liov, strerror(errno));
+		}
+	}
+	wrw->liov = 0;
+	wrw->cliov = 0;
+	wrw->niov = 0;
+	if (wrw->ciov < wrw->siov)
+		wrw->ciov = wrw->niov++;
+	return (wrw->werr);
+}
+
+unsigned
+WRW_FlushRelease(struct worker *w)
+{
+	unsigned u;
+
+	CHECK_OBJ_NOTNULL(w, WORKER_MAGIC);
+	AN(w->wrw.wfd);
+	u = WRW_Flush(w);
+	WRW_Release(w);
+	return (u);
+}
+
+unsigned
+WRW_WriteH(struct worker *w, const txt *hh, const char *suf)
+{
+	unsigned u;
+
+	CHECK_OBJ_NOTNULL(w, WORKER_MAGIC);
+	AN(w->wrw.wfd);
+	AN(w);
+	AN(hh);
+	AN(hh->b);
+	AN(hh->e);
+	u = WRW_Write(w, hh->b, hh->e - hh->b);
+	if (suf != NULL)
+		u += WRW_Write(w, suf, -1);
+	return (u);
+}
+
+unsigned
+WRW_Write(struct worker *w, const void *ptr, int len)
+{
+	struct wrw *wrw;
+
+	CHECK_OBJ_NOTNULL(w, WORKER_MAGIC);
+	wrw = &w->wrw;
+	AN(wrw->wfd);
+	if (len == 0 || *wrw->wfd < 0)
+		return (0);
+	if (len == -1)
+		len = strlen(ptr);
+	if (wrw->niov >= wrw->siov - (wrw->ciov < wrw->siov ? 1 : 0))
+		(void)WRW_Flush(w);
+	wrw->iov[wrw->niov].iov_base = TRUST_ME(ptr);
+	wrw->iov[wrw->niov].iov_len = len;
+	wrw->liov += len;
+	wrw->niov++;
+	if (wrw->ciov < wrw->siov) {
+		assert(wrw->niov < wrw->siov);
+		wrw->cliov += len;
+	}
+	return (len);
+}
+
+void
+WRW_Chunked(struct worker *w)
+{
+	struct wrw *wrw;
+
+	CHECK_OBJ_NOTNULL(w, WORKER_MAGIC);
+	wrw = &w->wrw;
+
+	assert(wrw->ciov == wrw->siov);
+	/*
+	 * If there are not space for chunked header, a chunk of data and
+	 * a chunk tail, we might as well flush right away.
+	 */
+	if (wrw->niov + 3 >= wrw->siov)
+		(void)WRW_Flush(w);
+	wrw->ciov = wrw->niov++;
+	wrw->cliov = 0;
+	assert(wrw->ciov < wrw->siov);
+	assert(wrw->niov < wrw->siov);
+}
+
+/*
+ * XXX: It is not worth the complexity to attempt to get the
+ * XXX: end of chunk into the WRW_Flush(), because most of the time
+ * XXX: if not always, that is a no-op anyway, because the calling
+ * XXX: code already called WRW_Flush() to release local storage.
+ */
+
+void
+WRW_EndChunk(struct worker *w)
+{
+	struct wrw *wrw;
+
+	CHECK_OBJ_NOTNULL(w, WORKER_MAGIC);
+	wrw = &w->wrw;
+
+	assert(wrw->ciov < wrw->siov);
+	(void)WRW_Flush(w);
+	wrw->ciov = wrw->siov;
+	wrw->niov = 0;
+	wrw->cliov = 0;
+	(void)WRW_Write(w, "0\r\n\r\n", -1);
+}
+
+
+#ifdef SENDFILE_WORKS
+void
+WRW_Sendfile(struct worker *w, int fd, off_t off, unsigned len)
+{
+	struct wrw *wrw;
+
+	CHECK_OBJ_NOTNULL(w, WORKER_MAGIC);
+	wrw = &w->wrw;
+	AN(wrw->wfd);
+	assert(fd >= 0);
+	assert(len > 0);
+
+#if defined(__FreeBSD__) || defined(__DragonFly__)
+	do {
+		struct sf_hdtr sfh;
+		memset(&sfh, 0, sizeof sfh);
+		if (wrw->niov > 0) {
+			sfh.headers = wrw->iov;
+			sfh.hdr_cnt = wrw->niov;
+		}
+		if (sendfile(fd, *wrw->wfd, off, len, &sfh, NULL, 0) != 0)
+			wrw->werr++;
+		wrw->liov = 0;
+		wrw->niov = 0;
+	} while (0);
+#elif defined(__linux__)
+	do {
+		if (WRW_Flush(w) == 0 &&
+		    sendfile(*wrw->wfd, fd, &off, len) != len)
+			wrw->werr++;
+	} while (0);
+#elif defined(__sun) && defined(HAVE_SENDFILEV)
+	do {
+		sendfilevec_t svvec[cache_param->http_headers * 2 + 1];
+		size_t xferred = 0, expected = 0;
+		int i;
+		for (i = 0; i < wrw->niov; i++) {
+			svvec[i].sfv_fd = SFV_FD_SELF;
+			svvec[i].sfv_flag = 0;
+			svvec[i].sfv_off = (off_t) wrw->iov[i].iov_base;
+			svvec[i].sfv_len = wrw->iov[i].iov_len;
+			expected += svvec[i].sfv_len;
+		}
+		svvec[i].sfv_fd = fd;
+		svvec[i].sfv_flag = 0;
+		svvec[i].sfv_off = off;
+		svvec[i].sfv_len = len;
+		expected += svvec[i].sfv_len;
+		if (sendfilev(*wrw->wfd, svvec, i, &xferred) == -1 ||
+		    xferred != expected)
+			wrw->werr++;
+		wrw->liov = 0;
+		wrw->niov = 0;
+	} while (0);
+#elif defined(__sun) && defined(HAVE_SENDFILE)
+	do {
+		if (WRW_Flush(w) == 0 &&
+		    sendfile(*wrw->wfd, fd, &off, len) != len)
+			wrw->werr++;
+	} while (0);
+#else
+#error Unknown sendfile() implementation
+#endif
+}
+#endif /* SENDFILE_WORKS */
+
diff --git a/bin/varnishd/cache/cache_ws.c b/bin/varnishd/cache/cache_ws.c
new file mode 100644
index 0000000..9fca215
--- /dev/null
+++ b/bin/varnishd/cache/cache_ws.c
@@ -0,0 +1,212 @@
+/*-
+ * Copyright (c) 2006 Verdens Gang AS
+ * Copyright (c) 2006-2010 Varnish Software AS
+ * All rights reserved.
+ *
+ * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ */
+
+#include "config.h"
+
+#include "cache.h"
+
+void
+WS_Assert(const struct ws *ws)
+{
+
+	CHECK_OBJ_NOTNULL(ws, WS_MAGIC);
+	DSL(0x02, SLT_Debug, 0, "WS(%p = (%s, %p %u %u %u)",
+	    ws, ws->id, ws->s, pdiff(ws->s, ws->f),
+	    ws->r == NULL ? 0 : pdiff(ws->f, ws->r),
+	    pdiff(ws->s, ws->e));
+	assert(ws->s != NULL);
+	assert(PAOK(ws->s));
+	assert(ws->e != NULL);
+	assert(PAOK(ws->e));
+	assert(ws->s < ws->e);
+	assert(ws->f >= ws->s);
+	assert(ws->f <= ws->e);
+	assert(PAOK(ws->f));
+	if (ws->r) {
+		assert(ws->r > ws->s);
+		assert(ws->r <= ws->e);
+		assert(PAOK(ws->r));
+	}
+}
+
+void
+WS_Init(struct ws *ws, const char *id, void *space, unsigned len)
+{
+
+	DSL(0x02, SLT_Debug, 0,
+	    "WS_Init(%p, \"%s\", %p, %u)", ws, id, space, len);
+	assert(space != NULL);
+	memset(ws, 0, sizeof *ws);
+	ws->magic = WS_MAGIC;
+	ws->s = space;
+	assert(PAOK(space));
+	ws->e = ws->s + len;
+	assert(PAOK(len));
+	ws->f = ws->s;
+	ws->id = id;
+	WS_Assert(ws);
+}
+
+/*
+ * Reset a WS to start or a given pointer, likely from WS_Snapshot
+ */
+
+void
+WS_Reset(struct ws *ws, char *p)
+{
+
+	WS_Assert(ws);
+	DSL(0x02, SLT_Debug, 0, "WS_Reset(%p, %p)", ws, p);
+	assert(ws->r == NULL);
+	if (p == NULL)
+		ws->f = ws->s;
+	else {
+		assert(p >= ws->s);
+		assert(p < ws->e);
+		ws->f = p;
+	}
+	WS_Assert(ws);
+}
+
+char *
+WS_Alloc(struct ws *ws, unsigned bytes)
+{
+	char *r;
+
+	WS_Assert(ws);
+	bytes = PRNDUP(bytes);
+
+	assert(ws->r == NULL);
+	if (ws->f + bytes > ws->e) {
+		ws->overflow++;
+		WS_Assert(ws);
+		return(NULL);
+	}
+	r = ws->f;
+	ws->f += bytes;
+	DSL(0x02, SLT_Debug, 0, "WS_Alloc(%p, %u) = %p", ws, bytes, r);
+	WS_Assert(ws);
+	return (r);
+}
+
+char *
+WS_Dup(struct ws *ws, const char *s)
+{
+	unsigned l;
+	char *p;
+
+	WS_Assert(ws);
+	l = strlen(s) + 1;
+	p = WS_Alloc(ws, l);
+	if (p != NULL)
+		memcpy(p, s, l);
+	DSL(0x02, SLT_Debug, 0, "WS_Dup(%p, \"%s\") = %p", ws, s, p);
+	WS_Assert(ws);
+	return (p);
+}
+
+unsigned
+WS_Free(const struct ws *ws)
+{
+
+	WS_Assert(ws);
+	return(ws->e - ws->f);
+}
+
+char *
+WS_Snapshot(struct ws *ws)
+{
+
+	WS_Assert(ws);
+	assert(ws->r == NULL);
+	DSL(0x02, SLT_Debug, 0, "WS_Snapshot(%p) = %p", ws, ws->f);
+	return (ws->f);
+}
+
+unsigned
+WS_Reserve(struct ws *ws, unsigned bytes)
+{
+	unsigned b2;
+
+	WS_Assert(ws);
+	assert(ws->r == NULL);
+	if (bytes == 0)
+		b2 = ws->e - ws->f;
+	else if (bytes > ws->e - ws->f)
+		b2 = ws->e - ws->f;
+	else
+		b2 = bytes;
+	b2 = PRNDDN(b2);
+	xxxassert(ws->f + b2 <= ws->e);
+	ws->r = ws->f + b2;
+	DSL(0x02, SLT_Debug, 0, "WS_Reserve(%p, %u/%u) = %u",
+	    ws, b2, bytes, pdiff(ws->f, ws->r));
+	WS_Assert(ws);
+	return (pdiff(ws->f, ws->r));
+}
+
+void
+WS_Release(struct ws *ws, unsigned bytes)
+{
+	WS_Assert(ws);
+	bytes = PRNDUP(bytes);
+	assert(bytes <= ws->e - ws->f);
+	DSL(0x02, SLT_Debug, 0, "WS_Release(%p, %u)", ws, bytes);
+	assert(ws->r != NULL);
+	assert(ws->f + bytes <= ws->r);
+	ws->f += bytes;
+	ws->r = NULL;
+	WS_Assert(ws);
+}
+
+void
+WS_ReleaseP(struct ws *ws, char *ptr)
+{
+	WS_Assert(ws);
+	DSL(0x02, SLT_Debug, 0, "WS_ReleaseP(%p, %p)", ws, ptr);
+	assert(ws->r != NULL);
+	assert(ptr >= ws->f);
+	assert(ptr <= ws->r);
+	ws->f += PRNDUP(ptr - ws->f);
+	ws->r = NULL;
+	WS_Assert(ws);
+}
+
+#if 0
+/* XXX: not used anywhere (yet) */
+void
+WS_Return(struct ws *ws, char *s, char *e)
+{
+
+	WS_Assert(ws);
+	if (e == ws->f)
+		ws->f = s;
+}
+#endif
diff --git a/bin/varnishd/cache_acceptor.c b/bin/varnishd/cache_acceptor.c
deleted file mode 100644
index 8c83121..0000000
--- a/bin/varnishd/cache_acceptor.c
+++ /dev/null
@@ -1,430 +0,0 @@
-/*-
- * Copyright (c) 2006 Verdens Gang AS
- * Copyright (c) 2006-2011 Varnish Software AS
- * All rights reserved.
- *
- * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- *
- */
-
-#include "config.h"
-
-#include "cache.h"
-#include "common/heritage.h"
-
-#include "vcli.h"
-#include "vcli_priv.h"
-#include "vtcp.h"
-#include "vtim.h"
-
-static pthread_t	VCA_thread;
-static struct timeval	tv_sndtimeo;
-static struct timeval	tv_rcvtimeo;
-
-/*--------------------------------------------------------------------
- * We want to get out of any kind of trouble-hit TCP connections as fast
- * as absolutely possible, so we set them LINGER enabled with zero timeout,
- * so that even if there are outstanding write data on the socket, a close(2)
- * will return immediately.
- */
-static const struct linger linger = {
-	.l_onoff	=	0,
-};
-
-static unsigned char	need_sndtimeo, need_rcvtimeo, need_linger, need_test;
-
-static void
-sock_test(int fd)
-{
-	struct linger lin;
-	struct timeval tv;
-	socklen_t l;
-	int i;
-
-	l = sizeof lin;
-	i = getsockopt(fd, SOL_SOCKET, SO_LINGER, &lin, &l);
-	if (i) {
-		VTCP_Assert(i);
-		return;
-	}
-	assert(l == sizeof lin);
-	if (memcmp(&lin, &linger, l))
-		need_linger = 1;
-
-#ifdef SO_SNDTIMEO_WORKS
-	l = sizeof tv;
-	i = getsockopt(fd, SOL_SOCKET, SO_SNDTIMEO, &tv, &l);
-	if (i) {
-		VTCP_Assert(i);
-		return;
-	}
-	assert(l == sizeof tv);
-	if (memcmp(&tv, &tv_sndtimeo, l))
-		need_sndtimeo = 1;
-#else
-	(void)tv;
-	(void)tv_sndtimeo;
-	(void)need_sndtimeo;
-#endif
-
-#ifdef SO_RCVTIMEO_WORKS
-	l = sizeof tv;
-	i = getsockopt(fd, SOL_SOCKET, SO_RCVTIMEO, &tv, &l);
-	if (i) {
-		VTCP_Assert(i);
-		return;
-	}
-	assert(l == sizeof tv);
-	if (memcmp(&tv, &tv_rcvtimeo, l))
-		need_rcvtimeo = 1;
-#else
-	(void)tv;
-	(void)tv_rcvtimeo;
-	(void)need_rcvtimeo;
-#endif
-
-	need_test = 0;
-}
-
-/*--------------------------------------------------------------------
- * Called once the workerthread gets hold of the session, to do setup
- * setup overhead, we don't want to bother the acceptor thread with.
- */
-
-void
-VCA_Prep(struct sess *sp)
-{
-	char addr[VTCP_ADDRBUFSIZE];
-	char port[VTCP_PORTBUFSIZE];
-
-	VTCP_name(&sp->sockaddr, sp->sockaddrlen,
-	    addr, sizeof addr, port, sizeof port);
-	sp->addr = WS_Dup(sp->ws, addr);
-	sp->port = WS_Dup(sp->ws, port);
-	if (cache_param->log_local_addr) {
-		AZ(getsockname(sp->fd, (void*)&sp->mysockaddr, &sp->mysockaddrlen));
-		VTCP_name(&sp->mysockaddr, sp->mysockaddrlen,
-		    addr, sizeof addr, port, sizeof port);
-		WSP(sp, SLT_SessionOpen, "%s %s %s %s",
-		    sp->addr, sp->port, addr, port);
-	} else {
-		WSP(sp, SLT_SessionOpen, "%s %s %s",
-		    sp->addr, sp->port, sp->mylsock->name);
-	}
-	sp->acct_ses.first = sp->t_open;
-	if (need_test)
-		sock_test(sp->fd);
-	if (need_linger)
-		VTCP_Assert(setsockopt(sp->fd, SOL_SOCKET, SO_LINGER,
-		    &linger, sizeof linger));
-#ifdef SO_SNDTIMEO_WORKS
-	if (need_sndtimeo)
-		VTCP_Assert(setsockopt(sp->fd, SOL_SOCKET, SO_SNDTIMEO,
-		    &tv_sndtimeo, sizeof tv_sndtimeo));
-#endif
-#ifdef SO_RCVTIMEO_WORKS
-	if (need_rcvtimeo)
-		VTCP_Assert(setsockopt(sp->fd, SOL_SOCKET, SO_RCVTIMEO,
-		    &tv_rcvtimeo, sizeof tv_rcvtimeo));
-#endif
-}
-
-/*--------------------------------------------------------------------
- * If accept(2)'ing fails, we pace ourselves to relive any resource
- * shortage if possible.
- */
-
-static double vca_pace = 0.0;
-static struct lock pace_mtx;
-
-static void
-vca_pace_check(void)
-{
-	double p;
-
-	if (vca_pace == 0.0)
-		return;
-	Lck_Lock(&pace_mtx);
-	p = vca_pace;
-	Lck_Unlock(&pace_mtx);
-	if (p > 0.0)
-		VTIM_sleep(p);
-}
-
-static void
-vca_pace_bad(void)
-{
-
-	Lck_Lock(&pace_mtx);
-	vca_pace += cache_param->acceptor_sleep_incr;
-	if (vca_pace > cache_param->acceptor_sleep_max)
-		vca_pace = cache_param->acceptor_sleep_max;
-	Lck_Unlock(&pace_mtx);
-}
-
-static void
-vca_pace_good(void)
-{
-
-	if (vca_pace == 0.0)
-		return;
-	Lck_Lock(&pace_mtx);
-	vca_pace *= cache_param->acceptor_sleep_decay;
-	if (vca_pace < cache_param->acceptor_sleep_incr)
-		vca_pace = 0.0;
-	Lck_Unlock(&pace_mtx);
-}
-
-/*--------------------------------------------------------------------
- * Accept on a listen socket, and handle error returns.
- */
-
-static int hack_ready;
-
-int
-VCA_Accept(struct listen_sock *ls, struct wrk_accept *wa)
-{
-	int i;
-
-	CHECK_OBJ_NOTNULL(ls, LISTEN_SOCK_MAGIC);
-	vca_pace_check();
-
-	while(!hack_ready)
-		(void)usleep(100*1000);
-
-	wa->acceptaddrlen = sizeof wa->acceptaddr;
-	i = accept(ls->sock, (void*)&wa->acceptaddr, &wa->acceptaddrlen);
-
-	if (i < 0) {
-		switch (errno) {
-		case ECONNABORTED:
-			break;
-		case EMFILE:
-			VSL(SLT_Debug, ls->sock, "Too many open files");
-			vca_pace_bad();
-			break;
-		default:
-			VSL(SLT_Debug, ls->sock, "Accept failed: %s",
-			    strerror(errno));
-			vca_pace_bad();
-			break;
-		}
-	}
-	wa->acceptlsock = ls;
-	wa->acceptsock = i;
-	return (i);
-}
-
-/*--------------------------------------------------------------------
- * Fail a session
- *
- * This happens if we accept the socket, but cannot get a session
- * structure.
- *
- * We consider this a DoS situation (false positive:  Extremely popular
- * busy objects) and silently close the connection with minimum effort
- * and fuzz, rather than try to send an intelligent message back.
- */
-
-void
-VCA_FailSess(struct worker *w)
-{
-	struct wrk_accept *wa;
-
-	CHECK_OBJ_NOTNULL(w, WORKER_MAGIC);
-	CAST_OBJ_NOTNULL(wa, (void*)w->ws->f, WRK_ACCEPT_MAGIC);
-	AZ(w->sp);
-	AZ(close(wa->acceptsock));
-	w->stats.sess_drop++;
-	vca_pace_bad();
-}
-
-/*--------------------------------------------------------------------*/
-
-void
-VCA_SetupSess(struct worker *w)
-{
-	struct sess *sp;
-	struct wrk_accept *wa;
-
-	CHECK_OBJ_NOTNULL(w, WORKER_MAGIC);
-	CAST_OBJ_NOTNULL(wa, (void*)w->ws->f, WRK_ACCEPT_MAGIC);
-	sp = w->sp;
-	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
-	sp->fd = wa->acceptsock;
-	sp->vsl_id = wa->acceptsock | VSL_CLIENTMARKER ;
-	wa->acceptsock = -1;
-	sp->t_open = VTIM_real();
-	sp->t_end = sp->t_open;
-	sp->mylsock = wa->acceptlsock;
-	CHECK_OBJ_NOTNULL(sp->mylsock, LISTEN_SOCK_MAGIC);
-	assert(wa->acceptaddrlen <= sp->sockaddrlen);
-	memcpy(&sp->sockaddr, &wa->acceptaddr, wa->acceptaddrlen);
-	sp->sockaddrlen = wa->acceptaddrlen;
-	sp->step = STP_FIRST;
-	vca_pace_good();
-	w->stats.sess_conn++;
-}
-
-/*--------------------------------------------------------------------*/
-
-static void *
-vca_acct(void *arg)
-{
-#ifdef SO_RCVTIMEO_WORKS
-	double sess_timeout = 0;
-#endif
-#ifdef SO_SNDTIMEO_WORKS
-	double send_timeout = 0;
-#endif
-	struct listen_sock *ls;
-	double t0, now;
-
-	THR_SetName("cache-acceptor");
-	(void)arg;
-
-	VTAILQ_FOREACH(ls, &heritage.socks, list) {
-		if (ls->sock < 0)
-			continue;
-		AZ(listen(ls->sock, cache_param->listen_depth));
-		AZ(setsockopt(ls->sock, SOL_SOCKET, SO_LINGER,
-		    &linger, sizeof linger));
-	}
-
-	hack_ready = 1;
-
-	need_test = 1;
-	t0 = VTIM_real();
-	while (1) {
-		(void)sleep(1);
-#ifdef SO_SNDTIMEO_WORKS
-		if (cache_param->idle_send_timeout != send_timeout) {
-			need_test = 1;
-			send_timeout = cache_param->idle_send_timeout;
-			tv_sndtimeo = VTIM_timeval(send_timeout);
-			VTAILQ_FOREACH(ls, &heritage.socks, list) {
-				if (ls->sock < 0)
-					continue;
-				AZ(setsockopt(ls->sock, SOL_SOCKET,
-				    SO_SNDTIMEO,
-				    &tv_sndtimeo, sizeof tv_sndtimeo));
-			}
-		}
-#endif
-#ifdef SO_RCVTIMEO_WORKS
-		if (cache_param->sess_timeout != sess_timeout) {
-			need_test = 1;
-			sess_timeout = cache_param->sess_timeout;
-			tv_rcvtimeo = VTIM_timeval(sess_timeout);
-			VTAILQ_FOREACH(ls, &heritage.socks, list) {
-				if (ls->sock < 0)
-					continue;
-				AZ(setsockopt(ls->sock, SOL_SOCKET,
-				    SO_RCVTIMEO,
-				    &tv_rcvtimeo, sizeof tv_rcvtimeo));
-			}
-		}
-#endif
-		now = VTIM_real();
-		VSC_C_main->uptime = (uint64_t)(now - t0);
-	}
-	NEEDLESS_RETURN(NULL);
-}
-
-
-/*--------------------------------------------------------------------*/
-
-static void
-ccf_start(struct cli *cli, const char * const *av, void *priv)
-{
-
-	(void)cli;
-	(void)av;
-	(void)priv;
-
-	AZ(pthread_create(&VCA_thread, NULL, vca_acct, NULL));
-}
-
-/*--------------------------------------------------------------------*/
-
-static void
-ccf_listen_address(struct cli *cli, const char * const *av, void *priv)
-{
-	struct listen_sock *ls;
-	char h[32], p[32];
-
-	(void)cli;
-	(void)av;
-	(void)priv;
-
-	/*
-	 * This CLI command is primarily used by varnishtest.  Don't
-	 * respond until liste(2) has been called, in order to avoid
-	 * a race where varnishtest::client would attempt to connect(2)
-	 * before listen(2) has been called.
-	 */
-	while(!hack_ready)
-		(void)usleep(100*1000);
-
-	VTAILQ_FOREACH(ls, &heritage.socks, list) {
-		if (ls->sock < 0)
-			continue;
-		VTCP_myname(ls->sock, h, sizeof h, p, sizeof p);
-		VCLI_Out(cli, "%s %s\n", h, p);
-	}
-}
-
-/*--------------------------------------------------------------------*/
-
-static struct cli_proto vca_cmds[] = {
-	{ CLI_SERVER_START,	"i", ccf_start },
-	{ "debug.listen_address",
-	    "debug.listen_address",
-	    "Report the actual listen address\n", 0, 0,
-	    "d", ccf_listen_address, NULL },
-	{ NULL }
-};
-
-void
-VCA_Init(void)
-{
-
-	CLI_AddFuncs(vca_cmds);
-	Lck_New(&pace_mtx, lck_vcapace);
-}
-
-void
-VCA_Shutdown(void)
-{
-	struct listen_sock *ls;
-	int i;
-
-	VTAILQ_FOREACH(ls, &heritage.socks, list) {
-		if (ls->sock < 0)
-			continue;
-		i = ls->sock;
-		ls->sock = -1;
-		(void)close(i);
-	}
-}
diff --git a/bin/varnishd/cache_backend.c b/bin/varnishd/cache_backend.c
deleted file mode 100644
index 1f290c3..0000000
--- a/bin/varnishd/cache_backend.c
+++ /dev/null
@@ -1,517 +0,0 @@
-/*-
- * Copyright (c) 2006 Verdens Gang AS
- * Copyright (c) 2006-2011 Varnish Software AS
- * All rights reserved.
- *
- * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- *
- * Handle backend connections and backend request structures.
- *
- */
-
-#include "config.h"
-
-#include <poll.h>
-#include <stdlib.h>
-
-#include "cache.h"
-
-#include "cache_backend.h"
-#include "vrt.h"
-#include "vtcp.h"
-
-/*--------------------------------------------------------------------
- * The "simple" director really isn't, since thats where all the actual
- * connections happen.  Nontheless, pretend it is simple by sequestering
- * the directoricity of it under this line.
- */
-
-struct vdi_simple {
-	unsigned		magic;
-#define VDI_SIMPLE_MAGIC	0x476d25b7
-	struct director		dir;
-	struct backend		*backend;
-	const struct vrt_backend *vrt;
-};
-
-/*--------------------------------------------------------------------
- * Create default Host: header for backend request
- */
-void
-VDI_AddHostHeader(const struct sess *sp)
-{
-
-	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
-	CHECK_OBJ_NOTNULL(sp->wrk->bereq, HTTP_MAGIC);
-	CHECK_OBJ_NOTNULL(sp->wrk->vbc, VBC_MAGIC);
-	CHECK_OBJ_NOTNULL(sp->wrk->vbc->vdis, VDI_SIMPLE_MAGIC);
-	http_PrintfHeader(sp->wrk, sp->vsl_id, sp->wrk->bereq,
-	    "Host: %s", sp->wrk->vbc->vdis->vrt->hosthdr);
-}
-
-/*--------------------------------------------------------------------*/
-
-/* Private interface from backend_cfg.c */
-void
-VBE_ReleaseConn(struct vbc *vc)
-{
-
-	CHECK_OBJ_NOTNULL(vc, VBC_MAGIC);
-	assert(vc->backend == NULL);
-	assert(vc->fd < 0);
-
-	vc->addr = NULL;
-	vc->addrlen = 0;
-	vc->recycled = 0;
-	Lck_Lock(&VBE_mtx);
-	VSC_C_main->n_vbc--;
-	Lck_Unlock(&VBE_mtx);
-	FREE_OBJ(vc);
-}
-
-#define FIND_TMO(tmx, dst, sp, be)		\
-	do {					\
-		dst = sp->wrk->tmx;		\
-		if (dst == 0.0)			\
-			dst = be->tmx;		\
-		if (dst == 0.0)			\
-			dst = cache_param->tmx;	\
-	} while (0)
-
-/*--------------------------------------------------------------------
- * Attempt to connect to a given addrinfo entry.
- *
- * Must be called with locked backend, but will release the backend
- * lock during the slow/sleeping stuff, so that other worker threads
- * can have a go, while we ponder.
- *
- */
-
-static int
-vbe_TryConnect(const struct sess *sp, int pf, const struct sockaddr_storage *sa,
-    socklen_t salen, const struct vdi_simple *vs)
-{
-	int s, i, tmo;
-	double tmod;
-
-	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
-	CHECK_OBJ_NOTNULL(vs, VDI_SIMPLE_MAGIC);
-
-	s = socket(pf, SOCK_STREAM, 0);
-	if (s < 0)
-		return (s);
-
-	FIND_TMO(connect_timeout, tmod, sp, vs->vrt);
-
-	tmo = (int)(tmod * 1000.0);
-
-	i = VTCP_connect(s, sa, salen, tmo);
-
-	if (i != 0) {
-		AZ(close(s));
-		return (-1);
-	}
-
-	return (s);
-}
-
-/*--------------------------------------------------------------------*/
-
-static void
-bes_conn_try(const struct sess *sp, struct vbc *vc, const struct vdi_simple *vs)
-{
-	int s;
-	struct backend *bp = vs->backend;
-	char abuf1[VTCP_ADDRBUFSIZE];
-	char pbuf1[VTCP_PORTBUFSIZE];
-
-	CHECK_OBJ_NOTNULL(vs, VDI_SIMPLE_MAGIC);
-
-	Lck_Lock(&bp->mtx);
-	bp->refcount++;
-	bp->n_conn++;		/* It mostly works */
-	Lck_Unlock(&bp->mtx);
-
-	s = -1;
-	assert(bp->ipv6 != NULL || bp->ipv4 != NULL);
-
-	/* release lock during stuff that can take a long time */
-
-	if (cache_param->prefer_ipv6 && bp->ipv6 != NULL) {
-		s = vbe_TryConnect(sp, PF_INET6, bp->ipv6, bp->ipv6len, vs);
-		vc->addr = bp->ipv6;
-		vc->addrlen = bp->ipv6len;
-	}
-	if (s == -1 && bp->ipv4 != NULL) {
-		s = vbe_TryConnect(sp, PF_INET, bp->ipv4, bp->ipv4len, vs);
-		vc->addr = bp->ipv4;
-		vc->addrlen = bp->ipv4len;
-	}
-	if (s == -1 && !cache_param->prefer_ipv6 && bp->ipv6 != NULL) {
-		s = vbe_TryConnect(sp, PF_INET6, bp->ipv6, bp->ipv6len, vs);
-		vc->addr = bp->ipv6;
-		vc->addrlen = bp->ipv6len;
-	}
-
-	vc->fd = s;
-	if (s < 0) {
-		Lck_Lock(&bp->mtx);
-		bp->n_conn--;
-		bp->refcount--;		/* Only keep ref on success */
-		Lck_Unlock(&bp->mtx);
-		vc->addr = NULL;
-		vc->addrlen = 0;
-	} else {
-		vc->vsl_id = s | VSL_BACKENDMARKER;
-		VTCP_myname(s, abuf1, sizeof abuf1, pbuf1, sizeof pbuf1);
-		WSL(sp->wrk, SLT_BackendOpen, vc->vsl_id, "%s %s %s ",
-		    vs->backend->display_name, abuf1, pbuf1);
-	}
-
-}
-
-/*--------------------------------------------------------------------
- * Check that there is still something at the far end of a given socket.
- * We poll the fd with instant timeout, if there are any events we can't
- * use it (backends are not allowed to pipeline).
- */
-
-static int
-vbe_CheckFd(int fd)
-{
-	struct pollfd pfd;
-
-	pfd.fd = fd;
-	pfd.events = POLLIN;
-	pfd.revents = 0;
-	return(poll(&pfd, 1, 0) == 0);
-}
-
-/*--------------------------------------------------------------------
- * Manage a pool of vbc structures.
- * XXX: as an experiment, make this caching controled by a parameter
- * XXX: so we can see if it has any effect.
- */
-
-static struct vbc *
-vbe_NewConn(void)
-{
-	struct vbc *vc;
-
-	ALLOC_OBJ(vc, VBC_MAGIC);
-	XXXAN(vc);
-	vc->fd = -1;
-	Lck_Lock(&VBE_mtx);
-	VSC_C_main->n_vbc++;
-	Lck_Unlock(&VBE_mtx);
-	return (vc);
-}
-
-/*--------------------------------------------------------------------
- * It evaluates if a backend is healthy _for_a_specific_object_.
- * That means that it relies on sp->objcore->objhead. This is mainly for
- * saint-mode, but also takes backend->healthy into account. If
- * cache_param->saintmode_threshold is 0, this is basically just a test of
- * backend->healthy.
- *
- * The threshold has to be evaluated _after_ the timeout check, otherwise
- * items would never time out once the threshold is reached.
- */
-
-static unsigned int
-vbe_Healthy(const struct vdi_simple *vs, const struct sess *sp)
-{
-	struct trouble *tr;
-	struct trouble *tr2;
-	struct trouble *old;
-	unsigned i = 0, retval;
-	unsigned int threshold;
-	struct backend *backend;
-	uintptr_t target;
-	double now;
-
-	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
-	CHECK_OBJ_NOTNULL(vs, VDI_SIMPLE_MAGIC);
-	backend = vs->backend;
-	CHECK_OBJ_NOTNULL(backend, BACKEND_MAGIC);
-
-	if (backend->admin_health == ah_probe && !backend->healthy)
-		return (0);
-
-	if (backend->admin_health == ah_sick)
-		return (0);
-
-	/* VRT/VCC sets threshold to UINT_MAX to mark that it's not
-	 * specified by VCL (thus use param).
-	 */
-	if (vs->vrt->saintmode_threshold == UINT_MAX)
-		threshold = cache_param->saintmode_threshold;
-	else
-		threshold = vs->vrt->saintmode_threshold;
-
-	if (backend->admin_health == ah_healthy)
-		threshold = UINT_MAX;
-
-	/* Saintmode is disabled */
-	if (threshold == 0)
-		return (1);
-
-	if (sp->objcore == NULL)
-		return (1);
-
-	now = sp->t_req;
-	target = (uintptr_t)(sp->objcore->objhead);
-
-	old = NULL;
-	retval = 1;
-	Lck_Lock(&backend->mtx);
-	VTAILQ_FOREACH_SAFE(tr, &backend->troublelist, list, tr2) {
-		CHECK_OBJ_NOTNULL(tr, TROUBLE_MAGIC);
-
-		if (tr->timeout < now) {
-			VTAILQ_REMOVE(&backend->troublelist, tr, list);
-			old = tr;
-			retval = 1;
-			break;
-		}
-
-		if (tr->target == target) {
-			retval = 0;
-			break;
-		}
-
-		/* If the threshold is at 1, a single entry on the list
-		 * will disable the backend. Since 0 is disable, ++i
-		 * instead of i++ to allow this behavior.
-		 */
-		if (++i >= threshold) {
-			retval = 0;
-			break;
-		}
-	}
-	Lck_Unlock(&backend->mtx);
-
-	if (old != NULL)
-		FREE_OBJ(old);
-
-	return (retval);
-}
-
-/*--------------------------------------------------------------------
- * Get a connection to a particular backend.
- */
-
-static struct vbc *
-vbe_GetVbe(const struct sess *sp, struct vdi_simple *vs)
-{
-	struct vbc *vc;
-	struct backend *bp;
-
-	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
-	CHECK_OBJ_NOTNULL(vs, VDI_SIMPLE_MAGIC);
-	bp = vs->backend;
-	CHECK_OBJ_NOTNULL(bp, BACKEND_MAGIC);
-
-	/* first look for vbc's we can recycle */
-	while (1) {
-		Lck_Lock(&bp->mtx);
-		vc = VTAILQ_FIRST(&bp->connlist);
-		if (vc != NULL) {
-			bp->refcount++;
-			assert(vc->backend == bp);
-			assert(vc->fd >= 0);
-			AN(vc->addr);
-			VTAILQ_REMOVE(&bp->connlist, vc, list);
-		}
-		Lck_Unlock(&bp->mtx);
-		if (vc == NULL)
-			break;
-		if (vbe_CheckFd(vc->fd)) {
-			/* XXX locking of stats */
-			VSC_C_main->backend_reuse += 1;
-			WSP(sp, SLT_Backend, "%d %s %s",
-			    vc->fd, sp->director->vcl_name, bp->display_name);
-			vc->vdis = vs;
-			vc->recycled = 1;
-			return (vc);
-		}
-		VSC_C_main->backend_toolate++;
-		WSL(sp->wrk, SLT_BackendClose, vc->vsl_id, "%s", bp->display_name);
-
-		/* Checkpoint log to flush all info related to this connection
-		   before the OS reuses the FD */
-		WSL_Flush(sp->wrk, 0);
-
-		VTCP_close(&vc->fd);
-		VBE_DropRefConn(bp);
-		vc->backend = NULL;
-		VBE_ReleaseConn(vc);
-	}
-
-	if (!vbe_Healthy(vs, sp)) {
-		VSC_C_main->backend_unhealthy++;
-		return (NULL);
-	}
-
-	if (vs->vrt->max_connections > 0 &&
-	    bp->n_conn >= vs->vrt->max_connections) {
-		VSC_C_main->backend_busy++;
-		return (NULL);
-	}
-
-	vc = vbe_NewConn();
-	assert(vc->fd == -1);
-	AZ(vc->backend);
-	bes_conn_try(sp, vc, vs);
-	if (vc->fd < 0) {
-		VBE_ReleaseConn(vc);
-		VSC_C_main->backend_fail++;
-		return (NULL);
-	}
-	vc->backend = bp;
-	VSC_C_main->backend_conn++;
-	WSP(sp, SLT_Backend, "%d %s %s",
-	    vc->fd, sp->director->vcl_name, bp->display_name);
-	vc->vdis = vs;
-	return (vc);
-}
-
-/*--------------------------------------------------------------------
- * Returns the backend if and only if the this is a simple director.
- * XXX: Needs a better name and possibly needs a better general approach.
- * XXX: This is mainly used by the DNS director to fetch the actual backend
- * XXX: so it can compare DNS lookups with the actual IP.
- */
-
-struct backend *
-vdi_get_backend_if_simple(const struct director *d)
-{
-	CHECK_OBJ_NOTNULL(d, DIRECTOR_MAGIC);
-	struct vdi_simple *vs, *vs2;
-
-	vs2 = d->priv;
-	if (vs2->magic != VDI_SIMPLE_MAGIC)
-		return (NULL);
-	CAST_OBJ_NOTNULL(vs, d->priv, VDI_SIMPLE_MAGIC);
-	return (vs->backend);
-}
-
-/*--------------------------------------------------------------------
- *
- */
-
-void
-VBE_UseHealth(const struct director *vdi)
-{
-	struct vdi_simple *vs;
-
-	ASSERT_CLI();
-
-	if (strcmp(vdi->name, "simple"))
-		return;
-	CAST_OBJ_NOTNULL(vs, vdi->priv, VDI_SIMPLE_MAGIC);
-	if (vs->vrt->probe == NULL)
-		return;
-	VBP_Use(vs->backend, vs->vrt->probe);
-}
-
-/*--------------------------------------------------------------------
- *
- */
-
-static struct vbc * __match_proto__(vdi_getfd_f)
-vdi_simple_getfd(const struct director *d, struct sess *sp)
-{
-	struct vdi_simple *vs;
-	struct vbc *vc;
-
-	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
-	CHECK_OBJ_NOTNULL(d, DIRECTOR_MAGIC);
-	CAST_OBJ_NOTNULL(vs, d->priv, VDI_SIMPLE_MAGIC);
-	vc = vbe_GetVbe(sp, vs);
-	if (vc != NULL) {
-		FIND_TMO(first_byte_timeout,
-		    vc->first_byte_timeout, sp, vs->vrt);
-		FIND_TMO(between_bytes_timeout,
-		    vc->between_bytes_timeout, sp, vs->vrt);
-	}
-	return (vc);
-}
-
-static unsigned
-vdi_simple_healthy(const struct director *d, const struct sess *sp)
-{
-	struct vdi_simple *vs;
-
-	CHECK_OBJ_NOTNULL(d, DIRECTOR_MAGIC);
-	CAST_OBJ_NOTNULL(vs, d->priv, VDI_SIMPLE_MAGIC);
-	return (vbe_Healthy(vs, sp));
-}
-
-static void
-vdi_simple_fini(const struct director *d)
-{
-	struct vdi_simple *vs;
-
-	ASSERT_CLI();
-	CHECK_OBJ_NOTNULL(d, DIRECTOR_MAGIC);
-	CAST_OBJ_NOTNULL(vs, d->priv, VDI_SIMPLE_MAGIC);
-
-	if (vs->vrt->probe != NULL)
-		VBP_Remove(vs->backend, vs->vrt->probe);
-	VBE_DropRefVcl(vs->backend);
-	free(vs->dir.vcl_name);
-	vs->dir.magic = 0;
-	FREE_OBJ(vs);
-}
-
-void
-VRT_init_dir_simple(struct cli *cli, struct director **bp, int idx,
-    const void *priv)
-{
-	const struct vrt_backend *t;
-	struct vdi_simple *vs;
-
-	ASSERT_CLI();
-	(void)cli;
-	t = priv;
-
-	ALLOC_OBJ(vs, VDI_SIMPLE_MAGIC);
-	XXXAN(vs);
-	vs->dir.magic = DIRECTOR_MAGIC;
-	vs->dir.priv = vs;
-	vs->dir.name = "simple";
-	REPLACE(vs->dir.vcl_name, t->vcl_name);
-	vs->dir.getfd = vdi_simple_getfd;
-	vs->dir.fini = vdi_simple_fini;
-	vs->dir.healthy = vdi_simple_healthy;
-
-	vs->vrt = t;
-
-	vs->backend = VBE_AddBackend(cli, t);
-	if (vs->vrt->probe != NULL)
-		VBP_Insert(vs->backend, vs->vrt->probe, vs->vrt->hosthdr);
-
-	bp[idx] = &vs->dir;
-}
diff --git a/bin/varnishd/cache_backend.h b/bin/varnishd/cache_backend.h
deleted file mode 100644
index 72a1283..0000000
--- a/bin/varnishd/cache_backend.h
+++ /dev/null
@@ -1,192 +0,0 @@
-/*-
- * Copyright (c) 2006 Verdens Gang AS
- * Copyright (c) 2006-2011 Varnish Software AS
- * All rights reserved.
- *
- * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- *
- * This is the central switch-board for backend connections and it is
- * slightly complicated by a number of optimizations.
- *
- * The data structures:
- *
- *    A vrt_backend is a definition of a backend in a VCL program.
- *
- *    A backend is a TCP destination, possibly multi-homed and it has a
- *    number of associated properties and statistics.
- *
- *    A vbc is an open TCP connection to a backend.
- *
- *    A bereq is a memory carrier for handling a HTTP transaction with
- *    a backend over a vbc.
- *
- *    A director is a piece of code that selects which backend to use,
- *    by whatever method or metric it chooses.
- *
- * The relationships:
- *
- *    Backends and directors get instantiated when VCL's are loaded,
- *    and this always happen in the CLI thread.
- *
- *    When a VCL tries to instantiate a backend, any existing backend
- *    with the same identity (== definition in VCL) will be used instead
- *    so that vbc's can be reused across VCL changes.
- *
- *    Directors disapper with the VCL that created them.
- *
- *    Backends disappear when their reference count drop to zero.
- *
- *    Backends have their host/port name looked up to addrinfo structures
- *    when they are instantiated, and we just cache that result and cycle
- *    through the entries (for multihomed backends) on failure only.
- *    XXX: add cli command to redo lookup.
- *
- *    bereq is sort of a step-child here, we just manage the pool of them.
- *
- */
-
-struct vbp_target;
-struct vbc;
-struct vrt_backend_probe;
-
-/*--------------------------------------------------------------------
- * A director is a piece of code which selects one of possibly multiple
- * backends to use.
- */
-
-typedef struct vbc *vdi_getfd_f(const struct director *, struct sess *sp);
-typedef void vdi_fini_f(const struct director *);
-typedef unsigned vdi_healthy(const struct director *, const struct sess *sp);
-
-struct director {
-	unsigned		magic;
-#define DIRECTOR_MAGIC		0x3336351d
-	const char		*name;
-	char			*vcl_name;
-	vdi_getfd_f		*getfd;
-	vdi_fini_f		*fini;
-	vdi_healthy		*healthy;
-	void			*priv;
-};
-
-/*--------------------------------------------------------------------
- * List of objectheads that have recently been rejected by VCL.
- */
-
-struct trouble {
-	unsigned		magic;
-#define TROUBLE_MAGIC		0x4211ab21
-	uintptr_t		target;
-	double			timeout;
-	VTAILQ_ENTRY(trouble)	list;
-};
-
-/*--------------------------------------------------------------------
- * An instance of a backend from a VCL program.
- */
-
-enum admin_health {
-	ah_invalid = 0,
-	ah_healthy,
-	ah_sick,
-	ah_probe
-};
-
-struct backend {
-	unsigned		magic;
-#define BACKEND_MAGIC		0x64c4c7c6
-
-	VTAILQ_ENTRY(backend)	list;
-	int			refcount;
-	struct lock		mtx;
-
-	char			*vcl_name;
-	char			*display_name;
-	char			*ipv4_addr;
-	char			*ipv6_addr;
-	char			*port;
-
-	struct sockaddr_storage	*ipv4;
-	socklen_t		ipv4len;
-	struct sockaddr_storage	*ipv6;
-	socklen_t		ipv6len;
-
-	unsigned		n_conn;
-	VTAILQ_HEAD(, vbc)	connlist;
-
-	struct vbp_target	*probe;
-	unsigned		healthy;
-	enum admin_health	admin_health;
-	VTAILQ_HEAD(, trouble)	troublelist;
-
-	struct VSC_C_vbe	*vsc;
-};
-
-/* -------------------------------------------------------------------*/
-
-/* Backend connection */
-struct vbc {
-	unsigned		magic;
-#define VBC_MAGIC		0x0c5e6592
-	VTAILQ_ENTRY(vbc)	list;
-	struct backend		*backend;
-	struct vdi_simple	*vdis;
-	unsigned		vsl_id;
-	int			fd;
-
-	struct sockaddr_storage	*addr;
-	socklen_t		addrlen;
-
-	uint8_t			recycled;
-
-	/* Timeouts */
-	double			first_byte_timeout;
-	double			between_bytes_timeout;
-};
-
-/* cache_backend.c */
-void VBE_ReleaseConn(struct vbc *vc);
-struct backend *vdi_get_backend_if_simple(const struct director *d);
-
-/* cache_backend_cfg.c */
-extern struct lock VBE_mtx;
-void VBE_DropRefConn(struct backend *);
-void VBE_DropRefVcl(struct backend *);
-void VBE_DropRefLocked(struct backend *b);
-
-/* cache_backend_poll.c */
-void VBP_Insert(struct backend *b, struct vrt_backend_probe const *p, const char *hosthdr);
-void VBP_Remove(struct backend *b, struct vrt_backend_probe const *p);
-void VBP_Use(const struct backend *b, const struct vrt_backend_probe const *p);
-void VBP_Summary(struct cli *cli, const struct vbp_target *vt);
-
-/* Init functions for directors */
-typedef void dir_init_f(struct cli *, struct director **, int , const void*);
-dir_init_f VRT_init_dir_simple;
-dir_init_f VRT_init_dir_dns;
-dir_init_f VRT_init_dir_hash;
-dir_init_f VRT_init_dir_random;
-dir_init_f VRT_init_dir_round_robin;
-dir_init_f VRT_init_dir_fallback;
-dir_init_f VRT_init_dir_client;
diff --git a/bin/varnishd/cache_backend_cfg.c b/bin/varnishd/cache_backend_cfg.c
deleted file mode 100644
index cbb8c85..0000000
--- a/bin/varnishd/cache_backend_cfg.c
+++ /dev/null
@@ -1,507 +0,0 @@
-/*-
- * Copyright (c) 2006 Verdens Gang AS
- * Copyright (c) 2006-2011 Varnish Software AS
- * All rights reserved.
- *
- * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- *
- * Handle configuration of backends from VCL programs.
- *
- */
-
-#include "config.h"
-
-#include <ctype.h>
-#include <stdio.h>
-#include <stdlib.h>
-
-#include "cache.h"
-
-#include "cache_backend.h"
-#include "vcli.h"
-#include "vcli_priv.h"
-#include "vrt.h"
-
-struct lock VBE_mtx;
-
-
-/*
- * The list of backends is not locked, it is only ever accessed from
- * the CLI thread, so there is no need.
- */
-static VTAILQ_HEAD(, backend) backends = VTAILQ_HEAD_INITIALIZER(backends);
-
-/*--------------------------------------------------------------------
- */
-
-static void
-VBE_Nuke(struct backend *b)
-{
-
-	ASSERT_CLI();
-	VTAILQ_REMOVE(&backends, b, list);
-	free(b->ipv4);
-	free(b->ipv4_addr);
-	free(b->ipv6);
-	free(b->ipv6_addr);
-	free(b->port);
-	VSM_Free(b->vsc);
-	FREE_OBJ(b);
-	VSC_C_main->n_backend--;
-}
-
-/*--------------------------------------------------------------------
- */
-
-void
-VBE_Poll(void)
-{
-	struct backend *b, *b2;
-
-	ASSERT_CLI();
-	VTAILQ_FOREACH_SAFE(b, &backends, list, b2) {
-		assert(
-			b->admin_health == ah_healthy ||
-			b->admin_health == ah_sick ||
-			b->admin_health == ah_probe
-		);
-		if (b->refcount == 0 && b->probe == NULL)
-			VBE_Nuke(b);
-	}
-}
-
-/*--------------------------------------------------------------------
- * Drop a reference to a backend.
- * The last reference must come from the watcher in the CLI thread,
- * as only that thread is allowed to clean up the backend list.
- */
-
-void
-VBE_DropRefLocked(struct backend *b)
-{
-	int i;
-	struct vbc *vbe, *vbe2;
-
-	CHECK_OBJ_NOTNULL(b, BACKEND_MAGIC);
-	assert(b->refcount > 0);
-
-	i = --b->refcount;
-	Lck_Unlock(&b->mtx);
-	if (i > 0)
-		return;
-
-	ASSERT_CLI();
-	VTAILQ_FOREACH_SAFE(vbe, &b->connlist, list, vbe2) {
-		VTAILQ_REMOVE(&b->connlist, vbe, list);
-		if (vbe->fd >= 0) {
-			AZ(close(vbe->fd));
-			vbe->fd = -1;
-		}
-		vbe->backend = NULL;
-		VBE_ReleaseConn(vbe);
-	}
-	VBE_Nuke(b);
-}
-
-void
-VBE_DropRefVcl(struct backend *b)
-{
-
-	CHECK_OBJ_NOTNULL(b, BACKEND_MAGIC);
-
-	Lck_Lock(&b->mtx);
-	b->vsc->vcls--;
-	VBE_DropRefLocked(b);
-}
-
-void
-VBE_DropRefConn(struct backend *b)
-{
-
-	CHECK_OBJ_NOTNULL(b, BACKEND_MAGIC);
-
-	Lck_Lock(&b->mtx);
-	assert(b->n_conn > 0);
-	b->n_conn--;
-	VBE_DropRefLocked(b);
-}
-
-/*--------------------------------------------------------------------
- * See lib/libvcl/vcc_backend.c::emit_sockaddr()
- */
-
-static void
-copy_sockaddr(struct sockaddr_storage **sa, socklen_t *len,
-    const unsigned char *src)
-{
-
-	assert(*src > 0);
-	*sa = calloc(sizeof **sa, 1);
-	XXXAN(*sa);
-	memcpy(*sa, src + 1, *src);
-	*len = *src;
-}
-
-/*--------------------------------------------------------------------
- * Add a backend/director instance when loading a VCL.
- * If an existing backend is matched, grab a refcount and return.
- * Else create a new backend structure with reference initialized to one.
- */
-
-struct backend *
-VBE_AddBackend(struct cli *cli, const struct vrt_backend *vb)
-{
-	struct backend *b;
-	char buf[128];
-
-	AN(vb->vcl_name);
-	assert(vb->ipv4_sockaddr != NULL || vb->ipv6_sockaddr != NULL);
-	(void)cli;
-	ASSERT_CLI();
-
-	/* Run through the list and see if we already have this backend */
-	VTAILQ_FOREACH(b, &backends, list) {
-		CHECK_OBJ_NOTNULL(b, BACKEND_MAGIC);
-		if (strcmp(b->vcl_name, vb->vcl_name))
-			continue;
-		if (vb->ipv4_sockaddr != NULL && (
-		    b->ipv4len != vb->ipv4_sockaddr[0] ||
-		    memcmp(b->ipv4, vb->ipv4_sockaddr + 1, b->ipv4len)))
-			continue;
-		if (vb->ipv6_sockaddr != NULL && (
-		    b->ipv6len != vb->ipv6_sockaddr[0] ||
-		    memcmp(b->ipv6, vb->ipv6_sockaddr + 1, b->ipv6len)))
-			continue;
-		b->refcount++;
-		b->vsc->vcls++;
-		return (b);
-	}
-
-	/* Create new backend */
-	ALLOC_OBJ(b, BACKEND_MAGIC);
-	XXXAN(b);
-	Lck_New(&b->mtx, lck_backend);
-	b->refcount = 1;
-
-	bprintf(buf, "%s(%s,%s,%s)",
-	    vb->vcl_name,
-	    vb->ipv4_addr == NULL ? "" : vb->ipv4_addr,
-	    vb->ipv6_addr == NULL ? "" : vb->ipv6_addr, vb->port);
-
-	b->vsc = VSM_Alloc(sizeof *b->vsc, VSC_CLASS, VSC_TYPE_VBE, buf);
-	b->vsc->vcls++;
-
-	VTAILQ_INIT(&b->connlist);
-
-	VTAILQ_INIT(&b->troublelist);
-
-	/*
-	 * This backend may live longer than the VCL that instantiated it
-	 * so we cannot simply reference the VCL's copy of things.
-	 */
-	REPLACE(b->vcl_name, vb->vcl_name);
-	REPLACE(b->display_name, buf);
-	REPLACE(b->ipv4_addr, vb->ipv4_addr);
-	REPLACE(b->ipv6_addr, vb->ipv6_addr);
-	REPLACE(b->port, vb->port);
-
-	/*
-	 * Copy over the sockaddrs
-	 */
-	if (vb->ipv4_sockaddr != NULL)
-		copy_sockaddr(&b->ipv4, &b->ipv4len, vb->ipv4_sockaddr);
-	if (vb->ipv6_sockaddr != NULL)
-		copy_sockaddr(&b->ipv6, &b->ipv6len, vb->ipv6_sockaddr);
-
-	assert(b->ipv4 != NULL || b->ipv6 != NULL);
-
-	b->healthy = 1;
-	b->admin_health = ah_probe;
-
-	VTAILQ_INSERT_TAIL(&backends, b, list);
-	VSC_C_main->n_backend++;
-	return (b);
-}
-
-/*--------------------------------------------------------------------*/
-
-void
-VRT_init_dir(struct cli *cli, struct director **dir, const char *name,
-    int idx, const void *priv)
-{
-
-	ASSERT_CLI();
-	if (!strcmp(name, "simple"))
-		VRT_init_dir_simple(cli, dir, idx, priv);
-	else if (!strcmp(name, "hash"))
-		VRT_init_dir_hash(cli, dir, idx, priv);
-	else if (!strcmp(name, "random"))
-		VRT_init_dir_random(cli, dir, idx, priv);
-	else if (!strcmp(name, "dns"))
-		VRT_init_dir_dns(cli, dir, idx, priv);
-	else if (!strcmp(name, "round-robin"))
-		VRT_init_dir_round_robin(cli, dir, idx, priv);
-	else if (!strcmp(name, "fallback"))
-		VRT_init_dir_fallback(cli, dir, idx, priv);
-	else if (!strcmp(name, "client"))
-		VRT_init_dir_client(cli, dir, idx, priv);
-	else
-		INCOMPL();
-}
-
-void
-VRT_fini_dir(struct cli *cli, struct director *b)
-{
-
-	(void)cli;
-	ASSERT_CLI();
-	CHECK_OBJ_NOTNULL(b, DIRECTOR_MAGIC);
-	b->fini(b);
-	b->priv = NULL;
-}
-
-/*---------------------------------------------------------------------
- * String to admin_health
- */
-
-static enum admin_health
-vbe_str2adminhealth(const char *wstate)
-{
-
-	if (strcasecmp(wstate, "healthy") == 0)
-		return(ah_healthy);
-	if (strcasecmp(wstate, "sick") == 0)
-		return(ah_sick);
-	if (strcmp(wstate, "auto") == 0)
-		return(ah_probe);
-	return (ah_invalid);
-}
-
-/*---------------------------------------------------------------------
- * A general function for finding backends and doing things with them.
- *
- * Return -1 on match-argument parse errors.
- *
- * If the call-back function returns non-zero, the search is terminated
- * and we relay that return value.
- *
- * Otherwise we return the number of matches.
- */
-
-typedef int bf_func(struct cli *cli, struct backend *b, void *priv);
-
-static int
-backend_find(struct cli *cli, const char *matcher, bf_func *func, void *priv)
-{
-	struct backend *b;
-	const char *s;
-	const char *name_b;
-	ssize_t name_l = 0;
-	const char *ip_b = NULL;
-	ssize_t ip_l = 0;
-	const char *port_b = NULL;
-	ssize_t port_l = 0;
-	int found = 0;
-	int i;
-
-	name_b = matcher;
-	if (matcher != NULL) {
-		s = strchr(matcher,'(');
-
-		if (s != NULL)
-			name_l = s - name_b;
-		else
-			name_l = strlen(name_b);
-
-		if (s != NULL) {
-			s++;
-			while (isspace(*s))
-				s++;
-			ip_b = s;
-			while (*s != '\0' &&
-			    *s != ')' &&
-			    *s != ':' &&
-			    !isspace(*s))
-				s++;
-			ip_l = s - ip_b;
-			while (isspace(*s))
-				s++;
-			if (*s == ':') {
-				s++;
-				while (isspace(*s))
-					s++;
-				port_b = s;
-				while (*s != '\0' && *s != ')' && !isspace(*s))
-					s++;
-				port_l = s - port_b;
-			}
-			while (isspace(*s))
-				s++;
-			if (*s != ')') {
-				VCLI_Out(cli,
-				    "Match string syntax error:"
-				    " ')' not found.");
-				VCLI_SetResult(cli, CLIS_CANT);
-				return (-1);
-			}
-			s++;
-			while (isspace(*s))
-				s++;
-			if (*s != '\0') {
-				VCLI_Out(cli,
-				    "Match string syntax error:"
-				    " junk after ')'");
-				VCLI_SetResult(cli, CLIS_CANT);
-				return (-1);
-			}
-		}
-	}
-	VTAILQ_FOREACH(b, &backends, list) {
-		CHECK_OBJ_NOTNULL(b, BACKEND_MAGIC);
-		if (port_b != NULL && strncmp(b->port, port_b, port_l) != 0)
-			continue;
-		if (name_b != NULL && strncmp(b->vcl_name, name_b, name_l) != 0)
-			continue;
-		if (ip_b != NULL &&
-		    (b->ipv4_addr == NULL ||
-		      strncmp(b->ipv4_addr, ip_b, ip_l)) &&
-		    (b->ipv6_addr == NULL ||
-		      strncmp(b->ipv6_addr, ip_b, ip_l)))
-			continue;
-		found++;
-		i = func(cli, b, priv);
-		if (i)
-			return(i);
-	}
-	return (found);
-}
-
-/*---------------------------------------------------------------------*/
-
-static int __match_proto__()
-do_list(struct cli *cli, struct backend *b, void *priv)
-{
-	int *hdr;
-
-	AN(priv);
-	hdr = priv;
-	if (!*hdr) {
-		VCLI_Out(cli, "%-30s %-6s %-10s %s",
-		    "Backend name", "Refs", "Admin", "Probe");
-		*hdr = 1;
-	}
-	CHECK_OBJ_NOTNULL(b, BACKEND_MAGIC);
-
-	VCLI_Out(cli, "\n%-30s %-6d", b->display_name, b->refcount);
-
-	if (b->admin_health == ah_probe)
-		VCLI_Out(cli, " %-10s", "probe");
-	else if (b->admin_health == ah_sick)
-		VCLI_Out(cli, " %-10s", "sick");
-	else if (b->admin_health == ah_healthy)
-		VCLI_Out(cli, " %-10s", "healthy");
-	else
-		VCLI_Out(cli, " %-10s", "invalid");
-
-	if (b->probe == NULL)
-		VCLI_Out(cli, " %s", "Healthy (no probe)");
-	else {
-		if (b->healthy)
-			VCLI_Out(cli, " %s", "Healthy ");
-		else
-			VCLI_Out(cli, " %s", "Sick ");
-		VBP_Summary(cli, b->probe);
-	}
-
-	return (0);
-}
-
-static void
-cli_backend_list(struct cli *cli, const char * const *av, void *priv)
-{
-	int hdr = 0;
-
-	(void)av;
-	(void)priv;
-	ASSERT_CLI();
-	(void)backend_find(cli, av[2], do_list, &hdr);
-}
-
-/*---------------------------------------------------------------------*/
-
-static int __match_proto__()
-do_set_health(struct cli *cli, struct backend *b, void *priv)
-{
-	enum admin_health state;
-
-	(void)cli;
-	state = *(enum admin_health*)priv;
-	CHECK_OBJ_NOTNULL(b, BACKEND_MAGIC);
-	b->admin_health = state;
-	return (0);
-}
-
-static void
-cli_backend_set_health(struct cli *cli, const char * const *av, void *priv)
-{
-	enum admin_health state;
-	int n;
-
-	(void)av;
-	(void)priv;
-	ASSERT_CLI();
-	AN(av[2]);
-	AN(av[3]);
-	state = vbe_str2adminhealth(av[3]);
-	if (state == ah_invalid) {
-		VCLI_Out(cli, "Invalid state %s", av[3]);
-		VCLI_SetResult(cli, CLIS_PARAM);
-		return;
-	}
-	n = backend_find(cli, av[2], do_set_health, &state);
-	if (n == 0) {
-		VCLI_Out(cli, "No Backends matches");
-		VCLI_SetResult(cli, CLIS_PARAM);
-	}
-}
-
-/*---------------------------------------------------------------------*/
-
-static struct cli_proto backend_cmds[] = {
-	{ "backend.list", "backend.list",
-	    "\tList all backends\n", 0, 1, "d", cli_backend_list },
-	{ "backend.set_health", "backend.set_health matcher state",
-	    "\tShow a backend\n", 2, 2, "d", cli_backend_set_health },
-	{ NULL }
-};
-
-/*---------------------------------------------------------------------*/
-
-void
-VBE_Init(void)
-{
-
-	Lck_New(&VBE_mtx, lck_vbe);
-	CLI_AddFuncs(backend_cmds);
-}
diff --git a/bin/varnishd/cache_backend_poll.c b/bin/varnishd/cache_backend_poll.c
deleted file mode 100644
index efd64cb..0000000
--- a/bin/varnishd/cache_backend_poll.c
+++ /dev/null
@@ -1,597 +0,0 @@
-/*-
- * Copyright (c) 2006 Verdens Gang AS
- * Copyright (c) 2006-2011 Varnish Software AS
- * All rights reserved.
- *
- * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- *
- * Poll backends for collection of health statistics
- *
- * We co-opt threads from the worker pool for probing the backends,
- * but we want to avoid a potentially messy cleanup operation when we
- * retire the backend, so the thread owns the health information, which
- * the backend references, rather than the other way around.
- *
- */
-
-#include "config.h"
-
-#include <math.h>
-#include <poll.h>
-#include <stdio.h>
-#include <stdlib.h>
-
-#include "cache.h"
-
-#include "cache_backend.h"
-#include "vcli_priv.h"
-#include "vrt.h"
-#include "vtcp.h"
-#include "vtim.h"
-
-/* Default averaging rate, we want something pretty responsive */
-#define AVG_RATE			4
-
-struct vbp_vcl {
-	unsigned			magic;
-#define VBP_VCL_MAGIC			0x70829764
-
-	VTAILQ_ENTRY(vbp_vcl)		list;
-	const struct vrt_backend_probe	*probep;
-	struct vrt_backend_probe	probe;
-	const char			*hosthdr;
-};
-
-struct vbp_target {
-	unsigned			magic;
-#define VBP_TARGET_MAGIC		0x6b7cb656
-
-	struct backend			*backend;
-	VTAILQ_HEAD( ,vbp_vcl)		vcls;
-
-	struct vrt_backend_probe	probe;
-	int				stop;
-	struct vsb			*vsb;
-	char				*req;
-	int				req_len;
-
-	char				resp_buf[128];
-	unsigned			good;
-
-	/* Collected statistics */
-#define BITMAP(n, c, t, b)	uint64_t	n;
-#include "tbl/backend_poll.h"
-#undef BITMAP
-
-	double				last;
-	double				avg;
-	double				rate;
-
-	VTAILQ_ENTRY(vbp_target)	list;
-	pthread_t			thread;
-};
-
-static VTAILQ_HEAD(, vbp_target)	vbp_list =
-    VTAILQ_HEAD_INITIALIZER(vbp_list);
-
-static struct lock			vbp_mtx;
-
-/*--------------------------------------------------------------------
- * Poke one backend, once, but possibly at both IPv4 and IPv6 addresses.
- *
- * We do deliberately not use the stuff in cache_backend.c, because we
- * want to measure the backends response without local distractions.
- */
-
-static int
-vbp_connect(int pf, const struct sockaddr_storage *sa, socklen_t salen, int tmo)
-{
-	int s, i;
-
-	s = socket(pf, SOCK_STREAM, 0);
-	if (s < 0)
-		return (s);
-
-	i = VTCP_connect(s, sa, salen, tmo);
-	if (i == 0)
-		return (s);
-	VTCP_close(&s);
-	return (-1);
-}
-
-static void
-vbp_poke(struct vbp_target *vt)
-{
-	int s, tmo, i;
-	double t_start, t_now, t_end;
-	unsigned rlen, resp;
-	struct backend *bp;
-	char buf[8192], *p;
-	struct pollfd pfda[1], *pfd = pfda;
-
-	bp = vt->backend;
-	CHECK_OBJ_NOTNULL(bp, BACKEND_MAGIC);
-
-	t_start = t_now = VTIM_real();
-	t_end = t_start + vt->probe.timeout;
-	tmo = (int)round((t_end - t_now) * 1e3);
-
-	s = -1;
-	if (cache_param->prefer_ipv6 && bp->ipv6 != NULL) {
-		s = vbp_connect(PF_INET6, bp->ipv6, bp->ipv6len, tmo);
-		t_now = VTIM_real();
-		tmo = (int)round((t_end - t_now) * 1e3);
-		if (s >= 0)
-			vt->good_ipv6 |= 1;
-	}
-	if (tmo > 0 && s < 0 && bp->ipv4 != NULL) {
-		s = vbp_connect(PF_INET, bp->ipv4, bp->ipv4len, tmo);
-		t_now = VTIM_real();
-		tmo = (int)round((t_end - t_now) * 1e3);
-		if (s >= 0)
-			vt->good_ipv4 |= 1;
-	}
-	if (tmo > 0 && s < 0 && bp->ipv6 != NULL) {
-		s = vbp_connect(PF_INET6, bp->ipv6, bp->ipv6len, tmo);
-		t_now = VTIM_real();
-		tmo = (int)round((t_end - t_now) * 1e3);
-		if (s >= 0)
-			vt->good_ipv6 |= 1;
-	}
-	if (s < 0) {
-		/* Got no connection: failed */
-		return;
-	}
-	if (tmo <= 0) {
-		/* Spent too long time getting it */
-		VTCP_close(&s);
-		return;
-	}
-
-	/* Send the request */
-	i = write(s, vt->req, vt->req_len);
-	if (i != vt->req_len) {
-		if (i < 0)
-			vt->err_xmit |= 1;
-		VTCP_close(&s);
-		return;
-	}
-	vt->good_xmit |= 1;
-
-	pfd->fd = s;
-	rlen = 0;
-	do {
-		pfd->events = POLLIN;
-		pfd->revents = 0;
-		tmo = (int)round((t_end - t_now) * 1e3);
-		if (tmo > 0)
-			i = poll(pfd, 1, tmo);
-		if (i == 0 || tmo <= 0) {
-			VTCP_close(&s);
-			return;
-		}
-		if (rlen < sizeof vt->resp_buf)
-			i = read(s, vt->resp_buf + rlen,
-			    sizeof vt->resp_buf - rlen);
-		else
-			i = read(s, buf, sizeof buf);
-		rlen += i;
-	} while (i > 0);
-
-	VTCP_close(&s);
-
-	if (i < 0) {
-		vt->err_recv |= 1;
-		return;
-	}
-
-	if (rlen == 0)
-		return;
-
-	/* So we have a good receive ... */
-	t_now = VTIM_real();
-	vt->last = t_now - t_start;
-	vt->good_recv |= 1;
-
-	/* Now find out if we like the response */
-	vt->resp_buf[sizeof vt->resp_buf - 1] = '\0';
-	p = strchr(vt->resp_buf, '\r');
-	if (p != NULL)
-		*p = '\0';
-	p = strchr(vt->resp_buf, '\n');
-	if (p != NULL)
-		*p = '\0';
-
-	i = sscanf(vt->resp_buf, "HTTP/%*f %u %s", &resp, buf);
-
-	if (i == 2 && resp == vt->probe.exp_status)
-		vt->happy |= 1;
-}
-
-/*--------------------------------------------------------------------
- * Record pokings...
- */
-
-static void
-vbp_start_poke(struct vbp_target *vt)
-{
-	CHECK_OBJ_NOTNULL(vt, VBP_TARGET_MAGIC);
-
-#define BITMAP(n, c, t, b)	vt->n <<= 1;
-#include "tbl/backend_poll.h"
-#undef BITMAP
-
-	vt->last = 0;
-	vt->resp_buf[0] = '\0';
-}
-
-static void
-vbp_has_poked(struct vbp_target *vt)
-{
-	unsigned i, j;
-	uint64_t u;
-	const char *logmsg;
-	char bits[10];
-
-	CHECK_OBJ_NOTNULL(vt, VBP_TARGET_MAGIC);
-
-	/* Calculate exponential average */
-	if (vt->happy & 1) {
-		if (vt->rate < AVG_RATE)
-			vt->rate += 1.0;
-		vt->avg += (vt->last - vt->avg) / vt->rate;
-	}
-
-	i = 0;
-#define BITMAP(n, c, t, b)	bits[i++] = (vt->n & 1) ? c : '-';
-#include "tbl/backend_poll.h"
-#undef BITMAP
-	bits[i] = '\0';
-
-	u = vt->happy;
-	for (i = j = 0; i < vt->probe.window; i++) {
-		if (u & 1)
-			j++;
-		u >>= 1;
-	}
-	vt->good = j;
-
-	if (vt->good >= vt->probe.threshold) {
-		if (vt->backend->healthy)
-			logmsg = "Still healthy";
-		else
-			logmsg = "Back healthy";
-		vt->backend->healthy = 1;
-	} else {
-		if (vt->backend->healthy)
-			logmsg = "Went sick";
-		else
-			logmsg = "Still sick";
-		vt->backend->healthy = 0;
-	}
-	VSL(SLT_Backend_health, 0, "%s %s %s %u %u %u %.6f %.6f %s",
-	    vt->backend->vcl_name, logmsg, bits,
-	    vt->good, vt->probe.threshold, vt->probe.window,
-	    vt->last, vt->avg, vt->resp_buf);
-	vt->backend->vsc->happy = vt->happy;
-}
-
-/*--------------------------------------------------------------------
- * Build request from probe spec
- */
-
-static void
-vbp_build_req(struct vsb *vsb, const struct vbp_vcl *vcl)
-{
-
-	XXXAN(vsb);
-	XXXAN(vcl);
-	VSB_clear(vsb);
-	if(vcl->probe.request != NULL) {
-		VSB_cat(vsb, vcl->probe.request);
-	} else {
-		VSB_printf(vsb, "GET %s HTTP/1.1\r\n",
-		    vcl->probe.url != NULL ?  vcl->probe.url : "/");
-		if (vcl->hosthdr != NULL)
-			VSB_printf(vsb, "Host: %s\r\n", vcl->hosthdr);
-		VSB_printf(vsb, "Connection: close\r\n");
-		VSB_printf(vsb, "\r\n");
-	}
-	AZ(VSB_finish(vsb));
-}
-
-/*--------------------------------------------------------------------
- * One thread per backend to be poked.
- */
-
-static void *
-vbp_wrk_poll_backend(void *priv)
-{
-	struct vbp_target *vt;
-	struct vbp_vcl *vcl = NULL;
-
-	THR_SetName("backend poll");
-
-	CAST_OBJ_NOTNULL(vt, priv, VBP_TARGET_MAGIC);
-
-	while (!vt->stop) {
-		Lck_Lock(&vbp_mtx);
-		if (VTAILQ_FIRST(&vt->vcls) != vcl) {
-			vcl = VTAILQ_FIRST(&vt->vcls);
-			vbp_build_req(vt->vsb, vcl);
-			vt->probe = vcl->probe;
-		}
-		Lck_Unlock(&vbp_mtx);
-
-		vt->req = VSB_data(vt->vsb);
-		vt->req_len = VSB_len(vt->vsb);
-
-		vbp_start_poke(vt);
-		vbp_poke(vt);
-		vbp_has_poked(vt);
-
-		if (!vt->stop)
-			VTIM_sleep(vt->probe.interval);
-	}
-	return (NULL);
-}
-
-/*--------------------------------------------------------------------
- * Cli functions
- */
-
-void
-VBP_Summary(struct cli *cli, const struct vbp_target *vt)
-{
-
-	CHECK_OBJ_NOTNULL(vt, VBP_TARGET_MAGIC);
-	VCLI_Out(cli, "%d/%d", vt->good, vt->probe.window);
-}
-
-static void
-vbp_bitmap(struct cli *cli, char c, uint64_t map, const char *lbl)
-{
-	int i;
-	uint64_t u = (1ULL << 63);
-
-	for (i = 0; i < 64; i++) {
-		if (map & u)
-			VCLI_Out(cli, "%c", c);
-		else
-			VCLI_Out(cli, "-");
-		map <<= 1;
-	}
-	VCLI_Out(cli, " %s\n", lbl);
-}
-
-/*lint -e{506} constant value boolean */
-/*lint -e{774} constant value boolean */
-static void
-vbp_health_one(struct cli *cli, const struct vbp_target *vt)
-{
-
-	VCLI_Out(cli, "Backend %s is %s\n",
-	    vt->backend->vcl_name,
-	    vt->backend->healthy ? "Healthy" : "Sick");
-	VCLI_Out(cli, "Current states  good: %2u threshold: %2u window: %2u\n",
-	    vt->good, vt->probe.threshold, vt->probe.window);
-	VCLI_Out(cli, "Average responsetime of good probes: %.6f\n", vt->avg);
-	VCLI_Out(cli,
-	    "Oldest                       "
-	    "                             Newest\n");
-	VCLI_Out(cli,
-	    "============================="
-	    "===================================\n");
-
-#define BITMAP(n, c, t, b)					\
-		if ((vt->n != 0) || (b))			\
-			vbp_bitmap(cli, (c), vt->n, (t));
-#include "tbl/backend_poll.h"
-#undef BITMAP
-}
-
-static void
-vbp_health(struct cli *cli, const char * const *av, void *priv)
-{
-	struct vbp_target *vt;
-
-	ASSERT_CLI();
-	(void)av;
-	(void)priv;
-
-	VTAILQ_FOREACH(vt, &vbp_list, list)
-		vbp_health_one(cli, vt);
-}
-
-static struct cli_proto debug_cmds[] = {
-	{ "debug.health", "debug.health",
-		"\tDump backend health stuff\n",
-		0, 0, "d", vbp_health },
-	{ NULL }
-};
-
-/*--------------------------------------------------------------------
- * A new VCL wants to probe this backend,
- */
-
-static struct vbp_vcl *
-vbp_new_vcl(const struct vrt_backend_probe *p, const char *hosthdr)
-{
-	struct vbp_vcl *vcl;
-
-	ALLOC_OBJ(vcl, VBP_VCL_MAGIC);
-	XXXAN(vcl);
-	vcl->probep = p;
-	vcl->probe = *p;
-	vcl->hosthdr = hosthdr;
-
-	/*
-	 * Sanitize and insert defaults
-	 * XXX: we could make these defaults parameters
-	 */
-	if (vcl->probe.timeout == 0.0)
-		vcl->probe.timeout = 2.0;
-	if (vcl->probe.interval == 0.0)
-		vcl->probe.interval = 5.0;
-	if (vcl->probe.window == 0)
-		vcl->probe.window = 8;
-	if (vcl->probe.threshold == 0)
-		vcl->probe.threshold = 3;
-	if (vcl->probe.exp_status == 0)
-		vcl->probe.exp_status = 200;
-
-	if (vcl->probe.threshold == ~0U)
-		vcl->probe.initial = vcl->probe.threshold - 1;
-
-	if (vcl->probe.initial > vcl->probe.threshold)
-		vcl->probe.initial = vcl->probe.threshold;
-	return (vcl);
-}
-
-/*--------------------------------------------------------------------
- * Insert/Remove/Use called from cache_backend.c
- */
-
-void
-VBP_Insert(struct backend *b, const struct vrt_backend_probe *p, const char *hosthdr)
-{
-	struct vbp_target *vt;
-	struct vbp_vcl *vcl;
-	int startthread = 0;
-	unsigned u;
-
-	ASSERT_CLI();
-	AN(p);
-
-	if (b->probe == NULL) {
-		ALLOC_OBJ(vt, VBP_TARGET_MAGIC);
-		XXXAN(vt);
-		VTAILQ_INIT(&vt->vcls);
-		vt->backend = b;
-		vt->vsb = VSB_new_auto();
-		XXXAN(vt->vsb);
-		b->probe = vt;
-		startthread = 1;
-		VTAILQ_INSERT_TAIL(&vbp_list, vt, list);
-	} else {
-		vt = b->probe;
-	}
-
-	VTAILQ_FOREACH(vcl, &vt->vcls, list)
-		assert (vcl->probep != p);
-
-	vcl = vbp_new_vcl(p, hosthdr);
-	Lck_Lock(&vbp_mtx);
-	VTAILQ_INSERT_TAIL(&vt->vcls, vcl, list);
-	Lck_Unlock(&vbp_mtx);
-
-	if (startthread) {
-		for (u = 0; u < vcl->probe.initial; u++) {
-			vbp_start_poke(vt);
-			vt->happy |= 1;
-			vbp_has_poked(vt);
-		}
-		AZ(pthread_create(&vt->thread, NULL, vbp_wrk_poll_backend, vt));
-	}
-}
-
-void
-VBP_Use(const struct backend *b, const struct vrt_backend_probe *p)
-{
-	struct vbp_target *vt;
-	struct vbp_vcl *vcl;
-
-	ASSERT_CLI();
-	AN(p);
-	CHECK_OBJ_NOTNULL(b, BACKEND_MAGIC);
-	AN(b->probe);
-	vt = b->probe;
-
-	VTAILQ_FOREACH(vcl, &vt->vcls, list) {
-		if (vcl->probep != p)
-			continue;
-
-		Lck_Lock(&vbp_mtx);
-		VTAILQ_REMOVE(&vt->vcls, vcl, list);
-		VTAILQ_INSERT_HEAD(&vt->vcls, vcl, list);
-		Lck_Unlock(&vbp_mtx);
-		return;
-	}
-}
-
-void
-VBP_Remove(struct backend *b, struct vrt_backend_probe const *p)
-{
-	struct vbp_target *vt;
-	struct vbp_vcl *vcl;
-	void *ret;
-
-	ASSERT_CLI();
-	AN(p);
-	CHECK_OBJ_NOTNULL(b, BACKEND_MAGIC);
-	AN(b->probe);
-	vt = b->probe;
-
-	VTAILQ_FOREACH(vcl, &vt->vcls, list)
-		if (vcl->probep == p)
-			break;
-
-	AN(vcl);
-
-	Lck_Lock(&vbp_mtx);
-	VTAILQ_REMOVE(&vt->vcls, vcl, list);
-	Lck_Unlock(&vbp_mtx);
-
-	FREE_OBJ(vcl);
-
-	if (!VTAILQ_EMPTY(&vt->vcls))
-		return;
-
-	/* No more polling for this backend */
-
-	b->healthy = 1;
-
-	vt->stop = 1;
-	AZ(pthread_cancel(vt->thread));
-	AZ(pthread_join(vt->thread, &ret));
-
-	b->healthy = 1;
-
-	VTAILQ_REMOVE(&vbp_list, vt, list);
-	b->probe = NULL;
-	VSB_delete(vt->vsb);
-	FREE_OBJ(vt);
-}
-
-/*--------------------------------------------------------------------
- * Initialize the backend probe subsystem
- */
-
-void
-VBP_Init(void)
-{
-
-	Lck_New(&vbp_mtx, lck_vbp);
-	CLI_AddFuncs(debug_cmds);
-}
diff --git a/bin/varnishd/cache_ban.c b/bin/varnishd/cache_ban.c
deleted file mode 100644
index 768e631..0000000
--- a/bin/varnishd/cache_ban.c
+++ /dev/null
@@ -1,1108 +0,0 @@
-/*-
- * Copyright (c) 2006 Verdens Gang AS
- * Copyright (c) 2006-2011 Varnish Software AS
- * All rights reserved.
- *
- * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- *
- * Ban processing
- *
- * A ban consists of a number of conditions (or tests), all of which must be
- * satisfied.  Here are some potential bans we could support:
- *
- *	req.url == "/foo"
- *	req.url ~ ".iso" && obj.size > 10MB
- *	req.http.host ~ "web1.com" && obj.set-cookie ~ "USER=29293"
- *
- * We make the "&&" mandatory from the start, leaving the syntax space
- * for latter handling of "||" as well.
- *
- * Bans are compiled into bytestrings as follows:
- *	8 bytes	- double: timestamp		XXX: Byteorder ?
- *	4 bytes - be32: length
- *	1 byte - flags: 0x01: BAN_F_REQ
- *	N tests
- * A test have this form:
- *	1 byte - arg (see ban_vars.h col 3 "BAN_ARG_XXX")
- *	(n bytes) - http header name, canonical encoding
- *	lump - comparison arg
- *	1 byte - operation (BAN_OPER_)
- *	(lump) - compiled regexp
- * A lump is:
- *	4 bytes - be32: length
- *	n bytes - content
- *
- * In a perfect world, we should vector through VRE to get to PCRE,
- * but since we rely on PCRE's ability to encode the regexp into a
- * byte string, that would be a little bit artificial, so this is
- * the exception that confirmes the rule.
- *
- */
-
-#include "config.h"
-
-#include <pcre.h>
-#include <stdio.h>
-
-#include "cache.h"
-
-#include "hash/hash_slinger.h"
-#include "vcli.h"
-#include "vcli_priv.h"
-#include "vend.h"
-#include "vtim.h"
-
-struct ban {
-	unsigned		magic;
-#define BAN_MAGIC		0x700b08ea
-	VTAILQ_ENTRY(ban)	list;
-	unsigned		refcount;
-	unsigned		flags;
-#define BAN_F_GONE		(1 << 0)
-#define BAN_F_REQ		(1 << 2)
-#define BAN_F_LURK		(3 << 6)	/* ban-lurker-color */
-	VTAILQ_HEAD(,objcore)	objcore;
-	struct vsb		*vsb;
-	uint8_t			*spec;
-};
-
-#define LURK_SHIFT 6
-
-struct ban_test {
-	uint8_t			arg1;
-	const char		*arg1_spec;
-	uint8_t			oper;
-	const char		*arg2;
-	const void		*arg2_spec;
-};
-
-static VTAILQ_HEAD(banhead_s,ban) ban_head = VTAILQ_HEAD_INITIALIZER(ban_head);
-static struct lock ban_mtx;
-static struct ban *ban_magic;
-static pthread_t ban_thread;
-static struct ban * volatile ban_start;
-static bgthread_t ban_lurker;
-
-/*--------------------------------------------------------------------
- * BAN string magic markers
- */
-
-#define	BAN_OPER_EQ	0x10
-#define	BAN_OPER_NEQ	0x11
-#define	BAN_OPER_MATCH	0x12
-#define	BAN_OPER_NMATCH	0x13
-
-#define BAN_ARG_URL	0x18
-#define BAN_ARG_REQHTTP	0x19
-#define BAN_ARG_OBJHTTP	0x1a
-
-/*--------------------------------------------------------------------
- * Variables we can purge on
- */
-
-static const struct pvar {
-	const char		*name;
-	unsigned		flag;
-	uint8_t			tag;
-} pvars[] = {
-#define PVAR(a, b, c)	{ (a), (b), (c) },
-#include "tbl/ban_vars.h"
-#undef PVAR
-	{ 0, 0, 0}
-};
-
-/*--------------------------------------------------------------------
- * Storage handling of bans
- */
-
-struct ban *
-BAN_New(void)
-{
-	struct ban *b;
-
-	ALLOC_OBJ(b, BAN_MAGIC);
-	if (b == NULL)
-		return (b);
-	b->vsb = VSB_new_auto();
-	if (b->vsb == NULL) {
-		FREE_OBJ(b);
-		return (NULL);
-	}
-	VTAILQ_INIT(&b->objcore);
-	return (b);
-}
-
-void
-BAN_Free(struct ban *b)
-{
-
-	CHECK_OBJ_NOTNULL(b, BAN_MAGIC);
-	AZ(b->refcount);
-	assert(VTAILQ_EMPTY(&b->objcore));
-
-	if (b->vsb != NULL)
-		VSB_delete(b->vsb);
-	if (b->spec != NULL)
-		free(b->spec);
-	FREE_OBJ(b);
-}
-
-/*--------------------------------------------------------------------
- * Get & Release a tail reference, used to hold the list stable for
- * traversals etc.
- */
-
-struct ban *
-BAN_TailRef(void)
-{
-	struct ban *b;
-
-	ASSERT_CLI();
-	Lck_Lock(&ban_mtx);
-	b = VTAILQ_LAST(&ban_head, banhead_s);
-	AN(b);
-	b->refcount++;
-	Lck_Unlock(&ban_mtx);
-	return (b);
-}
-
-void
-BAN_TailDeref(struct ban **bb)
-{
-	struct ban *b;
-
-	b = *bb;
-	*bb = NULL;
-	Lck_Lock(&ban_mtx);
-	b->refcount--;
-	Lck_Unlock(&ban_mtx);
-}
-
-/*--------------------------------------------------------------------
- * Extract time and length from ban-spec
- */
-
-static double
-ban_time(const uint8_t *banspec)
-{
-	double t;
-
-	assert(sizeof t == 8);
-	memcpy(&t, banspec, sizeof t);
-	return (t);
-}
-
-static unsigned
-ban_len(const uint8_t *banspec)
-{
-	unsigned u;
-
-	u = vbe32dec(banspec + 8);
-	return (u);
-}
-
-/*--------------------------------------------------------------------
- * Access a lump of bytes in a ban test spec
- */
-
-static void
-ban_add_lump(const struct ban *b, const void *p, uint32_t len)
-{
-	uint8_t buf[sizeof len];
-
-	vbe32enc(buf, len);
-	VSB_bcat(b->vsb, buf, sizeof buf);
-	VSB_bcat(b->vsb, p, len);
-}
-
-static const void *
-ban_get_lump(const uint8_t **bs)
-{
-	const void *r;
-	unsigned ln;
-
-	ln = vbe32dec(*bs);
-	*bs += 4;
-	r = (const void*)*bs;
-	*bs += ln;
-	return (r);
-}
-
-/*--------------------------------------------------------------------
- * Pick a test apart from a spec string
- */
-
-static void
-ban_iter(const uint8_t **bs, struct ban_test *bt)
-{
-
-	memset(bt, 0, sizeof *bt);
-	bt->arg1 = *(*bs)++;
-	if (bt->arg1 == BAN_ARG_REQHTTP || bt->arg1 == BAN_ARG_OBJHTTP) {
-		bt->arg1_spec = (const char *)*bs;
-		(*bs) += (*bs)[0] + 2;
-	}
-	bt->arg2 = ban_get_lump(bs);
-	bt->oper = *(*bs)++;
-	if (bt->oper == BAN_OPER_MATCH || bt->oper == BAN_OPER_NMATCH)
-		bt->arg2_spec = ban_get_lump(bs);
-}
-
-/*--------------------------------------------------------------------
- * Parse and add a http argument specification
- * Output something which HTTP_GetHdr understands
- */
-
-static void
-ban_parse_http(const struct ban *b, const char *a1)
-{
-	int l;
-
-	l = strlen(a1) + 1;
-	assert(l <= 127);
-	VSB_putc(b->vsb, (char)l);
-	VSB_cat(b->vsb, a1);
-	VSB_putc(b->vsb, ':');
-	VSB_putc(b->vsb, '\0');
-}
-
-/*--------------------------------------------------------------------
- * Parse and add a ban test specification
- */
-
-static int
-ban_parse_regexp(struct cli *cli, const struct ban *b, const char *a3)
-{
-	const char *error;
-	int erroroffset, rc;
-	size_t sz;
-	pcre *re;
-
-	re = pcre_compile(a3, 0, &error, &erroroffset, NULL);
-	if (re == NULL) {
-		VSL(SLT_Debug, 0, "REGEX: <%s>", error);
-		VCLI_Out(cli, "%s", error);
-		VCLI_SetResult(cli, CLIS_PARAM);
-		return (-1);
-	}
-	rc = pcre_fullinfo(re, NULL, PCRE_INFO_SIZE, &sz);
-	AZ(rc);
-	ban_add_lump(b, re, sz);
-	return (0);
-}
-
-/*--------------------------------------------------------------------
- * Add a (and'ed) test-condition to a ban
- */
-
-int
-BAN_AddTest(struct cli *cli, struct ban *b, const char *a1, const char *a2,
-    const char *a3)
-{
-	const struct pvar *pv;
-	int i;
-
-	CHECK_OBJ_NOTNULL(b, BAN_MAGIC);
-
-	for (pv = pvars; pv->name != NULL; pv++)
-		if (!strncmp(a1, pv->name, strlen(pv->name)))
-			break;
-	if (pv->name == NULL) {
-		VCLI_Out(cli, "unknown or unsupported field \"%s\"", a1);
-		VCLI_SetResult(cli, CLIS_PARAM);
-		return (-1);
-	}
-
-	if (pv->flag & PVAR_REQ)
-		b->flags |= BAN_F_REQ;
-
-	VSB_putc(b->vsb, pv->tag);
-	if (pv->flag & PVAR_HTTP)
-		ban_parse_http(b, a1 + strlen(pv->name));
-
-	ban_add_lump(b, a3, strlen(a3) + 1);
-	if (!strcmp(a2, "~")) {
-		VSB_putc(b->vsb, BAN_OPER_MATCH);
-		i = ban_parse_regexp(cli, b, a3);
-		if (i)
-			return (i);
-	} else if (!strcmp(a2, "!~")) {
-		VSB_putc(b->vsb, BAN_OPER_NMATCH);
-		i = ban_parse_regexp(cli, b, a3);
-		if (i)
-			return (i);
-	} else if (!strcmp(a2, "==")) {
-		VSB_putc(b->vsb, BAN_OPER_EQ);
-	} else if (!strcmp(a2, "!=")) {
-		VSB_putc(b->vsb, BAN_OPER_NEQ);
-	} else {
-		VCLI_Out(cli,
-		    "expected conditional (~, !~, == or !=) got \"%s\"", a2);
-		VCLI_SetResult(cli, CLIS_PARAM);
-		return (-1);
-	}
-	return (0);
-}
-
-/*--------------------------------------------------------------------
- * We maintain ban_start as a pointer to the first element of the list
- * as a separate variable from the VTAILQ, to avoid depending on the
- * internals of the VTAILQ macros.  We tacitly assume that a pointer
- * write is always atomic in doing so.
- */
-
-void
-BAN_Insert(struct ban *b)
-{
-	struct ban  *bi, *be;
-	ssize_t ln;
-	double t0;
-
-	CHECK_OBJ_NOTNULL(b, BAN_MAGIC);
-
-	AZ(VSB_finish(b->vsb));
-	ln = VSB_len(b->vsb);
-	assert(ln >= 0);
-
-	b->spec = malloc(ln + 13L);	/* XXX */
-	XXXAN(b->spec);
-
-	t0 = VTIM_real();
-	memcpy(b->spec, &t0, sizeof t0);
-	b->spec[12] = (b->flags & BAN_F_REQ) ? 1 : 0;
-	memcpy(b->spec + 13, VSB_data(b->vsb), ln);
-	ln += 13;
-	vbe32enc(b->spec + 8, ln);
-
-	VSB_delete(b->vsb);
-	b->vsb = NULL;
-
-	Lck_Lock(&ban_mtx);
-	VTAILQ_INSERT_HEAD(&ban_head, b, list);
-	ban_start = b;
-	VSC_C_main->bans++;
-	VSC_C_main->bans_added++;
-	if (b->flags & BAN_F_REQ)
-		VSC_C_main->bans_req++;
-
-	be = VTAILQ_LAST(&ban_head, banhead_s);
-	if (cache_param->ban_dups && be != b)
-		be->refcount++;
-	else
-		be = NULL;
-
-	SMP_NewBan(b->spec, ln);
-	Lck_Unlock(&ban_mtx);
-
-	if (be == NULL)
-		return;
-
-	/* Hunt down duplicates, and mark them as gone */
-	bi = b;
-	Lck_Lock(&ban_mtx);
-	while(bi != be) {
-		bi = VTAILQ_NEXT(bi, list);
-		if (bi->flags & BAN_F_GONE)
-			continue;
-		/* Safe because the length is part of the fixed size hdr */
-		if (memcmp(b->spec + 8, bi->spec + 8, ln - 8))
-			continue;
-		bi->flags |= BAN_F_GONE;
-		VSC_C_main->bans_gone++;
-		VSC_C_main->bans_dups++;
-	}
-	be->refcount--;
-	Lck_Unlock(&ban_mtx);
-}
-
-/*--------------------------------------------------------------------
- * A new object is created, grab a reference to the newest ban
- */
-
-void
-BAN_NewObjCore(struct objcore *oc)
-{
-
-	CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
-	AZ(oc->ban);
-	Lck_Lock(&ban_mtx);
-	oc->ban = ban_start;
-	ban_start->refcount++;
-	VTAILQ_INSERT_TAIL(&ban_start->objcore, oc, ban_list);
-	Lck_Unlock(&ban_mtx);
-}
-
-/*--------------------------------------------------------------------
- * An object is destroyed, release its ban reference
- */
-
-void
-BAN_DestroyObj(struct objcore *oc)
-{
-
-	CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
-	if (oc->ban == NULL)
-		return;
-	CHECK_OBJ_NOTNULL(oc->ban, BAN_MAGIC);
-	Lck_Lock(&ban_mtx);
-	assert(oc->ban->refcount > 0);
-	oc->ban->refcount--;
-	VTAILQ_REMOVE(&oc->ban->objcore, oc, ban_list);
-	oc->ban = NULL;
-	Lck_Unlock(&ban_mtx);
-}
-
-/*--------------------------------------------------------------------
- * Find and/or Grab a reference to an objects ban based on timestamp
- * Assume we hold a TailRef, so list traversal is safe.
- */
-
-struct ban *
-BAN_RefBan(struct objcore *oc, double t0, const struct ban *tail)
-{
-	struct ban *b;
-	double t1 = 0;
-
-	VTAILQ_FOREACH(b, &ban_head, list) {
-		t1 = ban_time(b->spec);
-		if (t1 <= t0)
-			break;
-		if (b == tail)
-			break;
-	}
-	AN(b);
-	assert(t1 == t0);
-	Lck_Lock(&ban_mtx);
-	b->refcount++;
-	VTAILQ_INSERT_TAIL(&b->objcore, oc, ban_list);
-	Lck_Unlock(&ban_mtx);
-	return (b);
-}
-
-/*--------------------------------------------------------------------
- * Put a skeleton ban in the list, unless there is an identical,
- * time & condition, ban already in place.
- *
- * If a newer ban has same condition, mark the new ban GONE.
- * mark any older bans, with the same condition, GONE as well.
- */
-
-void
-BAN_Reload(const uint8_t *ban, unsigned len)
-{
-	struct ban *b, *b2;
-	int gone = 0;
-	double t0, t1, t2 = 9e99;
-
-	ASSERT_CLI();
-
-	t0 = ban_time(ban);
-	assert(len == ban_len(ban));
-
-	Lck_Lock(&ban_mtx);
-
-	VTAILQ_FOREACH(b, &ban_head, list) {
-		t1 = ban_time(b->spec);
-		assert (t1 < t2);
-		t2 = t1;
-		if (t1 == t0) {
-			Lck_Unlock(&ban_mtx);
-			return;
-		}
-		if (t1 < t0)
-			break;
-		if (!memcmp(b->spec + 8, ban + 8, len - 8)) {
-			gone |= BAN_F_GONE;
-			VSC_C_main->bans_dups++;
-			VSC_C_main->bans_gone++;
-		}
-	}
-
-	VSC_C_main->bans++;
-	VSC_C_main->bans_added++;
-
-	b2 = BAN_New();
-	AN(b2);
-	b2->spec = malloc(len);
-	AN(b2->spec);
-	memcpy(b2->spec, ban, len);
-	b2->flags |= gone;
-	if (ban[12])
-		b2->flags |= BAN_F_REQ;
-	if (b == NULL)
-		VTAILQ_INSERT_TAIL(&ban_head, b2, list);
-	else
-		VTAILQ_INSERT_BEFORE(b, b2, list);
-
-	/* Hunt down older duplicates */
-	for (b = VTAILQ_NEXT(b2, list); b != NULL; b = VTAILQ_NEXT(b, list)) {
-		if (b->flags & BAN_F_GONE)
-			continue;
-		if (!memcmp(b->spec + 8, ban + 8, len - 8)) {
-			b->flags |= BAN_F_GONE;
-			VSC_C_main->bans_dups++;
-			VSC_C_main->bans_gone++;
-		}
-	}
-	Lck_Unlock(&ban_mtx);
-}
-
-/*--------------------------------------------------------------------
- * Get a bans timestamp
- */
-
-double
-BAN_Time(const struct ban *b)
-{
-
-	if (b == NULL)
-		return (0.0);
-
-	CHECK_OBJ_NOTNULL(b, BAN_MAGIC);
-	return (ban_time(b->spec));
-}
-
-/*--------------------------------------------------------------------
- * All silos have read their bans, ready for action
- */
-
-void
-BAN_Compile(void)
-{
-
-	ASSERT_CLI();
-
-	SMP_NewBan(ban_magic->spec, ban_len(ban_magic->spec));
-	ban_start = VTAILQ_FIRST(&ban_head);
-	WRK_BgThread(&ban_thread, "ban-lurker", ban_lurker, NULL);
-}
-
-/*--------------------------------------------------------------------
- * Evaluate ban-spec
- */
-
-static int
-ban_evaluate(const uint8_t *bs, const struct http *objhttp,
-    const struct http *reqhttp, unsigned *tests)
-{
-	struct ban_test bt;
-	const uint8_t *be;
-	char *arg1;
-
-	be = bs + ban_len(bs);
-	bs += 13;
-	while (bs < be) {
-		(*tests)++;
-		ban_iter(&bs, &bt);
-		arg1 = NULL;
-		switch (bt.arg1) {
-		case BAN_ARG_URL:
-			arg1 = reqhttp->hd[HTTP_HDR_URL].b;
-			break;
-		case BAN_ARG_REQHTTP:
-			(void)http_GetHdr(reqhttp, bt.arg1_spec, &arg1);
-			break;
-		case BAN_ARG_OBJHTTP:
-			(void)http_GetHdr(objhttp, bt.arg1_spec, &arg1);
-			break;
-		default:
-			INCOMPL();
-		}
-
-		switch (bt.oper) {
-		case BAN_OPER_EQ:
-			if (arg1 == NULL || strcmp(arg1, bt.arg2))
-				return (0);
-			break;
-		case BAN_OPER_NEQ:
-			if (arg1 != NULL && !strcmp(arg1, bt.arg2))
-				return (0);
-			break;
-		case BAN_OPER_MATCH:
-			if (arg1 == NULL ||
-			    pcre_exec(bt.arg2_spec, NULL, arg1, strlen(arg1),
-			    0, 0, NULL, 0) < 0)
-				return (0);
-			break;
-		case BAN_OPER_NMATCH:
-			if (arg1 != NULL &&
-			    pcre_exec(bt.arg2_spec, NULL, arg1, strlen(arg1),
-			    0, 0, NULL, 0) >= 0)
-				return (0);
-			break;
-		default:
-			INCOMPL();
-		}
-	}
-	return (1);
-}
-
-/*--------------------------------------------------------------------
- * Check an object against all applicable bans
- *
- * Return:
- *	-1 not all bans checked, but none of the checked matched
- *		Only if !has_req
- *	0 No bans matched, object moved to ban_start.
- *	1 Ban matched, object removed from ban list.
- */
-
-static int
-ban_check_object(struct object *o, const struct sess *sp, int has_req)
-{
-	struct ban *b;
-	struct objcore *oc;
-	struct ban * volatile b0;
-	unsigned tests, skipped;
-
-	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
-	CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC);
-	oc = o->objcore;
-	CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
-	CHECK_OBJ_NOTNULL(oc->ban, BAN_MAGIC);
-
-	b0 = ban_start;
-	CHECK_OBJ_NOTNULL(b0, BAN_MAGIC);
-
-	if (b0 == oc->ban)
-		return (0);
-
-	/*
-	 * This loop is safe without locks, because we know we hold
-	 * a refcount on a ban somewhere in the list and we do not
-	 * inspect the list past that ban.
-	 */
-	tests = 0;
-	skipped = 0;
-	for (b = b0; b != oc->ban; b = VTAILQ_NEXT(b, list)) {
-		CHECK_OBJ_NOTNULL(b, BAN_MAGIC);
-		if (b->flags & BAN_F_GONE)
-			continue;
-		if ((b->flags & BAN_F_LURK) &&
-		    (b->flags & BAN_F_LURK) == (oc->flags & OC_F_LURK)) {
-			AZ(b->flags & BAN_F_REQ);
-			/* Lurker already tested this */
-			continue;
-		}
-		if (!has_req && (b->flags & BAN_F_REQ)) {
-			/*
-			 * We cannot test this one, but there might
-			 * be other bans that match, so we soldier on
-			 */
-			skipped++;
-		} else if (ban_evaluate(b->spec, o->http, sp->http, &tests))
-			break;
-	}
-
-	Lck_Lock(&ban_mtx);
-	VSC_C_main->bans_tested++;
-	VSC_C_main->bans_tests_tested += tests;
-
-	if (b == oc->ban && skipped > 0) {
-		AZ(has_req);
-		Lck_Unlock(&ban_mtx);
-		/*
-		 * Not banned, but some tests were skipped, so we cannot know
-		 * for certain that it cannot be, so we just have to give up.
-		 */
-		return (-1);
-	}
-
-	oc->ban->refcount--;
-	VTAILQ_REMOVE(&oc->ban->objcore, oc, ban_list);
-	if (b == oc->ban) {	/* not banned */
-		b->flags &= ~BAN_F_LURK;
-		VTAILQ_INSERT_TAIL(&b0->objcore, oc, ban_list);
-		b0->refcount++;
-	}
-	Lck_Unlock(&ban_mtx);
-
-	if (b == oc->ban) {	/* not banned */
-		oc->ban = b0;
-		oc_updatemeta(oc);
-		return (0);
-	} else {
-		EXP_Clr(&o->exp);
-		oc->ban = NULL;
-		oc_updatemeta(oc);
-		/* BAN also changed, but that is not important any more */
-		WSP(sp, SLT_ExpBan, "%u was banned", o->xid);
-		EXP_Rearm(o);
-		return (1);
-	}
-}
-
-int
-BAN_CheckObject(struct object *o, const struct sess *sp)
-{
-
-	return (ban_check_object(o, sp, 1) > 0);
-}
-
-static struct ban *
-ban_CheckLast(void)
-{
-	struct ban *b;
-
-	Lck_AssertHeld(&ban_mtx);
-	b = VTAILQ_LAST(&ban_head, banhead_s);
-	if (b != VTAILQ_FIRST(&ban_head) && b->refcount == 0) {
-		if (b->flags & BAN_F_GONE)
-			VSC_C_main->bans_gone--;
-		if (b->flags & BAN_F_REQ)
-			VSC_C_main->bans_req--;
-		VSC_C_main->bans--;
-		VSC_C_main->bans_deleted++;
-		VTAILQ_REMOVE(&ban_head, b, list);
-	} else {
-		b = NULL;
-	}
-	return (b);
-}
-
-/*--------------------------------------------------------------------
- * Ban lurker thread
- */
-
-static int
-ban_lurker_work(const struct sess *sp, unsigned pass)
-{
-	struct ban *b, *b0, *b2;
-	struct objhead *oh;
-	struct objcore *oc, *oc2;
-	struct object *o;
-	int i;
-
-	AN(pass & BAN_F_LURK);
-	AZ(pass & ~BAN_F_LURK);
-
-	/* First route the last ban(s) */
-	do {
-		Lck_Lock(&ban_mtx);
-		b2 = ban_CheckLast();
-		Lck_Unlock(&ban_mtx);
-		if (b2 != NULL)
-			BAN_Free(b2);
-	} while (b2 != NULL);
-
-	/*
-	 * Find out if we have any bans we can do something about
-	 * If we find any, tag them with our pass number.
-	 */
-	i = 0;
-	b0 = NULL;
-	VTAILQ_FOREACH(b, &ban_head, list) {
-		if (b->flags & BAN_F_GONE)
-			continue;
-		if (b->flags & BAN_F_REQ)
-			continue;
-		if (b == VTAILQ_LAST(&ban_head, banhead_s))
-			continue;
-		if (b0 == NULL)
-			b0 = b;
-		i++;
-		b->flags &= ~BAN_F_LURK;
-		b->flags |= pass;
-	}
-	if (cache_param->diag_bitmap & 0x80000)
-		VSL(SLT_Debug, 0, "lurker: %d actionable bans", i);
-	if (i == 0)
-		return (0);
-
-	VTAILQ_FOREACH_REVERSE(b, &ban_head, banhead_s, list) {
-		if (cache_param->diag_bitmap & 0x80000)
-			VSL(SLT_Debug, 0, "lurker doing %f %d",
-			    ban_time(b->spec), b->refcount);
-		while (1) {
-			Lck_Lock(&ban_mtx);
-			oc = VTAILQ_FIRST(&b->objcore);
-			if (oc == NULL)
-				break;
-			CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
-			if (cache_param->diag_bitmap & 0x80000)
-				VSL(SLT_Debug, 0, "test: %p %d %d",
-				    oc, oc->flags & OC_F_LURK, pass);
-			if ((oc->flags & OC_F_LURK) == pass)
-				break;
-			oh = oc->objhead;
-			CHECK_OBJ_NOTNULL(oh, OBJHEAD_MAGIC);
-			if (Lck_Trylock(&oh->mtx)) {
-				Lck_Unlock(&ban_mtx);
-				VTIM_sleep(cache_param->ban_lurker_sleep);
-				continue;
-			}
-			/*
-			 * See if the objcore is still on the objhead since
-			 * we race against HSH_Deref() which comes in the
-			 * opposite locking order.
-			 */
-			VTAILQ_FOREACH(oc2, &oh->objcs, list)
-				if (oc == oc2)
-					break;
-			if (oc2 == NULL) {
-				Lck_Unlock(&oh->mtx);
-				Lck_Unlock(&ban_mtx);
-				VTIM_sleep(cache_param->ban_lurker_sleep);
-				continue;
-			}
-			/*
-			 * Grab a reference to the OC and we can let go of
-			 * the BAN mutex
-			 */
-			AN(oc->refcnt);
-			oc->refcnt++;
-			oc->flags &= ~OC_F_LURK;
-			Lck_Unlock(&ban_mtx);
-			/*
-			 * Get the object and check it against all relevant bans
-			 */
-			o = oc_getobj(sp->wrk, oc);
-			i = ban_check_object(o, sp, 0);
-			if (cache_param->diag_bitmap & 0x80000)
-				VSL(SLT_Debug, 0, "lurker got: %p %d",
-				    oc, i);
-			if (i == -1) {
-				/* Not banned, not moved */
-				oc->flags |= pass;
-				Lck_Lock(&ban_mtx);
-				VTAILQ_REMOVE(&b->objcore, oc, ban_list);
-				VTAILQ_INSERT_TAIL(&b->objcore, oc, ban_list);
-				Lck_Unlock(&ban_mtx);
-			}
-			Lck_Unlock(&oh->mtx);
-			if (cache_param->diag_bitmap & 0x80000)
-				VSL(SLT_Debug, 0, "lurker done: %p %d %d",
-				    oc, oc->flags & OC_F_LURK, pass);
-			(void)HSH_Deref(sp->wrk, NULL, &o);
-			VTIM_sleep(cache_param->ban_lurker_sleep);
-		}
-		Lck_AssertHeld(&ban_mtx);
-		if (!(b->flags & BAN_F_REQ)) {
-			if (!(b->flags & BAN_F_GONE)) {
-				b->flags |= BAN_F_GONE;
-				VSC_C_main->bans_gone++;
-			}
-			if (cache_param->diag_bitmap & 0x80000)
-				VSL(SLT_Debug, 0, "lurker BAN %f now gone",
-				    ban_time(b->spec));
-		}
-		Lck_Unlock(&ban_mtx);
-		VTIM_sleep(cache_param->ban_lurker_sleep);
-		if (b == b0)
-			break;
-	}
-	return (1);
-}
-
-static void * __match_proto__(bgthread_t)
-ban_lurker(struct sess *sp, void *priv)
-{
-	struct ban *bf;
-	unsigned pass = (1 << LURK_SHIFT);
-
-	int i = 0;
-	(void)priv;
-	while (1) {
-
-		while (cache_param->ban_lurker_sleep == 0.0) {
-			/*
-			 * Ban-lurker is disabled:
-			 * Clean the last ban, if possible, and sleep
-			 */
-			Lck_Lock(&ban_mtx);
-			bf = ban_CheckLast();
-			Lck_Unlock(&ban_mtx);
-			if (bf != NULL)
-				BAN_Free(bf);
-			else
-				VTIM_sleep(1.0);
-		}
-
-		i = ban_lurker_work(sp, pass);
-		WSL_Flush(sp->wrk, 0);
-		WRK_SumStat(sp->wrk);
-		if (i) {
-			pass += (1 << LURK_SHIFT);
-			pass &= BAN_F_LURK;
-			if (pass == 0)
-				pass += (1 << LURK_SHIFT);
-			VTIM_sleep(cache_param->ban_lurker_sleep);
-		} else {
-			VTIM_sleep(1.0);
-		}
-	}
-	NEEDLESS_RETURN(NULL);
-}
-
-/*--------------------------------------------------------------------
- * CLI functions to add bans
- */
-
-static void
-ccf_ban(struct cli *cli, const char * const *av, void *priv)
-{
-	int narg, i;
-	struct ban *b;
-
-	(void)priv;
-
-	/* First do some cheap checks on the arguments */
-	for (narg = 0; av[narg + 2] != NULL; narg++)
-		continue;
-	if ((narg % 4) != 3) {
-		VCLI_Out(cli, "Wrong number of arguments");
-		VCLI_SetResult(cli, CLIS_PARAM);
-		return;
-	}
-	for (i = 3; i < narg; i += 4) {
-		if (strcmp(av[i + 2], "&&")) {
-			VCLI_Out(cli, "Found \"%s\" expected &&", av[i + 2]);
-			VCLI_SetResult(cli, CLIS_PARAM);
-			return;
-		}
-	}
-
-	b = BAN_New();
-	if (b == NULL) {
-		VCLI_Out(cli, "Out of Memory");
-		VCLI_SetResult(cli, CLIS_CANT);
-		return;
-	}
-	for (i = 0; i < narg; i += 4)
-		if (BAN_AddTest(cli, b, av[i + 2], av[i + 3], av[i + 4])) {
-			BAN_Free(b);
-			return;
-		}
-	BAN_Insert(b);
-}
-
-static void
-ccf_ban_url(struct cli *cli, const char * const *av, void *priv)
-{
-	const char *aav[6];
-
-	(void)priv;
-	aav[0] = NULL;
-	aav[1] = "ban";
-	aav[2] = "req.url";
-	aav[3] = "~";
-	aav[4] = av[2];
-	aav[5] = NULL;
-	ccf_ban(cli, aav, priv);
-}
-
-static void
-ban_render(struct cli *cli, const uint8_t *bs)
-{
-	struct ban_test bt;
-	const uint8_t *be;
-
-	be = bs + ban_len(bs);
-	bs += 13;
-	while (bs < be) {
-		ban_iter(&bs, &bt);
-		switch (bt.arg1) {
-		case BAN_ARG_URL:
-			VCLI_Out(cli, "req.url");
-			break;
-		case BAN_ARG_REQHTTP:
-			VCLI_Out(cli, "req.http.%.*s",
-			    bt.arg1_spec[0] - 1, bt.arg1_spec + 1);
-			break;
-		case BAN_ARG_OBJHTTP:
-			VCLI_Out(cli, "obj.http.%.*s",
-			    bt.arg1_spec[0] - 1, bt.arg1_spec + 1);
-			break;
-		default:
-			INCOMPL();
-		}
-		switch (bt.oper) {
-		case BAN_OPER_EQ:	VCLI_Out(cli, " == "); break;
-		case BAN_OPER_NEQ:	VCLI_Out(cli, " != "); break;
-		case BAN_OPER_MATCH:	VCLI_Out(cli, " ~ "); break;
-		case BAN_OPER_NMATCH:	VCLI_Out(cli, " !~ "); break;
-		default:
-			INCOMPL();
-		}
-		VCLI_Out(cli, "%s", bt.arg2);
-		if (bs < be)
-			VCLI_Out(cli, " && ");
-	}
-}
-
-static void
-ccf_ban_list(struct cli *cli, const char * const *av, void *priv)
-{
-	struct ban *b, *bl;
-
-	(void)av;
-	(void)priv;
-
-	/* Get a reference so we are safe to traverse the list */
-	bl = BAN_TailRef();
-
-	VCLI_Out(cli, "Present bans:\n");
-	VTAILQ_FOREACH(b, &ban_head, list) {
-		if (b == bl && !(cache_param->diag_bitmap & 0x80000))
-			break;
-		VCLI_Out(cli, "%10.6f %5u%s\t", ban_time(b->spec),
-		    bl == b ? b->refcount - 1 : b->refcount,
-		    b->flags & BAN_F_GONE ? "G" : " ");
-		ban_render(cli, b->spec);
-		VCLI_Out(cli, "\n");
-		if (cache_param->diag_bitmap & 0x80000) {
-			Lck_Lock(&ban_mtx);
-			struct objcore *oc;
-			VTAILQ_FOREACH(oc, &b->objcore, ban_list)
-				VCLI_Out(cli, "     %p\n", oc);
-			Lck_Unlock(&ban_mtx);
-		}
-	}
-
-	BAN_TailDeref(&bl);
-}
-
-static struct cli_proto ban_cmds[] = {
-	{ CLI_BAN_URL,				"", ccf_ban_url },
-	{ CLI_BAN,				"", ccf_ban },
-	{ CLI_BAN_LIST,				"", ccf_ban_list },
-	{ NULL }
-};
-
-void
-BAN_Init(void)
-{
-
-	Lck_New(&ban_mtx, lck_ban);
-	CLI_AddFuncs(ban_cmds);
-	assert(BAN_F_LURK == OC_F_LURK);
-	AN((1 << LURK_SHIFT) & BAN_F_LURK);
-	AN((2 << LURK_SHIFT) & BAN_F_LURK);
-
-	ban_magic = BAN_New();
-	AN(ban_magic);
-	ban_magic->flags |= BAN_F_GONE;
-	VSC_C_main->bans_gone++;
-	BAN_Insert(ban_magic);
-}
diff --git a/bin/varnishd/cache_center.c b/bin/varnishd/cache_center.c
deleted file mode 100644
index e42fac8..0000000
--- a/bin/varnishd/cache_center.c
+++ /dev/null
@@ -1,1691 +0,0 @@
-/*-
- * Copyright (c) 2006 Verdens Gang AS
- * Copyright (c) 2006-2011 Varnish Software AS
- * All rights reserved.
- *
- * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- *
- * This file contains the central state machine for pushing requests.
- *
- * We cannot just use direct calls because it is possible to kick a
- * request back to the lookup stage (usually after a rewrite).  The
- * state engine also allows us to break the processing up into some
- * logical chunks which improves readability a little bit.
- *
- * Since the states are rather nasty in detail, I have decided to embedd
- * a dot(1) graph in the source code comments.  So to see the big picture,
- * extract the DOT lines and run though dot(1), for instance with the
- * command:
- *	sed -n '/^DOT/s///p' cache_center.c | dot -Tps > /tmp/_.ps
- */
-
-/*
-DOT digraph vcl_center {
-xDOT	page="8.2,11.5"
-DOT	size="7.2,10.5"
-DOT	margin="0.5"
-DOT	center="1"
-DOT acceptor [
-DOT	shape=hexagon
-DOT	label="Request received"
-DOT ]
-DOT ERROR [shape=plaintext]
-DOT RESTART [shape=plaintext]
-DOT acceptor -> start [style=bold,color=green]
- */
-
-#include "config.h"
-
-#include <math.h>
-#include <poll.h>
-#include <stdio.h>
-#include <stdlib.h>
-
-#include "cache.h"
-
-#include "hash/hash_slinger.h"
-#include "vcl.h"
-#include "vcli_priv.h"
-#include "vsha256.h"
-#include "vtcp.h"
-#include "vtim.h"
-
-#ifndef HAVE_SRANDOMDEV
-#include "compat/srandomdev.h"
-#endif
-
-static unsigned xids;
-
-/*--------------------------------------------------------------------
- * WAIT
- * Wait (briefly) until we have a full request in our htc.
- */
-
-static int
-cnt_wait(struct sess *sp)
-{
-	int i;
-	struct pollfd pfd[1];
-
-	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
-	AZ(sp->vcl);
-	AZ(sp->obj);
-	assert(sp->xid == 0);
-
-	i = HTC_Complete(sp->htc);
-	if (i == 0 && cache_param->session_linger > 0) {
-		pfd[0].fd = sp->fd;
-		pfd[0].events = POLLIN;
-		pfd[0].revents = 0;
-		i = poll(pfd, 1, cache_param->session_linger);
-		if (i)
-			i = HTC_Rx(sp->htc);
-	}
-	if (i == 0) {
-		WSP(sp, SLT_Debug, "herding");
-		sp->wrk->stats.sess_herd++;
-		SES_Charge(sp);
-		sp->wrk = NULL;
-		Pool_Wait(sp);
-		return (1);
-	}
-	if (i == 1) {
-		sp->step = STP_START;
-		return (0);
-	}
-	if (i == -2) {
-		SES_Close(sp, "overflow");
-		return (0);
-	}
-	if (i == -1 && Tlen(sp->htc->rxbuf) == 0 &&
-	    (errno == 0 || errno == ECONNRESET))
-		SES_Close(sp, "EOF");
-	else
-		SES_Close(sp, "error");
-	sp->step = STP_DONE;
-	return (0);
-}
-
-/*--------------------------------------------------------------------
- * We have a refcounted object on the session, now deliver it.
- *
-DOT subgraph xcluster_prepresp {
-DOT	prepresp [
-DOT		shape=ellipse
-DOT		label="Filter obj.->resp."
-DOT	]
-DOT	vcl_deliver [
-DOT		shape=record
-DOT		label="vcl_deliver()|resp."
-DOT	]
-DOT	prepresp -> vcl_deliver [style=bold,color=green]
-DOT	prepresp -> vcl_deliver [style=bold,color=cyan]
-DOT	prepresp -> vcl_deliver [style=bold,color=red]
-DOT	prepresp -> vcl_deliver [style=bold,color=blue,]
-DOT	vcl_deliver -> deliver [style=bold,color=green,label=deliver]
-DOT	vcl_deliver -> deliver [style=bold,color=red]
-DOT	vcl_deliver -> deliver [style=bold,color=blue]
-DOT     vcl_deliver -> errdeliver [label="error"]
-DOT     errdeliver [label="ERROR",shape=plaintext]
-DOT     vcl_deliver -> rstdeliver [label="restart",color=purple]
-DOT     rstdeliver [label="RESTART",shape=plaintext]
-DOT     vcl_deliver -> streambody [style=bold,color=cyan,label="deliver"]
-DOT }
- *
- */
-
-static int
-cnt_prepresp(struct sess *sp)
-{
-
-	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
-	CHECK_OBJ_NOTNULL(sp->obj, OBJECT_MAGIC);
-	CHECK_OBJ_NOTNULL(sp->vcl, VCL_CONF_MAGIC);
-
-	if (sp->wrk->do_stream)
-		AssertObjCorePassOrBusy(sp->obj->objcore);
-
-	sp->wrk->res_mode = 0;
-
-	if ((sp->wrk->h_content_length != NULL || !sp->wrk->do_stream) &&
-	    !sp->wrk->do_gzip && !sp->wrk->do_gunzip)
-		sp->wrk->res_mode |= RES_LEN;
-
-	if (!sp->disable_esi && sp->obj->esidata != NULL) {
-		/* In ESI mode, we don't know the aggregate length */
-		sp->wrk->res_mode &= ~RES_LEN;
-		sp->wrk->res_mode |= RES_ESI;
-	}
-
-	if (sp->esi_level > 0) {
-		sp->wrk->res_mode &= ~RES_LEN;
-		sp->wrk->res_mode |= RES_ESI_CHILD;
-	}
-
-	if (cache_param->http_gzip_support && sp->obj->gziped &&
-	    !RFC2616_Req_Gzip(sp)) {
-		/*
-		 * We don't know what it uncompresses to
-		 * XXX: we could cache that
-		 */
-		sp->wrk->res_mode &= ~RES_LEN;
-		sp->wrk->res_mode |= RES_GUNZIP;
-	}
-
-	if (!(sp->wrk->res_mode & (RES_LEN|RES_CHUNKED|RES_EOF))) {
-		if (sp->obj->len == 0 && !sp->wrk->do_stream)
-			/*
-			 * If the object is empty, neither ESI nor GUNZIP
-			 * can make it any different size
-			 */
-			sp->wrk->res_mode |= RES_LEN;
-		else if (!sp->wantbody) {
-			/* Nothing */
-		} else if (sp->http->protover >= 11) {
-			sp->wrk->res_mode |= RES_CHUNKED;
-		} else {
-			sp->wrk->res_mode |= RES_EOF;
-			sp->doclose = "EOF mode";
-		}
-	}
-
-	sp->t_resp = VTIM_real();
-	if (sp->obj->objcore != NULL) {
-		if ((sp->t_resp - sp->obj->last_lru) > cache_param->lru_timeout &&
-		    EXP_Touch(sp->obj->objcore))
-			sp->obj->last_lru = sp->t_resp;
-		sp->obj->last_use = sp->t_resp;	/* XXX: locking ? */
-	}
-	http_Setup(sp->wrk->resp, sp->wrk->ws);
-	RES_BuildHttp(sp);
-	VCL_deliver_method(sp);
-	switch (sp->handling) {
-	case VCL_RET_DELIVER:
-		break;
-	case VCL_RET_RESTART:
-		if (sp->restarts >= cache_param->max_restarts)
-			break;
-		if (sp->wrk->do_stream) {
-			VDI_CloseFd(sp->wrk);
-			HSH_Drop(sp);
-		} else {
-			(void)HSH_Deref(sp->wrk, NULL, &sp->obj);
-		}
-		AZ(sp->obj);
-		sp->restarts++;
-		sp->director = NULL;
-		sp->wrk->h_content_length = NULL;
-		http_Setup(sp->wrk->bereq, NULL);
-		http_Setup(sp->wrk->beresp, NULL);
-		http_Setup(sp->wrk->resp, NULL);
-		sp->step = STP_RECV;
-		return (0);
-	default:
-		WRONG("Illegal action in vcl_deliver{}");
-	}
-	if (sp->wrk->do_stream) {
-		AssertObjCorePassOrBusy(sp->obj->objcore);
-		sp->step = STP_STREAMBODY;
-	} else {
-		sp->step = STP_DELIVER;
-	}
-	return (0);
-}
-
-/*--------------------------------------------------------------------
- * Deliver an already stored object
- *
-DOT subgraph xcluster_deliver {
-DOT	deliver [
-DOT		shape=ellipse
-DOT		label="Send body"
-DOT	]
-DOT }
-DOT deliver -> DONE [style=bold,color=green]
-DOT deliver -> DONE [style=bold,color=red]
-DOT deliver -> DONE [style=bold,color=blue]
- *
- */
-
-static int
-cnt_deliver(struct sess *sp)
-{
-
-	sp->director = NULL;
-	sp->restarts = 0;
-
-	RES_WriteObj(sp);
-
-	assert(WRW_IsReleased(sp->wrk));
-	assert(sp->wrk->wrw.ciov == sp->wrk->wrw.siov);
-	(void)HSH_Deref(sp->wrk, NULL, &sp->obj);
-	http_Setup(sp->wrk->resp, NULL);
-	sp->step = STP_DONE;
-	return (0);
-}
-
-/*--------------------------------------------------------------------
- * This is the final state, figure out if we should close or recycle
- * the client connection
- *
-DOT	DONE [
-DOT		shape=hexagon
-DOT		label="Request completed"
-DOT	]
- */
-
-static int
-cnt_done(struct sess *sp)
-{
-	double dh, dp, da;
-	int i;
-
-	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
-	CHECK_OBJ_ORNULL(sp->vcl, VCL_CONF_MAGIC);
-
-	AZ(sp->obj);
-	AZ(sp->wrk->vbc);
-	sp->director = NULL;
-	sp->restarts = 0;
-
-	sp->wrk->do_esi = 0;
-	sp->wrk->do_gunzip = 0;
-	sp->wrk->do_gzip = 0;
-	sp->wrk->do_stream = 0;
-	sp->wrk->is_gunzip = 0;
-	sp->wrk->is_gzip = 0;
-
-	if (sp->vcl != NULL && sp->esi_level == 0) {
-		if (sp->wrk->vcl != NULL)
-			VCL_Rel(&sp->wrk->vcl);
-		sp->wrk->vcl = sp->vcl;
-		sp->vcl = NULL;
-	}
-
-	SES_Charge(sp);
-
-	sp->t_end = VTIM_real();
-	sp->wrk->lastused = sp->t_end;
-	if (sp->xid == 0) {
-		sp->t_req = sp->t_end;
-		sp->t_resp = sp->t_end;
-	} else if (sp->esi_level == 0) {
-		dp = sp->t_resp - sp->t_req;
-		da = sp->t_end - sp->t_resp;
-		dh = sp->t_req - sp->t_open;
-		/* XXX: Add StatReq == StatSess */
-		/* XXX: Workaround for pipe */
-		if (sp->fd >= 0) {
-			WSP(sp, SLT_Length, "%ju",
-			    (uintmax_t)sp->req_bodybytes);
-		}
-		WSP(sp, SLT_ReqEnd, "%u %.9f %.9f %.9f %.9f %.9f",
-		    sp->xid, sp->t_req, sp->t_end, dh, dp, da);
-	}
-	sp->xid = 0;
-	sp->t_open = sp->t_end;
-	sp->t_resp = NAN;
-	WSL_Flush(sp->wrk, 0);
-
-	/* If we did an ESI include, don't mess up our state */
-	if (sp->esi_level > 0)
-		return (1);
-
-	sp->req_bodybytes = 0;
-
-	sp->t_req = NAN;
-	sp->hash_always_miss = 0;
-	sp->hash_ignore_busy = 0;
-
-	if (sp->fd >= 0 && sp->doclose != NULL) {
-		/*
-		 * This is an orderly close of the connection; ditch nolinger
-		 * before we close, to get queued data transmitted.
-		 */
-		// XXX: not yet (void)VTCP_linger(sp->fd, 0);
-		SES_Close(sp, sp->doclose);
-	}
-
-	if (sp->fd < 0) {
-		sp->wrk->stats.sess_closed++;
-		SES_Delete(sp, NULL);
-		return (1);
-	}
-
-	if (sp->wrk->stats.client_req >= cache_param->wthread_stats_rate)
-		WRK_SumStat(sp->wrk);
-	/* Reset the workspace to the session-watermark */
-	WS_Reset(sp->ws, sp->ws_ses);
-	WS_Reset(sp->wrk->ws, NULL);
-
-	i = HTC_Reinit(sp->htc);
-	if (i == 1) {
-		sp->wrk->stats.sess_pipeline++;
-		sp->step = STP_START;
-		return (0);
-	}
-	if (Tlen(sp->htc->rxbuf)) {
-		sp->wrk->stats.sess_readahead++;
-		sp->step = STP_WAIT;
-		return (0);
-	}
-	if (cache_param->session_linger > 0) {
-		sp->wrk->stats.sess_linger++;
-		sp->step = STP_WAIT;
-		return (0);
-	}
-	sp->wrk->stats.sess_herd++;
-	sp->wrk = NULL;
-	Pool_Wait(sp);
-	return (1);
-}
-
-/*--------------------------------------------------------------------
- * Emit an error
- *
-DOT subgraph xcluster_error {
-DOT	vcl_error [
-DOT		shape=record
-DOT		label="vcl_error()|resp."
-DOT	]
-DOT	ERROR -> vcl_error
-DOT	vcl_error-> prepresp [label=deliver]
-DOT }
-DOT vcl_error-> rsterr [label="restart",color=purple]
-DOT rsterr [label="RESTART",shape=plaintext]
- */
-
-static int
-cnt_error(struct sess *sp)
-{
-	struct worker *w;
-	struct http *h;
-	char date[40];
-
-	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
-
-	sp->wrk->do_esi = 0;
-	sp->wrk->is_gzip = 0;
-	sp->wrk->is_gunzip = 0;
-	sp->wrk->do_gzip = 0;
-	sp->wrk->do_gunzip = 0;
-	sp->wrk->do_stream = 0;
-
-	w = sp->wrk;
-	if (sp->obj == NULL) {
-		HSH_Prealloc(sp);
-		EXP_Clr(&w->exp);
-		sp->obj = STV_NewObject(sp, NULL, cache_param->http_resp_size,
-		     &w->exp, (uint16_t)cache_param->http_max_hdr);
-		if (sp->obj == NULL)
-			sp->obj = STV_NewObject(sp, TRANSIENT_STORAGE,
-			    cache_param->http_resp_size, &w->exp,
-			    (uint16_t)cache_param->http_max_hdr);
-		if (sp->obj == NULL) {
-			sp->doclose = "Out of objects";
-			sp->director = NULL;
-			sp->wrk->h_content_length = NULL;
-			http_Setup(sp->wrk->beresp, NULL);
-			http_Setup(sp->wrk->bereq, NULL);
-			sp->step = STP_DONE;
-			return(0);
-		}
-		AN(sp->obj);
-		sp->obj->xid = sp->xid;
-		sp->obj->exp.entered = sp->t_req;
-	} else {
-		/* XXX: Null the headers ? */
-	}
-	CHECK_OBJ_NOTNULL(sp->obj, OBJECT_MAGIC);
-	h = sp->obj->http;
-
-	if (sp->err_code < 100 || sp->err_code > 999)
-		sp->err_code = 501;
-
-	http_PutProtocol(w, sp->vsl_id, h, "HTTP/1.1");
-	http_PutStatus(h, sp->err_code);
-	VTIM_format(VTIM_real(), date);
-	http_PrintfHeader(w, sp->vsl_id, h, "Date: %s", date);
-	http_SetHeader(w, sp->vsl_id, h, "Server: Varnish");
-
-	if (sp->err_reason != NULL)
-		http_PutResponse(w, sp->vsl_id, h, sp->err_reason);
-	else
-		http_PutResponse(w, sp->vsl_id, h,
-		    http_StatusMessage(sp->err_code));
-	VCL_error_method(sp);
-
-	if (sp->handling == VCL_RET_RESTART &&
-	    sp->restarts <  cache_param->max_restarts) {
-		HSH_Drop(sp);
-		sp->director = NULL;
-		sp->restarts++;
-		sp->step = STP_RECV;
-		return (0);
-	} else if (sp->handling == VCL_RET_RESTART)
-		sp->handling = VCL_RET_DELIVER;
-
-
-	/* We always close when we take this path */
-	sp->doclose = "error";
-	sp->wantbody = 1;
-
-	assert(sp->handling == VCL_RET_DELIVER);
-	sp->err_code = 0;
-	sp->err_reason = NULL;
-	http_Setup(sp->wrk->bereq, NULL);
-	sp->step = STP_PREPRESP;
-	return (0);
-}
-
-/*--------------------------------------------------------------------
- * Fetch response headers from the backend
- *
-DOT subgraph xcluster_fetch {
-DOT	fetch [
-DOT		shape=ellipse
-DOT		label="fetch hdr\nfrom backend\n(find obj.ttl)"
-DOT	]
-DOT	vcl_fetch [
-DOT		shape=record
-DOT		label="vcl_fetch()|req.\nbereq.\nberesp."
-DOT	]
-DOT	fetch -> vcl_fetch [style=bold,color=blue]
-DOT	fetch -> vcl_fetch [style=bold,color=red]
-DOT	fetch_pass [
-DOT		shape=ellipse
-DOT		label="obj.f.pass=true"
-DOT	]
-DOT	vcl_fetch -> fetch_pass [label="hit_for_pass",style=bold,color=red]
-DOT }
-DOT fetch_pass -> fetchbody [style=bold,color=red]
-DOT vcl_fetch -> fetchbody [label="deliver",style=bold,color=blue]
-DOT vcl_fetch -> rstfetch [label="restart",color=purple]
-DOT rstfetch [label="RESTART",shape=plaintext]
-DOT fetch -> errfetch
-DOT vcl_fetch -> errfetch [label="error"]
-DOT errfetch [label="ERROR",shape=plaintext]
- */
-
-static int
-cnt_fetch(struct sess *sp)
-{
-	int i;
-
-	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
-	CHECK_OBJ_NOTNULL(sp->vcl, VCL_CONF_MAGIC);
-
-	AN(sp->director);
-	AZ(sp->wrk->vbc);
-	AZ(sp->wrk->h_content_length);
-	AZ(sp->wrk->do_close);
-	AZ(sp->wrk->storage_hint);
-
-	http_Setup(sp->wrk->beresp, sp->wrk->ws);
-
-	i = FetchHdr(sp);
-	/*
-	 * If we recycle a backend connection, there is a finite chance
-	 * that the backend closed it before we get a request to it.
-	 * Do a single retry in that case.
-	 */
-	if (i == 1) {
-		VSC_C_main->backend_retry++;
-		i = FetchHdr(sp);
-	}
-
-	if (i) {
-		sp->handling = VCL_RET_ERROR;
-		sp->err_code = 503;
-	} else {
-		/*
-		 * These two headers can be spread over multiple actual headers
-		 * and we rely on their content outside of VCL, so collect them
-		 * into one line here.
-		 */
-		http_CollectHdr(sp->wrk->beresp, H_Cache_Control);
-		http_CollectHdr(sp->wrk->beresp, H_Vary);
-
-		/*
-		 * Figure out how the fetch is supposed to happen, before the
-		 * headers are adultered by VCL
-		 * NB: Also sets other sp->wrk variables
-		 */
-		sp->wrk->body_status = RFC2616_Body(sp);
-
-		sp->err_code = http_GetStatus(sp->wrk->beresp);
-
-		/*
-		 * What does RFC2616 think about TTL ?
-		 */
-		EXP_Clr(&sp->wrk->exp);
-		sp->wrk->exp.entered = VTIM_real();
-		RFC2616_Ttl(sp);
-
-		/* pass from vclrecv{} has negative TTL */
-		if (sp->objcore == NULL)
-			sp->wrk->exp.ttl = -1.;
-
-		AZ(sp->wrk->do_esi);
-
-		VCL_fetch_method(sp);
-
-		switch (sp->handling) {
-		case VCL_RET_HIT_FOR_PASS:
-			if (sp->objcore != NULL)
-				sp->objcore->flags |= OC_F_PASS;
-			sp->step = STP_FETCHBODY;
-			return (0);
-		case VCL_RET_DELIVER:
-			AssertObjCorePassOrBusy(sp->objcore);
-			sp->step = STP_FETCHBODY;
-			return (0);
-		default:
-			break;
-		}
-
-		/* We are not going to fetch the body, Close the connection */
-		VDI_CloseFd(sp->wrk);
-	}
-
-	/* Clean up partial fetch */
-	AZ(sp->wrk->vbc);
-
-	if (sp->objcore != NULL) {
-		CHECK_OBJ_NOTNULL(sp->objcore, OBJCORE_MAGIC);
-		AZ(HSH_Deref(sp->wrk, sp->objcore, NULL));
-		sp->objcore = NULL;
-	}
-	http_Setup(sp->wrk->bereq, NULL);
-	http_Setup(sp->wrk->beresp, NULL);
-	sp->wrk->h_content_length = NULL;
-	sp->director = NULL;
-	sp->wrk->storage_hint = NULL;
-
-	switch (sp->handling) {
-	case VCL_RET_RESTART:
-		sp->restarts++;
-		sp->step = STP_RECV;
-		return (0);
-	case VCL_RET_ERROR:
-		sp->step = STP_ERROR;
-		return (0);
-	default:
-		WRONG("Illegal action in vcl_fetch{}");
-	}
-}
-
-/*--------------------------------------------------------------------
- * Fetch response body from the backend
- *
-DOT subgraph xcluster_body {
-DOT	fetchbody [
-DOT		shape=diamond
-DOT		label="stream ?"
-DOT	]
-DOT	fetchbody2 [
-DOT		shape=ellipse
-DOT		label="fetch body\nfrom backend\n"
-DOT	]
-DOT }
-DOT fetchbody -> fetchbody2 [label=no,style=bold,color=red]
-DOT fetchbody -> fetchbody2 [style=bold,color=blue]
-DOT fetchbody -> prepresp [label=yes,style=bold,color=cyan]
-DOT fetchbody2 -> prepresp [style=bold,color=red]
-DOT fetchbody2 -> prepresp [style=bold,color=blue]
- */
-
-
-static int
-cnt_fetchbody(struct sess *sp)
-{
-	int i;
-	struct http *hp, *hp2;
-	char *b;
-	uint16_t nhttp;
-	unsigned l;
-	struct vsb *vary = NULL;
-	int varyl = 0, pass;
-
-	assert(sp->handling == VCL_RET_HIT_FOR_PASS ||
-	    sp->handling == VCL_RET_DELIVER);
-
-	if (sp->objcore == NULL) {
-		/* This is a pass from vcl_recv */
-		pass = 1;
-		/* VCL may have fiddled this, but that doesn't help */
-		sp->wrk->exp.ttl = -1.;
-	} else if (sp->handling == VCL_RET_HIT_FOR_PASS) {
-		/* pass from vcl_fetch{} -> hit-for-pass */
-		/* XXX: the bereq was not filtered pass... */
-		pass = 1;
-	} else {
-		/* regular object */
-		pass = 0;
-	}
-
-	/*
-	 * The VCL variables beresp.do_g[un]zip tells us how we want the
-	 * object processed before it is stored.
-	 *
-	 * The backend Content-Encoding header tells us what we are going
-	 * to receive, which we classify in the following three classes:
-	 *
-	 *	"Content-Encoding: gzip"	--> object is gzip'ed.
-	 *	no Content-Encoding		--> object is not gzip'ed.
-	 *	anything else			--> do nothing wrt gzip
-	 *
-	 */
-
-	AZ(sp->wrk->vfp);
-
-	/* We do nothing unless the param is set */
-	if (!cache_param->http_gzip_support)
-		sp->wrk->do_gzip = sp->wrk->do_gunzip = 0;
-
-	sp->wrk->is_gzip =
-	    http_HdrIs(sp->wrk->beresp, H_Content_Encoding, "gzip");
-
-	sp->wrk->is_gunzip =
-	    !http_GetHdr(sp->wrk->beresp, H_Content_Encoding, NULL);
-
-	/* It can't be both */
-	assert(sp->wrk->is_gzip == 0 || sp->wrk->is_gunzip == 0);
-
-	/* We won't gunzip unless it is gzip'ed */
-	if (sp->wrk->do_gunzip && !sp->wrk->is_gzip)
-		sp->wrk->do_gunzip = 0;
-
-	/* If we do gunzip, remove the C-E header */
-	if (sp->wrk->do_gunzip)
-		http_Unset(sp->wrk->beresp, H_Content_Encoding);
-
-	/* We wont gzip unless it is ungziped */
-	if (sp->wrk->do_gzip && !sp->wrk->is_gunzip)
-		sp->wrk->do_gzip = 0;
-
-	/* If we do gzip, add the C-E header */
-	if (sp->wrk->do_gzip)
-		http_SetHeader(sp->wrk, sp->vsl_id, sp->wrk->beresp,
-		    "Content-Encoding: gzip");
-
-	/* But we can't do both at the same time */
-	assert(sp->wrk->do_gzip == 0 || sp->wrk->do_gunzip == 0);
-
-	/* ESI takes precedence and handles gzip/gunzip itself */
-	if (sp->wrk->do_esi)
-		sp->wrk->vfp = &vfp_esi;
-	else if (sp->wrk->do_gunzip)
-		sp->wrk->vfp = &vfp_gunzip;
-	else if (sp->wrk->do_gzip)
-		sp->wrk->vfp = &vfp_gzip;
-	else if (sp->wrk->is_gzip)
-		sp->wrk->vfp = &vfp_testgzip;
-
-	if (sp->wrk->do_esi || sp->esi_level > 0)
-		sp->wrk->do_stream = 0;
-	if (!sp->wantbody)
-		sp->wrk->do_stream = 0;
-
-	l = http_EstimateWS(sp->wrk->beresp,
-	    pass ? HTTPH_R_PASS : HTTPH_A_INS, &nhttp);
-
-	/* Create Vary instructions */
-	if (sp->objcore != NULL) {
-		CHECK_OBJ_NOTNULL(sp->objcore, OBJCORE_MAGIC);
-		vary = VRY_Create(sp, sp->wrk->beresp);
-		if (vary != NULL) {
-			varyl = VSB_len(vary);
-			assert(varyl > 0);
-			l += varyl;
-		}
-	}
-
-	/*
-	 * Space for producing a Content-Length: header including padding
-	 * A billion gigabytes is enough for anybody.
-	 */
-	l += strlen("Content-Length: XxxXxxXxxXxxXxxXxx") + sizeof(void *);
-
-	if (sp->wrk->exp.ttl < cache_param->shortlived || sp->objcore == NULL)
-		sp->wrk->storage_hint = TRANSIENT_STORAGE;
-
-	sp->obj = STV_NewObject(sp, sp->wrk->storage_hint, l,
-	    &sp->wrk->exp, nhttp);
-	if (sp->obj == NULL) {
-		/*
-		 * Try to salvage the transaction by allocating a
-		 * shortlived object on Transient storage.
-		 */
-		sp->obj = STV_NewObject(sp, TRANSIENT_STORAGE, l,
-		    &sp->wrk->exp, nhttp);
-		if (sp->wrk->exp.ttl > cache_param->shortlived)
-			sp->wrk->exp.ttl = cache_param->shortlived;
-		sp->wrk->exp.grace = 0.0;
-		sp->wrk->exp.keep = 0.0;
-	}
-	if (sp->obj == NULL) {
-		sp->err_code = 503;
-		sp->step = STP_ERROR;
-		VDI_CloseFd(sp->wrk);
-		return (0);
-	}
-	CHECK_OBJ_NOTNULL(sp->obj, OBJECT_MAGIC);
-
-	sp->wrk->storage_hint = NULL;
-
-	if (sp->wrk->do_gzip || (sp->wrk->is_gzip && !sp->wrk->do_gunzip))
-		sp->obj->gziped = 1;
-
-	if (vary != NULL) {
-		sp->obj->vary =
-		    (void *)WS_Alloc(sp->obj->http->ws, varyl);
-		AN(sp->obj->vary);
-		memcpy(sp->obj->vary, VSB_data(vary), varyl);
-		VRY_Validate(sp->obj->vary);
-		VSB_delete(vary);
-	}
-
-	sp->obj->xid = sp->xid;
-	sp->obj->response = sp->err_code;
-	WS_Assert(sp->obj->ws_o);
-
-	/* Filter into object */
-	hp = sp->wrk->beresp;
-	hp2 = sp->obj->http;
-
-	hp2->logtag = HTTP_Obj;
-	http_CopyResp(hp2, hp);
-	http_FilterFields(sp->wrk, sp->vsl_id, hp2, hp,
-	    pass ? HTTPH_R_PASS : HTTPH_A_INS);
-	http_CopyHome(sp->wrk, sp->vsl_id, hp2);
-
-	if (http_GetHdr(hp, H_Last_Modified, &b))
-		sp->obj->last_modified = VTIM_parse(b);
-	else
-		sp->obj->last_modified = floor(sp->wrk->exp.entered);
-
-	assert(WRW_IsReleased(sp->wrk));
-
-	/*
-	 * If we can deliver a 304 reply, we don't bother streaming.
-	 * Notice that vcl_deliver{} could still nuke the headers
-	 * that allow the 304, in which case we return 200 non-stream.
-	 */
-	if (sp->obj->response == 200 &&
-	    sp->http->conds &&
-	    RFC2616_Do_Cond(sp))
-		sp->wrk->do_stream = 0;
-
-	AssertObjCorePassOrBusy(sp->obj->objcore);
-
-	if (sp->wrk->do_stream) {
-		sp->step = STP_PREPRESP;
-		return (0);
-	}
-
-	/* Use unmodified headers*/
-	i = FetchBody(sp->wrk, sp->obj);
-
-	sp->wrk->h_content_length = NULL;
-
-	http_Setup(sp->wrk->bereq, NULL);
-	http_Setup(sp->wrk->beresp, NULL);
-	sp->wrk->vfp = NULL;
-	assert(WRW_IsReleased(sp->wrk));
-	AZ(sp->wrk->vbc);
-	AN(sp->director);
-
-	if (i) {
-		HSH_Drop(sp);
-		AZ(sp->obj);
-		sp->err_code = 503;
-		sp->step = STP_ERROR;
-		return (0);
-	}
-
-	if (sp->obj->objcore != NULL) {
-		EXP_Insert(sp->obj);
-		AN(sp->obj->objcore);
-		AN(sp->obj->objcore->ban);
-		HSH_Unbusy(sp);
-	}
-	sp->wrk->acct_tmp.fetch++;
-	sp->step = STP_PREPRESP;
-	return (0);
-}
-
-/*--------------------------------------------------------------------
- * Stream the body as we fetch it
-DOT subgraph xstreambody {
-DOT	streambody [
-DOT		shape=ellipse
-DOT		label="streaming\nfetch/deliver"
-DOT	]
-DOT }
-DOT streambody -> DONE [style=bold,color=cyan]
- */
-
-static int
-cnt_streambody(struct sess *sp)
-{
-	int i;
-	struct stream_ctx sctx;
-	uint8_t obuf[sp->wrk->res_mode & RES_GUNZIP ?
-	    cache_param->gzip_stack_buffer : 1];
-
-	memset(&sctx, 0, sizeof sctx);
-	sctx.magic = STREAM_CTX_MAGIC;
-	AZ(sp->wrk->sctx);
-	sp->wrk->sctx = &sctx;
-
-	if (sp->wrk->res_mode & RES_GUNZIP) {
-		sctx.vgz = VGZ_NewUngzip(sp->wrk, "U S -");
-		sctx.obuf = obuf;
-		sctx.obuf_len = sizeof (obuf);
-	}
-
-	RES_StreamStart(sp);
-
-	AssertObjCorePassOrBusy(sp->obj->objcore);
-
-	i = FetchBody(sp->wrk, sp->obj);
-
-	sp->wrk->h_content_length = NULL;
-
-	http_Setup(sp->wrk->bereq, NULL);
-	http_Setup(sp->wrk->beresp, NULL);
-	sp->wrk->vfp = NULL;
-	AZ(sp->wrk->vbc);
-	AN(sp->director);
-
-	if (!i && sp->obj->objcore != NULL) {
-		EXP_Insert(sp->obj);
-		AN(sp->obj->objcore);
-		AN(sp->obj->objcore->ban);
-		HSH_Unbusy(sp);
-	} else {
-		sp->doclose = "Stream error";
-	}
-	sp->wrk->acct_tmp.fetch++;
-	sp->director = NULL;
-	sp->restarts = 0;
-
-	RES_StreamEnd(sp);
-	if (sp->wrk->res_mode & RES_GUNZIP)
-		(void)VGZ_Destroy(&sctx.vgz, sp->vsl_id);
-
-	sp->wrk->sctx = NULL;
-	assert(WRW_IsReleased(sp->wrk));
-	assert(sp->wrk->wrw.ciov == sp->wrk->wrw.siov);
-	(void)HSH_Deref(sp->wrk, NULL, &sp->obj);
-	http_Setup(sp->wrk->resp, NULL);
-	sp->step = STP_DONE;
-	return (0);
-}
-
-/*--------------------------------------------------------------------
- * The very first request
- */
-static int
-cnt_first(struct sess *sp)
-{
-
-	/*
-	 * XXX: If we don't have acceptfilters we are somewhat subject
-	 * XXX: to DoS'ing here.  One remedy would be to set a shorter
-	 * XXX: SO_RCVTIMEO and once we have received something here
-	 * XXX: increase it to the normal value.
-	 */
-
-	assert(sp->xid == 0);
-	assert(sp->restarts == 0);
-	VCA_Prep(sp);
-
-	/* Record the session watermark */
-	sp->ws_ses = WS_Snapshot(sp->ws);
-
-	/* Receive a HTTP protocol request */
-	HTC_Init(sp->htc, sp->ws, sp->fd, sp->vsl_id, cache_param->http_req_size,
-	    cache_param->http_req_hdr_len);
-	sp->wrk->lastused = sp->t_open;
-	sp->wrk->acct_tmp.sess++;
-
-	sp->step = STP_WAIT;
-	return (0);
-}
-
-/*--------------------------------------------------------------------
- * HIT
- * We had a cache hit.  Ask VCL, then march off as instructed.
- *
-DOT subgraph xcluster_hit {
-DOT	hit [
-DOT		shape=record
-DOT		label="vcl_hit()|req.\nobj."
-DOT	]
-DOT }
-DOT hit -> err_hit [label="error"]
-DOT err_hit [label="ERROR",shape=plaintext]
-DOT hit -> rst_hit [label="restart",color=purple]
-DOT rst_hit [label="RESTART",shape=plaintext]
-DOT hit -> pass [label=pass,style=bold,color=red]
-DOT hit -> prepresp [label="deliver",style=bold,color=green]
- */
-
-static int
-cnt_hit(struct sess *sp)
-{
-
-	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
-	CHECK_OBJ_NOTNULL(sp->obj, OBJECT_MAGIC);
-	CHECK_OBJ_NOTNULL(sp->vcl, VCL_CONF_MAGIC);
-
-	assert(!(sp->obj->objcore->flags & OC_F_PASS));
-
-	AZ(sp->wrk->do_stream);
-
-	VCL_hit_method(sp);
-
-	if (sp->handling == VCL_RET_DELIVER) {
-		/* Dispose of any body part of the request */
-		(void)FetchReqBody(sp);
-		AZ(sp->wrk->bereq->ws);
-		AZ(sp->wrk->beresp->ws);
-		sp->step = STP_PREPRESP;
-		return (0);
-	}
-
-	/* Drop our object, we won't need it */
-	(void)HSH_Deref(sp->wrk, NULL, &sp->obj);
-	sp->objcore = NULL;
-
-	switch(sp->handling) {
-	case VCL_RET_PASS:
-		sp->step = STP_PASS;
-		return (0);
-	case VCL_RET_ERROR:
-		sp->step = STP_ERROR;
-		return (0);
-	case VCL_RET_RESTART:
-		sp->director = NULL;
-		sp->restarts++;
-		sp->step = STP_RECV;
-		return (0);
-	default:
-		WRONG("Illegal action in vcl_hit{}");
-	}
-}
-
-/*--------------------------------------------------------------------
- * LOOKUP
- * Hash things together and look object up in hash-table.
- *
- * LOOKUP consists of two substates so that we can reenter if we
- * encounter a busy object.
- *
-DOT subgraph xcluster_lookup {
-DOT	hash [
-DOT		shape=record
-DOT		label="vcl_hash()|req."
-DOT	]
-DOT	lookup [
-DOT		shape=diamond
-DOT		label="obj in cache ?\ncreate if not"
-DOT	]
-DOT	lookup2 [
-DOT		shape=diamond
-DOT		label="obj.f.pass ?"
-DOT	]
-DOT	hash -> lookup [label="hash",style=bold,color=green]
-DOT	lookup -> lookup2 [label="yes",style=bold,color=green]
-DOT }
-DOT lookup2 -> hit [label="no", style=bold,color=green]
-DOT lookup2 -> pass [label="yes",style=bold,color=red]
-DOT lookup -> miss [label="no",style=bold,color=blue]
- */
-
-static int
-cnt_lookup(struct sess *sp)
-{
-	struct objcore *oc;
-	struct object *o;
-	struct objhead *oh;
-
-	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
-	CHECK_OBJ_NOTNULL(sp->vcl, VCL_CONF_MAGIC);
-
-	if (sp->hash_objhead == NULL) {
-		/* Not a waiting list return */
-		AZ(sp->vary_b);
-		AZ(sp->vary_l);
-		AZ(sp->vary_e);
-		(void)WS_Reserve(sp->ws, 0);
-	} else {
-		AN(sp->ws->r);
-	}
-	sp->vary_b = (void*)sp->ws->f;
-	sp->vary_e = (void*)sp->ws->r;
-	sp->vary_b[2] = '\0';
-
-	oc = HSH_Lookup(sp, &oh);
-
-	if (oc == NULL) {
-		/*
-		 * We lost the session to a busy object, disembark the
-		 * worker thread.   The hash code to restart the session,
-		 * still in STP_LOOKUP, later when the busy object isn't.
-		 * NB:  Do not access sp any more !
-		 */
-		return (1);
-	}
-
-
-	CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
-	CHECK_OBJ_NOTNULL(oh, OBJHEAD_MAGIC);
-
-	/* If we inserted a new object it's a miss */
-	if (oc->flags & OC_F_BUSY) {
-		sp->wrk->stats.cache_miss++;
-
-		if (sp->vary_l != NULL) {
-			assert(oc->busyobj->vary == sp->vary_b);
-			VRY_Validate(oc->busyobj->vary);
-			WS_ReleaseP(sp->ws, (void*)sp->vary_l);
-		} else {
-			AZ(oc->busyobj->vary);
-			WS_Release(sp->ws, 0);
-		}
-		sp->vary_b = NULL;
-		sp->vary_l = NULL;
-		sp->vary_e = NULL;
-
-		sp->objcore = oc;
-		sp->step = STP_MISS;
-		return (0);
-	}
-
-	o = oc_getobj(sp->wrk, oc);
-	CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC);
-	sp->obj = o;
-
-	WS_Release(sp->ws, 0);
-	sp->vary_b = NULL;
-	sp->vary_l = NULL;
-	sp->vary_e = NULL;
-
-	if (oc->flags & OC_F_PASS) {
-		sp->wrk->stats.cache_hitpass++;
-		WSP(sp, SLT_HitPass, "%u", sp->obj->xid);
-		(void)HSH_Deref(sp->wrk, NULL, &sp->obj);
-		sp->objcore = NULL;
-		sp->step = STP_PASS;
-		return (0);
-	}
-
-	sp->wrk->stats.cache_hit++;
-	WSP(sp, SLT_Hit, "%u", sp->obj->xid);
-	sp->step = STP_HIT;
-	return (0);
-}
-
-/*--------------------------------------------------------------------
- * We had a miss, ask VCL, proceed as instructed
- *
-DOT subgraph xcluster_miss {
-DOT	miss [
-DOT		shape=ellipse
-DOT		label="filter req.->bereq."
-DOT	]
-DOT	vcl_miss [
-DOT		shape=record
-DOT		label="vcl_miss()|req.\nbereq."
-DOT	]
-DOT	miss -> vcl_miss [style=bold,color=blue]
-DOT }
-DOT vcl_miss -> rst_miss [label="restart",color=purple]
-DOT rst_miss [label="RESTART",shape=plaintext]
-DOT vcl_miss -> err_miss [label="error"]
-DOT err_miss [label="ERROR",shape=plaintext]
-DOT vcl_miss -> fetch [label="fetch",style=bold,color=blue]
-DOT vcl_miss -> pass [label="pass",style=bold,color=red]
-DOT
- */
-
-static int
-cnt_miss(struct sess *sp)
-{
-
-	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
-	CHECK_OBJ_NOTNULL(sp->vcl, VCL_CONF_MAGIC);
-
-	AZ(sp->obj);
-	AN(sp->objcore);
-	WS_Reset(sp->wrk->ws, NULL);
-	http_Setup(sp->wrk->bereq, sp->wrk->ws);
-	http_FilterHeader(sp, HTTPH_R_FETCH);
-	http_ForceGet(sp->wrk->bereq);
-	if (cache_param->http_gzip_support) {
-		/*
-		 * We always ask the backend for gzip, even if the
-		 * client doesn't grok it.  We will uncompress for
-		 * the minority of clients which don't.
-		 */
-		http_Unset(sp->wrk->bereq, H_Accept_Encoding);
-		http_SetHeader(sp->wrk, sp->vsl_id, sp->wrk->bereq,
-		    "Accept-Encoding: gzip");
-	}
-	sp->wrk->connect_timeout = 0;
-	sp->wrk->first_byte_timeout = 0;
-	sp->wrk->between_bytes_timeout = 0;
-	VCL_miss_method(sp);
-	switch(sp->handling) {
-	case VCL_RET_ERROR:
-		AZ(HSH_Deref(sp->wrk, sp->objcore, NULL));
-		sp->objcore = NULL;
-		http_Setup(sp->wrk->bereq, NULL);
-		sp->step = STP_ERROR;
-		return (0);
-	case VCL_RET_PASS:
-		AZ(HSH_Deref(sp->wrk, sp->objcore, NULL));
-		sp->objcore = NULL;
-		sp->step = STP_PASS;
-		return (0);
-	case VCL_RET_FETCH:
-		sp->step = STP_FETCH;
-		return (0);
-	case VCL_RET_RESTART:
-		AZ(HSH_Deref(sp->wrk, sp->objcore, NULL));
-		sp->objcore = NULL;
-		INCOMPL();
-	default:
-		WRONG("Illegal action in vcl_miss{}");
-	}
-}
-
-/*--------------------------------------------------------------------
- * Start pass processing by getting headers from backend, then
- * continue in passbody.
- *
-DOT subgraph xcluster_pass {
-DOT	pass [
-DOT		shape=ellipse
-DOT		label="deref obj."
-DOT	]
-DOT	pass2 [
-DOT		shape=ellipse
-DOT		label="filter req.->bereq."
-DOT	]
-DOT	vcl_pass [
-DOT		shape=record
-DOT		label="vcl_pass()|req.\nbereq."
-DOT	]
-DOT	pass_do [
-DOT		shape=ellipse
-DOT		label="create anon object\n"
-DOT	]
-DOT	pass -> pass2 [style=bold, color=red]
-DOT	pass2 -> vcl_pass [style=bold, color=red]
-DOT	vcl_pass -> pass_do [label="pass"] [style=bold, color=red]
-DOT }
-DOT pass_do -> fetch [style=bold, color=red]
-DOT vcl_pass -> rst_pass [label="restart",color=purple]
-DOT rst_pass [label="RESTART",shape=plaintext]
-DOT vcl_pass -> err_pass [label="error"]
-DOT err_pass [label="ERROR",shape=plaintext]
- */
-
-static int
-cnt_pass(struct sess *sp)
-{
-
-	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
-	CHECK_OBJ_NOTNULL(sp->vcl, VCL_CONF_MAGIC);
-	AZ(sp->obj);
-
-	WS_Reset(sp->wrk->ws, NULL);
-	http_Setup(sp->wrk->bereq, sp->wrk->ws);
-	http_FilterHeader(sp, HTTPH_R_PASS);
-
-	sp->wrk->connect_timeout = 0;
-	sp->wrk->first_byte_timeout = 0;
-	sp->wrk->between_bytes_timeout = 0;
-	VCL_pass_method(sp);
-	if (sp->handling == VCL_RET_ERROR) {
-		http_Setup(sp->wrk->bereq, NULL);
-		sp->step = STP_ERROR;
-		return (0);
-	}
-	assert(sp->handling == VCL_RET_PASS);
-	sp->wrk->acct_tmp.pass++;
-	sp->sendbody = 1;
-	sp->step = STP_FETCH;
-	return (0);
-}
-
-/*--------------------------------------------------------------------
- * Ship the request header to the backend unchanged, then pipe
- * until one of the ends close the connection.
- *
-DOT subgraph xcluster_pipe {
-DOT	pipe [
-DOT		shape=ellipse
-DOT		label="Filter req.->bereq."
-DOT	]
-DOT	vcl_pipe [
-DOT		shape=record
-DOT		label="vcl_pipe()|req.\nbereq\."
-DOT	]
-DOT	pipe_do [
-DOT		shape=ellipse
-DOT		label="send bereq.\npipe until close"
-DOT	]
-DOT	vcl_pipe -> pipe_do [label="pipe",style=bold,color=orange]
-DOT	pipe -> vcl_pipe [style=bold,color=orange]
-DOT }
-DOT pipe_do -> DONE [style=bold,color=orange]
-DOT vcl_pipe -> err_pipe [label="error"]
-DOT err_pipe [label="ERROR",shape=plaintext]
- */
-
-static int
-cnt_pipe(struct sess *sp)
-{
-
-	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
-	CHECK_OBJ_NOTNULL(sp->vcl, VCL_CONF_MAGIC);
-
-	sp->wrk->acct_tmp.pipe++;
-	WS_Reset(sp->wrk->ws, NULL);
-	http_Setup(sp->wrk->bereq, sp->wrk->ws);
-	http_FilterHeader(sp, HTTPH_R_PIPE);
-
-	VCL_pipe_method(sp);
-
-	if (sp->handling == VCL_RET_ERROR)
-		INCOMPL();
-	assert(sp->handling == VCL_RET_PIPE);
-
-	PipeSession(sp);
-	assert(WRW_IsReleased(sp->wrk));
-	http_Setup(sp->wrk->bereq, NULL);
-	sp->step = STP_DONE;
-	return (0);
-}
-
-/*--------------------------------------------------------------------
- * RECV
- * We have a complete request, set everything up and start it.
- *
-DOT subgraph xcluster_recv {
-DOT	recv [
-DOT		shape=record
-DOT		label="vcl_recv()|req."
-DOT	]
-DOT }
-DOT RESTART -> recv
-DOT recv -> pipe [label="pipe",style=bold,color=orange]
-DOT recv -> pass2 [label="pass",style=bold,color=red]
-DOT recv -> err_recv [label="error"]
-DOT err_recv [label="ERROR",shape=plaintext]
-DOT recv -> hash [label="lookup",style=bold,color=green]
- */
-
-static int
-cnt_recv(struct sess *sp)
-{
-	unsigned recv_handling;
-
-	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
-	CHECK_OBJ_NOTNULL(sp->vcl, VCL_CONF_MAGIC);
-	AZ(sp->obj);
-	assert(sp->wrk->wrw.ciov == sp->wrk->wrw.siov);
-
-	/* By default we use the first backend */
-	AZ(sp->director);
-	sp->director = sp->vcl->director[0];
-	AN(sp->director);
-
-	sp->disable_esi = 0;
-	sp->hash_always_miss = 0;
-	sp->hash_ignore_busy = 0;
-	sp->client_identity = NULL;
-
-	http_CollectHdr(sp->http, H_Cache_Control);
-
-	VCL_recv_method(sp);
-	recv_handling = sp->handling;
-
-	if (sp->restarts >= cache_param->max_restarts) {
-		if (sp->err_code == 0)
-			sp->err_code = 503;
-		sp->step = STP_ERROR;
-		return (0);
-	}
-
-	/* Zap these, in case we came here through restart */
-	sp->wrk->do_esi = 0;
-	sp->wrk->is_gzip = 0;
-	sp->wrk->is_gunzip = 0;
-	sp->wrk->do_gzip = 0;
-	sp->wrk->do_gunzip = 0;
-	sp->wrk->do_stream = 0;
-
-	if (cache_param->http_gzip_support &&
-	     (recv_handling != VCL_RET_PIPE) &&
-	     (recv_handling != VCL_RET_PASS)) {
-		if (RFC2616_Req_Gzip(sp)) {
-			http_Unset(sp->http, H_Accept_Encoding);
-			http_SetHeader(sp->wrk, sp->vsl_id, sp->http,
-			    "Accept-Encoding: gzip");
-		} else {
-			http_Unset(sp->http, H_Accept_Encoding);
-		}
-	}
-
-	SHA256_Init(sp->wrk->sha256ctx);
-	VCL_hash_method(sp);
-	assert(sp->handling == VCL_RET_HASH);
-	SHA256_Final(sp->digest, sp->wrk->sha256ctx);
-
-	if (!strcmp(sp->http->hd[HTTP_HDR_REQ].b, "HEAD"))
-		sp->wantbody = 0;
-	else
-		sp->wantbody = 1;
-
-	sp->sendbody = 0;
-	switch(recv_handling) {
-	case VCL_RET_LOOKUP:
-		/* XXX: discard req body, if any */
-		sp->step = STP_LOOKUP;
-		return (0);
-	case VCL_RET_PIPE:
-		if (sp->esi_level > 0) {
-			/* XXX: VSL something */
-			INCOMPL();
-			/* sp->step = STP_DONE; */
-			return (1);
-		}
-		sp->step = STP_PIPE;
-		return (0);
-	case VCL_RET_PASS:
-		sp->step = STP_PASS;
-		return (0);
-	case VCL_RET_ERROR:
-		/* XXX: discard req body, if any */
-		sp->step = STP_ERROR;
-		return (0);
-	default:
-		WRONG("Illegal action in vcl_recv{}");
-	}
-}
-
-/*--------------------------------------------------------------------
- * START
- * Handle a request, wherever it came from recv/restart.
- *
-DOT start [shape=box,label="Dissect request"]
-DOT start -> recv [style=bold,color=green]
- */
-
-static int
-cnt_start(struct sess *sp)
-{
-	uint16_t done;
-	char *p;
-	const char *r = "HTTP/1.1 100 Continue\r\n\r\n";
-
-	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
-	AZ(sp->restarts);
-	AZ(sp->obj);
-	AZ(sp->vcl);
-
-	/* Update stats of various sorts */
-	sp->wrk->stats.client_req++;
-	sp->t_req = VTIM_real();
-	sp->wrk->lastused = sp->t_req;
-	sp->wrk->acct_tmp.req++;
-
-	/* Assign XID and log */
-	sp->xid = ++xids;				/* XXX not locked */
-	WSP(sp, SLT_ReqStart, "%s %s %u", sp->addr, sp->port,  sp->xid);
-
-	/* Borrow VCL reference from worker thread */
-	VCL_Refresh(&sp->wrk->vcl);
-	sp->vcl = sp->wrk->vcl;
-	sp->wrk->vcl = NULL;
-
-	http_Setup(sp->http, sp->ws);
-	done = http_DissectRequest(sp);
-
-	/* If we could not even parse the request, just close */
-	if (done == 400) {
-		sp->step = STP_DONE;
-		SES_Close(sp, "junk");
-		return (0);
-	}
-
-	/* Catch request snapshot */
-	sp->ws_req = WS_Snapshot(sp->ws);
-
-	/* Catch original request, before modification */
-	HTTP_Copy(sp->http0, sp->http);
-
-	if (done != 0) {
-		sp->err_code = done;
-		sp->step = STP_ERROR;
-		return (0);
-	}
-
-	sp->doclose = http_DoConnection(sp->http);
-
-	/* XXX: Handle TRACE & OPTIONS of Max-Forwards = 0 */
-
-	/*
-	 * Handle Expect headers
-	 */
-	if (http_GetHdr(sp->http, H_Expect, &p)) {
-		if (strcasecmp(p, "100-continue")) {
-			sp->err_code = 417;
-			sp->step = STP_ERROR;
-			return (0);
-		}
-
-		/* XXX: Don't bother with write failures for now */
-		(void)write(sp->fd, r, strlen(r));
-		/* XXX: When we do ESI includes, this is not removed
-		 * XXX: because we use http0 as our basis.  Believed
-		 * XXX: safe, but potentially confusing.
-		 */
-		http_Unset(sp->http, H_Expect);
-	}
-
-	sp->step = STP_RECV;
-	return (0);
-}
-
-/*--------------------------------------------------------------------
- * Central state engine dispatcher.
- *
- * Kick the session around until it has had enough.
- *
- */
-
-static void
-cnt_diag(struct sess *sp, const char *state)
-{
-	if (sp->wrk != NULL) {
-		WSP(sp, SLT_Debug, "thr %p STP_%s sp %p obj %p vcl %p",
-		    pthread_self(), state, sp, sp->obj, sp->vcl);
-		WSL_Flush(sp->wrk, 0);
-	} else {
-		VSL(SLT_Debug, sp->vsl_id,
-		    "thr %p STP_%s sp %p obj %p vcl %p",
-		    pthread_self(), state, sp, sp->obj, sp->vcl);
-	}
-}
-
-void
-CNT_Session(struct sess *sp)
-{
-	int done;
-	struct worker *w;
-
-	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
-	w = sp->wrk;
-	CHECK_OBJ_NOTNULL(w, WORKER_MAGIC);
-
-	/*
-	 * Possible entrance states
-	 */
-	assert(
-	    sp->step == STP_FIRST ||
-	    sp->step == STP_START ||
-	    sp->step == STP_LOOKUP ||
-	    sp->step == STP_RECV);
-
-	AZ(w->do_stream);
-	AZ(w->is_gzip);
-	AZ(w->do_gzip);
-	AZ(w->is_gunzip);
-	AZ(w->do_gunzip);
-	AZ(w->do_esi);
-
-	/*
-	 * Whenever we come in from the acceptor or waiter, we need to set
-	 * blocking mode, but there is no point in setting it when we come from
-	 * ESI or when a parked sessions returns.
-	 * It would be simpler to do this in the acceptor or waiter, but we'd
-	 * rather do the syscall in the worker thread.
-	 * On systems which return errors for ioctl, we close early
-	 */
-	if ((sp->step == STP_FIRST || sp->step == STP_START) &&
-	    VTCP_blocking(sp->fd)) {
-		if (errno == ECONNRESET)
-			SES_Close(sp, "remote closed");
-		else
-			SES_Close(sp, "error");
-		sp->step = STP_DONE;
-	}
-
-	/*
-	 * NB: Once done is set, we can no longer touch sp!
-	 */
-	for (done = 0; !done; ) {
-		assert(sp->wrk == w);
-		/*
-		 * This is a good place to be paranoid about the various
-		 * pointers still pointing to the things we expect.
-		 */
-		CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
-		CHECK_OBJ_ORNULL(sp->obj, OBJECT_MAGIC);
-		CHECK_OBJ_NOTNULL(sp->wrk, WORKER_MAGIC);
-		CHECK_OBJ_ORNULL(w->nobjhead, OBJHEAD_MAGIC);
-		WS_Assert(w->ws);
-
-		switch (sp->step) {
-#define STEP(l,u) \
-		    case STP_##u: \
-			if (cache_param->diag_bitmap & 0x01) \
-				cnt_diag(sp, #u); \
-			done = cnt_##l(sp); \
-		        break;
-#include "tbl/steps.h"
-#undef STEP
-		default:
-			WRONG("State engine misfire");
-		}
-		WS_Assert(w->ws);
-		CHECK_OBJ_ORNULL(w->nobjhead, OBJHEAD_MAGIC);
-	}
-	WSL_Flush(w, 0);
-	AZ(w->do_stream);
-	AZ(w->is_gzip);
-	AZ(w->do_gzip);
-	AZ(w->is_gunzip);
-	AZ(w->do_gunzip);
-	AZ(w->do_esi);
-#define ACCT(foo)	AZ(w->acct_tmp.foo);
-#include "tbl/acct_fields.h"
-#undef ACCT
-	assert(WRW_IsReleased(w));
-}
-
-/*
-DOT }
-*/
-
-/*--------------------------------------------------------------------
- * Debugging aids
- */
-
-static void
-cli_debug_xid(struct cli *cli, const char * const *av, void *priv)
-{
-	(void)priv;
-	if (av[2] != NULL)
-		xids = strtoul(av[2], NULL, 0);
-	VCLI_Out(cli, "XID is %u", xids);
-}
-
-/*
- * Default to seed=1, this is the only seed value POSIXl guarantees will
- * result in a reproducible random number sequence.
- */
-static void
-cli_debug_srandom(struct cli *cli, const char * const *av, void *priv)
-{
-	(void)priv;
-	unsigned seed = 1;
-
-	if (av[2] != NULL)
-		seed = strtoul(av[2], NULL, 0);
-	srandom(seed);
-	srand48(random());
-	VCLI_Out(cli, "Random(3) seeded with %lu", seed);
-}
-
-static struct cli_proto debug_cmds[] = {
-	{ "debug.xid", "debug.xid",
-		"\tExamine or set XID\n", 0, 1, "d", cli_debug_xid },
-	{ "debug.srandom", "debug.srandom",
-		"\tSeed the random(3) function\n", 0, 1, "d", cli_debug_srandom },
-	{ NULL }
-};
-
-/*--------------------------------------------------------------------
- *
- */
-
-void
-CNT_Init(void)
-{
-
-	srandomdev();
-	srand48(random());
-	xids = random();
-	CLI_AddFuncs(debug_cmds);
-}
-
-
diff --git a/bin/varnishd/cache_cli.c b/bin/varnishd/cache_cli.c
deleted file mode 100644
index f29f86a..0000000
--- a/bin/varnishd/cache_cli.c
+++ /dev/null
@@ -1,243 +0,0 @@
-/*-
- * Copyright (c) 2006 Verdens Gang AS
- * Copyright (c) 2006-2011 Varnish Software AS
- * All rights reserved.
- *
- * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- *
- * Caching process CLI handling.
- *
- * We only have one CLI source, the stdin/stdout pipes from the manager
- * process, but we complicate things by having undocumented commands that
- * we do not want to show in a plain help, and by having commands that the
- * manager has already shown in help before asking us.
- */
-
-#include "config.h"
-
-#include <stddef.h>			// offsetof
-
-#include "cache.h"
-#include "common/heritage.h"
-
-#include "cache_backend.h"		// struct vbc
-#include "hash/hash_slinger.h"		// struct objhead
-#include "vcli.h"
-#include "vcli_common.h"
-#include "vcli_priv.h"
-#include "vcli_serve.h"
-
-pthread_t		cli_thread;
-static struct lock	cli_mtx;
-static int		add_check;
-static struct VCLS	*cls;
-
-/*
- * The CLI commandlist is split in three:
- *  - Commands we get from/share with the manager, we don't show these
- *	in help, as the manager already did that.
- *  - Cache process commands, show in help
- *  - Undocumented debug commands, show in undocumented "help -d"
- */
-
-/*--------------------------------------------------------------------
- * Add CLI functions to the appropriate command set
- */
-
-void
-CLI_AddFuncs(struct cli_proto *p)
-{
-
-	AZ(add_check);
-	Lck_Lock(&cli_mtx);
-	AZ(VCLS_AddFunc(cls, 0, p));
-	Lck_Unlock(&cli_mtx);
-}
-
-static void
-cli_cb_before(const struct cli *cli)
-{
-
-	ASSERT_CLI();
-	VSL(SLT_CLI, 0, "Rd %s", cli->cmd);
-	VCL_Poll();
-	VBE_Poll();
-	Lck_Lock(&cli_mtx);
-}
-
-static void
-cli_cb_after(const struct cli *cli)
-{
-
-	ASSERT_CLI();
-	Lck_Unlock(&cli_mtx);
-	VSL(SLT_CLI, 0, "Wr %03u %u %s",
-	    cli->result, VSB_len(cli->sb), VSB_data(cli->sb));
-}
-
-void
-CLI_Run(void)
-{
-	int i;
-
-	add_check = 1;
-
-	AN(VCLS_AddFd(cls, heritage.cli_in, heritage.cli_out, NULL, NULL));
-
-	do {
-		i = VCLS_Poll(cls, -1);
-	} while(i > 0);
-	VSL(SLT_CLI, 0, "EOF on CLI connection, worker stops");
-	VCA_Shutdown();
-}
-
-/*--------------------------------------------------------------------*/
-
-static void
-cli_debug_sizeof(struct cli *cli, const char * const *av, void *priv)
-{
-	(void)av;
-	(void)priv;
-
-#define SZOF(foo)       VCLI_Out(cli, \
-    "sizeof(%s) = %zd = 0x%zx\n", #foo, sizeof(foo), sizeof(foo))
-	SZOF(struct ws);
-	SZOF(struct http);
-	SZOF(struct http_conn);
-	SZOF(struct acct);
-	SZOF(struct worker);
-	SZOF(struct storage);
-	SZOF(struct object);
-	SZOF(struct objcore);
-	SZOF(struct objhead);
-	SZOF(struct sess);
-	SZOF(struct vbc);
-	SZOF(struct VSC_C_main);
-	SZOF(struct lock);
-#if 0
-#define OFOF(foo, bar)	{ foo __foo; VCLI_Out(cli, \
-    "%-30s = 0x%4zx @ 0x%4zx\n", \
-	#foo "." #bar, sizeof(__foo.bar), offsetof(foo, bar)); }
-#if 0
-	OFOF(struct objhead, magic);
-	OFOF(struct objhead, refcnt);
-	OFOF(struct objhead, mtx);
-	OFOF(struct objhead, objcs);
-	OFOF(struct objhead, digest);
-	OFOF(struct objhead, waitinglist);
-	OFOF(struct objhead, _u);
-#endif
-#if 0
-	OFOF(struct http, magic);
-	OFOF(struct http, logtag);
-	OFOF(struct http, ws);
-	OFOF(struct http, hd);
-	OFOF(struct http, hdf);
-	OFOF(struct http, shd);
-	OFOF(struct http, nhd);
-	OFOF(struct http, status);
-	OFOF(struct http, protover);
-	OFOF(struct http, conds);
-#endif
-#if 0
-	OFOF(struct storage, magic);
-	OFOF(struct storage, fd);
-	OFOF(struct storage, where);
-	OFOF(struct storage, list);
-	OFOF(struct storage, stevedore);
-	OFOF(struct storage, priv);
-	OFOF(struct storage, ptr);
-	OFOF(struct storage, len);
-	OFOF(struct storage, space);
-#endif
-#if 0
-	OFOF(struct object, magic);
-	OFOF(struct object, xid);
-	OFOF(struct object, objstore);
-	OFOF(struct object, objcore);
-	OFOF(struct object, ws_o);
-	OFOF(struct object, vary);
-	OFOF(struct object, hits);
-	OFOF(struct object, response);
-	OFOF(struct object, gziped);
-	OFOF(struct object, gzip_start);
-	OFOF(struct object, gzip_last);
-	OFOF(struct object, gzip_stop);
-	OFOF(struct object, len);
-	OFOF(struct object, age);
-	OFOF(struct object, entered);
-	OFOF(struct object, exp);
-	OFOF(struct object, last_modified);
-	OFOF(struct object, last_lru);
-	OFOF(struct object, http);
-	OFOF(struct object, store);
-	OFOF(struct object, esidata);
-	OFOF(struct object, last_use);
-#endif
-#undef OFOF
-#endif
-}
-
-/*--------------------------------------------------------------------*/
-
-static void
-ccf_panic(struct cli *cli, const char * const *av, void *priv)
-{
-
-	(void)cli;
-	(void)av;
-	(void)priv;
-	assert(!strcmp("", "You asked for it"));
-}
-
-/*--------------------------------------------------------------------*/
-
-static struct cli_proto master_cmds[] = {
-	{ CLI_PING,		"i", VCLS_func_ping },
-	{ CLI_HELP,             "i", VCLS_func_help },
-	{ "debug.sizeof", "debug.sizeof",
-		"\tDump sizeof various data structures\n",
-		0, 0, "d", cli_debug_sizeof },
-	{ "debug.panic.worker", "debug.panic.worker",
-		"\tPanic the worker process.\n",
-		0, 0, "d", ccf_panic },
-	{ NULL }
-};
-
-/*--------------------------------------------------------------------
- * Initialize the CLI subsystem
- */
-
-void
-CLI_Init(void)
-{
-
-	Lck_New(&cli_mtx, lck_cli);
-	cli_thread = pthread_self();
-
-	cls = VCLS_New(cli_cb_before, cli_cb_after, cache_param->cli_buffer);
-	AN(cls);
-
-	CLI_AddFuncs(master_cmds);
-}
diff --git a/bin/varnishd/cache_dir.c b/bin/varnishd/cache_dir.c
deleted file mode 100644
index d4794f9..0000000
--- a/bin/varnishd/cache_dir.c
+++ /dev/null
@@ -1,121 +0,0 @@
-/*-
- * Copyright (c) 2006 Verdens Gang AS
- * Copyright (c) 2006-2011 Varnish Software AS
- * All rights reserved.
- *
- * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- *
- * Handle backend connections and backend request structures.
- *
- */
-
-#include "config.h"
-
-#include "cache.h"
-
-#include "cache_backend.h"
-#include "vtcp.h"
-
-/* Close a connection ------------------------------------------------*/
-
-void
-VDI_CloseFd(struct worker *wrk)
-{
-	struct backend *bp;
-
-	CHECK_OBJ_NOTNULL(wrk->vbc, VBC_MAGIC);
-	CHECK_OBJ_NOTNULL(wrk->vbc->backend, BACKEND_MAGIC);
-	assert(wrk->vbc->fd >= 0);
-
-	bp = wrk->vbc->backend;
-
-	WSL(wrk, SLT_BackendClose, wrk->vbc->vsl_id, "%s", bp->display_name);
-
-	/* Checkpoint log to flush all info related to this connection
-	   before the OS reuses the FD */
-	WSL_Flush(wrk, 0);
-
-	VTCP_close(&wrk->vbc->fd);
-	VBE_DropRefConn(bp);
-	wrk->vbc->backend = NULL;
-	VBE_ReleaseConn(wrk->vbc);
-	wrk->vbc = NULL;
-	wrk->do_close = 0;
-}
-
-/* Recycle a connection ----------------------------------------------*/
-
-void
-VDI_RecycleFd(struct worker *wrk)
-{
-	struct backend *bp;
-
-	CHECK_OBJ_NOTNULL(wrk->vbc, VBC_MAGIC);
-	CHECK_OBJ_NOTNULL(wrk->vbc->backend, BACKEND_MAGIC);
-	assert(wrk->vbc->fd >= 0);
-	AZ(wrk->do_close);
-
-	bp = wrk->vbc->backend;
-
-	WSL(wrk, SLT_BackendReuse, wrk->vbc->vsl_id, "%s", bp->display_name);
-	/*
-	 * Flush the shmlog, so that another session reusing this backend
-	 * will log chronologically later than our use of it.
-	 */
-	WSL_Flush(wrk, 0);
-	Lck_Lock(&bp->mtx);
-	VSC_C_main->backend_recycle++;
-	VTAILQ_INSERT_HEAD(&bp->connlist, wrk->vbc, list);
-	wrk->vbc = NULL;
-	VBE_DropRefLocked(bp);
-}
-
-/* Get a connection --------------------------------------------------*/
-
-struct vbc *
-VDI_GetFd(const struct director *d, struct sess *sp)
-{
-
-	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
-	if (d == NULL)
-		d = sp->director;
-	CHECK_OBJ_NOTNULL(d, DIRECTOR_MAGIC);
-	return (d->getfd(d, sp));
-}
-
-/* Check health ------------------------------------------------------
- *
- * The target is really an objhead pointer, but since it can not be
- * dereferenced during health-checks, we pass it as uintptr_t, which
- * hopefully will make people investigate, before mucking about with it.
- */
-
-int
-VDI_Healthy(const struct director *d, const struct sess *sp)
-{
-
-	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
-	CHECK_OBJ_NOTNULL(d, DIRECTOR_MAGIC);
-	return (d->healthy(d, sp));
-}
diff --git a/bin/varnishd/cache_dir_dns.c b/bin/varnishd/cache_dir_dns.c
deleted file mode 100644
index ae96dbb..0000000
--- a/bin/varnishd/cache_dir_dns.c
+++ /dev/null
@@ -1,469 +0,0 @@
-/*-
- * Copyright (c) 2009-2010 Varnish Software AS
- * All rights reserved.
- *
- * Author: Kristian Lyngstol <kristian at redpill-linpro.com>
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- *
- */
-
-#include "config.h"
-
-#include <netinet/in.h>
-
-#include <netdb.h>
-#include <stdio.h>
-#include <stdlib.h>
-
-#include "cache.h"
-
-#include "cache_backend.h"
-#include "vrt.h"
-
-/*--------------------------------------------------------------------*/
-
-/* FIXME: Should eventually be a configurable variable. */
-#define VDI_DNS_MAX_CACHE		1024
-#define VDI_DNS_GROUP_MAX_BACKENDS	1024
-
-/* DNS Cache entry
- */
-struct vdi_dns_hostgroup {
-	unsigned			magic;
-#define VDI_DNSDIR_MAGIC		0x1bacab21
-	char				*hostname;
-	struct director			*hosts[VDI_DNS_GROUP_MAX_BACKENDS];
-	unsigned			nhosts;
-	unsigned			next_host; /* Next to use...*/
-	double				ttl;
-	VTAILQ_ENTRY(vdi_dns_hostgroup)	list;
-};
-
-struct vdi_dns {
-	unsigned			magic;
-#define VDI_DNS_MAGIC			0x1337a178
-	struct director			dir;
-	struct director			**hosts;
-	unsigned			nhosts;
-	VTAILQ_HEAD(_cachelist,vdi_dns_hostgroup)	cachelist;
-	unsigned			ncachelist;
-	pthread_rwlock_t		rwlock;
-	const char			*suffix;
-	double			ttl;
-};
-
-/* Compare an IPv4 backend to a IPv4 addr/len */
-static int
-vdi_dns_comp_addrinfo4(const struct backend *bp,
-		       const struct sockaddr_storage *addr,
-		       const socklen_t len)
-{
-	uint32_t u, p;
-	const struct sockaddr_in *bps = (const void *)bp->ipv4;
-	const struct sockaddr_in *bpd = (const void *)addr;
-
-	if (bp->ipv4len != len || len <= 0)
-		return (0);
-
-	u = bpd->sin_addr.s_addr;
-	p = bps->sin_addr.s_addr;
-
-	return (u == p);
-}
-
-/* Compare an IPv6 backend to a IPv6 addr/len */
-static int
-vdi_dns_comp_addrinfo6(const struct backend *bp,
-		       const struct sockaddr_storage *addr,
-		       const socklen_t len)
-{
-	const uint8_t *u, *p;
-	const struct sockaddr_in6 *bps = (const void *)bp->ipv6;
-	const struct sockaddr_in6 *bpd = (const void *)addr;
-
-	if (bp->ipv6len != len || len <= 0)
-		return (0);
-
-	u = bpd->sin6_addr.s6_addr;
-	p = bps->sin6_addr.s6_addr;
-
-	return (!memcmp(u, p, 16));
-}
-
-/* Check if a backends socket is the same as addr */
-static int
-vdi_dns_comp_addrinfo(const struct director *dir,
-		      const struct sockaddr_storage *addr,
-		      const socklen_t len)
-{
-	struct backend *bp;
-
-	bp = vdi_get_backend_if_simple(dir);
-	AN(bp);
-	if (addr->ss_family == PF_INET && bp->ipv4) {
-		return (vdi_dns_comp_addrinfo4(bp, addr, len));
-	} else if (addr->ss_family == PF_INET6 && bp->ipv6) {
-		return (vdi_dns_comp_addrinfo6(bp, addr, len));
-	}
-	return (0);
-}
-
-/* Pick a host from an existing hostgroup.
- * Balance on round-robin if multiple backends are available and only pick
- * healthy ones.
- */
-static struct director *
-vdi_dns_pick_host(const struct sess *sp, struct vdi_dns_hostgroup *group) {
-	int initial, i, nhosts, current;
-	if (group->nhosts == 0)
-		return (NULL); // In case of error.
-	if (group->next_host >= group->nhosts)
-		group->next_host = 0;
-
-	/* Pick a healthy backend */
-	initial = group->next_host;
-	nhosts = group->nhosts;
-	for (i=0; i < nhosts; i++) {
-		if (i + initial >= nhosts)
-			current = i + initial - nhosts;
-		else
-			current = i + initial;
-		if (VDI_Healthy(group->hosts[current], sp)) {
-			group->next_host = current+1;
-			return (group->hosts[current]);
-		}
-	}
-
-	return (NULL);
-}
-
-/* Remove an item from the dns cache.
- * If *group is NULL, the head is popped.
- * Remember locking.
- */
-static void
-vdi_dns_pop_cache(struct vdi_dns *vs,
-		  struct vdi_dns_hostgroup *group)
-{
-	if (group == NULL)
-		group = VTAILQ_LAST( &vs->cachelist, _cachelist );
-	assert(group != NULL);
-	free(group->hostname);
-	VTAILQ_REMOVE(&vs->cachelist, group, list);
-	FREE_OBJ(group);
-	vs->ncachelist--;
-}
-
-/* Dummy in case someone feels like optimizing it? meh...
- */
-static inline int
-vdi_dns_groupmatch(const struct vdi_dns_hostgroup *group, const char *hostname)
-{
-	return (!strcmp(group->hostname, hostname));
-}
-
-/* Search the cache for 'hostname' and put a backend-pointer as necessary,
- * return true for cache hit. This could still be a NULL backend if we did
- * a lookup earlier and didn't find a host (ie: cache failed too)
- *
- * if rwlock is true, the first timed out object found (if any) is popped
- * and freed.
- */
-static int
-vdi_dns_cache_has(const struct sess *sp,
-		  struct vdi_dns *vs,
-		  const char *hostname,
-		  struct director **backend,
-		  int rwlock)
-{
-	struct director *ret;
-	struct vdi_dns_hostgroup *hostgr;
-	struct vdi_dns_hostgroup *hostgr2;
-	VTAILQ_FOREACH_SAFE(hostgr, &vs->cachelist, list, hostgr2) {
-		CHECK_OBJ_NOTNULL(hostgr, VDI_DNSDIR_MAGIC);
-		if (hostgr->ttl <= sp->t_req) {
-			if (rwlock)
-				vdi_dns_pop_cache(vs, hostgr);
-			return (0);
-		}
-		if (vdi_dns_groupmatch(hostgr, hostname)) {
-			ret = (vdi_dns_pick_host(sp, hostgr));
-			*backend = ret;
-			if (*backend != NULL)
-				CHECK_OBJ_NOTNULL(*backend, DIRECTOR_MAGIC);
-			return (1);
-		}
-	}
-	return (0);
-}
-
-/* Add a newly cached item to the dns cache list.
- * (Sorry for the list_add/_add confusion...)
- */
-static void
-vdi_dns_cache_list_add(const struct sess *sp,
-		       struct vdi_dns *vs,
-		       struct vdi_dns_hostgroup *new)
-{
-	if (vs->ncachelist >= VDI_DNS_MAX_CACHE) {
-		VSC_C_main->dir_dns_cache_full++;
-		vdi_dns_pop_cache(vs, NULL);
-	}
-	CHECK_OBJ_NOTNULL(new, VDI_DNSDIR_MAGIC);
-	assert(new->hostname != 0);
-	new->ttl = sp->t_req + vs->ttl;
-	VTAILQ_INSERT_HEAD(&vs->cachelist, new, list);
-	vs->ncachelist++;
-}
-
-/* Add an item to the dns cache.
- * XXX: Might want to factor the getaddrinfo() out of the lock and do the
- * cache_has() afterwards to do multiple dns lookups in parallel...
- */
-static int
-vdi_dns_cache_add(const struct sess *sp,
-		  struct vdi_dns *vs,
-		  const char *hostname,
-		  struct director **backend)
-{
-	int error, i, host = 0;
-	struct addrinfo *res0, *res, hint;
-	struct vdi_dns_hostgroup *new;
-
-	/* Due to possible race while upgrading the lock, we have to
-	 * recheck if the result is already looked up. The overhead for
-	 * this is insignificant unless dns isn't cached properly (all
-	 * unique names or something equally troublesome).
-	 */
-
-	if (vdi_dns_cache_has(sp, vs, hostname, backend, 1))
-		return (1);
-
-	memset(&hint, 0, sizeof hint);
-	hint.ai_family = PF_UNSPEC;
-	hint.ai_socktype = SOCK_STREAM;
-
-	ALLOC_OBJ(new, VDI_DNSDIR_MAGIC);
-	XXXAN(new);
-
-	REPLACE(new->hostname, hostname);
-
-	error = getaddrinfo(hostname, "80", &hint, &res0);
-	VSC_C_main->dir_dns_lookups++;
-	if (error) {
-		vdi_dns_cache_list_add(sp, vs, new);
-		VSC_C_main->dir_dns_failed++;
-		return (0);
-	}
-
-	for (res = res0; res; res = res->ai_next) {
-		if (res->ai_family != PF_INET && res->ai_family != PF_INET6)
-			continue;
-
-		for (i = 0; i < vs->nhosts; i++) {
-			struct sockaddr_storage ss_hack;
-			memcpy(&ss_hack, res->ai_addr, res->ai_addrlen);
-			if (vdi_dns_comp_addrinfo(vs->hosts[i],
-			    &ss_hack, res->ai_addrlen)) {
-				new->hosts[host] = vs->hosts[i];
-				CHECK_OBJ_NOTNULL(new->hosts[host],
-				    DIRECTOR_MAGIC);
-				host++;
-			}
-		}
-	}
-	freeaddrinfo(res0);
-
-	new->nhosts = host;
-	vdi_dns_cache_list_add(sp, vs, new);
-	*backend = vdi_dns_pick_host(sp, new);
-	return (1);
-}
-
-/* Walk through the cached lookups looking for the relevant host, add one
- * if it isn't already cached.
- *
- * Returns a backend or NULL.
- */
-static struct director *
-vdi_dns_walk_cache(const struct sess *sp,
-		   struct vdi_dns *vs,
-		   const char *hostname)
-{
-	struct director *backend = NULL;
-	int ret;
-
-	AZ(pthread_rwlock_rdlock(&vs->rwlock));
-	ret = vdi_dns_cache_has(sp, vs, hostname, &backend, 0);
-	AZ(pthread_rwlock_unlock(&vs->rwlock));
-	if (!ret) {
-		/*
-		 * XXX: Isn't there a race here where another thread
-		 * XXX: could grab the lock and add it before we do ?
-		 * XXX: Should 'ret' be checked for that ?
-		 */
-		AZ(pthread_rwlock_wrlock(&vs->rwlock));
-		ret = vdi_dns_cache_add(sp, vs, hostname, &backend);
-		AZ(pthread_rwlock_unlock(&vs->rwlock));
-	} else
-		VSC_C_main->dir_dns_hit++;
-
-	/* Bank backend == cached a failure, so to speak */
-	if (backend != NULL)
-		CHECK_OBJ_NOTNULL(backend, DIRECTOR_MAGIC);
-	return (backend);
-}
-
-/* Parses the Host:-header and heads out to find a backend.
- */
-static struct director *
-vdi_dns_find_backend(const struct sess *sp, struct vdi_dns *vs)
-{
-	struct director *ret;
-	struct http *hp;
-	char *p, *q;
-	char hostname[NI_MAXHOST];
-
-	/* bereq is only present after recv et. al, otherwise use req (ie:
-	 * use req for health checks in vcl_recv and such).
-	 */
-	if (sp->wrk->bereq)
-		hp = sp->wrk->bereq;
-	else
-		hp = sp->http;
-
-
-	CHECK_OBJ_NOTNULL(hp, HTTP_MAGIC);
-	if (http_GetHdr(hp, H_Host, &p) == 0)
-		return (NULL);
-
-	q = strchr(p, ':');
-	if (q == NULL)
-		q = strchr(p, '\0');
-	AN(q);
-
-	bprintf(hostname, "%.*s%s", (int)(q - p), p,
-	    vs->suffix ? vs->suffix : "");
-
-	ret = vdi_dns_walk_cache(sp, vs, hostname);
-	return (ret);
-}
-
-static struct vbc *
-vdi_dns_getfd(const struct director *director, struct sess *sp)
-{
-	struct vdi_dns *vs;
-	struct director *dir;
-	struct vbc *vbe;
-
-	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
-	CHECK_OBJ_NOTNULL(director, DIRECTOR_MAGIC);
-	CAST_OBJ_NOTNULL(vs, director->priv, VDI_DNS_MAGIC);
-
-	dir = vdi_dns_find_backend(sp, vs);
-	if (!dir || !VDI_Healthy(dir, sp))
-		return (NULL);
-
-	vbe = VDI_GetFd(dir, sp);
-	return (vbe);
-}
-
-static unsigned
-vdi_dns_healthy(const struct director *dir, const struct sess *sp)
-{
-	/* XXX: Fooling -Werror for a bit until it's actually implemented.
-	 */
-	(void)dir;
-	(void)sp;
-	return (1);
-
-	/*
-	struct vdi_dns *vs;
-	struct director *dir;
-	int i;
-
-	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
-	CHECK_OBJ_NOTNULL(sp->director, DIRECTOR_MAGIC);
-	CAST_OBJ_NOTNULL(vs, sp->director->priv, VDI_DNS_MAGIC);
-
-	dir = vdi_dns_find_backend(sp, vs);
-
-	if (dir)
-		return (1);
-	return (0);
-	*/
-}
-
-static void
-vdi_dns_fini(const struct director *d)
-{
-	struct vdi_dns *vs;
-
-	CHECK_OBJ_NOTNULL(d, DIRECTOR_MAGIC);
-	CAST_OBJ_NOTNULL(vs, d->priv, VDI_DNS_MAGIC);
-
-	free(vs->hosts);
-	free(vs->dir.vcl_name);
-	vs->dir.magic = 0;
-	/* FIXME: Free the cache */
-	AZ(pthread_rwlock_destroy(&vs->rwlock));
-	FREE_OBJ(vs);
-}
-
-void
-VRT_init_dir_dns(struct cli *cli, struct director **bp, int idx,
-    const void *priv)
-{
-	const struct vrt_dir_dns *t;
-	struct vdi_dns *vs;
-	const struct vrt_dir_dns_entry *te;
-	int i;
-
-	ASSERT_CLI();
-	(void)cli;
-	t = priv;
-	ALLOC_OBJ(vs, VDI_DNS_MAGIC);
-	XXXAN(vs);
-	vs->hosts = calloc(sizeof(struct director *), t->nmember);
-	XXXAN(vs->hosts);
-
-	vs->dir.magic = DIRECTOR_MAGIC;
-	vs->dir.priv = vs;
-	vs->dir.name = "dns";
-	REPLACE(vs->dir.vcl_name, t->name);
-	vs->dir.getfd = vdi_dns_getfd;
-	vs->dir.fini = vdi_dns_fini;
-	vs->dir.healthy = vdi_dns_healthy;
-
-	vs->suffix = t->suffix;
-	vs->ttl = t->ttl;
-
-	te = t->members;
-	for (i = 0; i < t->nmember; i++, te++)
-		vs->hosts[i] = bp[te->host];
-	vs->nhosts = t->nmember;
-	vs->ttl = t->ttl;
-	VTAILQ_INIT(&vs->cachelist);
-	AZ(pthread_rwlock_init(&vs->rwlock, NULL));
-	bp[idx] = &vs->dir;
-}
diff --git a/bin/varnishd/cache_dir_random.c b/bin/varnishd/cache_dir_random.c
deleted file mode 100644
index d6570ed..0000000
--- a/bin/varnishd/cache_dir_random.c
+++ /dev/null
@@ -1,285 +0,0 @@
-/*-
- * Copyright (c) 2006 Verdens Gang AS
- * Copyright (c) 2006-2011 Varnish Software AS
- * All rights reserved.
- *
- * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- *
- * This code is shared between the random, client and hash directors, because
- * they share the same properties and most of the same selection logic.
- *
- * The random director picks a backend on random.
- *
- * The hash director picks based on the hash from vcl_hash{}
- *
- * The client director picks based on client identity or IP-address
- *
- * In all cases, the choice is by weight of the healthy subset of
- * configured backends.
- *
- * Failures to get a connection are retried, here all three policies
- * fall back to a deterministically random choice, by weight in the
- * healthy subset.
- *
- */
-
-#include "config.h"
-
-#include <math.h>
-#include <stdlib.h>
-
-#include "cache.h"
-
-#include "cache_backend.h"
-#include "vend.h"
-#include "vrt.h"
-#include "vsha256.h"
-
-/*--------------------------------------------------------------------*/
-
-struct vdi_random_host {
-	struct director		*backend;
-	double			weight;
-};
-
-enum crit_e {c_random, c_hash, c_client};
-
-struct vdi_random {
-	unsigned		magic;
-#define VDI_RANDOM_MAGIC	0x3771ae23
-	struct director		dir;
-
-	enum crit_e		criteria;
-	unsigned		retries;
-	double			tot_weight;
-	struct vdi_random_host	*hosts;
-	unsigned		nhosts;
-};
-
-/*
- * Applies sha256 using the given context and input/length, and returns
- * a double in the range [0...1[ based on the hash.
- */
-static double
-vdi_random_sha(const char *input, ssize_t len)
-{
-	struct SHA256Context ctx;
-	uint8_t sign[SHA256_LEN];
-
-	AN(input);
-	SHA256_Init(&ctx);
-	SHA256_Update(&ctx, input, len);
-	SHA256_Final(sign, &ctx);
-	return (scalbn(vle32dec(sign), -32));
-}
-
-/*
- * Sets up the initial seed for picking a backend according to policy.
- */
-static double
-vdi_random_init_seed(const struct vdi_random *vs, const struct sess *sp)
-{
-	const char *p;
-	double retval;
-
-	switch (vs->criteria) {
-	case c_client:
-		if (sp->client_identity != NULL)
-			p = sp->client_identity;
-		else
-			p = sp->addr;
-		retval = vdi_random_sha(p, strlen(p));
-		break;
-	case c_hash:
-		AN(sp->digest);
-		retval = scalbn(vle32dec(sp->digest), -32);
-		break;
-	case c_random:
-	default:
-		retval = scalbn(random(), -31);
-		break;
-	}
-	return (retval);
-}
-
-/*
- * Find the healthy backend corresponding to the weight r [0...1[
- */
-static struct vbc *
-vdi_random_pick_one(struct sess *sp, const struct vdi_random *vs, double r)
-{
-	double w[vs->nhosts];
-	int i;
-	double s1;
-
-	assert(r >= 0.0 && r < 1.0);
-
-	memset(w, 0, sizeof w);
-	/* Sum up the weights of healty backends */
-	s1 = 0.0;
-	for (i = 0; i < vs->nhosts; i++) {
-		if (VDI_Healthy(vs->hosts[i].backend, sp))
-			w[i] = vs->hosts[i].weight;
-		s1 += w[i];
-	}
-
-	if (s1 == 0.0)
-		return (NULL);
-
-	r *= s1;
-	s1 = 0.0;
-	for (i = 0; i < vs->nhosts; i++)  {
-		s1 += w[i];
-		if (r < s1)
-			return(VDI_GetFd(vs->hosts[i].backend, sp));
-	}
-	return (NULL);
-}
-
-/*
- * Try the specified number of times to get a backend.
- * First one according to policy, after that, deterministically
- * random by rehashing the key.
- */
-static struct vbc *
-vdi_random_getfd(const struct director *d, struct sess *sp)
-{
-	int k;
-	struct vdi_random *vs;
-	double r;
-	struct vbc *vbe;
-
-	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
-	CHECK_OBJ_NOTNULL(d, DIRECTOR_MAGIC);
-	CAST_OBJ_NOTNULL(vs, d->priv, VDI_RANDOM_MAGIC);
-
-	r = vdi_random_init_seed(vs, sp);
-
-	for (k = 0; k < vs->retries; k++) {
-		vbe = vdi_random_pick_one(sp, vs, r);
-		if (vbe != NULL)
-			return (vbe);
-		r = vdi_random_sha((void *)&r, sizeof(r));
-	}
-	return (NULL);
-}
-
-/*
- * Healthy if just a single backend is...
- */
-static unsigned
-vdi_random_healthy(const struct director *d, const struct sess *sp)
-{
-	struct vdi_random *vs;
-	int i;
-
-	CHECK_OBJ_NOTNULL(d, DIRECTOR_MAGIC);
-	CAST_OBJ_NOTNULL(vs, d->priv, VDI_RANDOM_MAGIC);
-
-	for (i = 0; i < vs->nhosts; i++) {
-		if (VDI_Healthy(vs->hosts[i].backend, sp))
-			return (1);
-	}
-	return (0);
-}
-
-static void
-vdi_random_fini(const struct director *d)
-{
-	struct vdi_random *vs;
-
-	CHECK_OBJ_NOTNULL(d, DIRECTOR_MAGIC);
-	CAST_OBJ_NOTNULL(vs, d->priv, VDI_RANDOM_MAGIC);
-
-	free(vs->hosts);
-	free(vs->dir.vcl_name);
-	vs->dir.magic = 0;
-	FREE_OBJ(vs);
-}
-
-static void
-vrt_init(struct cli *cli, struct director **bp, int idx,
-    const void *priv, enum crit_e criteria)
-{
-	const struct vrt_dir_random *t;
-	struct vdi_random *vs;
-	const struct vrt_dir_random_entry *te;
-	struct vdi_random_host *vh;
-	int i;
-
-	ASSERT_CLI();
-	(void)cli;
-	t = priv;
-
-	ALLOC_OBJ(vs, VDI_RANDOM_MAGIC);
-	XXXAN(vs);
-	vs->hosts = calloc(sizeof *vh, t->nmember);
-	XXXAN(vs->hosts);
-
-	vs->dir.magic = DIRECTOR_MAGIC;
-	vs->dir.priv = vs;
-	vs->dir.name = "random";
-	REPLACE(vs->dir.vcl_name, t->name);
-	vs->dir.getfd = vdi_random_getfd;
-	vs->dir.fini = vdi_random_fini;
-	vs->dir.healthy = vdi_random_healthy;
-
-	vs->criteria = criteria;
-	vs->retries = t->retries;
-	if (vs->retries == 0)
-		vs->retries = t->nmember;
-	vh = vs->hosts;
-	te = t->members;
-	vs->tot_weight = 0.;
-	for (i = 0; i < t->nmember; i++, vh++, te++) {
-		assert(te->weight > 0.0);
-		vh->weight = te->weight;
-		vs->tot_weight += vh->weight;
-		vh->backend = bp[te->host];
-		AN(vh->backend);
-	}
-	vs->nhosts = t->nmember;
-	bp[idx] = &vs->dir;
-}
-
-void
-VRT_init_dir_random(struct cli *cli, struct director **bp, int idx,
-    const void *priv)
-{
-	vrt_init(cli, bp, idx, priv, c_random);
-}
-
-void
-VRT_init_dir_hash(struct cli *cli, struct director **bp, int idx,
-    const void *priv)
-{
-	vrt_init(cli, bp, idx, priv, c_hash);
-}
-
-void
-VRT_init_dir_client(struct cli *cli, struct director **bp, int idx,
-    const void *priv)
-{
-	vrt_init(cli, bp, idx, priv, c_client);
-}
diff --git a/bin/varnishd/cache_dir_round_robin.c b/bin/varnishd/cache_dir_round_robin.c
deleted file mode 100644
index 7d75473..0000000
--- a/bin/varnishd/cache_dir_round_robin.c
+++ /dev/null
@@ -1,176 +0,0 @@
-/*-
- * Copyright (c) 2008-2011 Varnish Software AS
- * All rights reserved.
- *
- * Author: Petter Knudsen <petter at linpro.no>
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- *
- */
-
-#include "config.h"
-
-#include <stdlib.h>
-
-#include "cache.h"
-
-#include "cache_backend.h"
-#include "vrt.h"
-
-/*--------------------------------------------------------------------*/
-
-struct vdi_round_robin_host {
-	struct director			*backend;
-};
-
-enum mode_e { m_round_robin, m_fallback };
-
-struct vdi_round_robin {
-	unsigned			magic;
-#define VDI_ROUND_ROBIN_MAGIC		0x2114a178
-	struct director			dir;
-	enum mode_e			mode;
-	struct vdi_round_robin_host	*hosts;
-	unsigned			nhosts;
-	unsigned			next_host;
-};
-
-static struct vbc *
-vdi_round_robin_getfd(const struct director *d, struct sess *sp)
-{
-	int i;
-	struct vdi_round_robin *vs;
-	struct director *backend;
-	struct vbc *vbe;
-
-	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
-	CHECK_OBJ_NOTNULL(d, DIRECTOR_MAGIC);
-	CAST_OBJ_NOTNULL(vs, d->priv, VDI_ROUND_ROBIN_MAGIC);
-
-	/*
-	 * In fallback mode we ignore the next_host and always grab the
-	 * first healthy backend we can find.
-	 */
-	for (i = 0; i < vs->nhosts; i++) {
-		if (vs->mode == m_round_robin) {
-			backend = vs->hosts[vs->next_host].backend;
-			vs->next_host = (vs->next_host + 1) % vs->nhosts;
-		} else /* m_fallback */ {
-			backend = vs->hosts[i].backend;
-		}
-		if (!VDI_Healthy(backend, sp))
-			continue;
-		vbe = VDI_GetFd(backend, sp);
-		if (vbe != NULL)
-			return (vbe);
-	}
-
-	return (NULL);
-}
-
-static unsigned
-vdi_round_robin_healthy(const struct director *d, const struct sess *sp)
-{
-	struct vdi_round_robin *vs;
-	struct director *backend;
-	int i;
-
-	CHECK_OBJ_NOTNULL(d, DIRECTOR_MAGIC);
-	CAST_OBJ_NOTNULL(vs, d->priv, VDI_ROUND_ROBIN_MAGIC);
-
-	for (i = 0; i < vs->nhosts; i++) {
-		backend = vs->hosts[i].backend;
-		if (VDI_Healthy(backend, sp))
-			return (1);
-	}
-	return (0);
-}
-
-static void
-vdi_round_robin_fini(const struct director *d)
-{
-	struct vdi_round_robin *vs;
-
-	CHECK_OBJ_NOTNULL(d, DIRECTOR_MAGIC);
-	CAST_OBJ_NOTNULL(vs, d->priv, VDI_ROUND_ROBIN_MAGIC);
-
-	free(vs->hosts);
-	free(vs->dir.vcl_name);
-	vs->dir.magic = 0;
-	vs->next_host = 0;
-	FREE_OBJ(vs);
-}
-
-static void
-vrt_init_dir(struct cli *cli, struct director **bp, int idx,
-    const void *priv, enum mode_e mode)
-{
-	const struct vrt_dir_round_robin *t;
-	struct vdi_round_robin *vs;
-	const struct vrt_dir_round_robin_entry *te;
-	struct vdi_round_robin_host *vh;
-	int i;
-
-	ASSERT_CLI();
-	(void)cli;
-	t = priv;
-
-	ALLOC_OBJ(vs, VDI_ROUND_ROBIN_MAGIC);
-	XXXAN(vs);
-	vs->hosts = calloc(sizeof *vh, t->nmember);
-	XXXAN(vs->hosts);
-
-	vs->dir.magic = DIRECTOR_MAGIC;
-	vs->dir.priv = vs;
-	vs->dir.name = "round_robin";
-	REPLACE(vs->dir.vcl_name, t->name);
-	vs->dir.getfd = vdi_round_robin_getfd;
-	vs->dir.fini = vdi_round_robin_fini;
-	vs->dir.healthy = vdi_round_robin_healthy;
-
-	vs->mode = mode;
-	vh = vs->hosts;
-	te = t->members;
-	for (i = 0; i < t->nmember; i++, vh++, te++) {
-		vh->backend = bp[te->host];
-		AN (vh->backend);
-	}
-	vs->nhosts = t->nmember;
-	vs->next_host = 0;
-
-	bp[idx] = &vs->dir;
-}
-
-void
-VRT_init_dir_round_robin(struct cli *cli, struct director **bp, int idx,
-    const void *priv)
-{
-	vrt_init_dir(cli, bp, idx, priv, m_round_robin);
-}
-
-void
-VRT_init_dir_fallback(struct cli *cli, struct director **bp, int idx,
-    const void *priv)
-{
-	vrt_init_dir(cli, bp, idx, priv, m_fallback);
-}
-
diff --git a/bin/varnishd/cache_esi.h b/bin/varnishd/cache_esi.h
deleted file mode 100644
index ff84d41..0000000
--- a/bin/varnishd/cache_esi.h
+++ /dev/null
@@ -1,48 +0,0 @@
-/*-
- * Copyright (c) 2011 Varnish Software AS
- * All rights reserved.
- *
- * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- *
- */
-
-#define	VEC_GZ	(0x21)
-#define	VEC_V1	(0x40 + 1)
-#define	VEC_V2	(0x40 + 2)
-#define	VEC_V8	(0x40 + 8)
-#define	VEC_C1	(0x50 + 1)
-#define	VEC_C2	(0x50 + 2)
-#define	VEC_C8	(0x50 + 8)
-#define	VEC_S1	(0x60 + 1)
-#define	VEC_S2	(0x60 + 2)
-#define	VEC_S8	(0x60 + 8)
-#define	VEC_INCL	'I'
-
-typedef ssize_t vep_callback_t(struct worker *w, ssize_t l, enum vgz_flag flg);
-
-void VEP_Init(struct worker *w, vep_callback_t *cb);
-void VEP_Parse(const struct worker *w, const char *p, size_t l);
-struct vsb *VEP_Finish(struct worker *w);
-
-
diff --git a/bin/varnishd/cache_esi_deliver.c b/bin/varnishd/cache_esi_deliver.c
deleted file mode 100644
index 4051027..0000000
--- a/bin/varnishd/cache_esi_deliver.c
+++ /dev/null
@@ -1,570 +0,0 @@
-/*-
- * Copyright (c) 2011 Varnish Software AS
- * All rights reserved.
- *
- * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- *
- * VED - Varnish Esi Delivery
- */
-
-#include "config.h"
-
-#include <stdio.h>
-#include <stdlib.h>
-
-#include "cache.h"
-
-#include "cache_esi.h"
-#include "vend.h"
-#include "vgz.h"
-
-/*--------------------------------------------------------------------*/
-
-static void
-ved_include(struct sess *sp, const char *src, const char *host)
-{
-	struct object *obj;
-	struct worker *w;
-	char *sp_ws_wm;
-	char *wrk_ws_wm;
-	unsigned sxid, res_mode;
-
-	w = sp->wrk;
-
-	if (sp->esi_level >= cache_param->max_esi_depth)
-		return;
-	sp->esi_level++;
-
-	(void)WRW_FlushRelease(w);
-
-	obj = sp->obj;
-	sp->obj = NULL;
-	res_mode = sp->wrk->res_mode;
-
-	/* Reset request to status before we started messing with it */
-	HTTP_Copy(sp->http, sp->http0);
-
-	/* Take a workspace snapshot */
-	sp_ws_wm = WS_Snapshot(sp->ws);
-	wrk_ws_wm = WS_Snapshot(w->ws);
-
-	http_SetH(sp->http, HTTP_HDR_URL, src);
-	if (host != NULL && *host != '\0')  {
-		http_Unset(sp->http, H_Host);
-		http_Unset(sp->http, H_If_Modified_Since);
-		http_SetHeader(w, sp->vsl_id, sp->http, host);
-	}
-	/*
-	 * XXX: We should decide if we should cache the director
-	 * XXX: or not (for session/backend coupling).  Until then
-	 * XXX: make sure we don't trip up the check in vcl_recv.
-	 */
-	sp->director = NULL;
-	sp->step = STP_RECV;
-	http_ForceGet(sp->http);
-
-	/* Don't do conditionals */
-	sp->http->conds = 0;
-	http_Unset(sp->http, H_If_Modified_Since);
-
-	/* Client content already taken care of */
-	http_Unset(sp->http, H_Content_Length);
-
-	sp->wrk->do_esi = 0;
-	sp->wrk->is_gzip = 0;
-	sp->wrk->is_gunzip = 0;
-	sp->wrk->do_gzip = 0;
-	sp->wrk->do_gunzip = 0;
-	sp->wrk->do_stream = 0;
-
-	sxid = sp->xid;
-	while (1) {
-		sp->wrk = w;
-		CNT_Session(sp);
-		if (sp->step == STP_DONE)
-			break;
-		AZ(sp->wrk);
-		WSL_Flush(w, 0);
-		DSL(0x20, SLT_Debug, sp->vsl_id, "loop waiting for ESI");
-		(void)usleep(10000);
-	}
-	sp->xid = sxid;
-	AN(sp->wrk);
-	assert(sp->step == STP_DONE);
-	sp->esi_level--;
-	sp->obj = obj;
-	sp->wrk->res_mode = res_mode;
-
-	/* Reset the workspace */
-	WS_Reset(sp->ws, sp_ws_wm);
-	WS_Reset(w->ws, wrk_ws_wm);
-
-	WRW_Reserve(sp->wrk, &sp->fd);
-	if (sp->wrk->res_mode & RES_CHUNKED)
-		WRW_Chunked(sp->wrk);
-}
-
-/*--------------------------------------------------------------------*/
-
-
-//#define Debug(fmt, ...) printf(fmt, __VA_ARGS__)
-#define Debug(fmt, ...) /**/
-
-static ssize_t
-ved_decode_len(uint8_t **pp)
-{
-	uint8_t *p;
-	ssize_t l;
-
-	p = *pp;
-	switch (*p & 15) {
-	case 1:
-		l = p[1];
-		p += 2;
-		break;
-	case 2:
-		l = vbe16dec(p + 1);
-		p += 3;
-		break;
-	case 8:
-		l = vbe64dec(p + 1);
-		p += 9;
-		break;
-	default:
-		printf("Illegal Length %d %d\n", *p, (*p & 15));
-		INCOMPL();
-	}
-	*pp = p;
-	assert(l > 0);
-	return (l);
-}
-
-/*---------------------------------------------------------------------
- * If a gzip'ed ESI object includes a ungzip'ed object, we need to make
- * it looked like a gzip'ed data stream.  The official way to do so would
- * be to fire up libvgz and gzip it, but we don't, we fake it.
- *
- * First, we cannot know if it is ungzip'ed on purpose, the admin may
- * know something we don't.
- *
- * What do you mean "BS ?"
- *
- * All right then...
- *
- * The matter of the fact is that we simply will not fire up a gzip in
- * the output path because it costs too much memory and CPU, so we simply
- * wrap the data in very convenient "gzip copy-blocks" and send it down
- * the stream with a bit more overhead.
- */
-
-static void
-ved_pretend_gzip(const struct sess *sp, const uint8_t *p, ssize_t l)
-{
-	uint8_t buf1[5], buf2[5];
-	uint16_t lx;
-
-	lx = 65535;
-	buf1[0] = 0;
-	vle16enc(buf1 + 1, lx);
-	vle16enc(buf1 + 3, ~lx);
-
-	while (l > 0) {
-		if (l >= 65535) {
-			lx = 65535;
-			(void)WRW_Write(sp->wrk, buf1, sizeof buf1);
-		} else {
-			lx = (uint16_t)l;
-			buf2[0] = 0;
-			vle16enc(buf2 + 1, lx);
-			vle16enc(buf2 + 3, ~lx);
-			(void)WRW_Write(sp->wrk, buf2, sizeof buf2);
-		}
-		(void)WRW_Write(sp->wrk, p, lx);
-		sp->wrk->crc = crc32(sp->wrk->crc, p, lx);
-		sp->wrk->l_crc += lx;
-		l -= lx;
-		p += lx;
-	}
-	/* buf2 is local, have to flush */
-	(void)WRW_Flush(sp->wrk);
-}
-
-/*---------------------------------------------------------------------
- */
-
-static const uint8_t gzip_hdr[] = {
-	0x1f, 0x8b, 0x08,
-	0x00, 0x00, 0x00, 0x00,
-	0x00,
-	0x02, 0x03
-};
-
-void
-ESI_Deliver(struct sess *sp)
-{
-	struct storage *st;
-	uint8_t *p, *e, *q, *r;
-	unsigned off;
-	ssize_t l, l2, l_icrc = 0;
-	uint32_t icrc = 0;
-	uint8_t tailbuf[8 + 5];
-	int isgzip;
-	struct vgz *vgz = NULL;
-	char obuf[cache_param->gzip_stack_buffer];
-	ssize_t obufl = 0;
-	size_t dl;
-	const void *dp;
-	int i;
-
-	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
-	st = sp->obj->esidata;
-	AN(st);
-	assert(sizeof obuf >= 1024);
-
-	obuf[0] = 0;	/* For flexelint */
-
-	p = st->ptr;
-	e = st->ptr + st->len;
-
-	if (*p == VEC_GZ) {
-		isgzip = 1;
-		p++;
-	} else {
-		isgzip = 0;
-	}
-
-	if (sp->esi_level == 0) {
-		/*
-		 * Only the top level document gets to decide this.
-		 */
-		sp->wrk->gzip_resp = 0;
-		if (isgzip && !(sp->wrk->res_mode & RES_GUNZIP)) {
-			assert(sizeof gzip_hdr == 10);
-			/* Send out the gzip header */
-			(void)WRW_Write(sp->wrk, gzip_hdr, 10);
-			sp->wrk->l_crc = 0;
-			sp->wrk->gzip_resp = 1;
-			sp->wrk->crc = crc32(0L, Z_NULL, 0);
-		}
-	}
-
-	if (isgzip && !sp->wrk->gzip_resp) {
-		vgz = VGZ_NewUngzip(sp->wrk, "U D E");
-
-		/* Feed a gzip header to gunzip to make it happy */
-		VGZ_Ibuf(vgz, gzip_hdr, sizeof gzip_hdr);
-		VGZ_Obuf(vgz, obuf, sizeof obuf);
-		i = VGZ_Gunzip(vgz, &dp, &dl);
-		assert(i == VGZ_OK);
-		assert(VGZ_IbufEmpty(vgz));
-		assert(dl == 0);
-
-		obufl = 0;
-	}
-
-	st = VTAILQ_FIRST(&sp->obj->store);
-	off = 0;
-
-	while (p < e) {
-		switch (*p) {
-		case VEC_V1:
-		case VEC_V2:
-		case VEC_V8:
-			l = ved_decode_len(&p);
-			if (isgzip) {
-				assert(*p == VEC_C1 || *p == VEC_C2 ||
-				    *p == VEC_C8);
-				l_icrc = ved_decode_len(&p);
-				icrc = vbe32dec(p);
-				p += 4;
-				if (sp->wrk->gzip_resp) {
-					sp->wrk->crc = crc32_combine(
-					    sp->wrk->crc, icrc, l_icrc);
-					sp->wrk->l_crc += l_icrc;
-				}
-			}
-			/*
-			 * There is no guarantee that the 'l' bytes are all
-			 * in the same storage segment, so loop over storage
-			 * until we have processed them all.
-			 */
-			while (l > 0) {
-				l2 = l;
-				if (l2 > st->len - off)
-					l2 = st->len - off;
-				l -= l2;
-
-				if (sp->wrk->gzip_resp && isgzip) {
-					/*
-					 * We have a gzip'ed VEC and delivers
-					 * a gzip'ed ESI response.
-					 */
-					(void)WRW_Write(sp->wrk, st->ptr + off, l2);
-				} else if (sp->wrk->gzip_resp) {
-					/*
-					 * A gzip'ed ESI response, but the VEC
-					 * was not gzip'ed.
-					 */
-					ved_pretend_gzip(sp, st->ptr + off, l2);
-				} else if (isgzip) {
-					/*
-					 * A gzip'ed VEC, but ungzip'ed ESI
-					 * response
-					 */
-					AN(vgz);
-					i = VGZ_WrwGunzip(sp->wrk, vgz,
-						st->ptr + off, l2,
-						obuf, sizeof obuf, &obufl);
-					if (WRW_Error(sp->wrk)) {
-						SES_Close(sp, "remote closed");
-						p = e;
-						break;
-					}
-					assert (i == VGZ_OK || i == VGZ_END);
-				} else {
-					/*
-					 * Ungzip'ed VEC, ungzip'ed ESI response
-					 */
-					(void)WRW_Write(sp->wrk, st->ptr + off, l2);
-				}
-				off += l2;
-				if (off == st->len) {
-					st = VTAILQ_NEXT(st, list);
-					off = 0;
-				}
-			}
-			break;
-		case VEC_S1:
-		case VEC_S2:
-		case VEC_S8:
-			l = ved_decode_len(&p);
-			Debug("SKIP1(%d)\n", (int)l);
-			/*
-			 * There is no guarantee that the 'l' bytes are all
-			 * in the same storage segment, so loop over storage
-			 * until we have processed them all.
-			 */
-			while (l > 0) {
-				l2 = l;
-				if (l2 > st->len - off)
-					l2 = st->len - off;
-				l -= l2;
-				off += l2;
-				if (off == st->len) {
-					st = VTAILQ_NEXT(st, list);
-					off = 0;
-				}
-			}
-			break;
-		case VEC_INCL:
-			p++;
-			q = (void*)strchr((const char*)p, '\0');
-			AN(q);
-			q++;
-			r = (void*)strchr((const char*)q, '\0');
-			AN(r);
-			if (obufl > 0) {
-				(void)WRW_Write(sp->wrk, obuf, obufl);
-				obufl = 0;
-			}
-			if (WRW_Flush(sp->wrk)) {
-				SES_Close(sp, "remote closed");
-				p = e;
-				break;
-			}
-			Debug("INCL [%s][%s] BEGIN\n", q, p);
-			ved_include(sp, (const char*)q, (const char*)p);
-			Debug("INCL [%s][%s] END\n", q, p);
-			p = r + 1;
-			break;
-		default:
-			printf("XXXX 0x%02x [%s]\n", *p, p);
-			INCOMPL();
-		}
-	}
-	if (vgz != NULL) {
-		if (obufl > 0)
-			(void)WRW_Write(sp->wrk, obuf, obufl);
-		(void)VGZ_Destroy(&vgz, sp->vsl_id);
-	}
-	if (sp->wrk->gzip_resp && sp->esi_level == 0) {
-		/* Emit a gzip literal block with finish bit set */
-		tailbuf[0] = 0x01;
-		tailbuf[1] = 0x00;
-		tailbuf[2] = 0x00;
-		tailbuf[3] = 0xff;
-		tailbuf[4] = 0xff;
-
-		/* Emit CRC32 */
-		vle32enc(tailbuf + 5, sp->wrk->crc);
-
-		/* MOD(2^32) length */
-		vle32enc(tailbuf + 9, sp->wrk->l_crc);
-
-		(void)WRW_Write(sp->wrk, tailbuf, 13);
-	}
-	(void)WRW_Flush(sp->wrk);
-}
-
-/*---------------------------------------------------------------------
- * Include an object in a gzip'ed ESI object delivery
- */
-
-static uint8_t
-ved_deliver_byterange(const struct sess *sp, ssize_t low, ssize_t high)
-{
-	struct storage *st;
-	ssize_t l, lx;
-	u_char *p;
-
-//printf("BR %jd %jd\n", low, high);
-	lx = 0;
-	VTAILQ_FOREACH(st, &sp->obj->store, list) {
-		p = st->ptr;
-		l = st->len;
-//printf("[0-] %jd %jd\n", lx, lx + l);
-		if (lx + l < low) {
-			lx += l;
-			continue;
-		}
-		if (lx == high)
-			return (p[0]);
-		assert(lx < high);
-		if (lx < low) {
-			p += (low - lx);
-			l -= (low - lx);
-			lx = low;
-		}
-//printf("[1-] %jd %jd\n", lx, lx + l);
-		if (lx + l >= high)
-			l = high - lx;
-//printf("[2-] %jd %jd\n", lx, lx + l);
-		assert(lx >= low && lx + l <= high);
-		if (l != 0)
-			(void)WRW_Write(sp->wrk, p, l);
-		if (lx + st->len > high)
-			return(p[l]);
-		lx += st->len;
-	}
-	INCOMPL();
-}
-
-void
-ESI_DeliverChild(const struct sess *sp)
-{
-	struct storage *st;
-	struct object *obj;
-	ssize_t start, last, stop, lpad;
-	u_char *p, cc;
-	uint32_t icrc;
-	uint32_t ilen;
-	uint8_t *dbits;
-
-	if (!sp->obj->gziped) {
-		VTAILQ_FOREACH(st, &sp->obj->store, list)
-			ved_pretend_gzip(sp, st->ptr, st->len);
-		return;
-	}
-	/*
-	 * This is the interesting case: Deliver all the deflate
-	 * blocks, stripping the "LAST" bit of the last one and
-	 * padding it, as necessary, to a byte boundary.
-	 */
-
-	dbits = (void*)WS_Alloc(sp->wrk->ws, 8);
-	AN(dbits);
-	obj = sp->obj;
-	CHECK_OBJ_NOTNULL(obj, OBJECT_MAGIC);
-	start = obj->gzip_start;
-	last = obj->gzip_last;
-	stop = obj->gzip_stop;
-	assert(start > 0 && start < obj->len * 8);
-	assert(last > 0 && last < obj->len * 8);
-	assert(stop > 0 && stop < obj->len * 8);
-	assert(last >= start);
-	assert(last < stop);
-
-	/* The start bit must be byte aligned. */
-	AZ(start & 7);
-
-	/*
-	 * XXX: optimize for the case where the 'last'
-	 * XXX: bit is in a empty copy block
-	 */
-	*dbits = ved_deliver_byterange(sp, start/8, last/8);
-	*dbits &= ~(1U << (last & 7));
-	(void)WRW_Write(sp->wrk, dbits, 1);
-	cc = ved_deliver_byterange(sp, 1 + last/8, stop/8);
-	switch((int)(stop & 7)) {
-	case 0: /* xxxxxxxx */
-		/* I think we have an off by one here, but that's OK */
-		lpad = 0;
-		break;
-	case 1: /* x000.... 00000000 00000000 11111111 11111111 */
-	case 3: /* xxx000.. 00000000 00000000 11111111 11111111 */
-	case 5: /* xxxxx000 00000000 00000000 11111111 11111111 */
-		dbits[1] = cc | 0x00;
-		dbits[2] = 0x00; dbits[3] = 0x00;
-	        dbits[4] = 0xff; dbits[5] = 0xff;
-		lpad = 5;
-		break;
-	case 2: /* xx010000 00000100 00000001 00000000 */
-		dbits[1] = cc | 0x08;
-		dbits[2] = 0x20;
-		dbits[3] = 0x80;
-		dbits[4] = 0x00;
-		lpad = 4;
-		break;
-	case 4: /* xxxx0100 00000001 00000000 */
-		dbits[1] = cc | 0x20;
-		dbits[2] = 0x80;
-		dbits[3] = 0x00;
-		lpad = 3;
-		break;
-	case 6: /* xxxxxx01 00000000 */
-		dbits[1] = cc | 0x80;
-		dbits[2] = 0x00;
-		lpad = 2;
-		break;
-	case 7:	/* xxxxxxx0 00...... 00000000 00000000 11111111 11111111 */
-		dbits[1] = cc | 0x00;
-		dbits[2] = 0x00;
-		dbits[3] = 0x00; dbits[4] = 0x00;
-	        dbits[5] = 0xff; dbits[6] = 0xff;
-		lpad = 6;
-		break;
-	default:
-		INCOMPL();
-	}
-	if (lpad > 0)
-		(void)WRW_Write(sp->wrk, dbits + 1, lpad);
-	st = VTAILQ_LAST(&sp->obj->store, storagehead);
-	assert(st->len > 8);
-
-	p = st->ptr + st->len - 8;
-	icrc = vle32dec(p);
-	ilen = vle32dec(p + 4);
-	sp->wrk->crc = crc32_combine(sp->wrk->crc, icrc, ilen);
-	sp->wrk->l_crc += ilen;
-}
diff --git a/bin/varnishd/cache_esi_fetch.c b/bin/varnishd/cache_esi_fetch.c
deleted file mode 100644
index 5ec8f6b..0000000
--- a/bin/varnishd/cache_esi_fetch.c
+++ /dev/null
@@ -1,405 +0,0 @@
-/*-
- * Copyright (c) 2011 Varnish Software AS
- * All rights reserved.
- *
- * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- *
- * VEF Varnish Esi Fetching
- */
-
-#include "config.h"
-
-#include <stdlib.h>
-
-#include "cache.h"
-
-#include "cache_esi.h"
-
-/*---------------------------------------------------------------------
- * Read some bytes.
- *
- * If the esi_syntax&8 bit is set, we read only a couple of bytes at
- * a time, in order to stress the parse/pending/callback code.
- */
-
-static ssize_t
-vef_read(struct worker *w, struct http_conn *htc, void *buf, ssize_t buflen,
-    ssize_t bytes)
-{
-	ssize_t d;
-
-	if (buflen < bytes)
-		bytes = buflen;
-	if (cache_param->esi_syntax & 0x8) {
-		d = (random() & 3) + 1;
-		if (d < bytes)
-			bytes = d;
-	}
-	return (HTC_Read(w, htc, buf, bytes));
-}
-
-/*---------------------------------------------------------------------
- * We receive a ungzip'ed object, and want to store it ungzip'ed.
- */
-
-static int
-vfp_esi_bytes_uu(struct worker *w, struct http_conn *htc, ssize_t bytes)
-{
-	ssize_t wl;
-	struct storage *st;
-
-	CHECK_OBJ_NOTNULL(w, WORKER_MAGIC);
-
-	while (bytes > 0) {
-		st = FetchStorage(w, 0);
-		if (st == NULL)
-			return (-1);
-		wl = vef_read(w, htc,
-		    st->ptr + st->len, st->space - st->len, bytes);
-		if (wl <= 0)
-			return (wl);
-		VEP_Parse(w, (const char *)st->ptr + st->len, wl);
-		st->len += wl;
-		w->fetch_obj->len += wl;
-		bytes -= wl;
-	}
-	return (1);
-}
-
-/*---------------------------------------------------------------------
- * We receive a gzip'ed object, and want to store it ungzip'ed.
- */
-
-static int
-vfp_esi_bytes_gu(struct worker *w, struct http_conn *htc, ssize_t bytes)
-{
-	struct vgz *vg;
-	ssize_t wl;
-	uint8_t	ibuf[cache_param->gzip_stack_buffer];
-	int i;
-	size_t dl;
-	const void *dp;
-
-	CHECK_OBJ_NOTNULL(w, WORKER_MAGIC);
-	vg = w->vgz_rx;
-
-	while (bytes > 0) {
-		if (VGZ_IbufEmpty(vg) && bytes > 0) {
-			wl = vef_read(w, htc, ibuf, sizeof ibuf, bytes);
-			if (wl <= 0)
-				return (wl);
-			VGZ_Ibuf(vg, ibuf, wl);
-			bytes -= wl;
-		}
-		if (VGZ_ObufStorage(w, vg))
-			return(-1);
-		i = VGZ_Gunzip(vg, &dp, &dl);
-		xxxassert(i == VGZ_OK || i == VGZ_END);
-		VEP_Parse(w, dp, dl);
-		w->fetch_obj->len += dl;
-	}
-	return (1);
-}
-
-/*---------------------------------------------------------------------
- */
-
-struct vef_priv {
-	unsigned		magic;
-#define VEF_MAGIC		0xf104b51f
-	struct vgz		*vgz;
-
-	char			*bufp;
-	ssize_t			tot;
-	int			error;
-	char			pending[20];
-	ssize_t			npend;
-};
-
-/*---------------------------------------------------------------------
- * We receive a [un]gzip'ed object, and want to store it gzip'ed.
- */
-
-static ssize_t
-vfp_vep_callback(struct worker *w, ssize_t l, enum vgz_flag flg)
-{
-	struct vef_priv *vef;
-	size_t dl, px;
-	const void *dp;
-	int i;
-
-	CHECK_OBJ_NOTNULL(w, WORKER_MAGIC);
-	vef = w->vef_priv;
-	CHECK_OBJ_NOTNULL(vef, VEF_MAGIC);
-	assert(l >= 0);
-
-	if (vef->error) {
-		vef->tot += l;
-		return (vef->tot);
-	}
-
-	/*
-	 * l == 0 is valid when 'flg' calls for action, but in the
-	 * normal case we can just ignore a l==0 request.
-	 * (It would cause Z_BUF_ERROR anyway)
-	 */
-	if (l == 0 && flg == VGZ_NORMAL)
-		return (vef->tot);
-
-	do {
-		px = vef->npend;
-		if (l < px)
-			px = l;
-		if (px != 0) {
-			VGZ_Ibuf(vef->vgz, vef->pending, px);
-			l -= px;
-		} else {
-			VGZ_Ibuf(vef->vgz, vef->bufp, l);
-			vef->bufp += l;
-			l = 0;
-		}
-		do {
-			if (VGZ_ObufStorage(w, vef->vgz)) {
-				vef->error = ENOMEM;
-				vef->tot += l;
-				return (vef->tot);
-			}
-			i = VGZ_Gzip(vef->vgz, &dp, &dl, flg);
-			vef->tot += dl;
-			w->fetch_obj->len += dl;
-		} while (!VGZ_IbufEmpty(vef->vgz) ||
-		    (flg != VGZ_NORMAL && VGZ_ObufFull(vef->vgz)));
-		if (px != 0) {
-			memmove(vef->pending, vef->pending + px,
-			    vef->npend - px);
-			vef->npend -= px;
-		}
-	} while (l > 0);
-	if (flg == VGZ_FINISH)
-		assert(i == 1);			/* XXX */
-	else
-		assert(i == 0);			/* XXX */
-	return (vef->tot);
-}
-
-static int
-vfp_esi_bytes_ug(struct worker *w, struct http_conn *htc, ssize_t bytes)
-{
-	ssize_t wl;
-	char ibuf[cache_param->gzip_stack_buffer];
-	struct vef_priv *vef;
-
-	CHECK_OBJ_NOTNULL(w, WORKER_MAGIC);
-	vef = w->vef_priv;
-	CHECK_OBJ_NOTNULL(vef, VEF_MAGIC);
-
-	while (bytes > 0) {
-		wl = vef_read(w, htc, ibuf, sizeof ibuf, bytes);
-		if (wl <= 0)
-			return (wl);
-		bytes -= wl;
-		vef->bufp = ibuf;
-		VEP_Parse(w, ibuf, wl);
-		assert(vef->bufp >= ibuf && vef->bufp <= ibuf + wl);
-		if (vef->error) {
-			errno = vef->error;
-			return (-1);
-		}
-		if (vef->bufp < ibuf + wl) {
-			wl = (ibuf + wl) - vef->bufp;
-			assert(wl + vef->npend < sizeof vef->pending);
-			memmove(vef->pending + vef->npend, vef->bufp, wl);
-			vef->npend += wl;
-		}
-	}
-	return (1);
-}
-
-/*---------------------------------------------------------------------
- * We receive a gzip'ed object, and want to store it gzip'ed.
- */
-
-static int
-vfp_esi_bytes_gg(struct worker *w, struct http_conn *htc, size_t bytes)
-{
-	ssize_t wl;
-	char ibuf[cache_param->gzip_stack_buffer];
-	char ibuf2[cache_param->gzip_stack_buffer];
-	struct vef_priv *vef;
-	size_t dl;
-	const void *dp;
-	int i;
-
-	CHECK_OBJ_NOTNULL(w, WORKER_MAGIC);
-	vef = w->vef_priv;
-	CHECK_OBJ_NOTNULL(vef, VEF_MAGIC);
-	assert(sizeof ibuf >= 1024);
-	ibuf2[0] = 0; /* For Flexelint */
-
-	while (bytes > 0) {
-		wl = vef_read(w, htc, ibuf, sizeof ibuf, bytes);
-		if (wl <= 0)
-			return (wl);
-		bytes -= wl;
-
-		vef->bufp = ibuf;
-		VGZ_Ibuf(w->vgz_rx, ibuf, wl);
-		do {
-			VGZ_Obuf(w->vgz_rx, ibuf2, sizeof ibuf2);
-			i = VGZ_Gunzip(w->vgz_rx, &dp, &dl);
-			/* XXX: check i */
-			assert(i >= VGZ_OK);
-			vef->bufp = ibuf2;
-			if (dl > 0)
-				VEP_Parse(w, ibuf2, dl);
-			if (vef->error) {
-				errno = vef->error;
-				return (-1);
-			}
-			if (vef->bufp < ibuf2 + dl) {
-				dl = (ibuf2 + dl) - vef->bufp;
-				assert(dl + vef->npend < sizeof vef->pending);
-				memmove(vef->pending + vef->npend,
-				    vef->bufp, dl);
-				vef->npend += dl;
-			}
-		} while (!VGZ_IbufEmpty(w->vgz_rx));
-	}
-	return (1);
-}
-
-
-/*---------------------------------------------------------------------*/
-
-static void __match_proto__()
-vfp_esi_begin(struct worker *w, size_t estimate)
-{
-	struct vef_priv *vef;
-
-	CHECK_OBJ_NOTNULL(w, WORKER_MAGIC);
-
-	AZ(w->vgz_rx);
-	if (w->is_gzip && w->do_gunzip) {
-		w->vgz_rx = VGZ_NewUngzip(w, "U F E");
-		VEP_Init(w, NULL);
-	} else if (w->is_gunzip && w->do_gzip) {
-		ALLOC_OBJ(vef, VEF_MAGIC);
-		AN(vef);
-		vef->vgz = VGZ_NewGzip(w, "G F E");
-		AZ(w->vef_priv);
-		w->vef_priv = vef;
-		VEP_Init(w, vfp_vep_callback);
-	} else if (w->is_gzip) {
-		w->vgz_rx = VGZ_NewUngzip(w, "U F E");
-		ALLOC_OBJ(vef, VEF_MAGIC);
-		AN(vef);
-		vef->vgz = VGZ_NewGzip(w, "G F E");
-		AZ(w->vef_priv);
-		w->vef_priv = vef;
-		VEP_Init(w, vfp_vep_callback);
-	} else {
-		AZ(w->vef_priv);
-		VEP_Init(w, NULL);
-	}
-
-	(void)estimate;
-	AN(w->vep);
-}
-
-static int __match_proto__()
-vfp_esi_bytes(struct worker *w, struct http_conn *htc, ssize_t bytes)
-{
-	int i;
-
-	CHECK_OBJ_NOTNULL(w, WORKER_MAGIC);
-	AZ(w->fetch_failed);
-	AN(w->vep);
-	assert(w->htc == htc);
-	if (w->is_gzip && w->do_gunzip)
-		i = vfp_esi_bytes_gu(w, htc, bytes);
-	else if (w->is_gunzip && w->do_gzip)
-		i = vfp_esi_bytes_ug(w, htc, bytes);
-	else if (w->is_gzip)
-		i = vfp_esi_bytes_gg(w, htc, bytes);
-	else
-		i = vfp_esi_bytes_uu(w, htc, bytes);
-	AN(w->vep);
-	return (i);
-}
-
-static int __match_proto__()
-vfp_esi_end(struct worker *w)
-{
-	struct vsb *vsb;
-	struct vef_priv *vef;
-	ssize_t l;
-	int retval;
-
-	CHECK_OBJ_NOTNULL(w, WORKER_MAGIC);
-	AN(w->vep);
-
-	retval = w->fetch_failed;
-
-	if (w->vgz_rx != NULL && VGZ_Destroy(&w->vgz_rx, -1) != VGZ_END)
-		retval = FetchError(w,
-		    "Gunzip+ESI Failed at the very end");
-
-	vsb = VEP_Finish(w);
-
-	if (vsb != NULL) {
-		if (!retval) {
-			l = VSB_len(vsb);
-			assert(l > 0);
-			/* XXX: This is a huge waste of storage... */
-			w->fetch_obj->esidata = STV_alloc(w, l);
-			if (w->fetch_obj->esidata != NULL) {
-				memcpy(w->fetch_obj->esidata->ptr,
-				    VSB_data(vsb), l);
-				w->fetch_obj->esidata->len = l;
-			} else {
-				retval = FetchError(w,
-				    "Could not allocate storage for esidata");
-			}
-		}
-		VSB_delete(vsb);
-	}
-
-	if (w->vef_priv != NULL) {
-		vef = w->vef_priv;
-		CHECK_OBJ_NOTNULL(vef, VEF_MAGIC);
-		w->vef_priv = NULL;
-		VGZ_UpdateObj(vef->vgz, w->fetch_obj);
-		if (VGZ_Destroy(&vef->vgz,  -1) != VGZ_END)
-			retval = FetchError(w,
-			    "ESI+Gzip Failed at the very end");
-		FREE_OBJ(vef);
-	}
-	return (retval);
-}
-
-struct vfp vfp_esi = {
-        .begin  =       vfp_esi_begin,
-        .bytes  =       vfp_esi_bytes,
-        .end    =       vfp_esi_end,
-};
diff --git a/bin/varnishd/cache_esi_parse.c b/bin/varnishd/cache_esi_parse.c
deleted file mode 100644
index 9e2b4f6..0000000
--- a/bin/varnishd/cache_esi_parse.c
+++ /dev/null
@@ -1,1189 +0,0 @@
-/*-
- * Copyright (c) 2011 Varnish Software AS
- * All rights reserved.
- *
- * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- *
- * VEP Varnish Esi Parsing
- */
-
-#include "config.h"
-
-#include <stdio.h>
-#include <stdlib.h>
-
-#include "cache.h"
-
-#include "cache_esi.h"
-#include "vct.h"
-#include "vend.h"
-#include "vgz.h"
-
-//#define Debug(fmt, ...) printf(fmt, __VA_ARGS__)
-#define Debug(fmt, ...) /**/
-
-struct vep_state;
-
-enum dowhat {DO_ATTR, DO_TAG};
-typedef void dostuff_f(struct vep_state *, enum dowhat);
-
-struct vep_match {
-	const char	*match;
-	const char	* const *state;
-};
-
-enum vep_mark { VERBATIM = 0, SKIP };
-
-struct vep_state {
-	unsigned		magic;
-#define VEP_MAGIC		0x55cb9b82
-	struct vsb		*vsb;
-
-	struct worker		*wrk;
-	int			dogzip;
-	vep_callback_t		*cb;
-
-	/* Internal Counter for default call-back function */
-	ssize_t			cb_x;
-
-	/* parser state */
-	const char		*state;
-	unsigned		startup;
-	unsigned		esi_found;
-
-	unsigned		endtag;
-	unsigned		emptytag;
-	unsigned		canattr;
-
-	unsigned		remove;
-
-	ssize_t			o_wait;
-	ssize_t			o_pending;
-	ssize_t			o_total;
-	uint32_t		crc;
-	ssize_t			o_crc;
-	uint32_t		crcp;
-	ssize_t			o_last;
-
-const char		*hack_p;
-	const char		*ver_p;
-
-	const char		*until;
-	const char		*until_p;
-	const char		*until_s;
-
-	int			in_esi_tag;
-
-	const char		*esicmt;
-	const char		*esicmt_p;
-
-	struct vep_match	*attr;
-	struct vsb		*attr_vsb;
-	int			attr_delim;
-
-	struct vep_match	*match;
-	struct vep_match	*match_hit;
-
-	char			tag[10];
-	int			tag_i;
-
-	dostuff_f		*dostuff;
-
-	struct vsb		*include_src;
-
-	unsigned		nm_skip;
-	unsigned		nm_verbatim;
-	unsigned		nm_pending;
-	enum vep_mark		last_mark;
-};
-
-/*---------------------------------------------------------------------*/
-
-static const char * const VEP_START =		"[Start]";
-static const char * const VEP_TESTXML =		"[TestXml]";
-static const char * const VEP_NOTXML =		"[NotXml]";
-
-static const char * const VEP_NEXTTAG =		"[NxtTag]";
-static const char * const VEP_NOTMYTAG =	"[NotMyTag]";
-
-static const char * const VEP_STARTTAG =	"[StartTag]";
-static const char * const VEP_COMMENT =		"[Comment]";
-static const char * const VEP_CDATA =		"[CDATA]";
-static const char * const VEP_ESITAG =		"[ESITag]";
-
-static const char * const VEP_ESIREMOVE =	"[ESI:Remove]";
-static const char * const VEP_ESIINCLUDE =	"[ESI:Include]";
-static const char * const VEP_ESICOMMENT =	"[ESI:Comment]";
-static const char * const VEP_ESIBOGON =	"[ESI:Bogon]";
-
-static const char * const VEP_INTAG =		"[InTag]";
-static const char * const VEP_TAGERROR =	"[TagError]";
-
-static const char * const VEP_ATTR =		"[Attribute]";
-static const char * const VEP_SKIPATTR =	"[SkipAttribute]";
-static const char * const VEP_ATTRDELIM =	"[AttrDelim]";
-static const char * const VEP_ATTRGETVAL =	"[AttrGetValue]";
-static const char * const VEP_ATTRVAL =		"[AttrValue]";
-
-static const char * const VEP_UNTIL =		"[Until]";
-static const char * const VEP_MATCHBUF =	"[MatchBuf]";
-static const char * const VEP_MATCH =		"[Match]";
-
-/*---------------------------------------------------------------------*/
-
-static struct vep_match vep_match_starttag[] = {
-	{ "!--",	&VEP_COMMENT },
-	{ "esi:",	&VEP_ESITAG },
-	{ "![CDATA[",	&VEP_CDATA },
-	{ NULL,		&VEP_NOTMYTAG }
-};
-
-/*---------------------------------------------------------------------*/
-
-static struct vep_match vep_match_esi[] = {
-	{ "include",	&VEP_ESIINCLUDE },
-	{ "remove",	&VEP_ESIREMOVE },
-	{ "comment",	&VEP_ESICOMMENT },
-	{ NULL,		&VEP_ESIBOGON }
-};
-
-/*---------------------------------------------------------------------*/
-
-static struct vep_match vep_match_attr_include[] = {
-	{ "src=",	&VEP_ATTRGETVAL },
-	{ NULL,		&VEP_SKIPATTR }
-};
-
-/*--------------------------------------------------------------------
- * Report a parsing error
- */
-
-static void
-vep_error(const struct vep_state *vep, const char *p)
-{
-	intmax_t l;
-
-	VSC_C_main->esi_errors++;
-	l = (intmax_t)(vep->ver_p - vep->hack_p);
-	WSLB(vep->wrk, SLT_ESI_xmlerror, "ERR at %jd %s", l, p);
-
-}
-
-/*--------------------------------------------------------------------
- * Report a parsing warning
- */
-
-static void
-vep_warn(const struct vep_state *vep, const char *p)
-{
-	intmax_t l;
-
-	VSC_C_main->esi_warnings++;
-	l = (intmax_t)(vep->ver_p - vep->hack_p);
-	printf("WARNING at %jd %s\n", l, p);
-	WSLB(vep->wrk, SLT_ESI_xmlerror, "WARN at %jd %s", l, p);
-
-}
-
-/*---------------------------------------------------------------------
- * return match or NULL if more input needed.
- */
-
-static struct vep_match *
-vep_match(struct vep_state *vep, const char *b, const char *e)
-{
-	struct vep_match *vm;
-	const char *q, *r;
-	ssize_t l;
-
-	for (vm = vep->match; vm->match; vm++) {
-		r = b;
-		for (q = vm->match; *q && r < e; q++, r++)
-			if (*q != *r)
-				break;
-		if (*q != '\0' && r == e) {
-			if (b != vep->tag) {
-				l = e - b;
-				assert(l < sizeof vep->tag);
-				memmove(vep->tag, b, l);
-				vep->tag_i = l;
-			}
-			return (NULL);
-		}
-		if (*q == '\0')
-			return (vm);
-	}
-	return (vm);
-}
-
-/*---------------------------------------------------------------------
- *
- */
-
-static void
-vep_emit_len(const struct vep_state *vep, ssize_t l, int m8, int m16, int m64)
-{
-	uint8_t buf[9];
-
-	assert(l > 0);
-	if (l < 256) {
-		buf[0] = (uint8_t)m8;
-		buf[1] = (uint8_t)l;
-		assert((ssize_t)buf[1] == l);
-		VSB_bcat(vep->vsb, buf, 2);
-	} else if (l < 65536) {
-		buf[0] = (uint8_t)m16;
-		vbe16enc(buf + 1, (uint16_t)l);
-		assert((ssize_t)vbe16dec(buf + 1) == l);
-		VSB_bcat(vep->vsb, buf, 3);
-	} else {
-		buf[0] = (uint8_t)m64;
-		vbe64enc(buf + 1, l);
-		assert((ssize_t)vbe64dec(buf + 1) == l);
-		VSB_bcat(vep->vsb, buf, 9);
-	}
-}
-
-static void
-vep_emit_skip(const struct vep_state *vep, ssize_t l)
-{
-
-	if (cache_param->esi_syntax & 0x20) {
-		Debug("---> SKIP(%jd)\n", (intmax_t)l);
-	}
-	vep_emit_len(vep, l, VEC_S1, VEC_S2, VEC_S8);
-}
-
-static void
-vep_emit_verbatim(const struct vep_state *vep, ssize_t l, ssize_t l_crc)
-{
-	uint8_t buf[4];
-
-	if (cache_param->esi_syntax & 0x20) {
-		Debug("---> VERBATIM(%jd)\n", (intmax_t)l);
-	}
-	vep_emit_len(vep, l, VEC_V1, VEC_V2, VEC_V8);
-	if (vep->dogzip) {
-		vep_emit_len(vep, l_crc, VEC_C1, VEC_C2, VEC_C8);
-		vbe32enc(buf, vep->crc);
-		VSB_bcat(vep->vsb, buf, sizeof buf);
-	}
-}
-
-static void
-vep_emit_common(struct vep_state *vep, ssize_t l, enum vep_mark mark)
-{
-
-	assert(l > 0);
-	assert(mark == SKIP || mark == VERBATIM);
-	if (mark == SKIP)
-		vep_emit_skip(vep, l);
-	else
-		vep_emit_verbatim(vep, l, vep->o_crc);
-
-	vep->crc = crc32(0L, Z_NULL, 0);
-	vep->o_crc = 0;
-	vep->o_total += l;
-}
-
-/*---------------------------------------------------------------------
- *
- */
-
-static void
-vep_mark_common(struct vep_state *vep, const char *p, enum vep_mark mark)
-{
-	ssize_t l, lcb;
-
-	assert(mark == SKIP || mark == VERBATIM);
-
-	/* The NO-OP case, no data, no pending data & no change of mode */
-	if (vep->last_mark == mark && p == vep->ver_p && vep->o_pending == 0)
-		return;
-
-	/*
-	 * If we changed mode, emit whatever the opposite mode
-	 * assembled before the pending bytes.
-	 */
-
-	if (vep->last_mark != mark && (vep->o_wait > 0 || vep->startup)) {
-		lcb = vep->cb(vep->wrk, 0,
-		    mark == VERBATIM ? VGZ_RESET : VGZ_ALIGN);
-		if (lcb - vep->o_last > 0)
-			vep_emit_common(vep, lcb - vep->o_last, vep->last_mark);
-		vep->o_last = lcb;
-		vep->o_wait = 0;
-	}
-
-	/* Transfer pending bytes CRC into active mode CRC */
-	if (vep->o_pending) {
-		(void)vep->cb(vep->wrk, vep->o_pending, VGZ_NORMAL);
-		if (vep->o_crc == 0) {
-			vep->crc = vep->crcp;
-			vep->o_crc = vep->o_pending;
-		} else {
-			vep->crc = crc32_combine(vep->crc,
-			    vep->crcp, vep->o_pending);
-			vep->o_crc += vep->o_pending;
-		}
-		vep->crcp = crc32(0L, Z_NULL, 0);
-		vep->o_wait += vep->o_pending;
-		vep->o_pending = 0;
-	}
-
-	/* * Process this bit of input */
-	AN(vep->ver_p);
-	l = p - vep->ver_p;
-	assert(l >= 0);
-	vep->crc = crc32(vep->crc, (const void*)vep->ver_p, l);
-	vep->o_crc += l;
-	vep->ver_p = p;
-
-	vep->o_wait += l;
-	vep->last_mark = mark;
-	(void)vep->cb(vep->wrk, l, VGZ_NORMAL);
-}
-
-static void
-vep_mark_verbatim(struct vep_state *vep, const char *p)
-{
-
-	vep_mark_common(vep, p, VERBATIM);
-	vep->nm_verbatim++;
-}
-
-static void
-vep_mark_skip(struct vep_state *vep, const char *p)
-{
-
-	vep_mark_common(vep, p, SKIP);
-	vep->nm_skip++;
-}
-
-static void
-vep_mark_pending(struct vep_state *vep, const char *p)
-{
-	ssize_t l;
-
-	AN(vep->ver_p);
-	l = p - vep->ver_p;
-	assert(l > 0);
-	assert(l >= 0);
-	vep->crcp = crc32(vep->crcp, (const void *)vep->ver_p, l);
-	vep->ver_p = p;
-
-	vep->o_pending += l;
-	vep->nm_pending++;
-}
-
-/*---------------------------------------------------------------------
- */
-
-static void __match_proto__()
-vep_do_comment(struct vep_state *vep, enum dowhat what)
-{
-	Debug("DO_COMMENT(%d)\n", what);
-	assert(what == DO_TAG);
-	if (!vep->emptytag)
-		vep_error(vep, "ESI 1.0 <esi:comment> needs final '/'");
-}
-
-/*---------------------------------------------------------------------
- */
-
-static void __match_proto__()
-vep_do_remove(struct vep_state *vep, enum dowhat what)
-{
-	Debug("DO_REMOVE(%d, end %d empty %d remove %d)\n",
-	    what, vep->endtag, vep->emptytag, vep->remove);
-	assert(what == DO_TAG);
-	if (vep->emptytag) {
-		vep_error(vep,
-		    "ESI 1.0 <esi:remove/> not legal");
-	} else {
-		if (vep->remove && !vep->endtag)
-			vep_error(vep,
-			    "ESI 1.0 <esi:remove> already open");
-		else if (!vep->remove && vep->endtag)
-			vep_error(vep,
-			    "ESI 1.0 <esi:remove> not open");
-		else
-			vep->remove = !vep->endtag;
-	}
-}
-
-/*---------------------------------------------------------------------
- */
-
-static void __match_proto__()
-vep_do_include(struct vep_state *vep, enum dowhat what)
-{
-	char *p, *q, *h;
-	ssize_t l;
-	txt url;
-
-	Debug("DO_INCLUDE(%d)\n", what);
-	if (what == DO_ATTR) {
-		Debug("ATTR (%s) (%s)\n", vep->match_hit->match,
-			VSB_data(vep->attr_vsb));
-		if (vep->include_src != NULL) {
-			vep_error(vep,
-			    "ESI 1.0 <esi:include> "
-			    "has multiple src= attributes");
-			vep->state = VEP_TAGERROR;
-			VSB_delete(vep->attr_vsb);
-			VSB_delete(vep->include_src);
-			vep->attr_vsb = NULL;
-			vep->include_src = NULL;
-			return;
-		}
-		XXXAZ(vep->include_src);	/* multiple src= */
-		vep->include_src = vep->attr_vsb;
-		return;
-	}
-	assert(what == DO_TAG);
-	if (!vep->emptytag)
-		vep_warn(vep,
-		    "ESI 1.0 <esi:include> lacks final '/'");
-	if (vep->include_src == NULL) {
-		vep_error(vep,
-		    "ESI 1.0 <esi:include> lacks src attr");
-		return;
-	}
-
-	/*
-	 * Strictly speaking, we ought to spit out any piled up skip before
-	 * emitting the VEC for the include, but objectively that makes no
-	 * difference and robs us of a chance to collapse another skip into
-	 * this on so we don't do that.
-	 * However, we cannot tolerate any verbatim stuff piling up.
-	 * The mark_skip() before calling dostuff should have taken
-	 * care of that.  Make sure.
-	 */
-	assert(vep->o_wait == 0 || vep->last_mark == SKIP);
-	/* XXX: what if it contains NUL bytes ?? */
-	p = VSB_data(vep->include_src);
-	l = VSB_len(vep->include_src);
-	h = 0;
-
-	VSB_printf(vep->vsb, "%c", VEC_INCL);
-	if (l > 7 && !memcmp(p, "http://", 7)) {
-		h = p + 7;
-		p = strchr(h, '/');
-		AN(p);
-		Debug("HOST <%.*s> PATH <%s>\n", (int)(p-h),h, p);
-		VSB_printf(vep->vsb, "Host: %.*s%c",
-		    (int)(p-h), h, 0);
-	} else if (*p == '/') {
-		VSB_printf(vep->vsb, "%c", 0);
-	} else {
-		VSB_printf(vep->vsb, "%c", 0);
-		url = vep->wrk->bereq->hd[HTTP_HDR_URL];
-		/* Look for the last / before a '?' */
-		h = NULL;
-		for (q = url.b; q < url.e && *q != '?'; q++)
-			if (*q == '/')
-				h = q;
-		if (h == NULL)
-			h = q + 1;
-
-		Debug("INCL:: [%.*s]/[%s]\n",
-		    (int)(h - url.b), url.b, p);
-		VSB_printf(vep->vsb, "%.*s/", (int)(h - url.b), url.b);
-	}
-	l -= (p - VSB_data(vep->include_src));
-	for (q = p; *q != '\0'; ) {
-		if (*q == '&') {
-#define R(w,f,r)							\
-			if (q + w <= p + l && !memcmp(q, f, w)) { \
-				VSB_printf(vep->vsb, "%c", r);	\
-				q += w;				\
-				continue;			\
-			}
-			R(6, "&apos;", '\'');
-			R(6, "&quot;", '"');
-			R(4, "&lt;", '<');
-			R(4, "&gt;", '>');
-			R(5, "&amp;", '&');
-		}
-		VSB_printf(vep->vsb, "%c", *q++);
-	}
-#undef R
-	VSB_printf(vep->vsb, "%c", 0);
-
-	VSB_delete(vep->include_src);
-	vep->include_src = NULL;
-}
-
-/*---------------------------------------------------------------------
- * Lex/Parse object for ESI instructions
- *
- * This function is called with the input object piecemal so do not
- * assume that we have more than one char available at at time, but
- * optimize for getting huge chunks.
- *
- * NB: At the bottom of this source-file, there is a dot-diagram matching
- * NB: the state-machine.  Please maintain it along with the code.
- */
-
-void
-VEP_Parse(const struct worker *w, const char *p, size_t l)
-{
-	struct vep_state *vep;
-	const char *e;
-	struct vep_match *vm;
-	int i;
-
-	CHECK_OBJ_NOTNULL(w, WORKER_MAGIC);
-	vep = w->vep;
-	CHECK_OBJ_NOTNULL(vep, VEP_MAGIC);
-	assert(l > 0);
-
-	/* XXX: Really need to fix this */
-	if (vep->hack_p == NULL)
-		vep->hack_p = p;
-
-	vep->ver_p = p;
-
-	e = p + l;
-
-	while (p < e) {
-		AN(vep->state);
-		i = e - p;
-		if (i > 10)
-			i = 10;
-		Debug("EP %s %d (%.*s) [%.*s]\n",
-		    vep->state,
-		    vep->remove,
-		    vep->tag_i, vep->tag,
-		    i, p);
-		assert(p >= vep->ver_p);
-
-		/******************************************************
-		 * SECTION A
-		 */
-
-		if (vep->state == VEP_START) {
-			if (cache_param->esi_syntax & 0x1)
-				vep->state = VEP_NEXTTAG;
-			else
-				vep->state = VEP_TESTXML;
-		} else if (vep->state == VEP_TESTXML) {
-			/*
-			 * If the first non-whitespace char is different
-			 * from '<' we assume this is not XML.
-			 */
-			while (p < e && vct_islws(*p))
-				p++;
-			vep_mark_verbatim(vep, p);
-			if (p < e && *p == '<') {
-				p++;
-				vep->state = VEP_STARTTAG;
-			} else if (p < e) {
-				WSLB(vep->wrk, SLT_ESI_xmlerror,
-				    "No ESI processing, first char not '<'");
-				vep->state = VEP_NOTXML;
-			}
-		} else if (vep->state == VEP_NOTXML) {
-			/*
-			 * This is not recognized as XML, just skip thru
-			 * vfp_esi_end() will handle the rest
-			 */
-			p = e;
-			vep_mark_verbatim(vep, p);
-
-		/******************************************************
-		 * SECTION B
-		 */
-
-		} else if (vep->state == VEP_NOTMYTAG) {
-			if (cache_param->esi_syntax & 0x2) {
-				p++;
-				vep->state = VEP_NEXTTAG;
-			} else {
-				vep->tag_i = 0;
-				while (p < e) {
-					if (*p++ == '>') {
-						vep->state = VEP_NEXTTAG;
-						break;
-					}
-				}
-			}
-			if (p == e && !vep->remove)
-				vep_mark_verbatim(vep, p);
-		} else if (vep->state == VEP_NEXTTAG) {
-			/*
-			 * Hunt for start of next tag and keep an eye
-			 * out for end of EsiCmt if armed.
-			 */
-			vep->emptytag = 0;
-			vep->endtag = 0;
-			vep->attr = NULL;
-			vep->dostuff = NULL;
-			while (p < e && *p != '<') {
-				if (vep->esicmt_p == NULL) {
-					p++;
-					continue;
-				}
-				if (*p != *vep->esicmt_p) {
-					p++;
-					vep->esicmt_p = vep->esicmt;
-					continue;
-				}
-				if (!vep->remove &&
-				    vep->esicmt_p == vep->esicmt)
-					vep_mark_verbatim(vep, p);
-				p++;
-				if (*++vep->esicmt_p == '\0') {
-					vep->esi_found = 1;
-					vep->esicmt = NULL;
-					vep->esicmt_p = NULL;
-					/*
-					 * The end of the esicmt
-					 * should not be emitted.
-					 * But the stuff before should
-					 */
-					vep_mark_skip(vep, p);
-				}
-			}
-			if (p < e) {
-				if (!vep->remove)
-					vep_mark_verbatim(vep, p);
-				assert(*p == '<');
-				p++;
-				vep->state = VEP_STARTTAG;
-			} else if (vep->esicmt_p == vep->esicmt && !vep->remove)
-				vep_mark_verbatim(vep, p);
-
-		/******************************************************
-		 * SECTION C
-		 */
-
-		} else if (vep->state == VEP_STARTTAG) {
-			/*
-			 * Start of tag, set up match table
-			 */
-			if (p < e) {
-				if (*p == '/') {
-					vep->endtag = 1;
-					p++;
-				}
-				vep->match = vep_match_starttag;
-				vep->state = VEP_MATCH;
-			}
-		} else if (vep->state == VEP_COMMENT) {
-			/*
-			 * We are in a comment, find out if it is an
-			 * ESI comment or a regular comment
-			 */
-			if (vep->esicmt == NULL)
-				vep->esicmt_p = vep->esicmt = "esi";
-			while (p < e) {
-				if (*p != *vep->esicmt_p) {
-					vep->esicmt_p = vep->esicmt = NULL;
-					vep->until_p = vep->until = "-->";
-					vep->until_s = VEP_NEXTTAG;
-					vep->state = VEP_UNTIL;
-					vep_mark_verbatim(vep, p);
-					break;
-				}
-				p++;
-				if (*++vep->esicmt_p != '\0')
-					continue;
-				if (vep->remove)
-					vep_error(vep,
-					    "ESI 1.0 Nested <!--esi"
-					    " element in <esi:remove>");
-				vep->esicmt_p = vep->esicmt = "-->";
-				vep->state = VEP_NEXTTAG;
-				vep_mark_skip(vep, p);
-				break;
-			}
-		} else if (vep->state == VEP_CDATA) {
-			/*
-			 * Easy: just look for the end of CDATA
-			 */
-			vep->until_p = vep->until = "]]>";
-			vep->until_s = VEP_NEXTTAG;
-			vep->state = VEP_UNTIL;
-		} else if (vep->state == VEP_ESITAG) {
-			vep->in_esi_tag = 1;
-			vep->esi_found = 1;
-			vep_mark_skip(vep, p);
-			vep->match = vep_match_esi;
-			vep->state = VEP_MATCH;
-		} else if (vep->state == VEP_ESIINCLUDE) {
-			if (vep->remove) {
-				vep_error(vep,
-				    "ESI 1.0 <esi:include> element"
-				    " nested in <esi:remove>");
-				vep->state = VEP_TAGERROR;
-			} else if (vep->endtag) {
-				vep_error(vep,
-				    "ESI 1.0 </esi:include> illegal end-tag");
-				vep->state = VEP_TAGERROR;
-			} else {
-				vep->dostuff = vep_do_include;
-				vep->state = VEP_INTAG;
-				vep->attr = vep_match_attr_include;
-			}
-		} else if (vep->state == VEP_ESIREMOVE) {
-			vep->dostuff = vep_do_remove;
-			vep->state = VEP_INTAG;
-		} else if (vep->state == VEP_ESICOMMENT) {
-			if (vep->remove) {
-				vep_error(vep,
-				    "ESI 1.0 <esi:comment> element"
-				    " nested in <esi:remove>");
-				vep->state = VEP_TAGERROR;
-			} else if (vep->endtag) {
-				vep_error(vep,
-				    "ESI 1.0 </esi:comment> illegal end-tag");
-				vep->state = VEP_TAGERROR;
-			} else {
-				vep->dostuff = vep_do_comment;
-				vep->state = VEP_INTAG;
-			}
-		} else if (vep->state == VEP_ESIBOGON) {
-			vep_error(vep,
-			    "ESI 1.0 <esi:bogus> element");
-			vep->state = VEP_TAGERROR;
-
-		/******************************************************
-		 * SECTION D
-		 */
-
-		} else if (vep->state == VEP_INTAG) {
-			vep->tag_i = 0;
-			while (p < e && vct_islws(*p) && !vep->emptytag) {
-				p++;
-				vep->canattr = 1;
-			}
-			if (p < e && *p == '/' && !vep->emptytag) {
-				p++;
-				vep->emptytag = 1;
-				vep->canattr = 0;
-			}
-			if (p < e && *p == '>') {
-				p++;
-				AN(vep->dostuff);
-				vep_mark_skip(vep, p);
-				vep->dostuff(vep, DO_TAG);
-				vep->in_esi_tag = 0;
-				vep->state = VEP_NEXTTAG;
-			} else if (p < e && vep->emptytag) {
-				vep_error(vep,
-				    "XML 1.0 '>' does not follow '/' in tag");
-				vep->state = VEP_TAGERROR;
-			} else if (p < e && vep->canattr &&
-			    vct_isxmlnamestart(*p)) {
-				vep->state = VEP_ATTR;
-			} else if (p < e) {
-				vep_error(vep,
-				    "XML 1.0 Illegal attribute start char");
-				vep->state = VEP_TAGERROR;
-			}
-		} else if (vep->state == VEP_TAGERROR) {
-			while (p < e && *p != '>')
-				p++;
-			if (p < e) {
-				p++;
-				vep_mark_skip(vep, p);
-				vep->in_esi_tag = 0;
-				vep->state = VEP_NEXTTAG;
-			}
-
-		/******************************************************
-		 * SECTION E
-		 */
-
-		} else if (vep->state == VEP_ATTR) {
-			AZ(vep->attr_delim);
-			if (vep->attr == NULL) {
-				p++;
-				AZ(vep->attr_vsb);
-				vep->state = VEP_SKIPATTR;
-			} else {
-				vep->match = vep->attr;
-				vep->state = VEP_MATCH;
-			}
-		} else if (vep->state == VEP_SKIPATTR) {
-			while (p < e && vct_isxmlname(*p))
-				p++;
-			if (p < e && *p == '=') {
-				p++;
-				vep->state = VEP_ATTRDELIM;
-			} else if (p < e && *p == '>') {
-				vep->state = VEP_INTAG;
-			} else if (p < e && *p == '/') {
-				vep->state = VEP_INTAG;
-			} else if (p < e && vct_issp(*p)) {
-				vep->state = VEP_INTAG;
-			} else if (p < e) {
-				vep_error(vep,
-				    "XML 1.0 Illegal attr char");
-				vep->state = VEP_TAGERROR;
-			}
-		} else if (vep->state == VEP_ATTRGETVAL) {
-			vep->attr_vsb = VSB_new_auto();
-			vep->state = VEP_ATTRDELIM;
-		} else if (vep->state == VEP_ATTRDELIM) {
-			AZ(vep->attr_delim);
-			if (*p == '"' || *p == '\'') {
-				vep->attr_delim = *p++;
-				vep->state = VEP_ATTRVAL;
-			} else if (!vct_issp(*p)) {
-				vep->attr_delim = ' ';
-				vep->state = VEP_ATTRVAL;
-			} else {
-				vep_error(vep,
-				    "XML 1.0 Illegal attribute delimiter");
-				vep->state = VEP_TAGERROR;
-			}
-
-		} else if (vep->state == VEP_ATTRVAL) {
-			while (p < e && *p != '>' && *p != vep->attr_delim &&
-			   (vep->attr_delim != ' ' || !vct_issp(*p))) {
-				if (vep->attr_vsb != NULL)
-					VSB_bcat(vep->attr_vsb, p, 1);
-				p++;
-			}
-			if (p < e && *p == '>') {
-				vep_error(vep,
-				    "XML 1.0 Missing end attribute delimiter");
-				vep->state = VEP_TAGERROR;
-				vep->attr_delim = 0;
-				if (vep->attr_vsb != NULL) {
-					AZ(VSB_finish(vep->attr_vsb));
-					VSB_delete(vep->attr_vsb);
-					vep->attr_vsb = NULL;
-				}
-			} else if (p < e) {
-				vep->attr_delim = 0;
-				p++;
-				vep->state = VEP_INTAG;
-				if (vep->attr_vsb != NULL) {
-					AZ(VSB_finish(vep->attr_vsb));
-					AN(vep->dostuff);
-					vep->dostuff(vep, DO_ATTR);
-					vep->attr_vsb = NULL;
-				}
-			}
-
-		/******************************************************
-		 * Utility Section
-		 */
-
-		} else if (vep->state == VEP_MATCH) {
-			/*
-			 * Match against a table
-			 */
-			vm = vep_match(vep, p, e);
-			vep->match_hit = vm;
-			if (vm != NULL) {
-				if (vm->match != NULL)
-					p += strlen(vm->match);
-				vep->state = *vm->state;
-				vep->match = NULL;
-				vep->tag_i = 0;
-			} else {
-				memcpy(vep->tag, p, e - p);
-				vep->tag_i = e - p;
-				vep->state = VEP_MATCHBUF;
-				p = e;
-			}
-		} else if (vep->state == VEP_MATCHBUF) {
-			/*
-			 * Match against a table while split over input
-			 * sections.
-			 */
-			do {
-				if (*p == '>') {
-					for (vm = vep->match;
-					    vm->match != NULL; vm++)
-						continue;
-					AZ(vm->match);
-				} else {
-					vep->tag[vep->tag_i++] = *p++;
-					vm = vep_match(vep,
-					    vep->tag, vep->tag + vep->tag_i);
-					if (vm && vm->match == NULL) {
-						vep->tag_i--;
-						p--;
-					}
-				}
-			} while (vm == NULL && p < e);
-			vep->match_hit = vm;
-			if (vm == NULL) {
-				assert(p == e);
-			} else {
-				vep->state = *vm->state;
-				vep->match = NULL;
-			}
-		} else if (vep->state == VEP_UNTIL) {
-			/*
-			 * Skip until we see magic string
-			 */
-			while (p < e) {
-				if (*p++ != *vep->until_p++) {
-					vep->until_p = vep->until;
-				} else if (*vep->until_p == '\0') {
-					vep->state = vep->until_s;
-					break;
-				}
-			}
-			if (p == e && !vep->remove)
-				vep_mark_verbatim(vep, p);
-		} else {
-			Debug("*** Unknown state %s\n", vep->state);
-			INCOMPL();
-		}
-	}
-	/*
-	 * We must always mark up the storage we got, try to do so
-	 * in the most efficient way, in particular with respect to
-	 * minimizing and limiting use of pending.
-	 */
-	if (p == vep->ver_p)
-		;
-	else if (vep->in_esi_tag)
-		vep_mark_skip(vep, p);
-	else if (vep->remove)
-		vep_mark_skip(vep, p);
-	else
-		vep_mark_pending(vep, p);
-}
-
-/*---------------------------------------------------------------------
- */
-
-static ssize_t __match_proto__()
-vep_default_cb(struct worker *w, ssize_t l, enum vgz_flag flg)
-{
-
-	(void)flg;
-	AN(w->vep);
-	w->vep->cb_x += l;
-Debug("CB(%jd,%d) = %jd\n", (intmax_t)l, flg, (intmax_t)w->vep->cb_x);
-	return (w->vep->cb_x);
-}
-
-/*---------------------------------------------------------------------
- */
-
-void
-VEP_Init(struct worker *w, vep_callback_t *cb)
-{
-	struct vep_state *vep;
-
-	CHECK_OBJ_NOTNULL(w, WORKER_MAGIC);
-	AZ(w->vep);
-	w->vep = (void*)WS_Alloc(w->ws, sizeof *vep);
-	AN(w->vep);
-
-	vep = w->vep;
-	memset(vep, 0, sizeof *vep);
-	vep->magic = VEP_MAGIC;
-	vep->wrk = w;
-	vep->vsb = VSB_new_auto();
-	AN(vep->vsb);
-
-	if (cb != NULL) {
-		vep->dogzip = 1;
-		/* XXX */
-		VSB_printf(vep->vsb, "%c", VEC_GZ);
-		vep->cb = cb;
-	} else {
-		vep->cb = vep_default_cb;
-	}
-
-	vep->state = VEP_START;
-	vep->crc = crc32(0L, Z_NULL, 0);
-	vep->crcp = crc32(0L, Z_NULL, 0);
-
-	/*
-	 * We must force the GZIP header out as a SKIP string, otherwise
-	 * an object starting with <esi:include would have its GZIP header
-	 * appear after the included object (e000026.vtc)
-	 */
-	vep->startup = 1;
-	vep->ver_p = "";
-	vep->last_mark = SKIP;
-	vep_mark_common(vep, vep->ver_p, VERBATIM);
-	vep->startup = 0;
-}
-
-/*---------------------------------------------------------------------
- */
-
-struct vsb *
-VEP_Finish(struct worker *w)
-{
-	struct vep_state *vep;
-	ssize_t l, lcb;
-
-	CHECK_OBJ_NOTNULL(w, WORKER_MAGIC);
-	vep = w->vep;
-	CHECK_OBJ_NOTNULL(vep, VEP_MAGIC);
-
-	if (vep->o_pending)
-		vep_mark_common(vep, vep->ver_p, vep->last_mark);
-	if (vep->o_wait > 0) {
-		lcb = vep->cb(vep->wrk, 0, VGZ_ALIGN);
-		vep_emit_common(vep, lcb - vep->o_last, vep->last_mark);
-	}
-	(void)vep->cb(vep->wrk, 0, VGZ_FINISH);
-
-	w->vep = NULL;
-
-	AZ(VSB_finish(vep->vsb));
-	l = VSB_len(vep->vsb);
-	if (vep->esi_found && l > 0)
-		return (vep->vsb);
-	VSB_delete(vep->vsb);
-	return (NULL);
-}
-
-#if 0
-
-digraph xml {
-	rankdir="LR"
-	size="7,10"
-#################################################################
-# SECTION A
-#
-
-START		[shape=ellipse]
-TESTXML		[shape=ellipse]
-NOTXML		[shape=ellipse]
-NEXTTAGa	[shape=hexagon, label="NEXTTAG"]
-STARTTAGa	[shape=hexagon, label="STARTTAG"]
-START		-> TESTXML
-START		-> NEXTTAGa	[style=dotted, label="syntax:1"]
-TESTXML		-> TESTXML	[label="lws"]
-TESTXML		-> NOTXML
-TESTXML		-> STARTTAGa	[label="'<'"]
-
-#################################################################
-# SECTION B
-
-NOTMYTAG	[shape=ellipse]
-NEXTTAG		[shape=ellipse]
-NOTMYTAG	-> NEXTTAG	[style=dotted, label="syntax:2"]
-STARTTAGb	[shape=hexagon, label="STARTTAG"]
-NOTMYTAG	-> NEXTTAG	[label="'>'"]
-NOTMYTAG	-> NOTMYTAG	[label="*"]
-NEXTTAG		-> NEXTTAG	[label="'-->'"]
-NEXTTAG		-> NEXTTAG	[label="*"]
-NEXTTAG		-> STARTTAGb	[label="'<'"]
-
-#################################################################
-# SECTION C
-
-STARTTAG	[shape=ellipse]
-COMMENT		[shape=ellipse]
-CDATA		[shape=ellipse]
-ESITAG		[shape=ellipse]
-ESIETAG		[shape=ellipse]
-ESIINCLUDE	[shape=ellipse]
-ESIREMOVE	[shape=ellipse]
-ESICOMMENT	[shape=ellipse]
-ESIBOGON	[shape=ellipse]
-INTAGc		[shape=hexagon, label="INTAG"]
-NOTMYTAGc	[shape=hexagon, label="NOTMYTAG"]
-NEXTTAGc	[shape=hexagon, label="NEXTTAG"]
-TAGERRORc	[shape=hexagon, label="TAGERROR"]
-C1		[shape=circle,label=""]
-STARTTAG	-> COMMENT	[label="'<!--'"]
-STARTTAG	-> ESITAG	[label="'<esi'"]
-STARTTAG	-> CDATA	[label="'<![CDATA['"]
-STARTTAG	-> NOTMYTAGc	[label="'*'"]
-COMMENT		-> NEXTTAGc	[label="'esi'"]
-COMMENT		-> C1		[label="*"]
-C1		-> C1		[label="*"]
-C1		-> NEXTTAGc	[label="-->"]
-CDATA		-> CDATA	[label="*"]
-CDATA		-> NEXTTAGc	[label="]]>"]
-ESITAG		-> ESIINCLUDE	[label="'include'"]
-ESITAG		-> ESIREMOVE	[label="'remove'"]
-ESITAG		-> ESICOMMENT	[label="'comment'"]
-ESITAG		-> ESIBOGON	[label="*"]
-ESICOMMENT	-> INTAGc
-ESICOMMENT	-> TAGERRORc
-ESICOMMENT	-> TAGERRORc	[style=dotted, label="nested\nin\nremove"]
-ESIREMOVE	-> INTAGc
-ESIREMOVE	-> TAGERRORc
-ESIINCLUDE	-> INTAGc
-ESIINCLUDE	-> TAGERRORc
-ESIINCLUDE	-> TAGERRORc	[style=dotted, label="nested\nin\nremove"]
-ESIBOGON	-> TAGERRORc
-
-#################################################################
-# SECTION D
-
-INTAG		[shape=ellipse]
-TAGERROR	[shape=ellipse]
-NEXTTAGd	[shape=hexagon, label="NEXTTAG"]
-ATTRd		[shape=hexagon, label="ATTR"]
-D1		[shape=circle, label=""]
-D2		[shape=circle, label=""]
-INTAG		-> D1		[label="lws"]
-D1		-> D2		[label="/"]
-INTAG		-> D2		[label="/"]
-INTAG		-> NEXTTAGd	[label=">"]
-D1		-> NEXTTAGd	[label=">"]
-D2		-> NEXTTAGd	[label=">"]
-D1		-> ATTRd	[label="XMLstartchar"]
-D1		-> TAGERROR	[label="*"]
-D2		-> TAGERROR	[label="*"]
-TAGERROR	-> TAGERROR	[label="*"]
-TAGERROR	-> NEXTTAGd	[label="'>'"]
-
-#################################################################
-# SECTION E
-
-ATTR		[shape=ellipse]
-SKIPATTR	[shape=ellipse]
-ATTRGETVAL	[shape=ellipse]
-ATTRDELIM	[shape=ellipse]
-ATTRVAL		[shape=ellipse]
-TAGERRORe	[shape=hexagon, label="TAGERROR"]
-INTAGe		[shape=hexagon, label="INTAG"]
-ATTR		-> SKIPATTR	[label="*"]
-ATTR		-> ATTRGETVAL	[label="wanted attr"]
-SKIPATTR	-> SKIPATTR	[label="XMLname"]
-SKIPATTR	-> ATTRDELIM	[label="'='"]
-SKIPATTR	-> TAGERRORe	[label="*"]
-ATTRGETVAL	-> ATTRDELIM
-ATTRDELIM	-> ATTRVAL	[label="\""]
-ATTRDELIM	-> ATTRVAL	[label="\'"]
-ATTRDELIM	-> ATTRVAL	[label="*"]
-ATTRDELIM	-> TAGERRORe	[label="lws"]
-ATTRVAL		-> TAGERRORe	[label="'>'"]
-ATTRVAL		-> INTAGe	[label="delim"]
-ATTRVAL		-> ATTRVAL	[label="*"]
-
-}
-
-#endif
diff --git a/bin/varnishd/cache_expire.c b/bin/varnishd/cache_expire.c
deleted file mode 100644
index 23e3fc6..0000000
--- a/bin/varnishd/cache_expire.c
+++ /dev/null
@@ -1,490 +0,0 @@
-/*-
- * Copyright (c) 2006 Verdens Gang AS
- * Copyright (c) 2006-2011 Varnish Software AS
- * All rights reserved.
- *
- * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- *
- * LRU and object timer handling.
- *
- * We have two data structures, a LRU-list and a binary heap for the timers
- * and two ways to kill objects: TTL-timeouts and LRU cleanups.
- *
- * Any object on the LRU is also on the binheap and vice versa.
- *
- * We hold a single object reference for both data structures.
- *
- * An attempted overview:
- *
- *	                        EXP_Ttl()      EXP_Grace()   EXP_Keep()
- *				   |                |            |
- *      entered                    v                v            |
- *         |                       +--------------->+            |
- *         v                       |      grace                  |
- *         +---------------------->+                             |
- *                  ttl            |                             v
- *                                 +---------------------------->+
- *                                     keep
- *
- */
-
-#include "config.h"
-
-#include <math.h>
-
-#include "cache.h"
-
-#include "binary_heap.h"
-#include "hash/hash_slinger.h"
-#include "vtim.h"
-
-static pthread_t exp_thread;
-static struct binheap *exp_heap;
-static struct lock exp_mtx;
-
-/*--------------------------------------------------------------------
- * struct exp manipulations
- *
- * The Get/Set functions encapsulate the mutual magic between the
- * fields in one single place.
- */
-
-void
-EXP_Clr(struct exp *e)
-{
-
-	e->ttl = -1;
-	e->grace = -1;
-	e->keep = -1;
-	e->age = 0;
-	e->entered = 0;
-}
-
-#define EXP_ACCESS(fld, low_val, extra)				\
-	double							\
-	EXP_Get_##fld(const struct exp *e)			\
-	{							\
-		return (e->fld > 0. ? e->fld : low_val);	\
-	}							\
-								\
-	void							\
-	EXP_Set_##fld(struct exp *e, double v)			\
-	{							\
-		if (v > 0.)					\
-			e->fld = v;				\
-		else {						\
-			e->fld = -1.;				\
-			extra;					\
-		}						\
-	}							\
-
-EXP_ACCESS(ttl, -1., (e->grace = e->keep = -1.))
-EXP_ACCESS(grace, 0., )
-EXP_ACCESS(keep, 0.,)
-
-/*--------------------------------------------------------------------
- * Calculate an objects effective keep, grace or ttl time, suitably
- * adjusted for defaults and by per-session limits.
- */
-
-static double
-EXP_Keep(const struct sess *sp, const struct object *o)
-{
-	double r;
-
-	r = (double)cache_param->default_keep;
-	if (o->exp.keep > 0.)
-		r = o->exp.keep;
-	if (sp != NULL && sp->exp.keep > 0. && sp->exp.keep < r)
-		r = sp->exp.keep;
-	return (EXP_Ttl(sp, o) + r);
-}
-
-double
-EXP_Grace(const struct sess *sp, const struct object *o)
-{
-	double r;
-
-	r = (double)cache_param->default_grace;
-	if (o->exp.grace >= 0.)
-		r = o->exp.grace;
-	if (sp != NULL && sp->exp.grace > 0. && sp->exp.grace < r)
-		r = sp->exp.grace;
-	return (EXP_Ttl(sp, o) + r);
-}
-
-double
-EXP_Ttl(const struct sess *sp, const struct object *o)
-{
-	double r;
-
-	r = o->exp.ttl;
-	if (sp != NULL && sp->exp.ttl > 0. && sp->exp.ttl < r)
-		r = sp->exp.ttl;
-	return (o->exp.entered + r);
-}
-
-/*--------------------------------------------------------------------
- * When & why does the timer fire for this object ?
- */
-
-static int
-update_object_when(const struct object *o)
-{
-	struct objcore *oc;
-	double when, w2;
-
-	CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC);
-	oc = o->objcore;
-	CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
-	Lck_AssertHeld(&exp_mtx);
-
-	when = EXP_Keep(NULL, o);
-	w2 = EXP_Grace(NULL, o);
-	if (w2 > when)
-		when = w2;
-	assert(!isnan(when));
-	if (when == oc->timer_when)
-		return (0);
-	oc->timer_when = when;
-	return (1);
-}
-
-/*--------------------------------------------------------------------*/
-
-static void
-exp_insert(struct objcore *oc, struct lru *lru)
-{
-	CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
-	CHECK_OBJ_NOTNULL(lru, LRU_MAGIC);
-
-	Lck_AssertHeld(&lru->mtx);
-	Lck_AssertHeld(&exp_mtx);
-	assert(oc->timer_idx == BINHEAP_NOIDX);
-	binheap_insert(exp_heap, oc);
-	assert(oc->timer_idx != BINHEAP_NOIDX);
-	VTAILQ_INSERT_TAIL(&lru->lru_head, oc, lru_list);
-}
-
-/*--------------------------------------------------------------------
- * Object has been added to cache, record in lru & binheap.
- *
- * The objcore comes with a reference, which we inherit.
- */
-
-void
-EXP_Inject(struct objcore *oc, struct lru *lru, double when)
-{
-
-	CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
-	CHECK_OBJ_NOTNULL(lru, LRU_MAGIC);
-
-	Lck_Lock(&lru->mtx);
-	Lck_Lock(&exp_mtx);
-	oc->timer_when = when;
-	exp_insert(oc, lru);
-	Lck_Unlock(&exp_mtx);
-	Lck_Unlock(&lru->mtx);
-}
-
-/*--------------------------------------------------------------------
- * Object has been added to cache, record in lru & binheap.
- *
- * We grab a reference to the object, which will keep it around until
- * we decide its time to let it go.
- */
-
-void
-EXP_Insert(struct object *o)
-{
-	struct objcore *oc;
-	struct lru *lru;
-
-	CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC);
-	oc = o->objcore;
-	CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
-	AssertObjBusy(o);
-	HSH_Ref(oc);
-
-	assert(o->exp.entered != 0 && !isnan(o->exp.entered));
-	o->last_lru = o->exp.entered;
-
-	lru = oc_getlru(oc);
-	CHECK_OBJ_NOTNULL(lru, LRU_MAGIC);
-	Lck_Lock(&lru->mtx);
-	Lck_Lock(&exp_mtx);
-	(void)update_object_when(o);
-	exp_insert(oc, lru);
-	Lck_Unlock(&exp_mtx);
-	Lck_Unlock(&lru->mtx);
-	oc_updatemeta(oc);
-}
-
-/*--------------------------------------------------------------------
- * Object was used, move to tail of LRU list.
- *
- * To avoid the exp_mtx becoming a hotspot, we only attempt to move
- * objects if they have not been moved recently and if the lock is available.
- * This optimization obviously leaves the LRU list imperfectly sorted.
- */
-
-int
-EXP_Touch(struct objcore *oc)
-{
-	struct lru *lru;
-
-	CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
-
-	/*
-	 * For -spersistent we don't move objects on the lru list.  Each
-	 * segment has its own LRU list, and the order on it is not material
-	 * for anything.  The code below would move the objects to the
-	 * LRU list of the currently open segment, which would prevent
-	 * the cleaner from doing its job.
-	 */
-	if (oc->flags & OC_F_LRUDONTMOVE)
-		return (0);
-
-	lru = oc_getlru(oc);
-	CHECK_OBJ_NOTNULL(lru, LRU_MAGIC);
-
-	/*
-	 * We only need the LRU lock here.  The locking order is LRU->EXP
-	 * so we can trust the content of the oc->timer_idx without the
-	 * EXP lock.   Since each lru list has its own lock, this should
-	 * reduce contention a fair bit
-	 */
-	if (Lck_Trylock(&lru->mtx))
-		return (0);
-
-	if (oc->timer_idx != BINHEAP_NOIDX) {
-		VTAILQ_REMOVE(&lru->lru_head, oc, lru_list);
-		VTAILQ_INSERT_TAIL(&lru->lru_head, oc, lru_list);
-		VSC_C_main->n_lru_moved++;
-	}
-	Lck_Unlock(&lru->mtx);
-	return (1);
-}
-
-/*--------------------------------------------------------------------
- * We have changed one or more of the object timers, shuffle it
- * accordingly in the binheap
- *
- * The VCL code can send us here on a non-cached object, just return.
- *
- * XXX: special case check for ttl = 0 ?
- */
-
-void
-EXP_Rearm(const struct object *o)
-{
-	struct objcore *oc;
-	struct lru *lru;
-
-	CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC);
-	oc = o->objcore;
-	if (oc == NULL)
-		return;
-	CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
-	lru = oc_getlru(oc);
-	Lck_Lock(&lru->mtx);
-	Lck_Lock(&exp_mtx);
-	/*
-	 * The hang-man might have this object of the binheap while
-	 * tending to a timer.  If so, we do not muck with it here.
-	 */
-	if (oc->timer_idx != BINHEAP_NOIDX && update_object_when(o)) {
-		assert(oc->timer_idx != BINHEAP_NOIDX);
-		binheap_reorder(exp_heap, oc->timer_idx);
-		assert(oc->timer_idx != BINHEAP_NOIDX);
-	}
-	Lck_Unlock(&exp_mtx);
-	Lck_Unlock(&lru->mtx);
-	oc_updatemeta(oc);
-}
-
-/*--------------------------------------------------------------------
- * This thread monitors the root of the binary heap and whenever an
- * object expires, accounting also for graceability, it is killed.
- */
-
-static void * __match_proto__(void *start_routine(void *))
-exp_timer(struct sess *sp, void *priv)
-{
-	struct objcore *oc;
-	struct lru *lru;
-	double t;
-	struct object *o;
-
-	(void)priv;
-	t = VTIM_real();
-	oc = NULL;
-	while (1) {
-		if (oc == NULL) {
-			WSL_Flush(sp->wrk, 0);
-			WRK_SumStat(sp->wrk);
-			VTIM_sleep(cache_param->expiry_sleep);
-			t = VTIM_real();
-		}
-
-		Lck_Lock(&exp_mtx);
-		oc = binheap_root(exp_heap);
-		if (oc == NULL) {
-			Lck_Unlock(&exp_mtx);
-			continue;
-		}
-		CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
-
-		/*
-		 * We may have expired so many objects that our timestamp
-		 * got out of date, refresh it and check again.
-		 */
-		if (oc->timer_when > t)
-			t = VTIM_real();
-		if (oc->timer_when > t) {
-			Lck_Unlock(&exp_mtx);
-			oc = NULL;
-			continue;
-		}
-
-		/*
-		 * It's time...
-		 * Technically we should drop the exp_mtx, get the lru->mtx
-		 * get the exp_mtx again and then check that the oc is still
-		 * on the binheap.  We take the shorter route and try to
-		 * get the lru->mtx and punt if we fail.
-		 */
-
-		lru = oc_getlru(oc);
-		CHECK_OBJ_NOTNULL(lru, LRU_MAGIC);
-		if (Lck_Trylock(&lru->mtx)) {
-			Lck_Unlock(&exp_mtx);
-			oc = NULL;
-			continue;
-		}
-
-		/* Remove from binheap */
-		assert(oc->timer_idx != BINHEAP_NOIDX);
-		binheap_delete(exp_heap, oc->timer_idx);
-		assert(oc->timer_idx == BINHEAP_NOIDX);
-
-		/* And from LRU */
-		lru = oc_getlru(oc);
-		VTAILQ_REMOVE(&lru->lru_head, oc, lru_list);
-
-		Lck_Unlock(&exp_mtx);
-		Lck_Unlock(&lru->mtx);
-
-		VSC_C_main->n_expired++;
-
-		CHECK_OBJ_NOTNULL(oc->objhead, OBJHEAD_MAGIC);
-		o = oc_getobj(sp->wrk, oc);
-		WSL(sp->wrk, SLT_ExpKill, 0, "%u %.0f",
-		    o->xid, EXP_Ttl(NULL, o) - t);
-		(void)HSH_Deref(sp->wrk, oc, NULL);
-	}
-	NEEDLESS_RETURN(NULL);
-}
-
-/*--------------------------------------------------------------------
- * Attempt to make space by nuking the oldest object on the LRU list
- * which isn't in use.
- * Returns: 1: did, 0: didn't, -1: can't
- */
-
-int
-EXP_NukeOne(struct worker *w, struct lru *lru)
-{
-	struct objcore *oc;
-	struct object *o;
-
-	/* Find the first currently unused object on the LRU.  */
-	Lck_Lock(&lru->mtx);
-	Lck_Lock(&exp_mtx);
-	VTAILQ_FOREACH(oc, &lru->lru_head, lru_list) {
-		CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
-		assert (oc->timer_idx != BINHEAP_NOIDX);
-		/*
-		 * It wont release any space if we cannot release the last
-		 * reference, besides, if somebody else has a reference,
-		 * it's a bad idea to nuke this object anyway.
-		 */
-		if (oc->refcnt == 1)
-			break;
-	}
-	if (oc != NULL) {
-		VTAILQ_REMOVE(&lru->lru_head, oc, lru_list);
-		binheap_delete(exp_heap, oc->timer_idx);
-		assert(oc->timer_idx == BINHEAP_NOIDX);
-		VSC_C_main->n_lru_nuked++;
-	}
-	Lck_Unlock(&exp_mtx);
-	Lck_Unlock(&lru->mtx);
-
-	if (oc == NULL)
-		return (-1);
-
-	/* XXX: bad idea for -spersistent */
-	o = oc_getobj(w, oc);
-	WSL(w, SLT_ExpKill, 0, "%u LRU", o->xid);
-	(void)HSH_Deref(w, NULL, &o);
-	return (1);
-}
-
-/*--------------------------------------------------------------------
- * BinHeap helper functions for objcore.
- */
-
-static int
-object_cmp(void *priv, void *a, void *b)
-{
-	struct objcore *aa, *bb;
-
-	(void)priv;
-	CAST_OBJ_NOTNULL(aa, a, OBJCORE_MAGIC);
-	CAST_OBJ_NOTNULL(bb, b, OBJCORE_MAGIC);
-	return (aa->timer_when < bb->timer_when);
-}
-
-static void
-object_update(void *priv, void *p, unsigned u)
-{
-	struct objcore *oc;
-
-	(void)priv;
-	CAST_OBJ_NOTNULL(oc, p, OBJCORE_MAGIC);
-	oc->timer_idx = u;
-}
-
-/*--------------------------------------------------------------------*/
-
-void
-EXP_Init(void)
-{
-
-	Lck_New(&exp_mtx, lck_exp);
-	exp_heap = binheap_new(NULL, object_cmp, object_update);
-	XXXAN(exp_heap);
-	WRK_BgThread(&exp_thread, "cache-timeout", exp_timer, NULL);
-}
diff --git a/bin/varnishd/cache_fetch.c b/bin/varnishd/cache_fetch.c
deleted file mode 100644
index a678dcc..0000000
--- a/bin/varnishd/cache_fetch.c
+++ /dev/null
@@ -1,645 +0,0 @@
-/*-
- * Copyright (c) 2006 Verdens Gang AS
- * Copyright (c) 2006-2011 Varnish Software AS
- * All rights reserved.
- *
- * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- */
-
-#include "config.h"
-
-#include <inttypes.h>
-#include <stdio.h>
-#include <stdlib.h>
-
-#include "cache.h"
-
-#include "cache_backend.h"
-#include "vcli_priv.h"
-#include "vct.h"
-#include "vtcp.h"
-
-static unsigned fetchfrag;
-
-/*--------------------------------------------------------------------
- * We want to issue the first error we encounter on fetching and
- * supress the rest.  This function does that.
- *
- * Other code is allowed to look at w->fetch_failed to bail out
- *
- * For convenience, always return -1
- */
-
-int
-FetchError2(struct worker *w, const char *error, const char *more)
-{
-
-	CHECK_OBJ_NOTNULL(w, WORKER_MAGIC);
-	if (!w->fetch_failed) {
-		if (more == NULL)
-			WSLB(w, SLT_FetchError, "%s", error);
-		else
-			WSLB(w, SLT_FetchError, "%s: %s", error, more);
-	}
-	w->fetch_failed = 1;
-	return (-1);
-}
-
-int
-FetchError(struct worker *w, const char *error)
-{
-	return(FetchError2(w, error, NULL));
-}
-
-/*--------------------------------------------------------------------
- * VFP_NOP
- *
- * This fetch-processor does nothing but store the object.
- * It also documents the API
- */
-
-/*--------------------------------------------------------------------
- * VFP_BEGIN
- *
- * Called to set up stuff.
- *
- * 'estimate' is the estimate of the number of bytes we expect to receive,
- * as seen on the socket, or zero if unknown.
- */
-static void __match_proto__()
-vfp_nop_begin(struct worker *w, size_t estimate)
-{
-
-	if (estimate > 0)
-		(void)FetchStorage(w, estimate);
-}
-
-/*--------------------------------------------------------------------
- * VFP_BYTES
- *
- * Process (up to) 'bytes' from the socket.
- *
- * Return -1 on error, issue FetchError()
- *	will not be called again, once error happens.
- * Return 0 on EOF on socket even if bytes not reached.
- * Return 1 when 'bytes' have been processed.
- */
-
-static int __match_proto__()
-vfp_nop_bytes(struct worker *w, struct http_conn *htc, ssize_t bytes)
-{
-	ssize_t l, wl;
-	struct storage *st;
-
-	AZ(w->fetch_failed);
-	while (bytes > 0) {
-		st = FetchStorage(w, 0);
-		if (st == NULL)
-			return(-1);
-		l = st->space - st->len;
-		if (l > bytes)
-			l = bytes;
-		wl = HTC_Read(w, htc, st->ptr + st->len, l);
-		if (wl <= 0)
-			return (wl);
-		st->len += wl;
-		w->fetch_obj->len += wl;
-		bytes -= wl;
-		if (w->do_stream)
-			RES_StreamPoll(w);
-	}
-	return (1);
-}
-
-/*--------------------------------------------------------------------
- * VFP_END
- *
- * Finish & cleanup
- *
- * Return -1 for error
- * Return 0 for OK
- */
-
-static int __match_proto__()
-vfp_nop_end(struct worker *w)
-{
-	struct storage *st;
-
-	st = VTAILQ_LAST(&w->fetch_obj->store, storagehead);
-	if (st == NULL)
-		return (0);
-
-	if (st->len == 0) {
-		VTAILQ_REMOVE(&w->fetch_obj->store, st, list);
-		STV_free(st);
-		return (0);
-	}
-	if (st->len < st->space)
-		STV_trim(st, st->len);
-	return (0);
-}
-
-static struct vfp vfp_nop = {
-	.begin	=	vfp_nop_begin,
-	.bytes	=	vfp_nop_bytes,
-	.end	=	vfp_nop_end,
-};
-
-/*--------------------------------------------------------------------
- * Fetch Storage to put object into.
- *
- */
-
-struct storage *
-FetchStorage(struct worker *w, ssize_t sz)
-{
-	ssize_t l;
-	struct storage *st;
-	struct object *obj;
-
-	obj = w->fetch_obj;
-	CHECK_OBJ_NOTNULL(obj, OBJECT_MAGIC);
-	st = VTAILQ_LAST(&obj->store, storagehead);
-	if (st != NULL && st->len < st->space)
-		return (st);
-
-	l = fetchfrag;
-	if (l == 0)
-		l = sz;
-	if (l == 0)
-		l = cache_param->fetch_chunksize * 1024LL;
-	st = STV_alloc(w, l);
-	if (st == NULL) {
-		(void)FetchError(w, "Could not get storage");
-		return (NULL);
-	}
-	AZ(st->len);
-	VTAILQ_INSERT_TAIL(&obj->store, st, list);
-	return (st);
-}
-
-/*--------------------------------------------------------------------
- * Convert a string to a size_t safely
- */
-
-static ssize_t
-fetch_number(const char *nbr, int radix)
-{
-	uintmax_t cll;
-	ssize_t cl;
-	char *q;
-
-	if (*nbr == '\0')
-		return (-1);
-	cll = strtoumax(nbr, &q, radix);
-	if (q == NULL || *q != '\0')
-		return (-1);
-
-	cl = (ssize_t)cll;
-	if((uintmax_t)cl != cll) /* Protect against bogusly large values */
-		return (-1);
-	return (cl);
-}
-
-/*--------------------------------------------------------------------*/
-
-static int
-fetch_straight(struct worker *w, struct http_conn *htc, ssize_t cl)
-{
-	int i;
-
-	assert(w->body_status == BS_LENGTH);
-
-	if (cl < 0) {
-		return (FetchError(w, "straight length field bogus"));
-	} else if (cl == 0)
-		return (0);
-
-	i = w->vfp->bytes(w, htc, cl);
-	if (i <= 0)
-		return (FetchError(w, "straight insufficient bytes"));
-	return (0);
-}
-
-/*--------------------------------------------------------------------
- * Read a chunked HTTP object.
- *
- * XXX: Reading one byte at a time is pretty pessimal.
- */
-
-static int
-fetch_chunked(struct worker *w, struct http_conn *htc)
-{
-	int i;
-	char buf[20];		/* XXX: 20 is arbitrary */
-	unsigned u;
-	ssize_t cl;
-
-	assert(w->body_status == BS_CHUNKED);
-	do {
-		/* Skip leading whitespace */
-		do {
-			if (HTC_Read(w, htc, buf, 1) <= 0)
-				return (-1);
-		} while (vct_islws(buf[0]));
-
-		if (!vct_ishex(buf[0]))
-			return (FetchError(w,"chunked header non-hex"));
-
-		/* Collect hex digits, skipping leading zeros */
-		for (u = 1; u < sizeof buf; u++) {
-			do {
-				if (HTC_Read(w, htc, buf + u, 1) <= 0)
-					return (-1);
-			} while (u == 1 && buf[0] == '0' && buf[u] == '0');
-			if (!vct_ishex(buf[u]))
-				break;
-		}
-
-		if (u >= sizeof buf)
-			return (FetchError(w,"chunked header too long"));
-
-		/* Skip trailing white space */
-		while(vct_islws(buf[u]) && buf[u] != '\n')
-			if (HTC_Read(w, htc, buf + u, 1) <= 0)
-				return (-1);
-
-		if (buf[u] != '\n')
-			return (FetchError(w,"chunked header no NL"));
-
-		buf[u] = '\0';
-		cl = fetch_number(buf, 16);
-		if (cl < 0)
-			return (FetchError(w,"chunked header number syntax"));
-
-		if (cl > 0 && w->vfp->bytes(w, htc, cl) <= 0)
-			return (-1);
-
-		i = HTC_Read(w, htc, buf, 1);
-		if (i <= 0)
-			return (-1);
-		if (buf[0] == '\r' && HTC_Read(w, htc, buf, 1) <= 0)
-			return (-1);
-		if (buf[0] != '\n')
-			return (FetchError(w,"chunked tail no NL"));
-	} while (cl > 0);
-	return (0);
-}
-
-/*--------------------------------------------------------------------*/
-
-static int
-fetch_eof(struct worker *w, struct http_conn *htc)
-{
-	int i;
-
-	assert(w->body_status == BS_EOF);
-	i = w->vfp->bytes(w, htc, SSIZE_MAX);
-	if (i < 0)
-		return (-1);
-	return (0);
-}
-
-/*--------------------------------------------------------------------
- * Fetch any body attached to the incoming request, and either write it
- * to the backend (if we pass) or discard it (anything else).
- * This is mainly a separate function to isolate the stack buffer and
- * to contain the complexity when we start handling chunked encoding.
- */
-
-int
-FetchReqBody(struct sess *sp)
-{
-	unsigned long content_length;
-	char buf[8192];
-	char *ptr, *endp;
-	int rdcnt;
-
-	if (http_GetHdr(sp->http, H_Content_Length, &ptr)) {
-
-		content_length = strtoul(ptr, &endp, 10);
-		/* XXX should check result of conversion */
-		while (content_length) {
-			if (content_length > sizeof buf)
-				rdcnt = sizeof buf;
-			else
-				rdcnt = content_length;
-			rdcnt = HTC_Read(sp->wrk, sp->htc, buf, rdcnt);
-			if (rdcnt <= 0)
-				return (1);
-			content_length -= rdcnt;
-			if (!sp->sendbody)
-				continue;
-			(void)WRW_Write(sp->wrk, buf, rdcnt); /* XXX: stats ? */
-			if (WRW_Flush(sp->wrk))
-				return (2);
-		}
-	}
-	if (http_GetHdr(sp->http, H_Transfer_Encoding, NULL)) {
-		/* XXX: Handle chunked encoding. */
-		WSP(sp, SLT_Debug, "Transfer-Encoding in request");
-		return (1);
-	}
-	return (0);
-}
-
-/*--------------------------------------------------------------------
- * Send request, and receive the HTTP protocol response, but not the
- * response body.
- *
- * Return value:
- *	-1 failure, not retryable
- *	 0 success
- *	 1 failure which can be retried.
- */
-
-int
-FetchHdr(struct sess *sp)
-{
-	struct vbc *vc;
-	struct worker *w;
-	char *b;
-	struct http *hp;
-	int retry = -1;
-	int i;
-
-	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
-	CHECK_OBJ_NOTNULL(sp->wrk, WORKER_MAGIC);
-	w = sp->wrk;
-
-	AN(sp->director);
-	AZ(sp->obj);
-
-	if (sp->objcore != NULL) {		/* pass has no objcore */
-		CHECK_OBJ_NOTNULL(sp->objcore, OBJCORE_MAGIC);
-		AN(sp->objcore->flags & OC_F_BUSY);
-	}
-
-	hp = w->bereq;
-
-	sp->wrk->vbc = VDI_GetFd(NULL, sp);
-	if (sp->wrk->vbc == NULL) {
-		WSP(sp, SLT_FetchError, "no backend connection");
-		return (-1);
-	}
-	vc = sp->wrk->vbc;
-	if (vc->recycled)
-		retry = 1;
-
-	/*
-	 * Now that we know our backend, we can set a default Host:
-	 * header if one is necessary.  This cannot be done in the VCL
-	 * because the backend may be chosen by a director.
-	 */
-	if (!http_GetHdr(hp, H_Host, &b))
-		VDI_AddHostHeader(sp);
-
-	(void)VTCP_blocking(vc->fd);	/* XXX: we should timeout instead */
-	WRW_Reserve(w, &vc->fd);
-	(void)http_Write(w, vc->vsl_id, hp, 0);	/* XXX: stats ? */
-
-	/* Deal with any message-body the request might have */
-	i = FetchReqBody(sp);
-	if (WRW_FlushRelease(w) || i > 0) {
-		WSP(sp, SLT_FetchError, "backend write error: %d (%s)",
-		    errno, strerror(errno));
-		VDI_CloseFd(sp->wrk);
-		/* XXX: other cleanup ? */
-		return (retry);
-	}
-
-	/* Checkpoint the vsl.here */
-	WSL_Flush(w, 0);
-
-	/* XXX is this the right place? */
-	VSC_C_main->backend_req++;
-
-	/* Receive response */
-
-	HTC_Init(w->htc, w->ws, vc->fd, vc->vsl_id, cache_param->http_resp_size,
-	    cache_param->http_resp_hdr_len);
-
-	VTCP_set_read_timeout(vc->fd, vc->first_byte_timeout);
-
-	i = HTC_Rx(w->htc);
-
-	if (i < 0) {
-		WSP(sp, SLT_FetchError, "http first read error: %d %d (%s)",
-		    i, errno, strerror(errno));
-		VDI_CloseFd(sp->wrk);
-		/* XXX: other cleanup ? */
-		/* Retryable if we never received anything */
-		return (i == -1 ? retry : -1);
-	}
-
-	VTCP_set_read_timeout(vc->fd, vc->between_bytes_timeout);
-
-	while (i == 0) {
-		i = HTC_Rx(w->htc);
-		if (i < 0) {
-			WSP(sp, SLT_FetchError,
-			    "http first read error: %d %d (%s)",
-			    i, errno, strerror(errno));
-			VDI_CloseFd(sp->wrk);
-			/* XXX: other cleanup ? */
-			return (-1);
-		}
-	}
-
-	hp = w->beresp;
-
-	if (http_DissectResponse(w, w->htc, hp)) {
-		WSP(sp, SLT_FetchError, "http format error");
-		VDI_CloseFd(sp->wrk);
-		/* XXX: other cleanup ? */
-		return (-1);
-	}
-	return (0);
-}
-
-/*--------------------------------------------------------------------*/
-
-int
-FetchBody(struct worker *w, struct object *obj)
-{
-	int cls;
-	struct storage *st;
-	int mklen;
-	ssize_t cl;
-
-	CHECK_OBJ_NOTNULL(w, WORKER_MAGIC);
-	AZ(w->fetch_obj);
-	CHECK_OBJ_NOTNULL(w->vbc, VBC_MAGIC);
-	CHECK_OBJ_NOTNULL(obj, OBJECT_MAGIC);
-	CHECK_OBJ_NOTNULL(obj->http, HTTP_MAGIC);
-
-	if (w->vfp == NULL)
-		w->vfp = &vfp_nop;
-
-	AssertObjCorePassOrBusy(obj->objcore);
-
-	AZ(w->vgz_rx);
-	AZ(VTAILQ_FIRST(&obj->store));
-
-	w->fetch_obj = obj;
-	w->fetch_failed = 0;
-
-	/* XXX: pick up estimate from objdr ? */
-	cl = 0;
-	switch (w->body_status) {
-	case BS_NONE:
-		cls = 0;
-		mklen = 0;
-		break;
-	case BS_ZERO:
-		cls = 0;
-		mklen = 1;
-		break;
-	case BS_LENGTH:
-		cl = fetch_number( w->h_content_length, 10);
-		w->vfp->begin(w, cl > 0 ? cl : 0);
-		cls = fetch_straight(w, w->htc, cl);
-		mklen = 1;
-		if (w->vfp->end(w))
-			cls = -1;
-		break;
-	case BS_CHUNKED:
-		w->vfp->begin(w, cl);
-		cls = fetch_chunked(w, w->htc);
-		mklen = 1;
-		if (w->vfp->end(w))
-			cls = -1;
-		break;
-	case BS_EOF:
-		w->vfp->begin(w, cl);
-		cls = fetch_eof(w, w->htc);
-		mklen = 1;
-		if (w->vfp->end(w))
-			cls = -1;
-		break;
-	case BS_ERROR:
-		cls = 1;
-		mklen = 0;
-		break;
-	default:
-		cls = 0;
-		mklen = 0;
-		INCOMPL();
-	}
-	AZ(w->vgz_rx);
-
-	/*
-	 * It is OK for ->end to just leave the last storage segment
-	 * sitting on w->storage, we will always call vfp_nop_end()
-	 * to get it trimmed or thrown out if empty.
-	 */
-	AZ(vfp_nop_end(w));
-
-	w->fetch_obj = NULL;
-
-	WSLB(w, SLT_Fetch_Body, "%u(%s) cls %d mklen %u",
-	    w->body_status, body_status(w->body_status),
-	    cls, mklen);
-
-	if (w->body_status == BS_ERROR) {
-		VDI_CloseFd(w);
-		return (__LINE__);
-	}
-
-	if (cls < 0) {
-		w->stats.fetch_failed++;
-		/* XXX: Wouldn't this store automatically be released ? */
-		while (!VTAILQ_EMPTY(&obj->store)) {
-			st = VTAILQ_FIRST(&obj->store);
-			VTAILQ_REMOVE(&obj->store, st, list);
-			STV_free(st);
-		}
-		VDI_CloseFd(w);
-		obj->len = 0;
-		return (__LINE__);
-	}
-	AZ(w->fetch_failed);
-
-	if (cls == 0 && w->do_close)
-		cls = 1;
-
-	WSLB(w, SLT_Length, "%u", obj->len);
-
-	{
-	/* Sanity check fetch methods accounting */
-		ssize_t uu;
-
-		uu = 0;
-		VTAILQ_FOREACH(st, &obj->store, list)
-			uu += st->len;
-		if (w->do_stream)
-			/* Streaming might have started freeing stuff */
-			assert (uu <= obj->len);
-
-		else
-			assert(uu == obj->len);
-	}
-
-	if (mklen > 0) {
-		http_Unset(obj->http, H_Content_Length);
-		http_PrintfHeader(w, w->vbc->vsl_id, obj->http,
-		    "Content-Length: %jd", (intmax_t)obj->len);
-	}
-
-	if (cls)
-		VDI_CloseFd(w);
-	else
-		VDI_RecycleFd(w);
-
-	return (0);
-}
-
-/*--------------------------------------------------------------------
- * Debugging aids
- */
-
-static void
-debug_fragfetch(struct cli *cli, const char * const *av, void *priv)
-{
-	(void)priv;
-	(void)cli;
-	fetchfrag = strtoul(av[2], NULL, 0);
-}
-
-static struct cli_proto debug_cmds[] = {
-	{ "debug.fragfetch", "debug.fragfetch",
-		"\tEnable fetch fragmentation\n", 1, 1, "d", debug_fragfetch },
-	{ NULL }
-};
-
-/*--------------------------------------------------------------------
- *
- */
-
-void
-Fetch_Init(void)
-{
-
-	CLI_AddFuncs(debug_cmds);
-}
diff --git a/bin/varnishd/cache_gzip.c b/bin/varnishd/cache_gzip.c
deleted file mode 100644
index 32a7413..0000000
--- a/bin/varnishd/cache_gzip.c
+++ /dev/null
@@ -1,697 +0,0 @@
-/*-
- * Copyright (c) 2011 Varnish Software AS
- * All rights reserved.
- *
- * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- *
- * Interaction with the linvgz (zlib) library.
- *
- * The zlib library pollutes namespace a LOT when you include the "vgz.h"
- * (aka (zlib.h") file so we contain the damage by vectoring all access
- * to libz through this source file.
- *
- * The API defined by this file, will also insulate the rest of the code,
- * should we find a better gzip library at a later date.
- *
- * The absolutely worst case gzip processing path, once we have pipe-lining,
- * will be the following, so we need to be a bit careful with the scratch
- * space we use:
- *
- *	Backend		Tmp	Input	Output
- *         |		----------------------
- *	   v
- *	 gunzip		wrk	stack	?
- *         |
- *	   v
- *	  esi
- *         |
- *	   v
- *	  gzip		wrk	?	storage
- *         |
- *	   v
- *	  cache
- *         |
- *	   v
- *	 gunzip		wrk	storage	stack
- *         |
- *	   v
- *	 client
- *
- * XXXX: The two '?' are obviously the same memory, but I have yet to decide
- * where it goes.   As usual we try to avoid the session->ws if we can but
- * I may have to use that.
- *
- */
-
-#include "config.h"
-
-#include <stdio.h>
-#include <stdlib.h>
-
-#include "cache.h"
-
-#include "vgz.h"
-
-struct vgz {
-	unsigned		magic;
-#define VGZ_MAGIC		0x162df0cb
-	enum {VGZ_GZ,VGZ_UN}	dir;
-	struct worker		*wrk;
-	const char		*id;
-	struct ws		*tmp;
-	char			*tmp_snapshot;
-	int			last_i;
-
-	struct storage		*obuf;
-
-	z_stream		vz;
-};
-
-/*--------------------------------------------------------------------*/
-
-static voidpf
-vgz_alloc(voidpf opaque, uInt items, uInt size)
-{
-	struct vgz *vg;
-
-	CAST_OBJ_NOTNULL(vg, opaque, VGZ_MAGIC);
-
-	return (WS_Alloc(vg->tmp, items * size));
-}
-
-static void
-vgz_free(voidpf opaque, voidpf address)
-{
-	struct vgz *vg;
-
-	CAST_OBJ_NOTNULL(vg, opaque, VGZ_MAGIC);
-	(void)address;
-}
-
-/*--------------------------------------------------------------------
- * Set up a gunzip instance
- */
-
-static struct vgz *
-vgz_alloc_vgz(struct worker *wrk, const char *id)
-{
-	struct vgz *vg;
-	struct ws *ws;
-
-	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
-	ws = wrk->ws;
-	WS_Assert(ws);
-	// XXX: we restore workspace in esi:include
-	// vg = (void*)WS_Alloc(ws, sizeof *vg);
-	ALLOC_OBJ(vg, VGZ_MAGIC);
-	AN(vg);
-	memset(vg, 0, sizeof *vg);
-	vg->magic = VGZ_MAGIC;
-	vg->wrk = wrk;
-	vg->id = id;
-
-	switch (cache_param->gzip_tmp_space) {
-	case 0:
-	case 1:
-		/* malloc, the default */
-		break;
-	case 2:
-		vg->tmp = wrk->ws;
-		vg->tmp_snapshot = WS_Snapshot(vg->tmp);
-		vg->vz.zalloc = vgz_alloc;
-		vg->vz.zfree = vgz_free;
-		vg->vz.opaque = vg;
-		break;
-	default:
-		assert(0 == __LINE__);
-	}
-	return (vg);
-}
-
-struct vgz *
-VGZ_NewUngzip(struct worker *wrk, const char *id)
-{
-	struct vgz *vg;
-
-	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
-	vg = vgz_alloc_vgz(wrk, id);
-	vg->dir = VGZ_UN;
-	VSC_C_main->n_gunzip++;
-
-	/*
-	 * Max memory usage according to zonf.h:
-	 *	mem_needed = "a few kb" + (1 << (windowBits))
-	 * Since we don't control windowBits, we have to assume
-	 * it is 15, so 34-35KB or so.
-	 */
-	assert(Z_OK == inflateInit2(&vg->vz, 31));
-	return (vg);
-}
-
-struct vgz *
-VGZ_NewGzip(struct worker *wrk, const char *id)
-{
-	struct vgz *vg;
-	int i;
-
-	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
-	vg = vgz_alloc_vgz(wrk, id);
-	vg->dir = VGZ_GZ;
-	VSC_C_main->n_gzip++;
-
-	/*
-	 * From zconf.h:
-	 *
-	 *	mem_needed = "a few kb"
-	 *		+ (1 << (windowBits+2))
-	 *		+  (1 << (memLevel+9))
-	 *
-	 * windowBits [8..15] (-> 1K..128K)
-	 * memLevel [1..9] (-> 1K->256K)
-	 *
-	 * XXX: They probably needs to be params...
-	 *
-	 * XXX: It may be more efficent to malloc them, rather than have
-	 * XXX: too many worker threads grow the stacks.
-	 */
-	i = deflateInit2(&vg->vz,
-	    cache_param->gzip_level,		/* Level */
-	    Z_DEFLATED,			/* Method */
-	    16 + cache_param->gzip_window,	/* Window bits (16=gzip + 15) */
-	    cache_param->gzip_memlevel,	/* memLevel */
-	    Z_DEFAULT_STRATEGY);
-	assert(Z_OK == i);
-	return (vg);
-}
-
-/*--------------------------------------------------------------------*/
-
-void
-VGZ_Ibuf(struct vgz *vg, const void *ptr, ssize_t len)
-{
-
-	CHECK_OBJ_NOTNULL(vg, VGZ_MAGIC);
-
-	AZ(vg->vz.avail_in);
-	vg->vz.next_in = TRUST_ME(ptr);
-	vg->vz.avail_in = len;
-}
-
-int
-VGZ_IbufEmpty(const struct vgz *vg)
-{
-
-	CHECK_OBJ_NOTNULL(vg, VGZ_MAGIC);
-	return (vg->vz.avail_in == 0);
-}
-
-/*--------------------------------------------------------------------*/
-
-void
-VGZ_Obuf(struct vgz *vg, void *ptr, ssize_t len)
-{
-
-	CHECK_OBJ_NOTNULL(vg, VGZ_MAGIC);
-
-	vg->vz.next_out = TRUST_ME(ptr);
-	vg->vz.avail_out = len;
-}
-
-int
-VGZ_ObufFull(const struct vgz *vg)
-{
-
-	CHECK_OBJ_NOTNULL(vg, VGZ_MAGIC);
-	return (vg->vz.avail_out == 0);
-}
-
-/*--------------------------------------------------------------------
- * Keep the outbuffer supplied with storage and file it under the
- * sp->obj as it fills.
- */
-
-int
-VGZ_ObufStorage(struct worker *w, struct vgz *vg)
-{
-	struct storage *st;
-
-	st = FetchStorage(w, 0);
-	if (st == NULL)
-		return (-1);
-
-	vg->obuf = st;
-	VGZ_Obuf(vg, st->ptr + st->len, st->space - st->len);
-
-	return (0);
-}
-
-/*--------------------------------------------------------------------*/
-
-int
-VGZ_Gunzip(struct vgz *vg, const void **pptr, size_t *plen)
-{
-	int i;
-	ssize_t l;
-	const uint8_t *before;
-
-	CHECK_OBJ_NOTNULL(vg, VGZ_MAGIC);
-
-	*pptr = NULL;
-	*plen = 0;
-	AN(vg->vz.next_out);
-	AN(vg->vz.avail_out);
-	before = vg->vz.next_out;
-	i = inflate(&vg->vz, 0);
-	if (i == Z_OK || i == Z_STREAM_END) {
-		*pptr = before;
-		l = (const uint8_t *)vg->vz.next_out - before;
-		*plen = l;
-		if (vg->obuf != NULL)
-			vg->obuf->len += l;
-	}
-	vg->last_i = i;
-	if (i == Z_OK)
-		return (VGZ_OK);
-	if (i == Z_STREAM_END)
-		return (VGZ_END);
-	if (i == Z_BUF_ERROR)
-		return (VGZ_STUCK);
-	VSL(SLT_Debug, 0, "Unknown INFLATE=%d (%s)\n", i, vg->vz.msg);
-	return (VGZ_ERROR);
-}
-
-/*--------------------------------------------------------------------*/
-
-int
-VGZ_Gzip(struct vgz *vg, const void **pptr, size_t *plen, enum vgz_flag flags)
-{
-	int i;
-	int zflg;
-	ssize_t l;
-	const uint8_t *before;
-
-	CHECK_OBJ_NOTNULL(vg, VGZ_MAGIC);
-
-	*pptr = NULL;
-	*plen = 0;
-	AN(vg->vz.next_out);
-	AN(vg->vz.avail_out);
-	before = vg->vz.next_out;
-	switch(flags) {
-	case VGZ_NORMAL:	zflg = Z_NO_FLUSH; break;
-	case VGZ_ALIGN:		zflg = Z_SYNC_FLUSH; break;
-	case VGZ_RESET:		zflg = Z_FULL_FLUSH; break;
-	case VGZ_FINISH:	zflg = Z_FINISH; break;
-	default:		INCOMPL();
-	}
-	i = deflate(&vg->vz, zflg);
-	if (i == Z_OK || i == Z_STREAM_END) {
-		*pptr = before;
-		l = (const uint8_t *)vg->vz.next_out - before;
-		*plen = l;
-		if (vg->obuf != NULL)
-			vg->obuf->len += l;
-	}
-	vg->last_i = i;
-	if (i == Z_OK)
-		return (0);
-	if (i == Z_STREAM_END)
-		return (1);
-	if (i == Z_BUF_ERROR)
-		return (2);
-	return (-1);
-}
-
-/*--------------------------------------------------------------------
- * Gunzip ibuf into outb, if it runs full, emit it with WRW.
- * Leave flushing to caller, more data may be coming.
- */
-
-int
-VGZ_WrwGunzip(struct worker *w, struct vgz *vg, const void *ibuf,
-    ssize_t ibufl, char *obuf, ssize_t obufl, ssize_t *obufp)
-{
-	int i;
-	size_t dl;
-	const void *dp;
-
-	CHECK_OBJ_NOTNULL(vg, VGZ_MAGIC);
-	assert(obufl > 16);
-	VGZ_Ibuf(vg, ibuf, ibufl);
-	if (ibufl == 0)
-		return (VGZ_OK);
-	VGZ_Obuf(vg, obuf + *obufp, obufl - *obufp);
-	do {
-		if (obufl == *obufp)
-			i = VGZ_STUCK;
-		else {
-			i = VGZ_Gunzip(vg, &dp, &dl);
-			*obufp += dl;
-		}
-		if (i < VGZ_OK) {
-			/* XXX: VSL ? */
-			return (-1);
-		}
-		if (obufl == *obufp || i == VGZ_STUCK) {
-			w->acct_tmp.bodybytes += *obufp;
-			(void)WRW_Write(w, obuf, *obufp);
-			(void)WRW_Flush(w);
-			*obufp = 0;
-			VGZ_Obuf(vg, obuf + *obufp, obufl - *obufp);
-		}
-	} while (!VGZ_IbufEmpty(vg));
-	if (i == VGZ_STUCK)
-		i = VGZ_OK;
-	return (i);
-}
-
-/*--------------------------------------------------------------------*/
-
-void
-VGZ_UpdateObj(const struct vgz *vg, struct object *obj)
-{
-
-	CHECK_OBJ_NOTNULL(vg, VGZ_MAGIC);
-	CHECK_OBJ_NOTNULL(obj, OBJECT_MAGIC);
-	obj->gzip_start	= vg->vz.start_bit;
-	obj->gzip_last	= vg->vz.last_bit;
-	obj->gzip_stop	= vg->vz.stop_bit;
-}
-
-/*--------------------------------------------------------------------
- * Passing a vsl_id of -1 means "use w->vbc->vsl_id"
- */
-
-int
-VGZ_Destroy(struct vgz **vgp, int vsl_id)
-{
-	struct vgz *vg;
-	int i;
-
-	vg = *vgp;
-	CHECK_OBJ_NOTNULL(vg, VGZ_MAGIC);
-	*vgp = NULL;
-
-	if (vsl_id < 0)
-		WSLB(vg->wrk, SLT_Gzip, "%s %jd %jd %jd %jd %jd",
-		    vg->id,
-		    (intmax_t)vg->vz.total_in,
-		    (intmax_t)vg->vz.total_out,
-		    (intmax_t)vg->vz.start_bit,
-		    (intmax_t)vg->vz.last_bit,
-		    (intmax_t)vg->vz.stop_bit);
-	else
-		WSL(vg->wrk, SLT_Gzip, vsl_id, "%s %jd %jd %jd %jd %jd",
-		    vg->id,
-		    (intmax_t)vg->vz.total_in,
-		    (intmax_t)vg->vz.total_out,
-		    (intmax_t)vg->vz.start_bit,
-		    (intmax_t)vg->vz.last_bit,
-		    (intmax_t)vg->vz.stop_bit);
-	if (vg->tmp != NULL)
-		WS_Reset(vg->tmp, vg->tmp_snapshot);
-	if (vg->dir == VGZ_GZ)
-		i = deflateEnd(&vg->vz);
-	else
-		i = inflateEnd(&vg->vz);
-	if (vg->last_i == Z_STREAM_END && i == Z_OK)
-		i = Z_STREAM_END;
-	FREE_OBJ(vg);
-	if (i == Z_OK)
-		return (VGZ_OK);
-	if (i == Z_STREAM_END)
-		return (VGZ_END);
-	if (i == Z_BUF_ERROR)
-		return (VGZ_STUCK);
-	return (VGZ_ERROR);
-}
-
-/*--------------------------------------------------------------------
- * VFP_GUNZIP
- *
- * A VFP for gunzip'ing an object as we receive it from the backend
- */
-
-static void __match_proto__()
-vfp_gunzip_begin(struct worker *w, size_t estimate)
-{
-	(void)estimate;
-	AZ(w->vgz_rx);
-	w->vgz_rx = VGZ_NewUngzip(w, "U F -");
-}
-
-static int __match_proto__()
-vfp_gunzip_bytes(struct worker *w, struct http_conn *htc, ssize_t bytes)
-{
-	struct vgz *vg;
-	ssize_t l, wl;
-	int i = -100;
-	uint8_t	ibuf[cache_param->gzip_stack_buffer];
-	size_t dl;
-	const void *dp;
-
-	AZ(w->fetch_failed);
-	vg = w->vgz_rx;
-	CHECK_OBJ_NOTNULL(vg, VGZ_MAGIC);
-	AZ(vg->vz.avail_in);
-	while (bytes > 0 || vg->vz.avail_in > 0) {
-		if (vg->vz.avail_in == 0 && bytes > 0) {
-			l = sizeof ibuf;
-			if (l > bytes)
-				l = bytes;
-			wl = HTC_Read(w, htc, ibuf, l);
-			if (wl <= 0)
-				return (wl);
-			VGZ_Ibuf(vg, ibuf, wl);
-			bytes -= wl;
-		}
-
-		if (VGZ_ObufStorage(w, vg))
-			return(-1);
-		i = VGZ_Gunzip(vg, &dp, &dl);
-		if (i != VGZ_OK && i != VGZ_END)
-			return(FetchError(w, "Gunzip data error"));
-		w->fetch_obj->len += dl;
-		if (w->do_stream)
-			RES_StreamPoll(w);
-	}
-	assert(i == Z_OK || i == Z_STREAM_END);
-	return (1);
-}
-
-static int __match_proto__()
-vfp_gunzip_end(struct worker *w)
-{
-	struct vgz *vg;
-
-	vg = w->vgz_rx;
-	w->vgz_rx = NULL;
-	CHECK_OBJ_NOTNULL(vg, VGZ_MAGIC);
-	if (w->fetch_failed) {
-		(void)VGZ_Destroy(&vg, -1);
-		return(0);
-	}
-	if (VGZ_Destroy(&vg, -1) != VGZ_END)
-		return(FetchError(w, "Gunzip error at the very end"));
-	return (0);
-}
-
-struct vfp vfp_gunzip = {
-        .begin  =       vfp_gunzip_begin,
-        .bytes  =       vfp_gunzip_bytes,
-        .end    =       vfp_gunzip_end,
-};
-
-
-/*--------------------------------------------------------------------
- * VFP_GZIP
- *
- * A VFP for gzip'ing an object as we receive it from the backend
- */
-
-static void __match_proto__()
-vfp_gzip_begin(struct worker *w, size_t estimate)
-{
-	(void)estimate;
-
-	AZ(w->vgz_rx);
-	w->vgz_rx = VGZ_NewGzip(w, "G F -");
-}
-
-static int __match_proto__()
-vfp_gzip_bytes(struct worker *w, struct http_conn *htc, ssize_t bytes)
-{
-	struct vgz *vg;
-	ssize_t l, wl;
-	int i = -100;
-	uint8_t ibuf[cache_param->gzip_stack_buffer];
-	size_t dl;
-	const void *dp;
-
-	AZ(w->fetch_failed);
-	vg = w->vgz_rx;
-	CHECK_OBJ_NOTNULL(vg, VGZ_MAGIC);
-	AZ(vg->vz.avail_in);
-	while (bytes > 0 || !VGZ_IbufEmpty(vg)) {
-		if (VGZ_IbufEmpty(vg) && bytes > 0) {
-			l = sizeof ibuf;
-			if (l > bytes)
-				l = bytes;
-			wl = HTC_Read(w, htc, ibuf, l);
-			if (wl <= 0)
-				return (wl);
-			VGZ_Ibuf(vg, ibuf, wl);
-			bytes -= wl;
-		}
-		if (VGZ_ObufStorage(w, vg))
-			return(-1);
-		i = VGZ_Gzip(vg, &dp, &dl, VGZ_NORMAL);
-		assert(i == Z_OK);
-		w->fetch_obj->len += dl;
-		if (w->do_stream)
-			RES_StreamPoll(w);
-	}
-	return (1);
-}
-
-static int __match_proto__()
-vfp_gzip_end(struct worker *w)
-{
-	struct vgz *vg;
-	size_t dl;
-	const void *dp;
-	int i;
-
-	vg = w->vgz_rx;
-	CHECK_OBJ_NOTNULL(vg, VGZ_MAGIC);
-	w->vgz_rx = NULL;
-	if (w->fetch_failed) {
-		(void)VGZ_Destroy(&vg, -1);
-		return(0);
-	}
-	do {
-		VGZ_Ibuf(vg, "", 0);
-		if (VGZ_ObufStorage(w, vg))
-			return(-1);
-		i = VGZ_Gzip(vg, &dp, &dl, VGZ_FINISH);
-		w->fetch_obj->len += dl;
-	} while (i != Z_STREAM_END);
-	if (w->do_stream)
-		RES_StreamPoll(w);
-	VGZ_UpdateObj(vg, w->fetch_obj);
-	if (VGZ_Destroy(&vg, -1) != VGZ_END)
-		return(FetchError(w, "Gzip error at the very end"));
-	return (0);
-}
-
-struct vfp vfp_gzip = {
-        .begin  =       vfp_gzip_begin,
-        .bytes  =       vfp_gzip_bytes,
-        .end    =       vfp_gzip_end,
-};
-
-/*--------------------------------------------------------------------
- * VFP_TESTGZIP
- *
- * A VFP for testing that received gzip data is valid, and for
- * collecting the magic bits while we're at it.
- */
-
-static void __match_proto__()
-vfp_testgzip_begin(struct worker *w, size_t estimate)
-{
-	(void)estimate;
-	w->vgz_rx = VGZ_NewUngzip(w, "u F -");
-	CHECK_OBJ_NOTNULL(w->vgz_rx, VGZ_MAGIC);
-}
-
-static int __match_proto__()
-vfp_testgzip_bytes(struct worker *w, struct http_conn *htc, ssize_t bytes)
-{
-	struct vgz *vg;
-	ssize_t l, wl;
-	int i = -100;
-	uint8_t	obuf[cache_param->gzip_stack_buffer];
-	size_t dl;
-	const void *dp;
-	struct storage *st;
-
-	AZ(w->fetch_failed);
-	vg = w->vgz_rx;
-	CHECK_OBJ_NOTNULL(vg, VGZ_MAGIC);
-	AZ(vg->vz.avail_in);
-	while (bytes > 0) {
-		st = FetchStorage(w, 0);
-		if (st == NULL)
-			return(-1);
-		l = st->space - st->len;
-		if (l > bytes)
-			l = bytes;
-		wl = HTC_Read(w, htc, st->ptr + st->len, l);
-		if (wl <= 0)
-			return (wl);
-		bytes -= wl;
-		VGZ_Ibuf(vg, st->ptr + st->len, wl);
-		st->len += wl;
-		w->fetch_obj->len += wl;
-		if (w->do_stream)
-			RES_StreamPoll(w);
-
-		while (!VGZ_IbufEmpty(vg)) {
-			VGZ_Obuf(vg, obuf, sizeof obuf);
-			i = VGZ_Gunzip(vg, &dp, &dl);
-			if (i == VGZ_END && !VGZ_IbufEmpty(vg))
-				return(FetchError(w, "Junk after gzip data"));
-			if (i != VGZ_OK && i != VGZ_END)
-				return(FetchError2(w,
-				    "Invalid Gzip data", vg->vz.msg));
-		}
-	}
-	assert(i == VGZ_OK || i == VGZ_END);
-	return (1);
-}
-
-static int __match_proto__()
-vfp_testgzip_end(struct worker *w)
-{
-	struct vgz *vg;
-
-	vg = w->vgz_rx;
-	w->vgz_rx = NULL;
-	CHECK_OBJ_NOTNULL(vg, VGZ_MAGIC);
-	if (w->fetch_failed) {
-		(void)VGZ_Destroy(&vg, -1);
-		return(0);
-	}
-	VGZ_UpdateObj(vg, w->fetch_obj);
-	if (VGZ_Destroy(&vg, -1) != VGZ_END)
-		return(FetchError(w, "TestGunzip error at the very end"));
-	return (0);
-}
-
-struct vfp vfp_testgzip = {
-        .begin  =       vfp_testgzip_begin,
-        .bytes  =       vfp_testgzip_bytes,
-        .end    =       vfp_testgzip_end,
-};
diff --git a/bin/varnishd/cache_hash.c b/bin/varnishd/cache_hash.c
deleted file mode 100644
index db865de..0000000
--- a/bin/varnishd/cache_hash.c
+++ /dev/null
@@ -1,752 +0,0 @@
-/*-
- * Copyright (c) 2006 Verdens Gang AS
- * Copyright (c) 2006-2011 Varnish Software AS
- * All rights reserved.
- *
- * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- *
- * This is the central hash-table code, it relies on a chosen hash
- * implementation only for the actual hashing, all the housekeeping
- * happens here.
- *
- * We have two kinds of structures, objecthead and object.  An objecthead
- * corresponds to a given (Host:, URL) tupple, and the objects hung from
- * the objecthead may represent various variations (ie: Vary: header,
- * different TTL etc) instances of that web-entity.
- *
- * Each objecthead has a mutex which locks both its own fields, the
- * list of objects and fields in the objects.
- *
- * The hash implementation must supply a reference count facility on
- * the objecthead, and return with a reference held after a lookup.
- *
- * Lookups in the hash implementation returns with a ref held and each
- * object hung from the objhead holds a ref as well.
- *
- * Objects have refcounts which are locked by the objecthead mutex.
- *
- * New objects are always marked busy, and they can go from busy to
- * not busy only once.
- */
-
-#include "config.h"
-
-#include <math.h>
-#include <stdio.h>
-#include <stdlib.h>
-
-#include "cache.h"
-
-#include "hash/hash_slinger.h"
-#include "vsha256.h"
-
-static const struct hash_slinger *hash;
-
-/*---------------------------------------------------------------------*/
-/* Precreate an objhead and object for later use */
-void
-HSH_Prealloc(const struct sess *sp)
-{
-	struct worker *w;
-	struct objhead *oh;
-	struct objcore *oc;
-	struct waitinglist *wl;
-
-	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
-	CHECK_OBJ_NOTNULL(sp->wrk, WORKER_MAGIC);
-	w = sp->wrk;
-
-	if (w->nobjcore == NULL) {
-		ALLOC_OBJ(oc, OBJCORE_MAGIC);
-		XXXAN(oc);
-		w->nobjcore = oc;
-		w->stats.n_objectcore++;
-		oc->flags |= OC_F_BUSY;
-	}
-	CHECK_OBJ_NOTNULL(w->nobjcore, OBJCORE_MAGIC);
-
-	if (w->nobjhead == NULL) {
-		ALLOC_OBJ(oh, OBJHEAD_MAGIC);
-		XXXAN(oh);
-		oh->refcnt = 1;
-		VTAILQ_INIT(&oh->objcs);
-		Lck_New(&oh->mtx, lck_objhdr);
-		w->nobjhead = oh;
-		w->stats.n_objecthead++;
-	}
-	CHECK_OBJ_NOTNULL(w->nobjhead, OBJHEAD_MAGIC);
-
-	if (w->nwaitinglist == NULL) {
-		ALLOC_OBJ(wl, WAITINGLIST_MAGIC);
-		XXXAN(wl);
-		VTAILQ_INIT(&wl->list);
-		w->nwaitinglist = wl;
-		w->stats.n_waitinglist++;
-	}
-	CHECK_OBJ_NOTNULL(w->nwaitinglist, WAITINGLIST_MAGIC);
-
-	if (w->nbusyobj == NULL) {
-		ALLOC_OBJ(w->nbusyobj, BUSYOBJ_MAGIC);
-		XXXAN(w->nbusyobj);
-	}
-
-	if (hash->prep != NULL)
-		hash->prep(sp);
-}
-
-void
-HSH_Cleanup(struct worker *w)
-{
-
-	if (w->nobjcore != NULL) {
-		FREE_OBJ(w->nobjcore);
-		w->stats.n_objectcore--;
-		w->nobjcore = NULL;
-	}
-	if (w->nobjhead != NULL) {
-		Lck_Delete(&w->nobjhead->mtx);
-		FREE_OBJ(w->nobjhead);
-		w->nobjhead = NULL;
-		w->stats.n_objecthead--;
-	}
-	if (w->nwaitinglist != NULL) {
-		FREE_OBJ(w->nwaitinglist);
-		w->nwaitinglist = NULL;
-	}
-	if (w->nhashpriv != NULL) {
-		/* XXX: If needed, add slinger method for this */
-		free(w->nhashpriv);
-		w->nhashpriv = NULL;
-	}
-	if (w->nbusyobj != NULL) {
-		FREE_OBJ(w->nbusyobj);
-		w->nbusyobj = NULL;
-	}
-}
-
-void
-HSH_DeleteObjHead(struct worker *w, struct objhead *oh)
-{
-
-	AZ(oh->refcnt);
-	assert(VTAILQ_EMPTY(&oh->objcs));
-	Lck_Delete(&oh->mtx);
-	w->stats.n_objecthead--;
-	FREE_OBJ(oh);
-}
-
-void
-HSH_AddString(const struct sess *sp, const char *str)
-{
-	int l;
-
-	if (str == NULL)
-		str = "";
-	l = strlen(str);
-
-	SHA256_Update(sp->wrk->sha256ctx, str, l);
-	SHA256_Update(sp->wrk->sha256ctx, "#", 1);
-
-	if (cache_param->log_hash)
-		WSP(sp, SLT_Hash, "%s", str);
-}
-
-/*---------------------------------------------------------------------
- * This is a debugging hack to enable testing of boundary conditions
- * in the hash algorithm.
- * We trap the first 9 different digests and translate them to different
- * digests with edge bit conditions
- */
-
-static struct hsh_magiclist {
-	unsigned char was[SHA256_LEN];
-	unsigned char now[SHA256_LEN];
-} hsh_magiclist[] = {
-	{ .now = {	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 } },
-	{ .now = {	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01 } },
-	{ .now = {	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02 } },
-	{ .now = {	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40 } },
-	{ .now = {	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80 } },
-	{ .now = {	0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 } },
-	{ .now = {	0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 } },
-	{ .now = {	0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 } },
-	{ .now = {	0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 } },
-};
-
-#define HSH_NMAGIC (sizeof hsh_magiclist / sizeof hsh_magiclist[0])
-
-static void
-hsh_testmagic(void *result)
-{
-	int i, j;
-	static int nused = 0;
-
-	for (i = 0; i < nused; i++)
-		if (!memcmp(hsh_magiclist[i].was, result, SHA256_LEN))
-			break;
-	if (i == nused && i < HSH_NMAGIC)
-		memcpy(hsh_magiclist[nused++].was, result, SHA256_LEN);
-	if (i == nused)
-		return;
-	assert(i < HSH_NMAGIC);
-	fprintf(stderr, "HASHMAGIC: <");
-	for (j = 0; j < SHA256_LEN; j++)
-		fprintf(stderr, "%02x", ((unsigned char*)result)[j]);
-	fprintf(stderr, "> -> <");
-	memcpy(result, hsh_magiclist[i].now, SHA256_LEN);
-	for (j = 0; j < SHA256_LEN; j++)
-		fprintf(stderr, "%02x", ((unsigned char*)result)[j]);
-	fprintf(stderr, ">\n");
-}
-
-/*---------------------------------------------------------------------
- * Insert an object which magically appears out of nowhere or, more likely,
- * comes off some persistent storage device.
- * Return it with a reference held.
- */
-
-struct objcore *
-HSH_Insert(const struct sess *sp)
-{
-	struct worker *w;
-	struct objhead *oh;
-	struct objcore *oc;
-
-	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
-	CHECK_OBJ_NOTNULL(sp->wrk, WORKER_MAGIC);
-	AN(hash);
-	w = sp->wrk;
-
-	HSH_Prealloc(sp);
-	if (cache_param->diag_bitmap & 0x80000000)
-		hsh_testmagic(sp->wrk->nobjhead->digest);
-
-	AZ(sp->hash_objhead);
-	AN(w->nobjhead);
-	oh = hash->lookup(sp, w->nobjhead);
-	CHECK_OBJ_NOTNULL(oh, OBJHEAD_MAGIC);
-	if (oh == w->nobjhead)
-		w->nobjhead = NULL;
-	Lck_Lock(&oh->mtx);
-	assert(oh->refcnt > 0);
-
-	/* Insert (precreated) objcore in objecthead */
-	oc = w->nobjcore;
-	w->nobjcore = NULL;
-	oc->refcnt = 1;
-	CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
-	AZ(oc->flags & OC_F_BUSY);
-
-	VTAILQ_INSERT_HEAD(&oh->objcs, oc, list);
-	/* NB: do not deref objhead the new object inherits our reference */
-	oc->objhead = oh;
-	Lck_Unlock(&oh->mtx);
-	sp->wrk->stats.n_vampireobject++;
-	return (oc);
-}
-
-/*---------------------------------------------------------------------
- */
-
-struct objcore *
-HSH_Lookup(struct sess *sp, struct objhead **poh)
-{
-	struct worker *w;
-	struct objhead *oh;
-	struct objcore *oc;
-	struct objcore *busy_oc, *grace_oc;
-	struct object *o;
-	double grace_ttl;
-
-	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
-	CHECK_OBJ_NOTNULL(sp->wrk, WORKER_MAGIC);
-	CHECK_OBJ_NOTNULL(sp->http, HTTP_MAGIC);
-	AN(sp->director);
-	AN(hash);
-	w = sp->wrk;
-
-	HSH_Prealloc(sp);
-	memcpy(sp->wrk->nobjhead->digest, sp->digest, sizeof sp->digest);
-	if (cache_param->diag_bitmap & 0x80000000)
-		hsh_testmagic(sp->wrk->nobjhead->digest);
-
-	if (sp->hash_objhead != NULL) {
-		/*
-		 * This sess came off the waiting list, and brings a
-		 * oh refcnt with it.
-		 */
-		CHECK_OBJ_NOTNULL(sp->hash_objhead, OBJHEAD_MAGIC);
-		oh = sp->hash_objhead;
-		sp->hash_objhead = NULL;
-	} else {
-		AN(w->nobjhead);
-		oh = hash->lookup(sp, w->nobjhead);
-		if (oh == w->nobjhead)
-			w->nobjhead = NULL;
-	}
-
-	CHECK_OBJ_NOTNULL(oh, OBJHEAD_MAGIC);
-	Lck_Lock(&oh->mtx);
-	assert(oh->refcnt > 0);
-	busy_oc = NULL;
-	grace_oc = NULL;
-	grace_ttl = NAN;
-	VTAILQ_FOREACH(oc, &oh->objcs, list) {
-		/* Must be at least our own ref + the objcore we examine */
-		assert(oh->refcnt > 1);
-		CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
-		assert(oc->objhead == oh);
-
-		if (oc->flags & OC_F_BUSY) {
-			CHECK_OBJ_NOTNULL(oc->busyobj, BUSYOBJ_MAGIC);
-			if (sp->hash_ignore_busy)
-				continue;
-
-			if (oc->busyobj->vary != NULL &&
-			    !VRY_Match(sp, oc->busyobj->vary))
-				continue;
-
-			busy_oc = oc;
-			continue;
-		}
-
-		o = oc_getobj(sp->wrk, oc);
-		CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC);
-
-		if (o->exp.ttl <= 0.)
-			continue;
-		if (BAN_CheckObject(o, sp))
-			continue;
-		if (o->vary != NULL && !VRY_Match(sp, o->vary))
-			continue;
-
-		/* If still valid, use it */
-		if (EXP_Ttl(sp, o) >= sp->t_req)
-			break;
-
-		/*
-		 * Remember any matching objects inside their grace period
-		 * and if there are several, use the least expired one.
-		 */
-		if (EXP_Grace(sp, o) >= sp->t_req) {
-			if (grace_oc == NULL ||
-			    grace_ttl < o->exp.entered + o->exp.ttl) {
-				grace_oc = oc;
-				grace_ttl = o->exp.entered + o->exp.ttl;
-			}
-		}
-	}
-
-	/*
-	 * If we have seen a busy object or the backend is unhealthy, and
-	 * we have an object in grace, use it, if req.grace is also
-	 * satisified.
-	 * XXX: Interesting footnote:  The busy object might be for a
-	 * XXX: different "Vary:" than we sought.  We have no way of knowing
-	 * XXX: this until the object is unbusy'ed, so in practice we
-	 * XXX: serialize fetch of all Vary's if grace is possible.
-	 */
-
-	AZ(sp->objcore);
-	sp->objcore = grace_oc;		/* XXX: Hack-ish */
-	if (oc == NULL			/* We found no live object */
-	    && grace_oc != NULL		/* There is a grace candidate */
-	    && (busy_oc != NULL		/* Somebody else is already busy */
-	    || !VDI_Healthy(sp->director, sp))) {
-					/* Or it is impossible to fetch */
-		o = oc_getobj(sp->wrk, grace_oc);
-		CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC);
-		oc = grace_oc;
-	}
-	sp->objcore = NULL;
-
-	if (oc != NULL && !sp->hash_always_miss) {
-		o = oc_getobj(sp->wrk, oc);
-		CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC);
-		assert(oc->objhead == oh);
-
-		/* We found an object we like */
-		oc->refcnt++;
-		if (o->hits < INT_MAX)
-			o->hits++;
-		assert(oh->refcnt > 1);
-		Lck_Unlock(&oh->mtx);
-		assert(hash->deref(oh));
-		*poh = oh;
-		return (oc);
-	}
-
-	if (busy_oc != NULL) {
-		/* There are one or more busy objects, wait for them */
-		if (sp->esi_level == 0) {
-			CHECK_OBJ_NOTNULL(sp->wrk->nwaitinglist,
-			    WAITINGLIST_MAGIC);
-			if (oh->waitinglist == NULL) {
-				oh->waitinglist = sp->wrk->nwaitinglist;
-				sp->wrk->nwaitinglist = NULL;
-			}
-			VTAILQ_INSERT_TAIL(&oh->waitinglist->list, sp, list);
-		}
-		if (cache_param->diag_bitmap & 0x20)
-			WSP(sp, SLT_Debug,
-				"on waiting list <%p>", oh);
-		SES_Charge(sp);
-		/*
-		 * The objhead reference transfers to the sess, we get it
-		 * back when the sess comes off the waiting list and
-		 * calls us again
-		 */
-		sp->hash_objhead = oh;
-		sp->wrk = NULL;
-		Lck_Unlock(&oh->mtx);
-		return (NULL);
-	}
-
-	/* Insert (precreated) objcore in objecthead */
-	oc = w->nobjcore;
-	w->nobjcore = NULL;
-	AN(oc->flags & OC_F_BUSY);
-	oc->refcnt = 1;
-
-	/* XXX: clear w->nbusyobj before use */
-	VRY_Validate(sp->vary_b);
-	if (sp->vary_l != NULL)
-		w->nbusyobj->vary = sp->vary_b;
-	else
-		w->nbusyobj->vary = NULL;
-	oc->busyobj = w->nbusyobj;
-	w->nbusyobj = NULL;
-
-	/*
-	 * Busy objects go on the tail, so they will not trip up searches.
-	 * HSH_Unbusy() will move them to the front.
-	 */
-	VTAILQ_INSERT_TAIL(&oh->objcs, oc, list);
-	oc->objhead = oh;
-	/* NB: do not deref objhead the new object inherits our reference */
-	Lck_Unlock(&oh->mtx);
-	*poh = oh;
-	return (oc);
-}
-
-/*---------------------------------------------------------------------
- */
-
-static void
-hsh_rush(struct objhead *oh)
-{
-	unsigned u;
-	struct sess *sp;
-	struct waitinglist *wl;
-
-	CHECK_OBJ_NOTNULL(oh, OBJHEAD_MAGIC);
-	Lck_AssertHeld(&oh->mtx);
-	wl = oh->waitinglist;
-	CHECK_OBJ_NOTNULL(wl, WAITINGLIST_MAGIC);
-	for (u = 0; u < cache_param->rush_exponent; u++) {
-		sp = VTAILQ_FIRST(&wl->list);
-		if (sp == NULL)
-			break;
-		CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
-		AZ(sp->wrk);
-		VTAILQ_REMOVE(&wl->list, sp, list);
-		DSL(0x20, SLT_Debug, sp->vsl_id, "off waiting list");
-		if (SES_Schedule(sp)) {
-			/*
-			 * We could not schedule the session, leave the
-			 * rest on the busy list.
-			 */
-			break;
-		}
-	}
-	if (VTAILQ_EMPTY(&wl->list)) {
-		oh->waitinglist = NULL;
-		FREE_OBJ(wl);
-	}
-}
-
-/*---------------------------------------------------------------------
- * Purge an entire objhead
- */
-
-void
-HSH_Purge(const struct sess *sp, struct objhead *oh, double ttl, double grace)
-{
-	struct objcore *oc, **ocp;
-	unsigned spc, nobj, n;
-	struct object *o;
-
-	CHECK_OBJ_NOTNULL(oh, OBJHEAD_MAGIC);
-	spc = WS_Reserve(sp->wrk->ws, 0);
-	ocp = (void*)sp->wrk->ws->f;
-	Lck_Lock(&oh->mtx);
-	assert(oh->refcnt > 0);
-	nobj = 0;
-	VTAILQ_FOREACH(oc, &oh->objcs, list) {
-		CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
-		assert(oc->objhead == oh);
-		if (oc->flags & OC_F_BUSY) {
-			/*
-			 * We cannot purge busy objects here, because their
-			 * owners have special rights to them, and may nuke
-			 * them without concern for the refcount, which by
-			 * definition always must be one, so they don't check.
-			 */
-			continue;
-		}
-
-		(void)oc_getobj(sp->wrk, oc); /* XXX: still needed ? */
-
-		xxxassert(spc >= sizeof *ocp);
-		oc->refcnt++;
-		spc -= sizeof *ocp;
-		ocp[nobj++] = oc;
-	}
-	Lck_Unlock(&oh->mtx);
-
-	/* NB: inverse test to catch NAN also */
-	if (!(ttl > 0.))
-		ttl = -1.;
-	if (!(grace > 0.))
-		grace = -1.;
-	for (n = 0; n < nobj; n++) {
-		oc = ocp[n];
-		CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
-		o = oc_getobj(sp->wrk, oc);
-		if (o == NULL)
-			continue;
-		CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC);
-		o->exp.ttl = ttl;
-		o->exp.grace = grace;
-		EXP_Rearm(o);
-		(void)HSH_Deref(sp->wrk, NULL, &o);
-	}
-	WS_Release(sp->wrk->ws, 0);
-}
-
-
-/*---------------------------------------------------------------------
- * Kill a busy object we don't need anyway.
- * There may be sessions on the waiting list, so we cannot just blow
- * it out of the water.
- */
-
-void
-HSH_Drop(struct sess *sp)
-{
-	struct object *o;
-
-	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
-	o = sp->obj;
-	CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC);
-	AssertObjCorePassOrBusy(o->objcore);
-	o->exp.ttl = -1.;
-	if (o->objcore != NULL)		/* Pass has no objcore */
-		HSH_Unbusy(sp);
-	(void)HSH_Deref(sp->wrk, NULL, &sp->obj);
-}
-
-void
-HSH_Unbusy(const struct sess *sp)
-{
-	struct object *o;
-	struct objhead *oh;
-	struct objcore *oc;
-
-	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
-	o = sp->obj;
-	CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC);
-	oc = o->objcore;
-	CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
-	oh = oc->objhead;
-	CHECK_OBJ(oh, OBJHEAD_MAGIC);
-
-	AssertObjBusy(o);
-	AN(oc->ban);
-	assert(oc->refcnt > 0);
-	assert(oh->refcnt > 0);
-	if (o->ws_o->overflow)
-		sp->wrk->stats.n_objoverflow++;
-	if (cache_param->diag_bitmap & 0x40)
-		WSP(sp, SLT_Debug,
-		    "Object %u workspace free %u", o->xid, WS_Free(o->ws_o));
-
-	/* XXX: pretouch neighbors on oh->objcs to prevent page-on under mtx */
-	Lck_Lock(&oh->mtx);
-	assert(oh->refcnt > 0);
-	/* XXX: strictly speaking, we should sort in Date: order. */
-	VTAILQ_REMOVE(&oh->objcs, oc, list);
-	VTAILQ_INSERT_HEAD(&oh->objcs, oc, list);
-	oc->flags &= ~OC_F_BUSY;
-	AZ(sp->wrk->nbusyobj);
-	sp->wrk->nbusyobj = oc->busyobj;
-	oc->busyobj = NULL;
-	if (oh->waitinglist != NULL)
-		hsh_rush(oh);
-	AN(oc->ban);
-	Lck_Unlock(&oh->mtx);
-	assert(oc_getobj(sp->wrk, oc) == o);
-}
-
-void
-HSH_Ref(struct objcore *oc)
-{
-	struct objhead *oh;
-
-	CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
-	oh = oc->objhead;
-	CHECK_OBJ_NOTNULL(oh, OBJHEAD_MAGIC);
-	Lck_Lock(&oh->mtx);
-	assert(oc->refcnt > 0);
-	oc->refcnt++;
-	Lck_Unlock(&oh->mtx);
-}
-
-/*--------------------------------------------------------------------
- * Dereference objcore and or object
- *
- * Can deal with:
- *	bare objcore (incomplete fetch)
- *	bare object (pass)
- *	object with objcore
- *	XXX later:  objcore with object (?)
- *
- * But you can only supply one of the two arguments at a time.
- *
- * Returns zero if target was destroyed.
- */
-
-int
-HSH_Deref(struct worker *w, struct objcore *oc, struct object **oo)
-{
-	struct object *o = NULL;
-	struct objhead *oh;
-	unsigned r;
-
-	/* Only one arg at a time */
-	assert(oc == NULL || oo == NULL);
-
-	if (oo != NULL) {
-		o = *oo;
-		*oo = NULL;
-		CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC);
-		oc = o->objcore;
-	}
-
-	if (o != NULL && oc == NULL) {
-		/*
-		 * A pass object with neither objcore nor objhdr reference.
-		 * -> simply free the (Transient) storage
-		 */
-		STV_Freestore(o);
-		STV_free(o->objstore);
-		w->stats.n_object--;
-		return (0);
-	}
-
-	CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
-
-	oh = oc->objhead;
-	CHECK_OBJ_NOTNULL(oh, OBJHEAD_MAGIC);
-
-	Lck_Lock(&oh->mtx);
-	assert(oh->refcnt > 0);
-	assert(oc->refcnt > 0);
-	r = --oc->refcnt;
-	if (!r)
-		VTAILQ_REMOVE(&oh->objcs, oc, list);
-	else {
-		/* Must have an object */
-		AN(oc->methods);
-	}
-	if (oh->waitinglist != NULL)
-		hsh_rush(oh);
-	Lck_Unlock(&oh->mtx);
-	if (r != 0)
-		return (r);
-
-	BAN_DestroyObj(oc);
-	AZ(oc->ban);
-
-	if (oc->flags & OC_F_BUSY) {
-		CHECK_OBJ_NOTNULL(oc->busyobj, BUSYOBJ_MAGIC);
-		if (w->nbusyobj == NULL)
-			w->nbusyobj = oc->busyobj;
-		else
-			FREE_OBJ(oc->busyobj);
-		oc->busyobj = NULL;
-	}
-	AZ(oc->busyobj);
-
-	if (oc->methods != NULL) {
-		oc_freeobj(oc);
-		w->stats.n_object--;
-	}
-	FREE_OBJ(oc);
-
-	w->stats.n_objectcore--;
-	/* Drop our ref on the objhead */
-	assert(oh->refcnt > 0);
-	if (hash->deref(oh))
-		return (0);
-	HSH_DeleteObjHead(w, oh);
-	return (0);
-}
-
-void
-HSH_Init(const struct hash_slinger *slinger)
-{
-
-	assert(DIGEST_LEN == SHA256_LEN);	/* avoid #include pollution */
-	hash = slinger;
-	if (hash->start != NULL)
-		hash->start();
-}
diff --git a/bin/varnishd/cache_http.c b/bin/varnishd/cache_http.c
deleted file mode 100644
index 784eb28..0000000
--- a/bin/varnishd/cache_http.c
+++ /dev/null
@@ -1,1119 +0,0 @@
-/*-
- * Copyright (c) 2006 Verdens Gang AS
- * Copyright (c) 2006-2011 Varnish Software AS
- * All rights reserved.
- *
- * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- *
- * HTTP request storage and manipulation
- */
-
-#include "config.h"
-
-#include <stdio.h>
-
-#include "cache.h"
-
-#include "vct.h"
-
-#define HTTPH(a, b, c, d, e, f, g) char b[] = "*" a ":";
-#include "tbl/http_headers.h"
-#undef HTTPH
-
-/*lint -save -e773 not () */
-#define LOGMTX2(ax, bx, cx)	[bx] = SLT_##ax##cx
-
-#define LOGMTX1(ax) {					\
-	LOGMTX2(ax, HTTP_HDR_REQ,	Request),	\
-	LOGMTX2(ax, HTTP_HDR_RESPONSE,	Response),	\
-	LOGMTX2(ax, HTTP_HDR_STATUS,	Status),	\
-	LOGMTX2(ax, HTTP_HDR_URL,	URL),		\
-	LOGMTX2(ax, HTTP_HDR_PROTO,	Protocol),	\
-	LOGMTX2(ax, HTTP_HDR_FIRST,	Header),	\
-	}
-
-static const enum VSL_tag_e logmtx[][HTTP_HDR_FIRST + 1] = {
-	[HTTP_Rx] = LOGMTX1(Rx),
-	[HTTP_Tx] = LOGMTX1(Tx),
-	[HTTP_Obj] = LOGMTX1(Obj)
-};
-/*lint -restore */
-
-static enum VSL_tag_e
-http2shmlog(const struct http *hp, int t)
-{
-
-	CHECK_OBJ_NOTNULL(hp, HTTP_MAGIC);
-	if (t > HTTP_HDR_FIRST)
-		t = HTTP_HDR_FIRST;
-	assert(hp->logtag >= HTTP_Rx && hp->logtag <= HTTP_Obj); /*lint !e685*/
-	assert(t >= HTTP_HDR_REQ && t <= HTTP_HDR_FIRST);
-	return (logmtx[hp->logtag][t]);
-}
-
-static void
-WSLH(struct worker *w, unsigned vsl_id, const struct http *hp, unsigned hdr)
-{
-
-	AN(vsl_id & (VSL_CLIENTMARKER|VSL_BACKENDMARKER));
-	WSLR(w, http2shmlog(hp, hdr), vsl_id, hp->hd[hdr]);
-}
-
-/*--------------------------------------------------------------------*/
-/* List of canonical HTTP response code names from RFC2616 */
-
-static struct http_msg {
-	unsigned	nbr;
-	const char	*txt;
-} http_msg[] = {
-#define HTTP_RESP(n, t)	{ n, t},
-#include "tbl/http_response.h"
-	{ 0, NULL }
-};
-
-const char *
-http_StatusMessage(unsigned status)
-{
-	struct http_msg *mp;
-
-	assert(status >= 100 && status <= 999);
-	for (mp = http_msg; mp->nbr != 0 && mp->nbr <= status; mp++)
-		if (mp->nbr == status)
-			return (mp->txt);
-	return ("Unknown Error");
-}
-
-/*--------------------------------------------------------------------*/
-
-unsigned
-HTTP_estimate(unsigned nhttp)
-{
-
-	/* XXX: We trust the structs to size-aligned as necessary */
-	return (sizeof (struct http) + (sizeof (txt) + 1) * nhttp);
-}
-
-struct http *
-HTTP_create(void *p, uint16_t nhttp)
-{
-	struct http *hp;
-
-	hp = p;
-	hp->magic = HTTP_MAGIC;
-	hp->hd = (void*)(hp + 1);
-	hp->shd = nhttp;
-	hp->hdf = (void*)(hp->hd + nhttp);
-	return (hp);
-}
-
-/*--------------------------------------------------------------------*/
-
-void
-http_Setup(struct http *hp, struct ws *ws)
-{
-	uint16_t shd;
-	txt *hd;
-	unsigned char *hdf;
-
-	/* XXX: This is not elegant, is it efficient ? */
-	shd = hp->shd;
-	hd = hp->hd;
-	hdf = hp->hdf;
-	memset(hp, 0, sizeof *hp);
-	memset(hd, 0, sizeof *hd * shd);
-	memset(hdf, 0, sizeof *hdf * shd);
-	hp->magic = HTTP_MAGIC;
-	hp->ws = ws;
-	hp->nhd = HTTP_HDR_FIRST;
-	hp->shd = shd;
-	hp->hd = hd;
-	hp->hdf = hdf;
-}
-
-/*--------------------------------------------------------------------*/
-
-static int
-http_IsHdr(const txt *hh, const char *hdr)
-{
-	unsigned l;
-
-	Tcheck(*hh);
-	AN(hdr);
-	l = hdr[0];
-	assert(l == strlen(hdr + 1));
-	assert(hdr[l] == ':');
-	hdr++;
-	return (!strncasecmp(hdr, hh->b, l));
-}
-
-/*--------------------------------------------------------------------
- * This function collapses multiple headerlines of the same name.
- * The lines are joined with a comma, according to [rfc2616, 4.2bot, p32]
- */
-
-void
-http_CollectHdr(struct http *hp, const char *hdr)
-{
-	unsigned u, v, ml, f = 0, x;
-	char *b = NULL, *e = NULL;
-
-	for (u = HTTP_HDR_FIRST; u < hp->nhd; u++) {
-		while (u < hp->nhd && http_IsHdr(&hp->hd[u], hdr)) {
-			Tcheck(hp->hd[u]);
-			if (f == 0) {
-				/* Found first header, just record the fact */
-				f = u;
-				break;
-			}
-			if (b == NULL) {
-				/* Found second header, start our collection */
-				ml = WS_Reserve(hp->ws, 0);
-				b = hp->ws->f;
-				e = b + ml;
-				x = Tlen(hp->hd[f]);
-				if (b + x < e) {
-					memcpy(b, hp->hd[f].b, x);
-					b += x;
-				} else
-					b = e;
-			}
-
-			AN(b);
-			AN(e);
-
-			/* Append the Nth header we found */
-			if (b < e)
-				*b++ = ',';
-			x = Tlen(hp->hd[u]) - *hdr;
-			if (b + x < e) {
-				memcpy(b, hp->hd[u].b + *hdr, x);
-				b += x;
-			} else
-				b = e;
-
-			/* Shift remaining headers up one slot */
-			for (v = u; v < hp->nhd - 1; v++)
-				hp->hd[v] = hp->hd[v + 1];
-			hp->nhd--;
-		}
-
-	}
-	if (b == NULL)
-		return;
-	AN(e);
-	if (b >= e) {
-		WS_Release(hp->ws, 0);
-		return;
-	}
-	*b = '\0';
-	hp->hd[f].b = hp->ws->f;
-	hp->hd[f].e = b;
-	WS_ReleaseP(hp->ws, b + 1);
-}
-
-
-/*--------------------------------------------------------------------*/
-
-static unsigned
-http_findhdr(const struct http *hp, unsigned l, const char *hdr)
-{
-	unsigned u;
-
-	for (u = HTTP_HDR_FIRST; u < hp->nhd; u++) {
-		Tcheck(hp->hd[u]);
-		if (hp->hd[u].e < hp->hd[u].b + l + 1)
-			continue;
-		if (hp->hd[u].b[l] != ':')
-			continue;
-		if (strncasecmp(hdr, hp->hd[u].b, l))
-			continue;
-		return (u);
-	}
-	return (0);
-}
-
-int
-http_GetHdr(const struct http *hp, const char *hdr, char **ptr)
-{
-	unsigned u, l;
-	char *p;
-
-	l = hdr[0];
-	diagnostic(l == strlen(hdr + 1));
-	assert(hdr[l] == ':');
-	hdr++;
-	u = http_findhdr(hp, l - 1, hdr);
-	if (u == 0) {
-		if (ptr != NULL)
-			*ptr = NULL;
-		return (0);
-	}
-	if (ptr != NULL) {
-		p = hp->hd[u].b + l;
-		while (vct_issp(*p))
-			p++;
-		*ptr = p;
-	}
-	return (1);
-}
-
-
-/*--------------------------------------------------------------------
- * Find a given data element in a header according to RFC2616's #rule
- * (section 2.1, p15)
- */
-
-int
-http_GetHdrData(const struct http *hp, const char *hdr,
-    const char *field, char **ptr)
-{
-	char *h, *e;
-	unsigned fl;
-
-	if (ptr != NULL)
-		*ptr = NULL;
-	if (!http_GetHdr(hp, hdr, &h))
-		return (0);
-	AN(h);
-	e = strchr(h, '\0');
-	fl = strlen(field);
-	while (h + fl <= e) {
-		/* Skip leading whitespace and commas */
-		if (vct_islws(*h) || *h == ',') {
-			h++;
-			continue;
-		}
-		/* Check for substrings before memcmp() */
-		if ((h + fl == e || vct_issepctl(h[fl])) &&
-		    !memcmp(h, field, fl)) {
-			if (ptr != NULL) {
-				h += fl;
-				while (vct_islws(*h))
-					h++;
-				*ptr = h;
-			}
-			return (1);
-		}
-		/* Skip until end of header or comma */
-		while (*h && *h != ',')
-			h++;
-	}
-	return (0);
-}
-
-/*--------------------------------------------------------------------
- * Find a given headerfields Q value.
- */
-
-double
-http_GetHdrQ(const struct http *hp, const char *hdr, const char *field)
-{
-	char *h;
-	int i;
-	double a, b;
-
-	h = NULL;
-	i = http_GetHdrData(hp, hdr, field, &h);
-	if (!i)
-		return (0.);
-
-	if (h == NULL)
-		return (1.);
-	/* Skip whitespace, looking for '=' */
-	while (*h && vct_issp(*h))
-		h++;
-	if (*h++ != ';')
-		return (1.);
-	while (*h && vct_issp(*h))
-		h++;
-	if (*h++ != 'q')
-		return (1.);
-	while (*h && vct_issp(*h))
-		h++;
-	if (*h++ != '=')
-		return (1.);
-	while (*h && vct_issp(*h))
-		h++;
-	a = 0.;
-	while (vct_isdigit(*h)) {
-		a *= 10.;
-		a += *h - '0';
-		h++;
-	}
-	if (*h++ != '.')
-		return (a);
-	b = .1;
-	while (vct_isdigit(*h)) {
-		a += b * (*h - '0');
-		b *= .1;
-		h++;
-	}
-	return (a);
-}
-
-/*--------------------------------------------------------------------
- * Find a given headerfields value.
- */
-
-int
-http_GetHdrField(const struct http *hp, const char *hdr,
-    const char *field, char **ptr)
-{
-	char *h;
-	int i;
-
-	if (ptr != NULL)
-		*ptr = NULL;
-
-	h = NULL;
-	i = http_GetHdrData(hp, hdr, field, &h);
-	if (!i)
-		return (i);
-
-	if (ptr != NULL && h != NULL) {
-		/* Skip whitespace, looking for '=' */
-		while (*h && vct_issp(*h))
-			h++;
-		if (*h == '=') {
-			h++;
-			while (*h && vct_issp(*h))
-				h++;
-			*ptr = h;
-		}
-	}
-	return (i);
-}
-
-/*--------------------------------------------------------------------
- * XXX: redo with http_GetHdrField() ?
- */
-
-const char *
-http_DoConnection(const struct http *hp)
-{
-	char *p, *q;
-	const char *ret;
-	unsigned u;
-
-	if (!http_GetHdr(hp, H_Connection, &p)) {
-		if (hp->protover < 11)
-			return ("not HTTP/1.1");
-		return (NULL);
-	}
-	ret = NULL;
-	AN(p);
-	for (; *p; p++) {
-		if (vct_issp(*p))
-			continue;
-		if (*p == ',')
-			continue;
-		for (q = p + 1; *q; q++)
-			if (*q == ',' || vct_issp(*q))
-				break;
-		u = pdiff(p, q);
-		if (u == 5 && !strncasecmp(p, "close", u))
-			ret = "Connection: close";
-		u = http_findhdr(hp, u, p);
-		if (u != 0)
-			hp->hdf[u] |= HDF_FILTER;
-		if (!*q)
-			break;
-		p = q;
-	}
-	return (ret);
-}
-
-/*--------------------------------------------------------------------*/
-
-int
-http_HdrIs(const struct http *hp, const char *hdr, const char *val)
-{
-	char *p;
-
-	if (!http_GetHdr(hp, hdr, &p))
-		return (0);
-	AN(p);
-	if (!strcasecmp(p, val))
-		return (1);
-	return (0);
-}
-
-/*--------------------------------------------------------------------*/
-
-uint16_t
-http_GetStatus(const struct http *hp)
-{
-
-	return (hp->status);
-}
-
-const char *
-http_GetReq(const struct http *hp)
-{
-
-	Tcheck(hp->hd[HTTP_HDR_REQ]);
-	return (hp->hd[HTTP_HDR_REQ].b);
-}
-
-/*--------------------------------------------------------------------
- * Dissect the headers of the HTTP protocol message.
- * Detect conditionals (headers which start with '^[Ii][Ff]-')
- */
-
-static uint16_t
-http_dissect_hdrs(struct worker *w, struct http *hp, unsigned vsl_id, char *p,
-    const struct http_conn *htc)
-{
-	char *q, *r;
-	txt t = htc->rxbuf;
-
-	if (*p == '\r')
-		p++;
-
-	hp->nhd = HTTP_HDR_FIRST;
-	hp->conds = 0;
-	r = NULL;		/* For FlexeLint */
-	for (; p < t.e; p = r) {
-
-		/* Find end of next header */
-		q = r = p;
-		while (r < t.e) {
-			if (!vct_iscrlf(*r)) {
-				r++;
-				continue;
-			}
-			q = r;
-			assert(r < t.e);
-			r += vct_skipcrlf(r);
-			if (r >= t.e)
-				break;
-			/* If line does not continue: got it. */
-			if (!vct_issp(*r))
-				break;
-
-			/* Clear line continuation LWS to spaces */
-			while (vct_islws(*q))
-				*q++ = ' ';
-		}
-
-		if (q - p > htc->maxhdr) {
-			VSC_C_main->losthdr++;
-			WSL(w, SLT_LostHeader, vsl_id, "%.*s",
-			    q - p > 20 ? 20 : q - p, p);
-			return (413);
-		}
-
-		/* Empty header = end of headers */
-		if (p == q)
-			break;
-
-		if ((p[0] == 'i' || p[0] == 'I') &&
-		    (p[1] == 'f' || p[1] == 'F') &&
-		    p[2] == '-')
-			hp->conds = 1;
-
-		while (q > p && vct_issp(q[-1]))
-			q--;
-		*q = '\0';
-
-		if (hp->nhd < hp->shd) {
-			hp->hdf[hp->nhd] = 0;
-			hp->hd[hp->nhd].b = p;
-			hp->hd[hp->nhd].e = q;
-			WSLH(w, vsl_id, hp, hp->nhd);
-			hp->nhd++;
-		} else {
-			VSC_C_main->losthdr++;
-			WSL(w, SLT_LostHeader, vsl_id, "%.*s",
-			    q - p > 20 ? 20 : q - p, p);
-			return (413);
-		}
-	}
-	return (0);
-}
-
-/*--------------------------------------------------------------------
- * Deal with first line of HTTP protocol message.
- */
-
-static uint16_t
-http_splitline(struct worker *w, unsigned vsl_id, struct http *hp,
-    const struct http_conn *htc, int h1, int h2, int h3)
-{
-	char *p, *q;
-
-	CHECK_OBJ_NOTNULL(htc, HTTP_CONN_MAGIC);
-	CHECK_OBJ_NOTNULL(hp, HTTP_MAGIC);
-
-	/* XXX: Assert a NUL at rx.e ? */
-	Tcheck(htc->rxbuf);
-
-	/* Skip leading LWS */
-	for (p = htc->rxbuf.b ; vct_islws(*p); p++)
-		continue;
-
-	/* First field cannot contain SP, CRLF or CTL */
-	q = p;
-	for (; !vct_issp(*p); p++) {
-		if (vct_isctl(*p))
-			return (400);
-	}
-	hp->hd[h1].b = q;
-	hp->hd[h1].e = p;
-
-	/* Skip SP */
-	for (; vct_issp(*p); p++) {
-		if (vct_isctl(*p))
-			return (400);
-	}
-
-	/* Second field cannot contain LWS or CTL */
-	q = p;
-	for (; !vct_islws(*p); p++) {
-		if (vct_isctl(*p))
-			return (400);
-	}
-	hp->hd[h2].b = q;
-	hp->hd[h2].e = p;
-
-	if (!Tlen(hp->hd[h2]))
-		return (413);
-
-	/* Skip SP */
-	for (; vct_issp(*p); p++) {
-		if (vct_isctl(*p))
-			return (400);
-	}
-
-	/* Third field is optional and cannot contain CTL */
-	q = p;
-	if (!vct_iscrlf(*p)) {
-		for (; !vct_iscrlf(*p); p++)
-			if (!vct_issep(*p) && vct_isctl(*p))
-				return (400);
-	}
-	hp->hd[h3].b = q;
-	hp->hd[h3].e = p;
-
-	/* Skip CRLF */
-	p += vct_skipcrlf(p);
-
-	*hp->hd[h1].e = '\0';
-	WSLH(w, vsl_id, hp, h1);
-
-	*hp->hd[h2].e = '\0';
-	WSLH(w, vsl_id, hp, h2);
-
-	if (hp->hd[h3].e != NULL) {
-		*hp->hd[h3].e = '\0';
-		WSLH(w, vsl_id, hp, h3);
-	}
-
-	return (http_dissect_hdrs(w, hp, vsl_id, p, htc));
-}
-
-/*--------------------------------------------------------------------*/
-
-static void
-http_ProtoVer(struct http *hp)
-{
-
-	if (!strcasecmp(hp->hd[HTTP_HDR_PROTO].b, "HTTP/1.0"))
-		hp->protover = 10;
-	else if (!strcasecmp(hp->hd[HTTP_HDR_PROTO].b, "HTTP/1.1"))
-		hp->protover = 11;
-	else
-		hp->protover = 9;
-}
-
-
-/*--------------------------------------------------------------------*/
-
-uint16_t
-http_DissectRequest(struct sess *sp)
-{
-	struct http_conn *htc;
-	struct http *hp;
-	uint16_t retval;
-
-	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
-	htc = sp->htc;
-	CHECK_OBJ_NOTNULL(htc, HTTP_CONN_MAGIC);
-	hp = sp->http;
-	CHECK_OBJ_NOTNULL(hp, HTTP_MAGIC);
-
-	hp->logtag = HTTP_Rx;
-
-	retval = http_splitline(sp->wrk, sp->vsl_id, hp, htc,
-	    HTTP_HDR_REQ, HTTP_HDR_URL, HTTP_HDR_PROTO);
-	if (retval != 0) {
-		WSPR(sp, SLT_HttpGarbage, htc->rxbuf);
-		return (retval);
-	}
-	http_ProtoVer(hp);
-	return (retval);
-}
-
-/*--------------------------------------------------------------------*/
-
-uint16_t
-http_DissectResponse(struct worker *w, const struct http_conn *htc,
-    struct http *hp)
-{
-	int j;
-	uint16_t retval = 0;
-	char *p;
-
-
-	CHECK_OBJ_NOTNULL(htc, HTTP_CONN_MAGIC);
-	CHECK_OBJ_NOTNULL(hp, HTTP_MAGIC);
-	hp->logtag = HTTP_Rx;
-
-	if (http_splitline(w, htc->vsl_id, hp, htc,
-	    HTTP_HDR_PROTO, HTTP_HDR_STATUS, HTTP_HDR_RESPONSE))
-		retval = 503;
-
-	if (retval == 0 && memcmp(hp->hd[HTTP_HDR_PROTO].b, "HTTP/1.", 7))
-		retval = 503;
-
-	if (retval == 0 && Tlen(hp->hd[HTTP_HDR_STATUS]) != 3)
-		retval = 503;
-
-	if (retval == 0) {
-		hp->status = 0;
-		p = hp->hd[HTTP_HDR_STATUS].b;
-		for (j = 100; j != 0; j /= 10) {
-			if (!vct_isdigit(*p)) {
-				retval = 503;
-				break;
-			}
-			hp->status += (uint16_t)(j * (*p - '0'));
-			p++;
-		}
-		if (*p != '\0')
-			retval = 503;
-	}
-
-	if (retval != 0) {
-		WSLR(w, SLT_HttpGarbage, htc->vsl_id, htc->rxbuf);
-		assert(retval >= 100 && retval <= 999);
-		hp->status = retval;
-	} else {
-		http_ProtoVer(hp);
-	}
-
-	if (hp->hd[HTTP_HDR_RESPONSE].b == NULL ||
-	    !Tlen(hp->hd[HTTP_HDR_RESPONSE])) {
-		/* Backend didn't send a response string, use the standard */
-		hp->hd[HTTP_HDR_RESPONSE].b =
-		    TRUST_ME(http_StatusMessage(hp->status));
-		hp->hd[HTTP_HDR_RESPONSE].e =
-		    strchr(hp->hd[HTTP_HDR_RESPONSE].b, '\0');
-	}
-	return (retval);
-}
-
-/*--------------------------------------------------------------------*/
-
-void
-http_SetH(const struct http *to, unsigned n, const char *fm)
-{
-
-	assert(n < to->shd);
-	AN(fm);
-	to->hd[n].b = TRUST_ME(fm);
-	to->hd[n].e = strchr(to->hd[n].b, '\0');
-	to->hdf[n] = 0;
-}
-
-static void
-http_copyh(const struct http *to, const struct http *fm, unsigned n)
-{
-
-	assert(n < HTTP_HDR_FIRST);
-	Tcheck(fm->hd[n]);
-	to->hd[n] = fm->hd[n];
-	to->hdf[n] = fm->hdf[n];
-}
-
-void
-http_ForceGet(const struct http *to)
-{
-	if (strcmp(http_GetReq(to), "GET"))
-		http_SetH(to, HTTP_HDR_REQ, "GET");
-}
-
-void
-http_CopyResp(struct http *to, const struct http *fm)
-{
-
-	CHECK_OBJ_NOTNULL(fm, HTTP_MAGIC);
-	CHECK_OBJ_NOTNULL(to, HTTP_MAGIC);
-	http_SetH(to, HTTP_HDR_PROTO, "HTTP/1.1");
-	to->status = fm->status;
-	http_copyh(to, fm, HTTP_HDR_RESPONSE);
-}
-
-void
-http_SetResp(struct http *to, const char *proto, uint16_t status,
-    const char *response)
-{
-
-	CHECK_OBJ_NOTNULL(to, HTTP_MAGIC);
-	http_SetH(to, HTTP_HDR_PROTO, proto);
-	assert(status >= 100 && status <= 999);
-	to->status = status;
-	http_SetH(to, HTTP_HDR_RESPONSE, response);
-}
-
-static void
-http_copyheader(struct worker *w, unsigned vsl_id, struct http *to,
-    const struct http *fm, unsigned n)
-{
-
-	CHECK_OBJ_NOTNULL(fm, HTTP_MAGIC);
-	CHECK_OBJ_NOTNULL(to, HTTP_MAGIC);
-	assert(n < fm->shd);
-	Tcheck(fm->hd[n]);
-	if (to->nhd < to->shd) {
-		to->hd[to->nhd] = fm->hd[n];
-		to->hdf[to->nhd] = 0;
-		to->nhd++;
-	} else  {
-		VSC_C_main->losthdr++;
-		WSLR(w, SLT_LostHeader, vsl_id, fm->hd[n]);
-	}
-}
-
-/*--------------------------------------------------------------------
- * Estimate how much workspace we need to Filter this header according
- * to 'how'.
- */
-
-unsigned
-http_EstimateWS(const struct http *fm, unsigned how, uint16_t *nhd)
-{
-	unsigned u, l;
-
-	l = 0;
-	*nhd = HTTP_HDR_FIRST;
-	CHECK_OBJ_NOTNULL(fm, HTTP_MAGIC);
-	for (u = 0; u < fm->nhd; u++) {
-		if (fm->hd[u].b == NULL)
-			continue;
-		if (fm->hdf[u] & HDF_FILTER)
-			continue;
-#define HTTPH(a, b, c, d, e, f, g) \
-		if (((e) & how) && http_IsHdr(&fm->hd[u], (b))) \
-			continue;
-#include "tbl/http_headers.h"
-#undef HTTPH
-		l += PRNDUP(Tlen(fm->hd[u]) + 1);
-		(*nhd)++;
-		// fm->hdf[u] |= HDF_COPY;
-	}
-	return (l);
-}
-
-/*--------------------------------------------------------------------*/
-
-void
-http_FilterFields(struct worker *w, unsigned vsl_id, struct http *to,
-    const struct http *fm, unsigned how)
-{
-	unsigned u;
-
-	CHECK_OBJ_NOTNULL(fm, HTTP_MAGIC);
-	CHECK_OBJ_NOTNULL(to, HTTP_MAGIC);
-	to->nhd = HTTP_HDR_FIRST;
-	to->status = fm->status;
-	for (u = HTTP_HDR_FIRST; u < fm->nhd; u++) {
-		if (fm->hd[u].b == NULL)
-			continue;
-		if (fm->hdf[u] & HDF_FILTER)
-			continue;
-#define HTTPH(a, b, c, d, e, f, g) \
-		if (((e) & how) && http_IsHdr(&fm->hd[u], (b))) \
-			continue;
-#include "tbl/http_headers.h"
-#undef HTTPH
-		http_copyheader(w, vsl_id, to, fm, u);
-	}
-}
-
-/*--------------------------------------------------------------------*/
-
-void
-http_FilterHeader(const struct sess *sp, unsigned how)
-{
-	struct http *hp;
-
-	hp = sp->wrk->bereq;
-	CHECK_OBJ_NOTNULL(hp, HTTP_MAGIC);
-	hp->logtag = HTTP_Tx;
-
-	http_copyh(hp, sp->http, HTTP_HDR_REQ);
-	http_copyh(hp, sp->http, HTTP_HDR_URL);
-	if (how == HTTPH_R_FETCH)
-		http_SetH(hp, HTTP_HDR_PROTO, "HTTP/1.1");
-	else
-		http_copyh(hp, sp->http, HTTP_HDR_PROTO);
-	http_FilterFields(sp->wrk, sp->vsl_id, hp, sp->http, how);
-	http_PrintfHeader(sp->wrk, sp->vsl_id, hp, "X-Varnish: %u", sp->xid);
-}
-
-/*--------------------------------------------------------------------
- * This function copies any header fields which reference foreign
- * storage into our own WS.
- */
-
-void
-http_CopyHome(struct worker *w, unsigned vsl_id, const struct http *hp)
-{
-	unsigned u, l;
-	char *p;
-
-	for (u = 0; u < hp->nhd; u++) {
-		if (hp->hd[u].b == NULL)
-			continue;
-		if (hp->hd[u].b >= hp->ws->s && hp->hd[u].e <= hp->ws->e) {
-			WSLH(w, vsl_id, hp, u);
-			continue;
-		}
-		l = Tlen(hp->hd[u]);
-		p = WS_Alloc(hp->ws, l + 1);
-		if (p != NULL) {
-			WSLH(w, vsl_id, hp, u);
-			memcpy(p, hp->hd[u].b, l + 1L);
-			hp->hd[u].b = p;
-			hp->hd[u].e = p + l;
-		} else {
-			/* XXX This leaves a slot empty */
-			VSC_C_main->losthdr++;
-			WSLR(w, SLT_LostHeader, vsl_id, hp->hd[u]);
-			hp->hd[u].b = NULL;
-			hp->hd[u].e = NULL;
-		}
-	}
-}
-
-/*--------------------------------------------------------------------*/
-
-void
-http_ClrHeader(struct http *to)
-{
-
-	CHECK_OBJ_NOTNULL(to, HTTP_MAGIC);
-	to->nhd = HTTP_HDR_FIRST;
-	to->status = 0;
-	to->protover = 0;
-	to->conds = 0;
-	memset(to->hd, 0, sizeof *to->hd * to->shd);
-}
-
-/*--------------------------------------------------------------------*/
-
-void
-http_SetHeader(struct worker *w, unsigned vsl_id, struct http *to,
-    const char *hdr)
-{
-
-	CHECK_OBJ_NOTNULL(to, HTTP_MAGIC);
-	if (to->nhd >= to->shd) {
-		VSC_C_main->losthdr++;
-		WSL(w, SLT_LostHeader, vsl_id, "%s", hdr);
-		return;
-	}
-	http_SetH(to, to->nhd++, hdr);
-}
-
-/*--------------------------------------------------------------------*/
-
-static void
-http_PutField(struct worker *w, unsigned vsl_id, const struct http *to,
-    int field, const char *string)
-{
-	char *p;
-	unsigned l;
-
-	CHECK_OBJ_NOTNULL(to, HTTP_MAGIC);
-	l = strlen(string);
-	p = WS_Alloc(to->ws, l + 1);
-	if (p == NULL) {
-		WSL(w, SLT_LostHeader, vsl_id, "%s", string);
-		to->hd[field].b = NULL;
-		to->hd[field].e = NULL;
-		to->hdf[field] = 0;
-	} else {
-		memcpy(p, string, l + 1L);
-		to->hd[field].b = p;
-		to->hd[field].e = p + l;
-		to->hdf[field] = 0;
-	}
-}
-
-void
-http_PutProtocol(struct worker *w, unsigned vsl_id, const struct http *to,
-    const char *protocol)
-{
-
-	http_PutField(w, vsl_id, to, HTTP_HDR_PROTO, protocol);
-	if (to->hd[HTTP_HDR_PROTO].b == NULL)
-		http_SetH(to, HTTP_HDR_PROTO, "HTTP/1.1");
-	Tcheck(to->hd[HTTP_HDR_PROTO]);
-}
-
-void
-http_PutStatus(struct http *to, uint16_t status)
-{
-
-	assert(status >= 100 && status <= 999);
-	to->status = status;
-}
-
-void
-http_PutResponse(struct worker *w, unsigned vsl_id, const struct http *to,
-    const char *response)
-{
-
-	http_PutField(w, vsl_id, to, HTTP_HDR_RESPONSE, response);
-	if (to->hd[HTTP_HDR_RESPONSE].b == NULL)
-		http_SetH(to, HTTP_HDR_RESPONSE, "Lost Response");
-	Tcheck(to->hd[HTTP_HDR_RESPONSE]);
-}
-
-void
-http_PrintfHeader(struct worker *w, unsigned vsl_id, struct http *to,
-    const char *fmt, ...)
-{
-	va_list ap;
-	unsigned l, n;
-
-	CHECK_OBJ_NOTNULL(to, HTTP_MAGIC);
-	l = WS_Reserve(to->ws, 0);
-	va_start(ap, fmt);
-	n = vsnprintf(to->ws->f, l, fmt, ap);
-	va_end(ap);
-	if (n + 1 >= l || to->nhd >= to->shd) {
-		VSC_C_main->losthdr++;
-		WSL(w, SLT_LostHeader, vsl_id, "%s", to->ws->f);
-		WS_Release(to->ws, 0);
-	} else {
-		to->hd[to->nhd].b = to->ws->f;
-		to->hd[to->nhd].e = to->ws->f + n;
-		to->hdf[to->nhd] = 0;
-		WS_Release(to->ws, n + 1);
-		to->nhd++;
-	}
-}
-/*--------------------------------------------------------------------*/
-
-void
-http_Unset(struct http *hp, const char *hdr)
-{
-	uint16_t u, v;
-
-	for (v = u = HTTP_HDR_FIRST; u < hp->nhd; u++) {
-		if (hp->hd[u].b == NULL)
-			continue;
-		if (http_IsHdr(&hp->hd[u], hdr))
-			continue;
-		if (v != u) {
-			memcpy(&hp->hd[v], &hp->hd[u], sizeof *hp->hd);
-			memcpy(&hp->hdf[v], &hp->hdf[u], sizeof *hp->hdf);
-		}
-		v++;
-	}
-	hp->nhd = v;
-}
-
-/*--------------------------------------------------------------------*/
-
-void
-HTTP_Copy(struct http *to, const struct http * const fm)
-{
-
-	to->conds = fm->conds;
-	to->logtag = fm->logtag;
-	to->status = fm->status;
-	to->protover = fm->protover;
-	to->nhd = fm->nhd;
-	assert(fm->nhd <= to->shd);
-	memcpy(to->hd, fm->hd, fm->nhd * sizeof *to->hd);
-	memcpy(to->hdf, fm->hdf, fm->nhd * sizeof *to->hdf);
-}
-
-/*--------------------------------------------------------------------*/
-
-unsigned
-http_Write(struct worker *w, unsigned vsl_id, const struct http *hp, int resp)
-{
-	unsigned u, l;
-
-	if (resp) {
-		l = WRW_WriteH(w, &hp->hd[HTTP_HDR_PROTO], " ");
-		WSLH(w, vsl_id, hp, HTTP_HDR_PROTO);
-
-		hp->hd[HTTP_HDR_STATUS].b = WS_Alloc(w->ws, 4);
-		AN(hp->hd[HTTP_HDR_STATUS].b);
-
-		sprintf(hp->hd[HTTP_HDR_STATUS].b, "%3d", hp->status);
-		hp->hd[HTTP_HDR_STATUS].e = hp->hd[HTTP_HDR_STATUS].b + 3;
-
-		l += WRW_WriteH(w, &hp->hd[HTTP_HDR_STATUS], " ");
-		WSLH(w, vsl_id, hp, HTTP_HDR_STATUS);
-
-		l += WRW_WriteH(w, &hp->hd[HTTP_HDR_RESPONSE], "\r\n");
-		WSLH(w, vsl_id, hp, HTTP_HDR_RESPONSE);
-	} else {
-		AN(hp->hd[HTTP_HDR_URL].b);
-		l = WRW_WriteH(w, &hp->hd[HTTP_HDR_REQ], " ");
-		WSLH(w, vsl_id, hp, HTTP_HDR_REQ);
-		l += WRW_WriteH(w, &hp->hd[HTTP_HDR_URL], " ");
-		WSLH(w, vsl_id, hp, HTTP_HDR_URL);
-		l += WRW_WriteH(w, &hp->hd[HTTP_HDR_PROTO], "\r\n");
-		WSLH(w, vsl_id, hp, HTTP_HDR_PROTO);
-	}
-	for (u = HTTP_HDR_FIRST; u < hp->nhd; u++) {
-		if (hp->hd[u].b == NULL)
-			continue;
-		AN(hp->hd[u].b);
-		AN(hp->hd[u].e);
-		l += WRW_WriteH(w, &hp->hd[u], "\r\n");
-		WSLH(w, vsl_id, hp, u);
-	}
-	l += WRW_Write(w, "\r\n", -1);
-	return (l);
-}
-
-/*--------------------------------------------------------------------*/
-
-void
-HTTP_Init(void)
-{
-
-#define HTTPH(a, b, c, d, e, f, g) b[0] = (char)strlen(b + 1);
-#include "tbl/http_headers.h"
-#undef HTTPH
-}
diff --git a/bin/varnishd/cache_httpconn.c b/bin/varnishd/cache_httpconn.c
deleted file mode 100644
index 9e0a052..0000000
--- a/bin/varnishd/cache_httpconn.c
+++ /dev/null
@@ -1,229 +0,0 @@
-/*-
- * Copyright (c) 2006 Verdens Gang AS
- * Copyright (c) 2006-2011 Varnish Software AS
- * All rights reserved.
- *
- * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- *
- * HTTP protocol requests
- *
- * The trouble with the "until magic sequence" design of HTTP protocol messages
- * is that either you have to read a single character at a time, which is
- * inefficient, or you risk reading too much, and pre-read some of the object,
- * or even the next pipelined request, which follows the one you want.
- *
- * HTC reads a HTTP protocol header into a workspace, subject to limits,
- * and stops when we see the magic marker (double [CR]NL), and if we overshoot,
- * it keeps track of the "pipelined" data.
- *
- * We use this both for client and backend connections.
- */
-
-#include "config.h"
-
-#include "cache.h"
-
-#include "vct.h"
-
-/*--------------------------------------------------------------------
- * Check if we have a complete HTTP request or response yet
- *
- * Return values:
- *	 0  No, keep trying
- *	>0  Yes, it is this many bytes long.
- */
-
-static int
-htc_header_complete(txt *t)
-{
-	const char *p;
-
-	Tcheck(*t);
-	assert(*t->e == '\0');
-	/* Skip any leading white space */
-	for (p = t->b ; vct_issp(*p); p++)
-		continue;
-	if (p == t->e) {
-		/* All white space */
-		t->e = t->b;
-		*t->e = '\0';
-		return (0);
-	}
-	while (1) {
-		p = strchr(p, '\n');
-		if (p == NULL)
-			return (0);
-		p++;
-		if (*p == '\r')
-			p++;
-		if (*p == '\n')
-			break;
-	}
-	p++;
-	return (p - t->b);
-}
-
-/*--------------------------------------------------------------------*/
-
-void
-HTC_Init(struct http_conn *htc, struct ws *ws, int fd, unsigned vsl_id,
-    unsigned maxbytes, unsigned maxhdr)
-{
-
-	htc->magic = HTTP_CONN_MAGIC;
-	htc->ws = ws;
-	htc->fd = fd;
-	htc->vsl_id = vsl_id;
-	htc->maxbytes = maxbytes;
-	htc->maxhdr = maxhdr;
-
-	(void)WS_Reserve(htc->ws, htc->maxbytes);
-	htc->rxbuf.b = ws->f;
-	htc->rxbuf.e = ws->f;
-	*htc->rxbuf.e = '\0';
-	htc->pipeline.b = NULL;
-	htc->pipeline.e = NULL;
-}
-
-/*--------------------------------------------------------------------
- * Start over, and recycle any pipelined input.
- * The WS_Reset is safe, even though the pipelined input is stored in
- * the ws somewhere, because WS_Reset only fiddles pointers.
- */
-
-int
-HTC_Reinit(struct http_conn *htc)
-{
-	unsigned l;
-
-	CHECK_OBJ_NOTNULL(htc, HTTP_CONN_MAGIC);
-	(void)WS_Reserve(htc->ws, htc->maxbytes);
-	htc->rxbuf.b = htc->ws->f;
-	htc->rxbuf.e = htc->ws->f;
-	if (htc->pipeline.b != NULL) {
-		l = Tlen(htc->pipeline);
-		memmove(htc->rxbuf.b, htc->pipeline.b, l);
-		htc->rxbuf.e += l;
-		htc->pipeline.b = NULL;
-		htc->pipeline.e = NULL;
-	}
-	*htc->rxbuf.e = '\0';
-	return (HTC_Complete(htc));
-}
-
-/*--------------------------------------------------------------------
- * Return 1 if we have a complete HTTP procol header
- */
-
-int
-HTC_Complete(struct http_conn *htc)
-{
-	int i;
-
-	CHECK_OBJ_NOTNULL(htc, HTTP_CONN_MAGIC);
-	i = htc_header_complete(&htc->rxbuf);
-	assert(i >= 0);
-	if (i == 0)
-		return (0);
-	WS_ReleaseP(htc->ws, htc->rxbuf.e);
-	AZ(htc->pipeline.b);
-	AZ(htc->pipeline.e);
-	if (htc->rxbuf.b + i < htc->rxbuf.e) {
-		htc->pipeline.b = htc->rxbuf.b + i;
-		htc->pipeline.e = htc->rxbuf.e;
-		htc->rxbuf.e = htc->pipeline.b;
-	}
-	return (1);
-}
-
-/*--------------------------------------------------------------------
- * Receive more HTTP protocol bytes
- * Returns:
- *	-2 overflow
- *	-1 error
- *	 0 more needed
- *	 1 got complete HTTP header
- */
-
-int
-HTC_Rx(struct http_conn *htc)
-{
-	int i;
-
-	CHECK_OBJ_NOTNULL(htc, HTTP_CONN_MAGIC);
-	AN(htc->ws->r);
-	i = (htc->ws->r - htc->rxbuf.e) - 1;	/* space for NUL */
-	if (i <= 0) {
-		WS_ReleaseP(htc->ws, htc->rxbuf.b);
-		return (-2);
-	}
-	i = read(htc->fd, htc->rxbuf.e, i);
-	if (i <= 0) {
-		/*
-		 * We wouldn't come here if we had a complete HTTP header
-		 * so consequently an EOF can not be OK
-		 */
-		WS_ReleaseP(htc->ws, htc->rxbuf.b);
-		return (-1);
-	}
-	htc->rxbuf.e += i;
-	*htc->rxbuf.e = '\0';
-	return (HTC_Complete(htc));
-}
-
-/*--------------------------------------------------------------------
- * Read up to len bytes, returning pipelined data first.
- */
-
-ssize_t
-HTC_Read(struct worker *w, struct http_conn *htc, void *d, size_t len)
-{
-	size_t l;
-	unsigned char *p;
-	ssize_t i;
-
-	CHECK_OBJ_NOTNULL(w, WORKER_MAGIC);
-	CHECK_OBJ_NOTNULL(htc, HTTP_CONN_MAGIC);
-	l = 0;
-	p = d;
-	if (htc->pipeline.b) {
-		l = Tlen(htc->pipeline);
-		if (l > len)
-			l = len;
-		memcpy(p, htc->pipeline.b, l);
-		p += l;
-		len -= l;
-		htc->pipeline.b += l;
-		if (htc->pipeline.b == htc->pipeline.e)
-			htc->pipeline.b = htc->pipeline.e = NULL;
-	}
-	if (len == 0)
-		return (l);
-	i = read(htc->fd, p, len);
-	if (i < 0) {
-		WSL(w, SLT_FetchError, htc->vsl_id, "%s", strerror(errno));
-		return (i);
-	}
-	return (i + l);
-}
diff --git a/bin/varnishd/cache_lck.c b/bin/varnishd/cache_lck.c
deleted file mode 100644
index 2aef6dc..0000000
--- a/bin/varnishd/cache_lck.c
+++ /dev/null
@@ -1,210 +0,0 @@
-/*-
- * Copyright (c) 2008-2010 Varnish Software AS
- * All rights reserved.
- *
- * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- *
- * The geniuses who came up with pthreads did not think operations like
- * pthread_assert_mutex_held() were important enough to include them in
- * the API.
- *
- * Build our own locks on top of pthread mutexes and hope that the next
- * civilization is better at such crucial details than this one.
- */
-
-#include "config.h"
-
-#include <stdlib.h>
-
-#include "cache.h"
-
-/*The constability of lck depends on platform pthreads implementation */
-
-struct ilck {
-	unsigned		magic;
-#define ILCK_MAGIC		0x7b86c8a5
-	pthread_mutex_t		mtx;
-	int			held;
-	pthread_t		owner;
-	VTAILQ_ENTRY(ilck)	list;
-	const char		*w;
-	struct VSC_C_lck	*stat;
-};
-
-static VTAILQ_HEAD(, ilck)	ilck_head =
-    VTAILQ_HEAD_INITIALIZER(ilck_head);
-
-static pthread_mutex_t		lck_mtx;
-
-void __match_proto__()
-Lck__Lock(struct lock *lck, const char *p, const char *f, int l)
-{
-	struct ilck *ilck;
-	int r;
-
-	CAST_OBJ_NOTNULL(ilck, lck->priv, ILCK_MAGIC);
-	if (!(cache_param->diag_bitmap & 0x18)) {
-		AZ(pthread_mutex_lock(&ilck->mtx));
-		AZ(ilck->held);
-		ilck->stat->locks++;
-		ilck->owner = pthread_self();
-		ilck->held = 1;
-		return;
-	}
-	r = pthread_mutex_trylock(&ilck->mtx);
-	assert(r == 0 || r == EBUSY);
-	if (r) {
-		ilck->stat->colls++;
-		if (cache_param->diag_bitmap & 0x8)
-			VSL(SLT_Debug, 0, "MTX_CONTEST(%s,%s,%d,%s)",
-			    p, f, l, ilck->w);
-		AZ(pthread_mutex_lock(&ilck->mtx));
-	} else if (cache_param->diag_bitmap & 0x8) {
-		VSL(SLT_Debug, 0, "MTX_LOCK(%s,%s,%d,%s)", p, f, l, ilck->w);
-	}
-	AZ(ilck->held);
-	ilck->stat->locks++;
-	ilck->owner = pthread_self();
-	ilck->held = 1;
-}
-
-void __match_proto__()
-Lck__Unlock(struct lock *lck, const char *p, const char *f, int l)
-{
-	struct ilck *ilck;
-
-	CAST_OBJ_NOTNULL(ilck, lck->priv, ILCK_MAGIC);
-	assert(pthread_equal(ilck->owner, pthread_self()));
-	AN(ilck->held);
-	ilck->held = 0;
-	AZ(pthread_mutex_unlock(&ilck->mtx));
-	if (cache_param->diag_bitmap & 0x8)
-		VSL(SLT_Debug, 0, "MTX_UNLOCK(%s,%s,%d,%s)", p, f, l, ilck->w);
-}
-
-int __match_proto__()
-Lck__Trylock(struct lock *lck, const char *p, const char *f, int l)
-{
-	struct ilck *ilck;
-	int r;
-
-	CAST_OBJ_NOTNULL(ilck, lck->priv, ILCK_MAGIC);
-	r = pthread_mutex_trylock(&ilck->mtx);
-	assert(r == 0 || r == EBUSY);
-	if (cache_param->diag_bitmap & 0x8)
-		VSL(SLT_Debug, 0,
-		    "MTX_TRYLOCK(%s,%s,%d,%s) = %d", p, f, l, ilck->w);
-	if (r == 0) {
-		AZ(ilck->held);
-		ilck->held = 1;
-		ilck->owner = pthread_self();
-	}
-	return (r);
-}
-
-void
-Lck__Assert(const struct lock *lck, int held)
-{
-	struct ilck *ilck;
-
-	CAST_OBJ_NOTNULL(ilck, lck->priv, ILCK_MAGIC);
-	if (held)
-		assert(ilck->held &&
-		    pthread_equal(ilck->owner, pthread_self()));
-	else
-		assert(!ilck->held ||
-		    !pthread_equal(ilck->owner, pthread_self()));
-}
-
-int __match_proto__()
-Lck_CondWait(pthread_cond_t *cond, struct lock *lck, struct timespec *ts)
-{
-	struct ilck *ilck;
-	int retval = 0;
-
-	CAST_OBJ_NOTNULL(ilck, lck->priv, ILCK_MAGIC);
-	AN(ilck->held);
-	assert(pthread_equal(ilck->owner, pthread_self()));
-	ilck->held = 0;
-	if (ts == NULL) {
-		AZ(pthread_cond_wait(cond, &ilck->mtx));
-	} else {
-		retval = pthread_cond_timedwait(cond, &ilck->mtx, ts);
-		assert(retval == 0 || retval == ETIMEDOUT);
-	}
-	AZ(ilck->held);
-	ilck->held = 1;
-	ilck->owner = pthread_self();
-	return (retval);
-}
-
-void
-Lck__New(struct lock *lck, struct VSC_C_lck *st, const char *w)
-{
-	struct ilck *ilck;
-
-	AN(st);
-	AZ(lck->priv);
-	ALLOC_OBJ(ilck, ILCK_MAGIC);
-	AN(ilck);
-	ilck->w = w;
-	ilck->stat = st;
-	ilck->stat->creat++;
-	AZ(pthread_mutex_init(&ilck->mtx, NULL));
-	AZ(pthread_mutex_lock(&lck_mtx));
-	VTAILQ_INSERT_TAIL(&ilck_head, ilck, list);
-	AZ(pthread_mutex_unlock(&lck_mtx));
-	lck->priv = ilck;
-}
-
-void
-Lck_Delete(struct lock *lck)
-{
-	struct ilck *ilck;
-
-	CAST_OBJ_NOTNULL(ilck, lck->priv, ILCK_MAGIC);
-	ilck->stat->destroy++;
-	lck->priv = NULL;
-	AZ(pthread_mutex_lock(&lck_mtx));
-	VTAILQ_REMOVE(&ilck_head, ilck, list);
-	AZ(pthread_mutex_unlock(&lck_mtx));
-	AZ(pthread_mutex_destroy(&ilck->mtx));
-	FREE_OBJ(ilck);
-}
-
-#define LOCK(nam) struct VSC_C_lck *lck_##nam;
-#include "tbl/locks.h"
-#undef LOCK
-
-void
-LCK_Init(void)
-{
-
-	AZ(pthread_mutex_init(&lck_mtx, NULL));
-#define LOCK(nam)						\
-	lck_##nam = VSM_Alloc(sizeof(struct VSC_C_lck),		\
-	   VSC_CLASS, VSC_TYPE_LCK, #nam);
-#include "tbl/locks.h"
-#undef LOCK
-}
diff --git a/bin/varnishd/cache_main.c b/bin/varnishd/cache_main.c
deleted file mode 100644
index eb3fa1d..0000000
--- a/bin/varnishd/cache_main.c
+++ /dev/null
@@ -1,146 +0,0 @@
-/*-
- * Copyright (c) 2006 Verdens Gang AS
- * Copyright (c) 2006-2009 Varnish Software AS
- * All rights reserved.
- *
- * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- */
-
-#include "config.h"
-
-#include <stdio.h>
-#include <stdlib.h>
-
-#include "cache.h"
-#include "common/heritage.h"
-
-#include "waiter/cache_waiter.h"
-#include "hash/hash_slinger.h"
-
-volatile struct params	*cache_param;
-
-/*--------------------------------------------------------------------
- * Per thread storage for the session currently being processed by
- * the thread.  This is used for panic messages.
- */
-
-static pthread_key_t sp_key;
-
-void
-THR_SetSession(const struct sess *sp)
-{
-
-	AZ(pthread_setspecific(sp_key, sp));
-}
-
-const struct sess *
-THR_GetSession(void)
-{
-
-	return (pthread_getspecific(sp_key));
-}
-
-/*--------------------------------------------------------------------
- * Name threads if our pthreads implementation supports it.
- */
-
-static pthread_key_t name_key;
-
-void
-THR_SetName(const char *name)
-{
-
-	AZ(pthread_setspecific(name_key, name));
-#ifdef HAVE_PTHREAD_SET_NAME_NP
-	pthread_set_name_np(pthread_self(), name);
-#endif
-}
-
-const char *
-THR_GetName(void)
-{
-
-	return (pthread_getspecific(name_key));
-}
-
-/*--------------------------------------------------------------------
- * XXX: Think more about which order we start things
- */
-
-void
-child_main(void)
-{
-
-	setbuf(stdout, NULL);
-	setbuf(stderr, NULL);
-	printf("Child starts\n");
-
-	AZ(pthread_key_create(&sp_key, NULL));
-	AZ(pthread_key_create(&name_key, NULL));
-
-	THR_SetName("cache-main");
-
-	VSL_Init();	/* First, LCK needs it. */
-
-	LCK_Init();	/* Second, locking */
-
-	WAIT_Init();
-	PAN_Init();
-	CLI_Init();
-	Fetch_Init();
-
-	CNT_Init();
-	VCL_Init();
-
-	HTTP_Init();
-
-	VBE_Init();
-	VBP_Init();
-	WRK_Init();
-	Pool_Init();
-
-	EXP_Init();
-	HSH_Init(heritage.hash);
-	BAN_Init();
-
-	VCA_Init();
-
-	SMS_Init();
-	SMP_Init();
-	STV_open();
-
-	VMOD_Init();
-
-	BAN_Compile();
-
-	/* Wait for persistent storage to load if asked to */
-	if (cache_param->diag_bitmap & 0x00020000)
-		SMP_Ready();
-
-	CLI_Run();
-
-	STV_close();
-
-	printf("Child dies\n");
-}
diff --git a/bin/varnishd/cache_panic.c b/bin/varnishd/cache_panic.c
deleted file mode 100644
index c626ae3..0000000
--- a/bin/varnishd/cache_panic.c
+++ /dev/null
@@ -1,387 +0,0 @@
-/*-
- * Copyright (c) 2006 Verdens Gang AS
- * Copyright (c) 2006-2011 Varnish Software AS
- * All rights reserved.
- *
- * Author: Dag-Erling Sm?rgrav <des at des.no>
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- */
-
-#include "config.h"
-
-#ifndef HAVE_EXECINFO_H
-#include "compat/execinfo.h"
-#else
-#include <execinfo.h>
-#endif
-
-#include <stdio.h>
-#include <stdlib.h>
-
-#include "cache.h"
-
-#include "vapi/vsm_int.h"
-
-#include "cache_backend.h"
-#include "waiter/cache_waiter.h"
-#include "vcl.h"
-
-/*
- * The panic string is constructed in memory, then copied to the
- * shared memory.
- *
- * It can be extracted post-mortem from a core dump using gdb:
- *
- * (gdb) printf "%s", panicstr
- */
-
-static struct vsb vsps, *vsp;
-static pthread_mutex_t panicstr_mtx = PTHREAD_MUTEX_INITIALIZER;
-
-/*--------------------------------------------------------------------*/
-
-static void
-pan_ws(const struct ws *ws, int indent)
-{
-
-	VSB_printf(vsp, "%*sws = %p { %s\n", indent, "",
-	    ws, ws->overflow ? "overflow" : "");
-	VSB_printf(vsp, "%*sid = \"%s\",\n", indent + 2, "", ws->id);
-	VSB_printf(vsp, "%*s{s,f,r,e} = {%p", indent + 2, "", ws->s);
-	if (ws->f > ws->s)
-		VSB_printf(vsp, ",+%ld", (long) (ws->f - ws->s));
-	else
-		VSB_printf(vsp, ",%p", ws->f);
-	if (ws->r > ws->s)
-		VSB_printf(vsp, ",+%ld", (long) (ws->r - ws->s));
-	else
-		VSB_printf(vsp, ",%p", ws->r);
-	if (ws->e > ws->s)
-		VSB_printf(vsp, ",+%ld", (long) (ws->e - ws->s));
-	else
-		VSB_printf(vsp, ",%p", ws->e);
-	VSB_printf(vsp, "},\n");
-	VSB_printf(vsp, "%*s},\n", indent, "" );
-}
-
-/*--------------------------------------------------------------------*/
-
-static void
-pan_vbc(const struct vbc *vbc)
-{
-
-	struct backend *be;
-
-	be = vbc->backend;
-
-	VSB_printf(vsp, "  backend = %p fd = %d {\n", be, vbc->fd);
-	VSB_printf(vsp, "    display_name = \"%s\",\n", be->display_name);
-	VSB_printf(vsp, "  },\n");
-}
-
-/*--------------------------------------------------------------------*/
-
-static void
-pan_storage(const struct storage *st)
-{
-	int i, j;
-
-#define MAX_BYTES (4*16)
-#define show(ch) (((ch) > 31 && (ch) < 127) ? (ch) : '.')
-
-	VSB_printf(vsp, "      %u {\n", st->len);
-	for (i = 0; i < MAX_BYTES && i < st->len; i += 16) {
-		VSB_printf(vsp, "        ");
-		for (j = 0; j < 16; ++j) {
-			if (i + j < st->len)
-				VSB_printf(vsp, "%02x ", st->ptr[i + j]);
-			else
-				VSB_printf(vsp, "   ");
-		}
-		VSB_printf(vsp, "|");
-		for (j = 0; j < 16; ++j)
-			if (i + j < st->len)
-				VSB_printf(vsp, "%c", show(st->ptr[i + j]));
-		VSB_printf(vsp, "|\n");
-	}
-	if (st->len > MAX_BYTES)
-		VSB_printf(vsp, "        [%u more]\n", st->len - MAX_BYTES);
-	VSB_printf(vsp, "      },\n");
-
-#undef show
-#undef MAX_BYTES
-}
-
-/*--------------------------------------------------------------------*/
-
-static void
-pan_http(const char *id, const struct http *h, int indent)
-{
-	int i;
-
-	VSB_printf(vsp, "%*shttp[%s] = {\n", indent, "", id);
-	VSB_printf(vsp, "%*sws = %p[%s]\n", indent + 2, "",
-	    h->ws, h->ws ? h->ws->id : "");
-	for (i = 0; i < h->nhd; ++i) {
-		if (h->hd[i].b == NULL && h->hd[i].e == NULL)
-			continue;
-		VSB_printf(vsp, "%*s\"%.*s\",\n", indent + 4, "",
-		    (int)(h->hd[i].e - h->hd[i].b),
-		    h->hd[i].b);
-	}
-	VSB_printf(vsp, "%*s},\n", indent, "");
-}
-
-
-/*--------------------------------------------------------------------*/
-
-static void
-pan_object(const struct object *o)
-{
-	const struct storage *st;
-
-	VSB_printf(vsp, "  obj = %p {\n", o);
-	VSB_printf(vsp, "    xid = %u,\n", o->xid);
-	pan_ws(o->ws_o, 4);
-	pan_http("obj", o->http, 4);
-	VSB_printf(vsp, "    len = %jd,\n", (intmax_t)o->len);
-	VSB_printf(vsp, "    store = {\n");
-	VTAILQ_FOREACH(st, &o->store, list)
-		pan_storage(st);
-	VSB_printf(vsp, "    },\n");
-	VSB_printf(vsp, "  },\n");
-}
-
-/*--------------------------------------------------------------------*/
-
-static void
-pan_vcl(const struct VCL_conf *vcl)
-{
-	int i;
-
-	VSB_printf(vsp, "    vcl = {\n");
-	VSB_printf(vsp, "      srcname = {\n");
-	for (i = 0; i < vcl->nsrc; ++i)
-		VSB_printf(vsp, "        \"%s\",\n", vcl->srcname[i]);
-	VSB_printf(vsp, "      },\n");
-	VSB_printf(vsp, "    },\n");
-}
-
-
-/*--------------------------------------------------------------------*/
-
-static void
-pan_wrk(const struct worker *wrk)
-{
-
-	VSB_printf(vsp, "  worker = %p {\n", wrk);
-	pan_ws(wrk->ws, 4);
-	if (wrk->bereq->ws != NULL)
-		pan_http("bereq", wrk->bereq, 4);
-	if (wrk->beresp->ws != NULL)
-		pan_http("beresp", wrk->beresp, 4);
-	if (wrk->resp->ws != NULL)
-		pan_http("resp", wrk->resp, 4);
-	VSB_printf(vsp, "    },\n");
-}
-
-/*--------------------------------------------------------------------*/
-
-static void
-pan_sess(const struct sess *sp)
-{
-	const char *stp, *hand;
-
-	VSB_printf(vsp, "sp = %p {\n", sp);
-	VSB_printf(vsp,
-	    "  fd = %d, id = %d, xid = %u,\n",
-	    sp->fd, sp->vsl_id & VSL_IDENTMASK, sp->xid);
-	VSB_printf(vsp, "  client = %s %s,\n",
-	    sp->addr ? sp->addr : "?.?.?.?",
-	    sp->port ? sp->port : "?");
-	switch (sp->step) {
-#define STEP(l, u) case STP_##u: stp = "STP_" #u; break;
-#include "tbl/steps.h"
-#undef STEP
-		default: stp = NULL;
-	}
-	hand = VCL_Return_Name(sp->handling);
-	if (stp != NULL)
-		VSB_printf(vsp, "  step = %s,\n", stp);
-	else
-		VSB_printf(vsp, "  step = 0x%x,\n", sp->step);
-	if (hand != NULL)
-		VSB_printf(vsp, "  handling = %s,\n", hand);
-	else
-		VSB_printf(vsp, "  handling = 0x%x,\n", sp->handling);
-	if (sp->err_code)
-		VSB_printf(vsp,
-		    "  err_code = %d, err_reason = %s,\n", sp->err_code,
-		    sp->err_reason ? sp->err_reason : "(null)");
-
-	VSB_printf(vsp, "  restarts = %d, esi_level = %d\n",
-	    sp->restarts, sp->esi_level);
-
-	VSB_printf(vsp, "  flags = ");
-	if (sp->wrk->do_stream)	VSB_printf(vsp, " do_stream");
-	if (sp->wrk->do_gzip)	VSB_printf(vsp, " do_gzip");
-	if (sp->wrk->do_gunzip)	VSB_printf(vsp, " do_gunzip");
-	if (sp->wrk->do_esi)	VSB_printf(vsp, " do_esi");
-	if (sp->wrk->do_close)	VSB_printf(vsp, " do_close");
-	if (sp->wrk->is_gzip)	VSB_printf(vsp, " is_gzip");
-	if (sp->wrk->is_gunzip)	VSB_printf(vsp, " is_gunzip");
-	VSB_printf(vsp, "\n");
-	VSB_printf(vsp, "  bodystatus = %d\n", sp->wrk->body_status);
-
-	pan_ws(sp->ws, 2);
-	pan_http("req", sp->http, 2);
-
-	if (sp->wrk != NULL)
-		pan_wrk(sp->wrk);
-
-	if (VALID_OBJ(sp->vcl, VCL_CONF_MAGIC))
-		pan_vcl(sp->vcl);
-
-	if (VALID_OBJ(sp->wrk->vbc, BACKEND_MAGIC))
-		pan_vbc(sp->wrk->vbc);
-
-	if (VALID_OBJ(sp->obj, OBJECT_MAGIC))
-		pan_object(sp->obj);
-
-	VSB_printf(vsp, "},\n");
-}
-
-/*--------------------------------------------------------------------*/
-
-static void
-pan_backtrace(void)
-{
-	void *array[10];
-	size_t size;
-	size_t i;
-
-	size = backtrace (array, 10);
-	if (size == 0)
-		return;
-	VSB_printf(vsp, "Backtrace:\n");
-	for (i = 0; i < size; i++) {
-		VSB_printf (vsp, "  ");
-		if (Symbol_Lookup(vsp, array[i]) < 0) {
-			char **strings;
-			strings = backtrace_symbols(&array[i], 1);
-			if (strings != NULL && strings[0] != NULL)
-				VSB_printf(vsp, "%p: %s", array[i], strings[0]);
-			else
-				VSB_printf(vsp, "%p: (?)", array[i]);
-		}
-		VSB_printf (vsp, "\n");
-	}
-}
-
-/*--------------------------------------------------------------------*/
-
-static void
-pan_ic(const char *func, const char *file, int line, const char *cond,
-    int err, int xxx)
-{
-	const char *q;
-	const struct sess *sp;
-
-	AZ(pthread_mutex_lock(&panicstr_mtx)); /* Won't be released,
-						  we're going to die
-						  anyway */
-	switch(xxx) {
-	case 3:
-		VSB_printf(vsp,
-		    "Wrong turn at %s:%d:\n%s\n", file, line, cond);
-		break;
-	case 2:
-		VSB_printf(vsp,
-		    "Panic from VCL:\n  %s\n", cond);
-		break;
-	case 1:
-		VSB_printf(vsp,
-		    "Missing errorhandling code in %s(), %s line %d:\n"
-		    "  Condition(%s) not true.",
-		    func, file, line, cond);
-		break;
-	default:
-	case 0:
-		VSB_printf(vsp,
-		    "Assert error in %s(), %s line %d:\n"
-		    "  Condition(%s) not true.\n",
-		    func, file, line, cond);
-		break;
-	}
-	if (err)
-		VSB_printf(vsp, "errno = %d (%s)\n", err, strerror(err));
-
-	q = THR_GetName();
-	if (q != NULL)
-		VSB_printf(vsp, "thread = (%s)\n", q);
-
-	VSB_printf(vsp, "ident = %s,%s\n",
-	    VSB_data(vident) + 1, WAIT_GetName());
-
-	pan_backtrace();
-
-	if (!(cache_param->diag_bitmap & 0x2000)) {
-		sp = THR_GetSession();
-		if (sp != NULL)
-			pan_sess(sp);
-	}
-	VSB_printf(vsp, "\n");
-	VSB_bcat(vsp, "", 1);	/* NUL termination */
-
-	if (cache_param->diag_bitmap & 0x4000)
-		(void)fputs(VSM_head->panicstr, stderr);
-
-#ifdef HAVE_ABORT2
-	if (cache_param->diag_bitmap & 0x8000) {
-		void *arg[1];
-		char *p;
-
-		for (p = VSM_head->panicstr; *p; p++)
-			if (*p == '\n')
-				*p = ' ';
-		arg[0] = VSM_head->panicstr;
-		abort2(VSM_head->panicstr, 1, arg);
-	}
-#endif
-	if (cache_param->diag_bitmap & 0x1000)
-		exit(4);
-	else
-		abort();
-}
-
-/*--------------------------------------------------------------------*/
-
-void
-PAN_Init(void)
-{
-
-	VAS_Fail = pan_ic;
-	vsp = &vsps;
-	AN(VSB_new(vsp, VSM_head->panicstr, sizeof VSM_head->panicstr,
-	    VSB_FIXEDLEN));
-}
diff --git a/bin/varnishd/cache_pipe.c b/bin/varnishd/cache_pipe.c
deleted file mode 100644
index 4180d39..0000000
--- a/bin/varnishd/cache_pipe.c
+++ /dev/null
@@ -1,133 +0,0 @@
-/*-
- * Copyright (c) 2006 Verdens Gang AS
- * Copyright (c) 2006-2011 Varnish Software AS
- * All rights reserved.
- *
- * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- *
- * XXX: charge bytes to srcaddr
- */
-
-#include "config.h"
-
-#include <poll.h>
-#include <stdio.h>
-
-#include "cache.h"
-
-#include "cache_backend.h"
-#include "vtcp.h"
-#include "vtim.h"
-
-static int
-rdf(int fd0, int fd1)
-{
-	int i, j;
-	char buf[BUFSIZ], *p;
-
-	i = read(fd0, buf, sizeof buf);
-	if (i <= 0)
-		return (1);
-	for (p = buf; i > 0; i -= j, p += j) {
-		j = write(fd1, p, i);
-		if (j <= 0)
-			return (1);
-		if (i != j)
-			(void)usleep(100000);		/* XXX hack */
-	}
-	return (0);
-}
-
-void
-PipeSession(struct sess *sp)
-{
-	struct vbc *vc;
-	struct worker *w;
-	struct pollfd fds[2];
-	int i;
-
-	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
-	CHECK_OBJ_NOTNULL(sp->wrk, WORKER_MAGIC);
-	w = sp->wrk;
-
-	sp->wrk->vbc = VDI_GetFd(NULL, sp);
-	if (sp->wrk->vbc == NULL)
-		return;
-	vc = sp->wrk->vbc;
-	(void)VTCP_blocking(vc->fd);
-
-	WRW_Reserve(w, &vc->fd);
-	sp->wrk->acct_tmp.hdrbytes +=
-	    http_Write(w, sp->vsl_id, sp->wrk->bereq, 0);
-
-	if (sp->htc->pipeline.b != NULL)
-		sp->wrk->acct_tmp.bodybytes +=
-		    WRW_Write(w, sp->htc->pipeline.b, Tlen(sp->htc->pipeline));
-
-	i = WRW_FlushRelease(w);
-
-	if (i) {
-		SES_Close(sp, "pipe");
-		VDI_CloseFd(sp->wrk);
-		return;
-	}
-
-	sp->t_resp = VTIM_real();
-
-	memset(fds, 0, sizeof fds);
-
-	// XXX: not yet (void)VTCP_linger(vc->fd, 0);
-	fds[0].fd = vc->fd;
-	fds[0].events = POLLIN | POLLERR;
-
-	// XXX: not yet (void)VTCP_linger(sp->fd, 0);
-	fds[1].fd = sp->fd;
-	fds[1].events = POLLIN | POLLERR;
-
-	while (fds[0].fd > -1 || fds[1].fd > -1) {
-		fds[0].revents = 0;
-		fds[1].revents = 0;
-		i = poll(fds, 2, cache_param->pipe_timeout * 1000);
-		if (i < 1)
-			break;
-		if (fds[0].revents && rdf(vc->fd, sp->fd)) {
-			if (fds[1].fd == -1)
-				break;
-			(void)shutdown(vc->fd, SHUT_RD);
-			(void)shutdown(sp->fd, SHUT_WR);
-			fds[0].events = 0;
-			fds[0].fd = -1;
-		}
-		if (fds[1].revents && rdf(sp->fd, vc->fd)) {
-			if (fds[0].fd == -1)
-				break;
-			(void)shutdown(sp->fd, SHUT_RD);
-			(void)shutdown(vc->fd, SHUT_WR);
-			fds[1].events = 0;
-			fds[1].fd = -1;
-		}
-	}
-	SES_Close(sp, "pipe");
-	VDI_CloseFd(sp->wrk);
-}
diff --git a/bin/varnishd/cache_pool.c b/bin/varnishd/cache_pool.c
deleted file mode 100644
index 79a5fcd..0000000
--- a/bin/varnishd/cache_pool.c
+++ /dev/null
@@ -1,594 +0,0 @@
-/*-
- * Copyright (c) 2006 Verdens Gang AS
- * Copyright (c) 2006-2011 Varnish Software AS
- * All rights reserved.
- *
- * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- *
- * We maintain a number of worker thread pools, to spread lock contention.
- *
- * Pools can be added on the fly, as a means to mitigate lock contention,
- * but can only be removed again by a restart. (XXX: we could fix that)
- *
- * Two threads herd the pools, one eliminates idle threads and aggregates
- * statistics for all the pools, the other thread creates new threads
- * on demand, subject to various numerical constraints.
- *
- * The algorithm for when to create threads needs to be reactive enough
- * to handle startup spikes, but sufficiently attenuated to not cause
- * thread pileups.  This remains subject for improvement.
- */
-
-#include "config.h"
-
-#include <math.h>
-#include <stdlib.h>
-
-#include "cache.h"
-#include "common/heritage.h"
-
-#include "waiter/cache_waiter.h"
-#include "vtcp.h"
-#include "vtim.h"
-
-/*--------------------------------------------------------------------
- * MAC OS/X is incredibly moronic when it comes to time and such...
- */
-
-#ifndef CLOCK_MONOTONIC
-#define CLOCK_MONOTONIC	0
-
-#include <sys/time.h>
-
-static int
-clock_gettime(int foo, struct timespec *ts)
-{
-	struct timeval tv;
-
-	(void)foo;
-	gettimeofday(&tv, NULL);
-	ts->tv_sec = tv.tv_sec;
-	ts->tv_nsec = tv.tv_usec * 1000;
-	return (0);
-}
-
-static int
-pthread_condattr_setclock(pthread_condattr_t *attr, int foo)
-{
-	(void)attr;
-	(void)foo;
-	return (0);
-}
-#endif /* !CLOCK_MONOTONIC */
-
-static void *waiter_priv;
-
-VTAILQ_HEAD(workerhead, worker);
-
-struct poolsock {
-	unsigned			magic;
-#define POOLSOCK_MAGIC			0x1b0a2d38
-	VTAILQ_ENTRY(poolsock)		list;
-	struct listen_sock		*lsock;
-};
-
-/* Number of work requests queued in excess of worker threads available */
-
-struct pool {
-	unsigned		magic;
-#define POOL_MAGIC		0x606658fa
-	VTAILQ_ENTRY(pool)	list;
-
-	pthread_cond_t		herder_cond;
-	struct lock		herder_mtx;
-	pthread_t		herder_thr;
-
-	struct lock		mtx;
-	struct workerhead	idle;
-	VTAILQ_HEAD(, sess)	queue;
-	VTAILQ_HEAD(, poolsock)	socks;
-	unsigned		nthr;
-	unsigned		lqueue;
-	unsigned		last_lqueue;
-	uintmax_t		ndropped;
-	uintmax_t		nqueued;
-	struct sesspool		*sesspool;
-};
-
-static struct lock		pool_mtx;
-static pthread_t		thr_pool_herder;
-
-/*--------------------------------------------------------------------
- * Nobody is accepting on this socket, so we do.
- *
- * As long as we can stick the accepted connection to another thread
- * we do so, otherwise we return and handle it ourselves.
- *
- * Notice calling convention:  Called locked and returns locked, but
- * works lock in the meantime.
- *
- * We store data about the accept in reserved workspace, it is only used
- * for a brief moment and it takes up around 144 bytes.
- */
-
-static int
-pool_accept(struct pool *pp, struct worker *w, const struct poolsock *ps)
-{
-	struct worker *w2;
-	struct wrk_accept *wa, *wa2;
-
-	CHECK_OBJ_NOTNULL(pp, POOL_MAGIC);
-	CHECK_OBJ_NOTNULL(w, WORKER_MAGIC);
-	CHECK_OBJ_NOTNULL(ps, POOLSOCK_MAGIC);
-
-	CHECK_OBJ_NOTNULL(ps->lsock, LISTEN_SOCK_MAGIC);
-	Lck_AssertHeld(&pp->mtx);
-	Lck_Unlock(&pp->mtx);
-	assert(sizeof *wa == WS_Reserve(w->ws, sizeof *wa));
-	wa = (void*)w->ws->f;
-	while (1) {
-		memset(wa, 0, sizeof *wa);
-		wa->magic = WRK_ACCEPT_MAGIC;
-
-		if (ps->lsock->sock < 0) {
-			/* Socket Shutdown */
-			Lck_Lock(&pp->mtx);
-			return (-1);
-		}
-		if (VCA_Accept(ps->lsock, wa) < 0) {
-			w->stats.sess_fail++;
-			/* We're going to pace in vca anyway... */
-			(void)WRK_TrySumStat(w);
-			continue;
-		}
-
-		Lck_Lock(&pp->mtx);
-		if (VTAILQ_EMPTY(&pp->idle))
-			return (0);
-		w2 = VTAILQ_FIRST(&pp->idle);
-		VTAILQ_REMOVE(&pp->idle, w2, list);
-		Lck_Unlock(&pp->mtx);
-		assert(sizeof *wa2 == WS_Reserve(w2->ws, sizeof *wa2));
-		wa2 = (void*)w2->ws->f;
-		memcpy(wa2, wa, sizeof *wa);
-		AZ(pthread_cond_signal(&w2->cond));
-	}
-}
-
-/*--------------------------------------------------------------------
- * This is the work function for worker threads in the pool.
- */
-
-void
-Pool_Work_Thread(void *priv, struct worker *w)
-{
-	struct pool *pp;
-	int stats_clean, i;
-	struct poolsock *ps;
-
-	CAST_OBJ_NOTNULL(pp, priv, POOL_MAGIC);
-	w->pool = pp;
-	Lck_Lock(&pp->mtx);
-	stats_clean = 1;
-	while (1) {
-
-		Lck_AssertHeld(&pp->mtx);
-
-		CHECK_OBJ_NOTNULL(w, WORKER_MAGIC);
-		CHECK_OBJ_NOTNULL(w->bereq, HTTP_MAGIC);
-		CHECK_OBJ_NOTNULL(w->beresp, HTTP_MAGIC);
-		CHECK_OBJ_NOTNULL(w->resp, HTTP_MAGIC);
-
-		WS_Reset(w->ws, NULL);
-
-		w->sp = VTAILQ_FIRST(&pp->queue);
-		if (w->sp != NULL) {
-			/* Process queued requests, if any */
-			assert(pp->lqueue > 0);
-			VTAILQ_REMOVE(&pp->queue, w->sp, poollist);
-			pp->lqueue--;
-		} else if (!VTAILQ_EMPTY(&pp->socks)) {
-			/* Accept on a socket */
-			ps = VTAILQ_FIRST(&pp->socks);
-			VTAILQ_REMOVE(&pp->socks, ps, list);
-			i = pool_accept(pp, w, ps);
-			Lck_AssertHeld(&pp->mtx);
-			if (i < 0) {
-				/* Socket Shutdown */
-				FREE_OBJ(ps);
-				WS_Release(w->ws, 0);
-				continue;
-			}
-			VTAILQ_INSERT_TAIL(&pp->socks, ps, list);
-		} else if (VTAILQ_EMPTY(&pp->socks)) {
-			/* Nothing to do: To sleep, perchance to dream ... */
-			if (isnan(w->lastused))
-				w->lastused = VTIM_real();
-			VTAILQ_INSERT_HEAD(&pp->idle, w, list);
-			if (!stats_clean)
-				WRK_SumStat(w);
-			(void)Lck_CondWait(&w->cond, &pp->mtx, NULL);
-		}
-
-		/*
-		 * If we got neither session or accepted a socket, we were
-		 * woken up to die to cull the herd.
-		 */
-		if (w->sp == NULL && w->ws->r == NULL)
-			break;
-
-		Lck_Unlock(&pp->mtx);
-
-		if (w->sp == NULL) {
-			/* Turn accepted socket into a session */
-			assert(w->ws->r != NULL);
-			w->sp = SES_New(w, pp->sesspool);
-			if (w->sp == NULL)
-				VCA_FailSess(w);
-			else
-				VCA_SetupSess(w);
-			WS_Release(w->ws, 0);
-		}
-		assert(w->ws->r == NULL);
-
-		if (w->sp != NULL) {
-			CHECK_OBJ_NOTNULL(w->sp, SESS_MAGIC);
-
-			stats_clean = 0;
-			w->lastused = NAN;
-			w->storage_hint = NULL;
-
-			AZ(w->sp->wrk);
-			THR_SetSession(w->sp);
-			w->sp->wrk = w;
-			CNT_Session(w->sp);
-			THR_SetSession(NULL);
-			w->sp = NULL;
-
-			WS_Assert(w->ws);
-			AZ(w->bereq->ws);
-			AZ(w->beresp->ws);
-			AZ(w->resp->ws);
-			AZ(w->wrw.wfd);
-			AZ(w->storage_hint);
-			assert(w->wlp == w->wlb);
-			if (cache_param->diag_bitmap & 0x00040000) {
-				if (w->vcl != NULL)
-					VCL_Rel(&w->vcl);
-			}
-		}
-		stats_clean = WRK_TrySumStat(w);
-		Lck_Lock(&pp->mtx);
-	}
-	Lck_Unlock(&pp->mtx);
-	w->pool = NULL;
-}
-
-/*--------------------------------------------------------------------
- * Queue a workrequest if possible.
- *
- * Return zero if the request was queued, negative if it wasn't.
- */
-
-static int
-pool_queue(struct pool *pp, struct sess *sp)
-{
-	struct worker *w;
-
-	Lck_Lock(&pp->mtx);
-
-	/* If there are idle threads, we tickle the first one into action */
-	w = VTAILQ_FIRST(&pp->idle);
-	if (w != NULL) {
-		VTAILQ_REMOVE(&pp->idle, w, list);
-		Lck_Unlock(&pp->mtx);
-		w->sp = sp;
-		AZ(pthread_cond_signal(&w->cond));
-		return (0);
-	}
-
-	/* If we have too much in the queue already, refuse. */
-	if (pp->lqueue > (cache_param->queue_max * pp->nthr) / 100) {
-		pp->ndropped++;
-		Lck_Unlock(&pp->mtx);
-		return (-1);
-	}
-
-	VTAILQ_INSERT_TAIL(&pp->queue, sp, poollist);
-	pp->nqueued++;
-	pp->lqueue++;
-	Lck_Unlock(&pp->mtx);
-	AZ(pthread_cond_signal(&pp->herder_cond));
-	return (0);
-}
-
-/*--------------------------------------------------------------------*/
-
-int
-Pool_Schedule(struct pool *pp, struct sess *sp)
-{
-	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
-	AZ(sp->wrk);
-	if (pool_queue(pp, sp) == 0)
-		return(0);
-
-	VSC_C_main->client_drop_late++;
-
-	/*
-	 * Couldn't queue it -- kill it.
-	 *
-	 * XXX: a notice might be polite, but would potentially
-	 * XXX: sleep whichever thread got us here
-	 */
-	sp->t_end = VTIM_real();
-	if (sp->vcl != NULL) {
-		/*
-		 * A session parked on a busy object can come here
-		 * after it wakes up.  Loose the VCL reference.
-		 */
-		VCL_Rel(&sp->vcl);
-	}
-	return (1);
-}
-
-/*--------------------------------------------------------------------
- * Wait for another request
- */
-
-void
-Pool_Wait(struct sess *sp)
-{
-
-	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
-	AZ(sp->obj);
-	AZ(sp->vcl);
-	assert(sp->fd >= 0);
-	/*
-	 * Set nonblocking in the worker-thread, before passing to the
-	 * acceptor thread, to reduce syscall density of the latter.
-	 */
-	if (VTCP_nonblocking(sp->fd))
-		SES_Close(sp, "remote closed");
-	waiter->pass(waiter_priv, sp);
-}
-
-/*--------------------------------------------------------------------
- * Create another thread, if necessary & possible
- */
-
-static void
-pool_breed(struct pool *qp, const pthread_attr_t *tp_attr)
-{
-	pthread_t tp;
-
-	/*
-	 * If we need more threads, and have space, create
-	 * one more thread.
-	 */
-	if (qp->nthr < cache_param->wthread_min ||	/* Not enough threads yet */
-	    (qp->lqueue > cache_param->wthread_add_threshold && /* more needed */
-	    qp->lqueue > qp->last_lqueue)) {	/* not getting better since last */
-		if (qp->nthr > cache_param->wthread_max) {
-			Lck_Lock(&pool_mtx);
-			VSC_C_main->threads_limited++;
-			Lck_Unlock(&pool_mtx);
-		} else if (pthread_create(&tp, tp_attr, WRK_thread, qp)) {
-			VSL(SLT_Debug, 0, "Create worker thread failed %d %s",
-			    errno, strerror(errno));
-			Lck_Lock(&pool_mtx);
-			VSC_C_main->threads_limited++;
-			Lck_Unlock(&pool_mtx);
-			VTIM_sleep(cache_param->wthread_fail_delay * 1e-3);
-		} else {
-			AZ(pthread_detach(tp));
-			VTIM_sleep(cache_param->wthread_add_delay * 1e-3);
-			qp->nthr++;
-			Lck_Lock(&pool_mtx);
-			VSC_C_main->threads++;
-			VSC_C_main->threads_created++;
-			Lck_Unlock(&pool_mtx);
-		}
-	}
-	qp->last_lqueue = qp->lqueue;
-}
-
-/*--------------------------------------------------------------------
- * Herd a single pool
- *
- * This thread wakes up whenever a pool queues.
- *
- * The trick here is to not be too aggressive about creating threads.
- * We do this by only examining one pool at a time, and by sleeping
- * a short while whenever we create a thread and a little while longer
- * whenever we fail to, hopefully missing a lot of cond_signals in
- * the meantime.
- *
- * XXX: probably need a lot more work.
- *
- */
-
-static void*
-pool_herder(void *priv)
-{
-	struct pool *pp;
-	pthread_attr_t tp_attr;
-	struct timespec ts;
-	double t_idle;
-	struct worker *w;
-	int i;
-
-	CAST_OBJ_NOTNULL(pp, priv, POOL_MAGIC);
-	AZ(pthread_attr_init(&tp_attr));
-
-	while (1) {
-		/* Set the stacksize for worker threads we create */
-		if (cache_param->wthread_stacksize != UINT_MAX)
-			AZ(pthread_attr_setstacksize(&tp_attr,
-			    cache_param->wthread_stacksize));
-		else {
-			AZ(pthread_attr_destroy(&tp_attr));
-			AZ(pthread_attr_init(&tp_attr));
-		}
-
-		pool_breed(pp, &tp_attr);
-
-		if (pp->nthr < cache_param->wthread_min)
-			continue;
-
-		AZ(clock_gettime(CLOCK_MONOTONIC, &ts));
-		ts.tv_sec += cache_param->wthread_purge_delay / 1000;
-		ts.tv_nsec +=
-		    (cache_param->wthread_purge_delay % 1000) * 1000000;
-		if (ts.tv_nsec >= 1000000000) {
-			ts.tv_sec++;
-			ts.tv_nsec -= 1000000000;
-		}
-
-		Lck_Lock(&pp->herder_mtx);
-		i = Lck_CondWait(&pp->herder_cond, &pp->herder_mtx, &ts);
-		Lck_Unlock(&pp->herder_mtx);
-		if (!i)
-			continue;
-
-		if (pp->nthr <= cache_param->wthread_min)
-			continue;
-
-		t_idle = VTIM_real() - cache_param->wthread_timeout;
-
-		Lck_Lock(&pp->mtx);
-		VSC_C_main->sess_queued += pp->nqueued;
-		VSC_C_main->sess_dropped += pp->ndropped;
-		pp->nqueued = pp->ndropped = 0;
-		w = VTAILQ_LAST(&pp->idle, workerhead);
-		if (w != NULL &&
-		    (w->lastused < t_idle || pp->nthr > cache_param->wthread_max)) {
-			VTAILQ_REMOVE(&pp->idle, w, list);
-		} else
-			w = NULL;
-		Lck_Unlock(&pp->mtx);
-
-		/* And give it a kiss on the cheek... */
-		if (w != NULL) {
-			pp->nthr--;
-			Lck_Lock(&pool_mtx);
-			VSC_C_main->threads--;
-			VSC_C_main->threads_destroyed++;
-			Lck_Unlock(&pool_mtx);
-			AZ(w->sp);
-			AZ(pthread_cond_signal(&w->cond));
-		}
-	}
-	NEEDLESS_RETURN(NULL);
-}
-
-/*--------------------------------------------------------------------
- * Add a thread pool
- */
-
-static struct pool *
-pool_mkpool(void)
-{
-	struct pool *pp;
-	struct listen_sock *ls;
-	struct poolsock *ps;
-	pthread_condattr_t cv_attr;
-
-	ALLOC_OBJ(pp, POOL_MAGIC);
-	XXXAN(pp);
-	Lck_New(&pp->mtx, lck_wq);
-
-	VTAILQ_INIT(&pp->queue);
-	VTAILQ_INIT(&pp->idle);
-	VTAILQ_INIT(&pp->socks);
-	pp->sesspool = SES_NewPool(pp);
-	AN(pp->sesspool);
-
-	VTAILQ_FOREACH(ls, &heritage.socks, list) {
-		if (ls->sock < 0)
-			continue;
-		ALLOC_OBJ(ps, POOLSOCK_MAGIC);
-		XXXAN(ps);
-		ps->lsock = ls;
-		VTAILQ_INSERT_TAIL(&pp->socks, ps, list);
-	}
-
-	AZ(pthread_condattr_init(&cv_attr));
-	AZ(pthread_condattr_setclock(&cv_attr, CLOCK_MONOTONIC));
-	AZ(pthread_cond_init(&pp->herder_cond, &cv_attr));
-	AZ(pthread_condattr_destroy(&cv_attr));
-	Lck_New(&pp->herder_mtx, lck_herder);
-	AZ(pthread_create(&pp->herder_thr, NULL, pool_herder, pp));
-
-	return (pp);
-}
-
-/*--------------------------------------------------------------------
- * This thread adjusts the number of pools to match the parameter.
- *
- */
-
-static void *
-pool_poolherder(void *priv)
-{
-	unsigned nwq;
-	VTAILQ_HEAD(,pool)	pools = VTAILQ_HEAD_INITIALIZER(pools);
-	struct pool *pp;
-	uint64_t u;
-
-	THR_SetName("pool_herder");
-	(void)priv;
-
-	nwq = 0;
-	while (1) {
-		if (nwq < cache_param->wthread_pools) {
-			pp = pool_mkpool();
-			if (pp != NULL) {
-				VTAILQ_INSERT_TAIL(&pools, pp, list);
-				VSC_C_main->pools++;
-				nwq++;
-				continue;
-			}
-		}
-		/* XXX: remove pools */
-		if (0)
-			SES_DeletePool(NULL, NULL);
-		(void)sleep(1);
-		u = 0;
-		VTAILQ_FOREACH(pp, &pools, list)
-			u += pp->lqueue;
-		VSC_C_main->thread_queue_len = u;
-	}
-	NEEDLESS_RETURN(NULL);
-}
-
-/*--------------------------------------------------------------------*/
-
-void
-Pool_Init(void)
-{
-
-	waiter_priv = waiter->init();
-	Lck_New(&pool_mtx, lck_wq);
-	AZ(pthread_create(&thr_pool_herder, NULL, pool_poolherder, NULL));
-}
diff --git a/bin/varnishd/cache_response.c b/bin/varnishd/cache_response.c
deleted file mode 100644
index 487a514..0000000
--- a/bin/varnishd/cache_response.c
+++ /dev/null
@@ -1,427 +0,0 @@
-/*-
- * Copyright (c) 2006 Verdens Gang AS
- * Copyright (c) 2006-2011 Varnish Software AS
- * All rights reserved.
- *
- * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- */
-
-#include "config.h"
-
-#include "cache.h"
-
-#include "vct.h"
-#include "vtim.h"
-
-/*--------------------------------------------------------------------*/
-
-static void
-res_dorange(const struct sess *sp, const char *r, ssize_t *plow, ssize_t *phigh)
-{
-	ssize_t low, high, has_low;
-
-	assert(sp->obj->response == 200);
-	if (strncmp(r, "bytes=", 6))
-		return;
-	r += 6;
-
-	/* The low end of range */
-	has_low = low = 0;
-	if (!vct_isdigit(*r) && *r != '-')
-		return;
-	while (vct_isdigit(*r)) {
-		has_low = 1;
-		low *= 10;
-		low += *r - '0';
-		r++;
-	}
-
-	if (low >= sp->obj->len)
-		return;
-
-	if (*r != '-')
-		return;
-	r++;
-
-	/* The high end of range */
-	if (vct_isdigit(*r)) {
-		high = 0;
-		while (vct_isdigit(*r)) {
-			high *= 10;
-			high += *r - '0';
-			r++;
-		}
-		if (!has_low) {
-			low = sp->obj->len - high;
-			high = sp->obj->len - 1;
-		}
-	} else
-		high = sp->obj->len - 1;
-	if (*r != '\0')
-		return;
-
-	if (high >= sp->obj->len)
-		high = sp->obj->len - 1;
-
-	if (low > high)
-		return;
-
-	http_PrintfHeader(sp->wrk, sp->vsl_id, sp->wrk->resp,
-	    "Content-Range: bytes %jd-%jd/%jd",
-	    (intmax_t)low, (intmax_t)high, (intmax_t)sp->obj->len);
-	http_Unset(sp->wrk->resp, H_Content_Length);
-	assert(sp->wrk->res_mode & RES_LEN);
-	http_PrintfHeader(sp->wrk, sp->vsl_id, sp->wrk->resp,
-	    "Content-Length: %jd", (intmax_t)(1 + high - low));
-	http_SetResp(sp->wrk->resp, "HTTP/1.1", 206, "Partial Content");
-
-	*plow = low;
-	*phigh = high;
-}
-
-/*--------------------------------------------------------------------*/
-
-void
-RES_BuildHttp(const struct sess *sp)
-{
-	char time_str[30];
-
-	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
-
-
-	http_ClrHeader(sp->wrk->resp);
-	sp->wrk->resp->logtag = HTTP_Tx;
-	http_CopyResp(sp->wrk->resp, sp->obj->http);
-	http_FilterFields(sp->wrk, sp->vsl_id, sp->wrk->resp, sp->obj->http,
-	    HTTPH_A_DELIVER);
-
-	if (!(sp->wrk->res_mode & RES_LEN)) {
-		http_Unset(sp->wrk->resp, H_Content_Length);
-	} else if (cache_param->http_range_support) {
-		/* We only accept ranges if we know the length */
-		http_SetHeader(sp->wrk, sp->vsl_id, sp->wrk->resp,
-		    "Accept-Ranges: bytes");
-	}
-
-	if (sp->wrk->res_mode & RES_CHUNKED)
-		http_SetHeader(sp->wrk, sp->vsl_id, sp->wrk->resp,
-		    "Transfer-Encoding: chunked");
-
-	VTIM_format(VTIM_real(), time_str);
-	http_PrintfHeader(sp->wrk, sp->vsl_id, sp->wrk->resp, "Date: %s", time_str);
-
-	if (sp->xid != sp->obj->xid)
-		http_PrintfHeader(sp->wrk, sp->vsl_id, sp->wrk->resp,
-		    "X-Varnish: %u %u", sp->xid, sp->obj->xid);
-	else
-		http_PrintfHeader(sp->wrk, sp->vsl_id, sp->wrk->resp,
-		    "X-Varnish: %u", sp->xid);
-	http_PrintfHeader(sp->wrk, sp->vsl_id, sp->wrk->resp, "Age: %.0f",
-	    sp->obj->exp.age + sp->t_resp - sp->obj->exp.entered);
-	http_SetHeader(sp->wrk, sp->vsl_id, sp->wrk->resp, "Via: 1.1 varnish");
-	http_PrintfHeader(sp->wrk, sp->vsl_id, sp->wrk->resp, "Connection: %s",
-	    sp->doclose ? "close" : "keep-alive");
-}
-
-/*--------------------------------------------------------------------
- * We have a gzip'ed object and need to ungzip it for a client which
- * does not understand gzip.
- * XXX: handle invalid gzip data better (how ?)
- */
-
-static void
-res_WriteGunzipObj(const struct sess *sp)
-{
-	struct storage *st;
-	unsigned u = 0;
-	struct vgz *vg;
-	char obuf[cache_param->gzip_stack_buffer];
-	ssize_t obufl = 0;
-	int i;
-
-	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
-
-	vg = VGZ_NewUngzip(sp->wrk, "U D -");
-
-	VGZ_Obuf(vg, obuf, sizeof obuf);
-	VTAILQ_FOREACH(st, &sp->obj->store, list) {
-		CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
-		CHECK_OBJ_NOTNULL(st, STORAGE_MAGIC);
-		u += st->len;
-
-		VSC_C_main->n_objwrite++;
-
-		i = VGZ_WrwGunzip(sp->wrk, vg,
-		    st->ptr, st->len,
-		    obuf, sizeof obuf, &obufl);
-		/* XXX: error check */
-		(void)i;
-	}
-	if (obufl) {
-		(void)WRW_Write(sp->wrk, obuf, obufl);
-		(void)WRW_Flush(sp->wrk);
-	}
-	(void)VGZ_Destroy(&vg, sp->vsl_id);
-	assert(u == sp->obj->len);
-}
-
-/*--------------------------------------------------------------------*/
-
-static void
-res_WriteDirObj(const struct sess *sp, ssize_t low, ssize_t high)
-{
-	ssize_t u = 0;
-	size_t ptr, off, len;
-	struct storage *st;
-
-	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
-
-	ptr = 0;
-	VTAILQ_FOREACH(st, &sp->obj->store, list) {
-		CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
-		CHECK_OBJ_NOTNULL(st, STORAGE_MAGIC);
-		u += st->len;
-		len = st->len;
-		off = 0;
-		if (ptr + len <= low) {
-			/* This segment is too early */
-			ptr += len;
-			continue;
-		}
-		if (ptr < low) {
-			/* Chop front of segment off */
-			off += (low - ptr);
-			len -= (low - ptr);
-			ptr += (low - ptr);
-		}
-		if (ptr + len > high)
-			/* Chop tail of segment off */
-			len = 1 + high - ptr;
-
-		ptr += len;
-
-		sp->wrk->acct_tmp.bodybytes += len;
-#ifdef SENDFILE_WORKS
-		/*
-		 * XXX: the overhead of setting up sendfile is not
-		 * XXX: epsilon and maybe not even delta, so avoid
-		 * XXX: engaging sendfile for small objects.
-		 * XXX: Should use getpagesize() ?
-		 */
-		if (st->fd >= 0 &&
-		    st->len >= cache_param->sendfile_threshold) {
-			VSC_C_main->n_objsendfile++;
-			WRW_Sendfile(sp->wrk, st->fd, st->where + off, len);
-			continue;
-		}
-#endif /* SENDFILE_WORKS */
-		VSC_C_main->n_objwrite++;
-		(void)WRW_Write(sp->wrk, st->ptr + off, len);
-	}
-	assert(u == sp->obj->len);
-}
-
-/*--------------------------------------------------------------------
- * Deliver an object.
- * Attempt optimizations like 304 and 206 here.
- */
-
-void
-RES_WriteObj(struct sess *sp)
-{
-	char *r;
-	ssize_t low, high;
-
-	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
-
-	WRW_Reserve(sp->wrk, &sp->fd);
-
-	if (sp->obj->response == 200 &&
-	    sp->http->conds &&
-	    RFC2616_Do_Cond(sp)) {
-		sp->wantbody = 0;
-		http_SetResp(sp->wrk->resp, "HTTP/1.1", 304, "Not Modified");
-		http_Unset(sp->wrk->resp, H_Content_Length);
-		http_Unset(sp->wrk->resp, H_Transfer_Encoding);
-	}
-
-	/*
-	 * If nothing special planned, we can attempt Range support
-	 */
-	low = 0;
-	high = sp->obj->len - 1;
-	if (
-	    sp->wantbody &&
-	    (sp->wrk->res_mode & RES_LEN) &&
-	    !(sp->wrk->res_mode & (RES_ESI|RES_ESI_CHILD|RES_GUNZIP)) &&
-	    cache_param->http_range_support &&
-	    sp->obj->response == 200 &&
-	    http_GetHdr(sp->http, H_Range, &r))
-		res_dorange(sp, r, &low, &high);
-
-	/*
-	 * Always remove C-E if client don't grok it
-	 */
-	if (sp->wrk->res_mode & RES_GUNZIP)
-		http_Unset(sp->wrk->resp, H_Content_Encoding);
-
-	/*
-	 * Send HTTP protocol header, unless interior ESI object
-	 */
-	if (!(sp->wrk->res_mode & RES_ESI_CHILD))
-		sp->wrk->acct_tmp.hdrbytes +=
-		    http_Write(sp->wrk, sp->vsl_id, sp->wrk->resp, 1);
-
-	if (!sp->wantbody)
-		sp->wrk->res_mode &= ~RES_CHUNKED;
-
-	if (sp->wrk->res_mode & RES_CHUNKED)
-		WRW_Chunked(sp->wrk);
-
-	if (!sp->wantbody) {
-		/* This was a HEAD or conditional request */
-	} else if (sp->obj->len == 0) {
-		/* Nothing to do here */
-	} else if (sp->wrk->res_mode & RES_ESI) {
-		ESI_Deliver(sp);
-	} else if (sp->wrk->res_mode & RES_ESI_CHILD && sp->wrk->gzip_resp) {
-		ESI_DeliverChild(sp);
-	} else if (sp->wrk->res_mode & RES_ESI_CHILD &&
-	    !sp->wrk->gzip_resp && sp->obj->gziped) {
-		res_WriteGunzipObj(sp);
-	} else if (sp->wrk->res_mode & RES_GUNZIP) {
-		res_WriteGunzipObj(sp);
-	} else {
-		res_WriteDirObj(sp, low, high);
-	}
-
-	if (sp->wrk->res_mode & RES_CHUNKED &&
-	    !(sp->wrk->res_mode & RES_ESI_CHILD))
-		WRW_EndChunk(sp->wrk);
-
-	if (WRW_FlushRelease(sp->wrk) && sp->fd >= 0)
-		SES_Close(sp, "remote closed");
-}
-
-/*--------------------------------------------------------------------*/
-
-void
-RES_StreamStart(struct sess *sp)
-{
-	struct stream_ctx *sctx;
-
-	sctx = sp->wrk->sctx;
-	CHECK_OBJ_NOTNULL(sctx, STREAM_CTX_MAGIC);
-
-	AZ(sp->wrk->res_mode & RES_ESI_CHILD);
-	AN(sp->wantbody);
-
-	WRW_Reserve(sp->wrk, &sp->fd);
-	/*
-	 * Always remove C-E if client don't grok it
-	 */
-	if (sp->wrk->res_mode & RES_GUNZIP)
-		http_Unset(sp->wrk->resp, H_Content_Encoding);
-
-	if (!(sp->wrk->res_mode & RES_CHUNKED) &&
-	    sp->wrk->h_content_length != NULL)
-		http_PrintfHeader(sp->wrk, sp->vsl_id, sp->wrk->resp,
-		    "Content-Length: %s", sp->wrk->h_content_length);
-
-	sp->wrk->acct_tmp.hdrbytes +=
-	    http_Write(sp->wrk, sp->vsl_id, sp->wrk->resp, 1);
-
-	if (sp->wrk->res_mode & RES_CHUNKED)
-		WRW_Chunked(sp->wrk);
-}
-
-void
-RES_StreamPoll(struct worker *w)
-{
-	struct stream_ctx *sctx;
-	struct storage *st;
-	ssize_t l, l2;
-	void *ptr;
-
-	CHECK_OBJ_NOTNULL(w, WORKER_MAGIC);
-	CHECK_OBJ_NOTNULL(w->fetch_obj, OBJECT_MAGIC);
-	sctx = w->sctx;
-	CHECK_OBJ_NOTNULL(sctx, STREAM_CTX_MAGIC);
-	if (w->fetch_obj->len == sctx->stream_next)
-		return;
-	assert(w->fetch_obj->len > sctx->stream_next);
-	l = sctx->stream_front;
-	VTAILQ_FOREACH(st, &w->fetch_obj->store, list) {
-		if (st->len + l <= sctx->stream_next) {
-			l += st->len;
-			continue;
-		}
-		l2 = st->len + l - sctx->stream_next;
-		ptr = st->ptr + (sctx->stream_next - l);
-		if (w->res_mode & RES_GUNZIP) {
-			(void)VGZ_WrwGunzip(w, sctx->vgz, ptr, l2,
-			    sctx->obuf, sctx->obuf_len, &sctx->obuf_ptr);
-		} else {
-			(void)WRW_Write(w, ptr, l2);
-		}
-		l += st->len;
-		sctx->stream_next += l2;
-	}
-	if (!(w->res_mode & RES_GUNZIP))
-		(void)WRW_Flush(w);
-
-	if (w->fetch_obj->objcore == NULL ||
-	    (w->fetch_obj->objcore->flags & OC_F_PASS)) {
-		/*
-		 * This is a pass object, release storage as soon as we
-		 * have delivered it.
-		 */
-		while (1) {
-			st = VTAILQ_FIRST(&w->fetch_obj->store);
-			if (st == NULL ||
-			    sctx->stream_front + st->len > sctx->stream_next)
-				break;
-			VTAILQ_REMOVE(&w->fetch_obj->store, st, list);
-			sctx->stream_front += st->len;
-			STV_free(st);
-		}
-	}
-}
-
-void
-RES_StreamEnd(struct sess *sp)
-{
-	struct stream_ctx *sctx;
-
-	sctx = sp->wrk->sctx;
-	CHECK_OBJ_NOTNULL(sctx, STREAM_CTX_MAGIC);
-
-	if (sp->wrk->res_mode & RES_GUNZIP && sctx->obuf_ptr > 0)
-		(void)WRW_Write(sp->wrk, sctx->obuf, sctx->obuf_ptr);
-	if (sp->wrk->res_mode & RES_CHUNKED &&
-	    !(sp->wrk->res_mode & RES_ESI_CHILD))
-		WRW_EndChunk(sp->wrk);
-	if (WRW_FlushRelease(sp->wrk))
-		SES_Close(sp, "remote closed");
-}
diff --git a/bin/varnishd/cache_rfc2616.c b/bin/varnishd/cache_rfc2616.c
deleted file mode 100644
index 4041f45..0000000
--- a/bin/varnishd/cache_rfc2616.c
+++ /dev/null
@@ -1,336 +0,0 @@
-/*-
- * Copyright (c) 2006 Verdens Gang AS
- * Copyright (c) 2006-2011 Varnish Software AS
- * All rights reserved.
- *
- * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- */
-
-#include "config.h"
-
-#include <math.h>
-#include <stdlib.h>
-
-#include "cache.h"
-
-#include "vtim.h"
-
-/*--------------------------------------------------------------------
- * TTL and Age calculation in Varnish
- *
- * RFC2616 has a lot to say about how caches should calculate the TTL
- * and expiry times of objects, but it sort of misses the case that
- * applies to Varnish:  the server-side cache.
- *
- * A normal cache, shared or single-client, has no symbiotic relationship
- * with the server, and therefore must take a very defensive attitude
- * if the Data/Expiry/Age/max-age data does not make sense.  Overall
- * the policy described in section 13 of RFC 2616 results in no caching
- * happening on the first little sign of trouble.
- *
- * Varnish on the other hand tries to offload as many transactions from
- * the backend as possible, and therefore just passing through everything
- * if there is a clock-skew between backend and Varnish is not a workable
- * choice.
- *
- * Varnish implements a policy which is RFC2616 compliant when there
- * is no clockskew, and falls as gracefully as possible otherwise.
- * Our "clockless cache" model is syntehsized from the bits of RFC2616
- * that talks about how a cache should react to a clockless origin server,
- * and more or less uses the inverse logic for the opposite relationship.
- *
- */
-
-void
-RFC2616_Ttl(const struct sess *sp)
-{
-	unsigned max_age, age;
-	double h_date, h_expires;
-	char *p;
-	const struct http *hp;
-
-	hp = sp->wrk->beresp;
-
-	assert(sp->wrk->exp.entered != 0.0 && !isnan(sp->wrk->exp.entered));
-	/* If all else fails, cache using default ttl */
-	sp->wrk->exp.ttl = cache_param->default_ttl;
-
-	max_age = age = 0;
-	h_expires = 0;
-	h_date = 0;
-
-	/*
-	 * Initial cacheability determination per [RFC2616, 13.4]
-	 * We do not support ranges yet, so 206 is out.
-	 */
-
-	if (http_GetHdr(hp, H_Age, &p)) {
-		age = strtoul(p, NULL, 0);
-		sp->wrk->exp.age = age;
-	}
-	if (http_GetHdr(hp, H_Expires, &p))
-		h_expires = VTIM_parse(p);
-
-	if (http_GetHdr(hp, H_Date, &p))
-		h_date = VTIM_parse(p);
-
-	switch (sp->err_code) {
-	default:
-		sp->wrk->exp.ttl = -1.;
-		break;
-	case 200: /* OK */
-	case 203: /* Non-Authoritative Information */
-	case 300: /* Multiple Choices */
-	case 301: /* Moved Permanently */
-	case 302: /* Moved Temporarily */
-	case 307: /* Temporary Redirect */
-	case 410: /* Gone */
-	case 404: /* Not Found */
-		/*
-		 * First find any relative specification from the backend
-		 * These take precedence according to RFC2616, 13.2.4
-		 */
-
-		if ((http_GetHdrField(hp, H_Cache_Control, "s-maxage", &p) ||
-		    http_GetHdrField(hp, H_Cache_Control, "max-age", &p)) &&
-		    p != NULL) {
-
-			if (*p == '-')
-				max_age = 0;
-			else
-				max_age = strtoul(p, NULL, 0);
-
-			if (age > max_age)
-				sp->wrk->exp.ttl = 0;
-			else
-				sp->wrk->exp.ttl = max_age - age;
-			break;
-		}
-
-		/* No expire header, fall back to default */
-		if (h_expires == 0)
-			break;
-
-
-		/* If backend told us it is expired already, don't cache. */
-		if (h_expires < h_date) {
-			sp->wrk->exp.ttl = 0;
-			break;
-		}
-
-		if (h_date == 0 ||
-		    fabs(h_date - sp->wrk->exp.entered) < cache_param->clock_skew) {
-			/*
-			 * If we have no Date: header or if it is
-			 * sufficiently close to our clock we will
-			 * trust Expires: relative to our own clock.
-			 */
-			if (h_expires < sp->wrk->exp.entered)
-				sp->wrk->exp.ttl = 0;
-			else
-				sp->wrk->exp.ttl = h_expires -
-				    sp->wrk->exp.entered;
-			break;
-		} else {
-			/*
-			 * But even if the clocks are out of whack we can still
-			 * derive a relative time from the two headers.
-			 * (the negative ttl case is caught above)
-			 */
-			sp->wrk->exp.ttl = (int)(h_expires - h_date);
-		}
-
-	}
-
-	/* calculated TTL, Our time, Date, Expires, max-age, age */
-	WSP(sp, SLT_TTL,
-	    "%u RFC %.0f %.0f %.0f %.0f %.0f %.0f %.0f %u",
-	    sp->xid, sp->wrk->exp.ttl, -1., -1., sp->wrk->exp.entered,
-	    sp->wrk->exp.age, h_date, h_expires, max_age);
-}
-
-/*--------------------------------------------------------------------
- * Body existence, fetch method and close policy.
- */
-
-enum body_status
-RFC2616_Body(const struct sess *sp)
-{
-	struct http *hp;
-	char *b;
-
-	hp = sp->wrk->beresp;
-
-	if (hp->protover < 11 && !http_HdrIs(hp, H_Connection, "keep-alive"))
-		sp->wrk->do_close = 1;
-	else if (http_HdrIs(hp, H_Connection, "close"))
-		sp->wrk->do_close = 1;
-	else
-		sp->wrk->do_close = 0;
-
-	if (!strcasecmp(http_GetReq(sp->wrk->bereq), "head")) {
-		/*
-		 * A HEAD request can never have a body in the reply,
-		 * no matter what the headers might say.
-		 * [RFC2516 4.3 p33]
-		 */
-		sp->wrk->stats.fetch_head++;
-		return (BS_NONE);
-	}
-
-	if (hp->status <= 199) {
-		/*
-		 * 1xx responses never have a body.
-		 * [RFC2616 4.3 p33]
-		 */
-		sp->wrk->stats.fetch_1xx++;
-		return (BS_NONE);
-	}
-
-	if (hp->status == 204) {
-		/*
-		 * 204 is "No Content", obviously don't expect a body.
-		 * [RFC2616 10.2.5 p60]
-		 */
-		sp->wrk->stats.fetch_204++;
-		return (BS_NONE);
-	}
-
-	if (hp->status == 304) {
-		/*
-		 * 304 is "Not Modified" it has no body.
-		 * [RFC2616 10.3.5 p63]
-		 */
-		sp->wrk->stats.fetch_304++;
-		return (BS_NONE);
-	}
-
-	if (http_HdrIs(hp, H_Transfer_Encoding, "chunked")) {
-		 sp->wrk->stats.fetch_chunked++;
-		return (BS_CHUNKED);
-	}
-
-	if (http_GetHdr(hp, H_Transfer_Encoding, &b)) {
-		sp->wrk->stats.fetch_bad++;
-		return (BS_ERROR);
-	}
-
-	if (http_GetHdr(hp, H_Content_Length, &sp->wrk->h_content_length)) {
-		sp->wrk->stats.fetch_length++;
-		return (BS_LENGTH);
-	}
-
-	if (http_HdrIs(hp, H_Connection, "keep-alive")) {
-		/*
-		 * Keep alive with neither TE=Chunked or C-Len is impossible.
-		 * We assume a zero length body.
-		 */
-		sp->wrk->stats.fetch_zero++;
-		return (BS_ZERO);
-	}
-
-	if (http_HdrIs(hp, H_Connection, "close")) {
-		/*
-		 * In this case, it is safe to just read what comes.
-		 */
-		sp->wrk->stats.fetch_close++;
-		return (BS_EOF);
-	}
-
-	if (hp->protover < 11) {
-		/*
-		 * With no Connection header, assume EOF.
-		 */
-		sp->wrk->stats.fetch_oldhttp++;
-		return (BS_EOF);
-	}
-
-	/*
-	 * Fall back to EOF transfer.
-	 */
-	sp->wrk->stats.fetch_eof++;
-	return (BS_EOF);
-}
-
-/*--------------------------------------------------------------------
- * Find out if the request can receive a gzip'ed response
- */
-
-unsigned
-RFC2616_Req_Gzip(const struct sess *sp)
-{
-
-
-	/*
-	 * "x-gzip" is for http/1.0 backwards compat, final note in 14.3
-	 * p104 says to not do q values for x-gzip, so we just test
-	 * for its existence.
-	 */
-	if (http_GetHdrData(sp->http, H_Accept_Encoding, "x-gzip", NULL))
-		return (1);
-
-	/*
-	 * "gzip" is the real thing, but the 'q' value must be nonzero.
-	 * We do not care a hoot if the client prefers some other
-	 * compression more than gzip: Varnish only does gzip.
-	 */
-	if (http_GetHdrQ(sp->http, H_Accept_Encoding, "gzip") > 0.)
-		return (1);
-
-	/* Bad client, no gzip. */
-	return (0);
-}
-
-/*--------------------------------------------------------------------*/
-
-int
-RFC2616_Do_Cond(const struct sess *sp)
-{
-	char *p, *e;
-	double ims;
-	int do_cond = 0;
-
-	/* RFC 2616 13.3.4 states we need to match both ETag
-	   and If-Modified-Since if present*/
-
-	if (http_GetHdr(sp->http, H_If_Modified_Since, &p) ) {
-		if (!sp->obj->last_modified)
-			return (0);
-		ims = VTIM_parse(p);
-		if (ims > sp->t_req)	/* [RFC2616 14.25] */
-			return (0);
-		if (sp->obj->last_modified > ims)
-			return (0);
-		do_cond = 1;
-	}
-
-	if (http_GetHdr(sp->http, H_If_None_Match, &p) &&
-	    http_GetHdr(sp->obj->http, H_ETag, &e)) {
-		if (strcmp(p,e) != 0)
-			return (0);
-		do_cond = 1;
-	}
-
-	return (do_cond);
-}
diff --git a/bin/varnishd/cache_session.c b/bin/varnishd/cache_session.c
deleted file mode 100644
index 7befbcc..0000000
--- a/bin/varnishd/cache_session.c
+++ /dev/null
@@ -1,419 +0,0 @@
-/*-
- * Copyright (c) 2006 Verdens Gang AS
- * Copyright (c) 2006-2010 Varnish Software AS
- * All rights reserved.
- *
- * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- *
- * Session management
- *
- * This is a little bit of a mixed back, containing both memory management
- * and various state-change functions.
- *
- */
-
-#include "config.h"
-
-#include <math.h>
-#include <stdlib.h>
-
-#include "cache.h"
-
-#include "waiter/cache_waiter.h"
-
-/*--------------------------------------------------------------------*/
-
-struct sessmem {
-	unsigned		magic;
-#define SESSMEM_MAGIC		0x555859c5
-
-	struct sesspool		*pool;
-
-	unsigned		workspace;
-	uint16_t		nhttp;
-	void			*wsp;
-	struct http		*http[2];
-	VTAILQ_ENTRY(sessmem)	list;
-
-	struct sess		sess;
-};
-
-struct sesspool {
-	unsigned		magic;
-#define SESSPOOL_MAGIC		0xd916e202
-	struct pool		*pool;
-	VTAILQ_HEAD(,sessmem)	freelist;
-	struct lock		mtx;
-	unsigned		nsess;
-	unsigned		dly_free_cnt;
-};
-
-/*--------------------------------------------------------------------
- * Charge statistics from worker to request and session.
- */
-
-void
-SES_Charge(struct sess *sp)
-{
-	struct acct *a = &sp->wrk->acct_tmp;
-
-	sp->req_bodybytes += a->bodybytes;
-
-#define ACCT(foo)				\
-	sp->wrk->stats.s_##foo += a->foo;	\
-	sp->acct_ses.foo += a->foo;		\
-	a->foo = 0;
-#include "tbl/acct_fields.h"
-#undef ACCT
-}
-
-/*--------------------------------------------------------------------
- * This function allocates a session + assorted peripheral data
- * structures in one single malloc operation.
- */
-
-static struct sessmem *
-ses_sm_alloc(void)
-{
-	struct sessmem *sm;
-	unsigned char *p, *q;
-	unsigned nws;
-	uint16_t nhttp;
-	unsigned l, hl;
-
-	/*
-	 * It is not necessary to lock these, but we need to
-	 * cache them locally, to make sure we get a consistent
-	 * view of the value.
-	 */
-	nws = cache_param->sess_workspace;
-	nhttp = (uint16_t)cache_param->http_max_hdr;
-
-	hl = HTTP_estimate(nhttp);
-	l = sizeof *sm + nws + 2 * hl;
-	VSC_C_main->sessmem_size = l;
-	p = malloc(l);
-	if (p == NULL)
-		return (NULL);
-	q = p + l;
-
-	/* Don't waste time zeroing the workspace */
-	memset(p, 0, l - nws);
-
-	sm = (void*)p;
-	p += sizeof *sm;
-
-	sm->magic = SESSMEM_MAGIC;
-	sm->workspace = nws;
-	sm->nhttp = nhttp;
-
-	sm->http[0] = HTTP_create(p, nhttp);
-	p += hl;
-
-	sm->http[1] = HTTP_create(p, nhttp);
-	p += hl;
-
-	sm->wsp = p;
-	p += nws;
-
-	assert(p == q);
-
-	return (sm);
-}
-
-/*--------------------------------------------------------------------
- * This prepares a session for use, based on its sessmem structure.
- */
-
-static void
-ses_setup(struct sessmem *sm)
-{
-	struct sess *sp;
-
-	CHECK_OBJ_NOTNULL(sm, SESSMEM_MAGIC);
-	sp = &sm->sess;
-	memset(sp, 0, sizeof *sp);
-
-	/* We assume that the sess has been zeroed by the time we get here */
-	AZ(sp->magic);
-
-	sp->magic = SESS_MAGIC;
-	sp->mem = sm;
-	sp->sockaddrlen = sizeof(sp->sockaddr);
-	sp->mysockaddrlen = sizeof(sp->mysockaddr);
-	sp->sockaddr.ss_family = sp->mysockaddr.ss_family = PF_UNSPEC;
-	sp->t_open = NAN;
-	sp->t_req = NAN;
-	sp->t_resp = NAN;
-	sp->t_end = NAN;
-	EXP_Clr(&sp->exp);
-
-	WS_Init(sp->ws, "sess", sm->wsp, sm->workspace);
-	sp->http = sm->http[0];
-	sp->http0 = sm->http[1];
-}
-
-/*--------------------------------------------------------------------
- * Get a new session, preferably by recycling an already ready one
- */
-
-struct sess *
-SES_New(struct worker *wrk, struct sesspool *pp)
-{
-	struct sessmem *sm;
-	struct sess *sp;
-	int do_alloc;
-
-	CHECK_OBJ_NOTNULL(pp, SESSPOOL_MAGIC);
-
-	do_alloc = 0;
-	Lck_Lock(&pp->mtx);
-	sm = VTAILQ_FIRST(&pp->freelist);
-	if (sm != NULL) {
-		VTAILQ_REMOVE(&pp->freelist, sm, list);
-	} else if (pp->nsess < cache_param->max_sess) {
-		pp->nsess++;
-		do_alloc = 1;
-	}
-	wrk->stats.sessmem_free += pp->dly_free_cnt;
-	pp->dly_free_cnt = 0;
-	Lck_Unlock(&pp->mtx);
-	if (do_alloc) {
-		sm = ses_sm_alloc();
-		if (sm != NULL) {
-			wrk->stats.sessmem_alloc++;
-			sm->pool = pp;
-			ses_setup(sm);
-		} else {
-			wrk->stats.sessmem_fail++;
-		}
-	} else if (sm == NULL) {
-		wrk->stats.sessmem_limit++;
-	}
-	if (sm == NULL)
-		return (NULL);
-	sp = &sm->sess;
-	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
-	return (sp);
-}
-
-/*--------------------------------------------------------------------
- * Allocate a session for use by background threads.
- */
-
-struct sess *
-SES_Alloc(void)
-{
-	struct sess *sp;
-	struct sessmem *sm;
-
-	sm = ses_sm_alloc();
-	AN(sm);
-	ses_setup(sm);
-	sp = &sm->sess;
-	sp->sockaddrlen = 0;
-	return (sp);
-}
-
-/*--------------------------------------------------------------------
- * Schedule a session back on a work-thread from its pool
- */
-
-int
-SES_Schedule(struct sess *sp)
-{
-	struct sessmem *sm;
-	struct sesspool *pp;
-
-	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
-
-	AZ(sp->wrk);
-
-	sm = sp->mem;
-	CHECK_OBJ_NOTNULL(sm, SESSMEM_MAGIC);
-
-	pp = sm->pool;
-	CHECK_OBJ_NOTNULL(pp, SESSPOOL_MAGIC);
-
-	AN(pp->pool);
-
-	if (Pool_Schedule(pp->pool, sp)) {
-		SES_Delete(sp, "dropped");
-		return (1);
-	}
-	return (0);
-}
-
-/*--------------------------------------------------------------------
- * Handle a session (from waiter)
- *
- * Status: see HTC_Rx()
- */
-
-void
-SES_Handle(struct sess *sp, int status)
-{
-
-	switch (status) {
-	case -2:
-		SES_Delete(sp, "blast");
-		break;
-	case -1:
-		SES_Delete(sp, "no request");
-		break;
-	case 1:
-		sp->step = STP_START;
-		(void)SES_Schedule(sp);
-		break;
-	default:
-		WRONG("Unexpected return from HTC_Rx()");
-	}
-}
-
-/*--------------------------------------------------------------------
- * Close a sessions connection.
- */
-
-void
-SES_Close(struct sess *sp, const char *reason)
-{
-	int i;
-
-	assert(sp->fd >= 0);
-	VSL(SLT_SessionClose, sp->vsl_id, "%s", reason);
-	i = close(sp->fd);
-	assert(i == 0 || errno != EBADF); /* XXX EINVAL seen */
-	sp->fd = -1;
-}
-
-/*--------------------------------------------------------------------
- * (Close &) Free or Recycle a session.
- *
- * If the workspace has changed, deleted it, otherwise wash it, and put
- * it up for adoption.
- *
- * XXX: We should also check nhttp
- */
-
-void
-SES_Delete(struct sess *sp, const char *reason)
-{
-	struct acct *b;
-	struct sessmem *sm;
-	static char noaddr[] = "-";
-	struct worker *wrk;
-	struct sesspool *pp;
-
-
-	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
-	sm = sp->mem;
-	CHECK_OBJ_NOTNULL(sm, SESSMEM_MAGIC);
-	pp = sm->pool;
-	CHECK_OBJ_NOTNULL(pp, SESSPOOL_MAGIC);
-	wrk = sp->wrk;
-	CHECK_OBJ_ORNULL(wrk, WORKER_MAGIC);
-
-
-	if (reason != NULL)
-		SES_Close(sp, reason);
-	assert(sp->fd < 0);
-
-	AZ(sp->obj);
-	AZ(sp->vcl);
-	if (sp->addr == NULL)
-		sp->addr = noaddr;
-	if (sp->port == NULL)
-		sp->port = noaddr;
-
-	b = &sp->acct_ses;
-	assert(!isnan(b->first));
-	assert(!isnan(sp->t_end));
-
-	VSL(SLT_StatSess, sp->vsl_id, "%s %s %.0f %ju %ju %ju %ju %ju %ju %ju",
-	    sp->addr, sp->port, sp->t_end - b->first,
-	    b->sess, b->req, b->pipe, b->pass,
-	    b->fetch, b->hdrbytes, b->bodybytes);
-
-	if (sm->workspace != cache_param->sess_workspace ||
-	    sm->nhttp != (uint16_t)cache_param->http_max_hdr ||
-	    pp->nsess > cache_param->max_sess) {
-		free(sm);
-		Lck_Lock(&pp->mtx);
-		if (wrk != NULL)
-			wrk->stats.sessmem_free++;
-		else
-			pp->dly_free_cnt++;
-		pp->nsess--;
-		Lck_Unlock(&pp->mtx);
-	} else {
-		/* Clean and prepare for reuse */
-		ses_setup(sm);
-		Lck_Lock(&pp->mtx);
-		if (wrk != NULL) {
-			wrk->stats.sessmem_free += pp->dly_free_cnt;
-			pp->dly_free_cnt = 0;
-		}
-		VTAILQ_INSERT_HEAD(&pp->freelist, sm, list);
-		Lck_Unlock(&pp->mtx);
-	}
-}
-
-/*--------------------------------------------------------------------
- * Create and delete pools
- */
-
-struct sesspool *
-SES_NewPool(struct pool *pp)
-{
-	struct sesspool *sp;
-
-	ALLOC_OBJ(sp, SESSPOOL_MAGIC);
-	AN(sp);
-	sp->pool = pp;
-	VTAILQ_INIT(&sp->freelist);
-	Lck_New(&sp->mtx, lck_sessmem);
-	return (sp);
-}
-
-void
-SES_DeletePool(struct sesspool *sp, struct worker *wrk)
-{
-	struct sessmem *sm;
-
-	CHECK_OBJ_NOTNULL(sp, SESSPOOL_MAGIC);
-	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
-	Lck_Lock(&sp->mtx);
-	while (!VTAILQ_EMPTY(&sp->freelist)) {
-		sm = VTAILQ_FIRST(&sp->freelist);
-		CHECK_OBJ_NOTNULL(sm, SESSMEM_MAGIC);
-		VTAILQ_REMOVE(&sp->freelist, sm, list);
-		FREE_OBJ(sm);
-		wrk->stats.sessmem_free++;
-		sp->nsess--;
-	}
-	AZ(sp->nsess);
-	Lck_Unlock(&sp->mtx);
-	Lck_Delete(&sp->mtx);
-	FREE_OBJ(sp);
-}
diff --git a/bin/varnishd/cache_shmlog.c b/bin/varnishd/cache_shmlog.c
deleted file mode 100644
index 1252fa3..0000000
--- a/bin/varnishd/cache_shmlog.c
+++ /dev/null
@@ -1,346 +0,0 @@
-/*-
- * Copyright (c) 2006 Verdens Gang AS
- * Copyright (c) 2006-2011 Varnish Software AS
- * All rights reserved.
- *
- * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- */
-
-#include "config.h"
-
-#include <stdio.h>
-
-#include "cache.h"
-
-#include "cache_backend.h"	// For w->vbc
-
-#include "vapi/vsm_int.h"
-#include "vmb.h"
-#include "vtim.h"
-
-/* These cannot be struct lock, which depends on vsm/vsl working */
-static pthread_mutex_t vsl_mtx;
-static pthread_mutex_t vsm_mtx;
-
-static uint32_t			*vsl_start;
-static const uint32_t		*vsl_end;
-static uint32_t			*vsl_ptr;
-
-static inline uint32_t
-vsl_w0(uint32_t type, uint32_t length)
-{
-
-	assert(length < 0x10000);
-        return (((type & 0xff) << 24) | length);
-}
-
-/*--------------------------------------------------------------------*/
-
-static inline void
-vsl_hdr(enum VSL_tag_e tag, uint32_t *p, unsigned len, unsigned id)
-{
-
-	assert(((uintptr_t)p & 0x3) == 0);
-
-	p[1] = id;
-	VMB();
-	p[0] = vsl_w0(tag, len);
-}
-
-/*--------------------------------------------------------------------*/
-
-static void
-vsl_wrap(void)
-{
-
-	assert(vsl_ptr >= vsl_start + 1);
-	assert(vsl_ptr < vsl_end);
-	vsl_start[1] = VSL_ENDMARKER;
-	do
-		vsl_start[0]++;
-	while (vsl_start[0] == 0);
-	VWMB();
-	if (vsl_ptr != vsl_start + 1) {
-		*vsl_ptr = VSL_WRAPMARKER;
-		vsl_ptr = vsl_start + 1;
-	}
-	VSC_C_main->shm_cycles++;
-}
-
-/*--------------------------------------------------------------------
- * Reserve bytes for a record, wrap if necessary
- */
-
-static uint32_t *
-vsl_get(unsigned len, unsigned records, unsigned flushes)
-{
-	uint32_t *p;
-
-	if (pthread_mutex_trylock(&vsl_mtx)) {
-		AZ(pthread_mutex_lock(&vsl_mtx));
-		VSC_C_main->shm_cont++;
-	}
-	assert(vsl_ptr < vsl_end);
-	assert(((uintptr_t)vsl_ptr & 0x3) == 0);
-
-	VSC_C_main->shm_writes++;
-	VSC_C_main->shm_flushes += flushes;
-	VSC_C_main->shm_records += records;
-
-	/* Wrap if necessary */
-	if (VSL_END(vsl_ptr, len) >= vsl_end)
-		vsl_wrap();
-
-	p = vsl_ptr;
-	vsl_ptr = VSL_END(vsl_ptr, len);
-
-	*vsl_ptr = VSL_ENDMARKER;
-
-	assert(vsl_ptr < vsl_end);
-	assert(((uintptr_t)vsl_ptr & 0x3) == 0);
-	AZ(pthread_mutex_unlock(&vsl_mtx));
-
-	return (p);
-}
-
-/*--------------------------------------------------------------------
- * This variant copies a byte-range directly to the log, without
- * taking the detour over sprintf()
- */
-
-static void
-VSLR(enum VSL_tag_e tag, int id, const char *b, unsigned len)
-{
-	uint32_t *p;
-	unsigned mlen;
-
-	mlen = cache_param->shm_reclen;
-
-	/* Truncate */
-	if (len > mlen)
-		len = mlen;
-
-	p = vsl_get(len, 1, 0);
-
-	memcpy(p + 2, b, len);
-	vsl_hdr(tag, p, len, id);
-}
-
-/*--------------------------------------------------------------------*/
-
-void
-VSL(enum VSL_tag_e tag, int id, const char *fmt, ...)
-{
-	va_list ap;
-	unsigned n, mlen = cache_param->shm_reclen;
-	char buf[mlen];
-
-	/*
-	 * XXX: consider formatting into a stack buffer then move into
-	 * XXX: shmlog with VSLR().
-	 */
-	AN(fmt);
-	va_start(ap, fmt);
-
-	if (strchr(fmt, '%') == NULL) {
-		VSLR(tag, id, fmt, strlen(fmt));
-	} else {
-		n = vsnprintf(buf, mlen, fmt, ap);
-		if (n > mlen)
-			n = mlen;
-		VSLR(tag, id, buf, n);
-	}
-	va_end(ap);
-}
-
-/*--------------------------------------------------------------------*/
-
-void
-WSL_Flush(struct worker *w, int overflow)
-{
-	uint32_t *p;
-	unsigned l;
-
-	l = pdiff(w->wlb, w->wlp);
-	if (l == 0)
-		return;
-
-	assert(l >= 8);
-
-	p = vsl_get(l - 8, w->wlr, overflow);
-
-	memcpy(p + 1, w->wlb + 1, l - 4);
-	VWMB();
-	p[0] = w->wlb[0];
-	w->wlp = w->wlb;
-	w->wlr = 0;
-}
-
-/*--------------------------------------------------------------------*/
-
-void
-WSLR(struct worker *w, enum VSL_tag_e tag, int id, txt t)
-{
-	unsigned l, mlen;
-
-	Tcheck(t);
-	mlen = cache_param->shm_reclen;
-
-	/* Truncate */
-	l = Tlen(t);
-	if (l > mlen) {
-		l = mlen;
-		t.e = t.b + l;
-	}
-
-	assert(w->wlp < w->wle);
-
-	/* Wrap if necessary */
-	if (VSL_END(w->wlp, l) >= w->wle)
-		WSL_Flush(w, 1);
-	assert (VSL_END(w->wlp, l) < w->wle);
-	memcpy(VSL_DATA(w->wlp), t.b, l);
-	vsl_hdr(tag, w->wlp, l, id);
-	w->wlp = VSL_END(w->wlp, l);
-	assert(w->wlp < w->wle);
-	w->wlr++;
-	if (cache_param->diag_bitmap & 0x10000)
-		WSL_Flush(w, 0);
-}
-
-/*--------------------------------------------------------------------*/
-
-static void
-wsl(struct worker *w, enum VSL_tag_e tag, int id, const char *fmt, va_list ap)
-{
-	char *p;
-	unsigned n, mlen;
-	txt t;
-
-	AN(fmt);
-	mlen = cache_param->shm_reclen;
-
-	if (strchr(fmt, '%') == NULL) {
-		t.b = TRUST_ME(fmt);
-		t.e = strchr(t.b, '\0');
-		WSLR(w, tag, id, t);
-	} else {
-		assert(w->wlp < w->wle);
-
-		/* Wrap if we cannot fit a full size record */
-		if (VSL_END(w->wlp, mlen) >= w->wle)
-			WSL_Flush(w, 1);
-
-		p = VSL_DATA(w->wlp);
-		n = vsnprintf(p, mlen, fmt, ap);
-		if (n > mlen)
-			n = mlen;	/* we truncate long fields */
-		vsl_hdr(tag, w->wlp, n, id);
-		w->wlp = VSL_END(w->wlp, n);
-		assert(w->wlp < w->wle);
-		w->wlr++;
-	}
-	if (cache_param->diag_bitmap & 0x10000)
-		WSL_Flush(w, 0);
-}
-
-/*--------------------------------------------------------------------*/
-
-void
-WSL(struct worker *w, enum VSL_tag_e tag, int id, const char *fmt, ...)
-{
-	va_list ap;
-
-	CHECK_OBJ_NOTNULL(w, WORKER_MAGIC);
-	AN(fmt);
-	va_start(ap, fmt);
-	wsl(w, tag, id, fmt, ap);
-	va_end(ap);
-}
-
-
-/*--------------------------------------------------------------------*/
-
-void
-WSLB(struct worker *w, enum VSL_tag_e tag, const char *fmt, ...)
-{
-	va_list ap;
-
-	CHECK_OBJ_NOTNULL(w, WORKER_MAGIC);
-	CHECK_OBJ_NOTNULL(w->vbc, VBC_MAGIC);
-	AN(fmt);
-	va_start(ap, fmt);
-	wsl(w, tag, w->vbc->vsl_id, fmt, ap);
-	va_end(ap);
-}
-
-/*--------------------------------------------------------------------*/
-
-void
-VSL_Init(void)
-{
-	struct VSM_chunk *vsc;
-
-	AZ(pthread_mutex_init(&vsl_mtx, NULL));
-	AZ(pthread_mutex_init(&vsm_mtx, NULL));
-
-	VSM__Clean();
-
-	VSM_ITER(vsc)
-		if (!strcmp(vsc->class, VSL_CLASS))
-			break;
-	AN(vsc);
-	vsl_start = VSM_PTR(vsc);
-	vsl_end = VSM_NEXT(vsc);
-	vsl_ptr = vsl_start + 1;
-
-	vsl_wrap();
-	VSM_head->starttime = (intmax_t)VTIM_real();
-	memset(VSM_head->panicstr, '\0', sizeof *VSM_head->panicstr);
-	memset(VSC_C_main, 0, sizeof *VSC_C_main);
-	VSM_head->child_pid = getpid();
-}
-
-/*--------------------------------------------------------------------*/
-
-void *
-VSM_Alloc(unsigned size, const char *class, const char *type,
-    const char *ident)
-{
-	void *p;
-
-	AZ(pthread_mutex_lock(&vsm_mtx));
-	p = VSM__Alloc(size, class, type, ident);
-	AZ(pthread_mutex_unlock(&vsm_mtx));
-	return (p);
-}
-
-void
-VSM_Free(const void *ptr)
-{
-
-	AZ(pthread_mutex_lock(&vsm_mtx));
-	VSM__Free(ptr);
-	AZ(pthread_mutex_unlock(&vsm_mtx));
-}
diff --git a/bin/varnishd/cache_vary.c b/bin/varnishd/cache_vary.c
deleted file mode 100644
index 026f937..0000000
--- a/bin/varnishd/cache_vary.c
+++ /dev/null
@@ -1,257 +0,0 @@
-/*-
- * Copyright (c) 2006-2011 Varnish Software AS
- * All rights reserved.
- *
- * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- *
- * Do Vary processing.
- *
- * When we insert an object into the cache which has a Vary: header,
- * we encode a vary matching string containing the headers mentioned
- * and their value.
- *
- * When we match an object in the cache, we check the present request
- * against the vary matching string.
- *
- * The only kind of header-munging we do is leading & trailing space
- * removal.  All the potential "q=foo" gymnastics is not worth the
- * effort.
- *
- * The vary matching string has the following format:
- *
- * Sequence of: {
- *	<msb>			\   Length of header contents.
- *	<lsb>			/
- *	<length of header + 1>	\
- *	<header>		 \  Same format as argument to http_GetHdr()
- *	':'			 /
- *	'\0'			/
- *      <header>		>   Only present if length != 0xffff
- * }
- *      '\0'
- */
-
-#include "config.h"
-
-#include "cache.h"
-
-#include "vct.h"
-#include "vend.h"
-
-struct vsb *
-VRY_Create(const struct sess *sp, const struct http *hp)
-{
-	char *v, *p, *q, *h, *e;
-	struct vsb *sb, *sbh;
-	int l;
-
-	/* No Vary: header, no worries */
-	if (!http_GetHdr(hp, H_Vary, &v))
-		return (NULL);
-
-	/* For vary matching string */
-	sb = VSB_new_auto();
-	AN(sb);
-
-	/* For header matching strings */
-	sbh = VSB_new_auto();
-	AN(sbh);
-
-	if (*v == ':') {
-		WSP(sp, SLT_Error, "Vary header had extra ':', fix backend");
-		v++;
-	}
-	for (p = v; *p; p++) {
-
-		/* Find next header-name */
-		if (vct_issp(*p))
-			continue;
-		for (q = p; *q && !vct_issp(*q) && *q != ','; q++)
-			continue;
-
-		/* Build a header-matching string out of it */
-		VSB_clear(sbh);
-		VSB_printf(sbh, "%c%.*s:%c",
-		    (char)(1 + (q - p)), (int)(q - p), p, 0);
-		AZ(VSB_finish(sbh));
-
-		if (http_GetHdr(sp->http, VSB_data(sbh), &h)) {
-			AZ(vct_issp(*h));
-			/* Trim trailing space */
-			e = strchr(h, '\0');
-			while (e > h && vct_issp(e[-1]))
-				e--;
-			/* Encode two byte length and contents */
-			l = e - h;
-			assert(!(l & ~0xffff));
-		} else {
-			e = h;
-			l = 0xffff;
-		}
-		VSB_printf(sb, "%c%c", (unsigned)l >> 8, l & 0xff);
-		/* Append to vary matching string */
-		VSB_bcat(sb, VSB_data(sbh), VSB_len(sbh));
-		if (e != h)
-			VSB_bcat(sb, h, e - h);
-
-		while (vct_issp(*q))
-			q++;
-		if (*q == '\0')
-			break;
-		xxxassert(*q == ',');
-		p = q;
-	}
-	/* Terminate vary matching string */
-	VSB_printf(sb, "%c%c%c", 0xff, 0xff, 0);
-
-	VSB_delete(sbh);
-	AZ(VSB_finish(sb));
-	return(sb);
-}
-
-/*
- * Find length of a vary entry
- */
-static unsigned
-vry_len(const uint8_t *p)
-{
-	unsigned l = vbe16dec(p);
-
-	return (2 + p[2] + 2 + (l == 0xffff ? 0 : l));
-}
-
-/*
- * Compare two vary entries
- */
-static int
-vry_cmp(const uint8_t * const *v1, uint8_t * const *v2)
-{
-	unsigned retval = 0;
-
-	if (!memcmp(*v1, *v2, vry_len(*v1))) {
-		/* Same same */
-		retval = 0;
-	} else if (memcmp((*v1) + 2, (*v2) + 2, (*v1)[2] + 2)) {
-		/* Different header */
-		retval = 1;
-	} else if (cache_param->http_gzip_support &&
-	    !strcasecmp(H_Accept_Encoding, (const char*)((*v1)+2))) {
-		/*
-		 * If we do gzip processing, we do not vary on Accept-Encoding,
-		 * because we want everybody to get the gzip'ed object, and
-		 * varnish will gunzip as necessary.  We implement the skip at
-		 * check time, rather than create time, so that object in
-		 * persistent storage can be used with either setting of
-		 * http_gzip_support.
-		 */
-		retval = 0;
-	} else {
-		/* Same header, different content */
-		retval = 2;
-	}
-	return (retval);
-}
-
-int
-VRY_Match(struct sess *sp, const uint8_t *vary)
-{
-	uint8_t *vsp = sp->vary_b;
-	char *h, *e;
-	unsigned lh, ln;
-	int i, retval = 1, oflo = 0;
-
-	AN(vsp);
-	while (vary[2]) {
-		i = vry_cmp(&vary, &vsp);
-		if (i == 1) {
-			/* Build a new entry */
-
-			i = http_GetHdr(sp->http, (const char*)(vary+2), &h);
-			if (i) {
-				/* Trim trailing space */
-				e = strchr(h, '\0');
-				while (e > h && vct_issp(e[-1]))
-					e--;
-				lh = e - h;
-				assert(lh < 0xffff);
-			} else {
-				e = h = NULL;
-				lh = 0xffff;
-			}
-
-			/* Length of the entire new vary entry */
-			ln = 2 + vary[2] + 2 + (lh == 0xffff ? 0 : lh);
-			if (vsp + ln >= sp->vary_e) {
-				vsp = sp->vary_b;
-				oflo = 1;
-			}
-
-			/*
-			 * We MUST have space for one entry and the end marker
-			 * after it, which prevents old junk from confusing us
-			 */
-			assert(vsp + ln + 2 < sp->vary_e);
-
-			vbe16enc(vsp, (uint16_t)lh);
-			memcpy(vsp + 2, vary + 2, vary[2] + 2);
-			if (h != NULL && e != NULL) {
-				memcpy(vsp + 2 + vsp[2] + 2, h, e - h);
-				vsp[2 + vary[2] + 2 + (e - h) + 2] = '\0';
-			} else
-				vsp[2 + vary[2] + 2 + 2] = '\0';
-
-			i = vry_cmp(&vary, &vsp);
-			assert(i != 1);	/* hdr must be the same now */
-		}
-		if (i != 0)
-			retval = 0;
-		vsp += vry_len(vsp);
-		vary += vry_len(vary);
-	}
-	if (vsp + 3 > sp->vary_e)
-		oflo = 1;
-
-	if (oflo) {
-		/* XXX: Should log this */
-		vsp = sp->vary_b;
-	}
-	vsp[0] = 0xff;
-	vsp[1] = 0xff;
-	vsp[2] = 0;
-	if (oflo)
-		sp->vary_l = NULL;
-	else
-		sp->vary_l = vsp + 3;
-	return (retval);
-}
-
-void
-VRY_Validate(const uint8_t *vary)
-{
-
-	while (vary[2] != 0) {
-		assert(strlen((const char*)vary+3) == vary[2]);
-		vary += vry_len(vary);
-	}
-}
diff --git a/bin/varnishd/cache_vcl.c b/bin/varnishd/cache_vcl.c
deleted file mode 100644
index 068b482..0000000
--- a/bin/varnishd/cache_vcl.c
+++ /dev/null
@@ -1,365 +0,0 @@
-/*-
- * Copyright (c) 2006 Verdens Gang AS
- * Copyright (c) 2006-2011 Varnish Software AS
- * All rights reserved.
- *
- * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- *
- * Interface *to* compiled VCL code:  Loading, unloading, calling into etc.
- *
- * The interface *from* the compiled VCL code is in cache_vrt.c.
- */
-
-#include "config.h"
-
-#include <dlfcn.h>
-#include <stdlib.h>
-
-#include "cache.h"
-
-#include "vcl.h"
-#include "vcli.h"
-#include "vcli_priv.h"
-
-struct vcls {
-	unsigned		magic;
-#define VVCLS_MAGIC		0x214188f2
-	VTAILQ_ENTRY(vcls)	list;
-	char			*name;
-	void			*dlh;
-	struct VCL_conf		conf[1];
-};
-
-/*
- * XXX: Presently all modifications to this list happen from the
- * CLI event-engine, so no locking is necessary
- */
-static VTAILQ_HEAD(, vcls)	vcl_head =
-    VTAILQ_HEAD_INITIALIZER(vcl_head);
-
-
-static struct lock		vcl_mtx;
-static struct vcls		*vcl_active; /* protected by vcl_mtx */
-
-/*--------------------------------------------------------------------*/
-
-const char *
-VCL_Return_Name(unsigned method)
-{
-
-	switch (method) {
-#define VCL_RET_MAC(l, U, B) case VCL_RET_##U: return(#l);
-#include "tbl/vcl_returns.h"
-#undef VCL_RET_MAC
-	default:
-		return (NULL);
-	}
-}
-
-/*--------------------------------------------------------------------*/
-
-static void
-VCL_Get(struct VCL_conf **vcc)
-{
-	static int once = 0;
-
-	while (!once && vcl_active == NULL) {
-		(void)sleep(1);
-	}
-	once = 1;
-
-	Lck_Lock(&vcl_mtx);
-	AN(vcl_active);
-	*vcc = vcl_active->conf;
-	AN(*vcc);
-	AZ((*vcc)->discard);
-	(*vcc)->busy++;
-	Lck_Unlock(&vcl_mtx);
-}
-
-void
-VCL_Refresh(struct VCL_conf **vcc)
-{
-	if (*vcc == vcl_active->conf)
-		return;
-	if (*vcc != NULL)
-		VCL_Rel(vcc);	/* XXX: optimize locking */
-	VCL_Get(vcc);
-}
-
-void
-VCL_Rel(struct VCL_conf **vcc)
-{
-	struct VCL_conf *vc;
-
-	AN(*vcc);
-	vc = *vcc;
-	*vcc = NULL;
-
-	Lck_Lock(&vcl_mtx);
-	assert(vc->busy > 0);
-	vc->busy--;
-	/*
-	 * We do not garbage collect discarded VCL's here, that happens
-	 * in VCL_Poll() which is called from the CLI thread.
-	 */
-	Lck_Unlock(&vcl_mtx);
-}
-
-/*--------------------------------------------------------------------*/
-
-static struct vcls *
-vcl_find(const char *name)
-{
-	struct vcls *vcl;
-
-	ASSERT_CLI();
-	VTAILQ_FOREACH(vcl, &vcl_head, list) {
-		if (vcl->conf->discard)
-			continue;
-		if (!strcmp(vcl->name, name))
-			return (vcl);
-	}
-	return (NULL);
-}
-
-static int
-VCL_Load(const char *fn, const char *name, struct cli *cli)
-{
-	struct vcls *vcl;
-	struct VCL_conf const *cnf;
-
-	ASSERT_CLI();
-	vcl = vcl_find(name);
-	if (vcl != NULL) {
-		VCLI_Out(cli, "Config '%s' already loaded", name);
-		return (1);
-	}
-
-	ALLOC_OBJ(vcl, VVCLS_MAGIC);
-	XXXAN(vcl);
-
-	vcl->dlh = dlopen(fn, RTLD_NOW | RTLD_LOCAL);
-
-	if (vcl->dlh == NULL) {
-		VCLI_Out(cli, "dlopen(%s): %s\n", fn, dlerror());
-		FREE_OBJ(vcl);
-		return (1);
-	}
-	cnf = dlsym(vcl->dlh, "VCL_conf");
-	if (cnf == NULL) {
-		VCLI_Out(cli, "Internal error: No VCL_conf symbol\n");
-		(void)dlclose(vcl->dlh);
-		FREE_OBJ(vcl);
-		return (1);
-	}
-	memcpy(vcl->conf, cnf, sizeof *cnf);
-
-	if (vcl->conf->magic != VCL_CONF_MAGIC) {
-		VCLI_Out(cli, "Wrong VCL_CONF_MAGIC\n");
-		(void)dlclose(vcl->dlh);
-		FREE_OBJ(vcl);
-		return (1);
-	}
-	if (vcl->conf->init_vcl(cli)) {
-		VCLI_Out(cli, "VCL \"%s\" Failed to initialize", name);
-		(void)dlclose(vcl->dlh);
-		FREE_OBJ(vcl);
-		return (1);
-	}
-	REPLACE(vcl->name, name);
-	VCLI_Out(cli, "Loaded \"%s\" as \"%s\"", fn , name);
-	VTAILQ_INSERT_TAIL(&vcl_head, vcl, list);
-	(void)vcl->conf->init_func(NULL);
-	Lck_Lock(&vcl_mtx);
-	if (vcl_active == NULL)
-		vcl_active = vcl;
-	Lck_Unlock(&vcl_mtx);
-	VSC_C_main->n_vcl++;
-	VSC_C_main->n_vcl_avail++;
-	return (0);
-}
-
-/*--------------------------------------------------------------------
- * This function is polled from the CLI thread to dispose of any non-busy
- * VCLs which have been discarded.
- */
-
-static void
-VCL_Nuke(struct vcls *vcl)
-{
-
-	ASSERT_CLI();
-	assert(vcl != vcl_active);
-	assert(vcl->conf->discard);
-	assert(vcl->conf->busy == 0);
-	VTAILQ_REMOVE(&vcl_head, vcl, list);
-	(void)vcl->conf->fini_func(NULL);
-	vcl->conf->fini_vcl(NULL);
-	free(vcl->name);
-	(void)dlclose(vcl->dlh);
-	FREE_OBJ(vcl);
-	VSC_C_main->n_vcl--;
-	VSC_C_main->n_vcl_discard--;
-}
-
-/*--------------------------------------------------------------------*/
-
-void
-VCL_Poll(void)
-{
-	struct vcls *vcl, *vcl2;
-
-	ASSERT_CLI();
-	VTAILQ_FOREACH_SAFE(vcl, &vcl_head, list, vcl2)
-		if (vcl->conf->discard && vcl->conf->busy == 0)
-			VCL_Nuke(vcl);
-}
-
-/*--------------------------------------------------------------------*/
-
-static void
-ccf_config_list(struct cli *cli, const char * const *av, void *priv)
-{
-	struct vcls *vcl;
-	const char *flg;
-
-	(void)av;
-	(void)priv;
-	ASSERT_CLI();
-	VTAILQ_FOREACH(vcl, &vcl_head, list) {
-		if (vcl == vcl_active) {
-			flg = "active";
-		} else if (vcl->conf->discard) {
-			flg = "discarded";
-		} else
-			flg = "available";
-		VCLI_Out(cli, "%-10s %6u %s\n",
-		    flg,
-		    vcl->conf->busy,
-		    vcl->name);
-	}
-}
-
-static void
-ccf_config_load(struct cli *cli, const char * const *av, void *priv)
-{
-
-	(void)av;
-	(void)priv;
-	ASSERT_CLI();
-	if (VCL_Load(av[3], av[2], cli))
-		VCLI_SetResult(cli, CLIS_PARAM);
-	return;
-}
-
-static void
-ccf_config_discard(struct cli *cli, const char * const *av, void *priv)
-{
-	struct vcls *vcl;
-
-	ASSERT_CLI();
-	(void)av;
-	(void)priv;
-	vcl = vcl_find(av[2]);
-	if (vcl == NULL) {
-		VCLI_SetResult(cli, CLIS_PARAM);
-		VCLI_Out(cli, "VCL '%s' unknown", av[2]);
-		return;
-	}
-	Lck_Lock(&vcl_mtx);
-	if (vcl == vcl_active) {
-		Lck_Unlock(&vcl_mtx);
-		VCLI_SetResult(cli, CLIS_PARAM);
-		VCLI_Out(cli, "VCL %s is the active VCL", av[2]);
-		return;
-	}
-	VSC_C_main->n_vcl_discard++;
-	VSC_C_main->n_vcl_avail--;
-	vcl->conf->discard = 1;
-	Lck_Unlock(&vcl_mtx);
-	if (vcl->conf->busy == 0)
-		VCL_Nuke(vcl);
-}
-
-static void
-ccf_config_use(struct cli *cli, const char * const *av, void *priv)
-{
-	struct vcls *vcl;
-	int i;
-
-	(void)av;
-	(void)priv;
-	vcl = vcl_find(av[2]);
-	if (vcl == NULL) {
-		VCLI_Out(cli, "No VCL named '%s'", av[2]);
-		VCLI_SetResult(cli, CLIS_PARAM);
-		return;
-	}
-	Lck_Lock(&vcl_mtx);
-	vcl_active = vcl;
-	Lck_Unlock(&vcl_mtx);
-
-	/* Tickle this VCL's backends to take over health polling */
-	for(i = 1; i < vcl->conf->ndirector; i++)
-		VBE_UseHealth(vcl->conf->director[i]);
-}
-
-/*--------------------------------------------------------------------*/
-
-#define VCL_MET_MAC(func, upper, bitmap)				\
-void									\
-VCL_##func##_method(struct sess *sp)					\
-{									\
-									\
-	sp->handling = 0;						\
-	sp->cur_method = VCL_MET_ ## upper;				\
-	WSP(sp, SLT_VCL_call, "%s", #func);				\
-	(void)sp->vcl->func##_func(sp);					\
-	WSP(sp, SLT_VCL_return, "%s", VCL_Return_Name(sp->handling));	\
-	sp->cur_method = 0;						\
-	assert((1U << sp->handling) & bitmap);				\
-	assert(!((1U << sp->handling) & ~bitmap));			\
-}
-
-#include "tbl/vcl_returns.h"
-#undef VCL_MET_MAC
-
-/*--------------------------------------------------------------------*/
-
-static struct cli_proto vcl_cmds[] = {
-	{ CLI_VCL_LOAD,         "i", ccf_config_load },
-	{ CLI_VCL_LIST,         "i", ccf_config_list },
-	{ CLI_VCL_DISCARD,      "i", ccf_config_discard },
-	{ CLI_VCL_USE,          "i", ccf_config_use },
-	{ NULL }
-};
-
-void
-VCL_Init()
-{
-
-	CLI_AddFuncs(vcl_cmds);
-	Lck_New(&vcl_mtx, lck_vcl);
-}
diff --git a/bin/varnishd/cache_vrt.c b/bin/varnishd/cache_vrt.c
deleted file mode 100644
index c20b552..0000000
--- a/bin/varnishd/cache_vrt.c
+++ /dev/null
@@ -1,535 +0,0 @@
-/*-
- * Copyright (c) 2006 Verdens Gang AS
- * Copyright (c) 2006-2010 Varnish Software AS
- * All rights reserved.
- *
- * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- *
- * Runtime support for compiled VCL programs
- */
-
-#include "config.h"
-
-#include <netinet/in.h>
-#include <arpa/inet.h>
-
-#include <stdio.h>
-#include <stdlib.h>
-
-#include "cache.h"
-
-#include "cache_backend.h"
-#include "hash/hash_slinger.h"
-#include "vav.h"
-#include "vcl.h"
-#include "vrt.h"
-#include "vrt_obj.h"
-#include "vtim.h"
-
-const void * const vrt_magic_string_end = &vrt_magic_string_end;
-
-/*--------------------------------------------------------------------*/
-
-void
-VRT_error(struct sess *sp, unsigned code, const char *reason)
-{
-
-	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
-	WSL(sp->wrk, SLT_Debug, 0, "VCL_error(%u, %s)", code, reason ?
-	    reason : "(null)");
-	if (code < 100 || code > 999)
-		code = 503;
-	sp->err_code = (uint16_t)code;
-	sp->err_reason = reason ? reason : http_StatusMessage(sp->err_code);
-}
-
-/*--------------------------------------------------------------------*/
-
-void
-VRT_count(const struct sess *sp, unsigned u)
-{
-
-	if (sp == NULL)
-		return;
-	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
-	if (cache_param->vcl_trace)
-		WSP(sp, SLT_VCL_trace, "%u %d.%d", u,
-		    sp->vcl->ref[u].line, sp->vcl->ref[u].pos);
-}
-
-/*--------------------------------------------------------------------*/
-
-void
-VRT_acl_log(const struct sess *sp, const char *msg)
-{
-	WSP(sp, SLT_VCL_acl, msg);
-}
-
-/*--------------------------------------------------------------------*/
-
-static struct http *
-vrt_selecthttp(const struct sess *sp, enum gethdr_e where)
-{
-	struct http *hp;
-
-	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
-	switch (where) {
-	case HDR_REQ:
-		hp = sp->http;
-		break;
-	case HDR_BEREQ:
-		hp = sp->wrk->bereq;
-		break;
-	case HDR_BERESP:
-		hp = sp->wrk->beresp;
-		break;
-	case HDR_RESP:
-		hp = sp->wrk->resp;
-		break;
-	case HDR_OBJ:
-		CHECK_OBJ_NOTNULL(sp->obj, OBJECT_MAGIC);
-		hp = sp->obj->http;
-		break;
-	default:
-		INCOMPL();
-	}
-	CHECK_OBJ_NOTNULL(hp, HTTP_MAGIC);
-	return (hp);
-}
-
-char *
-VRT_GetHdr(const struct sess *sp, enum gethdr_e where, const char *n)
-{
-	char *p;
-	struct http *hp;
-
-	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
-	hp = vrt_selecthttp(sp, where);
-	if (!http_GetHdr(hp, n, &p))
-		return (NULL);
-	return (p);
-}
-
-/*--------------------------------------------------------------------
- * XXX: Optimize the single element case ?
- */
-
-char *
-VRT_StringList(char *d, unsigned dl, const char *p, va_list ap)
-{
-	char *b, *e;
-	unsigned x;
-
-	b = d;
-	e = b + dl;
-	while (p != vrt_magic_string_end && b < e) {
-		if (p != NULL) {
-			x = strlen(p);
-			if (b + x < e)
-				memcpy(b, p, x);
-			b += x;
-		}
-		p = va_arg(ap, const char *);
-	}
-	if (b >= e)
-		return (NULL);
-	*b++ = '\0';
-	return (b);
-}
-
-/*--------------------------------------------------------------------
- * XXX: Optimize the single element case ?
- */
-
-char *
-VRT_String(struct ws *ws, const char *h, const char *p, va_list ap)
-{
-	char *b, *e;
-	unsigned u, x;
-
-	u = WS_Reserve(ws, 0);
-	e = b = ws->f;
-	e += u;
-	if (h != NULL) {
-		x = strlen(h);
-		if (b + x < e)
-			memcpy(b, h, x);
-		b += x;
-		if (b < e)
-			*b = ' ';
-		b++;
-	}
-	b = VRT_StringList(b, e > b ? e - b : 0, p, ap);
-	if (b == NULL || b == e) {
-		WS_Release(ws, 0);
-		return (NULL);
-	}
-	e = b;
-	b = ws->f;
-	WS_Release(ws, e - b);
-	return (b);
-}
-
-/*--------------------------------------------------------------------
- * Build a string on the worker threads workspace
- */
-
-const char *
-VRT_WrkString(const struct sess *sp, const char *p, ...)
-{
-	va_list ap;
-	char *b;
-
-	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
-	va_start(ap, p);
-	b = VRT_String(sp->wrk->ws, NULL, p, ap);
-	va_end(ap);
-	return (b);
-}
-
-/*--------------------------------------------------------------------*/
-
-void
-VRT_SetHdr(const struct sess *sp , enum gethdr_e where, const char *hdr,
-    const char *p, ...)
-{
-	struct http *hp;
-	va_list ap;
-	char *b;
-
-	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
-	hp = vrt_selecthttp(sp, where);
-	va_start(ap, p);
-	if (p == NULL) {
-		http_Unset(hp, hdr);
-	} else {
-		b = VRT_String(hp->ws, hdr + 1, p, ap);
-		if (b == NULL) {
-			WSP(sp, SLT_LostHeader, "%s", hdr + 1);
-		} else {
-			http_Unset(hp, hdr);
-			http_SetHeader(sp->wrk, sp->vsl_id, hp, b);
-		}
-	}
-	va_end(ap);
-}
-
-/*--------------------------------------------------------------------*/
-
-void
-VRT_handling(struct sess *sp, unsigned hand)
-{
-
-	if (sp == NULL) {
-		assert(hand == VCL_RET_OK);
-		return;
-	}
-	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
-	assert(hand < VCL_RET_MAX);
-	sp->handling = hand;
-}
-
-/*--------------------------------------------------------------------
- * Add an element to the array/list of hash bits.
- */
-
-void
-VRT_hashdata(const struct sess *sp, const char *str, ...)
-{
-	va_list ap;
-	const char *p;
-
-	HSH_AddString(sp, str);
-	va_start(ap, str);
-	while (1) {
-		p = va_arg(ap, const char *);
-		if (p == vrt_magic_string_end)
-			break;
-		HSH_AddString(sp, p);
-	}
-}
-
-/*--------------------------------------------------------------------*/
-
-double
-VRT_r_now(const struct sess *sp)
-{
-
-	(void)sp;
-	return (VTIM_real());
-}
-
-/*--------------------------------------------------------------------*/
-
-char *
-VRT_IP_string(const struct sess *sp, const struct sockaddr_storage *sa)
-{
-	char *p;
-	const struct sockaddr_in *si4;
-	const struct sockaddr_in6 *si6;
-	const void *addr;
-	int len;
-
-	switch (sa->ss_family) {
-	case AF_INET:
-		len = INET_ADDRSTRLEN;
-		si4 = (const void *)sa;
-		addr = &(si4->sin_addr);
-		break;
-	case AF_INET6:
-		len = INET6_ADDRSTRLEN;
-		si6 = (const void *)sa;
-		addr = &(si6->sin6_addr);
-		break;
-	default:
-		INCOMPL();
-	}
-	XXXAN(len);
-	AN(p = WS_Alloc(sp->http->ws, len));
-	AN(inet_ntop(sa->ss_family, addr, p, len));
-	return (p);
-}
-
-char *
-VRT_int_string(const struct sess *sp, int num)
-{
-	char *p;
-	int size;
-
-	size = snprintf(NULL, 0, "%d", num) + 1;
-	AN(p = WS_Alloc(sp->http->ws, size));
-	assert(snprintf(p, size, "%d", num) < size);
-	return (p);
-}
-
-char *
-VRT_double_string(const struct sess *sp, double num)
-{
-	char *p;
-	int size;
-
-	size = snprintf(NULL, 0, "%.3f", num) + 1;
-	AN(p = WS_Alloc(sp->http->ws, size));
-	assert(snprintf(p, size, "%.3f", num) < size);
-	return (p);
-}
-
-char *
-VRT_time_string(const struct sess *sp, double t)
-{
-	char *p;
-
-	AN(p = WS_Alloc(sp->http->ws, VTIM_FORMAT_SIZE));
-	VTIM_format(t, p);
-	return (p);
-}
-
-const char *
-VRT_backend_string(const struct sess *sp, const struct director *d)
-{
-	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
-	if (d == NULL)
-		d = sp->director;
-	if (d == NULL)
-		return (NULL);
-	return (d->vcl_name);
-}
-
-const char *
-VRT_bool_string(const struct sess *sp, unsigned val)
-{
-
-	(void)sp;
-	return (val ? "true" : "false");
-}
-
-/*--------------------------------------------------------------------*/
-
-void
-VRT_Rollback(struct sess *sp)
-{
-
-	HTTP_Copy(sp->http, sp->http0);
-	WS_Reset(sp->ws, sp->ws_req);
-}
-
-/*--------------------------------------------------------------------*/
-
-void
-VRT_panic(const struct sess *sp, const char *str, ...)
-{
-	va_list ap;
-	char *b;
-
-	va_start(ap, str);
-	b = VRT_String(sp->http->ws, "PANIC: ", str, ap);
-	va_end(ap);
-	VAS_Fail("VCL", "", 0, b, 0, 2);
-}
-
-/*--------------------------------------------------------------------*/
-
-void
-VRT_synth_page(const struct sess *sp, unsigned flags, const char *str, ...)
-{
-	va_list ap;
-	const char *p;
-	struct vsb *vsb;
-
-	(void)flags;
-	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
-	CHECK_OBJ_NOTNULL(sp->obj, OBJECT_MAGIC);
-	vsb = SMS_Makesynth(sp->obj);
-	AN(vsb);
-
-	VSB_cat(vsb, str);
-	va_start(ap, str);
-	p = va_arg(ap, const char *);
-	while (p != vrt_magic_string_end) {
-		if (p == NULL)
-			p = "(null)";
-		VSB_cat(vsb, p);
-		p = va_arg(ap, const char *);
-	}
-	va_end(ap);
-	SMS_Finish(sp->obj);
-	http_Unset(sp->obj->http, H_Content_Length);
-	http_PrintfHeader(sp->wrk, sp->vsl_id, sp->obj->http,
-	    "Content-Length: %d", sp->obj->len);
-}
-
-/*--------------------------------------------------------------------*/
-
-void
-VRT_ban(struct sess *sp, char *cmds, ...)
-{
-	char *a1, *a2, *a3;
-	va_list ap;
-	struct ban *b;
-	int good;
-
-	(void)sp;
-	b = BAN_New();
-	va_start(ap, cmds);
-	a1 = cmds;
-	good = 0;
-	while (a1 != NULL) {
-		good = 0;
-		a2 = va_arg(ap, char *);
-		if (a2 == NULL)
-			break;
-		a3 = va_arg(ap, char *);
-		if (a3 == NULL)
-			break;
-		if (BAN_AddTest(NULL, b, a1, a2, a3))
-			break;
-		a1 = va_arg(ap, char *);
-		good = 1;
-	}
-	if (!good)
-		/* XXX: report error how ? */
-		BAN_Free(b);
-	else
-		BAN_Insert(b);
-}
-
-/*--------------------------------------------------------------------*/
-
-void
-VRT_ban_string(struct sess *sp, const char *str)
-{
-	char *a1, *a2, *a3;
-	char **av;
-	struct ban *b;
-	int good;
-	int i;
-
-	(void)sp;
-	av = VAV_Parse(str, NULL, ARGV_NOESC);
-	if (av[0] != NULL) {
-		/* XXX: report error how ? */
-		VAV_Free(av);
-		return;
-	}
-	b = BAN_New();
-	good = 0;
-	for (i = 1; ;) {
-		a1 = av[i++];
-		if (a1 == NULL)
-			break;
-		good = 0;
-		a2 = av[i++];
-		if (a2 == NULL)
-			break;
-		a3 = av[i++];
-		if (a3 == NULL)
-			break;
-		if (BAN_AddTest(NULL, b, a1, a2, a3))
-			break;
-		good = 1;
-		if (av[i] == NULL)
-			break;
-		good = 0;
-		if (strcmp(av[i++], "&&"))
-			break;
-	}
-	if (!good)
-		/* XXX: report error how ? */
-		BAN_Free(b);
-	else
-		BAN_Insert(b);
-	VAV_Free(av);
-}
-
-/*--------------------------------------------------------------------
- * "real" purges
- */
-
-void
-VRT_purge(const struct sess *sp, double ttl, double grace)
-{
-	if (sp->cur_method == VCL_MET_HIT)
-		HSH_Purge(sp, sp->obj->objcore->objhead, ttl, grace);
-	else if (sp->cur_method == VCL_MET_MISS)
-		HSH_Purge(sp, sp->objcore->objhead, ttl, grace);
-}
-
-/*--------------------------------------------------------------------
- * Simple stuff
- */
-
-int
-VRT_strcmp(const char *s1, const char *s2)
-{
-	if (s1 == NULL || s2 == NULL)
-		return(1);
-	return (strcmp(s1, s2));
-}
-
-void
-VRT_memmove(void *dst, const void *src, unsigned len)
-{
-
-	(void)memmove(dst, src, len);
-}
diff --git a/bin/varnishd/cache_vrt_re.c b/bin/varnishd/cache_vrt_re.c
deleted file mode 100644
index 7759d0a..0000000
--- a/bin/varnishd/cache_vrt_re.c
+++ /dev/null
@@ -1,162 +0,0 @@
-/*-
- * Copyright (c) 2006 Verdens Gang AS
- * Copyright (c) 2006-2009 Varnish Software AS
- * All rights reserved.
- *
- * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- *
- * Runtime support for compiled VCL programs, regexps
- */
-
-#include "config.h"
-
-#include <ctype.h>
-
-#include "cache.h"
-
-#include "vre.h"
-#include "vrt.h"
-
-void
-VRT_re_init(void **rep, const char *re)
-{
-	vre_t *t;
-	const char *error;
-	int erroroffset;
-
-	/* This was already check-compiled by the VCL compiler */
-	t = VRE_compile(re, 0, &error, &erroroffset);
-	AN(t);
-	*rep = t;
-}
-
-void
-VRT_re_fini(void *rep)
-{
-	vre_t *vv;
-
-	vv = rep;
-	if (rep != NULL)
-		VRE_free(&vv);
-}
-
-int
-VRT_re_match(const struct sess *sp, const char *s, void *re)
-{
-	vre_t *t;
-	int i;
-
-	if (s == NULL)
-		s = "";
-	AN(re);
-	t = re;
-	i = VRE_exec(t, s, strlen(s), 0, 0, NULL, 0, &cache_param->vre_limits);
-	if (i >= 0)
-		return (1);
-	if (i < VRE_ERROR_NOMATCH )
-		WSP(sp, SLT_VCL_Error, "Regexp matching returned %d", i);
-	return (0);
-}
-
-const char *
-VRT_regsub(const struct sess *sp, int all, const char *str, void *re,
-    const char *sub)
-{
-	int ovector[30];
-	vre_t *t;
-	int i, l;
-	txt res;
-	char *b0;
-	const char *s;
-	unsigned u, x;
-	int options = 0;
-	size_t len;
-
-	AN(re);
-	if (str == NULL)
-		str = "";
-	t = re;
-	memset(ovector, 0, sizeof(ovector));
-	len = strlen(str);
-	i = VRE_exec(t, str, len, 0, options, ovector, 30,
-	    &cache_param->vre_limits);
-
-	/* If it didn't match, we can return the original string */
-	if (i == VRE_ERROR_NOMATCH)
-		return(str);
-	if (i < VRE_ERROR_NOMATCH ) {
-		WSP(sp, SLT_VCL_Error, "Regexp matching returned %d", i);
-		return(str);
-	}
-
-	u = WS_Reserve(sp->http->ws, 0);
-	res.e = res.b = b0 = sp->http->ws->f;
-	res.e += u;
-
-	do {
-		/* Copy prefix to match */
-		Tadd(&res, str, ovector[0]);
-		for (s = sub ; *s != '\0'; s++ ) {
-			if (*s != '\\' || s[1] == '\0') {
-				if (res.b < res.e)
-					*res.b++ = *s;
-				continue;
-			}
-			s++;
-			if (isdigit(*s)) {
-				x = *s - '0';
-				l = ovector[2*x+1] - ovector[2*x];
-				Tadd(&res, str + ovector[2*x], l);
-				continue;
-			} else {
-				if (res.b < res.e)
-					*res.b++ = *s;
-			}
-		}
-		str += ovector[1];
-		len -= ovector[1];
-		if (!all)
-			break;
-		memset(&ovector, 0, sizeof(ovector));
-		options |= VRE_NOTEMPTY_ATSTART;
-		i = VRE_exec(t, str, len, 0, options, ovector, 30,
-		    &cache_param->vre_limits);
-		if (i < VRE_ERROR_NOMATCH ) {
-			WS_Release(sp->http->ws, 0);
-			WSP(sp, SLT_VCL_Error,
-			    "Regexp matching returned %d", i);
-			return(str);
-		}
-	} while (i != VRE_ERROR_NOMATCH);
-
-	/* Copy suffix to match */
-	Tadd(&res, str, len+1);
-	if (res.b >= res.e) {
-		WS_Release(sp->http->ws, 0);
-		return (str);
-	}
-	Tcheck(res);
-	WS_ReleaseP(sp->http->ws, res.b);
-	return (b0);
-}
diff --git a/bin/varnishd/cache_vrt_var.c b/bin/varnishd/cache_vrt_var.c
deleted file mode 100644
index 860c7aa..0000000
--- a/bin/varnishd/cache_vrt_var.c
+++ /dev/null
@@ -1,550 +0,0 @@
-/*-
- * Copyright (c) 2006 Verdens Gang AS
- * Copyright (c) 2006-2011 Varnish Software AS
- * All rights reserved.
- *
- * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- *
- * Runtime support for compiled VCL programs
- */
-#include "config.h"
-
-#include <stdio.h>
-#include <stdlib.h>
-
-#include "cache.h"
-#include "common/heritage.h"
-
-#include "cache_backend.h"
-#include "vrt_obj.h"
-#include "vtcp.h"
-#include "vtim.h"
-
-static char vrt_hostname[255] = "";
-
-/*--------------------------------------------------------------------*/
-
-static void
-vrt_do_string(struct worker *w, int fd, const struct http *hp, int fld,
-    const char *err, const char *p, va_list ap)
-{
-	char *b;
-
-	// AN(p);
-	AN(hp);
-	b = VRT_String(hp->ws, NULL, p, ap);
-	if (b == NULL || *b == '\0') {
-		WSL(w, SLT_LostHeader, fd, err);
-	} else {
-		http_SetH(hp, fld, b);
-	}
-	va_end(ap);
-}
-
-#define VRT_DO_HDR(obj, hdr, http, fld)				\
-void								\
-VRT_l_##obj##_##hdr(const struct sess *sp, const char *p, ...)	\
-{								\
-	va_list ap;						\
-								\
-	va_start(ap, p);					\
-	vrt_do_string(sp->wrk, sp->fd,				\
-	    http, fld, #obj "." #hdr, p, ap);			\
-	va_end(ap);						\
-}								\
-								\
-const char *							\
-VRT_r_##obj##_##hdr(const struct sess *sp)			\
-{								\
-	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);			\
-	CHECK_OBJ_NOTNULL(http, HTTP_MAGIC);			\
-	return (http->hd[fld].b);				\
-}
-
-VRT_DO_HDR(req,   request,	sp->http,		HTTP_HDR_REQ)
-VRT_DO_HDR(req,   url,		sp->http,		HTTP_HDR_URL)
-VRT_DO_HDR(req,   proto,	sp->http,		HTTP_HDR_PROTO)
-VRT_DO_HDR(bereq, request,	sp->wrk->bereq,		HTTP_HDR_REQ)
-VRT_DO_HDR(bereq, url,		sp->wrk->bereq,		HTTP_HDR_URL)
-VRT_DO_HDR(bereq, proto,	sp->wrk->bereq,		HTTP_HDR_PROTO)
-VRT_DO_HDR(obj,   proto,	sp->obj->http,		HTTP_HDR_PROTO)
-VRT_DO_HDR(obj,   response,	sp->obj->http,		HTTP_HDR_RESPONSE)
-VRT_DO_HDR(resp,  proto,	sp->wrk->resp,		HTTP_HDR_PROTO)
-VRT_DO_HDR(resp,  response,	sp->wrk->resp,		HTTP_HDR_RESPONSE)
-VRT_DO_HDR(beresp,  proto,	sp->wrk->beresp,	HTTP_HDR_PROTO)
-VRT_DO_HDR(beresp,  response,	sp->wrk->beresp,	HTTP_HDR_RESPONSE)
-
-/*--------------------------------------------------------------------*/
-
-#define VRT_DO_STATUS(obj, http)				\
-void								\
-VRT_l_##obj##_status(const struct sess *sp, int num)		\
-{								\
-								\
-	assert(num >= 100 && num <= 999);			\
-	http->status = (uint16_t)num;				\
-}								\
-								\
-int								\
-VRT_r_##obj##_status(const struct sess *sp)			\
-{								\
-								\
-	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);			\
-	return(http->status);					\
-}
-
-VRT_DO_STATUS(obj, sp->obj->http)
-VRT_DO_STATUS(beresp, sp->wrk->beresp)
-VRT_DO_STATUS(resp, sp->wrk->resp)
-
-/*--------------------------------------------------------------------*/
-
-/* XXX: review this */
-/* Add an objecthead to the saintmode list for the (hopefully) relevant
- * backend. Some double-up asserting here to avoid assert-errors when there
- * is no object.
- */
-void
-VRT_l_beresp_saintmode(const struct sess *sp, double a)
-{
-	struct trouble *new;
-	struct trouble *tr;
-	struct trouble *tr2;
-	struct worker *wrk;
-
-	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
-	wrk = sp->wrk;
-	if (!wrk->vbc)
-		return;
-	CHECK_OBJ_NOTNULL(wrk->vbc, VBC_MAGIC);
-	if (!wrk->vbc->backend)
-		return;
-	CHECK_OBJ_NOTNULL(wrk->vbc->backend, BACKEND_MAGIC);
-	if (!sp->objcore)
-		return;
-	CHECK_OBJ_NOTNULL(sp->objcore, OBJCORE_MAGIC);
-
-	/* Setting a negative holdoff period is a mistake. Detecting this
-	 * when compiling the VCL would be better.
-	 */
-	assert(a > 0);
-
-	ALLOC_OBJ(new, TROUBLE_MAGIC);
-	AN(new);
-	new->target = (uintptr_t)(sp->objcore->objhead);
-	new->timeout = sp->t_req + a;
-
-	/* Insert the new item on the list before the first item with a
-	 * timeout at a later date (ie: sort by which entry will time out
-	 * from the list
-	 */
-	Lck_Lock(&wrk->vbc->backend->mtx);
-	VTAILQ_FOREACH_SAFE(tr, &wrk->vbc->backend->troublelist, list, tr2) {
-		if (tr->timeout < new->timeout) {
-			VTAILQ_INSERT_BEFORE(tr, new, list);
-			new = NULL;
-			break;
-		}
-	}
-
-	/* Insert the item at the end if the list is empty or all other
-	 * items have a longer timeout.
-	 */
-	if (new)
-		VTAILQ_INSERT_TAIL(&wrk->vbc->backend->troublelist, new, list);
-
-	Lck_Unlock(&wrk->vbc->backend->mtx);
-}
-
-/*--------------------------------------------------------------------*/
-
-#define VBERESP(dir, type, onm, field)					\
-void									\
-VRT_l_##dir##_##onm(const struct sess *sp, type a)			\
-{									\
-	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);				\
-	sp->wrk->field = a;						\
-}									\
-									\
-type									\
-VRT_r_##dir##_##onm(const struct sess *sp)				\
-{									\
-	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);				\
-	return (sp->wrk->field);					\
-}
-
-VBERESP(beresp, unsigned, do_esi, do_esi)
-VBERESP(beresp, unsigned, do_gzip, do_gzip)
-VBERESP(beresp, unsigned, do_gunzip, do_gunzip)
-VBERESP(beresp, unsigned, do_stream, do_stream)
-
-/*--------------------------------------------------------------------*/
-
-const char * __match_proto__()
-VRT_r_client_identity(struct sess *sp)
-{
-	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
-	if (sp->client_identity != NULL)
-		return (sp->client_identity);
-	else
-		return (sp->addr);
-}
-
-void
-VRT_l_client_identity(struct sess *sp, const char *str, ...)
-{
-	va_list ap;
-	char *b;
-
-	va_start(ap, str);
-	b = VRT_String(sp->http->ws, NULL, str, ap);
-	va_end(ap);
-	sp->client_identity = b;
-}
-
-/*--------------------------------------------------------------------*/
-
-#define BEREQ_TIMEOUT(which)					\
-void __match_proto__()						\
-VRT_l_bereq_##which(struct sess *sp, double num)		\
-{								\
-								\
-	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);			\
-	sp->wrk->which = (num > 0.0 ? num : 0.0);		\
-}								\
-								\
-double __match_proto__()					\
-VRT_r_bereq_##which(struct sess *sp)				\
-{								\
-								\
-	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);			\
-	return(sp->wrk->which);					\
-}
-
-BEREQ_TIMEOUT(connect_timeout)
-BEREQ_TIMEOUT(first_byte_timeout)
-BEREQ_TIMEOUT(between_bytes_timeout)
-
-/*--------------------------------------------------------------------*/
-
-const char *
-VRT_r_beresp_backend_name(const struct sess *sp)
-{
-
-	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
-	CHECK_OBJ_NOTNULL(sp->wrk->vbc, VBC_MAGIC);
-	return(sp->wrk->vbc->backend->vcl_name);
-}
-
-struct sockaddr_storage *
-VRT_r_beresp_backend_ip(const struct sess *sp)
-{
-
-	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
-	CHECK_OBJ_NOTNULL(sp->wrk->vbc, VBC_MAGIC);
-	return(sp->wrk->vbc->addr);
-}
-
-int
-VRT_r_beresp_backend_port(const struct sess *sp)
-{
-
-	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
-	CHECK_OBJ_NOTNULL(sp->wrk->vbc, VBC_MAGIC);
-	return (VTCP_port(sp->wrk->vbc->addr));
-}
-
-const char * __match_proto__()
-VRT_r_beresp_storage(struct sess *sp)
-{
-	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
-	if (sp->wrk->storage_hint != NULL)
-		return (sp->wrk->storage_hint);
-	else
-		return (NULL);
-}
-
-void __match_proto__()
-VRT_l_beresp_storage(struct sess *sp, const char *str, ...)
-{
-	va_list ap;
-	char *b;
-
-	va_start(ap, str);
-	b = VRT_String(sp->wrk->ws, NULL, str, ap);
-	va_end(ap);
-	sp->wrk->storage_hint = b;
-}
-
-/*--------------------------------------------------------------------*/
-
-void
-VRT_l_req_backend(struct sess *sp, struct director *be)
-{
-	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
-	sp->director = be;
-}
-
-struct director * __match_proto__()
-VRT_r_req_backend(struct sess *sp)
-{
-	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
-	return (sp->director);
-}
-
-/*--------------------------------------------------------------------*/
-
-void
-VRT_l_req_esi(struct sess *sp, unsigned process_esi)
-{
-	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
-	/*
-	 * Only allow you to turn of esi in the main request
-	 * else everything gets confused
-	 */
-	if(sp->esi_level == 0)
-		sp->disable_esi = !process_esi;
-}
-
-unsigned __match_proto__()
-VRT_r_req_esi(struct sess *sp)
-{
-	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
-	return (!sp->disable_esi);
-}
-
-int
-VRT_r_req_esi_level(const struct sess *sp)
-{
-
-	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
-	return(sp->esi_level);
-}
-
-/*--------------------------------------------------------------------*/
-
-unsigned __match_proto__()
-VRT_r_req_can_gzip(struct sess *sp)
-{
-
-	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
-	return (RFC2616_Req_Gzip(sp));
-}
-
-
-/*--------------------------------------------------------------------*/
-
-int
-VRT_r_req_restarts(const struct sess *sp)
-{
-
-	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
-	return (sp->restarts);
-}
-
-/*--------------------------------------------------------------------
- * NB: TTL is relative to when object was created, whereas grace and
- * keep are relative to ttl.
- */
-
-#define VRT_DO_EXP(which, exp, fld, offset, extra)		\
-								\
-void __match_proto__()						\
-VRT_l_##which##_##fld(struct sess *sp, double a)		\
-{								\
-								\
-	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);			\
-	if (a > 0.)						\
-		a += offset;					\
-	EXP_Set_##fld(&exp, a);					\
-	extra;							\
-}								\
-								\
-double __match_proto__()					\
-VRT_r_##which##_##fld(struct sess *sp)				\
-{								\
-								\
-	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);			\
-	return(EXP_Get_##fld(&exp) - offset);			\
-}
-
-static void
-vrt_wsp_exp(const struct sess *sp, unsigned xid, const struct exp *e)
-{
-	WSP(sp, SLT_TTL, "%u VCL %.0f %.0f %.0f %.0f %.0f",
-	    xid, e->ttl - (sp->t_req - e->entered), e->grace, e->keep,
-	    sp->t_req, e->age + (sp->t_req - e->entered));
-}
-
-VRT_DO_EXP(req, sp->exp, ttl, 0, )
-VRT_DO_EXP(req, sp->exp, grace, 0, )
-VRT_DO_EXP(req, sp->exp, keep, 0, )
-
-VRT_DO_EXP(obj, sp->obj->exp, grace, 0,
-   EXP_Rearm(sp->obj);
-   vrt_wsp_exp(sp, sp->obj->xid, &sp->obj->exp);)
-VRT_DO_EXP(obj, sp->obj->exp, ttl, (sp->t_req - sp->obj->exp.entered),
-   EXP_Rearm(sp->obj);
-   vrt_wsp_exp(sp, sp->obj->xid, &sp->obj->exp);)
-VRT_DO_EXP(obj, sp->obj->exp, keep, 0,
-   EXP_Rearm(sp->obj);
-   vrt_wsp_exp(sp, sp->obj->xid, &sp->obj->exp);)
-
-VRT_DO_EXP(beresp, sp->wrk->exp, grace, 0,
-   vrt_wsp_exp(sp, sp->xid, &sp->wrk->exp);)
-VRT_DO_EXP(beresp, sp->wrk->exp, ttl, 0,
-   vrt_wsp_exp(sp, sp->xid, &sp->wrk->exp);)
-VRT_DO_EXP(beresp, sp->wrk->exp, keep, 0,
-   vrt_wsp_exp(sp, sp->xid, &sp->wrk->exp);)
-
-/*--------------------------------------------------------------------
- * req.xid
- */
-
-const char * __match_proto__()
-VRT_r_req_xid(struct sess *sp)
-{
-	char *p;
-	int size;
-	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
-
-	size = snprintf(NULL, 0, "%u", sp->xid) + 1;
-	AN(p = WS_Alloc(sp->http->ws, size));
-	assert(snprintf(p, size, "%u", sp->xid) < size);
-	return (p);
-}
-
-/*--------------------------------------------------------------------*/
-
-#define REQ_BOOL(which)						\
-void __match_proto__()						\
-VRT_l_req_##which(struct sess *sp, unsigned val)		\
-{								\
-								\
-	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);			\
-	sp->which = val ? 1 : 0;				\
-}								\
-								\
-unsigned __match_proto__()					\
-VRT_r_req_##which(struct sess *sp)				\
-{								\
-								\
-	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);			\
-	return(sp->which);					\
-}
-
-REQ_BOOL(hash_ignore_busy)
-REQ_BOOL(hash_always_miss)
-
-/*--------------------------------------------------------------------*/
-
-struct sockaddr_storage *
-VRT_r_client_ip(struct sess *sp)
-{
-
-	return (&sp->sockaddr);
-}
-
-struct sockaddr_storage *
-VRT_r_server_ip(struct sess *sp)
-{
-	int i;
-
-	if (sp->mysockaddr.ss_family == AF_UNSPEC) {
-		i = getsockname(sp->fd,
-		    (void*)&sp->mysockaddr, &sp->mysockaddrlen);
-		assert(VTCP_Check(i));
-	}
-
-	return (&sp->mysockaddr);
-}
-
-const char*
-VRT_r_server_identity(struct sess *sp)
-{
-	(void)sp;
-
-	if (heritage.identity[0] != '\0')
-		return (heritage.identity);
-	else
-		return (heritage.name);
-}
-
-
-const char*
-VRT_r_server_hostname(struct sess *sp)
-{
-	(void)sp;
-
-	if (vrt_hostname[0] == '\0')
-		AZ(gethostname(vrt_hostname, sizeof(vrt_hostname)));
-
-	return (vrt_hostname);
-}
-
-/*--------------------------------------------------------------------
- * XXX: This is pessimistically silly
- */
-
-int
-VRT_r_server_port(struct sess *sp)
-{
-	int i;
-
-	if (sp->mysockaddr.ss_family == AF_UNSPEC) {
-		i = getsockname(sp->fd,
-		    (void*)&sp->mysockaddr, &sp->mysockaddrlen);
-		assert(VTCP_Check(i));
-	}
-	return (VTCP_port(&sp->mysockaddr));
-}
-
-/*--------------------------------------------------------------------*/
-
-int
-VRT_r_obj_hits(const struct sess *sp)
-{
-
-	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
-	CHECK_OBJ_NOTNULL(sp->obj, OBJECT_MAGIC);	/* XXX */
-	return (sp->obj->hits);
-}
-
-double
-VRT_r_obj_lastuse(const struct sess *sp)
-{
-
-	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
-	CHECK_OBJ_NOTNULL(sp->obj, OBJECT_MAGIC);	/* XXX */
-	return (VTIM_real() - sp->obj->last_use);
-}
-
-unsigned
-VRT_r_req_backend_healthy(const struct sess *sp)
-{
-	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
-	CHECK_OBJ_NOTNULL(sp->director, DIRECTOR_MAGIC);
-	return (VDI_Healthy(sp->director, sp));
-}
-
diff --git a/bin/varnishd/cache_vrt_vmod.c b/bin/varnishd/cache_vrt_vmod.c
deleted file mode 100644
index 6b3b846..0000000
--- a/bin/varnishd/cache_vrt_vmod.c
+++ /dev/null
@@ -1,180 +0,0 @@
-/*-
- * Copyright (c) 2006 Verdens Gang AS
- * Copyright (c) 2006-2011 Varnish Software AS
- * All rights reserved.
- *
- * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- *
- * Runtime support for compiled VCL programs
- */
-
-#include "config.h"
-
-#include <dlfcn.h>
-#include <stdlib.h>
-
-#include "cache.h"
-
-#include "vcli_priv.h"
-#include "vrt.h"
-
-/*--------------------------------------------------------------------
- * Modules stuff
- */
-
-struct vmod {
-	unsigned		magic;
-#define VMOD_MAGIC		0xb750219c
-
-	VTAILQ_ENTRY(vmod)	list;
-
-	int			ref;
-
-	char			*nm;
-	char			*path;
-	void			*hdl;
-	const void		*funcs;
-	int			funclen;
-	const void		*idptr;
-};
-
-static VTAILQ_HEAD(,vmod)	vmods = VTAILQ_HEAD_INITIALIZER(vmods);
-
-int
-VRT_Vmod_Init(void **hdl, void *ptr, int len, const char *nm,
-    const char *path, struct cli *cli)
-{
-	struct vmod *v;
-	void *x, *y, *z, *w;
-
-	ASSERT_CLI();
-
-	VTAILQ_FOREACH(v, &vmods, list)
-		if (!strcmp(v->nm, nm))	// Also path, len ?
-			break;
-	if (v == NULL) {
-		ALLOC_OBJ(v, VMOD_MAGIC);
-		AN(v);
-
-		v->hdl = dlopen(path, RTLD_NOW | RTLD_LOCAL);
-		if (v->hdl == NULL) {
-			VCLI_Out(cli, "Loading VMOD %s from %s:\n", nm, path);
-			VCLI_Out(cli, "dlopen() failed: %s\n", dlerror());
-			VCLI_Out(cli, "Check child process permissions.\n");
-			FREE_OBJ(v);
-			return (1);
-		}
-
-		x = dlsym(v->hdl, "Vmod_Name");
-		y = dlsym(v->hdl, "Vmod_Len");
-		z = dlsym(v->hdl, "Vmod_Func");
-		w = dlsym(v->hdl, "Vmod_Id");
-		if (x == NULL || y == NULL || z == NULL || w == NULL) {
-			VCLI_Out(cli, "Loading VMOD %s from %s:\n", nm, path);
-			VCLI_Out(cli, "VMOD symbols not found\n");
-			VCLI_Out(cli, "Check relative pathnames.\n");
-			(void)dlclose(v->hdl);
-			FREE_OBJ(v);
-			return (1);
-		}
-		AN(x);
-		AN(y);
-		AN(z);
-		AN(w);
-		if (strcmp(x, nm)) {
-			VCLI_Out(cli, "Loading VMOD %s from %s:\n", nm, path);
-			VCLI_Out(cli, "File contain wrong VMOD (\"%s\")\n", x);
-			VCLI_Out(cli, "Check relative pathnames ?.\n");
-			(void)dlclose(v->hdl);
-			FREE_OBJ(v);
-			return (1);
-		}
-
-		v->funclen = *(const int *)y;
-		v->funcs = z;
-
-		REPLACE(v->nm, nm);
-		REPLACE(v->path, path);
-
-		VSC_C_main->vmods++;
-		VTAILQ_INSERT_TAIL(&vmods, v, list);
-		v->idptr = w;
-	}
-
-	assert(len == v->funclen);
-	memcpy(ptr, v->funcs, v->funclen);
-	v->ref++;
-
-	*hdl = v;
-	return (0);
-}
-
-void
-VRT_Vmod_Fini(void **hdl)
-{
-	struct vmod *v;
-
-	ASSERT_CLI();
-
-	AN(*hdl);
-	CAST_OBJ_NOTNULL(v, *hdl, VMOD_MAGIC);
-	*hdl = NULL;
-	if (--v->ref != 0)
-		return;
-#ifndef DONT_DLCLOSE_VMODS
-	AZ(dlclose(v->hdl));
-#endif
-	free(v->nm);
-	free(v->path);
-	VTAILQ_REMOVE(&vmods, v, list);
-	VSC_C_main->vmods--;
-	FREE_OBJ(v);
-}
-
-/*---------------------------------------------------------------------*/
-
-static void
-ccf_debug_vmod(struct cli *cli, const char * const *av, void *priv)
-{
-	struct vmod *v;
-
-	(void)av;
-	(void)priv;
-	ASSERT_CLI();
-	VTAILQ_FOREACH(v, &vmods, list)
-		VCLI_Out(cli, "%5d %s (%s)\n", v->ref, v->nm, v->path);
-}
-
-static struct cli_proto vcl_cmds[] = {
-	{ "debug.vmod", "debug.vmod", "show loaded vmods", 0, 0,
-		"d", ccf_debug_vmod },
-	{ NULL }
-};
-
-void
-VMOD_Init(void)
-{
-
-	CLI_AddFuncs(vcl_cmds);
-}
diff --git a/bin/varnishd/cache_wrk.c b/bin/varnishd/cache_wrk.c
deleted file mode 100644
index bfee84c..0000000
--- a/bin/varnishd/cache_wrk.c
+++ /dev/null
@@ -1,204 +0,0 @@
-/*-
- * Copyright (c) 2006 Verdens Gang AS
- * Copyright (c) 2006-2011 Varnish Software AS
- * All rights reserved.
- *
- * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- *
- * Worker thread stuff unrelated to the worker thread pools.
- */
-
-#include "config.h"
-
-#include <math.h>
-#include <stdlib.h>
-
-#include "cache.h"
-
-#include "hash/hash_slinger.h"
-#include "vsha256.h"
-
-static struct lock		wstat_mtx;
-
-/*--------------------------------------------------------------------*/
-
-static void
-wrk_sumstat(struct worker *w)
-{
-
-	Lck_AssertHeld(&wstat_mtx);
-#define L0(n)
-#define L1(n) (VSC_C_main->n += w->stats.n)
-#define VSC_DO_MAIN
-#define VSC_F(n, t, l, f, d, e) L##l(n);
-#include "tbl/vsc_fields.h"
-#undef VSC_F
-#undef VSC_DO_MAIN
-#undef L0
-#undef L1
-	memset(&w->stats, 0, sizeof w->stats);
-}
-
-void
-WRK_SumStat(struct worker *w)
-{
-
-	Lck_Lock(&wstat_mtx);
-	wrk_sumstat(w);
-	Lck_Unlock(&wstat_mtx);
-}
-
-int
-WRK_TrySumStat(struct worker *w)
-{
-	if (Lck_Trylock(&wstat_mtx))
-		return (0);
-	wrk_sumstat(w);
-	Lck_Unlock(&wstat_mtx);
-	return (1);
-}
-
-/*--------------------------------------------------------------------
- * Create and starte a back-ground thread which as its own worker and
- * session data structures;
- */
-
-struct bgthread {
-	unsigned	magic;
-#define BGTHREAD_MAGIC	0x23b5152b
-	const char	*name;
-	bgthread_t	*func;
-	void		*priv;
-};
-
-static void *
-wrk_bgthread(void *arg)
-{
-	struct bgthread *bt;
-	struct worker ww;
-	struct sess *sp;
-	uint32_t logbuf[1024];	/* XXX:  size ? */
-
-	CAST_OBJ_NOTNULL(bt, arg, BGTHREAD_MAGIC);
-	THR_SetName(bt->name);
-	sp = SES_Alloc();
-	XXXAN(sp);
-	memset(&ww, 0, sizeof ww);
-	sp->wrk = &ww;
-	ww.magic = WORKER_MAGIC;
-	ww.wlp = ww.wlb = logbuf;
-	ww.wle = logbuf + (sizeof logbuf) / 4;
-
-	(void)bt->func(sp, bt->priv);
-
-	WRONG("BgThread terminated");
-
-	NEEDLESS_RETURN(NULL);
-}
-
-void
-WRK_BgThread(pthread_t *thr, const char *name, bgthread_t *func, void *priv)
-{
-	struct bgthread *bt;
-
-	ALLOC_OBJ(bt, BGTHREAD_MAGIC);
-	AN(bt);
-
-	bt->name = name;
-	bt->func = func;
-	bt->priv = priv;
-	AZ(pthread_create(thr, NULL, wrk_bgthread, bt));
-}
-
-/*--------------------------------------------------------------------*/
-
-static void *
-wrk_thread_real(void *priv, unsigned shm_workspace, unsigned sess_workspace,
-    uint16_t nhttp, unsigned http_space, unsigned siov)
-{
-	struct worker *w, ww;
-	uint32_t wlog[shm_workspace / 4];
-	/* XXX: can we trust these to be properly aligned ? */
-	unsigned char ws[sess_workspace];
-	unsigned char http0[http_space];
-	unsigned char http1[http_space];
-	unsigned char http2[http_space];
-	struct iovec iov[siov];
-	struct SHA256Context sha256;
-
-	THR_SetName("cache-worker");
-	w = &ww;
-	memset(w, 0, sizeof *w);
-	w->magic = WORKER_MAGIC;
-	w->lastused = NAN;
-	w->wlb = w->wlp = wlog;
-	w->wle = wlog + (sizeof wlog) / 4;
-	w->sha256ctx = &sha256;
-	w->bereq = HTTP_create(http0, nhttp);
-	w->beresp = HTTP_create(http1, nhttp);
-	w->resp = HTTP_create(http2, nhttp);
-	w->wrw.iov = iov;
-	w->wrw.siov = siov;
-	w->wrw.ciov = siov;
-	AZ(pthread_cond_init(&w->cond, NULL));
-
-	WS_Init(w->ws, "wrk", ws, sess_workspace);
-
-	VSL(SLT_WorkThread, 0, "%p start", w);
-
-	Pool_Work_Thread(priv, w);
-	AZ(w->pool);
-
-	VSL(SLT_WorkThread, 0, "%p end", w);
-	if (w->vcl != NULL)
-		VCL_Rel(&w->vcl);
-	AZ(pthread_cond_destroy(&w->cond));
-	HSH_Cleanup(w);
-	WRK_SumStat(w);
-	return (NULL);
-}
-
-void *
-WRK_thread(void *priv)
-{
-	uint16_t nhttp;
-	unsigned siov;
-
-	assert(cache_param->http_max_hdr <= 65535);
-	/* We need to snapshot these two for consistency */
-	nhttp = (uint16_t)cache_param->http_max_hdr;
-	siov = nhttp * 2;
-	if (siov > IOV_MAX)
-		siov = IOV_MAX;
-	return (wrk_thread_real(priv,
-	    cache_param->shm_workspace,
-	    cache_param->wthread_workspace,
-	    nhttp, HTTP_estimate(nhttp), siov));
-}
-
-void
-WRK_Init(void)
-{
-	Lck_New(&wstat_mtx, lck_wstat);
-}
diff --git a/bin/varnishd/cache_wrw.c b/bin/varnishd/cache_wrw.c
deleted file mode 100644
index 2160f69..0000000
--- a/bin/varnishd/cache_wrw.c
+++ /dev/null
@@ -1,344 +0,0 @@
-/*-
- * Copyright (c) 2006 Verdens Gang AS
- * Copyright (c) 2006-2011 Varnish Software AS
- * All rights reserved.
- *
- * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- *
- * Write data to fd
- * We try to use writev() if possible in order to minimize number of
- * syscalls made and packets sent.  It also just might allow the worker
- * thread to complete the request without holding stuff locked.
- */
-
-#include "config.h"
-
-#include <sys/types.h>
-#ifdef SENDFILE_WORKS
-#  if defined(__FreeBSD__) || defined(__DragonFly__)
-#    include <sys/socket.h>
-#  elif defined(__linux__)
-#    include <sys/sendfile.h>
-#  elif defined(__sun)
-#    include <sys/sendfile.h>
-#  else
-#     error Unknown sendfile() implementation
-#  endif
-#endif /* SENDFILE_WORKS */
-#include <sys/uio.h>
-
-#include <stdio.h>
-
-#include "cache.h"
-#include "vtim.h"
-
-/*--------------------------------------------------------------------
- */
-
-int
-WRW_Error(const struct worker *w)
-{
-
-	return (w->wrw.werr);
-}
-
-void
-WRW_Reserve(struct worker *w, int *fd)
-{
-	struct wrw *wrw;
-
-	CHECK_OBJ_NOTNULL(w, WORKER_MAGIC);
-	wrw = &w->wrw;
-	AZ(wrw->wfd);
-	wrw->werr = 0;
-	wrw->liov = 0;
-	wrw->niov = 0;
-	wrw->ciov = wrw->siov;
-	wrw->wfd = fd;
-}
-
-static void
-WRW_Release(struct worker *w)
-{
-	struct wrw *wrw;
-
-	CHECK_OBJ_NOTNULL(w, WORKER_MAGIC);
-	wrw = &w->wrw;
-	AN(wrw->wfd);
-	wrw->werr = 0;
-	wrw->liov = 0;
-	wrw->niov = 0;
-	wrw->ciov = wrw->siov;
-	wrw->wfd = NULL;
-}
-
-unsigned
-WRW_Flush(struct worker *w)
-{
-	ssize_t i;
-	struct wrw *wrw;
-	char cbuf[32];
-
-	CHECK_OBJ_NOTNULL(w, WORKER_MAGIC);
-	wrw = &w->wrw;
-	AN(wrw->wfd);
-
-	/* For chunked, there must be one slot reserved for the chunked tail */
-	if (wrw->ciov < wrw->siov)
-		assert(wrw->niov < wrw->siov);
-
-	if (*wrw->wfd >= 0 && wrw->liov > 0 && wrw->werr == 0) {
-		if (wrw->ciov < wrw->siov && wrw->cliov > 0) {
-			bprintf(cbuf, "00%jx\r\n", (intmax_t)wrw->cliov);
-			i = strlen(cbuf);
-			wrw->iov[wrw->ciov].iov_base = cbuf;
-			wrw->iov[wrw->ciov].iov_len = i;
-			wrw->liov += i;
-
-			wrw->iov[wrw->niov].iov_base = cbuf + i - 2;
-			wrw->iov[wrw->niov++].iov_len = 2;
-			wrw->liov += 2;
-		} else if (wrw->ciov < wrw->siov) {
-			wrw->iov[wrw->ciov].iov_base = cbuf;
-			wrw->iov[wrw->ciov].iov_len = 0;
-		}
-		i = writev(*wrw->wfd, wrw->iov, wrw->niov);
-		while (i != wrw->liov && i > 0) {
-			/* Remove sent data from start of I/O vector,
-			 * then retry; we hit a timeout, but some data
-			 * was sent.
-
-			 XXX: Add a "minimum sent data per timeout
-			 counter to prevent slowlaris attacks
-			*/
-			size_t used = 0;
-
-			if (VTIM_real() - w->sp->t_resp > cache_param->send_timeout) {
-				WSL(w, SLT_Debug, *wrw->wfd,
-				    "Hit total send timeout, wrote = %ld/%ld; not retrying",
-				    i, wrw->liov);
-				i = -1;
-				break;
-			}
-
-			WSL(w, SLT_Debug, *wrw->wfd,
-			    "Hit send timeout, wrote = %ld/%ld; retrying",
-			    i, wrw->liov);
-
-			for (int j = 0; j < wrw->niov; j++) {
-				if (used + wrw->iov[j].iov_len > i) {
-					/* Cutoff is in this iov */
-					int used_here = i - used;
-					wrw->iov[j].iov_len -= used_here;
-					wrw->iov[j].iov_base = (char*)wrw->iov[j].iov_base + used_here;
-					memmove(wrw->iov, &wrw->iov[j],
-						(wrw->niov - j) * sizeof(struct iovec));
-					wrw->niov -= j;
-					wrw->liov -= i;
-					break;
-				}
-				used += wrw->iov[j].iov_len;
-			}
-			i = writev(*wrw->wfd, wrw->iov, wrw->niov);
-		}
-		if (i <= 0) {
-			wrw->werr++;
-			WSL(w, SLT_Debug, *wrw->wfd,
-			    "Write error, retval = %d, len = %d, errno = %s",
-			    i, wrw->liov, strerror(errno));
-		}
-	}
-	wrw->liov = 0;
-	wrw->cliov = 0;
-	wrw->niov = 0;
-	if (wrw->ciov < wrw->siov)
-		wrw->ciov = wrw->niov++;
-	return (wrw->werr);
-}
-
-unsigned
-WRW_FlushRelease(struct worker *w)
-{
-	unsigned u;
-
-	CHECK_OBJ_NOTNULL(w, WORKER_MAGIC);
-	AN(w->wrw.wfd);
-	u = WRW_Flush(w);
-	WRW_Release(w);
-	return (u);
-}
-
-unsigned
-WRW_WriteH(struct worker *w, const txt *hh, const char *suf)
-{
-	unsigned u;
-
-	CHECK_OBJ_NOTNULL(w, WORKER_MAGIC);
-	AN(w->wrw.wfd);
-	AN(w);
-	AN(hh);
-	AN(hh->b);
-	AN(hh->e);
-	u = WRW_Write(w, hh->b, hh->e - hh->b);
-	if (suf != NULL)
-		u += WRW_Write(w, suf, -1);
-	return (u);
-}
-
-unsigned
-WRW_Write(struct worker *w, const void *ptr, int len)
-{
-	struct wrw *wrw;
-
-	CHECK_OBJ_NOTNULL(w, WORKER_MAGIC);
-	wrw = &w->wrw;
-	AN(wrw->wfd);
-	if (len == 0 || *wrw->wfd < 0)
-		return (0);
-	if (len == -1)
-		len = strlen(ptr);
-	if (wrw->niov >= wrw->siov - (wrw->ciov < wrw->siov ? 1 : 0))
-		(void)WRW_Flush(w);
-	wrw->iov[wrw->niov].iov_base = TRUST_ME(ptr);
-	wrw->iov[wrw->niov].iov_len = len;
-	wrw->liov += len;
-	wrw->niov++;
-	if (wrw->ciov < wrw->siov) {
-		assert(wrw->niov < wrw->siov);
-		wrw->cliov += len;
-	}
-	return (len);
-}
-
-void
-WRW_Chunked(struct worker *w)
-{
-	struct wrw *wrw;
-
-	CHECK_OBJ_NOTNULL(w, WORKER_MAGIC);
-	wrw = &w->wrw;
-
-	assert(wrw->ciov == wrw->siov);
-	/*
-	 * If there are not space for chunked header, a chunk of data and
-	 * a chunk tail, we might as well flush right away.
-	 */
-	if (wrw->niov + 3 >= wrw->siov)
-		(void)WRW_Flush(w);
-	wrw->ciov = wrw->niov++;
-	wrw->cliov = 0;
-	assert(wrw->ciov < wrw->siov);
-	assert(wrw->niov < wrw->siov);
-}
-
-/*
- * XXX: It is not worth the complexity to attempt to get the
- * XXX: end of chunk into the WRW_Flush(), because most of the time
- * XXX: if not always, that is a no-op anyway, because the calling
- * XXX: code already called WRW_Flush() to release local storage.
- */
-
-void
-WRW_EndChunk(struct worker *w)
-{
-	struct wrw *wrw;
-
-	CHECK_OBJ_NOTNULL(w, WORKER_MAGIC);
-	wrw = &w->wrw;
-
-	assert(wrw->ciov < wrw->siov);
-	(void)WRW_Flush(w);
-	wrw->ciov = wrw->siov;
-	wrw->niov = 0;
-	wrw->cliov = 0;
-	(void)WRW_Write(w, "0\r\n\r\n", -1);
-}
-
-
-#ifdef SENDFILE_WORKS
-void
-WRW_Sendfile(struct worker *w, int fd, off_t off, unsigned len)
-{
-	struct wrw *wrw;
-
-	CHECK_OBJ_NOTNULL(w, WORKER_MAGIC);
-	wrw = &w->wrw;
-	AN(wrw->wfd);
-	assert(fd >= 0);
-	assert(len > 0);
-
-#if defined(__FreeBSD__) || defined(__DragonFly__)
-	do {
-		struct sf_hdtr sfh;
-		memset(&sfh, 0, sizeof sfh);
-		if (wrw->niov > 0) {
-			sfh.headers = wrw->iov;
-			sfh.hdr_cnt = wrw->niov;
-		}
-		if (sendfile(fd, *wrw->wfd, off, len, &sfh, NULL, 0) != 0)
-			wrw->werr++;
-		wrw->liov = 0;
-		wrw->niov = 0;
-	} while (0);
-#elif defined(__linux__)
-	do {
-		if (WRW_Flush(w) == 0 &&
-		    sendfile(*wrw->wfd, fd, &off, len) != len)
-			wrw->werr++;
-	} while (0);
-#elif defined(__sun) && defined(HAVE_SENDFILEV)
-	do {
-		sendfilevec_t svvec[cache_param->http_headers * 2 + 1];
-		size_t xferred = 0, expected = 0;
-		int i;
-		for (i = 0; i < wrw->niov; i++) {
-			svvec[i].sfv_fd = SFV_FD_SELF;
-			svvec[i].sfv_flag = 0;
-			svvec[i].sfv_off = (off_t) wrw->iov[i].iov_base;
-			svvec[i].sfv_len = wrw->iov[i].iov_len;
-			expected += svvec[i].sfv_len;
-		}
-		svvec[i].sfv_fd = fd;
-		svvec[i].sfv_flag = 0;
-		svvec[i].sfv_off = off;
-		svvec[i].sfv_len = len;
-		expected += svvec[i].sfv_len;
-		if (sendfilev(*wrw->wfd, svvec, i, &xferred) == -1 ||
-		    xferred != expected)
-			wrw->werr++;
-		wrw->liov = 0;
-		wrw->niov = 0;
-	} while (0);
-#elif defined(__sun) && defined(HAVE_SENDFILE)
-	do {
-		if (WRW_Flush(w) == 0 &&
-		    sendfile(*wrw->wfd, fd, &off, len) != len)
-			wrw->werr++;
-	} while (0);
-#else
-#error Unknown sendfile() implementation
-#endif
-}
-#endif /* SENDFILE_WORKS */
-
diff --git a/bin/varnishd/cache_ws.c b/bin/varnishd/cache_ws.c
deleted file mode 100644
index 9fca215..0000000
--- a/bin/varnishd/cache_ws.c
+++ /dev/null
@@ -1,212 +0,0 @@
-/*-
- * Copyright (c) 2006 Verdens Gang AS
- * Copyright (c) 2006-2010 Varnish Software AS
- * All rights reserved.
- *
- * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- *
- */
-
-#include "config.h"
-
-#include "cache.h"
-
-void
-WS_Assert(const struct ws *ws)
-{
-
-	CHECK_OBJ_NOTNULL(ws, WS_MAGIC);
-	DSL(0x02, SLT_Debug, 0, "WS(%p = (%s, %p %u %u %u)",
-	    ws, ws->id, ws->s, pdiff(ws->s, ws->f),
-	    ws->r == NULL ? 0 : pdiff(ws->f, ws->r),
-	    pdiff(ws->s, ws->e));
-	assert(ws->s != NULL);
-	assert(PAOK(ws->s));
-	assert(ws->e != NULL);
-	assert(PAOK(ws->e));
-	assert(ws->s < ws->e);
-	assert(ws->f >= ws->s);
-	assert(ws->f <= ws->e);
-	assert(PAOK(ws->f));
-	if (ws->r) {
-		assert(ws->r > ws->s);
-		assert(ws->r <= ws->e);
-		assert(PAOK(ws->r));
-	}
-}
-
-void
-WS_Init(struct ws *ws, const char *id, void *space, unsigned len)
-{
-
-	DSL(0x02, SLT_Debug, 0,
-	    "WS_Init(%p, \"%s\", %p, %u)", ws, id, space, len);
-	assert(space != NULL);
-	memset(ws, 0, sizeof *ws);
-	ws->magic = WS_MAGIC;
-	ws->s = space;
-	assert(PAOK(space));
-	ws->e = ws->s + len;
-	assert(PAOK(len));
-	ws->f = ws->s;
-	ws->id = id;
-	WS_Assert(ws);
-}
-
-/*
- * Reset a WS to start or a given pointer, likely from WS_Snapshot
- */
-
-void
-WS_Reset(struct ws *ws, char *p)
-{
-
-	WS_Assert(ws);
-	DSL(0x02, SLT_Debug, 0, "WS_Reset(%p, %p)", ws, p);
-	assert(ws->r == NULL);
-	if (p == NULL)
-		ws->f = ws->s;
-	else {
-		assert(p >= ws->s);
-		assert(p < ws->e);
-		ws->f = p;
-	}
-	WS_Assert(ws);
-}
-
-char *
-WS_Alloc(struct ws *ws, unsigned bytes)
-{
-	char *r;
-
-	WS_Assert(ws);
-	bytes = PRNDUP(bytes);
-
-	assert(ws->r == NULL);
-	if (ws->f + bytes > ws->e) {
-		ws->overflow++;
-		WS_Assert(ws);
-		return(NULL);
-	}
-	r = ws->f;
-	ws->f += bytes;
-	DSL(0x02, SLT_Debug, 0, "WS_Alloc(%p, %u) = %p", ws, bytes, r);
-	WS_Assert(ws);
-	return (r);
-}
-
-char *
-WS_Dup(struct ws *ws, const char *s)
-{
-	unsigned l;
-	char *p;
-
-	WS_Assert(ws);
-	l = strlen(s) + 1;
-	p = WS_Alloc(ws, l);
-	if (p != NULL)
-		memcpy(p, s, l);
-	DSL(0x02, SLT_Debug, 0, "WS_Dup(%p, \"%s\") = %p", ws, s, p);
-	WS_Assert(ws);
-	return (p);
-}
-
-unsigned
-WS_Free(const struct ws *ws)
-{
-
-	WS_Assert(ws);
-	return(ws->e - ws->f);
-}
-
-char *
-WS_Snapshot(struct ws *ws)
-{
-
-	WS_Assert(ws);
-	assert(ws->r == NULL);
-	DSL(0x02, SLT_Debug, 0, "WS_Snapshot(%p) = %p", ws, ws->f);
-	return (ws->f);
-}
-
-unsigned
-WS_Reserve(struct ws *ws, unsigned bytes)
-{
-	unsigned b2;
-
-	WS_Assert(ws);
-	assert(ws->r == NULL);
-	if (bytes == 0)
-		b2 = ws->e - ws->f;
-	else if (bytes > ws->e - ws->f)
-		b2 = ws->e - ws->f;
-	else
-		b2 = bytes;
-	b2 = PRNDDN(b2);
-	xxxassert(ws->f + b2 <= ws->e);
-	ws->r = ws->f + b2;
-	DSL(0x02, SLT_Debug, 0, "WS_Reserve(%p, %u/%u) = %u",
-	    ws, b2, bytes, pdiff(ws->f, ws->r));
-	WS_Assert(ws);
-	return (pdiff(ws->f, ws->r));
-}
-
-void
-WS_Release(struct ws *ws, unsigned bytes)
-{
-	WS_Assert(ws);
-	bytes = PRNDUP(bytes);
-	assert(bytes <= ws->e - ws->f);
-	DSL(0x02, SLT_Debug, 0, "WS_Release(%p, %u)", ws, bytes);
-	assert(ws->r != NULL);
-	assert(ws->f + bytes <= ws->r);
-	ws->f += bytes;
-	ws->r = NULL;
-	WS_Assert(ws);
-}
-
-void
-WS_ReleaseP(struct ws *ws, char *ptr)
-{
-	WS_Assert(ws);
-	DSL(0x02, SLT_Debug, 0, "WS_ReleaseP(%p, %p)", ws, ptr);
-	assert(ws->r != NULL);
-	assert(ptr >= ws->f);
-	assert(ptr <= ws->r);
-	ws->f += PRNDUP(ptr - ws->f);
-	ws->r = NULL;
-	WS_Assert(ws);
-}
-
-#if 0
-/* XXX: not used anywhere (yet) */
-void
-WS_Return(struct ws *ws, char *s, char *e)
-{
-
-	WS_Assert(ws);
-	if (e == ws->f)
-		ws->f = s;
-}
-#endif
diff --git a/bin/varnishd/hash/hash_classic.c b/bin/varnishd/hash/hash_classic.c
index 6b1d589..9ea27de 100644
--- a/bin/varnishd/hash/hash_classic.c
+++ b/bin/varnishd/hash/hash_classic.c
@@ -34,7 +34,7 @@
 #include <stdio.h>
 #include <stdlib.h>
 
-#include "cache.h"
+#include "cache/cache.h"
 
 #include "hash/hash_slinger.h"
 
diff --git a/bin/varnishd/hash/hash_critbit.c b/bin/varnishd/hash/hash_critbit.c
index fd50b26..ecb7b62 100644
--- a/bin/varnishd/hash/hash_critbit.c
+++ b/bin/varnishd/hash/hash_critbit.c
@@ -34,7 +34,7 @@
 
 #include <stdlib.h>
 
-#include "cache.h"
+#include "cache/cache.h"
 
 #include "hash/hash_slinger.h"
 #include "vcli_priv.h"
diff --git a/bin/varnishd/hash/hash_simple_list.c b/bin/varnishd/hash/hash_simple_list.c
index 1a0cae5..4bdae02 100644
--- a/bin/varnishd/hash/hash_simple_list.c
+++ b/bin/varnishd/hash/hash_simple_list.c
@@ -31,7 +31,7 @@
 
 #include "config.h"
 
-#include "cache.h"
+#include "cache/cache.h"
 
 #include "hash/hash_slinger.h"
 
diff --git a/bin/varnishd/storage/stevedore.c b/bin/varnishd/storage/stevedore.c
index b119c06..3978f64 100644
--- a/bin/varnishd/storage/stevedore.c
+++ b/bin/varnishd/storage/stevedore.c
@@ -36,7 +36,7 @@
 #include <stdio.h>
 #include <stdlib.h>
 
-#include "cache.h"
+#include "cache/cache.h"
 
 #include "storage/storage.h"
 #include "vav.h"
diff --git a/bin/varnishd/storage/storage_file.c b/bin/varnishd/storage/storage_file.c
index 981d43e..2d9ccf4 100644
--- a/bin/varnishd/storage/storage_file.c
+++ b/bin/varnishd/storage/storage_file.c
@@ -36,7 +36,7 @@
 #include <stdio.h>
 #include <stdlib.h>
 
-#include "cache.h"
+#include "cache/cache.h"
 #include "storage/storage.h"
 
 #include "vnum.h"
diff --git a/bin/varnishd/storage/storage_malloc.c b/bin/varnishd/storage/storage_malloc.c
index 03d6a55..156c832 100644
--- a/bin/varnishd/storage/storage_malloc.c
+++ b/bin/varnishd/storage/storage_malloc.c
@@ -34,7 +34,7 @@
 #include <stdio.h>
 #include <stdlib.h>
 
-#include "cache.h"
+#include "cache/cache.h"
 #include "storage/storage.h"
 
 #include "vnum.h"
diff --git a/bin/varnishd/storage/storage_persistent.c b/bin/varnishd/storage/storage_persistent.c
index 02d244e..ded638b 100644
--- a/bin/varnishd/storage/storage_persistent.c
+++ b/bin/varnishd/storage/storage_persistent.c
@@ -43,7 +43,7 @@
 #include <stdlib.h>
 #include <string.h>
 
-#include "cache.h"
+#include "cache/cache.h"
 #include "storage/storage.h"
 
 #include "hash/hash_slinger.h"
diff --git a/bin/varnishd/storage/storage_persistent_mgt.c b/bin/varnishd/storage/storage_persistent_mgt.c
index 1631ea6..bb0eb38 100644
--- a/bin/varnishd/storage/storage_persistent_mgt.c
+++ b/bin/varnishd/storage/storage_persistent_mgt.c
@@ -41,7 +41,7 @@
 #include <stdio.h>
 #include <stdlib.h>
 
-#include "cache.h"
+#include "cache/cache.h"
 #include "storage/storage.h"
 
 #include "vsha256.h"
diff --git a/bin/varnishd/storage/storage_persistent_silo.c b/bin/varnishd/storage/storage_persistent_silo.c
index 27f0c98..b7ef952 100644
--- a/bin/varnishd/storage/storage_persistent_silo.c
+++ b/bin/varnishd/storage/storage_persistent_silo.c
@@ -37,7 +37,7 @@
 #include <stdio.h>
 #include <stdlib.h>
 
-#include "cache.h"
+#include "cache/cache.h"
 #include "storage/storage.h"
 
 #include "hash/hash_slinger.h"
diff --git a/bin/varnishd/storage/storage_persistent_subr.c b/bin/varnishd/storage/storage_persistent_subr.c
index 2066e72..004bba3 100644
--- a/bin/varnishd/storage/storage_persistent_subr.c
+++ b/bin/varnishd/storage/storage_persistent_subr.c
@@ -41,7 +41,7 @@
 #include <stdio.h>
 #include <stdlib.h>
 
-#include "cache.h"
+#include "cache/cache.h"
 #include "storage/storage.h"
 
 #include "vsha256.h"
diff --git a/bin/varnishd/storage/storage_synth.c b/bin/varnishd/storage/storage_synth.c
index 0519738..e9e9b2f 100644
--- a/bin/varnishd/storage/storage_synth.c
+++ b/bin/varnishd/storage/storage_synth.c
@@ -32,7 +32,7 @@
 
 #include <stdlib.h>
 
-#include "cache.h"
+#include "cache/cache.h"
 #include "storage/storage.h"
 
 
diff --git a/bin/varnishd/storage/storage_umem.c b/bin/varnishd/storage/storage_umem.c
index 78110d4..52d238d 100644
--- a/bin/varnishd/storage/storage_umem.c
+++ b/bin/varnishd/storage/storage_umem.c
@@ -39,7 +39,7 @@
 #include <stdlib.h>
 #include <umem.h>
 
-#include "cache.h"
+#include "cache/cache.h"
 #include "storage/storage.h"
 
 static size_t			smu_max = SIZE_MAX;
diff --git a/bin/varnishd/waiter/cache_waiter.c b/bin/varnishd/waiter/cache_waiter.c
index e6654fd..c428294 100644
--- a/bin/varnishd/waiter/cache_waiter.c
+++ b/bin/varnishd/waiter/cache_waiter.c
@@ -30,7 +30,7 @@
 
 #include "config.h"
 
-#include "cache.h"
+#include "cache/cache.h"
 
 #include "waiter/cache_waiter.h"
 #include "vcli.h"
diff --git a/bin/varnishd/waiter/cache_waiter_epoll.c b/bin/varnishd/waiter/cache_waiter_epoll.c
index 46cb288..b4b40f5 100644
--- a/bin/varnishd/waiter/cache_waiter_epoll.c
+++ b/bin/varnishd/waiter/cache_waiter_epoll.c
@@ -40,7 +40,7 @@
 #include <fcntl.h>
 #include <stdlib.h>
 
-#include "cache.h"
+#include "cache/cache.h"
 
 #include "waiter/cache_waiter.h"
 #include "vtim.h"
diff --git a/bin/varnishd/waiter/cache_waiter_kqueue.c b/bin/varnishd/waiter/cache_waiter_kqueue.c
index a631606..c6a03de 100644
--- a/bin/varnishd/waiter/cache_waiter_kqueue.c
+++ b/bin/varnishd/waiter/cache_waiter_kqueue.c
@@ -42,7 +42,7 @@
 #include <stdlib.h>
 #include <unistd.h>
 
-#include "cache.h"
+#include "cache/cache.h"
 
 #include "waiter/cache_waiter.h"
 #include "vtim.h"
diff --git a/bin/varnishd/waiter/cache_waiter_poll.c b/bin/varnishd/waiter/cache_waiter_poll.c
index 5f8dbd7..3f5cea3 100644
--- a/bin/varnishd/waiter/cache_waiter_poll.c
+++ b/bin/varnishd/waiter/cache_waiter_poll.c
@@ -33,7 +33,7 @@
 #include <poll.h>
 #include <stdlib.h>
 
-#include "cache.h"
+#include "cache/cache.h"
 
 #include "waiter/cache_waiter.h"
 #include "vtim.h"
diff --git a/bin/varnishd/waiter/cache_waiter_ports.c b/bin/varnishd/waiter/cache_waiter_ports.c
index aa9f86d..a1699d0 100644
--- a/bin/varnishd/waiter/cache_waiter_ports.c
+++ b/bin/varnishd/waiter/cache_waiter_ports.c
@@ -40,7 +40,7 @@
 #include <stdio.h>
 #include <stdlib.h>
 
-#include "cache.h"
+#include "cache/cache.h"
 
 #include "waiter/cache_waiter.h"
 #include "vtim.h"
diff --git a/lib/libvmod_std/Makefile.am b/lib/libvmod_std/Makefile.am
index c9979fa..065948a 100644
--- a/lib/libvmod_std/Makefile.am
+++ b/lib/libvmod_std/Makefile.am
@@ -1,6 +1,9 @@
 #
 
-INCLUDES = -I$(top_srcdir)/include -I$(top_builddir)/include
+INCLUDES = \
+	-I$(top_srcdir)/include \
+	-I$(top_srcdir)/bin/varnishd \
+	-I$(top_builddir)/include
 
 dist_man_MANS = vmod_std.3
 
diff --git a/lib/libvmod_std/vmod_std.c b/lib/libvmod_std/vmod_std.c
index f54958f..14d8503 100644
--- a/lib/libvmod_std/vmod_std.c
+++ b/lib/libvmod_std/vmod_std.c
@@ -39,7 +39,7 @@
 #include "vrt.h"
 #include "vtcp.h"
 
-#include "../../bin/varnishd/cache.h"
+#include "cache/cache.h"
 
 #include "vcc_if.h"
 
diff --git a/lib/libvmod_std/vmod_std_conversions.c b/lib/libvmod_std/vmod_std_conversions.c
index fe8ae96..149be81 100644
--- a/lib/libvmod_std/vmod_std_conversions.c
+++ b/lib/libvmod_std/vmod_std_conversions.c
@@ -33,7 +33,7 @@
 #include <math.h>
 #include <stdlib.h>
 
-#include "../../bin/varnishd/cache.h"
+#include "cache/cache.h"
 
 #include "vrt.h"
 #include "vcc_if.h"
diff --git a/lib/libvmod_std/vmod_std_fileread.c b/lib/libvmod_std/vmod_std_fileread.c
index 8476bd6..89e4f50 100644
--- a/lib/libvmod_std/vmod_std_fileread.c
+++ b/lib/libvmod_std/vmod_std_fileread.c
@@ -43,7 +43,7 @@
 #include <stdint.h>
 #include <stdlib.h>
 
-#include "../../bin/varnishd/cache.h"
+#include "cache/cache.h"
 
 #include "vrt.h"
 #include "vfil.h"



From geoff at varnish-cache.org  Mon Jan  9 20:52:33 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:33 +0100
Subject: [experimental-ims] ccf9f85 The CLI protocol and its implementation is
	not designed for transferring huge amounts of data,
	and it should not be used that way.
Message-ID: <E1RkMCn-0008B3-P0@project.varnish-software.com>

commit ccf9f85a217bb240550e9c2fac69c25ab23d8187
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Nov 14 10:18:18 2011 +0000

    The CLI protocol and its implementation is not designed for
    transferring huge amounts of data, and it should not be used that
    way.
    
    But we have commands like ban.list which potentially can return at
    metric shitload of data and this is a problem.
    
    This commit implements a cli_limit paramter which (with its upper
    limit) protects from responses that cannot be moved by the CLI
    protocol no matter what.
    
    We set the default to a low 4096 bytes, in order to make people
    specifically shoot their feet off if they really want to see 200k
    bans.

diff --git a/bin/varnishd/cache/cache_cli.c b/bin/varnishd/cache/cache_cli.c
index f29f86a..f299bef 100644
--- a/bin/varnishd/cache/cache_cli.c
+++ b/bin/varnishd/cache/cache_cli.c
@@ -236,7 +236,8 @@ CLI_Init(void)
 	Lck_New(&cli_mtx, lck_cli);
 	cli_thread = pthread_self();
 
-	cls = VCLS_New(cli_cb_before, cli_cb_after, cache_param->cli_buffer);
+	cls = VCLS_New(cli_cb_before, cli_cb_after,
+	    &cache_param->cli_buffer, &cache_param->cli_limit);
 	AN(cls);
 
 	CLI_AddFuncs(master_cmds);
diff --git a/bin/varnishd/common/params.h b/bin/varnishd/common/params.h
index fb71e33..ee8bad2 100644
--- a/bin/varnishd/common/params.h
+++ b/bin/varnishd/common/params.h
@@ -107,6 +107,7 @@ struct params {
 
 	/* CLI related */
 	unsigned		cli_timeout;
+	unsigned		cli_limit;
 	unsigned		ping_interval;
 
 	/* LRU list ordering interval */
diff --git a/bin/varnishd/mgt/mgt_cli.c b/bin/varnishd/mgt/mgt_cli.c
index 70e8557..1d46d82 100644
--- a/bin/varnishd/mgt/mgt_cli.c
+++ b/bin/varnishd/mgt/mgt_cli.c
@@ -336,7 +336,8 @@ static void
 mgt_cli_init_cls(void)
 {
 
-	cls = VCLS_New(mgt_cli_cb_before, mgt_cli_cb_after, mgt_param.cli_buffer);
+	cls = VCLS_New(mgt_cli_cb_before, mgt_cli_cb_after,
+	    &mgt_param.cli_buffer, &mgt_param.cli_limit);
 	AN(cls);
 	AZ(VCLS_AddFunc(cls, MCF_NOAUTH, cli_auth));
 	AZ(VCLS_AddFunc(cls, MCF_AUTH, cli_proto));
diff --git a/bin/varnishd/mgt/mgt_main.c b/bin/varnishd/mgt/mgt_main.c
index 981841b..8a9e1fc 100644
--- a/bin/varnishd/mgt/mgt_main.c
+++ b/bin/varnishd/mgt/mgt_main.c
@@ -349,6 +349,7 @@ main(int argc, char * const *argv)
 	struct cli cli[1];
 	struct vpf_fh *pfh = NULL;
 	char *dirname;
+	unsigned clilim;
 
 	/*
 	 * Start out by closing all unwanted file descriptors we might
@@ -389,9 +390,12 @@ main(int argc, char * const *argv)
 	SHA256_Test();
 
 	memset(cli, 0, sizeof cli);
+	cli[0].magic = CLI_MAGIC;
 	cli[0].sb = VSB_new_auto();
 	XXXAN(cli[0].sb);
 	cli[0].result = CLIS_OK;
+	clilim = 32768;
+	cli[0].limit = &clilim;
 
 	VTAILQ_INIT(&heritage.socks);
 
diff --git a/bin/varnishd/mgt/mgt_param.c b/bin/varnishd/mgt/mgt_param.c
index 8050d08..afaab84 100644
--- a/bin/varnishd/mgt/mgt_param.c
+++ b/bin/varnishd/mgt/mgt_param.c
@@ -673,6 +673,19 @@ static const struct parspec input_parspec[] = {
 		"Listen queue depth.",
 		MUST_RESTART,
 		"1024", "connections" },
+	{ "cli_buffer", tweak_uint, &mgt_param.cli_buffer, 4096, UINT_MAX,
+		"Size of buffer for CLI command input."
+		"\nYou may need to increase this if you have big VCL files "
+		"and use the vcl.inline CLI command.\n"
+		"NB: Must be specified with -p to have effect.\n",
+		0,
+		"8192", "bytes" },
+	{ "cli_limit", tweak_uint, &mgt_param.cli_limit, 128, 99999999,
+		"Maximum size of CLI response.  If the response exceeds"
+		" this limit, the reponse code will be 201 instead of"
+		" 200 and the last line will indicate the truncation.",
+		0,
+		"4096", "bytes" },
 	{ "cli_timeout", tweak_timeout, &mgt_param.cli_timeout, 0, 0,
 		"Timeout for the childs replies to CLI requests from "
 		"the mgt_param.",
@@ -805,13 +818,6 @@ static const struct parspec input_parspec[] = {
 		"more sessions take a detour around the waiter.",
 		EXPERIMENTAL,
 		"50", "ms" },
-	{ "cli_buffer", tweak_uint, &mgt_param.cli_buffer, 4096, UINT_MAX,
-		"Size of buffer for CLI input."
-		"\nYou may need to increase this if you have big VCL files "
-		"and use the vcl.inline CLI command.\n"
-		"NB: Must be specified with -p to have effect.\n",
-		0,
-		"8192", "bytes" },
 	{ "log_hashstring", tweak_bool, &mgt_param.log_hash, 0, 0,
 		"Log the hash string components to shared memory log.\n",
 		0,
diff --git a/bin/varnishtest/tests/b00008.vtc b/bin/varnishtest/tests/b00008.vtc
index 1e9aa9d..cee8d4c 100644
--- a/bin/varnishtest/tests/b00008.vtc
+++ b/bin/varnishtest/tests/b00008.vtc
@@ -33,3 +33,7 @@ varnish v1 -cliok "help"
 varnish v1 -cliok "param.set waiter default"
 
 varnish v1 -clierr 106 "param.set waiter HASH(0x8839c4c)"
+
+varnish v1 -cliok "param.set cli_limit 128"
+
+varnish v1 -clierr 201 "param.show"
diff --git a/include/vcli.h b/include/vcli.h
index cd98f58..04ac11c 100644
--- a/include/vcli.h
+++ b/include/vcli.h
@@ -199,6 +199,7 @@ enum VCLI_status_e {
 	CLIS_PARAM	= 106,
 	CLIS_AUTH	= 107,
 	CLIS_OK		= 200,
+	CLIS_TRUNCATED	= 201,
 	CLIS_CANT	= 300,
 	CLIS_COMMS	= 400,
 	CLIS_CLOSE	= 500
diff --git a/include/vcli_common.h b/include/vcli_common.h
index 80d680f..8d0a623 100644
--- a/include/vcli_common.h
+++ b/include/vcli_common.h
@@ -42,4 +42,5 @@ struct cli {
 	char			*ident;
 	struct vlu		*vlu;
 	struct VCLS		*cls;
+	volatile unsigned	*limit;
 };
diff --git a/include/vcli_serve.h b/include/vcli_serve.h
index 0ac29f8..7e4e6e4 100644
--- a/include/vcli_serve.h
+++ b/include/vcli_serve.h
@@ -30,7 +30,8 @@
 struct VCLS;
 typedef void cls_cb_f(void *priv);
 typedef void cls_cbc_f(const struct cli*);
-struct VCLS *VCLS_New(cls_cbc_f *before, cls_cbc_f *after, unsigned maxlen);
+struct VCLS *VCLS_New(cls_cbc_f *before, cls_cbc_f *after,
+    volatile unsigned *maxlen, volatile unsigned *limit);
 struct cli *VCLS_AddFd(struct VCLS *cs, int fdi, int fdo, cls_cb_f *closefunc,
     void *priv);
 int VCLS_AddFunc(struct VCLS *cs, unsigned auth, struct cli_proto *clp);
diff --git a/lib/libvarnish/cli_common.c b/lib/libvarnish/cli_common.c
index 4a17de5..7b09bdb 100644
--- a/lib/libvarnish/cli_common.c
+++ b/lib/libvarnish/cli_common.c
@@ -43,6 +43,7 @@
 #include <time.h>
 #include <unistd.h>
 
+#include "miniobj.h"
 #include "vas.h"
 #include "vcli.h"
 #include "vcli_common.h"
@@ -56,10 +57,15 @@ VCLI_Out(struct cli *cli, const char *fmt, ...)
 	va_list ap;
 
 	va_start(ap, fmt);
-	if (cli != NULL)
-		(void)VSB_vprintf(cli->sb, fmt, ap);
-	else
+	if (cli != NULL) {
+		CHECK_OBJ_NOTNULL(cli, CLI_MAGIC);
+		if (VSB_len(cli->sb) < *cli->limit)
+			(void)VSB_vprintf(cli->sb, fmt, ap);
+		else if (cli->result == CLIS_OK)
+			cli->result = CLIS_TRUNCATED;
+	} else {
 		(void)vfprintf(stdout, fmt, ap);
+	}
 	va_end(ap);
 }
 
@@ -68,6 +74,7 @@ void
 VCLI_Quote(struct cli *cli, const char *s)
 {
 
+	CHECK_OBJ_NOTNULL(cli, CLI_MAGIC);
 	VSB_quote(cli->sb, s, -1, 0);
 }
 
@@ -75,10 +82,13 @@ void
 VCLI_SetResult(struct cli *cli, unsigned res)
 {
 
-	if (cli != NULL)
-		cli->result = res;	/*lint !e64 type mismatch */
-	else
+	if (cli != NULL) {
+		CHECK_OBJ_NOTNULL(cli, CLI_MAGIC);
+		if (cli->result != CLIS_TRUNCATED || res != CLIS_OK)
+			cli->result = res;	/*lint !e64 type mismatch */
+	} else {
 		printf("CLI result = %u\n", res);
+	}
 }
 
 int
diff --git a/lib/libvarnish/cli_serve.c b/lib/libvarnish/cli_serve.c
index f3def96..547061c 100644
--- a/lib/libvarnish/cli_serve.c
+++ b/lib/libvarnish/cli_serve.c
@@ -81,7 +81,8 @@ struct VCLS {
 	unsigned			nfd;
 	VTAILQ_HEAD(,VCLS_func)		funcs;
 	cls_cbc_f			*before, *after;
-	unsigned			maxlen;
+	volatile unsigned		*maxlen;
+	volatile unsigned		*limit;
 };
 
 /*--------------------------------------------------------------------*/
@@ -245,6 +246,10 @@ cls_vlu2(void *priv, char * const *av)
 	struct VCLS_func *cfn;
 	struct cli *cli;
 	unsigned na;
+	ssize_t len;
+	char *s;
+	unsigned lim;
+	const char *trunc = "!\n[response was truncated]\n";
 
 	CAST_OBJ_NOTNULL(cfd, priv, VCLS_FD_MAGIC);
 	cs = cfd->cls;
@@ -297,7 +302,16 @@ cls_vlu2(void *priv, char * const *av)
 
 	cli->cls = NULL;
 
-	if (VCLI_WriteResult(cfd->fdo, cli->result, VSB_data(cli->sb)) ||
+	s = VSB_data(cli->sb);
+	len = VSB_len(cli->sb);
+	lim = *cs->limit;
+	if (len > lim) {
+		if (cli->result == CLIS_OK)
+			cli->result = CLIS_TRUNCATED;
+		strcpy(s + (lim - strlen(trunc)), trunc);
+		assert(strlen(s) <= lim);
+	}
+	if (VCLI_WriteResult(cfd->fdo, cli->result, s) ||
 	    cli->result == CLIS_CLOSE)
 		return (1);
 
@@ -380,7 +394,8 @@ cls_vlu(void *priv, const char *p)
 }
 
 struct VCLS *
-VCLS_New(cls_cbc_f *before, cls_cbc_f *after, unsigned maxlen)
+VCLS_New(cls_cbc_f *before, cls_cbc_f *after, volatile unsigned *maxlen,
+    volatile unsigned *limit)
 {
 	struct VCLS *cs;
 
@@ -391,6 +406,7 @@ VCLS_New(cls_cbc_f *before, cls_cbc_f *after, unsigned maxlen)
 	cs->before = before;
 	cs->after = after;
 	cs->maxlen = maxlen;
+	cs->limit = limit;
 	return (cs);
 }
 
@@ -409,8 +425,9 @@ VCLS_AddFd(struct VCLS *cs, int fdi, int fdo, cls_cb_f *closefunc, void *priv)
 	cfd->fdo = fdo;
 	cfd->cli = &cfd->clis;
 	cfd->cli->magic = CLI_MAGIC;
-	cfd->cli->vlu = VLU_New(cfd, cls_vlu, cs->maxlen);
+	cfd->cli->vlu = VLU_New(cfd, cls_vlu, *cs->maxlen);
 	cfd->cli->sb = VSB_new_auto();
+	cfd->cli->limit = cs->limit;
 	cfd->closefunc = closefunc;
 	cfd->priv = priv;
 	AN(cfd->cli->sb);



From geoff at varnish-cache.org  Mon Jan  9 20:52:33 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:33 +0100
Subject: [experimental-ims] ce8a772 Use PCRE_NOTEMPTY rather than
	NOTEMPTY_ATSTART, it suffices for us
Message-ID: <E1RkMCn-0008BE-SD@project.varnish-software.com>

commit ce8a772629dc143d49427f14e7e1b431be344633
Author: Tollef Fog Heen <tfheen at varnish-software.com>
Date:   Thu Nov 10 12:24:22 2011 +0100

    Use PCRE_NOTEMPTY rather than NOTEMPTY_ATSTART, it suffices for us

diff --git a/bin/varnishd/cache/cache_vrt_re.c b/bin/varnishd/cache/cache_vrt_re.c
index 7759d0a..9b4a2e6 100644
--- a/bin/varnishd/cache/cache_vrt_re.c
+++ b/bin/varnishd/cache/cache_vrt_re.c
@@ -139,7 +139,7 @@ VRT_regsub(const struct sess *sp, int all, const char *str, void *re,
 		if (!all)
 			break;
 		memset(&ovector, 0, sizeof(ovector));
-		options |= VRE_NOTEMPTY_ATSTART;
+		options |= VRE_NOTEMPTY;
 		i = VRE_exec(t, str, len, 0, options, ovector, 30,
 		    &cache_param->vre_limits);
 		if (i < VRE_ERROR_NOMATCH ) {
diff --git a/include/vre.h b/include/vre.h
index 59ffeb0..a59e8d7 100644
--- a/include/vre.h
+++ b/include/vre.h
@@ -49,7 +49,7 @@ typedef struct vre vre_t;
 
 /* And those to PCRE options */
 extern const unsigned VRE_CASELESS;
-extern const unsigned VRE_NOTEMPTY_ATSTART;
+extern const unsigned VRE_NOTEMPTY;
 
 vre_t *VRE_compile(const char *, int, const char **, int *);
 int VRE_exec(const vre_t *code, const char *subject, int length,
diff --git a/lib/libvarnish/vre.c b/lib/libvarnish/vre.c
index eac8121..fb2dcda 100644
--- a/lib/libvarnish/vre.c
+++ b/lib/libvarnish/vre.c
@@ -49,12 +49,7 @@ struct vre {
  * here.
  */
 const unsigned VRE_CASELESS = PCRE_CASELESS;
-const unsigned VRE_NOTEMPTY_ATSTART =
-#ifdef PCRE_NOTEMPTY_ATSTART
-	PCRE_NOTEMPTY_ATSTART;
-#else
-	0;
-#endif
+const unsigned VRE_NOTEMPTY = PCRE_NOTEMPTY;
 
 vre_t *
 VRE_compile(const char *pattern, int options,



From geoff at varnish-cache.org  Mon Jan  9 20:52:33 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:33 +0100
Subject: [experimental-ims] 60ddf62 fix a block of text rendering as part of
	source code
Message-ID: <E1RkMCn-0008BP-VC@project.varnish-software.com>

commit 60ddf626604a625a151c3d1dc011d3a9eba38bd2
Author: Kolia Morev <kolyuchiy at gmail.com>
Date:   Sun Nov 13 13:40:47 2011 +0400

    fix a block of text rendering as part of source code

diff --git a/doc/sphinx/reference/vcl.rst b/doc/sphinx/reference/vcl.rst
index 016e84f..6873fcf 100644
--- a/doc/sphinx/reference/vcl.rst
+++ b/doc/sphinx/reference/vcl.rst
@@ -976,10 +976,10 @@ based on the request URL:::
     }
   }
 
-  The following snippet demonstrates how to force a minimum TTL for
-  all documents.  Note that this is not the same as setting the
-  default_ttl run-time parameter, as that only affects document for
-  which the backend did not specify a TTL:::
+The following snippet demonstrates how to force a minimum TTL for
+all documents.  Note that this is not the same as setting the
+default_ttl run-time parameter, as that only affects document for
+which the backend did not specify a TTL:::
   
   import std; # needed for std.log
 



From geoff at varnish-cache.org  Mon Jan  9 20:52:34 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:34 +0100
Subject: [experimental-ims] e7694ea Merge branch 'master' of
	ssh://git.varnish-cache.org/git/varnish-cache
Message-ID: <E1RkMCo-0008Bt-5i@project.varnish-software.com>

commit e7694ea92d0c9e1953b70ea9059fb3cbfe3eb257
Merge: 9857918 4e9bf8c
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Nov 14 16:32:18 2011 +0000

    Merge branch 'master' of ssh://git.varnish-cache.org/git/varnish-cache



From geoff at varnish-cache.org  Mon Jan  9 20:52:34 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:34 +0100
Subject: [experimental-ims] 4e9bf8c Revert "Teach varnishtest about
	VRE_NOTEMPTY_ATSTART and skip c00047"
Message-ID: <E1RkMCo-0008BY-1O@project.varnish-software.com>

commit 4e9bf8c846308e3ead7b9f1051fe9865e0acf0c5
Author: Tollef Fog Heen <tfheen at varnish-software.com>
Date:   Mon Nov 14 11:59:17 2011 +0100

    Revert "Teach varnishtest about VRE_NOTEMPTY_ATSTART and skip c00047"
    
    This isn't needed since we no longer use VRE_NOTEMPTY_ATSTART
    
    This reverts commit 43b009a8454f18c7aaaa211b19d2e54004dce24e.

diff --git a/bin/varnishtest/tests/c00047.vtc b/bin/varnishtest/tests/c00047.vtc
index dd1d256..4b23194 100644
--- a/bin/varnishtest/tests/c00047.vtc
+++ b/bin/varnishtest/tests/c00047.vtc
@@ -1,7 +1,5 @@
 varnishtest "Test VCL regsuball()"
 
-feature VRE_NOTEMPTY_ATSTART
-
 server s1 {
 	rxreq 
 	txresp \
diff --git a/bin/varnishtest/vtc.c b/bin/varnishtest/vtc.c
index d6c3b35..9581b3c 100644
--- a/bin/varnishtest/vtc.c
+++ b/bin/varnishtest/vtc.c
@@ -43,7 +43,6 @@
 #include "vtc.h"
 
 #include "vav.h"
-#include "vre.h"
 #include "vtim.h"
 
 
@@ -495,9 +494,6 @@ cmd_feature(CMD_ARGS)
 			continue;
 #endif
 		}
-		if (!strcmp(av[i], "VRE_NOTEMPTY_ATSTART") &&
-		    VRE_NOTEMPTY_ATSTART)
-			continue;
 
 		vtc_log(vl, 1, "SKIPPING test, missing feature: %s", av[i]);
 		vtc_stop = 1;



From geoff at varnish-cache.org  Mon Jan  9 20:52:31 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:31 +0100
Subject: [experimental-ims] 3ef111c Merge branch 'master' of
	ssh://git.varnish-cache.org/git/varnish-cache
Message-ID: <E1RkMCl-00087w-Pd@project.varnish-software.com>

commit 3ef111cdef48a81e9a69da59764c5a796872cbc6
Merge: 9b8598d f72bf05
Author: Andreas Plesner Jacobsen <apj at mutt.dk>
Date:   Thu Nov 10 11:28:30 2011 +0100

    Merge branch 'master' of ssh://git.varnish-cache.org/git/varnish-cache



From geoff at varnish-cache.org  Mon Jan  9 20:52:32 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:32 +0100
Subject: [experimental-ims] 921f14b Split params from heritage,
	they have been different beasts for a long time.
Message-ID: <E1RkMCm-00088S-1d@project.varnish-software.com>

commit 921f14b2df75f9f285f24a1975d06a1876d82372
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu Nov 10 15:55:22 2011 +0000

    Split params from heritage, they have been different beasts for a long time.

diff --git a/bin/varnishd/params.h b/bin/varnishd/params.h
new file mode 100644
index 0000000..e6e6558
--- /dev/null
+++ b/bin/varnishd/params.h
@@ -0,0 +1,228 @@
+/*-
+ * Copyright (c) 2006 Verdens Gang AS
+ * Copyright (c) 2006-2011 Varnish Software AS
+ * All rights reserved.
+ *
+ * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * This file contains the heritage passed when mgt forks cache
+ */
+
+#include "vre.h"
+
+struct listen_sock {
+	unsigned			magic;
+#define LISTEN_SOCK_MAGIC		0x999e4b57
+	VTAILQ_ENTRY(listen_sock)	list;
+	int				sock;
+	char				*name;
+	struct vss_addr			*addr;
+};
+
+VTAILQ_HEAD(listen_sock_head, listen_sock);
+
+struct heritage {
+
+	/* Two pipe(2)'s for CLI connection between cache and mgt.  */
+	int				cli_in;
+	int				cli_out;
+
+	/* File descriptor for stdout/stderr */
+	int				std_fd;
+
+	/* Sockets from which to accept connections */
+	struct listen_sock_head		socks;
+	unsigned			nsocks;
+
+	/* Hash method */
+	const struct hash_slinger	*hash;
+
+	char				*name;
+	char                            identity[1024];
+};
+
+struct params {
+
+	/* Unprivileged user / group */
+	char			*user;
+	uid_t			uid;
+	char			*group;
+	gid_t			gid;
+
+	/* TTL used for lack of anything better */
+	double			default_ttl;
+
+	/* Default grace period */
+	double			default_grace;
+
+	/* Default keep period */
+	double			default_keep;
+
+	/* Maximum concurrent sessions */
+	unsigned		max_sess;
+
+	/* Worker threads and pool */
+	unsigned		wthread_min;
+	unsigned		wthread_max;
+	unsigned		wthread_timeout;
+	unsigned		wthread_pools;
+	unsigned		wthread_add_threshold;
+	unsigned		wthread_add_delay;
+	unsigned		wthread_fail_delay;
+	unsigned		wthread_purge_delay;
+	unsigned		wthread_stats_rate;
+	unsigned		wthread_stacksize;
+	unsigned		wthread_workspace;
+
+	unsigned		queue_max;
+
+	/* Memory allocation hints */
+	unsigned		sess_workspace;
+	unsigned		shm_workspace;
+	unsigned		http_req_size;
+	unsigned		http_req_hdr_len;
+	unsigned		http_resp_size;
+	unsigned		http_resp_hdr_len;
+	unsigned		http_max_hdr;
+
+	unsigned		shm_reclen;
+
+	/* Acceptor hints */
+	unsigned		sess_timeout;
+	unsigned		pipe_timeout;
+	unsigned		send_timeout;
+	unsigned		idle_send_timeout;
+
+	/* Management hints */
+	unsigned		auto_restart;
+
+	/* Fetcher hints */
+	unsigned		fetch_chunksize;
+	unsigned		fetch_maxchunksize;
+	unsigned		nuke_limit;
+
+#ifdef SENDFILE_WORKS
+	/* Sendfile object minimum size */
+	unsigned		sendfile_threshold;
+#endif
+
+	/* VCL traces */
+	unsigned		vcl_trace;
+
+	/* Listen address */
+	char			*listen_address;
+
+	/* Listen depth */
+	unsigned		listen_depth;
+
+	/* CLI related */
+	unsigned		cli_timeout;
+	unsigned		ping_interval;
+
+	/* LRU list ordering interval */
+	unsigned		lru_timeout;
+
+	/* Maximum restarts allowed */
+	unsigned		max_restarts;
+
+	/* Maximum esi:include depth allowed */
+	unsigned		max_esi_depth;
+
+	/* ESI parser hints */
+	unsigned		esi_syntax;
+
+	/* Rush exponent */
+	unsigned		rush_exponent;
+
+	/* Default connection_timeout */
+	double			connect_timeout;
+
+	/* Read timeouts for backend */
+	double			first_byte_timeout;
+	double			between_bytes_timeout;
+
+	/* How long to linger on sessions */
+	unsigned		session_linger;
+
+	/* CLI buffer size */
+	unsigned		cli_buffer;
+
+	/* Control diagnostic code */
+	unsigned		diag_bitmap;
+
+	/* Log hash string to shm */
+	unsigned		log_hash;
+
+	/* Log local socket address to shm */
+	unsigned		log_local_addr;
+
+	/* Prefer IPv6 connections to backend*/
+	unsigned		prefer_ipv6;
+
+	/* Acceptable clockskew with backends */
+	unsigned		clock_skew;
+
+	/* Expiry pacer parameters */
+	double			expiry_sleep;
+
+	/* Acceptor pacer parameters */
+	double			acceptor_sleep_max;
+	double			acceptor_sleep_incr;
+	double			acceptor_sleep_decay;
+
+	/* Get rid of duplicate bans */
+	unsigned		ban_dups;
+
+	/* How long time does the ban lurker sleep */
+	double			ban_lurker_sleep;
+
+	/* Max size of the saintmode list. 0 == no saint mode. */
+	unsigned		saintmode_threshold;
+
+	unsigned		syslog_cli_traffic;
+
+	unsigned		http_range_support;
+
+	unsigned		http_gzip_support;
+	unsigned		gzip_stack_buffer;
+	unsigned		gzip_tmp_space;
+	unsigned		gzip_level;
+	unsigned		gzip_window;
+	unsigned		gzip_memlevel;
+
+	double			critbit_cooloff;
+
+	double			shortlived;
+
+	struct vre_limits	vre_limits;
+};
+
+/*
+ * We declare this a volatile pointer, so that reads of parameters
+ * become atomic, leaving the CLI thread lattitude to change the values
+ */
+extern volatile struct params * cache_param;
+extern struct heritage heritage;
+
+void child_main(void);



From geoff at varnish-cache.org  Mon Jan  9 20:52:32 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:32 +0100
Subject: [experimental-ims] 9699424 Split params.h from heritage.h,
	they are different beasts now. Reduce scope of heritage.h Reduce
	scope of libvcl.h
Message-ID: <E1RkMCm-00088n-7t@project.varnish-software.com>

commit 9699424ccd9f7fc51a8d1c651889433fdd200ce5
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sun Nov 13 09:29:46 2011 +0000

    Split params.h from heritage.h, they are different beasts now.
    Reduce scope of heritage.h
    Reduce scope of libvcl.h

diff --git a/bin/varnishd/Makefile.am b/bin/varnishd/Makefile.am
index f7c7620..002447f 100644
--- a/bin/varnishd/Makefile.am
+++ b/bin/varnishd/Makefile.am
@@ -89,6 +89,7 @@ noinst_HEADERS = \
 	heritage.h \
 	mgt/mgt.h \
 	mgt/mgt_cli.h \
+	params.h \
 	storage/storage.h \
 	storage/storage_persistent.h \
 	vparam.h
diff --git a/bin/varnishd/cache.h b/bin/varnishd/cache.h
index 5941fcc..b21f2d4 100644
--- a/bin/varnishd/cache.h
+++ b/bin/varnishd/cache.h
@@ -55,7 +55,7 @@
 #endif
 
 
-#include "heritage.h"
+#include "params.h"
 
 enum body_status {
 #define BODYSTATUS(U,l)	BS_##U,
@@ -909,6 +909,7 @@ void VCL_Init(void);
 void VCL_Refresh(struct VCL_conf **vcc);
 void VCL_Rel(struct VCL_conf **vcc);
 void VCL_Poll(void);
+const char *VCL_Return_Name(unsigned method);
 
 #define VCL_MET_MAC(l,u,b) void VCL_##l##_method(struct sess *);
 #include "tbl/vcl_returns.h"
diff --git a/bin/varnishd/cache_acceptor.c b/bin/varnishd/cache_acceptor.c
index 550a53f..5161b09 100644
--- a/bin/varnishd/cache_acceptor.c
+++ b/bin/varnishd/cache_acceptor.c
@@ -31,6 +31,7 @@
 #include "config.h"
 
 #include "cache.h"
+#include "heritage.h"
 
 #include "vcli.h"
 #include "vcli_priv.h"
diff --git a/bin/varnishd/cache_cli.c b/bin/varnishd/cache_cli.c
index 1e09d23..0bbf94b 100644
--- a/bin/varnishd/cache_cli.c
+++ b/bin/varnishd/cache_cli.c
@@ -39,6 +39,7 @@
 #include <stddef.h>			// offsetof
 
 #include "cache.h"
+#include "heritage.h"
 
 #include "cache_backend.h"		// struct vbc
 #include "hash/hash_slinger.h"		// struct objhead
diff --git a/bin/varnishd/cache_main.c b/bin/varnishd/cache_main.c
index f8f06e1..7a3bcd9 100644
--- a/bin/varnishd/cache_main.c
+++ b/bin/varnishd/cache_main.c
@@ -33,6 +33,7 @@
 #include <stdlib.h>
 
 #include "cache.h"
+#include "heritage.h"
 
 #include "waiter/cache_waiter.h"
 #include "hash/hash_slinger.h"
diff --git a/bin/varnishd/cache_panic.c b/bin/varnishd/cache_panic.c
index f69e90b..c626ae3 100644
--- a/bin/varnishd/cache_panic.c
+++ b/bin/varnishd/cache_panic.c
@@ -44,7 +44,6 @@
 
 #include "cache_backend.h"
 #include "waiter/cache_waiter.h"
-#include "libvcl.h"
 #include "vcl.h"
 
 /*
@@ -226,7 +225,7 @@ pan_sess(const struct sess *sp)
 #undef STEP
 		default: stp = NULL;
 	}
-	hand = VCC_Return_Name(sp->handling);
+	hand = VCL_Return_Name(sp->handling);
 	if (stp != NULL)
 		VSB_printf(vsp, "  step = %s,\n", stp);
 	else
diff --git a/bin/varnishd/cache_pool.c b/bin/varnishd/cache_pool.c
index 62d58aa..eaf7602 100644
--- a/bin/varnishd/cache_pool.c
+++ b/bin/varnishd/cache_pool.c
@@ -46,6 +46,7 @@
 #include <stdlib.h>
 
 #include "cache.h"
+#include "heritage.h"
 
 #include "waiter/cache_waiter.h"
 #include "vtcp.h"
diff --git a/bin/varnishd/cache_vcl.c b/bin/varnishd/cache_vcl.c
index cc67a7f..068b482 100644
--- a/bin/varnishd/cache_vcl.c
+++ b/bin/varnishd/cache_vcl.c
@@ -38,7 +38,6 @@
 
 #include "cache.h"
 
-#include "libvcl.h"
 #include "vcl.h"
 #include "vcli.h"
 #include "vcli_priv.h"
@@ -65,6 +64,21 @@ static struct vcls		*vcl_active; /* protected by vcl_mtx */
 
 /*--------------------------------------------------------------------*/
 
+const char *
+VCL_Return_Name(unsigned method)
+{
+
+	switch (method) {
+#define VCL_RET_MAC(l, U, B) case VCL_RET_##U: return(#l);
+#include "tbl/vcl_returns.h"
+#undef VCL_RET_MAC
+	default:
+		return (NULL);
+	}
+}
+
+/*--------------------------------------------------------------------*/
+
 static void
 VCL_Get(struct VCL_conf **vcc)
 {
@@ -323,7 +337,7 @@ VCL_##func##_method(struct sess *sp)					\
 	sp->cur_method = VCL_MET_ ## upper;				\
 	WSP(sp, SLT_VCL_call, "%s", #func);				\
 	(void)sp->vcl->func##_func(sp);					\
-	WSP(sp, SLT_VCL_return, "%s", VCC_Return_Name(sp->handling));	\
+	WSP(sp, SLT_VCL_return, "%s", VCL_Return_Name(sp->handling));	\
 	sp->cur_method = 0;						\
 	assert((1U << sp->handling) & bitmap);				\
 	assert(!((1U << sp->handling) & ~bitmap));			\
diff --git a/bin/varnishd/cache_vrt_var.c b/bin/varnishd/cache_vrt_var.c
index 7a5aed0..a9bf87f 100644
--- a/bin/varnishd/cache_vrt_var.c
+++ b/bin/varnishd/cache_vrt_var.c
@@ -34,6 +34,7 @@
 #include <stdlib.h>
 
 #include "cache.h"
+#include "heritage.h"
 
 #include "cache_backend.h"
 #include "vrt_obj.h"
diff --git a/bin/varnishd/heritage.h b/bin/varnishd/heritage.h
index e6e6558..36433bb 100644
--- a/bin/varnishd/heritage.h
+++ b/bin/varnishd/heritage.h
@@ -29,8 +29,6 @@
  * This file contains the heritage passed when mgt forks cache
  */
 
-#include "vre.h"
-
 struct listen_sock {
 	unsigned			magic;
 #define LISTEN_SOCK_MAGIC		0x999e4b57
@@ -62,167 +60,6 @@ struct heritage {
 	char                            identity[1024];
 };
 
-struct params {
-
-	/* Unprivileged user / group */
-	char			*user;
-	uid_t			uid;
-	char			*group;
-	gid_t			gid;
-
-	/* TTL used for lack of anything better */
-	double			default_ttl;
-
-	/* Default grace period */
-	double			default_grace;
-
-	/* Default keep period */
-	double			default_keep;
-
-	/* Maximum concurrent sessions */
-	unsigned		max_sess;
-
-	/* Worker threads and pool */
-	unsigned		wthread_min;
-	unsigned		wthread_max;
-	unsigned		wthread_timeout;
-	unsigned		wthread_pools;
-	unsigned		wthread_add_threshold;
-	unsigned		wthread_add_delay;
-	unsigned		wthread_fail_delay;
-	unsigned		wthread_purge_delay;
-	unsigned		wthread_stats_rate;
-	unsigned		wthread_stacksize;
-	unsigned		wthread_workspace;
-
-	unsigned		queue_max;
-
-	/* Memory allocation hints */
-	unsigned		sess_workspace;
-	unsigned		shm_workspace;
-	unsigned		http_req_size;
-	unsigned		http_req_hdr_len;
-	unsigned		http_resp_size;
-	unsigned		http_resp_hdr_len;
-	unsigned		http_max_hdr;
-
-	unsigned		shm_reclen;
-
-	/* Acceptor hints */
-	unsigned		sess_timeout;
-	unsigned		pipe_timeout;
-	unsigned		send_timeout;
-	unsigned		idle_send_timeout;
-
-	/* Management hints */
-	unsigned		auto_restart;
-
-	/* Fetcher hints */
-	unsigned		fetch_chunksize;
-	unsigned		fetch_maxchunksize;
-	unsigned		nuke_limit;
-
-#ifdef SENDFILE_WORKS
-	/* Sendfile object minimum size */
-	unsigned		sendfile_threshold;
-#endif
-
-	/* VCL traces */
-	unsigned		vcl_trace;
-
-	/* Listen address */
-	char			*listen_address;
-
-	/* Listen depth */
-	unsigned		listen_depth;
-
-	/* CLI related */
-	unsigned		cli_timeout;
-	unsigned		ping_interval;
-
-	/* LRU list ordering interval */
-	unsigned		lru_timeout;
-
-	/* Maximum restarts allowed */
-	unsigned		max_restarts;
-
-	/* Maximum esi:include depth allowed */
-	unsigned		max_esi_depth;
-
-	/* ESI parser hints */
-	unsigned		esi_syntax;
-
-	/* Rush exponent */
-	unsigned		rush_exponent;
-
-	/* Default connection_timeout */
-	double			connect_timeout;
-
-	/* Read timeouts for backend */
-	double			first_byte_timeout;
-	double			between_bytes_timeout;
-
-	/* How long to linger on sessions */
-	unsigned		session_linger;
-
-	/* CLI buffer size */
-	unsigned		cli_buffer;
-
-	/* Control diagnostic code */
-	unsigned		diag_bitmap;
-
-	/* Log hash string to shm */
-	unsigned		log_hash;
-
-	/* Log local socket address to shm */
-	unsigned		log_local_addr;
-
-	/* Prefer IPv6 connections to backend*/
-	unsigned		prefer_ipv6;
-
-	/* Acceptable clockskew with backends */
-	unsigned		clock_skew;
-
-	/* Expiry pacer parameters */
-	double			expiry_sleep;
-
-	/* Acceptor pacer parameters */
-	double			acceptor_sleep_max;
-	double			acceptor_sleep_incr;
-	double			acceptor_sleep_decay;
-
-	/* Get rid of duplicate bans */
-	unsigned		ban_dups;
-
-	/* How long time does the ban lurker sleep */
-	double			ban_lurker_sleep;
-
-	/* Max size of the saintmode list. 0 == no saint mode. */
-	unsigned		saintmode_threshold;
-
-	unsigned		syslog_cli_traffic;
-
-	unsigned		http_range_support;
-
-	unsigned		http_gzip_support;
-	unsigned		gzip_stack_buffer;
-	unsigned		gzip_tmp_space;
-	unsigned		gzip_level;
-	unsigned		gzip_window;
-	unsigned		gzip_memlevel;
-
-	double			critbit_cooloff;
-
-	double			shortlived;
-
-	struct vre_limits	vre_limits;
-};
-
-/*
- * We declare this a volatile pointer, so that reads of parameters
- * become atomic, leaving the CLI thread lattitude to change the values
- */
-extern volatile struct params * cache_param;
 extern struct heritage heritage;
 
 void child_main(void);
diff --git a/bin/varnishd/mgt/mgt_child.c b/bin/varnishd/mgt/mgt_child.c
index cfe0b2f..d2e0abd 100644
--- a/bin/varnishd/mgt/mgt_child.c
+++ b/bin/varnishd/mgt/mgt_child.c
@@ -44,8 +44,9 @@
 #include <unistd.h>
 
 #include "mgt/mgt.h"
-
 #include "heritage.h"
+#include "params.h"
+
 #include "vapi/vsm_int.h"
 #include "vbm.h"
 #include "vcli.h"
diff --git a/bin/varnishd/mgt/mgt_cli.c b/bin/varnishd/mgt/mgt_cli.c
index ec8fd4d..3fdb432 100644
--- a/bin/varnishd/mgt/mgt_cli.c
+++ b/bin/varnishd/mgt/mgt_cli.c
@@ -43,8 +43,9 @@
 #include <unistd.h>
 
 #include "mgt/mgt.h"
-
 #include "heritage.h"
+#include "params.h"
+
 #include "vcli.h"
 #include "vcli_common.h"
 #include "vcli_priv.h"
diff --git a/bin/varnishd/mgt/mgt_param.c b/bin/varnishd/mgt/mgt_param.c
index dd898ee..cd96e8b 100644
--- a/bin/varnishd/mgt/mgt_param.c
+++ b/bin/varnishd/mgt/mgt_param.c
@@ -38,9 +38,10 @@
 #include <unistd.h>
 
 #include "mgt/mgt.h"
+#include "heritage.h"
+#include "params.h"
 
 #include "waiter/cache_waiter.h"
-#include "heritage.h"
 #include "vav.h"
 #include "vcli.h"
 #include "vcli_common.h"
diff --git a/bin/varnishd/mgt/mgt_pool.c b/bin/varnishd/mgt/mgt_pool.c
index 94459b4..548ad78 100644
--- a/bin/varnishd/mgt/mgt_pool.c
+++ b/bin/varnishd/mgt/mgt_pool.c
@@ -48,8 +48,9 @@
 #include <unistd.h>
 
 #include "mgt/mgt.h"
-
 #include "heritage.h"
+#include "params.h"
+
 #include "vparam.h"
 
 /*--------------------------------------------------------------------*/
diff --git a/bin/varnishd/mgt/mgt_sandbox.c b/bin/varnishd/mgt/mgt_sandbox.c
index e5fe99c..a9fce93 100644
--- a/bin/varnishd/mgt/mgt_sandbox.c
+++ b/bin/varnishd/mgt/mgt_sandbox.c
@@ -53,8 +53,8 @@
 #include <unistd.h>
 
 #include "mgt/mgt.h"
-
 #include "heritage.h"
+#include "params.h"
 
 /*--------------------------------------------------------------------*/
 
diff --git a/bin/varnishd/mgt/mgt_shmem.c b/bin/varnishd/mgt/mgt_shmem.c
index 3953bdb..bb87a8f 100644
--- a/bin/varnishd/mgt/mgt_shmem.c
+++ b/bin/varnishd/mgt/mgt_shmem.c
@@ -98,9 +98,10 @@
 #include <unistd.h>
 
 #include "mgt/mgt.h"
+#include "heritage.h"
+#include "params.h"
 
 #include "flopen.h"
-#include "heritage.h"
 #include "vapi/vsc_int.h"
 #include "vapi/vsl_int.h"
 #include "vapi/vsm_int.h"
diff --git a/bin/varnishd/params.h b/bin/varnishd/params.h
index e6e6558..fb71e33 100644
--- a/bin/varnishd/params.h
+++ b/bin/varnishd/params.h
@@ -31,37 +31,6 @@
 
 #include "vre.h"
 
-struct listen_sock {
-	unsigned			magic;
-#define LISTEN_SOCK_MAGIC		0x999e4b57
-	VTAILQ_ENTRY(listen_sock)	list;
-	int				sock;
-	char				*name;
-	struct vss_addr			*addr;
-};
-
-VTAILQ_HEAD(listen_sock_head, listen_sock);
-
-struct heritage {
-
-	/* Two pipe(2)'s for CLI connection between cache and mgt.  */
-	int				cli_in;
-	int				cli_out;
-
-	/* File descriptor for stdout/stderr */
-	int				std_fd;
-
-	/* Sockets from which to accept connections */
-	struct listen_sock_head		socks;
-	unsigned			nsocks;
-
-	/* Hash method */
-	const struct hash_slinger	*hash;
-
-	char				*name;
-	char                            identity[1024];
-};
-
 struct params {
 
 	/* Unprivileged user / group */
@@ -223,6 +192,3 @@ struct params {
  * become atomic, leaving the CLI thread lattitude to change the values
  */
 extern volatile struct params * cache_param;
-extern struct heritage heritage;
-
-void child_main(void);
diff --git a/bin/varnishd/varnishd.c b/bin/varnishd/varnishd.c
index 6ae1842..ccda130 100644
--- a/bin/varnishd/varnishd.c
+++ b/bin/varnishd/varnishd.c
@@ -44,9 +44,10 @@
 #include <unistd.h>
 
 #include "mgt/mgt.h"
+#include "heritage.h"
+#include "params.h"
 
 #include "hash/hash_slinger.h"
-#include "heritage.h"
 #include "vav.h"
 #include "vcli.h"
 #include "vcli_common.h"
diff --git a/include/libvcl.h b/include/libvcl.h
index a92906f..e046db9 100644
--- a/include/libvcl.h
+++ b/include/libvcl.h
@@ -37,4 +37,3 @@ void VCC_VMOD_dir(struct vcc *, const char *str);
 void VCC_Err_Unref(struct vcc *tl, unsigned u);
 
 char *VCC_Compile(const struct vcc *, struct vsb *sb, const char *b);
-const char *VCC_Return_Name(unsigned action);
diff --git a/lib/libvcl/vcc_compile.c b/lib/libvcl/vcc_compile.c
index a0c6232..5719cbf 100644
--- a/lib/libvcl/vcc_compile.c
+++ b/lib/libvcl/vcc_compile.c
@@ -710,21 +710,6 @@ VCC_Compile(const struct vcc *tl, struct vsb *sb, const char *b)
 	return (r);
 }
 
-/*--------------------------------------------------------------------*/
-
-const char *
-VCC_Return_Name(unsigned method)
-{
-
-	switch (method) {
-#define VCL_RET_MAC(l, U, B) case VCL_RET_##U: return(#l);
-#include "tbl/vcl_returns.h"
-#undef VCL_RET_MAC
-	default:
-		return (NULL);
-	}
-}
-
 /*--------------------------------------------------------------------
  * Allocate a compiler instance
  */



From geoff at varnish-cache.org  Mon Jan  9 20:52:32 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:32 +0100
Subject: [experimental-ims] 9393154 Remove some unused #includes
Message-ID: <E1RkMCm-00088x-BT@project.varnish-software.com>

commit 9393154348e1f28b7815ba9c7d9078243ea43645
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sun Nov 13 09:40:14 2011 +0000

    Remove some unused #includes

diff --git a/lib/libvarnish/binary_heap.c b/lib/libvarnish/binary_heap.c
index ed99f13..33616b5 100644
--- a/lib/libvarnish/binary_heap.c
+++ b/lib/libvarnish/binary_heap.c
@@ -38,12 +38,10 @@
 #include <errno.h>
 #include <limits.h>
 #include <stdint.h>
-#include <stdio.h>		// for testcase
 #include <stdlib.h>
 #include <unistd.h>
 
 #include "binary_heap.h"
-#include "miniobj.h"		// for testcase
 #include "vas.h"
 
 /* Parameters --------------------------------------------------------*/
@@ -457,6 +455,11 @@ binheap_reorder(const struct binheap *bh, unsigned idx)
 }
 
 #ifdef TEST_DRIVER
+
+#include <stdio.h>
+
+#include "miniobj.h"
+
 /* Test driver -------------------------------------------------------*/
 
 static void
diff --git a/lib/libvarnish/vmb.c b/lib/libvarnish/vmb.c
index 8f130f1..692aab2 100644
--- a/lib/libvarnish/vmb.c
+++ b/lib/libvarnish/vmb.c
@@ -58,4 +58,4 @@ vmb_pthread(void)
 	AZ(pthread_mutex_unlock(&mb_mtx));
 }
 
-#endif
+#endif /* VMB_NEEDS_PTHREAD_WORKAROUND_THIS_IS_BAD_FOR_PERFORMANCE */
diff --git a/lib/libvarnish/vnum.c b/lib/libvarnish/vnum.c
index d2f423b..605947e 100644
--- a/lib/libvarnish/vnum.c
+++ b/lib/libvarnish/vnum.c
@@ -35,7 +35,6 @@
 #include <stdio.h>
 #include <stdlib.h>
 
-#include "vas.h"
 #include "vnum.h"
 
 static const char err_miss_num[] = "Missing number";
diff --git a/lib/libvmod_std/vmod_std.c b/lib/libvmod_std/vmod_std.c
index 79cbf02..f54958f 100644
--- a/lib/libvmod_std/vmod_std.c
+++ b/lib/libvmod_std/vmod_std.c
@@ -33,7 +33,6 @@
 #include <ctype.h>
 #include <errno.h>
 #include <stdarg.h>
-#include <stdint.h>
 #include <stdlib.h>
 #include <syslog.h>
 
diff --git a/lib/libvmod_std/vmod_std_conversions.c b/lib/libvmod_std/vmod_std_conversions.c
index 156cf13..fe8ae96 100644
--- a/lib/libvmod_std/vmod_std_conversions.c
+++ b/lib/libvmod_std/vmod_std_conversions.c
@@ -30,9 +30,7 @@
 #include "config.h"
 
 #include <ctype.h>
-#include <errno.h>
 #include <math.h>
-#include <stdint.h>
 #include <stdlib.h>
 
 #include "../../bin/varnishd/cache.h"



From geoff at varnish-cache.org  Mon Jan  9 20:52:32 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:32 +0100
Subject: [experimental-ims] 43b009a Teach varnishtest about
	VRE_NOTEMPTY_ATSTART and skip c00047 which just hangs if we
	don't have it
Message-ID: <E1RkMCm-000898-Dz@project.varnish-software.com>

commit 43b009a8454f18c7aaaa211b19d2e54004dce24e
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sun Nov 13 09:45:49 2011 +0000

    Teach varnishtest about VRE_NOTEMPTY_ATSTART and skip c00047
    which just hangs if we don't have it

diff --git a/bin/varnishtest/tests/c00047.vtc b/bin/varnishtest/tests/c00047.vtc
index 4b23194..dd1d256 100644
--- a/bin/varnishtest/tests/c00047.vtc
+++ b/bin/varnishtest/tests/c00047.vtc
@@ -1,5 +1,7 @@
 varnishtest "Test VCL regsuball()"
 
+feature VRE_NOTEMPTY_ATSTART
+
 server s1 {
 	rxreq 
 	txresp \
diff --git a/bin/varnishtest/vtc.c b/bin/varnishtest/vtc.c
index 9581b3c..d6c3b35 100644
--- a/bin/varnishtest/vtc.c
+++ b/bin/varnishtest/vtc.c
@@ -43,6 +43,7 @@
 #include "vtc.h"
 
 #include "vav.h"
+#include "vre.h"
 #include "vtim.h"
 
 
@@ -494,6 +495,9 @@ cmd_feature(CMD_ARGS)
 			continue;
 #endif
 		}
+		if (!strcmp(av[i], "VRE_NOTEMPTY_ATSTART") &&
+		    VRE_NOTEMPTY_ATSTART)
+			continue;
 
 		vtc_log(vl, 1, "SKIPPING test, missing feature: %s", av[i]);
 		vtc_stop = 1;



From geoff at varnish-cache.org  Mon Jan  9 20:52:32 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:32 +0100
Subject: [experimental-ims] 91ac88d Silence a Flexelint warning
Message-ID: <E1RkMCm-00089J-Gn@project.varnish-software.com>

commit 91ac88d789af4bdd1f546e96b8e7601abf7f04f3
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sun Nov 13 10:05:39 2011 +0000

    Silence a Flexelint warning

diff --git a/bin/varnishd/storage/storage_file.c b/bin/varnishd/storage/storage_file.c
index 58e1d3d..981d43e 100644
--- a/bin/varnishd/storage/storage_file.c
+++ b/bin/varnishd/storage/storage_file.c
@@ -227,7 +227,8 @@ alloc_smf(struct smf_sc *sc, size_t bytes)
 	b = bytes / sc->pagesize;
 	if (b >= NBUCKET)
 		b = NBUCKET - 1;
-	for (sp = NULL; b < NBUCKET - 1; b++) {
+	sp = NULL;
+	for (; b < NBUCKET - 1; b++) {
 		sp = VTAILQ_FIRST(&sc->free[b]);
 		if (sp != NULL)
 			break;



From geoff at varnish-cache.org  Mon Jan  9 20:52:32 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:32 +0100
Subject: [experimental-ims] ac37476 Add a dead reference to SES_DeletePool()
	to remind ourselves where it will be used from.
Message-ID: <E1RkMCm-00089V-Jk@project.varnish-software.com>

commit ac3747664187fc3487107aee51a776be343745c5
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sun Nov 13 10:09:00 2011 +0000

    Add a dead reference to SES_DeletePool() to remind ourselves where
    it will be used from.

diff --git a/bin/varnishd/cache_pool.c b/bin/varnishd/cache_pool.c
index eaf7602..be49548 100644
--- a/bin/varnishd/cache_pool.c
+++ b/bin/varnishd/cache_pool.c
@@ -571,6 +571,8 @@ pool_poolherder(void *priv)
 			}
 		}
 		/* XXX: remove pools */
+		if (0)
+			SES_DeletePool(NULL, NULL);
 		(void)sleep(1);
 		u = 0;
 		VTAILQ_FOREACH(pp, &pools, list)



From geoff at varnish-cache.org  Mon Jan  9 20:52:32 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:32 +0100
Subject: [experimental-ims] 5c0eee3 rename varnishd.c to mgt/mgt_main.c
Message-ID: <E1RkMCm-00089f-NJ@project.varnish-software.com>

commit 5c0eee322b6ad646781f66f40d26f01dd7e0079d
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sun Nov 13 10:09:28 2011 +0000

    rename varnishd.c to mgt/mgt_main.c

diff --git a/bin/varnishd/Makefile.am b/bin/varnishd/Makefile.am
index 002447f..94b52a5 100644
--- a/bin/varnishd/Makefile.am
+++ b/bin/varnishd/Makefile.am
@@ -58,6 +58,7 @@ varnishd_SOURCES = \
 	hash/hash_simple_list.c \
 	mgt/mgt_child.c \
 	mgt/mgt_cli.c \
+	mgt/mgt_main.c \
 	mgt/mgt_param.c \
 	mgt/mgt_pool.c \
 	mgt/mgt_sandbox.c \
@@ -75,7 +76,6 @@ varnishd_SOURCES = \
 	storage/storage_persistent_subr.c \
 	storage/storage_synth.c \
 	storage/storage_umem.c \
-	varnishd.c \
 	vsm.c
 
 noinst_HEADERS = \
diff --git a/bin/varnishd/mgt/mgt_main.c b/bin/varnishd/mgt/mgt_main.c
new file mode 100644
index 0000000..ccda130
--- /dev/null
+++ b/bin/varnishd/mgt/mgt_main.c
@@ -0,0 +1,652 @@
+/*-
+ * Copyright (c) 2006 Verdens Gang AS
+ * Copyright (c) 2006-2011 Varnish Software AS
+ * All rights reserved.
+ *
+ * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * The management process and CLI handling
+ */
+
+#include "config.h"
+
+#include <sys/stat.h>
+#include <sys/utsname.h>
+
+#include <ctype.h>
+#include <fcntl.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <syslog.h>
+#include <time.h>
+#include <unistd.h>
+
+#include "mgt/mgt.h"
+#include "heritage.h"
+#include "params.h"
+
+#include "hash/hash_slinger.h"
+#include "vav.h"
+#include "vcli.h"
+#include "vcli_common.h"
+#include "vev.h"
+#include "vfil.h"
+#include "vin.h"
+#include "vpf.h"
+#include "vsha256.h"
+#include "vtim.h"
+
+#include "compat/daemon.h"
+
+#ifndef HAVE_SRANDOMDEV
+#include "compat/srandomdev.h"
+#endif
+
+struct heritage		heritage;
+unsigned		d_flag = 0;
+pid_t			mgt_pid;
+struct vev_base		*mgt_evb;
+int			exit_status = 0;
+struct vsb		*vident;
+
+static void
+build_vident(void)
+{
+	struct utsname uts;
+
+	vident = VSB_new_auto();
+	AN(vident);
+	if (!uname(&uts)) {
+		VSB_printf(vident, ",%s", uts.sysname);
+		VSB_printf(vident, ",%s", uts.release);
+		VSB_printf(vident, ",%s", uts.machine);
+	}
+}
+
+/*--------------------------------------------------------------------*/
+
+const void *
+pick(const struct choice *cp, const char *which, const char *kind)
+{
+
+	for(; cp->name != NULL; cp++) {
+		if (!strcmp(cp->name, which))
+			return (cp->ptr);
+	}
+	ARGV_ERR("Unknown %s method \"%s\"\n", kind, which);
+}
+
+/*--------------------------------------------------------------------*/
+
+static unsigned long
+arg_ul(const char *p)
+{
+	char *q;
+	unsigned long ul;
+
+	ul = strtoul(p, &q, 0);
+	if (*q != '\0')
+		ARGV_ERR("Invalid number: \"%s\"\n", p);
+	return (ul);
+}
+
+/*--------------------------------------------------------------------*/
+
+static void
+usage(void)
+{
+#define FMT "    %-28s # %s\n"
+
+	fprintf(stderr, "usage: varnishd [options]\n");
+	fprintf(stderr, FMT, "-a address:port", "HTTP listen address and port");
+	fprintf(stderr, FMT, "-b address:port", "backend address and port");
+	fprintf(stderr, FMT, "", "   -b <hostname_or_IP>");
+	fprintf(stderr, FMT, "", "   -b '<hostname_or_IP>:<port_or_service>'");
+	fprintf(stderr, FMT, "-C", "print VCL code compiled to C language");
+	fprintf(stderr, FMT, "-d", "debug");
+	fprintf(stderr, FMT, "-f file", "VCL script");
+	fprintf(stderr, FMT, "-F", "Run in foreground");
+	fprintf(stderr, FMT, "-h kind[,hashoptions]", "Hash specification");
+	fprintf(stderr, FMT, "", "  -h critbit [default]");
+	fprintf(stderr, FMT, "", "  -h simple_list");
+	fprintf(stderr, FMT, "", "  -h classic");
+	fprintf(stderr, FMT, "", "  -h classic,<buckets>");
+	fprintf(stderr, FMT, "-i identity", "Identity of varnish instance");
+	fprintf(stderr, FMT, "-l shl,free,fill", "Size of shared memory file");
+	fprintf(stderr, FMT, "", "  shl: space for SHL records [80m]");
+	fprintf(stderr, FMT, "", "  free: space for other allocations [1m]");
+	fprintf(stderr, FMT, "", "  fill: prefill new file [+]");
+	fprintf(stderr, FMT, "-M address:port", "Reverse CLI destination.");
+	fprintf(stderr, FMT, "-n dir", "varnishd working directory");
+	fprintf(stderr, FMT, "-P file", "PID file");
+	fprintf(stderr, FMT, "-p param=value", "set parameter");
+	fprintf(stderr, FMT,
+	    "-s kind[,storageoptions]", "Backend storage specification");
+	fprintf(stderr, FMT, "", "  -s malloc");
+#ifdef HAVE_LIBUMEM
+	fprintf(stderr, FMT, "", "  -s umem");
+#endif
+	fprintf(stderr, FMT, "", "  -s file  [default: use /tmp]");
+	fprintf(stderr, FMT, "", "  -s file,<dir_or_file>");
+	fprintf(stderr, FMT, "", "  -s file,<dir_or_file>,<size>");
+	fprintf(stderr, FMT, "", "  -s persist{experimenta}");
+	fprintf(stderr, FMT, "",
+	    "  -s file,<dir_or_file>,<size>,<granularity>");
+	fprintf(stderr, FMT, "-t", "Default TTL");
+	fprintf(stderr, FMT, "-S secret-file",
+	    "Secret file for CLI authentication");
+	fprintf(stderr, FMT, "-T address:port",
+	    "Telnet listen address and port");
+	fprintf(stderr, FMT, "-V", "version");
+	fprintf(stderr, FMT, "-w int[,int[,int]]", "Number of worker threads");
+	fprintf(stderr, FMT, "", "  -w <fixed_count>");
+	fprintf(stderr, FMT, "", "  -w min,max");
+	fprintf(stderr, FMT, "", "  -w min,max,timeout [default: -w2,500,300]");
+	fprintf(stderr, FMT, "-u user", "Priviledge separation user id");
+#undef FMT
+	exit(1);
+}
+
+
+/*--------------------------------------------------------------------*/
+
+static void
+tackle_warg(const char *argv)
+{
+	char **av;
+	unsigned int u;
+
+	av = VAV_Parse(argv, NULL, ARGV_COMMA);
+	AN(av);
+
+	if (av[0] != NULL)
+		ARGV_ERR("%s\n", av[0]);
+
+	if (av[1] == NULL)
+		usage();
+
+	u = arg_ul(av[1]);
+	if (u < 1)
+		usage();
+	mgt_param.wthread_max = mgt_param.wthread_min = u;
+
+	if (av[2] != NULL) {
+		u = arg_ul(av[2]);
+		if (u < mgt_param.wthread_min)
+			usage();
+		mgt_param.wthread_max = u;
+
+		if (av[3] != NULL) {
+			u = arg_ul(av[3]);
+			mgt_param.wthread_timeout = u;
+		}
+	}
+	VAV_Free(av);
+}
+
+/*--------------------------------------------------------------------*/
+
+static void
+cli_check(const struct cli *cli)
+{
+	if (cli->result == CLIS_OK) {
+		VSB_clear(cli->sb);
+		return;
+	}
+	AZ(VSB_finish(cli->sb));
+	fprintf(stderr, "Error:\n%s\n", VSB_data(cli->sb));
+	exit (2);
+}
+
+/*--------------------------------------------------------------------
+ * All praise POSIX!  Thanks to our glorious standards there are no
+ * standard way to get a back-trace of the stack, and even if we hack
+ * that together from spit and pieces of string, there is no way no
+ * standard way to translate a pointer to a symbol, which returns anything
+ * usable.  (See for instance FreeBSD PR-134391).
+ *
+ * Attempt to run nm(1) on our binary during startup, hoping it will
+ * give us a usable list of symbols.
+ */
+
+struct symbols {
+	uintptr_t		a;
+	char			*n;
+	VTAILQ_ENTRY(symbols)	list;
+};
+
+static VTAILQ_HEAD(,symbols) symbols = VTAILQ_HEAD_INITIALIZER(symbols);
+
+int
+Symbol_Lookup(struct vsb *vsb, void *ptr)
+{
+	struct symbols *s, *s0;
+	uintptr_t pp;
+
+	pp = (uintptr_t)ptr;
+	s0 = NULL;
+	VTAILQ_FOREACH(s, &symbols, list) {
+		if (s->a > pp)
+			continue;
+		if (s0 == NULL || s->a > s0->a)
+			s0 = s;
+	}
+	if (s0 == NULL)
+		return (-1);
+	VSB_printf(vsb, "%p: %s+%jx", ptr, s0->n, (uintmax_t)pp - s0->a);
+	return (0);
+}
+
+static void
+Symbol_hack(const char *a0)
+{
+	char buf[BUFSIZ], *p, *e;
+	FILE *fi;
+	uintptr_t a;
+	struct symbols *s;
+
+	bprintf(buf, "nm -an %s 2>/dev/null", a0);
+	fi = popen(buf, "r");
+	if (fi == NULL)
+		return;
+	while (fgets(buf, sizeof buf, fi)) {
+		if (buf[0] == ' ')
+			continue;
+		p = NULL;
+		a = strtoul(buf, &p, 16);
+		if (p == NULL)
+			continue;
+		if (a == 0)
+			continue;
+		if (*p++ != ' ')
+			continue;
+		p++;
+		if (*p++ != ' ')
+			continue;
+		if (*p <= ' ')
+			continue;
+		e = strchr(p, '\0');
+		AN(e);
+		while (e > p && isspace(e[-1]))
+			e--;
+		*e = '\0';
+		s = malloc(sizeof *s + strlen(p) + 1);
+		AN(s);
+		s->a = a;
+		s->n = (void*)(s + 1);
+		strcpy(s->n, p);
+		VTAILQ_INSERT_TAIL(&symbols, s, list);
+	}
+	(void)pclose(fi);
+}
+
+/*--------------------------------------------------------------------
+ * This function is called when the CLI on stdin is closed.
+ */
+
+static void
+cli_stdin_close(void *priv)
+{
+
+	(void)priv;
+	(void)close(0);
+	(void)close(1);
+	(void)close(2);
+	assert(open("/dev/null", O_RDONLY) == 0);
+	assert(open("/dev/null", O_WRONLY) == 1);
+	assert(open("/dev/null", O_WRONLY) == 2);
+
+	if (d_flag) {
+		mgt_stop_child();
+		mgt_cli_close_all();
+		exit(0);
+	}
+}
+
+/*--------------------------------------------------------------------*/
+
+int
+main(int argc, char * const *argv)
+{
+	int o;
+	unsigned C_flag = 0;
+	unsigned F_flag = 0;
+	const char *b_arg = NULL;
+	const char *f_arg = NULL;
+	const char *i_arg = NULL;
+	const char *l_arg = NULL;	/* default in mgt_shmem.c */
+	const char *h_arg = "critbit";
+	const char *M_arg = NULL;
+	const char *n_arg = NULL;
+	const char *P_arg = NULL;
+	const char *S_arg = NULL;
+	const char *s_arg = "file";
+	int s_arg_given = 0;
+	const char *T_arg = NULL;
+	char *p, *vcl = NULL;
+	struct cli cli[1];
+	struct vpf_fh *pfh = NULL;
+	char *dirname;
+
+	/*
+	 * Start out by closing all unwanted file descriptors we might
+	 * have inherited from sloppy process control daemons.
+	 */
+	for (o = getdtablesize(); o > STDERR_FILENO; o--)
+		(void)close(o);
+
+	srandomdev();
+
+	mgt_got_fd(STDERR_FILENO);
+
+	setbuf(stdout, NULL);
+	setbuf(stderr, NULL);
+
+	build_vident();
+
+	Symbol_hack(argv[0]);
+
+	/* for ASSERT_MGT() */
+	mgt_pid = getpid();
+
+	/*
+	 * Run in UTC timezone, on the off-chance that this operating
+	 * system does not have a timegm() function, and translates
+	 * timestamps on the local timescale.
+	 * See lib/libvarnish/time.c
+	 */
+	AZ(setenv("TZ", "UTC", 1));
+	tzset();
+	assert(VTIM_parse("Sun, 06 Nov 1994 08:49:37 GMT") == 784111777);
+	assert(VTIM_parse("Sunday, 06-Nov-94 08:49:37 GMT") == 784111777);
+	assert(VTIM_parse("Sun Nov  6 08:49:37 1994") == 784111777);
+
+	/*
+	 * Check that our SHA256 works
+	 */
+	SHA256_Test();
+
+	memset(cli, 0, sizeof cli);
+	cli[0].sb = VSB_new_auto();
+	XXXAN(cli[0].sb);
+	cli[0].result = CLIS_OK;
+
+	VTAILQ_INIT(&heritage.socks);
+
+	MCF_ParamInit(cli);
+
+	if (sizeof(void *) < 8) {
+		/*
+		 * Adjust default parameters for 32 bit systems to conserve
+		 * VM space.
+		 */
+		MCF_ParamSet(cli, "sess_workspace", "16384");
+		cli_check(cli);
+
+		MCF_ParamSet(cli, "thread_pool_workspace", "16384");
+		cli_check(cli);
+
+		MCF_ParamSet(cli, "http_resp_size", "8192");
+		cli_check(cli);
+
+		MCF_ParamSet(cli, "http_req_size", "12288");
+		cli_check(cli);
+
+		MCF_ParamSet(cli, "thread_pool_stack", "32bit");
+		cli_check(cli);
+
+		MCF_ParamSet(cli, "gzip_stack_buffer", "4096");
+		cli_check(cli);
+	}
+
+	cli_check(cli);
+
+	while ((o = getopt(argc, argv,
+	    "a:b:Cdf:Fg:h:i:l:L:M:n:P:p:S:s:T:t:u:Vx:w:")) != -1)
+		switch (o) {
+		case 'a':
+			MCF_ParamSet(cli, "listen_address", optarg);
+			cli_check(cli);
+			break;
+		case 'b':
+			b_arg = optarg;
+			break;
+		case 'C':
+			C_flag = 1 - C_flag;
+			break;
+		case 'd':
+			d_flag++;
+			break;
+		case 'f':
+			f_arg = optarg;
+			break;
+		case 'F':
+			F_flag = 1 - F_flag;
+			break;
+		case 'g':
+			MCF_ParamSet(cli, "group", optarg);
+			break;
+		case 'h':
+			h_arg = optarg;
+			break;
+		case 'i':
+			i_arg = optarg;
+			break;
+		case 'l':
+			l_arg = optarg;
+			break;
+		case 'M':
+			M_arg = optarg;
+			break;
+		case 'n':
+			n_arg = optarg;
+			break;
+		case 'P':
+			P_arg = optarg;
+			break;
+		case 'p':
+			p = strchr(optarg, '=');
+			if (p == NULL)
+				usage();
+			AN(p);
+			*p++ = '\0';
+			MCF_ParamSet(cli, optarg, p);
+			cli_check(cli);
+			break;
+		case 's':
+			s_arg_given = 1;
+			STV_Config(optarg);
+			break;
+		case 't':
+			MCF_ParamSet(cli, "default_ttl", optarg);
+			break;
+		case 'S':
+			S_arg = optarg;
+			break;
+		case 'T':
+			T_arg = optarg;
+			break;
+		case 'u':
+			MCF_ParamSet(cli, "user", optarg);
+			break;
+		case 'V':
+			/* XXX: we should print the ident here */
+			VCS_Message("varnishd");
+			exit(0);
+		case 'x':
+			if (!strcmp(optarg, "dumprst")) {
+				MCF_DumpRst();
+				exit (0);
+			}
+			usage();
+			break;
+		case 'w':
+			tackle_warg(optarg);
+			break;
+		default:
+			usage();
+		}
+
+	argc -= optind;
+	argv += optind;
+
+	mgt_vcc_init();
+
+	if (argc != 0) {
+		fprintf(stderr, "Too many arguments (%s...)\n", argv[0]);
+		usage();
+	}
+
+	/* XXX: we can have multiple CLI actions above, is this enough ? */
+	if (cli[0].result != CLIS_OK) {
+		fprintf(stderr, "Parameter errors:\n");
+		AZ(VSB_finish(cli[0].sb));
+		fprintf(stderr, "%s\n", VSB_data(cli[0].sb));
+		exit(1);
+	}
+
+	if (d_flag && F_flag) {
+		fprintf(stderr, "Only one of -d or -F can be specified\n");
+		usage();
+	}
+
+	if (b_arg != NULL && f_arg != NULL) {
+		fprintf(stderr, "Only one of -b or -f can be specified\n");
+		usage();
+	}
+	if (S_arg == NULL && T_arg == NULL && d_flag == 0 && b_arg == NULL &&
+	    f_arg == NULL && M_arg == NULL) {
+		fprintf(stderr,
+		    "At least one of -d, -b, -f, -M, -S or -T "
+		    "must be specified\n");
+		usage();
+	}
+
+	if (f_arg != NULL) {
+		vcl = VFIL_readfile(NULL, f_arg, NULL);
+		if (vcl == NULL) {
+			fprintf(stderr, "Cannot read '%s': %s\n",
+			    f_arg, strerror(errno));
+			exit(1);
+		}
+	}
+
+	if (VIN_N_Arg(n_arg, &heritage.name, &dirname, NULL) != 0) {
+		fprintf(stderr, "Invalid instance name: %s\n",
+		    strerror(errno));
+		exit(1);
+	}
+
+	if (i_arg != NULL) {
+		if (snprintf(heritage.identity, sizeof heritage.identity,
+		    "%s", i_arg) > sizeof heritage.identity) {
+			fprintf(stderr, "Invalid identity name: %s\n",
+			    strerror(ENAMETOOLONG));
+			exit(1);
+		}
+	}
+
+	if (n_arg != NULL)
+		openlog(n_arg, LOG_PID, LOG_LOCAL0);
+	else
+		openlog("varnishd", LOG_PID, LOG_LOCAL0);
+
+	if (mkdir(dirname, 0755) < 0 && errno != EEXIST) {
+		fprintf(stderr, "Cannot create working directory '%s': %s\n",
+		    dirname, strerror(errno));
+		exit(1);
+	}
+
+	if (chdir(dirname) < 0) {
+		fprintf(stderr, "Cannot change to working directory '%s': %s\n",
+		    dirname, strerror(errno));
+		exit(1);
+	}
+
+	/* XXX: should this be relative to the -n arg ? */
+	if (P_arg && (pfh = VPF_Open(P_arg, 0644, NULL)) == NULL) {
+		perror(P_arg);
+		exit(1);
+	}
+
+	if (b_arg != NULL || f_arg != NULL)
+		if (mgt_vcc_default(b_arg, f_arg, vcl, C_flag))
+			exit (2);
+
+	if (C_flag)
+		exit (0);
+
+	/* If no -s argument specified, process default -s argument */
+	if (!s_arg_given)
+		STV_Config(s_arg);
+
+	/* Configure Transient storage, if user did not */
+	STV_Config_Transient();
+
+	HSH_config(h_arg);
+
+	mgt_SHM_Init(l_arg);
+
+	AZ(VSB_finish(vident));
+
+	if (!d_flag && !F_flag)
+		AZ(varnish_daemon(1, 0));
+
+	mgt_SHM_Pid();
+
+	if (pfh != NULL && VPF_Write(pfh))
+		fprintf(stderr, "NOTE: Could not write PID file\n");
+
+	if (d_flag)
+		fprintf(stderr, "Platform: %s\n", VSB_data(vident) + 1);
+	syslog(LOG_NOTICE, "Platform: %s\n", VSB_data(vident) + 1);
+
+	/* Do this again after debugstunt and daemon has run */
+	mgt_pid = getpid();
+
+	mgt_evb = vev_new_base();
+	XXXAN(mgt_evb);
+
+	if (d_flag)
+		mgt_cli_setup(0, 1, 1, "debug", cli_stdin_close, NULL);
+	if (S_arg != NULL)
+		mgt_cli_secret(S_arg);
+	if (M_arg != NULL)
+		mgt_cli_master(M_arg);
+	if (T_arg != NULL)
+		mgt_cli_telnet(T_arg);
+
+	AN(VSM_Alloc(0, VSM_CLASS_MARK, "", ""));
+
+	MGT_Run();
+
+	if (pfh != NULL)
+		(void)VPF_Remove(pfh);
+	exit(exit_status);
+}
+
+#if defined(PTHREAD_CANCELED) || defined(PTHREAD_MUTEX_DEFAULT)
+#error "Keep pthreads out of in manager process"
+#endif
diff --git a/bin/varnishd/varnishd.c b/bin/varnishd/varnishd.c
deleted file mode 100644
index ccda130..0000000
--- a/bin/varnishd/varnishd.c
+++ /dev/null
@@ -1,652 +0,0 @@
-/*-
- * Copyright (c) 2006 Verdens Gang AS
- * Copyright (c) 2006-2011 Varnish Software AS
- * All rights reserved.
- *
- * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- *
- * The management process and CLI handling
- */
-
-#include "config.h"
-
-#include <sys/stat.h>
-#include <sys/utsname.h>
-
-#include <ctype.h>
-#include <fcntl.h>
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#include <syslog.h>
-#include <time.h>
-#include <unistd.h>
-
-#include "mgt/mgt.h"
-#include "heritage.h"
-#include "params.h"
-
-#include "hash/hash_slinger.h"
-#include "vav.h"
-#include "vcli.h"
-#include "vcli_common.h"
-#include "vev.h"
-#include "vfil.h"
-#include "vin.h"
-#include "vpf.h"
-#include "vsha256.h"
-#include "vtim.h"
-
-#include "compat/daemon.h"
-
-#ifndef HAVE_SRANDOMDEV
-#include "compat/srandomdev.h"
-#endif
-
-struct heritage		heritage;
-unsigned		d_flag = 0;
-pid_t			mgt_pid;
-struct vev_base		*mgt_evb;
-int			exit_status = 0;
-struct vsb		*vident;
-
-static void
-build_vident(void)
-{
-	struct utsname uts;
-
-	vident = VSB_new_auto();
-	AN(vident);
-	if (!uname(&uts)) {
-		VSB_printf(vident, ",%s", uts.sysname);
-		VSB_printf(vident, ",%s", uts.release);
-		VSB_printf(vident, ",%s", uts.machine);
-	}
-}
-
-/*--------------------------------------------------------------------*/
-
-const void *
-pick(const struct choice *cp, const char *which, const char *kind)
-{
-
-	for(; cp->name != NULL; cp++) {
-		if (!strcmp(cp->name, which))
-			return (cp->ptr);
-	}
-	ARGV_ERR("Unknown %s method \"%s\"\n", kind, which);
-}
-
-/*--------------------------------------------------------------------*/
-
-static unsigned long
-arg_ul(const char *p)
-{
-	char *q;
-	unsigned long ul;
-
-	ul = strtoul(p, &q, 0);
-	if (*q != '\0')
-		ARGV_ERR("Invalid number: \"%s\"\n", p);
-	return (ul);
-}
-
-/*--------------------------------------------------------------------*/
-
-static void
-usage(void)
-{
-#define FMT "    %-28s # %s\n"
-
-	fprintf(stderr, "usage: varnishd [options]\n");
-	fprintf(stderr, FMT, "-a address:port", "HTTP listen address and port");
-	fprintf(stderr, FMT, "-b address:port", "backend address and port");
-	fprintf(stderr, FMT, "", "   -b <hostname_or_IP>");
-	fprintf(stderr, FMT, "", "   -b '<hostname_or_IP>:<port_or_service>'");
-	fprintf(stderr, FMT, "-C", "print VCL code compiled to C language");
-	fprintf(stderr, FMT, "-d", "debug");
-	fprintf(stderr, FMT, "-f file", "VCL script");
-	fprintf(stderr, FMT, "-F", "Run in foreground");
-	fprintf(stderr, FMT, "-h kind[,hashoptions]", "Hash specification");
-	fprintf(stderr, FMT, "", "  -h critbit [default]");
-	fprintf(stderr, FMT, "", "  -h simple_list");
-	fprintf(stderr, FMT, "", "  -h classic");
-	fprintf(stderr, FMT, "", "  -h classic,<buckets>");
-	fprintf(stderr, FMT, "-i identity", "Identity of varnish instance");
-	fprintf(stderr, FMT, "-l shl,free,fill", "Size of shared memory file");
-	fprintf(stderr, FMT, "", "  shl: space for SHL records [80m]");
-	fprintf(stderr, FMT, "", "  free: space for other allocations [1m]");
-	fprintf(stderr, FMT, "", "  fill: prefill new file [+]");
-	fprintf(stderr, FMT, "-M address:port", "Reverse CLI destination.");
-	fprintf(stderr, FMT, "-n dir", "varnishd working directory");
-	fprintf(stderr, FMT, "-P file", "PID file");
-	fprintf(stderr, FMT, "-p param=value", "set parameter");
-	fprintf(stderr, FMT,
-	    "-s kind[,storageoptions]", "Backend storage specification");
-	fprintf(stderr, FMT, "", "  -s malloc");
-#ifdef HAVE_LIBUMEM
-	fprintf(stderr, FMT, "", "  -s umem");
-#endif
-	fprintf(stderr, FMT, "", "  -s file  [default: use /tmp]");
-	fprintf(stderr, FMT, "", "  -s file,<dir_or_file>");
-	fprintf(stderr, FMT, "", "  -s file,<dir_or_file>,<size>");
-	fprintf(stderr, FMT, "", "  -s persist{experimenta}");
-	fprintf(stderr, FMT, "",
-	    "  -s file,<dir_or_file>,<size>,<granularity>");
-	fprintf(stderr, FMT, "-t", "Default TTL");
-	fprintf(stderr, FMT, "-S secret-file",
-	    "Secret file for CLI authentication");
-	fprintf(stderr, FMT, "-T address:port",
-	    "Telnet listen address and port");
-	fprintf(stderr, FMT, "-V", "version");
-	fprintf(stderr, FMT, "-w int[,int[,int]]", "Number of worker threads");
-	fprintf(stderr, FMT, "", "  -w <fixed_count>");
-	fprintf(stderr, FMT, "", "  -w min,max");
-	fprintf(stderr, FMT, "", "  -w min,max,timeout [default: -w2,500,300]");
-	fprintf(stderr, FMT, "-u user", "Priviledge separation user id");
-#undef FMT
-	exit(1);
-}
-
-
-/*--------------------------------------------------------------------*/
-
-static void
-tackle_warg(const char *argv)
-{
-	char **av;
-	unsigned int u;
-
-	av = VAV_Parse(argv, NULL, ARGV_COMMA);
-	AN(av);
-
-	if (av[0] != NULL)
-		ARGV_ERR("%s\n", av[0]);
-
-	if (av[1] == NULL)
-		usage();
-
-	u = arg_ul(av[1]);
-	if (u < 1)
-		usage();
-	mgt_param.wthread_max = mgt_param.wthread_min = u;
-
-	if (av[2] != NULL) {
-		u = arg_ul(av[2]);
-		if (u < mgt_param.wthread_min)
-			usage();
-		mgt_param.wthread_max = u;
-
-		if (av[3] != NULL) {
-			u = arg_ul(av[3]);
-			mgt_param.wthread_timeout = u;
-		}
-	}
-	VAV_Free(av);
-}
-
-/*--------------------------------------------------------------------*/
-
-static void
-cli_check(const struct cli *cli)
-{
-	if (cli->result == CLIS_OK) {
-		VSB_clear(cli->sb);
-		return;
-	}
-	AZ(VSB_finish(cli->sb));
-	fprintf(stderr, "Error:\n%s\n", VSB_data(cli->sb));
-	exit (2);
-}
-
-/*--------------------------------------------------------------------
- * All praise POSIX!  Thanks to our glorious standards there are no
- * standard way to get a back-trace of the stack, and even if we hack
- * that together from spit and pieces of string, there is no way no
- * standard way to translate a pointer to a symbol, which returns anything
- * usable.  (See for instance FreeBSD PR-134391).
- *
- * Attempt to run nm(1) on our binary during startup, hoping it will
- * give us a usable list of symbols.
- */
-
-struct symbols {
-	uintptr_t		a;
-	char			*n;
-	VTAILQ_ENTRY(symbols)	list;
-};
-
-static VTAILQ_HEAD(,symbols) symbols = VTAILQ_HEAD_INITIALIZER(symbols);
-
-int
-Symbol_Lookup(struct vsb *vsb, void *ptr)
-{
-	struct symbols *s, *s0;
-	uintptr_t pp;
-
-	pp = (uintptr_t)ptr;
-	s0 = NULL;
-	VTAILQ_FOREACH(s, &symbols, list) {
-		if (s->a > pp)
-			continue;
-		if (s0 == NULL || s->a > s0->a)
-			s0 = s;
-	}
-	if (s0 == NULL)
-		return (-1);
-	VSB_printf(vsb, "%p: %s+%jx", ptr, s0->n, (uintmax_t)pp - s0->a);
-	return (0);
-}
-
-static void
-Symbol_hack(const char *a0)
-{
-	char buf[BUFSIZ], *p, *e;
-	FILE *fi;
-	uintptr_t a;
-	struct symbols *s;
-
-	bprintf(buf, "nm -an %s 2>/dev/null", a0);
-	fi = popen(buf, "r");
-	if (fi == NULL)
-		return;
-	while (fgets(buf, sizeof buf, fi)) {
-		if (buf[0] == ' ')
-			continue;
-		p = NULL;
-		a = strtoul(buf, &p, 16);
-		if (p == NULL)
-			continue;
-		if (a == 0)
-			continue;
-		if (*p++ != ' ')
-			continue;
-		p++;
-		if (*p++ != ' ')
-			continue;
-		if (*p <= ' ')
-			continue;
-		e = strchr(p, '\0');
-		AN(e);
-		while (e > p && isspace(e[-1]))
-			e--;
-		*e = '\0';
-		s = malloc(sizeof *s + strlen(p) + 1);
-		AN(s);
-		s->a = a;
-		s->n = (void*)(s + 1);
-		strcpy(s->n, p);
-		VTAILQ_INSERT_TAIL(&symbols, s, list);
-	}
-	(void)pclose(fi);
-}
-
-/*--------------------------------------------------------------------
- * This function is called when the CLI on stdin is closed.
- */
-
-static void
-cli_stdin_close(void *priv)
-{
-
-	(void)priv;
-	(void)close(0);
-	(void)close(1);
-	(void)close(2);
-	assert(open("/dev/null", O_RDONLY) == 0);
-	assert(open("/dev/null", O_WRONLY) == 1);
-	assert(open("/dev/null", O_WRONLY) == 2);
-
-	if (d_flag) {
-		mgt_stop_child();
-		mgt_cli_close_all();
-		exit(0);
-	}
-}
-
-/*--------------------------------------------------------------------*/
-
-int
-main(int argc, char * const *argv)
-{
-	int o;
-	unsigned C_flag = 0;
-	unsigned F_flag = 0;
-	const char *b_arg = NULL;
-	const char *f_arg = NULL;
-	const char *i_arg = NULL;
-	const char *l_arg = NULL;	/* default in mgt_shmem.c */
-	const char *h_arg = "critbit";
-	const char *M_arg = NULL;
-	const char *n_arg = NULL;
-	const char *P_arg = NULL;
-	const char *S_arg = NULL;
-	const char *s_arg = "file";
-	int s_arg_given = 0;
-	const char *T_arg = NULL;
-	char *p, *vcl = NULL;
-	struct cli cli[1];
-	struct vpf_fh *pfh = NULL;
-	char *dirname;
-
-	/*
-	 * Start out by closing all unwanted file descriptors we might
-	 * have inherited from sloppy process control daemons.
-	 */
-	for (o = getdtablesize(); o > STDERR_FILENO; o--)
-		(void)close(o);
-
-	srandomdev();
-
-	mgt_got_fd(STDERR_FILENO);
-
-	setbuf(stdout, NULL);
-	setbuf(stderr, NULL);
-
-	build_vident();
-
-	Symbol_hack(argv[0]);
-
-	/* for ASSERT_MGT() */
-	mgt_pid = getpid();
-
-	/*
-	 * Run in UTC timezone, on the off-chance that this operating
-	 * system does not have a timegm() function, and translates
-	 * timestamps on the local timescale.
-	 * See lib/libvarnish/time.c
-	 */
-	AZ(setenv("TZ", "UTC", 1));
-	tzset();
-	assert(VTIM_parse("Sun, 06 Nov 1994 08:49:37 GMT") == 784111777);
-	assert(VTIM_parse("Sunday, 06-Nov-94 08:49:37 GMT") == 784111777);
-	assert(VTIM_parse("Sun Nov  6 08:49:37 1994") == 784111777);
-
-	/*
-	 * Check that our SHA256 works
-	 */
-	SHA256_Test();
-
-	memset(cli, 0, sizeof cli);
-	cli[0].sb = VSB_new_auto();
-	XXXAN(cli[0].sb);
-	cli[0].result = CLIS_OK;
-
-	VTAILQ_INIT(&heritage.socks);
-
-	MCF_ParamInit(cli);
-
-	if (sizeof(void *) < 8) {
-		/*
-		 * Adjust default parameters for 32 bit systems to conserve
-		 * VM space.
-		 */
-		MCF_ParamSet(cli, "sess_workspace", "16384");
-		cli_check(cli);
-
-		MCF_ParamSet(cli, "thread_pool_workspace", "16384");
-		cli_check(cli);
-
-		MCF_ParamSet(cli, "http_resp_size", "8192");
-		cli_check(cli);
-
-		MCF_ParamSet(cli, "http_req_size", "12288");
-		cli_check(cli);
-
-		MCF_ParamSet(cli, "thread_pool_stack", "32bit");
-		cli_check(cli);
-
-		MCF_ParamSet(cli, "gzip_stack_buffer", "4096");
-		cli_check(cli);
-	}
-
-	cli_check(cli);
-
-	while ((o = getopt(argc, argv,
-	    "a:b:Cdf:Fg:h:i:l:L:M:n:P:p:S:s:T:t:u:Vx:w:")) != -1)
-		switch (o) {
-		case 'a':
-			MCF_ParamSet(cli, "listen_address", optarg);
-			cli_check(cli);
-			break;
-		case 'b':
-			b_arg = optarg;
-			break;
-		case 'C':
-			C_flag = 1 - C_flag;
-			break;
-		case 'd':
-			d_flag++;
-			break;
-		case 'f':
-			f_arg = optarg;
-			break;
-		case 'F':
-			F_flag = 1 - F_flag;
-			break;
-		case 'g':
-			MCF_ParamSet(cli, "group", optarg);
-			break;
-		case 'h':
-			h_arg = optarg;
-			break;
-		case 'i':
-			i_arg = optarg;
-			break;
-		case 'l':
-			l_arg = optarg;
-			break;
-		case 'M':
-			M_arg = optarg;
-			break;
-		case 'n':
-			n_arg = optarg;
-			break;
-		case 'P':
-			P_arg = optarg;
-			break;
-		case 'p':
-			p = strchr(optarg, '=');
-			if (p == NULL)
-				usage();
-			AN(p);
-			*p++ = '\0';
-			MCF_ParamSet(cli, optarg, p);
-			cli_check(cli);
-			break;
-		case 's':
-			s_arg_given = 1;
-			STV_Config(optarg);
-			break;
-		case 't':
-			MCF_ParamSet(cli, "default_ttl", optarg);
-			break;
-		case 'S':
-			S_arg = optarg;
-			break;
-		case 'T':
-			T_arg = optarg;
-			break;
-		case 'u':
-			MCF_ParamSet(cli, "user", optarg);
-			break;
-		case 'V':
-			/* XXX: we should print the ident here */
-			VCS_Message("varnishd");
-			exit(0);
-		case 'x':
-			if (!strcmp(optarg, "dumprst")) {
-				MCF_DumpRst();
-				exit (0);
-			}
-			usage();
-			break;
-		case 'w':
-			tackle_warg(optarg);
-			break;
-		default:
-			usage();
-		}
-
-	argc -= optind;
-	argv += optind;
-
-	mgt_vcc_init();
-
-	if (argc != 0) {
-		fprintf(stderr, "Too many arguments (%s...)\n", argv[0]);
-		usage();
-	}
-
-	/* XXX: we can have multiple CLI actions above, is this enough ? */
-	if (cli[0].result != CLIS_OK) {
-		fprintf(stderr, "Parameter errors:\n");
-		AZ(VSB_finish(cli[0].sb));
-		fprintf(stderr, "%s\n", VSB_data(cli[0].sb));
-		exit(1);
-	}
-
-	if (d_flag && F_flag) {
-		fprintf(stderr, "Only one of -d or -F can be specified\n");
-		usage();
-	}
-
-	if (b_arg != NULL && f_arg != NULL) {
-		fprintf(stderr, "Only one of -b or -f can be specified\n");
-		usage();
-	}
-	if (S_arg == NULL && T_arg == NULL && d_flag == 0 && b_arg == NULL &&
-	    f_arg == NULL && M_arg == NULL) {
-		fprintf(stderr,
-		    "At least one of -d, -b, -f, -M, -S or -T "
-		    "must be specified\n");
-		usage();
-	}
-
-	if (f_arg != NULL) {
-		vcl = VFIL_readfile(NULL, f_arg, NULL);
-		if (vcl == NULL) {
-			fprintf(stderr, "Cannot read '%s': %s\n",
-			    f_arg, strerror(errno));
-			exit(1);
-		}
-	}
-
-	if (VIN_N_Arg(n_arg, &heritage.name, &dirname, NULL) != 0) {
-		fprintf(stderr, "Invalid instance name: %s\n",
-		    strerror(errno));
-		exit(1);
-	}
-
-	if (i_arg != NULL) {
-		if (snprintf(heritage.identity, sizeof heritage.identity,
-		    "%s", i_arg) > sizeof heritage.identity) {
-			fprintf(stderr, "Invalid identity name: %s\n",
-			    strerror(ENAMETOOLONG));
-			exit(1);
-		}
-	}
-
-	if (n_arg != NULL)
-		openlog(n_arg, LOG_PID, LOG_LOCAL0);
-	else
-		openlog("varnishd", LOG_PID, LOG_LOCAL0);
-
-	if (mkdir(dirname, 0755) < 0 && errno != EEXIST) {
-		fprintf(stderr, "Cannot create working directory '%s': %s\n",
-		    dirname, strerror(errno));
-		exit(1);
-	}
-
-	if (chdir(dirname) < 0) {
-		fprintf(stderr, "Cannot change to working directory '%s': %s\n",
-		    dirname, strerror(errno));
-		exit(1);
-	}
-
-	/* XXX: should this be relative to the -n arg ? */
-	if (P_arg && (pfh = VPF_Open(P_arg, 0644, NULL)) == NULL) {
-		perror(P_arg);
-		exit(1);
-	}
-
-	if (b_arg != NULL || f_arg != NULL)
-		if (mgt_vcc_default(b_arg, f_arg, vcl, C_flag))
-			exit (2);
-
-	if (C_flag)
-		exit (0);
-
-	/* If no -s argument specified, process default -s argument */
-	if (!s_arg_given)
-		STV_Config(s_arg);
-
-	/* Configure Transient storage, if user did not */
-	STV_Config_Transient();
-
-	HSH_config(h_arg);
-
-	mgt_SHM_Init(l_arg);
-
-	AZ(VSB_finish(vident));
-
-	if (!d_flag && !F_flag)
-		AZ(varnish_daemon(1, 0));
-
-	mgt_SHM_Pid();
-
-	if (pfh != NULL && VPF_Write(pfh))
-		fprintf(stderr, "NOTE: Could not write PID file\n");
-
-	if (d_flag)
-		fprintf(stderr, "Platform: %s\n", VSB_data(vident) + 1);
-	syslog(LOG_NOTICE, "Platform: %s\n", VSB_data(vident) + 1);
-
-	/* Do this again after debugstunt and daemon has run */
-	mgt_pid = getpid();
-
-	mgt_evb = vev_new_base();
-	XXXAN(mgt_evb);
-
-	if (d_flag)
-		mgt_cli_setup(0, 1, 1, "debug", cli_stdin_close, NULL);
-	if (S_arg != NULL)
-		mgt_cli_secret(S_arg);
-	if (M_arg != NULL)
-		mgt_cli_master(M_arg);
-	if (T_arg != NULL)
-		mgt_cli_telnet(T_arg);
-
-	AN(VSM_Alloc(0, VSM_CLASS_MARK, "", ""));
-
-	MGT_Run();
-
-	if (pfh != NULL)
-		(void)VPF_Remove(pfh);
-	exit(exit_status);
-}
-
-#if defined(PTHREAD_CANCELED) || defined(PTHREAD_MUTEX_DEFAULT)
-#error "Keep pthreads out of in manager process"
-#endif



From geoff at varnish-cache.org  Mon Jan  9 20:52:32 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:32 +0100
Subject: [experimental-ims] 94c79bd Rename rfc2616.c to cache_rfc2616.c where
	it belongs. Move vparam.h to mgt/mgt_param.h
Message-ID: <E1RkMCm-00089o-PW@project.varnish-software.com>

commit 94c79bd35de7b47178ac670d9a80d7004168cbc2
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sun Nov 13 10:21:45 2011 +0000

    Rename rfc2616.c to cache_rfc2616.c where it belongs.
    Move vparam.h to mgt/mgt_param.h

diff --git a/bin/varnishd/Makefile.am b/bin/varnishd/Makefile.am
index 94b52a5..0460303 100644
--- a/bin/varnishd/Makefile.am
+++ b/bin/varnishd/Makefile.am
@@ -36,6 +36,7 @@ varnishd_SOURCES = \
 	cache_pipe.c \
 	cache_pool.c \
 	cache_response.c \
+	cache_rfc2616.c \
 	cache_session.c \
 	cache_shmlog.c \
 	cache_vary.c \
@@ -65,7 +66,6 @@ varnishd_SOURCES = \
 	mgt/mgt_sandbox_solaris.c \
 	mgt/mgt_shmem.c \
 	mgt/mgt_vcc.c \
-	rfc2616.c \
 	storage/stevedore.c \
 	storage/stevedore_utils.c \
 	storage/storage_file.c \
@@ -89,10 +89,10 @@ noinst_HEADERS = \
 	heritage.h \
 	mgt/mgt.h \
 	mgt/mgt_cli.h \
+	mgt/mgt_param.h \
 	params.h \
 	storage/storage.h \
-	storage/storage_persistent.h \
-	vparam.h
+	storage/storage_persistent.h
 
 varnishd_CFLAGS = \
 	@PCRE_CFLAGS@ \
diff --git a/bin/varnishd/cache_rfc2616.c b/bin/varnishd/cache_rfc2616.c
new file mode 100644
index 0000000..4041f45
--- /dev/null
+++ b/bin/varnishd/cache_rfc2616.c
@@ -0,0 +1,336 @@
+/*-
+ * Copyright (c) 2006 Verdens Gang AS
+ * Copyright (c) 2006-2011 Varnish Software AS
+ * All rights reserved.
+ *
+ * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include "config.h"
+
+#include <math.h>
+#include <stdlib.h>
+
+#include "cache.h"
+
+#include "vtim.h"
+
+/*--------------------------------------------------------------------
+ * TTL and Age calculation in Varnish
+ *
+ * RFC2616 has a lot to say about how caches should calculate the TTL
+ * and expiry times of objects, but it sort of misses the case that
+ * applies to Varnish:  the server-side cache.
+ *
+ * A normal cache, shared or single-client, has no symbiotic relationship
+ * with the server, and therefore must take a very defensive attitude
+ * if the Data/Expiry/Age/max-age data does not make sense.  Overall
+ * the policy described in section 13 of RFC 2616 results in no caching
+ * happening on the first little sign of trouble.
+ *
+ * Varnish on the other hand tries to offload as many transactions from
+ * the backend as possible, and therefore just passing through everything
+ * if there is a clock-skew between backend and Varnish is not a workable
+ * choice.
+ *
+ * Varnish implements a policy which is RFC2616 compliant when there
+ * is no clockskew, and falls as gracefully as possible otherwise.
+ * Our "clockless cache" model is syntehsized from the bits of RFC2616
+ * that talks about how a cache should react to a clockless origin server,
+ * and more or less uses the inverse logic for the opposite relationship.
+ *
+ */
+
+void
+RFC2616_Ttl(const struct sess *sp)
+{
+	unsigned max_age, age;
+	double h_date, h_expires;
+	char *p;
+	const struct http *hp;
+
+	hp = sp->wrk->beresp;
+
+	assert(sp->wrk->exp.entered != 0.0 && !isnan(sp->wrk->exp.entered));
+	/* If all else fails, cache using default ttl */
+	sp->wrk->exp.ttl = cache_param->default_ttl;
+
+	max_age = age = 0;
+	h_expires = 0;
+	h_date = 0;
+
+	/*
+	 * Initial cacheability determination per [RFC2616, 13.4]
+	 * We do not support ranges yet, so 206 is out.
+	 */
+
+	if (http_GetHdr(hp, H_Age, &p)) {
+		age = strtoul(p, NULL, 0);
+		sp->wrk->exp.age = age;
+	}
+	if (http_GetHdr(hp, H_Expires, &p))
+		h_expires = VTIM_parse(p);
+
+	if (http_GetHdr(hp, H_Date, &p))
+		h_date = VTIM_parse(p);
+
+	switch (sp->err_code) {
+	default:
+		sp->wrk->exp.ttl = -1.;
+		break;
+	case 200: /* OK */
+	case 203: /* Non-Authoritative Information */
+	case 300: /* Multiple Choices */
+	case 301: /* Moved Permanently */
+	case 302: /* Moved Temporarily */
+	case 307: /* Temporary Redirect */
+	case 410: /* Gone */
+	case 404: /* Not Found */
+		/*
+		 * First find any relative specification from the backend
+		 * These take precedence according to RFC2616, 13.2.4
+		 */
+
+		if ((http_GetHdrField(hp, H_Cache_Control, "s-maxage", &p) ||
+		    http_GetHdrField(hp, H_Cache_Control, "max-age", &p)) &&
+		    p != NULL) {
+
+			if (*p == '-')
+				max_age = 0;
+			else
+				max_age = strtoul(p, NULL, 0);
+
+			if (age > max_age)
+				sp->wrk->exp.ttl = 0;
+			else
+				sp->wrk->exp.ttl = max_age - age;
+			break;
+		}
+
+		/* No expire header, fall back to default */
+		if (h_expires == 0)
+			break;
+
+
+		/* If backend told us it is expired already, don't cache. */
+		if (h_expires < h_date) {
+			sp->wrk->exp.ttl = 0;
+			break;
+		}
+
+		if (h_date == 0 ||
+		    fabs(h_date - sp->wrk->exp.entered) < cache_param->clock_skew) {
+			/*
+			 * If we have no Date: header or if it is
+			 * sufficiently close to our clock we will
+			 * trust Expires: relative to our own clock.
+			 */
+			if (h_expires < sp->wrk->exp.entered)
+				sp->wrk->exp.ttl = 0;
+			else
+				sp->wrk->exp.ttl = h_expires -
+				    sp->wrk->exp.entered;
+			break;
+		} else {
+			/*
+			 * But even if the clocks are out of whack we can still
+			 * derive a relative time from the two headers.
+			 * (the negative ttl case is caught above)
+			 */
+			sp->wrk->exp.ttl = (int)(h_expires - h_date);
+		}
+
+	}
+
+	/* calculated TTL, Our time, Date, Expires, max-age, age */
+	WSP(sp, SLT_TTL,
+	    "%u RFC %.0f %.0f %.0f %.0f %.0f %.0f %.0f %u",
+	    sp->xid, sp->wrk->exp.ttl, -1., -1., sp->wrk->exp.entered,
+	    sp->wrk->exp.age, h_date, h_expires, max_age);
+}
+
+/*--------------------------------------------------------------------
+ * Body existence, fetch method and close policy.
+ */
+
+enum body_status
+RFC2616_Body(const struct sess *sp)
+{
+	struct http *hp;
+	char *b;
+
+	hp = sp->wrk->beresp;
+
+	if (hp->protover < 11 && !http_HdrIs(hp, H_Connection, "keep-alive"))
+		sp->wrk->do_close = 1;
+	else if (http_HdrIs(hp, H_Connection, "close"))
+		sp->wrk->do_close = 1;
+	else
+		sp->wrk->do_close = 0;
+
+	if (!strcasecmp(http_GetReq(sp->wrk->bereq), "head")) {
+		/*
+		 * A HEAD request can never have a body in the reply,
+		 * no matter what the headers might say.
+		 * [RFC2516 4.3 p33]
+		 */
+		sp->wrk->stats.fetch_head++;
+		return (BS_NONE);
+	}
+
+	if (hp->status <= 199) {
+		/*
+		 * 1xx responses never have a body.
+		 * [RFC2616 4.3 p33]
+		 */
+		sp->wrk->stats.fetch_1xx++;
+		return (BS_NONE);
+	}
+
+	if (hp->status == 204) {
+		/*
+		 * 204 is "No Content", obviously don't expect a body.
+		 * [RFC2616 10.2.5 p60]
+		 */
+		sp->wrk->stats.fetch_204++;
+		return (BS_NONE);
+	}
+
+	if (hp->status == 304) {
+		/*
+		 * 304 is "Not Modified" it has no body.
+		 * [RFC2616 10.3.5 p63]
+		 */
+		sp->wrk->stats.fetch_304++;
+		return (BS_NONE);
+	}
+
+	if (http_HdrIs(hp, H_Transfer_Encoding, "chunked")) {
+		 sp->wrk->stats.fetch_chunked++;
+		return (BS_CHUNKED);
+	}
+
+	if (http_GetHdr(hp, H_Transfer_Encoding, &b)) {
+		sp->wrk->stats.fetch_bad++;
+		return (BS_ERROR);
+	}
+
+	if (http_GetHdr(hp, H_Content_Length, &sp->wrk->h_content_length)) {
+		sp->wrk->stats.fetch_length++;
+		return (BS_LENGTH);
+	}
+
+	if (http_HdrIs(hp, H_Connection, "keep-alive")) {
+		/*
+		 * Keep alive with neither TE=Chunked or C-Len is impossible.
+		 * We assume a zero length body.
+		 */
+		sp->wrk->stats.fetch_zero++;
+		return (BS_ZERO);
+	}
+
+	if (http_HdrIs(hp, H_Connection, "close")) {
+		/*
+		 * In this case, it is safe to just read what comes.
+		 */
+		sp->wrk->stats.fetch_close++;
+		return (BS_EOF);
+	}
+
+	if (hp->protover < 11) {
+		/*
+		 * With no Connection header, assume EOF.
+		 */
+		sp->wrk->stats.fetch_oldhttp++;
+		return (BS_EOF);
+	}
+
+	/*
+	 * Fall back to EOF transfer.
+	 */
+	sp->wrk->stats.fetch_eof++;
+	return (BS_EOF);
+}
+
+/*--------------------------------------------------------------------
+ * Find out if the request can receive a gzip'ed response
+ */
+
+unsigned
+RFC2616_Req_Gzip(const struct sess *sp)
+{
+
+
+	/*
+	 * "x-gzip" is for http/1.0 backwards compat, final note in 14.3
+	 * p104 says to not do q values for x-gzip, so we just test
+	 * for its existence.
+	 */
+	if (http_GetHdrData(sp->http, H_Accept_Encoding, "x-gzip", NULL))
+		return (1);
+
+	/*
+	 * "gzip" is the real thing, but the 'q' value must be nonzero.
+	 * We do not care a hoot if the client prefers some other
+	 * compression more than gzip: Varnish only does gzip.
+	 */
+	if (http_GetHdrQ(sp->http, H_Accept_Encoding, "gzip") > 0.)
+		return (1);
+
+	/* Bad client, no gzip. */
+	return (0);
+}
+
+/*--------------------------------------------------------------------*/
+
+int
+RFC2616_Do_Cond(const struct sess *sp)
+{
+	char *p, *e;
+	double ims;
+	int do_cond = 0;
+
+	/* RFC 2616 13.3.4 states we need to match both ETag
+	   and If-Modified-Since if present*/
+
+	if (http_GetHdr(sp->http, H_If_Modified_Since, &p) ) {
+		if (!sp->obj->last_modified)
+			return (0);
+		ims = VTIM_parse(p);
+		if (ims > sp->t_req)	/* [RFC2616 14.25] */
+			return (0);
+		if (sp->obj->last_modified > ims)
+			return (0);
+		do_cond = 1;
+	}
+
+	if (http_GetHdr(sp->http, H_If_None_Match, &p) &&
+	    http_GetHdr(sp->obj->http, H_ETag, &e)) {
+		if (strcmp(p,e) != 0)
+			return (0);
+		do_cond = 1;
+	}
+
+	return (do_cond);
+}
diff --git a/bin/varnishd/mgt/mgt_param.c b/bin/varnishd/mgt/mgt_param.c
index cd96e8b..8fff352 100644
--- a/bin/varnishd/mgt/mgt_param.c
+++ b/bin/varnishd/mgt/mgt_param.c
@@ -41,12 +41,12 @@
 #include "heritage.h"
 #include "params.h"
 
+#include "mgt/mgt_param.h"
 #include "waiter/cache_waiter.h"
 #include "vav.h"
 #include "vcli.h"
 #include "vcli_common.h"
 #include "vcli_priv.h"
-#include "vparam.h"
 #include "vss.h"
 
 #include "mgt_cli.h"
diff --git a/bin/varnishd/mgt/mgt_param.h b/bin/varnishd/mgt/mgt_param.h
new file mode 100644
index 0000000..2d4a97f
--- /dev/null
+++ b/bin/varnishd/mgt/mgt_param.h
@@ -0,0 +1,59 @@
+/*-
+ * Copyright (c) 2006 Verdens Gang AS
+ * Copyright (c) 2006-2011 Varnish Software AS
+ * All rights reserved.
+ *
+ * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ */
+
+struct parspec;
+
+typedef void tweak_t(struct cli *, const struct parspec *, const char *arg);
+
+struct parspec {
+	const char	*name;
+	tweak_t		*func;
+	volatile void	*priv;
+	double		min;
+	double		max;
+	const char	*descr;
+	int		 flags;
+#define DELAYED_EFFECT	(1<<0)
+#define EXPERIMENTAL	(1<<1)
+#define MUST_RESTART	(1<<2)
+#define MUST_RELOAD	(1<<3)
+#define WIZARD		(1<<4)
+	const char	*def;
+	const char	*units;
+};
+
+void tweak_generic_uint(struct cli *cli,
+    volatile unsigned *dest, const char *arg, unsigned min, unsigned max);
+void tweak_uint(struct cli *cli, const struct parspec *par, const char *arg);
+void tweak_timeout(struct cli *cli,
+    const struct parspec *par, const char *arg);
+
+/* mgt_pool.c */
+extern const struct parspec WRK_parspec[];
diff --git a/bin/varnishd/mgt/mgt_pool.c b/bin/varnishd/mgt/mgt_pool.c
index 548ad78..6f61553 100644
--- a/bin/varnishd/mgt/mgt_pool.c
+++ b/bin/varnishd/mgt/mgt_pool.c
@@ -51,7 +51,7 @@
 #include "heritage.h"
 #include "params.h"
 
-#include "vparam.h"
+#include "mgt/mgt_param.h"
 
 /*--------------------------------------------------------------------*/
 
diff --git a/bin/varnishd/rfc2616.c b/bin/varnishd/rfc2616.c
deleted file mode 100644
index 4041f45..0000000
--- a/bin/varnishd/rfc2616.c
+++ /dev/null
@@ -1,336 +0,0 @@
-/*-
- * Copyright (c) 2006 Verdens Gang AS
- * Copyright (c) 2006-2011 Varnish Software AS
- * All rights reserved.
- *
- * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- */
-
-#include "config.h"
-
-#include <math.h>
-#include <stdlib.h>
-
-#include "cache.h"
-
-#include "vtim.h"
-
-/*--------------------------------------------------------------------
- * TTL and Age calculation in Varnish
- *
- * RFC2616 has a lot to say about how caches should calculate the TTL
- * and expiry times of objects, but it sort of misses the case that
- * applies to Varnish:  the server-side cache.
- *
- * A normal cache, shared or single-client, has no symbiotic relationship
- * with the server, and therefore must take a very defensive attitude
- * if the Data/Expiry/Age/max-age data does not make sense.  Overall
- * the policy described in section 13 of RFC 2616 results in no caching
- * happening on the first little sign of trouble.
- *
- * Varnish on the other hand tries to offload as many transactions from
- * the backend as possible, and therefore just passing through everything
- * if there is a clock-skew between backend and Varnish is not a workable
- * choice.
- *
- * Varnish implements a policy which is RFC2616 compliant when there
- * is no clockskew, and falls as gracefully as possible otherwise.
- * Our "clockless cache" model is syntehsized from the bits of RFC2616
- * that talks about how a cache should react to a clockless origin server,
- * and more or less uses the inverse logic for the opposite relationship.
- *
- */
-
-void
-RFC2616_Ttl(const struct sess *sp)
-{
-	unsigned max_age, age;
-	double h_date, h_expires;
-	char *p;
-	const struct http *hp;
-
-	hp = sp->wrk->beresp;
-
-	assert(sp->wrk->exp.entered != 0.0 && !isnan(sp->wrk->exp.entered));
-	/* If all else fails, cache using default ttl */
-	sp->wrk->exp.ttl = cache_param->default_ttl;
-
-	max_age = age = 0;
-	h_expires = 0;
-	h_date = 0;
-
-	/*
-	 * Initial cacheability determination per [RFC2616, 13.4]
-	 * We do not support ranges yet, so 206 is out.
-	 */
-
-	if (http_GetHdr(hp, H_Age, &p)) {
-		age = strtoul(p, NULL, 0);
-		sp->wrk->exp.age = age;
-	}
-	if (http_GetHdr(hp, H_Expires, &p))
-		h_expires = VTIM_parse(p);
-
-	if (http_GetHdr(hp, H_Date, &p))
-		h_date = VTIM_parse(p);
-
-	switch (sp->err_code) {
-	default:
-		sp->wrk->exp.ttl = -1.;
-		break;
-	case 200: /* OK */
-	case 203: /* Non-Authoritative Information */
-	case 300: /* Multiple Choices */
-	case 301: /* Moved Permanently */
-	case 302: /* Moved Temporarily */
-	case 307: /* Temporary Redirect */
-	case 410: /* Gone */
-	case 404: /* Not Found */
-		/*
-		 * First find any relative specification from the backend
-		 * These take precedence according to RFC2616, 13.2.4
-		 */
-
-		if ((http_GetHdrField(hp, H_Cache_Control, "s-maxage", &p) ||
-		    http_GetHdrField(hp, H_Cache_Control, "max-age", &p)) &&
-		    p != NULL) {
-
-			if (*p == '-')
-				max_age = 0;
-			else
-				max_age = strtoul(p, NULL, 0);
-
-			if (age > max_age)
-				sp->wrk->exp.ttl = 0;
-			else
-				sp->wrk->exp.ttl = max_age - age;
-			break;
-		}
-
-		/* No expire header, fall back to default */
-		if (h_expires == 0)
-			break;
-
-
-		/* If backend told us it is expired already, don't cache. */
-		if (h_expires < h_date) {
-			sp->wrk->exp.ttl = 0;
-			break;
-		}
-
-		if (h_date == 0 ||
-		    fabs(h_date - sp->wrk->exp.entered) < cache_param->clock_skew) {
-			/*
-			 * If we have no Date: header or if it is
-			 * sufficiently close to our clock we will
-			 * trust Expires: relative to our own clock.
-			 */
-			if (h_expires < sp->wrk->exp.entered)
-				sp->wrk->exp.ttl = 0;
-			else
-				sp->wrk->exp.ttl = h_expires -
-				    sp->wrk->exp.entered;
-			break;
-		} else {
-			/*
-			 * But even if the clocks are out of whack we can still
-			 * derive a relative time from the two headers.
-			 * (the negative ttl case is caught above)
-			 */
-			sp->wrk->exp.ttl = (int)(h_expires - h_date);
-		}
-
-	}
-
-	/* calculated TTL, Our time, Date, Expires, max-age, age */
-	WSP(sp, SLT_TTL,
-	    "%u RFC %.0f %.0f %.0f %.0f %.0f %.0f %.0f %u",
-	    sp->xid, sp->wrk->exp.ttl, -1., -1., sp->wrk->exp.entered,
-	    sp->wrk->exp.age, h_date, h_expires, max_age);
-}
-
-/*--------------------------------------------------------------------
- * Body existence, fetch method and close policy.
- */
-
-enum body_status
-RFC2616_Body(const struct sess *sp)
-{
-	struct http *hp;
-	char *b;
-
-	hp = sp->wrk->beresp;
-
-	if (hp->protover < 11 && !http_HdrIs(hp, H_Connection, "keep-alive"))
-		sp->wrk->do_close = 1;
-	else if (http_HdrIs(hp, H_Connection, "close"))
-		sp->wrk->do_close = 1;
-	else
-		sp->wrk->do_close = 0;
-
-	if (!strcasecmp(http_GetReq(sp->wrk->bereq), "head")) {
-		/*
-		 * A HEAD request can never have a body in the reply,
-		 * no matter what the headers might say.
-		 * [RFC2516 4.3 p33]
-		 */
-		sp->wrk->stats.fetch_head++;
-		return (BS_NONE);
-	}
-
-	if (hp->status <= 199) {
-		/*
-		 * 1xx responses never have a body.
-		 * [RFC2616 4.3 p33]
-		 */
-		sp->wrk->stats.fetch_1xx++;
-		return (BS_NONE);
-	}
-
-	if (hp->status == 204) {
-		/*
-		 * 204 is "No Content", obviously don't expect a body.
-		 * [RFC2616 10.2.5 p60]
-		 */
-		sp->wrk->stats.fetch_204++;
-		return (BS_NONE);
-	}
-
-	if (hp->status == 304) {
-		/*
-		 * 304 is "Not Modified" it has no body.
-		 * [RFC2616 10.3.5 p63]
-		 */
-		sp->wrk->stats.fetch_304++;
-		return (BS_NONE);
-	}
-
-	if (http_HdrIs(hp, H_Transfer_Encoding, "chunked")) {
-		 sp->wrk->stats.fetch_chunked++;
-		return (BS_CHUNKED);
-	}
-
-	if (http_GetHdr(hp, H_Transfer_Encoding, &b)) {
-		sp->wrk->stats.fetch_bad++;
-		return (BS_ERROR);
-	}
-
-	if (http_GetHdr(hp, H_Content_Length, &sp->wrk->h_content_length)) {
-		sp->wrk->stats.fetch_length++;
-		return (BS_LENGTH);
-	}
-
-	if (http_HdrIs(hp, H_Connection, "keep-alive")) {
-		/*
-		 * Keep alive with neither TE=Chunked or C-Len is impossible.
-		 * We assume a zero length body.
-		 */
-		sp->wrk->stats.fetch_zero++;
-		return (BS_ZERO);
-	}
-
-	if (http_HdrIs(hp, H_Connection, "close")) {
-		/*
-		 * In this case, it is safe to just read what comes.
-		 */
-		sp->wrk->stats.fetch_close++;
-		return (BS_EOF);
-	}
-
-	if (hp->protover < 11) {
-		/*
-		 * With no Connection header, assume EOF.
-		 */
-		sp->wrk->stats.fetch_oldhttp++;
-		return (BS_EOF);
-	}
-
-	/*
-	 * Fall back to EOF transfer.
-	 */
-	sp->wrk->stats.fetch_eof++;
-	return (BS_EOF);
-}
-
-/*--------------------------------------------------------------------
- * Find out if the request can receive a gzip'ed response
- */
-
-unsigned
-RFC2616_Req_Gzip(const struct sess *sp)
-{
-
-
-	/*
-	 * "x-gzip" is for http/1.0 backwards compat, final note in 14.3
-	 * p104 says to not do q values for x-gzip, so we just test
-	 * for its existence.
-	 */
-	if (http_GetHdrData(sp->http, H_Accept_Encoding, "x-gzip", NULL))
-		return (1);
-
-	/*
-	 * "gzip" is the real thing, but the 'q' value must be nonzero.
-	 * We do not care a hoot if the client prefers some other
-	 * compression more than gzip: Varnish only does gzip.
-	 */
-	if (http_GetHdrQ(sp->http, H_Accept_Encoding, "gzip") > 0.)
-		return (1);
-
-	/* Bad client, no gzip. */
-	return (0);
-}
-
-/*--------------------------------------------------------------------*/
-
-int
-RFC2616_Do_Cond(const struct sess *sp)
-{
-	char *p, *e;
-	double ims;
-	int do_cond = 0;
-
-	/* RFC 2616 13.3.4 states we need to match both ETag
-	   and If-Modified-Since if present*/
-
-	if (http_GetHdr(sp->http, H_If_Modified_Since, &p) ) {
-		if (!sp->obj->last_modified)
-			return (0);
-		ims = VTIM_parse(p);
-		if (ims > sp->t_req)	/* [RFC2616 14.25] */
-			return (0);
-		if (sp->obj->last_modified > ims)
-			return (0);
-		do_cond = 1;
-	}
-
-	if (http_GetHdr(sp->http, H_If_None_Match, &p) &&
-	    http_GetHdr(sp->obj->http, H_ETag, &e)) {
-		if (strcmp(p,e) != 0)
-			return (0);
-		do_cond = 1;
-	}
-
-	return (do_cond);
-}
diff --git a/bin/varnishd/vparam.h b/bin/varnishd/vparam.h
deleted file mode 100644
index 2d4a97f..0000000
--- a/bin/varnishd/vparam.h
+++ /dev/null
@@ -1,59 +0,0 @@
-/*-
- * Copyright (c) 2006 Verdens Gang AS
- * Copyright (c) 2006-2011 Varnish Software AS
- * All rights reserved.
- *
- * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- *
- */
-
-struct parspec;
-
-typedef void tweak_t(struct cli *, const struct parspec *, const char *arg);
-
-struct parspec {
-	const char	*name;
-	tweak_t		*func;
-	volatile void	*priv;
-	double		min;
-	double		max;
-	const char	*descr;
-	int		 flags;
-#define DELAYED_EFFECT	(1<<0)
-#define EXPERIMENTAL	(1<<1)
-#define MUST_RESTART	(1<<2)
-#define MUST_RELOAD	(1<<3)
-#define WIZARD		(1<<4)
-	const char	*def;
-	const char	*units;
-};
-
-void tweak_generic_uint(struct cli *cli,
-    volatile unsigned *dest, const char *arg, unsigned min, unsigned max);
-void tweak_uint(struct cli *cli, const struct parspec *par, const char *arg);
-void tweak_timeout(struct cli *cli,
-    const struct parspec *par, const char *arg);
-
-/* mgt_pool.c */
-extern const struct parspec WRK_parspec[];



From geoff at varnish-cache.org  Mon Jan  9 20:52:29 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:29 +0100
Subject: [experimental-ims] b32cfd0 Add some backend.list commands to exercise
	that code also
Message-ID: <E1RkMCj-00083G-G3@project.varnish-software.com>

commit b32cfd034291dd46157eb5f65146a00144485336
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Nov 7 11:44:06 2011 +0000

    Add some backend.list commands to exercise that code also

diff --git a/bin/varnishtest/tests/c00048.vtc b/bin/varnishtest/tests/c00048.vtc
index a4d0847..bf1c3c9 100644
--- a/bin/varnishtest/tests/c00048.vtc
+++ b/bin/varnishtest/tests/c00048.vtc
@@ -25,7 +25,9 @@ varnish v1 -vcl {
 
 delay 1
 
+varnish v1 -cliok "backend.list"
 varnish v1 -cliok "backend.set_health s1 auto"
+varnish v1 -cliok "backend.list s"
 
 client c1 {
 	txreq
@@ -33,7 +35,9 @@ client c1 {
 	expect resp.status == 200
 } -run
 
+varnish v1 -cliok "backend.list (:)"
 varnish v1 -cliok "backend.set_health s1 sick"
+varnish v1 -cliok "backend.list (1:)"
 
 client c1 {
 	txreq
@@ -41,7 +45,9 @@ client c1 {
 	expect resp.status == 503
 } -run
 
+varnish v1 -cliok "backend.list"
 varnish v1 -cliok "backend.set_health s1 healthy"
+varnish v1 -cliok "backend.list"
 
 client c1 {
 	txreq
@@ -52,4 +58,5 @@ client c1 {
 varnish v1 -clierr 106 "backend.set_health s1 foo"
 varnish v1 -clierr 106 "backend.set_health s2 foo"
 varnish v1 -clierr 106 "backend.set_health s2 auto"
+varnish v1 -cliok "backend.list (foo)"
 



From geoff at varnish-cache.org  Mon Jan  9 20:52:29 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:29 +0100
Subject: [experimental-ims] 6afe314 One more backend.list test to get all of
	the code.
Message-ID: <E1RkMCj-00084U-T1@project.varnish-software.com>

commit 6afe3140441f0b81723f2c78bf2cb26080df508b
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Nov 7 12:51:14 2011 +0000

    One more backend.list test to get all of the code.

diff --git a/bin/varnishtest/tests/c00048.vtc b/bin/varnishtest/tests/c00048.vtc
index 4ddb888..2502275 100644
--- a/bin/varnishtest/tests/c00048.vtc
+++ b/bin/varnishtest/tests/c00048.vtc
@@ -61,4 +61,5 @@ varnish v1 -clierr 106 "backend.set_health s2 auto"
 varnish v1 -cliok "backend.list (foo)"
 varnish v1 -clierr 300 "backend.list ("
 varnish v1 -clierr 300 {backend.list " ( : ) -"}
+varnish v1 -cliok {backend.list "a ( b : c )"}
 



From geoff at varnish-cache.org  Mon Jan  9 20:52:39 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:39 +0100
Subject: [experimental-ims] 5cfe903 Combine two small functions
Message-ID: <E1RkMCt-0008C1-8V@project.varnish-software.com>

commit 5cfe903c15a76c3b98fb1c42e8c862a5553286c2
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Nov 15 19:52:36 2011 +0000

    Combine two small functions

diff --git a/bin/varnishd/mgt/mgt_child.c b/bin/varnishd/mgt/mgt_child.c
index a2c0b58..7e60c1a 100644
--- a/bin/varnishd/mgt/mgt_child.c
+++ b/bin/varnishd/mgt/mgt_child.c
@@ -424,21 +424,14 @@ mgt_stop_child(void)
 /*--------------------------------------------------------------------*/
 
 static void
-mgt_report_panic(pid_t r)
+mgt_handle_panicstr(pid_t r)
 {
+	char time_str[30];
 
 	if (VSM_head->panicstr[0] == '\0')
 		return;
 	REPORT(LOG_ERR, "Child (%jd) Panic message: %s",
 	    (intmax_t)r, VSM_head->panicstr);
-}
-
-static void
-mgt_save_panic(void)
-{
-	char time_str[30];
-	if (VSM_head->panicstr[0] == '\0')
-		return;
 
 	if (child_panic)
 		VSB_delete(child_panic);
@@ -500,8 +493,7 @@ mgt_sigchld(const struct vev *e, int what)
 	REPORT(LOG_INFO, "%s", VSB_data(vsb));
 	VSB_delete(vsb);
 
-	mgt_report_panic(r);
-	mgt_save_panic();
+	mgt_handle_panicstr(r);
 
 	child_pid = -1;
 



From geoff at varnish-cache.org  Mon Jan  9 20:52:39 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:39 +0100
Subject: [experimental-ims] 54e7eb5 Move prototypes to smaller scope
Message-ID: <E1RkMCt-0008J4-DE@project.varnish-software.com>

commit 54e7eb52c299a09f2adada5c9fc9d4173a5886ff
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Nov 16 05:59:49 2011 +0000

    Move prototypes to smaller scope

diff --git a/bin/varnishd/common/common.h b/bin/varnishd/common/common.h
index ff06c64..f6d01db 100644
--- a/bin/varnishd/common/common.h
+++ b/bin/varnishd/common/common.h
@@ -73,9 +73,6 @@ void mgt_child_inherit(int fd, const char *what);
 
 #define NEEDLESS_RETURN(foo)	return (foo)
 
-/* stevedore.c */
-void STV_Config(const char *spec);
-void STV_Config_Transient(void);
 
 /* vsm.c */
 // extern struct VSM_head		*VSM_head;
diff --git a/bin/varnishd/mgt/mgt.h b/bin/varnishd/mgt/mgt.h
index 4aec482..1eabe10 100644
--- a/bin/varnishd/mgt/mgt.h
+++ b/bin/varnishd/mgt/mgt.h
@@ -84,6 +84,10 @@ void mgt_sandbox_solaris_privsep(void);
 void mgt_SHM_Init(const char *arg);
 void mgt_SHM_Pid(void);
 
+/* stevedore_mgt.c */
+void STV_Config(const char *spec);
+void STV_Config_Transient(void);
+
 /* mgt_vcc.c */
 void mgt_vcc_init(void);
 int mgt_vcc_default(const char *bflag, const char *f_arg, char *vcl, int Cflag);



From geoff at varnish-cache.org  Mon Jan  9 20:52:39 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:39 +0100
Subject: [experimental-ims] ceaf73d Make the panicstr a VSM allocation
Message-ID: <E1RkMCt-0008JE-FD@project.varnish-software.com>

commit ceaf73d70ae92b81440615709789cf6101f18a51
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Nov 16 06:22:49 2011 +0000

    Make the panicstr a VSM allocation

diff --git a/bin/varnishd/cache/cache_panic.c b/bin/varnishd/cache/cache_panic.c
index c626ae3..d112eb8 100644
--- a/bin/varnishd/cache/cache_panic.c
+++ b/bin/varnishd/cache/cache_panic.c
@@ -40,8 +40,6 @@
 
 #include "cache.h"
 
-#include "vapi/vsm_int.h"
-
 #include "cache_backend.h"
 #include "waiter/cache_waiter.h"
 #include "vcl.h"
@@ -58,6 +56,10 @@
 static struct vsb vsps, *vsp;
 static pthread_mutex_t panicstr_mtx = PTHREAD_MUTEX_INITIALIZER;
 
+/* Initialized in mgt_shmem.c, points into VSM */
+char *PAN_panicstr;
+unsigned PAN_panicstr_len;
+
 /*--------------------------------------------------------------------*/
 
 static void
@@ -354,20 +356,8 @@ pan_ic(const char *func, const char *file, int line, const char *cond,
 	VSB_bcat(vsp, "", 1);	/* NUL termination */
 
 	if (cache_param->diag_bitmap & 0x4000)
-		(void)fputs(VSM_head->panicstr, stderr);
-
-#ifdef HAVE_ABORT2
-	if (cache_param->diag_bitmap & 0x8000) {
-		void *arg[1];
-		char *p;
-
-		for (p = VSM_head->panicstr; *p; p++)
-			if (*p == '\n')
-				*p = ' ';
-		arg[0] = VSM_head->panicstr;
-		abort2(VSM_head->panicstr, 1, arg);
-	}
-#endif
+		(void)fputs(PAN_panicstr, stderr);
+
 	if (cache_param->diag_bitmap & 0x1000)
 		exit(4);
 	else
@@ -382,6 +372,8 @@ PAN_Init(void)
 
 	VAS_Fail = pan_ic;
 	vsp = &vsps;
-	AN(VSB_new(vsp, VSM_head->panicstr, sizeof VSM_head->panicstr,
+	AN(PAN_panicstr);
+	AN(PAN_panicstr_len);
+	AN(VSB_new(vsp, PAN_panicstr, PAN_panicstr_len,
 	    VSB_FIXEDLEN));
 }
diff --git a/bin/varnishd/cache/cache_shmlog.c b/bin/varnishd/cache/cache_shmlog.c
index 1252fa3..e004896 100644
--- a/bin/varnishd/cache/cache_shmlog.c
+++ b/bin/varnishd/cache/cache_shmlog.c
@@ -317,7 +317,6 @@ VSL_Init(void)
 
 	vsl_wrap();
 	VSM_head->starttime = (intmax_t)VTIM_real();
-	memset(VSM_head->panicstr, '\0', sizeof *VSM_head->panicstr);
 	memset(VSC_C_main, 0, sizeof *VSC_C_main);
 	VSM_head->child_pid = getpid();
 }
diff --git a/bin/varnishd/common/common.h b/bin/varnishd/common/common.h
index f6d01db..f59cfc9 100644
--- a/bin/varnishd/common/common.h
+++ b/bin/varnishd/common/common.h
@@ -50,6 +50,9 @@ extern pid_t mgt_pid;
 
 /* mgt_shmem.c */
 extern struct VSC_C_main *VSC_C_main;
+#define PAN_CLASS "Panic"
+extern char *PAN_panicstr;
+extern unsigned PAN_panicstr_len;
 
 /* varnishd.c */
 struct vsb;
diff --git a/bin/varnishd/mgt/mgt_child.c b/bin/varnishd/mgt/mgt_child.c
index 7e60c1a..b619b66 100644
--- a/bin/varnishd/mgt/mgt_child.c
+++ b/bin/varnishd/mgt/mgt_child.c
@@ -47,7 +47,6 @@
 #include "common/heritage.h"
 #include "common/params.h"
 
-#include "vapi/vsm_int.h"
 #include "vbm.h"
 #include "vcli.h"
 #include "vcli_priv.h"
@@ -428,10 +427,10 @@ mgt_handle_panicstr(pid_t r)
 {
 	char time_str[30];
 
-	if (VSM_head->panicstr[0] == '\0')
+	if (PAN_panicstr[0] == '\0')
 		return;
 	REPORT(LOG_ERR, "Child (%jd) Panic message: %s",
-	    (intmax_t)r, VSM_head->panicstr);
+	    (intmax_t)r, PAN_panicstr);
 
 	if (child_panic)
 		VSB_delete(child_panic);
@@ -439,7 +438,7 @@ mgt_handle_panicstr(pid_t r)
 	XXXAN(child_panic);
 	VTIM_format(VTIM_real(), time_str);
 	VSB_printf(child_panic, "Last panic at: %s\n", time_str);
-	VSB_cat(child_panic, VSM_head->panicstr);
+	VSB_cat(child_panic, PAN_panicstr);
 	AZ(VSB_finish(child_panic));
 }
 
diff --git a/bin/varnishd/mgt/mgt_param.c b/bin/varnishd/mgt/mgt_param.c
index afaab84..b7c18b2 100644
--- a/bin/varnishd/mgt/mgt_param.c
+++ b/bin/varnishd/mgt/mgt_param.c
@@ -843,9 +843,6 @@ static const struct parspec input_parspec[] = {
 		"  0x00001000 - do not core-dump child process.\n"
 		"  0x00002000 - only short panic message.\n"
 		"  0x00004000 - panic to stderr.\n"
-#ifdef HAVE_ABORT2
-		"  0x00008000 - panic to abort2().\n"
-#endif
 		"  0x00010000 - synchronize shmlog.\n"
 		"  0x00020000 - synchronous start of persistence.\n"
 		"  0x00040000 - release VCL early.\n"
diff --git a/bin/varnishd/mgt/mgt_shmem.c b/bin/varnishd/mgt/mgt_shmem.c
index d4346df..24bc396 100644
--- a/bin/varnishd/mgt/mgt_shmem.c
+++ b/bin/varnishd/mgt/mgt_shmem.c
@@ -331,6 +331,12 @@ mgt_SHM_Init(const char *l_arg)
 	vsl_log_start[1] = VSL_ENDMARKER;
 	VWMB();
 
+	PAN_panicstr_len = 64 * 1024;
+	PAN_panicstr = VSM_Alloc(PAN_panicstr_len, PAN_CLASS, "", "");
+	AN(PAN_panicstr);
+	/* XXX: shouldn't VSM_Alloc zero ? */
+	memset(PAN_panicstr, '\0', PAN_panicstr_len);
+
 	do
 		*vsl_log_start = random() & 0xffff;
 	while (*vsl_log_start == 0);
diff --git a/configure.ac b/configure.ac
index bbe44b5..8404de4 100644
--- a/configure.ac
+++ b/configure.ac
@@ -187,7 +187,6 @@ AC_CHECK_FUNCS([socket])
 AC_CHECK_FUNCS([strptime])
 AC_CHECK_FUNCS([fmtcheck])
 AC_CHECK_FUNCS([getdtablesize])
-AC_CHECK_FUNCS([abort2])
 AC_CHECK_FUNCS([timegm])
 AC_CHECK_FUNCS([nanosleep])
 AC_CHECK_FUNCS([setppriv])
diff --git a/include/vapi/vsm_int.h b/include/vapi/vsm_int.h
index 3554982..521f89d 100644
--- a/include/vapi/vsm_int.h
+++ b/include/vapi/vsm_int.h
@@ -65,9 +65,6 @@ struct VSM_head {
 
 	unsigned		shm_size;
 
-	/* Panic message buffer */
-	char			panicstr[64 * 1024];
-
 	unsigned		alloc_seq;
 	/* Must be last element */
 	struct VSM_chunk	head;



From geoff at varnish-cache.org  Mon Jan  9 20:52:39 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:39 +0100
Subject: [experimental-ims] 919696b More include file cleanup
Message-ID: <E1RkMCt-0008JO-HD@project.varnish-software.com>

commit 919696b0d969c7e2400434344c3c0658942160c8
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Nov 16 06:30:16 2011 +0000

    More include file cleanup

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index 129dbf0..bae2572 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -114,15 +114,6 @@ struct worker;
 
 #define DIGEST_LEN		32
 
-/*--------------------------------------------------------------------
- * Pointer aligment magic
- */
-
-#define PALGN		(sizeof(void *) - 1)
-#define PAOK(p)		(((uintptr_t)(p) & PALGN) == 0)
-#define PRNDDN(p)	((uintptr_t)(p) & ~PALGN)
-#define PRNDUP(p)	(((uintptr_t)(p) + PALGN) & ~PALGN)
-
 /*--------------------------------------------------------------------*/
 
 typedef struct {
@@ -138,6 +129,9 @@ enum step {
 #undef STEP
 };
 
+/*--------------------------------------------------------------------*/
+struct lock { void *priv; };	// Opaque
+
 /*--------------------------------------------------------------------
  * Workspace structure for quick memory allocation.
  */
diff --git a/bin/varnishd/common/common.h b/bin/varnishd/common/common.h
index f59cfc9..8e32ec9 100644
--- a/bin/varnishd/common/common.h
+++ b/bin/varnishd/common/common.h
@@ -55,13 +55,9 @@ extern char *PAN_panicstr;
 extern unsigned PAN_panicstr_len;
 
 /* varnishd.c */
-struct vsb;
-extern struct vsb *vident;
+extern struct vsb *vident;		// XXX: -> heritage ?
 int Symbol_Lookup(struct vsb *vsb, void *ptr);
 
-#define TRUST_ME(ptr)	((void*)(uintptr_t)(ptr))
-
-
 /* Help shut up FlexeLint */
 #define __match_proto__(xxx) /*lint -e{818} */
 
@@ -97,9 +93,6 @@ void VSM__Clean(void);
 #define VSM_CLASS_MARK	"MgrCld"
 #define VSM_COOL_TIME	5
 
-/* cache_lck.c */
-struct lock { void *priv; };		// Opaque
-
 /*---------------------------------------------------------------------
  * Generic power-2 rounding macros
  */
@@ -107,3 +100,17 @@ struct lock { void *priv; };		// Opaque
 #define PWR2(x)     ((((x)-1)&(x))==0)		/* Is a power of two */
 #define RDN2(x, y)  ((x)&(~((y)-1)))		/* if y is powers of two */
 #define RUP2(x, y)  (((x)+((y)-1))&(~((y)-1)))	/* if y is powers of two */
+
+/*--------------------------------------------------------------------
+ * Pointer aligment magic
+ */
+
+#define PALGN	    (sizeof(void *) - 1)	/* size of alignment */
+#define PAOK(p)	    (((uintptr_t)(p) & PALGN) == 0)	/* is aligned */
+#define PRNDDN(p)   ((uintptr_t)(p) & ~PALGN)		/* Round down */
+#define PRNDUP(p)   (((uintptr_t)(p) + PALGN) & ~PALGN)	/* Round up */
+
+/*--------------------------------------------------------------------
+ * To be used as little as possible to wash off const/volatile etc.
+ */
+#define TRUST_ME(ptr)	((void*)(uintptr_t)(ptr))



From geoff at varnish-cache.org  Mon Jan  9 20:52:39 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:39 +0100
Subject: [experimental-ims] 46b0d2d Move mgt cli stuff to _mgt.c file
Message-ID: <E1RkMCt-0008JU-Ip@project.varnish-software.com>

commit 46b0d2d22700ec66782591cbfcb4245baa978913
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Nov 16 06:42:09 2011 +0000

    Move mgt cli stuff to _mgt.c file

diff --git a/bin/varnishd/storage/stevedore.c b/bin/varnishd/storage/stevedore.c
index 9b22cd3..79b1292 100644
--- a/bin/varnishd/storage/stevedore.c
+++ b/bin/varnishd/storage/stevedore.c
@@ -39,7 +39,6 @@
 #include "cache/cache.h"
 
 #include "storage/storage.h"
-#include "vcli_priv.h"
 #include "vrt.h"
 #include "vrt_obj.h"
 
@@ -422,30 +421,6 @@ STV_close(void)
 		stv->close(stv);
 }
 
-/*--------------------------------------------------------------------*/
-
-static void
-stv_cli_list(struct cli *cli, const char * const *av, void *priv)
-{
-	struct stevedore *stv;
-
-	ASSERT_MGT();
-	(void)av;
-	(void)priv;
-	VCLI_Out(cli, "Storage devices:\n");
-	stv = stv_transient;
-		VCLI_Out(cli, "\tstorage.%s = %s\n", stv->ident, stv->name);
-	VTAILQ_FOREACH(stv, &stv_stevedores, list)
-		VCLI_Out(cli, "\tstorage.%s = %s\n", stv->ident, stv->name);
-}
-
-/*--------------------------------------------------------------------*/
-
-struct cli_proto cli_stv[] = {
-	{ "storage.list", "storage.list", "List storage devices\n",
-	    0, 0, "", stv_cli_list },
-	{ NULL}
-};
 
 /*--------------------------------------------------------------------
  * VRT functions for stevedores
diff --git a/bin/varnishd/storage/stevedore_mgt.c b/bin/varnishd/storage/stevedore_mgt.c
index fc0d6a6..4251839 100644
--- a/bin/varnishd/storage/stevedore_mgt.c
+++ b/bin/varnishd/storage/stevedore_mgt.c
@@ -42,12 +42,37 @@
 
 #include "storage/storage.h"
 #include "vav.h"
+#include "vcli_priv.h"
 
 struct stevedore_head stv_stevedores =
     VTAILQ_HEAD_INITIALIZER(stv_stevedores);
 
 struct stevedore *stv_transient;
 
+/*--------------------------------------------------------------------*/
+
+static void
+stv_cli_list(struct cli *cli, const char * const *av, void *priv)
+{
+	struct stevedore *stv;
+
+	ASSERT_MGT();
+	(void)av;
+	(void)priv;
+	VCLI_Out(cli, "Storage devices:\n");
+	stv = stv_transient;
+		VCLI_Out(cli, "\tstorage.%s = %s\n", stv->ident, stv->name);
+	VTAILQ_FOREACH(stv, &stv_stevedores, list)
+		VCLI_Out(cli, "\tstorage.%s = %s\n", stv->ident, stv->name);
+}
+
+/*--------------------------------------------------------------------*/
+
+struct cli_proto cli_stv[] = {
+	{ "storage.list", "storage.list", "List storage devices\n",
+	    0, 0, "", stv_cli_list },
+	{ NULL}
+};
 /*--------------------------------------------------------------------
  * Parse a stevedore argument on the form:
  *	[ name '=' ] strategy [ ',' arg ] *



From geoff at varnish-cache.org  Mon Jan  9 20:52:39 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:39 +0100
Subject: [experimental-ims] 86aa540 Reduce #include scope to common.h
Message-ID: <E1RkMCt-0008Jg-L3@project.varnish-software.com>

commit 86aa540f2bb65ddc0b6bb7adf5559d08ab3f8a6b
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu Nov 17 09:18:11 2011 +0000

    Reduce #include scope to common.h

diff --git a/bin/varnishd/waiter/cache_waiter.c b/bin/varnishd/waiter/cache_waiter.c
index c428294..e09a296 100644
--- a/bin/varnishd/waiter/cache_waiter.c
+++ b/bin/varnishd/waiter/cache_waiter.c
@@ -29,8 +29,11 @@
  */
 
 #include "config.h"
+#include <stdio.h>
+#include <unistd.h>
+#include <string.h>
 
-#include "cache/cache.h"
+#include "common/common.h"
 
 #include "waiter/cache_waiter.h"
 #include "vcli.h"



From geoff at varnish-cache.org  Mon Jan  9 20:52:39 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:39 +0100
Subject: [experimental-ims] aaad3cf Rename to clean up namespace a bit
Message-ID: <E1RkMCt-0008Jo-Mi@project.varnish-software.com>

commit aaad3cf561fc2f938155fad4f66c4ee46a69dc64
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu Nov 17 09:33:11 2011 +0000

    Rename to clean up namespace a bit

diff --git a/bin/varnishd/Makefile.am b/bin/varnishd/Makefile.am
index 359373d..2116701 100644
--- a/bin/varnishd/Makefile.am
+++ b/bin/varnishd/Makefile.am
@@ -74,7 +74,7 @@ varnishd_SOURCES = \
 	storage/storage_persistent_subr.c \
 	storage/storage_synth.c \
 	storage/storage_umem.c \
-	waiter/cache_waiter.c \
+	waiter/waiter_common.c \
 	waiter/cache_waiter_epoll.c \
 	waiter/cache_waiter_kqueue.c \
 	waiter/cache_waiter_poll.c \
diff --git a/bin/varnishd/waiter/cache_waiter.c b/bin/varnishd/waiter/cache_waiter.c
deleted file mode 100644
index e09a296..0000000
--- a/bin/varnishd/waiter/cache_waiter.c
+++ /dev/null
@@ -1,113 +0,0 @@
-/*-
- * Copyright (c) 2006 Verdens Gang AS
- * Copyright (c) 2006-2011 Varnish Software AS
- * All rights reserved.
- *
- * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- *
- */
-
-#include "config.h"
-#include <stdio.h>
-#include <unistd.h>
-#include <string.h>
-
-#include "common/common.h"
-
-#include "waiter/cache_waiter.h"
-#include "vcli.h"
-#include "vcli_priv.h"
-
-static const struct waiter * const vca_waiters[] = {
-    #if defined(HAVE_KQUEUE)
-	&waiter_kqueue,
-    #endif
-    #if defined(HAVE_EPOLL_CTL)
-	&waiter_epoll,
-    #endif
-    #if defined(HAVE_PORT_CREATE)
-	&waiter_ports,
-    #endif
-	&waiter_poll,
-	NULL,
-};
-
-struct waiter const * waiter;
-
-const char *
-WAIT_GetName(void)
-{
-
-	if (waiter != NULL)
-		return (waiter->name);
-	else
-		return ("no_waiter");
-}
-
-void
-WAIT_tweak_waiter(struct cli *cli, const char *arg)
-{
-	int i;
-
-	ASSERT_MGT();
-
-	if (arg == NULL) {
-		if (waiter == NULL)
-			VCLI_Out(cli, "default");
-		else
-			VCLI_Out(cli, "%s", waiter->name);
-
-		VCLI_Out(cli, " (");
-		for (i = 0; vca_waiters[i] != NULL; i++)
-			VCLI_Out(cli, "%s%s", i == 0 ? "" : ", ",
-			    vca_waiters[i]->name);
-		VCLI_Out(cli, ")");
-		return;
-	}
-	if (!strcmp(arg, "default")) {
-		waiter = NULL;
-		return;
-	}
-	for (i = 0; vca_waiters[i]; i++) {
-		if (!strcmp(arg, vca_waiters[i]->name)) {
-			waiter = vca_waiters[i];
-			return;
-		}
-	}
-	VCLI_Out(cli, "Unknown waiter");
-	VCLI_SetResult(cli, CLIS_PARAM);
-}
-
-void
-WAIT_Init(void)
-{
-
-	if (waiter == NULL)
-		waiter = vca_waiters[0];
-
-	AN(waiter);
-	AN(waiter->name);
-	AN(waiter->init);
-	AN(waiter->pass);
-}
diff --git a/bin/varnishd/waiter/waiter_common.c b/bin/varnishd/waiter/waiter_common.c
new file mode 100644
index 0000000..e09a296
--- /dev/null
+++ b/bin/varnishd/waiter/waiter_common.c
@@ -0,0 +1,113 @@
+/*-
+ * Copyright (c) 2006 Verdens Gang AS
+ * Copyright (c) 2006-2011 Varnish Software AS
+ * All rights reserved.
+ *
+ * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ */
+
+#include "config.h"
+#include <stdio.h>
+#include <unistd.h>
+#include <string.h>
+
+#include "common/common.h"
+
+#include "waiter/cache_waiter.h"
+#include "vcli.h"
+#include "vcli_priv.h"
+
+static const struct waiter * const vca_waiters[] = {
+    #if defined(HAVE_KQUEUE)
+	&waiter_kqueue,
+    #endif
+    #if defined(HAVE_EPOLL_CTL)
+	&waiter_epoll,
+    #endif
+    #if defined(HAVE_PORT_CREATE)
+	&waiter_ports,
+    #endif
+	&waiter_poll,
+	NULL,
+};
+
+struct waiter const * waiter;
+
+const char *
+WAIT_GetName(void)
+{
+
+	if (waiter != NULL)
+		return (waiter->name);
+	else
+		return ("no_waiter");
+}
+
+void
+WAIT_tweak_waiter(struct cli *cli, const char *arg)
+{
+	int i;
+
+	ASSERT_MGT();
+
+	if (arg == NULL) {
+		if (waiter == NULL)
+			VCLI_Out(cli, "default");
+		else
+			VCLI_Out(cli, "%s", waiter->name);
+
+		VCLI_Out(cli, " (");
+		for (i = 0; vca_waiters[i] != NULL; i++)
+			VCLI_Out(cli, "%s%s", i == 0 ? "" : ", ",
+			    vca_waiters[i]->name);
+		VCLI_Out(cli, ")");
+		return;
+	}
+	if (!strcmp(arg, "default")) {
+		waiter = NULL;
+		return;
+	}
+	for (i = 0; vca_waiters[i]; i++) {
+		if (!strcmp(arg, vca_waiters[i]->name)) {
+			waiter = vca_waiters[i];
+			return;
+		}
+	}
+	VCLI_Out(cli, "Unknown waiter");
+	VCLI_SetResult(cli, CLIS_PARAM);
+}
+
+void
+WAIT_Init(void)
+{
+
+	if (waiter == NULL)
+		waiter = vca_waiters[0];
+
+	AN(waiter);
+	AN(waiter->name);
+	AN(waiter->init);
+	AN(waiter->pass);
+}



From geoff at varnish-cache.org  Mon Jan  9 20:52:39 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:39 +0100
Subject: [experimental-ims] bdbb1d5 Use the right counter for directors index
Message-ID: <E1RkMCt-0008Jy-Oa@project.varnish-software.com>

commit bdbb1d59513cba8b268ed1dbe2d948619ef4ae07
Author: Rogier 'DocWilco' Mulhuijzen <github at bsdchicks.com>
Date:   Fri Nov 18 12:29:04 2011 +0100

    Use the right counter for directors index
    
    Fixes	#1060

diff --git a/bin/varnishtest/tests/r01060.vtc b/bin/varnishtest/tests/r01060.vtc
new file mode 100644
index 0000000..25a3988
--- /dev/null
+++ b/bin/varnishtest/tests/r01060.vtc
@@ -0,0 +1,48 @@
+varnishtest "DNS director tramples on other backends"
+
+server s1 {
+	rxreq
+	txresp
+} -start
+
+server s2 {
+	rxreq
+	txresp
+} -start
+
+varnish v1 -vcl {
+	backend b1 {
+		.host = "${s1_addr}";
+		.port = "${s1_port}";
+	}
+
+	director d1 dns {
+		.list = {
+			.port = "80";
+			"${bad_ip}"/32;
+		}
+		.ttl = 1m;
+	}
+
+	backend b2 {
+		.host = "${s2_addr}";
+		.port = "${s2_port}";
+	}
+
+	sub vcl_recv {
+		set req.backend = d1;
+		if (req.url == "/1") {
+			set req.backend = b1;
+		} else {
+			set req.backend = b2;
+		}
+	}
+} -start
+
+
+client c1 {
+	txreq -url "/1"
+	rxresp
+	txreq -url "/2"
+	rxresp
+} -run
diff --git a/lib/libvcl/vcc_dir_dns.c b/lib/libvcl/vcc_dir_dns.c
index a91c021..bdc0a21 100644
--- a/lib/libvcl/vcc_dir_dns.c
+++ b/lib/libvcl/vcc_dir_dns.c
@@ -79,7 +79,7 @@ print_backend(struct vcc *tl,
 	AN(vsb);
 	tl->fb = vsb;
 	Fc(tl, 0, "\t{ .host = VGC_backend_%s },\n",vgcname);
-	Fh(tl, 1, "\n#define VGC_backend_%s %d\n", vgcname, serial);
+	Fh(tl, 1, "\n#define VGC_backend_%s %d\n", vgcname, tl->ndirector);
 
 	Fb(tl, 0, "\nstatic const struct vrt_backend vgc_dir_priv_%s = {\n",
 	    vgcname);



From geoff at varnish-cache.org  Mon Jan  9 20:52:39 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:39 +0100
Subject: [experimental-ims] 4895d9f Update varnishtest(1) documentation
	somewhat
Message-ID: <E1RkMCt-0008K6-QM@project.varnish-software.com>

commit 4895d9f94730e5b5c4b73758c03708d21aa30f79
Author: Kristian Lyngstol <kristian at bohemians.org>
Date:   Fri Nov 18 15:09:51 2011 +0100

    Update varnishtest(1) documentation somewhat

diff --git a/doc/sphinx/reference/varnishtest.rst b/doc/sphinx/reference/varnishtest.rst
index c2aaae9..ff68337 100644
--- a/doc/sphinx/reference/varnishtest.rst
+++ b/doc/sphinx/reference/varnishtest.rst
@@ -7,14 +7,16 @@ Test program for Varnish
 ------------------------
 
 :Author: Stig Sandbeck Mathisen
-:Date:   2010-05-31
-:Version: 1.0
+:Author: Kristian Lyngst?l
+:Date:   2011-11-15
+:Version: 1.1
 :Manual section: 1
 
 
 SYNOPSIS
 ========
-     varnishtest [-n iter] [-q] [-v] file [file ...]
+
+     varnishtest [-iklLqv] [-n iter] [-D name=val] [-j jobs] [-t duration] file [file ...]
 
 DESCRIPTION
 ===========
@@ -24,160 +26,95 @@ Varnish Cache.
 
 The varnishtest program, when started and given one or more script
 files, can create a number of threads representing backends, some
-threads representing clients, and a varnishd process.
+threads representing clients, and a varnishd process. This is then used to
+simulate a transaction to provoke a specific behavior.
 
 The following options are available:
 
--n iter     Run iter number of iterations.
+-D name=val      Define macro for use in scripts
+
+-i               Find varnishd in build tree
+
+-j jobs          Run this many tests in parallel
+
+-k               Continue on test failure
+
+-l               Leave /tmp/vtc.* if test fails
+
+-L               Always leave /tmp/vtc.*
+
+-n iterations    Run tests this many times
+
+-q               Quiet mode: report only failures
+
+-t duration      Time tests out after this long
+
+-v               Verbose mode: always report test log
+
+-h               Show help
 
--q          Be quiet.
+file             File to use as a script
 
--v          Be verbose.
 
--t          Dunno.
+Macro definitions that can be overridden.
 
-file        File to use as a script
+varnishd         Path to varnishd to use [varnishd]
 
 
 SCRIPTS
 =======
 
-Example script
-~~~~~~~~~~~~~~
-::
-
-    # Start a varnish instance called "v1"
-    varnish v1 -arg "-b localhost:9080" -start
-    
-    # Create a server thread called "s1"
-    server s1 {
-        # Receive a request
-        rxreq
-        # Send a standard response
-        txresp -hdr "Connection: close" -body "012345\n"
-    }
-    
-    # Start the server thread
-    server s1 -start
-    
-    # Create a client thread called "c1"
-    client c1 {
-        # Send a request
-        txreq -url "/"
-        # Wait for a response
-        rxresp
-    # Insist that it be a success
-    expect resp.status == 200
-    }
-    
-    # Run the client
-    client c1 -run
-    
-    # Wait for the server to die
-    server s1 -wait
-
-    # (Forcefully) Stop the varnish instance.
-    varnish v1 -stop
-
-Example script output
-~~~~~~~~~~~~~~~~~~~~~
-
-The output, running this script looks as follows. The "bargraph" at
-the beginning of the line is an indication of the level of detail in
-the line. The second field where the message comes from. The rest of
-the line is anyones guess :-)
-::
-
-    #  TEST tests/b00000.vtc starting
-    ### v1  CMD: cd ../varnishd && ./varnishd -d -d -n v1 -a :9081 -T :9001 -b localhost:9080
-    ### v1  opening CLI connection
-    #### v1  debug| NB: Storage size limited to 2GB on 32 bit architecture,\n
-    #### v1  debug| NB: otherwise we could run out of address space.\n
-    #### v1  debug| storage_file: filename: ./varnish.Shkoq5 (unlinked) size 2047 MB.\n
-    ### v1  CLI connection fd = 3
-    #### v1  CLI TX| start
-    #### v1  debug| Using old SHMFILE\n
-    #### v1  debug| Notice: locking SHMFILE in core failed: Operation not permitted\n
-    #### v1  debug| bind(): Address already in use\n
-    #### v1  debug| rolling(1)...
-    #### v1  debug| \n
-    #### v1  debug| rolling(2)...\n
-    #### v1  debug| Debugging mode, enter "start" to start child\n
-    ### v1  CLI 200 <start>
-    ##  s1  Starting server
-    ### s1  listen on :9080 (fd 6)
-    ##  c1  Starting client
-    ##  c1  Waiting for client
-    ##  s1  started on :9080
-    ##  c1  started
-    ### c1  connect to :9081
-    ### c1  connected to :9081 fd is 8
-    #### c1  | GET / HTTP/1.1\r\n
-    #### c1  | \r\n
-    ### c1  rxresp
-    #### s1  Accepted socket 7
-    ### s1  rxreq
-    #### s1  | GET / HTTP/1.1\r\n
-    #### s1  | X-Varnish: 422080121\r\n
-    #### s1  | X-Forwarded-For: 127.0.0.1\r\n
-    #### s1  | Host: localhost\r\n
-    #### s1  | \r\n
-    #### s1  http[ 0] | GET
-    #### s1  http[ 1] | /
-    #### s1  http[ 2] | HTTP/1.1
-    #### s1  http[ 3] | X-Varnish: 422080121
-    #### s1  http[ 4] | X-Forwarded-For: 127.0.0.1
-    #### s1  http[ 5] | Host: localhost
-    #### s1  | HTTP/1.1 200 Ok\r\n
-    #### s1  | Connection: close\r\n
-    #### s1  | \r\n
-    #### s1  | 012345\n
-    #### s1  | \r\n
-    ##  s1  ending
-    #### c1  | HTTP/1.1 200 Ok\r\n
-    #### c1  | Content-Length: 9\r\n
-    #### c1  | Date: Mon, 16 Jun 2008 22:16:55 GMT\r\n
-    #### c1  | X-Varnish: 422080121\r\n
-    #### c1  | Age: 0\r\n
-    #### c1  | Via: 1.1 varnish\r\n
-    #### c1  | Connection: keep-alive\r\n
-    #### c1  | \r\n
-    #### c1  http[ 0] | HTTP/1.1
-    #### c1  http[ 1] | 200
-    #### c1  http[ 2] | Ok
-    #### c1  http[ 3] | Content-Length: 9
-    #### c1  http[ 4] | Date: Mon, 16 Jun 2008 22:16:55 GMT
-    #### c1  http[ 5] | X-Varnish: 422080121
-    #### c1  http[ 6] | Age: 0
-    #### c1  http[ 7] | Via: 1.1 varnish
-    #### c1  http[ 8] | Connection: keep-alive
-    #### c1  EXPECT resp.status (200) == 200 (200) match
-    ##  c1  ending
-    ##  s1  Waiting for server
-    #### v1  CLI TX| stop
-    ### v1  CLI 200 <stop>
-    #  TEST tests/b00000.vtc completed
-
-If instead of 200 we had expected 201 with the line:::
-
-  expect resp.status == 201
-
-The output would have ended with:::
-
-  #### c1  http[ 0] | HTTP/1.1
-  #### c1  http[ 1] | 200
-  #### c1  http[ 2] | Ok
-  #### c1  http[ 3] | Content-Length: 9
-  #### c1  http[ 4] | Date: Mon, 16 Jun 2008 22:26:35 GMT
-  #### c1  http[ 5] | X-Varnish: 648043653 648043652
-  #### c1  http[ 6] | Age: 6
-  #### c1  http[ 7] | Via: 1.1 varnish
-  #### c1  http[ 8] | Connection: keep-alive
-  ---- c1  EXPECT resp.status (200) == 201 (201) failed
+The script language used for Varnishtest is not a strictly defined
+language. The best reference for writing scripts is the varnishtest program
+itself. In the Varnish source code repository, under
+`bin/varnishtest/tests/`, all the regression tests for Varnish are kept.
+
+An example::
+
+        varnishtest "#1029"
+
+        server s1 {
+                rxreq
+                expect req.url == "/bar"
+                txresp -gzipbody {[bar]}
+
+                rxreq
+                expect req.url == "/foo"
+                txresp -body {<h1>FOO<esi:include src="/bar"/>BARF</h1>}
+
+        } -start
+
+        varnish v1 -vcl+backend {
+                sub vcl_fetch {
+                        set beresp.do_esi = true;
+                        if (req.url == "/foo") {
+                                set beresp.ttl = 0s;
+                        } else {
+                                set beresp.ttl = 10m;
+                        }
+                }
+        } -start
+
+        client c1 {
+                txreq -url "/bar" -hdr "Accept-Encoding: gzip"
+                rxresp
+                gunzip
+                expect resp.bodylen == 5
+
+                txreq -url "/foo" -hdr "Accept-Encoding: gzip"
+                rxresp
+                expect resp.bodylen == 21
+        } -run
+
+When run, the above script will simulate a server (s1) that expects two
+different requests. It will start a varnish server (v1) and add the backend
+definition to the VCL specified (-vcl+backend). Finally it starts the
+c1-client, which is a single client sending two requests.
 
 SEE ALSO
 ========
 
+* varnishtest source code repository with tests
 * varnishhist(1)
 * varnishlog(1)
 * varnishncsa(1)
@@ -190,9 +127,9 @@ HISTORY
 
 The varnishtest program was developed by Poul-Henning Kamp
 ?phk at phk.freebsd.dk? in cooperation with Varnish Software AS.
-This manual page
-was written by Stig Sandbeck Mathisen ?ssm at linpro.no? using examples
-by Poul-Henning Kamp ?phk at phk.freebsd.dk?.
+This manual page was originally written by Stig Sandbeck Mathisen
+?ssm at linpro.no? and updated by Kristian Lyngst?l
+(kristian at varnish-cache.org).
 
 COPYRIGHT
 =========
@@ -200,4 +137,4 @@ COPYRIGHT
 This document is licensed under the same licence as Varnish
 itself. See LICENCE for details.
 
-* Copyright (c) 2007-2008 Varnish Software AS
+* Copyright (c) 2007-2011 Varnish Software AS



From geoff at varnish-cache.org  Mon Jan  9 20:52:39 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:39 +0100
Subject: [experimental-ims] ab9cec7 Merge branch 'master' of
	git+ssh://git.varnish-cache.org/git/varnish-cache
Message-ID: <E1RkMCt-0008KE-SE@project.varnish-software.com>

commit ab9cec7c2c81bee78a388ade260f1b18beef8ba7
Merge: 4895d9f bdbb1d5
Author: Kristian Lyngstol <kristian at bohemians.org>
Date:   Fri Nov 18 15:10:05 2011 +0100

    Merge branch 'master' of git+ssh://git.varnish-cache.org/git/varnish-cache



From geoff at varnish-cache.org  Mon Jan  9 20:52:39 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:39 +0100
Subject: [experimental-ims] 8921e99 This started out as something entirely
	different,
	but I got tired of the fact that we have so many params in units of
	bytes which you cannot tell "64k" so now you can.
Message-ID: <E1RkMCt-0008KS-Up@project.varnish-software.com>

commit 8921e9908fb557b2128e99235bdddb4b23a48d64
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Fri Nov 18 19:02:16 2011 +0000

    This started out as something entirely different, but I got tired
    of the fact that we have so many params in units of bytes which
    you cannot tell "64k" so now you can.
    
    Notice that a couple of specialist params (fetch_chunksize and
    fetch_maxchunksize) have changed units from kilobytes to bytes,
    but the default values are the same.

diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c
index a678dcc..a5c0323 100644
--- a/bin/varnishd/cache/cache_fetch.c
+++ b/bin/varnishd/cache/cache_fetch.c
@@ -188,7 +188,7 @@ FetchStorage(struct worker *w, ssize_t sz)
 	if (l == 0)
 		l = sz;
 	if (l == 0)
-		l = cache_param->fetch_chunksize * 1024LL;
+		l = cache_param->fetch_chunksize;
 	st = STV_alloc(w, l);
 	if (st == NULL) {
 		(void)FetchError(w, "Could not get storage");
diff --git a/bin/varnishd/common/params.h b/bin/varnishd/common/params.h
index ee8bad2..51721b1 100644
--- a/bin/varnishd/common/params.h
+++ b/bin/varnishd/common/params.h
@@ -87,13 +87,13 @@ struct params {
 	unsigned		auto_restart;
 
 	/* Fetcher hints */
-	unsigned		fetch_chunksize;
-	unsigned		fetch_maxchunksize;
+	ssize_t			fetch_chunksize;
+	ssize_t			fetch_maxchunksize;
 	unsigned		nuke_limit;
 
 #ifdef SENDFILE_WORKS
 	/* Sendfile object minimum size */
-	unsigned		sendfile_threshold;
+	ssize_t			sendfile_threshold;
 #endif
 
 	/* VCL traces */
diff --git a/bin/varnishd/mgt/mgt_param.c b/bin/varnishd/mgt/mgt_param.c
index b7c18b2..10e64b5 100644
--- a/bin/varnishd/mgt/mgt_param.c
+++ b/bin/varnishd/mgt/mgt_param.c
@@ -31,6 +31,7 @@
 
 #include <grp.h>
 #include <limits.h>
+#include <math.h>
 #include <pwd.h>
 #include <stdio.h>
 #include <stdlib.h>
@@ -47,6 +48,7 @@
 #include "vcli.h"
 #include "vcli_common.h"
 #include "vcli_priv.h"
+#include "vnum.h"
 #include "vss.h"
 
 #include "mgt_cli.h"
@@ -244,6 +246,97 @@ tweak_uint(struct cli *cli, const struct parspec *par, const char *arg)
 	tweak_generic_uint(cli, dest, arg, (uint)par->min, (uint)par->max);
 }
 
+/*--------------------------------------------------------------------*/
+
+static void
+fmt_bytes(struct cli *cli, ssize_t t)
+{
+	const char *p;
+
+	if (t & 0xff) {
+		VCLI_Out(cli, "%zub", t);
+		return;
+	}
+	for (p = "kMGTPEZY"; *p; p++) {
+		if (t & 0x300) {
+			VCLI_Out(cli, "%.2f%c", t / 1024.0, *p);
+			return;
+		}
+		t /= 1024;
+		if (t & 0x0ff) {
+			VCLI_Out(cli, "%zu%c", t, *p);
+			return;
+		}
+	}
+	VCLI_Out(cli, "(bogus number)");
+}
+
+static void
+tweak_generic_bytes(struct cli *cli, volatile ssize_t *dest, const char *arg,
+    double min, double max)
+{
+	uintmax_t r;
+	const char *p;
+
+	if (arg != NULL) {
+		p = VNUM_2bytes(arg, &r, 0);
+		if (p != NULL) {
+			VCLI_Out(cli, "Could not convert to bytes.\n");
+			VCLI_Out(cli, "%s\n", p);
+			VCLI_Out(cli,
+			    "  Try something like '80k' or '120M'\n");
+			VCLI_SetResult(cli, CLIS_PARAM);
+			return;
+		}
+		if ((uintmax_t)((ssize_t)r) != r || r > max) {
+			VCLI_Out(cli, "Must be no more than ");
+			fmt_bytes(cli, (ssize_t)max);
+			VCLI_Out(cli, "\n");
+			VCLI_SetResult(cli, CLIS_PARAM);
+			return;
+		}
+		if (r < min) {
+			VCLI_Out(cli, "Must be at least ");
+			fmt_bytes(cli, (ssize_t)min);
+			VCLI_Out(cli, "\n");
+			VCLI_SetResult(cli, CLIS_PARAM);
+			return;
+		}
+		*dest = r;
+	} else {
+		fmt_bytes(cli, *dest);
+	}
+}
+
+/*--------------------------------------------------------------------*/
+
+static void
+tweak_bytes(struct cli *cli, const struct parspec *par, const char *arg)
+{
+	volatile ssize_t *dest;
+
+	assert(par->min >= 0);
+	dest = par->priv;
+	tweak_generic_bytes(cli, dest, arg, par->min, par->max);
+}
+
+
+/*--------------------------------------------------------------------*/
+
+static void
+tweak_bytes_u(struct cli *cli, const struct parspec *par, const char *arg)
+{
+	volatile unsigned *d1;
+	volatile ssize_t dest;
+
+	assert(par->max <= UINT_MAX);
+	assert(par->min >= 0);
+	d1 = par->priv;
+	dest = *d1;
+	tweak_generic_bytes(cli, &dest, arg, par->min, par->max);
+	*d1 = dest;
+}
+
 /*--------------------------------------------------------------------
  * XXX: slightly magic.  We want to initialize to "nobody" (XXX: shouldn't
  * XXX: that be something autocrap found for us ?) but we don't want to
@@ -511,21 +604,23 @@ static const struct parspec input_parspec[] = {
 		"flush of the cache use \"ban.url .\"",
 		0,
 		"120", "seconds" },
-	{ "sess_workspace", tweak_uint, &mgt_param.sess_workspace, 1024, UINT_MAX,
+	{ "sess_workspace",
+		tweak_bytes_u, &mgt_param.sess_workspace, 1024, UINT_MAX,
 		"Bytes of HTTP protocol workspace allocated for sessions. "
 		"This space must be big enough for the entire HTTP protocol "
 		"header and any edits done to it in the VCL code.\n"
 		"Minimum is 1024 bytes.",
 		DELAYED_EFFECT,
-		"65536",
-		"bytes" },
-	{ "http_req_hdr_len", tweak_uint, &mgt_param.http_req_hdr_len,
+		"64k", "bytes" },
+	{ "http_req_hdr_len",
+		tweak_bytes_u, &mgt_param.http_req_hdr_len,
 		40, UINT_MAX,
 		"Maximum length of any HTTP client request header we will "
 		"allow.  The limit is inclusive its continuation lines.\n",
 		0,
-		"8192", "bytes" },
-	{ "http_req_size", tweak_uint, &mgt_param.http_req_size,
+		"8k", "bytes" },
+	{ "http_req_size",
+		tweak_bytes_u, &mgt_param.http_req_size,
 		256, UINT_MAX,
 		"Maximum number of bytes of HTTP client request we will deal "
 		"with.  This is a limit on all bytes up to the double blank "
@@ -534,14 +629,16 @@ static const struct parspec input_parspec[] = {
 		"workspace (param: sess_workspace) and this parameter limits "
 		"how much of that the request is allowed to take up.",
 		0,
-		"32768", "bytes" },
-	{ "http_resp_hdr_len", tweak_uint, &mgt_param.http_resp_hdr_len,
+		"32k", "bytes" },
+	{ "http_resp_hdr_len",
+		tweak_bytes_u, &mgt_param.http_resp_hdr_len,
 		40, UINT_MAX,
 		"Maximum length of any HTTP backend response header we will "
 		"allow.  The limit is inclusive its continuation lines.\n",
 		0,
-		"8192", "bytes" },
-	{ "http_resp_size", tweak_uint, &mgt_param.http_resp_size,
+		"8k", "bytes" },
+	{ "http_resp_size",
+		tweak_bytes_u, &mgt_param.http_resp_size,
 		256, UINT_MAX,
 		"Maximum number of bytes of HTTP backend resonse we will deal "
 		"with.  This is a limit on all bytes up to the double blank "
@@ -550,7 +647,7 @@ static const struct parspec input_parspec[] = {
 		"workspace (param: sess_workspace) and this parameter limits "
 		"how much of that the request is allowed to take up.",
 		0,
-		"32768", "bytes" },
+		"32k", "bytes" },
 	{ "http_max_hdr", tweak_uint, &mgt_param.http_max_hdr, 32, 65535,
 		"Maximum number of HTTP headers we will deal with in "
 		"client request or backend reponses.  "
@@ -559,7 +656,8 @@ static const struct parspec input_parspec[] = {
 		"objects allocate exact space for the headers they store.\n",
 		0,
 		"64", "header lines" },
-	{ "shm_workspace", tweak_uint, &mgt_param.shm_workspace, 4096, UINT_MAX,
+	{ "shm_workspace",
+		tweak_bytes_u, &mgt_param.shm_workspace, 4096, UINT_MAX,
 		"Bytes of shmlog workspace allocated for worker threads. "
 		"If too big, it wastes some ram, if too small it causes "
 		"needless flushes of the SHM workspace.\n"
@@ -567,8 +665,9 @@ static const struct parspec input_parspec[] = {
 		"\"SHM flushes due to overflow\".\n"
 		"Minimum is 4096 bytes.",
 		DELAYED_EFFECT,
-		"8192", "bytes" },
-	{ "shm_reclen", tweak_uint, &mgt_param.shm_reclen, 16, 65535,
+		"8k", "bytes" },
+	{ "shm_reclen",
+		tweak_bytes_u, &mgt_param.shm_reclen, 16, 65535,
 		"Maximum number of bytes in SHM log record.\n"
 		"Maximum is 65535 bytes.",
 		0,
@@ -633,27 +732,29 @@ static const struct parspec input_parspec[] = {
 		EXPERIMENTAL,
 		"50", "allocations" },
 	{ "fetch_chunksize",
-		tweak_uint, &mgt_param.fetch_chunksize, 4, UINT_MAX / 1024.,
+		tweak_bytes_u,
+		    &mgt_param.fetch_chunksize, 4 * 1024, UINT_MAX,
 		"The default chunksize used by fetcher. "
 		"This should be bigger than the majority of objects with "
 		"short TTLs.\n"
 		"Internal limits in the storage_file module makes increases "
 		"above 128kb a dubious idea.",
 		EXPERIMENTAL,
-		"128", "kilobytes" },
+		"128k", "bytes" },
 	{ "fetch_maxchunksize",
-		tweak_uint, &mgt_param.fetch_maxchunksize, 64, UINT_MAX / 1024.,
+		tweak_bytes_u,
+		    &mgt_param.fetch_maxchunksize, 64 * 1024, UINT_MAX,
 		"The maximum chunksize we attempt to allocate from storage. "
 		"Making this too large may cause delays and storage "
 		"fragmentation.\n",
 		EXPERIMENTAL,
-		"262144", "kilobytes" },
+		"256m", "bytes" },
 #ifdef SENDFILE_WORKS
 	{ "sendfile_threshold",
-		tweak_uint, &mgt_param.sendfile_threshold, 0, UINT_MAX,
+		tweak_bytes, &mgt_param.sendfile_threshold, 0, HUGE_VAL,
 		"The minimum size of objects transmitted with sendfile.",
 		EXPERIMENTAL,
-		"-1", "bytes" },
+		"1E", "bytes" },
 #endif /* SENDFILE_WORKS */
 	{ "vcl_trace", tweak_bool,  &mgt_param.vcl_trace, 0, 0,
 		"Trace VCL execution in the shmlog.\n"
@@ -673,19 +774,21 @@ static const struct parspec input_parspec[] = {
 		"Listen queue depth.",
 		MUST_RESTART,
 		"1024", "connections" },
-	{ "cli_buffer", tweak_uint, &mgt_param.cli_buffer, 4096, UINT_MAX,
+	{ "cli_buffer",
+		tweak_bytes_u, &mgt_param.cli_buffer, 4096, UINT_MAX,
 		"Size of buffer for CLI command input."
 		"\nYou may need to increase this if you have big VCL files "
 		"and use the vcl.inline CLI command.\n"
 		"NB: Must be specified with -p to have effect.\n",
 		0,
-		"8192", "bytes" },
-	{ "cli_limit", tweak_uint, &mgt_param.cli_limit, 128, 99999999,
+		"8k", "bytes" },
+	{ "cli_limit",
+		tweak_bytes_u, &mgt_param.cli_limit, 128, 99999999,
 		"Maximum size of CLI response.  If the response exceeds"
 		" this limit, the reponse code will be 201 instead of"
 		" 200 and the last line will indicate the truncation.",
 		0,
-		"4096", "bytes" },
+		"4k", "bytes" },
 	{ "cli_timeout", tweak_timeout, &mgt_param.cli_timeout, 0, 0,
 		"Timeout for the childs replies to CLI requests from "
 		"the mgt_param.",
@@ -918,7 +1021,8 @@ static const struct parspec input_parspec[] = {
 		"Memory impact is 1=1k, 2=2k, ... 9=256k.",
 		0,
 		"8", ""},
-	{ "gzip_stack_buffer", tweak_uint, &mgt_param.gzip_stack_buffer,
+	{ "gzip_stack_buffer",
+		tweak_bytes_u, &mgt_param.gzip_stack_buffer,
 	        2048, UINT_MAX,
 		"Size of stack buffer used for gzip processing.\n"
 		"The stack buffers are used for in-transit data,"
@@ -927,7 +1031,7 @@ static const struct parspec input_parspec[] = {
 		" writes to sockets etc, making it too big is probably"
 		" just a waste of memory.",
 		EXPERIMENTAL,
-		"32768", "Bytes" },
+		"32k", "bytes" },
 	{ "shortlived", tweak_timeout_double,
 		&mgt_param.shortlived, 0, UINT_MAX,
 		"Objects created with TTL shorter than this are always "
diff --git a/bin/varnishd/storage/stevedore.c b/bin/varnishd/storage/stevedore.c
index 79b1292..860604e 100644
--- a/bin/varnishd/storage/stevedore.c
+++ b/bin/varnishd/storage/stevedore.c
@@ -162,8 +162,8 @@ stv_alloc(struct worker *w, const struct object *obj, size_t size)
 	stv = obj->objstore->stevedore;
 	CHECK_OBJ_NOTNULL(stv, STEVEDORE_MAGIC);
 
-	if (size > (size_t)(cache_param->fetch_maxchunksize) << 10)
-		size = (size_t)(cache_param->fetch_maxchunksize) << 10;
+	if (size > cache_param->fetch_maxchunksize)
+		size = cache_param->fetch_maxchunksize;
 
 	for (;;) {
 		/* try to allocate from it */
@@ -172,7 +172,7 @@ stv_alloc(struct worker *w, const struct object *obj, size_t size)
 		if (st != NULL)
 			break;
 
-		if (size > cache_param->fetch_chunksize * 1024LL) {
+		if (size > cache_param->fetch_chunksize) {
 			size >>= 1;
 			continue;
 		}
diff --git a/bin/varnishtest/tests/g00002.vtc b/bin/varnishtest/tests/g00002.vtc
index 1a0ab35..1bf3384 100644
--- a/bin/varnishtest/tests/g00002.vtc
+++ b/bin/varnishtest/tests/g00002.vtc
@@ -22,7 +22,7 @@ varnish v1 \
 	}
 } -start
 
-varnish v1 -cliok "param.set fetch_chunksize 4"
+varnish v1 -cliok "param.set fetch_chunksize 4k"
 
 client c1 {
 	txreq -url /foo -hdr "Accept-Encoding: gzip"
diff --git a/bin/varnishtest/tests/r00776.vtc b/bin/varnishtest/tests/r00776.vtc
index 776810e..53cc37e 100644
--- a/bin/varnishtest/tests/r00776.vtc
+++ b/bin/varnishtest/tests/r00776.vtc
@@ -8,7 +8,7 @@ server s1 {
 } -start
 
 varnish v1 \
-	-arg "-p fetch_chunksize=4" \
+	-arg "-p fetch_chunksize=4k" \
 	-arg "-s malloc,1m" -vcl+backend { } -start
 
 client c1 {



From geoff at varnish-cache.org  Mon Jan  9 20:52:40 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:40 +0100
Subject: [experimental-ims] 085f6d2 Merge branch 'master' of
	ssh://git.varnish-cache.org/git/varnish-cache
Message-ID: <E1RkMCu-0008Kb-0A@project.varnish-software.com>

commit 085f6d2dec9c05259f94a8282eac8a118b0e8baa
Merge: 8921e99 ab9cec7
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Fri Nov 18 19:03:48 2011 +0000

    Merge branch 'master' of ssh://git.varnish-cache.org/git/varnish-cache



From geoff at varnish-cache.org  Mon Jan  9 20:52:40 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:40 +0100
Subject: [experimental-ims] b40bdeb Convert the -l argument into two params.
	For now setting them any other way than -l does you no good.
Message-ID: <E1RkMCu-0008Ki-2W@project.varnish-software.com>

commit b40bdeb066636492b43eb863a002d191aeac557f
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Fri Nov 18 19:33:02 2011 +0000

    Convert the -l argument into two params.  For now setting them any
    other way than -l does you no good.

diff --git a/bin/varnishd/common/params.h b/bin/varnishd/common/params.h
index 51721b1..95258fb 100644
--- a/bin/varnishd/common/params.h
+++ b/bin/varnishd/common/params.h
@@ -186,6 +186,10 @@ struct params {
 	double			shortlived;
 
 	struct vre_limits	vre_limits;
+
+	/* VSM dimensions */
+	ssize_t			vsm_space;
+	ssize_t			vsl_space;
 };
 
 /*
diff --git a/bin/varnishd/mgt/mgt.h b/bin/varnishd/mgt/mgt.h
index 1eabe10..0650413 100644
--- a/bin/varnishd/mgt/mgt.h
+++ b/bin/varnishd/mgt/mgt.h
@@ -81,7 +81,7 @@ void mgt_sandbox_solaris_privsep(void);
 #endif
 
 /* mgt_shmem.c */
-void mgt_SHM_Init(const char *arg);
+void mgt_SHM_Init(void);
 void mgt_SHM_Pid(void);
 
 /* stevedore_mgt.c */
diff --git a/bin/varnishd/mgt/mgt_main.c b/bin/varnishd/mgt/mgt_main.c
index 8a9e1fc..ff8992e 100644
--- a/bin/varnishd/mgt/mgt_main.c
+++ b/bin/varnishd/mgt/mgt_main.c
@@ -336,7 +336,6 @@ main(int argc, char * const *argv)
 	const char *b_arg = NULL;
 	const char *f_arg = NULL;
 	const char *i_arg = NULL;
-	const char *l_arg = NULL;	/* default in mgt_shmem.c */
 	const char *h_arg = "critbit";
 	const char *M_arg = NULL;
 	const char *n_arg = NULL;
@@ -349,6 +348,7 @@ main(int argc, char * const *argv)
 	struct cli cli[1];
 	struct vpf_fh *pfh = NULL;
 	char *dirname;
+	char **av;
 	unsigned clilim;
 
 	/*
@@ -459,7 +459,19 @@ main(int argc, char * const *argv)
 			i_arg = optarg;
 			break;
 		case 'l':
-			l_arg = optarg;
+			av = VAV_Parse(optarg, NULL, ARGV_COMMA);
+			AN(av);
+			if (av[0] != NULL)
+				ARGV_ERR("\t-l ...: %s", av[0]);
+			if (av[1] != NULL) {
+				MCF_ParamSet(cli, "vsl_space", av[1]);
+				cli_check(cli);
+			}
+			if (av[1] != NULL && av[2] != NULL) {
+				MCF_ParamSet(cli, "vsm_space", av[2]);
+				cli_check(cli);
+			}
+			VAV_Free(av);
 			break;
 		case 'M':
 			M_arg = optarg;
@@ -611,7 +623,7 @@ main(int argc, char * const *argv)
 
 	HSH_config(h_arg);
 
-	mgt_SHM_Init(l_arg);
+	mgt_SHM_Init();
 
 	AZ(VSB_finish(vident));
 
diff --git a/bin/varnishd/mgt/mgt_param.c b/bin/varnishd/mgt/mgt_param.c
index 10e64b5..f6de562 100644
--- a/bin/varnishd/mgt/mgt_param.c
+++ b/bin/varnishd/mgt/mgt_param.c
@@ -1085,6 +1085,26 @@ static const struct parspec input_parspec[] = {
 		0,
 		"10000", ""},
 
+	{ "vsl_space", tweak_bytes,
+		&mgt_param.vsl_space, 1024*1024, HUGE_VAL,
+		"The amount of space to allocate for the VSL fifo buffer"
+		" in the VSM memory segment."
+		"  If you make this too small, varnish{ncsa|log} etc will"
+		" not be able to keep up."
+		"  Making it too large just costs memory resources.",
+		MUST_RESTART,
+		"80M", "bytes"},
+
+	{ "vsm_space", tweak_bytes,
+		&mgt_param.vsm_space, 1024*1024, HUGE_VAL,
+		"The amount of space to allocate for stats counters"
+		" in the VSM memory segment."
+		"  If you make this too small, some counters will be"
+		" invisible."
+		"  Making it too large just costs memory resources.",
+		MUST_RESTART,
+		"1M", "bytes"},
+
 	{ NULL, NULL, NULL }
 };
 
diff --git a/bin/varnishd/mgt/mgt_shmem.c b/bin/varnishd/mgt/mgt_shmem.c
index 24bc396..d1bf18e 100644
--- a/bin/varnishd/mgt/mgt_shmem.c
+++ b/bin/varnishd/mgt/mgt_shmem.c
@@ -105,9 +105,7 @@
 #include "vapi/vsc_int.h"
 #include "vapi/vsl_int.h"
 #include "vapi/vsm_int.h"
-#include "vav.h"
 #include "vmb.h"
-#include "vnum.h"
 
 #ifndef MAP_HASSEMAPHORE
 #define MAP_HASSEMAPHORE 0 /* XXX Linux */
@@ -227,67 +225,15 @@ mgt_shm_atexit(void)
 }
 
 void
-mgt_SHM_Init(const char *l_arg)
+mgt_SHM_Init(void)
 {
 	int i, fill;
-	const char *q;
-	uintmax_t size, s1, s2, ps;
-	char **av, **ap;
+	uintmax_t size, ps;
 	uint32_t *vsl_log_start;
 
-	if (l_arg == NULL)
-		l_arg = "";
+	fill = 1;
 
-	av = VAV_Parse(l_arg, NULL, ARGV_COMMA);
-	AN(av);
-	if (av[0] != NULL)
-		ARGV_ERR("\t-l ...: %s", av[0]);
-
-	ap = av + 1;
-
-	/* Size of SHMLOG */
-	if (*ap != NULL && **ap != '\0') {
-		q = VNUM_2bytes(*ap, &s1, 0);
-		if (q != NULL)
-			ARGV_ERR("\t-l[1] ...:  %s\n", q);
-	} else {
-		s1 = 80 * 1024 * 1024;
-	}
-	if (*ap != NULL)
-		ap++;
-
-	/* Size of space for other stuff */
-	if (*ap != NULL && **ap != '\0') {
-		q = VNUM_2bytes(*ap, &s2, 0);
-		if (q != NULL)
-			ARGV_ERR("\t-l[2] ...:  %s\n", q);
-	} else {
-		s2 = 1024 * 1024;
-	}
-	if (*ap != NULL)
-		ap++;
-
-	/* Fill or not ? */
-	if (*ap != NULL) {
-		if (**ap == '\0')
-			fill = 1;
-		else if (!strcmp(*ap, "-"))
-			fill = 0;
-		else if (!strcmp(*ap, "+"))
-			fill = 1;
-		else
-			ARGV_ERR("\t-l[3] ...:  Must be \"-\" or \"+\"\n");
-		ap++;
-	} else {
-		fill = 1;
-	}
-
-	if (*ap != NULL)
-		ARGV_ERR("\t-l ...:  Too many sub-args\n");
-
-	VAV_Free(av);
-
-	size = s1 + s2;
+	size = mgt_param.vsl_space + mgt_param.vsm_space;
 	ps = getpagesize();
 	size += ps - 1;
 	size &= ~(ps - 1);
@@ -326,7 +272,7 @@ mgt_SHM_Init(const char *l_arg)
 	AN(cache_param);
 	*cache_param = mgt_param;
 
-	vsl_log_start = VSM_Alloc(s1, VSL_CLASS, "", "");
+	vsl_log_start = VSM_Alloc(mgt_param.vsl_space, VSL_CLASS, "", "");
 	AN(vsl_log_start);
 	vsl_log_start[1] = VSL_ENDMARKER;
 	VWMB();



From geoff at varnish-cache.org  Mon Jan  9 20:52:40 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:40 +0100
Subject: [experimental-ims] 356d46a First part of VSM overhaul,
	this compiles and varnishd runs,
	but varnishapi and users do not work yet.
Message-ID: <E1RkMCu-0008Ky-6S@project.varnish-software.com>

commit 356d46a8fdb03f7179c757aa4d2160c238321f9d
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Fri Nov 18 22:18:06 2011 +0000

    First part of VSM overhaul, this compiles and varnishd runs,
    but varnishapi and users do not work yet.

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index bae2572..432e22b 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -853,7 +853,7 @@ int SES_Schedule(struct sess *sp);
 void VSL_Init(void);
 void *VSM_Alloc(unsigned size, const char *class, const char *type,
     const char *ident);
-void VSM_Free(const void *ptr);
+void VSM_Free(void *ptr);
 #ifdef VSL_ENDMARKER
 void VSL(enum VSL_tag_e tag, int id, const char *fmt, ...);
 void WSLR(struct worker *w, enum VSL_tag_e tag, int id, txt t);
diff --git a/bin/varnishd/cache/cache_shmlog.c b/bin/varnishd/cache/cache_shmlog.c
index e004896..63157c7 100644
--- a/bin/varnishd/cache/cache_shmlog.c
+++ b/bin/varnishd/cache/cache_shmlog.c
@@ -30,8 +30,10 @@
 #include "config.h"
 
 #include <stdio.h>
+#include <stdlib.h>
 
 #include "cache.h"
+#include "common/heritage.h"
 
 #include "cache_backend.h"	// For w->vbc
 
@@ -300,25 +302,32 @@ WSLB(struct worker *w, enum VSL_tag_e tag, const char *fmt, ...)
 void
 VSL_Init(void)
 {
-	struct VSM_chunk *vsc;
+	uint32_t *vsl_log_start;
 
 	AZ(pthread_mutex_init(&vsl_mtx, NULL));
 	AZ(pthread_mutex_init(&vsm_mtx, NULL));
 
-	VSM__Clean();
+	vsl_log_start = VSM_Alloc(cache_param->vsl_space, VSL_CLASS, "", "");
+	AN(vsl_log_start);
+	vsl_log_start[1] = VSL_ENDMARKER;
+	VWMB();
+	do
+		*vsl_log_start = random() & 0xffff;
+	while (*vsl_log_start == 0);
+	VWMB();
 
-	VSM_ITER(vsc)
-		if (!strcmp(vsc->class, VSL_CLASS))
-			break;
-	AN(vsc);
-	vsl_start = VSM_PTR(vsc);
-	vsl_end = VSM_NEXT(vsc);
+	vsl_start = vsl_log_start;
+	vsl_end = vsl_start + cache_param->vsl_space;
 	vsl_ptr = vsl_start + 1;
 
+	VSC_C_main = VSM_Alloc(sizeof *VSC_C_main,
+	    VSC_CLASS, VSC_TYPE_MAIN, "");
+	AN(VSC_C_main);
+
 	vsl_wrap();
-	VSM_head->starttime = (intmax_t)VTIM_real();
+	// VSM_head->starttime = (intmax_t)VTIM_real();
 	memset(VSC_C_main, 0, sizeof *VSC_C_main);
-	VSM_head->child_pid = getpid();
+	// VSM_head->child_pid = getpid();
 }
 
 /*--------------------------------------------------------------------*/
@@ -327,19 +336,19 @@ void *
 VSM_Alloc(unsigned size, const char *class, const char *type,
     const char *ident)
 {
-	void *p;
+	volatile void *p;
 
 	AZ(pthread_mutex_lock(&vsm_mtx));
-	p = VSM__Alloc(size, class, type, ident);
+	p = VSM_common_alloc(heritage.vsm, size, class, type, ident);
 	AZ(pthread_mutex_unlock(&vsm_mtx));
-	return (p);
+	return (TRUST_ME(p));
 }
 
 void
-VSM_Free(const void *ptr)
+VSM_Free(void *ptr)
 {
 
 	AZ(pthread_mutex_lock(&vsm_mtx));
-	VSM__Free(ptr);
+	VSM_common_free(heritage.vsm, ptr);
 	AZ(pthread_mutex_unlock(&vsm_mtx));
 }
diff --git a/bin/varnishd/common/common.h b/bin/varnishd/common/common.h
index 8e32ec9..a35e649 100644
--- a/bin/varnishd/common/common.h
+++ b/bin/varnishd/common/common.h
@@ -74,6 +74,13 @@ void mgt_child_inherit(int fd, const char *what);
 
 
 /* vsm.c */
+struct vsm_sc;
+struct vsm_sc *VSM_common_new(void *ptr, unsigned len);
+void *VSM_common_alloc(struct vsm_sc *sc, unsigned size,
+    const char *class, const char *type, const char *ident);
+void VSM_common_free(struct vsm_sc *sc, void *ptr);
+void VSM_common_delete(struct vsm_sc *sc);
+
 // extern struct VSM_head		*VSM_head;
 // extern const struct VSM_chunk	*vsm_end;
 
diff --git a/bin/varnishd/common/common_vsm.c b/bin/varnishd/common/common_vsm.c
index 5298381..682f542 100644
--- a/bin/varnishd/common/common_vsm.c
+++ b/bin/varnishd/common/common_vsm.c
@@ -27,28 +27,35 @@
  *
  * VSM stuff common to manager and child.
  *
- * We have three potential conflicts we need to lock against here:
+ * We have three potential conflicts we need to deal with:
  *
  * VSM-studying programs (varnishstat...) vs. everybody else
  *	The VSM studying programs only have read-only access to the VSM
  *	so everybody else must use memory barriers, stable storage and
  *	similar tricks to keep the VSM image in sync (long enough) for
  *	the studying programs.
+ *	It can not be prevented, and may indeed in some cases be
+ *	desirable for such programs to write to VSM, for instance to
+ *	zero counters.
+ *	Varnishd should never trust the integrity of VSM content.
  *
  * Manager process vs child process.
- *	Will only muck about in VSM when child process is not running
- *	Responsible for cleaning up any mess left behind by dying child.
+ *	The manager will create a fresh VSM for each child process launch
+ *	and not muck about with VSM while the child runs.  If the child
+ *	crashes, the panicstring will be evacuated and the VSM possibly
+ *	saved for debugging, and a new VSM created before the child is
+ *	started again.
  *
  * Child process threads
  *	Pthread locking necessary.
  *
- * XXX: not all of this is in place yet.
  */
 
 #include "config.h"
 
 #include <stdint.h>
 #include <stdio.h>
+#include <stdlib.h>
 #include <string.h>
 #include <unistd.h>
 
@@ -58,181 +65,267 @@
 #include "vmb.h"
 #include "vtim.h"
 
-/* These two come from beyond (mgt_shmem.c actually) */
-struct VSM_head		*VSM_head;
-const struct VSM_chunk	*vsm_end;
-
-static unsigned
-vsm_mark(void)
-{
-	unsigned seq;
-
-	seq = VSM_head->alloc_seq;
-	VSM_head->alloc_seq = 0;
-	VWMB();
-	return (seq);
-}
-
-static void
-vsm_release(unsigned seq)
-{
+/*--------------------------------------------------------------------*/
 
-	if (seq == 0)
-		return;
-	VWMB();
-	do
-		VSM_head->alloc_seq = ++seq;
-	while (VSM_head->alloc_seq == 0);
-}
+struct vsm_range {
+	unsigned 			magic;
+#define VSM_RANGE_MAGIC			0x8d30f14
+	VTAILQ_ENTRY(vsm_range)		list;
+	unsigned			off; 
+	unsigned			len;
+	double				cool;
+	struct VSM_chunk		*chunk;
+	void				*ptr;
+};
+
+struct vsm_sc {
+	unsigned 			magic;
+#define VSM_SC_MAGIC			0x8b83270d
+	char				*b;
+	unsigned			len;
+	struct VSM_head			*head;
+	VTAILQ_HEAD(,vsm_range)		r_used;
+	VTAILQ_HEAD(,vsm_range)		r_cooling;
+	VTAILQ_HEAD(,vsm_range)		r_free;
+	VTAILQ_HEAD(,vsm_range)		r_bogus;
+};
 
-/*--------------------------------------------------------------------*/
+/*--------------------------------------------------------------------
+ * The free list is sorted by size, which means that collapsing ranges
+ * on free becomes a multi-pass operation.
+ */
 
 static void
-vsm_cleanup(void)
+vsm_common_insert_free(struct vsm_sc *sc, struct vsm_range *vr)
 {
-	unsigned now = (unsigned)VTIM_mono();
-	struct VSM_chunk *sha, *sha2;
-	unsigned seq;
-
-	CHECK_OBJ_NOTNULL(VSM_head, VSM_HEAD_MAGIC);
-	VSM_ITER(sha) {
-		if (strcmp(sha->class, VSM_CLASS_COOL))
-			continue;
-		if (sha->state + VSM_COOL_TIME < now)
-			break;
-	}
-	if (sha == NULL)
-		return;
-	seq = vsm_mark();
-	/* First pass, free, and collapse with next if applicable */
-	VSM_ITER(sha) {
-		if (strcmp(sha->class, VSM_CLASS_COOL))
-			continue;
-		if (sha->state + VSM_COOL_TIME >= now)
-			continue;
-
-		bprintf(sha->class, "%s", VSM_CLASS_FREE);
-		bprintf(sha->type, "%s", "");
-		bprintf(sha->ident, "%s", "");
-		sha2 = VSM_NEXT(sha);
-		assert(sha2 <= vsm_end);
-		if (sha2 == vsm_end)
-			break;
-		CHECK_OBJ_NOTNULL(sha2, VSM_CHUNK_MAGIC);
-		if (!strcmp(sha2->class, VSM_CLASS_FREE)) {
-			sha->len += sha2->len;
-			memset(sha2, 0, sizeof *sha2);
+	struct vsm_range *vr2;
+
+	CHECK_OBJ_NOTNULL(sc, VSM_SC_MAGIC);
+	CHECK_OBJ_NOTNULL(vr, VSM_RANGE_MAGIC);
+
+	/* First try to see if we can collapse anything */
+	VTAILQ_FOREACH(vr2, &sc->r_free, list) {
+		if (vr2->off == vr->off + vr->len) {
+			vr2->off = vr->off;
+			vr2->len += vr->len;
+			FREE_OBJ(vr);
+			VTAILQ_REMOVE(&sc->r_free, vr2, list);
+			vsm_common_insert_free(sc, vr2);
+			return;
+		}
+		if (vr->off == vr2->off + vr2->len) {
+			vr2->len += vr->len;
+			FREE_OBJ(vr);
+			VTAILQ_REMOVE(&sc->r_free, vr2, list);
+			vsm_common_insert_free(sc, vr2);
+			return;
 		}
-		sha->state = 0;
 	}
-	/* Second pass, collaps with prev if applicable */
-	VSM_ITER(sha) {
-		if (strcmp(sha->class, VSM_CLASS_FREE))
-			continue;
-		sha2 = VSM_NEXT(sha);
-		assert(sha2 <= vsm_end);
-		if (sha2 == vsm_end)
-			break;
-		CHECK_OBJ_NOTNULL(sha2, VSM_CHUNK_MAGIC);
-		if (!strcmp(sha2->class, VSM_CLASS_FREE)) {
-			sha->len += sha2->len;
-			memset(sha2, 0, sizeof *sha2);
+	/* Insert in size order */
+	VTAILQ_FOREACH(vr2, &sc->r_free, list) {
+		if (vr2->len > vr->len) {
+			VTAILQ_INSERT_BEFORE(vr2, vr, list);
+			return;
 		}
 	}
-	vsm_release(seq);
+	/* At tail, if everything in the list is smaller */
+	VTAILQ_INSERT_TAIL(&sc->r_free, vr, list);
 }
 
-/*--------------------------------------------------------------------*/
+/*--------------------------------------------------------------------
+ * Initialize a new VSM segment
+ */
 
-void *
-VSM__Alloc(unsigned size, const char *class, const char *type, const char *ident)
+struct vsm_sc *
+VSM_common_new(void *p, unsigned l)
 {
-	struct VSM_chunk *sha, *sha2;
-	unsigned seq;
-
-	CHECK_OBJ_NOTNULL(VSM_head, VSM_HEAD_MAGIC);
-
-	vsm_cleanup();
+	struct vsm_sc *sc;
+	struct vsm_range *vr;
+
+	assert(PAOK(sizeof(struct VSM_chunk)));
+	assert(PAOK(p));
+	ALLOC_OBJ(sc, VSM_SC_MAGIC);
+	AN(sc);
+	VTAILQ_INIT(&sc->r_used);
+	VTAILQ_INIT(&sc->r_cooling);
+	VTAILQ_INIT(&sc->r_free);
+	VTAILQ_INIT(&sc->r_bogus);
+	sc->b = p;
+	sc->len = l;
+
+	sc->head = (void *)sc->b;
+	memset(TRUST_ME(sc->head), 0, sizeof *sc->head);
+	sc->head->magic = VSM_HEAD_MAGIC;
+	sc->head->hdrsize = sizeof *sc->head;
+	sc->head->shm_size = l;
+
+	ALLOC_OBJ(vr, VSM_RANGE_MAGIC);
+	AN(vr);
+	vr->off = PRNDUP(sizeof(*sc->head));
+	vr->len = l - vr->off;
+	VTAILQ_INSERT_TAIL(&sc->r_free, vr, list);
+	return (sc);
+}
 
-	/* Round up to pointersize */
-	size = RUP2(size, sizeof(void*));
+/*--------------------------------------------------------------------
+ * Allocate a chunk from VSM
+ */
 
-	size += sizeof *sha;		/* Make space for the header */
+void *
+VSM_common_alloc(struct vsm_sc *sc, unsigned size,
+    const char *class, const char *type, const char *ident)
+{
+	struct vsm_range *vr, *vr2, *vr3;
+	double now = VTIM_real();
+	unsigned l1, l2;
+
+	CHECK_OBJ_NOTNULL(sc, VSM_SC_MAGIC);
+	AN(size);
+
+	/* XXX: silent truncation instead of assert ? */
+	AN(class);
+	assert(strlen(class) < sizeof(vr->chunk->class));
+	AN(type);
+	assert(strlen(type) < sizeof(vr->chunk->type));
+	AN(ident);
+	assert(strlen(ident) < sizeof(vr->chunk->ident));
+
+	/* Move cooled off stuff to free list */
+	VTAILQ_FOREACH_SAFE(vr, &sc->r_cooling, list, vr2) {
+		if (vr->cool > now)
+			break;
+		VTAILQ_REMOVE(&sc->r_cooling, vr, list);
+		vsm_common_insert_free(sc, vr);
+	}
 
-	VSM_ITER(sha) {
-		CHECK_OBJ_NOTNULL(sha, VSM_CHUNK_MAGIC);
+	size = PRNDUP(size);
+	l1 = size + sizeof(struct VSM_chunk);
+	l2 = size + 2 * sizeof(struct VSM_chunk);
 
-		if (strcmp(sha->class, VSM_CLASS_FREE))
+	/* Find space in free-list */
+	VTAILQ_FOREACH_SAFE(vr, &sc->r_free, list, vr2) {
+		if (vr->len < l1)
 			continue;
+		if (vr->len <= l2) {
+			VTAILQ_REMOVE(&sc->r_free, vr, list);
+		} else {
+			ALLOC_OBJ(vr3, VSM_RANGE_MAGIC);
+			AN(vr3);
+			vr3->off = vr->off;
+			vr3->len = l1;
+			vr->off += l1;
+			vr->len -= l1;
+			VTAILQ_REMOVE(&sc->r_free, vr, list);
+			vsm_common_insert_free(sc, vr);
+			vr = vr3;
+		}
+		break;
+	}
 
-		if (size > sha->len)
-			continue;
+	if (vr == NULL) {
+		/*
+		 * No space in VSM, return malloc'd space
+		 */
+		ALLOC_OBJ(vr, VSM_RANGE_MAGIC);
+		AN(vr);
+		vr->ptr = malloc(size);
+		AN(vr->ptr);
+		VTAILQ_INSERT_TAIL(&sc->r_bogus, vr, list);
+		/* XXX: log + stats */
+		return (vr->ptr);
+	}
 
-		/* Mark as inconsistent while we write string fields */
-		seq = vsm_mark();
+	/* XXX: stats ? */
 
-		if (size + sizeof (*sha) < sha->len) {
-			sha2 = (void*)((uintptr_t)sha + size);
+	/* Zero the entire allocation, to avoid garbage confusing readers */
+	memset(TRUST_ME(sc->b + vr->off), 0, vr->len);
 
-			memset(sha2, 0, sizeof *sha2);
-			sha2->magic = VSM_CHUNK_MAGIC;
-			sha2->len = sha->len - size;
-			bprintf(sha2->class, "%s", VSM_CLASS_FREE);
-			sha->len = size;
-		}
+	vr->chunk = (void *)(sc->b + vr->off);
+	vr->ptr = (vr->chunk + 1);
+
+	vr->chunk->magic = VSM_CHUNK_MAGIC;
+	strcpy(TRUST_ME(vr->chunk->class), class);
+	strcpy(TRUST_ME(vr->chunk->type), type);
+	strcpy(TRUST_ME(vr->chunk->ident), ident);
+	VWMB();
 
-		bprintf(sha->class, "%s", class);
-		bprintf(sha->type, "%s", type);
-		bprintf(sha->ident, "%s", ident);
+	vr3 = VTAILQ_FIRST(&sc->r_used);
+	VTAILQ_INSERT_HEAD(&sc->r_used, vr, list);
 
-		vsm_release(seq);
-		return (VSM_PTR(sha));
+	if (vr3 != NULL) {	
+		AZ(vr3->chunk->next);
+		vr3->chunk->next = vr->off;
+	} else {
+		sc->head->first = vr->off;
 	}
-	return (NULL);
+	VWMB();
+	return (vr->ptr);
 }
 
-/*--------------------------------------------------------------------*/
+/*--------------------------------------------------------------------
+ * Free a chunk
+ */
 
 void
-VSM__Free(const void *ptr)
+VSM_common_free(struct vsm_sc *sc, void *ptr)
 {
-	struct VSM_chunk *sha;
-	unsigned seq;
+	struct vsm_range *vr, *vr2;
 
-	CHECK_OBJ_NOTNULL(VSM_head, VSM_HEAD_MAGIC);
-	VSM_ITER(sha)
-		if (VSM_PTR(sha) == ptr)
-			break;
-	AN(sha);
-	seq = vsm_mark();
-	bprintf(sha->class, "%s", VSM_CLASS_COOL);
-	sha->state = (unsigned)VTIM_mono();
-	vsm_release(seq);
+	CHECK_OBJ_NOTNULL(sc, VSM_SC_MAGIC);
+	AN(ptr);
+
+	/* Look in used list, move to cooling list */
+	VTAILQ_FOREACH(vr, &sc->r_used, list) {
+		if (vr->ptr != ptr)
+			continue;
+		/* XXX: stats ? */
+		vr2 = VTAILQ_NEXT(vr, list);
+		VTAILQ_REMOVE(&sc->r_used, vr, list);
+		VTAILQ_INSERT_TAIL(&sc->r_cooling, vr, list);
+		vr->cool = VTIM_real() + 60;	/* XXX: param ? */
+		if (vr2 != NULL)
+			vr2->chunk->next = vr->chunk->next;
+		else
+			sc->head->first = vr->chunk->next;
+		VWMB();
+		vr->chunk->len = 0;
+		VWMB();
+		return;
+	}
+	/* Look in bogus list, free */
+	VTAILQ_FOREACH(vr, &sc->r_bogus, list) {
+		if (vr->ptr == ptr) {
+			VTAILQ_REMOVE(&sc->r_bogus, vr, list);
+			FREE_OBJ(vr);
+			/* XXX: stats ? */
+			free(TRUST_ME(ptr));
+			return;
+		}
+	}
+	/* Panic */
+	assert(ptr == "Bogus pointer freed");
 }
 
 /*--------------------------------------------------------------------
- * Free all allocations after the mark (ie: allocated by child).
+ * Delete a VSM segment
  */
 
 void
-VSM__Clean(void)
+VSM_common_delete(struct vsm_sc *sc)
 {
-	struct VSM_chunk *sha;
-	unsigned f, seq;
-
-	CHECK_OBJ_NOTNULL(VSM_head, VSM_HEAD_MAGIC);
-	f = 0;
-	seq = vsm_mark();
-	VSM_ITER(sha) {
-		if (f == 0 && !strcmp(sha->class, VSM_CLASS_MARK)) {
-			f = 1;
-			continue;
-		}
-		if (f == 0)
-			continue;
-		if (strcmp(sha->class, VSM_CLASS_FREE) &&
-		    strcmp(sha->class, VSM_CLASS_COOL))
-			VSM__Free(VSM_PTR(sha));
+	struct vsm_range *vr, *vr2;
+
+	CHECK_OBJ_NOTNULL(sc, VSM_SC_MAGIC);
+	VTAILQ_FOREACH_SAFE(vr, &sc->r_free, list, vr2)
+		FREE_OBJ(vr);
+	VTAILQ_FOREACH_SAFE(vr, &sc->r_used, list, vr2)
+		FREE_OBJ(vr);
+	VTAILQ_FOREACH_SAFE(vr, &sc->r_cooling, list, vr2)
+		FREE_OBJ(vr);
+	VTAILQ_FOREACH_SAFE(vr, &sc->r_bogus, list, vr2) {
+		free(TRUST_ME(vr->ptr));
+		FREE_OBJ(vr);
 	}
-	vsm_release(seq);
+	sc->head->magic = 0;
+	FREE_OBJ(sc);
 }
diff --git a/bin/varnishd/common/heritage.h b/bin/varnishd/common/heritage.h
index 36433bb..db8b1f0 100644
--- a/bin/varnishd/common/heritage.h
+++ b/bin/varnishd/common/heritage.h
@@ -29,6 +29,8 @@
  * This file contains the heritage passed when mgt forks cache
  */
 
+struct vsm_sc;
+
 struct listen_sock {
 	unsigned			magic;
 #define LISTEN_SOCK_MAGIC		0x999e4b57
@@ -56,6 +58,8 @@ struct heritage {
 	/* Hash method */
 	const struct hash_slinger	*hash;
 
+	struct vsm_sc			*vsm;
+
 	char				*name;
 	char                            identity[1024];
 };
diff --git a/bin/varnishd/mgt/mgt.h b/bin/varnishd/mgt/mgt.h
index 0650413..571aa23 100644
--- a/bin/varnishd/mgt/mgt.h
+++ b/bin/varnishd/mgt/mgt.h
@@ -110,9 +110,8 @@ extern unsigned mgt_vcc_err_unref;
 		syslog(pri, fmt, __VA_ARGS__);		\
 	} while (0)
 
-#define VSM_Alloc(a, b, c, d)	VSM__Alloc(a,b,c,d)
-#define VSM_Free(a)		VSM__Free(a)
-#define VSM_Clean()		VSM__Clean()
+#define VSM_Alloc(a, b, c, d)	VSM_common_alloc(heritage.vsm, a,b,c,d)
+#define VSM_Free(a)		VSM_common_free(heritage.vsm, a)
 
 #if defined(PTHREAD_CANCELED) || defined(PTHREAD_MUTEX_DEFAULT)
 #error "Keep pthreads out of in manager process"
diff --git a/bin/varnishd/mgt/mgt_cli.c b/bin/varnishd/mgt/mgt_cli.c
index 1d46d82..aa4fffe 100644
--- a/bin/varnishd/mgt/mgt_cli.c
+++ b/bin/varnishd/mgt/mgt_cli.c
@@ -493,15 +493,15 @@ mgt_cli_secret(const char *S_arg)
 {
 	int i, fd;
 	char buf[BUFSIZ];
-	char *p;
+	volatile char *p;
 
 	/* Save in shmem */
 	i = strlen(S_arg);
 	p = VSM_Alloc(i + 1, "Arg", "-S", "");
 	AN(p);
-	strcpy(p, S_arg);
+	memcpy(TRUST_ME(p), S_arg, i + 1);
 
-	srandomdev();
+	srandomdev();			/* XXX: why here ??? */
 	fd = open(S_arg, O_RDONLY);
 	if (fd < 0) {
 		fprintf(stderr, "Can not open secret-file \"%s\"\n", S_arg);
@@ -527,7 +527,7 @@ mgt_cli_telnet(const char *T_arg)
 	struct vss_addr **ta;
 	int i, n, sock, good;
 	struct telnet *tn;
-	char *p;
+	volatile char *p;
 	struct vsb *vsb;
 	char abuf[VTCP_ADDRBUFSIZE];
 	char pbuf[VTCP_PORTBUFSIZE];
@@ -566,7 +566,7 @@ mgt_cli_telnet(const char *T_arg)
 	/* Save in shmem */
 	p = VSM_Alloc(VSB_len(vsb) + 1, "Arg", "-T", "");
 	AN(p);
-	strcpy(p, VSB_data(vsb));
+	memcpy(TRUST_ME(p), VSB_data(vsb), VSB_len(vsb) + 1);
 	VSB_delete(vsb);
 }
 
diff --git a/bin/varnishd/mgt/mgt_main.c b/bin/varnishd/mgt/mgt_main.c
index ff8992e..4e63c16 100644
--- a/bin/varnishd/mgt/mgt_main.c
+++ b/bin/varnishd/mgt/mgt_main.c
@@ -654,8 +654,6 @@ main(int argc, char * const *argv)
 	if (T_arg != NULL)
 		mgt_cli_telnet(T_arg);
 
-	AN(VSM_Alloc(0, VSM_CLASS_MARK, "", ""));
-
 	MGT_Run();
 
 	if (pfh != NULL)
diff --git a/bin/varnishd/mgt/mgt_shmem.c b/bin/varnishd/mgt/mgt_shmem.c
index d1bf18e..6095d9f 100644
--- a/bin/varnishd/mgt/mgt_shmem.c
+++ b/bin/varnishd/mgt/mgt_shmem.c
@@ -169,9 +169,8 @@ vsl_n_check(int fd)
  */
 
 static void
-vsl_buildnew(const char *fn, unsigned size, int fill)
+vsl_buildnew(const char *fn, ssize_t size)
 {
-	struct VSM_head slh;
 	int i;
 	unsigned u;
 	char buf[64*1024];
@@ -189,26 +188,16 @@ vsl_buildnew(const char *fn, unsigned size, int fill)
 	flags &= ~O_NONBLOCK;
 	AZ(fcntl(vsl_fd, F_SETFL, flags));
 
-	memset(&slh, 0, sizeof slh);
-	slh.magic = VSM_HEAD_MAGIC;
-	slh.hdrsize = sizeof slh;
-	slh.shm_size = size;
-	i = write(vsl_fd, &slh, sizeof slh);
-	xxxassert(i == sizeof slh);
-
-	if (fill) {
-		memset(buf, 0, sizeof buf);
-		for (u = sizeof slh; u < size; ) {
-			i = write(vsl_fd, buf, sizeof buf);
-			if (i <= 0) {
-				fprintf(stderr, "Write error %s: %s\n",
-				    fn, strerror(errno));
-				exit (1);
-			}
-			u += i;
+	memset(buf, 0, sizeof buf);
+	for (u = 0; u < size; ) {
+		i = write(vsl_fd, buf, sizeof buf);
+		if (i <= 0) {
+			fprintf(stderr, "Write error %s: %s\n",
+			    fn, strerror(errno));
+			exit (1);
 		}
+		u += i;
 	}
-
 	AZ(ftruncate(vsl_fd, (off_t)size));
 }
 
@@ -220,8 +209,10 @@ static
 void
 mgt_shm_atexit(void)
 {
+#if 0
 	if (getpid() == VSM_head->master_pid)
 		VSM_head->master_pid = 0;
+#endif
 }
 
 void
@@ -229,7 +220,10 @@ mgt_SHM_Init(void)
 {
 	int i, fill;
 	uintmax_t size, ps;
+	void *p;
+#if 0
 	uint32_t *vsl_log_start;
+#endif
 
 	fill = 1;
 
@@ -243,55 +237,49 @@ mgt_SHM_Init(void)
 		vsl_n_check(i);
 		(void)close(i);
 	}
-	vsl_buildnew(VSM_FILENAME, size, fill);
+	vsl_buildnew(VSM_FILENAME, size);
 
-	VSM_head = (void *)mmap(NULL, size,
+	p = (void *)mmap(NULL, size,
 	    PROT_READ|PROT_WRITE,
 	    MAP_HASSEMAPHORE | MAP_NOSYNC | MAP_SHARED,
 	    vsl_fd, 0);
-	VSM_head->master_pid = getpid();
-	AZ(atexit(mgt_shm_atexit));
-	xxxassert(VSM_head != MAP_FAILED);
-	(void)mlock((void*)VSM_head, size);
-
-	memset(&VSM_head->head, 0, sizeof VSM_head->head);
-	VSM_head->head.magic = VSM_CHUNK_MAGIC;
-	VSM_head->head.len =
-	    (uint8_t*)(VSM_head) + size - (uint8_t*)&VSM_head->head;
-	bprintf(VSM_head->head.class, "%s", VSM_CLASS_FREE);
-	VWMB();
+	xxxassert(p != MAP_FAILED);
 
-	vsm_end = (void*)((uint8_t*)VSM_head + size);
+	heritage.vsm = VSM_common_new(p, size);
 
-	VSC_C_main = VSM_Alloc(sizeof *VSC_C_main,
-	    VSC_CLASS, VSC_TYPE_MAIN, "");
-	AN(VSC_C_main);
+	(void)mlock(p, size);
+	AZ(atexit(mgt_shm_atexit));
 
 	/* XXX: We need to zero params if we dealloc/clean/wash */
 	cache_param = VSM_Alloc(sizeof *cache_param, VSM_CLASS_PARAM, "", "");
 	AN(cache_param);
 	*cache_param = mgt_param;
 
-	vsl_log_start = VSM_Alloc(mgt_param.vsl_space, VSL_CLASS, "", "");
-	AN(vsl_log_start);
-	vsl_log_start[1] = VSL_ENDMARKER;
-	VWMB();
-
 	PAN_panicstr_len = 64 * 1024;
 	PAN_panicstr = VSM_Alloc(PAN_panicstr_len, PAN_CLASS, "", "");
 	AN(PAN_panicstr);
-	/* XXX: shouldn't VSM_Alloc zero ? */
-	memset(PAN_panicstr, '\0', PAN_panicstr_len);
 
-	do
-		*vsl_log_start = random() & 0xffff;
-	while (*vsl_log_start == 0);
+#if 0
+
+	VSM_head->master_pid = getpid();
 
+	memset(&VSM_head->head, 0, sizeof VSM_head->head);
+	VSM_head->head.magic = VSM_CHUNK_MAGIC;
+	VSM_head->head.len =
+	    (uint8_t*)(VSM_head) + size - (uint8_t*)&VSM_head->head;
+	bprintf(VSM_head->head.class, "%s", VSM_CLASS_FREE);
 	VWMB();
 
+	vsm_end = (void*)((uint8_t*)VSM_head + size);
+
+	VSC_C_main = VSM_Alloc(sizeof *VSC_C_main,
+	    VSC_CLASS, VSC_TYPE_MAIN, "");
+	AN(VSC_C_main);
+
 	do
 		VSM_head->alloc_seq = random();
 	while (VSM_head->alloc_seq == 0);
+#endif
 
 }
 
@@ -299,5 +287,7 @@ void
 mgt_SHM_Pid(void)
 {
 
+#if 0
 	VSM_head->master_pid = getpid();
+#endif
 }
diff --git a/include/vapi/vsm_int.h b/include/vapi/vsm_int.h
index 521f89d..cad1e36 100644
--- a/include/vapi/vsm_int.h
+++ b/include/vapi/vsm_int.h
@@ -41,10 +41,11 @@
  */
 
 struct VSM_chunk {
-#define VSM_CHUNK_MAGIC		0x43907b6e	/* From /dev/random */
+#define VSM_CHUNK_MAGIC		0xa15712e5	/* From /dev/random */
 	unsigned		magic;
-	unsigned		len;
-	unsigned		state;
+	unsigned		len;		/* Incl VSM_chunk */
+	unsigned		next;		/* Offset in shmem */
+	unsigned		state;		/* XXX remove */
 	char			class[8];
 	char			type[8];
 	char			ident[64];
@@ -54,10 +55,11 @@ struct VSM_chunk {
 #define VSM_PTR(sha)		((void*)((uintptr_t)((sha) + 1)))
 
 struct VSM_head {
-#define VSM_HEAD_MAGIC		4185512502U	/* From /dev/random */
+#define VSM_HEAD_MAGIC		0xe75f7e91	/* From /dev/random */
 	unsigned		magic;
 
 	unsigned		hdrsize;
+	unsigned		first;		/* Offset, first chunk */
 
 	uint64_t		starttime;
 	int64_t			master_pid;



From geoff at varnish-cache.org  Mon Jan  9 20:52:40 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:40 +0100
Subject: [experimental-ims] e4d1d36 Polish round over varnishd part of VSM
Message-ID: <E1RkMCu-0008LA-91@project.varnish-software.com>

commit e4d1d360beca6917fe2ee1f866ca1ed71302b639
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sun Nov 20 12:44:19 2011 +0000

    Polish round over varnishd part of VSM

diff --git a/bin/varnishd/cache/cache_shmlog.c b/bin/varnishd/cache/cache_shmlog.c
index 63157c7..950f57f 100644
--- a/bin/varnishd/cache/cache_shmlog.c
+++ b/bin/varnishd/cache/cache_shmlog.c
@@ -37,9 +37,7 @@
 
 #include "cache_backend.h"	// For w->vbc
 
-#include "vapi/vsm_int.h"
 #include "vmb.h"
-#include "vtim.h"
 
 /* These cannot be struct lock, which depends on vsm/vsl working */
 static pthread_mutex_t vsl_mtx;
diff --git a/bin/varnishd/common/common.h b/bin/varnishd/common/common.h
index a35e649..a5d7633 100644
--- a/bin/varnishd/common/common.h
+++ b/bin/varnishd/common/common.h
@@ -75,30 +75,11 @@ void mgt_child_inherit(int fd, const char *what);
 
 /* vsm.c */
 struct vsm_sc;
-struct vsm_sc *VSM_common_new(void *ptr, unsigned len);
-void *VSM_common_alloc(struct vsm_sc *sc, unsigned size,
+struct vsm_sc *VSM_common_new(void *ptr, ssize_t len);
+void *VSM_common_alloc(struct vsm_sc *sc, ssize_t size,
     const char *class, const char *type, const char *ident);
 void VSM_common_free(struct vsm_sc *sc, void *ptr);
-void VSM_common_delete(struct vsm_sc *sc);
-
-// extern struct VSM_head		*VSM_head;
-// extern const struct VSM_chunk	*vsm_end;
-
-/*
- * These three should not be called directly, but only through
- * proper vectors in mgt.h/cache.h, hence the __
- */
-void *VSM__Alloc(unsigned size, const char *class, const char *type,
-    const char *ident);
-void VSM__Free(const void *ptr);
-void VSM__Clean(void);
-
-/* These classes are opaque to other programs, so we define the here */
-#define VSM_CLASS_FREE	"Free"
-#define VSM_CLASS_COOL	"Cool"
-#define VSM_CLASS_PARAM	"Params"
-#define VSM_CLASS_MARK	"MgrCld"
-#define VSM_COOL_TIME	5
+void VSM_common_delete(struct vsm_sc **sc);
 
 /*---------------------------------------------------------------------
  * Generic power-2 rounding macros
diff --git a/bin/varnishd/common/common_vsm.c b/bin/varnishd/common/common_vsm.c
index 682f542..f525a1e 100644
--- a/bin/varnishd/common/common_vsm.c
+++ b/bin/varnishd/common/common_vsm.c
@@ -27,27 +27,8 @@
  *
  * VSM stuff common to manager and child.
  *
- * We have three potential conflicts we need to deal with:
- *
- * VSM-studying programs (varnishstat...) vs. everybody else
- *	The VSM studying programs only have read-only access to the VSM
- *	so everybody else must use memory barriers, stable storage and
- *	similar tricks to keep the VSM image in sync (long enough) for
- *	the studying programs.
- *	It can not be prevented, and may indeed in some cases be
- *	desirable for such programs to write to VSM, for instance to
- *	zero counters.
- *	Varnishd should never trust the integrity of VSM content.
- *
- * Manager process vs child process.
- *	The manager will create a fresh VSM for each child process launch
- *	and not muck about with VSM while the child runs.  If the child
- *	crashes, the panicstring will be evacuated and the VSM possibly
- *	saved for debugging, and a new VSM created before the child is
- *	started again.
- *
- * Child process threads
- *	Pthread locking necessary.
+ * Please see comments in <vapi/vsm_int.h> for details of protocols and
+ * data consistency.
  *
  */
 
@@ -71,8 +52,8 @@ struct vsm_range {
 	unsigned 			magic;
 #define VSM_RANGE_MAGIC			0x8d30f14
 	VTAILQ_ENTRY(vsm_range)		list;
-	unsigned			off; 
-	unsigned			len;
+	ssize_t				off; 
+	ssize_t				len;
 	double				cool;
 	struct VSM_chunk		*chunk;
 	void				*ptr;
@@ -82,7 +63,7 @@ struct vsm_sc {
 	unsigned 			magic;
 #define VSM_SC_MAGIC			0x8b83270d
 	char				*b;
-	unsigned			len;
+	ssize_t				len;
 	struct VSM_head			*head;
 	VTAILQ_HEAD(,vsm_range)		r_used;
 	VTAILQ_HEAD(,vsm_range)		r_cooling;
@@ -137,7 +118,7 @@ vsm_common_insert_free(struct vsm_sc *sc, struct vsm_range *vr)
  */
 
 struct vsm_sc *
-VSM_common_new(void *p, unsigned l)
+VSM_common_new(void *p, ssize_t l)
 {
 	struct vsm_sc *sc;
 	struct vsm_range *vr;
@@ -154,10 +135,13 @@ VSM_common_new(void *p, unsigned l)
 	sc->len = l;
 
 	sc->head = (void *)sc->b;
-	memset(TRUST_ME(sc->head), 0, sizeof *sc->head);
+	/* This should not be necessary, but just in case...*/
+	memset(sc->head, 0, sizeof *sc->head);
 	sc->head->magic = VSM_HEAD_MAGIC;
 	sc->head->hdrsize = sizeof *sc->head;
 	sc->head->shm_size = l;
+	sc->head->alloc_seq = random() | 1;
+	VWMB();
 
 	ALLOC_OBJ(vr, VSM_RANGE_MAGIC);
 	AN(vr);
@@ -172,7 +156,7 @@ VSM_common_new(void *p, unsigned l)
  */
 
 void *
-VSM_common_alloc(struct vsm_sc *sc, unsigned size,
+VSM_common_alloc(struct vsm_sc *sc, ssize_t size,
     const char *class, const char *type, const char *ident)
 {
 	struct vsm_range *vr, *vr2, *vr3;
@@ -238,15 +222,15 @@ VSM_common_alloc(struct vsm_sc *sc, unsigned size,
 	/* XXX: stats ? */
 
 	/* Zero the entire allocation, to avoid garbage confusing readers */
-	memset(TRUST_ME(sc->b + vr->off), 0, vr->len);
+	memset(sc->b + vr->off, 0, vr->len);
 
 	vr->chunk = (void *)(sc->b + vr->off);
 	vr->ptr = (vr->chunk + 1);
 
 	vr->chunk->magic = VSM_CHUNK_MAGIC;
-	strcpy(TRUST_ME(vr->chunk->class), class);
-	strcpy(TRUST_ME(vr->chunk->type), type);
-	strcpy(TRUST_ME(vr->chunk->ident), ident);
+	strcpy(vr->chunk->class, class);
+	strcpy(vr->chunk->type, type);
+	strcpy(vr->chunk->ident, ident);
 	VWMB();
 
 	vr3 = VTAILQ_FIRST(&sc->r_used);
@@ -258,6 +242,7 @@ VSM_common_alloc(struct vsm_sc *sc, unsigned size,
 	} else {
 		sc->head->first = vr->off;
 	}
+	sc->head->alloc_seq += 2;
 	VWMB();
 	return (vr->ptr);
 }
@@ -289,6 +274,7 @@ VSM_common_free(struct vsm_sc *sc, void *ptr)
 			sc->head->first = vr->chunk->next;
 		VWMB();
 		vr->chunk->len = 0;
+		sc->head->alloc_seq += 2;
 		VWMB();
 		return;
 	}
@@ -298,7 +284,7 @@ VSM_common_free(struct vsm_sc *sc, void *ptr)
 			VTAILQ_REMOVE(&sc->r_bogus, vr, list);
 			FREE_OBJ(vr);
 			/* XXX: stats ? */
-			free(TRUST_ME(ptr));
+			free(ptr);
 			return;
 		}
 	}
@@ -311,9 +297,14 @@ VSM_common_free(struct vsm_sc *sc, void *ptr)
  */
 
 void
-VSM_common_delete(struct vsm_sc *sc)
+VSM_common_delete(struct vsm_sc **scp)
 {
 	struct vsm_range *vr, *vr2;
+	struct vsm_sc *sc;
+
+	AN(scp);
+	sc =*scp;
+	*scp = NULL;
 
 	CHECK_OBJ_NOTNULL(sc, VSM_SC_MAGIC);
 	VTAILQ_FOREACH_SAFE(vr, &sc->r_free, list, vr2)
@@ -323,9 +314,10 @@ VSM_common_delete(struct vsm_sc *sc)
 	VTAILQ_FOREACH_SAFE(vr, &sc->r_cooling, list, vr2)
 		FREE_OBJ(vr);
 	VTAILQ_FOREACH_SAFE(vr, &sc->r_bogus, list, vr2) {
-		free(TRUST_ME(vr->ptr));
+		free(vr->ptr);
 		FREE_OBJ(vr);
 	}
-	sc->head->magic = 0;
+	sc->head->alloc_seq = 0;
+	VWMB();
 	FREE_OBJ(sc);
 }
diff --git a/bin/varnishd/common/params.h b/bin/varnishd/common/params.h
index 95258fb..cab2f49 100644
--- a/bin/varnishd/common/params.h
+++ b/bin/varnishd/common/params.h
@@ -31,6 +31,8 @@
 
 #include "vre.h"
 
+#define VSM_CLASS_PARAM		"Params"
+
 struct params {
 
 	/* Unprivileged user / group */
diff --git a/bin/varnishd/mgt/mgt.h b/bin/varnishd/mgt/mgt.h
index 571aa23..775fe2c 100644
--- a/bin/varnishd/mgt/mgt.h
+++ b/bin/varnishd/mgt/mgt.h
@@ -82,7 +82,6 @@ void mgt_sandbox_solaris_privsep(void);
 
 /* mgt_shmem.c */
 void mgt_SHM_Init(void);
-void mgt_SHM_Pid(void);
 
 /* stevedore_mgt.c */
 void STV_Config(const char *spec);
diff --git a/bin/varnishd/mgt/mgt_cli.c b/bin/varnishd/mgt/mgt_cli.c
index aa4fffe..d3b7396 100644
--- a/bin/varnishd/mgt/mgt_cli.c
+++ b/bin/varnishd/mgt/mgt_cli.c
@@ -493,13 +493,13 @@ mgt_cli_secret(const char *S_arg)
 {
 	int i, fd;
 	char buf[BUFSIZ];
-	volatile char *p;
+	char *p;
 
 	/* Save in shmem */
 	i = strlen(S_arg);
-	p = VSM_Alloc(i + 1, "Arg", "-S", "");
+	p = VSM_Alloc(i + 1L, "Arg", "-S", "");
 	AN(p);
-	memcpy(TRUST_ME(p), S_arg, i + 1);
+	memcpy(p, S_arg, i + 1L);
 
 	srandomdev();			/* XXX: why here ??? */
 	fd = open(S_arg, O_RDONLY);
@@ -527,7 +527,7 @@ mgt_cli_telnet(const char *T_arg)
 	struct vss_addr **ta;
 	int i, n, sock, good;
 	struct telnet *tn;
-	volatile char *p;
+	char *p;
 	struct vsb *vsb;
 	char abuf[VTCP_ADDRBUFSIZE];
 	char pbuf[VTCP_PORTBUFSIZE];
@@ -566,7 +566,7 @@ mgt_cli_telnet(const char *T_arg)
 	/* Save in shmem */
 	p = VSM_Alloc(VSB_len(vsb) + 1, "Arg", "-T", "");
 	AN(p);
-	memcpy(TRUST_ME(p), VSB_data(vsb), VSB_len(vsb) + 1);
+	memcpy(p, VSB_data(vsb), VSB_len(vsb) + 1);
 	VSB_delete(vsb);
 }
 
diff --git a/bin/varnishd/mgt/mgt_main.c b/bin/varnishd/mgt/mgt_main.c
index 4e63c16..f782d03 100644
--- a/bin/varnishd/mgt/mgt_main.c
+++ b/bin/varnishd/mgt/mgt_main.c
@@ -630,8 +630,6 @@ main(int argc, char * const *argv)
 	if (!d_flag && !F_flag)
 		AZ(varnish_daemon(1, 0));
 
-	mgt_SHM_Pid();
-
 	if (pfh != NULL && VPF_Write(pfh))
 		fprintf(stderr, "NOTE: Could not write PID file\n");
 
diff --git a/bin/varnishd/mgt/mgt_pool.c b/bin/varnishd/mgt/mgt_pool.c
index e8c4f27..badb3f4 100644
--- a/bin/varnishd/mgt/mgt_pool.c
+++ b/bin/varnishd/mgt/mgt_pool.c
@@ -48,7 +48,6 @@
 #include <unistd.h>
 
 #include "mgt/mgt.h"
-#include "common/heritage.h"
 #include "common/params.h"
 
 #include "mgt/mgt_param.h"
diff --git a/bin/varnishd/mgt/mgt_sandbox.c b/bin/varnishd/mgt/mgt_sandbox.c
index cdba825..3cd1d98 100644
--- a/bin/varnishd/mgt/mgt_sandbox.c
+++ b/bin/varnishd/mgt/mgt_sandbox.c
@@ -53,7 +53,6 @@
 #include <unistd.h>
 
 #include "mgt/mgt.h"
-#include "common/heritage.h"
 #include "common/params.h"
 
 /*--------------------------------------------------------------------*/
diff --git a/bin/varnishd/mgt/mgt_shmem.c b/bin/varnishd/mgt/mgt_shmem.c
index 6095d9f..6bb8e13 100644
--- a/bin/varnishd/mgt/mgt_shmem.c
+++ b/bin/varnishd/mgt/mgt_shmem.c
@@ -90,7 +90,6 @@
 #include <sys/stat.h>
 
 #include <fcntl.h>
-#include <signal.h>
 #include <stdint.h>
 #include <stdio.h>
 #include <stdlib.h>
@@ -103,7 +102,6 @@
 
 #include "flopen.h"
 #include "vapi/vsc_int.h"
-#include "vapi/vsl_int.h"
 #include "vapi/vsm_int.h"
 #include "vmb.h"
 
@@ -117,88 +115,96 @@
 
 struct VSC_C_main	*VSC_C_main;
 
-static int vsl_fd = -1;
+static int vsm_fd = -1;
 
 /*--------------------------------------------------------------------
  * Check that we are not started with the same -n argument as an already
- * running varnishd
+ * running varnishd.
+ *
+ * Non-zero return means we should exit and not trample the file.
+ *
  */
 
-static void
-vsl_n_check(int fd)
+static int
+vsm_n_check(void)
 {
-	struct VSM_head slh;
-	int i;
+	int fd, i;
 	struct stat st;
 	pid_t pid;
+	struct VSM_head vsmh;
+	int retval = 2;
 
-	AZ(fstat(fd, &st));
-	if (!S_ISREG(st.st_mode))
-		ARGV_ERR("\tshmlog: Not a file\n");
-
-	/* Test if the SHMFILE is locked by other Varnish */
-	if (fltest(fd, &pid) > 0) {
-		fprintf(stderr,
-			"SHMFILE locked by running varnishd master (pid=%jd)\n",
-			(intmax_t)pid);
-		fprintf(stderr,
-			"(Use unique -n arguments if you want multiple "
-			"instances)\n");
-		exit(2);
-	}
+	fd = open(VSM_FILENAME, O_RDWR, 0644);
+	if (fd < 0)
+		return (0);
 
-	/* Read owning pid from SHMFILE */
-	memset(&slh, 0, sizeof slh);	/* XXX: for flexelint */
-	i = read(fd, &slh, sizeof slh);
-	if (i != sizeof slh)
-		return;
-	if (slh.magic != VSM_HEAD_MAGIC)
-		return;
-	if (slh.hdrsize != sizeof slh)
-		return;
-	if (slh.master_pid != 0 && !kill(slh.master_pid, 0)) {
+	AZ(fstat(fd, &st));
+	if (!S_ISREG(st.st_mode)) {
 		fprintf(stderr,
-			"WARNING: Taking over SHMFILE marked as owned by "
-			"running process (pid=%jd)\n",
-			(intmax_t)slh.master_pid);
+		    "VSM (%s) not a regular file.\n", VSM_FILENAME);
+	} else {
+		i = fltest(fd, &pid);
+		if (i < 0) {
+			fprintf(stderr,
+			    "Cannot determine locking status of VSM (%s)\n.",
+			    VSM_FILENAME);
+		} else if (i == 0) {
+			/*
+			 * File is unlocked, mark it as dead, to help any
+			 * consumers still stuck on it.
+			 */
+			if (pread(fd, &vsmh, sizeof vsmh, 0) == sizeof vsmh) {
+				vsmh.alloc_seq = 0;
+				(void)pwrite(fd, &vsmh, sizeof vsmh, 0);
+			}
+			retval = 0;
+		} else {
+			/* The VSM is locked, we won't touch it. */
+			fprintf(stderr,
+			    "VSM locked by running varnishd master (pid=%jd)\n"
+			    "(Use unique -n arguments if you want"
+			    "  multiple instances)\n", (intmax_t)pid);
+		}
 	}
+	(void)close(fd);
+	return (retval);
 }
 
 /*--------------------------------------------------------------------
- * Build a new shmlog file
+ * Build a zeroed file
  */
 
-static void
-vsl_buildnew(const char *fn, ssize_t size)
+static int
+vsm_zerofile(const char *fn, ssize_t size)
 {
-	int i;
-	unsigned u;
+	int fd;
+	ssize_t i, u;
 	char buf[64*1024];
 	int flags;
 
-	(void)unlink(fn);
-	vsl_fd = flopen(fn, O_RDWR | O_CREAT | O_EXCL | O_NONBLOCK, 0644);
-	if (vsl_fd < 0) {
+	fd = flopen(fn, O_RDWR | O_CREAT | O_EXCL | O_NONBLOCK, 0644);
+	if (fd < 0) {
 		fprintf(stderr, "Could not create %s: %s\n",
 		    fn, strerror(errno));
-		exit (1);
+		return (-1);
 	}
-	flags = fcntl(vsl_fd, F_GETFL);
+	flags = fcntl(fd, F_GETFL);
 	assert(flags != -1);
 	flags &= ~O_NONBLOCK;
-	AZ(fcntl(vsl_fd, F_SETFL, flags));
+	AZ(fcntl(fd, F_SETFL, flags));
 
 	memset(buf, 0, sizeof buf);
 	for (u = 0; u < size; ) {
-		i = write(vsl_fd, buf, sizeof buf);
+		i = write(fd, buf, sizeof buf);
 		if (i <= 0) {
 			fprintf(stderr, "Write error %s: %s\n",
 			    fn, strerror(errno));
-			exit (1);
+			return (-1);
 		}
 		u += i;
 	}
-	AZ(ftruncate(vsl_fd, (off_t)size));
+	AZ(ftruncate(fd, (off_t)size));
+	return (fd);
 }
 
 /*--------------------------------------------------------------------
@@ -209,48 +215,59 @@ static
 void
 mgt_shm_atexit(void)
 {
-#if 0
-	if (getpid() == VSM_head->master_pid)
-		VSM_head->master_pid = 0;
-#endif
+
+	if (heritage.vsm != NULL)
+		VSM_common_delete(&heritage.vsm);
 }
 
 void
 mgt_SHM_Init(void)
 {
-	int i, fill;
+	int i;
 	uintmax_t size, ps;
 	void *p;
-#if 0
-	uint32_t *vsl_log_start;
-#endif
-
-	fill = 1;
+	char fnbuf[64];
 
 	size = mgt_param.vsl_space + mgt_param.vsm_space;
 	ps = getpagesize();
 	size += ps - 1;
 	size &= ~(ps - 1);
 
-	i = open(VSM_FILENAME, O_RDWR, 0644);
-	if (i >= 0) {
-		vsl_n_check(i);
-		(void)close(i);
-	}
-	vsl_buildnew(VSM_FILENAME, size);
+	/* Collision check with already running varnishd */
+	i = vsm_n_check();
+	if (i)
+		exit(i);
+
+	bprintf(fnbuf, "%s.%jd", VSM_FILENAME, (intmax_t)getpid());
+
+	vsm_fd = vsm_zerofile(fnbuf, size);
+	if (vsm_fd < 0) 
+		exit(1);
 
 	p = (void *)mmap(NULL, size,
 	    PROT_READ|PROT_WRITE,
 	    MAP_HASSEMAPHORE | MAP_NOSYNC | MAP_SHARED,
-	    vsl_fd, 0);
-	xxxassert(p != MAP_FAILED);
+	    vsm_fd, 0);
 
-	heritage.vsm = VSM_common_new(p, size);
+	if (p == MAP_FAILED) {
+		fprintf(stderr, "Mmap error %s: %s\n", fnbuf, strerror(errno));
+		exit (-1);
+	}
 
+	/* This may or may not work */
 	(void)mlock(p, size);
+
+	heritage.vsm = VSM_common_new(p, size);
+
+	if (rename(fnbuf, VSM_FILENAME)) {
+		fprintf(stderr, "Rename failed %s -> %s: %s\n",
+		    fnbuf, VSM_FILENAME, strerror(errno));
+		(void)unlink(fnbuf);
+		exit (-1);
+	}
+
 	AZ(atexit(mgt_shm_atexit));
 
-	/* XXX: We need to zero params if we dealloc/clean/wash */
 	cache_param = VSM_Alloc(sizeof *cache_param, VSM_CLASS_PARAM, "", "");
 	AN(cache_param);
 	*cache_param = mgt_param;
@@ -258,36 +275,4 @@ mgt_SHM_Init(void)
 	PAN_panicstr_len = 64 * 1024;
 	PAN_panicstr = VSM_Alloc(PAN_panicstr_len, PAN_CLASS, "", "");
 	AN(PAN_panicstr);
-
-#if 0
-
-	VSM_head->master_pid = getpid();
-
-	memset(&VSM_head->head, 0, sizeof VSM_head->head);
-	VSM_head->head.magic = VSM_CHUNK_MAGIC;
-	VSM_head->head.len =
-	    (uint8_t*)(VSM_head) + size - (uint8_t*)&VSM_head->head;
-	bprintf(VSM_head->head.class, "%s", VSM_CLASS_FREE);
-	VWMB();
-
-	vsm_end = (void*)((uint8_t*)VSM_head + size);
-
-	VSC_C_main = VSM_Alloc(sizeof *VSC_C_main,
-	    VSC_CLASS, VSC_TYPE_MAIN, "");
-	AN(VSC_C_main);
-
-	do
-		VSM_head->alloc_seq = random();
-	while (VSM_head->alloc_seq == 0);
-#endif
-
-}
-
-void
-mgt_SHM_Pid(void)
-{
-
-#if 0
-	VSM_head->master_pid = getpid();
-#endif
 }
diff --git a/include/vapi/vsm.h b/include/vapi/vsm.h
index fd7fbee..275f6d9 100644
--- a/include/vapi/vsm.h
+++ b/include/vapi/vsm.h
@@ -32,8 +32,16 @@
 #define VAPI_VSM_H_INCLUDED
 
 struct VSM_head;
+struct VSM_chunk;
 struct VSM_data;
 
+struct VSM_fantom {
+	struct VSM_chunk	*chunk;
+	void			*b;
+	void			*e;
+	uintptr_t		priv;
+};
+
 /*---------------------------------------------------------------------
  * VSM level access functions
  */
@@ -46,6 +54,7 @@ struct VSM_data *VSM_New(void);
 	 * referencing the same or different shared memory files.
 	 * Returns:
 	 *	Pointer to usable VSL_data handle.
+	 *	NULL: malloc failed.
 	 */
 
 typedef void VSM_diag_f(void *priv, const char *fmt, ...);
@@ -66,7 +75,7 @@ int VSM_n_Arg(struct VSM_data *vd, const char *n_arg);
 	 * and VSC_Arg() functions.
 	 * Returns:
 	 *	 1 on success
-	 *	 -1 on failure, with diagnostic on stderr.
+	 *	 -1 on failure, with diagnostic.
 	 */
 
 const char *VSM_Name(const struct VSM_data *vd);
diff --git a/include/vapi/vsm_int.h b/include/vapi/vsm_int.h
index cad1e36..c33ba35 100644
--- a/include/vapi/vsm_int.h
+++ b/include/vapi/vsm_int.h
@@ -29,6 +29,67 @@
  * Define the layout of the shared memory log segment.
  *
  * NB: THIS IS NOT A PUBLIC API TO VARNISH!
+ *
+ * There is a lot of diplomacy and protocol involved with the VSM segment
+ * since there is no way to (and no desire to!) lock between the readers
+ * and the writer.
+ *
+ * In particular we want the readers to seamlessly jump from one VSM instance
+ * to another when the child restarts.
+ *
+ * The VSM life-cycle there is:
+ *
+ *	Manager creates VSM file under temp name
+ *
+ *	Temp VSM file is initialized such that VSM_head is consistent
+ *	with a non-zero alloc_seq
+ *
+ *	Manager renames Temp VSM file to correct filename as atomic
+ *	operation.
+ *
+ *	When manager abandons VSM file, alloc_seq is set to zero, which
+ *	never happens in any other circumstances.
+ *
+ *	If a manager is started and finds and old abandonned VSM segment
+ *	it will zero the alloc_seq in it, before replacing the file.
+ *
+ * Subscribers will have to monitor two things to make sure they have
+ * the current VSM instance:  The alloc_seq field and the inode number
+ * of the path-name.  The former check is by far the cheaper and the
+ * latter check should only be employed when lack of activity in the
+ * VSM segment raises suspicion that something has happened.
+ *
+ * The allocations ("chunks") in the VSM forms a linked list, starting with
+ * VSM_head->first, with the first/next fields being byte offsets relative
+ * to the start of the VSM segment.
+ *
+ * The last chunk on the list, has next == 0.
+ *
+ * New chunks are appended to the list, no matter where in the VSM
+ * they happen to be allocated.
+ *
+ * Chunk allocation sequence is:
+ *	Find free space
+ *	Zero payload
+ *	Init Chunk header
+ *	Write memory barrier
+ *	update hdr->first or $last->next pointer
+ *	hdr->alloc_seq changes
+ *	Write memory barrier
+ *
+ * Chunk contents should be designed so that zero bytes are not mistaken
+ * for valid contents.
+ *
+ * Chunk deallocation sequence is:
+ *	update hdr->first or $prev->next pointer
+ *	Write memory barrier
+ *	this->len = 0
+ *	hdr->alloc_seq changes
+ *	Write memory barrier
+ *
+ * The space occupied by the chunk is put on a cooling list and is not
+ * recycled for at least a minute.
+ *
  */
 
 #ifndef VSM_INT_H_INCLUDED
@@ -36,80 +97,23 @@
 
 #define VSM_FILENAME		"_.vsm"
 
-/*
- * This structure describes each allocation from the shmlog
- */
-
 struct VSM_chunk {
 #define VSM_CHUNK_MAGIC		0xa15712e5	/* From /dev/random */
 	unsigned		magic;
-	unsigned		len;		/* Incl VSM_chunk */
-	unsigned		next;		/* Offset in shmem */
-	unsigned		state;		/* XXX remove */
+	ssize_t			len;		/* Incl VSM_chunk */
+	ssize_t			next;		/* Offset in shmem */
 	char			class[8];
 	char			type[8];
 	char			ident[64];
 };
 
-#define VSM_NEXT(sha)		((void*)((uintptr_t)(sha) + (sha)->len))
-#define VSM_PTR(sha)		((void*)((uintptr_t)((sha) + 1)))
-
 struct VSM_head {
 #define VSM_HEAD_MAGIC		0xe75f7e91	/* From /dev/random */
 	unsigned		magic;
-
-	unsigned		hdrsize;
-	unsigned		first;		/* Offset, first chunk */
-
-	uint64_t		starttime;
-	int64_t			master_pid;
-	int64_t			child_pid;
-
-	unsigned		shm_size;
-
+	ssize_t			hdrsize;
+	ssize_t			shm_size;
+	ssize_t			first;		/* Offset, first chunk */
 	unsigned		alloc_seq;
-	/* Must be last element */
-	struct VSM_chunk	head;
 };
 
-/*
- * You must include "miniobj.h" and have an assert function to be
- * able to use the VSM_ITER() macro.
- */
-#ifdef CHECK_OBJ_NOTNULL
-
-extern struct VSM_head *VSM_head;
-extern const struct VSM_chunk *vsm_end;
-
-static inline struct VSM_chunk *
-vsm_iter_0(void)
-{
-
-	CHECK_OBJ_NOTNULL(VSM_head, VSM_HEAD_MAGIC);
-	CHECK_OBJ_NOTNULL(&VSM_head->head, VSM_CHUNK_MAGIC);
-	return (&VSM_head->head);
-}
-
-static inline void
-vsm_iter_n(struct VSM_chunk **pp)
-{
-
-	CHECK_OBJ_NOTNULL(VSM_head, VSM_HEAD_MAGIC);
-	CHECK_OBJ_NOTNULL(*pp, VSM_CHUNK_MAGIC);
-	*pp = VSM_NEXT(*pp);
-	if (*pp >= vsm_end) {
-		*pp = NULL;
-		return;
-	}
-	CHECK_OBJ_NOTNULL(*pp, VSM_CHUNK_MAGIC);
-}
-
-#define VSM_ITER(vd) for ((vd) = vsm_iter_0(); (vd) != NULL; vsm_iter_n(&vd))
-
-#else
-
-#define VSM_ITER(vd) while (YOU_NEED_MINIOBJ_TO_USE_VSM_ITER)
-
-#endif /* CHECK_OBJ_NOTNULL */
-
 #endif /* VSM_INT_H_INCLUDED */
diff --git a/lib/libvarnishapi/vsl.c b/lib/libvarnishapi/vsl.c
index 88bd9d6..b17cd36 100644
--- a/lib/libvarnishapi/vsl.c
+++ b/lib/libvarnishapi/vsl.c
@@ -390,7 +390,8 @@ VSL_Open(struct VSM_data *vd, int diag)
 
 /*--------------------------------------------------------------------*/
 
-int VSL_Matched(const struct VSM_data *vd, uint64_t bitmap)
+int
+VSL_Matched(const struct VSM_data *vd, uint64_t bitmap)
 {
 	if (vd->vsl->num_matchers > 0) {
 		uint64_t t;
diff --git a/lib/libvarnishapi/vsm.c b/lib/libvarnishapi/vsm.c
index 8e8b27b..97e3847 100644
--- a/lib/libvarnishapi/vsm.c
+++ b/lib/libvarnishapi/vsm.c
@@ -62,7 +62,8 @@ VSM_New(void)
 	struct VSM_data *vd;
 
 	ALLOC_OBJ(vd, VSM_MAGIC);
-	AN(vd);
+	if (vd == NULL)
+		return (vd);
 
 	vd->diag = (VSM_diag_f*)fprintf;
 	vd->priv = stderr;
@@ -132,15 +133,22 @@ VSM_Delete(struct VSM_data *vd)
 	if (vd->vsl != NULL)
 		VSL_Delete(vd);
 
-	free(vd);
+	FREE_OBJ(vd);
 }
 
-/*--------------------------------------------------------------------*/
+/*--------------------------------------------------------------------
+ * The internal VSM open function
+ *
+ * Return:
+ *	0 = sucess
+ * 	1 = failure
+ *
+ */
 
 static int
 vsm_open(struct VSM_data *vd, int diag)
 {
-	int i, j;
+	int i;
 	struct VSM_head slh;
 
 	CHECK_OBJ_NOTNULL(vd, VSM_MAGIC);
@@ -155,7 +163,7 @@ vsm_open(struct VSM_data *vd, int diag)
 		return (1);
 	}
 
-	assert(fstat(vd->vsm_fd, &vd->fstat) == 0);
+	AZ(fstat(vd->vsm_fd, &vd->fstat));
 	if (!S_ISREG(vd->fstat.st_mode)) {
 		if (diag)
 			vd->diag(vd->priv, "%s is not a regular file\n",
@@ -174,16 +182,16 @@ vsm_open(struct VSM_data *vd, int diag)
 		vd->vsm_fd = -1;
 		return (1);
 	}
-	if (slh.magic != VSM_HEAD_MAGIC) {
+	if (slh.magic != VSM_HEAD_MAGIC || slh.alloc_seq == 0) {
 		if (diag)
-			vd->diag(vd->priv, "Wrong magic number in file %s\n",
+			vd->diag(vd->priv, "Not a ready VSM file %s\n",
 			    vd->fname);
 		AZ(close(vd->vsm_fd));
 		vd->vsm_fd = -1;
 		return (1);
 	}
 
-	vd->VSM_head = (void *)mmap(NULL, slh.shm_size,
+	vd->VSM_head = mmap(NULL, slh.shm_size,
 	    PROT_READ, MAP_SHARED|MAP_HASSEMAPHORE, vd->vsm_fd, 0);
 	if (vd->VSM_head == MAP_FAILED) {
 		if (diag)
@@ -195,20 +203,7 @@ vsm_open(struct VSM_data *vd, int diag)
 		return (1);
 	}
 	vd->vsm_end = (uint8_t *)vd->VSM_head + slh.shm_size;
-
-	for (j = 0; j < 20 && vd->VSM_head->alloc_seq == 0; j++)
-		(void)usleep(50000);
-	if (vd->VSM_head->alloc_seq == 0) {
-		if (diag)
-			vd->diag(vd->priv, "File not initialized %s\n",
-			    vd->fname);
-		assert(0 == munmap((void*)vd->VSM_head, slh.shm_size));
-		AZ(close(vd->vsm_fd));
-		vd->vsm_fd = -1;
-		vd->VSM_head = NULL;
-		return (1);
-	}
-	vd->alloc_seq = vd->VSM_head->alloc_seq;
+	vd->my_alloc_seq = vd->VSM_head->alloc_seq;
 
 	if (vd->vsl != NULL)
 		VSL_Open_CallBack(vd);
@@ -238,10 +233,10 @@ VSM_Close(struct VSM_data *vd)
 	CHECK_OBJ_NOTNULL(vd, VSM_MAGIC);
 	if (vd->VSM_head == NULL)
 		return;
-	assert(0 == munmap((void*)vd->VSM_head, vd->VSM_head->shm_size));
+	AZ(munmap((void*)vd->VSM_head, vd->VSM_head->shm_size));
 	vd->VSM_head = NULL;
 	assert(vd->vsm_fd >= 0);
-	assert(0 == close(vd->vsm_fd));
+	AZ(close(vd->vsm_fd));
 	vd->vsm_fd = -1;
 }
 
@@ -330,10 +325,10 @@ VSM_iter0(struct VSM_data *vd)
 {
 
 	CHECK_OBJ_NOTNULL(vd, VSM_MAGIC);
-	vd->alloc_seq = vd->VSM_head->alloc_seq;
-	while (vd->alloc_seq == 0) {
+	vd->my_alloc_seq = vd->VSM_head->alloc_seq;
+	while (vd->my_alloc_seq == 0) {
 		(void)usleep(50000);
-		vd->alloc_seq = vd->VSM_head->alloc_seq;
+		vd->my_alloc_seq = vd->VSM_head->alloc_seq;
 	}
 	CHECK_OBJ_NOTNULL(&vd->VSM_head->head, VSM_CHUNK_MAGIC);
 	return (&vd->VSM_head->head);
@@ -344,7 +339,7 @@ VSM_itern(const struct VSM_data *vd, struct VSM_chunk **pp)
 {
 
 	CHECK_OBJ_NOTNULL(vd, VSM_MAGIC);
-	if (vd->alloc_seq != vd->VSM_head->alloc_seq) {
+	if (vd->my_alloc_seq != vd->VSM_head->alloc_seq) {
 		*pp = NULL;
 		return;
 	}
diff --git a/lib/libvarnishapi/vsm_api.h b/lib/libvarnishapi/vsm_api.h
index 519e46d..da5222e 100644
--- a/lib/libvarnishapi/vsm_api.h
+++ b/lib/libvarnishapi/vsm_api.h
@@ -44,13 +44,12 @@ struct VSM_data {
 	char			*n_opt;
 	char			*fname;
 
-
 	struct stat		fstat;
 
 	int			vsm_fd;
 	struct VSM_head		*VSM_head;
 	void			*vsm_end;
-	unsigned		alloc_seq;
+	unsigned		my_alloc_seq;
 
 	/* Stuff relating the stats fields start here */
 



From geoff at varnish-cache.org  Mon Jan  9 20:52:40 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:40 +0100
Subject: [experimental-ims] 9aaaf23 Typo
Message-ID: <E1RkMCu-0008LW-Da@project.varnish-software.com>

commit 9aaaf23a86853bf98494e0b69aafa311d98fbb0a
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sun Nov 20 14:21:17 2011 +0000

    Typo

diff --git a/include/vapi/vsm_int.h b/include/vapi/vsm_int.h
index c0a2df9..b45fdf7 100644
--- a/include/vapi/vsm_int.h
+++ b/include/vapi/vsm_int.h
@@ -108,7 +108,7 @@ struct VSM_chunk {
 };
 
 struct VSM_head {
-#define VSM_HEAD_MARKER		"VSMHEAD0"	/* Incr. as version# *
+#define VSM_HEAD_MARKER		"VSMHEAD0"	/* Incr. as version# */
 	char			marker[8];
 	ssize_t			hdrsize;
 	ssize_t			shm_size;



From geoff at varnish-cache.org  Mon Jan  9 20:52:40 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:40 +0100
Subject: [experimental-ims] 401ddc4 Make the VSM layout more hexdump friendly
Message-ID: <E1RkMCu-0008LH-B5@project.varnish-software.com>

commit 401ddc4e9462b2c8476e945c6e148982ea7325e7
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sun Nov 20 13:29:14 2011 +0000

    Make the VSM layout more hexdump friendly

diff --git a/bin/varnishd/common/common_vsm.c b/bin/varnishd/common/common_vsm.c
index f525a1e..781b01c 100644
--- a/bin/varnishd/common/common_vsm.c
+++ b/bin/varnishd/common/common_vsm.c
@@ -137,7 +137,7 @@ VSM_common_new(void *p, ssize_t l)
 	sc->head = (void *)sc->b;
 	/* This should not be necessary, but just in case...*/
 	memset(sc->head, 0, sizeof *sc->head);
-	sc->head->magic = VSM_HEAD_MAGIC;
+	memcpy(sc->head->marker, VSM_HEAD_MARKER, sizeof sc->head->marker);
 	sc->head->hdrsize = sizeof *sc->head;
 	sc->head->shm_size = l;
 	sc->head->alloc_seq = random() | 1;
@@ -145,8 +145,8 @@ VSM_common_new(void *p, ssize_t l)
 
 	ALLOC_OBJ(vr, VSM_RANGE_MAGIC);
 	AN(vr);
-	vr->off = PRNDUP(sizeof(*sc->head));
-	vr->len = l - vr->off;
+	vr->off = RUP2(sizeof(*sc->head), 16);
+	vr->len = RDN2(l - vr->off, 16);
 	VTAILQ_INSERT_TAIL(&sc->r_free, vr, list);
 	return (sc);
 }
@@ -182,9 +182,8 @@ VSM_common_alloc(struct vsm_sc *sc, ssize_t size,
 		vsm_common_insert_free(sc, vr);
 	}
 
-	size = PRNDUP(size);
-	l1 = size + sizeof(struct VSM_chunk);
-	l2 = size + 2 * sizeof(struct VSM_chunk);
+	l1 = RUP2(size + sizeof(struct VSM_chunk), 16);
+	l2 = RUP2(size + 2 * sizeof(struct VSM_chunk), 16);
 
 	/* Find space in free-list */
 	VTAILQ_FOREACH_SAFE(vr, &sc->r_free, list, vr2) {
@@ -227,7 +226,8 @@ VSM_common_alloc(struct vsm_sc *sc, ssize_t size,
 	vr->chunk = (void *)(sc->b + vr->off);
 	vr->ptr = (vr->chunk + 1);
 
-	vr->chunk->magic = VSM_CHUNK_MAGIC;
+	memcpy(vr->chunk->marker, VSM_CHUNK_MARKER, sizeof vr->chunk->marker);
+	vr->chunk->len = vr->len;
 	strcpy(vr->chunk->class, class);
 	strcpy(vr->chunk->type, type);
 	strcpy(vr->chunk->ident, ident);
diff --git a/include/vapi/vsm_int.h b/include/vapi/vsm_int.h
index c33ba35..c0a2df9 100644
--- a/include/vapi/vsm_int.h
+++ b/include/vapi/vsm_int.h
@@ -98,8 +98,8 @@
 #define VSM_FILENAME		"_.vsm"
 
 struct VSM_chunk {
-#define VSM_CHUNK_MAGIC		0xa15712e5	/* From /dev/random */
-	unsigned		magic;
+#define VSM_CHUNK_MARKER	"VSMCHUNK"
+	char			marker[8];
 	ssize_t			len;		/* Incl VSM_chunk */
 	ssize_t			next;		/* Offset in shmem */
 	char			class[8];
@@ -108,8 +108,8 @@ struct VSM_chunk {
 };
 
 struct VSM_head {
-#define VSM_HEAD_MAGIC		0xe75f7e91	/* From /dev/random */
-	unsigned		magic;
+#define VSM_HEAD_MARKER		"VSMHEAD0"	/* Incr. as version# *
+	char			marker[8];
 	ssize_t			hdrsize;
 	ssize_t			shm_size;
 	ssize_t			first;		/* Offset, first chunk */



From geoff at varnish-cache.org  Mon Jan  9 20:52:40 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:40 +0100
Subject: [experimental-ims] 889a8e4 White-space & comment cleanup
Message-ID: <E1RkMCu-0008Lv-Ho@project.varnish-software.com>

commit 889a8e4c05d8fb9dd5e17dfbd63f825e4822798e
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sun Nov 20 19:39:46 2011 +0000

    White-space & comment cleanup

diff --git a/bin/varnishd/common/common_vsm.c b/bin/varnishd/common/common_vsm.c
index 781b01c..a2a4fe4 100644
--- a/bin/varnishd/common/common_vsm.c
+++ b/bin/varnishd/common/common_vsm.c
@@ -49,10 +49,10 @@
 /*--------------------------------------------------------------------*/
 
 struct vsm_range {
-	unsigned 			magic;
+	unsigned			magic;
 #define VSM_RANGE_MAGIC			0x8d30f14
 	VTAILQ_ENTRY(vsm_range)		list;
-	ssize_t				off; 
+	ssize_t				off;
 	ssize_t				len;
 	double				cool;
 	struct VSM_chunk		*chunk;
@@ -60,7 +60,7 @@ struct vsm_range {
 };
 
 struct vsm_sc {
-	unsigned 			magic;
+	unsigned			magic;
 #define VSM_SC_MAGIC			0x8b83270d
 	char				*b;
 	ssize_t				len;
@@ -236,7 +236,7 @@ VSM_common_alloc(struct vsm_sc *sc, ssize_t size,
 	vr3 = VTAILQ_FIRST(&sc->r_used);
 	VTAILQ_INSERT_HEAD(&sc->r_used, vr, list);
 
-	if (vr3 != NULL) {	
+	if (vr3 != NULL) {
 		AZ(vr3->chunk->next);
 		vr3->chunk->next = vr->off;
 	} else {
diff --git a/bin/varnishd/mgt/mgt_shmem.c b/bin/varnishd/mgt/mgt_shmem.c
index 6bb8e13..40f0c24 100644
--- a/bin/varnishd/mgt/mgt_shmem.c
+++ b/bin/varnishd/mgt/mgt_shmem.c
@@ -26,62 +26,6 @@
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
  *
- *
- * TODO:
- *
- * There is a risk that the child process might corrupt the VSM segment
- * and we should capture that event and recover gracefully.
- *
- * A possible state diagram could be:
- *
- *	[manager start]
- *		|
- *		v
- *      Open old VSM,
- *	check pid	--------> exit/fail (-n message)
- *		|
- *		+<----------------------+
- *		|			^
- *		v			|
- *	Create new VSM			|
- *		|			|
- *		v			|
- *	Init header			|
- *	Alloc VSL			|
- *	Alloc VSC:Main			|
- *	Alloc Args etc.			|
- *		|			|
- *		+<--------------+	|
- *		|		^	|
- *		v		|	|
- *	start worker		|	|
- *		|		|	|
- *		|		|	+<---- worker crash
- *		v		|	^
- *	Reset VSL ptr.		|	|
- *	Reset VSC counters	|	|
- *		|		|	|
- *		+<------+	|	|
- *		|	^	|	|
- *		v	|	|	|
- *	alloc dynamics	|	|	|
- *	free dynamics	|	|	|
- *		|	|	|	|
- *		v	|	|	|
- *		+------>+	|	|
- *		|		|	|
- *		v		|	|
- *	stop worker		|	|
- *		|		|	|
- *		v		|	|
- *	Check consist---------- | ----->+
- *		|		|
- *		v		|
- *	Free dynamics		|
- *		|		|
- *		v		|
- *		+-------------->+
- *
  */
 
 #include "config.h"
@@ -241,7 +185,7 @@ mgt_SHM_Init(void)
 	bprintf(fnbuf, "%s.%jd", VSM_FILENAME, (intmax_t)getpid());
 
 	vsm_fd = vsm_zerofile(fnbuf, size);
-	if (vsm_fd < 0) 
+	if (vsm_fd < 0)
 		exit(1);
 
 	p = (void *)mmap(NULL, size,
diff --git a/lib/libvarnish/cli_common.c b/lib/libvarnish/cli_common.c
index aa5fd63..74c966f 100644
--- a/lib/libvarnish/cli_common.c
+++ b/lib/libvarnish/cli_common.c
@@ -212,7 +212,7 @@ VCLI_ReadResult(int fd, unsigned *status, char **ptr, double tmo)
 			break;
 		if (p[v] != '\n')
 			break;
-		
+
 		p[v] = '\0';
 		if (ptr == NULL)
 			free(p);
diff --git a/lib/libvarnishapi/vsc.c b/lib/libvarnishapi/vsc.c
index 1e2206b..4fc2233 100644
--- a/lib/libvarnishapi/vsc.c
+++ b/lib/libvarnishapi/vsc.c
@@ -292,7 +292,7 @@ iter_call(const struct vsc *vsc, VSC_iter_f *func, void *priv,
 		int i;							\
 									\
 		CHECK_OBJ_NOTNULL(vsc, VSC_MAGIC);			\
-		st = vf->b; 						\
+		st = vf->b;						\
 		sp.class = t;						\
 		sp.ident = vf->chunk->ident;
 
diff --git a/lib/libvarnishapi/vsm.c b/lib/libvarnishapi/vsm.c
index 1669278..81007d8 100644
--- a/lib/libvarnishapi/vsm.c
+++ b/lib/libvarnishapi/vsm.c
@@ -141,7 +141,7 @@ VSM_Delete(struct VSM_data *vd)
  *
  * Return:
  *	0 = sucess
- * 	<0 = failure
+ *	<0 = failure
  *
  */
 



From geoff at varnish-cache.org  Mon Jan  9 20:52:40 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:40 +0100
Subject: [experimental-ims] 9584dc0 Move cache_param entirely into the cache *
	namespace and communicate it via heritage.
Message-ID: <E1RkMCu-0008MA-Lh@project.varnish-software.com>

commit 9584dc0365f172d2d59569d15836f1c58c803771
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sun Nov 20 19:59:58 2011 +0000

    Move cache_param entirely into the cache * namespace and communicate
    it via heritage.

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index e08cc32..4b66309 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -784,6 +784,7 @@ int HTC_Complete(struct http_conn *htc);
 #undef HTTPH
 
 /* cache_main.c */
+extern volatile struct params * cache_param;
 void THR_SetName(const char *name);
 const char* THR_GetName(void);
 void THR_SetSession(const struct sess *sp);
diff --git a/bin/varnishd/cache/cache_main.c b/bin/varnishd/cache/cache_main.c
index eb3fa1d..568e5e5 100644
--- a/bin/varnishd/cache/cache_main.c
+++ b/bin/varnishd/cache/cache_main.c
@@ -96,6 +96,8 @@ child_main(void)
 	setbuf(stderr, NULL);
 	printf("Child starts\n");
 
+	cache_param = heritage.param;
+
 	AZ(pthread_key_create(&sp_key, NULL));
 	AZ(pthread_key_create(&name_key, NULL));
 
diff --git a/bin/varnishd/common/heritage.h b/bin/varnishd/common/heritage.h
index db8b1f0..4ad9deb 100644
--- a/bin/varnishd/common/heritage.h
+++ b/bin/varnishd/common/heritage.h
@@ -60,6 +60,8 @@ struct heritage {
 
 	struct vsm_sc			*vsm;
 
+	struct params			*param;
+
 	char				*name;
 	char                            identity[1024];
 };
diff --git a/bin/varnishd/common/params.h b/bin/varnishd/common/params.h
index cab2f49..b9b758f 100644
--- a/bin/varnishd/common/params.h
+++ b/bin/varnishd/common/params.h
@@ -193,9 +193,3 @@ struct params {
 	ssize_t			vsm_space;
 	ssize_t			vsl_space;
 };
-
-/*
- * We declare this a volatile pointer, so that reads of parameters
- * become atomic, leaving the CLI thread lattitude to change the values
- */
-extern volatile struct params * cache_param;
diff --git a/bin/varnishd/mgt/mgt_child.c b/bin/varnishd/mgt/mgt_child.c
index b619b66..4afcf81 100644
--- a/bin/varnishd/mgt/mgt_child.c
+++ b/bin/varnishd/mgt/mgt_child.c
@@ -315,7 +315,7 @@ start_child(struct cli *cli)
 	heritage.std_fd = cp[1];
 	child_output = cp[0];
 
-	AN(cache_param);
+	AN(heritage.param);
 	if ((pid = fork()) < 0) {
 		perror("Could not fork child");
 		exit(1);
diff --git a/bin/varnishd/mgt/mgt_param.c b/bin/varnishd/mgt/mgt_param.c
index f6de562..e4be5d8 100644
--- a/bin/varnishd/mgt/mgt_param.c
+++ b/bin/varnishd/mgt/mgt_param.c
@@ -1205,8 +1205,8 @@ MCF_ParamSet(struct cli *cli, const char *param, const char *val)
 	}
 	pp->func(cli, pp, val);
 
-	if (cli->result == CLIS_OK && cache_param != NULL)
-		*cache_param = mgt_param;
+	if (cli->result == CLIS_OK && heritage.param != NULL)
+		*heritage.param = mgt_param;
 
 	if (cli->result != CLIS_OK) {
 		VCLI_Out(cli, "(attempting to set param %s to %s)\n",
diff --git a/bin/varnishd/mgt/mgt_shmem.c b/bin/varnishd/mgt/mgt_shmem.c
index 5915b19..66eb035 100644
--- a/bin/varnishd/mgt/mgt_shmem.c
+++ b/bin/varnishd/mgt/mgt_shmem.c
@@ -210,9 +210,10 @@ mgt_SHM_Init(void)
 
 	AZ(atexit(mgt_shm_atexit));
 
-	cache_param = VSM_Alloc(sizeof *cache_param, VSM_CLASS_PARAM, "", "");
-	AN(cache_param);
-	*cache_param = mgt_param;
+	heritage.param =
+	    VSM_Alloc(sizeof *heritage.param, VSM_CLASS_PARAM, "", "");
+	AN(heritage.param);
+	*heritage.param = mgt_param;
 
 	PAN_panicstr_len = 64 * 1024;
 	PAN_panicstr = VSM_Alloc(PAN_panicstr_len, PAN_CLASS, "", "");



From geoff at varnish-cache.org  Mon Jan  9 20:52:40 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:40 +0100
Subject: [experimental-ims] d68a721 Introduce the new VSM "per-child" lifetime.
Message-ID: <E1RkMCu-0008Md-SR@project.varnish-software.com>

commit d68a7210c63e3caefb2f010d48dd4c5d2ce37b90
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sun Nov 20 21:26:25 2011 +0000

    Introduce the new VSM "per-child" lifetime.

diff --git a/bin/varnishd/common/common_vsm.c b/bin/varnishd/common/common_vsm.c
index b67e721..4eb4195 100644
--- a/bin/varnishd/common/common_vsm.c
+++ b/bin/varnishd/common/common_vsm.c
@@ -317,7 +317,10 @@ VSM_common_delete(struct vsm_sc **scp)
 		free(vr->ptr);
 		FREE_OBJ(vr);
 	}
+
+	/* Mark VSM as abandoned */
 	sc->head->alloc_seq = 0;
+
 	VWMB();
 	FREE_OBJ(sc);
 }
diff --git a/bin/varnishd/mgt/mgt.h b/bin/varnishd/mgt/mgt.h
index 1b5a872..52463e6 100644
--- a/bin/varnishd/mgt/mgt.h
+++ b/bin/varnishd/mgt/mgt.h
@@ -84,6 +84,10 @@ void mgt_sandbox_solaris_privsep(void);
 void mgt_SHM_Init(void);
 void mgt_SHM_static_alloc(const void *, ssize_t size,
     const char *class, const char *type, const char *ident);
+void mgt_SHM_Create(void);
+void mgt_SHM_Destroy(int keep);
+void mgt_SHM_Size_Adjust(void);
+
 
 /* stevedore_mgt.c */
 void STV_Config(const char *spec);
diff --git a/bin/varnishd/mgt/mgt_child.c b/bin/varnishd/mgt/mgt_child.c
index 6d42260..0f91d05 100644
--- a/bin/varnishd/mgt/mgt_child.c
+++ b/bin/varnishd/mgt/mgt_child.c
@@ -315,6 +315,9 @@ start_child(struct cli *cli)
 	heritage.std_fd = cp[1];
 	child_output = cp[0];
 
+	AN(heritage.vsm);
+	mgt_SHM_Size_Adjust();
+	AN(heritage.vsm);
 	AN(heritage.param);
 	if ((pid = fork()) < 0) {
 		perror("Could not fork child");
@@ -427,8 +430,7 @@ mgt_handle_panicstr(pid_t r)
 {
 	char time_str[30];
 
-	if (heritage.panic_str[0] == '\0')
-		return;
+	AN(heritage.panic_str[0]);
 	REPORT(LOG_ERR, "Child (%jd) Panic message: %s",
 	    (intmax_t)r, heritage.panic_str);
 
@@ -492,7 +494,13 @@ mgt_sigchld(const struct vev *e, int what)
 	REPORT(LOG_INFO, "%s", VSB_data(vsb));
 	VSB_delete(vsb);
 
-	mgt_handle_panicstr(r);
+	if (heritage.panic_str[0] != '\0') {
+		mgt_handle_panicstr(r);
+		mgt_SHM_Destroy(1);
+	} else {
+		mgt_SHM_Destroy(0);
+	}
+	mgt_SHM_Create();
 
 	child_pid = -1;
 
diff --git a/bin/varnishd/mgt/mgt_main.c b/bin/varnishd/mgt/mgt_main.c
index f782d03..cc918c1 100644
--- a/bin/varnishd/mgt/mgt_main.c
+++ b/bin/varnishd/mgt/mgt_main.c
@@ -652,6 +652,9 @@ main(int argc, char * const *argv)
 	if (T_arg != NULL)
 		mgt_cli_telnet(T_arg);
 
+	/* Instantiate VSM */
+	mgt_SHM_Create();
+
 	MGT_Run();
 
 	if (pfh != NULL)
diff --git a/bin/varnishd/mgt/mgt_shmem.c b/bin/varnishd/mgt/mgt_shmem.c
index 075b838..9f88f58 100644
--- a/bin/varnishd/mgt/mgt_shmem.c
+++ b/bin/varnishd/mgt/mgt_shmem.c
@@ -59,6 +59,9 @@
 
 #define PAN_CLASS "Panic"
 
+static void *mgt_vsm_p;
+static ssize_t mgt_vsm_l;
+
 /*--------------------------------------------------------------------
  * Use a bogo-VSM to hold master-copies of the VSM chunks the master
  * publishes, such as -S & -T arguments.
@@ -174,20 +177,31 @@ vsm_zerofile(const char *fn, ssize_t size)
 }
 
 /*--------------------------------------------------------------------
+ * Create a VSM instance
  */
 
-static void
-mgt_SHM_Setup(void)
+static size_t
+mgt_shm_size(void)
 {
-	uintmax_t size, ps;
-	void *p;
-	char fnbuf[64];
-	int vsm_fd;
+	size_t size, ps;
 
 	size = mgt_param.vsl_space + mgt_param.vsm_space;
 	ps = getpagesize();
 	size += ps - 1;
-	size &= ~(ps - 1);
+	size &= ~(ps - 1U);
+	return (size);
+}
+
+void
+mgt_SHM_Create(void)
+{
+	size_t size;
+	void *p;
+	char fnbuf[64];
+	int vsm_fd;
+
+	AZ(heritage.vsm);
+	size = mgt_shm_size();
 
 	bprintf(fnbuf, "%s.%jd", VSM_FILENAME, (intmax_t)getpid());
 
@@ -207,11 +221,26 @@ mgt_SHM_Setup(void)
 		exit (-1);
 	}
 
+	mgt_vsm_p = p;
+	mgt_vsm_l = size;
+
 	/* This may or may not work */
 	(void)mlock(p, size);
 
 	heritage.vsm = VSM_common_new(p, size);
 
+	VSM_common_copy(heritage.vsm, static_vsm);
+
+	heritage.param = VSM_common_alloc(heritage.vsm,
+	    sizeof *heritage.param, VSM_CLASS_PARAM, "", "");
+	AN(heritage.param);
+	*heritage.param = mgt_param;
+
+	heritage.panic_str_len = 64 * 1024;
+	heritage.panic_str = VSM_common_alloc(heritage.vsm,
+	    heritage.panic_str_len, PAN_CLASS, "", "");
+	AN(heritage.panic_str);
+
 	if (rename(fnbuf, VSM_FILENAME)) {
 		fprintf(stderr, "Rename failed %s -> %s: %s\n",
 		    fnbuf, VSM_FILENAME, strerror(errno));
@@ -221,6 +250,41 @@ mgt_SHM_Setup(void)
 }
 
 /*--------------------------------------------------------------------
+ * Destroy a VSM instance
+ */
+
+void
+mgt_SHM_Destroy(int keep)
+{
+
+	AN(heritage.vsm);
+	if (keep)
+		(void)rename(VSM_FILENAME, VSM_FILENAME ".keep");
+	heritage.panic_str = NULL;
+	heritage.panic_str_len = 0;
+	heritage.param = NULL;
+	VSM_common_delete(&heritage.vsm);
+	AZ(munmap(mgt_vsm_p, mgt_vsm_l));
+	mgt_vsm_p = NULL;
+	mgt_vsm_l = 0;
+}
+
+/*--------------------------------------------------------------------
+ * Destroy and recreate VSM if its size should change
+ */
+
+void
+mgt_SHM_Size_Adjust(void)
+{
+
+	AN(heritage.vsm);
+	if (mgt_vsm_l == mgt_shm_size())
+		return;
+	mgt_SHM_Destroy(0);
+	mgt_SHM_Create();
+}
+
+/*--------------------------------------------------------------------
  * Exit handler that clears the owning pid from the SHMLOG
  */
 
@@ -233,6 +297,10 @@ mgt_shm_atexit(void)
 		VSM_common_delete(&heritage.vsm);
 }
 
+/*--------------------------------------------------------------------
+ * Initialize VSM subsystem
+ */
+
 void
 mgt_SHM_Init(void)
 {
@@ -243,21 +311,9 @@ mgt_SHM_Init(void)
 	if (i)
 		exit(i);
 
+	/* Create our static VSM instance */
 	static_vsm = VSM_common_new(static_vsm_buf, sizeof static_vsm_buf);
 
-	mgt_SHM_Setup();
-
+	/* Setup atexit handler */
 	AZ(atexit(mgt_shm_atexit));
-
-	VSM_common_copy(heritage.vsm, static_vsm);
-
-	heritage.param = VSM_common_alloc(heritage.vsm,
-	    sizeof *heritage.param, VSM_CLASS_PARAM, "", "");
-	AN(heritage.param);
-	*heritage.param = mgt_param;
-
-	heritage.panic_str_len = 64 * 1024;
-	heritage.panic_str = VSM_common_alloc(heritage.vsm,
-	    heritage.panic_str_len, PAN_CLASS, "", "");
-	AN(heritage.panic_str);
 }



From geoff at varnish-cache.org  Mon Jan  9 20:52:42 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:42 +0100
Subject: [experimental-ims] 84b1ca0 Gcc isn't always very smart...
Message-ID: <E1RkMCw-0008Qt-HM@project.varnish-software.com>

commit 84b1ca029b635f47555495b46ef9153d3ac34567
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Fri Nov 25 11:20:40 2011 +0000

    Gcc isn't always very smart...

diff --git a/bin/varnishd/cache/cache_shmlog.c b/bin/varnishd/cache/cache_shmlog.c
index 893525b..a18a08e 100644
--- a/bin/varnishd/cache/cache_shmlog.c
+++ b/bin/varnishd/cache/cache_shmlog.c
@@ -311,6 +311,7 @@ vsm_cleaner(void *priv)
 		AZ(pthread_mutex_unlock(&vsm_mtx));
 		VTIM_sleep(1.1);
 	}
+	NEEDLESS_RETURN(NULL);
 }
 
 /*--------------------------------------------------------------------*/



From geoff at varnish-cache.org  Mon Jan  9 20:52:42 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:42 +0100
Subject: [experimental-ims] 42d9a31 Always remember what your pointers point
	to, when doing pointer addition.
Message-ID: <E1RkMCw-0008R2-Iv@project.varnish-software.com>

commit 42d9a31ac0dc63f0ae971974b4a4c36c20f421ef
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Fri Nov 25 11:40:54 2011 +0000

    Always remember what your pointers point to, when doing pointer addition.
    
    Fixes	#1065

diff --git a/bin/varnishd/cache/cache_shmlog.c b/bin/varnishd/cache/cache_shmlog.c
index a18a08e..1e414b8 100644
--- a/bin/varnishd/cache/cache_shmlog.c
+++ b/bin/varnishd/cache/cache_shmlog.c
@@ -335,7 +335,7 @@ VSM_Init(void)
 	VWMB();
 
 	vsl_start = vsl_log_start;
-	vsl_end = vsl_start + cache_param->vsl_space;
+	vsl_end = vsl_start + cache_param->vsl_space / sizeof *vsl_end;
 	vsl_ptr = vsl_start + 1;
 
 	VSC_C_main = VSM_Alloc(sizeof *VSC_C_main,



From geoff at varnish-cache.org  Mon Jan  9 20:52:42 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:42 +0100
Subject: [experimental-ims] f4ba8ee Make varnishlog not explode
Message-ID: <E1RkMCw-0008RE-KW@project.varnish-software.com>

commit f4ba8eeaf67cbd8f993ecf14ea5ade0862cf3ba2
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Fri Nov 25 11:43:08 2011 +0000

    Make varnishlog not explode

diff --git a/lib/libvarnishapi/vsl.c b/lib/libvarnishapi/vsl.c
index fdd93d8..6950c1b 100644
--- a/lib/libvarnishapi/vsl.c
+++ b/lib/libvarnishapi/vsl.c
@@ -124,9 +124,11 @@ vsl_open(struct VSM_data *vd)
 	int i;
 
 	assert(vsl->r_fd < 0);
-	i = VSM_Open(vd);
-	if (i)
-		return (i);
+	if (vd->head == NULL) {
+		i = VSM_Open(vd);
+		if (i)
+			return (i);
+	}
 	if (!VSM_Get(vd, &vsl->vf, VSL_CLASS, NULL, NULL)) {
 		VSM_Close(vd);
 		return (vsm_diag(vd, "No VSL chunk found "
@@ -254,10 +256,11 @@ VSL_NextSLT(struct VSM_data *vd, uint32_t **pp, uint64_t *bits)
 		i = vsl_nextslt(vd, &p);
 		if (i < 0)
 			return (i);
-		if (i == 0 && (vsl->d_opt || vsl->r_fd >= 0))
-			return (i);
-		if (i == 0 && !VSM_StillValid(vd, &vsl->vf)) {
-			vsl_close(vd);
+		if (i == 0) {
+			if (vsl->d_opt || vsl->r_fd >= 0)
+				return (i);
+			if (!VSM_StillValid(vd, &vsl->vf))
+				vsl_close(vd);
 			return (i);
 		}
 



From geoff at varnish-cache.org  Mon Jan  9 20:52:42 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:42 +0100
Subject: [experimental-ims] cd6f804 Remove a duplicate assert
Message-ID: <E1RkMCw-0008RL-Mk@project.varnish-software.com>

commit cd6f804c1567fc9e40a87231135a361d5d092cc8
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Fri Nov 25 11:50:46 2011 +0000

    Remove a duplicate assert
    
    Fixes	#1064

diff --git a/lib/libvcl/vcc_compile.c b/lib/libvcl/vcc_compile.c
index 5719cbf..fddd019 100644
--- a/lib/libvcl/vcc_compile.c
+++ b/lib/libvcl/vcc_compile.c
@@ -444,13 +444,13 @@ vcc_resolve_includes(struct vcc *tl)
 		}
 		t2 = VTAILQ_NEXT(t1, list);
 		assert(t2 != NULL);	/* There's always an EOI */
+
 		if (t2->tok != ';') {
 			VSB_printf(tl->sb,
 			    "include <string> not followed by semicolon.\n");
 			vcc_ErrWhere(tl, t1);
 			return;
 		}
-		assert(t2 != NULL);
 
 		sp = vcc_file_source(tl, tl->sb, t1->dec);
 		if (sp == NULL) {



From geoff at varnish-cache.org  Mon Jan  9 20:52:42 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:42 +0100
Subject: [experimental-ims] c4629a0 Make it possible to test for the
	non-definition of a http header.
Message-ID: <E1RkMCw-0008RW-O2@project.varnish-software.com>

commit c4629a0fb7731416f5f130d8ccefa55dc58e5191
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Nov 28 09:11:28 2011 +0000

    Make it possible to test for the non-definition of a http header.
    
    Fixes #1062

diff --git a/bin/varnishtest/tests/a00012.vtc b/bin/varnishtest/tests/a00012.vtc
new file mode 100644
index 0000000..07e51c1
--- /dev/null
+++ b/bin/varnishtest/tests/a00012.vtc
@@ -0,0 +1,13 @@
+varnishtest "Ensure that we can test non-existence of headers (#1062)"
+
+server s1 {
+        rxreq
+        txresp
+} -start
+
+client c1 -connect ${s1_sock} {
+        txreq
+        rxresp
+        expect resp.http.X-Test == <undef>
+} -run
+
diff --git a/bin/varnishtest/tests/c00016.vtc b/bin/varnishtest/tests/c00016.vtc
index d028a64..99e9c53 100644
--- a/bin/varnishtest/tests/c00016.vtc
+++ b/bin/varnishtest/tests/c00016.vtc
@@ -8,7 +8,7 @@ server s1 {
 
 	rxreq
 	expect req.url == "/bar"
-	expect req.http.Foo == "req.http.Foo"
+	expect req.http.Foo == <undef>
 	txresp -hdr "Bar: fnry,glyf, FOO ,brok" -hdr "Connection: bar" -body "foobar" 
 } -start
 
@@ -21,5 +21,5 @@ client c1 {
 
 	txreq -url "/bar" -hdr "Foo: bar2" -hdr "Connection: foo, close"
 	rxresp
-	expect req.http.Bar == "req.http.Bar"
+	expect req.http.Bar == <undef>
 } -run
diff --git a/bin/varnishtest/tests/e00024.vtc b/bin/varnishtest/tests/e00024.vtc
index 2aa8e3a..f728b3c 100644
--- a/bin/varnishtest/tests/e00024.vtc
+++ b/bin/varnishtest/tests/e00024.vtc
@@ -81,7 +81,7 @@ client c1 {
 
 	txreq 
 	rxresp
-	expect resp.http.content-encoding == resp.http.content-encoding
+	expect resp.http.content-encoding == <undef>
 	expect resp.status == 200
 	expect resp.bodylen == 252
 } -run
diff --git a/bin/varnishtest/tests/e00025.vtc b/bin/varnishtest/tests/e00025.vtc
index 5549f5c..5e8a12b 100644
--- a/bin/varnishtest/tests/e00025.vtc
+++ b/bin/varnishtest/tests/e00025.vtc
@@ -18,7 +18,7 @@ client c1 {
 
 	txreq 
 	rxresp
-	expect resp.http.content-encoding == resp.http.content-encoding
+	expect resp.http.content-encoding == <undef>
 	expect resp.status == 200
 	expect resp.bodylen == 3
 } -run
diff --git a/bin/varnishtest/tests/g00001.vtc b/bin/varnishtest/tests/g00001.vtc
index 34ad00f..e10b743 100644
--- a/bin/varnishtest/tests/g00001.vtc
+++ b/bin/varnishtest/tests/g00001.vtc
@@ -13,7 +13,7 @@ client c1 {
 	txreq 
 	rxresp
 	expect resp.bodylen == "3"
-	expect resp.http.content-encoding == "resp.http.content-encoding"
+	expect resp.http.content-encoding == <undef>
 
 	txreq -hdr "Accept-encoding: gzip;q=0.1"
 	rxresp
@@ -26,13 +26,13 @@ client c1 {
 	txreq -proto HTTP/1.0
 	rxresp
 	expect resp.bodylen == "3"
-	expect resp.http.content-encoding == "resp.http.content-encoding"
+	expect resp.http.content-encoding == <undef>
 } -run
 
 client c1 {
 	txreq -req HEAD
 	rxresp -no_obj
-	expect resp.http.content-encoding == "resp.http.content-encoding"
+	expect resp.http.content-encoding == <undef>
 
 	txreq -req HEAD -hdr "Accept-encoding: gzip;q=0.1"
 	rxresp -no_obj
diff --git a/bin/varnishtest/tests/g00002.vtc b/bin/varnishtest/tests/g00002.vtc
index 1bf3384..eb10ba0 100644
--- a/bin/varnishtest/tests/g00002.vtc
+++ b/bin/varnishtest/tests/g00002.vtc
@@ -39,12 +39,12 @@ client c1 {
 	# See varnish can gunzip it.
 	txreq -url /foo -hdr "Accept-Encoding: null"
 	rxresp
-	expect resp.http.content-encoding == "resp.http.content-encoding"
+	expect resp.http.content-encoding == <undef>
 	expect resp.bodylen == 4100
 
 	# See varnish can gunzip it, inside ESI
 	txreq -url /bar -hdr "Accept-Encoding: null"
 	rxresp
-	expect resp.http.content-encoding == "resp.http.content-encoding"
+	expect resp.http.content-encoding == <undef>
 	expect resp.bodylen == 4109
 } -run
diff --git a/bin/varnishtest/tests/g00003.vtc b/bin/varnishtest/tests/g00003.vtc
index dcd8936..05edb5d 100644
--- a/bin/varnishtest/tests/g00003.vtc
+++ b/bin/varnishtest/tests/g00003.vtc
@@ -30,12 +30,12 @@ varnish v1 -cliok "param.set http_gzip_support true" -vcl+backend {
 client c1 {
 	txreq -url /foo -hdr "Accept-Encoding: gzip"
 	rxresp
-	expect resp.http.content-encoding == "resp.http.content-encoding"
+	expect resp.http.content-encoding == <undef>
 	expect resp.bodylen == 41
 
 	txreq -url /bar -hdr "Accept-Encoding: gzip"
 	rxresp
-	expect resp.http.content-encoding == "resp.http.content-encoding"
+	expect resp.http.content-encoding == <undef>
 	expect resp.bodylen == 42
 
 	txreq -url /foobar -hdr "Accept-Encoding: gzip"
@@ -46,6 +46,6 @@ client c1 {
 
 	txreq -url /foobar 
 	rxresp
-	expect resp.http.content-encoding == "resp.http.content-encoding"
+	expect resp.http.content-encoding == <undef>
 	expect resp.bodylen == 43
 } -run
diff --git a/bin/varnishtest/tests/r00292.vtc b/bin/varnishtest/tests/r00292.vtc
index aaa914c..7739e85 100644
--- a/bin/varnishtest/tests/r00292.vtc
+++ b/bin/varnishtest/tests/r00292.vtc
@@ -6,11 +6,11 @@ varnishtest "Header deletion test"
 server s1 {
 	rxreq
 	expect req.url == "/foo"
-	expect req.http.hdr1 == "req.http.hdr1"
+	expect req.http.hdr1 == <undef>
 	expect req.http.hdr2 == "2"
-	expect req.http.hdr3 == "req.http.hdr3"
+	expect req.http.hdr3 == <undef>
 	expect req.http.hdr4 == "4"
-	expect req.http.hdr5 == "req.http.hdr5"
+	expect req.http.hdr5 == <undef>
 	expect req.http.hdr6 == "6"
 	txresp -body "foobar"
 } -start
diff --git a/bin/varnishtest/tests/r00466.vtc b/bin/varnishtest/tests/r00466.vtc
index acff68a..8d753ba 100644
--- a/bin/varnishtest/tests/r00466.vtc
+++ b/bin/varnishtest/tests/r00466.vtc
@@ -3,7 +3,7 @@ varnishtest "Check Range forwarding to backend"
 server s1 {
 	rxreq 
 	expect req.url == "/foo"
-	expect req.http.range == "req.http.range"
+	expect req.http.range == <undef>
 	txresp \
 		-hdr "Foobar: _barf_" \
 		-body "012345\n"
diff --git a/bin/varnishtest/tests/r00494.vtc b/bin/varnishtest/tests/r00494.vtc
index d1dedd1..1a7fec3 100644
--- a/bin/varnishtest/tests/r00494.vtc
+++ b/bin/varnishtest/tests/r00494.vtc
@@ -19,6 +19,6 @@ client c1 {
 	txreq
 	rxresp
 	expect resp.http.bar == "bar,  barf: fail"
-	expect resp.http.barf == resp.http.barf
-	expect resp.http.foo == resp.http.foo
+	expect resp.http.barf == <undef>
+	expect resp.http.foo == <undef>
 } -run
diff --git a/bin/varnishtest/tests/r00693.vtc b/bin/varnishtest/tests/r00693.vtc
index 6cf7b30..37b0a98 100644
--- a/bin/varnishtest/tests/r00693.vtc
+++ b/bin/varnishtest/tests/r00693.vtc
@@ -4,11 +4,11 @@ feature 64bit
 
 server s1 {
 	rxreq
-	expect req.http.baz == "req.http.baz"
+	expect req.http.baz == <undef>
 	txresp -status 201
 
 	rxreq
-	expect req.http.baz == "req.http.baz"
+	expect req.http.baz == <undef>
 	txresp -status 202
 
 	rxreq
@@ -52,7 +52,7 @@ varnish v1 -arg "-p sess_workspace=1024" -vcl+backend {
 			"0123456789abcdef" + "0123456789abcdef" +
 			"0123456789abcdef" + "0123456789abcdef" +
 			"0123456789abcdef" +
-			"01234567";
+			"0123456";
 		set req.http.baz = "BAZ";
 		return (pass);
 	}
diff --git a/bin/varnishtest/tests/r00861.vtc b/bin/varnishtest/tests/r00861.vtc
index 165795b..8a3e8a0 100644
--- a/bin/varnishtest/tests/r00861.vtc
+++ b/bin/varnishtest/tests/r00861.vtc
@@ -34,12 +34,12 @@ varnish v1 \
 client c1 {
 	txreq -url "/1"
 	rxresp
-	expect resp.http.Content-Encoding == resp.http.Content-Encoding
+	expect resp.http.Content-Encoding == <undef>
 	expect resp.bodylen == 22
 
 	txreq -url "/barf" -hdr "Accept-Encoding: gzip"
 	rxresp
-	expect resp.http.Content-Encoding == resp.http.Content-Encoding
+	expect resp.http.Content-Encoding == <undef>
 	expect resp.bodylen == 909
 
 	txreq -url "/2" -hdr "Accept-Encoding: gzip"
diff --git a/bin/varnishtest/tests/r00980.vtc b/bin/varnishtest/tests/r00980.vtc
index 9591786..b7d307e 100644
--- a/bin/varnishtest/tests/r00980.vtc
+++ b/bin/varnishtest/tests/r00980.vtc
@@ -24,6 +24,6 @@ client c1 {
 
 	txreq -url /foobar 
 	rxresp
-	expect resp.http.content-encoding == "resp.http.content-encoding"
+	expect resp.http.content-encoding == <undef>
 	expect resp.bodylen == 43
 } -run
diff --git a/bin/varnishtest/vtc_http.c b/bin/varnishtest/vtc_http.c
index 69485cc..d3753a5 100644
--- a/bin/varnishtest/vtc_http.c
+++ b/bin/varnishtest/vtc_http.c
@@ -169,7 +169,7 @@ http_find_header(char * const *hh, const char *hdr)
  * Expect
  */
 
-static char *
+static const char *
 cmd_var_resolve(struct http *hp, char *spec)
 {
 	char **hh, *hdr;
@@ -201,16 +201,16 @@ cmd_var_resolve(struct http *hp, char *spec)
 	hdr = http_find_header(hh, hdr);
 	if (hdr != NULL)
 		return (hdr);
-	return (spec);
+	return ("<undef>");
 }
 
 static void
 cmd_http_expect(CMD_ARGS)
 {
 	struct http *hp;
-	char *lhs;
+	const char *lhs;
 	char *cmp;
-	char *rhs;
+	const char *rhs;
 
 	(void)cmd;
 	(void)vl;



From geoff at varnish-cache.org  Mon Jan  9 20:52:43 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:43 +0100
Subject: [experimental-ims] 1bf8b02 Reorder code in cnt_done() for improved
	clarity
Message-ID: <E1RkMCx-0008SH-0Q@project.varnish-software.com>

commit 1bf8b02f961c2d0a2f567bce5bdd2374db017e67
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Nov 29 11:23:08 2011 +0000

    Reorder code in cnt_done() for improved clarity

diff --git a/bin/varnishd/cache/cache_center.c b/bin/varnishd/cache/cache_center.c
index 89886c2..b79aa31 100644
--- a/bin/varnishd/cache/cache_center.c
+++ b/bin/varnishd/cache/cache_center.c
@@ -330,14 +330,19 @@ cnt_done(struct sess *sp)
 	sp->wrk->is_gunzip = 0;
 	sp->wrk->is_gzip = 0;
 
-	if (sp->vcl != NULL && sp->esi_level == 0) {
+	SES_Charge(sp);
+
+	/* If we did an ESI include, don't mess up our state */
+	if (sp->esi_level > 0)
+		return (1);
+
+	if (sp->vcl != NULL) {
 		if (sp->wrk->vcl != NULL)
 			VCL_Rel(&sp->wrk->vcl);
 		sp->wrk->vcl = sp->vcl;
 		sp->vcl = NULL;
 	}
 
-	SES_Charge(sp);
 
 	sp->t_end = W_TIM_real(sp->wrk);
 WSP(sp, SLT_Debug, "PHK req %.9f resp %.9f end %.9f open %.9f",
@@ -345,7 +350,7 @@ WSP(sp, SLT_Debug, "PHK req %.9f resp %.9f end %.9f open %.9f",
 	if (sp->xid == 0) {
 		// sp->t_req = sp->t_end;
 		sp->t_resp = sp->t_end;
-	} else if (sp->esi_level == 0) {
+	} else {
 		dp = sp->t_resp - sp->t_req;
 		da = sp->t_end - sp->t_resp;
 		dh = sp->t_req - sp->t_open;
@@ -361,10 +366,6 @@ WSP(sp, SLT_Debug, "PHK req %.9f resp %.9f end %.9f open %.9f",
 	sp->xid = 0;
 	WSL_Flush(sp->wrk, 0);
 
-	/* If we did an ESI include, don't mess up our state */
-	if (sp->esi_level > 0)
-		return (1);
-
 	sp->t_open = sp->t_end;
 	sp->t_resp = NAN;
 



From geoff at varnish-cache.org  Mon Jan  9 20:52:43 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:43 +0100
Subject: [experimental-ims] 3150388 The obj and objcore are not really
	properties of the session,
	and we need to be able to reach them during fetching without worrying
	about a particular session.
Message-ID: <E1RkMCx-0008SQ-5S@project.varnish-software.com>

commit 3150388c776ce53d94451cf7364020000710e55e
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Nov 29 12:12:00 2011 +0000

    The obj and objcore are not really properties of the session, and we
    need to be able to reach them during fetching without worrying about
    a particular session.
    
    Move them to the worker thread, which corresponds much closer to their
    lifetime.
    
    This is mostly a brute force s/sp->obj/sp->wrk->obj/, with a few
    necessary edits.  Consequential fallouts and sp-scope-elimination to
    follow in subsequent commits.

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index 1475d6d..db001ba 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -321,6 +321,9 @@ struct worker {
 	struct http		*beresp;
 	struct http		*resp;
 
+	struct object		*obj;
+	struct objcore		*objcore;
+
 	struct exp		exp;
 
 	/* This is only here so VRT can find it */
@@ -599,8 +602,6 @@ struct sess {
 	VTAILQ_ENTRY(sess)	list;
 
 	struct director		*director;
-	struct object		*obj;
-	struct objcore		*objcore;
 	struct VCL_conf		*vcl;
 
 	/* The busy objhead we sleep on */
diff --git a/bin/varnishd/cache/cache_backend.c b/bin/varnishd/cache/cache_backend.c
index 1f290c3..de4318f 100644
--- a/bin/varnishd/cache/cache_backend.c
+++ b/bin/varnishd/cache/cache_backend.c
@@ -230,7 +230,7 @@ vbe_NewConn(void)
 
 /*--------------------------------------------------------------------
  * It evaluates if a backend is healthy _for_a_specific_object_.
- * That means that it relies on sp->objcore->objhead. This is mainly for
+ * That means that it relies on sp->wrk->objcore->objhead. This is mainly for
  * saint-mode, but also takes backend->healthy into account. If
  * cache_param->saintmode_threshold is 0, this is basically just a test of
  * backend->healthy.
@@ -277,11 +277,11 @@ vbe_Healthy(const struct vdi_simple *vs, const struct sess *sp)
 	if (threshold == 0)
 		return (1);
 
-	if (sp->objcore == NULL)
+	if (sp->wrk->objcore == NULL)
 		return (1);
 
 	now = sp->t_req;
-	target = (uintptr_t)(sp->objcore->objhead);
+	target = (uintptr_t)(sp->wrk->objcore->objhead);
 
 	old = NULL;
 	retval = 1;
diff --git a/bin/varnishd/cache/cache_center.c b/bin/varnishd/cache/cache_center.c
index b79aa31..28c4760 100644
--- a/bin/varnishd/cache/cache_center.c
+++ b/bin/varnishd/cache/cache_center.c
@@ -101,7 +101,7 @@ cnt_wait(struct sess *sp)
 
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 	AZ(sp->vcl);
-	AZ(sp->obj);
+	AZ(sp->wrk->obj);
 	AZ(sp->esi_level);
 	assert(sp->xid == 0);
 
@@ -170,11 +170,11 @@ cnt_prepresp(struct sess *sp)
 {
 
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
-	CHECK_OBJ_NOTNULL(sp->obj, OBJECT_MAGIC);
+	CHECK_OBJ_NOTNULL(sp->wrk->obj, OBJECT_MAGIC);
 	CHECK_OBJ_NOTNULL(sp->vcl, VCL_CONF_MAGIC);
 
 	if (sp->wrk->do_stream)
-		AssertObjCorePassOrBusy(sp->obj->objcore);
+		AssertObjCorePassOrBusy(sp->wrk->obj->objcore);
 
 	sp->wrk->res_mode = 0;
 
@@ -182,7 +182,7 @@ cnt_prepresp(struct sess *sp)
 	    !sp->wrk->do_gzip && !sp->wrk->do_gunzip)
 		sp->wrk->res_mode |= RES_LEN;
 
-	if (!sp->disable_esi && sp->obj->esidata != NULL) {
+	if (!sp->disable_esi && sp->wrk->obj->esidata != NULL) {
 		/* In ESI mode, we don't know the aggregate length */
 		sp->wrk->res_mode &= ~RES_LEN;
 		sp->wrk->res_mode |= RES_ESI;
@@ -193,7 +193,7 @@ cnt_prepresp(struct sess *sp)
 		sp->wrk->res_mode |= RES_ESI_CHILD;
 	}
 
-	if (cache_param->http_gzip_support && sp->obj->gziped &&
+	if (cache_param->http_gzip_support && sp->wrk->obj->gziped &&
 	    !RFC2616_Req_Gzip(sp)) {
 		/*
 		 * We don't know what it uncompresses to
@@ -204,7 +204,7 @@ cnt_prepresp(struct sess *sp)
 	}
 
 	if (!(sp->wrk->res_mode & (RES_LEN|RES_CHUNKED|RES_EOF))) {
-		if (sp->obj->len == 0 && !sp->wrk->do_stream)
+		if (sp->wrk->obj->len == 0 && !sp->wrk->do_stream)
 			/*
 			 * If the object is empty, neither ESI nor GUNZIP
 			 * can make it any different size
@@ -221,11 +221,11 @@ cnt_prepresp(struct sess *sp)
 	}
 
 	sp->t_resp = W_TIM_real(sp->wrk);
-	if (sp->obj->objcore != NULL) {
-		if ((sp->t_resp - sp->obj->last_lru) > cache_param->lru_timeout &&
-		    EXP_Touch(sp->obj->objcore))
-			sp->obj->last_lru = sp->t_resp;
-		sp->obj->last_use = sp->t_resp;	/* XXX: locking ? */
+	if (sp->wrk->obj->objcore != NULL) {
+		if ((sp->t_resp - sp->wrk->obj->last_lru) > cache_param->lru_timeout &&
+		    EXP_Touch(sp->wrk->obj->objcore))
+			sp->wrk->obj->last_lru = sp->t_resp;
+		sp->wrk->obj->last_use = sp->t_resp;	/* XXX: locking ? */
 	}
 	http_Setup(sp->wrk->resp, sp->wrk->ws);
 	RES_BuildHttp(sp);
@@ -240,9 +240,9 @@ cnt_prepresp(struct sess *sp)
 			VDI_CloseFd(sp->wrk);
 			HSH_Drop(sp);
 		} else {
-			(void)HSH_Deref(sp->wrk, NULL, &sp->obj);
+			(void)HSH_Deref(sp->wrk, NULL, &sp->wrk->obj);
 		}
-		AZ(sp->obj);
+		AZ(sp->wrk->obj);
 		sp->restarts++;
 		sp->director = NULL;
 		sp->wrk->h_content_length = NULL;
@@ -255,7 +255,7 @@ cnt_prepresp(struct sess *sp)
 		WRONG("Illegal action in vcl_deliver{}");
 	}
 	if (sp->wrk->do_stream) {
-		AssertObjCorePassOrBusy(sp->obj->objcore);
+		AssertObjCorePassOrBusy(sp->wrk->obj->objcore);
 		sp->step = STP_STREAMBODY;
 	} else {
 		sp->step = STP_DELIVER;
@@ -289,7 +289,7 @@ cnt_deliver(struct sess *sp)
 
 	assert(WRW_IsReleased(sp->wrk));
 	assert(sp->wrk->wrw.ciov == sp->wrk->wrw.siov);
-	(void)HSH_Deref(sp->wrk, NULL, &sp->obj);
+	(void)HSH_Deref(sp->wrk, NULL, &sp->wrk->obj);
 	http_Setup(sp->wrk->resp, NULL);
 	sp->step = STP_DONE;
 	return (0);
@@ -318,7 +318,7 @@ cnt_done(struct sess *sp)
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 	CHECK_OBJ_ORNULL(sp->vcl, VCL_CONF_MAGIC);
 
-	AZ(sp->obj);
+	AZ(sp->wrk->obj);
 	AZ(sp->wrk->vbc);
 	sp->director = NULL;
 	sp->restarts = 0;
@@ -450,16 +450,16 @@ cnt_error(struct sess *sp)
 	sp->wrk->do_stream = 0;
 
 	w = sp->wrk;
-	if (sp->obj == NULL) {
+	if (sp->wrk->obj == NULL) {
 		HSH_Prealloc(sp);
 		EXP_Clr(&w->exp);
-		sp->obj = STV_NewObject(sp, NULL, cache_param->http_resp_size,
+		sp->wrk->obj = STV_NewObject(sp, NULL, cache_param->http_resp_size,
 		     &w->exp, (uint16_t)cache_param->http_max_hdr);
-		if (sp->obj == NULL)
-			sp->obj = STV_NewObject(sp, TRANSIENT_STORAGE,
+		if (sp->wrk->obj == NULL)
+			sp->wrk->obj = STV_NewObject(sp, TRANSIENT_STORAGE,
 			    cache_param->http_resp_size, &w->exp,
 			    (uint16_t)cache_param->http_max_hdr);
-		if (sp->obj == NULL) {
+		if (sp->wrk->obj == NULL) {
 			sp->doclose = "Out of objects";
 			sp->director = NULL;
 			sp->wrk->h_content_length = NULL;
@@ -468,14 +468,14 @@ cnt_error(struct sess *sp)
 			sp->step = STP_DONE;
 			return(0);
 		}
-		AN(sp->obj);
-		sp->obj->xid = sp->xid;
-		sp->obj->exp.entered = sp->t_req;
+		AN(sp->wrk->obj);
+		sp->wrk->obj->xid = sp->xid;
+		sp->wrk->obj->exp.entered = sp->t_req;
 	} else {
 		/* XXX: Null the headers ? */
 	}
-	CHECK_OBJ_NOTNULL(sp->obj, OBJECT_MAGIC);
-	h = sp->obj->http;
+	CHECK_OBJ_NOTNULL(sp->wrk->obj, OBJECT_MAGIC);
+	h = sp->wrk->obj->http;
 
 	if (sp->err_code < 100 || sp->err_code > 999)
 		sp->err_code = 501;
@@ -601,7 +601,7 @@ cnt_fetch(struct sess *sp)
 		RFC2616_Ttl(sp);
 
 		/* pass from vclrecv{} has negative TTL */
-		if (sp->objcore == NULL)
+		if (sp->wrk->objcore == NULL)
 			sp->wrk->exp.ttl = -1.;
 
 		AZ(sp->wrk->do_esi);
@@ -610,12 +610,12 @@ cnt_fetch(struct sess *sp)
 
 		switch (sp->handling) {
 		case VCL_RET_HIT_FOR_PASS:
-			if (sp->objcore != NULL)
-				sp->objcore->flags |= OC_F_PASS;
+			if (sp->wrk->objcore != NULL)
+				sp->wrk->objcore->flags |= OC_F_PASS;
 			sp->step = STP_FETCHBODY;
 			return (0);
 		case VCL_RET_DELIVER:
-			AssertObjCorePassOrBusy(sp->objcore);
+			AssertObjCorePassOrBusy(sp->wrk->objcore);
 			sp->step = STP_FETCHBODY;
 			return (0);
 		default:
@@ -629,10 +629,10 @@ cnt_fetch(struct sess *sp)
 	/* Clean up partial fetch */
 	AZ(sp->wrk->vbc);
 
-	if (sp->objcore != NULL) {
-		CHECK_OBJ_NOTNULL(sp->objcore, OBJCORE_MAGIC);
-		AZ(HSH_Deref(sp->wrk, sp->objcore, NULL));
-		sp->objcore = NULL;
+	if (sp->wrk->objcore != NULL) {
+		CHECK_OBJ_NOTNULL(sp->wrk->objcore, OBJCORE_MAGIC);
+		AZ(HSH_Deref(sp->wrk, sp->wrk->objcore, NULL));
+		sp->wrk->objcore = NULL;
 	}
 	http_Setup(sp->wrk->bereq, NULL);
 	http_Setup(sp->wrk->beresp, NULL);
@@ -688,7 +688,7 @@ cnt_fetchbody(struct sess *sp)
 	assert(sp->handling == VCL_RET_HIT_FOR_PASS ||
 	    sp->handling == VCL_RET_DELIVER);
 
-	if (sp->objcore == NULL) {
+	if (sp->wrk->objcore == NULL) {
 		/* This is a pass from vcl_recv */
 		pass = 1;
 		/* VCL may have fiddled this, but that doesn't help */
@@ -769,8 +769,8 @@ cnt_fetchbody(struct sess *sp)
 	    pass ? HTTPH_R_PASS : HTTPH_A_INS, &nhttp);
 
 	/* Create Vary instructions */
-	if (sp->objcore != NULL) {
-		CHECK_OBJ_NOTNULL(sp->objcore, OBJCORE_MAGIC);
+	if (sp->wrk->objcore != NULL) {
+		CHECK_OBJ_NOTNULL(sp->wrk->objcore, OBJCORE_MAGIC);
 		vary = VRY_Create(sp, sp->wrk->beresp);
 		if (vary != NULL) {
 			varyl = VSB_len(vary);
@@ -785,52 +785,52 @@ cnt_fetchbody(struct sess *sp)
 	 */
 	l += strlen("Content-Length: XxxXxxXxxXxxXxxXxx") + sizeof(void *);
 
-	if (sp->wrk->exp.ttl < cache_param->shortlived || sp->objcore == NULL)
+	if (sp->wrk->exp.ttl < cache_param->shortlived || sp->wrk->objcore == NULL)
 		sp->wrk->storage_hint = TRANSIENT_STORAGE;
 
-	sp->obj = STV_NewObject(sp, sp->wrk->storage_hint, l,
+	sp->wrk->obj = STV_NewObject(sp, sp->wrk->storage_hint, l,
 	    &sp->wrk->exp, nhttp);
-	if (sp->obj == NULL) {
+	if (sp->wrk->obj == NULL) {
 		/*
 		 * Try to salvage the transaction by allocating a
 		 * shortlived object on Transient storage.
 		 */
-		sp->obj = STV_NewObject(sp, TRANSIENT_STORAGE, l,
+		sp->wrk->obj = STV_NewObject(sp, TRANSIENT_STORAGE, l,
 		    &sp->wrk->exp, nhttp);
 		if (sp->wrk->exp.ttl > cache_param->shortlived)
 			sp->wrk->exp.ttl = cache_param->shortlived;
 		sp->wrk->exp.grace = 0.0;
 		sp->wrk->exp.keep = 0.0;
 	}
-	if (sp->obj == NULL) {
+	if (sp->wrk->obj == NULL) {
 		sp->err_code = 503;
 		sp->step = STP_ERROR;
 		VDI_CloseFd(sp->wrk);
 		return (0);
 	}
-	CHECK_OBJ_NOTNULL(sp->obj, OBJECT_MAGIC);
+	CHECK_OBJ_NOTNULL(sp->wrk->obj, OBJECT_MAGIC);
 
 	sp->wrk->storage_hint = NULL;
 
 	if (sp->wrk->do_gzip || (sp->wrk->is_gzip && !sp->wrk->do_gunzip))
-		sp->obj->gziped = 1;
+		sp->wrk->obj->gziped = 1;
 
 	if (vary != NULL) {
-		sp->obj->vary =
-		    (void *)WS_Alloc(sp->obj->http->ws, varyl);
-		AN(sp->obj->vary);
-		memcpy(sp->obj->vary, VSB_data(vary), varyl);
-		VRY_Validate(sp->obj->vary);
+		sp->wrk->obj->vary =
+		    (void *)WS_Alloc(sp->wrk->obj->http->ws, varyl);
+		AN(sp->wrk->obj->vary);
+		memcpy(sp->wrk->obj->vary, VSB_data(vary), varyl);
+		VRY_Validate(sp->wrk->obj->vary);
 		VSB_delete(vary);
 	}
 
-	sp->obj->xid = sp->xid;
-	sp->obj->response = sp->err_code;
-	WS_Assert(sp->obj->ws_o);
+	sp->wrk->obj->xid = sp->xid;
+	sp->wrk->obj->response = sp->err_code;
+	WS_Assert(sp->wrk->obj->ws_o);
 
 	/* Filter into object */
 	hp = sp->wrk->beresp;
-	hp2 = sp->obj->http;
+	hp2 = sp->wrk->obj->http;
 
 	hp2->logtag = HTTP_Obj;
 	http_CopyResp(hp2, hp);
@@ -839,9 +839,9 @@ cnt_fetchbody(struct sess *sp)
 	http_CopyHome(sp->wrk, sp->vsl_id, hp2);
 
 	if (http_GetHdr(hp, H_Last_Modified, &b))
-		sp->obj->last_modified = VTIM_parse(b);
+		sp->wrk->obj->last_modified = VTIM_parse(b);
 	else
-		sp->obj->last_modified = floor(sp->wrk->exp.entered);
+		sp->wrk->obj->last_modified = floor(sp->wrk->exp.entered);
 
 	assert(WRW_IsReleased(sp->wrk));
 
@@ -850,12 +850,12 @@ cnt_fetchbody(struct sess *sp)
 	 * Notice that vcl_deliver{} could still nuke the headers
 	 * that allow the 304, in which case we return 200 non-stream.
 	 */
-	if (sp->obj->response == 200 &&
+	if (sp->wrk->obj->response == 200 &&
 	    sp->http->conds &&
 	    RFC2616_Do_Cond(sp))
 		sp->wrk->do_stream = 0;
 
-	AssertObjCorePassOrBusy(sp->obj->objcore);
+	AssertObjCorePassOrBusy(sp->wrk->obj->objcore);
 
 	if (sp->wrk->do_stream) {
 		sp->step = STP_PREPRESP;
@@ -863,7 +863,7 @@ cnt_fetchbody(struct sess *sp)
 	}
 
 	/* Use unmodified headers*/
-	i = FetchBody(sp->wrk, sp->obj);
+	i = FetchBody(sp->wrk, sp->wrk->obj);
 
 	sp->wrk->h_content_length = NULL;
 
@@ -876,16 +876,16 @@ cnt_fetchbody(struct sess *sp)
 
 	if (i) {
 		HSH_Drop(sp);
-		AZ(sp->obj);
+		AZ(sp->wrk->obj);
 		sp->err_code = 503;
 		sp->step = STP_ERROR;
 		return (0);
 	}
 
-	if (sp->obj->objcore != NULL) {
-		EXP_Insert(sp->obj);
-		AN(sp->obj->objcore);
-		AN(sp->obj->objcore->ban);
+	if (sp->wrk->obj->objcore != NULL) {
+		EXP_Insert(sp->wrk->obj);
+		AN(sp->wrk->obj->objcore);
+		AN(sp->wrk->obj->objcore->ban);
 		HSH_Unbusy(sp);
 	}
 	sp->wrk->acct_tmp.fetch++;
@@ -925,9 +925,9 @@ cnt_streambody(struct sess *sp)
 
 	RES_StreamStart(sp);
 
-	AssertObjCorePassOrBusy(sp->obj->objcore);
+	AssertObjCorePassOrBusy(sp->wrk->obj->objcore);
 
-	i = FetchBody(sp->wrk, sp->obj);
+	i = FetchBody(sp->wrk, sp->wrk->obj);
 
 	sp->wrk->h_content_length = NULL;
 
@@ -937,10 +937,10 @@ cnt_streambody(struct sess *sp)
 	AZ(sp->wrk->vbc);
 	AN(sp->director);
 
-	if (!i && sp->obj->objcore != NULL) {
-		EXP_Insert(sp->obj);
-		AN(sp->obj->objcore);
-		AN(sp->obj->objcore->ban);
+	if (!i && sp->wrk->obj->objcore != NULL) {
+		EXP_Insert(sp->wrk->obj);
+		AN(sp->wrk->obj->objcore);
+		AN(sp->wrk->obj->objcore->ban);
 		HSH_Unbusy(sp);
 	} else {
 		sp->doclose = "Stream error";
@@ -956,7 +956,7 @@ cnt_streambody(struct sess *sp)
 	sp->wrk->sctx = NULL;
 	assert(WRW_IsReleased(sp->wrk));
 	assert(sp->wrk->wrw.ciov == sp->wrk->wrw.siov);
-	(void)HSH_Deref(sp->wrk, NULL, &sp->obj);
+	(void)HSH_Deref(sp->wrk, NULL, &sp->wrk->obj);
 	http_Setup(sp->wrk->resp, NULL);
 	sp->step = STP_DONE;
 	return (0);
@@ -1024,10 +1024,10 @@ cnt_hit(struct sess *sp)
 {
 
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
-	CHECK_OBJ_NOTNULL(sp->obj, OBJECT_MAGIC);
+	CHECK_OBJ_NOTNULL(sp->wrk->obj, OBJECT_MAGIC);
 	CHECK_OBJ_NOTNULL(sp->vcl, VCL_CONF_MAGIC);
 
-	assert(!(sp->obj->objcore->flags & OC_F_PASS));
+	assert(!(sp->wrk->obj->objcore->flags & OC_F_PASS));
 
 	AZ(sp->wrk->do_stream);
 
@@ -1043,8 +1043,8 @@ cnt_hit(struct sess *sp)
 	}
 
 	/* Drop our object, we won't need it */
-	(void)HSH_Deref(sp->wrk, NULL, &sp->obj);
-	sp->objcore = NULL;
+	(void)HSH_Deref(sp->wrk, NULL, &sp->wrk->obj);
+	sp->wrk->objcore = NULL;
 
 	switch(sp->handling) {
 	case VCL_RET_PASS:
@@ -1146,14 +1146,14 @@ cnt_lookup(struct sess *sp)
 		sp->vary_l = NULL;
 		sp->vary_e = NULL;
 
-		sp->objcore = oc;
+		sp->wrk->objcore = oc;
 		sp->step = STP_MISS;
 		return (0);
 	}
 
 	o = oc_getobj(sp->wrk, oc);
 	CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC);
-	sp->obj = o;
+	sp->wrk->obj = o;
 
 	WS_Release(sp->ws, 0);
 	sp->vary_b = NULL;
@@ -1162,15 +1162,15 @@ cnt_lookup(struct sess *sp)
 
 	if (oc->flags & OC_F_PASS) {
 		sp->wrk->stats.cache_hitpass++;
-		WSP(sp, SLT_HitPass, "%u", sp->obj->xid);
-		(void)HSH_Deref(sp->wrk, NULL, &sp->obj);
-		sp->objcore = NULL;
+		WSP(sp, SLT_HitPass, "%u", sp->wrk->obj->xid);
+		(void)HSH_Deref(sp->wrk, NULL, &sp->wrk->obj);
+		sp->wrk->objcore = NULL;
 		sp->step = STP_PASS;
 		return (0);
 	}
 
 	sp->wrk->stats.cache_hit++;
-	WSP(sp, SLT_Hit, "%u", sp->obj->xid);
+	WSP(sp, SLT_Hit, "%u", sp->wrk->obj->xid);
 	sp->step = STP_HIT;
 	return (0);
 }
@@ -1205,8 +1205,8 @@ cnt_miss(struct sess *sp)
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 	CHECK_OBJ_NOTNULL(sp->vcl, VCL_CONF_MAGIC);
 
-	AZ(sp->obj);
-	AN(sp->objcore);
+	AZ(sp->wrk->obj);
+	AN(sp->wrk->objcore);
 	WS_Reset(sp->wrk->ws, NULL);
 	http_Setup(sp->wrk->bereq, sp->wrk->ws);
 	http_FilterHeader(sp, HTTPH_R_FETCH);
@@ -1227,22 +1227,22 @@ cnt_miss(struct sess *sp)
 	VCL_miss_method(sp);
 	switch(sp->handling) {
 	case VCL_RET_ERROR:
-		AZ(HSH_Deref(sp->wrk, sp->objcore, NULL));
-		sp->objcore = NULL;
+		AZ(HSH_Deref(sp->wrk, sp->wrk->objcore, NULL));
+		sp->wrk->objcore = NULL;
 		http_Setup(sp->wrk->bereq, NULL);
 		sp->step = STP_ERROR;
 		return (0);
 	case VCL_RET_PASS:
-		AZ(HSH_Deref(sp->wrk, sp->objcore, NULL));
-		sp->objcore = NULL;
+		AZ(HSH_Deref(sp->wrk, sp->wrk->objcore, NULL));
+		sp->wrk->objcore = NULL;
 		sp->step = STP_PASS;
 		return (0);
 	case VCL_RET_FETCH:
 		sp->step = STP_FETCH;
 		return (0);
 	case VCL_RET_RESTART:
-		AZ(HSH_Deref(sp->wrk, sp->objcore, NULL));
-		sp->objcore = NULL;
+		AZ(HSH_Deref(sp->wrk, sp->wrk->objcore, NULL));
+		sp->wrk->objcore = NULL;
 		INCOMPL();
 	default:
 		WRONG("Illegal action in vcl_miss{}");
@@ -1287,7 +1287,7 @@ cnt_pass(struct sess *sp)
 
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 	CHECK_OBJ_NOTNULL(sp->vcl, VCL_CONF_MAGIC);
-	AZ(sp->obj);
+	AZ(sp->wrk->obj);
 
 	WS_Reset(sp->wrk->ws, NULL);
 	http_Setup(sp->wrk->bereq, sp->wrk->ws);
@@ -1386,7 +1386,7 @@ cnt_recv(struct sess *sp)
 
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 	CHECK_OBJ_NOTNULL(sp->vcl, VCL_CONF_MAGIC);
-	AZ(sp->obj);
+	AZ(sp->wrk->obj);
 	assert(sp->wrk->wrw.ciov == sp->wrk->wrw.siov);
 
 	/* By default we use the first backend */
@@ -1489,7 +1489,7 @@ cnt_start(struct sess *sp)
 
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 	AZ(sp->restarts);
-	AZ(sp->obj);
+	AZ(sp->wrk->obj);
 	AZ(sp->vcl);
 	AZ(sp->esi_level);
 
@@ -1568,12 +1568,12 @@ cnt_diag(struct sess *sp, const char *state)
 {
 	if (sp->wrk != NULL) {
 		WSP(sp, SLT_Debug, "thr %p STP_%s sp %p obj %p vcl %p",
-		    pthread_self(), state, sp, sp->obj, sp->vcl);
+		    pthread_self(), state, sp, sp->wrk->obj, sp->vcl);
 		WSL_Flush(sp->wrk, 0);
 	} else {
 		VSL(SLT_Debug, sp->vsl_id,
 		    "thr %p STP_%s sp %p obj %p vcl %p",
-		    pthread_self(), state, sp, sp->obj, sp->vcl);
+		    pthread_self(), state, sp, sp->wrk->obj, sp->vcl);
 	}
 }
 
@@ -1602,6 +1602,8 @@ CNT_Session(struct sess *sp)
 	AZ(w->is_gunzip);
 	AZ(w->do_gunzip);
 	AZ(w->do_esi);
+	AZ(w->obj);
+	AZ(w->objcore);
 
 	/*
 	 * Whenever we come in from the acceptor or waiter, we need to set
@@ -1630,7 +1632,7 @@ CNT_Session(struct sess *sp)
 		 * pointers still pointing to the things we expect.
 		 */
 		CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
-		CHECK_OBJ_ORNULL(sp->obj, OBJECT_MAGIC);
+		CHECK_OBJ_ORNULL(sp->wrk->obj, OBJECT_MAGIC);
 		CHECK_OBJ_NOTNULL(sp->wrk, WORKER_MAGIC);
 		CHECK_OBJ_ORNULL(w->nobjhead, OBJHEAD_MAGIC);
 		WS_Assert(w->ws);
@@ -1651,6 +1653,8 @@ CNT_Session(struct sess *sp)
 		CHECK_OBJ_ORNULL(w->nobjhead, OBJHEAD_MAGIC);
 	}
 	WSL_Flush(w, 0);
+	AZ(w->obj);
+	AZ(w->objcore);
 	AZ(w->do_stream);
 	AZ(w->is_gzip);
 	AZ(w->do_gzip);
diff --git a/bin/varnishd/cache/cache_esi_deliver.c b/bin/varnishd/cache/cache_esi_deliver.c
index 4051027..5ac8e71 100644
--- a/bin/varnishd/cache/cache_esi_deliver.c
+++ b/bin/varnishd/cache/cache_esi_deliver.c
@@ -58,8 +58,8 @@ ved_include(struct sess *sp, const char *src, const char *host)
 
 	(void)WRW_FlushRelease(w);
 
-	obj = sp->obj;
-	sp->obj = NULL;
+	obj = sp->wrk->obj;
+	sp->wrk->obj = NULL;
 	res_mode = sp->wrk->res_mode;
 
 	/* Reset request to status before we started messing with it */
@@ -113,7 +113,7 @@ ved_include(struct sess *sp, const char *src, const char *host)
 	AN(sp->wrk);
 	assert(sp->step == STP_DONE);
 	sp->esi_level--;
-	sp->obj = obj;
+	sp->wrk->obj = obj;
 	sp->wrk->res_mode = res_mode;
 
 	/* Reset the workspace */
@@ -238,7 +238,7 @@ ESI_Deliver(struct sess *sp)
 	int i;
 
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
-	st = sp->obj->esidata;
+	st = sp->wrk->obj->esidata;
 	AN(st);
 	assert(sizeof obuf >= 1024);
 
@@ -283,7 +283,7 @@ ESI_Deliver(struct sess *sp)
 		obufl = 0;
 	}
 
-	st = VTAILQ_FIRST(&sp->obj->store);
+	st = VTAILQ_FIRST(&sp->wrk->obj->store);
 	off = 0;
 
 	while (p < e) {
@@ -440,7 +440,7 @@ ved_deliver_byterange(const struct sess *sp, ssize_t low, ssize_t high)
 
 //printf("BR %jd %jd\n", low, high);
 	lx = 0;
-	VTAILQ_FOREACH(st, &sp->obj->store, list) {
+	VTAILQ_FOREACH(st, &sp->wrk->obj->store, list) {
 		p = st->ptr;
 		l = st->len;
 //printf("[0-] %jd %jd\n", lx, lx + l);
@@ -481,8 +481,8 @@ ESI_DeliverChild(const struct sess *sp)
 	uint32_t ilen;
 	uint8_t *dbits;
 
-	if (!sp->obj->gziped) {
-		VTAILQ_FOREACH(st, &sp->obj->store, list)
+	if (!sp->wrk->obj->gziped) {
+		VTAILQ_FOREACH(st, &sp->wrk->obj->store, list)
 			ved_pretend_gzip(sp, st->ptr, st->len);
 		return;
 	}
@@ -494,7 +494,7 @@ ESI_DeliverChild(const struct sess *sp)
 
 	dbits = (void*)WS_Alloc(sp->wrk->ws, 8);
 	AN(dbits);
-	obj = sp->obj;
+	obj = sp->wrk->obj;
 	CHECK_OBJ_NOTNULL(obj, OBJECT_MAGIC);
 	start = obj->gzip_start;
 	last = obj->gzip_last;
@@ -559,7 +559,7 @@ ESI_DeliverChild(const struct sess *sp)
 	}
 	if (lpad > 0)
 		(void)WRW_Write(sp->wrk, dbits + 1, lpad);
-	st = VTAILQ_LAST(&sp->obj->store, storagehead);
+	st = VTAILQ_LAST(&sp->wrk->obj->store, storagehead);
 	assert(st->len > 8);
 
 	p = st->ptr + st->len - 8;
diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c
index a5c0323..66241bc 100644
--- a/bin/varnishd/cache/cache_fetch.c
+++ b/bin/varnishd/cache/cache_fetch.c
@@ -389,11 +389,11 @@ FetchHdr(struct sess *sp)
 	w = sp->wrk;
 
 	AN(sp->director);
-	AZ(sp->obj);
+	AZ(sp->wrk->obj);
 
-	if (sp->objcore != NULL) {		/* pass has no objcore */
-		CHECK_OBJ_NOTNULL(sp->objcore, OBJCORE_MAGIC);
-		AN(sp->objcore->flags & OC_F_BUSY);
+	if (sp->wrk->objcore != NULL) {		/* pass has no objcore */
+		CHECK_OBJ_NOTNULL(sp->wrk->objcore, OBJCORE_MAGIC);
+		AN(sp->wrk->objcore->flags & OC_F_BUSY);
 	}
 
 	hp = w->bereq;
diff --git a/bin/varnishd/cache/cache_gzip.c b/bin/varnishd/cache/cache_gzip.c
index 32a7413..57b25a5 100644
--- a/bin/varnishd/cache/cache_gzip.c
+++ b/bin/varnishd/cache/cache_gzip.c
@@ -248,7 +248,7 @@ VGZ_ObufFull(const struct vgz *vg)
 
 /*--------------------------------------------------------------------
  * Keep the outbuffer supplied with storage and file it under the
- * sp->obj as it fills.
+ * sp->wrk->obj as it fills.
  */
 
 int
diff --git a/bin/varnishd/cache/cache_hash.c b/bin/varnishd/cache/cache_hash.c
index db865de..2820c45 100644
--- a/bin/varnishd/cache/cache_hash.c
+++ b/bin/varnishd/cache/cache_hash.c
@@ -395,8 +395,8 @@ HSH_Lookup(struct sess *sp, struct objhead **poh)
 	 * XXX: serialize fetch of all Vary's if grace is possible.
 	 */
 
-	AZ(sp->objcore);
-	sp->objcore = grace_oc;		/* XXX: Hack-ish */
+	AZ(sp->wrk->objcore);
+	sp->wrk->objcore = grace_oc;		/* XXX: Hack-ish */
 	if (oc == NULL			/* We found no live object */
 	    && grace_oc != NULL		/* There is a grace candidate */
 	    && (busy_oc != NULL		/* Somebody else is already busy */
@@ -406,7 +406,7 @@ HSH_Lookup(struct sess *sp, struct objhead **poh)
 		CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC);
 		oc = grace_oc;
 	}
-	sp->objcore = NULL;
+	sp->wrk->objcore = NULL;
 
 	if (oc != NULL && !sp->hash_always_miss) {
 		o = oc_getobj(sp->wrk, oc);
@@ -585,13 +585,13 @@ HSH_Drop(struct sess *sp)
 	struct object *o;
 
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
-	o = sp->obj;
+	o = sp->wrk->obj;
 	CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC);
 	AssertObjCorePassOrBusy(o->objcore);
 	o->exp.ttl = -1.;
 	if (o->objcore != NULL)		/* Pass has no objcore */
 		HSH_Unbusy(sp);
-	(void)HSH_Deref(sp->wrk, NULL, &sp->obj);
+	(void)HSH_Deref(sp->wrk, NULL, &sp->wrk->obj);
 }
 
 void
@@ -602,7 +602,7 @@ HSH_Unbusy(const struct sess *sp)
 	struct objcore *oc;
 
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
-	o = sp->obj;
+	o = sp->wrk->obj;
 	CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC);
 	oc = o->objcore;
 	CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
diff --git a/bin/varnishd/cache/cache_panic.c b/bin/varnishd/cache/cache_panic.c
index 9d69c1b..f6f1ff5 100644
--- a/bin/varnishd/cache/cache_panic.c
+++ b/bin/varnishd/cache/cache_panic.c
@@ -266,8 +266,8 @@ pan_sess(const struct sess *sp)
 	if (VALID_OBJ(sp->wrk->vbc, BACKEND_MAGIC))
 		pan_vbc(sp->wrk->vbc);
 
-	if (VALID_OBJ(sp->obj, OBJECT_MAGIC))
-		pan_object(sp->obj);
+	if (VALID_OBJ(sp->wrk->obj, OBJECT_MAGIC))
+		pan_object(sp->wrk->obj);
 
 	VSB_printf(pan_vsp, "},\n");
 }
diff --git a/bin/varnishd/cache/cache_pool.c b/bin/varnishd/cache/cache_pool.c
index 79a5fcd..730e391 100644
--- a/bin/varnishd/cache/cache_pool.c
+++ b/bin/varnishd/cache/cache_pool.c
@@ -361,7 +361,6 @@ Pool_Wait(struct sess *sp)
 {
 
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
-	AZ(sp->obj);
 	AZ(sp->vcl);
 	assert(sp->fd >= 0);
 	/*
diff --git a/bin/varnishd/cache/cache_response.c b/bin/varnishd/cache/cache_response.c
index 487a514..bdc0fca 100644
--- a/bin/varnishd/cache/cache_response.c
+++ b/bin/varnishd/cache/cache_response.c
@@ -41,7 +41,7 @@ res_dorange(const struct sess *sp, const char *r, ssize_t *plow, ssize_t *phigh)
 {
 	ssize_t low, high, has_low;
 
-	assert(sp->obj->response == 200);
+	assert(sp->wrk->obj->response == 200);
 	if (strncmp(r, "bytes=", 6))
 		return;
 	r += 6;
@@ -57,7 +57,7 @@ res_dorange(const struct sess *sp, const char *r, ssize_t *plow, ssize_t *phigh)
 		r++;
 	}
 
-	if (low >= sp->obj->len)
+	if (low >= sp->wrk->obj->len)
 		return;
 
 	if (*r != '-')
@@ -73,23 +73,23 @@ res_dorange(const struct sess *sp, const char *r, ssize_t *plow, ssize_t *phigh)
 			r++;
 		}
 		if (!has_low) {
-			low = sp->obj->len - high;
-			high = sp->obj->len - 1;
+			low = sp->wrk->obj->len - high;
+			high = sp->wrk->obj->len - 1;
 		}
 	} else
-		high = sp->obj->len - 1;
+		high = sp->wrk->obj->len - 1;
 	if (*r != '\0')
 		return;
 
-	if (high >= sp->obj->len)
-		high = sp->obj->len - 1;
+	if (high >= sp->wrk->obj->len)
+		high = sp->wrk->obj->len - 1;
 
 	if (low > high)
 		return;
 
 	http_PrintfHeader(sp->wrk, sp->vsl_id, sp->wrk->resp,
 	    "Content-Range: bytes %jd-%jd/%jd",
-	    (intmax_t)low, (intmax_t)high, (intmax_t)sp->obj->len);
+	    (intmax_t)low, (intmax_t)high, (intmax_t)sp->wrk->obj->len);
 	http_Unset(sp->wrk->resp, H_Content_Length);
 	assert(sp->wrk->res_mode & RES_LEN);
 	http_PrintfHeader(sp->wrk, sp->vsl_id, sp->wrk->resp,
@@ -112,8 +112,8 @@ RES_BuildHttp(const struct sess *sp)
 
 	http_ClrHeader(sp->wrk->resp);
 	sp->wrk->resp->logtag = HTTP_Tx;
-	http_CopyResp(sp->wrk->resp, sp->obj->http);
-	http_FilterFields(sp->wrk, sp->vsl_id, sp->wrk->resp, sp->obj->http,
+	http_CopyResp(sp->wrk->resp, sp->wrk->obj->http);
+	http_FilterFields(sp->wrk, sp->vsl_id, sp->wrk->resp, sp->wrk->obj->http,
 	    HTTPH_A_DELIVER);
 
 	if (!(sp->wrk->res_mode & RES_LEN)) {
@@ -131,14 +131,14 @@ RES_BuildHttp(const struct sess *sp)
 	VTIM_format(VTIM_real(), time_str);
 	http_PrintfHeader(sp->wrk, sp->vsl_id, sp->wrk->resp, "Date: %s", time_str);
 
-	if (sp->xid != sp->obj->xid)
+	if (sp->xid != sp->wrk->obj->xid)
 		http_PrintfHeader(sp->wrk, sp->vsl_id, sp->wrk->resp,
-		    "X-Varnish: %u %u", sp->xid, sp->obj->xid);
+		    "X-Varnish: %u %u", sp->xid, sp->wrk->obj->xid);
 	else
 		http_PrintfHeader(sp->wrk, sp->vsl_id, sp->wrk->resp,
 		    "X-Varnish: %u", sp->xid);
 	http_PrintfHeader(sp->wrk, sp->vsl_id, sp->wrk->resp, "Age: %.0f",
-	    sp->obj->exp.age + sp->t_resp - sp->obj->exp.entered);
+	    sp->wrk->obj->exp.age + sp->t_resp - sp->wrk->obj->exp.entered);
 	http_SetHeader(sp->wrk, sp->vsl_id, sp->wrk->resp, "Via: 1.1 varnish");
 	http_PrintfHeader(sp->wrk, sp->vsl_id, sp->wrk->resp, "Connection: %s",
 	    sp->doclose ? "close" : "keep-alive");
@@ -165,7 +165,7 @@ res_WriteGunzipObj(const struct sess *sp)
 	vg = VGZ_NewUngzip(sp->wrk, "U D -");
 
 	VGZ_Obuf(vg, obuf, sizeof obuf);
-	VTAILQ_FOREACH(st, &sp->obj->store, list) {
+	VTAILQ_FOREACH(st, &sp->wrk->obj->store, list) {
 		CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 		CHECK_OBJ_NOTNULL(st, STORAGE_MAGIC);
 		u += st->len;
@@ -183,7 +183,7 @@ res_WriteGunzipObj(const struct sess *sp)
 		(void)WRW_Flush(sp->wrk);
 	}
 	(void)VGZ_Destroy(&vg, sp->vsl_id);
-	assert(u == sp->obj->len);
+	assert(u == sp->wrk->obj->len);
 }
 
 /*--------------------------------------------------------------------*/
@@ -198,7 +198,7 @@ res_WriteDirObj(const struct sess *sp, ssize_t low, ssize_t high)
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 
 	ptr = 0;
-	VTAILQ_FOREACH(st, &sp->obj->store, list) {
+	VTAILQ_FOREACH(st, &sp->wrk->obj->store, list) {
 		CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 		CHECK_OBJ_NOTNULL(st, STORAGE_MAGIC);
 		u += st->len;
@@ -239,7 +239,7 @@ res_WriteDirObj(const struct sess *sp, ssize_t low, ssize_t high)
 		VSC_C_main->n_objwrite++;
 		(void)WRW_Write(sp->wrk, st->ptr + off, len);
 	}
-	assert(u == sp->obj->len);
+	assert(u == sp->wrk->obj->len);
 }
 
 /*--------------------------------------------------------------------
@@ -257,7 +257,7 @@ RES_WriteObj(struct sess *sp)
 
 	WRW_Reserve(sp->wrk, &sp->fd);
 
-	if (sp->obj->response == 200 &&
+	if (sp->wrk->obj->response == 200 &&
 	    sp->http->conds &&
 	    RFC2616_Do_Cond(sp)) {
 		sp->wantbody = 0;
@@ -270,13 +270,13 @@ RES_WriteObj(struct sess *sp)
 	 * If nothing special planned, we can attempt Range support
 	 */
 	low = 0;
-	high = sp->obj->len - 1;
+	high = sp->wrk->obj->len - 1;
 	if (
 	    sp->wantbody &&
 	    (sp->wrk->res_mode & RES_LEN) &&
 	    !(sp->wrk->res_mode & (RES_ESI|RES_ESI_CHILD|RES_GUNZIP)) &&
 	    cache_param->http_range_support &&
-	    sp->obj->response == 200 &&
+	    sp->wrk->obj->response == 200 &&
 	    http_GetHdr(sp->http, H_Range, &r))
 		res_dorange(sp, r, &low, &high);
 
@@ -301,14 +301,14 @@ RES_WriteObj(struct sess *sp)
 
 	if (!sp->wantbody) {
 		/* This was a HEAD or conditional request */
-	} else if (sp->obj->len == 0) {
+	} else if (sp->wrk->obj->len == 0) {
 		/* Nothing to do here */
 	} else if (sp->wrk->res_mode & RES_ESI) {
 		ESI_Deliver(sp);
 	} else if (sp->wrk->res_mode & RES_ESI_CHILD && sp->wrk->gzip_resp) {
 		ESI_DeliverChild(sp);
 	} else if (sp->wrk->res_mode & RES_ESI_CHILD &&
-	    !sp->wrk->gzip_resp && sp->obj->gziped) {
+	    !sp->wrk->gzip_resp && sp->wrk->obj->gziped) {
 		res_WriteGunzipObj(sp);
 	} else if (sp->wrk->res_mode & RES_GUNZIP) {
 		res_WriteGunzipObj(sp);
diff --git a/bin/varnishd/cache/cache_rfc2616.c b/bin/varnishd/cache/cache_rfc2616.c
index 4041f45..e22da50 100644
--- a/bin/varnishd/cache/cache_rfc2616.c
+++ b/bin/varnishd/cache/cache_rfc2616.c
@@ -315,18 +315,18 @@ RFC2616_Do_Cond(const struct sess *sp)
 	   and If-Modified-Since if present*/
 
 	if (http_GetHdr(sp->http, H_If_Modified_Since, &p) ) {
-		if (!sp->obj->last_modified)
+		if (!sp->wrk->obj->last_modified)
 			return (0);
 		ims = VTIM_parse(p);
 		if (ims > sp->t_req)	/* [RFC2616 14.25] */
 			return (0);
-		if (sp->obj->last_modified > ims)
+		if (sp->wrk->obj->last_modified > ims)
 			return (0);
 		do_cond = 1;
 	}
 
 	if (http_GetHdr(sp->http, H_If_None_Match, &p) &&
-	    http_GetHdr(sp->obj->http, H_ETag, &e)) {
+	    http_GetHdr(sp->wrk->obj->http, H_ETag, &e)) {
 		if (strcmp(p,e) != 0)
 			return (0);
 		do_cond = 1;
diff --git a/bin/varnishd/cache/cache_session.c b/bin/varnishd/cache/cache_session.c
index eee8e9a..21d7f51 100644
--- a/bin/varnishd/cache/cache_session.c
+++ b/bin/varnishd/cache/cache_session.c
@@ -339,7 +339,6 @@ SES_Delete(struct sess *sp, const char *reason)
 		SES_Close(sp, reason);
 	assert(sp->fd < 0);
 
-	AZ(sp->obj);
 	AZ(sp->vcl);
 	if (sp->addr == NULL)
 		sp->addr = noaddr;
diff --git a/bin/varnishd/cache/cache_vrt.c b/bin/varnishd/cache/cache_vrt.c
index 5e19ccc..a0eb4c8 100644
--- a/bin/varnishd/cache/cache_vrt.c
+++ b/bin/varnishd/cache/cache_vrt.c
@@ -108,8 +108,8 @@ vrt_selecthttp(const struct sess *sp, enum gethdr_e where)
 		hp = sp->wrk->resp;
 		break;
 	case HDR_OBJ:
-		CHECK_OBJ_NOTNULL(sp->obj, OBJECT_MAGIC);
-		hp = sp->obj->http;
+		CHECK_OBJ_NOTNULL(sp->wrk->obj, OBJECT_MAGIC);
+		hp = sp->wrk->obj->http;
 		break;
 	default:
 		INCOMPL();
@@ -399,8 +399,8 @@ VRT_synth_page(const struct sess *sp, unsigned flags, const char *str, ...)
 
 	(void)flags;
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
-	CHECK_OBJ_NOTNULL(sp->obj, OBJECT_MAGIC);
-	vsb = SMS_Makesynth(sp->obj);
+	CHECK_OBJ_NOTNULL(sp->wrk->obj, OBJECT_MAGIC);
+	vsb = SMS_Makesynth(sp->wrk->obj);
 	AN(vsb);
 
 	VSB_cat(vsb, str);
@@ -413,10 +413,10 @@ VRT_synth_page(const struct sess *sp, unsigned flags, const char *str, ...)
 		p = va_arg(ap, const char *);
 	}
 	va_end(ap);
-	SMS_Finish(sp->obj);
-	http_Unset(sp->obj->http, H_Content_Length);
-	http_PrintfHeader(sp->wrk, sp->vsl_id, sp->obj->http,
-	    "Content-Length: %d", sp->obj->len);
+	SMS_Finish(sp->wrk->obj);
+	http_Unset(sp->wrk->obj->http, H_Content_Length);
+	http_PrintfHeader(sp->wrk, sp->vsl_id, sp->wrk->obj->http,
+	    "Content-Length: %d", sp->wrk->obj->len);
 }
 
 /*--------------------------------------------------------------------*/
@@ -510,9 +510,9 @@ void
 VRT_purge(const struct sess *sp, double ttl, double grace)
 {
 	if (sp->cur_method == VCL_MET_HIT)
-		HSH_Purge(sp, sp->obj->objcore->objhead, ttl, grace);
+		HSH_Purge(sp, sp->wrk->obj->objcore->objhead, ttl, grace);
 	else if (sp->cur_method == VCL_MET_MISS)
-		HSH_Purge(sp, sp->objcore->objhead, ttl, grace);
+		HSH_Purge(sp, sp->wrk->objcore->objhead, ttl, grace);
 }
 
 /*--------------------------------------------------------------------
diff --git a/bin/varnishd/cache/cache_vrt_var.c b/bin/varnishd/cache/cache_vrt_var.c
index 860c7aa..5951b9c 100644
--- a/bin/varnishd/cache/cache_vrt_var.c
+++ b/bin/varnishd/cache/cache_vrt_var.c
@@ -88,8 +88,8 @@ VRT_DO_HDR(req,   proto,	sp->http,		HTTP_HDR_PROTO)
 VRT_DO_HDR(bereq, request,	sp->wrk->bereq,		HTTP_HDR_REQ)
 VRT_DO_HDR(bereq, url,		sp->wrk->bereq,		HTTP_HDR_URL)
 VRT_DO_HDR(bereq, proto,	sp->wrk->bereq,		HTTP_HDR_PROTO)
-VRT_DO_HDR(obj,   proto,	sp->obj->http,		HTTP_HDR_PROTO)
-VRT_DO_HDR(obj,   response,	sp->obj->http,		HTTP_HDR_RESPONSE)
+VRT_DO_HDR(obj,   proto,	sp->wrk->obj->http,		HTTP_HDR_PROTO)
+VRT_DO_HDR(obj,   response,	sp->wrk->obj->http,		HTTP_HDR_RESPONSE)
 VRT_DO_HDR(resp,  proto,	sp->wrk->resp,		HTTP_HDR_PROTO)
 VRT_DO_HDR(resp,  response,	sp->wrk->resp,		HTTP_HDR_RESPONSE)
 VRT_DO_HDR(beresp,  proto,	sp->wrk->beresp,	HTTP_HDR_PROTO)
@@ -114,7 +114,7 @@ VRT_r_##obj##_status(const struct sess *sp)			\
 	return(http->status);					\
 }
 
-VRT_DO_STATUS(obj, sp->obj->http)
+VRT_DO_STATUS(obj, sp->wrk->obj->http)
 VRT_DO_STATUS(beresp, sp->wrk->beresp)
 VRT_DO_STATUS(resp, sp->wrk->resp)
 
@@ -141,9 +141,9 @@ VRT_l_beresp_saintmode(const struct sess *sp, double a)
 	if (!wrk->vbc->backend)
 		return;
 	CHECK_OBJ_NOTNULL(wrk->vbc->backend, BACKEND_MAGIC);
-	if (!sp->objcore)
+	if (!sp->wrk->objcore)
 		return;
-	CHECK_OBJ_NOTNULL(sp->objcore, OBJCORE_MAGIC);
+	CHECK_OBJ_NOTNULL(sp->wrk->objcore, OBJCORE_MAGIC);
 
 	/* Setting a negative holdoff period is a mistake. Detecting this
 	 * when compiling the VCL would be better.
@@ -152,7 +152,7 @@ VRT_l_beresp_saintmode(const struct sess *sp, double a)
 
 	ALLOC_OBJ(new, TROUBLE_MAGIC);
 	AN(new);
-	new->target = (uintptr_t)(sp->objcore->objhead);
+	new->target = (uintptr_t)(sp->wrk->objcore->objhead);
 	new->timeout = sp->t_req + a;
 
 	/* Insert the new item on the list before the first item with a
@@ -401,15 +401,15 @@ VRT_DO_EXP(req, sp->exp, ttl, 0, )
 VRT_DO_EXP(req, sp->exp, grace, 0, )
 VRT_DO_EXP(req, sp->exp, keep, 0, )
 
-VRT_DO_EXP(obj, sp->obj->exp, grace, 0,
-   EXP_Rearm(sp->obj);
-   vrt_wsp_exp(sp, sp->obj->xid, &sp->obj->exp);)
-VRT_DO_EXP(obj, sp->obj->exp, ttl, (sp->t_req - sp->obj->exp.entered),
-   EXP_Rearm(sp->obj);
-   vrt_wsp_exp(sp, sp->obj->xid, &sp->obj->exp);)
-VRT_DO_EXP(obj, sp->obj->exp, keep, 0,
-   EXP_Rearm(sp->obj);
-   vrt_wsp_exp(sp, sp->obj->xid, &sp->obj->exp);)
+VRT_DO_EXP(obj, sp->wrk->obj->exp, grace, 0,
+   EXP_Rearm(sp->wrk->obj);
+   vrt_wsp_exp(sp, sp->wrk->obj->xid, &sp->wrk->obj->exp);)
+VRT_DO_EXP(obj, sp->wrk->obj->exp, ttl, (sp->t_req - sp->wrk->obj->exp.entered),
+   EXP_Rearm(sp->wrk->obj);
+   vrt_wsp_exp(sp, sp->wrk->obj->xid, &sp->wrk->obj->exp);)
+VRT_DO_EXP(obj, sp->wrk->obj->exp, keep, 0,
+   EXP_Rearm(sp->wrk->obj);
+   vrt_wsp_exp(sp, sp->wrk->obj->xid, &sp->wrk->obj->exp);)
 
 VRT_DO_EXP(beresp, sp->wrk->exp, grace, 0,
    vrt_wsp_exp(sp, sp->xid, &sp->wrk->exp);)
@@ -527,8 +527,8 @@ VRT_r_obj_hits(const struct sess *sp)
 {
 
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
-	CHECK_OBJ_NOTNULL(sp->obj, OBJECT_MAGIC);	/* XXX */
-	return (sp->obj->hits);
+	CHECK_OBJ_NOTNULL(sp->wrk->obj, OBJECT_MAGIC);	/* XXX */
+	return (sp->wrk->obj->hits);
 }
 
 double
@@ -536,8 +536,8 @@ VRT_r_obj_lastuse(const struct sess *sp)
 {
 
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
-	CHECK_OBJ_NOTNULL(sp->obj, OBJECT_MAGIC);	/* XXX */
-	return (VTIM_real() - sp->obj->last_use);
+	CHECK_OBJ_NOTNULL(sp->wrk->obj, OBJECT_MAGIC);	/* XXX */
+	return (VTIM_real() - sp->wrk->obj->last_use);
 }
 
 unsigned
diff --git a/bin/varnishd/storage/stevedore.c b/bin/varnishd/storage/stevedore.c
index 860604e..42479c3 100644
--- a/bin/varnishd/storage/stevedore.c
+++ b/bin/varnishd/storage/stevedore.c
@@ -246,11 +246,11 @@ STV_MkObject(struct sess *sp, void *ptr, unsigned ltot,
 	VTAILQ_INIT(&o->store);
 	sp->wrk->stats.n_object++;
 
-	if (sp->objcore != NULL) {
-		CHECK_OBJ_NOTNULL(sp->objcore, OBJCORE_MAGIC);
+	if (sp->wrk->objcore != NULL) {
+		CHECK_OBJ_NOTNULL(sp->wrk->objcore, OBJCORE_MAGIC);
 
-		o->objcore = sp->objcore;
-		sp->objcore = NULL;     /* refcnt follows pointer. */
+		o->objcore = sp->wrk->objcore;
+		sp->wrk->objcore = NULL;     /* refcnt follows pointer. */
 		BAN_NewObjCore(o->objcore);
 
 		o->objcore->methods = &default_oc_methods;
diff --git a/bin/varnishd/storage/storage_persistent.c b/bin/varnishd/storage/storage_persistent.c
index ded638b..838dc8c 100644
--- a/bin/varnishd/storage/storage_persistent.c
+++ b/bin/varnishd/storage/storage_persistent.c
@@ -474,10 +474,10 @@ smp_allocobj(struct stevedore *stv, struct sess *sp, unsigned ltot,
 	struct objcore *oc;
 	unsigned objidx;
 
-	if (sp->objcore == NULL)
+	if (sp->wrk->objcore == NULL)
 		return (NULL);		/* from cnt_error */
 	CAST_OBJ_NOTNULL(sc, stv->priv, SMP_SC_MAGIC);
-	AN(sp->objcore);
+	AN(sp->wrk->objcore);
 	AN(sp->wrk->exp.ttl > 0.);
 
 	ltot = IRNUP(sc, ltot);
diff --git a/bin/varnishd/waiter/cache_waiter_kqueue.c b/bin/varnishd/waiter/cache_waiter_kqueue.c
index b331a31..23df3de 100644
--- a/bin/varnishd/waiter/cache_waiter_kqueue.c
+++ b/bin/varnishd/waiter/cache_waiter_kqueue.c
@@ -102,7 +102,6 @@ vwk_kev(struct vwk *vwk, const struct kevent *kp)
 		while (i >= sizeof ss[0]) {
 			CHECK_OBJ_NOTNULL(ss[j], SESS_MAGIC);
 			assert(ss[j]->fd >= 0);
-			AZ(ss[j]->obj);
 			VTAILQ_INSERT_TAIL(&vwk->sesshead, ss[j], list);
 			vwk_kq_sess(vwk, ss[j], EV_ADD | EV_ONESHOT);
 			j++;
diff --git a/bin/varnishd/waiter/cache_waiter_ports.c b/bin/varnishd/waiter/cache_waiter_ports.c
index a1699d0..17d7ce0 100644
--- a/bin/varnishd/waiter/cache_waiter_ports.c
+++ b/bin/varnishd/waiter/cache_waiter_ports.c
@@ -77,7 +77,6 @@ vws_port_ev(struct vws *vws, port_event_t *ev) {
 	if(ev->portev_source == PORT_SOURCE_USER) {
 		CAST_OBJ_NOTNULL(sp, ev->portev_user, SESS_MAGIC);
 		assert(sp->fd >= 0);
-		AZ(sp->obj);
 		VTAILQ_INSERT_TAIL(&vws->sesshead, sp, list);
 		vws_add(vws, sp->fd, sp);
 	} else {



From geoff at varnish-cache.org  Mon Jan  9 20:52:43 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:43 +0100
Subject: [experimental-ims] efd6a98 Consequence change of obj/objcore move
	from sp to wrk:
Message-ID: <E1RkMCx-0008Sn-9J@project.varnish-software.com>

commit efd6a9801ac6664d6d166880d73c75356dc47aeb
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Nov 29 12:38:36 2011 +0000

    Consequence change of obj/objcore move from sp to wrk:
    
    Take sess out of STV_NewObject
    eliminate a lot of sp->wrk with a local wrk variable

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index db001ba..8371278 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -948,8 +948,8 @@ unsigned RFC2616_Req_Gzip(const struct sess *sp);
 int RFC2616_Do_Cond(const struct sess *sp);
 
 /* stevedore.c */
-struct object *STV_NewObject(struct sess *sp, const char *hint, unsigned len,
-    struct exp *, uint16_t nhttp);
+struct object *STV_NewObject(struct worker *wrk, const char *hint, unsigned len,
+    uint16_t nhttp);
 struct storage *STV_alloc(struct worker *w, size_t size);
 void STV_trim(struct storage *st, size_t size);
 void STV_free(struct storage *st);
diff --git a/bin/varnishd/cache/cache_center.c b/bin/varnishd/cache/cache_center.c
index 28c4760..13db2e3 100644
--- a/bin/varnishd/cache/cache_center.c
+++ b/bin/varnishd/cache/cache_center.c
@@ -442,22 +442,22 @@ cnt_error(struct sess *sp)
 
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 
-	sp->wrk->do_esi = 0;
-	sp->wrk->is_gzip = 0;
-	sp->wrk->is_gunzip = 0;
-	sp->wrk->do_gzip = 0;
-	sp->wrk->do_gunzip = 0;
-	sp->wrk->do_stream = 0;
-
 	w = sp->wrk;
-	if (sp->wrk->obj == NULL) {
+	w->do_esi = 0;
+	w->is_gzip = 0;
+	w->is_gunzip = 0;
+	w->do_gzip = 0;
+	w->do_gunzip = 0;
+	w->do_stream = 0;
+
+	if (w->obj == NULL) {
 		HSH_Prealloc(sp);
 		EXP_Clr(&w->exp);
-		sp->wrk->obj = STV_NewObject(sp, NULL, cache_param->http_resp_size,
-		     &w->exp, (uint16_t)cache_param->http_max_hdr);
+		w->obj = STV_NewObject(w, NULL, cache_param->http_resp_size,
+		     (uint16_t)cache_param->http_max_hdr);
 		if (sp->wrk->obj == NULL)
-			sp->wrk->obj = STV_NewObject(sp, TRANSIENT_STORAGE,
-			    cache_param->http_resp_size, &w->exp,
+			w->obj = STV_NewObject(w, TRANSIENT_STORAGE,
+			    cache_param->http_resp_size,
 			    (uint16_t)cache_param->http_max_hdr);
 		if (sp->wrk->obj == NULL) {
 			sp->doclose = "Out of objects";
@@ -684,15 +684,18 @@ cnt_fetchbody(struct sess *sp)
 	unsigned l;
 	struct vsb *vary = NULL;
 	int varyl = 0, pass;
+	struct worker *wrk;
+
+	wrk = sp->wrk;
 
 	assert(sp->handling == VCL_RET_HIT_FOR_PASS ||
 	    sp->handling == VCL_RET_DELIVER);
 
-	if (sp->wrk->objcore == NULL) {
+	if (wrk->objcore == NULL) {
 		/* This is a pass from vcl_recv */
 		pass = 1;
 		/* VCL may have fiddled this, but that doesn't help */
-		sp->wrk->exp.ttl = -1.;
+		wrk->exp.ttl = -1.;
 	} else if (sp->handling == VCL_RET_HIT_FOR_PASS) {
 		/* pass from vcl_fetch{} -> hit-for-pass */
 		/* XXX: the bereq was not filtered pass... */
@@ -715,63 +718,61 @@ cnt_fetchbody(struct sess *sp)
 	 *
 	 */
 
-	AZ(sp->wrk->vfp);
+	AZ(wrk->vfp);
 
 	/* We do nothing unless the param is set */
 	if (!cache_param->http_gzip_support)
-		sp->wrk->do_gzip = sp->wrk->do_gunzip = 0;
+		wrk->do_gzip = wrk->do_gunzip = 0;
 
-	sp->wrk->is_gzip =
-	    http_HdrIs(sp->wrk->beresp, H_Content_Encoding, "gzip");
+	wrk->is_gzip = http_HdrIs(wrk->beresp, H_Content_Encoding, "gzip");
 
-	sp->wrk->is_gunzip =
-	    !http_GetHdr(sp->wrk->beresp, H_Content_Encoding, NULL);
+	wrk->is_gunzip = !http_GetHdr(wrk->beresp, H_Content_Encoding, NULL);
 
 	/* It can't be both */
-	assert(sp->wrk->is_gzip == 0 || sp->wrk->is_gunzip == 0);
+	assert(wrk->is_gzip == 0 || wrk->is_gunzip == 0);
 
 	/* We won't gunzip unless it is gzip'ed */
-	if (sp->wrk->do_gunzip && !sp->wrk->is_gzip)
-		sp->wrk->do_gunzip = 0;
+	if (wrk->do_gunzip && !wrk->is_gzip)
+		wrk->do_gunzip = 0;
 
 	/* If we do gunzip, remove the C-E header */
-	if (sp->wrk->do_gunzip)
-		http_Unset(sp->wrk->beresp, H_Content_Encoding);
+	if (wrk->do_gunzip)
+		http_Unset(wrk->beresp, H_Content_Encoding);
 
 	/* We wont gzip unless it is ungziped */
-	if (sp->wrk->do_gzip && !sp->wrk->is_gunzip)
-		sp->wrk->do_gzip = 0;
+	if (wrk->do_gzip && !wrk->is_gunzip)
+		wrk->do_gzip = 0;
 
 	/* If we do gzip, add the C-E header */
-	if (sp->wrk->do_gzip)
-		http_SetHeader(sp->wrk, sp->vsl_id, sp->wrk->beresp,
+	if (wrk->do_gzip)
+		http_SetHeader(wrk, sp->vsl_id, wrk->beresp,
 		    "Content-Encoding: gzip");
 
 	/* But we can't do both at the same time */
-	assert(sp->wrk->do_gzip == 0 || sp->wrk->do_gunzip == 0);
+	assert(wrk->do_gzip == 0 || wrk->do_gunzip == 0);
 
 	/* ESI takes precedence and handles gzip/gunzip itself */
-	if (sp->wrk->do_esi)
-		sp->wrk->vfp = &vfp_esi;
-	else if (sp->wrk->do_gunzip)
-		sp->wrk->vfp = &vfp_gunzip;
-	else if (sp->wrk->do_gzip)
-		sp->wrk->vfp = &vfp_gzip;
-	else if (sp->wrk->is_gzip)
-		sp->wrk->vfp = &vfp_testgzip;
-
-	if (sp->wrk->do_esi || sp->esi_level > 0)
-		sp->wrk->do_stream = 0;
+	if (wrk->do_esi)
+		wrk->vfp = &vfp_esi;
+	else if (wrk->do_gunzip)
+		wrk->vfp = &vfp_gunzip;
+	else if (wrk->do_gzip)
+		wrk->vfp = &vfp_gzip;
+	else if (wrk->is_gzip)
+		wrk->vfp = &vfp_testgzip;
+
+	if (wrk->do_esi || sp->esi_level > 0)
+		wrk->do_stream = 0;
 	if (!sp->wantbody)
-		sp->wrk->do_stream = 0;
+		wrk->do_stream = 0;
 
-	l = http_EstimateWS(sp->wrk->beresp,
+	l = http_EstimateWS(wrk->beresp,
 	    pass ? HTTPH_R_PASS : HTTPH_A_INS, &nhttp);
 
 	/* Create Vary instructions */
-	if (sp->wrk->objcore != NULL) {
-		CHECK_OBJ_NOTNULL(sp->wrk->objcore, OBJCORE_MAGIC);
-		vary = VRY_Create(sp, sp->wrk->beresp);
+	if (wrk->objcore != NULL) {
+		CHECK_OBJ_NOTNULL(wrk->objcore, OBJCORE_MAGIC);
+		vary = VRY_Create(sp, wrk->beresp);
 		if (vary != NULL) {
 			varyl = VSB_len(vary);
 			assert(varyl > 0);
@@ -785,110 +786,107 @@ cnt_fetchbody(struct sess *sp)
 	 */
 	l += strlen("Content-Length: XxxXxxXxxXxxXxxXxx") + sizeof(void *);
 
-	if (sp->wrk->exp.ttl < cache_param->shortlived || sp->wrk->objcore == NULL)
-		sp->wrk->storage_hint = TRANSIENT_STORAGE;
+	if (wrk->exp.ttl < cache_param->shortlived || wrk->objcore == NULL)
+		wrk->storage_hint = TRANSIENT_STORAGE;
 
-	sp->wrk->obj = STV_NewObject(sp, sp->wrk->storage_hint, l,
-	    &sp->wrk->exp, nhttp);
-	if (sp->wrk->obj == NULL) {
+	wrk->obj = STV_NewObject(wrk, wrk->storage_hint, l, nhttp);
+	if (wrk->obj == NULL) {
 		/*
 		 * Try to salvage the transaction by allocating a
 		 * shortlived object on Transient storage.
 		 */
-		sp->wrk->obj = STV_NewObject(sp, TRANSIENT_STORAGE, l,
-		    &sp->wrk->exp, nhttp);
-		if (sp->wrk->exp.ttl > cache_param->shortlived)
-			sp->wrk->exp.ttl = cache_param->shortlived;
-		sp->wrk->exp.grace = 0.0;
-		sp->wrk->exp.keep = 0.0;
+		wrk->obj = STV_NewObject(wrk, TRANSIENT_STORAGE, l, nhttp);
+		if (wrk->exp.ttl > cache_param->shortlived)
+			wrk->exp.ttl = cache_param->shortlived;
+		wrk->exp.grace = 0.0;
+		wrk->exp.keep = 0.0;
 	}
-	if (sp->wrk->obj == NULL) {
+	if (wrk->obj == NULL) {
 		sp->err_code = 503;
 		sp->step = STP_ERROR;
-		VDI_CloseFd(sp->wrk);
+		VDI_CloseFd(wrk);
 		return (0);
 	}
-	CHECK_OBJ_NOTNULL(sp->wrk->obj, OBJECT_MAGIC);
+	CHECK_OBJ_NOTNULL(wrk->obj, OBJECT_MAGIC);
 
-	sp->wrk->storage_hint = NULL;
+	wrk->storage_hint = NULL;
 
-	if (sp->wrk->do_gzip || (sp->wrk->is_gzip && !sp->wrk->do_gunzip))
-		sp->wrk->obj->gziped = 1;
+	if (wrk->do_gzip || (wrk->is_gzip && !wrk->do_gunzip))
+		wrk->obj->gziped = 1;
 
 	if (vary != NULL) {
-		sp->wrk->obj->vary =
-		    (void *)WS_Alloc(sp->wrk->obj->http->ws, varyl);
-		AN(sp->wrk->obj->vary);
-		memcpy(sp->wrk->obj->vary, VSB_data(vary), varyl);
-		VRY_Validate(sp->wrk->obj->vary);
+		wrk->obj->vary = (void *)WS_Alloc(wrk->obj->http->ws, varyl);
+		AN(wrk->obj->vary);
+		memcpy(wrk->obj->vary, VSB_data(vary), varyl);
+		VRY_Validate(wrk->obj->vary);
 		VSB_delete(vary);
 	}
 
-	sp->wrk->obj->xid = sp->xid;
-	sp->wrk->obj->response = sp->err_code;
-	WS_Assert(sp->wrk->obj->ws_o);
+	wrk->obj->xid = sp->xid;
+	wrk->obj->response = sp->err_code;
+	WS_Assert(wrk->obj->ws_o);
 
 	/* Filter into object */
-	hp = sp->wrk->beresp;
-	hp2 = sp->wrk->obj->http;
+	hp = wrk->beresp;
+	hp2 = wrk->obj->http;
 
 	hp2->logtag = HTTP_Obj;
 	http_CopyResp(hp2, hp);
-	http_FilterFields(sp->wrk, sp->vsl_id, hp2, hp,
+	http_FilterFields(wrk, sp->vsl_id, hp2, hp,
 	    pass ? HTTPH_R_PASS : HTTPH_A_INS);
-	http_CopyHome(sp->wrk, sp->vsl_id, hp2);
+	http_CopyHome(wrk, sp->vsl_id, hp2);
 
 	if (http_GetHdr(hp, H_Last_Modified, &b))
-		sp->wrk->obj->last_modified = VTIM_parse(b);
+		wrk->obj->last_modified = VTIM_parse(b);
 	else
-		sp->wrk->obj->last_modified = floor(sp->wrk->exp.entered);
+		wrk->obj->last_modified = floor(wrk->exp.entered);
 
-	assert(WRW_IsReleased(sp->wrk));
+	assert(WRW_IsReleased(wrk));
 
 	/*
 	 * If we can deliver a 304 reply, we don't bother streaming.
 	 * Notice that vcl_deliver{} could still nuke the headers
 	 * that allow the 304, in which case we return 200 non-stream.
 	 */
-	if (sp->wrk->obj->response == 200 &&
+	if (wrk->obj->response == 200 &&
 	    sp->http->conds &&
 	    RFC2616_Do_Cond(sp))
-		sp->wrk->do_stream = 0;
+		wrk->do_stream = 0;
 
-	AssertObjCorePassOrBusy(sp->wrk->obj->objcore);
+	AssertObjCorePassOrBusy(wrk->obj->objcore);
 
-	if (sp->wrk->do_stream) {
+	if (wrk->do_stream) {
 		sp->step = STP_PREPRESP;
 		return (0);
 	}
 
 	/* Use unmodified headers*/
-	i = FetchBody(sp->wrk, sp->wrk->obj);
+	i = FetchBody(wrk, wrk->obj);
 
-	sp->wrk->h_content_length = NULL;
+	wrk->h_content_length = NULL;
 
-	http_Setup(sp->wrk->bereq, NULL);
-	http_Setup(sp->wrk->beresp, NULL);
-	sp->wrk->vfp = NULL;
-	assert(WRW_IsReleased(sp->wrk));
-	AZ(sp->wrk->vbc);
+	http_Setup(wrk->bereq, NULL);
+	http_Setup(wrk->beresp, NULL);
+	wrk->vfp = NULL;
+	assert(WRW_IsReleased(wrk));
+	AZ(wrk->vbc);
 	AN(sp->director);
 
 	if (i) {
 		HSH_Drop(sp);
-		AZ(sp->wrk->obj);
+		AZ(wrk->obj);
 		sp->err_code = 503;
 		sp->step = STP_ERROR;
 		return (0);
 	}
 
-	if (sp->wrk->obj->objcore != NULL) {
-		EXP_Insert(sp->wrk->obj);
-		AN(sp->wrk->obj->objcore);
-		AN(sp->wrk->obj->objcore->ban);
+	if (wrk->obj->objcore != NULL) {
+		EXP_Insert(wrk->obj);
+		AN(wrk->obj->objcore);
+		AN(wrk->obj->objcore->ban);
 		HSH_Unbusy(sp);
 	}
-	sp->wrk->acct_tmp.fetch++;
+	wrk->acct_tmp.fetch++;
 	sp->step = STP_PREPRESP;
 	return (0);
 }
@@ -970,7 +968,7 @@ DOT		shape=box
 DOT		label="first\nConfigure data structures"
 DOT	]
 DOT }
-DOT first -> wait 
+DOT first -> wait
  */
 
 static int
diff --git a/bin/varnishd/storage/stevedore.c b/bin/varnishd/storage/stevedore.c
index 42479c3..1a4b8ac 100644
--- a/bin/varnishd/storage/stevedore.c
+++ b/bin/varnishd/storage/stevedore.c
@@ -117,7 +117,7 @@ LRU_Free(struct lru *lru)
  */
 
 static struct stevedore *
-stv_pick_stevedore(const struct sess *sp, const char **hint)
+stv_pick_stevedore(struct worker *wrk, const char **hint)
 {
 	struct stevedore *stv;
 
@@ -131,7 +131,8 @@ stv_pick_stevedore(const struct sess *sp, const char **hint)
 			return (stv_transient);
 
 		/* Hint was not valid, nuke it */
-		WSP(sp, SLT_Debug, "Storage hint not usable");
+		WSL(wrk, SLT_Debug, wrk->htc->vsl_id,
+		    "Storage hint not usable");
 		*hint = NULL;
 	}
 	/* pick a stevedore and bump the head along */
@@ -202,7 +203,6 @@ struct stv_objsecrets {
 	uint16_t	nhttp;
 	unsigned	lhttp;
 	unsigned	wsl;
-	struct exp	*exp;
 };
 
 /*--------------------------------------------------------------------
@@ -214,7 +214,7 @@ struct stv_objsecrets {
  */
 
 struct object *
-STV_MkObject(struct sess *sp, void *ptr, unsigned ltot,
+STV_MkObject(struct worker *wrk, void *ptr, unsigned ltot,
     const struct stv_objsecrets *soc)
 {
 	struct object *o;
@@ -242,15 +242,15 @@ STV_MkObject(struct sess *sp, void *ptr, unsigned ltot,
 
 	http_Setup(o->http, o->ws_o);
 	o->http->magic = HTTP_MAGIC;
-	o->exp = *soc->exp;
+	o->exp = wrk->exp;
 	VTAILQ_INIT(&o->store);
-	sp->wrk->stats.n_object++;
+	wrk->stats.n_object++;
 
-	if (sp->wrk->objcore != NULL) {
-		CHECK_OBJ_NOTNULL(sp->wrk->objcore, OBJCORE_MAGIC);
+	if (wrk->objcore != NULL) {
+		CHECK_OBJ_NOTNULL(wrk->objcore, OBJCORE_MAGIC);
 
-		o->objcore = sp->wrk->objcore;
-		sp->wrk->objcore = NULL;     /* refcnt follows pointer. */
+		o->objcore = wrk->objcore;
+		wrk->objcore = NULL;     /* refcnt follows pointer. */
 		BAN_NewObjCore(o->objcore);
 
 		o->objcore->methods = &default_oc_methods;
@@ -265,7 +265,7 @@ STV_MkObject(struct sess *sp, void *ptr, unsigned ltot,
  */
 
 struct object *
-stv_default_allocobj(struct stevedore *stv, struct sess *sp, unsigned ltot,
+stv_default_allocobj(struct stevedore *stv, struct worker *wrk, unsigned ltot,
     const struct stv_objsecrets *soc)
 {
 	struct object *o;
@@ -280,7 +280,7 @@ stv_default_allocobj(struct stevedore *stv, struct sess *sp, unsigned ltot,
 		return (NULL);
 	}
 	ltot = st->len = st->space;
-	o = STV_MkObject(sp, st->ptr, ltot, soc);
+	o = STV_MkObject(wrk, st->ptr, ltot, soc);
 	CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC);
 	o->objstore = st;
 	return (o);
@@ -293,8 +293,8 @@ stv_default_allocobj(struct stevedore *stv, struct sess *sp, unsigned ltot,
  */
 
 struct object *
-STV_NewObject(struct sess *sp, const char *hint, unsigned wsl, struct exp *ep,
-    uint16_t nhttp)
+STV_NewObject(struct worker *wrk, const char *hint, unsigned wsl,
+     uint16_t nhttp)
 {
 	struct object *o;
 	struct stevedore *stv, *stv0;
@@ -302,6 +302,7 @@ STV_NewObject(struct sess *sp, const char *hint, unsigned wsl, struct exp *ep,
 	struct stv_objsecrets soc;
 	int i;
 
+	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 	assert(wsl > 0);
 	wsl = PRNDUP(wsl);
 
@@ -313,26 +314,25 @@ STV_NewObject(struct sess *sp, const char *hint, unsigned wsl, struct exp *ep,
 	soc.nhttp = nhttp;
 	soc.lhttp = lhttp;
 	soc.wsl = wsl;
-	soc.exp = ep;
 
 	ltot = sizeof *o + wsl + lhttp;
 
-	stv = stv0 = stv_pick_stevedore(sp, &hint);
+	stv = stv0 = stv_pick_stevedore(wrk, &hint);
 	AN(stv->allocobj);
-	o = stv->allocobj(stv, sp, ltot, &soc);
+	o = stv->allocobj(stv, wrk, ltot, &soc);
 	if (o == NULL && hint == NULL) {
 		do {
-			stv = stv_pick_stevedore(sp, &hint);
+			stv = stv_pick_stevedore(wrk, &hint);
 			AN(stv->allocobj);
-			o = stv->allocobj(stv, sp, ltot, &soc);
+			o = stv->allocobj(stv, wrk, ltot, &soc);
 		} while (o == NULL && stv != stv0);
 	}
 	if (o == NULL) {
 		/* no luck; try to free some space and keep trying */
 		for (i = 0; o == NULL && i < cache_param->nuke_limit; i++) {
-			if (EXP_NukeOne(sp->wrk, stv->lru) == -1)
+			if (EXP_NukeOne(wrk, stv->lru) == -1)
 				break;
-			o = stv->allocobj(stv, sp, ltot, &soc);
+			o = stv->allocobj(stv, wrk, ltot, &soc);
 		}
 	}
 
diff --git a/bin/varnishd/storage/storage.h b/bin/varnishd/storage/storage.h
index a813a36..c8c3689 100644
--- a/bin/varnishd/storage/storage.h
+++ b/bin/varnishd/storage/storage.h
@@ -34,6 +34,7 @@
 struct stv_objsecrets;
 struct stevedore;
 struct sess;
+struct worker;
 struct lru;
 
 typedef void storage_init_f(struct stevedore *, int ac, char * const *av);
@@ -41,8 +42,8 @@ typedef void storage_open_f(const struct stevedore *);
 typedef struct storage *storage_alloc_f(struct stevedore *, size_t size);
 typedef void storage_trim_f(struct storage *, size_t size);
 typedef void storage_free_f(struct storage *);
-typedef struct object *storage_allocobj_f(struct stevedore *, struct sess *sp,
-    unsigned ltot, const struct stv_objsecrets *);
+typedef struct object *storage_allocobj_f(struct stevedore *,
+    struct worker *wrk, unsigned ltot, const struct stv_objsecrets *);
 typedef void storage_close_f(const struct stevedore *);
 
 /* Prototypes for VCL variable responders */
@@ -89,7 +90,7 @@ extern struct stevedore *stv_transient;
 int STV_GetFile(const char *fn, int *fdp, const char **fnp, const char *ctx);
 uintmax_t STV_FileSize(int fd, const char *size, unsigned *granularity,
     const char *ctx);
-struct object *STV_MkObject(struct sess *sp, void *ptr, unsigned ltot,
+struct object *STV_MkObject(struct worker *wrk, void *ptr, unsigned ltot,
     const struct stv_objsecrets *soc);
 
 struct lru *LRU_Alloc(void);
diff --git a/bin/varnishd/storage/storage_persistent.c b/bin/varnishd/storage/storage_persistent.c
index 838dc8c..15087b3 100644
--- a/bin/varnishd/storage/storage_persistent.c
+++ b/bin/varnishd/storage/storage_persistent.c
@@ -463,7 +463,7 @@ smp_allocx(struct stevedore *st, size_t min_size, size_t max_size,
  */
 
 static struct object *
-smp_allocobj(struct stevedore *stv, struct sess *sp, unsigned ltot,
+smp_allocobj(struct stevedore *stv, struct worker *wrk, unsigned ltot,
     const struct stv_objsecrets *soc)
 {
 	struct object *o;
@@ -474,11 +474,11 @@ smp_allocobj(struct stevedore *stv, struct sess *sp, unsigned ltot,
 	struct objcore *oc;
 	unsigned objidx;
 
-	if (sp->wrk->objcore == NULL)
+	if (wrk->objcore == NULL)
 		return (NULL);		/* from cnt_error */
 	CAST_OBJ_NOTNULL(sc, stv->priv, SMP_SC_MAGIC);
-	AN(sp->wrk->objcore);
-	AN(sp->wrk->exp.ttl > 0.);
+	AN(wrk->objcore);
+	AN(wrk->exp.ttl > 0.);
 
 	ltot = IRNUP(sc, ltot);
 
@@ -489,7 +489,7 @@ smp_allocobj(struct stevedore *stv, struct sess *sp, unsigned ltot,
 	assert(st->space >= ltot);
 	ltot = st->len = st->space;
 
-	o = STV_MkObject(sp, st->ptr, ltot, soc);
+	o = STV_MkObject(wrk, st->ptr, ltot, soc);
 	CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC);
 	o->objstore = st;
 



From geoff at varnish-cache.org  Mon Jan  9 20:52:43 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:43 +0100
Subject: [experimental-ims] ef234ec Start descoping the fetch-from-backend
	stuff into struct busyobj.
Message-ID: <E1RkMCx-0008TI-Gg@project.varnish-software.com>

commit ef234ec9c32a7c0d209df5501858c100d37235b2
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Nov 29 15:55:42 2011 +0000

    Start descoping the fetch-from-backend stuff into struct busyobj.
    
    The fetching worker has wrk->busyobj set
    
    Push is_g[un]zip flags into busyobj as proof of concept.

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index 8371278..7ad7fbd 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -323,6 +323,7 @@ struct worker {
 
 	struct object		*obj;
 	struct objcore		*objcore;
+	struct busyobj		*busyobj;
 
 	struct exp		exp;
 
@@ -340,9 +341,7 @@ struct worker {
 	unsigned		do_stream;
 	unsigned		do_esi;
 	unsigned		do_gzip;
-	unsigned		is_gzip;
 	unsigned		do_gunzip;
-	unsigned		is_gunzip;
 	unsigned		do_close;
 	char			*h_content_length;
 
@@ -492,6 +491,8 @@ struct busyobj {
 	unsigned		magic;
 #define BUSYOBJ_MAGIC		0x23b95567
 	uint8_t			*vary;
+	unsigned		is_gzip;
+	unsigned		is_gunzip;
 };
 
 /* Object structure --------------------------------------------------*/
@@ -967,6 +968,21 @@ void SMP_Init(void);
 void SMP_Ready(void);
 void SMP_NewBan(const uint8_t *ban, unsigned len);
 
+#define New_BusyObj(wrk)						\
+	do {								\
+		if (wrk->nbusyobj != NULL) {				\
+			CHECK_OBJ_NOTNULL(wrk->nbusyobj, BUSYOBJ_MAGIC);\
+			wrk->busyobj = wrk->nbusyobj;			\
+			wrk->nbusyobj = NULL;				\
+			memset(wrk->busyobj, 0, sizeof *wrk->busyobj);	\
+			wrk->busyobj->magic = BUSYOBJ_MAGIC;		\
+		} else {						\
+			ALLOC_OBJ(wrk->busyobj, BUSYOBJ_MAGIC);		\
+		}							\
+		CHECK_OBJ_NOTNULL(wrk->busyobj, BUSYOBJ_MAGIC);		\
+		AZ(wrk->nbusyobj);					\
+	} while (0)
+
 /*
  * A normal pointer difference is signed, but we never want a negative value
  * so this little tool will make sure we don't get that.
diff --git a/bin/varnishd/cache/cache_center.c b/bin/varnishd/cache/cache_center.c
index 3832116..8f9e527 100644
--- a/bin/varnishd/cache/cache_center.c
+++ b/bin/varnishd/cache/cache_center.c
@@ -323,12 +323,12 @@ cnt_done(struct sess *sp)
 	sp->director = NULL;
 	sp->restarts = 0;
 
+	sp->wrk->busyobj = NULL;
+
 	sp->wrk->do_esi = 0;
 	sp->wrk->do_gunzip = 0;
 	sp->wrk->do_gzip = 0;
 	sp->wrk->do_stream = 0;
-	sp->wrk->is_gunzip = 0;
-	sp->wrk->is_gzip = 0;
 
 	SES_Charge(sp);
 
@@ -444,8 +444,6 @@ cnt_error(struct sess *sp)
 
 	w = sp->wrk;
 	w->do_esi = 0;
-	w->is_gzip = 0;
-	w->is_gunzip = 0;
 	w->do_gzip = 0;
 	w->do_gunzip = 0;
 	w->do_stream = 0;
@@ -552,6 +550,8 @@ cnt_fetch(struct sess *sp)
 
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 	CHECK_OBJ_NOTNULL(sp->vcl, VCL_CONF_MAGIC);
+	CHECK_OBJ_NOTNULL(sp->wrk, WORKER_MAGIC);
+	CHECK_OBJ_NOTNULL(sp->wrk->busyobj, BUSYOBJ_MAGIC);
 
 	AN(sp->director);
 	AZ(sp->wrk->vbc);
@@ -688,6 +688,8 @@ cnt_fetchbody(struct sess *sp)
 
 	wrk = sp->wrk;
 
+	CHECK_OBJ_NOTNULL(wrk->busyobj, BUSYOBJ_MAGIC);
+
 	assert(sp->handling == VCL_RET_HIT_FOR_PASS ||
 	    sp->handling == VCL_RET_DELIVER);
 
@@ -724,15 +726,17 @@ cnt_fetchbody(struct sess *sp)
 	if (!cache_param->http_gzip_support)
 		wrk->do_gzip = wrk->do_gunzip = 0;
 
-	wrk->is_gzip = http_HdrIs(wrk->beresp, H_Content_Encoding, "gzip");
+	wrk->busyobj->is_gzip =
+	    http_HdrIs(wrk->beresp, H_Content_Encoding, "gzip");
 
-	wrk->is_gunzip = !http_GetHdr(wrk->beresp, H_Content_Encoding, NULL);
+	wrk->busyobj->is_gunzip =
+	    !http_GetHdr(wrk->beresp, H_Content_Encoding, NULL);
 
 	/* It can't be both */
-	assert(wrk->is_gzip == 0 || wrk->is_gunzip == 0);
+	assert(wrk->busyobj->is_gzip == 0 || wrk->busyobj->is_gunzip == 0);
 
 	/* We won't gunzip unless it is gzip'ed */
-	if (wrk->do_gunzip && !wrk->is_gzip)
+	if (wrk->do_gunzip && !wrk->busyobj->is_gzip)
 		wrk->do_gunzip = 0;
 
 	/* If we do gunzip, remove the C-E header */
@@ -740,7 +744,7 @@ cnt_fetchbody(struct sess *sp)
 		http_Unset(wrk->beresp, H_Content_Encoding);
 
 	/* We wont gzip unless it is ungziped */
-	if (wrk->do_gzip && !wrk->is_gunzip)
+	if (wrk->do_gzip && !wrk->busyobj->is_gunzip)
 		wrk->do_gzip = 0;
 
 	/* If we do gzip, add the C-E header */
@@ -758,7 +762,7 @@ cnt_fetchbody(struct sess *sp)
 		wrk->vfp = &vfp_gunzip;
 	else if (wrk->do_gzip)
 		wrk->vfp = &vfp_gzip;
-	else if (wrk->is_gzip)
+	else if (wrk->busyobj->is_gzip)
 		wrk->vfp = &vfp_testgzip;
 
 	if (wrk->do_esi || sp->esi_level > 0)
@@ -811,7 +815,7 @@ cnt_fetchbody(struct sess *sp)
 
 	wrk->storage_hint = NULL;
 
-	if (wrk->do_gzip || (wrk->is_gzip && !wrk->do_gunzip))
+	if (wrk->do_gzip || (wrk->busyobj->is_gzip && !wrk->do_gunzip))
 		wrk->obj->gziped = 1;
 
 	if (vary != NULL) {
@@ -910,6 +914,7 @@ cnt_streambody(struct sess *sp)
 	uint8_t obuf[sp->wrk->res_mode & RES_GUNZIP ?
 	    cache_param->gzip_stack_buffer : 1];
 
+	CHECK_OBJ_NOTNULL(sp->wrk->busyobj, BUSYOBJ_MAGIC);
 	memset(&sctx, 0, sizeof sctx);
 	sctx.magic = STREAM_CTX_MAGIC;
 	AZ(sp->wrk->sctx);
@@ -1020,29 +1025,34 @@ DOT hit -> prepresp [label="deliver",style=bold,color=green]
 static int
 cnt_hit(struct sess *sp)
 {
+	struct worker *wrk;
 
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
-	CHECK_OBJ_NOTNULL(sp->wrk->obj, OBJECT_MAGIC);
+	wrk = sp->wrk;
+	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
+
+	CHECK_OBJ_NOTNULL(wrk->obj, OBJECT_MAGIC);
 	CHECK_OBJ_NOTNULL(sp->vcl, VCL_CONF_MAGIC);
 
-	assert(!(sp->wrk->obj->objcore->flags & OC_F_PASS));
+	assert(!(wrk->obj->objcore->flags & OC_F_PASS));
 
-	AZ(sp->wrk->do_stream);
+	AZ(wrk->do_stream);
 
 	VCL_hit_method(sp);
 
 	if (sp->handling == VCL_RET_DELIVER) {
 		/* Dispose of any body part of the request */
 		(void)FetchReqBody(sp);
-		AZ(sp->wrk->bereq->ws);
-		AZ(sp->wrk->beresp->ws);
+		AZ(wrk->bereq->ws);
+		AZ(wrk->beresp->ws);
 		sp->step = STP_PREPRESP;
 		return (0);
 	}
 
 	/* Drop our object, we won't need it */
-	(void)HSH_Deref(sp->wrk, NULL, &sp->wrk->obj);
-	sp->wrk->objcore = NULL;
+	(void)HSH_Deref(wrk, NULL, &wrk->obj);
+	wrk->objcore = NULL;
+	wrk->busyobj = NULL;
 
 	switch(sp->handling) {
 	case VCL_RET_PASS:
@@ -1145,6 +1155,7 @@ cnt_lookup(struct sess *sp)
 		sp->vary_e = NULL;
 
 		sp->wrk->objcore = oc;
+		CHECK_OBJ_NOTNULL(sp->wrk->busyobj, BUSYOBJ_MAGIC);
 		sp->step = STP_MISS;
 		return (0);
 	}
@@ -1205,6 +1216,7 @@ cnt_miss(struct sess *sp)
 
 	AZ(sp->wrk->obj);
 	AN(sp->wrk->objcore);
+	CHECK_OBJ_NOTNULL(sp->wrk->busyobj, BUSYOBJ_MAGIC);
 	WS_Reset(sp->wrk->ws, NULL);
 	http_Setup(sp->wrk->bereq, sp->wrk->ws);
 	http_FilterHeader(sp, HTTPH_R_FETCH);
@@ -1222,7 +1234,10 @@ cnt_miss(struct sess *sp)
 	sp->wrk->connect_timeout = 0;
 	sp->wrk->first_byte_timeout = 0;
 	sp->wrk->between_bytes_timeout = 0;
+	CHECK_OBJ_NOTNULL(sp->wrk->busyobj, BUSYOBJ_MAGIC);
+
 	VCL_miss_method(sp);
+	CHECK_OBJ_NOTNULL(sp->wrk->busyobj, BUSYOBJ_MAGIC);
 	switch(sp->handling) {
 	case VCL_RET_ERROR:
 		AZ(HSH_Deref(sp->wrk, sp->wrk->objcore, NULL));
@@ -1236,6 +1251,7 @@ cnt_miss(struct sess *sp)
 		sp->step = STP_PASS;
 		return (0);
 	case VCL_RET_FETCH:
+		CHECK_OBJ_NOTNULL(sp->wrk->busyobj, BUSYOBJ_MAGIC);
 		sp->step = STP_FETCH;
 		return (0);
 	case VCL_RET_RESTART:
@@ -1304,6 +1320,7 @@ cnt_pass(struct sess *sp)
 	sp->wrk->acct_tmp.pass++;
 	sp->sendbody = 1;
 	sp->step = STP_FETCH;
+	New_BusyObj(sp->wrk);
 	return (0);
 }
 
@@ -1411,8 +1428,6 @@ cnt_recv(struct sess *sp)
 
 	/* Zap these, in case we came here through restart */
 	sp->wrk->do_esi = 0;
-	sp->wrk->is_gzip = 0;
-	sp->wrk->is_gunzip = 0;
 	sp->wrk->do_gzip = 0;
 	sp->wrk->do_gunzip = 0;
 	sp->wrk->do_stream = 0;
@@ -1595,9 +1610,7 @@ CNT_Session(struct sess *sp)
 	    sp->step == STP_RECV);
 
 	AZ(w->do_stream);
-	AZ(w->is_gzip);
 	AZ(w->do_gzip);
-	AZ(w->is_gunzip);
 	AZ(w->do_gunzip);
 	AZ(w->do_esi);
 	AZ(w->obj);
@@ -1654,9 +1667,7 @@ CNT_Session(struct sess *sp)
 	AZ(w->obj);
 	AZ(w->objcore);
 	AZ(w->do_stream);
-	AZ(w->is_gzip);
 	AZ(w->do_gzip);
-	AZ(w->is_gunzip);
 	AZ(w->do_gunzip);
 	AZ(w->do_esi);
 #define ACCT(foo)	AZ(w->acct_tmp.foo);
diff --git a/bin/varnishd/cache/cache_esi_deliver.c b/bin/varnishd/cache/cache_esi_deliver.c
index 5ac8e71..6c8a16f 100644
--- a/bin/varnishd/cache/cache_esi_deliver.c
+++ b/bin/varnishd/cache/cache_esi_deliver.c
@@ -92,8 +92,6 @@ ved_include(struct sess *sp, const char *src, const char *host)
 	http_Unset(sp->http, H_Content_Length);
 
 	sp->wrk->do_esi = 0;
-	sp->wrk->is_gzip = 0;
-	sp->wrk->is_gunzip = 0;
 	sp->wrk->do_gzip = 0;
 	sp->wrk->do_gunzip = 0;
 	sp->wrk->do_stream = 0;
diff --git a/bin/varnishd/cache/cache_esi_fetch.c b/bin/varnishd/cache/cache_esi_fetch.c
index 5ec8f6b..35166e7 100644
--- a/bin/varnishd/cache/cache_esi_fetch.c
+++ b/bin/varnishd/cache/cache_esi_fetch.c
@@ -298,19 +298,20 @@ vfp_esi_begin(struct worker *w, size_t estimate)
 	struct vef_priv *vef;
 
 	CHECK_OBJ_NOTNULL(w, WORKER_MAGIC);
+	CHECK_OBJ_NOTNULL(w->busyobj, BUSYOBJ_MAGIC);
 
 	AZ(w->vgz_rx);
-	if (w->is_gzip && w->do_gunzip) {
+	if (w->busyobj->is_gzip && w->do_gunzip) {
 		w->vgz_rx = VGZ_NewUngzip(w, "U F E");
 		VEP_Init(w, NULL);
-	} else if (w->is_gunzip && w->do_gzip) {
+	} else if (w->busyobj->is_gunzip && w->do_gzip) {
 		ALLOC_OBJ(vef, VEF_MAGIC);
 		AN(vef);
 		vef->vgz = VGZ_NewGzip(w, "G F E");
 		AZ(w->vef_priv);
 		w->vef_priv = vef;
 		VEP_Init(w, vfp_vep_callback);
-	} else if (w->is_gzip) {
+	} else if (w->busyobj->is_gzip) {
 		w->vgz_rx = VGZ_NewUngzip(w, "U F E");
 		ALLOC_OBJ(vef, VEF_MAGIC);
 		AN(vef);
@@ -336,11 +337,11 @@ vfp_esi_bytes(struct worker *w, struct http_conn *htc, ssize_t bytes)
 	AZ(w->fetch_failed);
 	AN(w->vep);
 	assert(w->htc == htc);
-	if (w->is_gzip && w->do_gunzip)
+	if (w->busyobj->is_gzip && w->do_gunzip)
 		i = vfp_esi_bytes_gu(w, htc, bytes);
-	else if (w->is_gunzip && w->do_gzip)
+	else if (w->busyobj->is_gunzip && w->do_gzip)
 		i = vfp_esi_bytes_ug(w, htc, bytes);
-	else if (w->is_gzip)
+	else if (w->busyobj->is_gzip)
 		i = vfp_esi_bytes_gg(w, htc, bytes);
 	else
 		i = vfp_esi_bytes_uu(w, htc, bytes);
diff --git a/bin/varnishd/cache/cache_hash.c b/bin/varnishd/cache/cache_hash.c
index a2fc985..1f1795c 100644
--- a/bin/varnishd/cache/cache_hash.c
+++ b/bin/varnishd/cache/cache_hash.c
@@ -456,14 +456,14 @@ HSH_Lookup(struct sess *sp, struct objhead **poh)
 	AN(oc->flags & OC_F_BUSY);
 	oc->refcnt = 1;
 
-	/* XXX: clear w->nbusyobj before use */
+	New_BusyObj(w);
+
 	VRY_Validate(sp->vary_b);
 	if (sp->vary_l != NULL)
-		w->nbusyobj->vary = sp->vary_b;
+		w->busyobj->vary = sp->vary_b;
 	else
-		w->nbusyobj->vary = NULL;
-	oc->busyobj = w->nbusyobj;
-	w->nbusyobj = NULL;
+		w->busyobj->vary = NULL;
+	oc->busyobj = w->busyobj;
 
 	/*
 	 * Busy objects go on the tail, so they will not trip up searches.
diff --git a/bin/varnishd/cache/cache_panic.c b/bin/varnishd/cache/cache_panic.c
index f6f1ff5..641fe99 100644
--- a/bin/varnishd/cache/cache_panic.c
+++ b/bin/varnishd/cache/cache_panic.c
@@ -206,6 +206,16 @@ pan_wrk(const struct worker *wrk)
 	VSB_printf(pan_vsp, "    },\n");
 }
 
+static void
+pan_busyobj(const struct busyobj *bo)
+{
+
+	VSB_printf(pan_vsp, "  busyobj = %p {\n", bo);
+	if (bo->is_gzip)	VSB_printf(pan_vsp, "    is_gzip\n");
+	if (bo->is_gunzip)	VSB_printf(pan_vsp, "    is_gunzip\n");
+	VSB_printf(pan_vsp, "    },\n");
+}
+
 /*--------------------------------------------------------------------*/
 
 static void
@@ -249,10 +259,9 @@ pan_sess(const struct sess *sp)
 	if (sp->wrk->do_gunzip)	VSB_printf(pan_vsp, " do_gunzip");
 	if (sp->wrk->do_esi)	VSB_printf(pan_vsp, " do_esi");
 	if (sp->wrk->do_close)	VSB_printf(pan_vsp, " do_close");
-	if (sp->wrk->is_gzip)	VSB_printf(pan_vsp, " is_gzip");
-	if (sp->wrk->is_gunzip)	VSB_printf(pan_vsp, " is_gunzip");
 	VSB_printf(pan_vsp, "\n");
 	VSB_printf(pan_vsp, "  bodystatus = %d\n", sp->wrk->body_status);
+	pan_busyobj(sp->wrk->busyobj);
 
 	pan_ws(sp->ws, 2);
 	pan_http("req", sp->http, 2);



From geoff at varnish-cache.org  Mon Jan  9 20:52:43 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:43 +0100
Subject: [experimental-ims] eb6fa2f Move vfp from worker to busyobj
Message-ID: <E1RkMCx-0008Tf-Ml@project.varnish-software.com>

commit eb6fa2f150979d12ede60d2075f76102f45eb1ee
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Nov 29 17:30:40 2011 +0000

    Move vfp from worker to busyobj

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index 7ad7fbd..21382d7 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -334,7 +334,6 @@ struct worker {
 	struct vbc		*vbc;
 	struct object		*fetch_obj;
 	enum body_status	body_status;
-	struct vfp		*vfp;
 	struct vgz		*vgz_rx;
 	struct vef_priv		*vef_priv;
 	unsigned		fetch_failed;
@@ -485,7 +484,16 @@ oc_getlru(const struct objcore *oc)
 	return (oc->methods->getlru(oc));
 }
 
-/* Busy Object structure ---------------------------------------------*/
+/* Busy Object structure ---------------------------------------------
+ *
+ * The busyobj structure captures the aspects of an object related to,
+ * and while it is being fetched from the backend.
+ *
+ * One of these aspects will be how much has been fetched, which
+ * streaming delivery will make use of.
+ *
+ * XXX: many fields from worker needs to move here.
+ */
 
 struct busyobj {
 	unsigned		magic;
@@ -493,6 +501,8 @@ struct busyobj {
 	uint8_t			*vary;
 	unsigned		is_gzip;
 	unsigned		is_gunzip;
+
+	struct vfp		*vfp;
 };
 
 /* Object structure --------------------------------------------------*/
diff --git a/bin/varnishd/cache/cache_center.c b/bin/varnishd/cache/cache_center.c
index 85224b6..27209de 100644
--- a/bin/varnishd/cache/cache_center.c
+++ b/bin/varnishd/cache/cache_center.c
@@ -706,7 +706,6 @@ cnt_fetchbody(struct sess *sp)
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 	wrk = sp->wrk;
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
-
 	CHECK_OBJ_NOTNULL(wrk->busyobj, BUSYOBJ_MAGIC);
 
 	assert(sp->handling == VCL_RET_HIT_FOR_PASS ||
@@ -739,8 +738,6 @@ cnt_fetchbody(struct sess *sp)
 	 *
 	 */
 
-	AZ(wrk->vfp);
-
 	/* We do nothing unless the param is set */
 	if (!cache_param->http_gzip_support)
 		wrk->do_gzip = wrk->do_gunzip = 0;
@@ -776,13 +773,13 @@ cnt_fetchbody(struct sess *sp)
 
 	/* ESI takes precedence and handles gzip/gunzip itself */
 	if (wrk->do_esi)
-		wrk->vfp = &vfp_esi;
+		wrk->busyobj->vfp = &vfp_esi;
 	else if (wrk->do_gunzip)
-		wrk->vfp = &vfp_gunzip;
+		wrk->busyobj->vfp = &vfp_gunzip;
 	else if (wrk->do_gzip)
-		wrk->vfp = &vfp_gzip;
+		wrk->busyobj->vfp = &vfp_gzip;
 	else if (wrk->busyobj->is_gzip)
-		wrk->vfp = &vfp_testgzip;
+		wrk->busyobj->vfp = &vfp_testgzip;
 
 	if (wrk->do_esi || sp->esi_level > 0)
 		wrk->do_stream = 0;
@@ -890,7 +887,7 @@ cnt_fetchbody(struct sess *sp)
 
 	http_Setup(wrk->bereq, NULL);
 	http_Setup(wrk->beresp, NULL);
-	wrk->vfp = NULL;
+	wrk->busyobj->vfp = NULL;
 	assert(WRW_IsReleased(wrk));
 	AZ(wrk->vbc);
 	AN(sp->director);
@@ -960,7 +957,7 @@ cnt_streambody(struct sess *sp)
 
 	http_Setup(wrk->bereq, NULL);
 	http_Setup(wrk->beresp, NULL);
-	wrk->vfp = NULL;
+	wrk->busyobj->vfp = NULL;
 	AZ(wrk->vbc);
 	AN(sp->director);
 
diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c
index 66241bc..9413300 100644
--- a/bin/varnishd/cache/cache_fetch.c
+++ b/bin/varnishd/cache/cache_fetch.c
@@ -236,7 +236,7 @@ fetch_straight(struct worker *w, struct http_conn *htc, ssize_t cl)
 	} else if (cl == 0)
 		return (0);
 
-	i = w->vfp->bytes(w, htc, cl);
+	i = w->busyobj->vfp->bytes(w, htc, cl);
 	if (i <= 0)
 		return (FetchError(w, "straight insufficient bytes"));
 	return (0);
@@ -293,7 +293,7 @@ fetch_chunked(struct worker *w, struct http_conn *htc)
 		if (cl < 0)
 			return (FetchError(w,"chunked header number syntax"));
 
-		if (cl > 0 && w->vfp->bytes(w, htc, cl) <= 0)
+		if (cl > 0 && w->busyobj->vfp->bytes(w, htc, cl) <= 0)
 			return (-1);
 
 		i = HTC_Read(w, htc, buf, 1);
@@ -315,7 +315,7 @@ fetch_eof(struct worker *w, struct http_conn *htc)
 	int i;
 
 	assert(w->body_status == BS_EOF);
-	i = w->vfp->bytes(w, htc, SSIZE_MAX);
+	i = w->busyobj->vfp->bytes(w, htc, SSIZE_MAX);
 	if (i < 0)
 		return (-1);
 	return (0);
@@ -494,8 +494,8 @@ FetchBody(struct worker *w, struct object *obj)
 	CHECK_OBJ_NOTNULL(obj, OBJECT_MAGIC);
 	CHECK_OBJ_NOTNULL(obj->http, HTTP_MAGIC);
 
-	if (w->vfp == NULL)
-		w->vfp = &vfp_nop;
+	if (w->busyobj->vfp == NULL)
+		w->busyobj->vfp = &vfp_nop;
 
 	AssertObjCorePassOrBusy(obj->objcore);
 
@@ -518,24 +518,24 @@ FetchBody(struct worker *w, struct object *obj)
 		break;
 	case BS_LENGTH:
 		cl = fetch_number( w->h_content_length, 10);
-		w->vfp->begin(w, cl > 0 ? cl : 0);
+		w->busyobj->vfp->begin(w, cl > 0 ? cl : 0);
 		cls = fetch_straight(w, w->htc, cl);
 		mklen = 1;
-		if (w->vfp->end(w))
+		if (w->busyobj->vfp->end(w))
 			cls = -1;
 		break;
 	case BS_CHUNKED:
-		w->vfp->begin(w, cl);
+		w->busyobj->vfp->begin(w, cl);
 		cls = fetch_chunked(w, w->htc);
 		mklen = 1;
-		if (w->vfp->end(w))
+		if (w->busyobj->vfp->end(w))
 			cls = -1;
 		break;
 	case BS_EOF:
-		w->vfp->begin(w, cl);
+		w->busyobj->vfp->begin(w, cl);
 		cls = fetch_eof(w, w->htc);
 		mklen = 1;
-		if (w->vfp->end(w))
+		if (w->busyobj->vfp->end(w))
 			cls = -1;
 		break;
 	case BS_ERROR:



From geoff at varnish-cache.org  Mon Jan  9 20:52:40 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:40 +0100
Subject: [experimental-ims] 635bc5f Give libvarnishapi a good flexelinting
Message-ID: <E1RkMCu-0008Mr-UQ@project.varnish-software.com>

commit 635bc5f3a291bf5717bad4c82499dda0e4fb1bb2
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sun Nov 20 21:40:50 2011 +0000

    Give libvarnishapi a good flexelinting

diff --git a/bin/varnishtest/flint.lnt b/bin/varnishtest/flint.lnt
index 8f3d880..53ad3dc 100644
--- a/bin/varnishtest/flint.lnt
+++ b/bin/varnishtest/flint.lnt
@@ -1,5 +1,7 @@
 
 -esym(850, av)
+-esym(785, VSL_tags)
+-esym(528, iter_call)		// Flexelint bug
 
 -e712	// 14      Info    712     Loss of precision (___) (___ to ___)
 -e747	// 16      Info    747     Significant prototype coercion (___) ___ to ___
diff --git a/bin/varnishtest/flint.sh b/bin/varnishtest/flint.sh
index eed1686..3a67bd7 100755
--- a/bin/varnishtest/flint.sh
+++ b/bin/varnishtest/flint.sh
@@ -9,4 +9,5 @@ flexelint \
 	-I../.. \
 	../flint.lnt \
 	flint.lnt \
-	*.c 
+	*.c \
+	../../lib/libvarnishapi/*.c
diff --git a/include/vapi/vsc.h b/include/vapi/vsc.h
index ac711ea..3cf16c6 100644
--- a/include/vapi/vsc.h
+++ b/include/vapi/vsc.h
@@ -66,7 +66,7 @@ int VSC_Open(struct VSM_data *vd, int diag);
 	 * args and returns as VSM_Open()
 	 */
 
-struct VSC_C_main *VSC_Main(struct VSM_data *vd);
+struct VSC_C_main *VSC_Main(const struct VSM_data *vd);
 	/*
 	 * return Main stats structure
 	 * returns NULL until child has been started.
@@ -84,7 +84,7 @@ struct VSC_point {
 
 typedef int VSC_iter_f(void *priv, const struct VSC_point *const pt);
 
-int VSC_Iter(struct VSM_data *vd, VSC_iter_f *func, void *priv);
+int VSC_Iter(const struct VSM_data *vd, VSC_iter_f *func, void *priv);
 	/*
 	 * Iterate over all statistics counters, calling "func" for
 	 * each counter not suppressed by any "-f" arguments.
diff --git a/include/vapi/vsm.h b/include/vapi/vsm.h
index 50a7dde..48cbe70 100644
--- a/include/vapi/vsm.h
+++ b/include/vapi/vsm.h
@@ -126,8 +126,8 @@ struct VSM_head *VSM_Head(const struct VSM_data *vd);
 	 * Return the head of the VSM.
 	 */
 
-void VSM__iter0(struct VSM_data *vd, struct VSM_fantom *vf);
-int VSM__itern(struct VSM_data *vd, struct VSM_fantom *vf);
+void VSM__iter0(const struct VSM_data *vd, struct VSM_fantom *vf);
+int VSM__itern(const struct VSM_data *vd, struct VSM_fantom *vf);
 
 #define VSM_FOREACH_SAFE(vf, vd) \
     for(VSM__iter0((vd), (vf)); VSM__itern((vd), (vf));)
@@ -137,7 +137,7 @@ int VSM__itern(struct VSM_data *vd, struct VSM_fantom *vf);
 	 * vd = "struct VSM_data *"
 	 */
 
-int VSM_StillValid(struct VSM_data *vd, struct VSM_fantom *vf);
+int VSM_StillValid(const struct VSM_data *vd, struct VSM_fantom *vf);
 	/*
 	 * Return:
 	 *	0: fantom is invalid now.
@@ -145,8 +145,8 @@ int VSM_StillValid(struct VSM_data *vd, struct VSM_fantom *vf);
 	 *	2: a fantom with same dimensions exist, check class/type/ident
 	 */
 
-int VSM_Get(struct VSM_data *vd, struct VSM_fantom *vf, const char *class,
-    const char *type, const char *ident);
+int VSM_Get(const struct VSM_data *vd, struct VSM_fantom *vf,
+    const char *class, const char *type, const char *ident);
 	/*
 	 * Find a chunk, produce fantom for it.
 	 * Returns zero on failure.
@@ -166,7 +166,7 @@ void VSM_Close(struct VSM_data *vd);
  */
 
 /* OBSOLETE: Will disappear from Varnish 4.x */
-void *VSM_Find_Chunk(struct VSM_data *vd, const char *class,
+void *VSM_Find_Chunk(const struct VSM_data *vd, const char *class,
     const char *type, const char *ident, unsigned *lenp);
 	/*
 	 * Find a given chunk in the shared memory.
diff --git a/lib/libvarnishapi/vsc.c b/lib/libvarnishapi/vsc.c
index 4fc2233..5791e85 100644
--- a/lib/libvarnishapi/vsc.c
+++ b/lib/libvarnishapi/vsc.c
@@ -222,7 +222,7 @@ VSC_Open(struct VSM_data *vd, int diag)
 /*--------------------------------------------------------------------*/
 
 struct VSC_C_main *
-VSC_Main(struct VSM_data *vd)
+VSC_Main(const struct VSM_data *vd)
 {
 	struct VSM_fantom vf;
 
@@ -316,7 +316,7 @@ iter_call(const struct vsc *vsc, VSC_iter_f *func, void *priv,
 #undef VSC_DONE
 
 int
-VSC_Iter(struct VSM_data *vd, VSC_iter_f *func, void *priv)
+VSC_Iter(const struct VSM_data *vd, VSC_iter_f *func, void *priv)
 {
 	struct vsc *vsc;
 	struct VSM_fantom vf;
diff --git a/lib/libvarnishapi/vsl.c b/lib/libvarnishapi/vsl.c
index d896301..3c90381 100644
--- a/lib/libvarnishapi/vsl.c
+++ b/lib/libvarnishapi/vsl.c
@@ -294,7 +294,7 @@ VSL_Dispatch(struct VSM_data *vd, VSL_handler_f *func, void *priv)
 			s |= VSL_S_CLIENT;
 		if (VSL_BACKEND(p))
 			s |= VSL_S_BACKEND;
-		if (func(priv, VSL_TAG(p), u, l, s, VSL_DATA(p), bitmap))
+		if (func(priv, (enum VSL_tag_e)VSL_TAG(p), u, l, s, VSL_DATA(p), bitmap))
 			return (1);
 	}
 }
@@ -326,7 +326,8 @@ VSL_H_Print(void *priv, enum VSL_tag_e tag, unsigned fd, unsigned len,
 		fprintf(fo, "\"\n");
 		return (0);
 	}
-	fprintf(fo, "%5u %-12s %c %.*s\n", fd, VSL_tags[tag], type, len, ptr);
+	fprintf(fo, "%5u %-12s %c %.*s\n",
+	    fd, VSL_tags[tag], type, (int)len, ptr);
 	return (0);
 }
 
diff --git a/lib/libvarnishapi/vsm.c b/lib/libvarnishapi/vsm.c
index 81007d8..c410fde 100644
--- a/lib/libvarnishapi/vsm.c
+++ b/lib/libvarnishapi/vsm.c
@@ -46,7 +46,6 @@
 
 #include "vapi/vsm.h"
 #include "vapi/vsm_int.h"
-#include "vbm.h"
 #include "vin.h"
 #include "vsm_api.h"
 
@@ -288,7 +287,7 @@ VSM_Head(const struct VSM_data *vd)
 /*--------------------------------------------------------------------*/
 
 void
-VSM__iter0(struct VSM_data *vd, struct VSM_fantom *vf)
+VSM__iter0(const struct VSM_data *vd, struct VSM_fantom *vf)
 {
 
 	CHECK_OBJ_NOTNULL(vd, VSM_MAGIC);
@@ -296,7 +295,7 @@ VSM__iter0(struct VSM_data *vd, struct VSM_fantom *vf)
 }
 
 int
-VSM__itern(struct VSM_data *vd, struct VSM_fantom *vf)
+VSM__itern(const struct VSM_data *vd, struct VSM_fantom *vf)
 {
 
 	CHECK_OBJ_NOTNULL(vd, VSM_MAGIC);
@@ -327,7 +326,7 @@ VSM__itern(struct VSM_data *vd, struct VSM_fantom *vf)
 /*--------------------------------------------------------------------*/
 
 int
-VSM_StillValid(struct VSM_data *vd, struct VSM_fantom *vf)
+VSM_StillValid(const struct VSM_data *vd, struct VSM_fantom *vf)
 {
 	struct VSM_fantom f2;
 
@@ -346,8 +345,8 @@ VSM_StillValid(struct VSM_data *vd, struct VSM_fantom *vf)
 }
 
 int
-VSM_Get(struct VSM_data *vd, struct VSM_fantom *vf, const char *class,
-    const char *type, const char *ident)
+VSM_Get(const struct VSM_data *vd, struct VSM_fantom *vf,
+    const char *class, const char *type, const char *ident)
 {
 
 	CHECK_OBJ_NOTNULL(vd, VSM_MAGIC);
@@ -367,8 +366,8 @@ VSM_Get(struct VSM_data *vd, struct VSM_fantom *vf, const char *class,
 /*--------------------------------------------------------------------*/
 
 void *
-VSM_Find_Chunk(struct VSM_data *vd, const char *class, const char *type,
-    const char *ident, unsigned *lenp)
+VSM_Find_Chunk(const struct VSM_data *vd, const char *class,
+    const char *type, const char *ident, unsigned *lenp)
 {
 	struct VSM_fantom vf;
 
diff --git a/lib/libvarnishapi/vsm_api.h b/lib/libvarnishapi/vsm_api.h
index 84ea523..3355ee4 100644
--- a/lib/libvarnishapi/vsm_api.h
+++ b/lib/libvarnishapi/vsm_api.h
@@ -56,8 +56,5 @@ struct VSM_data {
 	struct vsl		*vsl;
 };
 
-struct VSM_chunk *VSM_find_alloc(struct VSM_data *vd, const char *class,
-    const char *type, const char *ident);
-
 void VSC_Delete(struct VSM_data *vd);
 void VSL_Delete(struct VSM_data *vd);



From geoff at varnish-cache.org  Mon Jan  9 20:52:44 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:44 +0100
Subject: [experimental-ims] 22c90f1 Move do_gzip,
	do_gunzip and do_esi from worker to busyobj
Message-ID: <E1RkMCy-0008Vh-PX@project.varnish-software.com>

commit 22c90f14b4c95c6691a00815d76096056465e6af
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Nov 30 06:49:49 2011 +0000

    Move do_gzip, do_gunzip and do_esi from worker to busyobj

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index a0e2821..9a4e5f3 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -333,9 +333,6 @@ struct worker {
 	struct vbc		*vbc;
 
 	unsigned		do_stream;
-	unsigned		do_esi;
-	unsigned		do_gzip;
-	unsigned		do_gunzip;
 	unsigned		do_close;
 	char			*h_content_length;
 
@@ -507,6 +504,10 @@ struct busyobj {
 
 	enum body_status	body_status;
 	struct vef_priv		*vef_priv;
+
+	unsigned		do_esi;
+	unsigned		do_gzip;
+	unsigned		do_gunzip;
 };
 
 /* Object structure --------------------------------------------------*/
diff --git a/bin/varnishd/cache/cache_center.c b/bin/varnishd/cache/cache_center.c
index 7d4a280..6acd218 100644
--- a/bin/varnishd/cache/cache_center.c
+++ b/bin/varnishd/cache/cache_center.c
@@ -184,8 +184,12 @@ cnt_prepresp(struct sess *sp)
 
 	wrk->res_mode = 0;
 
-	if ((wrk->h_content_length != NULL || !wrk->do_stream) &&
-	    !wrk->do_gzip && !wrk->do_gunzip)
+	if (wrk->busyobj == NULL)
+		wrk->res_mode |= RES_LEN;
+
+	if (wrk->busyobj != NULL &&
+	    (wrk->h_content_length != NULL || !wrk->do_stream) &&
+	    !wrk->busyobj->do_gzip && !wrk->busyobj->do_gunzip)
 		wrk->res_mode |= RES_LEN;
 
 	if (!sp->disable_esi && wrk->obj->esidata != NULL) {
@@ -339,9 +343,6 @@ cnt_done(struct sess *sp)
 
 	wrk->busyobj = NULL;
 
-	wrk->do_esi = 0;
-	wrk->do_gunzip = 0;
-	wrk->do_gzip = 0;
 	wrk->do_stream = 0;
 
 	SES_Charge(sp);
@@ -457,9 +458,6 @@ cnt_error(struct sess *sp)
 	wrk = sp->wrk;
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 
-	wrk->do_esi = 0;
-	wrk->do_gzip = 0;
-	wrk->do_gunzip = 0;
 	wrk->do_stream = 0;
 
 	if (wrk->obj == NULL) {
@@ -621,7 +619,7 @@ cnt_fetch(struct sess *sp)
 		if (wrk->objcore == NULL)
 			wrk->busyobj->exp.ttl = -1.;
 
-		AZ(wrk->do_esi);
+		AZ(wrk->busyobj->do_esi);
 
 		VCL_fetch_method(sp);
 
@@ -740,7 +738,7 @@ cnt_fetchbody(struct sess *sp)
 
 	/* We do nothing unless the param is set */
 	if (!cache_param->http_gzip_support)
-		wrk->do_gzip = wrk->do_gunzip = 0;
+		wrk->busyobj->do_gzip = wrk->busyobj->do_gunzip = 0;
 
 	wrk->busyobj->is_gzip =
 	    http_HdrIs(wrk->beresp, H_Content_Encoding, "gzip");
@@ -752,36 +750,36 @@ cnt_fetchbody(struct sess *sp)
 	assert(wrk->busyobj->is_gzip == 0 || wrk->busyobj->is_gunzip == 0);
 
 	/* We won't gunzip unless it is gzip'ed */
-	if (wrk->do_gunzip && !wrk->busyobj->is_gzip)
-		wrk->do_gunzip = 0;
+	if (wrk->busyobj->do_gunzip && !wrk->busyobj->is_gzip)
+		wrk->busyobj->do_gunzip = 0;
 
 	/* If we do gunzip, remove the C-E header */
-	if (wrk->do_gunzip)
+	if (wrk->busyobj->do_gunzip)
 		http_Unset(wrk->beresp, H_Content_Encoding);
 
 	/* We wont gzip unless it is ungziped */
-	if (wrk->do_gzip && !wrk->busyobj->is_gunzip)
-		wrk->do_gzip = 0;
+	if (wrk->busyobj->do_gzip && !wrk->busyobj->is_gunzip)
+		wrk->busyobj->do_gzip = 0;
 
 	/* If we do gzip, add the C-E header */
-	if (wrk->do_gzip)
+	if (wrk->busyobj->do_gzip)
 		http_SetHeader(wrk, sp->vsl_id, wrk->beresp,
 		    "Content-Encoding: gzip");
 
 	/* But we can't do both at the same time */
-	assert(wrk->do_gzip == 0 || wrk->do_gunzip == 0);
+	assert(wrk->busyobj->do_gzip == 0 || wrk->busyobj->do_gunzip == 0);
 
 	/* ESI takes precedence and handles gzip/gunzip itself */
-	if (wrk->do_esi)
+	if (wrk->busyobj->do_esi)
 		wrk->busyobj->vfp = &vfp_esi;
-	else if (wrk->do_gunzip)
+	else if (wrk->busyobj->do_gunzip)
 		wrk->busyobj->vfp = &vfp_gunzip;
-	else if (wrk->do_gzip)
+	else if (wrk->busyobj->do_gzip)
 		wrk->busyobj->vfp = &vfp_gzip;
 	else if (wrk->busyobj->is_gzip)
 		wrk->busyobj->vfp = &vfp_testgzip;
 
-	if (wrk->do_esi || sp->esi_level > 0)
+	if (wrk->busyobj->do_esi || sp->esi_level > 0)
 		wrk->do_stream = 0;
 	if (!sp->wantbody)
 		wrk->do_stream = 0;
@@ -832,7 +830,8 @@ cnt_fetchbody(struct sess *sp)
 
 	wrk->storage_hint = NULL;
 
-	if (wrk->do_gzip || (wrk->busyobj->is_gzip && !wrk->do_gunzip))
+	if (wrk->busyobj->do_gzip ||
+	    (wrk->busyobj->is_gzip && !wrk->busyobj->do_gunzip))
 		wrk->obj->gziped = 1;
 
 	if (vary != NULL) {
@@ -1470,9 +1469,6 @@ cnt_recv(struct sess *sp)
 	}
 
 	/* Zap these, in case we came here through restart */
-	wrk->do_esi = 0;
-	wrk->do_gzip = 0;
-	wrk->do_gunzip = 0;
 	wrk->do_stream = 0;
 
 	if (cache_param->http_gzip_support &&
@@ -1656,9 +1652,6 @@ CNT_Session(struct sess *sp)
 	    sp->step == STP_RECV);
 
 	AZ(wrk->do_stream);
-	AZ(wrk->do_gzip);
-	AZ(wrk->do_gunzip);
-	AZ(wrk->do_esi);
 	AZ(wrk->obj);
 	AZ(wrk->objcore);
 
@@ -1713,9 +1706,6 @@ CNT_Session(struct sess *sp)
 	AZ(wrk->obj);
 	AZ(wrk->objcore);
 	AZ(wrk->do_stream);
-	AZ(wrk->do_gzip);
-	AZ(wrk->do_gunzip);
-	AZ(wrk->do_esi);
 #define ACCT(foo)	AZ(wrk->acct_tmp.foo);
 #include "tbl/acct_fields.h"
 #undef ACCT
diff --git a/bin/varnishd/cache/cache_esi_deliver.c b/bin/varnishd/cache/cache_esi_deliver.c
index 6c8a16f..518c5a9 100644
--- a/bin/varnishd/cache/cache_esi_deliver.c
+++ b/bin/varnishd/cache/cache_esi_deliver.c
@@ -91,9 +91,6 @@ ved_include(struct sess *sp, const char *src, const char *host)
 	/* Client content already taken care of */
 	http_Unset(sp->http, H_Content_Length);
 
-	sp->wrk->do_esi = 0;
-	sp->wrk->do_gzip = 0;
-	sp->wrk->do_gunzip = 0;
 	sp->wrk->do_stream = 0;
 
 	sxid = sp->xid;
diff --git a/bin/varnishd/cache/cache_esi_fetch.c b/bin/varnishd/cache/cache_esi_fetch.c
index e0765b6..aea28ed 100644
--- a/bin/varnishd/cache/cache_esi_fetch.c
+++ b/bin/varnishd/cache/cache_esi_fetch.c
@@ -304,10 +304,10 @@ vfp_esi_begin(struct worker *w, size_t estimate)
 	CHECK_OBJ_NOTNULL(w->busyobj, BUSYOBJ_MAGIC);
 
 	AZ(w->busyobj->vgz_rx);
-	if (w->busyobj->is_gzip && w->do_gunzip) {
+	if (w->busyobj->is_gzip && w->busyobj->do_gunzip) {
 		w->busyobj->vgz_rx = VGZ_NewUngzip(w, "U F E");
 		VEP_Init(w, NULL);
-	} else if (w->busyobj->is_gunzip && w->do_gzip) {
+	} else if (w->busyobj->is_gunzip && w->busyobj->do_gzip) {
 		ALLOC_OBJ(vef, VEF_MAGIC);
 		AN(vef);
 		vef->vgz = VGZ_NewGzip(w, "G F E");
@@ -340,9 +340,9 @@ vfp_esi_bytes(struct worker *w, struct http_conn *htc, ssize_t bytes)
 	AZ(w->busyobj->fetch_failed);
 	AN(w->busyobj->vep);
 	assert(&w->busyobj->htc == htc);
-	if (w->busyobj->is_gzip && w->do_gunzip)
+	if (w->busyobj->is_gzip && w->busyobj->do_gunzip)
 		i = vfp_esi_bytes_gu(w, htc, bytes);
-	else if (w->busyobj->is_gunzip && w->do_gzip)
+	else if (w->busyobj->is_gunzip && w->busyobj->do_gzip)
 		i = vfp_esi_bytes_ug(w, htc, bytes);
 	else if (w->busyobj->is_gzip)
 		i = vfp_esi_bytes_gg(w, htc, bytes);
diff --git a/bin/varnishd/cache/cache_panic.c b/bin/varnishd/cache/cache_panic.c
index 85fcae0..718a686 100644
--- a/bin/varnishd/cache/cache_panic.c
+++ b/bin/varnishd/cache/cache_panic.c
@@ -213,6 +213,9 @@ pan_busyobj(const struct busyobj *bo)
 	VSB_printf(pan_vsp, "  busyobj = %p {\n", bo);
 	if (bo->is_gzip)	VSB_printf(pan_vsp, "    is_gzip\n");
 	if (bo->is_gunzip)	VSB_printf(pan_vsp, "    is_gunzip\n");
+	if (bo->do_gzip)	VSB_printf(pan_vsp, "    do_gzip\n");
+	if (bo->do_gunzip)	VSB_printf(pan_vsp, "    do_gunzip\n");
+	if (bo->do_esi)		VSB_printf(pan_vsp, "    do_esi\n");
 	VSB_printf(pan_vsp, "    bodystatus = %d,\n", bo->body_status);
 	VSB_printf(pan_vsp, "    },\n");
 }
@@ -256,9 +259,6 @@ pan_sess(const struct sess *sp)
 
 	VSB_printf(pan_vsp, "  flags = ");
 	if (sp->wrk->do_stream)	VSB_printf(pan_vsp, " do_stream");
-	if (sp->wrk->do_gzip)	VSB_printf(pan_vsp, " do_gzip");
-	if (sp->wrk->do_gunzip)	VSB_printf(pan_vsp, " do_gunzip");
-	if (sp->wrk->do_esi)	VSB_printf(pan_vsp, " do_esi");
 	if (sp->wrk->do_close)	VSB_printf(pan_vsp, " do_close");
 	VSB_printf(pan_vsp, "\n");
 	pan_busyobj(sp->wrk->busyobj);
diff --git a/bin/varnishd/cache/cache_vrt_var.c b/bin/varnishd/cache/cache_vrt_var.c
index b889d69..adf1017 100644
--- a/bin/varnishd/cache/cache_vrt_var.c
+++ b/bin/varnishd/cache/cache_vrt_var.c
@@ -194,9 +194,9 @@ VRT_r_##dir##_##onm(const struct sess *sp)				\
 	return (sp->wrk->field);					\
 }
 
-VBERESP(beresp, unsigned, do_esi, do_esi)
-VBERESP(beresp, unsigned, do_gzip, do_gzip)
-VBERESP(beresp, unsigned, do_gunzip, do_gunzip)
+VBERESP(beresp, unsigned, do_esi, busyobj->do_esi)
+VBERESP(beresp, unsigned, do_gzip, busyobj->do_gzip)
+VBERESP(beresp, unsigned, do_gunzip, busyobj->do_gunzip)
 VBERESP(beresp, unsigned, do_stream, do_stream)
 
 /*--------------------------------------------------------------------*/



From geoff at varnish-cache.org  Mon Jan  9 20:52:47 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:47 +0100
Subject: [experimental-ims] 5ec8dea Give pipe its own local vbc.
Message-ID: <E1RkMD1-00009H-Eu@project.varnish-software.com>

commit 5ec8dead4585d87ff8742342cdf34cd4aa947eaf
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Dec 5 08:49:20 2011 +0000

    Give pipe its own local vbc.

diff --git a/bin/varnishd/cache/cache_pipe.c b/bin/varnishd/cache/cache_pipe.c
index 9bf8b79..5337eda 100644
--- a/bin/varnishd/cache/cache_pipe.c
+++ b/bin/varnishd/cache/cache_pipe.c
@@ -71,10 +71,9 @@ PipeSession(struct sess *sp)
 	CHECK_OBJ_NOTNULL(sp->wrk, WORKER_MAGIC);
 	w = sp->wrk;
 
-	sp->wrk->vbc = VDI_GetFd(NULL, sp);
-	if (sp->wrk->vbc == NULL)
+	vc = VDI_GetFd(NULL, sp);
+	if (vc == NULL)
 		return;
-	vc = sp->wrk->vbc;
 	(void)VTCP_blocking(vc->fd);
 
 	WRW_Reserve(w, &vc->fd);
@@ -89,7 +88,7 @@ PipeSession(struct sess *sp)
 
 	if (i) {
 		SES_Close(sp, "pipe");
-		VDI_CloseFd(sp->wrk, &sp->wrk->vbc);
+		VDI_CloseFd(sp->wrk, &vc);
 		return;
 	}
 
@@ -129,5 +128,5 @@ PipeSession(struct sess *sp)
 		}
 	}
 	SES_Close(sp, "pipe");
-	VDI_CloseFd(sp->wrk, &sp->wrk->vbc);
+	VDI_CloseFd(sp->wrk, &vc);
 }



From geoff at varnish-cache.org  Mon Jan  9 20:52:44 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:44 +0100
Subject: [experimental-ims] bd4d340 Merge branch 'master' of
	ssh://git.varnish-cache.org/git/varnish-cache
Message-ID: <E1RkMCy-0008VT-Lx@project.varnish-software.com>

commit bd4d3401adefe9f37968b955fec23cad0a697107
Merge: 7813b33 84628ca
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Nov 29 21:11:05 2011 +0000

    Merge branch 'master' of ssh://git.varnish-cache.org/git/varnish-cache



From geoff at varnish-cache.org  Mon Jan  9 20:52:41 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:41 +0100
Subject: [experimental-ims] 6b84197 Spelling cleanup
Message-ID: <E1RkMCv-0008PY-Vv@project.varnish-software.com>

commit 6b84197ee8b0763b43fddea284166298af7e0a73
Author: Andreas Plesner Jacobsen <apj at mutt.dk>
Date:   Wed Nov 23 23:48:45 2011 +0100

    Spelling cleanup

diff --git a/include/vapi/vsl.h b/include/vapi/vsl.h
index dfe5a83..db8b953 100644
--- a/include/vapi/vsl.h
+++ b/include/vapi/vsl.h
@@ -121,7 +121,7 @@ int VSL_Dispatch(struct VSM_data *vd, VSL_handler_f *func, void *priv);
 	 * Return values:
 	 *	!=0:	Non-zero return value from func()
 	 *	0:	no VSL records.
-	 *	-1:	VSL chunk was abandonned.
+	 *	-1:	VSL chunk was abandoned.
 	 *	-2:	End of file (-r) / -k arg exhausted / "done"
 	 */
 
@@ -132,7 +132,7 @@ int VSL_NextSLT(struct VSM_data *lh, uint32_t **pp, uint64_t *bitmap);
 	 * Return values:
 	 *	1:	Valid VSL record at *pp
 	 *	0:	no VSL records
-	 *	-1:	VSL cunkwas abandonned
+	 *	-1:	VSL chunk was abandoned
 	 *	-2:	End of file (-r) / -k arg exhausted / "done"
 	 */
 
diff --git a/include/vapi/vsm.h b/include/vapi/vsm.h
index 9635e8f..800cb76 100644
--- a/include/vapi/vsm.h
+++ b/include/vapi/vsm.h
@@ -100,16 +100,16 @@ int VSM_Open(struct VSM_data *vd);
 	 *	<0 on failure, VSM_Error() returns diagnostic string
 	 */
 
-int VSM_Abandonned(const struct VSM_data *vd);
+int VSM_Abandoned(const struct VSM_data *vd);
 	/*
-	 * Find out if the VSM file has been abandonned or closed and should
+	 * Find out if the VSM file has been abandoned or closed and should
 	 * be reopened.  This function calls stat(2) and should only be
 	 * used when lack of activity or invalidation of fantoms indicate
 	 * abandonment.
 	 *
 	 * Returns:
 	 *	0  No reopen needed.
-	 *	1  VSM abandonned.
+	 *	1  VSM abandoned.
 	 */
 
 void VSM_Close(struct VSM_data *vd);
@@ -132,7 +132,7 @@ int VSM__itern(const struct VSM_data *vd, struct VSM_fantom *vf);
 int VSM_StillValid(const struct VSM_data *vd, struct VSM_fantom *vf);
 	/*
 	 * This is a cheap syscall-less check to see if the fantom is still
-	 * valid.  Further checking with VSM_Abandonned() may be a good
+	 * valid.  Further checking with VSM_Abandoned() may be a good
 	 * idea.
 	 *
 	 * Return:
diff --git a/include/vapi/vsm_int.h b/include/vapi/vsm_int.h
index b45fdf7..237fabd 100644
--- a/include/vapi/vsm_int.h
+++ b/include/vapi/vsm_int.h
@@ -50,7 +50,7 @@
  *	When manager abandons VSM file, alloc_seq is set to zero, which
  *	never happens in any other circumstances.
  *
- *	If a manager is started and finds and old abandonned VSM segment
+ *	If a manager is started and finds and old abandoned VSM segment
  *	it will zero the alloc_seq in it, before replacing the file.
  *
  * Subscribers will have to monitor two things to make sure they have
diff --git a/lib/libvarnishapi/vsm.c b/lib/libvarnishapi/vsm.c
index 18de315..e11fe26 100644
--- a/lib/libvarnishapi/vsm.c
+++ b/lib/libvarnishapi/vsm.c
@@ -245,7 +245,7 @@ VSM_Close(struct VSM_data *vd)
 /*--------------------------------------------------------------------*/
 
 int
-VSM_Abandonned(const struct VSM_data *vd)
+VSM_Abandoned(const struct VSM_data *vd)
 {
 	struct stat st;
 
@@ -287,7 +287,7 @@ VSM__itern(const struct VSM_data *vd, struct VSM_fantom *vf)
 	AN(vf);
 
 	if (vd->head->alloc_seq == 0)
-		return (0);	/* abandonned VSM */
+		return (0);	/* abandoned VSM */
 	else if (vf->priv != 0) {
 		if (vf->priv != vd->head->alloc_seq)
 			return (0);
@@ -312,7 +312,7 @@ VSM__itern(const struct VSM_data *vd, struct VSM_fantom *vf)
 	vf->e = (char*)vf->b + vf->chunk->len;
 
 	if (vf->priv == 0)
-		return (0);	/* abandonned VSM */
+		return (0);	/* abandoned VSM */
 	if (vf->b == vf->e)
 		return (0);	/* freed chunk */
 	AN(vf->priv);



From geoff at varnish-cache.org  Mon Jan  9 20:52:48 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:48 +0100
Subject: [experimental-ims] 9cb9ba6 Move vbc from worker to busyobj
Message-ID: <E1RkMD2-0000AX-2V@project.varnish-software.com>

commit 9cb9ba6f6dbf4a3f9136f8b145c80be0d26418a9
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Dec 5 13:43:38 2011 +0000

    Move vbc from worker to busyobj

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index 4cc356e..46af4dd 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -330,7 +330,6 @@ struct worker {
 	/* Fetch stuff.  Here because pipe has no busyobj */
 	struct http		*bereq;
 	struct http		*beresp;
-	struct vbc		*vbc;
 
 	/* Stream state */
 	struct stream_ctx	*sctx;
@@ -494,6 +493,7 @@ struct busyobj {
 	unsigned		fetch_failed;
 	struct vgz		*vgz_rx;
 
+	struct vbc		*vbc;
 	struct object		*fetch_obj;
 	struct exp		exp;
 	struct http_conn	htc;
@@ -652,7 +652,7 @@ struct vbc *VDI_GetFd(const struct director *, struct sess *sp);
 int VDI_Healthy(const struct director *, const struct sess *sp);
 void VDI_CloseFd(struct worker *wrk, struct vbc **vbp);
 void VDI_RecycleFd(struct worker *wrk, struct vbc **vbp);
-void VDI_AddHostHeader(const struct sess *sp);
+void VDI_AddHostHeader(struct worker *wrk, const struct vbc *vbc);
 void VBE_Poll(void);
 
 /* cache_backend_cfg.c */
diff --git a/bin/varnishd/cache/cache_backend.c b/bin/varnishd/cache/cache_backend.c
index a435c1c..b661640 100644
--- a/bin/varnishd/cache/cache_backend.c
+++ b/bin/varnishd/cache/cache_backend.c
@@ -59,15 +59,15 @@ struct vdi_simple {
  * Create default Host: header for backend request
  */
 void
-VDI_AddHostHeader(const struct sess *sp)
+VDI_AddHostHeader(struct worker *wrk, const struct vbc *vbc)
 {
 
-	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
-	CHECK_OBJ_NOTNULL(sp->wrk->bereq, HTTP_MAGIC);
-	CHECK_OBJ_NOTNULL(sp->wrk->vbc, VBC_MAGIC);
-	CHECK_OBJ_NOTNULL(sp->wrk->vbc->vdis, VDI_SIMPLE_MAGIC);
-	http_PrintfHeader(sp->wrk, sp->wrk->vbc->vsl_id, sp->wrk->bereq,
-	    "Host: %s", sp->wrk->vbc->vdis->vrt->hosthdr);
+	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
+	CHECK_OBJ_NOTNULL(wrk->bereq, HTTP_MAGIC);
+	CHECK_OBJ_NOTNULL(vbc, VBC_MAGIC);
+	CHECK_OBJ_NOTNULL(vbc->vdis, VDI_SIMPLE_MAGIC);
+	http_PrintfHeader(wrk, vbc->vsl_id, wrk->bereq,
+	    "Host: %s", vbc->vdis->vrt->hosthdr);
 }
 
 /*--------------------------------------------------------------------*/
diff --git a/bin/varnishd/cache/cache_center.c b/bin/varnishd/cache/cache_center.c
index d2126ba..079df44 100644
--- a/bin/varnishd/cache/cache_center.c
+++ b/bin/varnishd/cache/cache_center.c
@@ -249,7 +249,7 @@ cnt_prepresp(struct sess *sp)
 		if (sp->restarts >= cache_param->max_restarts)
 			break;
 		if (wrk->busyobj->do_stream) {
-			VDI_CloseFd(wrk, &wrk->vbc);
+			VDI_CloseFd(wrk, &wrk->busyobj->vbc);
 			HSH_Drop(wrk);
 		} else {
 			(void)HSH_Deref(wrk, NULL, &wrk->obj);
@@ -338,7 +338,6 @@ cnt_done(struct sess *sp)
 	CHECK_OBJ_ORNULL(sp->vcl, VCL_CONF_MAGIC);
 
 	AZ(wrk->obj);
-	AZ(wrk->vbc);
 	sp->director = NULL;
 	sp->restarts = 0;
 
@@ -565,7 +564,7 @@ cnt_fetch(struct sess *sp)
 	CHECK_OBJ_NOTNULL(wrk->busyobj, BUSYOBJ_MAGIC);
 
 	AN(sp->director);
-	AZ(wrk->vbc);
+	AZ(wrk->busyobj->vbc);
 	AZ(wrk->busyobj->should_close);
 	AZ(wrk->storage_hint);
 
@@ -635,11 +634,11 @@ cnt_fetch(struct sess *sp)
 		}
 
 		/* We are not going to fetch the body, Close the connection */
-		VDI_CloseFd(wrk, &wrk->vbc);
+		VDI_CloseFd(wrk, &wrk->busyobj->vbc);
 	}
 
 	/* Clean up partial fetch */
-	AZ(wrk->vbc);
+	AZ(wrk->busyobj->vbc);
 
 	if (wrk->objcore != NULL) {
 		CHECK_OBJ_NOTNULL(wrk->objcore, OBJCORE_MAGIC);
@@ -819,7 +818,7 @@ cnt_fetchbody(struct sess *sp)
 	if (wrk->obj == NULL) {
 		sp->err_code = 503;
 		sp->step = STP_ERROR;
-		VDI_CloseFd(wrk, &wrk->vbc);
+		VDI_CloseFd(wrk, &wrk->busyobj->vbc);
 		return (0);
 	}
 	CHECK_OBJ_NOTNULL(wrk->obj, OBJECT_MAGIC);
@@ -883,7 +882,7 @@ cnt_fetchbody(struct sess *sp)
 	http_Setup(wrk->beresp, NULL);
 	wrk->busyobj->vfp = NULL;
 	assert(WRW_IsReleased(wrk));
-	AZ(wrk->vbc);
+	AZ(wrk->busyobj->vbc);
 	AN(sp->director);
 
 	if (i) {
@@ -950,7 +949,7 @@ cnt_streambody(struct sess *sp)
 	http_Setup(wrk->bereq, NULL);
 	http_Setup(wrk->beresp, NULL);
 	wrk->busyobj->vfp = NULL;
-	AZ(wrk->vbc);
+	AZ(wrk->busyobj->vbc);
 	AN(sp->director);
 
 	if (!i && wrk->obj->objcore != NULL) {
diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c
index d321d7a..6483a99 100644
--- a/bin/varnishd/cache/cache_fetch.c
+++ b/bin/varnishd/cache/cache_fetch.c
@@ -400,12 +400,12 @@ FetchHdr(struct sess *sp, int need_host_hdr)
 
 	hp = w->bereq;
 
-	sp->wrk->vbc = VDI_GetFd(NULL, sp);
-	if (sp->wrk->vbc == NULL) {
+	sp->wrk->busyobj->vbc = VDI_GetFd(NULL, sp);
+	if (sp->wrk->busyobj->vbc == NULL) {
 		WSP(sp, SLT_FetchError, "no backend connection");
 		return (-1);
 	}
-	vc = sp->wrk->vbc;
+	vc = sp->wrk->busyobj->vbc;
 	if (vc->recycled)
 		retry = 1;
 
@@ -415,7 +415,7 @@ FetchHdr(struct sess *sp, int need_host_hdr)
 	 * because the backend may be chosen by a director.
 	 */
 	if (need_host_hdr)
-		VDI_AddHostHeader(sp);
+		VDI_AddHostHeader(sp->wrk, vc);
 
 	(void)VTCP_blocking(vc->fd);	/* XXX: we should timeout instead */
 	WRW_Reserve(w, &vc->fd);
@@ -426,7 +426,7 @@ FetchHdr(struct sess *sp, int need_host_hdr)
 	if (WRW_FlushRelease(w) || i > 0) {
 		WSP(sp, SLT_FetchError, "backend write error: %d (%s)",
 		    errno, strerror(errno));
-		VDI_CloseFd(sp->wrk, &sp->wrk->vbc);
+		VDI_CloseFd(sp->wrk, &sp->wrk->busyobj->vbc);
 		/* XXX: other cleanup ? */
 		return (retry);
 	}
@@ -450,7 +450,7 @@ FetchHdr(struct sess *sp, int need_host_hdr)
 	if (i < 0) {
 		WSP(sp, SLT_FetchError, "http first read error: %d %d (%s)",
 		    i, errno, strerror(errno));
-		VDI_CloseFd(sp->wrk, &sp->wrk->vbc);
+		VDI_CloseFd(sp->wrk, &sp->wrk->busyobj->vbc);
 		/* XXX: other cleanup ? */
 		/* Retryable if we never received anything */
 		return (i == -1 ? retry : -1);
@@ -464,7 +464,7 @@ FetchHdr(struct sess *sp, int need_host_hdr)
 			WSP(sp, SLT_FetchError,
 			    "http first read error: %d %d (%s)",
 			    i, errno, strerror(errno));
-			VDI_CloseFd(sp->wrk, &sp->wrk->vbc);
+			VDI_CloseFd(sp->wrk, &sp->wrk->busyobj->vbc);
 			/* XXX: other cleanup ? */
 			return (-1);
 		}
@@ -474,7 +474,7 @@ FetchHdr(struct sess *sp, int need_host_hdr)
 
 	if (http_DissectResponse(w, htc, hp)) {
 		WSP(sp, SLT_FetchError, "http format error");
-		VDI_CloseFd(sp->wrk, &sp->wrk->vbc);
+		VDI_CloseFd(sp->wrk, &sp->wrk->busyobj->vbc);
 		/* XXX: other cleanup ? */
 		return (-1);
 	}
@@ -495,7 +495,7 @@ FetchBody(struct worker *w, struct object *obj)
 	CHECK_OBJ_NOTNULL(w, WORKER_MAGIC);
 	CHECK_OBJ_NOTNULL(w->busyobj, BUSYOBJ_MAGIC);
 	AZ(w->busyobj->fetch_obj);
-	CHECK_OBJ_NOTNULL(w->vbc, VBC_MAGIC);
+	CHECK_OBJ_NOTNULL(w->busyobj->vbc, VBC_MAGIC);
 	CHECK_OBJ_NOTNULL(obj, OBJECT_MAGIC);
 	CHECK_OBJ_NOTNULL(obj->http, HTTP_MAGIC);
 
@@ -570,7 +570,7 @@ FetchBody(struct worker *w, struct object *obj)
 	    cls, mklen);
 
 	if (w->busyobj->body_status == BS_ERROR) {
-		VDI_CloseFd(w, &w->vbc);
+		VDI_CloseFd(w, &w->busyobj->vbc);
 		return (__LINE__);
 	}
 
@@ -582,7 +582,7 @@ FetchBody(struct worker *w, struct object *obj)
 			VTAILQ_REMOVE(&obj->store, st, list);
 			STV_free(st);
 		}
-		VDI_CloseFd(w, &w->vbc);
+		VDI_CloseFd(w, &w->busyobj->vbc);
 		obj->len = 0;
 		return (__LINE__);
 	}
@@ -610,14 +610,14 @@ FetchBody(struct worker *w, struct object *obj)
 
 	if (mklen > 0) {
 		http_Unset(obj->http, H_Content_Length);
-		http_PrintfHeader(w, w->vbc->vsl_id, obj->http,
+		http_PrintfHeader(w, w->busyobj->vbc->vsl_id, obj->http,
 		    "Content-Length: %jd", (intmax_t)obj->len);
 	}
 
 	if (cls)
-		VDI_CloseFd(w, &w->vbc);
+		VDI_CloseFd(w, &w->busyobj->vbc);
 	else
-		VDI_RecycleFd(w, &w->vbc);
+		VDI_RecycleFd(w, &w->busyobj->vbc);
 
 	return (0);
 }
diff --git a/bin/varnishd/cache/cache_panic.c b/bin/varnishd/cache/cache_panic.c
index 798ba53..923abbd 100644
--- a/bin/varnishd/cache/cache_panic.c
+++ b/bin/varnishd/cache/cache_panic.c
@@ -220,6 +220,9 @@ pan_busyobj(const struct busyobj *bo)
 	if (bo->should_close)	VSB_printf(pan_vsp, "    should_close\n");
 	VSB_printf(pan_vsp, "    bodystatus = %d,\n", bo->body_status);
 	VSB_printf(pan_vsp, "    },\n");
+	if (VALID_OBJ(bo->vbc, BACKEND_MAGIC))
+		pan_vbc(bo->vbc);
+
 }
 
 /*--------------------------------------------------------------------*/
@@ -271,9 +274,6 @@ pan_sess(const struct sess *sp)
 	if (VALID_OBJ(sp->vcl, VCL_CONF_MAGIC))
 		pan_vcl(sp->vcl);
 
-	if (VALID_OBJ(sp->wrk->vbc, BACKEND_MAGIC))
-		pan_vbc(sp->wrk->vbc);
-
 	if (VALID_OBJ(sp->wrk->obj, OBJECT_MAGIC))
 		pan_object(sp->wrk->obj);
 
diff --git a/bin/varnishd/cache/cache_shmlog.c b/bin/varnishd/cache/cache_shmlog.c
index 3cfd451..b7c43a1 100644
--- a/bin/varnishd/cache/cache_shmlog.c
+++ b/bin/varnishd/cache/cache_shmlog.c
@@ -291,10 +291,11 @@ WSLB(struct worker *w, enum VSL_tag_e tag, const char *fmt, ...)
 	va_list ap;
 
 	CHECK_OBJ_NOTNULL(w, WORKER_MAGIC);
-	CHECK_OBJ_NOTNULL(w->vbc, VBC_MAGIC);
+	CHECK_OBJ_NOTNULL(w->busyobj, BUSYOBJ_MAGIC);
+	CHECK_OBJ_NOTNULL(w->busyobj->vbc, VBC_MAGIC);
 	AN(fmt);
 	va_start(ap, fmt);
-	wsl(w, tag, w->vbc->vsl_id, fmt, ap);
+	wsl(w, tag, w->busyobj->vbc->vsl_id, fmt, ap);
 	va_end(ap);
 }
 
diff --git a/bin/varnishd/cache/cache_vrt_var.c b/bin/varnishd/cache/cache_vrt_var.c
index 0342de8..fea5c70 100644
--- a/bin/varnishd/cache/cache_vrt_var.c
+++ b/bin/varnishd/cache/cache_vrt_var.c
@@ -132,15 +132,19 @@ VRT_l_beresp_saintmode(const struct sess *sp, double a)
 	struct trouble *tr;
 	struct trouble *tr2;
 	struct worker *wrk;
+	struct vbc *vbc;
 
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+	CHECK_OBJ_NOTNULL(sp->wrk, WORKER_MAGIC);
 	wrk = sp->wrk;
-	if (!wrk->vbc)
+	CHECK_OBJ_NOTNULL(wrk->busyobj, BUSYOBJ_MAGIC);
+	vbc = wrk->busyobj->vbc;
+	if (!vbc)
 		return;
-	CHECK_OBJ_NOTNULL(wrk->vbc, VBC_MAGIC);
-	if (!wrk->vbc->backend)
+	CHECK_OBJ_NOTNULL(vbc, VBC_MAGIC);
+	if (!vbc->backend)
 		return;
-	CHECK_OBJ_NOTNULL(wrk->vbc->backend, BACKEND_MAGIC);
+	CHECK_OBJ_NOTNULL(vbc->backend, BACKEND_MAGIC);
 	if (!sp->wrk->objcore)
 		return;
 	CHECK_OBJ_NOTNULL(sp->wrk->objcore, OBJCORE_MAGIC);
@@ -159,8 +163,8 @@ VRT_l_beresp_saintmode(const struct sess *sp, double a)
 	 * timeout at a later date (ie: sort by which entry will time out
 	 * from the list
 	 */
-	Lck_Lock(&wrk->vbc->backend->mtx);
-	VTAILQ_FOREACH_SAFE(tr, &wrk->vbc->backend->troublelist, list, tr2) {
+	Lck_Lock(&vbc->backend->mtx);
+	VTAILQ_FOREACH_SAFE(tr, &vbc->backend->troublelist, list, tr2) {
 		if (tr->timeout < new->timeout) {
 			VTAILQ_INSERT_BEFORE(tr, new, list);
 			new = NULL;
@@ -172,9 +176,9 @@ VRT_l_beresp_saintmode(const struct sess *sp, double a)
 	 * items have a longer timeout.
 	 */
 	if (new)
-		VTAILQ_INSERT_TAIL(&wrk->vbc->backend->troublelist, new, list);
+		VTAILQ_INSERT_TAIL(&vbc->backend->troublelist, new, list);
 
-	Lck_Unlock(&wrk->vbc->backend->mtx);
+	Lck_Unlock(&vbc->backend->mtx);
 }
 
 /*--------------------------------------------------------------------*/
@@ -253,8 +257,8 @@ VRT_r_beresp_backend_name(const struct sess *sp)
 {
 
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
-	CHECK_OBJ_NOTNULL(sp->wrk->vbc, VBC_MAGIC);
-	return(sp->wrk->vbc->backend->vcl_name);
+	CHECK_OBJ_NOTNULL(sp->wrk->busyobj->vbc, VBC_MAGIC);
+	return(sp->wrk->busyobj->vbc->backend->vcl_name);
 }
 
 struct sockaddr_storage *
@@ -262,8 +266,8 @@ VRT_r_beresp_backend_ip(const struct sess *sp)
 {
 
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
-	CHECK_OBJ_NOTNULL(sp->wrk->vbc, VBC_MAGIC);
-	return(sp->wrk->vbc->addr);
+	CHECK_OBJ_NOTNULL(sp->wrk->busyobj->vbc, VBC_MAGIC);
+	return(sp->wrk->busyobj->vbc->addr);
 }
 
 int
@@ -271,8 +275,8 @@ VRT_r_beresp_backend_port(const struct sess *sp)
 {
 
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
-	CHECK_OBJ_NOTNULL(sp->wrk->vbc, VBC_MAGIC);
-	return (VTCP_port(sp->wrk->vbc->addr));
+	CHECK_OBJ_NOTNULL(sp->wrk->busyobj->vbc, VBC_MAGIC);
+	return (VTCP_port(sp->wrk->busyobj->vbc->addr));
 }
 
 const char * __match_proto__()
diff --git a/bin/varnishd/storage/stevedore.c b/bin/varnishd/storage/stevedore.c
index 2748999..5de0e27 100644
--- a/bin/varnishd/storage/stevedore.c
+++ b/bin/varnishd/storage/stevedore.c
@@ -131,7 +131,7 @@ stv_pick_stevedore(struct worker *wrk, const char **hint)
 			return (stv_transient);
 
 		/* Hint was not valid, nuke it */
-		WSL(wrk, SLT_Debug, 0, 			/* XXX VSL_id ?? */
+		WSL(wrk, SLT_Debug, 0,			/* XXX VSL_id ?? */
 		    "Storage hint not usable");
 		*hint = NULL;
 	}



From geoff at varnish-cache.org  Mon Jan  9 20:52:48 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:48 +0100
Subject: [experimental-ims] 691dcc6 Minor polish
Message-ID: <E1RkMD2-0000B3-7v@project.varnish-software.com>

commit 691dcc69855ebe00fb9bdf5c9dd3ec2e5e4a816d
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Dec 6 07:57:13 2011 +0000

    Minor polish

diff --git a/bin/varnishd/cache/cache_gzip.c b/bin/varnishd/cache/cache_gzip.c
index cf07e1d..061348a 100644
--- a/bin/varnishd/cache/cache_gzip.c
+++ b/bin/varnishd/cache/cache_gzip.c
@@ -197,9 +197,9 @@ VGZ_NewGzip(struct worker *wrk, const char *id)
 	 */
 	i = deflateInit2(&vg->vz,
 	    cache_param->gzip_level,		/* Level */
-	    Z_DEFLATED,			/* Method */
+	    Z_DEFLATED,				/* Method */
 	    16 + cache_param->gzip_window,	/* Window bits (16=gzip + 15) */
-	    cache_param->gzip_memlevel,	/* memLevel */
+	    cache_param->gzip_memlevel,		/* memLevel */
 	    Z_DEFAULT_STRATEGY);
 	assert(Z_OK == i);
 	return (vg);
@@ -252,11 +252,11 @@ VGZ_ObufFull(const struct vgz *vg)
  */
 
 int
-VGZ_ObufStorage(struct worker *w, struct vgz *vg)
+VGZ_ObufStorage(struct worker *wrk, struct vgz *vg)
 {
 	struct storage *st;
 
-	st = FetchStorage(w, 0);
+	st = FetchStorage(wrk, 0);
 	if (st == NULL)
 		return (-1);
 
@@ -349,7 +349,7 @@ VGZ_Gzip(struct vgz *vg, const void **pptr, size_t *plen, enum vgz_flag flags)
  */
 
 int
-VGZ_WrwGunzip(struct worker *w, struct vgz *vg, const void *ibuf,
+VGZ_WrwGunzip(struct worker *wrk, struct vgz *vg, const void *ibuf,
     ssize_t ibufl, char *obuf, ssize_t obufl, ssize_t *obufp)
 {
 	int i;
@@ -374,9 +374,9 @@ VGZ_WrwGunzip(struct worker *w, struct vgz *vg, const void *ibuf,
 			return (-1);
 		}
 		if (obufl == *obufp || i == VGZ_STUCK) {
-			w->acct_tmp.bodybytes += *obufp;
-			(void)WRW_Write(w, obuf, *obufp);
-			(void)WRW_Flush(w);
+			wrk->acct_tmp.bodybytes += *obufp;
+			(void)WRW_Write(wrk, obuf, *obufp);
+			(void)WRW_Flush(wrk);
 			*obufp = 0;
 			VGZ_Obuf(vg, obuf + *obufp, obufl - *obufp);
 		}
@@ -400,7 +400,7 @@ VGZ_UpdateObj(const struct vgz *vg, struct object *obj)
 }
 
 /*--------------------------------------------------------------------
- * Passing a vsl_id of -1 means "use w->vbc->vsl_id"
+ * Passing a vsl_id of -1 means "use wrk->vbc->vsl_id"
  */
 
 int
@@ -454,15 +454,17 @@ VGZ_Destroy(struct vgz **vgp, int vsl_id)
  */
 
 static void __match_proto__()
-vfp_gunzip_begin(struct worker *w, size_t estimate)
+vfp_gunzip_begin(struct worker *wrk, size_t estimate)
 {
 	(void)estimate;
-	AZ(w->busyobj->vgz_rx);
-	w->busyobj->vgz_rx = VGZ_NewUngzip(w, "U F -");
+	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
+	CHECK_OBJ_NOTNULL(wrk->busyobj, BUSYOBJ_MAGIC);
+	AZ(wrk->busyobj->vgz_rx);
+	wrk->busyobj->vgz_rx = VGZ_NewUngzip(wrk, "U F -");
 }
 
 static int __match_proto__()
-vfp_gunzip_bytes(struct worker *w, struct http_conn *htc, ssize_t bytes)
+vfp_gunzip_bytes(struct worker *wrk, struct http_conn *htc, ssize_t bytes)
 {
 	struct vgz *vg;
 	ssize_t l, wl;
@@ -471,8 +473,10 @@ vfp_gunzip_bytes(struct worker *w, struct http_conn *htc, ssize_t bytes)
 	size_t dl;
 	const void *dp;
 
-	AZ(w->busyobj->fetch_failed);
-	vg = w->busyobj->vgz_rx;
+	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
+	CHECK_OBJ_NOTNULL(wrk->busyobj, BUSYOBJ_MAGIC);
+	AZ(wrk->busyobj->fetch_failed);
+	vg = wrk->busyobj->vgz_rx;
 	CHECK_OBJ_NOTNULL(vg, VGZ_MAGIC);
 	AZ(vg->vz.avail_in);
 	while (bytes > 0 || vg->vz.avail_in > 0) {
@@ -480,40 +484,42 @@ vfp_gunzip_bytes(struct worker *w, struct http_conn *htc, ssize_t bytes)
 			l = sizeof ibuf;
 			if (l > bytes)
 				l = bytes;
-			wl = HTC_Read(w, htc, ibuf, l);
+			wl = HTC_Read(wrk, htc, ibuf, l);
 			if (wl <= 0)
 				return (wl);
 			VGZ_Ibuf(vg, ibuf, wl);
 			bytes -= wl;
 		}
 
-		if (VGZ_ObufStorage(w, vg))
+		if (VGZ_ObufStorage(wrk, vg))
 			return(-1);
 		i = VGZ_Gunzip(vg, &dp, &dl);
 		if (i != VGZ_OK && i != VGZ_END)
-			return(FetchError(w, "Gunzip data error"));
-		w->busyobj->fetch_obj->len += dl;
-		if (w->busyobj->do_stream)
-			RES_StreamPoll(w);
+			return(FetchError(wrk, "Gunzip data error"));
+		wrk->busyobj->fetch_obj->len += dl;
+		if (wrk->busyobj->do_stream)
+			RES_StreamPoll(wrk);
 	}
 	assert(i == Z_OK || i == Z_STREAM_END);
 	return (1);
 }
 
 static int __match_proto__()
-vfp_gunzip_end(struct worker *w)
+vfp_gunzip_end(struct worker *wrk)
 {
 	struct vgz *vg;
 
-	vg = w->busyobj->vgz_rx;
-	w->busyobj->vgz_rx = NULL;
+	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
+	CHECK_OBJ_NOTNULL(wrk->busyobj, BUSYOBJ_MAGIC);
+	vg = wrk->busyobj->vgz_rx;
+	wrk->busyobj->vgz_rx = NULL;
 	CHECK_OBJ_NOTNULL(vg, VGZ_MAGIC);
-	if (w->busyobj->fetch_failed) {
+	if (wrk->busyobj->fetch_failed) {
 		(void)VGZ_Destroy(&vg, -1);
 		return(0);
 	}
 	if (VGZ_Destroy(&vg, -1) != VGZ_END)
-		return(FetchError(w, "Gunzip error at the very end"));
+		return(FetchError(wrk, "Gunzip error at the very end"));
 	return (0);
 }
 
@@ -531,16 +537,18 @@ struct vfp vfp_gunzip = {
  */
 
 static void __match_proto__()
-vfp_gzip_begin(struct worker *w, size_t estimate)
+vfp_gzip_begin(struct worker *wrk, size_t estimate)
 {
 	(void)estimate;
 
-	AZ(w->busyobj->vgz_rx);
-	w->busyobj->vgz_rx = VGZ_NewGzip(w, "G F -");
+	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
+	CHECK_OBJ_NOTNULL(wrk->busyobj, BUSYOBJ_MAGIC);
+	AZ(wrk->busyobj->vgz_rx);
+	wrk->busyobj->vgz_rx = VGZ_NewGzip(wrk, "G F -");
 }
 
 static int __match_proto__()
-vfp_gzip_bytes(struct worker *w, struct http_conn *htc, ssize_t bytes)
+vfp_gzip_bytes(struct worker *wrk, struct http_conn *htc, ssize_t bytes)
 {
 	struct vgz *vg;
 	ssize_t l, wl;
@@ -549,8 +557,10 @@ vfp_gzip_bytes(struct worker *w, struct http_conn *htc, ssize_t bytes)
 	size_t dl;
 	const void *dp;
 
-	AZ(w->busyobj->fetch_failed);
-	vg = w->busyobj->vgz_rx;
+	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
+	CHECK_OBJ_NOTNULL(wrk->busyobj, BUSYOBJ_MAGIC);
+	AZ(wrk->busyobj->fetch_failed);
+	vg = wrk->busyobj->vgz_rx;
 	CHECK_OBJ_NOTNULL(vg, VGZ_MAGIC);
 	AZ(vg->vz.avail_in);
 	while (bytes > 0 || !VGZ_IbufEmpty(vg)) {
@@ -558,50 +568,52 @@ vfp_gzip_bytes(struct worker *w, struct http_conn *htc, ssize_t bytes)
 			l = sizeof ibuf;
 			if (l > bytes)
 				l = bytes;
-			wl = HTC_Read(w, htc, ibuf, l);
+			wl = HTC_Read(wrk, htc, ibuf, l);
 			if (wl <= 0)
 				return (wl);
 			VGZ_Ibuf(vg, ibuf, wl);
 			bytes -= wl;
 		}
-		if (VGZ_ObufStorage(w, vg))
+		if (VGZ_ObufStorage(wrk, vg))
 			return(-1);
 		i = VGZ_Gzip(vg, &dp, &dl, VGZ_NORMAL);
 		assert(i == Z_OK);
-		w->busyobj->fetch_obj->len += dl;
-		if (w->busyobj->do_stream)
-			RES_StreamPoll(w);
+		wrk->busyobj->fetch_obj->len += dl;
+		if (wrk->busyobj->do_stream)
+			RES_StreamPoll(wrk);
 	}
 	return (1);
 }
 
 static int __match_proto__()
-vfp_gzip_end(struct worker *w)
+vfp_gzip_end(struct worker *wrk)
 {
 	struct vgz *vg;
 	size_t dl;
 	const void *dp;
 	int i;
 
-	vg = w->busyobj->vgz_rx;
+	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
+	CHECK_OBJ_NOTNULL(wrk->busyobj, BUSYOBJ_MAGIC);
+	vg = wrk->busyobj->vgz_rx;
 	CHECK_OBJ_NOTNULL(vg, VGZ_MAGIC);
-	w->busyobj->vgz_rx = NULL;
-	if (w->busyobj->fetch_failed) {
+	wrk->busyobj->vgz_rx = NULL;
+	if (wrk->busyobj->fetch_failed) {
 		(void)VGZ_Destroy(&vg, -1);
 		return(0);
 	}
 	do {
 		VGZ_Ibuf(vg, "", 0);
-		if (VGZ_ObufStorage(w, vg))
+		if (VGZ_ObufStorage(wrk, vg))
 			return(-1);
 		i = VGZ_Gzip(vg, &dp, &dl, VGZ_FINISH);
-		w->busyobj->fetch_obj->len += dl;
+		wrk->busyobj->fetch_obj->len += dl;
 	} while (i != Z_STREAM_END);
-	if (w->busyobj->do_stream)
-		RES_StreamPoll(w);
-	VGZ_UpdateObj(vg, w->busyobj->fetch_obj);
+	if (wrk->busyobj->do_stream)
+		RES_StreamPoll(wrk);
+	VGZ_UpdateObj(vg, wrk->busyobj->fetch_obj);
 	if (VGZ_Destroy(&vg, -1) != VGZ_END)
-		return(FetchError(w, "Gzip error at the very end"));
+		return(FetchError(wrk, "Gzip error at the very end"));
 	return (0);
 }
 
@@ -619,15 +631,17 @@ struct vfp vfp_gzip = {
  */
 
 static void __match_proto__()
-vfp_testgzip_begin(struct worker *w, size_t estimate)
+vfp_testgzip_begin(struct worker *wrk, size_t estimate)
 {
+	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
+	CHECK_OBJ_NOTNULL(wrk->busyobj, BUSYOBJ_MAGIC);
 	(void)estimate;
-	w->busyobj->vgz_rx = VGZ_NewUngzip(w, "u F -");
-	CHECK_OBJ_NOTNULL(w->busyobj->vgz_rx, VGZ_MAGIC);
+	wrk->busyobj->vgz_rx = VGZ_NewUngzip(wrk, "u F -");
+	CHECK_OBJ_NOTNULL(wrk->busyobj->vgz_rx, VGZ_MAGIC);
 }
 
 static int __match_proto__()
-vfp_testgzip_bytes(struct worker *w, struct http_conn *htc, ssize_t bytes)
+vfp_testgzip_bytes(struct worker *wrk, struct http_conn *htc, ssize_t bytes)
 {
 	struct vgz *vg;
 	ssize_t l, wl;
@@ -637,34 +651,36 @@ vfp_testgzip_bytes(struct worker *w, struct http_conn *htc, ssize_t bytes)
 	const void *dp;
 	struct storage *st;
 
-	AZ(w->busyobj->fetch_failed);
-	vg = w->busyobj->vgz_rx;
+	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
+	CHECK_OBJ_NOTNULL(wrk->busyobj, BUSYOBJ_MAGIC);
+	AZ(wrk->busyobj->fetch_failed);
+	vg = wrk->busyobj->vgz_rx;
 	CHECK_OBJ_NOTNULL(vg, VGZ_MAGIC);
 	AZ(vg->vz.avail_in);
 	while (bytes > 0) {
-		st = FetchStorage(w, 0);
+		st = FetchStorage(wrk, 0);
 		if (st == NULL)
 			return(-1);
 		l = st->space - st->len;
 		if (l > bytes)
 			l = bytes;
-		wl = HTC_Read(w, htc, st->ptr + st->len, l);
+		wl = HTC_Read(wrk, htc, st->ptr + st->len, l);
 		if (wl <= 0)
 			return (wl);
 		bytes -= wl;
 		VGZ_Ibuf(vg, st->ptr + st->len, wl);
 		st->len += wl;
-		w->busyobj->fetch_obj->len += wl;
-		if (w->busyobj->do_stream)
-			RES_StreamPoll(w);
+		wrk->busyobj->fetch_obj->len += wl;
+		if (wrk->busyobj->do_stream)
+			RES_StreamPoll(wrk);
 
 		while (!VGZ_IbufEmpty(vg)) {
 			VGZ_Obuf(vg, obuf, sizeof obuf);
 			i = VGZ_Gunzip(vg, &dp, &dl);
 			if (i == VGZ_END && !VGZ_IbufEmpty(vg))
-				return(FetchError(w, "Junk after gzip data"));
+				return(FetchError(wrk, "Junk after gzip data"));
 			if (i != VGZ_OK && i != VGZ_END)
-				return(FetchError2(w,
+				return(FetchError2(wrk,
 				    "Invalid Gzip data", vg->vz.msg));
 		}
 	}
@@ -673,20 +689,22 @@ vfp_testgzip_bytes(struct worker *w, struct http_conn *htc, ssize_t bytes)
 }
 
 static int __match_proto__()
-vfp_testgzip_end(struct worker *w)
+vfp_testgzip_end(struct worker *wrk)
 {
 	struct vgz *vg;
 
-	vg = w->busyobj->vgz_rx;
-	w->busyobj->vgz_rx = NULL;
+	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
+	CHECK_OBJ_NOTNULL(wrk->busyobj, BUSYOBJ_MAGIC);
+	vg = wrk->busyobj->vgz_rx;
+	wrk->busyobj->vgz_rx = NULL;
 	CHECK_OBJ_NOTNULL(vg, VGZ_MAGIC);
-	if (w->busyobj->fetch_failed) {
+	if (wrk->busyobj->fetch_failed) {
 		(void)VGZ_Destroy(&vg, -1);
 		return(0);
 	}
-	VGZ_UpdateObj(vg, w->busyobj->fetch_obj);
+	VGZ_UpdateObj(vg, wrk->busyobj->fetch_obj);
 	if (VGZ_Destroy(&vg, -1) != VGZ_END)
-		return(FetchError(w, "TestGunzip error at the very end"));
+		return(FetchError(wrk, "TestGunzip error at the very end"));
 	return (0);
 }
 



From geoff at varnish-cache.org  Mon Jan  9 20:52:48 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:48 +0100
Subject: [experimental-ims] f178da1 More code polishing
Message-ID: <E1RkMD2-0000Bk-GN@project.varnish-software.com>

commit f178da155efe9d4b7a1f65bcca2557032b94f42b
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Dec 6 08:39:44 2011 +0000

    More code polishing
    
    Revert two lines that got last in previous commit.

diff --git a/bin/varnishd/cache/cache_esi_fetch.c b/bin/varnishd/cache/cache_esi_fetch.c
index aea28ed..d2f4c01 100644
--- a/bin/varnishd/cache/cache_esi_fetch.c
+++ b/bin/varnishd/cache/cache_esi_fetch.c
@@ -44,7 +44,7 @@
  */
 
 static ssize_t
-vef_read(struct worker *w, struct http_conn *htc, void *buf, ssize_t buflen,
+vef_read(struct worker *wrk, struct http_conn *htc, void *buf, ssize_t buflen,
     ssize_t bytes)
 {
 	ssize_t d;
@@ -56,7 +56,7 @@ vef_read(struct worker *w, struct http_conn *htc, void *buf, ssize_t buflen,
 		if (d < bytes)
 			bytes = d;
 	}
-	return (HTC_Read(w, htc, buf, bytes));
+	return (HTC_Read(wrk, htc, buf, bytes));
 }
 
 /*---------------------------------------------------------------------
@@ -64,24 +64,24 @@ vef_read(struct worker *w, struct http_conn *htc, void *buf, ssize_t buflen,
  */
 
 static int
-vfp_esi_bytes_uu(struct worker *w, struct http_conn *htc, ssize_t bytes)
+vfp_esi_bytes_uu(struct worker *wrk, struct http_conn *htc, ssize_t bytes)
 {
 	ssize_t wl;
 	struct storage *st;
 
-	CHECK_OBJ_NOTNULL(w, WORKER_MAGIC);
+	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 
 	while (bytes > 0) {
-		st = FetchStorage(w, 0);
+		st = FetchStorage(wrk, 0);
 		if (st == NULL)
 			return (-1);
-		wl = vef_read(w, htc,
+		wl = vef_read(wrk, htc,
 		    st->ptr + st->len, st->space - st->len, bytes);
 		if (wl <= 0)
 			return (wl);
-		VEP_Parse(w, (const char *)st->ptr + st->len, wl);
+		VEP_Parse(wrk, (const char *)st->ptr + st->len, wl);
 		st->len += wl;
-		w->busyobj->fetch_obj->len += wl;
+		wrk->busyobj->fetch_obj->len += wl;
 		bytes -= wl;
 	}
 	return (1);
@@ -92,7 +92,7 @@ vfp_esi_bytes_uu(struct worker *w, struct http_conn *htc, ssize_t bytes)
  */
 
 static int
-vfp_esi_bytes_gu(struct worker *w, struct http_conn *htc, ssize_t bytes)
+vfp_esi_bytes_gu(struct worker *wrk, struct http_conn *htc, ssize_t bytes)
 {
 	struct vgz *vg;
 	ssize_t wl;
@@ -101,23 +101,23 @@ vfp_esi_bytes_gu(struct worker *w, struct http_conn *htc, ssize_t bytes)
 	size_t dl;
 	const void *dp;
 
-	CHECK_OBJ_NOTNULL(w, WORKER_MAGIC);
-	vg = w->busyobj->vgz_rx;
+	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
+	vg = wrk->busyobj->vgz_rx;
 
 	while (bytes > 0) {
 		if (VGZ_IbufEmpty(vg) && bytes > 0) {
-			wl = vef_read(w, htc, ibuf, sizeof ibuf, bytes);
+			wl = vef_read(wrk, htc, ibuf, sizeof ibuf, bytes);
 			if (wl <= 0)
 				return (wl);
 			VGZ_Ibuf(vg, ibuf, wl);
 			bytes -= wl;
 		}
-		if (VGZ_ObufStorage(w, vg))
+		if (VGZ_ObufStorage(wrk, vg))
 			return(-1);
 		i = VGZ_Gunzip(vg, &dp, &dl);
 		xxxassert(i == VGZ_OK || i == VGZ_END);
-		VEP_Parse(w, dp, dl);
-		w->busyobj->fetch_obj->len += dl;
+		VEP_Parse(wrk, dp, dl);
+		wrk->busyobj->fetch_obj->len += dl;
 	}
 	return (1);
 }
@@ -142,16 +142,16 @@ struct vef_priv {
  */
 
 static ssize_t
-vfp_vep_callback(struct worker *w, ssize_t l, enum vgz_flag flg)
+vfp_vep_callback(struct worker *wrk, ssize_t l, enum vgz_flag flg)
 {
 	struct vef_priv *vef;
 	size_t dl, px;
 	const void *dp;
 	int i;
 
-	CHECK_OBJ_NOTNULL(w, WORKER_MAGIC);
-	CHECK_OBJ_NOTNULL(w->busyobj, BUSYOBJ_MAGIC);
-	vef = w->busyobj->vef_priv;
+	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
+	CHECK_OBJ_NOTNULL(wrk->busyobj, BUSYOBJ_MAGIC);
+	vef = wrk->busyobj->vef_priv;
 	CHECK_OBJ_NOTNULL(vef, VEF_MAGIC);
 	assert(l >= 0);
 
@@ -181,14 +181,14 @@ vfp_vep_callback(struct worker *w, ssize_t l, enum vgz_flag flg)
 			l = 0;
 		}
 		do {
-			if (VGZ_ObufStorage(w, vef->vgz)) {
+			if (VGZ_ObufStorage(wrk, vef->vgz)) {
 				vef->error = ENOMEM;
 				vef->tot += l;
 				return (vef->tot);
 			}
 			i = VGZ_Gzip(vef->vgz, &dp, &dl, flg);
 			vef->tot += dl;
-			w->busyobj->fetch_obj->len += dl;
+			wrk->busyobj->fetch_obj->len += dl;
 		} while (!VGZ_IbufEmpty(vef->vgz) ||
 		    (flg != VGZ_NORMAL && VGZ_ObufFull(vef->vgz)));
 		if (px != 0) {
@@ -205,24 +205,24 @@ vfp_vep_callback(struct worker *w, ssize_t l, enum vgz_flag flg)
 }
 
 static int
-vfp_esi_bytes_ug(struct worker *w, struct http_conn *htc, ssize_t bytes)
+vfp_esi_bytes_ug(struct worker *wrk, struct http_conn *htc, ssize_t bytes)
 {
 	ssize_t wl;
 	char ibuf[cache_param->gzip_stack_buffer];
 	struct vef_priv *vef;
 
-	CHECK_OBJ_NOTNULL(w, WORKER_MAGIC);
-	CHECK_OBJ_NOTNULL(w->busyobj, BUSYOBJ_MAGIC);
-	vef = w->busyobj->vef_priv;
+	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
+	CHECK_OBJ_NOTNULL(wrk->busyobj, BUSYOBJ_MAGIC);
+	vef = wrk->busyobj->vef_priv;
 	CHECK_OBJ_NOTNULL(vef, VEF_MAGIC);
 
 	while (bytes > 0) {
-		wl = vef_read(w, htc, ibuf, sizeof ibuf, bytes);
+		wl = vef_read(wrk, htc, ibuf, sizeof ibuf, bytes);
 		if (wl <= 0)
 			return (wl);
 		bytes -= wl;
 		vef->bufp = ibuf;
-		VEP_Parse(w, ibuf, wl);
+		VEP_Parse(wrk, ibuf, wl);
 		assert(vef->bufp >= ibuf && vef->bufp <= ibuf + wl);
 		if (vef->error) {
 			errno = vef->error;
@@ -243,7 +243,7 @@ vfp_esi_bytes_ug(struct worker *w, struct http_conn *htc, ssize_t bytes)
  */
 
 static int
-vfp_esi_bytes_gg(struct worker *w, struct http_conn *htc, size_t bytes)
+vfp_esi_bytes_gg(struct worker *wrk, struct http_conn *htc, size_t bytes)
 {
 	ssize_t wl;
 	char ibuf[cache_param->gzip_stack_buffer];
@@ -253,29 +253,29 @@ vfp_esi_bytes_gg(struct worker *w, struct http_conn *htc, size_t bytes)
 	const void *dp;
 	int i;
 
-	CHECK_OBJ_NOTNULL(w, WORKER_MAGIC);
-	CHECK_OBJ_NOTNULL(w->busyobj, BUSYOBJ_MAGIC);
-	vef = w->busyobj->vef_priv;
+	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
+	CHECK_OBJ_NOTNULL(wrk->busyobj, BUSYOBJ_MAGIC);
+	vef = wrk->busyobj->vef_priv;
 	CHECK_OBJ_NOTNULL(vef, VEF_MAGIC);
 	assert(sizeof ibuf >= 1024);
 	ibuf2[0] = 0; /* For Flexelint */
 
 	while (bytes > 0) {
-		wl = vef_read(w, htc, ibuf, sizeof ibuf, bytes);
+		wl = vef_read(wrk, htc, ibuf, sizeof ibuf, bytes);
 		if (wl <= 0)
 			return (wl);
 		bytes -= wl;
 
 		vef->bufp = ibuf;
-		VGZ_Ibuf(w->busyobj->vgz_rx, ibuf, wl);
+		VGZ_Ibuf(wrk->busyobj->vgz_rx, ibuf, wl);
 		do {
-			VGZ_Obuf(w->busyobj->vgz_rx, ibuf2, sizeof ibuf2);
-			i = VGZ_Gunzip(w->busyobj->vgz_rx, &dp, &dl);
+			VGZ_Obuf(wrk->busyobj->vgz_rx, ibuf2, sizeof ibuf2);
+			i = VGZ_Gunzip(wrk->busyobj->vgz_rx, &dp, &dl);
 			/* XXX: check i */
 			assert(i >= VGZ_OK);
 			vef->bufp = ibuf2;
 			if (dl > 0)
-				VEP_Parse(w, ibuf2, dl);
+				VEP_Parse(wrk, ibuf2, dl);
 			if (vef->error) {
 				errno = vef->error;
 				return (-1);
@@ -287,7 +287,7 @@ vfp_esi_bytes_gg(struct worker *w, struct http_conn *htc, size_t bytes)
 				    vef->bufp, dl);
 				vef->npend += dl;
 			}
-		} while (!VGZ_IbufEmpty(w->busyobj->vgz_rx));
+		} while (!VGZ_IbufEmpty(wrk->busyobj->vgz_rx));
 	}
 	return (1);
 }
@@ -296,108 +296,114 @@ vfp_esi_bytes_gg(struct worker *w, struct http_conn *htc, size_t bytes)
 /*---------------------------------------------------------------------*/
 
 static void __match_proto__()
-vfp_esi_begin(struct worker *w, size_t estimate)
+vfp_esi_begin(struct worker *wrk, size_t estimate)
 {
+	struct busyobj *bo;
 	struct vef_priv *vef;
 
-	CHECK_OBJ_NOTNULL(w, WORKER_MAGIC);
-	CHECK_OBJ_NOTNULL(w->busyobj, BUSYOBJ_MAGIC);
-
-	AZ(w->busyobj->vgz_rx);
-	if (w->busyobj->is_gzip && w->busyobj->do_gunzip) {
-		w->busyobj->vgz_rx = VGZ_NewUngzip(w, "U F E");
-		VEP_Init(w, NULL);
-	} else if (w->busyobj->is_gunzip && w->busyobj->do_gzip) {
+	(void)estimate;
+	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
+	bo = wrk->busyobj;
+	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
+
+	AZ(bo->vgz_rx);
+	if (bo->is_gzip && bo->do_gunzip) {
+		bo->vgz_rx = VGZ_NewUngzip(wrk, "U F E");
+		VEP_Init(wrk, NULL);
+	} else if (bo->is_gunzip && bo->do_gzip) {
 		ALLOC_OBJ(vef, VEF_MAGIC);
 		AN(vef);
-		vef->vgz = VGZ_NewGzip(w, "G F E");
-		AZ(w->busyobj->vef_priv);
-		w->busyobj->vef_priv = vef;
-		VEP_Init(w, vfp_vep_callback);
-	} else if (w->busyobj->is_gzip) {
-		w->busyobj->vgz_rx = VGZ_NewUngzip(w, "U F E");
+		vef->vgz = VGZ_NewGzip(wrk, "G F E");
+		AZ(bo->vef_priv);
+		bo->vef_priv = vef;
+		VEP_Init(wrk, vfp_vep_callback);
+	} else if (bo->is_gzip) {
+		bo->vgz_rx = VGZ_NewUngzip(wrk, "U F E");
 		ALLOC_OBJ(vef, VEF_MAGIC);
 		AN(vef);
-		vef->vgz = VGZ_NewGzip(w, "G F E");
-		AZ(w->busyobj->vef_priv);
-		w->busyobj->vef_priv = vef;
-		VEP_Init(w, vfp_vep_callback);
+		vef->vgz = VGZ_NewGzip(wrk, "G F E");
+		AZ(bo->vef_priv);
+		bo->vef_priv = vef;
+		VEP_Init(wrk, vfp_vep_callback);
 	} else {
-		AZ(w->busyobj->vef_priv);
-		VEP_Init(w, NULL);
+		AZ(bo->vef_priv);
+		VEP_Init(wrk, NULL);
 	}
 
-	(void)estimate;
-	AN(w->busyobj->vep);
+	AN(bo->vep);
 }
 
 static int __match_proto__()
-vfp_esi_bytes(struct worker *w, struct http_conn *htc, ssize_t bytes)
+vfp_esi_bytes(struct worker *wrk, struct http_conn *htc, ssize_t bytes)
 {
+	struct busyobj *bo;
 	int i;
 
-	CHECK_OBJ_NOTNULL(w, WORKER_MAGIC);
-	AZ(w->busyobj->fetch_failed);
-	AN(w->busyobj->vep);
-	assert(&w->busyobj->htc == htc);
-	if (w->busyobj->is_gzip && w->busyobj->do_gunzip)
-		i = vfp_esi_bytes_gu(w, htc, bytes);
-	else if (w->busyobj->is_gunzip && w->busyobj->do_gzip)
-		i = vfp_esi_bytes_ug(w, htc, bytes);
-	else if (w->busyobj->is_gzip)
-		i = vfp_esi_bytes_gg(w, htc, bytes);
+	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
+	bo = wrk->busyobj;
+	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
+
+	AZ(bo->fetch_failed);
+	AN(bo->vep);
+	assert(&bo->htc == htc);
+	if (bo->is_gzip && bo->do_gunzip)
+		i = vfp_esi_bytes_gu(wrk, htc, bytes);
+	else if (bo->is_gunzip && bo->do_gzip)
+		i = vfp_esi_bytes_ug(wrk, htc, bytes);
+	else if (bo->is_gzip)
+		i = vfp_esi_bytes_gg(wrk, htc, bytes);
 	else
-		i = vfp_esi_bytes_uu(w, htc, bytes);
-	AN(w->busyobj->vep);
+		i = vfp_esi_bytes_uu(wrk, htc, bytes);
+	AN(bo->vep);
 	return (i);
 }
 
 static int __match_proto__()
-vfp_esi_end(struct worker *w)
+vfp_esi_end(struct worker *wrk)
 {
 	struct vsb *vsb;
 	struct vef_priv *vef;
+	struct busyobj *bo;
 	ssize_t l;
 	int retval;
 
-	CHECK_OBJ_NOTNULL(w, WORKER_MAGIC);
-	CHECK_OBJ_NOTNULL(w->busyobj, BUSYOBJ_MAGIC);
-	AN(w->busyobj->vep);
+	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
+	bo = wrk->busyobj;
+	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
+	AN(bo->vep);
 
-	retval = w->busyobj->fetch_failed;
+	retval = bo->fetch_failed;
 
-	if (w->busyobj->vgz_rx != NULL &&
-	     VGZ_Destroy(&w->busyobj->vgz_rx, -1) != VGZ_END)
-		retval = FetchError(w,
-		    "Gunzip+ESI Failed at the very end");
+	if (bo->vgz_rx != NULL && VGZ_Destroy(&bo->vgz_rx, -1) != VGZ_END)
+		retval = FetchError(wrk, "Gunzip+ESI Failed at the very end");
 
-	vsb = VEP_Finish(w);
+	vsb = VEP_Finish(wrk);
 
 	if (vsb != NULL) {
 		if (!retval) {
 			l = VSB_len(vsb);
 			assert(l > 0);
 			/* XXX: This is a huge waste of storage... */
-			w->busyobj->fetch_obj->esidata = STV_alloc(w, l);
-			if (w->busyobj->fetch_obj->esidata != NULL) {
-				memcpy(w->busyobj->fetch_obj->esidata->ptr,
+			bo->fetch_obj->esidata = STV_alloc(wrk, l);
+			if (bo->fetch_obj->esidata != NULL) {
+				memcpy(bo->fetch_obj->esidata->ptr,
 				    VSB_data(vsb), l);
-				w->busyobj->fetch_obj->esidata->len = l;
+				bo->fetch_obj->esidata->len = l;
 			} else {
-				retval = FetchError(w,
+				retval = FetchError(wrk,
 				    "Could not allocate storage for esidata");
 			}
 		}
 		VSB_delete(vsb);
 	}
 
-	vef = w->busyobj->vef_priv;
+	vef = bo->vef_priv;
 	if (vef != NULL) {
 		CHECK_OBJ_NOTNULL(vef, VEF_MAGIC);
-		w->busyobj->vef_priv = NULL;
-		VGZ_UpdateObj(vef->vgz, w->busyobj->fetch_obj);
+		bo->vef_priv = NULL;
+		VGZ_UpdateObj(vef->vgz, bo->fetch_obj);
 		if (VGZ_Destroy(&vef->vgz,  -1) != VGZ_END)
-			retval = FetchError(w,
+			retval = FetchError(wrk,
 			    "ESI+Gzip Failed at the very end");
 		FREE_OBJ(vef);
 	}
diff --git a/bin/varnishd/cache/cache_esi_parse.c b/bin/varnishd/cache/cache_esi_parse.c
index 41992ab..e431a8d 100644
--- a/bin/varnishd/cache/cache_esi_parse.c
+++ b/bin/varnishd/cache/cache_esi_parse.c
@@ -1007,6 +1007,8 @@ VEP_Init(struct worker *wrk, vep_callback_t *cb)
 	vep = (void*)WS_Alloc(wrk->ws, sizeof *vep);
 	AN(vep);
 
+	memset(vep, 0, sizeof *vep);
+	vep->magic = VEP_MAGIC;
 	vep->wrk = wrk;
 	vep->vsb = VSB_new_auto();
 	AN(vep->vsb);



From geoff at varnish-cache.org  Mon Jan  9 20:52:48 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:48 +0100
Subject: [experimental-ims] 3d723fd Rework the busyobj handling.
Message-ID: <E1RkMD2-0000CN-Ri@project.varnish-software.com>

commit 3d723fdab10f10c7f062e8350e1b81a341b28a21
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Tue Dec 6 10:37:40 2011 +0100

    Rework the busyobj handling.
    
    This patch reworks busyobj handling so that busyobjs are owned by the issuing worker, and the worker should explicitly release it when done with it. A busy objcore only lends a pointer to it. This is to fix the current situation where the owning party is murky, and the current code will leak busyobjs in at least one situation.
    
    BusyObj's are reference counted in preperation for the streaming code.
    
    Patch also adds several asserts to help make sure the busyobjs are sound.
    
    Fixes: #1068

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index 327682c..347de56 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -293,7 +293,7 @@ struct worker {
 	struct objhead		*nobjhead;
 	struct objcore		*nobjcore;
 	struct waitinglist	*nwaitinglist;
-	struct busyobj		*nbusyobj;
+	/* struct busyobj		*nbusyobj; */
 	void			*nhashpriv;
 	struct dstat		stats;
 
@@ -496,6 +496,8 @@ oc_getlru(const struct objcore *oc)
 struct busyobj {
 	unsigned		magic;
 #define BUSYOBJ_MAGIC		0x23b95567
+	unsigned		refcount;
+
 	uint8_t			*vary;
 	unsigned		is_gzip;
 	unsigned		is_gunzip;
@@ -667,8 +669,14 @@ void VDI_RecycleFd(struct worker *wrk, struct vbc **vbp);
 void VDI_AddHostHeader(struct worker *wrk, const struct vbc *vbc);
 void VBE_Poll(void);
 
-/* cache_backend_cfg.c */
+/* cache_backend.c */
 void VBE_Init(void);
+struct busyobj *VBE_GetBusyObj(void);
+struct busyobj *VBE_RefBusyObj(struct busyobj *busyobj);
+void VBE_DerefBusyObj(struct busyobj **busyobj);
+
+/* cache_backend_cfg.c */
+void VBE_InitCfg(void);
 struct backend *VBE_AddBackend(struct cli *cli, const struct vrt_backend *vb);
 
 /* cache_backend_poll.c */
@@ -995,21 +1003,6 @@ void SMP_Init(void);
 void SMP_Ready(void);
 void SMP_NewBan(const uint8_t *ban, unsigned len);
 
-#define New_BusyObj(wrk)						\
-	do {								\
-		if (wrk->nbusyobj != NULL) {				\
-			CHECK_OBJ_NOTNULL(wrk->nbusyobj, BUSYOBJ_MAGIC);\
-			wrk->busyobj = wrk->nbusyobj;			\
-			wrk->nbusyobj = NULL;				\
-			memset(wrk->busyobj, 0, sizeof *wrk->busyobj);	\
-			wrk->busyobj->magic = BUSYOBJ_MAGIC;		\
-		} else {						\
-			ALLOC_OBJ(wrk->busyobj, BUSYOBJ_MAGIC);		\
-		}							\
-		CHECK_OBJ_NOTNULL(wrk->busyobj, BUSYOBJ_MAGIC);		\
-		AZ(wrk->nbusyobj);					\
-	} while (0)
-
 /*
  * A normal pointer difference is signed, but we never want a negative value
  * so this little tool will make sure we don't get that.
@@ -1064,6 +1057,7 @@ AssertObjBusy(const struct object *o)
 {
 	AN(o->objcore);
 	AN (o->objcore->flags & OC_F_BUSY);
+	AN(o->objcore->busyobj);
 }
 
 static inline void
diff --git a/bin/varnishd/cache/cache_backend.c b/bin/varnishd/cache/cache_backend.c
index b661640..263aa17 100644
--- a/bin/varnishd/cache/cache_backend.c
+++ b/bin/varnishd/cache/cache_backend.c
@@ -41,6 +41,91 @@
 #include "vrt.h"
 #include "vtcp.h"
 
+static struct lock nbusyobj_mtx;
+static struct busyobj *nbusyobj;
+
+void
+VBE_Init(void)
+{
+	Lck_New(&nbusyobj_mtx, lck_nbusyobj);
+	nbusyobj = NULL;
+}
+
+/*--------------------------------------------------------------------
+ * BusyObj handling
+ */
+
+static struct busyobj *
+vbe_NewBusyObj(void)
+{
+	struct busyobj *busyobj;
+
+	ALLOC_OBJ(busyobj, BUSYOBJ_MAGIC);
+	AN(busyobj);
+	return (busyobj);
+}
+
+static void
+vbe_FreeBusyObj(struct busyobj *busyobj)
+{
+	CHECK_OBJ_NOTNULL(busyobj, BUSYOBJ_MAGIC);
+	AZ(busyobj->refcount);
+	FREE_OBJ(busyobj);
+}
+
+struct busyobj *
+VBE_GetBusyObj(void)
+{
+	struct busyobj *busyobj = NULL;
+
+	Lck_Lock(&nbusyobj_mtx);
+	if (nbusyobj != NULL) {
+		CHECK_OBJ_NOTNULL(nbusyobj, BUSYOBJ_MAGIC);
+		busyobj = nbusyobj;
+		nbusyobj = NULL;
+		memset(busyobj, 0, sizeof *busyobj);
+		busyobj->magic = BUSYOBJ_MAGIC;
+	}
+	Lck_Unlock(&nbusyobj_mtx);
+	if (busyobj == NULL)
+		busyobj = vbe_NewBusyObj();
+	AN(busyobj);
+	busyobj->refcount = 1;
+	return (busyobj);
+}
+
+struct busyobj *
+VBE_RefBusyObj(struct busyobj *busyobj)
+{
+	CHECK_OBJ_NOTNULL(busyobj, BUSYOBJ_MAGIC);
+	assert(busyobj->refcount > 0);
+	busyobj->refcount++;
+	return (busyobj);
+}
+
+void
+VBE_DerefBusyObj(struct busyobj **pbo)
+{
+	struct busyobj *busyobj;
+
+	busyobj = *pbo;
+	CHECK_OBJ_NOTNULL(busyobj, BUSYOBJ_MAGIC);
+	assert(busyobj->refcount > 0);
+	busyobj->refcount--;
+	*pbo = NULL;
+	if (busyobj->refcount > 0)
+		return;
+	/* XXX Sanity checks e.g. AZ(busyobj->vbc) */
+	Lck_Lock(&nbusyobj_mtx);
+	if (nbusyobj == NULL) {
+		nbusyobj = busyobj;
+		busyobj = NULL;
+	}
+	Lck_Unlock(&nbusyobj_mtx);
+	if (busyobj != NULL)
+		vbe_FreeBusyObj(busyobj);
+}
+
 /*--------------------------------------------------------------------
  * The "simple" director really isn't, since thats where all the actual
  * connections happen.  Nontheless, pretend it is simple by sequestering
diff --git a/bin/varnishd/cache/cache_backend_cfg.c b/bin/varnishd/cache/cache_backend_cfg.c
index cbb8c85..a47aa3c 100644
--- a/bin/varnishd/cache/cache_backend_cfg.c
+++ b/bin/varnishd/cache/cache_backend_cfg.c
@@ -499,7 +499,7 @@ static struct cli_proto backend_cmds[] = {
 /*---------------------------------------------------------------------*/
 
 void
-VBE_Init(void)
+VBE_InitCfg(void)
 {
 
 	Lck_New(&VBE_mtx, lck_vbe);
diff --git a/bin/varnishd/cache/cache_center.c b/bin/varnishd/cache/cache_center.c
index 079df44..995268a 100644
--- a/bin/varnishd/cache/cache_center.c
+++ b/bin/varnishd/cache/cache_center.c
@@ -179,8 +179,11 @@ cnt_prepresp(struct sess *sp)
 	CHECK_OBJ_NOTNULL(wrk->obj, OBJECT_MAGIC);
 	CHECK_OBJ_NOTNULL(sp->vcl, VCL_CONF_MAGIC);
 
-	if (wrk->busyobj != NULL && wrk->busyobj->do_stream)
+	if (wrk->busyobj != NULL) {
+		CHECK_OBJ_NOTNULL(wrk->busyobj, BUSYOBJ_MAGIC);
+		AN(wrk->busyobj->do_stream);
 		AssertObjCorePassOrBusy(wrk->obj->objcore);
+	}
 
 	wrk->res_mode = 0;
 
@@ -248,9 +251,11 @@ cnt_prepresp(struct sess *sp)
 	case VCL_RET_RESTART:
 		if (sp->restarts >= cache_param->max_restarts)
 			break;
-		if (wrk->busyobj->do_stream) {
+		if (wrk->busyobj != NULL) {
+			AN(wrk->busyobj->do_stream);
 			VDI_CloseFd(wrk, &wrk->busyobj->vbc);
 			HSH_Drop(wrk);
+			VBE_DerefBusyObj(&wrk->busyobj);
 		} else {
 			(void)HSH_Deref(wrk, NULL, &wrk->obj);
 		}
@@ -298,6 +303,7 @@ cnt_deliver(struct sess *sp)
 	wrk = sp->wrk;
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 
+	AZ(sp->wrk->busyobj);
 	sp->director = NULL;
 	sp->restarts = 0;
 
@@ -338,6 +344,7 @@ cnt_done(struct sess *sp)
 	CHECK_OBJ_ORNULL(sp->vcl, VCL_CONF_MAGIC);
 
 	AZ(wrk->obj);
+	AZ(wrk->busyobj);
 	sp->director = NULL;
 	sp->restarts = 0;
 
@@ -458,7 +465,8 @@ cnt_error(struct sess *sp)
 
 	if (wrk->obj == NULL) {
 		HSH_Prealloc(sp);
-		New_BusyObj(wrk);
+		AZ(wrk->busyobj);
+		wrk->busyobj = VBE_GetBusyObj();
 		wrk->obj = STV_NewObject(wrk, NULL, cache_param->http_resp_size,
 		     (uint16_t)cache_param->http_max_hdr);
 		if (wrk->obj == NULL)
@@ -477,6 +485,7 @@ cnt_error(struct sess *sp)
 		wrk->obj->xid = sp->xid;
 		wrk->obj->exp.entered = sp->t_req;
 	} else {
+		CHECK_OBJ_NOTNULL(wrk->busyobj, BUSYOBJ_MAGIC);
 		/* XXX: Null the headers ? */
 	}
 	CHECK_OBJ_NOTNULL(wrk->obj, OBJECT_MAGIC);
@@ -501,6 +510,7 @@ cnt_error(struct sess *sp)
 	if (sp->handling == VCL_RET_RESTART &&
 	    sp->restarts <  cache_param->max_restarts) {
 		HSH_Drop(wrk);
+		VBE_DerefBusyObj(&wrk->busyobj);
 		sp->director = NULL;
 		sp->restarts++;
 		sp->step = STP_RECV;
@@ -517,6 +527,7 @@ cnt_error(struct sess *sp)
 	sp->err_code = 0;
 	sp->err_reason = NULL;
 	http_Setup(wrk->bereq, NULL);
+	VBE_DerefBusyObj(&wrk->busyobj);
 	sp->step = STP_PREPRESP;
 	return (0);
 }
@@ -645,6 +656,7 @@ cnt_fetch(struct sess *sp)
 		AZ(HSH_Deref(wrk, wrk->objcore, NULL));
 		wrk->objcore = NULL;
 	}
+	VBE_DerefBusyObj(&wrk->busyobj);
 	http_Setup(wrk->bereq, NULL);
 	http_Setup(wrk->beresp, NULL);
 	sp->director = NULL;
@@ -819,6 +831,7 @@ cnt_fetchbody(struct sess *sp)
 		sp->err_code = 503;
 		sp->step = STP_ERROR;
 		VDI_CloseFd(wrk, &wrk->busyobj->vbc);
+		VBE_DerefBusyObj(&wrk->busyobj);
 		return (0);
 	}
 	CHECK_OBJ_NOTNULL(wrk->obj, OBJECT_MAGIC);
@@ -887,6 +900,7 @@ cnt_fetchbody(struct sess *sp)
 
 	if (i) {
 		HSH_Drop(wrk);
+		VBE_DerefBusyObj(&wrk->busyobj);
 		AZ(wrk->obj);
 		sp->err_code = 503;
 		sp->step = STP_ERROR;
@@ -899,6 +913,7 @@ cnt_fetchbody(struct sess *sp)
 		AN(wrk->obj->objcore->ban);
 		HSH_Unbusy(wrk);
 	}
+	VBE_DerefBusyObj(&wrk->busyobj);
 	wrk->acct_tmp.fetch++;
 	sp->step = STP_PREPRESP;
 	return (0);
@@ -972,6 +987,7 @@ cnt_streambody(struct sess *sp)
 	assert(WRW_IsReleased(wrk));
 	assert(wrk->wrw.ciov == wrk->wrw.siov);
 	(void)HSH_Deref(wrk, NULL, &wrk->obj);
+	VBE_DerefBusyObj(&wrk->busyobj);
 	http_Setup(wrk->resp, NULL);
 	sp->step = STP_DONE;
 	return (0);
@@ -1050,6 +1066,7 @@ cnt_hit(struct sess *sp)
 
 	CHECK_OBJ_NOTNULL(wrk->obj, OBJECT_MAGIC);
 	CHECK_OBJ_NOTNULL(sp->vcl, VCL_CONF_MAGIC);
+	AZ(wrk->busyobj);
 
 	assert(!(wrk->obj->objcore->flags & OC_F_PASS));
 
@@ -1067,7 +1084,6 @@ cnt_hit(struct sess *sp)
 	/* Drop our object, we won't need it */
 	(void)HSH_Deref(wrk, NULL, &wrk->obj);
 	wrk->objcore = NULL;
-	wrk->busyobj = NULL;
 
 	switch(sp->handling) {
 	case VCL_RET_PASS:
@@ -1127,6 +1143,7 @@ cnt_lookup(struct sess *sp)
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 
 	CHECK_OBJ_NOTNULL(sp->vcl, VCL_CONF_MAGIC);
+	AZ(wrk->busyobj);
 
 	if (sp->hash_objhead == NULL) {
 		/* Not a waiting list return */
@@ -1256,20 +1273,21 @@ cnt_miss(struct sess *sp)
 	wrk->connect_timeout = 0;
 	wrk->first_byte_timeout = 0;
 	wrk->between_bytes_timeout = 0;
-	CHECK_OBJ_NOTNULL(wrk->busyobj, BUSYOBJ_MAGIC);
 
 	VCL_miss_method(sp);
-	CHECK_OBJ_NOTNULL(wrk->busyobj, BUSYOBJ_MAGIC);
+
 	switch(sp->handling) {
 	case VCL_RET_ERROR:
 		AZ(HSH_Deref(wrk, wrk->objcore, NULL));
 		wrk->objcore = NULL;
 		http_Setup(wrk->bereq, NULL);
+		VBE_DerefBusyObj(&wrk->busyobj);
 		sp->step = STP_ERROR;
 		return (0);
 	case VCL_RET_PASS:
 		AZ(HSH_Deref(wrk, wrk->objcore, NULL));
 		wrk->objcore = NULL;
+		VBE_DerefBusyObj(&wrk->busyobj);
 		sp->step = STP_PASS;
 		return (0);
 	case VCL_RET_FETCH:
@@ -1279,6 +1297,7 @@ cnt_miss(struct sess *sp)
 	case VCL_RET_RESTART:
 		AZ(HSH_Deref(wrk, wrk->objcore, NULL));
 		wrk->objcore = NULL;
+		VBE_DerefBusyObj(&wrk->busyobj);
 		INCOMPL();
 	default:
 		WRONG("Illegal action in vcl_miss{}");
@@ -1327,6 +1346,7 @@ cnt_pass(struct sess *sp)
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 	CHECK_OBJ_NOTNULL(sp->vcl, VCL_CONF_MAGIC);
 	AZ(wrk->obj);
+	AZ(wrk->busyobj);
 
 	WS_Reset(wrk->ws, NULL);
 	http_Setup(wrk->bereq, wrk->ws);
@@ -1345,7 +1365,7 @@ cnt_pass(struct sess *sp)
 	wrk->acct_tmp.pass++;
 	sp->sendbody = 1;
 	sp->step = STP_FETCH;
-	New_BusyObj(wrk);
+	wrk->busyobj = VBE_GetBusyObj();
 	return (0);
 }
 
@@ -1433,6 +1453,7 @@ cnt_recv(struct sess *sp)
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 	CHECK_OBJ_NOTNULL(sp->vcl, VCL_CONF_MAGIC);
 	AZ(wrk->obj);
+	AZ(wrk->busyobj);
 	assert(wrk->wrw.ciov == wrk->wrw.siov);
 
 	/* By default we use the first backend */
diff --git a/bin/varnishd/cache/cache_hash.c b/bin/varnishd/cache/cache_hash.c
index 1f1795c..e33cdbe 100644
--- a/bin/varnishd/cache/cache_hash.c
+++ b/bin/varnishd/cache/cache_hash.c
@@ -106,11 +106,6 @@ HSH_Prealloc(const struct sess *sp)
 	}
 	CHECK_OBJ_NOTNULL(w->nwaitinglist, WAITINGLIST_MAGIC);
 
-	if (w->nbusyobj == NULL) {
-		ALLOC_OBJ(w->nbusyobj, BUSYOBJ_MAGIC);
-		XXXAN(w->nbusyobj);
-	}
-
 	if (hash->prep != NULL)
 		hash->prep(sp);
 }
@@ -139,10 +134,10 @@ HSH_Cleanup(struct worker *w)
 		free(w->nhashpriv);
 		w->nhashpriv = NULL;
 	}
-	if (w->nbusyobj != NULL) {
-		FREE_OBJ(w->nbusyobj);
-		w->nbusyobj = NULL;
-	}
+	/* if (w->nbusyobj != NULL) { */
+	/* 	FREE_OBJ(w->nbusyobj); */
+	/* 	w->nbusyobj = NULL; */
+	/* } */
 }
 
 void
@@ -456,7 +451,8 @@ HSH_Lookup(struct sess *sp, struct objhead **poh)
 	AN(oc->flags & OC_F_BUSY);
 	oc->refcnt = 1;
 
-	New_BusyObj(w);
+	AZ(w->busyobj);
+	w->busyobj = VBE_GetBusyObj();
 
 	VRY_Validate(sp->vary_b);
 	if (sp->vary_l != NULL)
@@ -626,8 +622,6 @@ HSH_Unbusy(struct worker *wrk)
 	VTAILQ_REMOVE(&oh->objcs, oc, list);
 	VTAILQ_INSERT_HEAD(&oh->objcs, oc, list);
 	oc->flags &= ~OC_F_BUSY;
-	AZ(wrk->nbusyobj);
-	wrk->nbusyobj = oc->busyobj;
 	oc->busyobj = NULL;
 	if (oh->waitinglist != NULL)
 		hsh_rush(oh);
@@ -716,16 +710,6 @@ HSH_Deref(struct worker *w, struct objcore *oc, struct object **oo)
 	BAN_DestroyObj(oc);
 	AZ(oc->ban);
 
-	if (oc->flags & OC_F_BUSY) {
-		CHECK_OBJ_NOTNULL(oc->busyobj, BUSYOBJ_MAGIC);
-		if (w->nbusyobj == NULL)
-			w->nbusyobj = oc->busyobj;
-		else
-			FREE_OBJ(oc->busyobj);
-		oc->busyobj = NULL;
-	}
-	AZ(oc->busyobj);
-
 	if (oc->methods != NULL) {
 		oc_freeobj(oc);
 		w->stats.n_object--;
diff --git a/bin/varnishd/cache/cache_main.c b/bin/varnishd/cache/cache_main.c
index fd31de4..fbe7624 100644
--- a/bin/varnishd/cache/cache_main.c
+++ b/bin/varnishd/cache/cache_main.c
@@ -118,6 +118,7 @@ child_main(void)
 	HTTP_Init();
 
 	VBE_Init();
+	VBE_InitCfg();
 	VBP_Init();
 	WRK_Init();
 	Pool_Init();
diff --git a/bin/varnishtest/tests/r01068.vtc b/bin/varnishtest/tests/r01068.vtc
new file mode 100644
index 0000000..c055d19
--- /dev/null
+++ b/bin/varnishtest/tests/r01068.vtc
@@ -0,0 +1,24 @@
+varnishtest "Bug 1068 restart on hit in vcl_deliver causes segfault"
+
+server s1 {
+	rxreq
+	txresp
+} -start
+
+varnish v1 -vcl+backend {
+	sub vcl_deliver {
+		if (req.http.x-restart && req.restarts == 0) {
+			return (restart);
+		}
+	}
+} -start
+
+client c1 {
+	txreq
+	rxresp
+	expect resp.status == 200
+
+	txreq -hdr "x-restart: true"
+	rxresp
+	expect resp.status == 200
+} -run
diff --git a/include/tbl/locks.h b/include/tbl/locks.h
index f1b634b..2ad717d 100644
--- a/include/tbl/locks.h
+++ b/include/tbl/locks.h
@@ -49,4 +49,5 @@ LOCK(vbp)
 LOCK(vbe)
 LOCK(backend)
 LOCK(vcapace)
+LOCK(nbusyobj)
 /*lint -restore */



From geoff at varnish-cache.org  Mon Jan  9 20:52:43 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:43 +0100
Subject: [experimental-ims] 5b77d34 Push sess out of HSH_Drop() and
	HSH_Unbusy()
Message-ID: <E1RkMCx-0008T6-CA@project.varnish-software.com>

commit 5b77d3496a174b30f1dadb21a8ce14293422f276
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Nov 29 12:49:08 2011 +0000

    Push sess out of HSH_Drop() and HSH_Unbusy()

diff --git a/bin/varnishd/cache/cache_center.c b/bin/varnishd/cache/cache_center.c
index 13db2e3..3832116 100644
--- a/bin/varnishd/cache/cache_center.c
+++ b/bin/varnishd/cache/cache_center.c
@@ -238,7 +238,7 @@ cnt_prepresp(struct sess *sp)
 			break;
 		if (sp->wrk->do_stream) {
 			VDI_CloseFd(sp->wrk);
-			HSH_Drop(sp);
+			HSH_Drop(sp->wrk);
 		} else {
 			(void)HSH_Deref(sp->wrk, NULL, &sp->wrk->obj);
 		}
@@ -495,7 +495,7 @@ cnt_error(struct sess *sp)
 
 	if (sp->handling == VCL_RET_RESTART &&
 	    sp->restarts <  cache_param->max_restarts) {
-		HSH_Drop(sp);
+		HSH_Drop(sp->wrk);
 		sp->director = NULL;
 		sp->restarts++;
 		sp->step = STP_RECV;
@@ -873,7 +873,7 @@ cnt_fetchbody(struct sess *sp)
 	AN(sp->director);
 
 	if (i) {
-		HSH_Drop(sp);
+		HSH_Drop(wrk);
 		AZ(wrk->obj);
 		sp->err_code = 503;
 		sp->step = STP_ERROR;
@@ -884,7 +884,7 @@ cnt_fetchbody(struct sess *sp)
 		EXP_Insert(wrk->obj);
 		AN(wrk->obj->objcore);
 		AN(wrk->obj->objcore->ban);
-		HSH_Unbusy(sp);
+		HSH_Unbusy(wrk);
 	}
 	wrk->acct_tmp.fetch++;
 	sp->step = STP_PREPRESP;
@@ -939,7 +939,7 @@ cnt_streambody(struct sess *sp)
 		EXP_Insert(sp->wrk->obj);
 		AN(sp->wrk->obj->objcore);
 		AN(sp->wrk->obj->objcore->ban);
-		HSH_Unbusy(sp);
+		HSH_Unbusy(sp->wrk);
 	} else {
 		sp->doclose = "Stream error";
 	}
diff --git a/bin/varnishd/cache/cache_hash.c b/bin/varnishd/cache/cache_hash.c
index 2820c45..a2fc985 100644
--- a/bin/varnishd/cache/cache_hash.c
+++ b/bin/varnishd/cache/cache_hash.c
@@ -580,29 +580,29 @@ HSH_Purge(const struct sess *sp, struct objhead *oh, double ttl, double grace)
  */
 
 void
-HSH_Drop(struct sess *sp)
+HSH_Drop(struct worker *wrk)
 {
 	struct object *o;
 
-	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
-	o = sp->wrk->obj;
+	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
+	o = wrk->obj;
 	CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC);
 	AssertObjCorePassOrBusy(o->objcore);
 	o->exp.ttl = -1.;
 	if (o->objcore != NULL)		/* Pass has no objcore */
-		HSH_Unbusy(sp);
-	(void)HSH_Deref(sp->wrk, NULL, &sp->wrk->obj);
+		HSH_Unbusy(wrk);
+	(void)HSH_Deref(wrk, NULL, &wrk->obj);
 }
 
 void
-HSH_Unbusy(const struct sess *sp)
+HSH_Unbusy(struct worker *wrk)
 {
 	struct object *o;
 	struct objhead *oh;
 	struct objcore *oc;
 
-	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
-	o = sp->wrk->obj;
+	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
+	o = wrk->obj;
 	CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC);
 	oc = o->objcore;
 	CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
@@ -614,9 +614,9 @@ HSH_Unbusy(const struct sess *sp)
 	assert(oc->refcnt > 0);
 	assert(oh->refcnt > 0);
 	if (o->ws_o->overflow)
-		sp->wrk->stats.n_objoverflow++;
+		wrk->stats.n_objoverflow++;
 	if (cache_param->diag_bitmap & 0x40)
-		WSP(sp, SLT_Debug,
+		WSL(wrk, SLT_Debug, 0,
 		    "Object %u workspace free %u", o->xid, WS_Free(o->ws_o));
 
 	/* XXX: pretouch neighbors on oh->objcs to prevent page-on under mtx */
@@ -626,14 +626,14 @@ HSH_Unbusy(const struct sess *sp)
 	VTAILQ_REMOVE(&oh->objcs, oc, list);
 	VTAILQ_INSERT_HEAD(&oh->objcs, oc, list);
 	oc->flags &= ~OC_F_BUSY;
-	AZ(sp->wrk->nbusyobj);
-	sp->wrk->nbusyobj = oc->busyobj;
+	AZ(wrk->nbusyobj);
+	wrk->nbusyobj = oc->busyobj;
 	oc->busyobj = NULL;
 	if (oh->waitinglist != NULL)
 		hsh_rush(oh);
 	AN(oc->ban);
 	Lck_Unlock(&oh->mtx);
-	assert(oc_getobj(sp->wrk, oc) == o);
+	assert(oc_getobj(wrk, oc) == o);
 }
 
 void
diff --git a/bin/varnishd/hash/hash_slinger.h b/bin/varnishd/hash/hash_slinger.h
index 9e3b13f..b45e604 100644
--- a/bin/varnishd/hash/hash_slinger.h
+++ b/bin/varnishd/hash/hash_slinger.h
@@ -54,9 +54,9 @@ struct hash_slinger {
 void HSH_Prealloc(const struct sess *sp);
 void HSH_Cleanup(struct worker *w);
 struct objcore *HSH_Lookup(struct sess *sp, struct objhead **poh);
-void HSH_Unbusy(const struct sess *sp);
+void HSH_Unbusy(struct worker *wrk);
 void HSH_Ref(struct objcore *o);
-void HSH_Drop(struct sess *sp);
+void HSH_Drop(struct worker *wrk);
 void HSH_Init(const struct hash_slinger *slinger);
 void HSH_AddString(const struct sess *sp, const char *str);
 struct objcore *HSH_Insert(const struct sess *sp);



From geoff at varnish-cache.org  Mon Jan  9 20:52:44 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:44 +0100
Subject: [experimental-ims] 9d67fce Move (beresp.)exp from worker to busyobj
Message-ID: <E1RkMCy-0008UK-10@project.varnish-software.com>

commit 9d67fcef723e583e56dbb43bf202259fe45f4b07
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Nov 29 18:10:22 2011 +0000

    Move (beresp.)exp from worker to busyobj

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index 7a0167c..5196575 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -325,8 +325,6 @@ struct worker {
 	struct objcore		*objcore;
 	struct busyobj		*busyobj;
 
-	struct exp		exp;
-
 	/* This is only here so VRT can find it */
 	const char		*storage_hint;
 
@@ -503,6 +501,8 @@ struct busyobj {
 	struct vep_state	*vep;
 	unsigned		fetch_failed;
 	struct vgz		*vgz_rx;
+
+	struct exp		exp;
 };
 
 /* Object structure --------------------------------------------------*/
diff --git a/bin/varnishd/cache/cache_center.c b/bin/varnishd/cache/cache_center.c
index 27209de..86b54d0 100644
--- a/bin/varnishd/cache/cache_center.c
+++ b/bin/varnishd/cache/cache_center.c
@@ -464,7 +464,7 @@ cnt_error(struct sess *sp)
 
 	if (wrk->obj == NULL) {
 		HSH_Prealloc(sp);
-		EXP_Clr(&wrk->exp);
+		New_BusyObj(wrk);
 		wrk->obj = STV_NewObject(wrk, NULL, cache_param->http_resp_size,
 		     (uint16_t)cache_param->http_max_hdr);
 		if (wrk->obj == NULL)
@@ -613,13 +613,13 @@ cnt_fetch(struct sess *sp)
 		/*
 		 * What does RFC2616 think about TTL ?
 		 */
-		EXP_Clr(&wrk->exp);
-		wrk->exp.entered = W_TIM_real(wrk);
+		EXP_Clr(&wrk->busyobj->exp);
+		wrk->busyobj->exp.entered = W_TIM_real(wrk);
 		RFC2616_Ttl(sp);
 
 		/* pass from vclrecv{} has negative TTL */
 		if (wrk->objcore == NULL)
-			wrk->exp.ttl = -1.;
+			wrk->busyobj->exp.ttl = -1.;
 
 		AZ(wrk->do_esi);
 
@@ -715,7 +715,7 @@ cnt_fetchbody(struct sess *sp)
 		/* This is a pass from vcl_recv */
 		pass = 1;
 		/* VCL may have fiddled this, but that doesn't help */
-		wrk->exp.ttl = -1.;
+		wrk->busyobj->exp.ttl = -1.;
 	} else if (sp->handling == VCL_RET_HIT_FOR_PASS) {
 		/* pass from vcl_fetch{} -> hit-for-pass */
 		/* XXX: the bereq was not filtered pass... */
@@ -806,7 +806,8 @@ cnt_fetchbody(struct sess *sp)
 	 */
 	l += strlen("Content-Length: XxxXxxXxxXxxXxxXxx") + sizeof(void *);
 
-	if (wrk->exp.ttl < cache_param->shortlived || wrk->objcore == NULL)
+	if (wrk->busyobj->exp.ttl < cache_param->shortlived ||
+	    wrk->objcore == NULL)
 		wrk->storage_hint = TRANSIENT_STORAGE;
 
 	wrk->obj = STV_NewObject(wrk, wrk->storage_hint, l, nhttp);
@@ -816,10 +817,10 @@ cnt_fetchbody(struct sess *sp)
 		 * shortlived object on Transient storage.
 		 */
 		wrk->obj = STV_NewObject(wrk, TRANSIENT_STORAGE, l, nhttp);
-		if (wrk->exp.ttl > cache_param->shortlived)
-			wrk->exp.ttl = cache_param->shortlived;
-		wrk->exp.grace = 0.0;
-		wrk->exp.keep = 0.0;
+		if (wrk->busyobj->exp.ttl > cache_param->shortlived)
+			wrk->busyobj->exp.ttl = cache_param->shortlived;
+		wrk->busyobj->exp.grace = 0.0;
+		wrk->busyobj->exp.keep = 0.0;
 	}
 	if (wrk->obj == NULL) {
 		sp->err_code = 503;
@@ -859,7 +860,7 @@ cnt_fetchbody(struct sess *sp)
 	if (http_GetHdr(hp, H_Last_Modified, &b))
 		wrk->obj->last_modified = VTIM_parse(b);
 	else
-		wrk->obj->last_modified = floor(wrk->exp.entered);
+		wrk->obj->last_modified = floor(wrk->busyobj->exp.entered);
 
 	assert(WRW_IsReleased(wrk));
 
diff --git a/bin/varnishd/cache/cache_rfc2616.c b/bin/varnishd/cache/cache_rfc2616.c
index e22da50..ae9614d 100644
--- a/bin/varnishd/cache/cache_rfc2616.c
+++ b/bin/varnishd/cache/cache_rfc2616.c
@@ -69,12 +69,15 @@ RFC2616_Ttl(const struct sess *sp)
 	double h_date, h_expires;
 	char *p;
 	const struct http *hp;
+	struct exp *expp;
+
+	expp = &sp->wrk->busyobj->exp;
 
 	hp = sp->wrk->beresp;
 
-	assert(sp->wrk->exp.entered != 0.0 && !isnan(sp->wrk->exp.entered));
+	assert(expp->entered != 0.0 && !isnan(expp->entered));
 	/* If all else fails, cache using default ttl */
-	sp->wrk->exp.ttl = cache_param->default_ttl;
+	expp->ttl = cache_param->default_ttl;
 
 	max_age = age = 0;
 	h_expires = 0;
@@ -87,7 +90,7 @@ RFC2616_Ttl(const struct sess *sp)
 
 	if (http_GetHdr(hp, H_Age, &p)) {
 		age = strtoul(p, NULL, 0);
-		sp->wrk->exp.age = age;
+		expp->age = age;
 	}
 	if (http_GetHdr(hp, H_Expires, &p))
 		h_expires = VTIM_parse(p);
@@ -97,7 +100,7 @@ RFC2616_Ttl(const struct sess *sp)
 
 	switch (sp->err_code) {
 	default:
-		sp->wrk->exp.ttl = -1.;
+		expp->ttl = -1.;
 		break;
 	case 200: /* OK */
 	case 203: /* Non-Authoritative Information */
@@ -122,9 +125,9 @@ RFC2616_Ttl(const struct sess *sp)
 				max_age = strtoul(p, NULL, 0);
 
 			if (age > max_age)
-				sp->wrk->exp.ttl = 0;
+				expp->ttl = 0;
 			else
-				sp->wrk->exp.ttl = max_age - age;
+				expp->ttl = max_age - age;
 			break;
 		}
 
@@ -135,22 +138,22 @@ RFC2616_Ttl(const struct sess *sp)
 
 		/* If backend told us it is expired already, don't cache. */
 		if (h_expires < h_date) {
-			sp->wrk->exp.ttl = 0;
+			expp->ttl = 0;
 			break;
 		}
 
 		if (h_date == 0 ||
-		    fabs(h_date - sp->wrk->exp.entered) < cache_param->clock_skew) {
+		    fabs(h_date - expp->entered) < cache_param->clock_skew) {
 			/*
 			 * If we have no Date: header or if it is
 			 * sufficiently close to our clock we will
 			 * trust Expires: relative to our own clock.
 			 */
-			if (h_expires < sp->wrk->exp.entered)
-				sp->wrk->exp.ttl = 0;
+			if (h_expires < expp->entered)
+				expp->ttl = 0;
 			else
-				sp->wrk->exp.ttl = h_expires -
-				    sp->wrk->exp.entered;
+				expp->ttl = h_expires -
+				    expp->entered;
 			break;
 		} else {
 			/*
@@ -158,7 +161,7 @@ RFC2616_Ttl(const struct sess *sp)
 			 * derive a relative time from the two headers.
 			 * (the negative ttl case is caught above)
 			 */
-			sp->wrk->exp.ttl = (int)(h_expires - h_date);
+			expp->ttl = (int)(h_expires - h_date);
 		}
 
 	}
@@ -166,8 +169,8 @@ RFC2616_Ttl(const struct sess *sp)
 	/* calculated TTL, Our time, Date, Expires, max-age, age */
 	WSP(sp, SLT_TTL,
 	    "%u RFC %.0f %.0f %.0f %.0f %.0f %.0f %.0f %u",
-	    sp->xid, sp->wrk->exp.ttl, -1., -1., sp->wrk->exp.entered,
-	    sp->wrk->exp.age, h_date, h_expires, max_age);
+	    sp->xid, expp->ttl, -1., -1., expp->entered,
+	    expp->age, h_date, h_expires, max_age);
 }
 
 /*--------------------------------------------------------------------
diff --git a/bin/varnishd/cache/cache_vrt_var.c b/bin/varnishd/cache/cache_vrt_var.c
index 5951b9c..b889d69 100644
--- a/bin/varnishd/cache/cache_vrt_var.c
+++ b/bin/varnishd/cache/cache_vrt_var.c
@@ -411,12 +411,12 @@ VRT_DO_EXP(obj, sp->wrk->obj->exp, keep, 0,
    EXP_Rearm(sp->wrk->obj);
    vrt_wsp_exp(sp, sp->wrk->obj->xid, &sp->wrk->obj->exp);)
 
-VRT_DO_EXP(beresp, sp->wrk->exp, grace, 0,
-   vrt_wsp_exp(sp, sp->xid, &sp->wrk->exp);)
-VRT_DO_EXP(beresp, sp->wrk->exp, ttl, 0,
-   vrt_wsp_exp(sp, sp->xid, &sp->wrk->exp);)
-VRT_DO_EXP(beresp, sp->wrk->exp, keep, 0,
-   vrt_wsp_exp(sp, sp->xid, &sp->wrk->exp);)
+VRT_DO_EXP(beresp, sp->wrk->busyobj->exp, grace, 0,
+   vrt_wsp_exp(sp, sp->xid, &sp->wrk->busyobj->exp);)
+VRT_DO_EXP(beresp, sp->wrk->busyobj->exp, ttl, 0,
+   vrt_wsp_exp(sp, sp->xid, &sp->wrk->busyobj->exp);)
+VRT_DO_EXP(beresp, sp->wrk->busyobj->exp, keep, 0,
+   vrt_wsp_exp(sp, sp->xid, &sp->wrk->busyobj->exp);)
 
 /*--------------------------------------------------------------------
  * req.xid
diff --git a/bin/varnishd/storage/stevedore.c b/bin/varnishd/storage/stevedore.c
index 1a4b8ac..7c364c0 100644
--- a/bin/varnishd/storage/stevedore.c
+++ b/bin/varnishd/storage/stevedore.c
@@ -242,7 +242,7 @@ STV_MkObject(struct worker *wrk, void *ptr, unsigned ltot,
 
 	http_Setup(o->http, o->ws_o);
 	o->http->magic = HTTP_MAGIC;
-	o->exp = wrk->exp;
+	o->exp = wrk->busyobj->exp;
 	VTAILQ_INIT(&o->store);
 	wrk->stats.n_object++;
 
diff --git a/bin/varnishd/storage/storage_persistent.c b/bin/varnishd/storage/storage_persistent.c
index 15087b3..0791941 100644
--- a/bin/varnishd/storage/storage_persistent.c
+++ b/bin/varnishd/storage/storage_persistent.c
@@ -478,7 +478,7 @@ smp_allocobj(struct stevedore *stv, struct worker *wrk, unsigned ltot,
 		return (NULL);		/* from cnt_error */
 	CAST_OBJ_NOTNULL(sc, stv->priv, SMP_SC_MAGIC);
 	AN(wrk->objcore);
-	AN(wrk->exp.ttl > 0.);
+	AN(wrk->busyobj->exp.ttl > 0.);
 
 	ltot = IRNUP(sc, ltot);
 



From geoff at varnish-cache.org  Mon Jan  9 20:52:49 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:49 +0100
Subject: [experimental-ims] 3025017 Add locks to busyobj's and use locks on
	refcounting.
Message-ID: <E1RkMD3-0000Cx-6C@project.varnish-software.com>

commit 3025017171ad9d0959dfee5b5760b044cd4501e7
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Tue Dec 6 11:55:58 2011 +0100

    Add locks to busyobj's and use locks on refcounting.

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index 347de56..cf0a6ee 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -496,6 +496,8 @@ oc_getlru(const struct objcore *oc)
 struct busyobj {
 	unsigned		magic;
 #define BUSYOBJ_MAGIC		0x23b95567
+	struct lock		mtx;
+	/* Members passed this line are cleared on reuse */
 	unsigned		refcount;
 
 	uint8_t			*vary;
diff --git a/bin/varnishd/cache/cache_backend.c b/bin/varnishd/cache/cache_backend.c
index 263aa17..ef937ea 100644
--- a/bin/varnishd/cache/cache_backend.c
+++ b/bin/varnishd/cache/cache_backend.c
@@ -34,6 +34,8 @@
 
 #include <poll.h>
 #include <stdlib.h>
+#include <stddef.h>
+#include <stdio.h>
 
 #include "cache.h"
 
@@ -62,6 +64,7 @@ vbe_NewBusyObj(void)
 
 	ALLOC_OBJ(busyobj, BUSYOBJ_MAGIC);
 	AN(busyobj);
+	Lck_New(&busyobj->mtx, lck_busyobj);
 	return (busyobj);
 }
 
@@ -70,6 +73,7 @@ vbe_FreeBusyObj(struct busyobj *busyobj)
 {
 	CHECK_OBJ_NOTNULL(busyobj, BUSYOBJ_MAGIC);
 	AZ(busyobj->refcount);
+	Lck_Delete(&busyobj->mtx);
 	FREE_OBJ(busyobj);
 }
 
@@ -83,8 +87,8 @@ VBE_GetBusyObj(void)
 		CHECK_OBJ_NOTNULL(nbusyobj, BUSYOBJ_MAGIC);
 		busyobj = nbusyobj;
 		nbusyobj = NULL;
-		memset(busyobj, 0, sizeof *busyobj);
-		busyobj->magic = BUSYOBJ_MAGIC;
+		memset((char *)busyobj + offsetof(struct busyobj, refcount), 0,
+		       sizeof *busyobj - offsetof(struct busyobj, refcount));
 	}
 	Lck_Unlock(&nbusyobj_mtx);
 	if (busyobj == NULL)
@@ -98,8 +102,10 @@ struct busyobj *
 VBE_RefBusyObj(struct busyobj *busyobj)
 {
 	CHECK_OBJ_NOTNULL(busyobj, BUSYOBJ_MAGIC);
+	Lck_Lock(&busyobj->mtx);
 	assert(busyobj->refcount > 0);
 	busyobj->refcount++;
+	Lck_Unlock(&busyobj->mtx);
 	return (busyobj);
 }
 
@@ -110,12 +116,18 @@ VBE_DerefBusyObj(struct busyobj **pbo)
 
 	busyobj = *pbo;
 	CHECK_OBJ_NOTNULL(busyobj, BUSYOBJ_MAGIC);
+	Lck_Lock(&busyobj->mtx);
 	assert(busyobj->refcount > 0);
 	busyobj->refcount--;
 	*pbo = NULL;
-	if (busyobj->refcount > 0)
+	if (busyobj->refcount > 0) {
+		Lck_Unlock(&busyobj->mtx);
 		return;
+	}
+	Lck_Unlock(&busyobj->mtx);
+
 	/* XXX Sanity checks e.g. AZ(busyobj->vbc) */
+
 	Lck_Lock(&nbusyobj_mtx);
 	if (nbusyobj == NULL) {
 		nbusyobj = busyobj;
diff --git a/include/tbl/locks.h b/include/tbl/locks.h
index 2ad717d..d86da01 100644
--- a/include/tbl/locks.h
+++ b/include/tbl/locks.h
@@ -50,4 +50,5 @@ LOCK(vbe)
 LOCK(backend)
 LOCK(vcapace)
 LOCK(nbusyobj)
+LOCK(busyobj)
 /*lint -restore */



From geoff at varnish-cache.org  Mon Jan  9 20:52:49 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:49 +0100
Subject: [experimental-ims] f483dd6 Replace 'w' with 'wrk' as the variable for
	worker thread.
Message-ID: <E1RkMD3-0000De-Fg@project.varnish-software.com>

commit f483dd61d454efd7b8b79f50496237ad73f8ad83
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Dec 7 10:17:36 2011 +0000

    Replace 'w' with 'wrk' as the variable for worker thread.

diff --git a/bin/varnishd/cache/cache_hash.c b/bin/varnishd/cache/cache_hash.c
index 0bd002d..8a73f6c 100644
--- a/bin/varnishd/cache/cache_hash.c
+++ b/bin/varnishd/cache/cache_hash.c
@@ -68,86 +68,86 @@ static const struct hash_slinger *hash;
 void
 HSH_Prealloc(const struct sess *sp)
 {
-	struct worker *w;
+	struct worker *wrk;
 	struct objhead *oh;
 	struct objcore *oc;
 	struct waitinglist *wl;
 
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 	CHECK_OBJ_NOTNULL(sp->wrk, WORKER_MAGIC);
-	w = sp->wrk;
+	wrk = sp->wrk;
 
-	if (w->nobjcore == NULL) {
+	if (wrk->nobjcore == NULL) {
 		ALLOC_OBJ(oc, OBJCORE_MAGIC);
 		XXXAN(oc);
-		w->nobjcore = oc;
-		w->stats.n_objectcore++;
+		wrk->nobjcore = oc;
+		wrk->stats.n_objectcore++;
 		oc->flags |= OC_F_BUSY;
 	}
-	CHECK_OBJ_NOTNULL(w->nobjcore, OBJCORE_MAGIC);
+	CHECK_OBJ_NOTNULL(wrk->nobjcore, OBJCORE_MAGIC);
 
-	if (w->nobjhead == NULL) {
+	if (wrk->nobjhead == NULL) {
 		ALLOC_OBJ(oh, OBJHEAD_MAGIC);
 		XXXAN(oh);
 		oh->refcnt = 1;
 		VTAILQ_INIT(&oh->objcs);
 		Lck_New(&oh->mtx, lck_objhdr);
-		w->nobjhead = oh;
-		w->stats.n_objecthead++;
+		wrk->nobjhead = oh;
+		wrk->stats.n_objecthead++;
 	}
-	CHECK_OBJ_NOTNULL(w->nobjhead, OBJHEAD_MAGIC);
+	CHECK_OBJ_NOTNULL(wrk->nobjhead, OBJHEAD_MAGIC);
 
-	if (w->nwaitinglist == NULL) {
+	if (wrk->nwaitinglist == NULL) {
 		ALLOC_OBJ(wl, WAITINGLIST_MAGIC);
 		XXXAN(wl);
 		VTAILQ_INIT(&wl->list);
-		w->nwaitinglist = wl;
-		w->stats.n_waitinglist++;
+		wrk->nwaitinglist = wl;
+		wrk->stats.n_waitinglist++;
 	}
-	CHECK_OBJ_NOTNULL(w->nwaitinglist, WAITINGLIST_MAGIC);
+	CHECK_OBJ_NOTNULL(wrk->nwaitinglist, WAITINGLIST_MAGIC);
 
 	if (hash->prep != NULL)
 		hash->prep(sp);
 }
 
 void
-HSH_Cleanup(struct worker *w)
+HSH_Cleanup(struct worker *wrk)
 {
 
-	if (w->nobjcore != NULL) {
-		FREE_OBJ(w->nobjcore);
-		w->stats.n_objectcore--;
-		w->nobjcore = NULL;
+	if (wrk->nobjcore != NULL) {
+		FREE_OBJ(wrk->nobjcore);
+		wrk->stats.n_objectcore--;
+		wrk->nobjcore = NULL;
 	}
-	if (w->nobjhead != NULL) {
-		Lck_Delete(&w->nobjhead->mtx);
-		FREE_OBJ(w->nobjhead);
-		w->nobjhead = NULL;
-		w->stats.n_objecthead--;
+	if (wrk->nobjhead != NULL) {
+		Lck_Delete(&wrk->nobjhead->mtx);
+		FREE_OBJ(wrk->nobjhead);
+		wrk->nobjhead = NULL;
+		wrk->stats.n_objecthead--;
 	}
-	if (w->nwaitinglist != NULL) {
-		FREE_OBJ(w->nwaitinglist);
-		w->nwaitinglist = NULL;
+	if (wrk->nwaitinglist != NULL) {
+		FREE_OBJ(wrk->nwaitinglist);
+		wrk->nwaitinglist = NULL;
 	}
-	if (w->nhashpriv != NULL) {
+	if (wrk->nhashpriv != NULL) {
 		/* XXX: If needed, add slinger method for this */
-		free(w->nhashpriv);
-		w->nhashpriv = NULL;
+		free(wrk->nhashpriv);
+		wrk->nhashpriv = NULL;
 	}
-	/* if (w->nbusyobj != NULL) { */
-	/* 	FREE_OBJ(w->nbusyobj); */
-	/* 	w->nbusyobj = NULL; */
+	/* if (wrk->nbusyobj != NULL) { */
+	/* 	FREE_OBJ(wrk->nbusyobj); */
+	/* 	wrk->nbusyobj = NULL; */
 	/* } */
 }
 
 void
-HSH_DeleteObjHead(struct worker *w, struct objhead *oh)
+HSH_DeleteObjHead(struct worker *wrk, struct objhead *oh)
 {
 
 	AZ(oh->refcnt);
 	assert(VTAILQ_EMPTY(&oh->objcs));
 	Lck_Delete(&oh->mtx);
-	w->stats.n_objecthead--;
+	wrk->stats.n_objecthead--;
 	FREE_OBJ(oh);
 }
 
@@ -251,31 +251,31 @@ hsh_testmagic(void *result)
 struct objcore *
 HSH_Insert(const struct sess *sp)
 {
-	struct worker *w;
+	struct worker *wrk;
 	struct objhead *oh;
 	struct objcore *oc;
 
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 	CHECK_OBJ_NOTNULL(sp->wrk, WORKER_MAGIC);
 	AN(hash);
-	w = sp->wrk;
+	wrk = sp->wrk;
 
 	HSH_Prealloc(sp);
 	if (cache_param->diag_bitmap & 0x80000000)
 		hsh_testmagic(sp->wrk->nobjhead->digest);
 
 	AZ(sp->hash_objhead);
-	AN(w->nobjhead);
-	oh = hash->lookup(sp, w->nobjhead);
+	AN(wrk->nobjhead);
+	oh = hash->lookup(sp, wrk->nobjhead);
 	CHECK_OBJ_NOTNULL(oh, OBJHEAD_MAGIC);
-	if (oh == w->nobjhead)
-		w->nobjhead = NULL;
+	if (oh == wrk->nobjhead)
+		wrk->nobjhead = NULL;
 	Lck_Lock(&oh->mtx);
 	assert(oh->refcnt > 0);
 
 	/* Insert (precreated) objcore in objecthead */
-	oc = w->nobjcore;
-	w->nobjcore = NULL;
+	oc = wrk->nobjcore;
+	wrk->nobjcore = NULL;
 	oc->refcnt = 1;
 	CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
 	AZ(oc->flags & OC_F_BUSY);
@@ -294,7 +294,7 @@ HSH_Insert(const struct sess *sp)
 struct objcore *
 HSH_Lookup(struct sess *sp, struct objhead **poh)
 {
-	struct worker *w;
+	struct worker *wrk;
 	struct objhead *oh;
 	struct objcore *oc;
 	struct objcore *busy_oc, *grace_oc;
@@ -306,7 +306,7 @@ HSH_Lookup(struct sess *sp, struct objhead **poh)
 	CHECK_OBJ_NOTNULL(sp->http, HTTP_MAGIC);
 	AN(sp->director);
 	AN(hash);
-	w = sp->wrk;
+	wrk = sp->wrk;
 
 	HSH_Prealloc(sp);
 	memcpy(sp->wrk->nobjhead->digest, sp->digest, sizeof sp->digest);
@@ -322,10 +322,10 @@ HSH_Lookup(struct sess *sp, struct objhead **poh)
 		oh = sp->hash_objhead;
 		sp->hash_objhead = NULL;
 	} else {
-		AN(w->nobjhead);
-		oh = hash->lookup(sp, w->nobjhead);
-		if (oh == w->nobjhead)
-			w->nobjhead = NULL;
+		AN(wrk->nobjhead);
+		oh = hash->lookup(sp, wrk->nobjhead);
+		if (oh == wrk->nobjhead)
+			wrk->nobjhead = NULL;
 	}
 
 	CHECK_OBJ_NOTNULL(oh, OBJHEAD_MAGIC);
@@ -446,20 +446,20 @@ HSH_Lookup(struct sess *sp, struct objhead **poh)
 	}
 
 	/* Insert (precreated) objcore in objecthead */
-	oc = w->nobjcore;
-	w->nobjcore = NULL;
+	oc = wrk->nobjcore;
+	wrk->nobjcore = NULL;
 	AN(oc->flags & OC_F_BUSY);
 	oc->refcnt = 1;
 
-	AZ(w->busyobj);
-	w->busyobj = VBE_GetBusyObj(w);
+	AZ(wrk->busyobj);
+	wrk->busyobj = VBE_GetBusyObj(wrk);
 
 	VRY_Validate(sp->vary_b);
 	if (sp->vary_l != NULL)
-		w->busyobj->vary = sp->vary_b;
+		wrk->busyobj->vary = sp->vary_b;
 	else
-		w->busyobj->vary = NULL;
-	oc->busyobj = w->busyobj;
+		wrk->busyobj->vary = NULL;
+	oc->busyobj = wrk->busyobj;
 
 	/*
 	 * Busy objects go on the tail, so they will not trip up searches.
@@ -659,7 +659,7 @@ HSH_Ref(struct objcore *oc)
  */
 
 int
-HSH_Deref(struct worker *w, struct objcore *oc, struct object **oo)
+HSH_Deref(struct worker *wrk, struct objcore *oc, struct object **oo)
 {
 	struct object *o = NULL;
 	struct objhead *oh;
@@ -682,7 +682,7 @@ HSH_Deref(struct worker *w, struct objcore *oc, struct object **oo)
 		 */
 		STV_Freestore(o);
 		STV_free(o->objstore);
-		w->stats.n_object--;
+		wrk->stats.n_object--;
 		return (0);
 	}
 
@@ -712,16 +712,16 @@ HSH_Deref(struct worker *w, struct objcore *oc, struct object **oo)
 
 	if (oc->methods != NULL) {
 		oc_freeobj(oc);
-		w->stats.n_object--;
+		wrk->stats.n_object--;
 	}
 	FREE_OBJ(oc);
 
-	w->stats.n_objectcore--;
+	wrk->stats.n_objectcore--;
 	/* Drop our ref on the objhead */
 	assert(oh->refcnt > 0);
 	if (hash->deref(oh))
 		return (0);
-	HSH_DeleteObjHead(w, oh);
+	HSH_DeleteObjHead(wrk, oh);
 	return (0);
 }
 



From geoff at varnish-cache.org  Mon Jan  9 20:52:49 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:49 +0100
Subject: [experimental-ims] 367c381 Missed part of last commit.
Message-ID: <E1RkMD3-0000E8-NG@project.varnish-software.com>

commit 367c3817b1fef87cf3c008f0014415d1339c5812
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Dec 7 10:31:56 2011 +0000

    Missed part of last commit.

diff --git a/lib/libvmod_std/vmod_std.c b/lib/libvmod_std/vmod_std.c
index 14d8503..fc371e7 100644
--- a/lib/libvmod_std/vmod_std.c
+++ b/lib/libvmod_std/vmod_std.c
@@ -192,7 +192,7 @@ vmod_collect(struct sess *sp, enum gethdr_e e, const char *h)
 	(void)h;
 	if (e == HDR_REQ)
 		http_CollectHdr(sp->http, h);
-	else if (e == HDR_BERESP)
-		http_CollectHdr(sp->wrk->beresp, h);
+	else if (e == HDR_BERESP && sp->wrk->busyobj != NULL)
+		http_CollectHdr(sp->wrk->busyobj->beresp, h);
 }
 



From geoff at varnish-cache.org  Mon Jan  9 20:52:40 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:40 +0100
Subject: [experimental-ims] 0ae79f5 Communicate the panic string through
	heritage too.
Message-ID: <E1RkMCu-0008MH-O4@project.varnish-software.com>

commit 0ae79f5e6603f0b7ff7abf4280c9d16da78f1f63
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sun Nov 20 20:06:52 2011 +0000

    Communicate the panic string through heritage too.

diff --git a/bin/varnishd/cache/cache_panic.c b/bin/varnishd/cache/cache_panic.c
index d112eb8..9d69c1b 100644
--- a/bin/varnishd/cache/cache_panic.c
+++ b/bin/varnishd/cache/cache_panic.c
@@ -39,6 +39,7 @@
 #include <stdlib.h>
 
 #include "cache.h"
+#include "common/heritage.h"
 
 #include "cache_backend.h"
 #include "waiter/cache_waiter.h"
@@ -53,37 +54,33 @@
  * (gdb) printf "%s", panicstr
  */
 
-static struct vsb vsps, *vsp;
+static struct vsb pan_vsp_storage, *pan_vsp;
 static pthread_mutex_t panicstr_mtx = PTHREAD_MUTEX_INITIALIZER;
 
-/* Initialized in mgt_shmem.c, points into VSM */
-char *PAN_panicstr;
-unsigned PAN_panicstr_len;
-
 /*--------------------------------------------------------------------*/
 
 static void
 pan_ws(const struct ws *ws, int indent)
 {
 
-	VSB_printf(vsp, "%*sws = %p { %s\n", indent, "",
+	VSB_printf(pan_vsp, "%*sws = %p { %s\n", indent, "",
 	    ws, ws->overflow ? "overflow" : "");
-	VSB_printf(vsp, "%*sid = \"%s\",\n", indent + 2, "", ws->id);
-	VSB_printf(vsp, "%*s{s,f,r,e} = {%p", indent + 2, "", ws->s);
+	VSB_printf(pan_vsp, "%*sid = \"%s\",\n", indent + 2, "", ws->id);
+	VSB_printf(pan_vsp, "%*s{s,f,r,e} = {%p", indent + 2, "", ws->s);
 	if (ws->f > ws->s)
-		VSB_printf(vsp, ",+%ld", (long) (ws->f - ws->s));
+		VSB_printf(pan_vsp, ",+%ld", (long) (ws->f - ws->s));
 	else
-		VSB_printf(vsp, ",%p", ws->f);
+		VSB_printf(pan_vsp, ",%p", ws->f);
 	if (ws->r > ws->s)
-		VSB_printf(vsp, ",+%ld", (long) (ws->r - ws->s));
+		VSB_printf(pan_vsp, ",+%ld", (long) (ws->r - ws->s));
 	else
-		VSB_printf(vsp, ",%p", ws->r);
+		VSB_printf(pan_vsp, ",%p", ws->r);
 	if (ws->e > ws->s)
-		VSB_printf(vsp, ",+%ld", (long) (ws->e - ws->s));
+		VSB_printf(pan_vsp, ",+%ld", (long) (ws->e - ws->s));
 	else
-		VSB_printf(vsp, ",%p", ws->e);
-	VSB_printf(vsp, "},\n");
-	VSB_printf(vsp, "%*s},\n", indent, "" );
+		VSB_printf(pan_vsp, ",%p", ws->e);
+	VSB_printf(pan_vsp, "},\n");
+	VSB_printf(pan_vsp, "%*s},\n", indent, "" );
 }
 
 /*--------------------------------------------------------------------*/
@@ -96,9 +93,9 @@ pan_vbc(const struct vbc *vbc)
 
 	be = vbc->backend;
 
-	VSB_printf(vsp, "  backend = %p fd = %d {\n", be, vbc->fd);
-	VSB_printf(vsp, "    display_name = \"%s\",\n", be->display_name);
-	VSB_printf(vsp, "  },\n");
+	VSB_printf(pan_vsp, "  backend = %p fd = %d {\n", be, vbc->fd);
+	VSB_printf(pan_vsp, "    display_name = \"%s\",\n", be->display_name);
+	VSB_printf(pan_vsp, "  },\n");
 }
 
 /*--------------------------------------------------------------------*/
@@ -111,24 +108,26 @@ pan_storage(const struct storage *st)
 #define MAX_BYTES (4*16)
 #define show(ch) (((ch) > 31 && (ch) < 127) ? (ch) : '.')
 
-	VSB_printf(vsp, "      %u {\n", st->len);
+	VSB_printf(pan_vsp, "      %u {\n", st->len);
 	for (i = 0; i < MAX_BYTES && i < st->len; i += 16) {
-		VSB_printf(vsp, "        ");
+		VSB_printf(pan_vsp, "        ");
 		for (j = 0; j < 16; ++j) {
 			if (i + j < st->len)
-				VSB_printf(vsp, "%02x ", st->ptr[i + j]);
+				VSB_printf(pan_vsp, "%02x ", st->ptr[i + j]);
 			else
-				VSB_printf(vsp, "   ");
+				VSB_printf(pan_vsp, "   ");
 		}
-		VSB_printf(vsp, "|");
+		VSB_printf(pan_vsp, "|");
 		for (j = 0; j < 16; ++j)
 			if (i + j < st->len)
-				VSB_printf(vsp, "%c", show(st->ptr[i + j]));
-		VSB_printf(vsp, "|\n");
+				VSB_printf(pan_vsp,
+				    "%c", show(st->ptr[i + j]));
+		VSB_printf(pan_vsp, "|\n");
 	}
 	if (st->len > MAX_BYTES)
-		VSB_printf(vsp, "        [%u more]\n", st->len - MAX_BYTES);
-	VSB_printf(vsp, "      },\n");
+		VSB_printf(pan_vsp,
+		    "        [%u more]\n", st->len - MAX_BYTES);
+	VSB_printf(pan_vsp, "      },\n");
 
 #undef show
 #undef MAX_BYTES
@@ -141,17 +140,17 @@ pan_http(const char *id, const struct http *h, int indent)
 {
 	int i;
 
-	VSB_printf(vsp, "%*shttp[%s] = {\n", indent, "", id);
-	VSB_printf(vsp, "%*sws = %p[%s]\n", indent + 2, "",
+	VSB_printf(pan_vsp, "%*shttp[%s] = {\n", indent, "", id);
+	VSB_printf(pan_vsp, "%*sws = %p[%s]\n", indent + 2, "",
 	    h->ws, h->ws ? h->ws->id : "");
 	for (i = 0; i < h->nhd; ++i) {
 		if (h->hd[i].b == NULL && h->hd[i].e == NULL)
 			continue;
-		VSB_printf(vsp, "%*s\"%.*s\",\n", indent + 4, "",
+		VSB_printf(pan_vsp, "%*s\"%.*s\",\n", indent + 4, "",
 		    (int)(h->hd[i].e - h->hd[i].b),
 		    h->hd[i].b);
 	}
-	VSB_printf(vsp, "%*s},\n", indent, "");
+	VSB_printf(pan_vsp, "%*s},\n", indent, "");
 }
 
 
@@ -162,16 +161,16 @@ pan_object(const struct object *o)
 {
 	const struct storage *st;
 
-	VSB_printf(vsp, "  obj = %p {\n", o);
-	VSB_printf(vsp, "    xid = %u,\n", o->xid);
+	VSB_printf(pan_vsp, "  obj = %p {\n", o);
+	VSB_printf(pan_vsp, "    xid = %u,\n", o->xid);
 	pan_ws(o->ws_o, 4);
 	pan_http("obj", o->http, 4);
-	VSB_printf(vsp, "    len = %jd,\n", (intmax_t)o->len);
-	VSB_printf(vsp, "    store = {\n");
+	VSB_printf(pan_vsp, "    len = %jd,\n", (intmax_t)o->len);
+	VSB_printf(pan_vsp, "    store = {\n");
 	VTAILQ_FOREACH(st, &o->store, list)
 		pan_storage(st);
-	VSB_printf(vsp, "    },\n");
-	VSB_printf(vsp, "  },\n");
+	VSB_printf(pan_vsp, "    },\n");
+	VSB_printf(pan_vsp, "  },\n");
 }
 
 /*--------------------------------------------------------------------*/
@@ -181,12 +180,12 @@ pan_vcl(const struct VCL_conf *vcl)
 {
 	int i;
 
-	VSB_printf(vsp, "    vcl = {\n");
-	VSB_printf(vsp, "      srcname = {\n");
+	VSB_printf(pan_vsp, "    vcl = {\n");
+	VSB_printf(pan_vsp, "      srcname = {\n");
 	for (i = 0; i < vcl->nsrc; ++i)
-		VSB_printf(vsp, "        \"%s\",\n", vcl->srcname[i]);
-	VSB_printf(vsp, "      },\n");
-	VSB_printf(vsp, "    },\n");
+		VSB_printf(pan_vsp, "        \"%s\",\n", vcl->srcname[i]);
+	VSB_printf(pan_vsp, "      },\n");
+	VSB_printf(pan_vsp, "    },\n");
 }
 
 
@@ -196,7 +195,7 @@ static void
 pan_wrk(const struct worker *wrk)
 {
 
-	VSB_printf(vsp, "  worker = %p {\n", wrk);
+	VSB_printf(pan_vsp, "  worker = %p {\n", wrk);
 	pan_ws(wrk->ws, 4);
 	if (wrk->bereq->ws != NULL)
 		pan_http("bereq", wrk->bereq, 4);
@@ -204,7 +203,7 @@ pan_wrk(const struct worker *wrk)
 		pan_http("beresp", wrk->beresp, 4);
 	if (wrk->resp->ws != NULL)
 		pan_http("resp", wrk->resp, 4);
-	VSB_printf(vsp, "    },\n");
+	VSB_printf(pan_vsp, "    },\n");
 }
 
 /*--------------------------------------------------------------------*/
@@ -214,11 +213,11 @@ pan_sess(const struct sess *sp)
 {
 	const char *stp, *hand;
 
-	VSB_printf(vsp, "sp = %p {\n", sp);
-	VSB_printf(vsp,
+	VSB_printf(pan_vsp, "sp = %p {\n", sp);
+	VSB_printf(pan_vsp,
 	    "  fd = %d, id = %d, xid = %u,\n",
 	    sp->fd, sp->vsl_id & VSL_IDENTMASK, sp->xid);
-	VSB_printf(vsp, "  client = %s %s,\n",
+	VSB_printf(pan_vsp, "  client = %s %s,\n",
 	    sp->addr ? sp->addr : "?.?.?.?",
 	    sp->port ? sp->port : "?");
 	switch (sp->step) {
@@ -229,31 +228,31 @@ pan_sess(const struct sess *sp)
 	}
 	hand = VCL_Return_Name(sp->handling);
 	if (stp != NULL)
-		VSB_printf(vsp, "  step = %s,\n", stp);
+		VSB_printf(pan_vsp, "  step = %s,\n", stp);
 	else
-		VSB_printf(vsp, "  step = 0x%x,\n", sp->step);
+		VSB_printf(pan_vsp, "  step = 0x%x,\n", sp->step);
 	if (hand != NULL)
-		VSB_printf(vsp, "  handling = %s,\n", hand);
+		VSB_printf(pan_vsp, "  handling = %s,\n", hand);
 	else
-		VSB_printf(vsp, "  handling = 0x%x,\n", sp->handling);
+		VSB_printf(pan_vsp, "  handling = 0x%x,\n", sp->handling);
 	if (sp->err_code)
-		VSB_printf(vsp,
+		VSB_printf(pan_vsp,
 		    "  err_code = %d, err_reason = %s,\n", sp->err_code,
 		    sp->err_reason ? sp->err_reason : "(null)");
 
-	VSB_printf(vsp, "  restarts = %d, esi_level = %d\n",
+	VSB_printf(pan_vsp, "  restarts = %d, esi_level = %d\n",
 	    sp->restarts, sp->esi_level);
 
-	VSB_printf(vsp, "  flags = ");
-	if (sp->wrk->do_stream)	VSB_printf(vsp, " do_stream");
-	if (sp->wrk->do_gzip)	VSB_printf(vsp, " do_gzip");
-	if (sp->wrk->do_gunzip)	VSB_printf(vsp, " do_gunzip");
-	if (sp->wrk->do_esi)	VSB_printf(vsp, " do_esi");
-	if (sp->wrk->do_close)	VSB_printf(vsp, " do_close");
-	if (sp->wrk->is_gzip)	VSB_printf(vsp, " is_gzip");
-	if (sp->wrk->is_gunzip)	VSB_printf(vsp, " is_gunzip");
-	VSB_printf(vsp, "\n");
-	VSB_printf(vsp, "  bodystatus = %d\n", sp->wrk->body_status);
+	VSB_printf(pan_vsp, "  flags = ");
+	if (sp->wrk->do_stream)	VSB_printf(pan_vsp, " do_stream");
+	if (sp->wrk->do_gzip)	VSB_printf(pan_vsp, " do_gzip");
+	if (sp->wrk->do_gunzip)	VSB_printf(pan_vsp, " do_gunzip");
+	if (sp->wrk->do_esi)	VSB_printf(pan_vsp, " do_esi");
+	if (sp->wrk->do_close)	VSB_printf(pan_vsp, " do_close");
+	if (sp->wrk->is_gzip)	VSB_printf(pan_vsp, " is_gzip");
+	if (sp->wrk->is_gunzip)	VSB_printf(pan_vsp, " is_gunzip");
+	VSB_printf(pan_vsp, "\n");
+	VSB_printf(pan_vsp, "  bodystatus = %d\n", sp->wrk->body_status);
 
 	pan_ws(sp->ws, 2);
 	pan_http("req", sp->http, 2);
@@ -270,7 +269,7 @@ pan_sess(const struct sess *sp)
 	if (VALID_OBJ(sp->obj, OBJECT_MAGIC))
 		pan_object(sp->obj);
 
-	VSB_printf(vsp, "},\n");
+	VSB_printf(pan_vsp, "},\n");
 }
 
 /*--------------------------------------------------------------------*/
@@ -285,18 +284,19 @@ pan_backtrace(void)
 	size = backtrace (array, 10);
 	if (size == 0)
 		return;
-	VSB_printf(vsp, "Backtrace:\n");
+	VSB_printf(pan_vsp, "Backtrace:\n");
 	for (i = 0; i < size; i++) {
-		VSB_printf (vsp, "  ");
-		if (Symbol_Lookup(vsp, array[i]) < 0) {
+		VSB_printf (pan_vsp, "  ");
+		if (Symbol_Lookup(pan_vsp, array[i]) < 0) {
 			char **strings;
 			strings = backtrace_symbols(&array[i], 1);
 			if (strings != NULL && strings[0] != NULL)
-				VSB_printf(vsp, "%p: %s", array[i], strings[0]);
+				VSB_printf(pan_vsp,
+				     "%p: %s", array[i], strings[0]);
 			else
-				VSB_printf(vsp, "%p: (?)", array[i]);
+				VSB_printf(pan_vsp, "%p: (?)", array[i]);
 		}
-		VSB_printf (vsp, "\n");
+		VSB_printf (pan_vsp, "\n");
 	}
 }
 
@@ -314,35 +314,35 @@ pan_ic(const char *func, const char *file, int line, const char *cond,
 						  anyway */
 	switch(xxx) {
 	case 3:
-		VSB_printf(vsp,
+		VSB_printf(pan_vsp,
 		    "Wrong turn at %s:%d:\n%s\n", file, line, cond);
 		break;
 	case 2:
-		VSB_printf(vsp,
+		VSB_printf(pan_vsp,
 		    "Panic from VCL:\n  %s\n", cond);
 		break;
 	case 1:
-		VSB_printf(vsp,
+		VSB_printf(pan_vsp,
 		    "Missing errorhandling code in %s(), %s line %d:\n"
 		    "  Condition(%s) not true.",
 		    func, file, line, cond);
 		break;
 	default:
 	case 0:
-		VSB_printf(vsp,
+		VSB_printf(pan_vsp,
 		    "Assert error in %s(), %s line %d:\n"
 		    "  Condition(%s) not true.\n",
 		    func, file, line, cond);
 		break;
 	}
 	if (err)
-		VSB_printf(vsp, "errno = %d (%s)\n", err, strerror(err));
+		VSB_printf(pan_vsp, "errno = %d (%s)\n", err, strerror(err));
 
 	q = THR_GetName();
 	if (q != NULL)
-		VSB_printf(vsp, "thread = (%s)\n", q);
+		VSB_printf(pan_vsp, "thread = (%s)\n", q);
 
-	VSB_printf(vsp, "ident = %s,%s\n",
+	VSB_printf(pan_vsp, "ident = %s,%s\n",
 	    VSB_data(vident) + 1, WAIT_GetName());
 
 	pan_backtrace();
@@ -352,11 +352,11 @@ pan_ic(const char *func, const char *file, int line, const char *cond,
 		if (sp != NULL)
 			pan_sess(sp);
 	}
-	VSB_printf(vsp, "\n");
-	VSB_bcat(vsp, "", 1);	/* NUL termination */
+	VSB_printf(pan_vsp, "\n");
+	VSB_bcat(pan_vsp, "", 1);	/* NUL termination */
 
 	if (cache_param->diag_bitmap & 0x4000)
-		(void)fputs(PAN_panicstr, stderr);
+		(void)fputs(heritage.panic_str, stderr);
 
 	if (cache_param->diag_bitmap & 0x1000)
 		exit(4);
@@ -371,9 +371,9 @@ PAN_Init(void)
 {
 
 	VAS_Fail = pan_ic;
-	vsp = &vsps;
-	AN(PAN_panicstr);
-	AN(PAN_panicstr_len);
-	AN(VSB_new(vsp, PAN_panicstr, PAN_panicstr_len,
+	pan_vsp = &pan_vsp_storage;
+	AN(heritage.panic_str);
+	AN(heritage.panic_str_len);
+	AN(VSB_new(pan_vsp, heritage.panic_str, heritage.panic_str_len,
 	    VSB_FIXEDLEN));
 }
diff --git a/bin/varnishd/common/common.h b/bin/varnishd/common/common.h
index 157e5f7..d6de47d 100644
--- a/bin/varnishd/common/common.h
+++ b/bin/varnishd/common/common.h
@@ -50,8 +50,6 @@ extern pid_t mgt_pid;
 
 /* mgt_shmem.c */
 #define PAN_CLASS "Panic"
-extern char *PAN_panicstr;
-extern unsigned PAN_panicstr_len;
 
 /* varnishd.c */
 extern struct vsb *vident;		// XXX: -> heritage ?
diff --git a/bin/varnishd/common/heritage.h b/bin/varnishd/common/heritage.h
index 4ad9deb..9169bd5 100644
--- a/bin/varnishd/common/heritage.h
+++ b/bin/varnishd/common/heritage.h
@@ -64,6 +64,10 @@ struct heritage {
 
 	char				*name;
 	char                            identity[1024];
+
+	char				*panic_str;
+	ssize_t				panic_str_len;
+	
 };
 
 extern struct heritage heritage;
diff --git a/bin/varnishd/mgt/mgt_child.c b/bin/varnishd/mgt/mgt_child.c
index 4afcf81..6d42260 100644
--- a/bin/varnishd/mgt/mgt_child.c
+++ b/bin/varnishd/mgt/mgt_child.c
@@ -427,10 +427,10 @@ mgt_handle_panicstr(pid_t r)
 {
 	char time_str[30];
 
-	if (PAN_panicstr[0] == '\0')
+	if (heritage.panic_str[0] == '\0')
 		return;
 	REPORT(LOG_ERR, "Child (%jd) Panic message: %s",
-	    (intmax_t)r, PAN_panicstr);
+	    (intmax_t)r, heritage.panic_str);
 
 	if (child_panic)
 		VSB_delete(child_panic);
@@ -438,7 +438,7 @@ mgt_handle_panicstr(pid_t r)
 	XXXAN(child_panic);
 	VTIM_format(VTIM_real(), time_str);
 	VSB_printf(child_panic, "Last panic at: %s\n", time_str);
-	VSB_cat(child_panic, PAN_panicstr);
+	VSB_cat(child_panic, heritage.panic_str);
 	AZ(VSB_finish(child_panic));
 }
 
diff --git a/bin/varnishd/mgt/mgt_shmem.c b/bin/varnishd/mgt/mgt_shmem.c
index 66eb035..a9a2a7e 100644
--- a/bin/varnishd/mgt/mgt_shmem.c
+++ b/bin/varnishd/mgt/mgt_shmem.c
@@ -215,7 +215,8 @@ mgt_SHM_Init(void)
 	AN(heritage.param);
 	*heritage.param = mgt_param;
 
-	PAN_panicstr_len = 64 * 1024;
-	PAN_panicstr = VSM_Alloc(PAN_panicstr_len, PAN_CLASS, "", "");
-	AN(PAN_panicstr);
+	heritage.panic_str_len = 64 * 1024;
+	heritage.panic_str =
+	    VSM_Alloc(heritage.panic_str_len, PAN_CLASS, "", "");
+	AN(heritage.panic_str);
 }



From geoff at varnish-cache.org  Mon Jan  9 20:52:43 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:43 +0100
Subject: [experimental-ims] 672fb60 Move vgz_rx from worker to busyobj
Message-ID: <E1RkMCx-0008UA-Uv@project.varnish-software.com>

commit 672fb604cd38bb81a8d65c5b1c24d7bfc019fdf9
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Nov 29 17:55:25 2011 +0000

    Move vgz_rx from worker to busyobj

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index 8cffdef..7a0167c 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -334,7 +334,6 @@ struct worker {
 	struct vbc		*vbc;
 	struct object		*fetch_obj;
 	enum body_status	body_status;
-	struct vgz		*vgz_rx;
 	struct vef_priv		*vef_priv;
 	unsigned		do_stream;
 	unsigned		do_esi;
@@ -503,6 +502,7 @@ struct busyobj {
 	struct vfp		*vfp;
 	struct vep_state	*vep;
 	unsigned		fetch_failed;
+	struct vgz		*vgz_rx;
 };
 
 /* Object structure --------------------------------------------------*/
diff --git a/bin/varnishd/cache/cache_esi_fetch.c b/bin/varnishd/cache/cache_esi_fetch.c
index b23538c..3dbc5ce 100644
--- a/bin/varnishd/cache/cache_esi_fetch.c
+++ b/bin/varnishd/cache/cache_esi_fetch.c
@@ -102,7 +102,7 @@ vfp_esi_bytes_gu(struct worker *w, struct http_conn *htc, ssize_t bytes)
 	const void *dp;
 
 	CHECK_OBJ_NOTNULL(w, WORKER_MAGIC);
-	vg = w->vgz_rx;
+	vg = w->busyobj->vgz_rx;
 
 	while (bytes > 0) {
 		if (VGZ_IbufEmpty(vg) && bytes > 0) {
@@ -264,10 +264,10 @@ vfp_esi_bytes_gg(struct worker *w, struct http_conn *htc, size_t bytes)
 		bytes -= wl;
 
 		vef->bufp = ibuf;
-		VGZ_Ibuf(w->vgz_rx, ibuf, wl);
+		VGZ_Ibuf(w->busyobj->vgz_rx, ibuf, wl);
 		do {
-			VGZ_Obuf(w->vgz_rx, ibuf2, sizeof ibuf2);
-			i = VGZ_Gunzip(w->vgz_rx, &dp, &dl);
+			VGZ_Obuf(w->busyobj->vgz_rx, ibuf2, sizeof ibuf2);
+			i = VGZ_Gunzip(w->busyobj->vgz_rx, &dp, &dl);
 			/* XXX: check i */
 			assert(i >= VGZ_OK);
 			vef->bufp = ibuf2;
@@ -284,7 +284,7 @@ vfp_esi_bytes_gg(struct worker *w, struct http_conn *htc, size_t bytes)
 				    vef->bufp, dl);
 				vef->npend += dl;
 			}
-		} while (!VGZ_IbufEmpty(w->vgz_rx));
+		} while (!VGZ_IbufEmpty(w->busyobj->vgz_rx));
 	}
 	return (1);
 }
@@ -300,9 +300,9 @@ vfp_esi_begin(struct worker *w, size_t estimate)
 	CHECK_OBJ_NOTNULL(w, WORKER_MAGIC);
 	CHECK_OBJ_NOTNULL(w->busyobj, BUSYOBJ_MAGIC);
 
-	AZ(w->vgz_rx);
+	AZ(w->busyobj->vgz_rx);
 	if (w->busyobj->is_gzip && w->do_gunzip) {
-		w->vgz_rx = VGZ_NewUngzip(w, "U F E");
+		w->busyobj->vgz_rx = VGZ_NewUngzip(w, "U F E");
 		VEP_Init(w, NULL);
 	} else if (w->busyobj->is_gunzip && w->do_gzip) {
 		ALLOC_OBJ(vef, VEF_MAGIC);
@@ -312,7 +312,7 @@ vfp_esi_begin(struct worker *w, size_t estimate)
 		w->vef_priv = vef;
 		VEP_Init(w, vfp_vep_callback);
 	} else if (w->busyobj->is_gzip) {
-		w->vgz_rx = VGZ_NewUngzip(w, "U F E");
+		w->busyobj->vgz_rx = VGZ_NewUngzip(w, "U F E");
 		ALLOC_OBJ(vef, VEF_MAGIC);
 		AN(vef);
 		vef->vgz = VGZ_NewGzip(w, "G F E");
@@ -362,7 +362,8 @@ vfp_esi_end(struct worker *w)
 
 	retval = w->busyobj->fetch_failed;
 
-	if (w->vgz_rx != NULL && VGZ_Destroy(&w->vgz_rx, -1) != VGZ_END)
+	if (w->busyobj->vgz_rx != NULL &&
+	     VGZ_Destroy(&w->busyobj->vgz_rx, -1) != VGZ_END)
 		retval = FetchError(w,
 		    "Gunzip+ESI Failed at the very end");
 
diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c
index 895a5cb..2a2de5b 100644
--- a/bin/varnishd/cache/cache_fetch.c
+++ b/bin/varnishd/cache/cache_fetch.c
@@ -499,7 +499,7 @@ FetchBody(struct worker *w, struct object *obj)
 
 	AssertObjCorePassOrBusy(obj->objcore);
 
-	AZ(w->vgz_rx);
+	AZ(w->busyobj->vgz_rx);
 	AZ(VTAILQ_FIRST(&obj->store));
 
 	w->fetch_obj = obj;
@@ -547,7 +547,7 @@ FetchBody(struct worker *w, struct object *obj)
 		mklen = 0;
 		INCOMPL();
 	}
-	AZ(w->vgz_rx);
+	AZ(w->busyobj->vgz_rx);
 
 	/*
 	 * It is OK for ->end to just leave the last storage segment
diff --git a/bin/varnishd/cache/cache_gzip.c b/bin/varnishd/cache/cache_gzip.c
index a175fdf..f325292 100644
--- a/bin/varnishd/cache/cache_gzip.c
+++ b/bin/varnishd/cache/cache_gzip.c
@@ -457,8 +457,8 @@ static void __match_proto__()
 vfp_gunzip_begin(struct worker *w, size_t estimate)
 {
 	(void)estimate;
-	AZ(w->vgz_rx);
-	w->vgz_rx = VGZ_NewUngzip(w, "U F -");
+	AZ(w->busyobj->vgz_rx);
+	w->busyobj->vgz_rx = VGZ_NewUngzip(w, "U F -");
 }
 
 static int __match_proto__()
@@ -472,7 +472,7 @@ vfp_gunzip_bytes(struct worker *w, struct http_conn *htc, ssize_t bytes)
 	const void *dp;
 
 	AZ(w->busyobj->fetch_failed);
-	vg = w->vgz_rx;
+	vg = w->busyobj->vgz_rx;
 	CHECK_OBJ_NOTNULL(vg, VGZ_MAGIC);
 	AZ(vg->vz.avail_in);
 	while (bytes > 0 || vg->vz.avail_in > 0) {
@@ -505,8 +505,8 @@ vfp_gunzip_end(struct worker *w)
 {
 	struct vgz *vg;
 
-	vg = w->vgz_rx;
-	w->vgz_rx = NULL;
+	vg = w->busyobj->vgz_rx;
+	w->busyobj->vgz_rx = NULL;
 	CHECK_OBJ_NOTNULL(vg, VGZ_MAGIC);
 	if (w->busyobj->fetch_failed) {
 		(void)VGZ_Destroy(&vg, -1);
@@ -535,8 +535,8 @@ vfp_gzip_begin(struct worker *w, size_t estimate)
 {
 	(void)estimate;
 
-	AZ(w->vgz_rx);
-	w->vgz_rx = VGZ_NewGzip(w, "G F -");
+	AZ(w->busyobj->vgz_rx);
+	w->busyobj->vgz_rx = VGZ_NewGzip(w, "G F -");
 }
 
 static int __match_proto__()
@@ -550,7 +550,7 @@ vfp_gzip_bytes(struct worker *w, struct http_conn *htc, ssize_t bytes)
 	const void *dp;
 
 	AZ(w->busyobj->fetch_failed);
-	vg = w->vgz_rx;
+	vg = w->busyobj->vgz_rx;
 	CHECK_OBJ_NOTNULL(vg, VGZ_MAGIC);
 	AZ(vg->vz.avail_in);
 	while (bytes > 0 || !VGZ_IbufEmpty(vg)) {
@@ -583,9 +583,9 @@ vfp_gzip_end(struct worker *w)
 	const void *dp;
 	int i;
 
-	vg = w->vgz_rx;
+	vg = w->busyobj->vgz_rx;
 	CHECK_OBJ_NOTNULL(vg, VGZ_MAGIC);
-	w->vgz_rx = NULL;
+	w->busyobj->vgz_rx = NULL;
 	if (w->busyobj->fetch_failed) {
 		(void)VGZ_Destroy(&vg, -1);
 		return(0);
@@ -622,8 +622,8 @@ static void __match_proto__()
 vfp_testgzip_begin(struct worker *w, size_t estimate)
 {
 	(void)estimate;
-	w->vgz_rx = VGZ_NewUngzip(w, "u F -");
-	CHECK_OBJ_NOTNULL(w->vgz_rx, VGZ_MAGIC);
+	w->busyobj->vgz_rx = VGZ_NewUngzip(w, "u F -");
+	CHECK_OBJ_NOTNULL(w->busyobj->vgz_rx, VGZ_MAGIC);
 }
 
 static int __match_proto__()
@@ -638,7 +638,7 @@ vfp_testgzip_bytes(struct worker *w, struct http_conn *htc, ssize_t bytes)
 	struct storage *st;
 
 	AZ(w->busyobj->fetch_failed);
-	vg = w->vgz_rx;
+	vg = w->busyobj->vgz_rx;
 	CHECK_OBJ_NOTNULL(vg, VGZ_MAGIC);
 	AZ(vg->vz.avail_in);
 	while (bytes > 0) {
@@ -677,8 +677,8 @@ vfp_testgzip_end(struct worker *w)
 {
 	struct vgz *vg;
 
-	vg = w->vgz_rx;
-	w->vgz_rx = NULL;
+	vg = w->busyobj->vgz_rx;
+	w->busyobj->vgz_rx = NULL;
 	CHECK_OBJ_NOTNULL(vg, VGZ_MAGIC);
 	if (w->busyobj->fetch_failed) {
 		(void)VGZ_Destroy(&vg, -1);



From geoff at varnish-cache.org  Mon Jan  9 20:52:46 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:46 +0100
Subject: [experimental-ims] ecd8f53 Only call pan_busyobj when we have a
	busyobj
Message-ID: <E1RkMD0-000082-TZ@project.varnish-software.com>

commit ecd8f535d8add7fc1a02f0b041939099a6183fa3
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Thu Dec 1 12:51:26 2011 +0100

    Only call pan_busyobj when we have a busyobj

diff --git a/bin/varnishd/cache/cache_panic.c b/bin/varnishd/cache/cache_panic.c
index 0908b74..cf39489 100644
--- a/bin/varnishd/cache/cache_panic.c
+++ b/bin/varnishd/cache/cache_panic.c
@@ -259,7 +259,8 @@ pan_sess(const struct sess *sp)
 	VSB_printf(pan_vsp, "  restarts = %d, esi_level = %d\n",
 	    sp->restarts, sp->esi_level);
 
-	pan_busyobj(sp->wrk->busyobj);
+	if (sp->wrk->busyobj != NULL)
+		pan_busyobj(sp->wrk->busyobj);
 
 	pan_ws(sp->ws, 2);
 	pan_http("req", sp->http, 2);



From geoff at varnish-cache.org  Mon Jan  9 20:52:46 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:46 +0100
Subject: [experimental-ims] 30eaac9 s/reponse/response/
Message-ID: <E1RkMD0-00007b-OB@project.varnish-software.com>

commit 30eaac997b61eeeb3e47778338a673086495135c
Author: Lasse Karstensen <lasse at varnish-software.com>
Date:   Wed Nov 30 15:23:37 2011 +0100

    s/reponse/response/

diff --git a/doc/sphinx/tutorial/increasing_your_hitrate.rst b/doc/sphinx/tutorial/increasing_your_hitrate.rst
index 437461a..31e32d0 100644
--- a/doc/sphinx/tutorial/increasing_your_hitrate.rst
+++ b/doc/sphinx/tutorial/increasing_your_hitrate.rst
@@ -94,7 +94,7 @@ googling "Live HTTP Headers".
 The role of HTTP Headers
 ~~~~~~~~~~~~~~~~~~~~~~~~
 
-Along with each HTTP request and reponse comes a bunch of headers
+Along with each HTTP request and response comes a bunch of headers
 carrying metadata. Varnish will look at these headers to determine if
 it is appropriate to cache the contents and how long Varnish can keep
 the content.



From geoff at varnish-cache.org  Mon Jan  9 20:52:50 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:50 +0100
Subject: [experimental-ims] d6b6428 Move the beresp and bereq from worker ot
	busyobj for good.
Message-ID: <E1RkMD4-0000Es-3Z@project.varnish-software.com>

commit d6b64285f179f8d6a66f29c60357f9c33cabb95c
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu Dec 8 07:45:21 2011 +0000

    Move the beresp and bereq from worker ot busyobj for good.

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index 4435c32..ed6a3fd 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -328,10 +328,6 @@ struct worker {
 	/* This is only here so VRT can find it */
 	const char		*storage_hint;
 
-	/* Fetch stuff.  Here because pipe has no busyobj */
-	struct http		*x_bereq;
-	struct http		*x_beresp;
-
 	/* Stream state */
 	struct stream_ctx	*sctx;
 
diff --git a/bin/varnishd/cache/cache_busyobj.c b/bin/varnishd/cache/cache_busyobj.c
index 4272725..ab4db76 100644
--- a/bin/varnishd/cache/cache_busyobj.c
+++ b/bin/varnishd/cache/cache_busyobj.c
@@ -43,6 +43,7 @@ struct vbo {
 #define VBO_MAGIC		0xde3d8223
 	struct lock		mtx;
 	unsigned		refcount;
+	uint16_t		nhttp;
 	struct busyobj		bo;
 };
 
@@ -64,9 +65,20 @@ static struct vbo *
 vbo_New(void)
 {
 	struct vbo *vbo;
+	uint16_t nhttp;
+	ssize_t http_space;
 
-	ALLOC_OBJ(vbo, VBO_MAGIC);
+	assert(cache_param->http_max_hdr < 65536);
+	nhttp = (uint16_t)cache_param->http_max_hdr;
+
+	http_space = HTTP_estimate(nhttp);
+
+	vbo = malloc(sizeof *vbo + 2 * http_space);
 	AN(vbo);
+
+	memset(vbo, 0, sizeof *vbo);
+	vbo->magic = VBO_MAGIC;
+	vbo->nhttp = nhttp;
 	Lck_New(&vbo->mtx, lck_busyobj);
 	return (vbo);
 }
@@ -89,6 +101,7 @@ struct busyobj *
 VBO_GetBusyObj(struct worker *wrk)
 {
 	struct vbo *vbo = NULL;
+	char *p;
 
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 
@@ -109,17 +122,25 @@ VBO_GetBusyObj(struct worker *wrk)
 		Lck_Unlock(&vbo_mtx);
 	}
 
+	if (vbo != NULL && vbo->nhttp != cache_param->http_max_hdr)
+		VBO_Free(&vbo);
+
 	if (vbo == NULL)
 		vbo = vbo_New();
 
 	CHECK_OBJ_NOTNULL(vbo, VBO_MAGIC);
 	AZ(vbo->refcount);
+
 	AZ(vbo->bo.magic);
 	vbo->refcount = 1;
 	vbo->bo.magic = BUSYOBJ_MAGIC;
 	vbo->bo.vbo = vbo;
-	vbo->bo.beresp = wrk->x_beresp;
-	vbo->bo.bereq = wrk->x_bereq;
+
+	p = (void*)(vbo + 1);
+	vbo->bo.bereq = HTTP_create(p, vbo->nhttp);
+	p += HTTP_estimate(vbo->nhttp);
+	vbo->bo.beresp = HTTP_create(p, vbo->nhttp);
+
 	return (&vbo->bo);
 }
 
diff --git a/bin/varnishd/cache/cache_pool.c b/bin/varnishd/cache/cache_pool.c
index cf40e85..a3cb805 100644
--- a/bin/varnishd/cache/cache_pool.c
+++ b/bin/varnishd/cache/cache_pool.c
@@ -196,8 +196,6 @@ Pool_Work_Thread(void *priv, struct worker *w)
 		Lck_AssertHeld(&pp->mtx);
 
 		CHECK_OBJ_NOTNULL(w, WORKER_MAGIC);
-		CHECK_OBJ_NOTNULL(w->x_bereq, HTTP_MAGIC);
-		CHECK_OBJ_NOTNULL(w->x_beresp, HTTP_MAGIC);
 		CHECK_OBJ_NOTNULL(w->resp, HTTP_MAGIC);
 
 		WS_Reset(w->ws, NULL);
diff --git a/bin/varnishd/cache/cache_wrk.c b/bin/varnishd/cache/cache_wrk.c
index 73eedb2..6b1d9dd 100644
--- a/bin/varnishd/cache/cache_wrk.c
+++ b/bin/varnishd/cache/cache_wrk.c
@@ -139,8 +139,6 @@ wrk_thread_real(void *priv, unsigned shm_workspace, unsigned sess_workspace,
 	uint32_t wlog[shm_workspace / 4];
 	/* XXX: can we trust these to be properly aligned ? */
 	unsigned char ws[sess_workspace];
-	unsigned char http0[http_space];
-	unsigned char http1[http_space];
 	unsigned char http2[http_space];
 	struct iovec iov[siov];
 	struct SHA256Context sha256;
@@ -153,8 +151,6 @@ wrk_thread_real(void *priv, unsigned shm_workspace, unsigned sess_workspace,
 	w->wlb = w->wlp = wlog;
 	w->wle = wlog + (sizeof wlog) / 4;
 	w->sha256ctx = &sha256;
-	w->x_bereq = HTTP_create(http0, nhttp);
-	w->x_beresp = HTTP_create(http1, nhttp);
 	w->resp = HTTP_create(http2, nhttp);
 	w->wrw.iov = iov;
 	w->wrw.siov = siov;



From geoff at varnish-cache.org  Mon Jan  9 20:52:49 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:49 +0100
Subject: [experimental-ims] 0dd1ee2 Complete the busyobj management code by
	adding a (default-on) per worker cache and a wrapper structure
	(needed for $param sized elements) and two new stats counters.
Message-ID: <E1RkMD3-0000Ec-Vu@project.varnish-software.com>

commit 0dd1ee21b69f043ad55219c592454d17dd8739db
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu Dec 8 07:12:33 2011 +0000

    Complete the busyobj management code by adding a (default-on) per
    worker cache and a wrapper structure (needed for $param sized elements)
    and two new stats counters.

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index 0ee855b..4435c32 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -106,6 +106,7 @@ struct pool;
 struct sess;
 struct sesspool;
 struct vbc;
+struct vbo;
 struct vef_priv;
 struct vrt_backend;
 struct vsb;
@@ -293,7 +294,7 @@ struct worker {
 	struct objhead		*nobjhead;
 	struct objcore		*nobjcore;
 	struct waitinglist	*nwaitinglist;
-	/* struct busyobj		*nbusyobj; */
+	struct vbo		*nvbo;
 	void			*nhashpriv;
 	struct dstat		stats;
 
@@ -496,9 +497,7 @@ oc_getlru(const struct objcore *oc)
 struct busyobj {
 	unsigned		magic;
 #define BUSYOBJ_MAGIC		0x23b95567
-	struct lock		mtx;
-	/* Members passed this line are cleared on reuse */
-	unsigned		refcount;
+	struct vbo		*vbo;
 
 	uint8_t			*vary;
 	unsigned		is_gzip;
@@ -700,8 +699,9 @@ double BAN_Time(const struct ban *ban);
 /* cache_busyobj.c */
 void VBO_Init(void);
 struct busyobj *VBO_GetBusyObj(struct worker *wrk);
-struct busyobj *VBO_RefBusyObj(struct busyobj *busyobj);
+void VBO_RefBusyObj(const struct busyobj *busyobj);
 void VBO_DerefBusyObj(struct worker *wrk, struct busyobj **busyobj);
+void VBO_Free(struct vbo **vbo);
 
 /* cache_center.c [CNT] */
 void CNT_Session(struct sess *sp);
diff --git a/bin/varnishd/cache/cache_busyobj.c b/bin/varnishd/cache/cache_busyobj.c
index 32bad7e..4272725 100644
--- a/bin/varnishd/cache/cache_busyobj.c
+++ b/bin/varnishd/cache/cache_busyobj.c
@@ -1,8 +1,8 @@
 /*-
- * Copyright (c) 2006 Verdens Gang AS
- * Copyright (c) 2006-2011 Varnish Software AS
+ * Copyright (c) 2011 Varnish Software AS
  * All rights reserved.
  *
+ * Author: Martin Blix Grydeland <martin at varnish-software.com>
  * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
  *
  * Redistribution and use in source and binary forms, with or without
@@ -38,101 +38,141 @@
 
 #include "cache.h"
 
-static struct lock nbusyobj_mtx;
-static struct busyobj *nbusyobj;
+struct vbo {
+	unsigned		magic;
+#define VBO_MAGIC		0xde3d8223
+	struct lock		mtx;
+	unsigned		refcount;
+	struct busyobj		bo;
+};
+
+static struct lock vbo_mtx;
+static struct vbo *nvbo;
 
 void
 VBO_Init(void)
 {
-	Lck_New(&nbusyobj_mtx, lck_nbusyobj);
-	nbusyobj = NULL;
+	Lck_New(&vbo_mtx, lck_busyobj);
+	nvbo = NULL;
 }
 
 /*--------------------------------------------------------------------
  * BusyObj handling
  */
 
-static struct busyobj *
-vbo_NewBusyObj(void)
+static struct vbo *
+vbo_New(void)
 {
-	struct busyobj *busyobj;
+	struct vbo *vbo;
 
-	ALLOC_OBJ(busyobj, BUSYOBJ_MAGIC);
-	AN(busyobj);
-	Lck_New(&busyobj->mtx, lck_busyobj);
-	return (busyobj);
+	ALLOC_OBJ(vbo, VBO_MAGIC);
+	AN(vbo);
+	Lck_New(&vbo->mtx, lck_busyobj);
+	return (vbo);
 }
 
-static void
-vbe_FreeBusyObj(struct busyobj *busyobj)
+void
+VBO_Free(struct vbo **vbop)
 {
-	CHECK_OBJ_NOTNULL(busyobj, BUSYOBJ_MAGIC);
-	AZ(busyobj->refcount);
-	Lck_Delete(&busyobj->mtx);
-	FREE_OBJ(busyobj);
+	struct vbo *vbo;
+
+	AN(vbop);
+	vbo = *vbop;
+	*vbop = NULL;
+	CHECK_OBJ_NOTNULL(vbo, VBO_MAGIC);
+	AZ(vbo->refcount);
+	Lck_Delete(&vbo->mtx);
+	FREE_OBJ(vbo);
 }
 
 struct busyobj *
 VBO_GetBusyObj(struct worker *wrk)
 {
-	struct busyobj *busyobj = NULL;
-
-	(void)wrk;
-	Lck_Lock(&nbusyobj_mtx);
-	if (nbusyobj != NULL) {
-		CHECK_OBJ_NOTNULL(nbusyobj, BUSYOBJ_MAGIC);
-		busyobj = nbusyobj;
-		nbusyobj = NULL;
-		memset((char *)busyobj + offsetof(struct busyobj, refcount), 0,
-		       sizeof *busyobj - offsetof(struct busyobj, refcount));
+	struct vbo *vbo = NULL;
+
+	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
+
+	if (wrk->nvbo != NULL) {
+		vbo = wrk->nvbo;
+		wrk->nvbo = NULL;
+	}
+
+	if (vbo == NULL) {
+		Lck_Lock(&vbo_mtx);
+
+		vbo = nvbo;
+		nvbo = NULL;
+
+		if (vbo == NULL)
+			VSC_C_main->busyobj_alloc++;
+
+		Lck_Unlock(&vbo_mtx);
 	}
-	Lck_Unlock(&nbusyobj_mtx);
-	if (busyobj == NULL)
-		busyobj = vbo_NewBusyObj();
-	AN(busyobj);
-	busyobj->refcount = 1;
-	busyobj->beresp = wrk->x_beresp;
-	busyobj->bereq = wrk->x_bereq;
-	return (busyobj);
+
+	if (vbo == NULL)
+		vbo = vbo_New();
+
+	CHECK_OBJ_NOTNULL(vbo, VBO_MAGIC);
+	AZ(vbo->refcount);
+	AZ(vbo->bo.magic);
+	vbo->refcount = 1;
+	vbo->bo.magic = BUSYOBJ_MAGIC;
+	vbo->bo.vbo = vbo;
+	vbo->bo.beresp = wrk->x_beresp;
+	vbo->bo.bereq = wrk->x_bereq;
+	return (&vbo->bo);
 }
 
-struct busyobj *
-VBO_RefBusyObj(struct busyobj *busyobj)
+void
+VBO_RefBusyObj(const struct busyobj *busyobj)
 {
+	struct vbo *vbo;
+
 	CHECK_OBJ_NOTNULL(busyobj, BUSYOBJ_MAGIC);
-	Lck_Lock(&busyobj->mtx);
-	assert(busyobj->refcount > 0);
-	busyobj->refcount++;
-	Lck_Unlock(&busyobj->mtx);
-	return (busyobj);
+	vbo = busyobj->vbo;
+	CHECK_OBJ_NOTNULL(vbo, VBO_MAGIC);
+	Lck_Lock(&vbo->mtx);
+	assert(vbo->refcount > 0);
+	vbo->refcount++;
+	Lck_Unlock(&vbo->mtx);
 }
 
 void
 VBO_DerefBusyObj(struct worker *wrk, struct busyobj **pbo)
 {
-	struct busyobj *busyobj;
+	struct busyobj *bo;
+	struct vbo *vbo;
+	unsigned r;
 
-	(void)wrk;
-	busyobj = *pbo;
-	CHECK_OBJ_NOTNULL(busyobj, BUSYOBJ_MAGIC);
-	Lck_Lock(&busyobj->mtx);
-	assert(busyobj->refcount > 0);
-	busyobj->refcount--;
+	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
+	AN(pbo);
+	bo = *pbo;
 	*pbo = NULL;
-	if (busyobj->refcount > 0) {
-		Lck_Unlock(&busyobj->mtx);
-		return;
-	}
-	Lck_Unlock(&busyobj->mtx);
-
-	/* XXX Sanity checks e.g. AZ(busyobj->vbc) */
-
-	Lck_Lock(&nbusyobj_mtx);
-	if (nbusyobj == NULL) {
-		nbusyobj = busyobj;
-		busyobj = NULL;
+	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
+	vbo = bo->vbo;
+	CHECK_OBJ_NOTNULL(vbo, VBO_MAGIC);
+	Lck_Lock(&vbo->mtx);
+	assert(vbo->refcount > 0);
+	r = --vbo->refcount;
+	Lck_Unlock(&vbo->mtx);
+
+	if (r == 0) {
+		/* XXX: Sanity checks & cleanup */
+		memset(&vbo->bo, 0, sizeof vbo->bo);
+
+		if (cache_param->bo_cache && wrk->nvbo == NULL) {
+			wrk->nvbo = vbo;
+		} else {
+			Lck_Lock(&vbo_mtx);
+			if (nvbo == NULL) {
+				nvbo = vbo;
+				vbo = NULL;
+			} else
+				VSC_C_main->busyobj_free++;
+			Lck_Unlock(&vbo_mtx);
+
+			if (vbo != NULL)
+				VBO_Free(&vbo);
+		}
 	}
-	Lck_Unlock(&nbusyobj_mtx);
-	if (busyobj != NULL)
-		vbe_FreeBusyObj(busyobj);
 }
diff --git a/bin/varnishd/cache/cache_hash.c b/bin/varnishd/cache/cache_hash.c
index 675afea..6f6a85f 100644
--- a/bin/varnishd/cache/cache_hash.c
+++ b/bin/varnishd/cache/cache_hash.c
@@ -134,10 +134,8 @@ HSH_Cleanup(struct worker *wrk)
 		free(wrk->nhashpriv);
 		wrk->nhashpriv = NULL;
 	}
-	/* if (wrk->nbusyobj != NULL) { */
-	/* 	FREE_OBJ(wrk->nbusyobj); */
-	/* 	wrk->nbusyobj = NULL; */
-	/* } */
+	if (wrk->nvbo != NULL)
+		VBO_Free(&wrk->nvbo);
 }
 
 void
diff --git a/bin/varnishd/common/params.h b/bin/varnishd/common/params.h
index b9b758f..8fd777f 100644
--- a/bin/varnishd/common/params.h
+++ b/bin/varnishd/common/params.h
@@ -189,6 +189,8 @@ struct params {
 
 	struct vre_limits	vre_limits;
 
+	unsigned		bo_cache;
+
 	/* VSM dimensions */
 	ssize_t			vsm_space;
 	ssize_t			vsl_space;
diff --git a/bin/varnishd/mgt/mgt_param.c b/bin/varnishd/mgt/mgt_param.c
index e4be5d8..cc14e41 100644
--- a/bin/varnishd/mgt/mgt_param.c
+++ b/bin/varnishd/mgt/mgt_param.c
@@ -1105,6 +1105,14 @@ static const struct parspec input_parspec[] = {
 		MUST_RESTART,
 		"1M", "bytes"},
 
+	{ "busyobj_worker_cache", tweak_bool,
+		&mgt_param.bo_cache, 0, 0,
+		"Cache free busyobj per worker thread."
+		"Disable this if you have very high hitrates and want"
+		"to save the memory of one busyobj per worker thread.",
+		0,
+		"true", ""},
+
 	{ NULL, NULL, NULL }
 };
 
diff --git a/include/tbl/vsc_f_main.h b/include/tbl/vsc_f_main.h
index a04767d..ab8415c 100644
--- a/include/tbl/vsc_f_main.h
+++ b/include/tbl/vsc_f_main.h
@@ -235,6 +235,20 @@ VSC_F(sess_dropped,		uint64_t, 1, 'c',
 	"  See also param queue_max."
 )
 
+/*---------------------------------------------------------------------
+ * BusyObj
+ */
+
+VSC_F(busyobj_alloc,		uint64_t, 1, 'c',
+    "Busyobj allocations",
+	"Number of busyobj structures allocated."
+)
+
+VSC_F(busyobj_free,		uint64_t, 1, 'c',
+    "Busyobj freed",
+	"Number of busyobj structures freed."
+)
+
 /*---------------------------------------------------------------------*/
 
 VSC_F(n_sess_mem,		uint64_t, 0, 'i', "N struct sess_mem", "")



From geoff at varnish-cache.org  Mon Jan  9 20:52:45 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:45 +0100
Subject: [experimental-ims] a2921da Move h_content_length from worker to
	busyobj
Message-ID: <E1RkMCz-000056-DR@project.varnish-software.com>

commit a2921da31f78a108322ce69bc5d476e09dbb29a6
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Nov 30 07:52:14 2011 +0000

    Move h_content_length from worker to busyobj

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index 819e701..ff9dcea 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -332,8 +332,6 @@ struct worker {
 	struct http		*beresp;
 	struct vbc		*vbc;
 
-	char			*h_content_length;
-
 	/* Stream state */
 	struct stream_ctx	*sctx;
 
@@ -504,6 +502,7 @@ struct busyobj {
 	struct vef_priv		*vef_priv;
 
 	unsigned		should_close;
+	char			*h_content_length;
 
 	unsigned		do_esi;
 	unsigned		do_gzip;
diff --git a/bin/varnishd/cache/cache_center.c b/bin/varnishd/cache/cache_center.c
index 6bda81f..501293d 100644
--- a/bin/varnishd/cache/cache_center.c
+++ b/bin/varnishd/cache/cache_center.c
@@ -188,7 +188,8 @@ cnt_prepresp(struct sess *sp)
 		wrk->res_mode |= RES_LEN;
 
 	if (wrk->busyobj != NULL &&
-	    (wrk->h_content_length != NULL || !wrk->busyobj->do_stream) &&
+	    (wrk->busyobj->h_content_length != NULL ||
+	    !wrk->busyobj->do_stream) &&
 	    !wrk->busyobj->do_gzip && !wrk->busyobj->do_gunzip)
 		wrk->res_mode |= RES_LEN;
 
@@ -256,7 +257,6 @@ cnt_prepresp(struct sess *sp)
 		AZ(wrk->obj);
 		sp->restarts++;
 		sp->director = NULL;
-		wrk->h_content_length = NULL;
 		http_Setup(wrk->bereq, NULL);
 		http_Setup(wrk->beresp, NULL);
 		http_Setup(wrk->resp, NULL);
@@ -469,7 +469,6 @@ cnt_error(struct sess *sp)
 		if (wrk->obj == NULL) {
 			sp->doclose = "Out of objects";
 			sp->director = NULL;
-			wrk->h_content_length = NULL;
 			http_Setup(wrk->beresp, NULL);
 			http_Setup(wrk->bereq, NULL);
 			sp->step = STP_DONE;
@@ -567,7 +566,6 @@ cnt_fetch(struct sess *sp)
 
 	AN(sp->director);
 	AZ(wrk->vbc);
-	AZ(wrk->h_content_length);
 	AZ(wrk->busyobj->should_close);
 	AZ(wrk->storage_hint);
 
@@ -648,7 +646,6 @@ cnt_fetch(struct sess *sp)
 	}
 	http_Setup(wrk->bereq, NULL);
 	http_Setup(wrk->beresp, NULL);
-	wrk->h_content_length = NULL;
 	sp->director = NULL;
 	wrk->storage_hint = NULL;
 
@@ -880,8 +877,6 @@ cnt_fetchbody(struct sess *sp)
 	/* Use unmodified headers*/
 	i = FetchBody(wrk, wrk->obj);
 
-	wrk->h_content_length = NULL;
-
 	http_Setup(wrk->bereq, NULL);
 	http_Setup(wrk->beresp, NULL);
 	wrk->busyobj->vfp = NULL;
@@ -950,8 +945,6 @@ cnt_streambody(struct sess *sp)
 
 	i = FetchBody(wrk, wrk->obj);
 
-	wrk->h_content_length = NULL;
-
 	http_Setup(wrk->bereq, NULL);
 	http_Setup(wrk->beresp, NULL);
 	wrk->busyobj->vfp = NULL;
diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c
index 16509f0..38a2cf3 100644
--- a/bin/varnishd/cache/cache_fetch.c
+++ b/bin/varnishd/cache/cache_fetch.c
@@ -525,7 +525,7 @@ FetchBody(struct worker *w, struct object *obj)
 		mklen = 1;
 		break;
 	case BS_LENGTH:
-		cl = fetch_number( w->h_content_length, 10);
+		cl = fetch_number( w->busyobj->h_content_length, 10);
 		w->busyobj->vfp->begin(w, cl > 0 ? cl : 0);
 		cls = fetch_straight(w, htc, cl);
 		mklen = 1;
diff --git a/bin/varnishd/cache/cache_response.c b/bin/varnishd/cache/cache_response.c
index 39c0ee6..7c63d51 100644
--- a/bin/varnishd/cache/cache_response.c
+++ b/bin/varnishd/cache/cache_response.c
@@ -345,9 +345,9 @@ RES_StreamStart(struct sess *sp)
 		http_Unset(sp->wrk->resp, H_Content_Encoding);
 
 	if (!(sp->wrk->res_mode & RES_CHUNKED) &&
-	    sp->wrk->h_content_length != NULL)
+	    sp->wrk->busyobj->h_content_length != NULL)
 		http_PrintfHeader(sp->wrk, sp->vsl_id, sp->wrk->resp,
-		    "Content-Length: %s", sp->wrk->h_content_length);
+		    "Content-Length: %s", sp->wrk->busyobj->h_content_length);
 
 	sp->wrk->acct_tmp.hdrbytes +=
 	    http_Write(sp->wrk, sp->vsl_id, sp->wrk->resp, 1);
diff --git a/bin/varnishd/cache/cache_rfc2616.c b/bin/varnishd/cache/cache_rfc2616.c
index c2a795a..599ffbb 100644
--- a/bin/varnishd/cache/cache_rfc2616.c
+++ b/bin/varnishd/cache/cache_rfc2616.c
@@ -239,7 +239,8 @@ RFC2616_Body(const struct sess *sp)
 		return (BS_ERROR);
 	}
 
-	if (http_GetHdr(hp, H_Content_Length, &sp->wrk->h_content_length)) {
+	if (http_GetHdr(hp, H_Content_Length,
+	    &sp->wrk->busyobj->h_content_length)) {
 		sp->wrk->stats.fetch_length++;
 		return (BS_LENGTH);
 	}



From geoff at varnish-cache.org  Mon Jan  9 20:52:45 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:45 +0100
Subject: [experimental-ims] 1dfe07e typo. Thanks to 张学彬
Message-ID: <E1RkMCz-00005i-Ua@project.varnish-software.com>

commit 1dfe07e0d7a78c9cf048ce3df9d22c22c739d3d8
Author: Per Buer <perbu at varnish-software.com>
Date:   Wed Nov 30 10:38:43 2011 +0100

    typo. Thanks to ???

diff --git a/doc/sphinx/reference/vcl.rst b/doc/sphinx/reference/vcl.rst
index 743d5f2..b74e691 100644
--- a/doc/sphinx/reference/vcl.rst
+++ b/doc/sphinx/reference/vcl.rst
@@ -498,7 +498,7 @@ vcl_pass
   client, but is not entered into the cache.  Subsequent requests sub?
   mitted over the same client connection are handled normally.
   
-  The vcl_recv subroutine may terminate with calling return() with one of
+  The vcl_pass subroutine may terminate with calling return() with one of
   the following keywords:
   
   error code [reason]



From geoff at varnish-cache.org  Mon Jan  9 20:52:48 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:48 +0100
Subject: [experimental-ims] 4f47929 Add an objcore->getxid() method.
Message-ID: <E1RkMD2-0000C9-MN@project.varnish-software.com>

commit 4f47929731b3dc637c0ac756e2babb4787fcb9bb
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Dec 6 10:00:25 2011 +0000

    Add an objcore->getxid() method.

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index 46af4dd..327682c 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -395,12 +395,14 @@ struct storage {
  */
 
 typedef struct object *getobj_f(struct worker *wrk, struct objcore *oc);
+typedef unsigned getxid_f(struct worker *wrk, struct objcore *oc);
 typedef void updatemeta_f(struct objcore *oc);
 typedef void freeobj_f(struct objcore *oc);
 typedef struct lru *getlru_f(const struct objcore *oc);
 
 struct objcore_methods {
 	getobj_f	*getobj;
+	getxid_f	*getxid;
 	updatemeta_f	*updatemeta;
 	freeobj_f	*freeobj;
 	getlru_f	*getlru;
@@ -429,6 +431,16 @@ struct objcore {
 	struct ban		*ban;
 };
 
+static inline unsigned
+oc_getxid(struct worker *wrk, struct objcore *oc)
+{
+	CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
+
+	AN(oc->methods);
+	AN(oc->methods->getxid);
+	return (oc->methods->getxid(wrk, oc));
+}
+
 static inline struct object *
 oc_getobj(struct worker *wrk, struct objcore *oc)
 {
diff --git a/bin/varnishd/cache/cache_expire.c b/bin/varnishd/cache/cache_expire.c
index 23e3fc6..761a62e 100644
--- a/bin/varnishd/cache/cache_expire.c
+++ b/bin/varnishd/cache/cache_expire.c
@@ -401,7 +401,7 @@ exp_timer(struct sess *sp, void *priv)
 		CHECK_OBJ_NOTNULL(oc->objhead, OBJHEAD_MAGIC);
 		o = oc_getobj(sp->wrk, oc);
 		WSL(sp->wrk, SLT_ExpKill, 0, "%u %.0f",
-		    o->xid, EXP_Ttl(NULL, o) - t);
+		    oc_getxid(sp->wrk, oc), EXP_Ttl(NULL, o) - t);
 		(void)HSH_Deref(sp->wrk, oc, NULL);
 	}
 	NEEDLESS_RETURN(NULL);
@@ -414,10 +414,9 @@ exp_timer(struct sess *sp, void *priv)
  */
 
 int
-EXP_NukeOne(struct worker *w, struct lru *lru)
+EXP_NukeOne(struct worker *wrk, struct lru *lru)
 {
 	struct objcore *oc;
-	struct object *o;
 
 	/* Find the first currently unused object on the LRU.  */
 	Lck_Lock(&lru->mtx);
@@ -446,9 +445,8 @@ EXP_NukeOne(struct worker *w, struct lru *lru)
 		return (-1);
 
 	/* XXX: bad idea for -spersistent */
-	o = oc_getobj(w, oc);
-	WSL(w, SLT_ExpKill, 0, "%u LRU", o->xid);
-	(void)HSH_Deref(w, NULL, &o);
+	WSL(wrk, SLT_ExpKill, 0, "%u LRU", oc_getxid(wrk, oc));
+	(void)HSH_Deref(wrk, oc, NULL);
 	return (1);
 }
 
diff --git a/bin/varnishd/storage/stevedore.c b/bin/varnishd/storage/stevedore.c
index 5de0e27..a6156d4 100644
--- a/bin/varnishd/storage/stevedore.c
+++ b/bin/varnishd/storage/stevedore.c
@@ -48,6 +48,15 @@ static const struct stevedore * volatile stv_next;
  * Default objcore methods
  */
 
+static unsigned __match_proto__(getxid_f)
+default_oc_getxid(struct worker *wrk, struct objcore *oc)
+{
+	struct object *o;
+
+	o = oc_getobj(wrk, oc);
+	return (o->xid);
+}
+
 static struct object * __match_proto__(getobj_f)
 default_oc_getobj(struct worker *wrk, struct objcore *oc)
 {
@@ -84,6 +93,7 @@ default_oc_getlru(const struct objcore *oc)
 
 static struct objcore_methods default_oc_methods = {
 	.getobj = default_oc_getobj,
+	.getxid = default_oc_getxid,
 	.freeobj = default_oc_freeobj,
 	.getlru = default_oc_getlru,
 };



From geoff at varnish-cache.org  Mon Jan  9 20:52:46 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:46 +0100
Subject: [experimental-ims] c54185f Merge branch 'master' of
	ssh://git.varnish-cache.org/git/varnish-cache
Message-ID: <E1RkMD0-00006F-3i@project.varnish-software.com>

commit c54185fabd889c25331ddd404d40f3849828165e
Merge: 1dfe07e a2921da
Author: Per Buer <perbu at varnish-software.com>
Date:   Wed Nov 30 10:38:57 2011 +0100

    Merge branch 'master' of ssh://git.varnish-cache.org/git/varnish-cache



From geoff at varnish-cache.org  Mon Jan  9 20:52:46 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:46 +0100
Subject: [experimental-ims] e62db2e Clean up examples a bit
Message-ID: <E1RkMD0-00006a-7r@project.varnish-software.com>

commit e62db2ea30d6b36fac5ce645f1f0db2c90dddd51
Author: Lasse Karstensen <lasse at varnish-software.com>
Date:   Wed Nov 30 15:01:40 2011 +0100

    Clean up examples a bit

diff --git a/doc/sphinx/tutorial/vary.rst b/doc/sphinx/tutorial/vary.rst
index 4da6744..a31e449 100644
--- a/doc/sphinx/tutorial/vary.rst
+++ b/doc/sphinx/tutorial/vary.rst
@@ -14,11 +14,11 @@ the page encoded with the deflate encoding.
 The problem is that the Accept-Encoding field contains a lot of
 different encodings. If one browser sends::
 
-  Accept-Encodign: gzip,deflate
+  Accept-Encoding: gzip,deflate
 
 And another one sends::
 
-  Accept-Encoding:: deflate,gzip
+  Accept-Encoding: deflate,gzip
 
 Varnish will keep two variants of the page requested due to the
 different Accept-Encoding headers. Normalizing the accept-encoding



From geoff at varnish-cache.org  Mon Jan  9 20:52:46 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:46 +0100
Subject: [experimental-ims] a1de487 Clean up examples a bit (#2)
Message-ID: <E1RkMD0-000071-Ed@project.varnish-software.com>

commit a1de487811a739388169d3b97b34ed6a413d2c00
Author: Lasse Karstensen <lasse at varnish-software.com>
Date:   Wed Nov 30 15:05:31 2011 +0100

    Clean up examples a bit (#2)

diff --git a/doc/sphinx/tutorial/vary.rst b/doc/sphinx/tutorial/vary.rst
index a31e449..ad7b48d 100644
--- a/doc/sphinx/tutorial/vary.rst
+++ b/doc/sphinx/tutorial/vary.rst
@@ -23,7 +23,7 @@ And another one sends::
 Varnish will keep two variants of the page requested due to the
 different Accept-Encoding headers. Normalizing the accept-encoding
 header will sure that you have as few variants as possible. The
-following VCL code will normalize the Accept-Encoding headers.::
+following VCL code will normalize the Accept-Encoding headers::
 
     if (req.http.Accept-Encoding) {
         if (req.url ~ "\.(jpg|png|gif|gz|tgz|bz2|tbz|mp3|ogg)$") {
@@ -34,7 +34,7 @@ following VCL code will normalize the Accept-Encoding headers.::
         } elsif (req.http.Accept-Encoding ~ "deflate") {
             set req.http.Accept-Encoding = "deflate";
         } else {
-            # unkown algorithm
+            # unknown algorithm
             remove req.http.Accept-Encoding;
         }
     }



From geoff at varnish-cache.org  Mon Jan  9 20:52:47 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:47 +0100
Subject: [experimental-ims] 9633e66 Led VDI_CloseFd() take an the vbc to be
	closed as argument.
Message-ID: <E1RkMD1-00008s-A3@project.varnish-software.com>

commit 9633e660b238f3456baa004caf58a10e1403cb41
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Dec 5 08:34:27 2011 +0000

    Led VDI_CloseFd() take an the vbc to be closed as argument.

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index ff9dcea..4cfad78 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -650,7 +650,7 @@ void VBE_UseHealth(const struct director *vdi);
 
 struct vbc *VDI_GetFd(const struct director *, struct sess *sp);
 int VDI_Healthy(const struct director *, const struct sess *sp);
-void VDI_CloseFd(struct worker *wrk);
+void VDI_CloseFd(struct worker *wrk, struct vbc **vbp);
 void VDI_RecycleFd(struct worker *wrk);
 void VDI_AddHostHeader(const struct sess *sp);
 void VBE_Poll(void);
diff --git a/bin/varnishd/cache/cache_center.c b/bin/varnishd/cache/cache_center.c
index 501293d..997ecac 100644
--- a/bin/varnishd/cache/cache_center.c
+++ b/bin/varnishd/cache/cache_center.c
@@ -249,7 +249,7 @@ cnt_prepresp(struct sess *sp)
 		if (sp->restarts >= cache_param->max_restarts)
 			break;
 		if (wrk->busyobj->do_stream) {
-			VDI_CloseFd(wrk);
+			VDI_CloseFd(wrk, &wrk->vbc);
 			HSH_Drop(wrk);
 		} else {
 			(void)HSH_Deref(wrk, NULL, &wrk->obj);
@@ -633,7 +633,7 @@ cnt_fetch(struct sess *sp)
 		}
 
 		/* We are not going to fetch the body, Close the connection */
-		VDI_CloseFd(wrk);
+		VDI_CloseFd(wrk, &wrk->vbc);
 	}
 
 	/* Clean up partial fetch */
@@ -817,7 +817,7 @@ cnt_fetchbody(struct sess *sp)
 	if (wrk->obj == NULL) {
 		sp->err_code = 503;
 		sp->step = STP_ERROR;
-		VDI_CloseFd(wrk);
+		VDI_CloseFd(wrk, &wrk->vbc);
 		return (0);
 	}
 	CHECK_OBJ_NOTNULL(wrk->obj, OBJECT_MAGIC);
diff --git a/bin/varnishd/cache/cache_dir.c b/bin/varnishd/cache/cache_dir.c
index eba6183..434277e 100644
--- a/bin/varnishd/cache/cache_dir.c
+++ b/bin/varnishd/cache/cache_dir.c
@@ -40,27 +40,31 @@
 /* Close a connection ------------------------------------------------*/
 
 void
-VDI_CloseFd(struct worker *wrk)
+VDI_CloseFd(struct worker *wrk, struct vbc **vbp)
 {
 	struct backend *bp;
+	struct vbc *vc;
 
-	CHECK_OBJ_NOTNULL(wrk->vbc, VBC_MAGIC);
-	CHECK_OBJ_NOTNULL(wrk->vbc->backend, BACKEND_MAGIC);
-	assert(wrk->vbc->fd >= 0);
+	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
+	AN(vbp);
+	vc = *vbp;
+	*vbp = NULL;
+	CHECK_OBJ_NOTNULL(vc, VBC_MAGIC);
+	CHECK_OBJ_NOTNULL(vc->backend, BACKEND_MAGIC);
+	assert(vc->fd >= 0);
 
-	bp = wrk->vbc->backend;
+	bp = vc->backend;
 
-	WSL(wrk, SLT_BackendClose, wrk->vbc->vsl_id, "%s", bp->display_name);
+	WSL(wrk, SLT_BackendClose, vc->vsl_id, "%s", bp->display_name);
 
 	/* Checkpoint log to flush all info related to this connection
 	   before the OS reuses the FD */
 	WSL_Flush(wrk, 0);
 
-	VTCP_close(&wrk->vbc->fd);
+	VTCP_close(&vc->fd);
 	VBE_DropRefConn(bp);
-	wrk->vbc->backend = NULL;
-	VBE_ReleaseConn(wrk->vbc);
-	wrk->vbc = NULL;
+	vc->backend = NULL;
+	VBE_ReleaseConn(vc);
 }
 
 /* Recycle a connection ----------------------------------------------*/
diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c
index 38a2cf3..1d4eff3 100644
--- a/bin/varnishd/cache/cache_fetch.c
+++ b/bin/varnishd/cache/cache_fetch.c
@@ -427,7 +427,7 @@ FetchHdr(struct sess *sp)
 	if (WRW_FlushRelease(w) || i > 0) {
 		WSP(sp, SLT_FetchError, "backend write error: %d (%s)",
 		    errno, strerror(errno));
-		VDI_CloseFd(sp->wrk);
+		VDI_CloseFd(sp->wrk, &sp->wrk->vbc);
 		/* XXX: other cleanup ? */
 		return (retry);
 	}
@@ -451,7 +451,7 @@ FetchHdr(struct sess *sp)
 	if (i < 0) {
 		WSP(sp, SLT_FetchError, "http first read error: %d %d (%s)",
 		    i, errno, strerror(errno));
-		VDI_CloseFd(sp->wrk);
+		VDI_CloseFd(sp->wrk, &sp->wrk->vbc);
 		/* XXX: other cleanup ? */
 		/* Retryable if we never received anything */
 		return (i == -1 ? retry : -1);
@@ -465,7 +465,7 @@ FetchHdr(struct sess *sp)
 			WSP(sp, SLT_FetchError,
 			    "http first read error: %d %d (%s)",
 			    i, errno, strerror(errno));
-			VDI_CloseFd(sp->wrk);
+			VDI_CloseFd(sp->wrk, &sp->wrk->vbc);
 			/* XXX: other cleanup ? */
 			return (-1);
 		}
@@ -475,7 +475,7 @@ FetchHdr(struct sess *sp)
 
 	if (http_DissectResponse(w, htc, hp)) {
 		WSP(sp, SLT_FetchError, "http format error");
-		VDI_CloseFd(sp->wrk);
+		VDI_CloseFd(sp->wrk, &sp->wrk->vbc);
 		/* XXX: other cleanup ? */
 		return (-1);
 	}
@@ -571,7 +571,7 @@ FetchBody(struct worker *w, struct object *obj)
 	    cls, mklen);
 
 	if (w->busyobj->body_status == BS_ERROR) {
-		VDI_CloseFd(w);
+		VDI_CloseFd(w, &w->vbc);
 		return (__LINE__);
 	}
 
@@ -583,7 +583,7 @@ FetchBody(struct worker *w, struct object *obj)
 			VTAILQ_REMOVE(&obj->store, st, list);
 			STV_free(st);
 		}
-		VDI_CloseFd(w);
+		VDI_CloseFd(w, &w->vbc);
 		obj->len = 0;
 		return (__LINE__);
 	}
@@ -616,7 +616,7 @@ FetchBody(struct worker *w, struct object *obj)
 	}
 
 	if (cls)
-		VDI_CloseFd(w);
+		VDI_CloseFd(w, &w->vbc);
 	else
 		VDI_RecycleFd(w);
 
diff --git a/bin/varnishd/cache/cache_pipe.c b/bin/varnishd/cache/cache_pipe.c
index 4180d39..9bf8b79 100644
--- a/bin/varnishd/cache/cache_pipe.c
+++ b/bin/varnishd/cache/cache_pipe.c
@@ -89,7 +89,7 @@ PipeSession(struct sess *sp)
 
 	if (i) {
 		SES_Close(sp, "pipe");
-		VDI_CloseFd(sp->wrk);
+		VDI_CloseFd(sp->wrk, &sp->wrk->vbc);
 		return;
 	}
 
@@ -129,5 +129,5 @@ PipeSession(struct sess *sp)
 		}
 	}
 	SES_Close(sp, "pipe");
-	VDI_CloseFd(sp->wrk);
+	VDI_CloseFd(sp->wrk, &sp->wrk->vbc);
 }



From geoff at varnish-cache.org  Mon Jan  9 20:52:47 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:47 +0100
Subject: [experimental-ims] 7e11625 Use correct vsl_id.
Message-ID: <E1RkMD1-00009a-Ja@project.varnish-software.com>

commit 7e11625e27494bb91f9129a99709d855c2ba469f
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Dec 5 09:10:33 2011 +0000

    Use correct vsl_id.

diff --git a/bin/varnishd/cache/cache_backend.c b/bin/varnishd/cache/cache_backend.c
index de4318f..a435c1c 100644
--- a/bin/varnishd/cache/cache_backend.c
+++ b/bin/varnishd/cache/cache_backend.c
@@ -66,7 +66,7 @@ VDI_AddHostHeader(const struct sess *sp)
 	CHECK_OBJ_NOTNULL(sp->wrk->bereq, HTTP_MAGIC);
 	CHECK_OBJ_NOTNULL(sp->wrk->vbc, VBC_MAGIC);
 	CHECK_OBJ_NOTNULL(sp->wrk->vbc->vdis, VDI_SIMPLE_MAGIC);
-	http_PrintfHeader(sp->wrk, sp->vsl_id, sp->wrk->bereq,
+	http_PrintfHeader(sp->wrk, sp->wrk->vbc->vsl_id, sp->wrk->bereq,
 	    "Host: %s", sp->wrk->vbc->vdis->vrt->hosthdr);
 }
 



From geoff at varnish-cache.org  Mon Jan  9 20:52:48 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:48 +0100
Subject: [experimental-ims] d763ce0 Tell FlexeLint to check printf-like
	arguments and fix what it found.
Message-ID: <E1RkMD2-0000Ao-5e@project.varnish-software.com>

commit d763ce00fb8599e52b1c06e99b91f257dc1bf822
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Dec 5 22:02:22 2011 +0000

    Tell FlexeLint to check printf-like arguments and fix what it found.
    
    Prompted to by:	DocWilco

diff --git a/bin/flint.lnt b/bin/flint.lnt
index 4ad63d1..18a8d27 100644
--- a/bin/flint.lnt
+++ b/bin/flint.lnt
@@ -2,6 +2,9 @@
 
 -ffc			// No automatic custody
 
+-printf(2, VSB_printf)
+
+
 ///////////////////////////////////////////////////////////////////////
 // Weirdness relating to varnish includes etc.
 
diff --git a/bin/varnishd/cache/cache_ban.c b/bin/varnishd/cache/cache_ban.c
index ca2b616..2badb76 100644
--- a/bin/varnishd/cache/cache_ban.c
+++ b/bin/varnishd/cache/cache_ban.c
@@ -77,7 +77,7 @@ struct ban {
 	unsigned		magic;
 #define BAN_MAGIC		0x700b08ea
 	VTAILQ_ENTRY(ban)	list;
-	unsigned		refcount;
+	int			refcount;
 	unsigned		flags;
 #define BAN_F_GONE		(1 << 0)
 #define BAN_F_REQ		(1 << 2)
@@ -841,7 +841,7 @@ ban_lurker_work(const struct sess *sp, unsigned pass)
 				break;
 			CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
 			if (cache_param->diag_bitmap & 0x80000)
-				VSL(SLT_Debug, 0, "test: %p %d %d",
+				VSL(SLT_Debug, 0, "test: %p %u %u",
 				    oc, oc->flags & OC_F_LURK, pass);
 			if ((oc->flags & OC_F_LURK) == pass)
 				break;
@@ -892,7 +892,7 @@ ban_lurker_work(const struct sess *sp, unsigned pass)
 			}
 			Lck_Unlock(&oh->mtx);
 			if (cache_param->diag_bitmap & 0x80000)
-				VSL(SLT_Debug, 0, "lurker done: %p %d %d",
+				VSL(SLT_Debug, 0, "lurker done: %p %u %u",
 				    oc, oc->flags & OC_F_LURK, pass);
 			(void)HSH_Deref(sp->wrk, NULL, &o);
 			VTIM_sleep(cache_param->ban_lurker_sleep);
diff --git a/bin/varnishd/cache/cache_cli.c b/bin/varnishd/cache/cache_cli.c
index f299bef..98d89c3 100644
--- a/bin/varnishd/cache/cache_cli.c
+++ b/bin/varnishd/cache/cache_cli.c
@@ -92,7 +92,7 @@ cli_cb_after(const struct cli *cli)
 
 	ASSERT_CLI();
 	Lck_Unlock(&cli_mtx);
-	VSL(SLT_CLI, 0, "Wr %03u %u %s",
+	VSL(SLT_CLI, 0, "Wr %03u %zd %s",
 	    cli->result, VSB_len(cli->sb), VSB_data(cli->sb));
 }
 
diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c
index 6483a99..5da06a7 100644
--- a/bin/varnishd/cache/cache_fetch.c
+++ b/bin/varnishd/cache/cache_fetch.c
@@ -565,7 +565,7 @@ FetchBody(struct worker *w, struct object *obj)
 
 	w->busyobj->fetch_obj = NULL;
 
-	WSLB(w, SLT_Fetch_Body, "%u(%s) cls %d mklen %u",
+	WSLB(w, SLT_Fetch_Body, "%u(%s) cls %d mklen %d",
 	    w->busyobj->body_status, body_status(w->busyobj->body_status),
 	    cls, mklen);
 
@@ -591,7 +591,7 @@ FetchBody(struct worker *w, struct object *obj)
 	if (cls == 0 && w->busyobj->should_close)
 		cls = 1;
 
-	WSLB(w, SLT_Length, "%u", obj->len);
+	WSLB(w, SLT_Length, "%zd", obj->len);
 
 	{
 	/* Sanity check fetch methods accounting */
diff --git a/bin/varnishd/cache/cache_http.c b/bin/varnishd/cache/cache_http.c
index 784eb28..267af00 100644
--- a/bin/varnishd/cache/cache_http.c
+++ b/bin/varnishd/cache/cache_http.c
@@ -520,7 +520,7 @@ http_dissect_hdrs(struct worker *w, struct http *hp, unsigned vsl_id, char *p,
 		if (q - p > htc->maxhdr) {
 			VSC_C_main->losthdr++;
 			WSL(w, SLT_LostHeader, vsl_id, "%.*s",
-			    q - p > 20 ? 20 : q - p, p);
+			    (int)(q - p > 20 ? 20 : q - p), p);
 			return (413);
 		}
 
@@ -546,7 +546,7 @@ http_dissect_hdrs(struct worker *w, struct http *hp, unsigned vsl_id, char *p,
 		} else {
 			VSC_C_main->losthdr++;
 			WSL(w, SLT_LostHeader, vsl_id, "%.*s",
-			    q - p > 20 ? 20 : q - p, p);
+			    (int)(q - p > 20 ? 20 : q - p), p);
 			return (413);
 		}
 	}
diff --git a/bin/varnishd/cache/cache_lck.c b/bin/varnishd/cache/cache_lck.c
index d74e31a..78b5e42 100644
--- a/bin/varnishd/cache/cache_lck.c
+++ b/bin/varnishd/cache/cache_lck.c
@@ -114,7 +114,7 @@ Lck__Trylock(struct lock *lck, const char *p, const char *f, int l)
 	assert(r == 0 || r == EBUSY);
 	if (cache_param->diag_bitmap & 0x8)
 		VSL(SLT_Debug, 0,
-		    "MTX_TRYLOCK(%s,%s,%d,%s) = %d", p, f, l, ilck->w);
+		    "MTX_TRYLOCK(%s,%s,%d,%s) = %d", p, f, l, ilck->w, r);
 	if (r == 0) {
 		AZ(ilck->held);
 		ilck->held = 1;
diff --git a/bin/varnishd/cache/cache_panic.c b/bin/varnishd/cache/cache_panic.c
index 923abbd..9fa14e0 100644
--- a/bin/varnishd/cache/cache_panic.c
+++ b/bin/varnishd/cache/cache_panic.c
@@ -234,7 +234,7 @@ pan_sess(const struct sess *sp)
 
 	VSB_printf(pan_vsp, "sp = %p {\n", sp);
 	VSB_printf(pan_vsp,
-	    "  fd = %d, id = %d, xid = %u,\n",
+	    "  fd = %d, id = %u, xid = %u,\n",
 	    sp->fd, sp->vsl_id & VSL_IDENTMASK, sp->xid);
 	VSB_printf(pan_vsp, "  client = %s %s,\n",
 	    sp->addr ? sp->addr : "?.?.?.?",
diff --git a/bin/varnishd/cache/cache_vary.c b/bin/varnishd/cache/cache_vary.c
index 026f937..f55607b 100644
--- a/bin/varnishd/cache/cache_vary.c
+++ b/bin/varnishd/cache/cache_vary.c
@@ -64,7 +64,7 @@ VRY_Create(const struct sess *sp, const struct http *hp)
 {
 	char *v, *p, *q, *h, *e;
 	struct vsb *sb, *sbh;
-	int l;
+	unsigned l;
 
 	/* No Vary: header, no worries */
 	if (!http_GetHdr(hp, H_Vary, &v))
@@ -109,7 +109,7 @@ VRY_Create(const struct sess *sp, const struct http *hp)
 			e = h;
 			l = 0xffff;
 		}
-		VSB_printf(sb, "%c%c", (unsigned)l >> 8, l & 0xff);
+		VSB_printf(sb, "%c%c", (int)(l >> 8), (int)(l & 0xff));
 		/* Append to vary matching string */
 		VSB_bcat(sb, VSB_data(sbh), VSB_len(sbh));
 		if (e != h)
diff --git a/bin/varnishd/cache/cache_vrt.c b/bin/varnishd/cache/cache_vrt.c
index a0eb4c8..bbf93b5 100644
--- a/bin/varnishd/cache/cache_vrt.c
+++ b/bin/varnishd/cache/cache_vrt.c
@@ -74,7 +74,7 @@ VRT_count(const struct sess *sp, unsigned u)
 		return;
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 	if (cache_param->vcl_trace)
-		WSP(sp, SLT_VCL_trace, "%u %d.%d", u,
+		WSP(sp, SLT_VCL_trace, "%u %u.%u", u,
 		    sp->vcl->ref[u].line, sp->vcl->ref[u].pos);
 }
 
@@ -83,7 +83,8 @@ VRT_count(const struct sess *sp, unsigned u)
 void
 VRT_acl_log(const struct sess *sp, const char *msg)
 {
-	WSP(sp, SLT_VCL_acl, msg);
+
+	WSP(sp, SLT_VCL_acl, "%s", msg);
 }
 
 /*--------------------------------------------------------------------*/
@@ -416,7 +417,7 @@ VRT_synth_page(const struct sess *sp, unsigned flags, const char *str, ...)
 	SMS_Finish(sp->wrk->obj);
 	http_Unset(sp->wrk->obj->http, H_Content_Length);
 	http_PrintfHeader(sp->wrk, sp->vsl_id, sp->wrk->obj->http,
-	    "Content-Length: %d", sp->wrk->obj->len);
+	    "Content-Length: %zd", sp->wrk->obj->len);
 }
 
 /*--------------------------------------------------------------------*/
diff --git a/bin/varnishd/cache/cache_vrt_var.c b/bin/varnishd/cache/cache_vrt_var.c
index fea5c70..cb54837 100644
--- a/bin/varnishd/cache/cache_vrt_var.c
+++ b/bin/varnishd/cache/cache_vrt_var.c
@@ -55,7 +55,7 @@ vrt_do_string(struct worker *w, int fd, const struct http *hp, int fld,
 	AN(hp);
 	b = VRT_String(hp->ws, NULL, p, ap);
 	if (b == NULL || *b == '\0') {
-		WSL(w, SLT_LostHeader, fd, err);
+		WSL(w, SLT_LostHeader, fd, "%s", err);
 	} else {
 		http_SetH(hp, fld, b);
 	}
diff --git a/bin/varnishd/cache/cache_wrw.c b/bin/varnishd/cache/cache_wrw.c
index 2160f69..2ffb45e 100644
--- a/bin/varnishd/cache/cache_wrw.c
+++ b/bin/varnishd/cache/cache_wrw.c
@@ -165,7 +165,7 @@ WRW_Flush(struct worker *w)
 		if (i <= 0) {
 			wrw->werr++;
 			WSL(w, SLT_Debug, *wrw->wfd,
-			    "Write error, retval = %d, len = %d, errno = %s",
+			    "Write error, retval = %zd, len = %zd, errno = %s",
 			    i, wrw->liov, strerror(errno));
 		}
 	}
diff --git a/bin/varnishd/flint.lnt b/bin/varnishd/flint.lnt
index ec0d955..1942c63 100644
--- a/bin/varnishd/flint.lnt
+++ b/bin/varnishd/flint.lnt
@@ -1,5 +1,11 @@
 -d__flexelint_v9__=1
 
+-printf(3, VSL)
+-printf(4, http_PrintfHeader)
+-printf(4, WSL)
+-printf(3, WSLB)
+-printf(2, VSB_printf)
+
 -esym(755, vct_*)
 -esym(759, vev_*)
 -esym(765, vev_*)
diff --git a/lib/libvcl/vcc_compile.h b/lib/libvcl/vcc_compile.h
index 8088509..b67e4ca 100644
--- a/lib/libvcl/vcc_compile.h
+++ b/lib/libvcl/vcc_compile.h
@@ -308,7 +308,7 @@ void vcc_ErrToken(const struct vcc *tl, const struct token *t);
 void vcc_ErrWhere(struct vcc *, const struct token *);
 void vcc_ErrWhere2(struct vcc *, const struct token *, const struct token *);
 
-void vcc__Expect(struct vcc *tl, unsigned tok, int line);
+void vcc__Expect(struct vcc *tl, unsigned tok, unsigned line);
 int vcc_Teq(const struct token *t1, const struct token *t2);
 int vcc_IdIs(const struct token *t, const char *p);
 void vcc_ExpectCid(struct vcc *tl);
diff --git a/lib/libvcl/vcc_token.c b/lib/libvcl/vcc_token.c
index 63ebbf3..26f2545 100644
--- a/lib/libvcl/vcc_token.c
+++ b/lib/libvcl/vcc_token.c
@@ -104,7 +104,7 @@ vcc_icoord(struct vsb *vsb, const struct token *t, int tail)
 		} else
 			pos++;
 	}
-	VSB_printf(vsb, "('%s' Line %d Pos %d)", t->src->name, lin, pos + 1);
+	VSB_printf(vsb, "('%s' Line %u Pos %u)", t->src->name, lin, pos + 1);
 }
 
 /*--------------------------------------------------------------------*/
@@ -261,7 +261,7 @@ vcc_NextToken(struct vcc *tl)
 }
 
 void
-vcc__Expect(struct vcc *tl, unsigned tok, int line)
+vcc__Expect(struct vcc *tl, unsigned tok, unsigned line)
 {
 	if (tl->t->tok == tok)
 		return;



From geoff at varnish-cache.org  Mon Jan  9 20:52:48 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:48 +0100
Subject: [experimental-ims] 6ac5644 Code polishing
Message-ID: <E1RkMD2-0000BI-B3@project.varnish-software.com>

commit 6ac5644bc8610840b59e52f183eed533c73f65d1
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Dec 6 08:09:50 2011 +0000

    Code polishing

diff --git a/bin/varnishd/cache/cache_wrw.c b/bin/varnishd/cache/cache_wrw.c
index 2ffb45e..96fa3e5 100644
--- a/bin/varnishd/cache/cache_wrw.c
+++ b/bin/varnishd/cache/cache_wrw.c
@@ -57,19 +57,19 @@
  */
 
 int
-WRW_Error(const struct worker *w)
+WRW_Error(const struct worker *wrk)
 {
 
-	return (w->wrw.werr);
+	return (wrk->wrw.werr);
 }
 
 void
-WRW_Reserve(struct worker *w, int *fd)
+WRW_Reserve(struct worker *wrk, int *fd)
 {
 	struct wrw *wrw;
 
-	CHECK_OBJ_NOTNULL(w, WORKER_MAGIC);
-	wrw = &w->wrw;
+	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
+	wrw = &wrk->wrw;
 	AZ(wrw->wfd);
 	wrw->werr = 0;
 	wrw->liov = 0;
@@ -79,12 +79,12 @@ WRW_Reserve(struct worker *w, int *fd)
 }
 
 static void
-WRW_Release(struct worker *w)
+WRW_Release(struct worker *wrk)
 {
 	struct wrw *wrw;
 
-	CHECK_OBJ_NOTNULL(w, WORKER_MAGIC);
-	wrw = &w->wrw;
+	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
+	wrw = &wrk->wrw;
 	AN(wrw->wfd);
 	wrw->werr = 0;
 	wrw->liov = 0;
@@ -93,15 +93,39 @@ WRW_Release(struct worker *w)
 	wrw->wfd = NULL;
 }
 
+static void
+wrw_prune(struct wrw *wrw, ssize_t bytes)
+{
+	ssize_t used = 0;
+	ssize_t j, used_here;
+
+	for (j = 0; j < wrw->niov; j++) {
+		if (used + wrw->iov[j].iov_len > bytes) {
+			/* Cutoff is in this iov */
+			used_here = bytes - used;
+			wrw->iov[j].iov_len -= used_here;
+			wrw->iov[j].iov_base =
+			    (char*)wrw->iov[j].iov_base + used_here;
+			memmove(wrw->iov, &wrw->iov[j],
+			    (wrw->niov - j) * sizeof(struct iovec));
+			wrw->niov -= j;
+			wrw->liov -= bytes;
+			return;
+		}
+		used += wrw->iov[j].iov_len;
+	}
+	assert(wrw->liov == 0);
+}
+
 unsigned
-WRW_Flush(struct worker *w)
+WRW_Flush(struct worker *wrk)
 {
 	ssize_t i;
 	struct wrw *wrw;
 	char cbuf[32];
 
-	CHECK_OBJ_NOTNULL(w, WORKER_MAGIC);
-	wrw = &w->wrw;
+	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
+	wrw = &wrk->wrw;
 	AN(wrw->wfd);
 
 	/* For chunked, there must be one slot reserved for the chunked tail */
@@ -110,7 +134,8 @@ WRW_Flush(struct worker *w)
 
 	if (*wrw->wfd >= 0 && wrw->liov > 0 && wrw->werr == 0) {
 		if (wrw->ciov < wrw->siov && wrw->cliov > 0) {
-			bprintf(cbuf, "00%jx\r\n", (intmax_t)wrw->cliov);
+			/* Add chunk head & tail */
+			bprintf(cbuf, "00%zx\r\n", wrw->cliov);
 			i = strlen(cbuf);
 			wrw->iov[wrw->ciov].iov_base = cbuf;
 			wrw->iov[wrw->ciov].iov_len = i;
@@ -123,48 +148,37 @@ WRW_Flush(struct worker *w)
 			wrw->iov[wrw->ciov].iov_base = cbuf;
 			wrw->iov[wrw->ciov].iov_len = 0;
 		}
+
 		i = writev(*wrw->wfd, wrw->iov, wrw->niov);
 		while (i != wrw->liov && i > 0) {
 			/* Remove sent data from start of I/O vector,
 			 * then retry; we hit a timeout, but some data
 			 * was sent.
-
-			 XXX: Add a "minimum sent data per timeout
-			 counter to prevent slowlaris attacks
+			 *
+			 * XXX: Add a "minimum sent data per timeout
+			 * counter to prevent slowlaris attacks
 			*/
-			size_t used = 0;
 
-			if (VTIM_real() - w->sp->t_resp > cache_param->send_timeout) {
-				WSL(w, SLT_Debug, *wrw->wfd,
-				    "Hit total send timeout, wrote = %ld/%ld; not retrying",
+			if (VTIM_real() - wrk->sp->t_resp >
+			    cache_param->send_timeout) {
+				WSL(wrk, SLT_Debug, *wrw->wfd,
+				    "Hit total send timeout, "
+				    "wrote = %ld/%ld; not retrying",
 				    i, wrw->liov);
 				i = -1;
 				break;
 			}
 
-			WSL(w, SLT_Debug, *wrw->wfd,
+			WSL(wrk, SLT_Debug, *wrw->wfd,
 			    "Hit send timeout, wrote = %ld/%ld; retrying",
 			    i, wrw->liov);
 
-			for (int j = 0; j < wrw->niov; j++) {
-				if (used + wrw->iov[j].iov_len > i) {
-					/* Cutoff is in this iov */
-					int used_here = i - used;
-					wrw->iov[j].iov_len -= used_here;
-					wrw->iov[j].iov_base = (char*)wrw->iov[j].iov_base + used_here;
-					memmove(wrw->iov, &wrw->iov[j],
-						(wrw->niov - j) * sizeof(struct iovec));
-					wrw->niov -= j;
-					wrw->liov -= i;
-					break;
-				}
-				used += wrw->iov[j].iov_len;
-			}
+			wrw_prune(wrw, i);
 			i = writev(*wrw->wfd, wrw->iov, wrw->niov);
 		}
 		if (i <= 0) {
 			wrw->werr++;
-			WSL(w, SLT_Debug, *wrw->wfd,
+			WSL(wrk, SLT_Debug, *wrw->wfd,
 			    "Write error, retval = %zd, len = %zd, errno = %s",
 			    i, wrw->liov, strerror(errno));
 		}
@@ -178,48 +192,48 @@ WRW_Flush(struct worker *w)
 }
 
 unsigned
-WRW_FlushRelease(struct worker *w)
+WRW_FlushRelease(struct worker *wrk)
 {
 	unsigned u;
 
-	CHECK_OBJ_NOTNULL(w, WORKER_MAGIC);
-	AN(w->wrw.wfd);
-	u = WRW_Flush(w);
-	WRW_Release(w);
+	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
+	AN(wrk->wrw.wfd);
+	u = WRW_Flush(wrk);
+	WRW_Release(wrk);
 	return (u);
 }
 
 unsigned
-WRW_WriteH(struct worker *w, const txt *hh, const char *suf)
+WRW_WriteH(struct worker *wrk, const txt *hh, const char *suf)
 {
 	unsigned u;
 
-	CHECK_OBJ_NOTNULL(w, WORKER_MAGIC);
-	AN(w->wrw.wfd);
-	AN(w);
+	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
+	AN(wrk->wrw.wfd);
+	AN(wrk);
 	AN(hh);
 	AN(hh->b);
 	AN(hh->e);
-	u = WRW_Write(w, hh->b, hh->e - hh->b);
+	u = WRW_Write(wrk, hh->b, hh->e - hh->b);
 	if (suf != NULL)
-		u += WRW_Write(w, suf, -1);
+		u += WRW_Write(wrk, suf, -1);
 	return (u);
 }
 
 unsigned
-WRW_Write(struct worker *w, const void *ptr, int len)
+WRW_Write(struct worker *wrk, const void *ptr, int len)
 {
 	struct wrw *wrw;
 
-	CHECK_OBJ_NOTNULL(w, WORKER_MAGIC);
-	wrw = &w->wrw;
+	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
+	wrw = &wrk->wrw;
 	AN(wrw->wfd);
 	if (len == 0 || *wrw->wfd < 0)
 		return (0);
 	if (len == -1)
 		len = strlen(ptr);
 	if (wrw->niov >= wrw->siov - (wrw->ciov < wrw->siov ? 1 : 0))
-		(void)WRW_Flush(w);
+		(void)WRW_Flush(wrk);
 	wrw->iov[wrw->niov].iov_base = TRUST_ME(ptr);
 	wrw->iov[wrw->niov].iov_len = len;
 	wrw->liov += len;
@@ -232,12 +246,12 @@ WRW_Write(struct worker *w, const void *ptr, int len)
 }
 
 void
-WRW_Chunked(struct worker *w)
+WRW_Chunked(struct worker *wrk)
 {
 	struct wrw *wrw;
 
-	CHECK_OBJ_NOTNULL(w, WORKER_MAGIC);
-	wrw = &w->wrw;
+	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
+	wrw = &wrk->wrw;
 
 	assert(wrw->ciov == wrw->siov);
 	/*
@@ -245,7 +259,7 @@ WRW_Chunked(struct worker *w)
 	 * a chunk tail, we might as well flush right away.
 	 */
 	if (wrw->niov + 3 >= wrw->siov)
-		(void)WRW_Flush(w);
+		(void)WRW_Flush(wrk);
 	wrw->ciov = wrw->niov++;
 	wrw->cliov = 0;
 	assert(wrw->ciov < wrw->siov);
@@ -260,30 +274,30 @@ WRW_Chunked(struct worker *w)
  */
 
 void
-WRW_EndChunk(struct worker *w)
+WRW_EndChunk(struct worker *wrk)
 {
 	struct wrw *wrw;
 
-	CHECK_OBJ_NOTNULL(w, WORKER_MAGIC);
-	wrw = &w->wrw;
+	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
+	wrw = &wrk->wrw;
 
 	assert(wrw->ciov < wrw->siov);
-	(void)WRW_Flush(w);
+	(void)WRW_Flush(wrk);
 	wrw->ciov = wrw->siov;
 	wrw->niov = 0;
 	wrw->cliov = 0;
-	(void)WRW_Write(w, "0\r\n\r\n", -1);
+	(void)WRW_Write(wrk, "0\r\n\r\n", -1);
 }
 
 
 #ifdef SENDFILE_WORKS
 void
-WRW_Sendfile(struct worker *w, int fd, off_t off, unsigned len)
+WRW_Sendfile(struct worker *wrk, int fd, off_t off, unsigned len)
 {
 	struct wrw *wrw;
 
-	CHECK_OBJ_NOTNULL(w, WORKER_MAGIC);
-	wrw = &w->wrw;
+	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
+	wrw = &wrk->wrw;
 	AN(wrw->wfd);
 	assert(fd >= 0);
 	assert(len > 0);
@@ -303,7 +317,7 @@ WRW_Sendfile(struct worker *w, int fd, off_t off, unsigned len)
 	} while (0);
 #elif defined(__linux__)
 	do {
-		if (WRW_Flush(w) == 0 &&
+		if (WRW_Flush(wrk) == 0 &&
 		    sendfile(*wrw->wfd, fd, &off, len) != len)
 			wrw->werr++;
 	} while (0);
@@ -332,7 +346,7 @@ WRW_Sendfile(struct worker *w, int fd, off_t off, unsigned len)
 	} while (0);
 #elif defined(__sun) && defined(HAVE_SENDFILE)
 	do {
-		if (WRW_Flush(w) == 0 &&
+		if (WRW_Flush(wrk) == 0 &&
 		    sendfile(*wrw->wfd, fd, &off, len) != len)
 			wrw->werr++;
 	} while (0);



From geoff at varnish-cache.org  Mon Jan  9 20:52:44 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:44 +0100
Subject: [experimental-ims] 69f785e move do_stream from worker to busyobj
Message-ID: <E1RkMCy-0008W0-Uj@project.varnish-software.com>

commit 69f785edcf843f5386444cad4d3a94c9b82ac303
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Nov 30 07:10:28 2011 +0000

    move do_stream from worker to busyobj

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index 9a4e5f3..8491150 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -332,7 +332,6 @@ struct worker {
 	struct http		*beresp;
 	struct vbc		*vbc;
 
-	unsigned		do_stream;
 	unsigned		do_close;
 	char			*h_content_length;
 
@@ -508,6 +507,7 @@ struct busyobj {
 	unsigned		do_esi;
 	unsigned		do_gzip;
 	unsigned		do_gunzip;
+	unsigned		do_stream;
 };
 
 /* Object structure --------------------------------------------------*/
diff --git a/bin/varnishd/cache/cache_center.c b/bin/varnishd/cache/cache_center.c
index 6acd218..a8138da 100644
--- a/bin/varnishd/cache/cache_center.c
+++ b/bin/varnishd/cache/cache_center.c
@@ -179,7 +179,7 @@ cnt_prepresp(struct sess *sp)
 	CHECK_OBJ_NOTNULL(wrk->obj, OBJECT_MAGIC);
 	CHECK_OBJ_NOTNULL(sp->vcl, VCL_CONF_MAGIC);
 
-	if (wrk->do_stream)
+	if (wrk->busyobj != NULL && wrk->busyobj->do_stream)
 		AssertObjCorePassOrBusy(wrk->obj->objcore);
 
 	wrk->res_mode = 0;
@@ -188,7 +188,7 @@ cnt_prepresp(struct sess *sp)
 		wrk->res_mode |= RES_LEN;
 
 	if (wrk->busyobj != NULL &&
-	    (wrk->h_content_length != NULL || !wrk->do_stream) &&
+	    (wrk->h_content_length != NULL || !wrk->busyobj->do_stream) &&
 	    !wrk->busyobj->do_gzip && !wrk->busyobj->do_gunzip)
 		wrk->res_mode |= RES_LEN;
 
@@ -214,7 +214,8 @@ cnt_prepresp(struct sess *sp)
 	}
 
 	if (!(wrk->res_mode & (RES_LEN|RES_CHUNKED|RES_EOF))) {
-		if (wrk->obj->len == 0 && !wrk->do_stream)
+		if (wrk->obj->len == 0 &&
+		    (wrk->busyobj == NULL || !wrk->busyobj->do_stream))
 			/*
 			 * If the object is empty, neither ESI nor GUNZIP
 			 * can make it any different size
@@ -246,7 +247,7 @@ cnt_prepresp(struct sess *sp)
 	case VCL_RET_RESTART:
 		if (sp->restarts >= cache_param->max_restarts)
 			break;
-		if (wrk->do_stream) {
+		if (wrk->busyobj->do_stream) {
 			VDI_CloseFd(wrk);
 			HSH_Drop(wrk);
 		} else {
@@ -264,7 +265,7 @@ cnt_prepresp(struct sess *sp)
 	default:
 		WRONG("Illegal action in vcl_deliver{}");
 	}
-	if (wrk->do_stream) {
+	if (wrk->busyobj != NULL && wrk->busyobj->do_stream) {
 		AssertObjCorePassOrBusy(wrk->obj->objcore);
 		sp->step = STP_STREAMBODY;
 	} else {
@@ -343,8 +344,6 @@ cnt_done(struct sess *sp)
 
 	wrk->busyobj = NULL;
 
-	wrk->do_stream = 0;
-
 	SES_Charge(sp);
 
 	/* If we did an ESI include, don't mess up our state */
@@ -458,8 +457,6 @@ cnt_error(struct sess *sp)
 	wrk = sp->wrk;
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 
-	wrk->do_stream = 0;
-
 	if (wrk->obj == NULL) {
 		HSH_Prealloc(sp);
 		New_BusyObj(wrk);
@@ -780,9 +777,9 @@ cnt_fetchbody(struct sess *sp)
 		wrk->busyobj->vfp = &vfp_testgzip;
 
 	if (wrk->busyobj->do_esi || sp->esi_level > 0)
-		wrk->do_stream = 0;
+		wrk->busyobj->do_stream = 0;
 	if (!sp->wantbody)
-		wrk->do_stream = 0;
+		wrk->busyobj->do_stream = 0;
 
 	l = http_EstimateWS(wrk->beresp,
 	    pass ? HTTPH_R_PASS : HTTPH_A_INS, &nhttp);
@@ -871,11 +868,11 @@ cnt_fetchbody(struct sess *sp)
 	if (wrk->obj->response == 200 &&
 	    sp->http->conds &&
 	    RFC2616_Do_Cond(sp))
-		wrk->do_stream = 0;
+		wrk->busyobj->do_stream = 0;
 
 	AssertObjCorePassOrBusy(wrk->obj->objcore);
 
-	if (wrk->do_stream) {
+	if (wrk->busyobj->do_stream) {
 		sp->step = STP_PREPRESP;
 		return (0);
 	}
@@ -1062,8 +1059,6 @@ cnt_hit(struct sess *sp)
 
 	assert(!(wrk->obj->objcore->flags & OC_F_PASS));
 
-	AZ(wrk->do_stream);
-
 	VCL_hit_method(sp);
 
 	if (sp->handling == VCL_RET_DELIVER) {
@@ -1468,9 +1463,6 @@ cnt_recv(struct sess *sp)
 		return (0);
 	}
 
-	/* Zap these, in case we came here through restart */
-	wrk->do_stream = 0;
-
 	if (cache_param->http_gzip_support &&
 	     (recv_handling != VCL_RET_PIPE) &&
 	     (recv_handling != VCL_RET_PASS)) {
@@ -1651,7 +1643,6 @@ CNT_Session(struct sess *sp)
 	    sp->step == STP_LOOKUP ||
 	    sp->step == STP_RECV);
 
-	AZ(wrk->do_stream);
 	AZ(wrk->obj);
 	AZ(wrk->objcore);
 
@@ -1705,7 +1696,6 @@ CNT_Session(struct sess *sp)
 	WSL_Flush(wrk, 0);
 	AZ(wrk->obj);
 	AZ(wrk->objcore);
-	AZ(wrk->do_stream);
 #define ACCT(foo)	AZ(wrk->acct_tmp.foo);
 #include "tbl/acct_fields.h"
 #undef ACCT
diff --git a/bin/varnishd/cache/cache_esi_deliver.c b/bin/varnishd/cache/cache_esi_deliver.c
index 518c5a9..0d6b74c 100644
--- a/bin/varnishd/cache/cache_esi_deliver.c
+++ b/bin/varnishd/cache/cache_esi_deliver.c
@@ -91,8 +91,6 @@ ved_include(struct sess *sp, const char *src, const char *host)
 	/* Client content already taken care of */
 	http_Unset(sp->http, H_Content_Length);
 
-	sp->wrk->do_stream = 0;
-
 	sxid = sp->xid;
 	while (1) {
 		sp->wrk = w;
diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c
index 15eeb9b..a198650 100644
--- a/bin/varnishd/cache/cache_fetch.c
+++ b/bin/varnishd/cache/cache_fetch.c
@@ -126,7 +126,7 @@ vfp_nop_bytes(struct worker *w, struct http_conn *htc, ssize_t bytes)
 		st->len += wl;
 		w->busyobj->fetch_obj->len += wl;
 		bytes -= wl;
-		if (w->do_stream)
+		if (w->busyobj->do_stream)
 			RES_StreamPoll(w);
 	}
 	return (1);
@@ -601,7 +601,7 @@ FetchBody(struct worker *w, struct object *obj)
 		uu = 0;
 		VTAILQ_FOREACH(st, &obj->store, list)
 			uu += st->len;
-		if (w->do_stream)
+		if (w->busyobj->do_stream)
 			/* Streaming might have started freeing stuff */
 			assert (uu <= obj->len);
 
diff --git a/bin/varnishd/cache/cache_gzip.c b/bin/varnishd/cache/cache_gzip.c
index 4076081..cf07e1d 100644
--- a/bin/varnishd/cache/cache_gzip.c
+++ b/bin/varnishd/cache/cache_gzip.c
@@ -493,7 +493,7 @@ vfp_gunzip_bytes(struct worker *w, struct http_conn *htc, ssize_t bytes)
 		if (i != VGZ_OK && i != VGZ_END)
 			return(FetchError(w, "Gunzip data error"));
 		w->busyobj->fetch_obj->len += dl;
-		if (w->do_stream)
+		if (w->busyobj->do_stream)
 			RES_StreamPoll(w);
 	}
 	assert(i == Z_OK || i == Z_STREAM_END);
@@ -569,7 +569,7 @@ vfp_gzip_bytes(struct worker *w, struct http_conn *htc, ssize_t bytes)
 		i = VGZ_Gzip(vg, &dp, &dl, VGZ_NORMAL);
 		assert(i == Z_OK);
 		w->busyobj->fetch_obj->len += dl;
-		if (w->do_stream)
+		if (w->busyobj->do_stream)
 			RES_StreamPoll(w);
 	}
 	return (1);
@@ -597,7 +597,7 @@ vfp_gzip_end(struct worker *w)
 		i = VGZ_Gzip(vg, &dp, &dl, VGZ_FINISH);
 		w->busyobj->fetch_obj->len += dl;
 	} while (i != Z_STREAM_END);
-	if (w->do_stream)
+	if (w->busyobj->do_stream)
 		RES_StreamPoll(w);
 	VGZ_UpdateObj(vg, w->busyobj->fetch_obj);
 	if (VGZ_Destroy(&vg, -1) != VGZ_END)
@@ -655,7 +655,7 @@ vfp_testgzip_bytes(struct worker *w, struct http_conn *htc, ssize_t bytes)
 		VGZ_Ibuf(vg, st->ptr + st->len, wl);
 		st->len += wl;
 		w->busyobj->fetch_obj->len += wl;
-		if (w->do_stream)
+		if (w->busyobj->do_stream)
 			RES_StreamPoll(w);
 
 		while (!VGZ_IbufEmpty(vg)) {
diff --git a/bin/varnishd/cache/cache_panic.c b/bin/varnishd/cache/cache_panic.c
index 718a686..58fb8e4 100644
--- a/bin/varnishd/cache/cache_panic.c
+++ b/bin/varnishd/cache/cache_panic.c
@@ -216,6 +216,7 @@ pan_busyobj(const struct busyobj *bo)
 	if (bo->do_gzip)	VSB_printf(pan_vsp, "    do_gzip\n");
 	if (bo->do_gunzip)	VSB_printf(pan_vsp, "    do_gunzip\n");
 	if (bo->do_esi)		VSB_printf(pan_vsp, "    do_esi\n");
+	if (bo->do_stream)	VSB_printf(pan_vsp, "    do_stream\n");
 	VSB_printf(pan_vsp, "    bodystatus = %d,\n", bo->body_status);
 	VSB_printf(pan_vsp, "    },\n");
 }
@@ -258,7 +259,6 @@ pan_sess(const struct sess *sp)
 	    sp->restarts, sp->esi_level);
 
 	VSB_printf(pan_vsp, "  flags = ");
-	if (sp->wrk->do_stream)	VSB_printf(pan_vsp, " do_stream");
 	if (sp->wrk->do_close)	VSB_printf(pan_vsp, " do_close");
 	VSB_printf(pan_vsp, "\n");
 	pan_busyobj(sp->wrk->busyobj);
diff --git a/bin/varnishd/cache/cache_vrt_var.c b/bin/varnishd/cache/cache_vrt_var.c
index adf1017..0342de8 100644
--- a/bin/varnishd/cache/cache_vrt_var.c
+++ b/bin/varnishd/cache/cache_vrt_var.c
@@ -197,7 +197,7 @@ VRT_r_##dir##_##onm(const struct sess *sp)				\
 VBERESP(beresp, unsigned, do_esi, busyobj->do_esi)
 VBERESP(beresp, unsigned, do_gzip, busyobj->do_gzip)
 VBERESP(beresp, unsigned, do_gunzip, busyobj->do_gunzip)
-VBERESP(beresp, unsigned, do_stream, do_stream)
+VBERESP(beresp, unsigned, do_stream, busyobj->do_stream)
 
 /*--------------------------------------------------------------------*/
 



From geoff at varnish-cache.org  Mon Jan  9 20:52:48 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:48 +0100
Subject: [experimental-ims] 3ef44da More code polishing
Message-ID: <E1RkMD2-0000Bu-JG@project.varnish-software.com>

commit 3ef44da5d44710270164d6970c3891f2b0e8ed8a
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Dec 6 08:59:16 2011 +0000

    More code polishing

diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c
index 5da06a7..8a1ac5b 100644
--- a/bin/varnishd/cache/cache_fetch.c
+++ b/bin/varnishd/cache/cache_fetch.c
@@ -46,30 +46,30 @@ static unsigned fetchfrag;
  * We want to issue the first error we encounter on fetching and
  * supress the rest.  This function does that.
  *
- * Other code is allowed to look at w->busyobj->fetch_failed to bail out
+ * Other code is allowed to look at wrk->busyobj->fetch_failed to bail out
  *
  * For convenience, always return -1
  */
 
 int
-FetchError2(struct worker *w, const char *error, const char *more)
+FetchError2(struct worker *wrk, const char *error, const char *more)
 {
 
-	CHECK_OBJ_NOTNULL(w, WORKER_MAGIC);
-	if (!w->busyobj->fetch_failed) {
+	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
+	if (!wrk->busyobj->fetch_failed) {
 		if (more == NULL)
-			WSLB(w, SLT_FetchError, "%s", error);
+			WSLB(wrk, SLT_FetchError, "%s", error);
 		else
-			WSLB(w, SLT_FetchError, "%s: %s", error, more);
+			WSLB(wrk, SLT_FetchError, "%s: %s", error, more);
 	}
-	w->busyobj->fetch_failed = 1;
+	wrk->busyobj->fetch_failed = 1;
 	return (-1);
 }
 
 int
-FetchError(struct worker *w, const char *error)
+FetchError(struct worker *wrk, const char *error)
 {
-	return(FetchError2(w, error, NULL));
+	return(FetchError2(wrk, error, NULL));
 }
 
 /*--------------------------------------------------------------------
@@ -88,11 +88,11 @@ FetchError(struct worker *w, const char *error)
  * as seen on the socket, or zero if unknown.
  */
 static void __match_proto__()
-vfp_nop_begin(struct worker *w, size_t estimate)
+vfp_nop_begin(struct worker *wrk, size_t estimate)
 {
 
 	if (estimate > 0)
-		(void)FetchStorage(w, estimate);
+		(void)FetchStorage(wrk, estimate);
 }
 
 /*--------------------------------------------------------------------
@@ -107,27 +107,27 @@ vfp_nop_begin(struct worker *w, size_t estimate)
  */
 
 static int __match_proto__()
-vfp_nop_bytes(struct worker *w, struct http_conn *htc, ssize_t bytes)
+vfp_nop_bytes(struct worker *wrk, struct http_conn *htc, ssize_t bytes)
 {
 	ssize_t l, wl;
 	struct storage *st;
 
-	AZ(w->busyobj->fetch_failed);
+	AZ(wrk->busyobj->fetch_failed);
 	while (bytes > 0) {
-		st = FetchStorage(w, 0);
+		st = FetchStorage(wrk, 0);
 		if (st == NULL)
 			return(-1);
 		l = st->space - st->len;
 		if (l > bytes)
 			l = bytes;
-		wl = HTC_Read(w, htc, st->ptr + st->len, l);
+		wl = HTC_Read(wrk, htc, st->ptr + st->len, l);
 		if (wl <= 0)
 			return (wl);
 		st->len += wl;
-		w->busyobj->fetch_obj->len += wl;
+		wrk->busyobj->fetch_obj->len += wl;
 		bytes -= wl;
-		if (w->busyobj->do_stream)
-			RES_StreamPoll(w);
+		if (wrk->busyobj->do_stream)
+			RES_StreamPoll(wrk);
 	}
 	return (1);
 }
@@ -142,16 +142,16 @@ vfp_nop_bytes(struct worker *w, struct http_conn *htc, ssize_t bytes)
  */
 
 static int __match_proto__()
-vfp_nop_end(struct worker *w)
+vfp_nop_end(struct worker *wrk)
 {
 	struct storage *st;
 
-	st = VTAILQ_LAST(&w->busyobj->fetch_obj->store, storagehead);
+	st = VTAILQ_LAST(&wrk->busyobj->fetch_obj->store, storagehead);
 	if (st == NULL)
 		return (0);
 
 	if (st->len == 0) {
-		VTAILQ_REMOVE(&w->busyobj->fetch_obj->store, st, list);
+		VTAILQ_REMOVE(&wrk->busyobj->fetch_obj->store, st, list);
 		STV_free(st);
 		return (0);
 	}
@@ -172,13 +172,13 @@ static struct vfp vfp_nop = {
  */
 
 struct storage *
-FetchStorage(struct worker *w, ssize_t sz)
+FetchStorage(struct worker *wrk, ssize_t sz)
 {
 	ssize_t l;
 	struct storage *st;
 	struct object *obj;
 
-	obj = w->busyobj->fetch_obj;
+	obj = wrk->busyobj->fetch_obj;
 	CHECK_OBJ_NOTNULL(obj, OBJECT_MAGIC);
 	st = VTAILQ_LAST(&obj->store, storagehead);
 	if (st != NULL && st->len < st->space)
@@ -189,9 +189,9 @@ FetchStorage(struct worker *w, ssize_t sz)
 		l = sz;
 	if (l == 0)
 		l = cache_param->fetch_chunksize;
-	st = STV_alloc(w, l);
+	st = STV_alloc(wrk, l);
 	if (st == NULL) {
-		(void)FetchError(w, "Could not get storage");
+		(void)FetchError(wrk, "Could not get storage");
 		return (NULL);
 	}
 	AZ(st->len);
@@ -225,20 +225,20 @@ fetch_number(const char *nbr, int radix)
 /*--------------------------------------------------------------------*/
 
 static int
-fetch_straight(struct worker *w, struct http_conn *htc, ssize_t cl)
+fetch_straight(struct worker *wrk, struct http_conn *htc, ssize_t cl)
 {
 	int i;
 
-	assert(w->busyobj->body_status == BS_LENGTH);
+	assert(wrk->busyobj->body_status == BS_LENGTH);
 
 	if (cl < 0) {
-		return (FetchError(w, "straight length field bogus"));
+		return (FetchError(wrk, "straight length field bogus"));
 	} else if (cl == 0)
 		return (0);
 
-	i = w->busyobj->vfp->bytes(w, htc, cl);
+	i = wrk->busyobj->vfp->bytes(wrk, htc, cl);
 	if (i <= 0)
-		return (FetchError(w, "straight insufficient bytes"));
+		return (FetchError(wrk, "straight insufficient bytes"));
 	return (0);
 }
 
@@ -249,28 +249,28 @@ fetch_straight(struct worker *w, struct http_conn *htc, ssize_t cl)
  */
 
 static int
-fetch_chunked(struct worker *w, struct http_conn *htc)
+fetch_chunked(struct worker *wrk, struct http_conn *htc)
 {
 	int i;
 	char buf[20];		/* XXX: 20 is arbitrary */
 	unsigned u;
 	ssize_t cl;
 
-	assert(w->busyobj->body_status == BS_CHUNKED);
+	assert(wrk->busyobj->body_status == BS_CHUNKED);
 	do {
 		/* Skip leading whitespace */
 		do {
-			if (HTC_Read(w, htc, buf, 1) <= 0)
+			if (HTC_Read(wrk, htc, buf, 1) <= 0)
 				return (-1);
 		} while (vct_islws(buf[0]));
 
 		if (!vct_ishex(buf[0]))
-			return (FetchError(w,"chunked header non-hex"));
+			return (FetchError(wrk,"chunked header non-hex"));
 
 		/* Collect hex digits, skipping leading zeros */
 		for (u = 1; u < sizeof buf; u++) {
 			do {
-				if (HTC_Read(w, htc, buf + u, 1) <= 0)
+				if (HTC_Read(wrk, htc, buf + u, 1) <= 0)
 					return (-1);
 			} while (u == 1 && buf[0] == '0' && buf[u] == '0');
 			if (!vct_ishex(buf[u]))
@@ -278,31 +278,31 @@ fetch_chunked(struct worker *w, struct http_conn *htc)
 		}
 
 		if (u >= sizeof buf)
-			return (FetchError(w,"chunked header too long"));
+			return (FetchError(wrk,"chunked header too long"));
 
 		/* Skip trailing white space */
 		while(vct_islws(buf[u]) && buf[u] != '\n')
-			if (HTC_Read(w, htc, buf + u, 1) <= 0)
+			if (HTC_Read(wrk, htc, buf + u, 1) <= 0)
 				return (-1);
 
 		if (buf[u] != '\n')
-			return (FetchError(w,"chunked header no NL"));
+			return (FetchError(wrk,"chunked header no NL"));
 
 		buf[u] = '\0';
 		cl = fetch_number(buf, 16);
 		if (cl < 0)
-			return (FetchError(w,"chunked header number syntax"));
+			return (FetchError(wrk,"chunked header number syntax"));
 
-		if (cl > 0 && w->busyobj->vfp->bytes(w, htc, cl) <= 0)
+		if (cl > 0 && wrk->busyobj->vfp->bytes(wrk, htc, cl) <= 0)
 			return (-1);
 
-		i = HTC_Read(w, htc, buf, 1);
+		i = HTC_Read(wrk, htc, buf, 1);
 		if (i <= 0)
 			return (-1);
-		if (buf[0] == '\r' && HTC_Read(w, htc, buf, 1) <= 0)
+		if (buf[0] == '\r' && HTC_Read(wrk, htc, buf, 1) <= 0)
 			return (-1);
 		if (buf[0] != '\n')
-			return (FetchError(w,"chunked tail no NL"));
+			return (FetchError(wrk,"chunked tail no NL"));
 	} while (cl > 0);
 	return (0);
 }
@@ -310,12 +310,12 @@ fetch_chunked(struct worker *w, struct http_conn *htc)
 /*--------------------------------------------------------------------*/
 
 static int
-fetch_eof(struct worker *w, struct http_conn *htc)
+fetch_eof(struct worker *wrk, struct http_conn *htc)
 {
 	int i;
 
-	assert(w->busyobj->body_status == BS_EOF);
-	i = w->busyobj->vfp->bytes(w, htc, SSIZE_MAX);
+	assert(wrk->busyobj->body_status == BS_EOF);
+	i = wrk->busyobj->vfp->bytes(wrk, htc, SSIZE_MAX);
 	if (i < 0)
 		return (-1);
 	return (0);
@@ -378,7 +378,7 @@ int
 FetchHdr(struct sess *sp, int need_host_hdr)
 {
 	struct vbc *vc;
-	struct worker *w;
+	struct worker *wrk;
 	struct http *hp;
 	int retry = -1;
 	int i;
@@ -386,9 +386,9 @@ FetchHdr(struct sess *sp, int need_host_hdr)
 
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 	CHECK_OBJ_NOTNULL(sp->wrk, WORKER_MAGIC);
-	w = sp->wrk;
-	CHECK_OBJ_NOTNULL(w->busyobj, BUSYOBJ_MAGIC);
-	htc = &w->busyobj->htc;
+	wrk = sp->wrk;
+	CHECK_OBJ_NOTNULL(wrk->busyobj, BUSYOBJ_MAGIC);
+	htc = &wrk->busyobj->htc;
 
 	AN(sp->director);
 	AZ(sp->wrk->obj);
@@ -398,7 +398,7 @@ FetchHdr(struct sess *sp, int need_host_hdr)
 		AN(sp->wrk->objcore->flags & OC_F_BUSY);
 	}
 
-	hp = w->bereq;
+	hp = wrk->bereq;
 
 	sp->wrk->busyobj->vbc = VDI_GetFd(NULL, sp);
 	if (sp->wrk->busyobj->vbc == NULL) {
@@ -418,12 +418,12 @@ FetchHdr(struct sess *sp, int need_host_hdr)
 		VDI_AddHostHeader(sp->wrk, vc);
 
 	(void)VTCP_blocking(vc->fd);	/* XXX: we should timeout instead */
-	WRW_Reserve(w, &vc->fd);
-	(void)http_Write(w, vc->vsl_id, hp, 0);	/* XXX: stats ? */
+	WRW_Reserve(wrk, &vc->fd);
+	(void)http_Write(wrk, vc->vsl_id, hp, 0);	/* XXX: stats ? */
 
 	/* Deal with any message-body the request might have */
 	i = FetchReqBody(sp);
-	if (WRW_FlushRelease(w) || i > 0) {
+	if (WRW_FlushRelease(wrk) || i > 0) {
 		WSP(sp, SLT_FetchError, "backend write error: %d (%s)",
 		    errno, strerror(errno));
 		VDI_CloseFd(sp->wrk, &sp->wrk->busyobj->vbc);
@@ -432,14 +432,14 @@ FetchHdr(struct sess *sp, int need_host_hdr)
 	}
 
 	/* Checkpoint the vsl.here */
-	WSL_Flush(w, 0);
+	WSL_Flush(wrk, 0);
 
 	/* XXX is this the right place? */
 	VSC_C_main->backend_req++;
 
 	/* Receive response */
 
-	HTC_Init(htc, w->ws, vc->fd, vc->vsl_id,
+	HTC_Init(htc, wrk->ws, vc->fd, vc->vsl_id,
 	    cache_param->http_resp_size,
 	    cache_param->http_resp_hdr_len);
 
@@ -470,9 +470,9 @@ FetchHdr(struct sess *sp, int need_host_hdr)
 		}
 	}
 
-	hp = w->beresp;
+	hp = wrk->beresp;
 
-	if (http_DissectResponse(w, htc, hp)) {
+	if (http_DissectResponse(wrk, htc, hp)) {
 		WSP(sp, SLT_FetchError, "http format error");
 		VDI_CloseFd(sp->wrk, &sp->wrk->busyobj->vbc);
 		/* XXX: other cleanup ? */
@@ -484,37 +484,39 @@ FetchHdr(struct sess *sp, int need_host_hdr)
 /*--------------------------------------------------------------------*/
 
 int
-FetchBody(struct worker *w, struct object *obj)
+FetchBody(struct worker *wrk, struct object *obj)
 {
 	int cls;
 	struct storage *st;
 	int mklen;
 	ssize_t cl;
 	struct http_conn *htc;
+	struct busyobj *bo;
 
-	CHECK_OBJ_NOTNULL(w, WORKER_MAGIC);
-	CHECK_OBJ_NOTNULL(w->busyobj, BUSYOBJ_MAGIC);
-	AZ(w->busyobj->fetch_obj);
-	CHECK_OBJ_NOTNULL(w->busyobj->vbc, VBC_MAGIC);
+	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
+	bo = wrk->busyobj;
+	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
+	AZ(bo->fetch_obj);
+	CHECK_OBJ_NOTNULL(bo->vbc, VBC_MAGIC);
 	CHECK_OBJ_NOTNULL(obj, OBJECT_MAGIC);
 	CHECK_OBJ_NOTNULL(obj->http, HTTP_MAGIC);
 
-	htc = &w->busyobj->htc;
+	htc = &bo->htc;
 
-	if (w->busyobj->vfp == NULL)
-		w->busyobj->vfp = &vfp_nop;
+	if (bo->vfp == NULL)
+		bo->vfp = &vfp_nop;
 
 	AssertObjCorePassOrBusy(obj->objcore);
 
-	AZ(w->busyobj->vgz_rx);
+	AZ(bo->vgz_rx);
 	AZ(VTAILQ_FIRST(&obj->store));
 
-	w->busyobj->fetch_obj = obj;
-	w->busyobj->fetch_failed = 0;
+	bo->fetch_obj = obj;
+	bo->fetch_failed = 0;
 
 	/* XXX: pick up estimate from objdr ? */
 	cl = 0;
-	switch (w->busyobj->body_status) {
+	switch (bo->body_status) {
 	case BS_NONE:
 		cls = 0;
 		mklen = 0;
@@ -524,25 +526,25 @@ FetchBody(struct worker *w, struct object *obj)
 		mklen = 1;
 		break;
 	case BS_LENGTH:
-		cl = fetch_number( w->busyobj->h_content_length, 10);
-		w->busyobj->vfp->begin(w, cl > 0 ? cl : 0);
-		cls = fetch_straight(w, htc, cl);
+		cl = fetch_number(bo->h_content_length, 10);
+		bo->vfp->begin(wrk, cl > 0 ? cl : 0);
+		cls = fetch_straight(wrk, htc, cl);
 		mklen = 1;
-		if (w->busyobj->vfp->end(w))
+		if (bo->vfp->end(wrk))
 			cls = -1;
 		break;
 	case BS_CHUNKED:
-		w->busyobj->vfp->begin(w, cl);
-		cls = fetch_chunked(w, htc);
+		bo->vfp->begin(wrk, cl);
+		cls = fetch_chunked(wrk, htc);
 		mklen = 1;
-		if (w->busyobj->vfp->end(w))
+		if (bo->vfp->end(wrk))
 			cls = -1;
 		break;
 	case BS_EOF:
-		w->busyobj->vfp->begin(w, cl);
-		cls = fetch_eof(w, htc);
+		bo->vfp->begin(wrk, cl);
+		cls = fetch_eof(wrk, htc);
 		mklen = 1;
-		if (w->busyobj->vfp->end(w))
+		if (bo->vfp->end(wrk))
 			cls = -1;
 		break;
 	case BS_ERROR:
@@ -554,44 +556,44 @@ FetchBody(struct worker *w, struct object *obj)
 		mklen = 0;
 		INCOMPL();
 	}
-	AZ(w->busyobj->vgz_rx);
+	AZ(bo->vgz_rx);
 
 	/*
 	 * It is OK for ->end to just leave the last storage segment
-	 * sitting on w->storage, we will always call vfp_nop_end()
+	 * sitting on wrk->storage, we will always call vfp_nop_end()
 	 * to get it trimmed or thrown out if empty.
 	 */
-	AZ(vfp_nop_end(w));
+	AZ(vfp_nop_end(wrk));
 
-	w->busyobj->fetch_obj = NULL;
+	bo->fetch_obj = NULL;
 
-	WSLB(w, SLT_Fetch_Body, "%u(%s) cls %d mklen %d",
-	    w->busyobj->body_status, body_status(w->busyobj->body_status),
+	WSLB(wrk, SLT_Fetch_Body, "%u(%s) cls %d mklen %d",
+	    bo->body_status, body_status(bo->body_status),
 	    cls, mklen);
 
-	if (w->busyobj->body_status == BS_ERROR) {
-		VDI_CloseFd(w, &w->busyobj->vbc);
+	if (bo->body_status == BS_ERROR) {
+		VDI_CloseFd(wrk, &bo->vbc);
 		return (__LINE__);
 	}
 
 	if (cls < 0) {
-		w->stats.fetch_failed++;
+		wrk->stats.fetch_failed++;
 		/* XXX: Wouldn't this store automatically be released ? */
 		while (!VTAILQ_EMPTY(&obj->store)) {
 			st = VTAILQ_FIRST(&obj->store);
 			VTAILQ_REMOVE(&obj->store, st, list);
 			STV_free(st);
 		}
-		VDI_CloseFd(w, &w->busyobj->vbc);
+		VDI_CloseFd(wrk, &bo->vbc);
 		obj->len = 0;
 		return (__LINE__);
 	}
-	AZ(w->busyobj->fetch_failed);
+	AZ(bo->fetch_failed);
 
-	if (cls == 0 && w->busyobj->should_close)
+	if (cls == 0 && bo->should_close)
 		cls = 1;
 
-	WSLB(w, SLT_Length, "%zd", obj->len);
+	WSLB(wrk, SLT_Length, "%zd", obj->len);
 
 	{
 	/* Sanity check fetch methods accounting */
@@ -600,7 +602,7 @@ FetchBody(struct worker *w, struct object *obj)
 		uu = 0;
 		VTAILQ_FOREACH(st, &obj->store, list)
 			uu += st->len;
-		if (w->busyobj->do_stream)
+		if (bo->do_stream)
 			/* Streaming might have started freeing stuff */
 			assert (uu <= obj->len);
 
@@ -610,14 +612,14 @@ FetchBody(struct worker *w, struct object *obj)
 
 	if (mklen > 0) {
 		http_Unset(obj->http, H_Content_Length);
-		http_PrintfHeader(w, w->busyobj->vbc->vsl_id, obj->http,
-		    "Content-Length: %jd", (intmax_t)obj->len);
+		http_PrintfHeader(wrk, bo->vbc->vsl_id, obj->http,
+		    "Content-Length: %zd", obj->len);
 	}
 
 	if (cls)
-		VDI_CloseFd(w, &w->busyobj->vbc);
+		VDI_CloseFd(wrk, &bo->vbc);
 	else
-		VDI_RecycleFd(w, &w->busyobj->vbc);
+		VDI_RecycleFd(wrk, &bo->vbc);
 
 	return (0);
 }



From geoff at varnish-cache.org  Mon Jan  9 20:51:47 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:51:47 +0100
Subject: [experimental-ims] ef0387e Fix escapes,
	and workaround man-page formatting issues
Message-ID: <E1RkMC3-0006nX-IO@project.varnish-software.com>

commit ef0387ea8c123a757e36cf6ef60985440e56a37d
Author: Andreas Plesner Jacobsen <apj at mutt.dk>
Date:   Sat Sep 10 15:16:37 2011 +0200

    Fix escapes, and workaround man-page formatting issues

diff --git a/doc/sphinx/reference/vcl.rst b/doc/sphinx/reference/vcl.rst
index 517f4fb..e3169ea 100644
--- a/doc/sphinx/reference/vcl.rst
+++ b/doc/sphinx/reference/vcl.rst
@@ -288,8 +288,8 @@ Probes take the following parameters:
   Specify a URL to request from the backend.
   Defaults to "/".
 .request
-  Specify a full HTTP request using multiple strings.
-  .request will have \r\n automatically inserted after every string.
+  Specify a full HTTP request using multiple strings. .request will
+  have \\r\\n automatically inserted after every string.
   If specified, .request will take precedence over .url.
 .window
   How many of the latest polls we examine to determine backend health.



From geoff at varnish-cache.org  Mon Jan  9 20:52:25 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:25 +0100
Subject: [experimental-ims] 78f18e8 Give the worker a "fetch_obj" to avoid
	passing it around as a separate parameter all the time.
Message-ID: <E1RkMCf-0007uR-Qu@project.varnish-software.com>

commit 78f18e855bf04ddc1cef31405bdc6f8e3e58b6cc
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Oct 24 12:54:55 2011 +0000

    Give the worker a "fetch_obj" to avoid passing it around as a separate
    parameter all the time.
    
    Also decontaminate FetchStorage to not need session

diff --git a/bin/varnishd/cache.h b/bin/varnishd/cache.h
index a8dff7b..46d4067 100644
--- a/bin/varnishd/cache.h
+++ b/bin/varnishd/cache.h
@@ -339,6 +339,7 @@ struct worker {
 
 	/* Fetch stuff */
 	struct vbc		*vbc;
+	struct object		*fetch_obj;
 	enum body_status	body_status;
 	struct vfp		*vfp;
 	struct vgz		*vgz_rx;
@@ -700,7 +701,7 @@ int EXP_Touch(struct objcore *oc);
 int EXP_NukeOne(struct worker *w, struct lru *lru);
 
 /* cache_fetch.c */
-struct storage *FetchStorage(const struct sess *sp, ssize_t sz);
+struct storage *FetchStorage(struct worker *w, ssize_t sz);
 int FetchHdr(struct sess *sp);
 int FetchBody(struct sess *sp, struct object *obj);
 int FetchReqBody(struct sess *sp);
@@ -950,8 +951,7 @@ int RFC2616_Do_Cond(const struct sess *sp);
 /* stevedore.c */
 struct object *STV_NewObject(struct sess *sp, const char *hint, unsigned len,
     struct exp *, uint16_t nhttp);
-struct storage *STV_alloc(struct worker *w, const struct object *obj,
-    size_t size);
+struct storage *STV_alloc(struct worker *w, size_t size);
 void STV_trim(struct storage *st, size_t size);
 void STV_free(struct storage *st);
 void STV_open(void);
diff --git a/bin/varnishd/cache_esi_fetch.c b/bin/varnishd/cache_esi_fetch.c
index 5d8f8dc..00c7432 100644
--- a/bin/varnishd/cache_esi_fetch.c
+++ b/bin/varnishd/cache_esi_fetch.c
@@ -71,7 +71,7 @@ vfp_esi_bytes_uu(struct sess *sp, struct http_conn *htc, ssize_t bytes)
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 
 	while (bytes > 0) {
-		st = FetchStorage(sp, 0);
+		st = FetchStorage(sp->wrk, 0);
 		if (st == NULL)
 			return (-1);
 		w = vef_read(htc,
@@ -368,7 +368,7 @@ vfp_esi_end(struct sess *sp)
 		l = VSB_len(vsb);
 		assert(l > 0);
 		/* XXX: This is a huge waste of storage... */
-		sp->obj->esidata = STV_alloc(sp->wrk, sp->obj, l);
+		sp->obj->esidata = STV_alloc(sp->wrk, l);
 		XXXAN(sp->obj->esidata);
 		memcpy(sp->obj->esidata->ptr, VSB_data(vsb), l);
 		sp->obj->esidata->len = l;
diff --git a/bin/varnishd/cache_fetch.c b/bin/varnishd/cache_fetch.c
index ccc2ec3..d71b2ec 100644
--- a/bin/varnishd/cache_fetch.c
+++ b/bin/varnishd/cache_fetch.c
@@ -66,7 +66,7 @@ vfp_nop_begin(struct sess *sp, size_t estimate)
 		WSP(sp, SLT_Debug, "Fetch %d byte segments:", fetchfrag);
 	}
 	if (estimate > 0)
-		(void)FetchStorage(sp, estimate);
+		(void)FetchStorage(sp->wrk, estimate);
 }
 
 /*--------------------------------------------------------------------
@@ -86,7 +86,7 @@ vfp_nop_bytes(struct sess *sp, struct http_conn *htc, ssize_t bytes)
 	struct storage *st;
 
 	while (bytes > 0) {
-		st = FetchStorage(sp, 0);
+		st = FetchStorage(sp->wrk, 0);
 		if (st == NULL) {
 			htc->error = "Could not get storage";
 			return (-1);
@@ -145,12 +145,15 @@ static struct vfp vfp_nop = {
  */
 
 struct storage *
-FetchStorage(const struct sess *sp, ssize_t sz)
+FetchStorage(struct worker *w, ssize_t sz)
 {
 	ssize_t l;
 	struct storage *st;
+	struct object *obj;
 
-	st = VTAILQ_LAST(&sp->obj->store, storagehead);
+	obj = w->fetch_obj;
+	CHECK_OBJ_NOTNULL(obj, OBJECT_MAGIC);
+	st = VTAILQ_LAST(&obj->store, storagehead);
 	if (st != NULL && st->len < st->space)
 		return (st);
 
@@ -159,13 +162,13 @@ FetchStorage(const struct sess *sp, ssize_t sz)
 		l = sz;
 	if (l == 0)
 		l = params->fetch_chunksize * 1024LL;
-	st = STV_alloc(sp->wrk, sp->obj, l);
+	st = STV_alloc(w, l);
 	if (st == NULL) {
 		errno = ENOMEM;
 		return (NULL);
 	}
 	AZ(st->len);
-	VTAILQ_INSERT_TAIL(&sp->obj->store, st, list);
+	VTAILQ_INSERT_TAIL(&obj->store, st, list);
 	return (st);
 }
 
@@ -487,6 +490,7 @@ FetchBody(struct sess *sp, struct object *obj)
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 	CHECK_OBJ_NOTNULL(sp->wrk, WORKER_MAGIC);
 	w = sp->wrk;
+	AZ(w->fetch_obj);
 	CHECK_OBJ_NOTNULL(obj, OBJECT_MAGIC);
 	CHECK_OBJ_NOTNULL(obj->http, HTTP_MAGIC);
 
@@ -498,6 +502,9 @@ FetchBody(struct sess *sp, struct object *obj)
 
 	AZ(w->vgz_rx);
 	AZ(VTAILQ_FIRST(&obj->store));
+
+	w->fetch_obj = obj;
+
 	switch (w->body_status) {
 	case BS_NONE:
 		cls = 0;
@@ -541,6 +548,8 @@ FetchBody(struct sess *sp, struct object *obj)
 	 */
 	AZ(vfp_nop_end(sp));
 
+	w->fetch_obj = NULL;
+
 	WSL(w, SLT_Fetch_Body, w->vbc->vsl_id, "%u(%s) cls %d mklen %u",
 	    w->body_status, body_status(w->body_status),
 	    cls, mklen);
diff --git a/bin/varnishd/cache_gzip.c b/bin/varnishd/cache_gzip.c
index 787ff16..e40676c 100644
--- a/bin/varnishd/cache_gzip.c
+++ b/bin/varnishd/cache_gzip.c
@@ -259,7 +259,7 @@ VGZ_ObufStorage(const struct sess *sp, struct vgz *vg)
 {
 	struct storage *st;
 
-	st = FetchStorage(sp, 0);
+	st = FetchStorage(sp->wrk, 0);
 	if (st == NULL)
 		return (-1);
 
@@ -615,7 +615,7 @@ vfp_testgzip_bytes(struct sess *sp, struct http_conn *htc, ssize_t bytes)
 	CHECK_OBJ_NOTNULL(vg, VGZ_MAGIC);
 	AZ(vg->vz.avail_in);
 	while (bytes > 0) {
-		st = FetchStorage(sp, 0);
+		st = FetchStorage(sp->wrk, 0);
 		if (st == NULL) {
 			htc->error = "Could not get storage";
 			return (-1);
diff --git a/bin/varnishd/storage/stevedore.c b/bin/varnishd/storage/stevedore.c
index 93e2bf8..4048836 100644
--- a/bin/varnishd/storage/stevedore.c
+++ b/bin/varnishd/storage/stevedore.c
@@ -371,10 +371,10 @@ STV_Freestore(struct object *o)
 /*-------------------------------------------------------------------*/
 
 struct storage *
-STV_alloc(struct worker *w, const struct object *obj, size_t size)
+STV_alloc(struct worker *w, size_t size)
 {
 
-	return (stv_alloc(w, obj, size));
+	return (stv_alloc(w, w->fetch_obj, size));
 }
 
 void



From geoff at varnish-cache.org  Mon Jan  9 20:51:55 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:51:55 +0100
Subject: [experimental-ims] ea74343 A little bit of spit and polish...
Message-ID: <E1RkMCB-0006ze-VY@project.varnish-software.com>

commit ea7434334778be8c38052b6090c25abc63bcc37b
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sat Sep 17 20:25:20 2011 +0000

    A little bit of spit and polish...

diff --git a/bin/varnishd/cache_session.c b/bin/varnishd/cache_session.c
index 6b34928..a930c07 100644
--- a/bin/varnishd/cache_session.c
+++ b/bin/varnishd/cache_session.c
@@ -319,27 +319,36 @@ SES_Delete(struct sess *sp, const char *reason)
 		Lck_Unlock(&stat_mtx);
 		free(sm);
 		Lck_Lock(&pp->mtx);
-		sesspool->nsess--;
+		pp->nsess--;
 		Lck_Unlock(&pp->mtx);
 	} else {
 		/* Clean and prepare for reuse */
 		ses_setup(sm);
 		Lck_Lock(&pp->mtx);
-		VTAILQ_INSERT_HEAD(&sesspool->freelist, sm, list);
+		VTAILQ_INSERT_HEAD(&pp->freelist, sm, list);
 		Lck_Unlock(&pp->mtx);
 	}
 }
 
 /*--------------------------------------------------------------------*/
 
+static struct sesspool *
+SES_NewPool(unsigned maxsess)
+{
+	struct sesspool *sp;
+
+	ALLOC_OBJ(sp, SESSPOOL_MAGIC);
+	VTAILQ_INIT(&sp->freelist);
+	Lck_New(&sp->mtx, lck_sessmem);
+	sp->maxsess = maxsess;
+	return (sp);
+}
+
+
 void
 SES_Init()
 {
 
-	ALLOC_OBJ(sesspool, SESSPOOL_MAGIC);
-	VTAILQ_INIT(&sesspool->freelist);
-	Lck_New(&sesspool->mtx, lck_sessmem);
-	sesspool->maxsess = params->max_sess;
-
+	sesspool = SES_NewPool(params->max_sess);
 	Lck_New(&stat_mtx, lck_stat);
 }



From geoff at varnish-cache.org  Mon Jan  9 20:52:00 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:00 +0100
Subject: [experimental-ims] abc5dd2 Go over this code once more,
	and fix various nits.
Message-ID: <E1RkMCG-00079R-L0@project.varnish-software.com>

commit abc5dd2a72249fe49d3f1071d8268d3708d7fd3e
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Sep 20 22:00:12 2011 +0000

    Go over this code once more, and fix various nits.

diff --git a/bin/varnishtest/vtc_http.c b/bin/varnishtest/vtc_http.c
index 8f96577..e6aaf33 100644
--- a/bin/varnishtest/vtc_http.c
+++ b/bin/varnishtest/vtc_http.c
@@ -107,7 +107,7 @@ synth_body(const char *len, int rnd)
 	AN(len);
 	i = strtoul(len, NULL, 0);
 	assert(i > 0);
-	b = malloc(i + 1);
+	b = malloc(i + 1L);
 	AN(b);
 	l = k = '!';
 	for (j = 0; j < i; j++) {
@@ -333,7 +333,7 @@ http_splitheader(struct http *hp, int req)
  */
 
 static int
-http_rxchar_eof(struct http *hp, int n)
+http_rxchar(struct http *hp, int n, int eof)
 {
 	int i;
 	struct pollfd pfd[1];
@@ -343,21 +343,25 @@ http_rxchar_eof(struct http *hp, int n)
 		pfd[0].events = POLLIN;
 		pfd[0].revents = 0;
 		i = poll(pfd, 1, hp->timeout);
-		if (i < 0)
+		if (i == 0)
 			vtc_log(hp->vl, 0, "HTTP rx timeout (fd:%d %u ms)",
 			    hp->fd, hp->timeout);
 		if (i < 0)
 			vtc_log(hp->vl, 0, "HTTP rx failed (fd:%d poll: %s)",
 			    hp->fd, strerror(errno));
 		assert(i > 0);
-		if (pfd[0].revents & ~POLLIN)
-			vtc_log(hp->vl, 4, "HTTP rx poll (fd:%d revents: %x)",
-			    hp->fd, pfd[0].revents);
 		assert(hp->prxbuf + n < hp->nrxbuf);
 		i = read(hp->fd, hp->rxbuf + hp->prxbuf, n);
-		if (i == 0)
+		if (!(pfd[0].revents & POLLIN))
+			vtc_log(hp->vl, 4,
+			    "HTTP rx poll (fd:%d revents: %x n=%d, i=%d)",
+			    hp->fd, pfd[0].revents, n, i);
+		if (i == 0 && eof)
 			return (i);
-		if (i <= 0)
+		if (i == 0)
+			vtc_log(hp->vl, 0, "HTTP rx EOF (fd:%d read: %s)",
+			    hp->fd, strerror(errno));
+		if (i < 0)
 			vtc_log(hp->vl, 0, "HTTP rx failed (fd:%d read: %s)",
 			    hp->fd, strerror(errno));
 		hp->prxbuf += i;
@@ -367,17 +371,6 @@ http_rxchar_eof(struct http *hp, int n)
 	return (1);
 }
 
-static void
-http_rxchar(struct http *hp, int n)
-{
-	int i;
-
-	i = http_rxchar_eof(hp, n);
-	if (i <= 0)
-		vtc_log(hp->vl, 0, "HTTP rx failed (%s)", strerror(errno));
-	assert(i > 0);
-}
-
 static int
 http_rxchunk(struct http *hp)
 {
@@ -386,7 +379,7 @@ http_rxchunk(struct http *hp)
 
 	l = hp->prxbuf;
 	do
-		http_rxchar(hp, 1);
+		(void)http_rxchar(hp, 1, 0);
 	while (hp->rxbuf[hp->prxbuf - 1] != '\n');
 	vtc_dump(hp->vl, 4, "len", hp->rxbuf + l, -1);
 	i = strtoul(hp->rxbuf + l, &q, 16);
@@ -400,12 +393,12 @@ http_rxchunk(struct http *hp)
 	assert(*q == '\0' || vct_islws(*q));
 	hp->prxbuf = l;
 	if (i > 0) {
-		http_rxchar(hp, i);
+		(void)http_rxchar(hp, i, 0);
 		vtc_dump(hp->vl, 4, "chunk",
 		    hp->rxbuf + l, i);
 	}
 	l = hp->prxbuf;
-	http_rxchar(hp, 2);
+	(void)http_rxchar(hp, 2, 0);
 	if(!vct_iscrlf(hp->rxbuf[l]))
 		vtc_log(hp->vl, 0,
 		    "Wrong chunk tail[0] = %02x",
@@ -433,7 +426,7 @@ http_swallow_body(struct http *hp, char * const *hh, int body)
 	p = http_find_header(hh, "content-length");
 	if (p != NULL) {
 		l = strtoul(p, NULL, 0);
-		http_rxchar(hp, l);
+		(void)http_rxchar(hp, l, 0);
 		vtc_dump(hp->vl, 4, "body", hp->body, l);
 		hp->bodyl = l;
 		sprintf(hp->bodylen, "%d", l);
@@ -452,7 +445,7 @@ http_swallow_body(struct http *hp, char * const *hh, int body)
 	if (body) {
 		hp->body = hp->rxbuf + hp->prxbuf;
 		do  {
-			i = http_rxchar_eof(hp, 1);
+			i = http_rxchar(hp, 1, 1);
 			ll += i;
 		} while (i > 0);
 		vtc_dump(hp->vl, 4, "rxeof", hp->body, ll);
@@ -475,7 +468,7 @@ http_rxhdr(struct http *hp)
 	hp->prxbuf = 0;
 	hp->body = NULL;
 	while (1) {
-		http_rxchar(hp, 1);
+		(void)http_rxchar(hp, 1, 0);
 		p = hp->rxbuf + hp->prxbuf - 1;
 		for (i = 0; p > hp->rxbuf; p--) {
 			if (*p != '\n')
@@ -533,7 +526,7 @@ cmd_http_rxresp(CMD_ARGS)
 
 #define TRUST_ME(ptr)   ((void*)(uintptr_t)(ptr))
 
-#define OVERHEAD 64
+#define OVERHEAD 64L
 
 
 static void
@@ -942,7 +935,7 @@ cmd_http_sendhex(CMD_ARGS)
 		if (!vct_ishex(buf[0]) || !vct_ishex(buf[1]))
 			vtc_log(hp->vl, 0, "Illegal Hex char \"%c%c\"",
 			    buf[0], buf[1]);
-		p[i] = strtoul(buf, NULL, 16);
+		p[i] = (uint8_t)strtoul(buf, NULL, 16);
 	}
 	vtc_hexdump(hp->vl, 4, "sendhex", (void*)p, i);
 	j = write(hp->fd, p, i);



From geoff at varnish-cache.org  Mon Jan  9 20:52:26 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:26 +0100
Subject: [experimental-ims] 6460036 Minor adjustment to asserts
Message-ID: <E1RkMCg-0007wP-LO@project.varnish-software.com>

commit 6460036898caf29c6be27e5bec9722f27b5edf44
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Oct 25 09:39:12 2011 +0000

    Minor adjustment to asserts

diff --git a/bin/varnishtest/vtc_log.c b/bin/varnishtest/vtc_log.c
index 313d3ab..651bd89 100644
--- a/bin/varnishtest/vtc_log.c
+++ b/bin/varnishtest/vtc_log.c
@@ -132,7 +132,7 @@ vtc_log(struct vtclog *vl, int lvl, const char *fmt, ...)
 	tx = VTIM_mono() - t0;
 	AZ(pthread_mutex_lock(&vl->mtx));
 	vl->act = 1;
-	assert(lvl < NLEAD);
+	assert(lvl < (int)NLEAD);
 	VSB_clear(vl->vsb);
 	VSB_printf(vl->vsb, "%s %-4s %4.1f ",
 	    lead[lvl < 0 ? 1: lvl], vl->id, tx);
@@ -170,8 +170,8 @@ vtc_dump(struct vtclog *vl, int lvl, const char *pfx, const char *str, int len)
 
 	CHECK_OBJ_NOTNULL(vl, VTCLOG_MAGIC);
 	tx = VTIM_mono() - t0;
-	assert(lvl < NLEAD);
 	assert(lvl >= 0);
+	assert(lvl < NLEAD);
 	AZ(pthread_mutex_lock(&vl->mtx));
 	vl->act = 1;
 	VSB_clear(vl->vsb);
@@ -237,8 +237,8 @@ vtc_hexdump(struct vtclog *vl, int lvl, const char *pfx, const unsigned char *st
 	CHECK_OBJ_NOTNULL(vl, VTCLOG_MAGIC);
 	tx = VTIM_mono() - t0;
 	assert(len >= 0);
-	assert(lvl < NLEAD);
 	assert(lvl >= 0);
+	assert(lvl < NLEAD);
 	AZ(pthread_mutex_lock(&vl->mtx));
 	vl->act = 1;
 	VSB_clear(vl->vsb);



From geoff at varnish-cache.org  Mon Jan  9 20:52:47 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:47 +0100
Subject: [experimental-ims] 9a5b477 Correct indent level of closing bracket in
	pan_wrk()
Message-ID: <E1RkMD1-00008S-2r@project.varnish-software.com>

commit 9a5b4779043aa90af7d5b5f4a13d6e34f787ecb5
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Thu Dec 1 12:53:51 2011 +0100

    Correct indent level of closing bracket in pan_wrk()

diff --git a/bin/varnishd/cache/cache_panic.c b/bin/varnishd/cache/cache_panic.c
index cf39489..798ba53 100644
--- a/bin/varnishd/cache/cache_panic.c
+++ b/bin/varnishd/cache/cache_panic.c
@@ -203,7 +203,7 @@ pan_wrk(const struct worker *wrk)
 		pan_http("beresp", wrk->beresp, 4);
 	if (wrk->resp->ws != NULL)
 		pan_http("resp", wrk->resp, 4);
-	VSB_printf(pan_vsp, "    },\n");
+	VSB_printf(pan_vsp, "  },\n");
 }
 
 static void



From geoff at varnish-cache.org  Mon Jan  9 20:52:46 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:46 +0100
Subject: [experimental-ims] 4c4ba07 remove minor typos
Message-ID: <E1RkMD0-00007H-JJ@project.varnish-software.com>

commit 4c4ba072982c4f18c4455f50fe77e3be1c8149a0
Author: Lasse Karstensen <lasse at varnish-software.com>
Date:   Wed Nov 30 15:07:59 2011 +0100

    remove minor typos

diff --git a/doc/sphinx/tutorial/cookies.rst b/doc/sphinx/tutorial/cookies.rst
index 2ee14cd..ab59e21 100644
--- a/doc/sphinx/tutorial/cookies.rst
+++ b/doc/sphinx/tutorial/cookies.rst
@@ -10,7 +10,7 @@ the backend.
 
 This can be overly conservative. A lot of sites use Google Analytics
 (GA) to analyze their traffic. GA sets a cookie to track you. This
-cookie is used by the client side java script and is therefore of no
+cookie is used by the client side javascript and is therefore of no
 interest to the server. 
 
 For a lot of web application it makes sense to completely disregard the
@@ -62,4 +62,4 @@ cookies named COOKIE1 and COOKIE2 and you can marvel at it::
   }
 
 The example is taken from the Varnish Wiki, where you can find other
-scary examples of what can be done i VCL.
+scary examples of what can be done in VCL.



From geoff at varnish-cache.org  Mon Jan  9 20:51:56 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:51:56 +0100
Subject: [experimental-ims] dd0e30e Finish polishing the session memory
	management:
Message-ID: <E1RkMCC-00070n-Cx@project.varnish-software.com>

commit dd0e30ee0b53f1a2a0f2f7de460a2dd16431efc3
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sun Sep 18 08:29:58 2011 +0000

    Finish polishing the session memory management:
    
    Also check if the http_max_hdr param changes.
    
    Describe http_max_hdr as per pool.
    
    Add SES_DeletePool() function.

diff --git a/bin/varnishd/cache.h b/bin/varnishd/cache.h
index 122ab34..2a7578d 100644
--- a/bin/varnishd/cache.h
+++ b/bin/varnishd/cache.h
@@ -849,7 +849,6 @@ unsigned WRW_WriteH(struct worker *w, const txt *hh, const char *suf);
 void WRW_Sendfile(struct worker *w, int fd, off_t off, unsigned len);
 #endif  /* SENDFILE_WORKS */
 
-
 /* cache_session.c [SES] */
 struct sess *SES_New(struct worker *wrk, struct sesspool *pp);
 struct sess *SES_Alloc(void);
@@ -857,7 +856,7 @@ void SES_Close(struct sess *sp, const char *reason);
 void SES_Delete(struct sess *sp, const char *reason);
 void SES_Charge(struct sess *sp);
 struct sesspool *SES_NewPool(void);
-
+void SES_DeletePool(struct sesspool *sp, struct worker *wrk);
 
 /* cache_shmlog.c */
 void VSL_Init(void);
diff --git a/bin/varnishd/cache_session.c b/bin/varnishd/cache_session.c
index 7012263..23d15e7 100644
--- a/bin/varnishd/cache_session.c
+++ b/bin/varnishd/cache_session.c
@@ -51,11 +51,14 @@ struct sessmem {
 #define SESSMEM_MAGIC		0x555859c5
 
 	struct sesspool		*pool;
-	struct sess		sess;
+
 	unsigned		workspace;
+	uint16_t		nhttp;
 	void			*wsp;
 	struct http		*http[2];
 	VTAILQ_ENTRY(sessmem)	list;
+
+	struct sess		sess;
 };
 
 struct sesspool {
@@ -120,14 +123,20 @@ ses_sm_alloc(void)
 
 	sm = (void*)p;
 	p += sizeof *sm;
+
 	sm->magic = SESSMEM_MAGIC;
 	sm->workspace = nws;
+	sm->nhttp = nhttp;
+
 	sm->http[0] = HTTP_create(p, nhttp);
 	p += hl;
+
 	sm->http[1] = HTTP_create(p, nhttp);
 	p += hl;
+
 	sm->wsp = p;
 	p += nws;
+
 	assert(p == q);
 
 	return (sm);
@@ -318,6 +327,7 @@ SES_Delete(struct sess *sp, const char *reason)
 	    b->fetch, b->hdrbytes, b->bodybytes);
 
 	if (sm->workspace != params->sess_workspace ||
+	    sm->nhttp != (uint16_t)params->http_max_hdr ||
 	    pp->nsess > params->max_sess) {
 		free(sm);
 		Lck_Lock(&pp->mtx);
@@ -341,7 +351,7 @@ SES_Delete(struct sess *sp, const char *reason)
 }
 
 /*--------------------------------------------------------------------
- * Create a new pool to allocate from
+ * Create and delete pools
  */
 
 struct sesspool *
@@ -355,3 +365,25 @@ SES_NewPool(void)
 	Lck_New(&sp->mtx, lck_sessmem);
 	return (sp);
 }
+
+void
+SES_DeletePool(struct sesspool *sp, struct worker *wrk)
+{
+	struct sessmem *sm;
+
+	CHECK_OBJ_NOTNULL(sp, SESSPOOL_MAGIC);
+	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
+	Lck_Lock(&sp->mtx);
+	while (!VTAILQ_EMPTY(&sp->freelist)) {
+		sm = VTAILQ_FIRST(&sp->freelist);
+		CHECK_OBJ_NOTNULL(sm, SESSMEM_MAGIC);
+		VTAILQ_REMOVE(&sp->freelist, sm, list);
+		FREE_OBJ(sm);
+		wrk->stats.sessmem_free++;
+		sp->nsess--;
+	}
+	AZ(sp->nsess);
+	Lck_Unlock(&sp->mtx);
+	Lck_Delete(&sp->mtx);
+	FREE_OBJ(sp);
+}
diff --git a/bin/varnishd/mgt_param.c b/bin/varnishd/mgt_param.c
index b9067e7..ec38b11 100644
--- a/bin/varnishd/mgt_param.c
+++ b/bin/varnishd/mgt_param.c
@@ -777,7 +777,7 @@ static const struct parspec input_parspec[] = {
 		"off", "bool" },
 	{ "session_max", tweak_uint,
 		&master.max_sess, 1000, UINT_MAX,
-		"Maximum number of sessions we will allocate "
+		"Maximum number of sessions we will allocate from one pool "
 		"before just dropping connections.\n"
 		"This is mostly an anti-DoS measure, and setting it plenty "
 		"high should not hurt, as long as you have the memory for "



From geoff at varnish-cache.org  Mon Jan  9 20:52:29 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:29 +0100
Subject: [experimental-ims] 5f45e4c Add support for JSON output format
Message-ID: <E1RkMCj-000828-2y@project.varnish-software.com>

commit 5f45e4ca4996027638a0b28949aed68ffe7eaf8c
Author: Lasse Karstensen <lasse at varnish-software.com>
Date:   Thu Nov 3 19:02:06 2011 +0100

    Add support for JSON output format

diff --git a/bin/varnishstat/varnishstat.c b/bin/varnishstat/varnishstat.c
index 3c6c4b5..00ddda8 100644
--- a/bin/varnishstat/varnishstat.c
+++ b/bin/varnishstat/varnishstat.c
@@ -82,6 +82,60 @@ do_xml(struct VSM_data *vd)
 	printf("</varnishstat>\n");
 }
 
+
+/*--------------------------------------------------------------------*/
+
+struct json_priv {
+	int first;
+};
+
+static int
+do_json_cb(void *priv, const struct VSC_point * const pt)
+{
+	uint64_t val;
+	struct json_priv *jp;
+	jp = priv;
+
+	assert(!strcmp(pt->fmt, "uint64_t"));
+	val = *(const volatile uint64_t*)pt->ptr;
+
+	if (jp->first ) jp->first = 0; else printf(",\n");
+	printf("\t\"%s\": {", pt->name);
+	if (strcmp(pt->class, ""))
+		printf("\t\t\"type\": \"%s\",\n", pt->class);
+	if (strcmp(pt->ident, ""))
+		printf("\t\t\"ident\": \"%s\",\n", pt->ident);
+	printf("\"value\": %ju, ", val);
+
+	printf("\"flag\": \"%c\",", pt->flag);
+	printf("\"description\": \"%s\"", pt->desc);
+	printf("}"); // closes name section.
+	if (jp->first) printf("\n");
+	return (0);
+}
+
+static void
+do_json(struct VSM_data *vd)
+{
+	/// TODO: support jsonp?
+	char time_stamp[20];
+	time_t now;
+
+	struct json_priv jp;
+
+	memset(&jp, 0, sizeof jp);
+	jp.first = 1;
+
+	printf("{\n"); 
+	now = time(NULL);
+
+	(void)strftime(time_stamp, 20, "%Y-%m-%dT%H:%M:%S", localtime(&now));
+	printf("\t\"timestamp\": \"%s\",\n", time_stamp);
+	(void)VSC_Iter(vd, do_json_cb, &jp);
+	printf("\n}");
+}
+
+
 /*--------------------------------------------------------------------*/
 
 struct once_priv {
@@ -89,6 +143,7 @@ struct once_priv {
 	int pad;
 };
 
+
 static int
 do_once_cb(void *priv, const struct VSC_point * const pt)
 {
@@ -191,12 +246,12 @@ main(int argc, char * const *argv)
 	int c;
 	struct VSM_data *vd;
 	const struct VSC_C_main *VSC_C_main;
-	int delay = 1, once = 0, xml = 0;
+	int delay = 1, once = 0, xml = 0, json = 0;
 
 	vd = VSM_New();
 	VSC_Setup(vd);
 
-	while ((c = getopt(argc, argv, VSC_ARGS "1f:lVw:x")) != -1) {
+	while ((c = getopt(argc, argv, VSC_ARGS "1f:lVw:xj")) != -1) {
 		switch (c) {
 		case '1':
 			once = 1;
@@ -215,6 +270,9 @@ main(int argc, char * const *argv)
 		case 'x':
 			xml = 1;
 			break;
+		case 'j':
+			json = 1;
+			break;
 		default:
 			if (VSC_Arg(vd, c, optarg) > 0)
 				break;
@@ -229,6 +287,8 @@ main(int argc, char * const *argv)
 
 	if (xml)
 		do_xml(vd);
+	else if (json)
+		do_json(vd);
 	else if (once)
 		do_once(vd, VSC_C_main);
 	else



From geoff at varnish-cache.org  Mon Jan  9 20:52:18 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:18 +0100
Subject: [experimental-ims] 1a80a01 Solaris sandbox changes.
Message-ID: <E1RkMCY-0007fN-27@project.varnish-software.com>

commit 1a80a0197d49b231d4fd97d9b01609c187db569b
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Oct 12 08:29:41 2011 +0000

    Solaris sandbox changes.
    
    Submitted by:	Nils Goroll

diff --git a/bin/varnishd/mgt.h b/bin/varnishd/mgt.h
index 2b17052..1656af8 100644
--- a/bin/varnishd/mgt.h
+++ b/bin/varnishd/mgt.h
@@ -70,6 +70,7 @@ void mgt_sandbox(void);
 #ifdef HAVE_SETPPRIV
 void mgt_sandbox_solaris_init(void);
 void mgt_sandbox_solaris_fini(void);
+void mgt_sandbox_solaris_privsep(void);
 #endif
 
 /* mgt_shmem.c */
diff --git a/bin/varnishd/mgt_sandbox.c b/bin/varnishd/mgt_sandbox.c
index 3aba7cc..8ac827d 100644
--- a/bin/varnishd/mgt_sandbox.c
+++ b/bin/varnishd/mgt_sandbox.c
@@ -63,17 +63,17 @@
 void
 mgt_sandbox(void)
 {
-
 #ifdef HAVE_SETPPRIV
 	mgt_sandbox_solaris_init();
-#endif
-
+	mgt_sandbox_solaris_privsep();
+#else
 	if (geteuid() == 0) {
 		XXXAZ(setgid(params->gid));
 		XXXAZ(setuid(params->uid));
 	} else {
 		REPORT0(LOG_INFO, "Not running as root, no priv-sep");
 	}
+#endif
 
 	/* On Linux >= 2.4, you need to set the dumpable flag
 	   to get core dumps after you have done a setuid. */
diff --git a/bin/varnishd/mgt_sandbox_solaris.c b/bin/varnishd/mgt_sandbox_solaris.c
index 5c50c83..534f609 100644
--- a/bin/varnishd/mgt_sandbox_solaris.c
+++ b/bin/varnishd/mgt_sandbox_solaris.c
@@ -40,6 +40,7 @@
 #include <stdio.h>
 #include <string.h>
 #include <syslog.h>
+#include <unistd.h>
 
 #include "mgt.h"
 
@@ -153,6 +154,20 @@ mgt_sandbox_solaris_init(void)
 	priv_freeset(priv_all);
 }
 
+void
+mgt_sandbox_solaris_privsep(void)
+{
+	if (priv_ineffect(PRIV_PROC_SETID)) {
+                if (getgid() != params->gid)
+                        XXXAZ(setgid(params->gid));
+                if (getuid() != params->uid)
+                        XXXAZ(setuid(params->uid));
+        } else {
+                REPORT(LOG_INFO, "Privilege %s missing, will not change uid/gid",
+		    PRIV_PROC_SETID);
+        }
+}
+
 /* 
  * Waive most privileges in the child
  *



From geoff at varnish-cache.org  Mon Jan  9 20:52:41 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:41 +0100
Subject: [experimental-ims] 07c4179 ...and make it all work again.
Message-ID: <E1RkMCv-0008NF-30@project.varnish-software.com>

commit 07c41790c25e31914a5f6e6297f3591537efd9fd
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sun Nov 20 23:38:24 2011 +0000

    ...and make it all work again.
    
    At least "work" in the sense that varnishtest does not obviously
    break, but since it doesn't test the utils at all (but does use
    the varnishapi) more work & testing will be needed.

diff --git a/bin/varnishd/common/heritage.h b/bin/varnishd/common/heritage.h
index 9169bd5..b62a6ca 100644
--- a/bin/varnishd/common/heritage.h
+++ b/bin/varnishd/common/heritage.h
@@ -67,7 +67,6 @@ struct heritage {
 
 	char				*panic_str;
 	ssize_t				panic_str_len;
-	
 };
 
 extern struct heritage heritage;
diff --git a/bin/varnishtest/vtc_varnish.c b/bin/varnishtest/vtc_varnish.c
index cf1742d..080f48a 100644
--- a/bin/varnishtest/vtc_varnish.c
+++ b/bin/varnishtest/vtc_varnish.c
@@ -200,17 +200,12 @@ varnishlog_thread(void *priv)
 	vsl = VSM_New();
 	VSL_Setup(vsl);
 	(void)VSL_Arg(vsl, 'n', v->workdir);
-	VSL_NonBlocking(vsl, 1);
 	while (v->pid  && VSL_Open(vsl, 0) != 0) {
 		assert(usleep(VSL_SLEEP_USEC) == 0 || errno == EINTR);
 	}
 	while (v->pid) {
-		if (VSL_Dispatch(vsl, h_addlog, v) < 0) {
-			assert(usleep(v->vsl_sleep) == 0 || errno == EINTR);
-			v->vsl_sleep += v->vsl_sleep;
-			if (v->vsl_sleep > VSL_SLEEP_USEC)
-				v->vsl_sleep = VSL_SLEEP_USEC;
-		}
+		if (VSL_Dispatch(vsl, h_addlog, v) <= 0)
+			break;
 	}
 	VSM_Delete(vsl);
 	return (NULL);
@@ -732,7 +727,7 @@ varnish_expect(const struct varnish *v, char * const *av) {
 	uint64_t ref;
 	int good;
 	char *p;
-	int i;
+	int i, j;
 	struct stat_priv sp;
 
 	good = -1;
@@ -742,9 +737,20 @@ varnish_expect(const struct varnish *v, char * const *av) {
 	ref = 0;
 	for (i = 0; i < 10; i++, (void)usleep(100000)) {
 
-		good = -1;
-		if (!VSC_Iter(v->vd, do_stat_cb, &sp))
-			continue;
+		good = VSC_Iter(v->vd, do_stat_cb, &sp);
+		if (good < 0) {
+			VSM_Close(v->vd);
+			j = VSM_Open(v->vd, 0);
+			if (j == 0)
+				continue;
+			do {
+				(void)usleep(100000);
+				j = VSM_Open(v->vd, 0);
+				i++;
+			} while(i < 10 && j < 0);
+			if (j < 0)
+				break;
+		}
 		good = 0;
 
 		ref = strtoumax(av[2], &p, 0);
diff --git a/lib/libvarnishapi/vsc.c b/lib/libvarnishapi/vsc.c
index 5791e85..c9facfd 100644
--- a/lib/libvarnishapi/vsc.c
+++ b/lib/libvarnishapi/vsc.c
@@ -326,6 +326,8 @@ VSC_Iter(const struct VSM_data *vd, VSC_iter_f *func, void *priv)
 	vsc = vd->vsc;
 	CHECK_OBJ_NOTNULL(vsc, VSC_MAGIC);
 	i = 0;
+	if (!VSM_StillValid(vd, NULL))
+		return (-1);
 	VSM_FOREACH_SAFE(&vf, vd) {
 		if (strcmp(vf.chunk->class, VSC_CLASS))
 			continue;
diff --git a/lib/libvarnishapi/vsl.c b/lib/libvarnishapi/vsl.c
index 3c90381..3e7eb40 100644
--- a/lib/libvarnishapi/vsl.c
+++ b/lib/libvarnishapi/vsl.c
@@ -51,6 +51,8 @@
 #include "vsl_api.h"
 #include "vsm_api.h"
 
+static void VSL_Close(struct VSM_data *vd);
+
 /*--------------------------------------------------------------------*/
 
 const char *VSL_tags[256] = {
@@ -128,20 +130,30 @@ VSL_NonBlocking(const struct VSM_data *vd, int nb)
 		vsl->flags &= ~F_NON_BLOCKING;
 }
 
-/*--------------------------------------------------------------------*/
+/*--------------------------------------------------------------------
+ * Return the next log record, if there is one
+ *
+ * Return:
+ *	<0: error
+ *	0: no record
+ *	>0: record available at pp
+ */
 
 static int
 vsl_nextlog(struct vsl *vsl, uint32_t **pp)
 {
-	unsigned w, l;
+	unsigned l;
 	uint32_t t;
 	int i;
 
 	CHECK_OBJ_NOTNULL(vsl, VSL_MAGIC);
 
+	*pp = NULL;
 	if (vsl->r_fd != -1) {
 		assert(vsl->rbuflen >= 8);
 		i = read(vsl->r_fd, vsl->rbuf, 8);
+		if (i == 0)
+			return (0);
 		if (i != 8)
 			return (-1);
 		l = 2 + VSL_WORDS(VSL_LEN(vsl->rbuf));
@@ -157,38 +169,36 @@ vsl_nextlog(struct vsl *vsl, uint32_t **pp)
 		*pp = vsl->rbuf;
 		return (1);
 	}
-	for (w = 0; w < TIMEOUT_USEC;) {
+	while (1) {
+		if (vsl->log_ptr == NULL)
+			return (0);
+		assert(vsl->log_ptr >= vsl->log_start + 1);
+		assert(vsl->log_ptr < vsl->log_end);
 		t = *vsl->log_ptr;
 
 		if (t == VSL_WRAPMARKER) {
 			/* Wrap around not possible at front */
-			assert(vsl->log_ptr != vsl->log_start + 1);
+			if (vsl->log_ptr == vsl->log_start + 1)
+				return (-1);
 			vsl->log_ptr = vsl->log_start + 1;
-			VRMB();
 			continue;
 		}
+
 		if (t == VSL_ENDMARKER) {
 			if (vsl->log_ptr != vsl->log_start + 1 &&
 			    vsl->last_seq != vsl->log_start[0]) {
 				/* ENDMARKER not at front and seq wrapped */
 				vsl->log_ptr = vsl->log_start + 1;
-				VRMB();
 				continue;
 			}
-			if (vsl->flags & F_NON_BLOCKING)
-				return (-1);
-			w += SLEEP_USEC;
-			assert(usleep(SLEEP_USEC) == 0 || errno == EINTR);
-			VRMB();
-			continue;
+			return (0);
 		}
+
 		if (t == 0) {
-			/* Zero-initialized VSL */
-			w += SLEEP_USEC;
-			assert(usleep(SLEEP_USEC) == 0 || errno == EINTR);
-			VRMB();
-			continue;
+			/* Uninitialized VSL */
+			return (0);
 		}
+
 		if (vsl->log_ptr == vsl->log_start + 1)
 			vsl->last_seq = vsl->log_start[0];
 
@@ -196,8 +206,6 @@ vsl_nextlog(struct vsl *vsl, uint32_t **pp)
 		vsl->log_ptr = VSL_NEXT(vsl->log_ptr);
 		return (1);
 	}
-	*pp = NULL;
-	return (0);
 }
 
 int
@@ -214,7 +222,7 @@ VSL_NextLog(const struct VSM_data *vd, uint32_t **pp, uint64_t *bits)
 
 	while (1) {
 		i = vsl_nextlog(vsl, &p);
-		if (i != 1)
+		if (i <= 0)
 			return (i);
 		t = VSL_TAG(p);
 		if (vsl->skip) {
@@ -275,17 +283,37 @@ VSL_Dispatch(struct VSM_data *vd, VSL_handler_f *func, void *priv)
 	unsigned u, l, s;
 	uint32_t *p;
 	uint64_t bitmap;
+	int tmo;
 
 	CHECK_OBJ_NOTNULL(vd, VSM_MAGIC);
 	vsl = vd->vsl;
 	CHECK_OBJ_NOTNULL(vsl, VSL_MAGIC);
 
+	tmo = 0;
 	while (1) {
 		bitmap = 0;
 		i = VSL_NextLog(vd, &p, &bitmap);
-		if (i == 0 && VSM_ReOpen(vd, 0) == 1)
+		if (i == 0) {
+			if (vsl->r_fd != -1)
+				return(0);
+			if (vsl->flags & F_NON_BLOCKING)
+				return (0);
+			if (VSM_StillValid(vd, &vsl->vf) != 1) {
+				VSL_Close(vd);
+				if (VSL_Open(vd, 0))
+					return (-1);
+				AN(vsl->log_ptr);
+				assert(vsl->log_ptr >= vsl->log_start + 1);
+				assert(vsl->log_ptr < vsl->log_end);
+				continue;
+			}
+			tmo += SLEEP_USEC;
+			if (tmo > TIMEOUT_USEC)
+				return (0);
+			(void)usleep(SLEEP_USEC);
 			continue;
-		if (i != 1)
+		}
+		if (i <= 0)
 			return (i);
 		u = VSL_ID(p);
 		l = VSL_LEN(p);
@@ -294,7 +322,8 @@ VSL_Dispatch(struct VSM_data *vd, VSL_handler_f *func, void *priv)
 			s |= VSL_S_CLIENT;
 		if (VSL_BACKEND(p))
 			s |= VSL_S_BACKEND;
-		if (func(priv, (enum VSL_tag_e)VSL_TAG(p), u, l, s, VSL_DATA(p), bitmap))
+		if (func(priv, (enum VSL_tag_e)VSL_TAG(p),
+		    u, l, s, VSL_DATA(p), bitmap))
 			return (1);
 	}
 }
@@ -333,6 +362,23 @@ VSL_H_Print(void *priv, enum VSL_tag_e tag, unsigned fd, unsigned len,
 
 /*--------------------------------------------------------------------*/
 
+static void
+VSL_Close(struct VSM_data *vd)
+{
+	struct vsl *vsl;
+
+	CHECK_OBJ_NOTNULL(vd, VSM_MAGIC);
+	vsl = vd->vsl;
+	CHECK_OBJ_NOTNULL(vsl, VSL_MAGIC);
+
+	VSM_Close(vd);
+	vsl->log_start = NULL;
+	vsl->log_end = NULL;
+	vsl->log_ptr = NULL;
+}
+
+/*--------------------------------------------------------------------*/
+
 int
 VSL_Open(struct VSM_data *vd, int diag)
 {
diff --git a/lib/libvarnishapi/vsm.c b/lib/libvarnishapi/vsm.c
index c410fde..6c78baa 100644
--- a/lib/libvarnishapi/vsm.c
+++ b/lib/libvarnishapi/vsm.c
@@ -297,19 +297,25 @@ VSM__iter0(const struct VSM_data *vd, struct VSM_fantom *vf)
 int
 VSM__itern(const struct VSM_data *vd, struct VSM_fantom *vf)
 {
+	void *p;
 
 	CHECK_OBJ_NOTNULL(vd, VSM_MAGIC);
-	if (vf->priv != 0) {
+	if (vd->head->alloc_seq == 0)
+		return (0);	/* abandonned VSM */
+	else if (vf->priv != 0) {
 		if (vf->priv != vd->head->alloc_seq)
 			return (0);
 		if (vf->chunk->len == 0)
 			return (0);
 		if (vf->chunk->next == 0)
 			return (0);
-		vf->chunk = (void*)(vd->b + vf->chunk->next);
+		p = (void*)(vd->b + vf->chunk->next);
+		assert(p != vf->chunk);
+		vf->chunk = p;
 	} else if (vd->head->first == 0) {
 		return (0);
 	} else {
+		AZ(vf->chunk);
 		vf->chunk = (void*)(vd->b + vd->head->first);
 	}
 	if (memcmp(vf->chunk->marker, VSM_CHUNK_MARKER,
@@ -318,8 +324,13 @@ VSM__itern(const struct VSM_data *vd, struct VSM_fantom *vf)
 	vf->priv = vd->head->alloc_seq;
 	vf->b = (void*)(vf->chunk + 1);
 	vf->e = (char*)vf->b + vf->chunk->len;
+
+	if (vd->priv == 0)
+		return (0);	/* abandonned VSM */
 	if (vf->b == vf->e)
-		return (0);
+		return (0);	/* freed chunk */
+	AN(vf->priv);
+	AN(vf->chunk);
 	return (1);
 }
 
@@ -331,6 +342,8 @@ VSM_StillValid(const struct VSM_data *vd, struct VSM_fantom *vf)
 	struct VSM_fantom f2;
 
 	CHECK_OBJ_NOTNULL(vd, VSM_MAGIC);
+	if (vf == NULL)
+		return (vd->head->alloc_seq ? 1 : 0);
 	if (vf->priv == vd->head->alloc_seq)
 		return (1);
 	VSM_FOREACH_SAFE(&f2, vd) {



From geoff at varnish-cache.org  Mon Jan  9 20:51:46 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:51:46 +0100
Subject: [experimental-ims] c397697 Add text representation to Fetch_Body VSL
	message.
Message-ID: <E1RkMC2-0006l5-2R@project.varnish-software.com>

commit c397697d40e90e7848b04009891a264802b8b8d8
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Sep 7 11:03:35 2011 +0000

    Add text representation to Fetch_Body VSL message.

diff --git a/bin/varnishd/body_status.h b/bin/varnishd/body_status.h
new file mode 100644
index 0000000..33803f9
--- /dev/null
+++ b/bin/varnishd/body_status.h
@@ -0,0 +1,38 @@
+/*-
+ * Copyright (c) 2011 Varnish Software AS
+ * All rights reserved.
+ *
+ * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * Various ways to handle the body coming from the backend.
+ */
+
+/*lint -save -e525 -e539 */
+BODYSTATUS(NONE, none)
+BODYSTATUS(ZERO, zero)
+BODYSTATUS(ERROR, error)
+BODYSTATUS(CHUNKED, chunked)
+BODYSTATUS(LENGTH, length)
+BODYSTATUS(EOF, eof)
+/*lint -restore */
diff --git a/bin/varnishd/cache.h b/bin/varnishd/cache.h
index f9989fb..09c94f3 100644
--- a/bin/varnishd/cache.h
+++ b/bin/varnishd/cache.h
@@ -66,14 +66,23 @@
 #include "vsl.h"
 
 enum body_status {
-	BS_NONE,
-	BS_ZERO,
-	BS_ERROR,
-	BS_CHUNKED,
-	BS_LENGTH,
-	BS_EOF
+#define BODYSTATUS(U,l)	BS_##U,
+#include "body_status.h"
+#undef BODYSTATUS
 };
 
+static inline const char *
+body_status(enum body_status e)
+{
+	switch(e) {
+#define BODYSTATUS(U,l)	case BS_##U: return (#l);
+#include "body_status.h"
+#undef BODYSTATUS
+	default:
+		return ("?");
+	}
+}
+
 /*
  * NB: HDR_STATUS is only used in cache_http.c, everybody else uses the
  * http->status integer field.
diff --git a/bin/varnishd/cache_fetch.c b/bin/varnishd/cache_fetch.c
index 657392f..24463f5 100644
--- a/bin/varnishd/cache_fetch.c
+++ b/bin/varnishd/cache_fetch.c
@@ -543,8 +543,9 @@ FetchBody(struct sess *sp)
 	 */
 	AZ(vfp_nop_end(sp));
 
-	WSL(sp->wrk, SLT_Fetch_Body, sp->vbc->fd, "%u %d %u",
-	    sp->wrk->body_status, cls, mklen);
+	WSL(sp->wrk, SLT_Fetch_Body, sp->vbc->fd, "%u(%s) %d %u",
+	    sp->wrk->body_status, body_status(sp->wrk->body_status),
+	    cls, mklen);
 
 	if (sp->wrk->body_status == BS_ERROR) {
 		VDI_CloseFd(sp);
diff --git a/bin/varnishd/flint.lnt b/bin/varnishd/flint.lnt
index 1e88f1b..f2d2a6b 100644
--- a/bin/varnishd/flint.lnt
+++ b/bin/varnishd/flint.lnt
@@ -86,6 +86,7 @@
 -efile(451, "sys/\*.h")	// No include guard
 -efile(451, "machine/\*.h")	// No include guard
 -efile(451, "vcl_returns.h")	// No include guard
+-efile(451, "body_status.h")	// No include guard
 -efile(451, "locks.h")	// No include guard
 -efile(451, "cache_backend_poll.h")	// No include guard
 -efile(451, "steps.h")	// No include guard



From geoff at varnish-cache.org  Mon Jan  9 20:51:49 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:51:49 +0100
Subject: [experimental-ims] 83753e2 Change the calling convention for waiters,
	to make it OO
Message-ID: <E1RkMC5-0006qk-Eo@project.varnish-software.com>

commit 83753e28342e4925e52ebf0b8e36ad9600c1e9a3
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sat Sep 17 10:11:40 2011 +0000

    Change the calling convention for waiters, to make it OO

diff --git a/bin/varnishd/cache_acceptor.c b/bin/varnishd/cache_acceptor.c
index 0692d46..2afc975 100644
--- a/bin/varnishd/cache_acceptor.c
+++ b/bin/varnishd/cache_acceptor.c
@@ -62,6 +62,8 @@ static struct waiter * const vca_waiters[] = {
 
 static struct waiter const *vca_act;
 
+static void *waiter_priv;
+
 pthread_t		VCA_thread;
 static struct timeval	tv_sndtimeo;
 static struct timeval	tv_rcvtimeo;
@@ -384,10 +386,7 @@ vca_return_session(struct sess *sp)
 	 */
 	if (VTCP_nonblocking(sp->fd))
 		vca_close_session(sp, "remote closed");
-	else if (vca_act->pass == NULL)
-		assert(sizeof sp == write(vca_pipes[1], &sp, sizeof sp));
-	else
-		vca_act->pass(sp);
+	vca_act->pass(waiter_priv, sp);
 }
 
 /*--------------------------------------------------------------------*/
@@ -405,10 +404,11 @@ ccf_start(struct cli *cli, const char * const *av, void *priv)
 
 	AN(vca_act);
 	AN(vca_act->name);
+	AN(vca_act->init);
+	AN(vca_act->pass);
 
-	if (vca_act->pass == NULL)
-		AZ(pipe(vca_pipes));
-	vca_act->init();
+	AZ(pipe(vca_pipes)); 	/* XXX */
+	waiter_priv = vca_act->init();
 	AZ(pthread_create(&VCA_thread, NULL, vca_acct, NULL));
 	VSL(SLT_Debug, 0, "Acceptor is %s", vca_act->name);
 }
diff --git a/bin/varnishd/cache_waiter.h b/bin/varnishd/cache_waiter.h
index 2e0953b..9d6eee9 100644
--- a/bin/varnishd/cache_waiter.h
+++ b/bin/varnishd/cache_waiter.h
@@ -30,8 +30,8 @@
 
 struct sess;
 
-typedef void waiter_init_f(void);
-typedef void waiter_pass_f(struct sess *);
+typedef void* waiter_init_f(void);
+typedef void waiter_pass_f(void *priv, struct sess *);
 
 extern int vca_pipes[2];
 
diff --git a/bin/varnishd/cache_waiter_epoll.c b/bin/varnishd/cache_waiter_epoll.c
index fe64961..5726b03 100644
--- a/bin/varnishd/cache_waiter_epoll.c
+++ b/bin/varnishd/cache_waiter_epoll.c
@@ -222,6 +222,16 @@ vca_sess_timeout_ticker(void *arg)
 /*--------------------------------------------------------------------*/
 
 static void
+vca_epoll_pass(void *priv, hhstruct sess *sp)
+{
+
+	(void)priv;
+	assert(sizeof sp == write(vca_pipes[1], &sp, sizeof sp));
+}
+
+/*--------------------------------------------------------------------*/
+
+static void *
 vca_epoll_init(void)
 {
 	int i;
@@ -242,11 +252,15 @@ vca_epoll_init(void)
 	AZ(pthread_create(&vca_epoll_timeout_thread,
 	    NULL, vca_sess_timeout_ticker, NULL));
 	AZ(pthread_create(&vca_epoll_thread, NULL, vca_main, NULL));
+	return(NULL);
 }
 
+/*--------------------------------------------------------------------*/
+
 struct waiter waiter_epoll = {
 	.name =		"epoll",
 	.init =		vca_epoll_init,
+	.pass =		vca_epoll_pass,
 };
 
 #endif /* defined(HAVE_EPOLL_CTL) */
diff --git a/bin/varnishd/cache_waiter_kqueue.c b/bin/varnishd/cache_waiter_kqueue.c
index ea0e2fc..12e7bd4 100644
--- a/bin/varnishd/cache_waiter_kqueue.c
+++ b/bin/varnishd/cache_waiter_kqueue.c
@@ -204,6 +204,16 @@ vca_kqueue_main(void *arg)
 /*--------------------------------------------------------------------*/
 
 static void
+vca_kqueue_pass(void *priv, struct sess *sp)
+{
+
+	(void)priv;
+	assert(sizeof sp == write(vca_pipes[1], &sp, sizeof sp));
+}
+
+/*--------------------------------------------------------------------*/
+
+static void *
 vca_kqueue_init(void)
 {
 	int i;
@@ -215,11 +225,15 @@ vca_kqueue_init(void)
 	assert(i != -1);
 
 	AZ(pthread_create(&vca_kqueue_thread, NULL, vca_kqueue_main, NULL));
+	return (NULL);
 }
 
+/*--------------------------------------------------------------------*/
+
 struct waiter waiter_kqueue = {
 	.name =		"kqueue",
 	.init =		vca_kqueue_init,
+	.pass =		vca_kqueue_pass,
 };
 
 #endif /* defined(HAVE_KQUEUE) */
diff --git a/bin/varnishd/cache_waiter_poll.c b/bin/varnishd/cache_waiter_poll.c
index 42b54dc..528a880 100644
--- a/bin/varnishd/cache_waiter_poll.c
+++ b/bin/varnishd/cache_waiter_poll.c
@@ -191,16 +191,29 @@ vca_main(void *arg)
 }
 
 /*--------------------------------------------------------------------*/
-
 static void
+vca_poll_pass(void *priv, struct sess *sp)
+{
+
+	(void)priv;
+	assert(sizeof sp == write(vca_pipes[1], &sp, sizeof sp));
+}
+
+/*--------------------------------------------------------------------*/
+
+static void *
 vca_poll_init(void)
 {
 
 	vca_pollspace(256);
 	AZ(pthread_create(&vca_poll_thread, NULL, vca_main, NULL));
+	return (NULL);
 }
 
+/*--------------------------------------------------------------------*/
+
 struct waiter waiter_poll = {
 	.name =		"poll",
 	.init =		vca_poll_init,
+	.pass =		vca_poll_pass,
 };
diff --git a/bin/varnishd/cache_waiter_ports.c b/bin/varnishd/cache_waiter_ports.c
index 9ee3aea..f1cc55c 100644
--- a/bin/varnishd/cache_waiter_ports.c
+++ b/bin/varnishd/cache_waiter_ports.c
@@ -243,10 +243,14 @@ vca_main(void *arg)
 	}
 }
 
+/*--------------------------------------------------------------------*/
+
 static void
-vca_ports_pass(struct sess *sp)
+vca_ports_pass(void *priv, struct sess *sp)
 {
 	int r;
+
+	(void)priv;
 	while((r = port_send(solaris_dport, 0, sp)) == -1 &&
 		errno == EAGAIN);
 	AZ(r);
@@ -254,13 +258,16 @@ vca_ports_pass(struct sess *sp)
 
 /*--------------------------------------------------------------------*/
 
-static void
+static void *
 vca_ports_init(void)
 {
 
 	AZ(pthread_create(&vca_ports_thread, NULL, vca_main, NULL));
+	return (NULL);
 }
 
+/*--------------------------------------------------------------------*/
+
 struct waiter waiter_ports = {
 	.name =		"ports",
 	.init =		vca_ports_init,



From geoff at varnish-cache.org  Mon Jan  9 20:51:56 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:51:56 +0100
Subject: [experimental-ims] 260574a Make listen_sock a miniobj
Message-ID: <E1RkMCC-000710-FQ@project.varnish-software.com>

commit 260574ac96af6db89a5374144d84f168b8e76645
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sun Sep 18 09:04:55 2011 +0000

    Make listen_sock a miniobj

diff --git a/bin/varnishd/heritage.h b/bin/varnishd/heritage.h
index 6571e31..13fce78 100644
--- a/bin/varnishd/heritage.h
+++ b/bin/varnishd/heritage.h
@@ -32,6 +32,8 @@
 #include <pthread.h>
 
 struct listen_sock {
+	unsigned			magic;
+#define LISTEN_SOCK_MAGIC		0x999e4b57
 	VTAILQ_ENTRY(listen_sock)	list;
 	int				sock;
 	char				*name;
diff --git a/bin/varnishd/mgt_param.c b/bin/varnishd/mgt_param.c
index ec38b11..96a83fc 100644
--- a/bin/varnishd/mgt_param.c
+++ b/bin/varnishd/mgt_param.c
@@ -333,10 +333,11 @@ clean_listen_sock_head(struct listen_sock_head *lsh)
 	struct listen_sock *ls, *ls2;
 
 	VTAILQ_FOREACH_SAFE(ls, lsh, list, ls2) {
+		CHECK_OBJ_NOTNULL(ls, LISTEN_SOCK_MAGIC);
 		VTAILQ_REMOVE(lsh, ls, list);
 		free(ls->name);
 		free(ls->addr);
-		free(ls);
+		FREE_OBJ(ls);
 	}
 }
 
@@ -386,7 +387,7 @@ tweak_listen_address(struct cli *cli, const struct parspec *par,
 			break;
 		}
 		for (j = 0; j < n; ++j) {
-			ls = calloc(sizeof *ls, 1);
+			ALLOC_OBJ(ls, LISTEN_SOCK_MAGIC);
 			AN(ls);
 			ls->sock = -1;
 			ls->addr = ta[j];
@@ -410,6 +411,7 @@ tweak_listen_address(struct cli *cli, const struct parspec *par,
 	while (!VTAILQ_EMPTY(&lsh)) {
 		ls = VTAILQ_FIRST(&lsh);
 		VTAILQ_REMOVE(&lsh, ls, list);
+		CHECK_OBJ_NOTNULL(ls, LISTEN_SOCK_MAGIC);
 		VTAILQ_INSERT_TAIL(&heritage.socks, ls, list);
 		heritage.nsocks++;
 	}



From geoff at varnish-cache.org  Mon Jan  9 20:52:41 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:41 +0100
Subject: [experimental-ims] a8aa0ba Complete the VSC filtering,
	and make everything compile and pass tests.
Message-ID: <E1RkMCv-0008PM-U3@project.varnish-software.com>

commit a8aa0bafdcf6ffbabb4bd5a21fa2fd5d1c612f39
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Nov 23 21:09:18 2011 +0000

    Complete the VSC filtering, and make everything compile and pass tests.
    
    Still not done, in particular: Do not roll any releases until
    libvarnishapi symbo/version stuff has been polished.
    
    Fixes	#829

diff --git a/bin/varnishadm/varnishadm.c b/bin/varnishadm/varnishadm.c
index f5d6054..4570e3c 100644
--- a/bin/varnishadm/varnishadm.c
+++ b/bin/varnishadm/varnishadm.c
@@ -282,25 +282,27 @@ n_arg_sock(const char *n_arg)
 	struct VSM_data *vsd;
 	char *p;
 	int sock;
+	struct VSM_fantom vt;
 
 	vsd = VSM_New();
 	assert(VSL_Arg(vsd, 'n', n_arg));
-	if (VSM_Open(vsd, 1)) {
-		fprintf(stderr, "Could not open shared memory\n");
+	if (VSM_Open(vsd)) {
+		fprintf(stderr, "%s\n", VSM_Error(vsd));
 		return (-1);
 	}
 	if (T_arg == NULL) {
-		p = VSM_Find_Chunk(vsd, "Arg", "-T", "", NULL);
-		if (p == NULL)  {
+		if (VSM_Get(vsd, &vt, "Arg", "-T", "")) {
 			fprintf(stderr, "No -T arg in shared memory\n");
 			return (-1);
 		}
-		T_start = T_arg = strdup(p);
+		T_start = T_arg = strdup(vt.b);
 	}
 	if (S_arg == NULL) {
-		p = VSM_Find_Chunk(vsd, "Arg", "-S", "", NULL);
-		if (p != NULL)
-			S_arg = strdup(p);
+		if (VSM_Get(vsd, &vt, "Arg", "-S", "")) {
+			fprintf(stderr, "No -S arg in shared memory\n");
+			return (-1);
+		}
+		S_arg = strdup(vt.b);
 	}
 	sock = -1;
 	while (*T_arg) {
diff --git a/bin/varnishhist/varnishhist.c b/bin/varnishhist/varnishhist.c
index a4bc6b7..852ff41 100644
--- a/bin/varnishhist/varnishhist.c
+++ b/bin/varnishhist/varnishhist.c
@@ -351,7 +351,6 @@ main(int argc, char **argv)
 	match_tag = -1;
 
 	vd = VSM_New();
-	VSL_Setup(vd);
 
 	while ((o = getopt(argc, argv, VSL_ARGS "Vw:r:R:f:p:")) != -1) {
 		switch (o) {
@@ -424,8 +423,10 @@ main(int argc, char **argv)
 	}
 	strcpy(format + 4*(fnum-1), "%lf");
 
-	if (VSL_Open(vd, 1))
+	if (VSM_Open(vd)) {
+		fprintf(stderr, "%s\n", VSM_Error(vd));
 		exit(1);
+	}
 
 	log_ten = log(10.0);
 
diff --git a/bin/varnishlog/varnishlog.c b/bin/varnishlog/varnishlog.c
index e15d102..45b4327 100644
--- a/bin/varnishlog/varnishlog.c
+++ b/bin/varnishlog/varnishlog.c
@@ -60,7 +60,7 @@ static uint64_t       bitmap[65536];
 #define F_INVCL		(1 << 0)
 
 static void
-h_order_finish(int fd, const struct VSM_data *vd)
+h_order_finish(int fd, struct VSM_data *vd)
 {
 
 	AZ(VSB_finish(ob[fd]));
@@ -72,7 +72,7 @@ h_order_finish(int fd, const struct VSM_data *vd)
 }
 
 static void
-clean_order(const struct VSM_data *vd)
+clean_order(struct VSM_data *vd)
 {
 	unsigned u;
 
@@ -234,7 +234,7 @@ open_log(const char *w_arg, int a_flag)
 }
 
 static void
-do_write(const struct VSM_data *vd, const char *w_arg, int a_flag)
+do_write(struct VSM_data *vd, const char *w_arg, int a_flag)
 {
 	int fd, i, l;
 	uint32_t *p;
@@ -243,7 +243,7 @@ do_write(const struct VSM_data *vd, const char *w_arg, int a_flag)
 	XXXAN(fd >= 0);
 	(void)signal(SIGHUP, sighup);
 	while (1) {
-		i = VSL_NextLog(vd, &p, NULL);
+		i = VSL_NextSLT(vd, &p, NULL);
 		if (i < 0)
 			break;
 		if (i > 0) {
@@ -285,7 +285,6 @@ main(int argc, char * const *argv)
 	struct VSM_data *vd;
 
 	vd = VSM_New();
-	VSL_Setup(vd);
 
 	while ((c = getopt(argc, argv, VSL_ARGS "aDP:uVw:oO")) != -1) {
 		switch (c) {
@@ -336,8 +335,10 @@ main(int argc, char * const *argv)
 	if ((argc - optind) > 0)
 		usage();
 
-	if (VSL_Open(vd, 1))
+	if (VSM_Open(vd)) {
+		fprintf(stderr, "%s\n", VSM_Error(vd));
 		exit(1);
+	}
 
 	if (P_arg && (pfh = VPF_Open(P_arg, 0644, NULL)) == NULL) {
 		perror(P_arg);
diff --git a/bin/varnishncsa/varnishncsa.c b/bin/varnishncsa/varnishncsa.c
index 9765796..5561adc 100644
--- a/bin/varnishncsa/varnishncsa.c
+++ b/bin/varnishncsa/varnishncsa.c
@@ -846,7 +846,6 @@ main(int argc, char *argv[])
 	format = "%h %l %u %t \"%r\" %s %b \"%{Referer}i\" \"%{User-agent}i\"";
 
 	vd = VSM_New();
-	VSL_Setup(vd);
 
 	while ((c = getopt(argc, argv, VSL_ARGS "aDP:Vw:fF:")) != -1) {
 		switch (c) {
@@ -907,8 +906,10 @@ main(int argc, char *argv[])
 
 	VSL_Arg(vd, 'c', optarg);
 
-	if (VSL_Open(vd, 1))
-		exit(1);
+	if (VSM_Open(vd)) {
+		fprintf(stderr, "%s\n", VSM_Error(vd));
+		return (-1);
+	}
 
 	if (P_arg && (pfh = VPF_Open(P_arg, 0644, NULL)) == NULL) {
 		perror(P_arg);
diff --git a/bin/varnishreplay/varnishreplay.c b/bin/varnishreplay/varnishreplay.c
index 12b6c41..8e694da 100644
--- a/bin/varnishreplay/varnishreplay.c
+++ b/bin/varnishreplay/varnishreplay.c
@@ -707,7 +707,6 @@ main(int argc, char *argv[])
 	const char *address = NULL;
 
 	vd = VSM_New();
-	VSL_Setup(vd);
 	debug = 0;
 
 	VSL_Arg(vd, 'c', NULL);
@@ -730,8 +729,10 @@ main(int argc, char *argv[])
 		usage();
 	}
 
-	if (VSL_Open(vd, 1))
+	if (VSM_Open(vd)) {
+		fprintf(stderr, "%s\n", VSM_Error(vd));
 		exit(1);
+	}
 
 	addr_info = init_connection(address);
 
diff --git a/bin/varnishtest/flint.lnt b/bin/varnishtest/flint.lnt
index 53ad3dc..9ff3c68 100644
--- a/bin/varnishtest/flint.lnt
+++ b/bin/varnishtest/flint.lnt
@@ -25,3 +25,5 @@
 -e713	// Loss of precision (assignment) (unsigned long long to long long)
 -e574	// Signed-unsigned mix with relational
 -e835	//  A zero has been given as ___ argument to operator '___'  (<<)
+-e786	// String concatenation within initializer
+
diff --git a/bin/varnishtest/vtc_varnish.c b/bin/varnishtest/vtc_varnish.c
index 080f48a..2f7514a 100644
--- a/bin/varnishtest/vtc_varnish.c
+++ b/bin/varnishtest/vtc_varnish.c
@@ -198,11 +198,7 @@ varnishlog_thread(void *priv)
 
 	CAST_OBJ_NOTNULL(v, priv, VARNISH_MAGIC);
 	vsl = VSM_New();
-	VSL_Setup(vsl);
 	(void)VSL_Arg(vsl, 'n', v->workdir);
-	while (v->pid  && VSL_Open(vsl, 0) != 0) {
-		assert(usleep(VSL_SLEEP_USEC) == 0 || errno == EINTR);
-	}
 	while (v->pid) {
 		if (VSL_Dispatch(vsl, h_addlog, v) <= 0)
 			break;
@@ -332,7 +328,6 @@ varnish_launch(struct varnish *v)
 	char *r;
 
 	v->vd = VSM_New();
-	VSC_Setup(v->vd);
 
 	/* Create listener socket */
 	nap = VSS_resolve("127.0.0.1", "0", &ap);
@@ -457,7 +452,7 @@ varnish_launch(struct varnish *v)
 	free(r);
 
 	(void)VSL_Arg(v->vd, 'n', v->workdir);
-	AZ(VSC_Open(v->vd, 1));
+	AZ(VSM_Open(v->vd));
 }
 
 /**********************************************************************
@@ -696,6 +691,8 @@ do_stat_cb(void *priv, const struct VSC_point * const pt)
 	const char *p = sp->target;
 	int i;
 
+	if (pt == NULL)
+		return(0);
 	if (strcmp(pt->class, "")) {
 		i = strlen(pt->class);
 		if (memcmp(pt->class, p, i))
@@ -714,10 +711,10 @@ do_stat_cb(void *priv, const struct VSC_point * const pt)
 			return (0);
 		p++;
 	}
-	if (strcmp(pt->name, p))
+	if (strcmp(pt->desc->name, p))
 		return (0);
 
-	assert(!strcmp(pt->fmt, "uint64_t"));
+	assert(!strcmp(pt->desc->fmt, "uint64_t"));
 	sp->val = *(const volatile uint64_t*)pt->ptr;
 	return (1);
 }
@@ -740,12 +737,12 @@ varnish_expect(const struct varnish *v, char * const *av) {
 		good = VSC_Iter(v->vd, do_stat_cb, &sp);
 		if (good < 0) {
 			VSM_Close(v->vd);
-			j = VSM_Open(v->vd, 0);
+			j = VSM_Open(v->vd);
 			if (j == 0)
 				continue;
 			do {
 				(void)usleep(100000);
-				j = VSM_Open(v->vd, 0);
+				j = VSM_Open(v->vd);
 				i++;
 			} while(i < 10 && j < 0);
 			if (j < 0)
diff --git a/bin/varnishtop/varnishtop.c b/bin/varnishtop/varnishtop.c
index efe312e..62efb15 100644
--- a/bin/varnishtop/varnishtop.c
+++ b/bin/varnishtop/varnishtop.c
@@ -195,7 +195,7 @@ accumulate_thread(void *arg)
 
 	for (;;) {
 
-		i = VSL_NextLog(vd, &p, NULL);
+		i = VSL_NextSLT(vd, &p, NULL);
 		if (i < 0)
 			break;
 		if (i == 0) {
@@ -292,7 +292,7 @@ do_once(struct VSM_data *vd)
 {
 	uint32_t *p;
 
-	while (VSL_NextLog(vd, &p, NULL) > 0)
+	while (VSL_NextSLT(vd, &p, NULL) > 0)
 		accumulate(p);
 	dump();
 }
@@ -313,7 +313,6 @@ main(int argc, char **argv)
 	float period = 60; /* seconds */
 
 	vd = VSM_New();
-	VSL_Setup(vd);
 
 	while ((o = getopt(argc, argv, VSL_ARGS "1fVp:")) != -1) {
 		switch (o) {
@@ -345,11 +344,12 @@ main(int argc, char **argv)
 		}
 	}
 
-	if (VSL_Open(vd, 1))
+	if (VSM_Open(vd)) {
+		fprintf(stderr, "%s\n", VSM_Error(vd));
 		exit (1);
+	}
 
 	if (once) {
-		VSL_NonBlocking(vd, 1);
 		do_once(vd);
 	} else {
 		do_curses(vd, period);
diff --git a/include/vapi/vsc.h b/include/vapi/vsc.h
index 6ebab31..16a1583 100644
--- a/include/vapi/vsc.h
+++ b/include/vapi/vsc.h
@@ -88,13 +88,13 @@ int VSC_Iter(struct VSM_data *vd, VSC_iter_f *func, void *priv);
 	 * Iterate over all statistics counters, calling "func" for
 	 * each counter not suppressed by any "-f" arguments.
 	 *
- 	 * Func is called with pt == NULL, whenever VSM allocations
+	 * Func is called with pt == NULL, whenever VSM allocations
 	 * change (child restart, allocations/deallocations)
 	 *
 	 * Returns:
-	 * 	!=0:	func returned non-zero
+	 *	!=0:	func returned non-zero
 	 *	-1:	No VSC's available
-	 *	0: 	Done
+	 *	0:	Done
 	 */
 
 /**********************************************************************
diff --git a/include/vapi/vsl.h b/include/vapi/vsl.h
index 35e09dc..dfe5a83 100644
--- a/include/vapi/vsl.h
+++ b/include/vapi/vsl.h
@@ -36,7 +36,7 @@
  *	VSL_Arg(vd, "r", "/some/file");
  * and once VSL_Dispatch()/VSL_NextSLT() will indicate EOF by returning -2.
  * Another file can then be opened with VSL_Arg() and processed.
- * 
+ *
  */
 
 #ifndef VAPI_VSL_H_INCLUDED
diff --git a/lib/libvarnishapi/libvarnishapi.map b/lib/libvarnishapi/libvarnishapi.map
index 83f5169..5640868 100644
--- a/lib/libvarnishapi/libvarnishapi.map
+++ b/lib/libvarnishapi/libvarnishapi.map
@@ -75,3 +75,12 @@ LIBVARNISHAPI_1.1 {
 	VSL_Name2Tag;
 	# Variables:
 } LIBVARNISHAPI_1.0;
+
+LIBVARNISHAPI_1.2 {
+  global:
+	# Functions:
+	VSL_NextSLT;
+	VSM_Error;
+	VSM_Get;
+	# Variables:
+} LIBVARNISHAPI_1.0;
diff --git a/lib/libvarnishapi/vsc.c b/lib/libvarnishapi/vsc.c
index 54bf20f..b399c98 100644
--- a/lib/libvarnishapi/vsc.c
+++ b/lib/libvarnishapi/vsc.c
@@ -50,8 +50,9 @@
 struct vsc_pt {
 	unsigned		magic;
 #define VSC_PT_MAGIC		0xa4ff159a
-	struct VSC_point	point;
 	VTAILQ_ENTRY(vsc_pt)	list;
+	struct VSM_fantom	vf;
+	struct VSC_point	point;
 };
 
 struct vsc_sf {
@@ -101,15 +102,10 @@ static void
 vsc_delete_pts(struct vsc *vsc)
 {
 	struct vsc_pt *pt;
-	struct VSM_fantom *vf = NULL;
 
 	while(!VTAILQ_EMPTY(&vsc->pt_list)) {
 		pt = VTAILQ_FIRST(&vsc->pt_list);
 		VTAILQ_REMOVE(&vsc->pt_list, pt, list);
-		if (pt->point.vf != vf) {
-			vf = pt->point.vf;
-			free(vf);
-		}
 		FREE_OBJ(pt);
 	}
 }
@@ -243,129 +239,13 @@ VSC_Main(struct VSM_data *vd)
 	return ((void*)vsc->main_fantom.b);
 }
 
-#if 0
-/*--------------------------------------------------------------------
- * -1 -> unknown stats encountered.
- */
-
-static inline int
-iter_test(const char *s1, const char *s2, int wc)
-{
-
-	if (s1 == NULL)
-		return (0);
-	if (!wc)
-		return (strcmp(s1, s2));
-	for (; *s1 != '\0' && *s1 == *s2; s1++, s2++)
-		continue;
-	return (*s1 != '\0');
-}
-
-static int
-iter_call(const struct vsc *vsc, VSC_iter_f *func, void *priv,
-    const struct VSC_point *const sp)
-{
-	struct vsc_sf *sf;
-	struct vsc_pt *pt;
-	int good;
-
-	CHECK_OBJ_NOTNULL(vsc, VSC_MAGIC);
-
-	ALLOC_OBJ(pt, VSC_PT_MAGIC);
-	AN(pt);
-
-	if (VTAILQ_EMPTY(&vsc->sf_list)) {
-		VTAILQ_INSERT_TAIL(&vsc->pt_list, pt, list);
-		return (func(priv, sp));
-	}
-
-	good = 0;
-	VTAILQ_FOREACH(sf, &vsc->sf_list, list) {
-		if (iter_test(sf->class, sp->class, sf->flags & VSC_SF_CL_WC))
-			continue;
-		if (iter_test(sf->ident, sp->ident, sf->flags & VSC_SF_ID_WC))
-			continue;
-		if (iter_test(sf->name, sp->desc->name, sf->flags & VSC_SF_NM_WC))
-			continue;
-		if (sf->flags & VSC_SF_EXCL)
-			good = 0;
-		else
-			good = 1;
-	}
-	if (!good)
-		return (0);
-	return (func(priv, sp));
-}
-
-#define VSC_DO(U,l,t)							\
-	static void							\
-	iter_##l(const struct vsc *vsc, struct VSM_fantom *vf,		\
-	    const struct VSC_desc *descs)				\
-	{								\
-		struct VSC_C_##l *st;					\
-		struct VSM_fantom *vf2;					\
-		int i;							\
-									\
-		CHECK_OBJ_NOTNULL(vsc, VSC_MAGIC);			\
-		st = vf->b;						\
-		sp.class = t;						\
-		sp.ident = vf->chunk->ident;				\
-		sp.desc = descs++;					\
-		vf2 = malloc(sizeof *vf2);				\
-		AN(vf2);						\
-		memcpy(vf2, vf, sizeof *vf2);
-
-#define VSC_F(nn,tt,ll,ff,dd,ee)					\
-		sp.ptr = &st->nn;					\
-		sp.vf = vf2;						\
-		i = iter_call(vsc, &sp);				\
-		if (i)							\
-			return(i);
-
-#define VSC_DONE(U,l,t)							\
-		return (0);						\
-	}
-
-#include "tbl/vsc_all.h"
-#undef VSC_DO
-#undef VSC_F
-#undef VSC_DONE
-
-static void
-vsc_build_pt_list(struct VSM_data *vd)
-{
-	struct vsc *vsc = vsc_setup(vd);
-	struct VSM_fantom vf;
-
-	vsc_delete_pts(vsc *vsc);
-
-	VSM_FOREACH_SAFE(&vf, vd) {
-		if (strcmp(vf.chunk->class, VSC_CLASS))
-			continue;
-		/*lint -save -e525 -e539 */
-#define VSC_F(n,t,l,f,d,e)
-#define VSC_DONE(a,b,c)
-#define VSC_DO(U,l,t)						\
-		if (!strcmp(vf.chunk->type, t))			\
-			iter_##l(vsc, &vf, VSC_desc_##l);
-#include "tbl/vsc_all.h"
-#undef VSC_F
-#undef VSC_DO
-#undef VSC_DONE
-		/*lint -restore */
-		break;
-	}
-	return (i);
-}
-#endif
-
 /*--------------------------------------------------------------------
  */
 
 static void
 vsc_add_pt(struct vsc *vsc, const char *class, const char *ident,
     const struct VSC_desc *desc, const volatile void *ptr,
-    struct VSM_fantom *vf)
+    const struct VSM_fantom *vf)
 {
 	struct vsc_pt *pt;
 
@@ -375,7 +255,8 @@ vsc_add_pt(struct vsc *vsc, const char *class, const char *ident,
 	pt->point.ident = ident;
 	pt->point.desc = desc;
 	pt->point.ptr = ptr;
-	pt->point.vf = vf;
+	pt->point.vf = &pt->vf;
+	pt->vf = *vf;
 	VTAILQ_INSERT_TAIL(&vsc->pt_list, pt, list);
 }
 
@@ -385,18 +266,14 @@ vsc_add_pt(struct vsc *vsc, const char *class, const char *ident,
 	    const struct VSC_desc *descs)				\
 	{								\
 		struct VSC_C_##l *st;					\
-		struct VSM_fantom *vf2;					\
 		const char *class = t;					\
 									\
 		CHECK_OBJ_NOTNULL(vsc, VSC_MAGIC);			\
 		st = vf->b;						\
-		vf2 = malloc(sizeof *vf2);				\
-		AN(vf2);						\
-		memcpy(vf2, vf, sizeof *vf2);
 
 #define VSC_F(nn,tt,ll,ff,dd,ee)					\
 		vsc_add_pt(vsc, class, vf->chunk->ident, descs++,	\
-		    &st->nn, vf2);
+		    &st->nn, vf);
 
 #define VSC_DONE(U,l,t)							\
 	}
@@ -409,6 +286,8 @@ vsc_add_pt(struct vsc *vsc, const char *class, const char *ident,
 /*--------------------------------------------------------------------
  */
 
+#include <stdio.h>
+
 static void
 vsc_build_pt_list(struct VSM_data *vd)
 {
@@ -432,8 +311,57 @@ vsc_build_pt_list(struct VSM_data *vd)
 #undef VSC_DONE
 		/*lint -restore */
 	}
+}
 
-	/* XXX: filter pt list */
+/*--------------------------------------------------------------------
+ */
+
+static inline int
+iter_test(const char *s1, const char *s2, int wc)
+{
+
+	if (s1 == NULL)
+		return (0);
+	if (!wc)
+		return (strcmp(s1, s2));
+	for (; *s1 != '\0' && *s1 == *s2; s1++, s2++)
+		continue;
+	return (*s1 != '\0');
+}
+
+static void
+vsc_filter_pt_list(struct VSM_data *vd)
+{
+	struct vsc *vsc = vsc_setup(vd);
+	struct vsc_sf *sf;
+	struct vsc_pt *pt, *pt2;
+	VTAILQ_HEAD(, vsc_pt)	pt_list;
+
+	if (VTAILQ_EMPTY(&vsc->sf_list))
+		return;
+
+	VTAILQ_INIT(&pt_list);
+	VTAILQ_FOREACH(sf, &vsc->sf_list, list) {
+		VTAILQ_FOREACH_SAFE(pt, &vsc->pt_list, list, pt2) {
+			if (iter_test(sf->class, pt->point.class,
+			    sf->flags & VSC_SF_CL_WC))
+				continue;
+			if (iter_test(sf->ident, pt->point.ident,
+			    sf->flags & VSC_SF_ID_WC))
+				continue;
+			if (iter_test(sf->name, pt->point.desc->name,
+			    sf->flags & VSC_SF_NM_WC))
+				continue;
+			VTAILQ_REMOVE(&vsc->pt_list, pt, list);
+			if (sf->flags & VSC_SF_EXCL) {
+				FREE_OBJ(pt);
+			} else {
+				VTAILQ_INSERT_TAIL(&pt_list, pt, list);
+			}
+		}
+	}
+	vsc_delete_pts(vsc);
+	VTAILQ_CONCAT(&vsc->pt_list, &pt_list, list);
 }
 
 /*--------------------------------------------------------------------
@@ -456,8 +384,10 @@ VSC_Iter(struct VSM_data *vd, VSC_iter_f *func, void *priv)
 			}
 		}
 		AN(vd->head);
-		func(priv, NULL);
+		/* Tell app that list will be nuked */
+		(void)func(priv, NULL);
 		vsc_build_pt_list(vd);
+		vsc_filter_pt_list(vd);
 	}
 	AN(vd->head);
 	VTAILQ_FOREACH(pt, &vsc->pt_list, list) {
diff --git a/lib/libvarnishapi/vsl.c b/lib/libvarnishapi/vsl.c
index d45e1c3..fdd93d8 100644
--- a/lib/libvarnishapi/vsl.c
+++ b/lib/libvarnishapi/vsl.c
@@ -296,7 +296,7 @@ VSL_NextSLT(struct VSM_data *vd, uint32_t **pp, uint64_t *bits)
 				if (vrm->tag == t) {
 					i = VRE_exec(vrm->re, VSL_DATA(p),
 					    VSL_LEN(p), 0, 0, NULL, 0, NULL);
-					if (i >= 0) 	/* XXX ?? */
+					if (i >= 0)	/* XXX ?? */
 						*bits |= (uintmax_t)1 << j;
 				}
 				j++;
diff --git a/lib/libvarnishapi/vsl_arg.c b/lib/libvarnishapi/vsl_arg.c
index 100d75a..068d539 100644
--- a/lib/libvarnishapi/vsl_arg.c
+++ b/lib/libvarnishapi/vsl_arg.c
@@ -160,10 +160,10 @@ vsl_ix_arg(struct VSM_data *vd, const char *opt, int arg)
 			else
 				vbit_clr(vsl->vbm_supress, i);
 		} else if (i == -2) {
-			return (vsm_diag(vd, 
+			return (vsm_diag(vd,
 			    "\"%*.*s\" matches multiple tags\n", l, l, b));
 		} else {
-			return (vsm_diag(vd, 
+			return (vsm_diag(vd,
 			    "Could not match \"%*.*s\" to any tag\n", l, l, b));
 		}
 	}
@@ -243,10 +243,10 @@ vsl_k_arg(struct VSM_data *vd, const char *opt)
 	char *end;
 
 	CHECK_OBJ_NOTNULL(vd, VSM_MAGIC);
-	if (*opt == '\0') 
+	if (*opt == '\0')
 		return (vsm_diag(vd, "number required for -k\n"));
 	vsl->keep = strtoul(opt, &end, 10);
-	if (*end != '\0') 
+	if (*end != '\0')
 		return (vsm_diag(vd, "invalid number for -k\n"));
 	return (1);
 }
diff --git a/lib/libvarnishapi/vsm.c b/lib/libvarnishapi/vsm.c
index a9f176c..18de315 100644
--- a/lib/libvarnishapi/vsm.c
+++ b/lib/libvarnishapi/vsm.c
@@ -86,6 +86,7 @@ vsm_diag(struct VSM_data *vd, const char *fmt, ...)
 	if (vd->diag == NULL)
 		vd->diag = VSB_new_auto();
 	AN(vd->diag);
+	VSB_clear(vd->diag);
 	va_start(ap, fmt);
 	VSB_vprintf(vd->diag, fmt, ap);
 	va_end(ap);
@@ -328,6 +329,8 @@ VSM_StillValid(const struct VSM_data *vd, struct VSM_fantom *vf)
 
 	CHECK_OBJ_NOTNULL(vd, VSM_MAGIC);
 	AN(vf);
+	if (!vd->head)
+		return (0);
 	if (!vd->head->alloc_seq)
 		return (0);
 	if (vf->priv == vd->head->alloc_seq)



From geoff at varnish-cache.org  Mon Jan  9 20:51:56 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:51:56 +0100
Subject: [experimental-ims] ac393c4 Revamp the code which rides herd on the
	thread pools:
Message-ID: <E1RkMCC-00071y-VF@project.varnish-software.com>

commit ac393c4b825a23abc428b4226920a0adc8bec67e
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sun Sep 18 22:48:01 2011 +0000

    Revamp the code which rides herd on the thread pools:
    
    Make one single thread, which creates thread pools, and since that
    is not particularly ardous work, also have it maintain the stats
    gauge of number of sessions queued.
    
    (Removing pools should now be possible, but the code is not there yet.)
    
    Each pool has its own herder thread, which creates and destroys
    threads as required, and updates stats for dropped and queued
    sessions.
    
    Sanitize VSC fields associated with this area.

diff --git a/bin/varnishd/cache.h b/bin/varnishd/cache.h
index 9de746c..4f18a08 100644
--- a/bin/varnishd/cache.h
+++ b/bin/varnishd/cache.h
@@ -821,7 +821,7 @@ void Lck__Assert(const struct lock *lck, int held);
 /* public interface: */
 void LCK_Init(void);
 void Lck_Delete(struct lock *lck);
-void Lck_CondWait(pthread_cond_t *cond, struct lock *lck);
+int Lck_CondWait(pthread_cond_t *cond, struct lock *lck, struct timespec *ts);
 
 #define Lck_New(a, b) Lck__New(a, b, #b)
 #define Lck_Lock(a) Lck__Lock(a, __func__, __FILE__, __LINE__)
diff --git a/bin/varnishd/cache_lck.c b/bin/varnishd/cache_lck.c
index 3e0c640..b9229b3 100644
--- a/bin/varnishd/cache_lck.c
+++ b/bin/varnishd/cache_lck.c
@@ -139,19 +139,26 @@ Lck__Assert(const struct lock *lck, int held)
 		    !pthread_equal(ilck->owner, pthread_self()));
 }
 
-void __match_proto__()
-Lck_CondWait(pthread_cond_t *cond, struct lock *lck)
+int __match_proto__()
+Lck_CondWait(pthread_cond_t *cond, struct lock *lck, struct timespec *ts)
 {
 	struct ilck *ilck;
+	int retval = 0;
 
 	CAST_OBJ_NOTNULL(ilck, lck->priv, ILCK_MAGIC);
 	AN(ilck->held);
 	assert(pthread_equal(ilck->owner, pthread_self()));
 	ilck->held = 0;
-	AZ(pthread_cond_wait(cond, &ilck->mtx));
+	if (ts == NULL) {
+		AZ(pthread_cond_wait(cond, &ilck->mtx));
+	} else {
+		retval = pthread_cond_timedwait(cond, &ilck->mtx, ts);
+		assert(retval == 0 || retval == ETIMEDOUT);
+	}
 	AZ(ilck->held);
 	ilck->held = 1;
 	ilck->owner = pthread_self();
+	return (retval);
 }
 
 void
diff --git a/bin/varnishd/cache_pool.c b/bin/varnishd/cache_pool.c
index b593617..2aeb1b9 100644
--- a/bin/varnishd/cache_pool.c
+++ b/bin/varnishd/cache_pool.c
@@ -75,6 +75,11 @@ struct pool {
 	unsigned		magic;
 #define POOL_MAGIC		0x606658fa
 	VTAILQ_ENTRY(pool)	list;
+
+	pthread_cond_t		herder_cond;
+	struct lock		herder_mtx;
+	pthread_t		herder_thr;
+
 	struct lock		mtx;
 	struct workerhead	idle;
 	VTAILQ_HEAD(, sess)	queue;
@@ -82,20 +87,13 @@ struct pool {
 	unsigned		nthr;
 	unsigned		lqueue;
 	unsigned		last_lqueue;
-	uintmax_t		ndrop;
-	uintmax_t		nqueue;
+	uintmax_t		ndropped;
+	uintmax_t		nqueued;
 	struct sesspool		*sesspool;
 };
 
-static VTAILQ_HEAD(,pool)	pools = VTAILQ_HEAD_INITIALIZER(pools);
-
-static unsigned			queue_max;
-static unsigned			nthr_max;
-
-static unsigned			nwq;
-
-static pthread_cond_t		herder_cond;
-static struct lock		herder_mtx;
+static struct lock		pool_mtx;
+static pthread_t		thr_pool_herder;
 
 /*--------------------------------------------------------------------
  * Nobody is accepting on this socket, so we do.
@@ -163,7 +161,6 @@ Pool_Work_Thread(void *priv, struct worker *w)
 	CAST_OBJ_NOTNULL(pp, priv, POOL_MAGIC);
 	w->pool = pp;
 	Lck_Lock(&pp->mtx);
-	pp->nthr++;
 	stats_clean = 1;
 	while (1) {
 
@@ -196,7 +193,7 @@ Pool_Work_Thread(void *priv, struct worker *w)
 			VTAILQ_INSERT_HEAD(&pp->idle, w, list);
 			if (!stats_clean)
 				WRK_SumStat(w);
-			Lck_CondWait(&w->cond, &pp->mtx);
+			(void)Lck_CondWait(&w->cond, &pp->mtx, NULL);
 		}
 
 		/*
@@ -251,8 +248,6 @@ Pool_Work_Thread(void *priv, struct worker *w)
 		stats_clean = WRK_TrySumStat(w);
 		Lck_Lock(&pp->mtx);
 	}
-	assert(pp->nthr > 0);
-	pp->nthr--;
 	Lck_Unlock(&pp->mtx);
 	w->pool = NULL;
 }
@@ -281,17 +276,17 @@ WRK_Queue(struct pool *pp, struct sess *sp)
 	}
 
 	/* If we have too much in the queue already, refuse. */
-	if (pp->lqueue > queue_max) {
-		pp->ndrop++;
+	if (pp->lqueue > (params->queue_max * pp->nthr) / 100) {
+		pp->ndropped++;
 		Lck_Unlock(&pp->mtx);
 		return (-1);
 	}
 
 	VTAILQ_INSERT_TAIL(&pp->queue, sp, poollist);
-	pp->nqueue++;
+	pp->nqueued++;
 	pp->lqueue++;
 	Lck_Unlock(&pp->mtx);
-	AZ(pthread_cond_signal(&herder_cond));
+	AZ(pthread_cond_signal(&pp->herder_cond));
 	return (0);
 }
 
@@ -325,149 +320,24 @@ Pool_Schedule(struct pool *pp, struct sess *sp)
 }
 
 /*--------------------------------------------------------------------
- * Add (more) thread pools
- */
-
-static struct pool *
-pool_mkpool(void)
-{
-	struct pool *pp;
-	struct listen_sock *ls;
-	struct poolsock *ps;
-
-	ALLOC_OBJ(pp, POOL_MAGIC);
-	XXXAN(pp);
-	Lck_New(&pp->mtx, lck_wq);
-	VTAILQ_INIT(&pp->queue);
-	VTAILQ_INIT(&pp->idle);
-	VTAILQ_INIT(&pp->socks);
-	pp->sesspool = SES_NewPool(pp);
-	AN(pp->sesspool);
-
-	VTAILQ_FOREACH(ls, &heritage.socks, list) {
-		if (ls->sock < 0)
-			continue;
-		ALLOC_OBJ(ps, POOLSOCK_MAGIC);
-		XXXAN(ps);
-		ps->lsock = ls;
-		VTAILQ_INSERT_TAIL(&pp->socks, ps, list);
-	}
-	VTAILQ_INSERT_TAIL(&pools, pp, list);
-	return (pp);
-}
-
-static void
-wrk_addpools(const unsigned npools)
-{
-	struct pool *pp;
-	unsigned u;
-
-	for (u = nwq; u < npools; u++) {
-		pp = pool_mkpool();
-		XXXAN(pp);
-	}
-	nwq = npools;
-}
-
-/*--------------------------------------------------------------------
- * If a thread is idle or excess, pick it out of the pool.
- */
-
-static void
-wrk_decimate_flock(struct pool *qp, double t_idle, struct VSC_C_main *vs)
-{
-	struct worker *w = NULL;
-
-	Lck_Lock(&qp->mtx);
-	vs->n_wrk += qp->nthr;
-	vs->n_wrk_lqueue += qp->lqueue;
-	vs->n_wrk_drop += qp->ndrop;
-	vs->n_wrk_queued += qp->nqueue;
-
-	if (qp->nthr > params->wthread_min) {
-		w = VTAILQ_LAST(&qp->idle, workerhead);
-		if (w != NULL && (w->lastused < t_idle || qp->nthr > nthr_max))
-			VTAILQ_REMOVE(&qp->idle, w, list);
-		else
-			w = NULL;
-	}
-	Lck_Unlock(&qp->mtx);
-
-	/* And give it a kiss on the cheek... */
-	if (w != NULL) {
-		AZ(w->sp);
-		AZ(pthread_cond_signal(&w->cond));
-		TIM_sleep(params->wthread_purge_delay * 1e-3);
-	}
-}
-
-/*--------------------------------------------------------------------
- * Periodic pool herding thread
- *
- * Do things which we can do at our leisure:
- *  Add pools
- *  Scale constants
- *  Get rid of excess threads
- *  Aggregate stats across pools
+ * Wait for another request
  */
 
-static void *
-wrk_herdtimer_thread(void *priv)
+void
+Pool_Wait(struct sess *sp)
 {
-	volatile unsigned u;
-	double t_idle;
-	struct VSC_C_main vsm, *vs;
-	int errno_is_multi_threaded;
-	struct pool *pp;
-
-	THR_SetName("wrk_herdtimer");
 
+	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+	AZ(sp->obj);
+	AZ(sp->vcl);
+	assert(sp->fd >= 0);
 	/*
-	 * This is one of the first threads created, test to see that
-	 * errno is really per thread.  If this fails, your C-compiler
-	 * needs some magic argument (-mt, -pthread, -pthreads etc etc).
+	 * Set nonblocking in the worker-thread, before passing to the
+	 * acceptor thread, to reduce syscall density of the latter.
 	 */
-	errno = 0;
-	AN(unlink("/"));		/* This had better fail */
-	errno_is_multi_threaded = errno;
-	assert(errno_is_multi_threaded != 0);
-
-	memset(&vsm, 0, sizeof vsm);
-	vs = &vsm;
-
-	(void)priv;
-	while (1) {
-		/* Add Pools */
-		u = params->wthread_pools;
-		if (u > nwq)
-			wrk_addpools(u);
-
-		/* Scale parameters */
-
-		u = params->wthread_max;
-		if (u < params->wthread_min)
-			u = params->wthread_min;
-		nthr_max = u;
-
-		queue_max = (nthr_max * params->queue_max) / 100;
-
-		vs->n_wrk = 0;
-		vs->n_wrk_lqueue = 0;
-		vs->n_wrk_drop = 0;
-		vs->n_wrk_queued = 0;
-
-		t_idle = TIM_real() - params->wthread_timeout;
-		VTAILQ_FOREACH(pp, &pools, list)
-			wrk_decimate_flock(pp, t_idle, vs);
-
-		VSC_C_main->n_wrk= vs->n_wrk;
-		VSC_C_main->n_wrk_lqueue = vs->n_wrk_lqueue;
-		VSC_C_main->n_wrk_drop = vs->n_wrk_drop;
-		VSC_C_main->n_wrk_queued = vs->n_wrk_queued;
-
-		TIM_sleep(params->wthread_purge_delay * 1e-3);
-	}
-	NEEDLESS_RETURN(NULL);
+	if (VTCP_nonblocking(sp->fd))
+		SES_Close(sp, "remote closed");
+	waiter->pass(waiter_priv, sp);
 }
 
 /*--------------------------------------------------------------------
@@ -486,23 +356,33 @@ wrk_breed_flock(struct pool *qp, const pthread_attr_t *tp_attr)
 	if (qp->nthr < params->wthread_min ||	/* Not enough threads yet */
 	    (qp->lqueue > params->wthread_add_threshold && /* more needed */
 	    qp->lqueue > qp->last_lqueue)) {	/* not getting better since last */
-		if (qp->nthr >= nthr_max) {
-			VSC_C_main->n_wrk_max++;
+		if (qp->nthr > params->wthread_max) {
+			Lck_Lock(&pool_mtx);
+			VSC_C_main->threads_limited++;
+			Lck_Unlock(&pool_mtx);
 		} else if (pthread_create(&tp, tp_attr, WRK_thread, qp)) {
 			VSL(SLT_Debug, 0, "Create worker thread failed %d %s",
 			    errno, strerror(errno));
-			VSC_C_main->n_wrk_failed++;
+			Lck_Lock(&pool_mtx);
+			VSC_C_main->threads_limited++;
+			Lck_Unlock(&pool_mtx);
 			TIM_sleep(params->wthread_fail_delay * 1e-3);
 		} else {
 			AZ(pthread_detach(tp));
-			VSC_C_main->n_wrk_create++;
 			TIM_sleep(params->wthread_add_delay * 1e-3);
+			qp->nthr++;
+			Lck_Lock(&pool_mtx);
+			VSC_C_main->threads++;
+			VSC_C_main->threads_created++;
+			Lck_Unlock(&pool_mtx);
 		}
 	}
 	qp->last_lqueue = qp->lqueue;
 }
 
 /*--------------------------------------------------------------------
+ * Herd a single pool
+ *
  * This thread wakes up whenever a pool queues.
  *
  * The trick here is to not be too aggressive about creating threads.
@@ -515,62 +395,154 @@ wrk_breed_flock(struct pool *qp, const pthread_attr_t *tp_attr)
  *
  */
 
-static void *
-wrk_herder_thread(void *priv)
+static void*
+pool_herder(void *priv)
 {
+	struct pool *pp;
 	pthread_attr_t tp_attr;
-	struct pool *pp, *pp2;
+	struct timespec ts;
+	double t_idle;
+	struct worker *w;
+	int i;
 
-	/* Set the stacksize for worker threads */
+	CAST_OBJ_NOTNULL(pp, priv, POOL_MAGIC);
 	AZ(pthread_attr_init(&tp_attr));
 
-	THR_SetName("wrk_herder");
-	(void)priv;
 	while (1) {
-		VTAILQ_FOREACH(pp, &pools, list) {
-			if (params->wthread_stacksize != UINT_MAX)
-				AZ(pthread_attr_setstacksize(&tp_attr,
-				    params->wthread_stacksize));
-
-			wrk_breed_flock(pp, &tp_attr);
-
-			/*
-			 * Make sure all pools have their minimum complement
-			 */
-			VTAILQ_FOREACH(pp2, &pools, list)
-				while (pp2->nthr < params->wthread_min)
-					wrk_breed_flock(pp2, &tp_attr);
-			/*
-			 * We cannot avoid getting a mutex, so we have a
-			 * bogo mutex just for POSIX_STUPIDITY
-			 */
-			Lck_Lock(&herder_mtx);
-			Lck_CondWait(&herder_cond, &herder_mtx);
-			Lck_Unlock(&herder_mtx);
+		/* Set the stacksize for worker threads we create */
+		if (params->wthread_stacksize != UINT_MAX)
+			AZ(pthread_attr_setstacksize(&tp_attr,
+			    params->wthread_stacksize));
+		else {
+			AZ(pthread_attr_destroy(&tp_attr));
+			AZ(pthread_attr_init(&tp_attr));
+		}
+
+		wrk_breed_flock(pp, &tp_attr);
+		
+		if (pp->nthr < params->wthread_min)
+			continue;
+
+		AZ(clock_gettime(CLOCK_MONOTONIC, &ts));
+		ts.tv_sec += params->wthread_purge_delay / 1000;
+		ts.tv_nsec += 
+		    (params->wthread_purge_delay % 1000) * 1000000;
+		if (ts.tv_nsec >= 1000000000) {
+			ts.tv_sec++;
+			ts.tv_nsec -= 1000000000;
+		}
+
+		Lck_Lock(&pp->herder_mtx);
+		i = Lck_CondWait(&pp->herder_cond, &pp->herder_mtx, &ts);
+		Lck_Unlock(&pp->herder_mtx);
+		if (!i)
+			continue;
+
+		if (pp->nthr <= params->wthread_min) 
+			continue;
+
+		t_idle = TIM_real() - params->wthread_timeout;
+
+		Lck_Lock(&pp->mtx);
+		VSC_C_main->sess_queued += pp->nqueued;
+		VSC_C_main->sess_dropped += pp->ndropped;
+		pp->nqueued = pp->ndropped = 0;
+		w = VTAILQ_LAST(&pp->idle, workerhead);
+		if (w != NULL &&  
+		    (w->lastused < t_idle || pp->nthr > params->wthread_max)) {
+			VTAILQ_REMOVE(&pp->idle, w, list);
+		} else
+			w = NULL;
+		Lck_Unlock(&pp->mtx);
+
+		/* And give it a kiss on the cheek... */
+		if (w != NULL) {
+			pp->nthr--;
+			Lck_Lock(&pool_mtx);
+			VSC_C_main->threads--;
+			VSC_C_main->threads_destroyed++;
+			Lck_Unlock(&pool_mtx);
+			AZ(w->sp);
+			AZ(pthread_cond_signal(&w->cond));
 		}
 	}
-	NEEDLESS_RETURN(NULL);
 }
 
 /*--------------------------------------------------------------------
- * Wait for another request
+ * Add a thread pool
  */
 
-void
-Pool_Wait(struct sess *sp)
+static struct pool *
+pool_mkpool(void)
 {
+	struct pool *pp;
+	struct listen_sock *ls;
+	struct poolsock *ps;
+	pthread_condattr_t cv_attr;
 
-	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
-	AZ(sp->obj);
-	AZ(sp->vcl);
-	assert(sp->fd >= 0);
-	/*
-	 * Set nonblocking in the worker-thread, before passing to the
-	 * acceptor thread, to reduce syscall density of the latter.
-	 */
-	if (VTCP_nonblocking(sp->fd))
-		SES_Close(sp, "remote closed");
-	waiter->pass(waiter_priv, sp);
+	ALLOC_OBJ(pp, POOL_MAGIC);
+	XXXAN(pp);
+	Lck_New(&pp->mtx, lck_wq);
+
+	VTAILQ_INIT(&pp->queue);
+	VTAILQ_INIT(&pp->idle);
+	VTAILQ_INIT(&pp->socks);
+	pp->sesspool = SES_NewPool(pp);
+	AN(pp->sesspool);
+
+	VTAILQ_FOREACH(ls, &heritage.socks, list) {
+		if (ls->sock < 0)
+			continue;
+		ALLOC_OBJ(ps, POOLSOCK_MAGIC);
+		XXXAN(ps);
+		ps->lsock = ls;
+		VTAILQ_INSERT_TAIL(&pp->socks, ps, list);
+	}
+
+	AZ(pthread_condattr_init(&cv_attr));
+	AZ(pthread_condattr_setclock(&cv_attr, CLOCK_MONOTONIC));
+	AZ(pthread_cond_init(&pp->herder_cond, &cv_attr));
+	AZ(pthread_condattr_destroy(&cv_attr));
+	Lck_New(&pp->herder_mtx, lck_herder);
+	AZ(pthread_create(&pp->herder_thr, NULL, pool_herder, pp));
+
+	return (pp);
+}
+
+/*--------------------------------------------------------------------
+ * This thread adjusts the number of pools to match the parameter.
+ *
+ */
+
+static void *
+pool_poolherder(void *priv)
+{
+	unsigned nwq;
+	VTAILQ_HEAD(,pool)	pools = VTAILQ_HEAD_INITIALIZER(pools);
+	struct pool *pp;
+	uint64_t u;
+
+	THR_SetName("pool_herder");
+	(void)priv;
+
+	nwq = 0;
+	while (1) {
+		if (nwq < params->wthread_pools) {
+			pp = pool_mkpool();
+			if (pp != NULL) {
+				VTAILQ_INSERT_TAIL(&pools, pp, list);
+				VSC_C_main->pools++;
+				nwq++;
+				continue;
+			} 
+		}
+		/* XXX: remove pools */
+		(void)sleep(1);
+		u = 0;
+		VTAILQ_FOREACH(pp, &pools, list) 
+			u += pp->lqueue;
+		VSC_C_main->thread_queue_len = u;
+	}
 }
 
 /*--------------------------------------------------------------------*/
@@ -578,18 +550,10 @@ Pool_Wait(struct sess *sp)
 void
 Pool_Init(void)
 {
-	pthread_t tp;
-
-	AZ(pthread_cond_init(&herder_cond, NULL));
-	Lck_New(&herder_mtx, lck_herder);
 
 	waiter_priv = waiter->init();
-
-	wrk_addpools(params->wthread_pools);
-	AZ(pthread_create(&tp, NULL, wrk_herdtimer_thread, NULL));
-	AZ(pthread_detach(tp));
-	AZ(pthread_create(&tp, NULL, wrk_herder_thread, NULL));
-	AZ(pthread_detach(tp));
+	Lck_New(&pool_mtx, lck_wq);
+	AZ(pthread_create(&thr_pool_herder, NULL, pool_poolherder, NULL));
 }
 
 /*--------------------------------------------------------------------*/
diff --git a/bin/varnishtest/tests/c00002.vtc b/bin/varnishtest/tests/c00002.vtc
index 88862b3..6914736 100644
--- a/bin/varnishtest/tests/c00002.vtc
+++ b/bin/varnishtest/tests/c00002.vtc
@@ -16,4 +16,4 @@ client c1 {
 	expect resp.status == 200
 } -run
 
-varnish v1 -expect n_wrk == 8
+varnish v1 -expect threads == 8
diff --git a/include/vsc_fields.h b/include/vsc_fields.h
index fc81919..ea668cc 100644
--- a/include/vsc_fields.h
+++ b/include/vsc_fields.h
@@ -139,6 +139,68 @@ VSC_F(sessmem_limit,		uint64_t, 1, 'c',
 	"Count of session memory allocations blocked by limit (max_sess)."
 )
 
+/*---------------------------------------------------------------------
+ * Pools, threads, and sessions
+ *    see: cache_pool.c
+ *
+ */
+
+VSC_F(pools,			uint64_t, 1, 'g',
+    "Number of thread pools",
+	"Number of thread pools.  See also param wthread_pools."
+	"  NB: Presently pools cannot be removed once created."
+)
+
+VSC_F(threads,			uint64_t, 1, 'g',
+    "Total number of threads",
+	"Number of threads in all pools."
+	"  See also params thread_pools, thread_pool_min & thread_pool_max."
+)
+
+VSC_F(threads_limited,		uint64_t, 1, 'c',
+    "Threads hit max",
+	"Number of times more threads were needed, but limit was reached"
+	" in a thread pool."
+	"  See also param thread_pool_max."
+)
+
+VSC_F(threads_created,		uint64_t, 1, 'c',
+    "Threads created",
+	"Total number of threads created in all pools."
+)
+
+VSC_F(threads_destroyed,	uint64_t, 1, 'c',
+    "Threads destoryed",
+	"Total number of threads destroyed in all pools."
+)
+
+VSC_F(threads_failed,		uint64_t, 1, 'c',
+    "Thread creation failed",
+	"Number of times creating a thread failed."
+	"  See VSL::Debug for diagnostics."
+	"  See also param thread_fail_delay."
+)
+
+VSC_F(thread_queue_len,		uint64_t, 1, 'g',
+    "Length of session queue",
+	"Length of session queue waiting for threads."
+	"  NB: Only updates once per second."
+	"  See also param queue_max."
+)
+
+VSC_F(sess_queued,		uint64_t, 1, 'c',
+    "Sessions queued for thread",
+	"Number of times session was queued waiting for a thread."
+	"  See also param queue_max."
+)
+
+VSC_F(sess_dropped,		uint64_t, 1, 'c',
+    "Sessions dropped for thread",
+	"Number of times session was dropped because the queue were too"
+	" long already."
+	"  See also param queue_max."
+)
+
 /*---------------------------------------------------------------------*/
 
 VSC_F(n_sess_mem,		uint64_t, 0, 'i', "N struct sess_mem", "")
@@ -150,14 +212,7 @@ VSC_F(n_objecthead,		uint64_t, 1, 'i', "N struct objecthead", "")
 VSC_F(n_waitinglist,		uint64_t, 1, 'i', "N struct waitinglist", "")
 
 VSC_F(n_vbc,		uint64_t, 0, 'i', "N struct vbc", "")
-VSC_F(n_wrk,		uint64_t, 0, 'i', "N worker threads", "")
-VSC_F(n_wrk_create,	uint64_t, 0, 'a', "N worker threads created", "")
-VSC_F(n_wrk_failed,	uint64_t, 0, 'a',
-					"N worker threads not created", "")
-VSC_F(n_wrk_max,		uint64_t, 0, 'a', "N worker threads limited", "")
-VSC_F(n_wrk_lqueue,		uint64_t, 0, 'a', "work request queue length", "")
-VSC_F(n_wrk_queued,		uint64_t, 0, 'a', "N queued work requests", "")
-VSC_F(n_wrk_drop,		uint64_t, 0, 'a', "N dropped work requests", "")
+
 VSC_F(n_backend,		uint64_t, 0, 'i', "N backends", "")
 
 VSC_F(n_expired,		uint64_t, 0, 'i', "N expired objects", "")



From geoff at varnish-cache.org  Mon Jan  9 20:52:29 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:29 +0100
Subject: [experimental-ims] 8f304ab Merge branch 'master' of
	ssh://git.varnish-cache.org/git/varnish-cache
Message-ID: <E1RkMCj-00084d-V0@project.varnish-software.com>

commit 8f304abc63cf0d3e88fa634c8e62ead5014a02f4
Merge: 6afe314 ae14008
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Nov 7 12:54:09 2011 +0000

    Merge branch 'master' of ssh://git.varnish-cache.org/git/varnish-cache



From geoff at varnish-cache.org  Mon Jan  9 20:51:56 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:51:56 +0100
Subject: [experimental-ims] 56c353f Give session memory pools a back pointer
	to their (thread) pool.
Message-ID: <E1RkMCC-00071T-P9@project.varnish-software.com>

commit 56c353f7242d483ad710a0599f6d0000d16126dc
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sun Sep 18 20:09:43 2011 +0000

    Give session memory pools a back pointer to their (thread) pool.
    
    Schedule sessions from busy-lists and from the waiter via
    SES_Schedule(), to Pool_Schedule() in the right pool.

diff --git a/bin/varnishd/cache.h b/bin/varnishd/cache.h
index d5e4311..9de746c 100644
--- a/bin/varnishd/cache.h
+++ b/bin/varnishd/cache.h
@@ -841,9 +841,9 @@ void PipeSession(struct sess *sp);
 
 /* cache_pool.c */
 void Pool_Init(void);
-int Pool_QueueSession(struct sess *sp);
 void Pool_Work_Thread(void *priv, struct worker *w);
 void Pool_Wait(struct sess *sp);
+int Pool_Schedule(struct pool *pp, struct sess *sp);
 
 #define WRW_IsReleased(w)	((w)->wrw.wfd == NULL)
 int WRW_Error(const struct worker *w);
@@ -864,8 +864,10 @@ struct sess *SES_Alloc(void);
 void SES_Close(struct sess *sp, const char *reason);
 void SES_Delete(struct sess *sp, const char *reason);
 void SES_Charge(struct sess *sp);
-struct sesspool *SES_NewPool(void);
+struct sesspool *SES_NewPool(struct pool *pp);
 void SES_DeletePool(struct sesspool *sp, struct worker *wrk);
+int SES_Schedule(struct sess *sp);
+
 
 /* cache_shmlog.c */
 void VSL_Init(void);
diff --git a/bin/varnishd/cache_hash.c b/bin/varnishd/cache_hash.c
index 8e0fd4e..faf867a 100644
--- a/bin/varnishd/cache_hash.c
+++ b/bin/varnishd/cache_hash.c
@@ -504,7 +504,7 @@ hsh_rush(struct objhead *oh)
 		AZ(sp->wrk);
 		VTAILQ_REMOVE(&wl->list, sp, list);
 		DSL(0x20, SLT_Debug, sp->id, "off waiting list");
-		if (Pool_QueueSession(sp)) {
+		if (SES_Schedule(sp)) {
 			/*
 			 * We could not schedule the session, leave the
 			 * rest on the busy list.
diff --git a/bin/varnishd/cache_pool.c b/bin/varnishd/cache_pool.c
index 64aad05..f8c6871 100644
--- a/bin/varnishd/cache_pool.c
+++ b/bin/varnishd/cache_pool.c
@@ -261,47 +261,33 @@ Pool_Work_Thread(void *priv, struct worker *w)
  */
 
 static int
-WRK_Queue(struct sess *sp)
+WRK_Queue(struct pool *pp, struct sess *sp)
 {
 	struct worker *w;
-	struct pool *qp;
-	static unsigned nq = 0;
-	unsigned onq;
 
-	/*
-	 * Select which pool we issue to
-	 * XXX: better alg ?
-	 * XXX: per CPU ?
-	 */
-	onq = nq + 1;
-	if (onq >= nwq)
-		onq = 0;
-	qp = wq[onq];
-	nq = onq;
-
-	Lck_Lock(&qp->mtx);
+	Lck_Lock(&pp->mtx);
 
 	/* If there are idle threads, we tickle the first one into action */
-	w = VTAILQ_FIRST(&qp->idle);
+	w = VTAILQ_FIRST(&pp->idle);
 	if (w != NULL) {
-		VTAILQ_REMOVE(&qp->idle, w, list);
-		Lck_Unlock(&qp->mtx);
+		VTAILQ_REMOVE(&pp->idle, w, list);
+		Lck_Unlock(&pp->mtx);
 		w->sp = sp;
 		AZ(pthread_cond_signal(&w->cond));
 		return (0);
 	}
 
 	/* If we have too much in the queue already, refuse. */
-	if (qp->lqueue > queue_max) {
-		qp->ndrop++;
-		Lck_Unlock(&qp->mtx);
+	if (pp->lqueue > queue_max) {
+		pp->ndrop++;
+		Lck_Unlock(&pp->mtx);
 		return (-1);
 	}
 
-	VTAILQ_INSERT_TAIL(&qp->queue, sp, poollist);
-	qp->nqueue++;
-	qp->lqueue++;
-	Lck_Unlock(&qp->mtx);
+	VTAILQ_INSERT_TAIL(&pp->queue, sp, poollist);
+	pp->nqueue++;
+	pp->lqueue++;
+	Lck_Unlock(&pp->mtx);
 	AZ(pthread_cond_signal(&herder_cond));
 	return (0);
 }
@@ -309,11 +295,11 @@ WRK_Queue(struct sess *sp)
 /*--------------------------------------------------------------------*/
 
 int
-Pool_QueueSession(struct sess *sp)
+Pool_Schedule(struct pool *pp, struct sess *sp)
 {
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 	AZ(sp->wrk);
-	if (WRK_Queue(sp) == 0)
+	if (WRK_Queue(pp, sp) == 0)
 		return(0);
 
 	VSC_C_main->client_drop_late++;
@@ -332,7 +318,6 @@ Pool_QueueSession(struct sess *sp)
 		 */
 		VCL_Rel(&sp->vcl);
 	}
-	SES_Delete(sp, "dropped");
 	return (1);
 }
 
@@ -353,7 +338,7 @@ pool_mkpool(void)
 	VTAILQ_INIT(&pp->queue);
 	VTAILQ_INIT(&pp->idle);
 	VTAILQ_INIT(&pp->socks);
-	pp->sesspool = SES_NewPool();
+	pp->sesspool = SES_NewPool(pp);
 	AN(pp->sesspool);
 
 	VTAILQ_FOREACH(ls, &heritage.socks, list) {
diff --git a/bin/varnishd/cache_session.c b/bin/varnishd/cache_session.c
index 23d15e7..e8352f6 100644
--- a/bin/varnishd/cache_session.c
+++ b/bin/varnishd/cache_session.c
@@ -64,6 +64,7 @@ struct sessmem {
 struct sesspool {
 	unsigned		magic;
 #define SESSPOOL_MAGIC		0xd916e202
+	struct pool		*pool;
 	VTAILQ_HEAD(,sessmem)	freelist;
 	struct lock		mtx;
 	unsigned		nsess;
@@ -237,6 +238,35 @@ SES_Alloc(void)
 }
 
 /*--------------------------------------------------------------------
+ * Schedule a session back on a work-thread from its pool
+ */
+
+int
+SES_Schedule(struct sess *sp)
+{
+	struct sessmem *sm;
+	struct sesspool *pp;
+
+	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+
+	AZ(sp->wrk);
+
+	sm = sp->mem;
+	CHECK_OBJ_NOTNULL(sm, SESSMEM_MAGIC);
+
+	pp = sm->pool;
+	CHECK_OBJ_NOTNULL(pp, SESSPOOL_MAGIC);
+
+	AN(pp->pool);
+
+	if (Pool_Schedule(pp->pool, sp)) {
+		SES_Delete(sp, "dropped");
+		return (1);
+	}
+	return (0);
+}
+
+/*--------------------------------------------------------------------
  * Handle a session (from waiter)
  *
  * Status: see HTC_Rx()
@@ -255,7 +285,7 @@ SES_Handle(struct sess *sp, int status)
 		break;
 	case 1:
 		sp->step = STP_START;
-		(void)Pool_QueueSession(sp);
+		(void)SES_Schedule(sp);
 		break;
 	default:
 		WRONG("Unexpected return from HTC_Rx()");
@@ -355,12 +385,13 @@ SES_Delete(struct sess *sp, const char *reason)
  */
 
 struct sesspool *
-SES_NewPool(void)
+SES_NewPool(struct pool *pp)
 {
 	struct sesspool *sp;
 
 	ALLOC_OBJ(sp, SESSPOOL_MAGIC);
 	AN(sp);
+	sp->pool = pp;
 	VTAILQ_INIT(&sp->freelist);
 	Lck_New(&sp->mtx, lck_sessmem);
 	return (sp);



From geoff at varnish-cache.org  Mon Jan  9 20:52:24 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:24 +0100
Subject: [experimental-ims] fa3b136 Ensure ban lurker sleeps 1.0s on failure
Message-ID: <E1RkMCe-0007sa-Np@project.varnish-software.com>

commit fa3b136f2169a71b63603835c69441ca37913507
Author: Kristian Lyngstol <kristian at bohemians.org>
Date:   Mon Oct 17 14:09:11 2011 +0200

    Ensure ban lurker sleeps 1.0s on failure
    
    As per documentation, the ban lurker sleeps ban_lurker_sleep when it is
    successful, but on failure it should only sleep 1.0s. No point hammering
    the ban list every 0.01s if bans aren't even used.
    
    Fixes #1030

diff --git a/bin/varnishd/cache_ban.c b/bin/varnishd/cache_ban.c
index 736cbf7..5b7d457 100644
--- a/bin/varnishd/cache_ban.c
+++ b/bin/varnishd/cache_ban.c
@@ -754,7 +754,7 @@ BAN_CheckObject(struct object *o, const struct sess *sp)
  * Ban tail lurker thread
  */
 
-static void
+static int
 ban_lurker_work(const struct sess *sp)
 {
 	struct ban *b, *bf;
@@ -773,21 +773,21 @@ ban_lurker_work(const struct sess *sp)
 	if (bf != NULL) {
 		Lck_Unlock(&ban_mtx);
 		BAN_Free(bf);
-		return;
+		return (0);
 	}
 
 	/* Find the last ban give up, if we have only one */
 	b = VTAILQ_LAST(&ban_head, banhead_s);
 	if (b == ban_start) {
 		Lck_Unlock(&ban_mtx);
-		return;
+		return (0);
 	}
 
 	/* Find the first object on it, if any */
 	oc = VTAILQ_FIRST(&b->objcore);
 	if (oc == NULL) {
 		Lck_Unlock(&ban_mtx);
-		return;
+		return (0);
 	}
 
 	/* Try to lock the objhead */
@@ -795,7 +795,7 @@ ban_lurker_work(const struct sess *sp)
 	CHECK_OBJ_NOTNULL(oh, OBJHEAD_MAGIC);
 	if (Lck_Trylock(&oh->mtx)) {
 		Lck_Unlock(&ban_mtx);
-		return;
+		return (0);
 	}
 
 	/*
@@ -808,7 +808,7 @@ ban_lurker_work(const struct sess *sp)
 	if (oc2 == NULL) {
 		Lck_Unlock(&oh->mtx);
 		Lck_Unlock(&ban_mtx);
-		return;
+		return (0);
 	}
 	/*
 	 * Grab a reference to the OC and we can let go of the BAN mutex
@@ -825,12 +825,13 @@ ban_lurker_work(const struct sess *sp)
 	Lck_Unlock(&oh->mtx);
 	WSP(sp, SLT_Debug, "lurker: %p %g %d", oc, o->exp.ttl, i);
 	(void)HSH_Deref(sp->wrk, NULL, &o);
+	return (i);
 }
 
 static void * __match_proto__(bgthread_t)
 ban_lurker(struct sess *sp, void *priv)
 {
-
+	int i = 0;
 	(void)priv;
 	while (1) {
 		if (params->ban_lurker_sleep == 0.0) {
@@ -838,8 +839,11 @@ ban_lurker(struct sess *sp, void *priv)
 			VTIM_sleep(1.0);
 			continue;
 		}
-		VTIM_sleep(params->ban_lurker_sleep);
-		ban_lurker_work(sp);
+		if (i != 0)
+			VTIM_sleep(params->ban_lurker_sleep);
+		else
+			VTIM_sleep(1.0);
+		i = ban_lurker_work(sp);
 		WSL_Flush(sp->wrk, 0);
 		WRK_SumStat(sp->wrk);
 	}
diff --git a/bin/varnishtest/tests/r01030.vtc b/bin/varnishtest/tests/r01030.vtc
new file mode 100644
index 0000000..97ef3d7
--- /dev/null
+++ b/bin/varnishtest/tests/r01030.vtc
@@ -0,0 +1,64 @@
+varnishtest "Test ban_lurker_sleep vs failed ban lurker"
+
+# The idea here is that the ban lurker should always wait 1 second when it
+# can't proceed, as per documentation and original intent. The
+# ban_lurker_sleep should not affect sleep-times when the lurker fails.
+
+server s1 {
+	rxreq
+	txresp -status 200
+
+	rxreq
+	txresp -status 200
+} -start
+
+varnish v1 -vcl+backend {
+	sub vcl_recv {
+		if (req.request == "BAN") {
+			ban("obj.http.url ~ /");
+			error 201 "banned";
+		}
+		return (lookup);
+	}
+	sub vcl_fetch {
+		set beresp.http.url = req.url;
+	}
+} -start
+
+varnish v1 -cliok "param.set ban_lurker_sleep 0.01"
+varnish v1 -expect n_ban_obj_test == 0
+
+delay 0.01
+client c1 {
+	txreq -req GET
+	rxresp
+	expect resp.status == 200
+
+	txreq -req BAN
+	rxresp
+	expect resp.status == 201
+} -run
+
+delay 0.1
+varnish v1 -expect n_ban_obj_test == 0
+
+delay 1.0
+varnish v1 -expect n_ban_obj_test == 1
+
+varnish v1 -cliok "param.set ban_lurker_sleep 5.01"
+
+client c2 {
+	txreq -req GET
+	rxresp
+	expect resp.status == 200
+
+	txreq -req BAN
+	rxresp
+	expect resp.status == 201
+} -run
+
+delay 0.1
+varnish v1 -expect n_ban_obj_test == 1
+
+delay 1.1
+varnish v1 -expect n_ban_obj_test == 2



From geoff at varnish-cache.org  Mon Jan  9 20:52:41 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:41 +0100
Subject: [experimental-ims] 852d2a4 Add missing #include
Message-ID: <E1RkMCv-0008PD-Ru@project.varnish-software.com>

commit 852d2a42a35e47c770d03e95c03e2254ff405597
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Nov 23 20:15:40 2011 +0000

    Add missing #include

diff --git a/bin/varnishd/mgt/mgt_sandbox_solaris.c b/bin/varnishd/mgt/mgt_sandbox_solaris.c
index 79f6650..114d6a4 100644
--- a/bin/varnishd/mgt/mgt_sandbox_solaris.c
+++ b/bin/varnishd/mgt/mgt_sandbox_solaris.c
@@ -45,6 +45,7 @@
 #include "mgt/mgt.h"
 
 #include "common/heritage.h"
+#include "common/params.h"
 
 /*--------------------------------------------------------------------
  * SOLARIS PRIVILEGES: Note on use of symbolic PRIV_* constants



From geoff at varnish-cache.org  Mon Jan  9 20:52:01 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:01 +0100
Subject: [experimental-ims] e0ee2a2 Add file_read to the privilege set we need
	on Solaris
Message-ID: <E1RkMCH-0007Af-09@project.varnish-software.com>

commit e0ee2a2e69654a9df74aaf3dcadc9639659cf42b
Author: Tollef Fog Heen <tfheen at varnish-software.com>
Date:   Wed Sep 21 13:52:03 2011 +0200

    Add file_read to the privilege set we need on Solaris
    
    Fixes: #912

diff --git a/bin/varnishd/mgt_sandbox.c b/bin/varnishd/mgt_sandbox.c
index 0299cfd..a5eee2f 100644
--- a/bin/varnishd/mgt_sandbox.c
+++ b/bin/varnishd/mgt_sandbox.c
@@ -98,6 +98,7 @@ mgt_sandbox(void)
 		 * silently ignore any errors if it doesn't exist
 		 */
 		priv_addset(minimal, "net_access");
+		priv_addset(minimal, "file_read");
 
 #define SETPPRIV(which, set)						\
 		if (setppriv(PRIV_SET, which, set))			\



From geoff at varnish-cache.org  Mon Jan  9 20:52:43 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:43 +0100
Subject: [experimental-ims] 58dc8e5 Introduce a consistent wrk local variable
	with in all cnt_*() steps.
Message-ID: <E1RkMCx-0008TS-KM@project.varnish-software.com>

commit 58dc8e5326adeab4af6b1172dbf8bd2dabb327ef
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Nov 29 17:00:41 2011 +0000

    Introduce a consistent wrk local variable with in all cnt_*()
    steps.
    
    A couple of minor polishishes.

diff --git a/bin/varnishd/cache/cache_center.c b/bin/varnishd/cache/cache_center.c
index 8f9e527..85224b6 100644
--- a/bin/varnishd/cache/cache_center.c
+++ b/bin/varnishd/cache/cache_center.c
@@ -98,10 +98,13 @@ cnt_wait(struct sess *sp)
 {
 	int i;
 	struct pollfd pfd[1];
+	struct worker *wrk;
 
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+	wrk = sp->wrk;
+	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 	AZ(sp->vcl);
-	AZ(sp->wrk->obj);
+	AZ(wrk->obj);
 	AZ(sp->esi_level);
 	assert(sp->xid == 0);
 
@@ -116,9 +119,8 @@ cnt_wait(struct sess *sp)
 	}
 	if (i == 0) {
 		WSP(sp, SLT_Debug, "herding");
-		sp->wrk->stats.sess_herd++;
+		wrk->stats.sess_herd++;
 		SES_Charge(sp);
-		sp->wrk = NULL;
 		Pool_Wait(sp);
 		return (1);
 	}
@@ -168,66 +170,70 @@ DOT }
 static int
 cnt_prepresp(struct sess *sp)
 {
+	struct worker *wrk;
 
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
-	CHECK_OBJ_NOTNULL(sp->wrk->obj, OBJECT_MAGIC);
+	wrk = sp->wrk;
+	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
+
+	CHECK_OBJ_NOTNULL(wrk->obj, OBJECT_MAGIC);
 	CHECK_OBJ_NOTNULL(sp->vcl, VCL_CONF_MAGIC);
 
-	if (sp->wrk->do_stream)
-		AssertObjCorePassOrBusy(sp->wrk->obj->objcore);
+	if (wrk->do_stream)
+		AssertObjCorePassOrBusy(wrk->obj->objcore);
 
-	sp->wrk->res_mode = 0;
+	wrk->res_mode = 0;
 
-	if ((sp->wrk->h_content_length != NULL || !sp->wrk->do_stream) &&
-	    !sp->wrk->do_gzip && !sp->wrk->do_gunzip)
-		sp->wrk->res_mode |= RES_LEN;
+	if ((wrk->h_content_length != NULL || !wrk->do_stream) &&
+	    !wrk->do_gzip && !wrk->do_gunzip)
+		wrk->res_mode |= RES_LEN;
 
-	if (!sp->disable_esi && sp->wrk->obj->esidata != NULL) {
+	if (!sp->disable_esi && wrk->obj->esidata != NULL) {
 		/* In ESI mode, we don't know the aggregate length */
-		sp->wrk->res_mode &= ~RES_LEN;
-		sp->wrk->res_mode |= RES_ESI;
+		wrk->res_mode &= ~RES_LEN;
+		wrk->res_mode |= RES_ESI;
 	}
 
 	if (sp->esi_level > 0) {
-		sp->wrk->res_mode &= ~RES_LEN;
-		sp->wrk->res_mode |= RES_ESI_CHILD;
+		wrk->res_mode &= ~RES_LEN;
+		wrk->res_mode |= RES_ESI_CHILD;
 	}
 
-	if (cache_param->http_gzip_support && sp->wrk->obj->gziped &&
+	if (cache_param->http_gzip_support && wrk->obj->gziped &&
 	    !RFC2616_Req_Gzip(sp)) {
 		/*
 		 * We don't know what it uncompresses to
 		 * XXX: we could cache that
 		 */
-		sp->wrk->res_mode &= ~RES_LEN;
-		sp->wrk->res_mode |= RES_GUNZIP;
+		wrk->res_mode &= ~RES_LEN;
+		wrk->res_mode |= RES_GUNZIP;
 	}
 
-	if (!(sp->wrk->res_mode & (RES_LEN|RES_CHUNKED|RES_EOF))) {
-		if (sp->wrk->obj->len == 0 && !sp->wrk->do_stream)
+	if (!(wrk->res_mode & (RES_LEN|RES_CHUNKED|RES_EOF))) {
+		if (wrk->obj->len == 0 && !wrk->do_stream)
 			/*
 			 * If the object is empty, neither ESI nor GUNZIP
 			 * can make it any different size
 			 */
-			sp->wrk->res_mode |= RES_LEN;
+			wrk->res_mode |= RES_LEN;
 		else if (!sp->wantbody) {
 			/* Nothing */
 		} else if (sp->http->protover >= 11) {
-			sp->wrk->res_mode |= RES_CHUNKED;
+			wrk->res_mode |= RES_CHUNKED;
 		} else {
-			sp->wrk->res_mode |= RES_EOF;
+			wrk->res_mode |= RES_EOF;
 			sp->doclose = "EOF mode";
 		}
 	}
 
-	sp->t_resp = W_TIM_real(sp->wrk);
-	if (sp->wrk->obj->objcore != NULL) {
-		if ((sp->t_resp - sp->wrk->obj->last_lru) > cache_param->lru_timeout &&
-		    EXP_Touch(sp->wrk->obj->objcore))
-			sp->wrk->obj->last_lru = sp->t_resp;
-		sp->wrk->obj->last_use = sp->t_resp;	/* XXX: locking ? */
+	sp->t_resp = W_TIM_real(wrk);
+	if (wrk->obj->objcore != NULL) {
+		if ((sp->t_resp - wrk->obj->last_lru) > cache_param->lru_timeout &&
+		    EXP_Touch(wrk->obj->objcore))
+			wrk->obj->last_lru = sp->t_resp;
+		wrk->obj->last_use = sp->t_resp;	/* XXX: locking ? */
 	}
-	http_Setup(sp->wrk->resp, sp->wrk->ws);
+	http_Setup(wrk->resp, wrk->ws);
 	RES_BuildHttp(sp);
 	VCL_deliver_method(sp);
 	switch (sp->handling) {
@@ -236,26 +242,26 @@ cnt_prepresp(struct sess *sp)
 	case VCL_RET_RESTART:
 		if (sp->restarts >= cache_param->max_restarts)
 			break;
-		if (sp->wrk->do_stream) {
-			VDI_CloseFd(sp->wrk);
-			HSH_Drop(sp->wrk);
+		if (wrk->do_stream) {
+			VDI_CloseFd(wrk);
+			HSH_Drop(wrk);
 		} else {
-			(void)HSH_Deref(sp->wrk, NULL, &sp->wrk->obj);
+			(void)HSH_Deref(wrk, NULL, &wrk->obj);
 		}
-		AZ(sp->wrk->obj);
+		AZ(wrk->obj);
 		sp->restarts++;
 		sp->director = NULL;
-		sp->wrk->h_content_length = NULL;
-		http_Setup(sp->wrk->bereq, NULL);
-		http_Setup(sp->wrk->beresp, NULL);
-		http_Setup(sp->wrk->resp, NULL);
+		wrk->h_content_length = NULL;
+		http_Setup(wrk->bereq, NULL);
+		http_Setup(wrk->beresp, NULL);
+		http_Setup(wrk->resp, NULL);
 		sp->step = STP_RECV;
 		return (0);
 	default:
 		WRONG("Illegal action in vcl_deliver{}");
 	}
-	if (sp->wrk->do_stream) {
-		AssertObjCorePassOrBusy(sp->wrk->obj->objcore);
+	if (wrk->do_stream) {
+		AssertObjCorePassOrBusy(wrk->obj->objcore);
 		sp->step = STP_STREAMBODY;
 	} else {
 		sp->step = STP_DELIVER;
@@ -281,16 +287,21 @@ DOT deliver -> DONE [style=bold,color=blue]
 static int
 cnt_deliver(struct sess *sp)
 {
+	struct worker *wrk;
+
+	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+	wrk = sp->wrk;
+	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 
 	sp->director = NULL;
 	sp->restarts = 0;
 
 	RES_WriteObj(sp);
 
-	assert(WRW_IsReleased(sp->wrk));
-	assert(sp->wrk->wrw.ciov == sp->wrk->wrw.siov);
-	(void)HSH_Deref(sp->wrk, NULL, &sp->wrk->obj);
-	http_Setup(sp->wrk->resp, NULL);
+	assert(WRW_IsReleased(wrk));
+	assert(wrk->wrw.ciov == wrk->wrw.siov);
+	(void)HSH_Deref(wrk, NULL, &wrk->obj);
+	http_Setup(wrk->resp, NULL);
 	sp->step = STP_DONE;
 	return (0);
 }
@@ -314,21 +325,24 @@ cnt_done(struct sess *sp)
 {
 	double dh, dp, da;
 	int i;
+	struct worker *wrk;
 
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+	wrk = sp->wrk;
+	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 	CHECK_OBJ_ORNULL(sp->vcl, VCL_CONF_MAGIC);
 
-	AZ(sp->wrk->obj);
-	AZ(sp->wrk->vbc);
+	AZ(wrk->obj);
+	AZ(wrk->vbc);
 	sp->director = NULL;
 	sp->restarts = 0;
 
-	sp->wrk->busyobj = NULL;
+	wrk->busyobj = NULL;
 
-	sp->wrk->do_esi = 0;
-	sp->wrk->do_gunzip = 0;
-	sp->wrk->do_gzip = 0;
-	sp->wrk->do_stream = 0;
+	wrk->do_esi = 0;
+	wrk->do_gunzip = 0;
+	wrk->do_gzip = 0;
+	wrk->do_stream = 0;
 
 	SES_Charge(sp);
 
@@ -337,14 +351,14 @@ cnt_done(struct sess *sp)
 		return (1);
 
 	if (sp->vcl != NULL) {
-		if (sp->wrk->vcl != NULL)
-			VCL_Rel(&sp->wrk->vcl);
-		sp->wrk->vcl = sp->vcl;
+		if (wrk->vcl != NULL)
+			VCL_Rel(&wrk->vcl);
+		wrk->vcl = sp->vcl;
 		sp->vcl = NULL;
 	}
 
 
-	sp->t_end = W_TIM_real(sp->wrk);
+	sp->t_end = W_TIM_real(wrk);
 WSP(sp, SLT_Debug, "PHK req %.9f resp %.9f end %.9f open %.9f",
     sp->t_req, sp->t_resp, sp->t_end,  sp->t_open);
 	if (sp->xid == 0) {
@@ -364,7 +378,7 @@ WSP(sp, SLT_Debug, "PHK req %.9f resp %.9f end %.9f open %.9f",
 		    sp->xid, sp->t_req, sp->t_end, dh, dp, da);
 	}
 	sp->xid = 0;
-	WSL_Flush(sp->wrk, 0);
+	WSL_Flush(wrk, 0);
 
 	sp->t_open = sp->t_end;
 	sp->t_resp = NAN;
@@ -385,35 +399,34 @@ WSP(sp, SLT_Debug, "PHK req %.9f resp %.9f end %.9f open %.9f",
 	}
 
 	if (sp->fd < 0) {
-		sp->wrk->stats.sess_closed++;
+		wrk->stats.sess_closed++;
 		SES_Delete(sp, NULL);
 		return (1);
 	}
 
-	if (sp->wrk->stats.client_req >= cache_param->wthread_stats_rate)
-		WRK_SumStat(sp->wrk);
+	if (wrk->stats.client_req >= cache_param->wthread_stats_rate)
+		WRK_SumStat(wrk);
 	/* Reset the workspace to the session-watermark */
 	WS_Reset(sp->ws, sp->ws_ses);
-	WS_Reset(sp->wrk->ws, NULL);
+	WS_Reset(wrk->ws, NULL);
 
 	i = HTC_Reinit(sp->htc);
 	if (i == 1) {
-		sp->wrk->stats.sess_pipeline++;
+		wrk->stats.sess_pipeline++;
 		sp->step = STP_START;
 		return (0);
 	}
 	if (Tlen(sp->htc->rxbuf)) {
-		sp->wrk->stats.sess_readahead++;
+		wrk->stats.sess_readahead++;
 		sp->step = STP_WAIT;
 		return (0);
 	}
 	if (cache_param->session_linger > 0) {
-		sp->wrk->stats.sess_linger++;
+		wrk->stats.sess_linger++;
 		sp->step = STP_WAIT;
 		return (0);
 	}
-	sp->wrk->stats.sess_herd++;
-	sp->wrk = NULL;
+	wrk->stats.sess_herd++;
 	Pool_Wait(sp);
 	return (1);
 }
@@ -436,64 +449,65 @@ DOT rsterr [label="RESTART",shape=plaintext]
 static int
 cnt_error(struct sess *sp)
 {
-	struct worker *w;
 	struct http *h;
 	char date[40];
+	struct worker *wrk;
 
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+	wrk = sp->wrk;
+	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 
-	w = sp->wrk;
-	w->do_esi = 0;
-	w->do_gzip = 0;
-	w->do_gunzip = 0;
-	w->do_stream = 0;
+	wrk->do_esi = 0;
+	wrk->do_gzip = 0;
+	wrk->do_gunzip = 0;
+	wrk->do_stream = 0;
 
-	if (w->obj == NULL) {
+	if (wrk->obj == NULL) {
 		HSH_Prealloc(sp);
-		EXP_Clr(&w->exp);
-		w->obj = STV_NewObject(w, NULL, cache_param->http_resp_size,
+		EXP_Clr(&wrk->exp);
+		wrk->obj = STV_NewObject(wrk, NULL, cache_param->http_resp_size,
 		     (uint16_t)cache_param->http_max_hdr);
-		if (sp->wrk->obj == NULL)
-			w->obj = STV_NewObject(w, TRANSIENT_STORAGE,
+		if (wrk->obj == NULL)
+			wrk->obj = STV_NewObject(wrk, TRANSIENT_STORAGE,
 			    cache_param->http_resp_size,
 			    (uint16_t)cache_param->http_max_hdr);
-		if (sp->wrk->obj == NULL) {
+		if (wrk->obj == NULL) {
 			sp->doclose = "Out of objects";
 			sp->director = NULL;
-			sp->wrk->h_content_length = NULL;
-			http_Setup(sp->wrk->beresp, NULL);
-			http_Setup(sp->wrk->bereq, NULL);
+			wrk->h_content_length = NULL;
+			http_Setup(wrk->beresp, NULL);
+			http_Setup(wrk->bereq, NULL);
 			sp->step = STP_DONE;
 			return(0);
 		}
-		AN(sp->wrk->obj);
-		sp->wrk->obj->xid = sp->xid;
-		sp->wrk->obj->exp.entered = sp->t_req;
+		AN(wrk->obj);
+		wrk->obj->xid = sp->xid;
+		wrk->obj->exp.entered = sp->t_req;
 	} else {
 		/* XXX: Null the headers ? */
 	}
-	CHECK_OBJ_NOTNULL(sp->wrk->obj, OBJECT_MAGIC);
-	h = sp->wrk->obj->http;
+	CHECK_OBJ_NOTNULL(wrk->obj, OBJECT_MAGIC);
+	h = wrk->obj->http;
 
 	if (sp->err_code < 100 || sp->err_code > 999)
 		sp->err_code = 501;
 
-	http_PutProtocol(w, sp->vsl_id, h, "HTTP/1.1");
+	http_PutProtocol(wrk, sp->vsl_id, h, "HTTP/1.1");
 	http_PutStatus(h, sp->err_code);
-	VTIM_format(W_TIM_real(sp->wrk), date);
-	http_PrintfHeader(w, sp->vsl_id, h, "Date: %s", date);
-	http_SetHeader(w, sp->vsl_id, h, "Server: Varnish");
+	VTIM_format(W_TIM_real(wrk), date);
+	http_PrintfHeader(wrk, sp->vsl_id, h, "Date: %s", date);
+	http_SetHeader(wrk, sp->vsl_id, h, "Server: Varnish");
 
 	if (sp->err_reason != NULL)
-		http_PutResponse(w, sp->vsl_id, h, sp->err_reason);
+		http_PutResponse(wrk, sp->vsl_id, h, sp->err_reason);
 	else
-		http_PutResponse(w, sp->vsl_id, h,
+		http_PutResponse(wrk, sp->vsl_id, h,
 		    http_StatusMessage(sp->err_code));
 	VCL_error_method(sp);
 
 	if (sp->handling == VCL_RET_RESTART &&
 	    sp->restarts <  cache_param->max_restarts) {
-		HSH_Drop(sp->wrk);
+		HSH_Drop(wrk);
 		sp->director = NULL;
 		sp->restarts++;
 		sp->step = STP_RECV;
@@ -509,7 +523,7 @@ cnt_error(struct sess *sp)
 	assert(sp->handling == VCL_RET_DELIVER);
 	sp->err_code = 0;
 	sp->err_reason = NULL;
-	http_Setup(sp->wrk->bereq, NULL);
+	http_Setup(wrk->bereq, NULL);
 	sp->step = STP_PREPRESP;
 	return (0);
 }
@@ -547,19 +561,22 @@ static int
 cnt_fetch(struct sess *sp)
 {
 	int i;
+	struct worker *wrk;
 
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+	wrk = sp->wrk;
+	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
+
 	CHECK_OBJ_NOTNULL(sp->vcl, VCL_CONF_MAGIC);
-	CHECK_OBJ_NOTNULL(sp->wrk, WORKER_MAGIC);
-	CHECK_OBJ_NOTNULL(sp->wrk->busyobj, BUSYOBJ_MAGIC);
+	CHECK_OBJ_NOTNULL(wrk->busyobj, BUSYOBJ_MAGIC);
 
 	AN(sp->director);
-	AZ(sp->wrk->vbc);
-	AZ(sp->wrk->h_content_length);
-	AZ(sp->wrk->do_close);
-	AZ(sp->wrk->storage_hint);
+	AZ(wrk->vbc);
+	AZ(wrk->h_content_length);
+	AZ(wrk->do_close);
+	AZ(wrk->storage_hint);
 
-	http_Setup(sp->wrk->beresp, sp->wrk->ws);
+	http_Setup(wrk->beresp, wrk->ws);
 
 	i = FetchHdr(sp);
 	/*
@@ -581,41 +598,41 @@ cnt_fetch(struct sess *sp)
 		 * and we rely on their content outside of VCL, so collect them
 		 * into one line here.
 		 */
-		http_CollectHdr(sp->wrk->beresp, H_Cache_Control);
-		http_CollectHdr(sp->wrk->beresp, H_Vary);
+		http_CollectHdr(wrk->beresp, H_Cache_Control);
+		http_CollectHdr(wrk->beresp, H_Vary);
 
 		/*
 		 * Figure out how the fetch is supposed to happen, before the
 		 * headers are adultered by VCL
-		 * NB: Also sets other sp->wrk variables
+		 * NB: Also sets other wrk variables
 		 */
-		sp->wrk->body_status = RFC2616_Body(sp);
+		wrk->body_status = RFC2616_Body(sp);
 
-		sp->err_code = http_GetStatus(sp->wrk->beresp);
+		sp->err_code = http_GetStatus(wrk->beresp);
 
 		/*
 		 * What does RFC2616 think about TTL ?
 		 */
-		EXP_Clr(&sp->wrk->exp);
-		sp->wrk->exp.entered = W_TIM_real(sp->wrk);
+		EXP_Clr(&wrk->exp);
+		wrk->exp.entered = W_TIM_real(wrk);
 		RFC2616_Ttl(sp);
 
 		/* pass from vclrecv{} has negative TTL */
-		if (sp->wrk->objcore == NULL)
-			sp->wrk->exp.ttl = -1.;
+		if (wrk->objcore == NULL)
+			wrk->exp.ttl = -1.;
 
-		AZ(sp->wrk->do_esi);
+		AZ(wrk->do_esi);
 
 		VCL_fetch_method(sp);
 
 		switch (sp->handling) {
 		case VCL_RET_HIT_FOR_PASS:
-			if (sp->wrk->objcore != NULL)
-				sp->wrk->objcore->flags |= OC_F_PASS;
+			if (wrk->objcore != NULL)
+				wrk->objcore->flags |= OC_F_PASS;
 			sp->step = STP_FETCHBODY;
 			return (0);
 		case VCL_RET_DELIVER:
-			AssertObjCorePassOrBusy(sp->wrk->objcore);
+			AssertObjCorePassOrBusy(wrk->objcore);
 			sp->step = STP_FETCHBODY;
 			return (0);
 		default:
@@ -623,22 +640,22 @@ cnt_fetch(struct sess *sp)
 		}
 
 		/* We are not going to fetch the body, Close the connection */
-		VDI_CloseFd(sp->wrk);
+		VDI_CloseFd(wrk);
 	}
 
 	/* Clean up partial fetch */
-	AZ(sp->wrk->vbc);
+	AZ(wrk->vbc);
 
-	if (sp->wrk->objcore != NULL) {
-		CHECK_OBJ_NOTNULL(sp->wrk->objcore, OBJCORE_MAGIC);
-		AZ(HSH_Deref(sp->wrk, sp->wrk->objcore, NULL));
-		sp->wrk->objcore = NULL;
+	if (wrk->objcore != NULL) {
+		CHECK_OBJ_NOTNULL(wrk->objcore, OBJCORE_MAGIC);
+		AZ(HSH_Deref(wrk, wrk->objcore, NULL));
+		wrk->objcore = NULL;
 	}
-	http_Setup(sp->wrk->bereq, NULL);
-	http_Setup(sp->wrk->beresp, NULL);
-	sp->wrk->h_content_length = NULL;
+	http_Setup(wrk->bereq, NULL);
+	http_Setup(wrk->beresp, NULL);
+	wrk->h_content_length = NULL;
 	sp->director = NULL;
-	sp->wrk->storage_hint = NULL;
+	wrk->storage_hint = NULL;
 
 	switch (sp->handling) {
 	case VCL_RET_RESTART:
@@ -686,7 +703,9 @@ cnt_fetchbody(struct sess *sp)
 	int varyl = 0, pass;
 	struct worker *wrk;
 
+	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 	wrk = sp->wrk;
+	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 
 	CHECK_OBJ_NOTNULL(wrk->busyobj, BUSYOBJ_MAGIC);
 
@@ -913,54 +932,59 @@ cnt_streambody(struct sess *sp)
 	struct stream_ctx sctx;
 	uint8_t obuf[sp->wrk->res_mode & RES_GUNZIP ?
 	    cache_param->gzip_stack_buffer : 1];
+	struct worker *wrk;
+
+	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+	wrk = sp->wrk;
+	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 
-	CHECK_OBJ_NOTNULL(sp->wrk->busyobj, BUSYOBJ_MAGIC);
+	CHECK_OBJ_NOTNULL(wrk->busyobj, BUSYOBJ_MAGIC);
 	memset(&sctx, 0, sizeof sctx);
 	sctx.magic = STREAM_CTX_MAGIC;
-	AZ(sp->wrk->sctx);
-	sp->wrk->sctx = &sctx;
+	AZ(wrk->sctx);
+	wrk->sctx = &sctx;
 
-	if (sp->wrk->res_mode & RES_GUNZIP) {
-		sctx.vgz = VGZ_NewUngzip(sp->wrk, "U S -");
+	if (wrk->res_mode & RES_GUNZIP) {
+		sctx.vgz = VGZ_NewUngzip(wrk, "U S -");
 		sctx.obuf = obuf;
 		sctx.obuf_len = sizeof (obuf);
 	}
 
 	RES_StreamStart(sp);
 
-	AssertObjCorePassOrBusy(sp->wrk->obj->objcore);
+	AssertObjCorePassOrBusy(wrk->obj->objcore);
 
-	i = FetchBody(sp->wrk, sp->wrk->obj);
+	i = FetchBody(wrk, wrk->obj);
 
-	sp->wrk->h_content_length = NULL;
+	wrk->h_content_length = NULL;
 
-	http_Setup(sp->wrk->bereq, NULL);
-	http_Setup(sp->wrk->beresp, NULL);
-	sp->wrk->vfp = NULL;
-	AZ(sp->wrk->vbc);
+	http_Setup(wrk->bereq, NULL);
+	http_Setup(wrk->beresp, NULL);
+	wrk->vfp = NULL;
+	AZ(wrk->vbc);
 	AN(sp->director);
 
-	if (!i && sp->wrk->obj->objcore != NULL) {
-		EXP_Insert(sp->wrk->obj);
-		AN(sp->wrk->obj->objcore);
-		AN(sp->wrk->obj->objcore->ban);
-		HSH_Unbusy(sp->wrk);
+	if (!i && wrk->obj->objcore != NULL) {
+		EXP_Insert(wrk->obj);
+		AN(wrk->obj->objcore);
+		AN(wrk->obj->objcore->ban);
+		HSH_Unbusy(wrk);
 	} else {
 		sp->doclose = "Stream error";
 	}
-	sp->wrk->acct_tmp.fetch++;
+	wrk->acct_tmp.fetch++;
 	sp->director = NULL;
 	sp->restarts = 0;
 
 	RES_StreamEnd(sp);
-	if (sp->wrk->res_mode & RES_GUNZIP)
+	if (wrk->res_mode & RES_GUNZIP)
 		(void)VGZ_Destroy(&sctx.vgz, sp->vsl_id);
 
-	sp->wrk->sctx = NULL;
-	assert(WRW_IsReleased(sp->wrk));
-	assert(sp->wrk->wrw.ciov == sp->wrk->wrw.siov);
-	(void)HSH_Deref(sp->wrk, NULL, &sp->wrk->obj);
-	http_Setup(sp->wrk->resp, NULL);
+	wrk->sctx = NULL;
+	assert(WRW_IsReleased(wrk));
+	assert(wrk->wrw.ciov == wrk->wrw.siov);
+	(void)HSH_Deref(wrk, NULL, &wrk->obj);
+	http_Setup(wrk->resp, NULL);
 	sp->step = STP_DONE;
 	return (0);
 }
@@ -979,6 +1003,11 @@ DOT first -> wait
 static int
 cnt_first(struct sess *sp)
 {
+	struct worker *wrk;
+
+	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+	wrk = sp->wrk;
+	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 
 	/*
 	 * XXX: If we don't have acceptfilters we are somewhat subject
@@ -998,7 +1027,7 @@ cnt_first(struct sess *sp)
 	/* Receive a HTTP protocol request */
 	HTC_Init(sp->htc, sp->ws, sp->fd, sp->vsl_id, cache_param->http_req_size,
 	    cache_param->http_req_hdr_len);
-	sp->wrk->acct_tmp.sess++;
+	wrk->acct_tmp.sess++;
 
 	sp->step = STP_WAIT;
 	return (0);
@@ -1105,8 +1134,12 @@ cnt_lookup(struct sess *sp)
 	struct objcore *oc;
 	struct object *o;
 	struct objhead *oh;
+	struct worker *wrk;
 
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+	wrk = sp->wrk;
+	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
+
 	CHECK_OBJ_NOTNULL(sp->vcl, VCL_CONF_MAGIC);
 
 	if (sp->hash_objhead == NULL) {
@@ -1140,7 +1173,7 @@ cnt_lookup(struct sess *sp)
 
 	/* If we inserted a new object it's a miss */
 	if (oc->flags & OC_F_BUSY) {
-		sp->wrk->stats.cache_miss++;
+		wrk->stats.cache_miss++;
 
 		if (sp->vary_l != NULL) {
 			assert(oc->busyobj->vary == sp->vary_b);
@@ -1154,15 +1187,15 @@ cnt_lookup(struct sess *sp)
 		sp->vary_l = NULL;
 		sp->vary_e = NULL;
 
-		sp->wrk->objcore = oc;
-		CHECK_OBJ_NOTNULL(sp->wrk->busyobj, BUSYOBJ_MAGIC);
+		wrk->objcore = oc;
+		CHECK_OBJ_NOTNULL(wrk->busyobj, BUSYOBJ_MAGIC);
 		sp->step = STP_MISS;
 		return (0);
 	}
 
-	o = oc_getobj(sp->wrk, oc);
+	o = oc_getobj(wrk, oc);
 	CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC);
-	sp->wrk->obj = o;
+	wrk->obj = o;
 
 	WS_Release(sp->ws, 0);
 	sp->vary_b = NULL;
@@ -1170,16 +1203,16 @@ cnt_lookup(struct sess *sp)
 	sp->vary_e = NULL;
 
 	if (oc->flags & OC_F_PASS) {
-		sp->wrk->stats.cache_hitpass++;
-		WSP(sp, SLT_HitPass, "%u", sp->wrk->obj->xid);
-		(void)HSH_Deref(sp->wrk, NULL, &sp->wrk->obj);
-		sp->wrk->objcore = NULL;
+		wrk->stats.cache_hitpass++;
+		WSP(sp, SLT_HitPass, "%u", wrk->obj->xid);
+		(void)HSH_Deref(wrk, NULL, &wrk->obj);
+		wrk->objcore = NULL;
 		sp->step = STP_PASS;
 		return (0);
 	}
 
-	sp->wrk->stats.cache_hit++;
-	WSP(sp, SLT_Hit, "%u", sp->wrk->obj->xid);
+	wrk->stats.cache_hit++;
+	WSP(sp, SLT_Hit, "%u", wrk->obj->xid);
 	sp->step = STP_HIT;
 	return (0);
 }
@@ -1210,53 +1243,56 @@ DOT
 static int
 cnt_miss(struct sess *sp)
 {
+	struct worker *wrk;
 
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+	wrk = sp->wrk;
+	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 	CHECK_OBJ_NOTNULL(sp->vcl, VCL_CONF_MAGIC);
 
-	AZ(sp->wrk->obj);
-	AN(sp->wrk->objcore);
-	CHECK_OBJ_NOTNULL(sp->wrk->busyobj, BUSYOBJ_MAGIC);
-	WS_Reset(sp->wrk->ws, NULL);
-	http_Setup(sp->wrk->bereq, sp->wrk->ws);
+	AZ(wrk->obj);
+	AN(wrk->objcore);
+	CHECK_OBJ_NOTNULL(wrk->busyobj, BUSYOBJ_MAGIC);
+	WS_Reset(wrk->ws, NULL);
+	http_Setup(wrk->bereq, wrk->ws);
 	http_FilterHeader(sp, HTTPH_R_FETCH);
-	http_ForceGet(sp->wrk->bereq);
+	http_ForceGet(wrk->bereq);
 	if (cache_param->http_gzip_support) {
 		/*
 		 * We always ask the backend for gzip, even if the
 		 * client doesn't grok it.  We will uncompress for
 		 * the minority of clients which don't.
 		 */
-		http_Unset(sp->wrk->bereq, H_Accept_Encoding);
-		http_SetHeader(sp->wrk, sp->vsl_id, sp->wrk->bereq,
+		http_Unset(wrk->bereq, H_Accept_Encoding);
+		http_SetHeader(wrk, sp->vsl_id, wrk->bereq,
 		    "Accept-Encoding: gzip");
 	}
-	sp->wrk->connect_timeout = 0;
-	sp->wrk->first_byte_timeout = 0;
-	sp->wrk->between_bytes_timeout = 0;
-	CHECK_OBJ_NOTNULL(sp->wrk->busyobj, BUSYOBJ_MAGIC);
+	wrk->connect_timeout = 0;
+	wrk->first_byte_timeout = 0;
+	wrk->between_bytes_timeout = 0;
+	CHECK_OBJ_NOTNULL(wrk->busyobj, BUSYOBJ_MAGIC);
 
 	VCL_miss_method(sp);
-	CHECK_OBJ_NOTNULL(sp->wrk->busyobj, BUSYOBJ_MAGIC);
+	CHECK_OBJ_NOTNULL(wrk->busyobj, BUSYOBJ_MAGIC);
 	switch(sp->handling) {
 	case VCL_RET_ERROR:
-		AZ(HSH_Deref(sp->wrk, sp->wrk->objcore, NULL));
-		sp->wrk->objcore = NULL;
-		http_Setup(sp->wrk->bereq, NULL);
+		AZ(HSH_Deref(wrk, wrk->objcore, NULL));
+		wrk->objcore = NULL;
+		http_Setup(wrk->bereq, NULL);
 		sp->step = STP_ERROR;
 		return (0);
 	case VCL_RET_PASS:
-		AZ(HSH_Deref(sp->wrk, sp->wrk->objcore, NULL));
-		sp->wrk->objcore = NULL;
+		AZ(HSH_Deref(wrk, wrk->objcore, NULL));
+		wrk->objcore = NULL;
 		sp->step = STP_PASS;
 		return (0);
 	case VCL_RET_FETCH:
-		CHECK_OBJ_NOTNULL(sp->wrk->busyobj, BUSYOBJ_MAGIC);
+		CHECK_OBJ_NOTNULL(wrk->busyobj, BUSYOBJ_MAGIC);
 		sp->step = STP_FETCH;
 		return (0);
 	case VCL_RET_RESTART:
-		AZ(HSH_Deref(sp->wrk, sp->wrk->objcore, NULL));
-		sp->wrk->objcore = NULL;
+		AZ(HSH_Deref(wrk, wrk->objcore, NULL));
+		wrk->objcore = NULL;
 		INCOMPL();
 	default:
 		WRONG("Illegal action in vcl_miss{}");
@@ -1298,29 +1334,32 @@ DOT err_pass [label="ERROR",shape=plaintext]
 static int
 cnt_pass(struct sess *sp)
 {
+	struct worker *wrk;
 
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+	wrk = sp->wrk;
+	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 	CHECK_OBJ_NOTNULL(sp->vcl, VCL_CONF_MAGIC);
-	AZ(sp->wrk->obj);
+	AZ(wrk->obj);
 
-	WS_Reset(sp->wrk->ws, NULL);
-	http_Setup(sp->wrk->bereq, sp->wrk->ws);
+	WS_Reset(wrk->ws, NULL);
+	http_Setup(wrk->bereq, wrk->ws);
 	http_FilterHeader(sp, HTTPH_R_PASS);
 
-	sp->wrk->connect_timeout = 0;
-	sp->wrk->first_byte_timeout = 0;
-	sp->wrk->between_bytes_timeout = 0;
+	wrk->connect_timeout = 0;
+	wrk->first_byte_timeout = 0;
+	wrk->between_bytes_timeout = 0;
 	VCL_pass_method(sp);
 	if (sp->handling == VCL_RET_ERROR) {
-		http_Setup(sp->wrk->bereq, NULL);
+		http_Setup(wrk->bereq, NULL);
 		sp->step = STP_ERROR;
 		return (0);
 	}
 	assert(sp->handling == VCL_RET_PASS);
-	sp->wrk->acct_tmp.pass++;
+	wrk->acct_tmp.pass++;
 	sp->sendbody = 1;
 	sp->step = STP_FETCH;
-	New_BusyObj(sp->wrk);
+	New_BusyObj(wrk);
 	return (0);
 }
 
@@ -1352,13 +1391,16 @@ DOT err_pipe [label="ERROR",shape=plaintext]
 static int
 cnt_pipe(struct sess *sp)
 {
+	struct worker *wrk;
 
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+	wrk = sp->wrk;
+	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 	CHECK_OBJ_NOTNULL(sp->vcl, VCL_CONF_MAGIC);
 
-	sp->wrk->acct_tmp.pipe++;
-	WS_Reset(sp->wrk->ws, NULL);
-	http_Setup(sp->wrk->bereq, sp->wrk->ws);
+	wrk->acct_tmp.pipe++;
+	WS_Reset(wrk->ws, NULL);
+	http_Setup(wrk->bereq, wrk->ws);
 	http_FilterHeader(sp, HTTPH_R_PIPE);
 
 	VCL_pipe_method(sp);
@@ -1368,8 +1410,8 @@ cnt_pipe(struct sess *sp)
 	assert(sp->handling == VCL_RET_PIPE);
 
 	PipeSession(sp);
-	assert(WRW_IsReleased(sp->wrk));
-	http_Setup(sp->wrk->bereq, NULL);
+	assert(WRW_IsReleased(wrk));
+	http_Setup(wrk->bereq, NULL);
 	sp->step = STP_DONE;
 	return (0);
 }
@@ -1397,12 +1439,15 @@ DOT recv -> hash [label="lookup",style=bold,color=green]
 static int
 cnt_recv(struct sess *sp)
 {
+	struct worker *wrk;
 	unsigned recv_handling;
 
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+	wrk = sp->wrk;
+	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 	CHECK_OBJ_NOTNULL(sp->vcl, VCL_CONF_MAGIC);
-	AZ(sp->wrk->obj);
-	assert(sp->wrk->wrw.ciov == sp->wrk->wrw.siov);
+	AZ(wrk->obj);
+	assert(wrk->wrw.ciov == wrk->wrw.siov);
 
 	/* By default we use the first backend */
 	AZ(sp->director);
@@ -1427,27 +1472,27 @@ cnt_recv(struct sess *sp)
 	}
 
 	/* Zap these, in case we came here through restart */
-	sp->wrk->do_esi = 0;
-	sp->wrk->do_gzip = 0;
-	sp->wrk->do_gunzip = 0;
-	sp->wrk->do_stream = 0;
+	wrk->do_esi = 0;
+	wrk->do_gzip = 0;
+	wrk->do_gunzip = 0;
+	wrk->do_stream = 0;
 
 	if (cache_param->http_gzip_support &&
 	     (recv_handling != VCL_RET_PIPE) &&
 	     (recv_handling != VCL_RET_PASS)) {
 		if (RFC2616_Req_Gzip(sp)) {
 			http_Unset(sp->http, H_Accept_Encoding);
-			http_SetHeader(sp->wrk, sp->vsl_id, sp->http,
+			http_SetHeader(wrk, sp->vsl_id, sp->http,
 			    "Accept-Encoding: gzip");
 		} else {
 			http_Unset(sp->http, H_Accept_Encoding);
 		}
 	}
 
-	SHA256_Init(sp->wrk->sha256ctx);
+	SHA256_Init(wrk->sha256ctx);
 	VCL_hash_method(sp);
 	assert(sp->handling == VCL_RET_HASH);
-	SHA256_Final(sp->digest, sp->wrk->sha256ctx);
+	SHA256_Final(sp->digest, wrk->sha256ctx);
 
 	if (!strcmp(sp->http->hd[HTTP_HDR_REQ].b, "HEAD"))
 		sp->wantbody = 0;
@@ -1499,26 +1544,29 @@ cnt_start(struct sess *sp)
 	uint16_t done;
 	char *p;
 	const char *r = "HTTP/1.1 100 Continue\r\n\r\n";
+	struct worker *wrk;
 
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+	wrk = sp->wrk;
+	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 	AZ(sp->restarts);
-	AZ(sp->wrk->obj);
+	AZ(wrk->obj);
 	AZ(sp->vcl);
 	AZ(sp->esi_level);
 
 	/* Update stats of various sorts */
-	sp->wrk->stats.client_req++;
-	sp->t_req = W_TIM_real(sp->wrk);
-	sp->wrk->acct_tmp.req++;
+	wrk->stats.client_req++;
+	sp->t_req = W_TIM_real(wrk);
+	wrk->acct_tmp.req++;
 
 	/* Assign XID and log */
 	sp->xid = ++xids;				/* XXX not locked */
 	WSP(sp, SLT_ReqStart, "%s %s %u", sp->addr, sp->port,  sp->xid);
 
 	/* Borrow VCL reference from worker thread */
-	VCL_Refresh(&sp->wrk->vcl);
-	sp->vcl = sp->wrk->vcl;
-	sp->wrk->vcl = NULL;
+	VCL_Refresh(&wrk->vcl);
+	sp->vcl = wrk->vcl;
+	wrk->vcl = NULL;
 
 	http_Setup(sp->http, sp->ws);
 	done = http_DissectRequest(sp);
@@ -1594,11 +1642,11 @@ void
 CNT_Session(struct sess *sp)
 {
 	int done;
-	struct worker *w;
+	struct worker *wrk;
 
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
-	w = sp->wrk;
-	CHECK_OBJ_NOTNULL(w, WORKER_MAGIC);
+	wrk = sp->wrk;
+	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 
 	/*
 	 * Possible entrance states
@@ -1609,12 +1657,12 @@ CNT_Session(struct sess *sp)
 	    sp->step == STP_LOOKUP ||
 	    sp->step == STP_RECV);
 
-	AZ(w->do_stream);
-	AZ(w->do_gzip);
-	AZ(w->do_gunzip);
-	AZ(w->do_esi);
-	AZ(w->obj);
-	AZ(w->objcore);
+	AZ(wrk->do_stream);
+	AZ(wrk->do_gzip);
+	AZ(wrk->do_gunzip);
+	AZ(wrk->do_esi);
+	AZ(wrk->obj);
+	AZ(wrk->objcore);
 
 	/*
 	 * Whenever we come in from the acceptor or waiter, we need to set
@@ -1637,16 +1685,16 @@ CNT_Session(struct sess *sp)
 	 * NB: Once done is set, we can no longer touch sp!
 	 */
 	for (done = 0; !done; ) {
-		assert(sp->wrk == w);
+		assert(sp->wrk == wrk);
 		/*
 		 * This is a good place to be paranoid about the various
 		 * pointers still pointing to the things we expect.
 		 */
 		CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
-		CHECK_OBJ_ORNULL(sp->wrk->obj, OBJECT_MAGIC);
-		CHECK_OBJ_NOTNULL(sp->wrk, WORKER_MAGIC);
-		CHECK_OBJ_ORNULL(w->nobjhead, OBJHEAD_MAGIC);
-		WS_Assert(w->ws);
+		CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
+		CHECK_OBJ_ORNULL(wrk->obj, OBJECT_MAGIC);
+		CHECK_OBJ_ORNULL(wrk->nobjhead, OBJHEAD_MAGIC);
+		WS_Assert(wrk->ws);
 
 		switch (sp->step) {
 #define STEP(l,u) \
@@ -1660,20 +1708,20 @@ CNT_Session(struct sess *sp)
 		default:
 			WRONG("State engine misfire");
 		}
-		WS_Assert(w->ws);
-		CHECK_OBJ_ORNULL(w->nobjhead, OBJHEAD_MAGIC);
+		WS_Assert(wrk->ws);
+		CHECK_OBJ_ORNULL(wrk->nobjhead, OBJHEAD_MAGIC);
 	}
-	WSL_Flush(w, 0);
-	AZ(w->obj);
-	AZ(w->objcore);
-	AZ(w->do_stream);
-	AZ(w->do_gzip);
-	AZ(w->do_gunzip);
-	AZ(w->do_esi);
-#define ACCT(foo)	AZ(w->acct_tmp.foo);
+	WSL_Flush(wrk, 0);
+	AZ(wrk->obj);
+	AZ(wrk->objcore);
+	AZ(wrk->do_stream);
+	AZ(wrk->do_gzip);
+	AZ(wrk->do_gunzip);
+	AZ(wrk->do_esi);
+#define ACCT(foo)	AZ(wrk->acct_tmp.foo);
 #include "tbl/acct_fields.h"
 #undef ACCT
-	assert(WRW_IsReleased(w));
+	assert(WRW_IsReleased(wrk));
 }
 
 /*
diff --git a/bin/varnishd/cache/cache_pool.c b/bin/varnishd/cache/cache_pool.c
index 730e391..582bb70 100644
--- a/bin/varnishd/cache/cache_pool.c
+++ b/bin/varnishd/cache/cache_pool.c
@@ -361,8 +361,11 @@ Pool_Wait(struct sess *sp)
 {
 
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+	CHECK_OBJ_NOTNULL(sp->wrk, WORKER_MAGIC);
 	AZ(sp->vcl);
 	assert(sp->fd >= 0);
+	sp->wrk = NULL;
+
 	/*
 	 * Set nonblocking in the worker-thread, before passing to the
 	 * acceptor thread, to reduce syscall density of the latter.
diff --git a/bin/varnishd/cache/cache_shmlog.c b/bin/varnishd/cache/cache_shmlog.c
index 1e414b8..3cfd451 100644
--- a/bin/varnishd/cache/cache_shmlog.c
+++ b/bin/varnishd/cache/cache_shmlog.c
@@ -335,7 +335,8 @@ VSM_Init(void)
 	VWMB();
 
 	vsl_start = vsl_log_start;
-	vsl_end = vsl_start + cache_param->vsl_space / sizeof *vsl_end;
+	vsl_end = vsl_start +
+	    cache_param->vsl_space / (unsigned)sizeof *vsl_end;
 	vsl_ptr = vsl_start + 1;
 
 	VSC_C_main = VSM_Alloc(sizeof *VSC_C_main,



From geoff at varnish-cache.org  Mon Jan  9 20:52:44 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:44 +0100
Subject: [experimental-ims] 4faa178 Move body_status from worker to busyobj
Message-ID: <E1RkMCy-0008Um-AA@project.varnish-software.com>

commit 4faa178440e6b73cc7e7924a845e1f26bf1f386e
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Nov 29 19:00:10 2011 +0000

    Move body_status from worker to busyobj

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index cb2a68e..c4d6f08 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -316,8 +316,8 @@ struct worker {
 	struct SHA256Context	*sha256ctx;
 
 	struct ws		ws[1];
-	struct http		*bereq;
-	struct http		*beresp;
+
+
 	struct http		*resp;
 
 	struct object		*obj;
@@ -327,9 +327,11 @@ struct worker {
 	/* This is only here so VRT can find it */
 	const char		*storage_hint;
 
-	/* Fetch stuff */
+	/* Fetch stuff.  Here because pipe has no busyobj */
+	struct http		*bereq;
+	struct http		*beresp;
 	struct vbc		*vbc;
-	enum body_status	body_status;
+
 	struct vef_priv		*vef_priv;
 	unsigned		do_stream;
 	unsigned		do_esi;
@@ -503,6 +505,8 @@ struct busyobj {
 	struct object		*fetch_obj;
 	struct exp		exp;
 	struct http_conn	htc;
+
+	enum body_status	body_status;
 };
 
 /* Object structure --------------------------------------------------*/
diff --git a/bin/varnishd/cache/cache_center.c b/bin/varnishd/cache/cache_center.c
index 86b54d0..7d4a280 100644
--- a/bin/varnishd/cache/cache_center.c
+++ b/bin/varnishd/cache/cache_center.c
@@ -606,7 +606,7 @@ cnt_fetch(struct sess *sp)
 		 * headers are adultered by VCL
 		 * NB: Also sets other wrk variables
 		 */
-		wrk->body_status = RFC2616_Body(sp);
+		wrk->busyobj->body_status = RFC2616_Body(sp);
 
 		sp->err_code = http_GetStatus(wrk->beresp);
 
diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c
index 7180a5b..15eeb9b 100644
--- a/bin/varnishd/cache/cache_fetch.c
+++ b/bin/varnishd/cache/cache_fetch.c
@@ -229,7 +229,7 @@ fetch_straight(struct worker *w, struct http_conn *htc, ssize_t cl)
 {
 	int i;
 
-	assert(w->body_status == BS_LENGTH);
+	assert(w->busyobj->body_status == BS_LENGTH);
 
 	if (cl < 0) {
 		return (FetchError(w, "straight length field bogus"));
@@ -256,7 +256,7 @@ fetch_chunked(struct worker *w, struct http_conn *htc)
 	unsigned u;
 	ssize_t cl;
 
-	assert(w->body_status == BS_CHUNKED);
+	assert(w->busyobj->body_status == BS_CHUNKED);
 	do {
 		/* Skip leading whitespace */
 		do {
@@ -314,7 +314,7 @@ fetch_eof(struct worker *w, struct http_conn *htc)
 {
 	int i;
 
-	assert(w->body_status == BS_EOF);
+	assert(w->busyobj->body_status == BS_EOF);
 	i = w->busyobj->vfp->bytes(w, htc, SSIZE_MAX);
 	if (i < 0)
 		return (-1);
@@ -515,7 +515,7 @@ FetchBody(struct worker *w, struct object *obj)
 
 	/* XXX: pick up estimate from objdr ? */
 	cl = 0;
-	switch (w->body_status) {
+	switch (w->busyobj->body_status) {
 	case BS_NONE:
 		cls = 0;
 		mklen = 0;
@@ -567,10 +567,10 @@ FetchBody(struct worker *w, struct object *obj)
 	w->busyobj->fetch_obj = NULL;
 
 	WSLB(w, SLT_Fetch_Body, "%u(%s) cls %d mklen %u",
-	    w->body_status, body_status(w->body_status),
+	    w->busyobj->body_status, body_status(w->busyobj->body_status),
 	    cls, mklen);
 
-	if (w->body_status == BS_ERROR) {
+	if (w->busyobj->body_status == BS_ERROR) {
 		VDI_CloseFd(w);
 		return (__LINE__);
 	}
diff --git a/bin/varnishd/cache/cache_panic.c b/bin/varnishd/cache/cache_panic.c
index 641fe99..85fcae0 100644
--- a/bin/varnishd/cache/cache_panic.c
+++ b/bin/varnishd/cache/cache_panic.c
@@ -213,6 +213,7 @@ pan_busyobj(const struct busyobj *bo)
 	VSB_printf(pan_vsp, "  busyobj = %p {\n", bo);
 	if (bo->is_gzip)	VSB_printf(pan_vsp, "    is_gzip\n");
 	if (bo->is_gunzip)	VSB_printf(pan_vsp, "    is_gunzip\n");
+	VSB_printf(pan_vsp, "    bodystatus = %d,\n", bo->body_status);
 	VSB_printf(pan_vsp, "    },\n");
 }
 
@@ -260,7 +261,6 @@ pan_sess(const struct sess *sp)
 	if (sp->wrk->do_esi)	VSB_printf(pan_vsp, " do_esi");
 	if (sp->wrk->do_close)	VSB_printf(pan_vsp, " do_close");
 	VSB_printf(pan_vsp, "\n");
-	VSB_printf(pan_vsp, "  bodystatus = %d\n", sp->wrk->body_status);
 	pan_busyobj(sp->wrk->busyobj);
 
 	pan_ws(sp->ws, 2);



From geoff at varnish-cache.org  Mon Jan  9 20:52:19 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:19 +0100
Subject: [experimental-ims] a641a46 Move the manager process sources into a
	subdirectory, away from the gaze of prying VMODs.
Message-ID: <E1RkMCZ-0007hr-0t@project.varnish-software.com>

commit a641a46cfe0e00e7104db050f459faeaa324a046
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu Oct 13 09:42:36 2011 +0000

    Move the manager process sources into a subdirectory, away from
    the gaze of prying VMODs.

diff --git a/bin/varnishd/Makefile.am b/bin/varnishd/Makefile.am
index 499cb6c..f2e9d04 100644
--- a/bin/varnishd/Makefile.am
+++ b/bin/varnishd/Makefile.am
@@ -55,14 +55,14 @@ varnishd_SOURCES = \
 	hash/hash_classic.c \
 	hash/hash_critbit.c \
 	hash/hash_simple_list.c \
-	mgt_child.c \
-	mgt_cli.c \
-	mgt_param.c \
-	mgt_pool.c \
-	mgt_sandbox.c \
-	mgt_sandbox_solaris.c \
-	mgt_shmem.c \
-	mgt_vcc.c \
+	mgt/mgt_child.c \
+	mgt/mgt_cli.c \
+	mgt/mgt_param.c \
+	mgt/mgt_pool.c \
+	mgt/mgt_sandbox.c \
+	mgt/mgt_sandbox_solaris.c \
+	mgt/mgt_shmem.c \
+	mgt/mgt_vcc.c \
 	rfc2616.c \
 	storage/stevedore.c \
 	storage/stevedore_utils.c \
@@ -86,8 +86,8 @@ noinst_HEADERS = \
 	default_vcl.h \
 	hash/hash_slinger.h \
 	heritage.h \
-	mgt.h \
-	mgt_cli.h \
+	mgt/mgt.h \
+	mgt/mgt_cli.h \
 	storage/storage.h \
 	storage/storage_persistent.h \
 	vparam.h
@@ -128,7 +128,7 @@ default_vcl.h:	default.vcl Makefile
 	    -e 's/^/ "/' $(srcdir)/default.vcl >> $@
 
 # Explicitly record dependency
-mgt_vcc.c:	default_vcl.h
+mgt/mgt_vcc.c:	default_vcl.h
 
 varnishd.1: $(top_srcdir)/doc/sphinx/reference/varnishd.rst
 if HAVE_RST2MAN
diff --git a/bin/varnishd/flint.sh b/bin/varnishd/flint.sh
index 357d280..4beb9d5 100755
--- a/bin/varnishd/flint.sh
+++ b/bin/varnishd/flint.sh
@@ -18,6 +18,8 @@ flexelint \
 	*.c \
 	storage/*.c \
 	waiter/*.c \
+	hash/*.c \
+	mgt/*.c \
 	../../lib/libvarnish/*.c \
 	../../lib/libvarnishcompat/execinfo.c \
 	../../lib/libvcl/*.c \
diff --git a/bin/varnishd/mgt.h b/bin/varnishd/mgt.h
deleted file mode 100644
index 1656af8..0000000
--- a/bin/varnishd/mgt.h
+++ /dev/null
@@ -1,110 +0,0 @@
-/*-
- * Copyright (c) 2006 Verdens Gang AS
- * Copyright (c) 2006-2011 Varnish Software AS
- * All rights reserved.
- *
- * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- *
- */
-
-#include <stdint.h>
-
-#include "common.h"
-
-struct cli;
-
-extern struct vev_base	*mgt_evb;
-extern unsigned		d_flag;
-extern int		exit_status;
-
-/* mgt_child.c */
-extern pid_t child_pid;
-void MGT_Run(void);
-void mgt_stop_child(void);
-void mgt_got_fd(int fd);
-void MGT_Child_Cli_Fail(void);
-
-/* mgt_cli.c */
-
-typedef void mgt_cli_close_f(void *priv);
-void mgt_cli_setup(int fdi, int fdo, int verbose, const char *ident, mgt_cli_close_f *close_func, void *priv);
-int mgt_cli_askchild(unsigned *status, char **resp, const char *fmt, ...);
-void mgt_cli_start_child(int fdi, int fdo);
-void mgt_cli_stop_child(void);
-void mgt_cli_telnet(const char *T_arg);
-void mgt_cli_master(const char *M_arg);
-void mgt_cli_secret(const char *S_arg);
-void mgt_cli_close_all(void);
-
-/* mgt_param.c */
-void MCF_ParamSync(void);
-void MCF_ParamInit(struct cli *);
-void MCF_ParamSet(struct cli *, const char *param, const char *val);
-void MCF_DumpRst(void);
-
-/* mgt_sandbox.c */
-void mgt_sandbox(void);
-
-/* mgt_sandbox_solaris.c */
-#ifdef HAVE_SETPPRIV
-void mgt_sandbox_solaris_init(void);
-void mgt_sandbox_solaris_fini(void);
-void mgt_sandbox_solaris_privsep(void);
-#endif
-
-/* mgt_shmem.c */
-void mgt_SHM_Init(const char *arg);
-void mgt_SHM_Pid(void);
-
-/* mgt_vcc.c */
-void mgt_vcc_init(void);
-int mgt_vcc_default(const char *bflag, const char *f_arg, char *vcl, int Cflag);
-int mgt_push_vcls_and_start(unsigned *status, char **p);
-int mgt_has_vcl(void);
-extern char *mgt_cc_cmd;
-extern const char *mgt_vcl_dir;
-extern const char *mgt_vmod_dir;
-extern unsigned mgt_vcc_err_unref;
-
-
-#define REPORT0(pri, fmt)				\
-	do {						\
-		fprintf(stderr, fmt "\n");		\
-		syslog(pri, fmt);			\
-	} while (0)
-
-#define REPORT(pri, fmt, ...)				\
-	do {						\
-		fprintf(stderr, fmt "\n", __VA_ARGS__);	\
-		syslog(pri, fmt, __VA_ARGS__);		\
-	} while (0)
-
-#define VSM_Alloc(a, b, c, d)	VSM__Alloc(a,b,c,d)
-#define VSM_Free(a)		VSM__Free(a)
-#define VSM_Clean()		VSM__Clean()
-
-
-#if defined(PTHREAD_CANCELED) || defined(PTHREAD_MUTEX_DEFAULT)
-#error "Keep pthreads out of in manager process"
-#endif
diff --git a/bin/varnishd/mgt/mgt.h b/bin/varnishd/mgt/mgt.h
new file mode 100644
index 0000000..1656af8
--- /dev/null
+++ b/bin/varnishd/mgt/mgt.h
@@ -0,0 +1,110 @@
+/*-
+ * Copyright (c) 2006 Verdens Gang AS
+ * Copyright (c) 2006-2011 Varnish Software AS
+ * All rights reserved.
+ *
+ * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ */
+
+#include <stdint.h>
+
+#include "common.h"
+
+struct cli;
+
+extern struct vev_base	*mgt_evb;
+extern unsigned		d_flag;
+extern int		exit_status;
+
+/* mgt_child.c */
+extern pid_t child_pid;
+void MGT_Run(void);
+void mgt_stop_child(void);
+void mgt_got_fd(int fd);
+void MGT_Child_Cli_Fail(void);
+
+/* mgt_cli.c */
+
+typedef void mgt_cli_close_f(void *priv);
+void mgt_cli_setup(int fdi, int fdo, int verbose, const char *ident, mgt_cli_close_f *close_func, void *priv);
+int mgt_cli_askchild(unsigned *status, char **resp, const char *fmt, ...);
+void mgt_cli_start_child(int fdi, int fdo);
+void mgt_cli_stop_child(void);
+void mgt_cli_telnet(const char *T_arg);
+void mgt_cli_master(const char *M_arg);
+void mgt_cli_secret(const char *S_arg);
+void mgt_cli_close_all(void);
+
+/* mgt_param.c */
+void MCF_ParamSync(void);
+void MCF_ParamInit(struct cli *);
+void MCF_ParamSet(struct cli *, const char *param, const char *val);
+void MCF_DumpRst(void);
+
+/* mgt_sandbox.c */
+void mgt_sandbox(void);
+
+/* mgt_sandbox_solaris.c */
+#ifdef HAVE_SETPPRIV
+void mgt_sandbox_solaris_init(void);
+void mgt_sandbox_solaris_fini(void);
+void mgt_sandbox_solaris_privsep(void);
+#endif
+
+/* mgt_shmem.c */
+void mgt_SHM_Init(const char *arg);
+void mgt_SHM_Pid(void);
+
+/* mgt_vcc.c */
+void mgt_vcc_init(void);
+int mgt_vcc_default(const char *bflag, const char *f_arg, char *vcl, int Cflag);
+int mgt_push_vcls_and_start(unsigned *status, char **p);
+int mgt_has_vcl(void);
+extern char *mgt_cc_cmd;
+extern const char *mgt_vcl_dir;
+extern const char *mgt_vmod_dir;
+extern unsigned mgt_vcc_err_unref;
+
+
+#define REPORT0(pri, fmt)				\
+	do {						\
+		fprintf(stderr, fmt "\n");		\
+		syslog(pri, fmt);			\
+	} while (0)
+
+#define REPORT(pri, fmt, ...)				\
+	do {						\
+		fprintf(stderr, fmt "\n", __VA_ARGS__);	\
+		syslog(pri, fmt, __VA_ARGS__);		\
+	} while (0)
+
+#define VSM_Alloc(a, b, c, d)	VSM__Alloc(a,b,c,d)
+#define VSM_Free(a)		VSM__Free(a)
+#define VSM_Clean()		VSM__Clean()
+
+
+#if defined(PTHREAD_CANCELED) || defined(PTHREAD_MUTEX_DEFAULT)
+#error "Keep pthreads out of in manager process"
+#endif
diff --git a/bin/varnishd/mgt/mgt_child.c b/bin/varnishd/mgt/mgt_child.c
new file mode 100644
index 0000000..ff8a776
--- /dev/null
+++ b/bin/varnishd/mgt/mgt_child.c
@@ -0,0 +1,673 @@
+/*-
+ * Copyright (c) 2006 Verdens Gang AS
+ * Copyright (c) 2006-2011 Varnish Software AS
+ * All rights reserved.
+ *
+ * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * The mechanics of handling the child process
+ */
+
+#include "config.h"
+
+#include <sys/types.h>
+#include <sys/wait.h>
+
+#include <fcntl.h>
+#include <poll.h>
+#include <signal.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <syslog.h>
+#include <unistd.h>
+
+#include "mgt/mgt.h"
+
+#include "heritage.h"
+#include "vapi/vsm_int.h"
+#include "vbm.h"
+#include "vcli.h"
+#include "vcli_priv.h"
+#include "vev.h"
+#include "vlu.h"
+#include "vss.h"
+#include "vtcp.h"
+#include "vtim.h"
+
+#include "mgt_cli.h"
+
+pid_t		child_pid = -1;
+
+
+static struct vbitmap	*fd_map;
+
+static int		child_cli_in = -1;
+static int		child_VCLI_Out = -1;
+static int		child_output = -1;
+
+static enum {
+	CH_STOPPED = 0,
+	CH_STARTING = 1,
+	CH_RUNNING = 2,
+	CH_STOPPING = 3,
+	CH_DIED = 4
+}			child_state = CH_STOPPED;
+
+static const char * const ch_state[] = {
+	[CH_STOPPED] =	"stopped",
+	[CH_STARTING] =	"starting",
+	[CH_RUNNING] =	"running",
+	[CH_STOPPING] =	"stopping",
+	[CH_DIED] =	"died, (restarting)",
+};
+
+static struct vev	*ev_poker;
+static struct vev	*ev_listen;
+static struct vlu	*vlu;
+
+static struct vsb *child_panic = NULL;
+
+/*--------------------------------------------------------------------
+ * Track the highest file descriptor the parent knows is being used.
+ *
+ * This allows the child process to clean/close only a small fraction
+ * of the possible file descriptors after exec(2).
+ *
+ * This is likely to a bit on the low side, as libc and other libraries
+ * has a tendency to cache file descriptors (syslog, resolver, etc.)
+ * so we add a margin of 100 fds.
+ */
+
+static int		mgt_max_fd;
+
+#define CLOSE_FD_UP_TO	(mgt_max_fd + 100)
+
+void
+mgt_got_fd(int fd)
+{
+	/*
+	 * Assert > 0, to catch bogus opens, we know where stdin goes
+	 * in the master process.
+	 */
+	assert(fd > 0);
+	if (fd > mgt_max_fd)
+		mgt_max_fd = fd;
+}
+
+/*--------------------------------------------------------------------
+ * A handy little function
+ */
+
+static inline void
+closex(int *fd)
+{
+
+	assert(*fd >= 0);
+	AZ(close(*fd));
+	*fd = -1;
+}
+
+/*--------------------------------------------------------------------
+ * Keep track of which filedescriptors the child should inherit and
+ * which should be closed after fork()
+ */
+
+void
+mgt_child_inherit(int fd, const char *what)
+{
+
+	assert(fd >= 0);
+	if (fd_map == NULL)
+		fd_map = vbit_init(128);
+	AN(fd_map);
+	if (what != NULL)
+		vbit_set(fd_map, fd);
+	else
+		vbit_clr(fd_map, fd);
+}
+
+/*--------------------------------------------------------------------*/
+
+static int
+child_line(void *priv, const char *p)
+{
+	(void)priv;
+
+	REPORT(LOG_NOTICE, "Child (%jd) said %s", (intmax_t)child_pid, p);
+	return (0);
+}
+
+/*--------------------------------------------------------------------*/
+
+static int
+child_listener(const struct vev *e, int what)
+{
+
+	(void)e;
+	if ((what & ~EV_RD)) {
+		ev_listen = NULL;
+		return (1);
+	}
+	if (VLU_Fd(child_output, vlu)) {
+		ev_listen = NULL;
+		return (1);
+	}
+	return (0);
+}
+
+/*--------------------------------------------------------------------*/
+
+static int
+child_poker(const struct vev *e, int what)
+{
+
+	(void)e;
+	(void)what;
+	if (child_state != CH_RUNNING)
+		return (1);
+	if (child_pid < 0)
+		return (0);
+	if (!mgt_cli_askchild(NULL, NULL, "ping\n"))
+		return (0);
+	return (0);
+}
+
+/*--------------------------------------------------------------------
+ * If CLI communications with the child process fails, there is nothing
+ * for us to do but to drag it behind the barn and get it over with.
+ *
+ * The typical case is where the child process fails to return a reply
+ * before the cli_timeout expires.  This invalidates the CLI pipes for
+ * all future use, as we don't know if the child was just slow and the
+ * result gets piped later on, or if the child is catatonic.
+ */
+
+void
+MGT_Child_Cli_Fail(void)
+{
+
+	if (child_state != CH_RUNNING)
+		return;
+	if (child_pid < 0)
+		return;
+	REPORT(LOG_ERR, "Child (%jd) not responding to CLI, killing it.",
+	    (intmax_t)child_pid);
+	if (params->diag_bitmap & 0x1000)
+		(void)kill(child_pid, SIGKILL);
+	else
+		(void)kill(child_pid, SIGQUIT);
+}
+
+/*--------------------------------------------------------------------*/
+
+static int
+open_sockets(void)
+{
+	struct listen_sock *ls, *ls2;
+	int good = 0;
+
+	VTAILQ_FOREACH_SAFE(ls, &heritage.socks, list, ls2) {
+		if (ls->sock >= 0) {
+			good++;
+			continue;
+		}
+		ls->sock = VSS_bind(ls->addr);
+		if (ls->sock < 0)
+			continue;
+
+		mgt_child_inherit(ls->sock, "sock");
+
+		/*
+		 * Set nonblocking mode to avoid a race where a client
+		 * closes before we call accept(2) and nobody else are in
+		 * the listen queue to release us.
+		 */
+		(void)VTCP_filter_http(ls->sock);
+		good++;
+	}
+	if (!good)
+		return (1);
+	return (0);
+}
+
+/*--------------------------------------------------------------------*/
+
+static void
+close_sockets(void)
+{
+	struct listen_sock *ls;
+
+	VTAILQ_FOREACH(ls, &heritage.socks, list) {
+		if (ls->sock < 0)
+			continue;
+		mgt_child_inherit(ls->sock, NULL);
+		closex(&ls->sock);
+	}
+}
+
+/*--------------------------------------------------------------------*/
+
+static void
+start_child(struct cli *cli)
+{
+	pid_t pid;
+	unsigned u;
+	char *p;
+	struct vev *e;
+	int i, cp[2];
+
+	if (child_state != CH_STOPPED && child_state != CH_DIED)
+		return;
+
+	if (open_sockets() != 0) {
+		child_state = CH_STOPPED;
+		if (cli != NULL) {
+			VCLI_SetResult(cli, CLIS_CANT);
+			VCLI_Out(cli, "Could not open sockets");
+			return;
+		}
+		REPORT0(LOG_ERR,
+		    "Child start failed: could not open sockets");
+		return;
+	}
+
+	child_state = CH_STARTING;
+
+	/* Open pipe for mgr->child CLI */
+	AZ(pipe(cp));
+	heritage.cli_in = cp[0];
+	mgt_child_inherit(heritage.cli_in, "cli_in");
+	child_VCLI_Out = cp[1];
+
+	/* Open pipe for child->mgr CLI */
+	AZ(pipe(cp));
+	heritage.VCLI_Out = cp[1];
+	mgt_child_inherit(heritage.VCLI_Out, "VCLI_Out");
+	child_cli_in = cp[0];
+
+	/*
+	 * Open pipe for child stdout/err
+	 * NB: not inherited, because we dup2() it to stdout/stderr in child
+	 */
+	AZ(pipe(cp));
+	heritage.std_fd = cp[1];
+	child_output = cp[0];
+
+	MCF_ParamSync();
+	if ((pid = fork()) < 0) {
+		perror("Could not fork child");
+		exit(1);
+	}
+	if (pid == 0) {
+
+		/* Redirect stdin/out/err */
+		AZ(close(STDIN_FILENO));
+		assert(open("/dev/null", O_RDONLY) == STDIN_FILENO);
+		assert(dup2(heritage.std_fd, STDOUT_FILENO) == STDOUT_FILENO);
+		assert(dup2(heritage.std_fd, STDERR_FILENO) == STDERR_FILENO);
+
+		/* Close anything we shouldn't know about */
+		closelog();
+		for (i = STDERR_FILENO + 1; i < CLOSE_FD_UP_TO; i++) {
+			if (vbit_test(fd_map, i))
+				continue;
+			(void)(close(i) == 0);
+		}
+#ifdef HAVE_SETPROCTITLE
+		setproctitle("Varnish-Chld %s", heritage.name);
+#endif
+
+		(void)signal(SIGINT, SIG_DFL);
+		(void)signal(SIGTERM, SIG_DFL);
+
+		mgt_sandbox();
+
+		child_main();
+
+		exit(1);
+	}
+	REPORT(LOG_NOTICE, "child (%jd) Started", (intmax_t)pid);
+
+	/* Close stuff the child got */
+	closex(&heritage.std_fd);
+
+	mgt_child_inherit(heritage.cli_in, NULL);
+	closex(&heritage.cli_in);
+
+	mgt_child_inherit(heritage.VCLI_Out, NULL);
+	closex(&heritage.VCLI_Out);
+
+	close_sockets();
+
+	vlu = VLU_New(NULL, child_line, 0);
+	AN(vlu);
+
+	AZ(ev_listen);
+	e = vev_new();
+	XXXAN(e);
+	e->fd = child_output;
+	e->fd_flags = EV_RD;
+	e->name = "Child listener";
+	e->callback = child_listener;
+	AZ(vev_add(mgt_evb, e));
+	ev_listen = e;
+	AZ(ev_poker);
+	if (params->ping_interval > 0) {
+		e = vev_new();
+		XXXAN(e);
+		e->timeout = params->ping_interval;
+		e->callback = child_poker;
+		e->name = "child poker";
+		AZ(vev_add(mgt_evb, e));
+		ev_poker = e;
+	}
+
+	mgt_cli_start_child(child_cli_in, child_VCLI_Out);
+	child_pid = pid;
+	if (mgt_push_vcls_and_start(&u, &p)) {
+		REPORT(LOG_ERR, "Pushing vcls failed:\n%s", p);
+		free(p);
+		child_state = CH_RUNNING;
+		mgt_stop_child();
+	} else
+		child_state = CH_RUNNING;
+}
+
+/*--------------------------------------------------------------------*/
+
+void
+mgt_stop_child(void)
+{
+
+	if (child_state != CH_RUNNING)
+		return;
+
+	child_state = CH_STOPPING;
+
+	REPORT0(LOG_DEBUG, "Stopping Child");
+	if (ev_poker != NULL) {
+		vev_del(mgt_evb, ev_poker);
+		free(ev_poker);
+	}
+	ev_poker = NULL;
+
+	mgt_cli_stop_child();
+
+	/* We tell the child to die gracefully by closing the CLI */
+	closex(&child_VCLI_Out);
+	closex(&child_cli_in);
+}
+
+/*--------------------------------------------------------------------*/
+
+static void
+mgt_report_panic(pid_t r)
+{
+
+	if (VSM_head->panicstr[0] == '\0')
+		return;
+	REPORT(LOG_ERR, "Child (%jd) Panic message: %s",
+	    (intmax_t)r, VSM_head->panicstr);
+}
+
+static void
+mgt_save_panic(void)
+{
+	char time_str[30];
+	if (VSM_head->panicstr[0] == '\0')
+		return;
+
+	if (child_panic)
+		VSB_delete(child_panic);
+	child_panic = VSB_new_auto();
+	XXXAN(child_panic);
+	VTIM_format(VTIM_real(), time_str);
+	VSB_printf(child_panic, "Last panic at: %s\n", time_str);
+	VSB_cat(child_panic, VSM_head->panicstr);
+	AZ(VSB_finish(child_panic));
+}
+
+static void
+mgt_clear_panic(void)
+{
+	VSB_delete(child_panic);
+	child_panic = NULL;
+}
+
+/*--------------------------------------------------------------------*/
+
+static int
+mgt_sigchld(const struct vev *e, int what)
+{
+	int status;
+	struct vsb *vsb;
+	pid_t r;
+
+	(void)e;
+	(void)what;
+
+	if (ev_poker != NULL) {
+		vev_del(mgt_evb, ev_poker);
+		free(ev_poker);
+	}
+	ev_poker = NULL;
+
+	r = waitpid(child_pid, &status, WNOHANG);
+	if (r == 0 || (r == -1 && errno == ECHILD))
+		return (0);
+	assert(r == child_pid);
+	vsb = VSB_new_auto();
+	XXXAN(vsb);
+	VSB_printf(vsb, "Child (%d) %s", r, status ? "died" : "ended");
+	if (WIFEXITED(status) && WEXITSTATUS(status)) {
+		VSB_printf(vsb, " status=%d", WEXITSTATUS(status));
+		exit_status |= 0x20;
+	}
+	if (WIFSIGNALED(status)) {
+		VSB_printf(vsb, " signal=%d", WTERMSIG(status));
+		exit_status |= 0x40;
+	}
+#ifdef WCOREDUMP
+	if (WCOREDUMP(status)) {
+		VSB_printf(vsb, " (core dumped)");
+		exit_status |= 0x80;
+	}
+#endif
+	AZ(VSB_finish(vsb));
+	REPORT(LOG_INFO, "%s", VSB_data(vsb));
+	VSB_delete(vsb);
+
+	mgt_report_panic(r);
+	mgt_save_panic();
+
+	child_pid = -1;
+
+	if (child_state == CH_RUNNING) {
+		child_state = CH_DIED;
+		mgt_cli_stop_child();
+		closex(&child_VCLI_Out);
+		closex(&child_cli_in);
+	}
+
+	if (ev_listen != NULL) {
+		vev_del(mgt_evb, ev_listen);
+		free(ev_listen);
+		ev_listen = NULL;
+	}
+	/* Pick up any stuff lingering on stdout/stderr */
+	(void)child_listener(NULL, EV_RD);
+	closex(&child_output);
+
+	REPORT0(LOG_DEBUG, "Child cleanup complete");
+
+	if (child_state == CH_DIED && params->auto_restart)
+		start_child(NULL);
+	else if (child_state == CH_DIED) {
+		child_state = CH_STOPPED;
+	} else if (child_state == CH_STOPPING)
+		child_state = CH_STOPPED;
+
+	return (0);
+}
+
+/*--------------------------------------------------------------------*/
+
+static int
+mgt_sigint(const struct vev *e, int what)
+{
+
+	(void)e;
+	(void)what;
+	REPORT0(LOG_ERR, "Manager got SIGINT");
+	(void)fflush(stdout);
+	if (child_pid >= 0)
+		mgt_stop_child();
+	exit (2);
+}
+
+/*--------------------------------------------------------------------
+ * This thread is the master thread in the management process.
+ * The relatively simple task is to start and stop the child process
+ * and to reincarnate it in case of trouble.
+ */
+
+void
+MGT_Run(void)
+{
+	struct sigaction sac;
+	struct vev *e;
+	int i;
+
+	e = vev_new();
+	XXXAN(e);
+	e->sig = SIGTERM;
+	e->callback = mgt_sigint;
+	e->name = "mgt_sigterm";
+	AZ(vev_add(mgt_evb, e));
+
+	e = vev_new();
+	XXXAN(e);
+	e->sig = SIGINT;
+	e->callback = mgt_sigint;
+	e->name = "mgt_sigint";
+	AZ(vev_add(mgt_evb, e));
+
+	e = vev_new();
+	XXXAN(e);
+	e->sig = SIGCHLD;
+	e->sig_flags = SA_NOCLDSTOP;
+	e->callback = mgt_sigchld;
+	e->name = "mgt_sigchild";
+	AZ(vev_add(mgt_evb, e));
+
+#ifdef HAVE_SETPROCTITLE
+	setproctitle("Varnish-Mgr %s", heritage.name);
+#endif
+
+	memset(&sac, 0, sizeof sac);
+	sac.sa_handler = SIG_IGN;
+	sac.sa_flags = SA_RESTART;
+
+	AZ(sigaction(SIGPIPE, &sac, NULL));
+	AZ(sigaction(SIGHUP, &sac, NULL));
+
+	if (!d_flag && !mgt_has_vcl())
+		REPORT0(LOG_ERR, "No VCL loaded yet");
+	else if (!d_flag) {
+		start_child(NULL);
+		if (child_state == CH_STOPPED) {
+			exit_status = 2;
+			return;
+		}
+	}
+
+	i = vev_schedule(mgt_evb);
+	if (i != 0)
+		REPORT(LOG_ERR, "vev_schedule() = %d", i);
+
+	REPORT0(LOG_ERR, "manager dies");
+}
+
+/*--------------------------------------------------------------------*/
+
+void __match_proto__(cli_func_t)
+mcf_server_startstop(struct cli *cli, const char * const *av, void *priv)
+{
+
+	(void)av;
+	if (priv != NULL && child_state == CH_RUNNING)
+		mgt_stop_child();
+	else if (priv == NULL && child_state == CH_STOPPED) {
+		if (mgt_has_vcl()) {
+			start_child(cli);
+		} else {
+			VCLI_SetResult(cli, CLIS_CANT);
+			VCLI_Out(cli, "No VCL available");
+		}
+	} else {
+		VCLI_SetResult(cli, CLIS_CANT);
+		VCLI_Out(cli, "Child in state %s", ch_state[child_state]);
+	}
+}
+
+/*--------------------------------------------------------------------*/
+
+void
+mcf_server_status(struct cli *cli, const char * const *av, void *priv)
+{
+	(void)av;
+	(void)priv;
+	VCLI_Out(cli, "Child in state %s", ch_state[child_state]);
+}
+
+void
+mcf_panic_show(struct cli *cli, const char * const *av, void *priv)
+{
+	(void)av;
+	(void)priv;
+
+	if (!child_panic) {
+	  VCLI_SetResult(cli, CLIS_CANT);
+	  VCLI_Out(cli, "Child has not panicked or panic has been cleared");
+	  return;
+	}
+
+	VCLI_Out(cli, "%s\n", VSB_data(child_panic));
+}
+
+void
+mcf_panic_clear(struct cli *cli, const char * const *av, void *priv)
+{
+	(void)av;
+	(void)priv;
+
+	if (!child_panic) {
+	  VCLI_SetResult(cli, CLIS_CANT);
+	  VCLI_Out(cli, "No panic to clear");
+	  return;
+	}
+
+	mgt_clear_panic();
+}
diff --git a/bin/varnishd/mgt/mgt_cli.c b/bin/varnishd/mgt/mgt_cli.c
new file mode 100644
index 0000000..c8f22c3
--- /dev/null
+++ b/bin/varnishd/mgt/mgt_cli.c
@@ -0,0 +1,663 @@
+/*-
+ * Copyright (c) 2006 Verdens Gang AS
+ * Copyright (c) 2006-2011 Varnish Software AS
+ * All rights reserved.
+ *
+ * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * The management process' CLI handling
+ */
+
+#include "config.h"
+
+#include <sys/socket.h>
+
+#include <fcntl.h>
+#include <poll.h>
+#include <stdarg.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <syslog.h>
+#include <unistd.h>
+
+#include "mgt/mgt.h"
+
+#include "heritage.h"
+#include "vcli.h"
+#include "vcli_common.h"
+#include "vcli_priv.h"
+#include "vcli_serve.h"
+#include "vev.h"
+#include "vlu.h"
+#include "vss.h"
+#include "vtcp.h"
+
+#include "mgt_cli.h"
+
+#ifndef HAVE_SRANDOMDEV
+#include "compat/srandomdev.h"
+#endif
+
+static int		cli_i = -1, cli_o = -1;
+static struct VCLS	*cls;
+static const char	*secret_file;
+
+#define	MCF_NOAUTH	0	/* NB: zero disables here-documents */
+#define MCF_AUTH	16
+
+/*--------------------------------------------------------------------*/
+
+static void
+mcf_banner(struct cli *cli, const char *const *av, void *priv)
+{
+
+	(void)av;
+	(void)priv;
+	VCLI_Out(cli, "-----------------------------\n");
+	VCLI_Out(cli, "Varnish Cache CLI 1.0\n");
+	VCLI_Out(cli, "-----------------------------\n");
+	VCLI_Out(cli, "%s\n", VSB_data(vident) + 1);
+	VCLI_Out(cli, "\n");
+	VCLI_Out(cli, "Type 'help' for command list.\n");
+	VCLI_Out(cli, "Type 'quit' to close CLI session.\n");
+	if (child_pid < 0)
+		VCLI_Out(cli, "Type 'start' to launch worker process.\n");
+	VCLI_SetResult(cli, CLIS_OK);
+}
+
+/*--------------------------------------------------------------------*/
+
+/* XXX: what order should this list be in ? */
+static struct cli_proto cli_proto[] = {
+	{ CLI_BANNER,		"", mcf_banner, NULL },
+	{ CLI_SERVER_STATUS,	"", mcf_server_status, NULL },
+	{ CLI_SERVER_START,	"", mcf_server_startstop, NULL },
+	{ CLI_SERVER_STOP,	"", mcf_server_startstop, cli_proto },
+	{ CLI_VCL_LOAD,		"", mcf_config_load, NULL },
+	{ CLI_VCL_INLINE,	"", mcf_config_inline, NULL },
+	{ CLI_VCL_USE,		"", mcf_config_use, NULL },
+	{ CLI_VCL_DISCARD,	"", mcf_config_discard, NULL },
+	{ CLI_VCL_LIST,		"", mcf_config_list, NULL },
+	{ CLI_VCL_SHOW,		"", mcf_config_show, NULL },
+	{ CLI_PARAM_SHOW,	"", mcf_param_show, NULL },
+	{ CLI_PARAM_SET,	"", mcf_param_set, NULL },
+	{ CLI_PANIC_SHOW,	"", mcf_panic_show, NULL },
+	{ CLI_PANIC_CLEAR,	"", mcf_panic_clear, NULL },
+	{ NULL }
+};
+
+/*--------------------------------------------------------------------*/
+
+static void
+mcf_panic(struct cli *cli, const char * const *av, void *priv)
+{
+
+	(void)cli;
+	(void)av;
+	(void)priv;
+	assert(!strcmp("", "You asked for it"));
+}
+
+static struct cli_proto cli_debug[] = {
+	{ "debug.panic.master", "debug.panic.master",
+		"\tPanic the master process.\n",
+		0, 0, "d", mcf_panic, NULL},
+	{ NULL }
+};
+
+/*--------------------------------------------------------------------*/
+
+static void
+mcf_askchild(struct cli *cli, const char * const *av, void *priv)
+{
+	int i;
+	char *q;
+	unsigned u;
+	struct vsb *vsb;
+
+	(void)priv;
+	/*
+	 * Command not recognized in master, try cacher if it is
+	 * running.
+	 */
+	if (cli_o <= 0) {
+		if (!strcmp(av[1], "help")) {
+			VCLI_Out(cli, "No help from child, (not running).\n");
+			return;
+		}
+		VCLI_SetResult(cli, CLIS_UNKNOWN);
+		VCLI_Out(cli,
+		    "Unknown request in manager process "
+		    "(child not running).\n"
+		    "Type 'help' for more info.");
+		return;
+	}
+	vsb = VSB_new_auto();
+	for (i = 1; av[i] != NULL; i++) {
+		VSB_quote(vsb, av[i], strlen(av[i]), 0);
+		VSB_putc(vsb, ' ');
+	}
+	VSB_putc(vsb, '\n');
+	AZ(VSB_finish(vsb));
+	i = write(cli_o, VSB_data(vsb), VSB_len(vsb));
+	if (i != VSB_len(vsb)) {
+		VSB_delete(vsb);
+		VCLI_SetResult(cli, CLIS_COMMS);
+		VCLI_Out(cli, "CLI communication error");
+		MGT_Child_Cli_Fail();
+		return;
+	}
+	VSB_delete(vsb);
+	(void)VCLI_ReadResult(cli_i, &u, &q, params->cli_timeout);
+	VCLI_SetResult(cli, u);
+	VCLI_Out(cli, "%s", q);
+	free(q);
+}
+
+static struct cli_proto cli_askchild[] = {
+	{ "*", "<wild-card-entry>", "\t<fall through to cacher>\n",
+		0, 9999, "h*", mcf_askchild, NULL},
+	{ NULL }
+};
+
+/*--------------------------------------------------------------------
+ * Ask the child something over CLI, return zero only if everything is
+ * happy happy.
+ */
+
+int
+mgt_cli_askchild(unsigned *status, char **resp, const char *fmt, ...) {
+	int i, j;
+	va_list ap;
+	unsigned u;
+	char buf[params->cli_buffer], *p;
+
+	if (resp != NULL)
+		*resp = NULL;
+	if (status != NULL)
+		*status = 0;
+	if (cli_i < 0|| cli_o < 0) {
+		if (status != NULL)
+			*status = CLIS_CANT;
+		return (CLIS_CANT);
+	}
+	va_start(ap, fmt);
+	vbprintf(buf, fmt, ap);
+	va_end(ap);
+	p = strchr(buf, '\0');
+	assert(p != NULL && p > buf && p[-1] == '\n');
+	i = p - buf;
+	j = write(cli_o, buf, i);
+	if (j != i) {
+		if (status != NULL)
+			*status = CLIS_COMMS;
+		if (resp != NULL)
+			*resp = strdup("CLI communication error");
+		MGT_Child_Cli_Fail();
+		return (CLIS_COMMS);
+	}
+
+	(void)VCLI_ReadResult(cli_i, &u, resp, params->cli_timeout);
+	if (status != NULL)
+		*status = u;
+	if (u == CLIS_COMMS)
+		MGT_Child_Cli_Fail();
+	return (u == CLIS_OK ? 0 : u);
+}
+
+/*--------------------------------------------------------------------*/
+
+void
+mgt_cli_start_child(int fdi, int fdo)
+{
+
+	cli_i = fdi;
+	cli_o = fdo;
+}
+
+/*--------------------------------------------------------------------*/
+
+void
+mgt_cli_stop_child(void)
+{
+
+	cli_i = -1;
+	cli_o = -1;
+	/* XXX: kick any users */
+}
+
+/*--------------------------------------------------------------------
+ * Generate a random challenge
+ */
+
+static void
+mgt_cli_challenge(struct cli *cli)
+{
+	int i;
+
+	for (i = 0; i + 2L < sizeof cli->challenge; i++)
+		cli->challenge[i] = (random() % 26) + 'a';
+	cli->challenge[i++] = '\n';
+	cli->challenge[i] = '\0';
+	VCLI_Out(cli, "%s", cli->challenge);
+	VCLI_Out(cli, "\nAuthentication required.\n");
+	VCLI_SetResult(cli, CLIS_AUTH);
+}
+
+/*--------------------------------------------------------------------
+ * Validate the authentication
+ */
+
+static void
+mcf_auth(struct cli *cli, const char *const *av, void *priv)
+{
+	int fd;
+	char buf[CLI_AUTH_RESPONSE_LEN + 1];
+
+	AN(av[2]);
+	(void)priv;
+	if (secret_file == NULL) {
+		VCLI_Out(cli, "Secret file not configured\n");
+		VCLI_SetResult(cli, CLIS_CANT);
+		return;
+	}
+	fd = open(secret_file, O_RDONLY);
+	if (fd < 0) {
+		VCLI_Out(cli, "Cannot open secret file (%s)\n",
+		    strerror(errno));
+		VCLI_SetResult(cli, CLIS_CANT);
+		return;
+	}
+	mgt_got_fd(fd);
+	VCLI_AuthResponse(fd, cli->challenge, buf);
+	AZ(close(fd));
+	if (strcasecmp(buf, av[2])) {
+		mgt_cli_challenge(cli);
+		return;
+	}
+	cli->auth = MCF_AUTH;
+	memset(cli->challenge, 0, sizeof cli->challenge);
+	VCLI_SetResult(cli, CLIS_OK);
+	mcf_banner(cli, av, priv);
+}
+
+static struct cli_proto cli_auth[] = {
+	{ CLI_HELP,		"", VCLS_func_help, NULL },
+	{ CLI_PING,		"", VCLS_func_ping },
+	{ CLI_AUTH,		"", mcf_auth, NULL },
+	{ CLI_QUIT,		"", VCLS_func_close, NULL},
+	{ NULL }
+};
+
+/*--------------------------------------------------------------------*/
+static void
+mgt_cli_cb_before(const struct cli *cli)
+{
+
+	if (params->syslog_cli_traffic)
+		syslog(LOG_NOTICE, "CLI %s Rd %s", cli->ident, cli->cmd);
+}
+
+static void
+mgt_cli_cb_after(const struct cli *cli)
+{
+
+	if (params->syslog_cli_traffic)
+		syslog(LOG_NOTICE, "CLI %s Wr %03u %s",
+		    cli->ident, cli->result, VSB_data(cli->sb));
+}
+
+/*--------------------------------------------------------------------*/
+
+static void
+mgt_cli_init_cls(void)
+{
+
+	cls = VCLS_New(mgt_cli_cb_before, mgt_cli_cb_after, params->cli_buffer);
+	AN(cls);
+	AZ(VCLS_AddFunc(cls, MCF_NOAUTH, cli_auth));
+	AZ(VCLS_AddFunc(cls, MCF_AUTH, cli_proto));
+	AZ(VCLS_AddFunc(cls, MCF_AUTH, cli_debug));
+	AZ(VCLS_AddFunc(cls, MCF_AUTH, cli_stv));
+	AZ(VCLS_AddFunc(cls, MCF_AUTH, cli_askchild));
+}
+
+/*--------------------------------------------------------------------
+ * Get rid of all CLI sessions
+ */
+
+void
+mgt_cli_close_all(void)
+{
+
+	VCLS_Destroy(&cls);
+}
+
+/*--------------------------------------------------------------------
+ * Callback whenever something happens to the input fd of the session.
+ */
+
+static int
+mgt_cli_callback2(const struct vev *e, int what)
+{
+	int i;
+
+	(void)e;
+	(void)what;
+	i = VCLS_PollFd(cls, e->fd, 0);
+	return (i);
+}
+
+/*--------------------------------------------------------------------*/
+
+void
+mgt_cli_setup(int fdi, int fdo, int verbose, const char *ident, mgt_cli_close_f *closefunc, void *priv)
+{
+	struct cli *cli;
+	struct vev *ev;
+
+	(void)ident;
+	(void)verbose;
+	if (cls == NULL)
+		mgt_cli_init_cls();
+
+	cli = VCLS_AddFd(cls, fdi, fdo, closefunc, priv);
+
+	cli->ident = strdup(ident);
+
+	/* Deal with TELNET options */
+	if (fdi != 0)
+		VLU_SetTelnet(cli->vlu, fdo);
+
+	if (fdi != 0 && secret_file != NULL) {
+		cli->auth = MCF_NOAUTH;
+		mgt_cli_challenge(cli);
+	} else {
+		cli->auth = MCF_AUTH;
+		mcf_banner(cli, NULL, NULL);
+	}
+	AZ(VSB_finish(cli->sb));
+	(void)VCLI_WriteResult(fdo, cli->result, VSB_data(cli->sb));
+
+
+	ev = vev_new();
+	AN(ev);
+	ev->name = cli->ident;
+	ev->fd = fdi;
+	ev->fd_flags = EV_RD;
+	ev->callback = mgt_cli_callback2;
+	ev->priv = cli;
+	AZ(vev_add(mgt_evb, ev));
+}
+
+/*--------------------------------------------------------------------*/
+
+static struct vsb *
+sock_id(const char *pfx, int fd)
+{
+	struct vsb *vsb;
+
+	char abuf1[VTCP_ADDRBUFSIZE], abuf2[VTCP_ADDRBUFSIZE];
+	char pbuf1[VTCP_PORTBUFSIZE], pbuf2[VTCP_PORTBUFSIZE];
+
+	vsb = VSB_new_auto();
+	AN(vsb);
+	VTCP_myname(fd, abuf1, sizeof abuf1, pbuf1, sizeof pbuf1);
+	VTCP_hisname(fd, abuf2, sizeof abuf2, pbuf2, sizeof pbuf2);
+	VSB_printf(vsb, "%s %s %s %s %s", pfx, abuf2, pbuf2, abuf1, pbuf1);
+	AZ(VSB_finish(vsb));
+	return (vsb);
+}
+
+/*--------------------------------------------------------------------*/
+
+struct telnet {
+	unsigned		magic;
+#define TELNET_MAGIC		0x53ec3ac0
+	int			fd;
+	struct vev		*ev;
+};
+
+static void
+telnet_close(void *priv)
+{
+	struct telnet *tn;
+
+	CAST_OBJ_NOTNULL(tn, priv, TELNET_MAGIC);
+	(void)close(tn->fd);
+	FREE_OBJ(tn);
+}
+
+static struct telnet *
+telnet_new(int fd)
+{
+	struct telnet *tn;
+
+	ALLOC_OBJ(tn, TELNET_MAGIC);
+	AN(tn);
+	tn->fd = fd;
+	return (tn);
+}
+
+static int
+telnet_accept(const struct vev *ev, int what)
+{
+	struct vsb *vsb;
+	struct sockaddr_storage addr;
+	socklen_t addrlen;
+	struct telnet *tn;
+	int i;
+
+	(void)what;
+	addrlen = sizeof addr;
+	i = accept(ev->fd, (void *)&addr, &addrlen);
+	if (i < 0 && errno == EBADF)
+		return (1);
+	if (i < 0)
+		return (0);
+
+	mgt_got_fd(i);
+	tn = telnet_new(i);
+	vsb = sock_id("telnet", i);
+	mgt_cli_setup(i, i, 0, VSB_data(vsb), telnet_close, tn);
+	VSB_delete(vsb);
+	return (0);
+}
+
+void
+mgt_cli_secret(const char *S_arg)
+{
+	int i, fd;
+	char buf[BUFSIZ];
+	char *p;
+
+	/* Save in shmem */
+	i = strlen(S_arg);
+	p = VSM_Alloc(i + 1, "Arg", "-S", "");
+	AN(p);
+	strcpy(p, S_arg);
+
+	srandomdev();
+	fd = open(S_arg, O_RDONLY);
+	if (fd < 0) {
+		fprintf(stderr, "Can not open secret-file \"%s\"\n", S_arg);
+		exit (2);
+	}
+	mgt_got_fd(fd);
+	i = read(fd, buf, sizeof buf);
+	if (i == 0) {
+		fprintf(stderr, "Empty secret-file \"%s\"\n", S_arg);
+		exit (2);
+	}
+	if (i < 0) {
+		fprintf(stderr, "Can not read secret-file \"%s\"\n", S_arg);
+		exit (2);
+	}
+	AZ(close(fd));
+	secret_file = S_arg;
+}
+
+void
+mgt_cli_telnet(const char *T_arg)
+{
+	struct vss_addr **ta;
+	int i, n, sock, good;
+	struct telnet *tn;
+	char *p;
+	struct vsb *vsb;
+	char abuf[VTCP_ADDRBUFSIZE];
+	char pbuf[VTCP_PORTBUFSIZE];
+
+	n = VSS_resolve(T_arg, NULL, &ta);
+	if (n == 0) {
+		REPORT(LOG_ERR, "-T %s Could not be resolved\n", T_arg);
+		exit(2);
+	}
+	good = 0;
+	vsb = VSB_new_auto();
+	XXXAN(vsb);
+	for (i = 0; i < n; ++i) {
+		sock = VSS_listen(ta[i], 10);
+		if (sock < 0)
+			continue;
+		VTCP_myname(sock, abuf, sizeof abuf, pbuf, sizeof pbuf);
+		VSB_printf(vsb, "%s %s\n", abuf, pbuf);
+		good++;
+		tn = telnet_new(sock);
+		tn->ev = vev_new();
+		XXXAN(tn->ev);
+		tn->ev->fd = sock;
+		tn->ev->fd_flags = POLLIN;
+		tn->ev->callback = telnet_accept;
+		AZ(vev_add(mgt_evb, tn->ev));
+		free(ta[i]);
+		ta[i] = NULL;
+	}
+	free(ta);
+	if (good == 0) {
+		REPORT(LOG_ERR, "-T %s could not be listened on.", T_arg);
+		exit(2);
+	}
+	AZ(VSB_finish(vsb));
+	/* Save in shmem */
+	p = VSM_Alloc(VSB_len(vsb) + 1, "Arg", "-T", "");
+	AN(p);
+	strcpy(p, VSB_data(vsb));
+	VSB_delete(vsb);
+}
+
+/* Reverse CLI ("Master") connections --------------------------------*/
+
+static int M_fd = -1;
+static struct vev *M_poker, *M_conn;
+static struct vss_addr **M_ta;
+static int M_nta, M_nxt;
+static double M_poll = 0.1;
+
+static void
+Marg_closer(void *priv)
+{
+
+	(void)priv;
+	(void)close(M_fd);
+	M_fd = -1;
+}
+
+static int
+Marg_poker(const struct vev *e, int what)
+{
+	struct vsb *vsb;
+	int s, k;
+	socklen_t l;
+
+	(void)what;	/* XXX: ??? */
+
+	if (e == M_conn) {
+		/* Our connect(2) returned, check result */
+		l = sizeof k;
+		AZ(getsockopt(M_fd, SOL_SOCKET, SO_ERROR, &k, &l));
+		if (k) {
+			errno = k;
+			syslog(LOG_INFO, "Could not connect to CLI-master: %m");
+			(void)close(M_fd);
+			M_fd = -1;
+			/* Try next address */
+			if (++M_nxt >= M_nta) {
+				M_nxt = 0;
+				if (M_poll < 10)
+					M_poll *= 2;
+			}
+			return (1);
+		}
+		vsb = sock_id("master", M_fd);
+		mgt_cli_setup(M_fd, M_fd, 0, VSB_data(vsb), Marg_closer, NULL);
+		VSB_delete(vsb);
+		M_poll = 1;
+		return (1);
+	}
+
+	assert(e == M_poker);
+
+	M_poker->timeout = M_poll;	/* XXX nasty ? */
+	if (M_fd >= 0)
+		return (0);
+
+	/* Try to connect asynchronously */
+	s = VSS_connect(M_ta[M_nxt], 1);
+	if (s < 0)
+		return (0);
+
+	mgt_got_fd(s);
+
+	M_conn = vev_new();
+	AN(M_conn);
+	M_conn->callback = Marg_poker;
+	M_conn->name = "-M connector";
+	M_conn->fd_flags = EV_WR;
+	M_conn->fd = s;
+	M_fd = s;
+	AZ(vev_add(mgt_evb, M_conn));
+	return (0);
+}
+
+void
+mgt_cli_master(const char *M_arg)
+{
+	(void)M_arg;
+
+	M_nta = VSS_resolve(M_arg, NULL, &M_ta);
+	if (M_nta <= 0) {
+		fprintf(stderr, "Could resolve -M argument to address\n");
+		exit (1);
+	}
+	M_nxt = 0;
+	AZ(M_poker);
+	M_poker = vev_new();
+	AN(M_poker);
+	M_poker->timeout = M_poll;
+	M_poker->callback = Marg_poker;
+	M_poker->name = "-M poker";
+	AZ(vev_add(mgt_evb, M_poker));
+}
diff --git a/bin/varnishd/mgt/mgt_cli.h b/bin/varnishd/mgt/mgt_cli.h
new file mode 100644
index 0000000..8c49abb
--- /dev/null
+++ b/bin/varnishd/mgt/mgt_cli.h
@@ -0,0 +1,50 @@
+/*-
+ * Copyright (c) 2006 Verdens Gang AS
+ * Copyright (c) 2006-2009 Varnish Software AS
+ * All rights reserved.
+ *
+ * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ */
+
+/* mgt_child.c */
+cli_func_t mcf_server_startstop;
+cli_func_t mcf_server_status;
+cli_func_t mcf_panic_show;
+cli_func_t mcf_panic_clear;
+
+/* mgt_param.c */
+cli_func_t mcf_param_show;
+cli_func_t mcf_param_set;
+
+/* mgt_vcc.c */
+cli_func_t mcf_config_load;
+cli_func_t mcf_config_inline;
+cli_func_t mcf_config_use;
+cli_func_t mcf_config_discard;
+cli_func_t mcf_config_list;
+cli_func_t mcf_config_show;
+
+/* stevedore.c */
+extern struct cli_proto cli_stv[];
diff --git a/bin/varnishd/mgt/mgt_param.c b/bin/varnishd/mgt/mgt_param.c
new file mode 100644
index 0000000..1ca6a74
--- /dev/null
+++ b/bin/varnishd/mgt/mgt_param.c
@@ -0,0 +1,1237 @@
+/*-
+ * Copyright (c) 2006 Verdens Gang AS
+ * Copyright (c) 2006-2011 Varnish Software AS
+ * All rights reserved.
+ *
+ * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include "config.h"
+
+#include <grp.h>
+#include <limits.h>
+#include <pwd.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+
+#include "mgt/mgt.h"
+
+#include "waiter/cache_waiter.h"
+#include "heritage.h"
+#include "vav.h"
+#include "vcli.h"
+#include "vcli_common.h"
+#include "vcli_priv.h"
+#include "vparam.h"
+#include "vss.h"
+
+#include "mgt_cli.h"
+
+#define MAGIC_INIT_STRING	"\001"
+struct params master;
+static int nparspec;
+static struct parspec const ** parspec;
+static int margin;
+
+/*--------------------------------------------------------------------*/
+
+static const struct parspec *
+mcf_findpar(const char *name)
+{
+	int i;
+
+	for (i = 0; i < nparspec; i++)
+		if (!strcmp(parspec[i]->name, name))
+			return (parspec[i]);
+	return (NULL);
+}
+
+/*--------------------------------------------------------------------*/
+
+static void
+tweak_generic_timeout(struct cli *cli, volatile unsigned *dst, const char *arg)
+{
+	unsigned u;
+
+	if (arg != NULL) {
+		u = strtoul(arg, NULL, 0);
+		if (u == 0) {
+			VCLI_Out(cli, "Timeout must be greater than zero\n");
+			VCLI_SetResult(cli, CLIS_PARAM);
+			return;
+		}
+		*dst = u;
+	} else
+		VCLI_Out(cli, "%u", *dst);
+}
+
+/*--------------------------------------------------------------------*/
+
+void
+tweak_timeout(struct cli *cli, const struct parspec *par, const char *arg)
+{
+	volatile unsigned *dest;
+
+	dest = par->priv;
+	tweak_generic_timeout(cli, dest, arg);
+}
+
+static void
+tweak_timeout_double(struct cli *cli, const struct parspec *par,
+    const char *arg)
+{
+	volatile double *dest;
+	double u;
+
+	dest = par->priv;
+	if (arg != NULL) {
+		u = strtod(arg, NULL);
+		if (u < par->min) {
+			VCLI_Out(cli,
+			    "Timeout must be greater or equal to %.g\n",
+				 par->min);
+			VCLI_SetResult(cli, CLIS_PARAM);
+			return;
+		}
+		if (u > par->max) {
+			VCLI_Out(cli,
+			    "Timeout must be less than or equal to %.g\n",
+				 par->max);
+			VCLI_SetResult(cli, CLIS_PARAM);
+			return;
+		}
+		*dest = u;
+	} else
+		VCLI_Out(cli, "%.6f", *dest);
+}
+
+/*--------------------------------------------------------------------*/
+
+static void
+tweak_generic_double(struct cli *cli, const struct parspec *par,
+    const char *arg)
+{
+	volatile double *dest;
+	double u;
+
+	dest = par->priv;
+	if (arg != NULL) {
+		u = strtod(arg, NULL);
+		if (u < par->min) {
+			VCLI_Out(cli,
+			    "Must be greater or equal to %.g\n",
+				 par->min);
+			VCLI_SetResult(cli, CLIS_PARAM);
+			return;
+		}
+		if (u > par->max) {
+			VCLI_Out(cli,
+			    "Must be less than or equal to %.g\n",
+				 par->max);
+			VCLI_SetResult(cli, CLIS_PARAM);
+			return;
+		}
+		*dest = u;
+	} else
+		VCLI_Out(cli, "%f", *dest);
+}
+
+/*--------------------------------------------------------------------*/
+
+static void
+tweak_generic_bool(struct cli *cli, volatile unsigned *dest, const char *arg)
+{
+	if (arg != NULL) {
+		if (!strcasecmp(arg, "off"))
+			*dest = 0;
+		else if (!strcasecmp(arg, "disable"))
+			*dest = 0;
+		else if (!strcasecmp(arg, "no"))
+			*dest = 0;
+		else if (!strcasecmp(arg, "false"))
+			*dest = 0;
+		else if (!strcasecmp(arg, "on"))
+			*dest = 1;
+		else if (!strcasecmp(arg, "enable"))
+			*dest = 1;
+		else if (!strcasecmp(arg, "yes"))
+			*dest = 1;
+		else if (!strcasecmp(arg, "true"))
+			*dest = 1;
+		else {
+			VCLI_Out(cli, "use \"on\" or \"off\"\n");
+			VCLI_SetResult(cli, CLIS_PARAM);
+			return;
+		}
+	} else
+		VCLI_Out(cli, *dest ? "on" : "off");
+}
+
+/*--------------------------------------------------------------------*/
+
+static void
+tweak_bool(struct cli *cli, const struct parspec *par, const char *arg)
+{
+	volatile unsigned *dest;
+
+	dest = par->priv;
+	tweak_generic_bool(cli, dest, arg);
+}
+
+/*--------------------------------------------------------------------*/
+
+void
+tweak_generic_uint(struct cli *cli, volatile unsigned *dest, const char *arg,
+    unsigned min, unsigned max)
+{
+	unsigned u;
+
+	if (arg != NULL) {
+		if (!strcasecmp(arg, "unlimited"))
+			u = UINT_MAX;
+		else
+			u = strtoul(arg, NULL, 0);
+		if (u < min) {
+			VCLI_Out(cli, "Must be at least %u\n", min);
+			VCLI_SetResult(cli, CLIS_PARAM);
+			return;
+		}
+		if (u > max) {
+			VCLI_Out(cli, "Must be no more than %u\n", max);
+			VCLI_SetResult(cli, CLIS_PARAM);
+			return;
+		}
+		*dest = u;
+	} else if (*dest == UINT_MAX) {
+		VCLI_Out(cli, "unlimited", *dest);
+	} else {
+		VCLI_Out(cli, "%u", *dest);
+	}
+}
+
+/*--------------------------------------------------------------------*/
+
+void
+tweak_uint(struct cli *cli, const struct parspec *par, const char *arg)
+{
+	volatile unsigned *dest;
+
+	dest = par->priv;
+	tweak_generic_uint(cli, dest, arg, (uint)par->min, (uint)par->max);
+}
+
+/*--------------------------------------------------------------------
+ * XXX: slightly magic.  We want to initialize to "nobody" (XXX: shouldn't
+ * XXX: that be something autocrap found for us ?) but we don't want to
+ * XXX: fail initialization if that user doesn't exists, even though we
+ * XXX: do want to fail it, in subsequent sets.
+ * XXX: The magic init string is a hack for this.
+ */
+
+static void
+tweak_user(struct cli *cli, const struct parspec *par, const char *arg)
+{
+	struct passwd *pw;
+	struct group *gr;
+
+	(void)par;
+	if (arg != NULL) {
+		if (!strcmp(arg, MAGIC_INIT_STRING)) {
+			pw = getpwnam("nobody");
+			if (pw == NULL) {
+				master.uid = getuid();
+				return;
+			}
+		} else
+			pw = getpwnam(arg);
+		if (pw == NULL) {
+			VCLI_Out(cli, "Unknown user");
+			VCLI_SetResult(cli, CLIS_PARAM);
+			return;
+		}
+		REPLACE(master.user, pw->pw_name);
+		master.uid = pw->pw_uid;
+		master.gid = pw->pw_gid;
+
+		/* set group to user's primary group */
+		if ((gr = getgrgid(pw->pw_gid)) != NULL &&
+		    (gr = getgrnam(gr->gr_name)) != NULL &&
+		    gr->gr_gid == pw->pw_gid)
+			REPLACE(master.group, gr->gr_name);
+	} else if (master.user) {
+		VCLI_Out(cli, "%s (%d)", master.user, (int)master.uid);
+	} else {
+		VCLI_Out(cli, "%d", (int)master.uid);
+	}
+}
+
+/*--------------------------------------------------------------------
+ * XXX: see comment for tweak_user, same thing here.
+ */
+
+static void
+tweak_group(struct cli *cli, const struct parspec *par, const char *arg)
+{
+	struct group *gr;
+
+	(void)par;
+	if (arg != NULL) {
+		if (!strcmp(arg, MAGIC_INIT_STRING)) {
+			gr = getgrnam("nogroup");
+			if (gr == NULL) {
+				/* Only replace if tweak_user didn't */
+				if (master.gid == 0)
+					master.gid = getgid();
+				return;
+			}
+		} else
+			gr = getgrnam(arg);
+		if (gr == NULL) {
+			VCLI_Out(cli, "Unknown group");
+			VCLI_SetResult(cli, CLIS_PARAM);
+			return;
+		}
+		REPLACE(master.group, gr->gr_name);
+		master.gid = gr->gr_gid;
+	} else if (master.group) {
+		VCLI_Out(cli, "%s (%d)", master.group, (int)master.gid);
+	} else {
+		VCLI_Out(cli, "%d", (int)master.gid);
+	}
+}
+
+/*--------------------------------------------------------------------*/
+
+static void
+clean_listen_sock_head(struct listen_sock_head *lsh)
+{
+	struct listen_sock *ls, *ls2;
+
+	VTAILQ_FOREACH_SAFE(ls, lsh, list, ls2) {
+		CHECK_OBJ_NOTNULL(ls, LISTEN_SOCK_MAGIC);
+		VTAILQ_REMOVE(lsh, ls, list);
+		free(ls->name);
+		free(ls->addr);
+		FREE_OBJ(ls);
+	}
+}
+
+static void
+tweak_listen_address(struct cli *cli, const struct parspec *par,
+    const char *arg)
+{
+	char **av;
+	int i;
+	struct listen_sock		*ls;
+	struct listen_sock_head		lsh;
+
+	(void)par;
+	if (arg == NULL) {
+		VCLI_Quote(cli, master.listen_address);
+		return;
+	}
+
+	av = VAV_Parse(arg, NULL, ARGV_COMMA);
+	if (av == NULL) {
+		VCLI_Out(cli, "Parse error: out of memory");
+		VCLI_SetResult(cli, CLIS_PARAM);
+		return;
+	}
+	if (av[0] != NULL) {
+		VCLI_Out(cli, "Parse error: %s", av[0]);
+		VCLI_SetResult(cli, CLIS_PARAM);
+		VAV_Free(av);
+		return;
+	}
+	if (av[1] == NULL) {
+		VCLI_Out(cli, "Empty listen address");
+		VCLI_SetResult(cli, CLIS_PARAM);
+		VAV_Free(av);
+		return;
+	}
+	VTAILQ_INIT(&lsh);
+	for (i = 1; av[i] != NULL; i++) {
+		struct vss_addr **ta;
+		int j, n;
+
+		n = VSS_resolve(av[i], "http", &ta);
+		if (n == 0) {
+			VCLI_Out(cli, "Invalid listen address ");
+			VCLI_Quote(cli, av[i]);
+			VCLI_SetResult(cli, CLIS_PARAM);
+			break;
+		}
+		for (j = 0; j < n; ++j) {
+			ALLOC_OBJ(ls, LISTEN_SOCK_MAGIC);
+			AN(ls);
+			ls->sock = -1;
+			ls->addr = ta[j];
+			ls->name = strdup(av[i]);
+			AN(ls->name);
+			VTAILQ_INSERT_TAIL(&lsh, ls, list);
+		}
+		free(ta);
+	}
+	VAV_Free(av);
+	if (cli != NULL && cli->result != CLIS_OK) {
+		clean_listen_sock_head(&lsh);
+		return;
+	}
+
+	REPLACE(master.listen_address, arg);
+
+	clean_listen_sock_head(&heritage.socks);
+	heritage.nsocks = 0;
+
+	while (!VTAILQ_EMPTY(&lsh)) {
+		ls = VTAILQ_FIRST(&lsh);
+		VTAILQ_REMOVE(&lsh, ls, list);
+		CHECK_OBJ_NOTNULL(ls, LISTEN_SOCK_MAGIC);
+		VTAILQ_INSERT_TAIL(&heritage.socks, ls, list);
+		heritage.nsocks++;
+	}
+}
+
+/*--------------------------------------------------------------------*/
+
+static void
+tweak_string(struct cli *cli, const struct parspec *par, const char *arg)
+{
+	char **p = TRUST_ME(par->priv);
+
+	AN(p);
+	/* XXX should have tweak_generic_string */
+	if (arg == NULL) {
+		VCLI_Quote(cli, *p);
+	} else {
+		REPLACE(*p, arg);
+	}
+}
+
+/*--------------------------------------------------------------------*/
+
+static void
+tweak_waiter(struct cli *cli, const struct parspec *par, const char *arg)
+{
+
+	/* XXX should have tweak_generic_string */
+	(void)par;
+	WAIT_tweak_waiter(cli, arg);
+}
+
+/*--------------------------------------------------------------------*/
+
+static void
+tweak_diag_bitmap(struct cli *cli, const struct parspec *par, const char *arg)
+{
+	unsigned u;
+
+	(void)par;
+	if (arg != NULL) {
+		u = strtoul(arg, NULL, 0);
+		master.diag_bitmap = u;
+	} else {
+		VCLI_Out(cli, "0x%x", master.diag_bitmap);
+	}
+}
+
+/*--------------------------------------------------------------------*/
+
+/*
+ * Make sure to end all lines with either a space or newline of the
+ * formatting will go haywire.
+ */
+
+#define DELAYED_EFFECT_TEXT \
+	"\nNB: This parameter may take quite some time to take (full) effect."
+
+#define MUST_RESTART_TEXT \
+	"\nNB: This parameter will not take any effect until the " \
+	"child process has been restarted."
+
+#define MUST_RELOAD_TEXT \
+	"\nNB: This parameter will not take any effect until the " \
+	"VCL programs have been reloaded."
+
+#define EXPERIMENTAL_TEXT \
+	"\nNB: We do not know yet if it is a good idea to change " \
+	"this parameter, or if the default value is even sensible.  " \
+	"Caution is advised, and feedback is most welcome."
+
+#define WIZARD_TEXT \
+	"\nNB: Do not change this parameter, unless a developer tell " \
+	"you to do so."
+
+/*
+ * Remember to update varnishd.1 whenever you add / remove a parameter or
+ * change its default value.
+ * XXX: we should generate the relevant section of varnishd.1 from here.
+ */
+static const struct parspec input_parspec[] = {
+	{ "user", tweak_user, NULL, 0, 0,
+		"The unprivileged user to run as.  Setting this will "
+		"also set \"group\" to the specified user's primary group.",
+		MUST_RESTART,
+		MAGIC_INIT_STRING },
+	{ "group", tweak_group, NULL, 0, 0,
+		"The unprivileged group to run as.",
+		MUST_RESTART,
+		MAGIC_INIT_STRING },
+	{ "default_ttl", tweak_timeout_double, &master.default_ttl,
+		0, UINT_MAX,
+		"The TTL assigned to objects if neither the backend nor "
+		"the VCL code assigns one.\n"
+		"Objects already cached will not be affected by changes "
+		"made until they are fetched from the backend again.\n"
+		"To force an immediate effect at the expense of a total "
+		"flush of the cache use \"ban.url .\"",
+		0,
+		"120", "seconds" },
+	{ "sess_workspace", tweak_uint, &master.sess_workspace, 1024, UINT_MAX,
+		"Bytes of HTTP protocol workspace allocated for sessions. "
+		"This space must be big enough for the entire HTTP protocol "
+		"header and any edits done to it in the VCL code.\n"
+		"Minimum is 1024 bytes.",
+		DELAYED_EFFECT,
+		"65536",
+		"bytes" },
+	{ "http_req_hdr_len", tweak_uint, &master.http_req_hdr_len,
+		40, UINT_MAX,
+		"Maximum length of any HTTP client request header we will "
+		"allow.  The limit is inclusive its continuation lines.\n",
+		0,
+		"8192", "bytes" },
+	{ "http_req_size", tweak_uint, &master.http_req_size,
+		256, UINT_MAX,
+		"Maximum number of bytes of HTTP client request we will deal "
+		"with.  This is a limit on all bytes up to the double blank "
+		"line which ends the HTTP request.\n"
+		"The memory for the request is allocated from the session "
+		"workspace (param: sess_workspace) and this parameter limits "
+		"how much of that the request is allowed to take up.",
+		0,
+		"32768", "bytes" },
+	{ "http_resp_hdr_len", tweak_uint, &master.http_resp_hdr_len,
+		40, UINT_MAX,
+		"Maximum length of any HTTP backend response header we will "
+		"allow.  The limit is inclusive its continuation lines.\n",
+		0,
+		"8192", "bytes" },
+	{ "http_resp_size", tweak_uint, &master.http_resp_size,
+		256, UINT_MAX,
+		"Maximum number of bytes of HTTP backend resonse we will deal "
+		"with.  This is a limit on all bytes up to the double blank "
+		"line which ends the HTTP request.\n"
+		"The memory for the request is allocated from the worker "
+		"workspace (param: sess_workspace) and this parameter limits "
+		"how much of that the request is allowed to take up.",
+		0,
+		"32768", "bytes" },
+	{ "http_max_hdr", tweak_uint, &master.http_max_hdr, 32, 65535,
+		"Maximum number of HTTP headers we will deal with in "
+		"client request or backend reponses.  "
+		"Note that the first line occupies five header fields.\n"
+		"This paramter does not influence storage consumption, "
+		"objects allocate exact space for the headers they store.\n",
+		0,
+		"64", "header lines" },
+	{ "shm_workspace", tweak_uint, &master.shm_workspace, 4096, UINT_MAX,
+		"Bytes of shmlog workspace allocated for worker threads. "
+		"If too big, it wastes some ram, if too small it causes "
+		"needless flushes of the SHM workspace.\n"
+		"These flushes show up in stats as "
+		"\"SHM flushes due to overflow\".\n"
+		"Minimum is 4096 bytes.",
+		DELAYED_EFFECT,
+		"8192", "bytes" },
+	{ "shm_reclen", tweak_uint, &master.shm_reclen, 16, 65535,
+		"Maximum number of bytes in SHM log record.\n"
+		"Maximum is 65535 bytes.",
+		0,
+		"255", "bytes" },
+	{ "default_grace", tweak_timeout_double, &master.default_grace,
+		0, UINT_MAX,
+		"Default grace period.  We will deliver an object "
+		"this long after it has expired, provided another thread "
+		"is attempting to get a new copy.\n"
+		"Objects already cached will not be affected by changes "
+		"made until they are fetched from the backend again.\n",
+		DELAYED_EFFECT,
+		"10", "seconds" },
+	{ "default_keep", tweak_timeout_double, &master.default_keep,
+		0, UINT_MAX,
+		"Default keep period.  We will keep a useless object "
+		"around this long, making it available for conditional "
+		"backend fetches.  "
+		"That means that the object will be removed from the "
+		"cache at the end of ttl+grace+keep.",
+		DELAYED_EFFECT,
+		"0", "seconds" },
+	{ "sess_timeout", tweak_timeout, &master.sess_timeout, 0, 0,
+		"Idle timeout for persistent sessions. "
+		"If a HTTP request has not been received in this many "
+		"seconds, the session is closed.",
+		0,
+		"5", "seconds" },
+	{ "expiry_sleep", tweak_timeout_double, &master.expiry_sleep, 0, 60,
+		"How long the expiry thread sleeps when there is nothing "
+		"for it to do.\n",
+		0,
+		"1", "seconds" },
+	{ "pipe_timeout", tweak_timeout, &master.pipe_timeout, 0, 0,
+		"Idle timeout for PIPE sessions. "
+		"If nothing have been received in either direction for "
+		"this many seconds, the session is closed.\n",
+		0,
+		"60", "seconds" },
+	{ "send_timeout", tweak_timeout, &master.send_timeout, 0, 0,
+		"Send timeout for client connections. "
+		"If the HTTP response hasn't been transmitted in this many\n"
+                "seconds the session is closed. \n"
+		"See setsockopt(2) under SO_SNDTIMEO for more information.",
+		DELAYED_EFFECT,
+		"60", "seconds" },
+	{ "auto_restart", tweak_bool, &master.auto_restart, 0, 0,
+		"Restart child process automatically if it dies.\n",
+		0,
+		"on", "bool" },
+	{ "nuke_limit",
+		tweak_uint, &master.nuke_limit, 0, UINT_MAX,
+		"Maximum number of objects we attempt to nuke in order"
+		"to make space for a object body.",
+		EXPERIMENTAL,
+		"50", "allocations" },
+	{ "fetch_chunksize",
+		tweak_uint, &master.fetch_chunksize, 4, UINT_MAX / 1024.,
+		"The default chunksize used by fetcher. "
+		"This should be bigger than the majority of objects with "
+		"short TTLs.\n"
+		"Internal limits in the storage_file module makes increases "
+		"above 128kb a dubious idea.",
+		EXPERIMENTAL,
+		"128", "kilobytes" },
+	{ "fetch_maxchunksize",
+		tweak_uint, &master.fetch_maxchunksize, 64, UINT_MAX / 1024.,
+		"The maximum chunksize we attempt to allocate from storage. "
+		"Making this too large may cause delays and storage "
+		"fragmentation.\n",
+		EXPERIMENTAL,
+		"262144", "kilobytes" },
+#ifdef SENDFILE_WORKS
+	{ "sendfile_threshold",
+		tweak_uint, &master.sendfile_threshold, 0, UINT_MAX,
+		"The minimum size of objects transmitted with sendfile.",
+		EXPERIMENTAL,
+		"-1", "bytes" },
+#endif /* SENDFILE_WORKS */
+	{ "vcl_trace", tweak_bool,  &master.vcl_trace, 0, 0,
+		"Trace VCL execution in the shmlog.\n"
+		"Enabling this will allow you to see the path each "
+		"request has taken through the VCL program.\n"
+		"This generates a lot of logrecords so it is off by "
+		"default.",
+		0,
+		"off", "bool" },
+	{ "listen_address", tweak_listen_address, NULL, 0, 0,
+		"Whitespace separated list of network endpoints where "
+		"Varnish will accept requests.\n"
+		"Possible formats: host, host:port, :port",
+		MUST_RESTART,
+		":80" },
+	{ "listen_depth", tweak_uint, &master.listen_depth, 0, UINT_MAX,
+		"Listen queue depth.",
+		MUST_RESTART,
+		"1024", "connections" },
+	{ "cli_timeout", tweak_timeout, &master.cli_timeout, 0, 0,
+		"Timeout for the childs replies to CLI requests from "
+		"the master.",
+		0,
+		"10", "seconds" },
+	{ "ping_interval", tweak_uint, &master.ping_interval, 0, UINT_MAX,
+		"Interval between pings from parent to child.\n"
+		"Zero will disable pinging entirely, which makes "
+		"it possible to attach a debugger to the child.",
+		MUST_RESTART,
+		"3", "seconds" },
+	{ "lru_interval", tweak_timeout, &master.lru_timeout, 0, 0,
+		"Grace period before object moves on LRU list.\n"
+		"Objects are only moved to the front of the LRU "
+		"list if they have not been moved there already inside "
+		"this timeout period.  This reduces the amount of lock "
+		"operations necessary for LRU list access.",
+		EXPERIMENTAL,
+		"2", "seconds" },
+	{ "cc_command", tweak_string, &mgt_cc_cmd, 0, 0,
+		"Command used for compiling the C source code to a "
+		"dlopen(3) loadable object.  Any occurrence of %s in "
+		"the string will be replaced with the source file name, "
+		"and %o will be replaced with the output file name.",
+		MUST_RELOAD,
+		VCC_CC , NULL },
+	{ "max_restarts", tweak_uint, &master.max_restarts, 0, UINT_MAX,
+		"Upper limit on how many times a request can restart."
+		"\nBe aware that restarts are likely to cause a hit against "
+		"the backend, so don't increase thoughtlessly.\n",
+		0,
+		"4", "restarts" },
+	{ "esi_syntax",
+		tweak_uint, &master.esi_syntax, 0, UINT_MAX,
+		"Bitmap controlling ESI parsing code:\n"
+		"  0x00000001 - Don't check if it looks like XML\n"
+		"  0x00000002 - Ignore non-esi elements\n"
+		"  0x00000004 - Emit parsing debug records\n"
+		"  0x00000008 - Force-split parser input (debugging)\n"
+		"Use 0x notation and do the bitor in your head :-)\n",
+		0,
+		"0", "bitmap" },
+	{ "max_esi_depth",
+		tweak_uint, &master.max_esi_depth, 0, UINT_MAX,
+		"Maximum depth of esi:include processing.\n",
+		0,
+		"5", "levels" },
+	{ "connect_timeout", tweak_timeout_double,
+		&master.connect_timeout,0, UINT_MAX,
+		"Default connection timeout for backend connections. "
+		"We only try to connect to the backend for this many "
+		"seconds before giving up. "
+		"VCL can override this default value for each backend and "
+		"backend request.",
+		0,
+		"0.7", "s" },
+	{ "first_byte_timeout", tweak_timeout_double,
+		&master.first_byte_timeout,0, UINT_MAX,
+		"Default timeout for receiving first byte from backend. "
+		"We only wait for this many seconds for the first "
+		"byte before giving up. A value of 0 means it will never time "
+		"out. "
+		"VCL can override this default value for each backend and "
+		"backend request. This parameter does not apply to pipe.",
+		0,
+		"60", "s" },
+	{ "between_bytes_timeout", tweak_timeout_double,
+		&master.between_bytes_timeout,0, UINT_MAX,
+		"Default timeout between bytes when receiving data from "
+		"backend. "
+		"We only wait for this many seconds between bytes "
+		"before giving up. A value of 0 means it will never time out. "
+		"VCL can override this default value for each backend request "
+		"and backend request. This parameter does not apply to pipe.",
+		0,
+		"60", "s" },
+	{ "acceptor_sleep_max", tweak_timeout_double,
+		&master.acceptor_sleep_max, 0,  10,
+		"If we run out of resources, such as file descriptors or "
+		"worker threads, the acceptor will sleep between accepts.\n"
+		"This parameter limits how long it can sleep between "
+		"attempts to accept new connections.",
+		EXPERIMENTAL,
+		"0.050", "s" },
+	{ "acceptor_sleep_incr", tweak_timeout_double,
+		&master.acceptor_sleep_incr, 0,  1,
+		"If we run out of resources, such as file descriptors or "
+		"worker threads, the acceptor will sleep between accepts.\n"
+		"This parameter control how much longer we sleep, each time "
+		"we fail to accept a new connection.",
+		EXPERIMENTAL,
+		"0.001", "s" },
+	{ "acceptor_sleep_decay", tweak_generic_double,
+		&master.acceptor_sleep_decay, 0,  1,
+		"If we run out of resources, such as file descriptors or "
+		"worker threads, the acceptor will sleep between accepts.\n"
+		"This parameter (multiplicatively) reduce the sleep duration "
+		"for each succesfull accept. (ie: 0.9 = reduce by 10%)",
+		EXPERIMENTAL,
+		"0.900", "" },
+	{ "clock_skew", tweak_uint, &master.clock_skew, 0, UINT_MAX,
+		"How much clockskew we are willing to accept between the "
+		"backend and our own clock.",
+		0,
+		"10", "s" },
+	{ "prefer_ipv6", tweak_bool, &master.prefer_ipv6, 0, 0,
+		"Prefer IPv6 address when connecting to backends which "
+		"have both IPv4 and IPv6 addresses.",
+		0,
+		"off", "bool" },
+	{ "session_max", tweak_uint,
+		&master.max_sess, 1000, UINT_MAX,
+		"Maximum number of sessions we will allocate from one pool "
+		"before just dropping connections.\n"
+		"This is mostly an anti-DoS measure, and setting it plenty "
+		"high should not hurt, as long as you have the memory for "
+		"it.\n",
+		0,
+		"100000", "sessions" },
+	{ "session_linger", tweak_uint,
+		&master.session_linger,0, UINT_MAX,
+		"How long time the workerthread lingers on the session "
+		"to see if a new request appears right away.\n"
+		"If sessions are reused, as much as half of all reuses "
+		"happen within the first 100 msec of the previous request "
+		"completing.\n"
+		"Setting this too high results in worker threads not doing "
+		"anything for their keep, setting it too low just means that "
+		"more sessions take a detour around the waiter.",
+		EXPERIMENTAL,
+		"50", "ms" },
+	{ "cli_buffer", tweak_uint, &master.cli_buffer, 4096, UINT_MAX,
+		"Size of buffer for CLI input."
+		"\nYou may need to increase this if you have big VCL files "
+		"and use the vcl.inline CLI command.\n"
+		"NB: Must be specified with -p to have effect.\n",
+		0,
+		"8192", "bytes" },
+	{ "log_hashstring", tweak_bool, &master.log_hash, 0, 0,
+		"Log the hash string components to shared memory log.\n",
+		0,
+		"on", "bool" },
+	{ "log_local_address", tweak_bool, &master.log_local_addr, 0, 0,
+		"Log the local address on the TCP connection in the "
+		"SessionOpen shared memory record.\n",
+		0,
+		"off", "bool" },
+	{ "waiter", tweak_waiter, NULL, 0, 0,
+		"Select the waiter kernel interface.\n",
+		EXPERIMENTAL | MUST_RESTART,
+		"default", NULL },
+	{ "diag_bitmap", tweak_diag_bitmap, 0, 0, 0,
+		"Bitmap controlling diagnostics code:\n"
+		"  0x00000001 - CNT_Session states.\n"
+		"  0x00000002 - workspace debugging.\n"
+		"  0x00000004 - kqueue debugging.\n"
+		"  0x00000008 - mutex logging.\n"
+		"  0x00000010 - mutex contests.\n"
+		"  0x00000020 - waiting list.\n"
+		"  0x00000040 - object workspace.\n"
+		"  0x00001000 - do not core-dump child process.\n"
+		"  0x00002000 - only short panic message.\n"
+		"  0x00004000 - panic to stderr.\n"
+#ifdef HAVE_ABORT2
+		"  0x00008000 - panic to abort2().\n"
+#endif
+		"  0x00010000 - synchronize shmlog.\n"
+		"  0x00020000 - synchronous start of persistence.\n"
+		"  0x00040000 - release VCL early.\n"
+		"  0x80000000 - do edge-detection on digest.\n"
+		"Use 0x notation and do the bitor in your head :-)\n",
+		0,
+		"0", "bitmap" },
+	{ "ban_dups", tweak_bool, &master.ban_dups, 0, 0,
+		"Detect and eliminate duplicate bans.\n",
+		0,
+		"on", "bool" },
+	{ "syslog_cli_traffic", tweak_bool, &master.syslog_cli_traffic, 0, 0,
+		"Log all CLI traffic to syslog(LOG_INFO).\n",
+		0,
+		"on", "bool" },
+	{ "ban_lurker_sleep", tweak_timeout_double,
+		&master.ban_lurker_sleep, 0, UINT_MAX,
+		"How long time does the ban lurker thread sleeps between "
+		"successful attempts to push the last item up the ban "
+		" list.  It always sleeps a second when nothing can be done.\n"
+		"A value of zero disables the ban lurker.",
+		0,
+		"0.01", "s" },
+	{ "saintmode_threshold", tweak_uint,
+		&master.saintmode_threshold, 0, UINT_MAX,
+		"The maximum number of objects held off by saint mode before "
+		"no further will be made to the backend until one times out.  "
+		"A value of 0 disables saintmode.",
+		EXPERIMENTAL,
+		"10", "objects" },
+	{ "http_range_support", tweak_bool, &master.http_range_support, 0, 0,
+		"Enable support for HTTP Range headers.\n",
+		EXPERIMENTAL,
+		"on", "bool" },
+	{ "http_gzip_support", tweak_bool, &master.http_gzip_support, 0, 0,
+		"Enable gzip support. When enabled Varnish will compress "
+		"uncompressed objects before they are stored in the cache. "
+		"If a client does not support gzip encoding Varnish will "
+		"uncompress compressed objects on demand. Varnish will also "
+		"rewrite the Accept-Encoding header of clients indicating "
+		"support for gzip to:\n"
+		"Accept-Encoding: gzip\n\n"
+		"Clients that do not support gzip will have their "
+		"Accept-Encoding header removed. For more information on how "
+		"gzip is implemented please see the chapter on gzip in the "
+		"Varnish reference.",
+		EXPERIMENTAL,
+		"on", "bool" },
+	{ "gzip_tmp_space", tweak_uint, &master.gzip_tmp_space, 0, 2,
+		"Where temporary space for gzip/gunzip is allocated:\n"
+		"  0 - malloc\n"
+		"  1 - session workspace\n"
+		"  2 - thread workspace\n"
+		"If you have much gzip/gunzip activity, it may be an"
+		" advantage to use workspace for these allocations to reduce"
+		" malloc activity.  Be aware that gzip needs 256+KB and gunzip"
+		" needs 32+KB of workspace (64+KB if ESI processing).",
+		EXPERIMENTAL,
+		"0", "" },
+	{ "gzip_level", tweak_uint, &master.gzip_level, 0, 9,
+		"Gzip compression level: 0=debug, 1=fast, 9=best",
+		0,
+		"6", ""},
+	{ "gzip_window", tweak_uint, &master.gzip_window, 8, 15,
+		"Gzip window size 8=least, 15=most compression.\n"
+		"Memory impact is 8=1k, 9=2k, ... 15=128k.",
+		0,
+		"15", ""},
+	{ "gzip_memlevel", tweak_uint, &master.gzip_memlevel, 1, 9,
+		"Gzip memory level 1=slow/least, 9=fast/most compression.\n"
+		"Memory impact is 1=1k, 2=2k, ... 9=256k.",
+		0,
+		"8", ""},
+	{ "gzip_stack_buffer", tweak_uint, &master.gzip_stack_buffer,
+	        2048, UINT_MAX,
+		"Size of stack buffer used for gzip processing.\n"
+		"The stack buffers are used for in-transit data,"
+		" for instance gunzip'ed data being sent to a client."
+		"Making this space to small results in more overhead,"
+		" writes to sockets etc, making it too big is probably"
+		" just a waste of memory.",
+		EXPERIMENTAL,
+		"32768", "Bytes" },
+	{ "shortlived", tweak_timeout_double,
+		&master.shortlived, 0, UINT_MAX,
+		"Objects created with TTL shorter than this are always "
+		"put in transient storage.\n",
+		0,
+		"10.0", "s" },
+	{ "critbit_cooloff", tweak_timeout_double,
+		&master.critbit_cooloff, 60, 254,
+		"How long time the critbit hasher keeps deleted objheads "
+		"on the cooloff list.\n",
+		WIZARD,
+		"180.0", "s" },
+	{ "vcl_dir", tweak_string, &mgt_vcl_dir, 0, 0,
+		"Directory from which relative VCL filenames (vcl.load and "
+		"include) are opened.",
+		0,
+#ifdef VARNISH_VCL_DIR
+		VARNISH_VCL_DIR,
+#else
+		".",
+#endif
+		NULL },
+	{ "vmod_dir", tweak_string, &mgt_vmod_dir, 0, 0,
+		"Directory where VCL modules are to be found.",
+		0,
+#ifdef VARNISH_VMOD_DIR
+		VARNISH_VMOD_DIR,
+#else
+		".",
+#endif
+		NULL },
+	{ "vcc_err_unref", tweak_bool, &mgt_vcc_err_unref, 0, 0,
+		"Unreferenced VCL objects result in error.\n",
+		0,
+		"on", "bool" },
+
+
+	{ "pcre_match_limit", tweak_uint,
+		&master.vre_limits.match,
+		1, UINT_MAX,
+		"The limit for the  number of internal matching function"
+		" calls in a pcre_exec() execution.",
+		0,
+		"10000", ""},
+
+	{ "pcre_match_limit_recursion", tweak_uint,
+		&master.vre_limits.match_recursion,
+		1, UINT_MAX,
+		"The limit for the  number of internal matching function"
+		" recursions in a pcre_exec() execution.",
+		0,
+		"10000", ""},
+
+	{ NULL, NULL, NULL }
+};
+
+/*--------------------------------------------------------------------*/
+
+#define WIDTH 76
+
+static void
+mcf_wrap(struct cli *cli, const char *text)
+{
+	const char *p, *q;
+
+	/* Format text to COLUMNS width */
+	for (p = text; *p != '\0'; ) {
+		q = strchr(p, '\n');
+		if (q == NULL)
+			q = strchr(p, '\0');
+		if (q > p + WIDTH - margin) {
+			q = p + WIDTH - margin;
+			while (q > p && *q != ' ')
+				q--;
+			AN(q);
+		}
+		VCLI_Out(cli, "%*s %.*s\n", margin, "", (int)(q - p), p);
+		p = q;
+		if (*p == ' ' || *p == '\n')
+			p++;
+	}
+}
+
+void
+mcf_param_show(struct cli *cli, const char * const *av, void *priv)
+{
+	int i;
+	const struct parspec *pp;
+	int lfmt;
+
+	(void)priv;
+	if (av[2] == NULL || strcmp(av[2], "-l"))
+		lfmt = 0;
+	else
+		lfmt = 1;
+	for (i = 0; i < nparspec; i++) {
+		pp = parspec[i];
+		if (av[2] != NULL && !lfmt && strcmp(pp->name, av[2]))
+			continue;
+		VCLI_Out(cli, "%-*s ", margin, pp->name);
+		if (pp->func == NULL) {
+			VCLI_Out(cli, "Not implemented.\n");
+			if (av[2] != NULL && !lfmt)
+				return;
+			else
+				continue;
+		}
+		pp->func(cli, pp, NULL);
+		if (pp->units != NULL)
+			VCLI_Out(cli, " [%s]\n", pp->units);
+		else
+			VCLI_Out(cli, "\n");
+		if (av[2] != NULL) {
+			VCLI_Out(cli, "%-*s Default is %s\n",
+			    margin, "", pp->def);
+			mcf_wrap(cli, pp->descr);
+			if (pp->flags & DELAYED_EFFECT)
+				mcf_wrap(cli, DELAYED_EFFECT_TEXT);
+			if (pp->flags & EXPERIMENTAL)
+				mcf_wrap(cli, EXPERIMENTAL_TEXT);
+			if (pp->flags & MUST_RELOAD)
+				mcf_wrap(cli, MUST_RELOAD_TEXT);
+			if (pp->flags & MUST_RESTART)
+				mcf_wrap(cli, MUST_RESTART_TEXT);
+			if (pp->flags & WIZARD)
+				mcf_wrap(cli, WIZARD_TEXT);
+			if (!lfmt)
+				return;
+			else
+				VCLI_Out(cli, "\n");
+		}
+	}
+	if (av[2] != NULL && !lfmt) {
+		VCLI_SetResult(cli, CLIS_PARAM);
+		VCLI_Out(cli, "Unknown parameter \"%s\".", av[2]);
+	}
+}
+
+/*--------------------------------------------------------------------*/
+
+void
+MCF_ParamSync(void)
+{
+	if (params != &master)
+		*params = master;
+}
+
+/*--------------------------------------------------------------------*/
+
+void
+MCF_ParamSet(struct cli *cli, const char *param, const char *val)
+{
+	const struct parspec *pp;
+
+	pp = mcf_findpar(param);
+	if (pp != NULL) {
+		pp->func(cli, pp, val);
+		if (cli->result != CLIS_OK) {
+			VCLI_Out(cli, "(attempting to set param %s to %s)\n",
+			    pp->name, val);
+		} else if (child_pid >= 0 && pp->flags & MUST_RESTART) {
+			VCLI_Out(cli, "Change will take effect"
+			    " when child is restarted");
+		} else if (pp->flags & MUST_RELOAD) {
+			VCLI_Out(cli, "Change will take effect"
+			    " when VCL script is reloaded");
+		}
+		MCF_ParamSync();
+		return;
+	}
+	VCLI_SetResult(cli, CLIS_PARAM);
+	VCLI_Out(cli, "Unknown parameter \"%s\".", param);
+}
+
+
+/*--------------------------------------------------------------------*/
+
+void
+mcf_param_set(struct cli *cli, const char * const *av, void *priv)
+{
+
+	(void)priv;
+	MCF_ParamSet(cli, av[2], av[3]);
+}
+
+/*--------------------------------------------------------------------
+ * Add a group of parameters to the global set and sort by name.
+ */
+
+static int
+parspec_cmp(const void *a, const void *b)
+{
+	struct parspec * const * pa = a;
+	struct parspec * const * pb = b;
+	return (strcmp((*pa)->name, (*pb)->name));
+}
+
+static void
+MCF_AddParams(const struct parspec *ps)
+{
+	const struct parspec *pp;
+	int n;
+
+	n = 0;
+	for (pp = ps; pp->name != NULL; pp++) {
+		if (mcf_findpar(pp->name) != NULL)
+			fprintf(stderr, "Duplicate param: %s\n", pp->name);
+		if (strlen(pp->name) + 1 > margin)
+			margin = strlen(pp->name) + 1;
+		n++;
+	}
+	parspec = realloc(parspec, (1L + nparspec + n) * sizeof *parspec);
+	XXXAN(parspec);
+	for (pp = ps; pp->name != NULL; pp++)
+		parspec[nparspec++] = pp;
+	parspec[nparspec] = NULL;
+	qsort (parspec, nparspec, sizeof parspec[0], parspec_cmp);
+}
+
+/*--------------------------------------------------------------------
+ * Set defaults for all parameters
+ */
+
+static void
+MCF_SetDefaults(struct cli *cli)
+{
+	const struct parspec *pp;
+	int i;
+
+	for (i = 0; i < nparspec; i++) {
+		pp = parspec[i];
+		if (cli != NULL)
+			VCLI_Out(cli,
+			    "Set Default for %s = %s\n", pp->name, pp->def);
+		pp->func(cli, pp, pp->def);
+		if (cli != NULL && cli->result != CLIS_OK)
+			return;
+	}
+}
+
+/*--------------------------------------------------------------------*/
+
+void
+MCF_ParamInit(struct cli *cli)
+{
+
+	MCF_AddParams(input_parspec);
+	MCF_AddParams(WRK_parspec);
+
+	/* XXX: We do this twice, to get past any interdependencies */
+	MCF_SetDefaults(NULL);
+	MCF_SetDefaults(cli);
+
+	params = &master;
+}
+
+/*--------------------------------------------------------------------*/
+
+void
+MCF_DumpRst(void)
+{
+	const struct parspec *pp;
+	const char *p, *q;
+	int i;
+
+	for (i = 0; i < nparspec; i++) {
+		pp = parspec[i];
+		printf("%s\n", pp->name);
+		if (pp->units != NULL && *pp->units != '\0')
+			printf("\t- Units: %s\n", pp->units);
+		printf("\t- Default: %s\n",
+		    strcmp(pp->def,MAGIC_INIT_STRING) == 0 ? "magic" : pp->def);
+		/*
+		 * XXX: we should mark the params with one/two flags
+		 * XXX: that say if ->min/->max are valid, so we
+		 * XXX: can emit those also in help texts.
+		 */
+		if (pp->flags) {
+			printf("\t- Flags: ");
+			q = "";
+			if (pp->flags & DELAYED_EFFECT) {
+				printf("%sdelayed", q);
+				q = ", ";
+			}
+			if (pp->flags & MUST_RESTART) {
+				printf("%smust_restart", q);
+				q = ", ";
+			}
+			if (pp->flags & MUST_RELOAD) {
+				printf("%smust_reload", q);
+				q = ", ";
+			}
+			if (pp->flags & EXPERIMENTAL) {
+				printf("%sexperimental", q);
+				q = ", ";
+			}
+			printf("\n");
+		}
+		printf("\n\t");
+		for (p = pp->descr; *p; p++) {
+			if (*p == '\n' && p[1] =='\0')
+				break;
+			if (*p == '\n' && p[1] =='\n') {
+				printf("\n\n\t");
+				p++;
+			} else if (*p == '\n') {
+				printf("\n\t");
+			} else if (*p == ':' && p[1] == '\n') {
+				/*
+				 * Start of definition list,
+				 * use RSTs code mode for this
+				 */
+				printf("::\n");
+			} else {
+				printf("%c", *p);
+			}
+		}
+		printf("\n\n");
+	}
+	printf("\n");
+}
diff --git a/bin/varnishd/mgt/mgt_pool.c b/bin/varnishd/mgt/mgt_pool.c
new file mode 100644
index 0000000..978e60a
--- /dev/null
+++ b/bin/varnishd/mgt/mgt_pool.c
@@ -0,0 +1,238 @@
+/*-
+ * Copyright (c) 2006 Verdens Gang AS
+ * Copyright (c) 2006-2011 Varnish Software AS
+ * All rights reserved.
+ *
+ * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * We maintain a number of worker thread pools, to spread lock contention.
+ *
+ * Pools can be added on the fly, as a means to mitigate lock contention,
+ * but can only be removed again by a restart. (XXX: we could fix that)
+ *
+ * Two threads herd the pools, one eliminates idle threads and aggregates
+ * statistics for all the pools, the other thread creates new threads
+ * on demand, subject to various numerical constraints.
+ *
+ * The algorithm for when to create threads needs to be reactive enough
+ * to handle startup spikes, but sufficiently attenuated to not cause
+ * thread pileups.  This remains subject for improvement.
+ */
+
+#include "config.h"
+
+#include <limits.h>
+#include <stdio.h>
+#include <string.h>
+#include <unistd.h>
+
+#include "mgt/mgt.h"
+
+#include "heritage.h"
+#include "vparam.h"
+
+/*--------------------------------------------------------------------*/
+
+static void
+tweak_thread_pool_min(struct cli *cli, const struct parspec *par,
+    const char *arg)
+{
+
+	tweak_generic_uint(cli, &master.wthread_min, arg,
+	    (unsigned)par->min, master.wthread_max);
+}
+
+/*--------------------------------------------------------------------
+ * This is utterly ridiculous:  POSIX does not guarantee that the
+ * minimum thread stack size is a compile time constant.
+ * XXX: "32" is a magic marker for 32bit systems.
+ */
+
+static void
+tweak_stack_size(struct cli *cli, const struct parspec *par,
+    const char *arg)
+{
+	unsigned low, u;
+	char buf[12];
+
+	low = sysconf(_SC_THREAD_STACK_MIN);
+
+	if (arg != NULL && !strcmp(arg, "32bit")) {
+		u = 65536;
+		if (u < low)
+			u = low;
+		sprintf(buf, "%u", u);
+		arg = buf;
+	}
+
+	tweak_generic_uint(cli, &master.wthread_stacksize, arg,
+	    low, (uint)par->max);
+}
+
+/*--------------------------------------------------------------------*/
+
+static void
+tweak_thread_pool_max(struct cli *cli, const struct parspec *par,
+    const char *arg)
+{
+
+	(void)par;
+	tweak_generic_uint(cli, &master.wthread_max, arg,
+	    master.wthread_min, UINT_MAX);
+}
+
+/*--------------------------------------------------------------------*/
+
+const struct parspec WRK_parspec[] = {
+	{ "thread_pools", tweak_uint, &master.wthread_pools, 1, UINT_MAX,
+		"Number of worker thread pools.\n"
+		"\n"
+		"Increasing number of worker pools decreases lock "
+		"contention.\n"
+		"\n"
+		"Too many pools waste CPU and RAM resources, and more than "
+		"one pool for each CPU is probably detrimal to performance.\n"
+		"\n"
+		"Can be increased on the fly, but decreases require a "
+		"restart to take effect.",
+		EXPERIMENTAL | DELAYED_EFFECT,
+		"2", "pools" },
+	{ "thread_pool_max", tweak_thread_pool_max, NULL, 1, 0,
+		"The maximum number of worker threads in each pool.\n"
+		"\n"
+		"Do not set this higher than you have to, since excess "
+		"worker threads soak up RAM and CPU and generally just get "
+		"in the way of getting work done.\n",
+		EXPERIMENTAL | DELAYED_EFFECT,
+		"500", "threads" },
+	{ "thread_pool_min", tweak_thread_pool_min, NULL, 2, 0,
+		"The minimum number of worker threads in each pool.\n"
+		"\n"
+		"Increasing this may help ramp up faster from low load "
+		"situations where threads have expired.\n"
+		"\n"
+		"Minimum is 2 threads.",
+		EXPERIMENTAL | DELAYED_EFFECT,
+		"5", "threads" },
+	{ "thread_pool_timeout", tweak_timeout, &master.wthread_timeout, 1, 0,
+		"Thread idle threshold.\n"
+		"\n"
+		"Threads in excess of thread_pool_min, which have been idle "
+		"for at least this long are candidates for purging.\n"
+		"\n"
+		"Minimum is 1 second.",
+		EXPERIMENTAL | DELAYED_EFFECT,
+		"300", "seconds" },
+	{ "thread_pool_purge_delay",
+		tweak_timeout, &master.wthread_purge_delay, 100, 0,
+		"Wait this long between purging threads.\n"
+		"\n"
+		"This controls the decay of thread pools when idle(-ish).\n"
+		"\n"
+		"Minimum is 100 milliseconds.",
+		EXPERIMENTAL | DELAYED_EFFECT,
+		"1000", "milliseconds" },
+	{ "thread_pool_add_threshold",
+		tweak_uint, &master.wthread_add_threshold, 0, UINT_MAX,
+		"Overflow threshold for worker thread creation.\n"
+		"\n"
+		"Setting this too low, will result in excess worker threads, "
+		"which is generally a bad idea.\n"
+		"\n"
+		"Setting it too high results in insuffient worker threads.\n",
+		EXPERIMENTAL,
+		"2", "requests" },
+	{ "thread_pool_add_delay",
+		tweak_timeout, &master.wthread_add_delay, 0, UINT_MAX,
+		"Wait at least this long between creating threads.\n"
+		"\n"
+		"Setting this too long results in insuffient worker threads.\n"
+		"\n"
+		"Setting this too short increases the risk of worker "
+		"thread pile-up.\n",
+		0,
+		"2", "milliseconds" },
+	{ "thread_pool_fail_delay",
+		tweak_timeout, &master.wthread_fail_delay, 100, UINT_MAX,
+		"Wait at least this long after a failed thread creation "
+		"before trying to create another thread.\n"
+		"\n"
+		"Failure to create a worker thread is often a sign that "
+		" the end is near, because the process is running out of "
+		"RAM resources for thread stacks.\n"
+		"This delay tries to not rush it on needlessly.\n"
+		"\n"
+		"If thread creation failures are a problem, check that "
+		"thread_pool_max is not too high.\n"
+		"\n"
+		"It may also help to increase thread_pool_timeout and "
+		"thread_pool_min, to reduce the rate at which treads are "
+		"destroyed and later recreated.\n",
+		EXPERIMENTAL,
+		"200", "milliseconds" },
+	{ "thread_stats_rate",
+		tweak_uint, &master.wthread_stats_rate, 0, UINT_MAX,
+		"Worker threads accumulate statistics, and dump these into "
+		"the global stats counters if the lock is free when they "
+		"finish a request.\n"
+		"This parameters defines the maximum number of requests "
+		"a worker thread may handle, before it is forced to dump "
+		"its accumulated stats into the global counters.\n",
+		EXPERIMENTAL,
+		"10", "requests" },
+	{ "queue_max", tweak_uint, &master.queue_max, 0, UINT_MAX,
+		"Percentage permitted queue length.\n"
+		"\n"
+		"This sets the ratio of queued requests to worker threads, "
+		"above which sessions will be dropped instead of queued.\n",
+		EXPERIMENTAL,
+		"100", "%" },
+	{ "rush_exponent", tweak_uint, &master.rush_exponent, 2, UINT_MAX,
+		"How many parked request we start for each completed "
+		"request on the object.\n"
+		"NB: Even with the implict delay of delivery, "
+		"this parameter controls an exponential increase in "
+		"number of worker threads.",
+		EXPERIMENTAL,
+		"3", "requests per request" },
+	{ "thread_pool_stack",
+		tweak_stack_size, &master.wthread_stacksize, 0, UINT_MAX,
+		"Worker thread stack size.\n"
+		"On 32bit systems you may need to tweak this down to fit "
+		"many threads into the limited address space.\n",
+		EXPERIMENTAL,
+		"-1", "bytes" },
+	{ "thread_pool_workspace", tweak_uint, &master.wthread_workspace,
+		1024, UINT_MAX,
+		"Bytes of HTTP protocol workspace allocated for worker "
+		"threads. "
+		"This space must be big enough for the backend request "
+		"and responses, and response to the client plus any other "
+		"memory needs in the VCL code."
+		"Minimum is 1024 bytes.",
+		DELAYED_EFFECT,
+		"65536",
+		"bytes" },
+	{ NULL, NULL, NULL }
+};
diff --git a/bin/varnishd/mgt/mgt_sandbox.c b/bin/varnishd/mgt/mgt_sandbox.c
new file mode 100644
index 0000000..b01d243
--- /dev/null
+++ b/bin/varnishd/mgt/mgt_sandbox.c
@@ -0,0 +1,91 @@
+/*-
+ * Copyright (c) 2006-2011 Varnish Software AS
+ * All rights reserved.
+ *
+ * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * Sandboxing child processes
+ *
+ * The worker/manager process border is one of the major security barriers
+ * in Varnish, and therefore subject to whatever restrictions we have access
+ * to under the given operating system.
+ *
+ * Unfortunately there is no consensus on APIs for this purpose, so each
+ * operating system will require its own methods.
+ *
+ * This sourcefile tries to encapsulate the resulting mess on place.
+ *
+ * TODO:
+ *	Unix:	chroot
+ *	FreeBSD: jail
+ *	FreeBSD: capsicum
+ */
+
+#include "config.h"
+
+#ifdef __linux__
+#include <sys/prctl.h>
+#endif
+
+#include <stdio.h>
+#include <syslog.h>
+#include <unistd.h>
+
+#include "mgt/mgt.h"
+
+#include "heritage.h"
+
+/*--------------------------------------------------------------------*/
+
+/* Waive all privileges in the child, it does not need any */
+
+void
+mgt_sandbox(void)
+{
+#ifdef HAVE_SETPPRIV
+	mgt_sandbox_solaris_init();
+	mgt_sandbox_solaris_privsep();
+#else
+	if (geteuid() == 0) {
+		XXXAZ(setgid(params->gid));
+		XXXAZ(setuid(params->uid));
+	} else {
+		REPORT0(LOG_INFO, "Not running as root, no priv-sep");
+	}
+#endif
+
+	/* On Linux >= 2.4, you need to set the dumpable flag
+	   to get core dumps after you have done a setuid. */
+
+#ifdef __linux__
+	if (prctl(PR_SET_DUMPABLE, 1) != 0)
+		REPORT0(LOG_INFO,
+		    "Could not set dumpable bit.  Core dumps turned off\n");
+#endif
+
+#ifdef HAVE_SETPPRIV
+	mgt_sandbox_solaris_fini();
+#endif
+
+}
diff --git a/bin/varnishd/mgt/mgt_sandbox_solaris.c b/bin/varnishd/mgt/mgt_sandbox_solaris.c
new file mode 100644
index 0000000..715408e
--- /dev/null
+++ b/bin/varnishd/mgt/mgt_sandbox_solaris.c
@@ -0,0 +1,233 @@
+/*-
+ * Copyright (c) 2006-2011 Varnish Software AS
+ * All rights reserved.
+ *
+ * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
+ *	   Nils Goroll <nils.goroll at uplex.de>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * Sandboxing child processes on Solaris
+ *
+ */
+
+#include "config.h"
+
+#ifdef HAVE_SETPPRIV
+
+#ifdef HAVE_PRIV_H
+#include <priv.h>
+#endif
+#include <stdio.h>
+#include <string.h>
+#include <syslog.h>
+#include <unistd.h>
+
+#include "mgt/mgt.h"
+
+#include "heritage.h"
+
+/*--------------------------------------------------------------------
+ * SOLARIS PRIVILEGES: Note on use of symbolic PRIV_* constants
+ * 
+ * For privileges which existed in Solaris 10 FCS, we may use the constants from
+ * sys/priv_names.h
+ *
+ * For privileges which have been added later, we need to use strings in order
+ * not to break builds of varnish on these platforms. To remain binary
+ * compatible, we need to silently ignore errors from priv_addset when using
+ * these strings.
+ *
+ * For optimal build and binary forward comatibility, we could use subtractive
+ * set specs like
+ *
+ *       basic,!file_link_any,!proc_exec,!proc_fork,!proc_info,!proc_session
+ *
+ * but I (Nils) have a preference for making an informed decision about which
+ * privileges the varnish child should have and which it shouldn't.
+ *
+ * Newly introduced privileges should be annotated with their PSARC / commit ID
+ * (as long as Oracle reveils these :/ )
+ *
+ * SOLARIS PRIVILEGES: Note on accidentally setting the SNOCD flag
+ *
+ * When setting privileges, we need to take care not to accidentally set the
+ * SNOCD flag which will disable core dumps unnecessarily. (see
+ * https://www.varnish-cache.org/trac/ticket/671 )
+ *
+ * When changing the logic herein, always check with mdb -k. Replace _PID_ with
+ * the pid of your varnish child, the result should be 0, otherwise a regression
+ * has been introduced.
+ *
+ * > 0t_PID_::pid2proc | ::print proc_t p_flag | >a
+ * > (<a & 0x10000000)=X
+ *                 0
+ *
+ * (a value of 0x10000000 indicates that SNOCD is set)
+ *
+ * NOTE that on Solaris changing the uid will _always_ set SNOCD, so make sure
+ * you run this test with appropriate privileges, but without proc_setid, so
+ * varnish won't setuid(), e.g.
+ *
+ * pfexec ppriv -e -s A=basic,net_privaddr,sys_resource varnish ...
+ *
+ * SOLARIS COREDUMPS with setuid(): See coreadm(1M) - global-setid / proc-setid
+ *
+ */
+
+/* effective during runtime of the child */
+static inline void
+mgt_sandbox_solaris_add_effective(priv_set_t *pset)
+{
+	/* PSARC/2009/685 - 8eca52188202 - onnv_132 */
+	priv_addset(pset, "net_access");
+
+	/* PSARC/2009/378 - 63678502e95e - onnv_140 */
+	priv_addset(pset, "file_read");
+	priv_addset(pset, "file_write");
+}
+
+/* permitted during runtime of the child - for privilege bracketing */
+static inline void
+mgt_sandbox_solaris_add_permitted(priv_set_t *pset)
+{
+	/* for raising limits in cache_waiter_ports.c */
+	priv_addset(pset, PRIV_SYS_RESOURCE);
+}
+
+/* effective during mgt_sandbox */
+static inline void
+mgt_sandbox_solaris_add_initial(priv_set_t *pset)
+{
+	/* for setgid/setuid */
+	priv_addset(pset, PRIV_PROC_SETID);
+}
+
+/*
+ * if we are not yet privilege-aware already (ie we have been started
+ * not-privilege aware wird euid 0), we need to grab any additional privileges
+ * needed during mgt_standbox, until we reduce to least privileges in
+ * mgt_sandbox_waive, otherwise we would loose them with setuid()
+ */
+
+void
+mgt_sandbox_solaris_init(void)
+{
+	priv_set_t *priv_all;
+
+	if (! (priv_all = priv_allocset())) {
+		REPORT(LOG_ERR,
+		    "Child start warning: mgt_sandbox_init - priv_allocset failed: errno=%d (%s)",
+		    errno, strerror(errno));
+		return;
+	}
+	
+	priv_emptyset(priv_all);
+
+	mgt_sandbox_solaris_add_effective(priv_all);
+	mgt_sandbox_solaris_add_permitted(priv_all);
+	mgt_sandbox_solaris_add_initial(priv_all);
+
+	setppriv(PRIV_ON, PRIV_PERMITTED, priv_all);
+	setppriv(PRIV_ON, PRIV_EFFECTIVE, priv_all);
+	setppriv(PRIV_ON, PRIV_INHERITABLE, priv_all);
+
+	priv_freeset(priv_all);
+}
+
+void
+mgt_sandbox_solaris_privsep(void)
+{
+	if (priv_ineffect(PRIV_PROC_SETID)) {
+                if (getgid() != params->gid)
+                        XXXAZ(setgid(params->gid));
+                if (getuid() != params->uid)
+                        XXXAZ(setuid(params->uid));
+        } else {
+                REPORT(LOG_INFO, "Privilege %s missing, will not change uid/gid",
+		    PRIV_PROC_SETID);
+        }
+}
+
+/* 
+ * Waive most privileges in the child
+ *
+ * as of onnv_151a, we should end up with:
+ *
+ * > ppriv -v #pid of varnish child
+ * PID:  .../varnishd ...
+ * flags = PRIV_AWARE
+ *      E: file_read,file_write,net_access
+ *      I: none
+ *      P: file_read,file_write,net_access,sys_resource
+ *      L: file_read,file_write,net_access,sys_resource
+ *
+ * We should keep sys_resource in P in order to adjust our limits if we need to
+ */
+
+void
+mgt_sandbox_solaris_fini(void)
+{
+	priv_set_t *effective, *inheritable, *permitted;
+
+	if (!(effective = priv_allocset()) ||
+	    !(inheritable = priv_allocset()) ||
+	    !(permitted = priv_allocset())) {
+		REPORT(LOG_ERR,
+		    "Child start warning: mgt_sandbox_waive - priv_allocset failed: errno=%d (%s)",
+		    errno, strerror(errno));
+		return;
+	}
+
+	priv_emptyset(inheritable);
+
+	priv_emptyset(effective);
+	mgt_sandbox_solaris_add_effective(effective);
+
+	priv_copyset(effective, permitted);
+	mgt_sandbox_solaris_add_permitted(permitted);
+
+	/* 
+	 * invert the sets and clear privileges such that setppriv will always
+	 * succeed
+	 */
+	priv_inverse(inheritable);
+	priv_inverse(effective);
+	priv_inverse(permitted);
+
+#define SETPPRIV(which, set)						\
+	if (setppriv(PRIV_OFF, which, set))				\
+		REPORT(LOG_ERR,						\
+		    "Child start warning: Waiving privileges failed on %s: errno=%d (%s)", \
+		    #which, errno, strerror(errno));
+
+	SETPPRIV(PRIV_INHERITABLE, inheritable);
+	SETPPRIV(PRIV_EFFECTIVE, effective);
+	SETPPRIV(PRIV_PERMITTED, permitted);
+	SETPPRIV(PRIV_LIMIT, permitted);
+#undef SETPPRIV
+
+	priv_freeset(inheritable);
+	priv_freeset(effective);
+}
+
+#endif /* HAVE_SETPPRIV */
diff --git a/bin/varnishd/mgt/mgt_shmem.c b/bin/varnishd/mgt/mgt_shmem.c
new file mode 100644
index 0000000..630865d
--- /dev/null
+++ b/bin/varnishd/mgt/mgt_shmem.c
@@ -0,0 +1,351 @@
+/*-
+ * Copyright (c) 2006 Verdens Gang AS
+ * Copyright (c) 2006-2011 Varnish Software AS
+ * All rights reserved.
+ *
+ * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ *
+ * TODO:
+ *
+ * There is a risk that the child process might corrupt the VSM segment
+ * and we should capture that event and recover gracefully.
+ *
+ * A possible state diagram could be:
+ *
+ *	[manager start]
+ *		|
+ *		v
+ *      Open old VSM,
+ *	check pid	--------> exit/fail (-n message)
+ *		|
+ *		+<----------------------+
+ *		|			^
+ *		v			|
+ *	Create new VSM			|
+ *		|			|
+ *		v			|
+ *	Init header			|
+ *	Alloc VSL			|
+ *	Alloc VSC:Main			|
+ *	Alloc Args etc.			|
+ *		|			|
+ *		+<--------------+	|
+ *		|		^	|
+ *		v		|	|
+ *	start worker		|	|
+ *		|		|	|
+ *		|		|	+<---- worker crash
+ *		v		|	^
+ *	Reset VSL ptr.		|	|
+ *	Reset VSC counters	|	|
+ *		|		|	|
+ *		+<------+	|	|
+ *		|	^	|	|
+ *		v	|	|	|
+ *	alloc dynamics	|	|	|
+ *	free dynamics	|	|	|
+ *		|	|	|	|
+ *		v	|	|	|
+ *		+------>+	|	|
+ *		|		|	|
+ *		v		|	|
+ *	stop worker		|	|
+ *		|		|	|
+ *		v		|	|
+ *	Check consist---------- | ----->+
+ *		|		|
+ *		v		|
+ *	Free dynamics		|
+ *		|		|
+ *		v		|
+ *		+-------------->+
+ *
+ */
+
+#include "config.h"
+
+#include <sys/mman.h>
+#include <sys/stat.h>
+
+#include <fcntl.h>
+#include <signal.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+
+#include "mgt/mgt.h"
+
+#include "flopen.h"
+#include "heritage.h"
+#include "vapi/vsc_int.h"
+#include "vapi/vsl_int.h"
+#include "vapi/vsm_int.h"
+#include "vav.h"
+#include "vmb.h"
+#include "vnum.h"
+
+#ifndef MAP_HASSEMAPHORE
+#define MAP_HASSEMAPHORE 0 /* XXX Linux */
+#endif
+
+#ifndef MAP_NOSYNC
+#define MAP_NOSYNC 0 /* XXX Linux */
+#endif
+
+struct VSC_C_main	*VSC_C_main;
+
+static int vsl_fd = -1;
+
+/*--------------------------------------------------------------------
+ * Check that we are not started with the same -n argument as an already
+ * running varnishd
+ */
+
+static void
+vsl_n_check(int fd)
+{
+	struct VSM_head slh;
+	int i;
+	struct stat st;
+	pid_t pid;
+
+	AZ(fstat(fd, &st));
+	if (!S_ISREG(st.st_mode))
+		ARGV_ERR("\tshmlog: Not a file\n");
+
+	/* Test if the SHMFILE is locked by other Varnish */
+	if (fltest(fd, &pid) > 0) {
+		fprintf(stderr,
+			"SHMFILE locked by running varnishd master (pid=%jd)\n",
+			(intmax_t)pid);
+		fprintf(stderr,
+			"(Use unique -n arguments if you want multiple "
+			"instances)\n");
+		exit(2);
+	}
+
+	/* Read owning pid from SHMFILE */
+	memset(&slh, 0, sizeof slh);	/* XXX: for flexelint */
+	i = read(fd, &slh, sizeof slh);
+	if (i != sizeof slh)
+		return;
+	if (slh.magic != VSM_HEAD_MAGIC)
+		return;
+	if (slh.hdrsize != sizeof slh)
+		return;
+	if (slh.master_pid != 0 && !kill(slh.master_pid, 0)) {
+		fprintf(stderr,
+			"WARNING: Taking over SHMFILE marked as owned by "
+			"running process (pid=%jd)\n",
+			(intmax_t)slh.master_pid);
+	}
+}
+
+/*--------------------------------------------------------------------
+ * Build a new shmlog file
+ */
+
+static void
+vsl_buildnew(const char *fn, unsigned size, int fill)
+{
+	struct VSM_head slh;
+	int i;
+	unsigned u;
+	char buf[64*1024];
+	int flags;
+
+	(void)unlink(fn);
+	vsl_fd = flopen(fn, O_RDWR | O_CREAT | O_EXCL | O_NONBLOCK, 0644);
+	if (vsl_fd < 0) {
+		fprintf(stderr, "Could not create %s: %s\n",
+		    fn, strerror(errno));
+		exit (1);
+	}
+	flags = fcntl(vsl_fd, F_GETFL);
+	assert(flags != -1);
+	flags &= ~O_NONBLOCK;
+	AZ(fcntl(vsl_fd, F_SETFL, flags));
+
+	memset(&slh, 0, sizeof slh);
+	slh.magic = VSM_HEAD_MAGIC;
+	slh.hdrsize = sizeof slh;
+	slh.shm_size = size;
+	i = write(vsl_fd, &slh, sizeof slh);
+	xxxassert(i == sizeof slh);
+
+	if (fill) {
+		memset(buf, 0, sizeof buf);
+		for (u = sizeof slh; u < size; ) {
+			i = write(vsl_fd, buf, sizeof buf);
+			if (i <= 0) {
+				fprintf(stderr, "Write error %s: %s\n",
+				    fn, strerror(errno));
+				exit (1);
+			}
+			u += i;
+		}
+	}
+
+	AZ(ftruncate(vsl_fd, (off_t)size));
+}
+
+/*--------------------------------------------------------------------
+ * Exit handler that clears the owning pid from the SHMLOG
+ */
+
+static
+void
+mgt_shm_atexit(void)
+{
+	if (getpid() == VSM_head->master_pid)
+		VSM_head->master_pid = 0;
+}
+
+void
+mgt_SHM_Init(const char *l_arg)
+{
+	int i, fill;
+	struct params *pp;
+	const char *q;
+	uintmax_t size, s1, s2, ps;
+	char **av, **ap;
+	uint32_t *vsl_log_start;
+
+	if (l_arg == NULL)
+		l_arg = "";
+
+	av = VAV_Parse(l_arg, NULL, ARGV_COMMA);
+	AN(av);
+	if (av[0] != NULL)
+		ARGV_ERR("\t-l ...: %s", av[0]);
+
+	ap = av + 1;
+
+	/* Size of SHMLOG */
+	if (*ap != NULL && **ap != '\0') {
+		q = VNUM_2bytes(*ap, &s1, 0);
+		if (q != NULL)
+			ARGV_ERR("\t-l[1] ...:  %s\n", q);
+	} else {
+		s1 = 80 * 1024 * 1024;
+	}
+	if (*ap != NULL)
+		ap++;
+
+	/* Size of space for other stuff */
+	if (*ap != NULL && **ap != '\0') {
+		q = VNUM_2bytes(*ap, &s2, 0);
+		if (q != NULL)
+			ARGV_ERR("\t-l[2] ...:  %s\n", q);
+	} else {
+		s2 = 1024 * 1024;
+	}
+	if (*ap != NULL)
+		ap++;
+
+	/* Fill or not ? */
+	if (*ap != NULL) {
+		if (**ap == '\0')
+			fill = 1;
+		else if (!strcmp(*ap, "-"))
+			fill = 0;
+		else if (!strcmp(*ap, "+"))
+			fill = 1;
+		else
+			ARGV_ERR("\t-l[3] ...:  Must be \"-\" or \"+\"\n");
+		ap++;
+	} else {
+		fill = 1;
+	}
+
+	if (*ap != NULL)
+		ARGV_ERR("\t-l ...:  Too many sub-args\n");
+
+	VAV_Free(av);
+
+	size = s1 + s2;
+	ps = getpagesize();
+	size += ps - 1;
+	size &= ~(ps - 1);
+
+	i = open(VSM_FILENAME, O_RDWR, 0644);
+	if (i >= 0) {
+		vsl_n_check(i);
+		(void)close(i);
+	}
+	vsl_buildnew(VSM_FILENAME, size, fill);
+
+	VSM_head = (void *)mmap(NULL, size,
+	    PROT_READ|PROT_WRITE,
+	    MAP_HASSEMAPHORE | MAP_NOSYNC | MAP_SHARED,
+	    vsl_fd, 0);
+	VSM_head->master_pid = getpid();
+	AZ(atexit(mgt_shm_atexit));
+	xxxassert(VSM_head != MAP_FAILED);
+	(void)mlock((void*)VSM_head, size);
+
+	memset(&VSM_head->head, 0, sizeof VSM_head->head);
+	VSM_head->head.magic = VSM_CHUNK_MAGIC;
+	VSM_head->head.len =
+	    (uint8_t*)(VSM_head) + size - (uint8_t*)&VSM_head->head;
+	bprintf(VSM_head->head.class, "%s", VSM_CLASS_FREE);
+	VWMB();
+
+	vsm_end = (void*)((uint8_t*)VSM_head + size);
+
+	VSC_C_main = VSM_Alloc(sizeof *VSC_C_main,
+	    VSC_CLASS, VSC_TYPE_MAIN, "");
+	AN(VSC_C_main);
+
+	pp = VSM_Alloc(sizeof *pp, VSM_CLASS_PARAM, "", "");
+	AN(pp);
+	*pp = *params;
+	params = pp;
+
+	vsl_log_start = VSM_Alloc(s1, VSL_CLASS, "", "");
+	AN(vsl_log_start);
+	vsl_log_start[1] = VSL_ENDMARKER;
+	VWMB();
+
+	do
+		*vsl_log_start = random() & 0xffff;
+	while (*vsl_log_start == 0);
+
+	VWMB();
+
+	do
+		VSM_head->alloc_seq = random();
+	while (VSM_head->alloc_seq == 0);
+
+}
+
+void
+mgt_SHM_Pid(void)
+{
+
+	VSM_head->master_pid = getpid();
+}
diff --git a/bin/varnishd/mgt/mgt_vcc.c b/bin/varnishd/mgt/mgt_vcc.c
new file mode 100644
index 0000000..ec117db
--- /dev/null
+++ b/bin/varnishd/mgt/mgt_vcc.c
@@ -0,0 +1,676 @@
+/*-
+ * Copyright (c) 2006 Verdens Gang AS
+ * Copyright (c) 2006-2011 Varnish Software AS
+ * All rights reserved.
+ *
+ * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * VCL compiler stuff
+ */
+
+#include "config.h"
+
+#include <dlfcn.h>
+#include <fcntl.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+
+#include "mgt/mgt.h"
+
+#include "libvcl.h"
+#include "vcl.h"
+#include "vcli.h"
+#include "vcli_priv.h"
+#include "vfil.h"
+#include "vsub.h"
+
+#include "mgt_cli.h"
+
+struct vclprog {
+	VTAILQ_ENTRY(vclprog)	list;
+	char			*name;
+	char			*fname;
+	int			active;
+};
+
+static VTAILQ_HEAD(, vclprog) vclhead = VTAILQ_HEAD_INITIALIZER(vclhead);
+
+char *mgt_cc_cmd;
+const char *mgt_vcl_dir;
+const char *mgt_vmod_dir;
+unsigned mgt_vcc_err_unref;
+
+static struct vcc *vcc;
+
+/*--------------------------------------------------------------------*/
+
+static const char * const default_vcl =
+#include "default_vcl.h"
+    ""	;
+
+/*--------------------------------------------------------------------
+ * Prepare the compiler command line
+ */
+static struct vsb *
+mgt_make_cc_cmd(const char *sf, const char *of)
+{
+	struct vsb *sb;
+	int pct;
+	char *p;
+
+	sb = VSB_new_auto();
+	XXXAN(sb);
+	for (p = mgt_cc_cmd, pct = 0; *p; ++p) {
+		if (pct) {
+			switch (*p) {
+			case 's':
+				VSB_cat(sb, sf);
+				break;
+			case 'o':
+				VSB_cat(sb, of);
+				break;
+			case '%':
+				VSB_putc(sb, '%');
+				break;
+			default:
+				VSB_putc(sb, '%');
+				VSB_putc(sb, *p);
+				break;
+			}
+			pct = 0;
+		} else if (*p == '%') {
+			pct = 1;
+		} else {
+			VSB_putc(sb, *p);
+		}
+	}
+	if (pct)
+		VSB_putc(sb, '%');
+	AZ(VSB_finish(sb));
+	return (sb);
+}
+
+/*--------------------------------------------------------------------
+ * Invoke system VCC compiler in a sub-process
+ */
+
+struct vcc_priv {
+	unsigned	magic;
+#define VCC_PRIV_MAGIC	0x70080cb8
+	char		*sf;
+	const char	*vcl;
+};
+
+static void
+run_vcc(void *priv)
+{
+	char *csrc;
+	struct vsb *sb;
+	struct vcc_priv *vp;
+	int fd, i, l;
+
+	CAST_OBJ_NOTNULL(vp, priv, VCC_PRIV_MAGIC);
+	sb = VSB_new_auto();
+	XXXAN(sb);
+	VCC_VCL_dir(vcc, mgt_vcl_dir);
+	VCC_VMOD_dir(vcc, mgt_vmod_dir);
+	VCC_Err_Unref(vcc, mgt_vcc_err_unref);
+	csrc = VCC_Compile(vcc, sb, vp->vcl);
+	AZ(VSB_finish(sb));
+	if (VSB_len(sb))
+		printf("%s", VSB_data(sb));
+	VSB_delete(sb);
+	if (csrc == NULL)
+		exit (1);
+
+	fd = open(vp->sf, O_WRONLY);
+	if (fd < 0) {
+		fprintf(stderr, "Cannot open %s", vp->sf);
+		exit (1);
+	}
+	l = strlen(csrc);
+	i = write(fd, csrc, l);
+	if (i != l) {
+		fprintf(stderr, "Cannot write %s", vp->sf);
+		exit (1);
+	}
+	AZ(close(fd));
+	free(csrc);
+	exit (0);
+}
+
+/*--------------------------------------------------------------------
+ * Invoke system C compiler in a sub-process
+ */
+
+static void
+run_cc(void *priv)
+{
+	(void)execl("/bin/sh", "/bin/sh", "-c", priv, NULL);
+}
+
+/*--------------------------------------------------------------------
+ * Attempt to open compiled VCL in a sub-process
+ */
+
+static void __match_proto__(sub_func_f)
+run_dlopen(void *priv)
+{
+	const char *of;
+	void *dlh;
+	struct VCL_conf const *cnf;
+
+	of = priv;
+
+	/* Try to load the object into the management process */
+	if ((dlh = dlopen(of, RTLD_NOW | RTLD_LOCAL)) == NULL) {
+		fprintf(stderr,
+		    "Compiled VCL program failed to load:\n  %s\n",
+		    dlerror());
+		exit(1);
+	}
+
+	cnf = dlsym(dlh, "VCL_conf");
+	if (cnf == NULL) {
+		fprintf(stderr, "Compiled VCL program, metadata not found\n");
+		exit(1);
+	}
+
+	if (cnf->magic != VCL_CONF_MAGIC) {
+		fprintf(stderr, "Compiled VCL program, mangled metadata\n");
+		exit(1);
+	}
+
+	if (dlclose(dlh)) {
+		fprintf(stderr,
+		    "Compiled VCL program failed to unload:\n  %s\n",
+		    dlerror());
+		exit(1);
+	}
+	exit(0);
+}
+
+/*--------------------------------------------------------------------
+ * Compile a VCL program, return shared object, errors in sb.
+ */
+
+static char *
+mgt_run_cc(const char *vcl, struct vsb *sb, int C_flag)
+{
+	char *csrc;
+	struct vsb *cmdsb;
+	char sf[] = "./vcl.########.c";
+	char of[sizeof sf + 1];
+	char *retval;
+	int sfd, i;
+	struct vcc_priv vp;
+
+	/* Create temporary C source file */
+	sfd = VFIL_tmpfile(sf);
+	if (sfd < 0) {
+		VSB_printf(sb, "Failed to create %s: %s", sf, strerror(errno));
+		return (NULL);
+	}
+	AZ(close(sfd));
+
+	/* Run the VCC compiler in a sub-process */
+	memset(&vp, 0, sizeof vp);
+	vp.magic = VCC_PRIV_MAGIC;
+	vp.sf = sf;
+	vp.vcl = vcl;
+	if (VSUB_run(sb, run_vcc, &vp, "VCC-compiler", -1)) {
+		(void)unlink(sf);
+		return (NULL);
+	}
+
+	if (C_flag) {
+		csrc = VFIL_readfile(NULL, sf, NULL);
+		XXXAN(csrc);
+		(void)fputs(csrc, stdout);
+		free(csrc);
+	}
+
+	/* Name the output shared library by "s/[.]c$/[.]so/" */
+	memcpy(of, sf, sizeof sf);
+	assert(sf[sizeof sf - 2] == 'c');
+	of[sizeof sf - 2] = 's';
+	of[sizeof sf - 1] = 'o';
+	of[sizeof sf] = '\0';
+
+	/* Build the C-compiler command line */
+	cmdsb = mgt_make_cc_cmd(sf, of);
+
+	/* Run the C-compiler in a sub-shell */
+	i = VSUB_run(sb, run_cc, VSB_data(cmdsb), "C-compiler", 10);
+
+	(void)unlink(sf);
+	VSB_delete(cmdsb);
+
+	if (!i)
+		i = VSUB_run(sb, run_dlopen, of, "dlopen", 10);
+
+	if (i) {
+		(void)unlink(of);
+		return (NULL);
+	}
+
+	retval = strdup(of);
+	XXXAN(retval);
+	return (retval);
+}
+
+/*--------------------------------------------------------------------*/
+
+static char *
+mgt_VccCompile(struct vsb **sb, const char *b, int C_flag)
+{
+	char *vf;
+
+	*sb = VSB_new_auto();
+	XXXAN(*sb);
+	vf = mgt_run_cc(b, *sb, C_flag);
+	AZ(VSB_finish(*sb));
+	return (vf);
+}
+
+/*--------------------------------------------------------------------*/
+
+static struct vclprog *
+mgt_vcc_add(const char *name, char *file)
+{
+	struct vclprog *vp;
+
+	vp = calloc(sizeof *vp, 1);
+	XXXAN(vp);
+	vp->name = strdup(name);
+	XXXAN(vp->name);
+	vp->fname = file;
+	VTAILQ_INSERT_TAIL(&vclhead, vp, list);
+	return (vp);
+}
+
+static void
+mgt_vcc_del(struct vclprog *vp)
+{
+	VTAILQ_REMOVE(&vclhead, vp, list);
+	printf("unlink %s\n", vp->fname);
+	XXXAZ(unlink(vp->fname));
+	free(vp->fname);
+	free(vp->name);
+	free(vp);
+}
+
+static struct vclprog *
+mgt_vcc_byname(const char *name)
+{
+	struct vclprog *vp;
+
+	VTAILQ_FOREACH(vp, &vclhead, list)
+		if (!strcmp(name, vp->name))
+			return (vp);
+	return (NULL);
+}
+
+
+static int
+mgt_vcc_delbyname(const char *name)
+{
+	struct vclprog *vp;
+
+	vp = mgt_vcc_byname(name);
+	if (vp != NULL) {
+		mgt_vcc_del(vp);
+		return (0);
+	}
+	return (1);
+}
+
+/*--------------------------------------------------------------------*/
+
+int
+mgt_vcc_default(const char *b_arg, const char *f_arg, char *vcl, int C_flag)
+{
+	char *vf;
+	struct vsb *sb;
+	struct vclprog *vp;
+	char buf[BUFSIZ];
+
+	/* XXX: annotate vcl with -b/-f arg so people know where it came from */
+	(void)f_arg;
+
+	if (b_arg != NULL) {
+		AZ(vcl);
+		/*
+		 * XXX: should do a "HEAD /" on the -b argument to see that
+		 * XXX: it even works.  On the other hand, we should do that
+		 * XXX: for all backends in the cache process whenever we
+		 * XXX: change config, but for a complex VCL, it might not be
+		 * XXX: a bug for a backend to not reply at that time, so then
+		 * XXX: again: we should check it here in the "trivial" case.
+		 */
+		bprintf(buf,
+		    "backend default {\n"
+		    "    .host = \"%s\";\n"
+		    "}\n", b_arg);
+		vcl = strdup(buf);
+		AN(vcl);
+	}
+	strcpy(buf, "boot");
+
+	vf = mgt_VccCompile(&sb, vcl, C_flag);
+	free(vcl);
+	if (VSB_len(sb) > 0)
+		fprintf(stderr, "%s", VSB_data(sb));
+	VSB_delete(sb);
+	if (C_flag) {
+		if (vf != NULL)
+			AZ(unlink(vf));
+		return (0);
+	}
+	if (vf == NULL) {
+		fprintf(stderr, "\nVCL compilation failed\n");
+		return (1);
+	}
+	vp = mgt_vcc_add(buf, vf);
+	vp->active = 1;
+	return (0);
+}
+
+/*--------------------------------------------------------------------*/
+
+int
+mgt_has_vcl()
+{
+
+	return (!VTAILQ_EMPTY(&vclhead));
+}
+
+/*--------------------------------------------------------------------*/
+
+int
+mgt_push_vcls_and_start(unsigned *status, char **p)
+{
+	struct vclprog *vp;
+
+	VTAILQ_FOREACH(vp, &vclhead, list) {
+		if (mgt_cli_askchild(status, p,
+		    "vcl.load \"%s\" %s\n", vp->name, vp->fname))
+			return (1);
+		free(*p);
+		if (!vp->active)
+			continue;
+		if (mgt_cli_askchild(status, p,
+		    "vcl.use \"%s\"\n", vp->name))
+			return (1);
+		free(*p);
+	}
+	if (mgt_cli_askchild(status, p, "start\n"))
+		return (1);
+	free(*p);
+	*p = NULL;
+	return (0);
+}
+
+/*--------------------------------------------------------------------*/
+
+static
+void
+mgt_vcc_atexit(void)
+{
+	struct vclprog *vp;
+
+	if (getpid() != mgt_pid)
+		return;
+	while (1) {
+		vp = VTAILQ_FIRST(&vclhead);
+		if (vp == NULL)
+			break;
+		(void)unlink(vp->fname);
+		VTAILQ_REMOVE(&vclhead, vp, list);
+	}
+}
+
+void
+mgt_vcc_init(void)
+{
+
+	vcc = VCC_New();
+	AN(vcc);
+	VCC_Default_VCL(vcc, default_vcl);
+	AZ(atexit(mgt_vcc_atexit));
+}
+
+/*--------------------------------------------------------------------*/
+
+void
+mcf_config_inline(struct cli *cli, const char * const *av, void *priv)
+{
+	char *vf, *p = NULL;
+	struct vsb *sb;
+	unsigned status;
+	struct vclprog *vp;
+
+	(void)priv;
+
+	vp = mgt_vcc_byname(av[2]);
+	if (vp != NULL) {
+		VCLI_Out(cli, "Already a VCL program named %s", av[2]);
+		VCLI_SetResult(cli, CLIS_PARAM);
+		return;
+	}
+
+	vf = mgt_VccCompile(&sb, av[3], 0);
+	if (VSB_len(sb) > 0)
+		VCLI_Out(cli, "%s\n", VSB_data(sb));
+	VSB_delete(sb);
+	if (vf == NULL) {
+		VCLI_Out(cli, "VCL compilation failed");
+		VCLI_SetResult(cli, CLIS_PARAM);
+		return;
+	}
+	VCLI_Out(cli, "VCL compiled.");
+	if (child_pid >= 0 &&
+	    mgt_cli_askchild(&status, &p, "vcl.load %s %s\n", av[2], vf)) {
+		VCLI_SetResult(cli, status);
+		VCLI_Out(cli, "%s", p);
+	} else {
+		(void)mgt_vcc_add(av[2], vf);
+	}
+	free(p);
+}
+
+void
+mcf_config_load(struct cli *cli, const char * const *av, void *priv)
+{
+	char *vf, *vcl;
+	struct vsb *sb;
+	unsigned status;
+	char *p = NULL;
+	struct vclprog *vp;
+
+	(void)priv;
+	vp = mgt_vcc_byname(av[2]);
+	if (vp != NULL) {
+		VCLI_Out(cli, "Already a VCL program named %s", av[2]);
+		VCLI_SetResult(cli, CLIS_PARAM);
+		return;
+	}
+
+	vcl = VFIL_readfile(mgt_vcl_dir, av[3], NULL);
+	if (vcl == NULL) {
+		VCLI_Out(cli, "Cannot open '%s'", av[3]);
+		VCLI_SetResult(cli, CLIS_PARAM);
+		return;
+	}
+
+	vf = mgt_VccCompile(&sb, vcl, 0);
+	free(vcl);
+
+	if (VSB_len(sb) > 0)
+		VCLI_Out(cli, "%s", VSB_data(sb));
+	VSB_delete(sb);
+	if (vf == NULL) {
+		VCLI_Out(cli, "VCL compilation failed");
+		VCLI_SetResult(cli, CLIS_PARAM);
+		return;
+	}
+	VCLI_Out(cli, "VCL compiled.");
+	if (child_pid >= 0 &&
+	    mgt_cli_askchild(&status, &p, "vcl.load %s %s\n", av[2], vf)) {
+		VCLI_SetResult(cli, status);
+		VCLI_Out(cli, "%s", p);
+	} else {
+		(void)mgt_vcc_add(av[2], vf);
+	}
+	free(p);
+}
+
+static struct vclprog *
+mcf_find_vcl(struct cli *cli, const char *name)
+{
+	struct vclprog *vp;
+
+	vp = mgt_vcc_byname(name);
+	if (vp != NULL)
+		return (vp);
+	VCLI_SetResult(cli, CLIS_PARAM);
+	VCLI_Out(cli, "No configuration named %s known.", name);
+	return (NULL);
+}
+
+void
+mcf_config_use(struct cli *cli, const char * const *av, void *priv)
+{
+	unsigned status;
+	char *p = NULL;
+	struct vclprog *vp;
+
+	(void)priv;
+	vp = mcf_find_vcl(cli, av[2]);
+	if (vp == NULL)
+		return;
+	if (vp->active != 0)
+		return;
+	if (child_pid >= 0 &&
+	    mgt_cli_askchild(&status, &p, "vcl.use %s\n", av[2])) {
+		VCLI_SetResult(cli, status);
+		VCLI_Out(cli, "%s", p);
+	} else {
+		vp->active = 2;
+		VTAILQ_FOREACH(vp, &vclhead, list) {
+			if (vp->active == 1)
+				vp->active = 0;
+			else if (vp->active == 2)
+				vp->active = 1;
+		}
+	}
+	free(p);
+}
+
+void
+mcf_config_discard(struct cli *cli, const char * const *av, void *priv)
+{
+	unsigned status;
+	char *p = NULL;
+	struct vclprog *vp;
+
+	(void)priv;
+	vp = mcf_find_vcl(cli, av[2]);
+	if (vp != NULL && vp->active) {
+		VCLI_SetResult(cli, CLIS_PARAM);
+		VCLI_Out(cli, "Cannot discard active VCL program\n");
+	} else if (vp != NULL) {
+		if (child_pid >= 0 &&
+		    mgt_cli_askchild(&status, &p,
+		    "vcl.discard %s\n", av[2])) {
+			VCLI_SetResult(cli, status);
+			VCLI_Out(cli, "%s", p);
+		} else {
+			AZ(mgt_vcc_delbyname(av[2]));
+		}
+	}
+	free(p);
+}
+
+void
+mcf_config_list(struct cli *cli, const char * const *av, void *priv)
+{
+	unsigned status;
+	char *p;
+	const char *flg;
+	struct vclprog *vp;
+
+	(void)av;
+	(void)priv;
+	if (child_pid >= 0) {
+		if (!mgt_cli_askchild(&status, &p, "vcl.list\n")) {
+			VCLI_SetResult(cli, status);
+			VCLI_Out(cli, "%s", p);
+		}
+		free(p);
+	} else {
+		VTAILQ_FOREACH(vp, &vclhead, list) {
+			if (vp->active) {
+				flg = "active";
+			} else
+				flg = "available";
+			VCLI_Out(cli, "%-10s %6s %s\n",
+			    flg, "N/A", vp->name);
+		}
+	}
+}
+
+/*
+ * XXX: This should take an option argument to show all (include) files
+ * XXX: This violates the principle of not loading VCL's in the master
+ * XXX: process.
+ */
+void
+mcf_config_show(struct cli *cli, const char * const *av, void *priv)
+{
+	struct vclprog *vp;
+	void *dlh, *sym;
+	const char **src;
+
+	(void)priv;
+	if ((vp = mcf_find_vcl(cli, av[2])) != NULL) {
+		if ((dlh = dlopen(vp->fname, RTLD_NOW | RTLD_LOCAL)) == NULL) {
+			VCLI_Out(cli, "failed to load %s: %s\n",
+			    vp->name, dlerror());
+			VCLI_SetResult(cli, CLIS_CANT);
+		} else if ((sym = dlsym(dlh, "srcbody")) == NULL) {
+			VCLI_Out(cli, "failed to locate source for %s: %s\n",
+			    vp->name, dlerror());
+			VCLI_SetResult(cli, CLIS_CANT);
+			AZ(dlclose(dlh));
+		} else {
+			src = sym;
+			VCLI_Out(cli, "%s", src[0]);
+			/* VCLI_Out(cli, src[1]); */
+			AZ(dlclose(dlh));
+		}
+	}
+}
diff --git a/bin/varnishd/mgt_child.c b/bin/varnishd/mgt_child.c
deleted file mode 100644
index 2108230..0000000
--- a/bin/varnishd/mgt_child.c
+++ /dev/null
@@ -1,673 +0,0 @@
-/*-
- * Copyright (c) 2006 Verdens Gang AS
- * Copyright (c) 2006-2011 Varnish Software AS
- * All rights reserved.
- *
- * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- *
- * The mechanics of handling the child process
- */
-
-#include "config.h"
-
-#include <sys/types.h>
-#include <sys/wait.h>
-
-#include <fcntl.h>
-#include <poll.h>
-#include <signal.h>
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#include <syslog.h>
-#include <unistd.h>
-
-#include "mgt.h"
-
-#include "heritage.h"
-#include "vapi/vsm_int.h"
-#include "vbm.h"
-#include "vcli.h"
-#include "vcli_priv.h"
-#include "vev.h"
-#include "vlu.h"
-#include "vss.h"
-#include "vtcp.h"
-#include "vtim.h"
-
-#include "mgt_cli.h"
-
-pid_t		child_pid = -1;
-
-
-static struct vbitmap	*fd_map;
-
-static int		child_cli_in = -1;
-static int		child_VCLI_Out = -1;
-static int		child_output = -1;
-
-static enum {
-	CH_STOPPED = 0,
-	CH_STARTING = 1,
-	CH_RUNNING = 2,
-	CH_STOPPING = 3,
-	CH_DIED = 4
-}			child_state = CH_STOPPED;
-
-static const char * const ch_state[] = {
-	[CH_STOPPED] =	"stopped",
-	[CH_STARTING] =	"starting",
-	[CH_RUNNING] =	"running",
-	[CH_STOPPING] =	"stopping",
-	[CH_DIED] =	"died, (restarting)",
-};
-
-static struct vev	*ev_poker;
-static struct vev	*ev_listen;
-static struct vlu	*vlu;
-
-static struct vsb *child_panic = NULL;
-
-/*--------------------------------------------------------------------
- * Track the highest file descriptor the parent knows is being used.
- *
- * This allows the child process to clean/close only a small fraction
- * of the possible file descriptors after exec(2).
- *
- * This is likely to a bit on the low side, as libc and other libraries
- * has a tendency to cache file descriptors (syslog, resolver, etc.)
- * so we add a margin of 100 fds.
- */
-
-static int		mgt_max_fd;
-
-#define CLOSE_FD_UP_TO	(mgt_max_fd + 100)
-
-void
-mgt_got_fd(int fd)
-{
-	/*
-	 * Assert > 0, to catch bogus opens, we know where stdin goes
-	 * in the master process.
-	 */
-	assert(fd > 0);
-	if (fd > mgt_max_fd)
-		mgt_max_fd = fd;
-}
-
-/*--------------------------------------------------------------------
- * A handy little function
- */
-
-static inline void
-closex(int *fd)
-{
-
-	assert(*fd >= 0);
-	AZ(close(*fd));
-	*fd = -1;
-}
-
-/*--------------------------------------------------------------------
- * Keep track of which filedescriptors the child should inherit and
- * which should be closed after fork()
- */
-
-void
-mgt_child_inherit(int fd, const char *what)
-{
-
-	assert(fd >= 0);
-	if (fd_map == NULL)
-		fd_map = vbit_init(128);
-	AN(fd_map);
-	if (what != NULL)
-		vbit_set(fd_map, fd);
-	else
-		vbit_clr(fd_map, fd);
-}
-
-/*--------------------------------------------------------------------*/
-
-static int
-child_line(void *priv, const char *p)
-{
-	(void)priv;
-
-	REPORT(LOG_NOTICE, "Child (%jd) said %s", (intmax_t)child_pid, p);
-	return (0);
-}
-
-/*--------------------------------------------------------------------*/
-
-static int
-child_listener(const struct vev *e, int what)
-{
-
-	(void)e;
-	if ((what & ~EV_RD)) {
-		ev_listen = NULL;
-		return (1);
-	}
-	if (VLU_Fd(child_output, vlu)) {
-		ev_listen = NULL;
-		return (1);
-	}
-	return (0);
-}
-
-/*--------------------------------------------------------------------*/
-
-static int
-child_poker(const struct vev *e, int what)
-{
-
-	(void)e;
-	(void)what;
-	if (child_state != CH_RUNNING)
-		return (1);
-	if (child_pid < 0)
-		return (0);
-	if (!mgt_cli_askchild(NULL, NULL, "ping\n"))
-		return (0);
-	return (0);
-}
-
-/*--------------------------------------------------------------------
- * If CLI communications with the child process fails, there is nothing
- * for us to do but to drag it behind the barn and get it over with.
- *
- * The typical case is where the child process fails to return a reply
- * before the cli_timeout expires.  This invalidates the CLI pipes for
- * all future use, as we don't know if the child was just slow and the
- * result gets piped later on, or if the child is catatonic.
- */
-
-void
-MGT_Child_Cli_Fail(void)
-{
-
-	if (child_state != CH_RUNNING)
-		return;
-	if (child_pid < 0)
-		return;
-	REPORT(LOG_ERR, "Child (%jd) not responding to CLI, killing it.",
-	    (intmax_t)child_pid);
-	if (params->diag_bitmap & 0x1000)
-		(void)kill(child_pid, SIGKILL);
-	else
-		(void)kill(child_pid, SIGQUIT);
-}
-
-/*--------------------------------------------------------------------*/
-
-static int
-open_sockets(void)
-{
-	struct listen_sock *ls, *ls2;
-	int good = 0;
-
-	VTAILQ_FOREACH_SAFE(ls, &heritage.socks, list, ls2) {
-		if (ls->sock >= 0) {
-			good++;
-			continue;
-		}
-		ls->sock = VSS_bind(ls->addr);
-		if (ls->sock < 0)
-			continue;
-
-		mgt_child_inherit(ls->sock, "sock");
-
-		/*
-		 * Set nonblocking mode to avoid a race where a client
-		 * closes before we call accept(2) and nobody else are in
-		 * the listen queue to release us.
-		 */
-		(void)VTCP_filter_http(ls->sock);
-		good++;
-	}
-	if (!good)
-		return (1);
-	return (0);
-}
-
-/*--------------------------------------------------------------------*/
-
-static void
-close_sockets(void)
-{
-	struct listen_sock *ls;
-
-	VTAILQ_FOREACH(ls, &heritage.socks, list) {
-		if (ls->sock < 0)
-			continue;
-		mgt_child_inherit(ls->sock, NULL);
-		closex(&ls->sock);
-	}
-}
-
-/*--------------------------------------------------------------------*/
-
-static void
-start_child(struct cli *cli)
-{
-	pid_t pid;
-	unsigned u;
-	char *p;
-	struct vev *e;
-	int i, cp[2];
-
-	if (child_state != CH_STOPPED && child_state != CH_DIED)
-		return;
-
-	if (open_sockets() != 0) {
-		child_state = CH_STOPPED;
-		if (cli != NULL) {
-			VCLI_SetResult(cli, CLIS_CANT);
-			VCLI_Out(cli, "Could not open sockets");
-			return;
-		}
-		REPORT0(LOG_ERR,
-		    "Child start failed: could not open sockets");
-		return;
-	}
-
-	child_state = CH_STARTING;
-
-	/* Open pipe for mgr->child CLI */
-	AZ(pipe(cp));
-	heritage.cli_in = cp[0];
-	mgt_child_inherit(heritage.cli_in, "cli_in");
-	child_VCLI_Out = cp[1];
-
-	/* Open pipe for child->mgr CLI */
-	AZ(pipe(cp));
-	heritage.VCLI_Out = cp[1];
-	mgt_child_inherit(heritage.VCLI_Out, "VCLI_Out");
-	child_cli_in = cp[0];
-
-	/*
-	 * Open pipe for child stdout/err
-	 * NB: not inherited, because we dup2() it to stdout/stderr in child
-	 */
-	AZ(pipe(cp));
-	heritage.std_fd = cp[1];
-	child_output = cp[0];
-
-	MCF_ParamSync();
-	if ((pid = fork()) < 0) {
-		perror("Could not fork child");
-		exit(1);
-	}
-	if (pid == 0) {
-
-		/* Redirect stdin/out/err */
-		AZ(close(STDIN_FILENO));
-		assert(open("/dev/null", O_RDONLY) == STDIN_FILENO);
-		assert(dup2(heritage.std_fd, STDOUT_FILENO) == STDOUT_FILENO);
-		assert(dup2(heritage.std_fd, STDERR_FILENO) == STDERR_FILENO);
-
-		/* Close anything we shouldn't know about */
-		closelog();
-		for (i = STDERR_FILENO + 1; i < CLOSE_FD_UP_TO; i++) {
-			if (vbit_test(fd_map, i))
-				continue;
-			(void)(close(i) == 0);
-		}
-#ifdef HAVE_SETPROCTITLE
-		setproctitle("Varnish-Chld %s", heritage.name);
-#endif
-
-		(void)signal(SIGINT, SIG_DFL);
-		(void)signal(SIGTERM, SIG_DFL);
-
-		mgt_sandbox();
-
-		child_main();
-
-		exit(1);
-	}
-	REPORT(LOG_NOTICE, "child (%jd) Started", (intmax_t)pid);
-
-	/* Close stuff the child got */
-	closex(&heritage.std_fd);
-
-	mgt_child_inherit(heritage.cli_in, NULL);
-	closex(&heritage.cli_in);
-
-	mgt_child_inherit(heritage.VCLI_Out, NULL);
-	closex(&heritage.VCLI_Out);
-
-	close_sockets();
-
-	vlu = VLU_New(NULL, child_line, 0);
-	AN(vlu);
-
-	AZ(ev_listen);
-	e = vev_new();
-	XXXAN(e);
-	e->fd = child_output;
-	e->fd_flags = EV_RD;
-	e->name = "Child listener";
-	e->callback = child_listener;
-	AZ(vev_add(mgt_evb, e));
-	ev_listen = e;
-	AZ(ev_poker);
-	if (params->ping_interval > 0) {
-		e = vev_new();
-		XXXAN(e);
-		e->timeout = params->ping_interval;
-		e->callback = child_poker;
-		e->name = "child poker";
-		AZ(vev_add(mgt_evb, e));
-		ev_poker = e;
-	}
-
-	mgt_cli_start_child(child_cli_in, child_VCLI_Out);
-	child_pid = pid;
-	if (mgt_push_vcls_and_start(&u, &p)) {
-		REPORT(LOG_ERR, "Pushing vcls failed:\n%s", p);
-		free(p);
-		child_state = CH_RUNNING;
-		mgt_stop_child();
-	} else
-		child_state = CH_RUNNING;
-}
-
-/*--------------------------------------------------------------------*/
-
-void
-mgt_stop_child(void)
-{
-
-	if (child_state != CH_RUNNING)
-		return;
-
-	child_state = CH_STOPPING;
-
-	REPORT0(LOG_DEBUG, "Stopping Child");
-	if (ev_poker != NULL) {
-		vev_del(mgt_evb, ev_poker);
-		free(ev_poker);
-	}
-	ev_poker = NULL;
-
-	mgt_cli_stop_child();
-
-	/* We tell the child to die gracefully by closing the CLI */
-	closex(&child_VCLI_Out);
-	closex(&child_cli_in);
-}
-
-/*--------------------------------------------------------------------*/
-
-static void
-mgt_report_panic(pid_t r)
-{
-
-	if (VSM_head->panicstr[0] == '\0')
-		return;
-	REPORT(LOG_ERR, "Child (%jd) Panic message: %s",
-	    (intmax_t)r, VSM_head->panicstr);
-}
-
-static void
-mgt_save_panic(void)
-{
-	char time_str[30];
-	if (VSM_head->panicstr[0] == '\0')
-		return;
-
-	if (child_panic)
-		VSB_delete(child_panic);
-	child_panic = VSB_new_auto();
-	XXXAN(child_panic);
-	VTIM_format(VTIM_real(), time_str);
-	VSB_printf(child_panic, "Last panic at: %s\n", time_str);
-	VSB_cat(child_panic, VSM_head->panicstr);
-	AZ(VSB_finish(child_panic));
-}
-
-static void
-mgt_clear_panic(void)
-{
-	VSB_delete(child_panic);
-	child_panic = NULL;
-}
-
-/*--------------------------------------------------------------------*/
-
-static int
-mgt_sigchld(const struct vev *e, int what)
-{
-	int status;
-	struct vsb *vsb;
-	pid_t r;
-
-	(void)e;
-	(void)what;
-
-	if (ev_poker != NULL) {
-		vev_del(mgt_evb, ev_poker);
-		free(ev_poker);
-	}
-	ev_poker = NULL;
-
-	r = waitpid(child_pid, &status, WNOHANG);
-	if (r == 0 || (r == -1 && errno == ECHILD))
-		return (0);
-	assert(r == child_pid);
-	vsb = VSB_new_auto();
-	XXXAN(vsb);
-	VSB_printf(vsb, "Child (%d) %s", r, status ? "died" : "ended");
-	if (WIFEXITED(status) && WEXITSTATUS(status)) {
-		VSB_printf(vsb, " status=%d", WEXITSTATUS(status));
-		exit_status |= 0x20;
-	}
-	if (WIFSIGNALED(status)) {
-		VSB_printf(vsb, " signal=%d", WTERMSIG(status));
-		exit_status |= 0x40;
-	}
-#ifdef WCOREDUMP
-	if (WCOREDUMP(status)) {
-		VSB_printf(vsb, " (core dumped)");
-		exit_status |= 0x80;
-	}
-#endif
-	AZ(VSB_finish(vsb));
-	REPORT(LOG_INFO, "%s", VSB_data(vsb));
-	VSB_delete(vsb);
-
-	mgt_report_panic(r);
-	mgt_save_panic();
-
-	child_pid = -1;
-
-	if (child_state == CH_RUNNING) {
-		child_state = CH_DIED;
-		mgt_cli_stop_child();
-		closex(&child_VCLI_Out);
-		closex(&child_cli_in);
-	}
-
-	if (ev_listen != NULL) {
-		vev_del(mgt_evb, ev_listen);
-		free(ev_listen);
-		ev_listen = NULL;
-	}
-	/* Pick up any stuff lingering on stdout/stderr */
-	(void)child_listener(NULL, EV_RD);
-	closex(&child_output);
-
-	REPORT0(LOG_DEBUG, "Child cleanup complete");
-
-	if (child_state == CH_DIED && params->auto_restart)
-		start_child(NULL);
-	else if (child_state == CH_DIED) {
-		child_state = CH_STOPPED;
-	} else if (child_state == CH_STOPPING)
-		child_state = CH_STOPPED;
-
-	return (0);
-}
-
-/*--------------------------------------------------------------------*/
-
-static int
-mgt_sigint(const struct vev *e, int what)
-{
-
-	(void)e;
-	(void)what;
-	REPORT0(LOG_ERR, "Manager got SIGINT");
-	(void)fflush(stdout);
-	if (child_pid >= 0)
-		mgt_stop_child();
-	exit (2);
-}
-
-/*--------------------------------------------------------------------
- * This thread is the master thread in the management process.
- * The relatively simple task is to start and stop the child process
- * and to reincarnate it in case of trouble.
- */
-
-void
-MGT_Run(void)
-{
-	struct sigaction sac;
-	struct vev *e;
-	int i;
-
-	e = vev_new();
-	XXXAN(e);
-	e->sig = SIGTERM;
-	e->callback = mgt_sigint;
-	e->name = "mgt_sigterm";
-	AZ(vev_add(mgt_evb, e));
-
-	e = vev_new();
-	XXXAN(e);
-	e->sig = SIGINT;
-	e->callback = mgt_sigint;
-	e->name = "mgt_sigint";
-	AZ(vev_add(mgt_evb, e));
-
-	e = vev_new();
-	XXXAN(e);
-	e->sig = SIGCHLD;
-	e->sig_flags = SA_NOCLDSTOP;
-	e->callback = mgt_sigchld;
-	e->name = "mgt_sigchild";
-	AZ(vev_add(mgt_evb, e));
-
-#ifdef HAVE_SETPROCTITLE
-	setproctitle("Varnish-Mgr %s", heritage.name);
-#endif
-
-	memset(&sac, 0, sizeof sac);
-	sac.sa_handler = SIG_IGN;
-	sac.sa_flags = SA_RESTART;
-
-	AZ(sigaction(SIGPIPE, &sac, NULL));
-	AZ(sigaction(SIGHUP, &sac, NULL));
-
-	if (!d_flag && !mgt_has_vcl())
-		REPORT0(LOG_ERR, "No VCL loaded yet");
-	else if (!d_flag) {
-		start_child(NULL);
-		if (child_state == CH_STOPPED) {
-			exit_status = 2;
-			return;
-		}
-	}
-
-	i = vev_schedule(mgt_evb);
-	if (i != 0)
-		REPORT(LOG_ERR, "vev_schedule() = %d", i);
-
-	REPORT0(LOG_ERR, "manager dies");
-}
-
-/*--------------------------------------------------------------------*/
-
-void __match_proto__(cli_func_t)
-mcf_server_startstop(struct cli *cli, const char * const *av, void *priv)
-{
-
-	(void)av;
-	if (priv != NULL && child_state == CH_RUNNING)
-		mgt_stop_child();
-	else if (priv == NULL && child_state == CH_STOPPED) {
-		if (mgt_has_vcl()) {
-			start_child(cli);
-		} else {
-			VCLI_SetResult(cli, CLIS_CANT);
-			VCLI_Out(cli, "No VCL available");
-		}
-	} else {
-		VCLI_SetResult(cli, CLIS_CANT);
-		VCLI_Out(cli, "Child in state %s", ch_state[child_state]);
-	}
-}
-
-/*--------------------------------------------------------------------*/
-
-void
-mcf_server_status(struct cli *cli, const char * const *av, void *priv)
-{
-	(void)av;
-	(void)priv;
-	VCLI_Out(cli, "Child in state %s", ch_state[child_state]);
-}
-
-void
-mcf_panic_show(struct cli *cli, const char * const *av, void *priv)
-{
-	(void)av;
-	(void)priv;
-
-	if (!child_panic) {
-	  VCLI_SetResult(cli, CLIS_CANT);
-	  VCLI_Out(cli, "Child has not panicked or panic has been cleared");
-	  return;
-	}
-
-	VCLI_Out(cli, "%s\n", VSB_data(child_panic));
-}
-
-void
-mcf_panic_clear(struct cli *cli, const char * const *av, void *priv)
-{
-	(void)av;
-	(void)priv;
-
-	if (!child_panic) {
-	  VCLI_SetResult(cli, CLIS_CANT);
-	  VCLI_Out(cli, "No panic to clear");
-	  return;
-	}
-
-	mgt_clear_panic();
-}
diff --git a/bin/varnishd/mgt_cli.c b/bin/varnishd/mgt_cli.c
deleted file mode 100644
index dea5e7f..0000000
--- a/bin/varnishd/mgt_cli.c
+++ /dev/null
@@ -1,663 +0,0 @@
-/*-
- * Copyright (c) 2006 Verdens Gang AS
- * Copyright (c) 2006-2011 Varnish Software AS
- * All rights reserved.
- *
- * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- *
- * The management process' CLI handling
- */
-
-#include "config.h"
-
-#include <sys/socket.h>
-
-#include <fcntl.h>
-#include <poll.h>
-#include <stdarg.h>
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#include <syslog.h>
-#include <unistd.h>
-
-#include "mgt.h"
-
-#include "heritage.h"
-#include "vcli.h"
-#include "vcli_common.h"
-#include "vcli_priv.h"
-#include "vcli_serve.h"
-#include "vev.h"
-#include "vlu.h"
-#include "vss.h"
-#include "vtcp.h"
-
-#include "mgt_cli.h"
-
-#ifndef HAVE_SRANDOMDEV
-#include "compat/srandomdev.h"
-#endif
-
-static int		cli_i = -1, cli_o = -1;
-static struct VCLS	*cls;
-static const char	*secret_file;
-
-#define	MCF_NOAUTH	0	/* NB: zero disables here-documents */
-#define MCF_AUTH	16
-
-/*--------------------------------------------------------------------*/
-
-static void
-mcf_banner(struct cli *cli, const char *const *av, void *priv)
-{
-
-	(void)av;
-	(void)priv;
-	VCLI_Out(cli, "-----------------------------\n");
-	VCLI_Out(cli, "Varnish Cache CLI 1.0\n");
-	VCLI_Out(cli, "-----------------------------\n");
-	VCLI_Out(cli, "%s\n", VSB_data(vident) + 1);
-	VCLI_Out(cli, "\n");
-	VCLI_Out(cli, "Type 'help' for command list.\n");
-	VCLI_Out(cli, "Type 'quit' to close CLI session.\n");
-	if (child_pid < 0)
-		VCLI_Out(cli, "Type 'start' to launch worker process.\n");
-	VCLI_SetResult(cli, CLIS_OK);
-}
-
-/*--------------------------------------------------------------------*/
-
-/* XXX: what order should this list be in ? */
-static struct cli_proto cli_proto[] = {
-	{ CLI_BANNER,		"", mcf_banner, NULL },
-	{ CLI_SERVER_STATUS,	"", mcf_server_status, NULL },
-	{ CLI_SERVER_START,	"", mcf_server_startstop, NULL },
-	{ CLI_SERVER_STOP,	"", mcf_server_startstop, cli_proto },
-	{ CLI_VCL_LOAD,		"", mcf_config_load, NULL },
-	{ CLI_VCL_INLINE,	"", mcf_config_inline, NULL },
-	{ CLI_VCL_USE,		"", mcf_config_use, NULL },
-	{ CLI_VCL_DISCARD,	"", mcf_config_discard, NULL },
-	{ CLI_VCL_LIST,		"", mcf_config_list, NULL },
-	{ CLI_VCL_SHOW,		"", mcf_config_show, NULL },
-	{ CLI_PARAM_SHOW,	"", mcf_param_show, NULL },
-	{ CLI_PARAM_SET,	"", mcf_param_set, NULL },
-	{ CLI_PANIC_SHOW,	"", mcf_panic_show, NULL },
-	{ CLI_PANIC_CLEAR,	"", mcf_panic_clear, NULL },
-	{ NULL }
-};
-
-/*--------------------------------------------------------------------*/
-
-static void
-mcf_panic(struct cli *cli, const char * const *av, void *priv)
-{
-
-	(void)cli;
-	(void)av;
-	(void)priv;
-	assert(!strcmp("", "You asked for it"));
-}
-
-static struct cli_proto cli_debug[] = {
-	{ "debug.panic.master", "debug.panic.master",
-		"\tPanic the master process.\n",
-		0, 0, "d", mcf_panic, NULL},
-	{ NULL }
-};
-
-/*--------------------------------------------------------------------*/
-
-static void
-mcf_askchild(struct cli *cli, const char * const *av, void *priv)
-{
-	int i;
-	char *q;
-	unsigned u;
-	struct vsb *vsb;
-
-	(void)priv;
-	/*
-	 * Command not recognized in master, try cacher if it is
-	 * running.
-	 */
-	if (cli_o <= 0) {
-		if (!strcmp(av[1], "help")) {
-			VCLI_Out(cli, "No help from child, (not running).\n");
-			return;
-		}
-		VCLI_SetResult(cli, CLIS_UNKNOWN);
-		VCLI_Out(cli,
-		    "Unknown request in manager process "
-		    "(child not running).\n"
-		    "Type 'help' for more info.");
-		return;
-	}
-	vsb = VSB_new_auto();
-	for (i = 1; av[i] != NULL; i++) {
-		VSB_quote(vsb, av[i], strlen(av[i]), 0);
-		VSB_putc(vsb, ' ');
-	}
-	VSB_putc(vsb, '\n');
-	AZ(VSB_finish(vsb));
-	i = write(cli_o, VSB_data(vsb), VSB_len(vsb));
-	if (i != VSB_len(vsb)) {
-		VSB_delete(vsb);
-		VCLI_SetResult(cli, CLIS_COMMS);
-		VCLI_Out(cli, "CLI communication error");
-		MGT_Child_Cli_Fail();
-		return;
-	}
-	VSB_delete(vsb);
-	(void)VCLI_ReadResult(cli_i, &u, &q, params->cli_timeout);
-	VCLI_SetResult(cli, u);
-	VCLI_Out(cli, "%s", q);
-	free(q);
-}
-
-static struct cli_proto cli_askchild[] = {
-	{ "*", "<wild-card-entry>", "\t<fall through to cacher>\n",
-		0, 9999, "h*", mcf_askchild, NULL},
-	{ NULL }
-};
-
-/*--------------------------------------------------------------------
- * Ask the child something over CLI, return zero only if everything is
- * happy happy.
- */
-
-int
-mgt_cli_askchild(unsigned *status, char **resp, const char *fmt, ...) {
-	int i, j;
-	va_list ap;
-	unsigned u;
-	char buf[params->cli_buffer], *p;
-
-	if (resp != NULL)
-		*resp = NULL;
-	if (status != NULL)
-		*status = 0;
-	if (cli_i < 0|| cli_o < 0) {
-		if (status != NULL)
-			*status = CLIS_CANT;
-		return (CLIS_CANT);
-	}
-	va_start(ap, fmt);
-	vbprintf(buf, fmt, ap);
-	va_end(ap);
-	p = strchr(buf, '\0');
-	assert(p != NULL && p > buf && p[-1] == '\n');
-	i = p - buf;
-	j = write(cli_o, buf, i);
-	if (j != i) {
-		if (status != NULL)
-			*status = CLIS_COMMS;
-		if (resp != NULL)
-			*resp = strdup("CLI communication error");
-		MGT_Child_Cli_Fail();
-		return (CLIS_COMMS);
-	}
-
-	(void)VCLI_ReadResult(cli_i, &u, resp, params->cli_timeout);
-	if (status != NULL)
-		*status = u;
-	if (u == CLIS_COMMS)
-		MGT_Child_Cli_Fail();
-	return (u == CLIS_OK ? 0 : u);
-}
-
-/*--------------------------------------------------------------------*/
-
-void
-mgt_cli_start_child(int fdi, int fdo)
-{
-
-	cli_i = fdi;
-	cli_o = fdo;
-}
-
-/*--------------------------------------------------------------------*/
-
-void
-mgt_cli_stop_child(void)
-{
-
-	cli_i = -1;
-	cli_o = -1;
-	/* XXX: kick any users */
-}
-
-/*--------------------------------------------------------------------
- * Generate a random challenge
- */
-
-static void
-mgt_cli_challenge(struct cli *cli)
-{
-	int i;
-
-	for (i = 0; i + 2L < sizeof cli->challenge; i++)
-		cli->challenge[i] = (random() % 26) + 'a';
-	cli->challenge[i++] = '\n';
-	cli->challenge[i] = '\0';
-	VCLI_Out(cli, "%s", cli->challenge);
-	VCLI_Out(cli, "\nAuthentication required.\n");
-	VCLI_SetResult(cli, CLIS_AUTH);
-}
-
-/*--------------------------------------------------------------------
- * Validate the authentication
- */
-
-static void
-mcf_auth(struct cli *cli, const char *const *av, void *priv)
-{
-	int fd;
-	char buf[CLI_AUTH_RESPONSE_LEN + 1];
-
-	AN(av[2]);
-	(void)priv;
-	if (secret_file == NULL) {
-		VCLI_Out(cli, "Secret file not configured\n");
-		VCLI_SetResult(cli, CLIS_CANT);
-		return;
-	}
-	fd = open(secret_file, O_RDONLY);
-	if (fd < 0) {
-		VCLI_Out(cli, "Cannot open secret file (%s)\n",
-		    strerror(errno));
-		VCLI_SetResult(cli, CLIS_CANT);
-		return;
-	}
-	mgt_got_fd(fd);
-	VCLI_AuthResponse(fd, cli->challenge, buf);
-	AZ(close(fd));
-	if (strcasecmp(buf, av[2])) {
-		mgt_cli_challenge(cli);
-		return;
-	}
-	cli->auth = MCF_AUTH;
-	memset(cli->challenge, 0, sizeof cli->challenge);
-	VCLI_SetResult(cli, CLIS_OK);
-	mcf_banner(cli, av, priv);
-}
-
-static struct cli_proto cli_auth[] = {
-	{ CLI_HELP,		"", VCLS_func_help, NULL },
-	{ CLI_PING,		"", VCLS_func_ping },
-	{ CLI_AUTH,		"", mcf_auth, NULL },
-	{ CLI_QUIT,		"", VCLS_func_close, NULL},
-	{ NULL }
-};
-
-/*--------------------------------------------------------------------*/
-static void
-mgt_cli_cb_before(const struct cli *cli)
-{
-
-	if (params->syslog_cli_traffic)
-		syslog(LOG_NOTICE, "CLI %s Rd %s", cli->ident, cli->cmd);
-}
-
-static void
-mgt_cli_cb_after(const struct cli *cli)
-{
-
-	if (params->syslog_cli_traffic)
-		syslog(LOG_NOTICE, "CLI %s Wr %03u %s",
-		    cli->ident, cli->result, VSB_data(cli->sb));
-}
-
-/*--------------------------------------------------------------------*/
-
-static void
-mgt_cli_init_cls(void)
-{
-
-	cls = VCLS_New(mgt_cli_cb_before, mgt_cli_cb_after, params->cli_buffer);
-	AN(cls);
-	AZ(VCLS_AddFunc(cls, MCF_NOAUTH, cli_auth));
-	AZ(VCLS_AddFunc(cls, MCF_AUTH, cli_proto));
-	AZ(VCLS_AddFunc(cls, MCF_AUTH, cli_debug));
-	AZ(VCLS_AddFunc(cls, MCF_AUTH, cli_stv));
-	AZ(VCLS_AddFunc(cls, MCF_AUTH, cli_askchild));
-}
-
-/*--------------------------------------------------------------------
- * Get rid of all CLI sessions
- */
-
-void
-mgt_cli_close_all(void)
-{
-
-	VCLS_Destroy(&cls);
-}
-
-/*--------------------------------------------------------------------
- * Callback whenever something happens to the input fd of the session.
- */
-
-static int
-mgt_cli_callback2(const struct vev *e, int what)
-{
-	int i;
-
-	(void)e;
-	(void)what;
-	i = VCLS_PollFd(cls, e->fd, 0);
-	return (i);
-}
-
-/*--------------------------------------------------------------------*/
-
-void
-mgt_cli_setup(int fdi, int fdo, int verbose, const char *ident, mgt_cli_close_f *closefunc, void *priv)
-{
-	struct cli *cli;
-	struct vev *ev;
-
-	(void)ident;
-	(void)verbose;
-	if (cls == NULL)
-		mgt_cli_init_cls();
-
-	cli = VCLS_AddFd(cls, fdi, fdo, closefunc, priv);
-
-	cli->ident = strdup(ident);
-
-	/* Deal with TELNET options */
-	if (fdi != 0)
-		VLU_SetTelnet(cli->vlu, fdo);
-
-	if (fdi != 0 && secret_file != NULL) {
-		cli->auth = MCF_NOAUTH;
-		mgt_cli_challenge(cli);
-	} else {
-		cli->auth = MCF_AUTH;
-		mcf_banner(cli, NULL, NULL);
-	}
-	AZ(VSB_finish(cli->sb));
-	(void)VCLI_WriteResult(fdo, cli->result, VSB_data(cli->sb));
-
-
-	ev = vev_new();
-	AN(ev);
-	ev->name = cli->ident;
-	ev->fd = fdi;
-	ev->fd_flags = EV_RD;
-	ev->callback = mgt_cli_callback2;
-	ev->priv = cli;
-	AZ(vev_add(mgt_evb, ev));
-}
-
-/*--------------------------------------------------------------------*/
-
-static struct vsb *
-sock_id(const char *pfx, int fd)
-{
-	struct vsb *vsb;
-
-	char abuf1[VTCP_ADDRBUFSIZE], abuf2[VTCP_ADDRBUFSIZE];
-	char pbuf1[VTCP_PORTBUFSIZE], pbuf2[VTCP_PORTBUFSIZE];
-
-	vsb = VSB_new_auto();
-	AN(vsb);
-	VTCP_myname(fd, abuf1, sizeof abuf1, pbuf1, sizeof pbuf1);
-	VTCP_hisname(fd, abuf2, sizeof abuf2, pbuf2, sizeof pbuf2);
-	VSB_printf(vsb, "%s %s %s %s %s", pfx, abuf2, pbuf2, abuf1, pbuf1);
-	AZ(VSB_finish(vsb));
-	return (vsb);
-}
-
-/*--------------------------------------------------------------------*/
-
-struct telnet {
-	unsigned		magic;
-#define TELNET_MAGIC		0x53ec3ac0
-	int			fd;
-	struct vev		*ev;
-};
-
-static void
-telnet_close(void *priv)
-{
-	struct telnet *tn;
-
-	CAST_OBJ_NOTNULL(tn, priv, TELNET_MAGIC);
-	(void)close(tn->fd);
-	FREE_OBJ(tn);
-}
-
-static struct telnet *
-telnet_new(int fd)
-{
-	struct telnet *tn;
-
-	ALLOC_OBJ(tn, TELNET_MAGIC);
-	AN(tn);
-	tn->fd = fd;
-	return (tn);
-}
-
-static int
-telnet_accept(const struct vev *ev, int what)
-{
-	struct vsb *vsb;
-	struct sockaddr_storage addr;
-	socklen_t addrlen;
-	struct telnet *tn;
-	int i;
-
-	(void)what;
-	addrlen = sizeof addr;
-	i = accept(ev->fd, (void *)&addr, &addrlen);
-	if (i < 0 && errno == EBADF)
-		return (1);
-	if (i < 0)
-		return (0);
-
-	mgt_got_fd(i);
-	tn = telnet_new(i);
-	vsb = sock_id("telnet", i);
-	mgt_cli_setup(i, i, 0, VSB_data(vsb), telnet_close, tn);
-	VSB_delete(vsb);
-	return (0);
-}
-
-void
-mgt_cli_secret(const char *S_arg)
-{
-	int i, fd;
-	char buf[BUFSIZ];
-	char *p;
-
-	/* Save in shmem */
-	i = strlen(S_arg);
-	p = VSM_Alloc(i + 1, "Arg", "-S", "");
-	AN(p);
-	strcpy(p, S_arg);
-
-	srandomdev();
-	fd = open(S_arg, O_RDONLY);
-	if (fd < 0) {
-		fprintf(stderr, "Can not open secret-file \"%s\"\n", S_arg);
-		exit (2);
-	}
-	mgt_got_fd(fd);
-	i = read(fd, buf, sizeof buf);
-	if (i == 0) {
-		fprintf(stderr, "Empty secret-file \"%s\"\n", S_arg);
-		exit (2);
-	}
-	if (i < 0) {
-		fprintf(stderr, "Can not read secret-file \"%s\"\n", S_arg);
-		exit (2);
-	}
-	AZ(close(fd));
-	secret_file = S_arg;
-}
-
-void
-mgt_cli_telnet(const char *T_arg)
-{
-	struct vss_addr **ta;
-	int i, n, sock, good;
-	struct telnet *tn;
-	char *p;
-	struct vsb *vsb;
-	char abuf[VTCP_ADDRBUFSIZE];
-	char pbuf[VTCP_PORTBUFSIZE];
-
-	n = VSS_resolve(T_arg, NULL, &ta);
-	if (n == 0) {
-		REPORT(LOG_ERR, "-T %s Could not be resolved\n", T_arg);
-		exit(2);
-	}
-	good = 0;
-	vsb = VSB_new_auto();
-	XXXAN(vsb);
-	for (i = 0; i < n; ++i) {
-		sock = VSS_listen(ta[i], 10);
-		if (sock < 0)
-			continue;
-		VTCP_myname(sock, abuf, sizeof abuf, pbuf, sizeof pbuf);
-		VSB_printf(vsb, "%s %s\n", abuf, pbuf);
-		good++;
-		tn = telnet_new(sock);
-		tn->ev = vev_new();
-		XXXAN(tn->ev);
-		tn->ev->fd = sock;
-		tn->ev->fd_flags = POLLIN;
-		tn->ev->callback = telnet_accept;
-		AZ(vev_add(mgt_evb, tn->ev));
-		free(ta[i]);
-		ta[i] = NULL;
-	}
-	free(ta);
-	if (good == 0) {
-		REPORT(LOG_ERR, "-T %s could not be listened on.", T_arg);
-		exit(2);
-	}
-	AZ(VSB_finish(vsb));
-	/* Save in shmem */
-	p = VSM_Alloc(VSB_len(vsb) + 1, "Arg", "-T", "");
-	AN(p);
-	strcpy(p, VSB_data(vsb));
-	VSB_delete(vsb);
-}
-
-/* Reverse CLI ("Master") connections --------------------------------*/
-
-static int M_fd = -1;
-static struct vev *M_poker, *M_conn;
-static struct vss_addr **M_ta;
-static int M_nta, M_nxt;
-static double M_poll = 0.1;
-
-static void
-Marg_closer(void *priv)
-{
-
-	(void)priv;
-	(void)close(M_fd);
-	M_fd = -1;
-}
-
-static int
-Marg_poker(const struct vev *e, int what)
-{
-	struct vsb *vsb;
-	int s, k;
-	socklen_t l;
-
-	(void)what;	/* XXX: ??? */
-
-	if (e == M_conn) {
-		/* Our connect(2) returned, check result */
-		l = sizeof k;
-		AZ(getsockopt(M_fd, SOL_SOCKET, SO_ERROR, &k, &l));
-		if (k) {
-			errno = k;
-			syslog(LOG_INFO, "Could not connect to CLI-master: %m");
-			(void)close(M_fd);
-			M_fd = -1;
-			/* Try next address */
-			if (++M_nxt >= M_nta) {
-				M_nxt = 0;
-				if (M_poll < 10)
-					M_poll *= 2;
-			}
-			return (1);
-		}
-		vsb = sock_id("master", M_fd);
-		mgt_cli_setup(M_fd, M_fd, 0, VSB_data(vsb), Marg_closer, NULL);
-		VSB_delete(vsb);
-		M_poll = 1;
-		return (1);
-	}
-
-	assert(e == M_poker);
-
-	M_poker->timeout = M_poll;	/* XXX nasty ? */
-	if (M_fd >= 0)
-		return (0);
-
-	/* Try to connect asynchronously */
-	s = VSS_connect(M_ta[M_nxt], 1);
-	if (s < 0)
-		return (0);
-
-	mgt_got_fd(s);
-
-	M_conn = vev_new();
-	AN(M_conn);
-	M_conn->callback = Marg_poker;
-	M_conn->name = "-M connector";
-	M_conn->fd_flags = EV_WR;
-	M_conn->fd = s;
-	M_fd = s;
-	AZ(vev_add(mgt_evb, M_conn));
-	return (0);
-}
-
-void
-mgt_cli_master(const char *M_arg)
-{
-	(void)M_arg;
-
-	M_nta = VSS_resolve(M_arg, NULL, &M_ta);
-	if (M_nta <= 0) {
-		fprintf(stderr, "Could resolve -M argument to address\n");
-		exit (1);
-	}
-	M_nxt = 0;
-	AZ(M_poker);
-	M_poker = vev_new();
-	AN(M_poker);
-	M_poker->timeout = M_poll;
-	M_poker->callback = Marg_poker;
-	M_poker->name = "-M poker";
-	AZ(vev_add(mgt_evb, M_poker));
-}
diff --git a/bin/varnishd/mgt_cli.h b/bin/varnishd/mgt_cli.h
deleted file mode 100644
index 8c49abb..0000000
--- a/bin/varnishd/mgt_cli.h
+++ /dev/null
@@ -1,50 +0,0 @@
-/*-
- * Copyright (c) 2006 Verdens Gang AS
- * Copyright (c) 2006-2009 Varnish Software AS
- * All rights reserved.
- *
- * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- *
- */
-
-/* mgt_child.c */
-cli_func_t mcf_server_startstop;
-cli_func_t mcf_server_status;
-cli_func_t mcf_panic_show;
-cli_func_t mcf_panic_clear;
-
-/* mgt_param.c */
-cli_func_t mcf_param_show;
-cli_func_t mcf_param_set;
-
-/* mgt_vcc.c */
-cli_func_t mcf_config_load;
-cli_func_t mcf_config_inline;
-cli_func_t mcf_config_use;
-cli_func_t mcf_config_discard;
-cli_func_t mcf_config_list;
-cli_func_t mcf_config_show;
-
-/* stevedore.c */
-extern struct cli_proto cli_stv[];
diff --git a/bin/varnishd/mgt_param.c b/bin/varnishd/mgt_param.c
deleted file mode 100644
index 43749b6..0000000
--- a/bin/varnishd/mgt_param.c
+++ /dev/null
@@ -1,1237 +0,0 @@
-/*-
- * Copyright (c) 2006 Verdens Gang AS
- * Copyright (c) 2006-2011 Varnish Software AS
- * All rights reserved.
- *
- * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- */
-
-#include "config.h"
-
-#include <grp.h>
-#include <limits.h>
-#include <pwd.h>
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#include <unistd.h>
-
-#include "mgt.h"
-
-#include "waiter/cache_waiter.h"
-#include "heritage.h"
-#include "vav.h"
-#include "vcli.h"
-#include "vcli_common.h"
-#include "vcli_priv.h"
-#include "vparam.h"
-#include "vss.h"
-
-#include "mgt_cli.h"
-
-#define MAGIC_INIT_STRING	"\001"
-struct params master;
-static int nparspec;
-static struct parspec const ** parspec;
-static int margin;
-
-/*--------------------------------------------------------------------*/
-
-static const struct parspec *
-mcf_findpar(const char *name)
-{
-	int i;
-
-	for (i = 0; i < nparspec; i++)
-		if (!strcmp(parspec[i]->name, name))
-			return (parspec[i]);
-	return (NULL);
-}
-
-/*--------------------------------------------------------------------*/
-
-static void
-tweak_generic_timeout(struct cli *cli, volatile unsigned *dst, const char *arg)
-{
-	unsigned u;
-
-	if (arg != NULL) {
-		u = strtoul(arg, NULL, 0);
-		if (u == 0) {
-			VCLI_Out(cli, "Timeout must be greater than zero\n");
-			VCLI_SetResult(cli, CLIS_PARAM);
-			return;
-		}
-		*dst = u;
-	} else
-		VCLI_Out(cli, "%u", *dst);
-}
-
-/*--------------------------------------------------------------------*/
-
-void
-tweak_timeout(struct cli *cli, const struct parspec *par, const char *arg)
-{
-	volatile unsigned *dest;
-
-	dest = par->priv;
-	tweak_generic_timeout(cli, dest, arg);
-}
-
-static void
-tweak_timeout_double(struct cli *cli, const struct parspec *par,
-    const char *arg)
-{
-	volatile double *dest;
-	double u;
-
-	dest = par->priv;
-	if (arg != NULL) {
-		u = strtod(arg, NULL);
-		if (u < par->min) {
-			VCLI_Out(cli,
-			    "Timeout must be greater or equal to %.g\n",
-				 par->min);
-			VCLI_SetResult(cli, CLIS_PARAM);
-			return;
-		}
-		if (u > par->max) {
-			VCLI_Out(cli,
-			    "Timeout must be less than or equal to %.g\n",
-				 par->max);
-			VCLI_SetResult(cli, CLIS_PARAM);
-			return;
-		}
-		*dest = u;
-	} else
-		VCLI_Out(cli, "%.6f", *dest);
-}
-
-/*--------------------------------------------------------------------*/
-
-static void
-tweak_generic_double(struct cli *cli, const struct parspec *par,
-    const char *arg)
-{
-	volatile double *dest;
-	double u;
-
-	dest = par->priv;
-	if (arg != NULL) {
-		u = strtod(arg, NULL);
-		if (u < par->min) {
-			VCLI_Out(cli,
-			    "Must be greater or equal to %.g\n",
-				 par->min);
-			VCLI_SetResult(cli, CLIS_PARAM);
-			return;
-		}
-		if (u > par->max) {
-			VCLI_Out(cli,
-			    "Must be less than or equal to %.g\n",
-				 par->max);
-			VCLI_SetResult(cli, CLIS_PARAM);
-			return;
-		}
-		*dest = u;
-	} else
-		VCLI_Out(cli, "%f", *dest);
-}
-
-/*--------------------------------------------------------------------*/
-
-static void
-tweak_generic_bool(struct cli *cli, volatile unsigned *dest, const char *arg)
-{
-	if (arg != NULL) {
-		if (!strcasecmp(arg, "off"))
-			*dest = 0;
-		else if (!strcasecmp(arg, "disable"))
-			*dest = 0;
-		else if (!strcasecmp(arg, "no"))
-			*dest = 0;
-		else if (!strcasecmp(arg, "false"))
-			*dest = 0;
-		else if (!strcasecmp(arg, "on"))
-			*dest = 1;
-		else if (!strcasecmp(arg, "enable"))
-			*dest = 1;
-		else if (!strcasecmp(arg, "yes"))
-			*dest = 1;
-		else if (!strcasecmp(arg, "true"))
-			*dest = 1;
-		else {
-			VCLI_Out(cli, "use \"on\" or \"off\"\n");
-			VCLI_SetResult(cli, CLIS_PARAM);
-			return;
-		}
-	} else
-		VCLI_Out(cli, *dest ? "on" : "off");
-}
-
-/*--------------------------------------------------------------------*/
-
-static void
-tweak_bool(struct cli *cli, const struct parspec *par, const char *arg)
-{
-	volatile unsigned *dest;
-
-	dest = par->priv;
-	tweak_generic_bool(cli, dest, arg);
-}
-
-/*--------------------------------------------------------------------*/
-
-void
-tweak_generic_uint(struct cli *cli, volatile unsigned *dest, const char *arg,
-    unsigned min, unsigned max)
-{
-	unsigned u;
-
-	if (arg != NULL) {
-		if (!strcasecmp(arg, "unlimited"))
-			u = UINT_MAX;
-		else
-			u = strtoul(arg, NULL, 0);
-		if (u < min) {
-			VCLI_Out(cli, "Must be at least %u\n", min);
-			VCLI_SetResult(cli, CLIS_PARAM);
-			return;
-		}
-		if (u > max) {
-			VCLI_Out(cli, "Must be no more than %u\n", max);
-			VCLI_SetResult(cli, CLIS_PARAM);
-			return;
-		}
-		*dest = u;
-	} else if (*dest == UINT_MAX) {
-		VCLI_Out(cli, "unlimited", *dest);
-	} else {
-		VCLI_Out(cli, "%u", *dest);
-	}
-}
-
-/*--------------------------------------------------------------------*/
-
-void
-tweak_uint(struct cli *cli, const struct parspec *par, const char *arg)
-{
-	volatile unsigned *dest;
-
-	dest = par->priv;
-	tweak_generic_uint(cli, dest, arg, (uint)par->min, (uint)par->max);
-}
-
-/*--------------------------------------------------------------------
- * XXX: slightly magic.  We want to initialize to "nobody" (XXX: shouldn't
- * XXX: that be something autocrap found for us ?) but we don't want to
- * XXX: fail initialization if that user doesn't exists, even though we
- * XXX: do want to fail it, in subsequent sets.
- * XXX: The magic init string is a hack for this.
- */
-
-static void
-tweak_user(struct cli *cli, const struct parspec *par, const char *arg)
-{
-	struct passwd *pw;
-	struct group *gr;
-
-	(void)par;
-	if (arg != NULL) {
-		if (!strcmp(arg, MAGIC_INIT_STRING)) {
-			pw = getpwnam("nobody");
-			if (pw == NULL) {
-				master.uid = getuid();
-				return;
-			}
-		} else
-			pw = getpwnam(arg);
-		if (pw == NULL) {
-			VCLI_Out(cli, "Unknown user");
-			VCLI_SetResult(cli, CLIS_PARAM);
-			return;
-		}
-		REPLACE(master.user, pw->pw_name);
-		master.uid = pw->pw_uid;
-		master.gid = pw->pw_gid;
-
-		/* set group to user's primary group */
-		if ((gr = getgrgid(pw->pw_gid)) != NULL &&
-		    (gr = getgrnam(gr->gr_name)) != NULL &&
-		    gr->gr_gid == pw->pw_gid)
-			REPLACE(master.group, gr->gr_name);
-	} else if (master.user) {
-		VCLI_Out(cli, "%s (%d)", master.user, (int)master.uid);
-	} else {
-		VCLI_Out(cli, "%d", (int)master.uid);
-	}
-}
-
-/*--------------------------------------------------------------------
- * XXX: see comment for tweak_user, same thing here.
- */
-
-static void
-tweak_group(struct cli *cli, const struct parspec *par, const char *arg)
-{
-	struct group *gr;
-
-	(void)par;
-	if (arg != NULL) {
-		if (!strcmp(arg, MAGIC_INIT_STRING)) {
-			gr = getgrnam("nogroup");
-			if (gr == NULL) {
-				/* Only replace if tweak_user didn't */
-				if (master.gid == 0)
-					master.gid = getgid();
-				return;
-			}
-		} else
-			gr = getgrnam(arg);
-		if (gr == NULL) {
-			VCLI_Out(cli, "Unknown group");
-			VCLI_SetResult(cli, CLIS_PARAM);
-			return;
-		}
-		REPLACE(master.group, gr->gr_name);
-		master.gid = gr->gr_gid;
-	} else if (master.group) {
-		VCLI_Out(cli, "%s (%d)", master.group, (int)master.gid);
-	} else {
-		VCLI_Out(cli, "%d", (int)master.gid);
-	}
-}
-
-/*--------------------------------------------------------------------*/
-
-static void
-clean_listen_sock_head(struct listen_sock_head *lsh)
-{
-	struct listen_sock *ls, *ls2;
-
-	VTAILQ_FOREACH_SAFE(ls, lsh, list, ls2) {
-		CHECK_OBJ_NOTNULL(ls, LISTEN_SOCK_MAGIC);
-		VTAILQ_REMOVE(lsh, ls, list);
-		free(ls->name);
-		free(ls->addr);
-		FREE_OBJ(ls);
-	}
-}
-
-static void
-tweak_listen_address(struct cli *cli, const struct parspec *par,
-    const char *arg)
-{
-	char **av;
-	int i;
-	struct listen_sock		*ls;
-	struct listen_sock_head		lsh;
-
-	(void)par;
-	if (arg == NULL) {
-		VCLI_Quote(cli, master.listen_address);
-		return;
-	}
-
-	av = VAV_Parse(arg, NULL, ARGV_COMMA);
-	if (av == NULL) {
-		VCLI_Out(cli, "Parse error: out of memory");
-		VCLI_SetResult(cli, CLIS_PARAM);
-		return;
-	}
-	if (av[0] != NULL) {
-		VCLI_Out(cli, "Parse error: %s", av[0]);
-		VCLI_SetResult(cli, CLIS_PARAM);
-		VAV_Free(av);
-		return;
-	}
-	if (av[1] == NULL) {
-		VCLI_Out(cli, "Empty listen address");
-		VCLI_SetResult(cli, CLIS_PARAM);
-		VAV_Free(av);
-		return;
-	}
-	VTAILQ_INIT(&lsh);
-	for (i = 1; av[i] != NULL; i++) {
-		struct vss_addr **ta;
-		int j, n;
-
-		n = VSS_resolve(av[i], "http", &ta);
-		if (n == 0) {
-			VCLI_Out(cli, "Invalid listen address ");
-			VCLI_Quote(cli, av[i]);
-			VCLI_SetResult(cli, CLIS_PARAM);
-			break;
-		}
-		for (j = 0; j < n; ++j) {
-			ALLOC_OBJ(ls, LISTEN_SOCK_MAGIC);
-			AN(ls);
-			ls->sock = -1;
-			ls->addr = ta[j];
-			ls->name = strdup(av[i]);
-			AN(ls->name);
-			VTAILQ_INSERT_TAIL(&lsh, ls, list);
-		}
-		free(ta);
-	}
-	VAV_Free(av);
-	if (cli != NULL && cli->result != CLIS_OK) {
-		clean_listen_sock_head(&lsh);
-		return;
-	}
-
-	REPLACE(master.listen_address, arg);
-
-	clean_listen_sock_head(&heritage.socks);
-	heritage.nsocks = 0;
-
-	while (!VTAILQ_EMPTY(&lsh)) {
-		ls = VTAILQ_FIRST(&lsh);
-		VTAILQ_REMOVE(&lsh, ls, list);
-		CHECK_OBJ_NOTNULL(ls, LISTEN_SOCK_MAGIC);
-		VTAILQ_INSERT_TAIL(&heritage.socks, ls, list);
-		heritage.nsocks++;
-	}
-}
-
-/*--------------------------------------------------------------------*/
-
-static void
-tweak_string(struct cli *cli, const struct parspec *par, const char *arg)
-{
-	char **p = TRUST_ME(par->priv);
-
-	AN(p);
-	/* XXX should have tweak_generic_string */
-	if (arg == NULL) {
-		VCLI_Quote(cli, *p);
-	} else {
-		REPLACE(*p, arg);
-	}
-}
-
-/*--------------------------------------------------------------------*/
-
-static void
-tweak_waiter(struct cli *cli, const struct parspec *par, const char *arg)
-{
-
-	/* XXX should have tweak_generic_string */
-	(void)par;
-	WAIT_tweak_waiter(cli, arg);
-}
-
-/*--------------------------------------------------------------------*/
-
-static void
-tweak_diag_bitmap(struct cli *cli, const struct parspec *par, const char *arg)
-{
-	unsigned u;
-
-	(void)par;
-	if (arg != NULL) {
-		u = strtoul(arg, NULL, 0);
-		master.diag_bitmap = u;
-	} else {
-		VCLI_Out(cli, "0x%x", master.diag_bitmap);
-	}
-}
-
-/*--------------------------------------------------------------------*/
-
-/*
- * Make sure to end all lines with either a space or newline of the
- * formatting will go haywire.
- */
-
-#define DELAYED_EFFECT_TEXT \
-	"\nNB: This parameter may take quite some time to take (full) effect."
-
-#define MUST_RESTART_TEXT \
-	"\nNB: This parameter will not take any effect until the " \
-	"child process has been restarted."
-
-#define MUST_RELOAD_TEXT \
-	"\nNB: This parameter will not take any effect until the " \
-	"VCL programs have been reloaded."
-
-#define EXPERIMENTAL_TEXT \
-	"\nNB: We do not know yet if it is a good idea to change " \
-	"this parameter, or if the default value is even sensible.  " \
-	"Caution is advised, and feedback is most welcome."
-
-#define WIZARD_TEXT \
-	"\nNB: Do not change this parameter, unless a developer tell " \
-	"you to do so."
-
-/*
- * Remember to update varnishd.1 whenever you add / remove a parameter or
- * change its default value.
- * XXX: we should generate the relevant section of varnishd.1 from here.
- */
-static const struct parspec input_parspec[] = {
-	{ "user", tweak_user, NULL, 0, 0,
-		"The unprivileged user to run as.  Setting this will "
-		"also set \"group\" to the specified user's primary group.",
-		MUST_RESTART,
-		MAGIC_INIT_STRING },
-	{ "group", tweak_group, NULL, 0, 0,
-		"The unprivileged group to run as.",
-		MUST_RESTART,
-		MAGIC_INIT_STRING },
-	{ "default_ttl", tweak_timeout_double, &master.default_ttl,
-		0, UINT_MAX,
-		"The TTL assigned to objects if neither the backend nor "
-		"the VCL code assigns one.\n"
-		"Objects already cached will not be affected by changes "
-		"made until they are fetched from the backend again.\n"
-		"To force an immediate effect at the expense of a total "
-		"flush of the cache use \"ban.url .\"",
-		0,
-		"120", "seconds" },
-	{ "sess_workspace", tweak_uint, &master.sess_workspace, 1024, UINT_MAX,
-		"Bytes of HTTP protocol workspace allocated for sessions. "
-		"This space must be big enough for the entire HTTP protocol "
-		"header and any edits done to it in the VCL code.\n"
-		"Minimum is 1024 bytes.",
-		DELAYED_EFFECT,
-		"65536",
-		"bytes" },
-	{ "http_req_hdr_len", tweak_uint, &master.http_req_hdr_len,
-		40, UINT_MAX,
-		"Maximum length of any HTTP client request header we will "
-		"allow.  The limit is inclusive its continuation lines.\n",
-		0,
-		"8192", "bytes" },
-	{ "http_req_size", tweak_uint, &master.http_req_size,
-		256, UINT_MAX,
-		"Maximum number of bytes of HTTP client request we will deal "
-		"with.  This is a limit on all bytes up to the double blank "
-		"line which ends the HTTP request.\n"
-		"The memory for the request is allocated from the session "
-		"workspace (param: sess_workspace) and this parameter limits "
-		"how much of that the request is allowed to take up.",
-		0,
-		"32768", "bytes" },
-	{ "http_resp_hdr_len", tweak_uint, &master.http_resp_hdr_len,
-		40, UINT_MAX,
-		"Maximum length of any HTTP backend response header we will "
-		"allow.  The limit is inclusive its continuation lines.\n",
-		0,
-		"8192", "bytes" },
-	{ "http_resp_size", tweak_uint, &master.http_resp_size,
-		256, UINT_MAX,
-		"Maximum number of bytes of HTTP backend resonse we will deal "
-		"with.  This is a limit on all bytes up to the double blank "
-		"line which ends the HTTP request.\n"
-		"The memory for the request is allocated from the worker "
-		"workspace (param: sess_workspace) and this parameter limits "
-		"how much of that the request is allowed to take up.",
-		0,
-		"32768", "bytes" },
-	{ "http_max_hdr", tweak_uint, &master.http_max_hdr, 32, 65535,
-		"Maximum number of HTTP headers we will deal with in "
-		"client request or backend reponses.  "
-		"Note that the first line occupies five header fields.\n"
-		"This paramter does not influence storage consumption, "
-		"objects allocate exact space for the headers they store.\n",
-		0,
-		"64", "header lines" },
-	{ "shm_workspace", tweak_uint, &master.shm_workspace, 4096, UINT_MAX,
-		"Bytes of shmlog workspace allocated for worker threads. "
-		"If too big, it wastes some ram, if too small it causes "
-		"needless flushes of the SHM workspace.\n"
-		"These flushes show up in stats as "
-		"\"SHM flushes due to overflow\".\n"
-		"Minimum is 4096 bytes.",
-		DELAYED_EFFECT,
-		"8192", "bytes" },
-	{ "shm_reclen", tweak_uint, &master.shm_reclen, 16, 65535,
-		"Maximum number of bytes in SHM log record.\n"
-		"Maximum is 65535 bytes.",
-		0,
-		"255", "bytes" },
-	{ "default_grace", tweak_timeout_double, &master.default_grace,
-		0, UINT_MAX,
-		"Default grace period.  We will deliver an object "
-		"this long after it has expired, provided another thread "
-		"is attempting to get a new copy.\n"
-		"Objects already cached will not be affected by changes "
-		"made until they are fetched from the backend again.\n",
-		DELAYED_EFFECT,
-		"10", "seconds" },
-	{ "default_keep", tweak_timeout_double, &master.default_keep,
-		0, UINT_MAX,
-		"Default keep period.  We will keep a useless object "
-		"around this long, making it available for conditional "
-		"backend fetches.  "
-		"That means that the object will be removed from the "
-		"cache at the end of ttl+grace+keep.",
-		DELAYED_EFFECT,
-		"0", "seconds" },
-	{ "sess_timeout", tweak_timeout, &master.sess_timeout, 0, 0,
-		"Idle timeout for persistent sessions. "
-		"If a HTTP request has not been received in this many "
-		"seconds, the session is closed.",
-		0,
-		"5", "seconds" },
-	{ "expiry_sleep", tweak_timeout_double, &master.expiry_sleep, 0, 60,
-		"How long the expiry thread sleeps when there is nothing "
-		"for it to do.\n",
-		0,
-		"1", "seconds" },
-	{ "pipe_timeout", tweak_timeout, &master.pipe_timeout, 0, 0,
-		"Idle timeout for PIPE sessions. "
-		"If nothing have been received in either direction for "
-		"this many seconds, the session is closed.\n",
-		0,
-		"60", "seconds" },
-	{ "send_timeout", tweak_timeout, &master.send_timeout, 0, 0,
-		"Send timeout for client connections. "
-		"If the HTTP response hasn't been transmitted in this many\n"
-                "seconds the session is closed. \n"
-		"See setsockopt(2) under SO_SNDTIMEO for more information.",
-		DELAYED_EFFECT,
-		"60", "seconds" },
-	{ "auto_restart", tweak_bool, &master.auto_restart, 0, 0,
-		"Restart child process automatically if it dies.\n",
-		0,
-		"on", "bool" },
-	{ "nuke_limit",
-		tweak_uint, &master.nuke_limit, 0, UINT_MAX,
-		"Maximum number of objects we attempt to nuke in order"
-		"to make space for a object body.",
-		EXPERIMENTAL,
-		"50", "allocations" },
-	{ "fetch_chunksize",
-		tweak_uint, &master.fetch_chunksize, 4, UINT_MAX / 1024.,
-		"The default chunksize used by fetcher. "
-		"This should be bigger than the majority of objects with "
-		"short TTLs.\n"
-		"Internal limits in the storage_file module makes increases "
-		"above 128kb a dubious idea.",
-		EXPERIMENTAL,
-		"128", "kilobytes" },
-	{ "fetch_maxchunksize",
-		tweak_uint, &master.fetch_maxchunksize, 64, UINT_MAX / 1024.,
-		"The maximum chunksize we attempt to allocate from storage. "
-		"Making this too large may cause delays and storage "
-		"fragmentation.\n",
-		EXPERIMENTAL,
-		"262144", "kilobytes" },
-#ifdef SENDFILE_WORKS
-	{ "sendfile_threshold",
-		tweak_uint, &master.sendfile_threshold, 0, UINT_MAX,
-		"The minimum size of objects transmitted with sendfile.",
-		EXPERIMENTAL,
-		"-1", "bytes" },
-#endif /* SENDFILE_WORKS */
-	{ "vcl_trace", tweak_bool,  &master.vcl_trace, 0, 0,
-		"Trace VCL execution in the shmlog.\n"
-		"Enabling this will allow you to see the path each "
-		"request has taken through the VCL program.\n"
-		"This generates a lot of logrecords so it is off by "
-		"default.",
-		0,
-		"off", "bool" },
-	{ "listen_address", tweak_listen_address, NULL, 0, 0,
-		"Whitespace separated list of network endpoints where "
-		"Varnish will accept requests.\n"
-		"Possible formats: host, host:port, :port",
-		MUST_RESTART,
-		":80" },
-	{ "listen_depth", tweak_uint, &master.listen_depth, 0, UINT_MAX,
-		"Listen queue depth.",
-		MUST_RESTART,
-		"1024", "connections" },
-	{ "cli_timeout", tweak_timeout, &master.cli_timeout, 0, 0,
-		"Timeout for the childs replies to CLI requests from "
-		"the master.",
-		0,
-		"10", "seconds" },
-	{ "ping_interval", tweak_uint, &master.ping_interval, 0, UINT_MAX,
-		"Interval between pings from parent to child.\n"
-		"Zero will disable pinging entirely, which makes "
-		"it possible to attach a debugger to the child.",
-		MUST_RESTART,
-		"3", "seconds" },
-	{ "lru_interval", tweak_timeout, &master.lru_timeout, 0, 0,
-		"Grace period before object moves on LRU list.\n"
-		"Objects are only moved to the front of the LRU "
-		"list if they have not been moved there already inside "
-		"this timeout period.  This reduces the amount of lock "
-		"operations necessary for LRU list access.",
-		EXPERIMENTAL,
-		"2", "seconds" },
-	{ "cc_command", tweak_string, &mgt_cc_cmd, 0, 0,
-		"Command used for compiling the C source code to a "
-		"dlopen(3) loadable object.  Any occurrence of %s in "
-		"the string will be replaced with the source file name, "
-		"and %o will be replaced with the output file name.",
-		MUST_RELOAD,
-		VCC_CC , NULL },
-	{ "max_restarts", tweak_uint, &master.max_restarts, 0, UINT_MAX,
-		"Upper limit on how many times a request can restart."
-		"\nBe aware that restarts are likely to cause a hit against "
-		"the backend, so don't increase thoughtlessly.\n",
-		0,
-		"4", "restarts" },
-	{ "esi_syntax",
-		tweak_uint, &master.esi_syntax, 0, UINT_MAX,
-		"Bitmap controlling ESI parsing code:\n"
-		"  0x00000001 - Don't check if it looks like XML\n"
-		"  0x00000002 - Ignore non-esi elements\n"
-		"  0x00000004 - Emit parsing debug records\n"
-		"  0x00000008 - Force-split parser input (debugging)\n"
-		"Use 0x notation and do the bitor in your head :-)\n",
-		0,
-		"0", "bitmap" },
-	{ "max_esi_depth",
-		tweak_uint, &master.max_esi_depth, 0, UINT_MAX,
-		"Maximum depth of esi:include processing.\n",
-		0,
-		"5", "levels" },
-	{ "connect_timeout", tweak_timeout_double,
-		&master.connect_timeout,0, UINT_MAX,
-		"Default connection timeout for backend connections. "
-		"We only try to connect to the backend for this many "
-		"seconds before giving up. "
-		"VCL can override this default value for each backend and "
-		"backend request.",
-		0,
-		"0.7", "s" },
-	{ "first_byte_timeout", tweak_timeout_double,
-		&master.first_byte_timeout,0, UINT_MAX,
-		"Default timeout for receiving first byte from backend. "
-		"We only wait for this many seconds for the first "
-		"byte before giving up. A value of 0 means it will never time "
-		"out. "
-		"VCL can override this default value for each backend and "
-		"backend request. This parameter does not apply to pipe.",
-		0,
-		"60", "s" },
-	{ "between_bytes_timeout", tweak_timeout_double,
-		&master.between_bytes_timeout,0, UINT_MAX,
-		"Default timeout between bytes when receiving data from "
-		"backend. "
-		"We only wait for this many seconds between bytes "
-		"before giving up. A value of 0 means it will never time out. "
-		"VCL can override this default value for each backend request "
-		"and backend request. This parameter does not apply to pipe.",
-		0,
-		"60", "s" },
-	{ "acceptor_sleep_max", tweak_timeout_double,
-		&master.acceptor_sleep_max, 0,  10,
-		"If we run out of resources, such as file descriptors or "
-		"worker threads, the acceptor will sleep between accepts.\n"
-		"This parameter limits how long it can sleep between "
-		"attempts to accept new connections.",
-		EXPERIMENTAL,
-		"0.050", "s" },
-	{ "acceptor_sleep_incr", tweak_timeout_double,
-		&master.acceptor_sleep_incr, 0,  1,
-		"If we run out of resources, such as file descriptors or "
-		"worker threads, the acceptor will sleep between accepts.\n"
-		"This parameter control how much longer we sleep, each time "
-		"we fail to accept a new connection.",
-		EXPERIMENTAL,
-		"0.001", "s" },
-	{ "acceptor_sleep_decay", tweak_generic_double,
-		&master.acceptor_sleep_decay, 0,  1,
-		"If we run out of resources, such as file descriptors or "
-		"worker threads, the acceptor will sleep between accepts.\n"
-		"This parameter (multiplicatively) reduce the sleep duration "
-		"for each succesfull accept. (ie: 0.9 = reduce by 10%)",
-		EXPERIMENTAL,
-		"0.900", "" },
-	{ "clock_skew", tweak_uint, &master.clock_skew, 0, UINT_MAX,
-		"How much clockskew we are willing to accept between the "
-		"backend and our own clock.",
-		0,
-		"10", "s" },
-	{ "prefer_ipv6", tweak_bool, &master.prefer_ipv6, 0, 0,
-		"Prefer IPv6 address when connecting to backends which "
-		"have both IPv4 and IPv6 addresses.",
-		0,
-		"off", "bool" },
-	{ "session_max", tweak_uint,
-		&master.max_sess, 1000, UINT_MAX,
-		"Maximum number of sessions we will allocate from one pool "
-		"before just dropping connections.\n"
-		"This is mostly an anti-DoS measure, and setting it plenty "
-		"high should not hurt, as long as you have the memory for "
-		"it.\n",
-		0,
-		"100000", "sessions" },
-	{ "session_linger", tweak_uint,
-		&master.session_linger,0, UINT_MAX,
-		"How long time the workerthread lingers on the session "
-		"to see if a new request appears right away.\n"
-		"If sessions are reused, as much as half of all reuses "
-		"happen within the first 100 msec of the previous request "
-		"completing.\n"
-		"Setting this too high results in worker threads not doing "
-		"anything for their keep, setting it too low just means that "
-		"more sessions take a detour around the waiter.",
-		EXPERIMENTAL,
-		"50", "ms" },
-	{ "cli_buffer", tweak_uint, &master.cli_buffer, 4096, UINT_MAX,
-		"Size of buffer for CLI input."
-		"\nYou may need to increase this if you have big VCL files "
-		"and use the vcl.inline CLI command.\n"
-		"NB: Must be specified with -p to have effect.\n",
-		0,
-		"8192", "bytes" },
-	{ "log_hashstring", tweak_bool, &master.log_hash, 0, 0,
-		"Log the hash string components to shared memory log.\n",
-		0,
-		"on", "bool" },
-	{ "log_local_address", tweak_bool, &master.log_local_addr, 0, 0,
-		"Log the local address on the TCP connection in the "
-		"SessionOpen shared memory record.\n",
-		0,
-		"off", "bool" },
-	{ "waiter", tweak_waiter, NULL, 0, 0,
-		"Select the waiter kernel interface.\n",
-		EXPERIMENTAL | MUST_RESTART,
-		"default", NULL },
-	{ "diag_bitmap", tweak_diag_bitmap, 0, 0, 0,
-		"Bitmap controlling diagnostics code:\n"
-		"  0x00000001 - CNT_Session states.\n"
-		"  0x00000002 - workspace debugging.\n"
-		"  0x00000004 - kqueue debugging.\n"
-		"  0x00000008 - mutex logging.\n"
-		"  0x00000010 - mutex contests.\n"
-		"  0x00000020 - waiting list.\n"
-		"  0x00000040 - object workspace.\n"
-		"  0x00001000 - do not core-dump child process.\n"
-		"  0x00002000 - only short panic message.\n"
-		"  0x00004000 - panic to stderr.\n"
-#ifdef HAVE_ABORT2
-		"  0x00008000 - panic to abort2().\n"
-#endif
-		"  0x00010000 - synchronize shmlog.\n"
-		"  0x00020000 - synchronous start of persistence.\n"
-		"  0x00040000 - release VCL early.\n"
-		"  0x80000000 - do edge-detection on digest.\n"
-		"Use 0x notation and do the bitor in your head :-)\n",
-		0,
-		"0", "bitmap" },
-	{ "ban_dups", tweak_bool, &master.ban_dups, 0, 0,
-		"Detect and eliminate duplicate bans.\n",
-		0,
-		"on", "bool" },
-	{ "syslog_cli_traffic", tweak_bool, &master.syslog_cli_traffic, 0, 0,
-		"Log all CLI traffic to syslog(LOG_INFO).\n",
-		0,
-		"on", "bool" },
-	{ "ban_lurker_sleep", tweak_timeout_double,
-		&master.ban_lurker_sleep, 0, UINT_MAX,
-		"How long time does the ban lurker thread sleeps between "
-		"successful attempts to push the last item up the ban "
-		" list.  It always sleeps a second when nothing can be done.\n"
-		"A value of zero disables the ban lurker.",
-		0,
-		"0.01", "s" },
-	{ "saintmode_threshold", tweak_uint,
-		&master.saintmode_threshold, 0, UINT_MAX,
-		"The maximum number of objects held off by saint mode before "
-		"no further will be made to the backend until one times out.  "
-		"A value of 0 disables saintmode.",
-		EXPERIMENTAL,
-		"10", "objects" },
-	{ "http_range_support", tweak_bool, &master.http_range_support, 0, 0,
-		"Enable support for HTTP Range headers.\n",
-		EXPERIMENTAL,
-		"on", "bool" },
-	{ "http_gzip_support", tweak_bool, &master.http_gzip_support, 0, 0,
-		"Enable gzip support. When enabled Varnish will compress "
-		"uncompressed objects before they are stored in the cache. "
-		"If a client does not support gzip encoding Varnish will "
-		"uncompress compressed objects on demand. Varnish will also "
-		"rewrite the Accept-Encoding header of clients indicating "
-		"support for gzip to:\n"
-		"Accept-Encoding: gzip\n\n"
-		"Clients that do not support gzip will have their "
-		"Accept-Encoding header removed. For more information on how "
-		"gzip is implemented please see the chapter on gzip in the "
-		"Varnish reference.",
-		EXPERIMENTAL,
-		"on", "bool" },
-	{ "gzip_tmp_space", tweak_uint, &master.gzip_tmp_space, 0, 2,
-		"Where temporary space for gzip/gunzip is allocated:\n"
-		"  0 - malloc\n"
-		"  1 - session workspace\n"
-		"  2 - thread workspace\n"
-		"If you have much gzip/gunzip activity, it may be an"
-		" advantage to use workspace for these allocations to reduce"
-		" malloc activity.  Be aware that gzip needs 256+KB and gunzip"
-		" needs 32+KB of workspace (64+KB if ESI processing).",
-		EXPERIMENTAL,
-		"0", "" },
-	{ "gzip_level", tweak_uint, &master.gzip_level, 0, 9,
-		"Gzip compression level: 0=debug, 1=fast, 9=best",
-		0,
-		"6", ""},
-	{ "gzip_window", tweak_uint, &master.gzip_window, 8, 15,
-		"Gzip window size 8=least, 15=most compression.\n"
-		"Memory impact is 8=1k, 9=2k, ... 15=128k.",
-		0,
-		"15", ""},
-	{ "gzip_memlevel", tweak_uint, &master.gzip_memlevel, 1, 9,
-		"Gzip memory level 1=slow/least, 9=fast/most compression.\n"
-		"Memory impact is 1=1k, 2=2k, ... 9=256k.",
-		0,
-		"8", ""},
-	{ "gzip_stack_buffer", tweak_uint, &master.gzip_stack_buffer,
-	        2048, UINT_MAX,
-		"Size of stack buffer used for gzip processing.\n"
-		"The stack buffers are used for in-transit data,"
-		" for instance gunzip'ed data being sent to a client."
-		"Making this space to small results in more overhead,"
-		" writes to sockets etc, making it too big is probably"
-		" just a waste of memory.",
-		EXPERIMENTAL,
-		"32768", "Bytes" },
-	{ "shortlived", tweak_timeout_double,
-		&master.shortlived, 0, UINT_MAX,
-		"Objects created with TTL shorter than this are always "
-		"put in transient storage.\n",
-		0,
-		"10.0", "s" },
-	{ "critbit_cooloff", tweak_timeout_double,
-		&master.critbit_cooloff, 60, 254,
-		"How long time the critbit hasher keeps deleted objheads "
-		"on the cooloff list.\n",
-		WIZARD,
-		"180.0", "s" },
-	{ "vcl_dir", tweak_string, &mgt_vcl_dir, 0, 0,
-		"Directory from which relative VCL filenames (vcl.load and "
-		"include) are opened.",
-		0,
-#ifdef VARNISH_VCL_DIR
-		VARNISH_VCL_DIR,
-#else
-		".",
-#endif
-		NULL },
-	{ "vmod_dir", tweak_string, &mgt_vmod_dir, 0, 0,
-		"Directory where VCL modules are to be found.",
-		0,
-#ifdef VARNISH_VMOD_DIR
-		VARNISH_VMOD_DIR,
-#else
-		".",
-#endif
-		NULL },
-	{ "vcc_err_unref", tweak_bool, &mgt_vcc_err_unref, 0, 0,
-		"Unreferenced VCL objects result in error.\n",
-		0,
-		"on", "bool" },
-
-
-	{ "pcre_match_limit", tweak_uint,
-		&master.vre_limits.match,
-		1, UINT_MAX,
-		"The limit for the  number of internal matching function"
-		" calls in a pcre_exec() execution.",
-		0,
-		"10000", ""},
-
-	{ "pcre_match_limit_recursion", tweak_uint,
-		&master.vre_limits.match_recursion,
-		1, UINT_MAX,
-		"The limit for the  number of internal matching function"
-		" recursions in a pcre_exec() execution.",
-		0,
-		"10000", ""},
-
-	{ NULL, NULL, NULL }
-};
-
-/*--------------------------------------------------------------------*/
-
-#define WIDTH 76
-
-static void
-mcf_wrap(struct cli *cli, const char *text)
-{
-	const char *p, *q;
-
-	/* Format text to COLUMNS width */
-	for (p = text; *p != '\0'; ) {
-		q = strchr(p, '\n');
-		if (q == NULL)
-			q = strchr(p, '\0');
-		if (q > p + WIDTH - margin) {
-			q = p + WIDTH - margin;
-			while (q > p && *q != ' ')
-				q--;
-			AN(q);
-		}
-		VCLI_Out(cli, "%*s %.*s\n", margin, "", (int)(q - p), p);
-		p = q;
-		if (*p == ' ' || *p == '\n')
-			p++;
-	}
-}
-
-void
-mcf_param_show(struct cli *cli, const char * const *av, void *priv)
-{
-	int i;
-	const struct parspec *pp;
-	int lfmt;
-
-	(void)priv;
-	if (av[2] == NULL || strcmp(av[2], "-l"))
-		lfmt = 0;
-	else
-		lfmt = 1;
-	for (i = 0; i < nparspec; i++) {
-		pp = parspec[i];
-		if (av[2] != NULL && !lfmt && strcmp(pp->name, av[2]))
-			continue;
-		VCLI_Out(cli, "%-*s ", margin, pp->name);
-		if (pp->func == NULL) {
-			VCLI_Out(cli, "Not implemented.\n");
-			if (av[2] != NULL && !lfmt)
-				return;
-			else
-				continue;
-		}
-		pp->func(cli, pp, NULL);
-		if (pp->units != NULL)
-			VCLI_Out(cli, " [%s]\n", pp->units);
-		else
-			VCLI_Out(cli, "\n");
-		if (av[2] != NULL) {
-			VCLI_Out(cli, "%-*s Default is %s\n",
-			    margin, "", pp->def);
-			mcf_wrap(cli, pp->descr);
-			if (pp->flags & DELAYED_EFFECT)
-				mcf_wrap(cli, DELAYED_EFFECT_TEXT);
-			if (pp->flags & EXPERIMENTAL)
-				mcf_wrap(cli, EXPERIMENTAL_TEXT);
-			if (pp->flags & MUST_RELOAD)
-				mcf_wrap(cli, MUST_RELOAD_TEXT);
-			if (pp->flags & MUST_RESTART)
-				mcf_wrap(cli, MUST_RESTART_TEXT);
-			if (pp->flags & WIZARD)
-				mcf_wrap(cli, WIZARD_TEXT);
-			if (!lfmt)
-				return;
-			else
-				VCLI_Out(cli, "\n");
-		}
-	}
-	if (av[2] != NULL && !lfmt) {
-		VCLI_SetResult(cli, CLIS_PARAM);
-		VCLI_Out(cli, "Unknown parameter \"%s\".", av[2]);
-	}
-}
-
-/*--------------------------------------------------------------------*/
-
-void
-MCF_ParamSync(void)
-{
-	if (params != &master)
-		*params = master;
-}
-
-/*--------------------------------------------------------------------*/
-
-void
-MCF_ParamSet(struct cli *cli, const char *param, const char *val)
-{
-	const struct parspec *pp;
-
-	pp = mcf_findpar(param);
-	if (pp != NULL) {
-		pp->func(cli, pp, val);
-		if (cli->result != CLIS_OK) {
-			VCLI_Out(cli, "(attempting to set param %s to %s)\n",
-			    pp->name, val);
-		} else if (child_pid >= 0 && pp->flags & MUST_RESTART) {
-			VCLI_Out(cli, "Change will take effect"
-			    " when child is restarted");
-		} else if (pp->flags & MUST_RELOAD) {
-			VCLI_Out(cli, "Change will take effect"
-			    " when VCL script is reloaded");
-		}
-		MCF_ParamSync();
-		return;
-	}
-	VCLI_SetResult(cli, CLIS_PARAM);
-	VCLI_Out(cli, "Unknown parameter \"%s\".", param);
-}
-
-
-/*--------------------------------------------------------------------*/
-
-void
-mcf_param_set(struct cli *cli, const char * const *av, void *priv)
-{
-
-	(void)priv;
-	MCF_ParamSet(cli, av[2], av[3]);
-}
-
-/*--------------------------------------------------------------------
- * Add a group of parameters to the global set and sort by name.
- */
-
-static int
-parspec_cmp(const void *a, const void *b)
-{
-	struct parspec * const * pa = a;
-	struct parspec * const * pb = b;
-	return (strcmp((*pa)->name, (*pb)->name));
-}
-
-static void
-MCF_AddParams(const struct parspec *ps)
-{
-	const struct parspec *pp;
-	int n;
-
-	n = 0;
-	for (pp = ps; pp->name != NULL; pp++) {
-		if (mcf_findpar(pp->name) != NULL)
-			fprintf(stderr, "Duplicate param: %s\n", pp->name);
-		if (strlen(pp->name) + 1 > margin)
-			margin = strlen(pp->name) + 1;
-		n++;
-	}
-	parspec = realloc(parspec, (1L + nparspec + n) * sizeof *parspec);
-	XXXAN(parspec);
-	for (pp = ps; pp->name != NULL; pp++)
-		parspec[nparspec++] = pp;
-	parspec[nparspec] = NULL;
-	qsort (parspec, nparspec, sizeof parspec[0], parspec_cmp);
-}
-
-/*--------------------------------------------------------------------
- * Set defaults for all parameters
- */
-
-static void
-MCF_SetDefaults(struct cli *cli)
-{
-	const struct parspec *pp;
-	int i;
-
-	for (i = 0; i < nparspec; i++) {
-		pp = parspec[i];
-		if (cli != NULL)
-			VCLI_Out(cli,
-			    "Set Default for %s = %s\n", pp->name, pp->def);
-		pp->func(cli, pp, pp->def);
-		if (cli != NULL && cli->result != CLIS_OK)
-			return;
-	}
-}
-
-/*--------------------------------------------------------------------*/
-
-void
-MCF_ParamInit(struct cli *cli)
-{
-
-	MCF_AddParams(input_parspec);
-	MCF_AddParams(WRK_parspec);
-
-	/* XXX: We do this twice, to get past any interdependencies */
-	MCF_SetDefaults(NULL);
-	MCF_SetDefaults(cli);
-
-	params = &master;
-}
-
-/*--------------------------------------------------------------------*/
-
-void
-MCF_DumpRst(void)
-{
-	const struct parspec *pp;
-	const char *p, *q;
-	int i;
-
-	for (i = 0; i < nparspec; i++) {
-		pp = parspec[i];
-		printf("%s\n", pp->name);
-		if (pp->units != NULL && *pp->units != '\0')
-			printf("\t- Units: %s\n", pp->units);
-		printf("\t- Default: %s\n",
-		    strcmp(pp->def,MAGIC_INIT_STRING) == 0 ? "magic" : pp->def);
-		/*
-		 * XXX: we should mark the params with one/two flags
-		 * XXX: that say if ->min/->max are valid, so we
-		 * XXX: can emit those also in help texts.
-		 */
-		if (pp->flags) {
-			printf("\t- Flags: ");
-			q = "";
-			if (pp->flags & DELAYED_EFFECT) {
-				printf("%sdelayed", q);
-				q = ", ";
-			}
-			if (pp->flags & MUST_RESTART) {
-				printf("%smust_restart", q);
-				q = ", ";
-			}
-			if (pp->flags & MUST_RELOAD) {
-				printf("%smust_reload", q);
-				q = ", ";
-			}
-			if (pp->flags & EXPERIMENTAL) {
-				printf("%sexperimental", q);
-				q = ", ";
-			}
-			printf("\n");
-		}
-		printf("\n\t");
-		for (p = pp->descr; *p; p++) {
-			if (*p == '\n' && p[1] =='\0')
-				break;
-			if (*p == '\n' && p[1] =='\n') {
-				printf("\n\n\t");
-				p++;
-			} else if (*p == '\n') {
-				printf("\n\t");
-			} else if (*p == ':' && p[1] == '\n') {
-				/*
-				 * Start of definition list,
-				 * use RSTs code mode for this
-				 */
-				printf("::\n");
-			} else {
-				printf("%c", *p);
-			}
-		}
-		printf("\n\n");
-	}
-	printf("\n");
-}
diff --git a/bin/varnishd/mgt_pool.c b/bin/varnishd/mgt_pool.c
deleted file mode 100644
index 489fab8..0000000
--- a/bin/varnishd/mgt_pool.c
+++ /dev/null
@@ -1,238 +0,0 @@
-/*-
- * Copyright (c) 2006 Verdens Gang AS
- * Copyright (c) 2006-2011 Varnish Software AS
- * All rights reserved.
- *
- * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- *
- * We maintain a number of worker thread pools, to spread lock contention.
- *
- * Pools can be added on the fly, as a means to mitigate lock contention,
- * but can only be removed again by a restart. (XXX: we could fix that)
- *
- * Two threads herd the pools, one eliminates idle threads and aggregates
- * statistics for all the pools, the other thread creates new threads
- * on demand, subject to various numerical constraints.
- *
- * The algorithm for when to create threads needs to be reactive enough
- * to handle startup spikes, but sufficiently attenuated to not cause
- * thread pileups.  This remains subject for improvement.
- */
-
-#include "config.h"
-
-#include <limits.h>
-#include <stdio.h>
-#include <string.h>
-#include <unistd.h>
-
-#include "mgt.h"
-
-#include "heritage.h"
-#include "vparam.h"
-
-/*--------------------------------------------------------------------*/
-
-static void
-tweak_thread_pool_min(struct cli *cli, const struct parspec *par,
-    const char *arg)
-{
-
-	tweak_generic_uint(cli, &master.wthread_min, arg,
-	    (unsigned)par->min, master.wthread_max);
-}
-
-/*--------------------------------------------------------------------
- * This is utterly ridiculous:  POSIX does not guarantee that the
- * minimum thread stack size is a compile time constant.
- * XXX: "32" is a magic marker for 32bit systems.
- */
-
-static void
-tweak_stack_size(struct cli *cli, const struct parspec *par,
-    const char *arg)
-{
-	unsigned low, u;
-	char buf[12];
-
-	low = sysconf(_SC_THREAD_STACK_MIN);
-
-	if (arg != NULL && !strcmp(arg, "32bit")) {
-		u = 65536;
-		if (u < low)
-			u = low;
-		sprintf(buf, "%u", u);
-		arg = buf;
-	}
-
-	tweak_generic_uint(cli, &master.wthread_stacksize, arg,
-	    low, (uint)par->max);
-}
-
-/*--------------------------------------------------------------------*/
-
-static void
-tweak_thread_pool_max(struct cli *cli, const struct parspec *par,
-    const char *arg)
-{
-
-	(void)par;
-	tweak_generic_uint(cli, &master.wthread_max, arg,
-	    master.wthread_min, UINT_MAX);
-}
-
-/*--------------------------------------------------------------------*/
-
-const struct parspec WRK_parspec[] = {
-	{ "thread_pools", tweak_uint, &master.wthread_pools, 1, UINT_MAX,
-		"Number of worker thread pools.\n"
-		"\n"
-		"Increasing number of worker pools decreases lock "
-		"contention.\n"
-		"\n"
-		"Too many pools waste CPU and RAM resources, and more than "
-		"one pool for each CPU is probably detrimal to performance.\n"
-		"\n"
-		"Can be increased on the fly, but decreases require a "
-		"restart to take effect.",
-		EXPERIMENTAL | DELAYED_EFFECT,
-		"2", "pools" },
-	{ "thread_pool_max", tweak_thread_pool_max, NULL, 1, 0,
-		"The maximum number of worker threads in each pool.\n"
-		"\n"
-		"Do not set this higher than you have to, since excess "
-		"worker threads soak up RAM and CPU and generally just get "
-		"in the way of getting work done.\n",
-		EXPERIMENTAL | DELAYED_EFFECT,
-		"500", "threads" },
-	{ "thread_pool_min", tweak_thread_pool_min, NULL, 2, 0,
-		"The minimum number of worker threads in each pool.\n"
-		"\n"
-		"Increasing this may help ramp up faster from low load "
-		"situations where threads have expired.\n"
-		"\n"
-		"Minimum is 2 threads.",
-		EXPERIMENTAL | DELAYED_EFFECT,
-		"5", "threads" },
-	{ "thread_pool_timeout", tweak_timeout, &master.wthread_timeout, 1, 0,
-		"Thread idle threshold.\n"
-		"\n"
-		"Threads in excess of thread_pool_min, which have been idle "
-		"for at least this long are candidates for purging.\n"
-		"\n"
-		"Minimum is 1 second.",
-		EXPERIMENTAL | DELAYED_EFFECT,
-		"300", "seconds" },
-	{ "thread_pool_purge_delay",
-		tweak_timeout, &master.wthread_purge_delay, 100, 0,
-		"Wait this long between purging threads.\n"
-		"\n"
-		"This controls the decay of thread pools when idle(-ish).\n"
-		"\n"
-		"Minimum is 100 milliseconds.",
-		EXPERIMENTAL | DELAYED_EFFECT,
-		"1000", "milliseconds" },
-	{ "thread_pool_add_threshold",
-		tweak_uint, &master.wthread_add_threshold, 0, UINT_MAX,
-		"Overflow threshold for worker thread creation.\n"
-		"\n"
-		"Setting this too low, will result in excess worker threads, "
-		"which is generally a bad idea.\n"
-		"\n"
-		"Setting it too high results in insuffient worker threads.\n",
-		EXPERIMENTAL,
-		"2", "requests" },
-	{ "thread_pool_add_delay",
-		tweak_timeout, &master.wthread_add_delay, 0, UINT_MAX,
-		"Wait at least this long between creating threads.\n"
-		"\n"
-		"Setting this too long results in insuffient worker threads.\n"
-		"\n"
-		"Setting this too short increases the risk of worker "
-		"thread pile-up.\n",
-		0,
-		"2", "milliseconds" },
-	{ "thread_pool_fail_delay",
-		tweak_timeout, &master.wthread_fail_delay, 100, UINT_MAX,
-		"Wait at least this long after a failed thread creation "
-		"before trying to create another thread.\n"
-		"\n"
-		"Failure to create a worker thread is often a sign that "
-		" the end is near, because the process is running out of "
-		"RAM resources for thread stacks.\n"
-		"This delay tries to not rush it on needlessly.\n"
-		"\n"
-		"If thread creation failures are a problem, check that "
-		"thread_pool_max is not too high.\n"
-		"\n"
-		"It may also help to increase thread_pool_timeout and "
-		"thread_pool_min, to reduce the rate at which treads are "
-		"destroyed and later recreated.\n",
-		EXPERIMENTAL,
-		"200", "milliseconds" },
-	{ "thread_stats_rate",
-		tweak_uint, &master.wthread_stats_rate, 0, UINT_MAX,
-		"Worker threads accumulate statistics, and dump these into "
-		"the global stats counters if the lock is free when they "
-		"finish a request.\n"
-		"This parameters defines the maximum number of requests "
-		"a worker thread may handle, before it is forced to dump "
-		"its accumulated stats into the global counters.\n",
-		EXPERIMENTAL,
-		"10", "requests" },
-	{ "queue_max", tweak_uint, &master.queue_max, 0, UINT_MAX,
-		"Percentage permitted queue length.\n"
-		"\n"
-		"This sets the ratio of queued requests to worker threads, "
-		"above which sessions will be dropped instead of queued.\n",
-		EXPERIMENTAL,
-		"100", "%" },
-	{ "rush_exponent", tweak_uint, &master.rush_exponent, 2, UINT_MAX,
-		"How many parked request we start for each completed "
-		"request on the object.\n"
-		"NB: Even with the implict delay of delivery, "
-		"this parameter controls an exponential increase in "
-		"number of worker threads.",
-		EXPERIMENTAL,
-		"3", "requests per request" },
-	{ "thread_pool_stack",
-		tweak_stack_size, &master.wthread_stacksize, 0, UINT_MAX,
-		"Worker thread stack size.\n"
-		"On 32bit systems you may need to tweak this down to fit "
-		"many threads into the limited address space.\n",
-		EXPERIMENTAL,
-		"-1", "bytes" },
-	{ "thread_pool_workspace", tweak_uint, &master.wthread_workspace,
-		1024, UINT_MAX,
-		"Bytes of HTTP protocol workspace allocated for worker "
-		"threads. "
-		"This space must be big enough for the backend request "
-		"and responses, and response to the client plus any other "
-		"memory needs in the VCL code."
-		"Minimum is 1024 bytes.",
-		DELAYED_EFFECT,
-		"65536",
-		"bytes" },
-	{ NULL, NULL, NULL }
-};
diff --git a/bin/varnishd/mgt_sandbox.c b/bin/varnishd/mgt_sandbox.c
deleted file mode 100644
index 8ac827d..0000000
--- a/bin/varnishd/mgt_sandbox.c
+++ /dev/null
@@ -1,91 +0,0 @@
-/*-
- * Copyright (c) 2006-2011 Varnish Software AS
- * All rights reserved.
- *
- * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- *
- * Sandboxing child processes
- *
- * The worker/manager process border is one of the major security barriers
- * in Varnish, and therefore subject to whatever restrictions we have access
- * to under the given operating system.
- *
- * Unfortunately there is no consensus on APIs for this purpose, so each
- * operating system will require its own methods.
- *
- * This sourcefile tries to encapsulate the resulting mess on place.
- *
- * TODO:
- *	Unix:	chroot
- *	FreeBSD: jail
- *	FreeBSD: capsicum
- */
-
-#include "config.h"
-
-#ifdef __linux__
-#include <sys/prctl.h>
-#endif
-
-#include <stdio.h>
-#include <syslog.h>
-#include <unistd.h>
-
-#include "mgt.h"
-
-#include "heritage.h"
-
-/*--------------------------------------------------------------------*/
-
-/* Waive all privileges in the child, it does not need any */
-
-void
-mgt_sandbox(void)
-{
-#ifdef HAVE_SETPPRIV
-	mgt_sandbox_solaris_init();
-	mgt_sandbox_solaris_privsep();
-#else
-	if (geteuid() == 0) {
-		XXXAZ(setgid(params->gid));
-		XXXAZ(setuid(params->uid));
-	} else {
-		REPORT0(LOG_INFO, "Not running as root, no priv-sep");
-	}
-#endif
-
-	/* On Linux >= 2.4, you need to set the dumpable flag
-	   to get core dumps after you have done a setuid. */
-
-#ifdef __linux__
-	if (prctl(PR_SET_DUMPABLE, 1) != 0)
-		REPORT0(LOG_INFO,
-		    "Could not set dumpable bit.  Core dumps turned off\n");
-#endif
-
-#ifdef HAVE_SETPPRIV
-	mgt_sandbox_solaris_fini();
-#endif
-
-}
diff --git a/bin/varnishd/mgt_sandbox_solaris.c b/bin/varnishd/mgt_sandbox_solaris.c
deleted file mode 100644
index 534f609..0000000
--- a/bin/varnishd/mgt_sandbox_solaris.c
+++ /dev/null
@@ -1,233 +0,0 @@
-/*-
- * Copyright (c) 2006-2011 Varnish Software AS
- * All rights reserved.
- *
- * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
- *	   Nils Goroll <nils.goroll at uplex.de>
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- *
- * Sandboxing child processes on Solaris
- *
- */
-
-#include "config.h"
-
-#ifdef HAVE_SETPPRIV
-
-#ifdef HAVE_PRIV_H
-#include <priv.h>
-#endif
-#include <stdio.h>
-#include <string.h>
-#include <syslog.h>
-#include <unistd.h>
-
-#include "mgt.h"
-
-#include "heritage.h"
-
-/*--------------------------------------------------------------------
- * SOLARIS PRIVILEGES: Note on use of symbolic PRIV_* constants
- * 
- * For privileges which existed in Solaris 10 FCS, we may use the constants from
- * sys/priv_names.h
- *
- * For privileges which have been added later, we need to use strings in order
- * not to break builds of varnish on these platforms. To remain binary
- * compatible, we need to silently ignore errors from priv_addset when using
- * these strings.
- *
- * For optimal build and binary forward comatibility, we could use subtractive
- * set specs like
- *
- *       basic,!file_link_any,!proc_exec,!proc_fork,!proc_info,!proc_session
- *
- * but I (Nils) have a preference for making an informed decision about which
- * privileges the varnish child should have and which it shouldn't.
- *
- * Newly introduced privileges should be annotated with their PSARC / commit ID
- * (as long as Oracle reveils these :/ )
- *
- * SOLARIS PRIVILEGES: Note on accidentally setting the SNOCD flag
- *
- * When setting privileges, we need to take care not to accidentally set the
- * SNOCD flag which will disable core dumps unnecessarily. (see
- * https://www.varnish-cache.org/trac/ticket/671 )
- *
- * When changing the logic herein, always check with mdb -k. Replace _PID_ with
- * the pid of your varnish child, the result should be 0, otherwise a regression
- * has been introduced.
- *
- * > 0t_PID_::pid2proc | ::print proc_t p_flag | >a
- * > (<a & 0x10000000)=X
- *                 0
- *
- * (a value of 0x10000000 indicates that SNOCD is set)
- *
- * NOTE that on Solaris changing the uid will _always_ set SNOCD, so make sure
- * you run this test with appropriate privileges, but without proc_setid, so
- * varnish won't setuid(), e.g.
- *
- * pfexec ppriv -e -s A=basic,net_privaddr,sys_resource varnish ...
- *
- * SOLARIS COREDUMPS with setuid(): See coreadm(1M) - global-setid / proc-setid
- *
- */
-
-/* effective during runtime of the child */
-static inline void
-mgt_sandbox_solaris_add_effective(priv_set_t *pset)
-{
-	/* PSARC/2009/685 - 8eca52188202 - onnv_132 */
-	priv_addset(pset, "net_access");
-
-	/* PSARC/2009/378 - 63678502e95e - onnv_140 */
-	priv_addset(pset, "file_read");
-	priv_addset(pset, "file_write");
-}
-
-/* permitted during runtime of the child - for privilege bracketing */
-static inline void
-mgt_sandbox_solaris_add_permitted(priv_set_t *pset)
-{
-	/* for raising limits in cache_waiter_ports.c */
-	priv_addset(pset, PRIV_SYS_RESOURCE);
-}
-
-/* effective during mgt_sandbox */
-static inline void
-mgt_sandbox_solaris_add_initial(priv_set_t *pset)
-{
-	/* for setgid/setuid */
-	priv_addset(pset, PRIV_PROC_SETID);
-}
-
-/*
- * if we are not yet privilege-aware already (ie we have been started
- * not-privilege aware wird euid 0), we need to grab any additional privileges
- * needed during mgt_standbox, until we reduce to least privileges in
- * mgt_sandbox_waive, otherwise we would loose them with setuid()
- */
-
-void
-mgt_sandbox_solaris_init(void)
-{
-	priv_set_t *priv_all;
-
-	if (! (priv_all = priv_allocset())) {
-		REPORT(LOG_ERR,
-		    "Child start warning: mgt_sandbox_init - priv_allocset failed: errno=%d (%s)",
-		    errno, strerror(errno));
-		return;
-	}
-	
-	priv_emptyset(priv_all);
-
-	mgt_sandbox_solaris_add_effective(priv_all);
-	mgt_sandbox_solaris_add_permitted(priv_all);
-	mgt_sandbox_solaris_add_initial(priv_all);
-
-	setppriv(PRIV_ON, PRIV_PERMITTED, priv_all);
-	setppriv(PRIV_ON, PRIV_EFFECTIVE, priv_all);
-	setppriv(PRIV_ON, PRIV_INHERITABLE, priv_all);
-
-	priv_freeset(priv_all);
-}
-
-void
-mgt_sandbox_solaris_privsep(void)
-{
-	if (priv_ineffect(PRIV_PROC_SETID)) {
-                if (getgid() != params->gid)
-                        XXXAZ(setgid(params->gid));
-                if (getuid() != params->uid)
-                        XXXAZ(setuid(params->uid));
-        } else {
-                REPORT(LOG_INFO, "Privilege %s missing, will not change uid/gid",
-		    PRIV_PROC_SETID);
-        }
-}
-
-/* 
- * Waive most privileges in the child
- *
- * as of onnv_151a, we should end up with:
- *
- * > ppriv -v #pid of varnish child
- * PID:  .../varnishd ...
- * flags = PRIV_AWARE
- *      E: file_read,file_write,net_access
- *      I: none
- *      P: file_read,file_write,net_access,sys_resource
- *      L: file_read,file_write,net_access,sys_resource
- *
- * We should keep sys_resource in P in order to adjust our limits if we need to
- */
-
-void
-mgt_sandbox_solaris_fini(void)
-{
-	priv_set_t *effective, *inheritable, *permitted;
-
-	if (!(effective = priv_allocset()) ||
-	    !(inheritable = priv_allocset()) ||
-	    !(permitted = priv_allocset())) {
-		REPORT(LOG_ERR,
-		    "Child start warning: mgt_sandbox_waive - priv_allocset failed: errno=%d (%s)",
-		    errno, strerror(errno));
-		return;
-	}
-
-	priv_emptyset(inheritable);
-
-	priv_emptyset(effective);
-	mgt_sandbox_solaris_add_effective(effective);
-
-	priv_copyset(effective, permitted);
-	mgt_sandbox_solaris_add_permitted(permitted);
-
-	/* 
-	 * invert the sets and clear privileges such that setppriv will always
-	 * succeed
-	 */
-	priv_inverse(inheritable);
-	priv_inverse(effective);
-	priv_inverse(permitted);
-
-#define SETPPRIV(which, set)						\
-	if (setppriv(PRIV_OFF, which, set))				\
-		REPORT(LOG_ERR,						\
-		    "Child start warning: Waiving privileges failed on %s: errno=%d (%s)", \
-		    #which, errno, strerror(errno));
-
-	SETPPRIV(PRIV_INHERITABLE, inheritable);
-	SETPPRIV(PRIV_EFFECTIVE, effective);
-	SETPPRIV(PRIV_PERMITTED, permitted);
-	SETPPRIV(PRIV_LIMIT, permitted);
-#undef SETPPRIV
-
-	priv_freeset(inheritable);
-	priv_freeset(effective);
-}
-
-#endif /* HAVE_SETPPRIV */
diff --git a/bin/varnishd/mgt_shmem.c b/bin/varnishd/mgt_shmem.c
deleted file mode 100644
index ae0a50d..0000000
--- a/bin/varnishd/mgt_shmem.c
+++ /dev/null
@@ -1,351 +0,0 @@
-/*-
- * Copyright (c) 2006 Verdens Gang AS
- * Copyright (c) 2006-2011 Varnish Software AS
- * All rights reserved.
- *
- * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- *
- *
- * TODO:
- *
- * There is a risk that the child process might corrupt the VSM segment
- * and we should capture that event and recover gracefully.
- *
- * A possible state diagram could be:
- *
- *	[manager start]
- *		|
- *		v
- *      Open old VSM,
- *	check pid	--------> exit/fail (-n message)
- *		|
- *		+<----------------------+
- *		|			^
- *		v			|
- *	Create new VSM			|
- *		|			|
- *		v			|
- *	Init header			|
- *	Alloc VSL			|
- *	Alloc VSC:Main			|
- *	Alloc Args etc.			|
- *		|			|
- *		+<--------------+	|
- *		|		^	|
- *		v		|	|
- *	start worker		|	|
- *		|		|	|
- *		|		|	+<---- worker crash
- *		v		|	^
- *	Reset VSL ptr.		|	|
- *	Reset VSC counters	|	|
- *		|		|	|
- *		+<------+	|	|
- *		|	^	|	|
- *		v	|	|	|
- *	alloc dynamics	|	|	|
- *	free dynamics	|	|	|
- *		|	|	|	|
- *		v	|	|	|
- *		+------>+	|	|
- *		|		|	|
- *		v		|	|
- *	stop worker		|	|
- *		|		|	|
- *		v		|	|
- *	Check consist---------- | ----->+
- *		|		|
- *		v		|
- *	Free dynamics		|
- *		|		|
- *		v		|
- *		+-------------->+
- *
- */
-
-#include "config.h"
-
-#include <sys/mman.h>
-#include <sys/stat.h>
-
-#include <fcntl.h>
-#include <signal.h>
-#include <stdint.h>
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#include <unistd.h>
-
-#include "mgt.h"
-
-#include "flopen.h"
-#include "heritage.h"
-#include "vapi/vsc_int.h"
-#include "vapi/vsl_int.h"
-#include "vapi/vsm_int.h"
-#include "vav.h"
-#include "vmb.h"
-#include "vnum.h"
-
-#ifndef MAP_HASSEMAPHORE
-#define MAP_HASSEMAPHORE 0 /* XXX Linux */
-#endif
-
-#ifndef MAP_NOSYNC
-#define MAP_NOSYNC 0 /* XXX Linux */
-#endif
-
-struct VSC_C_main	*VSC_C_main;
-
-static int vsl_fd = -1;
-
-/*--------------------------------------------------------------------
- * Check that we are not started with the same -n argument as an already
- * running varnishd
- */
-
-static void
-vsl_n_check(int fd)
-{
-	struct VSM_head slh;
-	int i;
-	struct stat st;
-	pid_t pid;
-
-	AZ(fstat(fd, &st));
-	if (!S_ISREG(st.st_mode))
-		ARGV_ERR("\tshmlog: Not a file\n");
-
-	/* Test if the SHMFILE is locked by other Varnish */
-	if (fltest(fd, &pid) > 0) {
-		fprintf(stderr,
-			"SHMFILE locked by running varnishd master (pid=%jd)\n",
-			(intmax_t)pid);
-		fprintf(stderr,
-			"(Use unique -n arguments if you want multiple "
-			"instances)\n");
-		exit(2);
-	}
-
-	/* Read owning pid from SHMFILE */
-	memset(&slh, 0, sizeof slh);	/* XXX: for flexelint */
-	i = read(fd, &slh, sizeof slh);
-	if (i != sizeof slh)
-		return;
-	if (slh.magic != VSM_HEAD_MAGIC)
-		return;
-	if (slh.hdrsize != sizeof slh)
-		return;
-	if (slh.master_pid != 0 && !kill(slh.master_pid, 0)) {
-		fprintf(stderr,
-			"WARNING: Taking over SHMFILE marked as owned by "
-			"running process (pid=%jd)\n",
-			(intmax_t)slh.master_pid);
-	}
-}
-
-/*--------------------------------------------------------------------
- * Build a new shmlog file
- */
-
-static void
-vsl_buildnew(const char *fn, unsigned size, int fill)
-{
-	struct VSM_head slh;
-	int i;
-	unsigned u;
-	char buf[64*1024];
-	int flags;
-
-	(void)unlink(fn);
-	vsl_fd = flopen(fn, O_RDWR | O_CREAT | O_EXCL | O_NONBLOCK, 0644);
-	if (vsl_fd < 0) {
-		fprintf(stderr, "Could not create %s: %s\n",
-		    fn, strerror(errno));
-		exit (1);
-	}
-	flags = fcntl(vsl_fd, F_GETFL);
-	assert(flags != -1);
-	flags &= ~O_NONBLOCK;
-	AZ(fcntl(vsl_fd, F_SETFL, flags));
-
-	memset(&slh, 0, sizeof slh);
-	slh.magic = VSM_HEAD_MAGIC;
-	slh.hdrsize = sizeof slh;
-	slh.shm_size = size;
-	i = write(vsl_fd, &slh, sizeof slh);
-	xxxassert(i == sizeof slh);
-
-	if (fill) {
-		memset(buf, 0, sizeof buf);
-		for (u = sizeof slh; u < size; ) {
-			i = write(vsl_fd, buf, sizeof buf);
-			if (i <= 0) {
-				fprintf(stderr, "Write error %s: %s\n",
-				    fn, strerror(errno));
-				exit (1);
-			}
-			u += i;
-		}
-	}
-
-	AZ(ftruncate(vsl_fd, (off_t)size));
-}
-
-/*--------------------------------------------------------------------
- * Exit handler that clears the owning pid from the SHMLOG
- */
-
-static
-void
-mgt_shm_atexit(void)
-{
-	if (getpid() == VSM_head->master_pid)
-		VSM_head->master_pid = 0;
-}
-
-void
-mgt_SHM_Init(const char *l_arg)
-{
-	int i, fill;
-	struct params *pp;
-	const char *q;
-	uintmax_t size, s1, s2, ps;
-	char **av, **ap;
-	uint32_t *vsl_log_start;
-
-	if (l_arg == NULL)
-		l_arg = "";
-
-	av = VAV_Parse(l_arg, NULL, ARGV_COMMA);
-	AN(av);
-	if (av[0] != NULL)
-		ARGV_ERR("\t-l ...: %s", av[0]);
-
-	ap = av + 1;
-
-	/* Size of SHMLOG */
-	if (*ap != NULL && **ap != '\0') {
-		q = VNUM_2bytes(*ap, &s1, 0);
-		if (q != NULL)
-			ARGV_ERR("\t-l[1] ...:  %s\n", q);
-	} else {
-		s1 = 80 * 1024 * 1024;
-	}
-	if (*ap != NULL)
-		ap++;
-
-	/* Size of space for other stuff */
-	if (*ap != NULL && **ap != '\0') {
-		q = VNUM_2bytes(*ap, &s2, 0);
-		if (q != NULL)
-			ARGV_ERR("\t-l[2] ...:  %s\n", q);
-	} else {
-		s2 = 1024 * 1024;
-	}
-	if (*ap != NULL)
-		ap++;
-
-	/* Fill or not ? */
-	if (*ap != NULL) {
-		if (**ap == '\0')
-			fill = 1;
-		else if (!strcmp(*ap, "-"))
-			fill = 0;
-		else if (!strcmp(*ap, "+"))
-			fill = 1;
-		else
-			ARGV_ERR("\t-l[3] ...:  Must be \"-\" or \"+\"\n");
-		ap++;
-	} else {
-		fill = 1;
-	}
-
-	if (*ap != NULL)
-		ARGV_ERR("\t-l ...:  Too many sub-args\n");
-
-	VAV_Free(av);
-
-	size = s1 + s2;
-	ps = getpagesize();
-	size += ps - 1;
-	size &= ~(ps - 1);
-
-	i = open(VSM_FILENAME, O_RDWR, 0644);
-	if (i >= 0) {
-		vsl_n_check(i);
-		(void)close(i);
-	}
-	vsl_buildnew(VSM_FILENAME, size, fill);
-
-	VSM_head = (void *)mmap(NULL, size,
-	    PROT_READ|PROT_WRITE,
-	    MAP_HASSEMAPHORE | MAP_NOSYNC | MAP_SHARED,
-	    vsl_fd, 0);
-	VSM_head->master_pid = getpid();
-	AZ(atexit(mgt_shm_atexit));
-	xxxassert(VSM_head != MAP_FAILED);
-	(void)mlock((void*)VSM_head, size);
-
-	memset(&VSM_head->head, 0, sizeof VSM_head->head);
-	VSM_head->head.magic = VSM_CHUNK_MAGIC;
-	VSM_head->head.len =
-	    (uint8_t*)(VSM_head) + size - (uint8_t*)&VSM_head->head;
-	bprintf(VSM_head->head.class, "%s", VSM_CLASS_FREE);
-	VWMB();
-
-	vsm_end = (void*)((uint8_t*)VSM_head + size);
-
-	VSC_C_main = VSM_Alloc(sizeof *VSC_C_main,
-	    VSC_CLASS, VSC_TYPE_MAIN, "");
-	AN(VSC_C_main);
-
-	pp = VSM_Alloc(sizeof *pp, VSM_CLASS_PARAM, "", "");
-	AN(pp);
-	*pp = *params;
-	params = pp;
-
-	vsl_log_start = VSM_Alloc(s1, VSL_CLASS, "", "");
-	AN(vsl_log_start);
-	vsl_log_start[1] = VSL_ENDMARKER;
-	VWMB();
-
-	do
-		*vsl_log_start = random() & 0xffff;
-	while (*vsl_log_start == 0);
-
-	VWMB();
-
-	do
-		VSM_head->alloc_seq = random();
-	while (VSM_head->alloc_seq == 0);
-
-}
-
-void
-mgt_SHM_Pid(void)
-{
-
-	VSM_head->master_pid = getpid();
-}
diff --git a/bin/varnishd/mgt_vcc.c b/bin/varnishd/mgt_vcc.c
deleted file mode 100644
index 5901408..0000000
--- a/bin/varnishd/mgt_vcc.c
+++ /dev/null
@@ -1,676 +0,0 @@
-/*-
- * Copyright (c) 2006 Verdens Gang AS
- * Copyright (c) 2006-2011 Varnish Software AS
- * All rights reserved.
- *
- * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- *
- * VCL compiler stuff
- */
-
-#include "config.h"
-
-#include <dlfcn.h>
-#include <fcntl.h>
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#include <unistd.h>
-
-#include "mgt.h"
-
-#include "libvcl.h"
-#include "vcl.h"
-#include "vcli.h"
-#include "vcli_priv.h"
-#include "vfil.h"
-#include "vsub.h"
-
-#include "mgt_cli.h"
-
-struct vclprog {
-	VTAILQ_ENTRY(vclprog)	list;
-	char			*name;
-	char			*fname;
-	int			active;
-};
-
-static VTAILQ_HEAD(, vclprog) vclhead = VTAILQ_HEAD_INITIALIZER(vclhead);
-
-char *mgt_cc_cmd;
-const char *mgt_vcl_dir;
-const char *mgt_vmod_dir;
-unsigned mgt_vcc_err_unref;
-
-static struct vcc *vcc;
-
-/*--------------------------------------------------------------------*/
-
-static const char * const default_vcl =
-#include "default_vcl.h"
-    ""	;
-
-/*--------------------------------------------------------------------
- * Prepare the compiler command line
- */
-static struct vsb *
-mgt_make_cc_cmd(const char *sf, const char *of)
-{
-	struct vsb *sb;
-	int pct;
-	char *p;
-
-	sb = VSB_new_auto();
-	XXXAN(sb);
-	for (p = mgt_cc_cmd, pct = 0; *p; ++p) {
-		if (pct) {
-			switch (*p) {
-			case 's':
-				VSB_cat(sb, sf);
-				break;
-			case 'o':
-				VSB_cat(sb, of);
-				break;
-			case '%':
-				VSB_putc(sb, '%');
-				break;
-			default:
-				VSB_putc(sb, '%');
-				VSB_putc(sb, *p);
-				break;
-			}
-			pct = 0;
-		} else if (*p == '%') {
-			pct = 1;
-		} else {
-			VSB_putc(sb, *p);
-		}
-	}
-	if (pct)
-		VSB_putc(sb, '%');
-	AZ(VSB_finish(sb));
-	return (sb);
-}
-
-/*--------------------------------------------------------------------
- * Invoke system VCC compiler in a sub-process
- */
-
-struct vcc_priv {
-	unsigned	magic;
-#define VCC_PRIV_MAGIC	0x70080cb8
-	char		*sf;
-	const char	*vcl;
-};
-
-static void
-run_vcc(void *priv)
-{
-	char *csrc;
-	struct vsb *sb;
-	struct vcc_priv *vp;
-	int fd, i, l;
-
-	CAST_OBJ_NOTNULL(vp, priv, VCC_PRIV_MAGIC);
-	sb = VSB_new_auto();
-	XXXAN(sb);
-	VCC_VCL_dir(vcc, mgt_vcl_dir);
-	VCC_VMOD_dir(vcc, mgt_vmod_dir);
-	VCC_Err_Unref(vcc, mgt_vcc_err_unref);
-	csrc = VCC_Compile(vcc, sb, vp->vcl);
-	AZ(VSB_finish(sb));
-	if (VSB_len(sb))
-		printf("%s", VSB_data(sb));
-	VSB_delete(sb);
-	if (csrc == NULL)
-		exit (1);
-
-	fd = open(vp->sf, O_WRONLY);
-	if (fd < 0) {
-		fprintf(stderr, "Cannot open %s", vp->sf);
-		exit (1);
-	}
-	l = strlen(csrc);
-	i = write(fd, csrc, l);
-	if (i != l) {
-		fprintf(stderr, "Cannot write %s", vp->sf);
-		exit (1);
-	}
-	AZ(close(fd));
-	free(csrc);
-	exit (0);
-}
-
-/*--------------------------------------------------------------------
- * Invoke system C compiler in a sub-process
- */
-
-static void
-run_cc(void *priv)
-{
-	(void)execl("/bin/sh", "/bin/sh", "-c", priv, NULL);
-}
-
-/*--------------------------------------------------------------------
- * Attempt to open compiled VCL in a sub-process
- */
-
-static void __match_proto__(sub_func_f)
-run_dlopen(void *priv)
-{
-	const char *of;
-	void *dlh;
-	struct VCL_conf const *cnf;
-
-	of = priv;
-
-	/* Try to load the object into the management process */
-	if ((dlh = dlopen(of, RTLD_NOW | RTLD_LOCAL)) == NULL) {
-		fprintf(stderr,
-		    "Compiled VCL program failed to load:\n  %s\n",
-		    dlerror());
-		exit(1);
-	}
-
-	cnf = dlsym(dlh, "VCL_conf");
-	if (cnf == NULL) {
-		fprintf(stderr, "Compiled VCL program, metadata not found\n");
-		exit(1);
-	}
-
-	if (cnf->magic != VCL_CONF_MAGIC) {
-		fprintf(stderr, "Compiled VCL program, mangled metadata\n");
-		exit(1);
-	}
-
-	if (dlclose(dlh)) {
-		fprintf(stderr,
-		    "Compiled VCL program failed to unload:\n  %s\n",
-		    dlerror());
-		exit(1);
-	}
-	exit(0);
-}
-
-/*--------------------------------------------------------------------
- * Compile a VCL program, return shared object, errors in sb.
- */
-
-static char *
-mgt_run_cc(const char *vcl, struct vsb *sb, int C_flag)
-{
-	char *csrc;
-	struct vsb *cmdsb;
-	char sf[] = "./vcl.########.c";
-	char of[sizeof sf + 1];
-	char *retval;
-	int sfd, i;
-	struct vcc_priv vp;
-
-	/* Create temporary C source file */
-	sfd = VFIL_tmpfile(sf);
-	if (sfd < 0) {
-		VSB_printf(sb, "Failed to create %s: %s", sf, strerror(errno));
-		return (NULL);
-	}
-	AZ(close(sfd));
-
-	/* Run the VCC compiler in a sub-process */
-	memset(&vp, 0, sizeof vp);
-	vp.magic = VCC_PRIV_MAGIC;
-	vp.sf = sf;
-	vp.vcl = vcl;
-	if (VSUB_run(sb, run_vcc, &vp, "VCC-compiler", -1)) {
-		(void)unlink(sf);
-		return (NULL);
-	}
-
-	if (C_flag) {
-		csrc = VFIL_readfile(NULL, sf, NULL);
-		XXXAN(csrc);
-		(void)fputs(csrc, stdout);
-		free(csrc);
-	}
-
-	/* Name the output shared library by "s/[.]c$/[.]so/" */
-	memcpy(of, sf, sizeof sf);
-	assert(sf[sizeof sf - 2] == 'c');
-	of[sizeof sf - 2] = 's';
-	of[sizeof sf - 1] = 'o';
-	of[sizeof sf] = '\0';
-
-	/* Build the C-compiler command line */
-	cmdsb = mgt_make_cc_cmd(sf, of);
-
-	/* Run the C-compiler in a sub-shell */
-	i = VSUB_run(sb, run_cc, VSB_data(cmdsb), "C-compiler", 10);
-
-	(void)unlink(sf);
-	VSB_delete(cmdsb);
-
-	if (!i)
-		i = VSUB_run(sb, run_dlopen, of, "dlopen", 10);
-
-	if (i) {
-		(void)unlink(of);
-		return (NULL);
-	}
-
-	retval = strdup(of);
-	XXXAN(retval);
-	return (retval);
-}
-
-/*--------------------------------------------------------------------*/
-
-static char *
-mgt_VccCompile(struct vsb **sb, const char *b, int C_flag)
-{
-	char *vf;
-
-	*sb = VSB_new_auto();
-	XXXAN(*sb);
-	vf = mgt_run_cc(b, *sb, C_flag);
-	AZ(VSB_finish(*sb));
-	return (vf);
-}
-
-/*--------------------------------------------------------------------*/
-
-static struct vclprog *
-mgt_vcc_add(const char *name, char *file)
-{
-	struct vclprog *vp;
-
-	vp = calloc(sizeof *vp, 1);
-	XXXAN(vp);
-	vp->name = strdup(name);
-	XXXAN(vp->name);
-	vp->fname = file;
-	VTAILQ_INSERT_TAIL(&vclhead, vp, list);
-	return (vp);
-}
-
-static void
-mgt_vcc_del(struct vclprog *vp)
-{
-	VTAILQ_REMOVE(&vclhead, vp, list);
-	printf("unlink %s\n", vp->fname);
-	XXXAZ(unlink(vp->fname));
-	free(vp->fname);
-	free(vp->name);
-	free(vp);
-}
-
-static struct vclprog *
-mgt_vcc_byname(const char *name)
-{
-	struct vclprog *vp;
-
-	VTAILQ_FOREACH(vp, &vclhead, list)
-		if (!strcmp(name, vp->name))
-			return (vp);
-	return (NULL);
-}
-
-
-static int
-mgt_vcc_delbyname(const char *name)
-{
-	struct vclprog *vp;
-
-	vp = mgt_vcc_byname(name);
-	if (vp != NULL) {
-		mgt_vcc_del(vp);
-		return (0);
-	}
-	return (1);
-}
-
-/*--------------------------------------------------------------------*/
-
-int
-mgt_vcc_default(const char *b_arg, const char *f_arg, char *vcl, int C_flag)
-{
-	char *vf;
-	struct vsb *sb;
-	struct vclprog *vp;
-	char buf[BUFSIZ];
-
-	/* XXX: annotate vcl with -b/-f arg so people know where it came from */
-	(void)f_arg;
-
-	if (b_arg != NULL) {
-		AZ(vcl);
-		/*
-		 * XXX: should do a "HEAD /" on the -b argument to see that
-		 * XXX: it even works.  On the other hand, we should do that
-		 * XXX: for all backends in the cache process whenever we
-		 * XXX: change config, but for a complex VCL, it might not be
-		 * XXX: a bug for a backend to not reply at that time, so then
-		 * XXX: again: we should check it here in the "trivial" case.
-		 */
-		bprintf(buf,
-		    "backend default {\n"
-		    "    .host = \"%s\";\n"
-		    "}\n", b_arg);
-		vcl = strdup(buf);
-		AN(vcl);
-	}
-	strcpy(buf, "boot");
-
-	vf = mgt_VccCompile(&sb, vcl, C_flag);
-	free(vcl);
-	if (VSB_len(sb) > 0)
-		fprintf(stderr, "%s", VSB_data(sb));
-	VSB_delete(sb);
-	if (C_flag) {
-		if (vf != NULL)
-			AZ(unlink(vf));
-		return (0);
-	}
-	if (vf == NULL) {
-		fprintf(stderr, "\nVCL compilation failed\n");
-		return (1);
-	}
-	vp = mgt_vcc_add(buf, vf);
-	vp->active = 1;
-	return (0);
-}
-
-/*--------------------------------------------------------------------*/
-
-int
-mgt_has_vcl()
-{
-
-	return (!VTAILQ_EMPTY(&vclhead));
-}
-
-/*--------------------------------------------------------------------*/
-
-int
-mgt_push_vcls_and_start(unsigned *status, char **p)
-{
-	struct vclprog *vp;
-
-	VTAILQ_FOREACH(vp, &vclhead, list) {
-		if (mgt_cli_askchild(status, p,
-		    "vcl.load \"%s\" %s\n", vp->name, vp->fname))
-			return (1);
-		free(*p);
-		if (!vp->active)
-			continue;
-		if (mgt_cli_askchild(status, p,
-		    "vcl.use \"%s\"\n", vp->name))
-			return (1);
-		free(*p);
-	}
-	if (mgt_cli_askchild(status, p, "start\n"))
-		return (1);
-	free(*p);
-	*p = NULL;
-	return (0);
-}
-
-/*--------------------------------------------------------------------*/
-
-static
-void
-mgt_vcc_atexit(void)
-{
-	struct vclprog *vp;
-
-	if (getpid() != mgt_pid)
-		return;
-	while (1) {
-		vp = VTAILQ_FIRST(&vclhead);
-		if (vp == NULL)
-			break;
-		(void)unlink(vp->fname);
-		VTAILQ_REMOVE(&vclhead, vp, list);
-	}
-}
-
-void
-mgt_vcc_init(void)
-{
-
-	vcc = VCC_New();
-	AN(vcc);
-	VCC_Default_VCL(vcc, default_vcl);
-	AZ(atexit(mgt_vcc_atexit));
-}
-
-/*--------------------------------------------------------------------*/
-
-void
-mcf_config_inline(struct cli *cli, const char * const *av, void *priv)
-{
-	char *vf, *p = NULL;
-	struct vsb *sb;
-	unsigned status;
-	struct vclprog *vp;
-
-	(void)priv;
-
-	vp = mgt_vcc_byname(av[2]);
-	if (vp != NULL) {
-		VCLI_Out(cli, "Already a VCL program named %s", av[2]);
-		VCLI_SetResult(cli, CLIS_PARAM);
-		return;
-	}
-
-	vf = mgt_VccCompile(&sb, av[3], 0);
-	if (VSB_len(sb) > 0)
-		VCLI_Out(cli, "%s\n", VSB_data(sb));
-	VSB_delete(sb);
-	if (vf == NULL) {
-		VCLI_Out(cli, "VCL compilation failed");
-		VCLI_SetResult(cli, CLIS_PARAM);
-		return;
-	}
-	VCLI_Out(cli, "VCL compiled.");
-	if (child_pid >= 0 &&
-	    mgt_cli_askchild(&status, &p, "vcl.load %s %s\n", av[2], vf)) {
-		VCLI_SetResult(cli, status);
-		VCLI_Out(cli, "%s", p);
-	} else {
-		(void)mgt_vcc_add(av[2], vf);
-	}
-	free(p);
-}
-
-void
-mcf_config_load(struct cli *cli, const char * const *av, void *priv)
-{
-	char *vf, *vcl;
-	struct vsb *sb;
-	unsigned status;
-	char *p = NULL;
-	struct vclprog *vp;
-
-	(void)priv;
-	vp = mgt_vcc_byname(av[2]);
-	if (vp != NULL) {
-		VCLI_Out(cli, "Already a VCL program named %s", av[2]);
-		VCLI_SetResult(cli, CLIS_PARAM);
-		return;
-	}
-
-	vcl = VFIL_readfile(mgt_vcl_dir, av[3], NULL);
-	if (vcl == NULL) {
-		VCLI_Out(cli, "Cannot open '%s'", av[3]);
-		VCLI_SetResult(cli, CLIS_PARAM);
-		return;
-	}
-
-	vf = mgt_VccCompile(&sb, vcl, 0);
-	free(vcl);
-
-	if (VSB_len(sb) > 0)
-		VCLI_Out(cli, "%s", VSB_data(sb));
-	VSB_delete(sb);
-	if (vf == NULL) {
-		VCLI_Out(cli, "VCL compilation failed");
-		VCLI_SetResult(cli, CLIS_PARAM);
-		return;
-	}
-	VCLI_Out(cli, "VCL compiled.");
-	if (child_pid >= 0 &&
-	    mgt_cli_askchild(&status, &p, "vcl.load %s %s\n", av[2], vf)) {
-		VCLI_SetResult(cli, status);
-		VCLI_Out(cli, "%s", p);
-	} else {
-		(void)mgt_vcc_add(av[2], vf);
-	}
-	free(p);
-}
-
-static struct vclprog *
-mcf_find_vcl(struct cli *cli, const char *name)
-{
-	struct vclprog *vp;
-
-	vp = mgt_vcc_byname(name);
-	if (vp != NULL)
-		return (vp);
-	VCLI_SetResult(cli, CLIS_PARAM);
-	VCLI_Out(cli, "No configuration named %s known.", name);
-	return (NULL);
-}
-
-void
-mcf_config_use(struct cli *cli, const char * const *av, void *priv)
-{
-	unsigned status;
-	char *p = NULL;
-	struct vclprog *vp;
-
-	(void)priv;
-	vp = mcf_find_vcl(cli, av[2]);
-	if (vp == NULL)
-		return;
-	if (vp->active != 0)
-		return;
-	if (child_pid >= 0 &&
-	    mgt_cli_askchild(&status, &p, "vcl.use %s\n", av[2])) {
-		VCLI_SetResult(cli, status);
-		VCLI_Out(cli, "%s", p);
-	} else {
-		vp->active = 2;
-		VTAILQ_FOREACH(vp, &vclhead, list) {
-			if (vp->active == 1)
-				vp->active = 0;
-			else if (vp->active == 2)
-				vp->active = 1;
-		}
-	}
-	free(p);
-}
-
-void
-mcf_config_discard(struct cli *cli, const char * const *av, void *priv)
-{
-	unsigned status;
-	char *p = NULL;
-	struct vclprog *vp;
-
-	(void)priv;
-	vp = mcf_find_vcl(cli, av[2]);
-	if (vp != NULL && vp->active) {
-		VCLI_SetResult(cli, CLIS_PARAM);
-		VCLI_Out(cli, "Cannot discard active VCL program\n");
-	} else if (vp != NULL) {
-		if (child_pid >= 0 &&
-		    mgt_cli_askchild(&status, &p,
-		    "vcl.discard %s\n", av[2])) {
-			VCLI_SetResult(cli, status);
-			VCLI_Out(cli, "%s", p);
-		} else {
-			AZ(mgt_vcc_delbyname(av[2]));
-		}
-	}
-	free(p);
-}
-
-void
-mcf_config_list(struct cli *cli, const char * const *av, void *priv)
-{
-	unsigned status;
-	char *p;
-	const char *flg;
-	struct vclprog *vp;
-
-	(void)av;
-	(void)priv;
-	if (child_pid >= 0) {
-		if (!mgt_cli_askchild(&status, &p, "vcl.list\n")) {
-			VCLI_SetResult(cli, status);
-			VCLI_Out(cli, "%s", p);
-		}
-		free(p);
-	} else {
-		VTAILQ_FOREACH(vp, &vclhead, list) {
-			if (vp->active) {
-				flg = "active";
-			} else
-				flg = "available";
-			VCLI_Out(cli, "%-10s %6s %s\n",
-			    flg, "N/A", vp->name);
-		}
-	}
-}
-
-/*
- * XXX: This should take an option argument to show all (include) files
- * XXX: This violates the principle of not loading VCL's in the master
- * XXX: process.
- */
-void
-mcf_config_show(struct cli *cli, const char * const *av, void *priv)
-{
-	struct vclprog *vp;
-	void *dlh, *sym;
-	const char **src;
-
-	(void)priv;
-	if ((vp = mcf_find_vcl(cli, av[2])) != NULL) {
-		if ((dlh = dlopen(vp->fname, RTLD_NOW | RTLD_LOCAL)) == NULL) {
-			VCLI_Out(cli, "failed to load %s: %s\n",
-			    vp->name, dlerror());
-			VCLI_SetResult(cli, CLIS_CANT);
-		} else if ((sym = dlsym(dlh, "srcbody")) == NULL) {
-			VCLI_Out(cli, "failed to locate source for %s: %s\n",
-			    vp->name, dlerror());
-			VCLI_SetResult(cli, CLIS_CANT);
-			AZ(dlclose(dlh));
-		} else {
-			src = sym;
-			VCLI_Out(cli, "%s", src[0]);
-			/* VCLI_Out(cli, src[1]); */
-			AZ(dlclose(dlh));
-		}
-	}
-}
diff --git a/bin/varnishd/storage/stevedore_utils.c b/bin/varnishd/storage/stevedore_utils.c
index 3daebf8..7e9c7d5 100644
--- a/bin/varnishd/storage/stevedore_utils.c
+++ b/bin/varnishd/storage/stevedore_utils.c
@@ -49,7 +49,7 @@
 #include <string.h>
 #include <unistd.h>
 
-#include "mgt.h"
+#include "mgt/mgt.h"
 
 #include "storage/storage.h"
 #include "vnum.h"
diff --git a/bin/varnishd/varnishd.c b/bin/varnishd/varnishd.c
index e0654a3..e0ba759 100644
--- a/bin/varnishd/varnishd.c
+++ b/bin/varnishd/varnishd.c
@@ -43,7 +43,7 @@
 #include <time.h>
 #include <unistd.h>
 
-#include "mgt.h"
+#include "mgt/mgt.h"
 
 #include "hash/hash_slinger.h"
 #include "heritage.h"



From geoff at varnish-cache.org  Mon Jan  9 20:52:26 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:26 +0100
Subject: [experimental-ims] adf381c Ignore SIGPIPE,
	it causes the test-executing subprocess to bail out before all
	diagnostics have been gathered.
Message-ID: <E1RkMCg-0007w1-GK@project.varnish-software.com>

commit adf381c8a1159580a8635bfb1de0e259eff9e81e
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Oct 25 08:15:35 2011 +0000

    Ignore SIGPIPE, it causes the test-executing subprocess to bail out
    before all diagnostics have been gathered.

diff --git a/bin/varnishtest/vtc.c b/bin/varnishtest/vtc.c
index b496f9e..9581b3c 100644
--- a/bin/varnishtest/vtc.c
+++ b/bin/varnishtest/vtc.c
@@ -32,6 +32,7 @@
 #include <sys/wait.h>
 
 #include <ctype.h>
+#include <signal.h>
 #include <stdarg.h>
 #include <stdint.h>
 #include <stdio.h>
@@ -526,6 +527,8 @@ exec_file(const char *fn, const char *script, const char *tmpdir,
 	FILE *f;
 	struct extmacro *m;
 
+	signal(SIGPIPE, SIG_IGN);
+
 	vtc_loginit(logbuf, loglen);
 	vltop = vtc_logopen("top");
 	AN(vltop);



From geoff at varnish-cache.org  Mon Jan  9 20:52:28 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:28 +0100
Subject: [experimental-ims] bced3f5 Merge branch 'master' of
	ssh://git.varnish-cache.org/git/varnish-cache
Message-ID: <E1RkMCi-00080c-GD@project.varnish-software.com>

commit bced3f5bdadee976bf8208ee61d4b6d293ba07be
Merge: f6536dac fe425ce
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Nov 1 09:41:40 2011 +0000

    Merge branch 'master' of ssh://git.varnish-cache.org/git/varnish-cache



From geoff at varnish-cache.org  Mon Jan  9 20:52:15 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:15 +0100
Subject: [experimental-ims] 2ac54a0 Remove unncessary #includes
Message-ID: <E1RkMCV-0007Z2-9t@project.varnish-software.com>

commit 2ac54a0443c15f1031fbfd447029de8a7fb01fbe
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sat Oct 8 21:30:13 2011 +0000

    Remove unncessary #includes

diff --git a/bin/varnishtest/vtc.c b/bin/varnishtest/vtc.c
index f39b628..506aee1 100644
--- a/bin/varnishtest/vtc.c
+++ b/bin/varnishtest/vtc.c
@@ -31,20 +31,16 @@
 #include <stdio.h>
 #include <string.h>
 #include <ctype.h>
-#include <fcntl.h>
-#include <limits.h>
-#include <signal.h>
 #include <stdarg.h>
 #include <stdlib.h>
 #include <unistd.h>
-#include <sys/stat.h>
 #include <sys/types.h>
 #include <sys/wait.h>
 
+#include "vtc.h"
+
 #include "libvarnish.h"
-#include "vev.h"
 
-#include "vtc.h"
 
 #ifndef HAVE_SRANDOMDEV
 #include "compat/srandomdev.h"
diff --git a/bin/varnishtest/vtc_client.c b/bin/varnishtest/vtc_client.c
index d06b478..2c75622 100644
--- a/bin/varnishtest/vtc_client.c
+++ b/bin/varnishtest/vtc_client.c
@@ -30,7 +30,6 @@
 
 #include <stdio.h>
 #include <stdlib.h>
-#include <unistd.h>
 #include <string.h>
 
 #include <sys/types.h>
diff --git a/bin/varnishtest/vtc_http.c b/bin/varnishtest/vtc_http.c
index a0ee01c..4582d4c 100644
--- a/bin/varnishtest/vtc_http.c
+++ b/bin/varnishtest/vtc_http.c
@@ -28,20 +28,18 @@
 
 #include "config.h"
 
-#include <poll.h>
+#include <string.h>
 #include <stdio.h>
 #include <poll.h>
-#include <stdio.h>
-#include <string.h>
 #include <stdlib.h>
-#include <unistd.h>
 
 #include <sys/types.h>
 #include <sys/socket.h>
 
-#include "libvarnish.h"
 #include "vct.h"
 
+#include "libvarnish.h"
+
 #include "vtc.h"
 
 #include "vgz.h"
diff --git a/bin/varnishtest/vtc_log.c b/bin/varnishtest/vtc_log.c
index 6b36aa5..51c6646 100644
--- a/bin/varnishtest/vtc_log.c
+++ b/bin/varnishtest/vtc_log.c
@@ -30,16 +30,13 @@
 
 #include <stdio.h>
 #include <string.h>
-#include <fcntl.h>
 #include <stdlib.h>
-#include <unistd.h>
 #include <stdarg.h>
 
-#include "libvarnish.h"
-#include "vas.h"
-
 #include "vtc.h"
 
+#include "libvarnish.h"
+
 static pthread_mutex_t	vtclog_mtx;
 static char		*vtclog_buf;
 static unsigned		vtclog_left;
diff --git a/bin/varnishtest/vtc_main.c b/bin/varnishtest/vtc_main.c
index d80aa6a..d201946 100644
--- a/bin/varnishtest/vtc_main.c
+++ b/bin/varnishtest/vtc_main.c
@@ -31,8 +31,6 @@
 #include <stdio.h>
 #include <string.h>
 #include <fcntl.h>
-#include <signal.h>
-#include <stdarg.h>
 #include <stdlib.h>
 #include <unistd.h>
 #include <sys/mman.h>
@@ -40,11 +38,11 @@
 #include <sys/types.h>
 #include <sys/wait.h>
 
+#include "vtc.h"
+
 #include "libvarnish.h"
 #include "vev.h"
 
-#include "vtc.h"
-
 #ifndef HAVE_SRANDOMDEV
 #include "compat/srandomdev.h"
 #endif
diff --git a/bin/varnishtest/vtc_sema.c b/bin/varnishtest/vtc_sema.c
index 4df1979..8a9228a 100644
--- a/bin/varnishtest/vtc_sema.c
+++ b/bin/varnishtest/vtc_sema.c
@@ -35,7 +35,6 @@
 
 #include "vtc.h"
 
-#include "miniobj.h"
 #include "libvarnish.h"
 
 struct sema {
diff --git a/bin/varnishtest/vtc_server.c b/bin/varnishtest/vtc_server.c
index 9390e5b..d541d67 100644
--- a/bin/varnishtest/vtc_server.c
+++ b/bin/varnishtest/vtc_server.c
@@ -31,7 +31,6 @@
 #include <stdio.h>
 #include <errno.h>
 #include <stdlib.h>
-#include <unistd.h>
 #include <string.h>
 
 #include <sys/types.h>
@@ -39,8 +38,8 @@
 
 #include "vtc.h"
 
-#include "vss.h"
 #include "libvarnish.h"
+#include "vss.h"
 
 struct server {
 	unsigned		magic;
diff --git a/bin/varnishtest/vtc_varnish.c b/bin/varnishtest/vtc_varnish.c
index ce6a45a..aea29a4 100644
--- a/bin/varnishtest/vtc_varnish.c
+++ b/bin/varnishtest/vtc_varnish.c
@@ -34,7 +34,6 @@
 #include <stdlib.h>
 #include <unistd.h>
 #include <string.h>
-#include <signal.h>
 #include <poll.h>
 #include <inttypes.h>
 
@@ -42,13 +41,14 @@
 #include <sys/wait.h>
 #include <sys/socket.h>
 
+#include "vtc.h"
+
 #include "libvarnish.h"
 #include "varnishapi.h"
+
 #include "vcli.h"
 #include "vss.h"
 
-#include "vtc.h"
-
 struct varnish {
 	unsigned		magic;
 #define VARNISH_MAGIC		0x208cd8e3



From geoff at varnish-cache.org  Mon Jan  9 20:51:49 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:51:49 +0100
Subject: [experimental-ims] 0e9e4d1 Use VTLA VWP for poll waiter,
	turn it into an object.
Message-ID: <E1RkMC5-0006r1-JG@project.varnish-software.com>

commit 0e9e4d1bf784e9cc875ca5928ed926667710f101
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sat Sep 17 10:31:36 2011 +0000

    Use VTLA VWP for poll waiter, turn it into an object.

diff --git a/bin/varnishd/cache_waiter_poll.c b/bin/varnishd/cache_waiter_poll.c
index 528a880..44b4033 100644
--- a/bin/varnishd/cache_waiter_poll.c
+++ b/bin/varnishd/cache_waiter_poll.c
@@ -42,23 +42,29 @@
 
 #define NEEV	128
 
-static pthread_t vca_poll_thread;
-static struct pollfd *pollfd;
-static unsigned npoll, hpoll;
-
-static VTAILQ_HEAD(,sess) sesshead = VTAILQ_HEAD_INITIALIZER(sesshead);
+struct vwp {
+	unsigned		magic;
+#define VWP_MAGIC		0x4b2cc735
+	int			pipes[2];
+	pthread_t		poll_thread;
+	struct pollfd		*pollfd;
+	unsigned		npoll;
+	unsigned		hpoll;
+
+	VTAILQ_HEAD(,sess)	sesshead;
+};
 
 /*--------------------------------------------------------------------*/
 
 static void
-vca_pollspace(unsigned fd)
+vwp_pollspace(struct vwp *vwp, unsigned fd)
 {
-	struct pollfd *newpollfd = pollfd;
+	struct pollfd *newpollfd = vwp->pollfd;
 	unsigned newnpoll;
 
-	if (fd < npoll)
+	if (fd < vwp->npoll)
 		return;
-	newnpoll = npoll;
+	newnpoll = vwp->npoll;
 	if (newnpoll == 0)
 		newnpoll = 1;
 	while (fd >= newnpoll)
@@ -66,123 +72,125 @@ vca_pollspace(unsigned fd)
 	VSL(SLT_Debug, 0, "Acceptor poll space increased to %u", newnpoll);
 	newpollfd = realloc(newpollfd, newnpoll * sizeof *newpollfd);
 	XXXAN(newpollfd);
-	memset(newpollfd + npoll, 0, (newnpoll - npoll) * sizeof *newpollfd);
-	pollfd = newpollfd;
-	while (npoll < newnpoll)
-		pollfd[npoll++].fd = -1;
-	assert(fd < npoll);
+	memset(newpollfd + vwp->npoll, 0,
+	    (newnpoll - vwp->npoll) * sizeof *newpollfd);
+	vwp->pollfd = newpollfd;
+	while (vwp->npoll < newnpoll)
+		vwp->pollfd[vwp->npoll++].fd = -1;
+	assert(fd < vwp->npoll);
 }
 
 /*--------------------------------------------------------------------*/
 
 static void
-vca_poll(int fd)
+vwp_poll(struct vwp *vwp, int fd)
 {
 
 	assert(fd >= 0);
-	vca_pollspace((unsigned)fd);
-	assert(fd < npoll);
+	vwp_pollspace(vwp, (unsigned)fd);
+	assert(fd < vwp->npoll);
 
-	if (hpoll < fd)
-		hpoll = fd;
+	if (vwp->hpoll < fd)
+		vwp->hpoll = fd;
 
-	assert(pollfd[fd].fd == -1);
-	assert(pollfd[fd].events == 0);
-	assert(pollfd[fd].revents == 0);
+	assert(vwp->pollfd[fd].fd == -1);
+	assert(vwp->pollfd[fd].events == 0);
+	assert(vwp->pollfd[fd].revents == 0);
 
-	pollfd[fd].fd = fd;
-	pollfd[fd].events = POLLIN;
+	vwp->pollfd[fd].fd = fd;
+	vwp->pollfd[fd].events = POLLIN;
 }
 
 static void
-vca_unpoll(int fd)
+vwp_unpoll(struct vwp *vwp, int fd)
 {
 
 	assert(fd >= 0);
-	assert(fd < npoll);
-	vca_pollspace((unsigned)fd);
+	assert(fd < vwp->npoll);
+	vwp_pollspace(vwp, (unsigned)fd);
 
-	assert(pollfd[fd].fd == fd);
-	assert(pollfd[fd].events == POLLIN);
-	assert(pollfd[fd].revents == 0);
+	assert(vwp->pollfd[fd].fd == fd);
+	assert(vwp->pollfd[fd].events == POLLIN);
+	assert(vwp->pollfd[fd].revents == 0);
 
-	pollfd[fd].fd = -1;
-	pollfd[fd].events = 0;
+	vwp->pollfd[fd].fd = -1;
+	vwp->pollfd[fd].events = 0;
 }
 
 /*--------------------------------------------------------------------*/
 
 static void *
-vca_main(void *arg)
+vwp_main(void *priv)
 {
 	int v;
+	struct vwp *vwp;
 	struct sess *ss[NEEV], *sp, *sp2;
 	double deadline;
 	int i, j, fd;
 
+	CAST_OBJ_NOTNULL(vwp, priv, VWP_MAGIC);
 	THR_SetName("cache-poll");
-	(void)arg;
 
-	vca_poll(vca_pipes[0]);
+	vwp_poll(vwp, vwp->pipes[0]);
 
 	while (1) {
-		assert(hpoll < npoll);
-		while (hpoll > 0 && pollfd[hpoll].fd == -1)
-			hpoll--;
-		assert(vca_pipes[0] <= hpoll);
-		assert(pollfd[vca_pipes[0]].fd == vca_pipes[0]);
-		assert(pollfd[vca_pipes[1]].fd == -1);
-		v = poll(pollfd, hpoll + 1, 100);
+		assert(vwp->hpoll < vwp->npoll);
+		while (vwp->hpoll > 0 && vwp->pollfd[vwp->hpoll].fd == -1)
+			vwp->hpoll--;
+		assert(vwp->pipes[0] <= vwp->hpoll);
+		assert(vwp->pollfd[vwp->pipes[0]].fd == vwp->pipes[0]);
+		assert(vwp->pollfd[vwp->pipes[1]].fd == -1);
+		v = poll(vwp->pollfd, vwp->hpoll + 1, 100);
 		assert(v >= 0);
 		deadline = TIM_real() - params->sess_timeout;
-		VTAILQ_FOREACH_SAFE(sp, &sesshead, list, sp2) {
+		VTAILQ_FOREACH_SAFE(sp, &vwp->sesshead, list, sp2) {
 			if (v == 0)
 				break;
 			CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 			fd = sp->fd;
 			assert(fd >= 0);
-			assert(fd <= hpoll);
-			assert(fd < npoll);
-			assert(pollfd[fd].fd == fd);
-			if (pollfd[fd].revents) {
+			assert(fd <= vwp->hpoll);
+			assert(fd < vwp->npoll);
+			assert(vwp->pollfd[fd].fd == fd);
+			if (vwp->pollfd[fd].revents) {
 				v--;
 				i = HTC_Rx(sp->htc);
-				if (pollfd[fd].revents != POLLIN)
+				if (vwp->pollfd[fd].revents != POLLIN)
 					VSL(SLT_Debug, fd, "Poll: %x / %d",
-					    pollfd[fd].revents, i);
-				pollfd[fd].revents = 0;
-				VTAILQ_REMOVE(&sesshead, sp, list);
+					    vwp->pollfd[fd].revents, i);
+				vwp->pollfd[fd].revents = 0;
+				VTAILQ_REMOVE(&vwp->sesshead, sp, list);
 				if (i == 0) {
 					/* Mov to front of list for speed */
-					VTAILQ_INSERT_HEAD(&sesshead, sp, list);
+					VTAILQ_INSERT_HEAD(&vwp->sesshead, sp, list);
 				} else {
-					vca_unpoll(fd);
+					vwp_unpoll(vwp, fd);
 					vca_handover(sp, i);
 				}
 			} else if (sp->t_open <= deadline) {
-				VTAILQ_REMOVE(&sesshead, sp, list);
-				vca_unpoll(fd);
+				VTAILQ_REMOVE(&vwp->sesshead, sp, list);
+				vwp_unpoll(vwp, fd);
 				// XXX: not yet (void)VTCP_linger(sp->fd, 0);
 				vca_close_session(sp, "timeout");
 				SES_Delete(sp);
 			}
 		}
-		if (v && pollfd[vca_pipes[0]].revents) {
+		if (v && vwp->pollfd[vwp->pipes[0]].revents) {
 
-			if (pollfd[vca_pipes[0]].revents != POLLIN)
+			if (vwp->pollfd[vwp->pipes[0]].revents != POLLIN)
 				VSL(SLT_Debug, 0, "pipe.revents= 0x%x",
-				    pollfd[vca_pipes[0]].revents);
-			assert(pollfd[vca_pipes[0]].revents == POLLIN);
-			pollfd[vca_pipes[0]].revents = 0;
+				    vwp->pollfd[vwp->pipes[0]].revents);
+			assert(vwp->pollfd[vwp->pipes[0]].revents == POLLIN);
+			vwp->pollfd[vwp->pipes[0]].revents = 0;
 			v--;
-			i = read(vca_pipes[0], ss, sizeof ss);
+			i = read(vwp->pipes[0], ss, sizeof ss);
 			assert(i >= 0);
 			assert(((unsigned)i % sizeof ss[0]) == 0);
 			for (j = 0; j * sizeof ss[0] < i; j++) {
 				CHECK_OBJ_NOTNULL(ss[j], SESS_MAGIC);
 				assert(ss[j]->fd >= 0);
-				VTAILQ_INSERT_TAIL(&sesshead, ss[j], list);
-				vca_poll(ss[j]->fd);
+				VTAILQ_INSERT_TAIL(&vwp->sesshead, ss[j], list);
+				vwp_poll(vwp, ss[j]->fd);
 			}
 		}
 		assert(v == 0);
@@ -191,29 +199,37 @@ vca_main(void *arg)
 }
 
 /*--------------------------------------------------------------------*/
+
 static void
-vca_poll_pass(void *priv, struct sess *sp)
+vwp_poll_pass(void *priv, struct sess *sp)
 {
+	struct vwp *vwp;
 
-	(void)priv;
-	assert(sizeof sp == write(vca_pipes[1], &sp, sizeof sp));
+	CAST_OBJ_NOTNULL(vwp, priv, VWP_MAGIC);
+
+	assert(sizeof sp == write(vwp->pipes[1], &sp, sizeof sp));
 }
 
 /*--------------------------------------------------------------------*/
 
 static void *
-vca_poll_init(void)
+vwp_poll_init(void)
 {
-
-	vca_pollspace(256);
-	AZ(pthread_create(&vca_poll_thread, NULL, vca_main, NULL));
-	return (NULL);
+	struct vwp *vwp;
+
+	ALLOC_OBJ(vwp, VWP_MAGIC);
+	AN(vwp);
+	VTAILQ_INIT(&vwp->sesshead);
+	AZ(pipe(vwp->pipes));
+	vwp_pollspace(vwp, 256);
+	AZ(pthread_create(&vwp->poll_thread, NULL, vwp_main, vwp));
+	return (vwp);
 }
 
 /*--------------------------------------------------------------------*/
 
 struct waiter waiter_poll = {
 	.name =		"poll",
-	.init =		vca_poll_init,
-	.pass =		vca_poll_pass,
+	.init =		vwp_poll_init,
+	.pass =		vwp_poll_pass,
 };



From geoff at varnish-cache.org  Mon Jan  9 20:52:02 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:02 +0100
Subject: [experimental-ims] 7f56b9d Try harder to make v00036 deterministic
Message-ID: <E1RkMCI-0007Ex-Fi@project.varnish-software.com>

commit 7f56b9d20f0516ae6b78e7c46fa7c010ba1be87b
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Fri Sep 30 19:09:16 2011 +0000

    Try harder to make v00036 deterministic

diff --git a/bin/varnishtest/tests/v00036.vtc b/bin/varnishtest/tests/v00036.vtc
index 46bcca8..e83acef 100644
--- a/bin/varnishtest/tests/v00036.vtc
+++ b/bin/varnishtest/tests/v00036.vtc
@@ -4,22 +4,25 @@ server s1 {
 	rxreq
 	expect req.url == "/"
 	txresp -body "slash"
-	accept
+	close
 
 	sema r1 sync 3
 
+	accept
 	rxreq
 	expect req.url == "/"
 	txresp -body "slash"
-	accept
+	close
 
+	accept
 	rxreq
 	expect req.url == "/"
 	txresp -body "slash"
+	close
 
 	sema r3 sync 2
-	accept
 
+	accept
 	rxreq
 	expect req.url == "/foo"
 	txresp -hdr "Foo: 1" -body "foobar"
@@ -30,17 +33,19 @@ server s2 {
 	rxreq
 	expect req.url == "/"
 	txresp -body "slash"
-	accept
+	close
 
 	sema r1 sync 3
 
+	accept
 	rxreq
 	expect req.url == "/"
 	txresp -body "slash"
+	close
 
 	sema r2 sync 2
-	accept
 
+	accept
 	rxreq
 	expect req.url == "/foo"
 	txresp -hdr "Foo: 2" -body "foobar"
diff --git a/bin/varnishtest/vtc_http.c b/bin/varnishtest/vtc_http.c
index 024e376..fddcd26 100644
--- a/bin/varnishtest/vtc_http.c
+++ b/bin/varnishtest/vtc_http.c
@@ -1059,6 +1059,25 @@ cmd_http_expect_close(CMD_ARGS)
 }
 
 /**********************************************************************
+ * close a new connection  (server only)
+ */
+
+static void
+cmd_http_close(CMD_ARGS)
+{
+	struct http *hp;
+
+	(void)cmd;
+	(void)vl;
+	CAST_OBJ_NOTNULL(hp, priv, HTTP_MAGIC);
+	AZ(av[1]);
+	assert(hp->sfd != NULL);
+	assert(*hp->sfd >= 0);
+	VTCP_close(&hp->fd);
+	vtc_log(vl, 4, "Closed");
+}
+
+/**********************************************************************
  * close and accept a new connection  (server only)
  */
 
@@ -1073,7 +1092,8 @@ cmd_http_accept(CMD_ARGS)
 	AZ(av[1]);
 	assert(hp->sfd != NULL);
 	assert(*hp->sfd >= 0);
-	VTCP_close(&hp->fd);
+	if (hp->fd >= 0)
+		VTCP_close(&hp->fd);
 	vtc_log(vl, 4, "Accepting");
 	hp->fd = accept(*hp->sfd, NULL, NULL);
 	if (hp->fd < 0)
@@ -1129,6 +1149,7 @@ static const struct cmds http_cmds[] = {
 	{ "delay",		cmd_delay },
 	{ "sema",		cmd_sema },
 	{ "expect_close",	cmd_http_expect_close },
+	{ "close",		cmd_http_close },
 	{ "accept",		cmd_http_accept },
 	{ "loop",		cmd_http_loop },
 	{ NULL,			NULL }



From geoff at varnish-cache.org  Mon Jan  9 20:52:39 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:39 +0100
Subject: [experimental-ims] 0f13e7e Split stevedore.c in a cache and a worker
	source file.
Message-ID: <E1RkMCt-0008Ir-Av@project.varnish-software.com>

commit 0f13e7ebf1bd4f2e6f9e3b91353c407ba9b8295e
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Nov 15 21:07:39 2011 +0000

    Split stevedore.c in a cache and a worker source file.

diff --git a/bin/varnishd/Makefile.am b/bin/varnishd/Makefile.am
index 850a18a..359373d 100644
--- a/bin/varnishd/Makefile.am
+++ b/bin/varnishd/Makefile.am
@@ -64,6 +64,7 @@ varnishd_SOURCES = \
 	mgt/mgt_shmem.c \
 	mgt/mgt_vcc.c \
 	storage/stevedore.c \
+	storage/stevedore_mgt.c \
 	storage/stevedore_utils.c \
 	storage/storage_file.c \
 	storage/storage_malloc.c \
diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index 653fe77..129dbf0 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -114,9 +114,6 @@ struct worker;
 
 #define DIGEST_LEN		32
 
-/* Name of transient storage */
-#define TRANSIENT_STORAGE	"Transient"
-
 /*--------------------------------------------------------------------
  * Pointer aligment magic
  */
diff --git a/bin/varnishd/common/common.h b/bin/varnishd/common/common.h
index 4e56a40..ff06c64 100644
--- a/bin/varnishd/common/common.h
+++ b/bin/varnishd/common/common.h
@@ -42,6 +42,9 @@
 
 struct cli;
 
+/* Name of transient storage */
+#define TRANSIENT_STORAGE	"Transient"
+
 extern pid_t mgt_pid;
 #define ASSERT_MGT() do { assert(getpid() == mgt_pid);} while (0)
 
@@ -68,13 +71,6 @@ void mgt_child_inherit(int fd, const char *what);
 		exit(2);					\
 	} while (0);
 
-/* A tiny helper for choosing hash/storage modules */
-struct choice {
-	const char      *name;
-	const void	*ptr;
-};
-const void *pick(const struct choice *cp, const char *which, const char *kind);
-
 #define NEEDLESS_RETURN(foo)	return (foo)
 
 /* stevedore.c */
diff --git a/bin/varnishd/mgt/mgt.h b/bin/varnishd/mgt/mgt.h
index 54352a0..4aec482 100644
--- a/bin/varnishd/mgt/mgt.h
+++ b/bin/varnishd/mgt/mgt.h
@@ -57,13 +57,19 @@ void mgt_cli_master(const char *M_arg);
 void mgt_cli_secret(const char *S_arg);
 void mgt_cli_close_all(void);
 
+/* mgt_main.c */
+struct choice {
+	const char      *name;
+	const void	*ptr;
+};
+const void *pick(const struct choice *cp, const char *which, const char *kind);
+
 /* mgt_param.c */
 void MCF_ParamInit(struct cli *);
 void MCF_ParamSet(struct cli *, const char *param, const char *val);
 void MCF_DumpRst(void);
 extern struct params mgt_param;
 
-
 /* mgt_sandbox.c */
 void mgt_sandbox(void);
 
@@ -88,7 +94,6 @@ extern const char *mgt_vcl_dir;
 extern const char *mgt_vmod_dir;
 extern unsigned mgt_vcc_err_unref;
 
-
 #define REPORT0(pri, fmt)				\
 	do {						\
 		fprintf(stderr, fmt "\n");		\
@@ -105,7 +110,6 @@ extern unsigned mgt_vcc_err_unref;
 #define VSM_Free(a)		VSM__Free(a)
 #define VSM_Clean()		VSM__Clean()
 
-
 #if defined(PTHREAD_CANCELED) || defined(PTHREAD_MUTEX_DEFAULT)
 #error "Keep pthreads out of in manager process"
 #endif
diff --git a/bin/varnishd/storage/stevedore.c b/bin/varnishd/storage/stevedore.c
index 3978f64..9b22cd3 100644
--- a/bin/varnishd/storage/stevedore.c
+++ b/bin/varnishd/storage/stevedore.c
@@ -39,18 +39,12 @@
 #include "cache/cache.h"
 
 #include "storage/storage.h"
-#include "vav.h"
 #include "vcli_priv.h"
 #include "vrt.h"
 #include "vrt_obj.h"
 
-static VTAILQ_HEAD(, stevedore)	stevedores =
-    VTAILQ_HEAD_INITIALIZER(stevedores);
-
 static const struct stevedore * volatile stv_next;
 
-static struct stevedore *stv_transient;
-
 /*---------------------------------------------------------------------
  * Default objcore methods
  */
@@ -130,7 +124,7 @@ stv_pick_stevedore(const struct sess *sp, const char **hint)
 
 	AN(hint);
 	if (*hint != NULL && **hint != '\0') {
-		VTAILQ_FOREACH(stv, &stevedores, list) {
+		VTAILQ_FOREACH(stv, &stv_stevedores, list) {
 			if (!strcmp(stv->ident, *hint))
 				return (stv);
 		}
@@ -144,7 +138,7 @@ stv_pick_stevedore(const struct sess *sp, const char **hint)
 	/* pick a stevedore and bump the head along */
 	stv = VTAILQ_NEXT(stv_next, list);
 	if (stv == NULL)
-		stv = VTAILQ_FIRST(&stevedores);
+		stv = VTAILQ_FIRST(&stv_stevedores);
 	AN(stv);
 	AN(stv->name);
 	stv_next = stv;
@@ -271,7 +265,7 @@ STV_MkObject(struct sess *sp, void *ptr, unsigned ltot,
  * implement persistent storage can rely on.
  */
 
-static struct object *
+struct object *
 stv_default_allocobj(struct stevedore *stv, struct sess *sp, unsigned ltot,
     const struct stv_objsecrets *soc)
 {
@@ -402,7 +396,7 @@ STV_open(void)
 {
 	struct stevedore *stv;
 
-	VTAILQ_FOREACH(stv, &stevedores, list) {
+	VTAILQ_FOREACH(stv, &stv_stevedores, list) {
 		stv->lru = LRU_Alloc();
 		if (stv->open != NULL)
 			stv->open(stv);
@@ -412,6 +406,7 @@ STV_open(void)
 		stv->lru = LRU_Alloc();
 		stv->open(stv);
 	}
+	stv_next = VTAILQ_FIRST(&stv_stevedores);
 }
 
 void
@@ -419,7 +414,7 @@ STV_close(void)
 {
 	struct stevedore *stv;
 
-	VTAILQ_FOREACH(stv, &stevedores, list)
+	VTAILQ_FOREACH(stv, &stv_stevedores, list)
 		if (stv->close != NULL)
 			stv->close(stv);
 	stv = stv_transient;
@@ -427,113 +422,6 @@ STV_close(void)
 		stv->close(stv);
 }
 
-/*--------------------------------------------------------------------
- * Parse a stevedore argument on the form:
- *	[ name '=' ] strategy [ ',' arg ] *
- */
-
-static const struct choice STV_choice[] = {
-	{ "file",	&smf_stevedore },
-	{ "malloc",	&sma_stevedore },
-	{ "persistent",	&smp_stevedore },
-#ifdef HAVE_LIBUMEM
-	{ "umem",	&smu_stevedore },
-#endif
-	{ NULL,		NULL }
-};
-
-void
-STV_Config(const char *spec)
-{
-	char **av;
-	const char *p, *q;
-	struct stevedore *stv;
-	const struct stevedore *stv2;
-	int ac, l;
-	static unsigned seq = 0;
-
-	ASSERT_MGT();
-	p = strchr(spec, '=');
-	q = strchr(spec, ',');
-	if (p != NULL && (q == NULL || q > p)) {
-		av = VAV_Parse(p + 1, NULL, ARGV_COMMA);
-	} else {
-		av = VAV_Parse(spec, NULL, ARGV_COMMA);
-		p = NULL;
-	}
-	AN(av);
-
-	if (av[0] != NULL)
-		ARGV_ERR("%s\n", av[0]);
-
-	if (av[1] == NULL)
-		ARGV_ERR("-s argument lacks strategy {malloc, file, ...}\n");
-
-	for (ac = 0; av[ac + 2] != NULL; ac++)
-		continue;
-
-	stv2 = pick(STV_choice, av[1], "storage");
-	AN(stv2);
-
-	/* Append strategy to ident string */
-	VSB_printf(vident, ",-s%s", av[1]);
-
-	av += 2;
-
-	CHECK_OBJ_NOTNULL(stv2, STEVEDORE_MAGIC);
-	ALLOC_OBJ(stv, STEVEDORE_MAGIC);
-	AN(stv);
-
-	*stv = *stv2;
-	AN(stv->name);
-	AN(stv->alloc);
-	if (stv->allocobj == NULL)
-		stv->allocobj = stv_default_allocobj;
-
-	if (p == NULL)
-		bprintf(stv->ident, "s%u", seq++);
-	else {
-		l = p - spec;
-		if (l > sizeof stv->ident - 1)
-			l = sizeof stv->ident - 1;
-		bprintf(stv->ident, "%.*s", l, spec);
-	}
-
-	VTAILQ_FOREACH(stv2, &stevedores, list) {
-		if (strcmp(stv2->ident, stv->ident))
-			continue;
-		ARGV_ERR("(-s%s=%s) already defined once\n",
-		    stv->ident, stv->name);
-	}
-
-	if (stv->init != NULL)
-		stv->init(stv, ac, av);
-	else if (ac != 0)
-		ARGV_ERR("(-s%s) too many arguments\n", stv->name);
-
-	if (!strcmp(stv->ident, TRANSIENT_STORAGE)) {
-		stv->transient = 1;
-		AZ(stv_transient);
-		stv_transient = stv;
-	} else {
-		VTAILQ_INSERT_TAIL(&stevedores, stv, list);
-		if (!stv_next)
-			stv_next = VTAILQ_FIRST(&stevedores);
-	}
-}
-
-/*--------------------------------------------------------------------*/
-
-void
-STV_Config_Transient(void)
-{
-
-	ASSERT_MGT();
-
-	if (stv_transient == NULL)
-		STV_Config(TRANSIENT_STORAGE "=malloc");
-}
-
 /*--------------------------------------------------------------------*/
 
 static void
@@ -547,7 +435,7 @@ stv_cli_list(struct cli *cli, const char * const *av, void *priv)
 	VCLI_Out(cli, "Storage devices:\n");
 	stv = stv_transient;
 		VCLI_Out(cli, "\tstorage.%s = %s\n", stv->ident, stv->name);
-	VTAILQ_FOREACH(stv, &stevedores, list)
+	VTAILQ_FOREACH(stv, &stv_stevedores, list)
 		VCLI_Out(cli, "\tstorage.%s = %s\n", stv->ident, stv->name);
 }
 
@@ -568,7 +456,7 @@ stv_find(const char *nm)
 {
 	const struct stevedore *stv;
 
-	VTAILQ_FOREACH(stv, &stevedores, list)
+	VTAILQ_FOREACH(stv, &stv_stevedores, list)
 		if (!strcmp(stv->ident, nm))
 			return (stv);
 	if (!strcmp(TRANSIENT_STORAGE, nm))
diff --git a/bin/varnishd/storage/stevedore_mgt.c b/bin/varnishd/storage/stevedore_mgt.c
new file mode 100644
index 0000000..fc0d6a6
--- /dev/null
+++ b/bin/varnishd/storage/stevedore_mgt.c
@@ -0,0 +1,156 @@
+/*-
+ * Copyright (c) 2007-2011 Varnish Software AS
+ * All rights reserved.
+ *
+ * Author: Dag-Erling Sm?rgav <des at des.no>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * STEVEDORE: one who works at or is responsible for loading and
+ * unloading ships in port.  Example: "on the wharves, stevedores were
+ * unloading cargo from the far corners of the world." Origin: Spanish
+ * estibador, from estibar to pack.  First Known Use: 1788
+ */
+
+#include "config.h"
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+
+#include "mgt/mgt.h"
+
+#include "storage/storage.h"
+#include "vav.h"
+
+struct stevedore_head stv_stevedores =
+    VTAILQ_HEAD_INITIALIZER(stv_stevedores);
+
+struct stevedore *stv_transient;
+
+/*--------------------------------------------------------------------
+ * Parse a stevedore argument on the form:
+ *	[ name '=' ] strategy [ ',' arg ] *
+ */
+
+static const struct choice STV_choice[] = {
+	{ "file",	&smf_stevedore },
+	{ "malloc",	&sma_stevedore },
+	{ "persistent",	&smp_stevedore },
+#ifdef HAVE_LIBUMEM
+	{ "umem",	&smu_stevedore },
+#endif
+	{ NULL,		NULL }
+};
+
+void
+STV_Config(const char *spec)
+{
+	char **av;
+	const char *p, *q;
+	struct stevedore *stv;
+	const struct stevedore *stv2;
+	int ac, l;
+	static unsigned seq = 0;
+
+	ASSERT_MGT();
+	p = strchr(spec, '=');
+	q = strchr(spec, ',');
+	if (p != NULL && (q == NULL || q > p)) {
+		av = VAV_Parse(p + 1, NULL, ARGV_COMMA);
+	} else {
+		av = VAV_Parse(spec, NULL, ARGV_COMMA);
+		p = NULL;
+	}
+	AN(av);
+
+	if (av[0] != NULL)
+		ARGV_ERR("%s\n", av[0]);
+
+	if (av[1] == NULL)
+		ARGV_ERR("-s argument lacks strategy {malloc, file, ...}\n");
+
+	for (ac = 0; av[ac + 2] != NULL; ac++)
+		continue;
+
+	stv2 = pick(STV_choice, av[1], "storage");
+	AN(stv2);
+
+	/* Append strategy to ident string */
+	VSB_printf(vident, ",-s%s", av[1]);
+
+	av += 2;
+
+	CHECK_OBJ_NOTNULL(stv2, STEVEDORE_MAGIC);
+	ALLOC_OBJ(stv, STEVEDORE_MAGIC);
+	AN(stv);
+
+	*stv = *stv2;
+	AN(stv->name);
+	AN(stv->alloc);
+	if (stv->allocobj == NULL)
+		stv->allocobj = stv_default_allocobj;
+
+	if (p == NULL)
+		bprintf(stv->ident, "s%u", seq++);
+	else {
+		l = p - spec;
+		if (l > sizeof stv->ident - 1)
+			l = sizeof stv->ident - 1;
+		bprintf(stv->ident, "%.*s", l, spec);
+	}
+
+	VTAILQ_FOREACH(stv2, &stv_stevedores, list) {
+		if (strcmp(stv2->ident, stv->ident))
+			continue;
+		ARGV_ERR("(-s%s=%s) already defined once\n",
+		    stv->ident, stv->name);
+	}
+
+	if (stv->init != NULL)
+		stv->init(stv, ac, av);
+	else if (ac != 0)
+		ARGV_ERR("(-s%s) too many arguments\n", stv->name);
+
+	if (!strcmp(stv->ident, TRANSIENT_STORAGE)) {
+		stv->transient = 1;
+		AZ(stv_transient);
+		stv_transient = stv;
+	} else {
+		VTAILQ_INSERT_TAIL(&stv_stevedores, stv, list);
+	}
+}
+
+/*--------------------------------------------------------------------*/
+
+void
+STV_Config_Transient(void)
+{
+
+	ASSERT_MGT();
+
+	if (stv_transient == NULL)
+		STV_Config(TRANSIENT_STORAGE "=malloc");
+}
+
+/*--------------------------------------------------------------------*/
diff --git a/bin/varnishd/storage/storage.h b/bin/varnishd/storage/storage.h
index 80cad13..a813a36 100644
--- a/bin/varnishd/storage/storage.h
+++ b/bin/varnishd/storage/storage.h
@@ -50,6 +50,8 @@ typedef void storage_close_f(const struct stevedore *);
 #include "tbl/vrt_stv_var.h"
 #undef VRTSTVTYPE
 
+extern storage_allocobj_f stv_default_allocobj;
+
 /*--------------------------------------------------------------------*/
 
 struct stevedore {
@@ -78,6 +80,11 @@ struct stevedore {
 	char			ident[16];	/* XXX: match VSM_chunk.ident */
 };
 
+VTAILQ_HEAD(stevedore_head, stevedore);
+
+extern struct stevedore_head stv_stevedores;
+extern struct stevedore *stv_transient;
+
 /*--------------------------------------------------------------------*/
 int STV_GetFile(const char *fn, int *fdp, const char **fnp, const char *ctx);
 uintmax_t STV_FileSize(int fd, const char *size, unsigned *granularity,



From geoff at varnish-cache.org  Mon Jan  9 20:51:46 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:51:46 +0100
Subject: [experimental-ims] 8b4c13c Don't leave pid file in weird error case.
Message-ID: <E1RkMC2-0006ln-Qr@project.varnish-software.com>

commit 8b4c13c4f3c4aa73d2466445459d4c950b6689d6
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Sep 7 20:16:30 2011 +0000

    Don't leave pid file in weird error case.

diff --git a/bin/varnishd/mgt_child.c b/bin/varnishd/mgt_child.c
index ebdf57b..9d6966a 100644
--- a/bin/varnishd/mgt_child.c
+++ b/bin/varnishd/mgt_child.c
@@ -595,8 +595,10 @@ MGT_Run(void)
 		REPORT0(LOG_ERR, "No VCL loaded yet");
 	else if (!d_flag) {
 		start_child(NULL);
-		if (child_state == CH_STOPPED)
-			exit(2);
+		if (child_state == CH_STOPPED) {
+			exit_status = 2;
+			return;
+		}
 	}
 
 	i = vev_schedule(mgt_evb);



From geoff at varnish-cache.org  Mon Jan  9 20:51:49 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:51:49 +0100
Subject: [experimental-ims] 8bebce8 Use VTLA VWK for Kqueue waiter and make it
	an object
Message-ID: <E1RkMC5-0006rG-N3@project.varnish-software.com>

commit 8bebce803ffe25049fa4c5694dcc4b642f5155f6
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sat Sep 17 10:46:30 2011 +0000

    Use VTLA VWK for Kqueue waiter and make it an object

diff --git a/bin/varnishd/cache_waiter.h b/bin/varnishd/cache_waiter.h
index 9d6eee9..9bf139c 100644
--- a/bin/varnishd/cache_waiter.h
+++ b/bin/varnishd/cache_waiter.h
@@ -31,7 +31,7 @@
 struct sess;
 
 typedef void* waiter_init_f(void);
-typedef void waiter_pass_f(void *priv, struct sess *);
+typedef void waiter_pass_f(void *priv, const struct sess *);
 
 extern int vca_pipes[2];
 
diff --git a/bin/varnishd/cache_waiter_epoll.c b/bin/varnishd/cache_waiter_epoll.c
index 5726b03..c405bfc 100644
--- a/bin/varnishd/cache_waiter_epoll.c
+++ b/bin/varnishd/cache_waiter_epoll.c
@@ -222,7 +222,7 @@ vca_sess_timeout_ticker(void *arg)
 /*--------------------------------------------------------------------*/
 
 static void
-vca_epoll_pass(void *priv, hhstruct sess *sp)
+vca_epoll_pass(void *priv, const struct sess *sp)
 {
 
 	(void)priv;
diff --git a/bin/varnishd/cache_waiter_kqueue.c b/bin/varnishd/cache_waiter_kqueue.c
index 12e7bd4..cbea248 100644
--- a/bin/varnishd/cache_waiter_kqueue.c
+++ b/bin/varnishd/cache_waiter_kqueue.c
@@ -47,64 +47,64 @@
 #include "cache.h"
 #include "cache_waiter.h"
 
-
-/*--------------------------------------------------------------------*/
-
-
-static pthread_t vca_kqueue_thread;
-static int kq = -1;
-
-
-static VTAILQ_HEAD(,sess) sesshead = VTAILQ_HEAD_INITIALIZER(sesshead);
-
 #define NKEV	100
 
-static struct kevent ki[NKEV];
-static unsigned nki;
+struct vwk {
+	unsigned		magic;
+#define VWK_MAGIC		0x1cc2acc2
+	pthread_t 		thread;
+	int			pipes[2];
+	int 			kq;
+	struct kevent		ki[NKEV];
+	unsigned		nki;
+	VTAILQ_HEAD(,sess)	sesshead;
+};
+
+/*--------------------------------------------------------------------*/
 
 static void
-vca_kq_flush(void)
+vwk_kq_flush(struct vwk *vwk)
 {
 	int i;
 
-	if (nki == 0)
+	if (vwk->nki == 0)
 		return;
-	i = kevent(kq, ki, nki, NULL, 0, NULL);
+	i = kevent(vwk->kq, vwk->ki, vwk->nki, NULL, 0, NULL);
 	assert(i == 0);
-	nki = 0;
+	vwk->nki = 0;
 }
 
 static void
-vca_kq_sess(struct sess *sp, short arm)
+vwk_kq_sess(struct vwk *vwk, struct sess *sp, short arm)
 {
 
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 	assert(sp->fd >= 0);
 	DSL(0x04, SLT_Debug, sp->fd, "KQ: EV_SET sp %p arm %x", sp, arm);
-	EV_SET(&ki[nki], sp->fd, EVFILT_READ, arm, 0, 0, sp);
-	if (++nki == NKEV)
-		vca_kq_flush();
+	EV_SET(&vwk->ki[vwk->nki], sp->fd, EVFILT_READ, arm, 0, 0, sp);
+	if (++vwk->nki == NKEV)
+		vwk_kq_flush(vwk);
 }
 
 static void
-vca_kev(const struct kevent *kp)
+vwk_kev(struct vwk *vwk, const struct kevent *kp)
 {
 	int i, j;
 	struct sess *sp;
 	struct sess *ss[NKEV];
 
 	AN(kp->udata);
-	if (kp->udata == vca_pipes) {
+	if (kp->udata == vwk->pipes) {
 		j = 0;
-		i = read(vca_pipes[0], ss, sizeof ss);
+		i = read(vwk->pipes[0], ss, sizeof ss);
 		if (i == -1 && errno == EAGAIN)
 			return;
 		while (i >= sizeof ss[0]) {
 			CHECK_OBJ_NOTNULL(ss[j], SESS_MAGIC);
 			assert(ss[j]->fd >= 0);
 			AZ(ss[j]->obj);
-			VTAILQ_INSERT_TAIL(&sesshead, ss[j], list);
-			vca_kq_sess(ss[j], EV_ADD | EV_ONESHOT);
+			VTAILQ_INSERT_TAIL(&vwk->sesshead, ss[j], list);
+			vwk_kq_sess(vwk, ss[j], EV_ADD | EV_ONESHOT);
 			j++;
 			i -= sizeof ss[0];
 		}
@@ -121,14 +121,14 @@ vca_kev(const struct kevent *kp)
 	if (kp->data > 0) {
 		i = HTC_Rx(sp->htc);
 		if (i == 0) {
-			vca_kq_sess(sp, EV_ADD | EV_ONESHOT);
+			vwk_kq_sess(vwk, sp, EV_ADD | EV_ONESHOT);
 			return;	/* more needed */
 		}
-		VTAILQ_REMOVE(&sesshead, sp, list);
+		VTAILQ_REMOVE(&vwk->sesshead, sp, list);
 		vca_handover(sp, i);
 		return;
 	} else if (kp->flags & EV_EOF) {
-		VTAILQ_REMOVE(&sesshead, sp, list);
+		VTAILQ_REMOVE(&vwk->sesshead, sp, list);
 		vca_close_session(sp, "EOF");
 		SES_Delete(sp);
 		return;
@@ -142,39 +142,40 @@ vca_kev(const struct kevent *kp)
 /*--------------------------------------------------------------------*/
 
 static void *
-vca_kqueue_main(void *arg)
+vwk_thread(void *priv)
 {
+	struct vwk *vwk;
 	struct kevent ke[NKEV], *kp;
 	int j, n, dotimer;
 	double deadline;
 	struct sess *sp;
 
+	CAST_OBJ_NOTNULL(vwk, priv, VWK_MAGIC);
 	THR_SetName("cache-kqueue");
-	(void)arg;
 
-	kq = kqueue();
-	assert(kq >= 0);
+	vwk->kq = kqueue();
+	assert(vwk->kq >= 0);
 
 	j = 0;
 	EV_SET(&ke[j], 0, EVFILT_TIMER, EV_ADD, 0, 100, NULL);
 	j++;
-	EV_SET(&ke[j], vca_pipes[0], EVFILT_READ, EV_ADD, 0, 0, vca_pipes);
+	EV_SET(&ke[j], vwk->pipes[0], EVFILT_READ, EV_ADD, 0, 0, vwk->pipes);
 	j++;
-	AZ(kevent(kq, ke, j, NULL, 0, NULL));
+	AZ(kevent(vwk->kq, ke, j, NULL, 0, NULL));
 
-	nki = 0;
+	vwk->nki = 0;
 	while (1) {
 		dotimer = 0;
-		n = kevent(kq, ki, nki, ke, NKEV, NULL);
+		n = kevent(vwk->kq, vwk->ki, vwk->nki, ke, NKEV, NULL);
 		assert(n >= 1 && n <= NKEV);
-		nki = 0;
+		vwk->nki = 0;
 		for (kp = ke, j = 0; j < n; j++, kp++) {
 			if (kp->filter == EVFILT_TIMER) {
 				dotimer = 1;
 				continue;
 			}
 			assert(kp->filter == EVFILT_READ);
-			vca_kev(kp);
+			vwk_kev(vwk, kp);
 		}
 		if (!dotimer)
 			continue;
@@ -185,15 +186,15 @@ vca_kqueue_main(void *arg)
 		 * the kevent(2) at the top of this loop, the kernel
 		 * would not know we meant "the old fd of this number".
 		 */
-		vca_kq_flush();
+		vwk_kq_flush(vwk);
 		deadline = TIM_real() - params->sess_timeout;
 		for (;;) {
-			sp = VTAILQ_FIRST(&sesshead);
+			sp = VTAILQ_FIRST(&vwk->sesshead);
 			if (sp == NULL)
 				break;
 			if (sp->t_open > deadline)
 				break;
-			VTAILQ_REMOVE(&sesshead, sp, list);
+			VTAILQ_REMOVE(&vwk->sesshead, sp, list);
 			// XXX: not yet (void)VTCP_linger(sp->fd, 0);
 			vca_close_session(sp, "timeout");
 			SES_Delete(sp);
@@ -204,11 +205,12 @@ vca_kqueue_main(void *arg)
 /*--------------------------------------------------------------------*/
 
 static void
-vca_kqueue_pass(void *priv, struct sess *sp)
+vca_kqueue_pass(void *priv, const struct sess *sp)
 {
+	struct vwk *vwk;
 
-	(void)priv;
-	assert(sizeof sp == write(vca_pipes[1], &sp, sizeof sp));
+	CAST_OBJ_NOTNULL(vwk, priv, VWK_MAGIC);
+	assert(sizeof sp == write(vwk->pipes[1], &sp, sizeof sp));
 }
 
 /*--------------------------------------------------------------------*/
@@ -217,15 +219,22 @@ static void *
 vca_kqueue_init(void)
 {
 	int i;
+	struct vwk *vwk;
+
+	ALLOC_OBJ(vwk, VWK_MAGIC);
+	AN(vwk);
+
+	VTAILQ_INIT(&vwk->sesshead);
+	AZ(pipe(vwk->pipes));
 
-	i = fcntl(vca_pipes[0], F_GETFL);
+	i = fcntl(vwk->pipes[0], F_GETFL);
 	assert(i != -1);
 	i |= O_NONBLOCK;
-	i = fcntl(vca_pipes[0], F_SETFL, i);
+	i = fcntl(vwk->pipes[0], F_SETFL, i);
 	assert(i != -1);
 
-	AZ(pthread_create(&vca_kqueue_thread, NULL, vca_kqueue_main, NULL));
-	return (NULL);
+	AZ(pthread_create(&vwk->thread, NULL, vwk_thread, vwk));
+	return (vwk);
 }
 
 /*--------------------------------------------------------------------*/
diff --git a/bin/varnishd/cache_waiter_poll.c b/bin/varnishd/cache_waiter_poll.c
index 44b4033..c51776d 100644
--- a/bin/varnishd/cache_waiter_poll.c
+++ b/bin/varnishd/cache_waiter_poll.c
@@ -201,7 +201,7 @@ vwp_main(void *priv)
 /*--------------------------------------------------------------------*/
 
 static void
-vwp_poll_pass(void *priv, struct sess *sp)
+vwp_poll_pass(void *priv, const struct sess *sp)
 {
 	struct vwp *vwp;
 
diff --git a/bin/varnishd/cache_waiter_ports.c b/bin/varnishd/cache_waiter_ports.c
index f1cc55c..6a587cc 100644
--- a/bin/varnishd/cache_waiter_ports.c
+++ b/bin/varnishd/cache_waiter_ports.c
@@ -246,7 +246,7 @@ vca_main(void *arg)
 /*--------------------------------------------------------------------*/
 
 static void
-vca_ports_pass(void *priv, struct sess *sp)
+vca_ports_pass(void *priv, const struct sess *sp)
 {
 	int r;
 



From geoff at varnish-cache.org  Mon Jan  9 20:52:41 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:41 +0100
Subject: [experimental-ims] 7090d80 Allocate rbuf if -r is specified
Message-ID: <E1RkMCv-0008Nx-B7@project.varnish-software.com>

commit 7090d8029742c592014dba0f34fb9db1ec4e999a
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Nov 21 09:11:13 2011 +0000

    Allocate rbuf if -r is specified

diff --git a/lib/libvarnishapi/vsl_arg.c b/lib/libvarnishapi/vsl_arg.c
index 4bcd44a..b855628 100644
--- a/lib/libvarnishapi/vsl_arg.c
+++ b/lib/libvarnishapi/vsl_arg.c
@@ -93,6 +93,10 @@ vsl_r_arg(const struct VSM_data *vd, const char *opt)
 	if (vd->vsl->r_fd < 0) {
 		perror(opt);
 		return (-1);
+	} else if (vd->vsl->rbuflen == 0) {
+		vd->vsl->rbuf = malloc(1024);
+		AN(vd->vsl->rbuf);
+		vd->vsl->rbuflen = 1024;
 	}
 	return (1);
 }



From geoff at varnish-cache.org  Mon Jan  9 20:52:29 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:29 +0100
Subject: [experimental-ims] 98a29cb Implement support for custom output and
	profiles in varnishhist
Message-ID: <E1RkMCj-00082m-As@project.varnish-software.com>

commit 98a29cbc3e345f008a3c169e522992445a95365d
Author: Tollef Fog Heen <tfheen at varnish-software.com>
Date:   Fri Nov 4 12:00:47 2011 +0100

    Implement support for custom output and profiles in varnishhist
    
    Drop varnishsizes, since it's pointless now (you can get the same
    functionality with varnishhist -p size)

diff --git a/bin/Makefile.am b/bin/Makefile.am
index 7bdddbb..9906c5e 100644
--- a/bin/Makefile.am
+++ b/bin/Makefile.am
@@ -3,5 +3,5 @@
 SUBDIRS = varnishadm varnishd varnishlog varnishncsa varnishreplay varnishtest
 
 if HAVE_CURSES
-SUBDIRS += varnishhist varnishstat varnishtop varnishsizes
+SUBDIRS += varnishhist varnishstat varnishtop
 endif
diff --git a/bin/varnishhist/varnishhist.c b/bin/varnishhist/varnishhist.c
index 72bc84b..692eb81 100644
--- a/bin/varnishhist/varnishhist.c
+++ b/bin/varnishhist/varnishhist.c
@@ -52,11 +52,12 @@
 #include "vcs.h"
 
 #define HIST_N 2000 /* how far back we remember */
-#define HIST_LOW -6 /* low end of log range */
-#define HIST_HIGH 3 /* high end of log range */
-#define HIST_RANGE (HIST_HIGH - HIST_LOW)
 #define HIST_RES 100 /* bucket resolution */
-#define HIST_BUCKETS (HIST_RANGE * HIST_RES)
+
+static int hist_low;
+static int hist_high;
+static int hist_range;
+static int hist_buckets;
 
 static pthread_mutex_t mtx = PTHREAD_MUTEX_INITIALIZER;
 
@@ -64,10 +65,13 @@ static int delay = 1;
 static unsigned rr_hist[HIST_N];
 static unsigned nhist;
 static unsigned next_hist;
-static unsigned bucket_miss[HIST_BUCKETS];
-static unsigned bucket_hit[HIST_BUCKETS];
+static unsigned *bucket_miss;
+static unsigned *bucket_hit;
 static unsigned char hh[FD_SETSIZE];
 static uint64_t bitmap[FD_SETSIZE];
+static double  values[FD_SETSIZE];
+static char *format;
+static int match_tag;
 
 static double log_ten;
 
@@ -95,11 +99,25 @@ static int scales[] = {
 	INT_MAX
 };
 
+struct profile {
+	const char *name;
+	enum VSL_tag_e tag;
+	int field;
+	int hist_low;
+	int hist_high;
+} profiles[] = {
+	{ .name = "responsetime", .tag = SLT_ReqEnd, .field = 5, .hist_low = -6, .hist_high = 3 },
+	{ .name = "size", .tag = SLT_Length, .field = 1, .hist_low = 1, .hist_high = 8 },
+	{ 0 }
+};
+
+static struct profile *active_profile;
+
 static void
 update(struct VSM_data *vd)
 {
-	int w = COLS / HIST_RANGE;
-	int n = w * HIST_RANGE;
+	int w = COLS / hist_range;
+	int n = w * hist_range;
 	unsigned bm[n], bh[n];
 	unsigned max;
 	int i, j, scale;
@@ -107,11 +125,11 @@ update(struct VSM_data *vd)
 	erase();
 
 	/* Draw horizontal axis */
-	w = COLS / HIST_RANGE;
-	n = w * HIST_RANGE;
+	w = COLS / hist_range;
+	n = w * hist_range;
 	for (i = 0; i < n; ++i)
 		(void)mvaddch(LINES - 2, i, '-');
-	for (i = 0, j = HIST_LOW; i < HIST_RANGE; ++i, ++j) {
+	for (i = 0, j = hist_low; i < hist_range; ++i, ++j) {
 		(void)mvaddch(LINES - 2, w * i, '+');
 		mvprintw(LINES - 1, w * i, "|1e%d", j);
 	}
@@ -121,8 +139,8 @@ update(struct VSM_data *vd)
 	/* count our flock */
 	for (i = 0; i < n; ++i)
 		bm[i] = bh[i] = 0;
-	for (i = 0, max = 1; i < HIST_BUCKETS; ++i) {
-		j = i * n / HIST_BUCKETS;
+	for (i = 0, max = 1; i < hist_buckets; ++i) {
+		j = i * n / hist_buckets;
 		bm[j] += bucket_miss[i];
 		bh[j] += bucket_hit[i];
 		if (bm[j] + bh[j] > max)
@@ -151,11 +169,8 @@ static int
 h_hist(void *priv, enum VSL_tag_e tag, unsigned fd, unsigned len,
     unsigned spec, const char *ptr, uint64_t bm)
 {
-	double b;
 	int i, j;
 	struct VSM_data *vd = priv;
-
-	(void)len;
 	(void)spec;
 
 	if (fd >= FD_SETSIZE)
@@ -168,6 +183,15 @@ h_hist(void *priv, enum VSL_tag_e tag, unsigned fd, unsigned len,
 		hh[fd] = 1;
 		return (0);
 	}
+	if (tag == match_tag) {
+		char buf[1024]; /* size? */
+		assert(len < sizeof(buf));
+		memcpy(buf, ptr, len);
+		buf[len] = '\0';
+		i = sscanf(buf, format, &values[fd]);
+		assert(i == 1);
+	}
+
 	if (tag != SLT_ReqEnd)
 		return (0);
 
@@ -177,24 +201,15 @@ h_hist(void *priv, enum VSL_tag_e tag, unsigned fd, unsigned len,
 		return (0);
 	}
 
-	/* determine processing time */
-#if 1
-	i = sscanf(ptr, "%*d %*f %*f %*f %lf", &b);
-#else
-	i = sscanf(ptr, "%*d %*f %*f %lf", &b);
-#endif
-	assert(i == 1);
-
 	/* select bucket */
-	i = HIST_RES * (log(b) / log_ten);
-	if (i < HIST_LOW * HIST_RES)
-		i = HIST_LOW * HIST_RES;
-	if (i >= HIST_HIGH * HIST_RES)
-		i = HIST_HIGH * HIST_RES - 1;
-	i -= HIST_LOW * HIST_RES;
+	i = HIST_RES * (log(values[fd]) / log_ten);
+	if (i < hist_low * HIST_RES)
+		i = hist_low * HIST_RES;
+	if (i >= hist_high * HIST_RES)
+		i = hist_high * HIST_RES - 1;
+	i -= hist_low * HIST_RES;
 	assert(i >= 0);
-	assert(i < HIST_BUCKETS);
-
+	assert(i < hist_buckets);
 	pthread_mutex_lock(&mtx);
 
 	/* phase out old data */
@@ -320,33 +335,94 @@ static void
 usage(void)
 {
 	fprintf(stderr, "usage: varnishhist "
-	    "%s [-n varnish_name] [-V] [-w delay]\n", VSL_USAGE);
+	    "%s [-p profile] [-f field_num] [-R max] [-r min] [-V] [-w delay]\n", VSL_USAGE);
 	exit(1);
 }
 
 int
 main(int argc, char **argv)
 {
-	int o;
+	int o, i;
 	struct VSM_data *vd;
+	const char *profile = "responsetime";
+	int fnum = -1;
+	hist_low = -1;
+	hist_high = -1;
+	match_tag = -1;
 
 	vd = VSM_New();
 	VSL_Setup(vd);
 
-	while ((o = getopt(argc, argv, VSL_ARGS "Vw:")) != -1) {
+	while ((o = getopt(argc, argv, VSL_ARGS "Vw:r:R:f:p:")) != -1) {
 		switch (o) {
 		case 'V':
 			VCS_Message("varnishhist");
 			exit(0);
+		case 'i':
+			match_tag = VSL_Name2Tag(optarg, -1);
+			if (match_tag < 0) {
+				fprintf(stderr, "No such tag %s\n", optarg);
+				exit(1);
+			}
+			break;
 		case 'w':
 			delay = atoi(optarg);
 			break;
+		case 'f':
+			fnum = atoi(optarg);
+			break;
+		case 'R':
+			hist_high = atoi(optarg);
+			break;
+		case 'r':
+			hist_low = atoi(optarg);
+			break;
+		case 'p':
+			profile = optarg;
+			break;
 		default:
 			if (VSL_Arg(vd, o, optarg) > 0)
 				break;
 			usage();
 		}
 	}
+	if (profile) {
+		for (active_profile = profiles; active_profile->name; active_profile++) {
+			if (strcmp(active_profile->name, profile) == 0) {
+				break;
+			}
+		}
+	}
+	if (! active_profile->name) {
+		fprintf(stderr, "No such profile %s\n", profile);
+		exit(1);
+	}
+	if (match_tag < 0) {
+		match_tag = active_profile->tag;
+	}
+
+	if (fnum < 0) {
+		fnum = active_profile->field;
+	}
+
+	if (hist_low < 0) {
+		hist_low = active_profile->hist_low;
+	}
+
+	if (hist_high < 0) {
+		hist_high = active_profile->hist_high;
+	}
+
+	hist_range = hist_high - hist_low;
+	hist_buckets = hist_range * HIST_RES;
+	bucket_hit = calloc(sizeof bucket_hit, hist_buckets);
+	bucket_miss = calloc(sizeof bucket_miss, hist_buckets);
+
+	format = malloc(4 * fnum);
+	for (i = 0; i < fnum-1; i++) {
+		strcpy(format + 4*i, "%*s ");
+	}
+	strcpy(format + 4*(fnum-1), "%lf");
 
 	if (VSL_Open(vd, 1))
 		exit(1);
diff --git a/bin/varnishsizes/Makefile.am b/bin/varnishsizes/Makefile.am
deleted file mode 100644
index 8029741..0000000
--- a/bin/varnishsizes/Makefile.am
+++ /dev/null
@@ -1,27 +0,0 @@
-#
-
-INCLUDES = -I$(top_srcdir)/include
-
-bin_PROGRAMS = varnishsizes
-
-dist_man_MANS = varnishsizes.1
-
-varnishsizes_SOURCES = varnishsizes.c \
-	$(top_builddir)/lib/libvarnish/vas.c \
-	$(top_builddir)/lib/libvarnish/version.c
-
-varnishsizes_LDADD = \
-	$(top_builddir)/lib/libvarnishcompat/libvarnishcompat.la \
-	$(top_builddir)/lib/libvarnishapi/libvarnishapi.la \
-	-lm \
-	${CURSES_LIBS} ${PTHREAD_LIBS}
-
-varnishsizes.1: $(top_srcdir)/doc/sphinx/reference/varnishsizes.rst
-if HAVE_RST2MAN
-	${RST2MAN} $? $@
-else
-	@echo "========================================"
-	@echo "You need rst2man installed to make dist"
-	@echo "========================================"
-	@false
-endif
diff --git a/bin/varnishsizes/varnishsizes.c b/bin/varnishsizes/varnishsizes.c
deleted file mode 100644
index ce68a9d..0000000
--- a/bin/varnishsizes/varnishsizes.c
+++ /dev/null
@@ -1,360 +0,0 @@
-/*-
- * Copyright (c) 2006 Verdens Gang AS
- * Copyright (c) 2006-2010 Varnish Software AS
- * All rights reserved.
- *
- * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
- * Author: Dag-Erling Sm?rgrav <des at des.no>
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- *
- * Log tailer for Varnish
- */
-
-#include "config.h"
-
-#include <sys/types.h>
-
-#include <curses.h>
-#include <errno.h>
-#include <limits.h>
-#include <math.h>
-#include <pthread.h>
-#include <regex.h>
-#include <signal.h>
-#include <stdint.h>
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#include <unistd.h>
-
-#include "vapi/vsl.h"
-#include "vapi/vsm.h"
-#include "vas.h"
-#include "vcs.h"
-
-#define HIST_N 2000 /* how far back we remember */
-#define HIST_LOW 1 /* low end of log range */
-#define HIST_HIGH 8 /* high end of log range */
-#define HIST_RANGE (HIST_HIGH - HIST_LOW)
-#define HIST_RES 100 /* bucket resolution */
-#define HIST_BUCKETS (HIST_RANGE * HIST_RES)
-
-static pthread_mutex_t mtx = PTHREAD_MUTEX_INITIALIZER;
-
-static int delay = 1;
-static unsigned rr_hist[HIST_N];
-static unsigned nhist;
-static unsigned next_hist;
-static unsigned bucket_miss[HIST_BUCKETS];
-static unsigned bucket_hit[HIST_BUCKETS];
-static unsigned char hh[FD_SETSIZE];
-static uint64_t bitmap[FD_SETSIZE];
-
-static double log_ten;
-
-static int scales[] = {
-	1,
-	2,
-	3,
-	4,
-	5,
-	10,
-	15,
-	20,
-	25,
-	50,
-	100,
-	250,
-	500,
-	1000,
-	2500,
-	5000,
-	10000,
-	25000,
-	50000,
-	100000,
-	INT_MAX
-};
-
-static void
-update(struct VSM_data *vd)
-{
-	int w = COLS / HIST_RANGE;
-	int n = w * HIST_RANGE;
-	unsigned bm[n], bh[n];
-	unsigned max;
-	int i, j, scale;
-
-	erase();
-
-	/* Draw horizontal axis */
-	w = COLS / HIST_RANGE;
-	n = w * HIST_RANGE;
-	for (i = 0; i < n; ++i)
-		(void)mvaddch(LINES - 2, i, '-');
-	for (i = 0, j = HIST_LOW; i < HIST_RANGE; ++i, ++j) {
-		(void)mvaddch(LINES - 2, w * i, '+');
-		mvprintw(LINES - 1, w * i, "|1e%d", j);
-	}
-
-	mvprintw(0, 0, "%*s", COLS - 1, VSM_Name(vd));
-
-	/* count our flock */
-	for (i = 0; i < n; ++i)
-		bm[i] = bh[i] = 0;
-	for (i = 0, max = 1; i < HIST_BUCKETS; ++i) {
-		j = i * n / HIST_BUCKETS;
-		bm[j] += bucket_miss[i];
-		bh[j] += bucket_hit[i];
-		if (bm[j] + bh[j] > max)
-			max = bm[j] + bh[j];
-	}
-
-	/* scale */
-	for (i = 0; max / scales[i] > LINES - 3; ++i)
-		/* nothing */ ;
-	scale = scales[i];
-
-	mvprintw(0, 0, "1:%d, n = %d", scale, nhist);
-
-	/* show them */
-	for (i = 0; i < n; ++i) {
-		for (j = 0; j < bm[i] / scale; ++j)
-			(void)mvaddch(LINES - 3 - j, i, '#');
-		for (; j < (bm[i] + bh[i]) / scale; ++j)
-			(void)mvaddch(LINES - 3 - j, i, '|');
-	}
-
-	refresh();
-}
-
-static int
-h_hist(void *priv, enum VSL_tag_e tag, unsigned fd, unsigned len,
-    unsigned spec, const char *ptr, uint64_t bm)
-{
-	double b;
-	int i, j, tmp;
-	struct VSM_data *vd = priv;
-
-	(void)len;
-	(void)spec;
-
-	if (fd >= FD_SETSIZE)
-		/* oops */
-		return (0);
-
-	bitmap[fd] |= bm;
-
-	if (tag == SLT_Hit) {
-		hh[fd] = 1;
-		return (0);
-	}
-	if (tag != SLT_Length)
-		return (0);
-
-	if (!VSL_Matched(vd, bitmap[fd])) {
-		bitmap[fd] = 0;
-		hh[fd] = 0;
-		return (0);
-	}
-
-	/* determine processing time */
-	i = sscanf(ptr, "%d", &tmp);
-	assert(i == 1);
-
-	/* Typically 304s and tend to throw the scaling off */
-	if (tmp == 0)
-		return 0;
-
-	b = tmp;
-	/* select bucket */
-	i = HIST_RES * (log(b) / log_ten);
-	if (i < HIST_LOW * HIST_RES)
-		i = HIST_LOW * HIST_RES;
-	if (i >= HIST_HIGH * HIST_RES)
-		i = HIST_HIGH * HIST_RES - 1;
-	i -= HIST_LOW * HIST_RES;
-	assert(i >= 0);
-	assert(i < HIST_BUCKETS);
-
-	pthread_mutex_lock(&mtx);
-
-	/* phase out old data */
-	if (nhist == HIST_N) {
-		j = rr_hist[next_hist];
-		if (j < 0)  {
-			assert(bucket_miss[-j] > 0);
-			bucket_miss[-j]--;
-		} else {
-			assert(bucket_hit[j] > 0);
-			bucket_hit[j]--;
-		}
-	} else {
-		++nhist;
-	}
-
-	/* phase in new data */
-	if (hh[fd] || i == 0) {
-		bucket_hit[i]++;
-		rr_hist[next_hist] = i;
-	} else {
-		bucket_miss[i]++;
-		rr_hist[next_hist] = -i;
-	}
-	if (++next_hist == HIST_N) {
-		next_hist = 0;
-	}
-	hh[fd] = 0;
-	bitmap[fd] = 0;
-
-	pthread_mutex_unlock(&mtx);
-
-	return (0);
-}
-
-static void *
-accumulate_thread(void *arg)
-{
-	struct VSM_data *vd = arg;
-	int i;
-
-	for (;;) {
-		i = VSL_Dispatch(vd, h_hist, vd);
-		if (i < 0)
-			break;
-		if (i == 0)
-			usleep(50000);
-	}
-	return (arg);
-}
-
-static void
-do_curses(struct VSM_data *vd)
-{
-	pthread_t thr;
-	int ch;
-
-	if (pthread_create(&thr, NULL, accumulate_thread, vd) != 0) {
-		fprintf(stderr, "pthread_create(): %s\n", strerror(errno));
-		exit(1);
-	}
-
-	initscr();
-	raw();
-	noecho();
-	nonl();
-	intrflush(stdscr, FALSE);
-	curs_set(0);
-	erase();
-	for (;;) {
-		pthread_mutex_lock(&mtx);
-		update(vd);
-		pthread_mutex_unlock(&mtx);
-
-		timeout(delay * 1000);
-		switch ((ch = getch())) {
-		case ERR:
-			break;
-#ifdef KEY_RESIZE
-		case KEY_RESIZE:
-			erase();
-			break;
-#endif
-		case '\014': /* Ctrl-L */
-		case '\024': /* Ctrl-T */
-			redrawwin(stdscr);
-			refresh();
-			break;
-		case '\003': /* Ctrl-C */
-			raise(SIGINT);
-			break;
-		case '\032': /* Ctrl-Z */
-			endwin();
-			raise(SIGTSTP);
-			break;
-		case '\021': /* Ctrl-Q */
-		case 'Q':
-		case 'q':
-			endwin();
-			return;
-		case '0':
-		case '1':
-		case '2':
-		case '3':
-		case '4':
-		case '5':
-		case '6':
-		case '7':
-		case '8':
-		case '9':
-			delay = 1 << (ch - '0');
-			break;
-		default:
-			beep();
-			break;
-		}
-	}
-}
-
-/*--------------------------------------------------------------------*/
-
-static void
-usage(void)
-{
-	fprintf(stderr, "usage: varnishsizes "
-	    "%s [-n varnish_name] [-V] [-w delay]\n", VSL_USAGE);
-	exit(1);
-}
-
-int
-main(int argc, char **argv)
-{
-	int o;
-	struct VSM_data *vd;
-
-	vd = VSM_New();
-	VSL_Setup(vd);
-
-	while ((o = getopt(argc, argv, VSL_ARGS "Vw:")) != -1) {
-		switch (o) {
-		case 'V':
-			VCS_Message("varnishsizes");
-			exit(0);
-		case 'w':
-			delay = atoi(optarg);
-			break;
-		default:
-			if (VSL_Arg(vd, o, optarg) > 0)
-				break;
-			usage();
-		}
-	}
-
-	if (VSL_Open(vd, 1))
-		exit(1);
-
-	log_ten = log(10.0);
-
-	do_curses(vd);
-	exit(0);
-}



From geoff at varnish-cache.org  Mon Jan  9 20:52:15 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:15 +0100
Subject: [experimental-ims] b4bb850 More #include cleanuppery
Message-ID: <E1RkMCV-0007ZQ-Ed@project.varnish-software.com>

commit b4bb85057482f1fcb79d86f340a4e6665ca4352d
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sun Oct 9 08:24:21 2011 +0000

    More #include cleanuppery

diff --git a/include/Makefile.am b/include/Makefile.am
index e40ea2a..c68e07c 100644
--- a/include/Makefile.am
+++ b/include/Makefile.am
@@ -3,9 +3,15 @@
 pkginclude_HEADERS = \
 	tbl/acct_fields.h \
 	tbl/backend_poll.h \
+	tbl/ban_vars.h \
 	tbl/body_status.h \
+	tbl/http_headers.h \
+	tbl/http_response.h \
 	tbl/locks.h \
 	tbl/steps.h \
+	tbl/vcc_types.h \
+	tbl/vcl_returns.h \
+	tbl/vrt_stv_var.h \
 	tbl/vsc_all.h \
 	tbl/vsc_fields.h \
 	tbl/vsl_tags.h \
@@ -28,11 +34,6 @@ nobase_noinst_HEADERS = \
 	libvcl.h \
 	miniobj.h \
 	persistent.h \
-	tbl/ban_vars.h \
-	tbl/http_headers.h \
-	tbl/http_response.h \
-	tbl/vcl_returns.h \
-	tbl/vrt_stv_var.h \
 	vas.h \
 	vav.h \
 	vbm.h \
diff --git a/include/tbl/vcc_types.h b/include/tbl/vcc_types.h
new file mode 100644
index 0000000..469b56c
--- /dev/null
+++ b/include/tbl/vcc_types.h
@@ -0,0 +1,44 @@
+/*-
+ * Copyright (c) 2010 Varnish Software AS
+ * All rights reserved.
+ *
+ * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ */
+
+/*lint -save -e525 -e539 */
+VCC_TYPE(VOID)
+VCC_TYPE(BACKEND)
+VCC_TYPE(BOOL)
+VCC_TYPE(INT)
+VCC_TYPE(TIME)
+VCC_TYPE(DURATION)
+VCC_TYPE(STRING)
+VCC_TYPE(STRING_LIST)
+VCC_TYPE(IP)
+VCC_TYPE(HEADER)
+VCC_TYPE(BYTES)
+VCC_TYPE(REAL)
+VCC_TYPE(ENUM)
+/*lint -restore */
diff --git a/lib/libvcl/Makefile.am b/lib/libvcl/Makefile.am
index fa61423..099ad36 100644
--- a/lib/libvcl/Makefile.am
+++ b/lib/libvcl/Makefile.am
@@ -10,7 +10,6 @@ libvcl_la_SOURCES = \
 	vcc_priv.h \
 	vcc_compile.h \
 	vcc_token_defs.h \
-	vcc_types.h \
 	symbol_kind.h \
 	\
 	vcc_acl.c \
diff --git a/lib/libvcl/vcc_acl.c b/lib/libvcl/vcc_acl.c
index 6eb451c..bd94a5b 100644
--- a/lib/libvcl/vcc_acl.c
+++ b/lib/libvcl/vcc_acl.c
@@ -30,7 +30,6 @@
 #include "config.h"
 
 #include <stdio.h>
-#include <stdlib.h>
 #include <stddef.h>
 #include <string.h>
 #include <sys/types.h>
@@ -38,10 +37,10 @@
 #include <netdb.h>
 #include <netinet/in.h>
 
-#include "vrt.h"
 
-#include "vcc_priv.h"
 #include "vcc_compile.h"
+#include "vrt.h"
+#include "vcc_priv.h"
 #include "libvarnish.h"
 
 struct acl_e {
diff --git a/lib/libvcl/vcc_action.c b/lib/libvcl/vcc_action.c
index c11f5b2..1fafe98 100644
--- a/lib/libvcl/vcc_action.c
+++ b/lib/libvcl/vcc_action.c
@@ -32,11 +32,9 @@
 
 #include "config.h"
 
-#include <stdio.h>
-#include <string.h>
 
-#include "vcc_priv.h"
 #include "vcc_compile.h"
+#include "vcc_priv.h"
 #include "libvarnish.h"
 
 /*--------------------------------------------------------------------*/
diff --git a/lib/libvcl/vcc_backend.c b/lib/libvcl/vcc_backend.c
index dfe58d2..67fecd6 100644
--- a/lib/libvcl/vcc_backend.c
+++ b/lib/libvcl/vcc_backend.c
@@ -55,16 +55,13 @@
 
 #include <limits.h>
 #include <netdb.h>
-#include <stdio.h>
-#include <stdarg.h>
 #include <stdlib.h>
 #include <string.h>
 
-#include "vss.h"
-
-#include "vcc_priv.h"
 #include "vcc_compile.h"
+#include "vcc_priv.h"
 #include "libvarnish.h"
+#include "vss.h"
 
 struct host {
 	VTAILQ_ENTRY(host)      list;
diff --git a/lib/libvcl/vcc_compile.c b/lib/libvcl/vcc_compile.c
index a09f396..be3ec63 100644
--- a/lib/libvcl/vcc_compile.c
+++ b/lib/libvcl/vcc_compile.c
@@ -52,18 +52,15 @@
 
 #include "config.h"
 
-#include <sys/stat.h>
-
 #include <ctype.h>
 #include <errno.h>
 #include <stdio.h>
 #include <stdarg.h>
 #include <stdlib.h>
 #include <string.h>
-#include <unistd.h>
 
-#include "vcc_priv.h"
 #include "vcc_compile.h"
+#include "vcc_priv.h"
 
 #include "libvcl.h"
 #include "libvarnish.h"
diff --git a/lib/libvcl/vcc_compile.h b/lib/libvcl/vcc_compile.h
index a3aa800..438fe58 100644
--- a/lib/libvcl/vcc_compile.h
+++ b/lib/libvcl/vcc_compile.h
@@ -44,7 +44,7 @@ struct symbol;
 
 enum var_type {
 #define VCC_TYPE(foo)		foo,
-#include "vcc_types.h"
+#include "tbl/vcc_types.h"
 #undef VCC_TYPE
 };
 
diff --git a/lib/libvcl/vcc_dir_dns.c b/lib/libvcl/vcc_dir_dns.c
index a8cb24b..a5952c8 100644
--- a/lib/libvcl/vcc_dir_dns.c
+++ b/lib/libvcl/vcc_dir_dns.c
@@ -28,16 +28,12 @@
 
 #include "config.h"
 
-#include <sys/types.h>
-#include <sys/socket.h>
-
 #include <stdio.h>
-#include <stdarg.h>
 #include <string.h>
 #include <limits.h>
 
-#include "vcc_priv.h"
 #include "vcc_compile.h"
+#include "vcc_priv.h"
 #include "libvarnish.h"
 
 /*--------------------------------------------------------------------
diff --git a/lib/libvcl/vcc_dir_random.c b/lib/libvcl/vcc_dir_random.c
index aa362f9..46f74c2 100644
--- a/lib/libvcl/vcc_dir_random.c
+++ b/lib/libvcl/vcc_dir_random.c
@@ -29,15 +29,8 @@
 
 #include "config.h"
 
-#include <sys/types.h>
-#include <sys/socket.h>
-
-#include <stdio.h>
-#include <stdarg.h>
-#include <string.h>
-
-#include "vcc_priv.h"
 #include "vcc_compile.h"
+#include "vcc_priv.h"
 #include "libvarnish.h"
 
 /*--------------------------------------------------------------------
diff --git a/lib/libvcl/vcc_dir_round_robin.c b/lib/libvcl/vcc_dir_round_robin.c
index 67fb118..b1094fe 100644
--- a/lib/libvcl/vcc_dir_round_robin.c
+++ b/lib/libvcl/vcc_dir_round_robin.c
@@ -28,15 +28,8 @@
 
 #include "config.h"
 
-#include <sys/types.h>
-#include <sys/socket.h>
-
-#include <stdio.h>
-#include <stdarg.h>
-#include <string.h>
-
-#include "vcc_priv.h"
 #include "vcc_compile.h"
+#include "vcc_priv.h"
 #include "libvarnish.h"
 
 /*--------------------------------------------------------------------
diff --git a/lib/libvcl/vcc_expr.c b/lib/libvcl/vcc_expr.c
index b7b6ba6..4fbcd85 100644
--- a/lib/libvcl/vcc_expr.c
+++ b/lib/libvcl/vcc_expr.c
@@ -46,7 +46,7 @@ vcc_Type(enum var_type fmt)
 {
 	switch(fmt) {
 #define VCC_TYPE(a)	case a: return(#a);
-#include "vcc_types.h"
+#include "tbl/vcc_types.h"
 #undef VCC_TYPE
 	default:
 		assert("Unknwon Type");
@@ -403,7 +403,7 @@ vcc_arg_type(const char **p)
 {
 
 #define VCC_TYPE(a) if (!strcmp(#a, *p)) { *p += strlen(#a) + 1; return (a);}
-#include "vcc_types.h"
+#include "tbl/vcc_types.h"
 #undef VCC_TYPE
 	return (VOID);
 }
diff --git a/lib/libvcl/vcc_types.h b/lib/libvcl/vcc_types.h
deleted file mode 100644
index 469b56c..0000000
--- a/lib/libvcl/vcc_types.h
+++ /dev/null
@@ -1,44 +0,0 @@
-/*-
- * Copyright (c) 2010 Varnish Software AS
- * All rights reserved.
- *
- * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- *
- */
-
-/*lint -save -e525 -e539 */
-VCC_TYPE(VOID)
-VCC_TYPE(BACKEND)
-VCC_TYPE(BOOL)
-VCC_TYPE(INT)
-VCC_TYPE(TIME)
-VCC_TYPE(DURATION)
-VCC_TYPE(STRING)
-VCC_TYPE(STRING_LIST)
-VCC_TYPE(IP)
-VCC_TYPE(HEADER)
-VCC_TYPE(BYTES)
-VCC_TYPE(REAL)
-VCC_TYPE(ENUM)
-/*lint -restore */



From geoff at varnish-cache.org  Mon Jan  9 20:51:59 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:51:59 +0100
Subject: [experimental-ims] 8b4cc65 Remember to release workspace on listen
	socket close.
Message-ID: <E1RkMCF-00077g-Ul@project.varnish-software.com>

commit 8b4cc6576a5e4869cd133fef7d2b002dffc49af9
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Sep 20 14:20:51 2011 +0000

    Remember to release workspace on listen socket close.

diff --git a/bin/varnishd/cache_pool.c b/bin/varnishd/cache_pool.c
index 5ff635b..6be7613 100644
--- a/bin/varnishd/cache_pool.c
+++ b/bin/varnishd/cache_pool.c
@@ -222,6 +222,7 @@ Pool_Work_Thread(void *priv, struct worker *w)
 			if (i < 0) {
 				/* Socket Shutdown */
 				FREE_OBJ(ps);
+				WS_Release(w->ws, 0);
 				continue;
 			}
 			VTAILQ_INSERT_TAIL(&pp->socks, ps, list);



From geoff at varnish-cache.org  Mon Jan  9 20:52:18 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:18 +0100
Subject: [experimental-ims] 2bcbc00 Use = rather than := in Makefile to make
	solaris make stop whining on distcheck
Message-ID: <E1RkMCY-0007hT-U9@project.varnish-software.com>

commit 2bcbc0032e76cd037e47887bc5d40628128818e6
Author: Tollef Fog Heen <tfheen at varnish-software.com>
Date:   Thu Oct 13 08:33:36 2011 +0200

    Use = rather than := in Makefile to make solaris make stop whining on distcheck

diff --git a/lib/libjemalloc/Makefile.am b/lib/libjemalloc/Makefile.am
index 32b3462..63f5c7a 100644
--- a/lib/libjemalloc/Makefile.am
+++ b/lib/libjemalloc/Makefile.am
@@ -1,6 +1,6 @@
 # See source code comments to avoid memory leaks when enabling MALLOC_MAG.
-#CPPFLAGS := -DMALLOC_PRODUCTION -DMALLOC_MAG
-AM_CPPFLAGS := -DMALLOC_PRODUCTION
+#CPPFLAGS = -DMALLOC_PRODUCTION -DMALLOC_MAG
+AM_CPPFLAGS = -DMALLOC_PRODUCTION
 
 #all: libjemalloc.so.0 libjemalloc_mt.so.0
 



From geoff at varnish-cache.org  Mon Jan  9 20:52:16 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:16 +0100
Subject: [experimental-ims] 368197a And now it is VNUM's turn...
Message-ID: <E1RkMCW-0007bo-Cl@project.varnish-software.com>

commit 368197ad6241b398fd6a50d25385d4bf9ba2cc81
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sun Oct 9 19:06:34 2011 +0000

    And now it is VNUM's turn...

diff --git a/bin/varnishd/mgt_shmem.c b/bin/varnishd/mgt_shmem.c
index 38dc59a..e42487a 100644
--- a/bin/varnishd/mgt_shmem.c
+++ b/bin/varnishd/mgt_shmem.c
@@ -103,6 +103,7 @@
 #include "vmb.h"
 #include "vsm.h"
 #include "vav.h"
+#include "vnum.h"
 #include "flopen.h"
 
 #ifndef MAP_HASSEMAPHORE
@@ -244,7 +245,7 @@ mgt_SHM_Init(const char *l_arg)
 
 	/* Size of SHMLOG */
 	if (*ap != NULL && **ap != '\0') {
-		q = str2bytes(*ap, &s1, 0);
+		q = VNUM_2bytes(*ap, &s1, 0);
 		if (q != NULL)
 			ARGV_ERR("\t-l[1] ...:  %s\n", q);
 	} else {
@@ -255,7 +256,7 @@ mgt_SHM_Init(const char *l_arg)
 
 	/* Size of space for other stuff */
 	if (*ap != NULL && **ap != '\0') {
-		q = str2bytes(*ap, &s2, 0);
+		q = VNUM_2bytes(*ap, &s2, 0);
 		if (q != NULL)
 			ARGV_ERR("\t-l[2] ...:  %s\n", q);
 	} else {
diff --git a/bin/varnishd/stevedore_utils.c b/bin/varnishd/stevedore_utils.c
index af7fc7f..725a086 100644
--- a/bin/varnishd/stevedore_utils.c
+++ b/bin/varnishd/stevedore_utils.c
@@ -53,6 +53,7 @@
 
 #include "mgt.h"
 #include "stevedore.h"
+#include "vnum.h"
 
 #ifndef O_LARGEFILE
 #define O_LARGEFILE	0
@@ -197,7 +198,7 @@ STV_FileSize(int fd, const char *size, unsigned *granularity, const char *ctx)
 		l = st.st_size;
 	} else {
 		AN(size);
-		q = str2bytes(size, &l, fssize);
+		q = VNUM_2bytes(size, &l, fssize);
 
 		if (q != NULL)
 			ARGV_ERR("(%s) size \"%s\": %s\n", size, ctx, q);
diff --git a/bin/varnishd/storage_file.c b/bin/varnishd/storage_file.c
index c363f4e..25c9947 100644
--- a/bin/varnishd/storage_file.c
+++ b/bin/varnishd/storage_file.c
@@ -38,6 +38,7 @@
 #include "cache.h"
 
 #include "stevedore.h"
+#include "vnum.h"
 
 #ifndef MAP_NOCORE
 #define MAP_NOCORE 0 /* XXX Linux */
@@ -131,7 +132,7 @@ smf_init(struct stevedore *parent, int ac, char * const *av)
 		size = av[1];
 	if (ac > 2 && *av[2] != '\0') {
 
-		r = str2bytes(av[2], &page_size, 0);
+		r = VNUM_2bytes(av[2], &page_size, 0);
 		if (r != NULL)
 			ARGV_ERR("(-sfile) granularity \"%s\": %s\n", av[2], r);
 	}
diff --git a/bin/varnishd/storage_malloc.c b/bin/varnishd/storage_malloc.c
index 500e74c..bc5a565 100644
--- a/bin/varnishd/storage_malloc.c
+++ b/bin/varnishd/storage_malloc.c
@@ -37,6 +37,7 @@
 #include "cache.h"
 
 #include "stevedore.h"
+#include "vnum.h"
 
 struct sma_sc {
 	unsigned		magic;
@@ -216,7 +217,7 @@ sma_init(struct stevedore *parent, int ac, char * const *av)
 	if (ac == 0 || *av[0] == '\0')
 		 return;
 
-	e = str2bytes(av[0], &u, 0);
+	e = VNUM_2bytes(av[0], &u, 0);
 	if (e != NULL)
 		ARGV_ERR("(-smalloc) size \"%s\": %s\n", av[0], e);
 	if ((u != (uintmax_t)(size_t)u))
diff --git a/bin/varnishd/storage_umem.c b/bin/varnishd/storage_umem.c
index 6e28a94..197d0ac 100644
--- a/bin/varnishd/storage_umem.c
+++ b/bin/varnishd/storage_umem.c
@@ -140,7 +140,7 @@ smu_init(struct stevedore *parent, int ac, char * const *av)
 	if (ac == 0 || *av[0] == '\0')
 		 return;
 
-	e = str2bytes(av[0], &u, 0);
+	e = VNUM_2bytes(av[0], &u, 0);
 	if (e != NULL)
 		ARGV_ERR("(-sumem) size \"%s\": %s\n", av[0], e);
 	if ((u != (uintmax_t)(size_t)u))
diff --git a/include/Makefile.am b/include/Makefile.am
index d62cab3..e852c58 100644
--- a/include/Makefile.am
+++ b/include/Makefile.am
@@ -48,6 +48,7 @@ nobase_noinst_HEADERS = \
 	vlu.h \
 	vmb.h \
 	vmod_abi.h \
+	vnum.h \
 	vpf.h \
 	vqueue.h \
 	vre.h \
diff --git a/include/libvarnish.h b/include/libvarnish.h
index c0851c8..1450578 100644
--- a/include/libvarnish.h
+++ b/include/libvarnish.h
@@ -38,9 +38,6 @@
 
 struct vsb;
 
-/* from libvarnish/num.c */
-const char *str2bytes(const char *p, uintmax_t *r, uintmax_t rel);
-
 /* from libvarnish/version.c */
 void VCS_Message(const char *);
 
diff --git a/lib/libvarnish/Makefile.am b/lib/libvarnish/Makefile.am
index 3c045e6..e890904 100644
--- a/lib/libvarnish/Makefile.am
+++ b/lib/libvarnish/Makefile.am
@@ -13,7 +13,7 @@ libvarnish_la_SOURCES = \
 	cli_common.c \
 	cli_serve.c \
 	flopen.c \
-	num.c \
+	vnum.c \
 	vtim.c \
 	vtcp.c \
 	vct.c \
@@ -33,13 +33,13 @@ libvarnish_la_CFLAGS = -DVARNISH_STATE_DIR='"${VARNISH_STATE_DIR}"'
 libvarnish_la_LIBADD = ${RT_LIBS} ${NET_LIBS} ${LIBM} @PCRE_LIBS@
 
 if ENABLE_TESTS
-TESTS = num_c_test
+TESTS = vnum_c_test
 
 noinst_PROGRAMS = ${TESTS}
 
-num_c_test_SOURCES = num.c
-num_c_test_CFLAGS = -DNUM_C_TEST -include config.h
-num_c_test_LDADD = ${LIBM}
+vnum_c_test_SOURCES = vnum.c
+vnum_c_test_CFLAGS = -DNUM_C_TEST -include config.h
+vnum_c_test_LDADD = ${LIBM}
 
 test: ${TESTS}
 	@for test in ${TESTS} ; do ./$${test} ; done
diff --git a/lib/libvarnish/num.c b/lib/libvarnish/num.c
deleted file mode 100644
index 70255db..0000000
--- a/lib/libvarnish/num.c
+++ /dev/null
@@ -1,197 +0,0 @@
-/*-
- * Copyright (c) 2008-2009 Varnish Software AS
- * All rights reserved.
- *
- * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- *
- * Deal with numbers with data storage suffix scaling
- */
-
-#include "config.h"
-
-#include <math.h>
-#include <stdio.h>
-#include <stdlib.h>
-#include <stdint.h>
-
-#include "libvarnish.h"
-
-static const char err_miss_num[] = "Missing number";
-static const char err_invalid_num[] = "Invalid number";
-static const char err_abs_req[] = "Absolute number required";
-static const char err_invalid_suff[] = "Invalid suffix";
-
-const char *
-str2bytes(const char *p, uintmax_t *r, uintmax_t rel)
-{
-	double fval;
-	char *end;
-
-	if (p == NULL || *p == '\0')
-		return (err_miss_num);
-
-	fval = strtod(p, &end);
-	if (end == p || !isfinite(fval))
-		return (err_invalid_num);
-
-	if (*end == '\0') {
-		*r = (uintmax_t)fval;
-		return (NULL);
-	}
-
-	if (end[0] == '%' && end[1] == '\0') {
-		if (rel == 0)
-			return (err_abs_req);
-		fval *= rel / 100.0;
-	} else {
-		/* accept a space before the multiplier */
-		if (end[0] == ' ' && end[1] != '\0')
-			++end;
-
-		switch (end[0]) {
-		case 'k': case 'K':
-			fval *= (uintmax_t)1 << 10;
-			++end;
-			break;
-		case 'm': case 'M':
-			fval *= (uintmax_t)1 << 20;
-			++end;
-			break;
-		case 'g': case 'G':
-			fval *= (uintmax_t)1 << 30;
-			++end;
-			break;
-		case 't': case 'T':
-			fval *= (uintmax_t)1 << 40;
-			++end;
-			break;
-		case 'p': case 'P':
-			fval *= (uintmax_t)1 << 50;
-			++end;
-			break;
-		case 'e': case 'E':
-			fval *= (uintmax_t)1 << 60;
-			++end;
-			break;
-		default:
-			break;
-		}
-
-		/* [bB] is a generic suffix of no effect */
-		if (end[0] == 'b' || end[0] == 'B')
-			end++;
-
-		if (end[0] != '\0')
-			return (err_invalid_suff);
-	}
-
-	*r = (uintmax_t)round(fval);
-	return (NULL);
-}
-
-#ifdef NUM_C_TEST
-/* Compile with: "cc -o foo -DNUM_C_TEST -I../.. -I../../include num.c -lm" */
-#include "vas.h"
-#include <math.h>
-#include <stdio.h>
-#include <string.h>
-
-struct test_case {
-	const char *str;
-	uintmax_t rel;
-	uintmax_t val;
-	const char *err;
-} test_cases[] = {
-	{ "1",			(uintmax_t)0,	(uintmax_t)1 },
-	{ "1B",			(uintmax_t)0,	(uintmax_t)1<<0 },
-	{ "1 B",		(uintmax_t)0,	(uintmax_t)1<<0 },
-	{ "1.3B",		(uintmax_t)0,	(uintmax_t)1 },
-	{ "1.7B",		(uintmax_t)0,	(uintmax_t)2 },
-
-	{ "1024",		(uintmax_t)0,	(uintmax_t)1024 },
-	{ "1k",			(uintmax_t)0,	(uintmax_t)1<<10 },
-	{ "1kB",		(uintmax_t)0,	(uintmax_t)1<<10 },
-	{ "1.3kB",		(uintmax_t)0,	(uintmax_t)1331 },
-	{ "1.7kB",		(uintmax_t)0,	(uintmax_t)1741 },
-
-	{ "1048576",		(uintmax_t)0,	(uintmax_t)1048576 },
-	{ "1M",			(uintmax_t)0,	(uintmax_t)1<<20 },
-	{ "1MB",		(uintmax_t)0,	(uintmax_t)1<<20 },
-	{ "1.3MB",		(uintmax_t)0,	(uintmax_t)1363149 },
-	{ "1.7MB",		(uintmax_t)0,	(uintmax_t)1782579 },
-
-	{ "1073741824",		(uintmax_t)0,	(uintmax_t)1073741824 },
-	{ "1G",			(uintmax_t)0,	(uintmax_t)1<<30 },
-	{ "1GB",		(uintmax_t)0,	(uintmax_t)1<<30 },
-	{ "1.3GB",		(uintmax_t)0,	(uintmax_t)1395864371 },
-	{ "1.7GB",		(uintmax_t)0,	(uintmax_t)1825361101 },
-
-	{ "1099511627776",	(uintmax_t)0,	(uintmax_t)1099511627776ULL },
-	{ "1T",			(uintmax_t)0,	(uintmax_t)1<<40 },
-	{ "1TB",		(uintmax_t)0,	(uintmax_t)1<<40 },
-	{ "1.3TB",		(uintmax_t)0,	(uintmax_t)1429365116109ULL },
-	{ "1.7TB",		(uintmax_t)0,	(uintmax_t)1869169767219ULL },
-
-	{ "1%",			(uintmax_t)1024,	(uintmax_t)10 },
-	{ "2%",			(uintmax_t)1024,	(uintmax_t)20 },
-	{ "3%",			(uintmax_t)1024,	(uintmax_t)31 },
-
-	/* Check the error checks */
-	{ "",			0,	0,	err_miss_num },
-	{ "m",			0,	0,	err_invalid_num },
-	{ "4%",			0,	0,	err_abs_req },
-	{ "3*",			0,	0,	err_invalid_suff },
-
-	/* TODO: add more */
-
-	{ 0, 0, 0 },
-};
-
-int
-main(int argc, char *argv[])
-{
-	struct test_case *tc;
-	uintmax_t val;
-	int ec;
-	const char *e;
-
-	(void)argc;
-	for (ec = 0, tc = test_cases; tc->str; ++tc) {
-		e = str2bytes(tc->str, &val, tc->rel);
-		if (e != tc->err) {
-			printf("%s: str2bytes(\"%s\", %ju) (%s) != (%s)\n",
-			    *argv, tc->str, tc->rel, tc->err, e);
-			++ec;
-		} else if (e == NULL && val != tc->val) {
-			printf("%s: str2bytes(\"%s\", %ju) %ju != %ju (%s)\n",
-			    *argv, tc->str, tc->rel, val, tc->val, e);
-			++ec;
-		}
-	}
-	/* TODO: test invalid strings */
-	if (!ec)
-		printf("OK\n");
-	return (ec > 0);
-}
-#endif
diff --git a/lib/libvarnish/vnum.c b/lib/libvarnish/vnum.c
new file mode 100644
index 0000000..7940e9b
--- /dev/null
+++ b/lib/libvarnish/vnum.c
@@ -0,0 +1,197 @@
+/*-
+ * Copyright (c) 2008-2009 Varnish Software AS
+ * All rights reserved.
+ *
+ * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * Deal with numbers with data storage suffix scaling
+ */
+
+#include "config.h"
+
+#include <math.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <stdint.h>
+
+#include "vnum.h"
+
+static const char err_miss_num[] = "Missing number";
+static const char err_invalid_num[] = "Invalid number";
+static const char err_abs_req[] = "Absolute number required";
+static const char err_invalid_suff[] = "Invalid suffix";
+
+const char *
+VNUM_2bytes(const char *p, uintmax_t *r, uintmax_t rel)
+{
+	double fval;
+	char *end;
+
+	if (p == NULL || *p == '\0')
+		return (err_miss_num);
+
+	fval = strtod(p, &end);
+	if (end == p || !isfinite(fval))
+		return (err_invalid_num);
+
+	if (*end == '\0') {
+		*r = (uintmax_t)fval;
+		return (NULL);
+	}
+
+	if (end[0] == '%' && end[1] == '\0') {
+		if (rel == 0)
+			return (err_abs_req);
+		fval *= rel / 100.0;
+	} else {
+		/* accept a space before the multiplier */
+		if (end[0] == ' ' && end[1] != '\0')
+			++end;
+
+		switch (end[0]) {
+		case 'k': case 'K':
+			fval *= (uintmax_t)1 << 10;
+			++end;
+			break;
+		case 'm': case 'M':
+			fval *= (uintmax_t)1 << 20;
+			++end;
+			break;
+		case 'g': case 'G':
+			fval *= (uintmax_t)1 << 30;
+			++end;
+			break;
+		case 't': case 'T':
+			fval *= (uintmax_t)1 << 40;
+			++end;
+			break;
+		case 'p': case 'P':
+			fval *= (uintmax_t)1 << 50;
+			++end;
+			break;
+		case 'e': case 'E':
+			fval *= (uintmax_t)1 << 60;
+			++end;
+			break;
+		default:
+			break;
+		}
+
+		/* [bB] is a generic suffix of no effect */
+		if (end[0] == 'b' || end[0] == 'B')
+			end++;
+
+		if (end[0] != '\0')
+			return (err_invalid_suff);
+	}
+
+	*r = (uintmax_t)round(fval);
+	return (NULL);
+}
+
+#ifdef NUM_C_TEST
+/* Compile with: "cc -o foo -DNUM_C_TEST -I../.. -I../../include num.c -lm" */
+#include "vas.h"
+#include <math.h>
+#include <stdio.h>
+#include <string.h>
+
+struct test_case {
+	const char *str;
+	uintmax_t rel;
+	uintmax_t val;
+	const char *err;
+} test_cases[] = {
+	{ "1",			(uintmax_t)0,	(uintmax_t)1 },
+	{ "1B",			(uintmax_t)0,	(uintmax_t)1<<0 },
+	{ "1 B",		(uintmax_t)0,	(uintmax_t)1<<0 },
+	{ "1.3B",		(uintmax_t)0,	(uintmax_t)1 },
+	{ "1.7B",		(uintmax_t)0,	(uintmax_t)2 },
+
+	{ "1024",		(uintmax_t)0,	(uintmax_t)1024 },
+	{ "1k",			(uintmax_t)0,	(uintmax_t)1<<10 },
+	{ "1kB",		(uintmax_t)0,	(uintmax_t)1<<10 },
+	{ "1.3kB",		(uintmax_t)0,	(uintmax_t)1331 },
+	{ "1.7kB",		(uintmax_t)0,	(uintmax_t)1741 },
+
+	{ "1048576",		(uintmax_t)0,	(uintmax_t)1048576 },
+	{ "1M",			(uintmax_t)0,	(uintmax_t)1<<20 },
+	{ "1MB",		(uintmax_t)0,	(uintmax_t)1<<20 },
+	{ "1.3MB",		(uintmax_t)0,	(uintmax_t)1363149 },
+	{ "1.7MB",		(uintmax_t)0,	(uintmax_t)1782579 },
+
+	{ "1073741824",		(uintmax_t)0,	(uintmax_t)1073741824 },
+	{ "1G",			(uintmax_t)0,	(uintmax_t)1<<30 },
+	{ "1GB",		(uintmax_t)0,	(uintmax_t)1<<30 },
+	{ "1.3GB",		(uintmax_t)0,	(uintmax_t)1395864371 },
+	{ "1.7GB",		(uintmax_t)0,	(uintmax_t)1825361101 },
+
+	{ "1099511627776",	(uintmax_t)0,	(uintmax_t)1099511627776ULL },
+	{ "1T",			(uintmax_t)0,	(uintmax_t)1<<40 },
+	{ "1TB",		(uintmax_t)0,	(uintmax_t)1<<40 },
+	{ "1.3TB",		(uintmax_t)0,	(uintmax_t)1429365116109ULL },
+	{ "1.7TB",		(uintmax_t)0,	(uintmax_t)1869169767219ULL },
+
+	{ "1%",			(uintmax_t)1024,	(uintmax_t)10 },
+	{ "2%",			(uintmax_t)1024,	(uintmax_t)20 },
+	{ "3%",			(uintmax_t)1024,	(uintmax_t)31 },
+
+	/* Check the error checks */
+	{ "",			0,	0,	err_miss_num },
+	{ "m",			0,	0,	err_invalid_num },
+	{ "4%",			0,	0,	err_abs_req },
+	{ "3*",			0,	0,	err_invalid_suff },
+
+	/* TODO: add more */
+
+	{ 0, 0, 0 },
+};
+
+int
+main(int argc, char *argv[])
+{
+	struct test_case *tc;
+	uintmax_t val;
+	int ec;
+	const char *e;
+
+	(void)argc;
+	for (ec = 0, tc = test_cases; tc->str; ++tc) {
+		e = VNUM_2bytes(tc->str, &val, tc->rel);
+		if (e != tc->err) {
+			printf("%s: VNUM_2bytes(\"%s\", %ju) (%s) != (%s)\n",
+			    *argv, tc->str, tc->rel, tc->err, e);
+			++ec;
+		} else if (e == NULL && val != tc->val) {
+			printf("%s: VNUM_2bytes(\"%s\", %ju) %ju != %ju (%s)\n",
+			    *argv, tc->str, tc->rel, val, tc->val, e);
+			++ec;
+		}
+	}
+	/* TODO: test invalid strings */
+	if (!ec)
+		printf("OK\n");
+	return (ec > 0);
+}
+#endif



From geoff at varnish-cache.org  Mon Jan  9 20:51:47 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:51:47 +0100
Subject: [experimental-ims] e13b98c Add undocumented variables
Message-ID: <E1RkMC3-0006nm-Lh@project.varnish-software.com>

commit e13b98cc8deff63a6d1470eb0f795d200e83929f
Author: Andreas Plesner Jacobsen <apj at mutt.dk>
Date:   Sun Sep 11 12:52:01 2011 +0200

    Add undocumented variables

diff --git a/doc/sphinx/reference/vcl.rst b/doc/sphinx/reference/vcl.rst
index e3169ea..9fd526c 100644
--- a/doc/sphinx/reference/vcl.rst
+++ b/doc/sphinx/reference/vcl.rst
@@ -687,6 +687,9 @@ The following variables are available while processing a request:
 client.ip
   The client's IP address.
 
+client.identity
+  Identification of the client, used to load balance in the client director.
+
 server.hostname
   The host name of the server.
 
@@ -733,6 +736,21 @@ req.hash_ignore_busy
 req.can_gzip
   Does the client accept the gzip transfer encoding.
 
+req.restarts
+  A count of how many times this request has been restarted.
+
+req.esi
+  True if the request is an ESI request.
+
+req.esi_level
+  A count of how many levels of ESI requests we're currently at.
+
+req.grace
+  Set to a period to enable grace.
+
+req.xid
+  Unique ID of this request.
+
 The following variables are available while preparing a backend
 request (either for a cache miss or for pass or pipe mode):
 
@@ -792,6 +810,25 @@ beresp.response
 beresp.ttl
   The object's remaining time to live, in seconds. beresp.ttl is writable.
 
+beresp.grace
+  Set to a period to enable grace.
+
+beresp.saintmode
+  Set to a period to enable saint mode.
+
+beresp.backend.name
+  Name of the backend this response was fetched from.
+
+beresp.backend.ip
+  IP of the backend this response was fetched from.
+
+beresp.backend.port
+  Port of the backend this response was fetched from.
+
+beresp.storage
+  Set to force Varnish to save this object to a particular storage
+  backend.
+
 After the object is entered into the cache, the following (mostly
 read-only) variables are available when the object has been located in
 cache, typically in vcl_hit and vcl_deliver.
@@ -816,6 +853,12 @@ obj.hits
   The approximate number of times the object has been delivered. A value 
   of 0 indicates a cache miss.
 
+obj.grace
+  The object's grace period in seconds. obj.grace is writable.
+
+obj.http.header
+  The corresponding HTTP header.
+
 The following variables are available while determining the hash key
 of an object:
 
@@ -1000,8 +1043,8 @@ and Per Buer.
 COPYRIGHT
 =========
 
-This document is licensed under the same licence as Varnish
-itself. See LICENCE for details.
+This document is licensed under the same license as Varnish
+itself. See LICENSE for details.
 
 * Copyright (c) 2006 Verdens Gang AS
 * Copyright (c) 2006-2011 Varnish Software AS



From geoff at varnish-cache.org  Mon Jan  9 20:52:16 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:16 +0100
Subject: [experimental-ims] 7ce0d47 Rename source files to their VTLA names
Message-ID: <E1RkMCW-0007bD-5a@project.varnish-software.com>

commit 7ce0d474d5b36e4feaf38df30594611060397e0f
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sun Oct 9 18:20:10 2011 +0000

    Rename source files to their VTLA names

diff --git a/bin/varnishadm/Makefile.am b/bin/varnishadm/Makefile.am
index 93997b9..201a361 100644
--- a/bin/varnishadm/Makefile.am
+++ b/bin/varnishadm/Makefile.am
@@ -8,8 +8,8 @@ dist_man_MANS = varnishadm.1
 
 varnishadm_SOURCES = \
 	varnishadm.c \
-	$(top_builddir)/lib/libvarnish/assert.c \
-	$(top_builddir)/lib/libvarnish/tcp.c \
+	$(top_builddir)/lib/libvarnish/vas.c \
+	$(top_builddir)/lib/libvarnish/vtcp.c \
 	$(top_builddir)/lib/libvarnish/vss.c
 
 varnishadm_CFLAGS = @LIBEDIT_CFLAGS@
diff --git a/bin/varnishhist/Makefile.am b/bin/varnishhist/Makefile.am
index 8f19313..8aafa2d 100644
--- a/bin/varnishhist/Makefile.am
+++ b/bin/varnishhist/Makefile.am
@@ -7,7 +7,7 @@ bin_PROGRAMS = varnishhist
 dist_man_MANS = varnishhist.1
 
 varnishhist_SOURCES = varnishhist.c \
-	$(top_builddir)/lib/libvarnish/assert.c \
+	$(top_builddir)/lib/libvarnish/vas.c \
 	$(top_builddir)/lib/libvarnish/version.c
 
 varnishhist_LDADD = \
diff --git a/bin/varnishlog/Makefile.am b/bin/varnishlog/Makefile.am
index 79bf2ad..e21ee35 100644
--- a/bin/varnishlog/Makefile.am
+++ b/bin/varnishlog/Makefile.am
@@ -8,7 +8,7 @@ dist_man_MANS = varnishlog.1
 
 varnishlog_SOURCES = \
 	varnishlog.c \
-	$(top_builddir)/lib/libvarnish/assert.c \
+	$(top_builddir)/lib/libvarnish/vas.c \
 	$(top_builddir)/lib/libvarnish/flopen.c \
 	$(top_builddir)/lib/libvarnish/version.c \
 	$(top_builddir)/lib/libvarnish/vsb.c \
diff --git a/bin/varnishncsa/Makefile.am b/bin/varnishncsa/Makefile.am
index 9b652da..8b3f617 100644
--- a/bin/varnishncsa/Makefile.am
+++ b/bin/varnishncsa/Makefile.am
@@ -10,7 +10,7 @@ varnishncsa_SOURCES = \
 	varnishncsa.c \
 	base64.c \
 	base64.h \
-	$(top_builddir)/lib/libvarnish/assert.c \
+	$(top_builddir)/lib/libvarnish/vas.c \
 	$(top_builddir)/lib/libvarnish/flopen.c \
 	$(top_builddir)/lib/libvarnish/version.c \
 	$(top_builddir)/lib/libvarnish/vsb.c \
diff --git a/bin/varnishreplay/Makefile.am b/bin/varnishreplay/Makefile.am
index 553dfda..2be20f7 100644
--- a/bin/varnishreplay/Makefile.am
+++ b/bin/varnishreplay/Makefile.am
@@ -8,8 +8,8 @@ dist_man_MANS = varnishreplay.1
 
 varnishreplay_SOURCES = \
 	varnishreplay.c \
-	$(top_builddir)/lib/libvarnish/assert.c \
-	$(top_builddir)/lib/libvarnish/tcp.c \
+	$(top_builddir)/lib/libvarnish/vas.c \
+	$(top_builddir)/lib/libvarnish/vtcp.c \
 	$(top_builddir)/lib/libvarnish/vss.c
 
 varnishreplay_LDADD = \
diff --git a/bin/varnishsizes/Makefile.am b/bin/varnishsizes/Makefile.am
index a25b280..8029741 100644
--- a/bin/varnishsizes/Makefile.am
+++ b/bin/varnishsizes/Makefile.am
@@ -7,7 +7,7 @@ bin_PROGRAMS = varnishsizes
 dist_man_MANS = varnishsizes.1
 
 varnishsizes_SOURCES = varnishsizes.c \
-	$(top_builddir)/lib/libvarnish/assert.c \
+	$(top_builddir)/lib/libvarnish/vas.c \
 	$(top_builddir)/lib/libvarnish/version.c
 
 varnishsizes_LDADD = \
diff --git a/bin/varnishstat/Makefile.am b/bin/varnishstat/Makefile.am
index 8407eb2..83588c9 100644
--- a/bin/varnishstat/Makefile.am
+++ b/bin/varnishstat/Makefile.am
@@ -11,7 +11,7 @@ varnishstat_SOURCES = \
 	\
 	varnishstat.c \
 	varnishstat_curses.c \
-	$(top_builddir)/lib/libvarnish/assert.c \
+	$(top_builddir)/lib/libvarnish/vas.c \
 	$(top_builddir)/lib/libvarnish/version.c
 
 varnishstat_LDADD = \
diff --git a/bin/varnishtop/Makefile.am b/bin/varnishtop/Makefile.am
index dab412f..630fbf7 100644
--- a/bin/varnishtop/Makefile.am
+++ b/bin/varnishtop/Makefile.am
@@ -7,7 +7,7 @@ bin_PROGRAMS = varnishtop
 dist_man_MANS = varnishtop.1
 
 varnishtop_SOURCES = varnishtop.c \
-	$(top_builddir)/lib/libvarnish/assert.c \
+	$(top_builddir)/lib/libvarnish/vas.c \
 	$(top_builddir)/lib/libvarnish/version.c
 
 varnishtop_LDADD = \
diff --git a/lib/libvarnish/Makefile.am b/lib/libvarnish/Makefile.am
index 21a9d98..b8651cc 100644
--- a/lib/libvarnish/Makefile.am
+++ b/lib/libvarnish/Makefile.am
@@ -5,8 +5,8 @@ pkglib_LTLIBRARIES = libvarnish.la
 libvarnish_la_LDFLAGS = -avoid-version
 
 libvarnish_la_SOURCES = \
-	argv.c \
-	assert.c \
+	vav.c \
+	vas.c \
 	binary_heap.c \
 	subproc.c \
 	cli_auth.c \
@@ -15,7 +15,7 @@ libvarnish_la_SOURCES = \
 	flopen.c \
 	num.c \
 	time.c \
-	tcp.c \
+	vtcp.c \
 	vct.c \
 	version.c \
 	vev.c \
diff --git a/lib/libvarnish/argv.c b/lib/libvarnish/argv.c
deleted file mode 100644
index a0c57bb..0000000
--- a/lib/libvarnish/argv.c
+++ /dev/null
@@ -1,264 +0,0 @@
-/*-
- * Copyright (c) 2006 Verdens Gang AS
- * Copyright (c) 2006-2011 Varnish Software AS
- * All rights reserved.
- *
- * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- *
- * const char **VAV_Parse(const char *s, int comment)
- *	Parse a command like line into an argv[]
- *	Index zero contains NULL or an error message
- *	"double quotes" and backslash substitution is handled.
- *
- * void VAV_Free(const char **argv)
- *	Free the result of VAV_Parse()
- *
- */
-
-#include "config.h"
-
-#include <ctype.h>
-#include <stdlib.h>
-#include <stdio.h>
-#include <string.h>
-
-#include "vas.h"
-#include "vav.h"
-
-int
-VAV_BackSlash(const char *s, char *res)
-{
-	int r;
-	char c;
-	unsigned u;
-
-	assert(*s == '\\');
-	r = c = 0;
-	switch(s[1]) {
-	case 'n':
-		c = '\n';
-		r = 2;
-		break;
-	case 'r':
-		c = '\r';
-		r = 2;
-		break;
-	case 't':
-		c = '\t';
-		r = 2;
-		break;
-	case '"':
-		c = '"';
-		r = 2;
-		break;
-	case '\\':
-		c = '\\';
-		r = 2;
-		break;
-	case '0': case '1': case '2': case '3':
-	case '4': case '5': case '6': case '7':
-		for (r = 1; r < 4; r++) {
-			if (!isdigit(s[r]))
-				break;
-			if (s[r] - '0' > 7)
-				break;
-			c <<= 3;	/*lint !e701 signed left shift */
-			c |= s[r] - '0';
-		}
-		break;
-	case 'x':
-		if (1 == sscanf(s + 1, "x%02x", &u)) {
-			assert(!(u & ~0xff));
-			c = u;	/*lint !e734 loss of precision */
-			r = 4;
-		}
-		break;
-	default:
-		break;
-	}
-	if (res != NULL)
-		*res = c;
-	return (r);
-}
-
-char *
-VAV_BackSlashDecode(const char *s, const char *e)
-{
-	const char *q;
-	char *p, *r;
-	int i;
-
-	if (e == NULL)
-		e = strchr(s, '\0');
-	assert(e != NULL);
-	p = calloc((e - s) + 1, 1);
-	if (p == NULL)
-		return (p);
-	for (r = p, q = s; q < e; ) {
-		if (*q != '\\') {
-			*r++ = *q++;
-			continue;
-		}
-		i = VAV_BackSlash(q, r);
-		q += i;
-		r++;
-	}
-	*r = '\0';
-	return (p);
-}
-
-static char err_invalid_backslash[] = "Invalid backslash sequence";
-static char err_missing_quote[] = "Missing '\"'";
-
-char **
-VAV_Parse(const char *s, int *argc, int flag)
-{
-	char **argv;
-	const char *p;
-	int nargv, largv;
-	int i, quote;
-
-	assert(s != NULL);
-	nargv = 1;
-	largv = 16;
-	argv = calloc(sizeof *argv, largv);
-	if (argv == NULL)
-		return (NULL);
-
-	for (;;) {
-		if (*s == '\0')
-			break;
-		if (isspace(*s)) {
-			s++;
-			continue;
-		}
-		if ((flag & ARGV_COMMENT) && *s == '#')
-			break;
-		if (*s == '"' && !(flag & ARGV_NOESC)) {
-			p = ++s;
-			quote = 1;
-		} else {
-			p = s;
-			quote = 0;
-		}
-		while (1) {
-			if (*s == '\\' && !(flag & ARGV_NOESC)) {
-				i = VAV_BackSlash(s, NULL);
-				if (i == 0) {
-					argv[0] = err_invalid_backslash;
-					return (argv);
-				}
-				s += i;
-				continue;
-			}
-			if (!quote) {
-				if (*s == '\0' || isspace(*s))
-					break;
-				if ((flag & ARGV_COMMA) && *s == ',')
-					break;
-				s++;
-				continue;
-			}
-			if (*s == '"' && !(flag & ARGV_NOESC))
-				break;
-			if (*s == '\0') {
-				argv[0] = err_missing_quote;
-				return (argv);
-			}
-			s++;
-		}
-		if (nargv + 1 >= largv) {
-			argv = realloc(argv, sizeof (*argv) * (largv += largv));
-			assert(argv != NULL);
-		}
-		if (flag & ARGV_NOESC) {
-			argv[nargv] = malloc(1 + (s - p));
-			assert(argv[nargv] != NULL);
-			memcpy(argv[nargv], p, s - p);
-			argv[nargv][s - p] = '\0';
-			nargv++;
-		} else {
-			argv[nargv++] = VAV_BackSlashDecode(p, s);
-		}
-		if (*s != '\0')
-			s++;
-	}
-	argv[nargv] = NULL;
-	if (argc != NULL)
-		*argc = nargv;
-	return (argv);
-}
-
-void
-VAV_Free(char **argv)
-{
-	int i;
-
-	for (i = 1; argv[i] != NULL; i++)
-		free(argv[i]);
-	free(argv);
-}
-
-#ifdef TESTPROG
-
-#include <printf.h>
-
-static void
-VAV_Print(char **argv)
-{
-	int i;
-
-	printf("---- %p\n", argv);
-	if (argv[0] != NULL)
-		printf("err %V\n", argv[0]);
-	for (i = 1; argv[i] != NULL; i++)
-		printf("%3d %V\n", i, argv[i]);
-}
-
-static void
-Test(const char *str)
-{
-	char **av;
-
-	printf("Test: <%V>\n", str);
-	av = VAV_Parse(str, 0);
-	VAV_Print(av);
-}
-
-int
-main(int argc, char **argv)
-{
-	char buf[BUFSIZ];
-
-	(void)argc;
-	(void)argv;
-
-	register_printf_render_std("V");
-
-	while (fgets(buf, sizeof buf, stdin))
-		Test(buf);
-
-	return (0);
-}
-#endif
diff --git a/lib/libvarnish/assert.c b/lib/libvarnish/assert.c
deleted file mode 100644
index 82351ce..0000000
--- a/lib/libvarnish/assert.c
+++ /dev/null
@@ -1,62 +0,0 @@
-/*-
- * Copyright (c) 2006 Verdens Gang AS
- * Copyright (c) 2006-2011 Varnish Software AS
- * All rights reserved.
- *
- * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- *
- * This is the default backend function for libvarnish' assert facilities.
- */
-
-#include "config.h"
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-
-#include "libvarnish.h"
-
-static void
-VAS_Fail_default(const char *func, const char *file, int line,
-    const char *cond, int err, int xxx)
-{
-
-	if (xxx) {
-		fprintf(stderr,
-		    "Missing errorhandling code in %s(), %s line %d:\n"
-		    "  Condition(%s) not true.\n",
-		    func, file, line, cond);
-	} else {
-		fprintf(stderr,
-		    "Assert error in %s(), %s line %d:\n"
-		    "  Condition(%s) not true.\n",
-		    func, file, line, cond);
-	}
-	if (err)
-		fprintf(stderr,
-		    "  errno = %d (%s)\n", err, strerror(err));
-	abort();
-}
-
-vas_f *VAS_Fail = VAS_Fail_default;
diff --git a/lib/libvarnish/tcp.c b/lib/libvarnish/tcp.c
deleted file mode 100644
index e8932e8..0000000
--- a/lib/libvarnish/tcp.c
+++ /dev/null
@@ -1,290 +0,0 @@
-/*-
- * Copyright (c) 2006 Verdens Gang AS
- * Copyright (c) 2006-2011 Varnish Software AS
- * All rights reserved.
- *
- * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- */
-
-#include "config.h"
-
-#include <sys/types.h>
-#include <sys/socket.h>
-
-#include <netinet/in.h>
-
-#ifdef __linux
-#include <netinet/tcp.h>
-#endif
-
-#include <errno.h>
-#include <sys/ioctl.h>
-#ifdef HAVE_SYS_FILIO_H
-#include <sys/filio.h>
-#endif
-#include <netdb.h>
-#include <poll.h>
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#include <unistd.h>
-#include <math.h>
-
-#include "vtcp.h"
-#include "libvarnish.h"
-
-/*--------------------------------------------------------------------*/
-
-int
-VTCP_port(const struct sockaddr_storage *addr)
-{
-
-	if (addr->ss_family == AF_INET) {
-		const struct sockaddr_in *ain = (const void *)addr;
-		return ntohs((ain->sin_port));
-	}
-	if (addr->ss_family == AF_INET6) {
-		const struct sockaddr_in6 *ain = (const void *)addr;
-		return ntohs((ain->sin6_port));
-	}
-	return (-1);
-}
-
-
-/*--------------------------------------------------------------------*/
-
-void
-VTCP_name(const struct sockaddr_storage *addr, unsigned l,
-    char *abuf, unsigned alen, char *pbuf, unsigned plen)
-{
-	int i;
-
-	i = getnameinfo((const void *)addr, l, abuf, alen, pbuf, plen,
-	   NI_NUMERICHOST | NI_NUMERICSERV);
-	if (i) {
-		/*
-		 * XXX this printf is shitty, but we may not have space
-		 * for the gai_strerror in the bufffer :-(
-		 */
-		printf("getnameinfo = %d %s\n", i, gai_strerror(i));
-		(void)snprintf(abuf, alen, "Conversion");
-		(void)snprintf(pbuf, plen, "Failed");
-		return;
-	}
-	/* XXX dirty hack for v4-to-v6 mapped addresses */
-	if (strncmp(abuf, "::ffff:", 7) == 0) {
-		for (i = 0; abuf[i + 7]; ++i)
-			abuf[i] = abuf[i + 7];
-		abuf[i] = '\0';
-	}
-}
-
-/*--------------------------------------------------------------------*/
-
-void
-VTCP_myname(int sock, char *abuf, unsigned alen, char *pbuf, unsigned plen)
-{
-	struct sockaddr_storage addr_s;
-	socklen_t l;
-
-	l = sizeof addr_s;
-	AZ(getsockname(sock, (void *)&addr_s, &l));
-	VTCP_name(&addr_s, l, abuf, alen, pbuf, plen);
-}
-/*--------------------------------------------------------------------*/
-
-void
-VTCP_hisname(int sock, char *abuf, unsigned alen, char *pbuf, unsigned plen)
-{
-	struct sockaddr_storage addr_s;
-	socklen_t l;
-
-	l = sizeof addr_s;
-	if (!getpeername(sock, (void*)&addr_s, &l))
-		VTCP_name(&addr_s, l, abuf, alen, pbuf, plen);
-	else {
-		(void)snprintf(abuf, alen, "<none>");
-		(void)snprintf(pbuf, plen, "<none>");
-	}
-}
-
-/*--------------------------------------------------------------------*/
-
-int
-VTCP_filter_http(int sock)
-{
-#ifdef HAVE_ACCEPT_FILTERS
-	struct accept_filter_arg afa;
-	int i;
-
-	memset(&afa, 0, sizeof(afa));
-	strcpy(afa.af_name, "httpready");
-	errno = 0;
-	i = setsockopt(sock, SOL_SOCKET, SO_ACCEPTFILTER,
-	    &afa, sizeof(afa));
-	/* XXX ugly */
-	if (i)
-		printf("Acceptfilter(%d, httpready): %d %s\n",
-		    sock, i, strerror(errno));
-	return (i);
-#elif defined(__linux)
-	int defer = 1;
-	setsockopt(sock, SOL_TCP,TCP_DEFER_ACCEPT,(char *) &defer, sizeof(int));
-	return (0);
-#else
-	(void)sock;
-	return (0);
-#endif
-}
-
-/*--------------------------------------------------------------------
- * Functions for controlling NONBLOCK mode.
- *
- * We use FIONBIO because it is cheaper than fcntl(2), which requires
- * us to do two syscalls, one to get and one to set, the latter of
- * which mucks about a bit before it ends up calling ioctl(FIONBIO),
- * at least on FreeBSD.
- */
-
-int
-VTCP_blocking(int sock)
-{
-	int i, j;
-
-	i = 0;
-	j = ioctl(sock, FIONBIO, &i);
-	VTCP_Assert(j);
-	return (j);
-}
-
-int
-VTCP_nonblocking(int sock)
-{
-	int i, j;
-
-	i = 1;
-	j = ioctl(sock, FIONBIO, &i);
-	VTCP_Assert(j);
-	return (j);
-}
-
-/*--------------------------------------------------------------------
- * On TCP a connect(2) can block for a looong time, and we don't want that.
- * Unfortunately, the SocketWizards back in those days were happy to wait
- * any amount of time for a connection, so the connect(2) syscall does not
- * take an argument for patience.
- *
- * There is a little used work-around, and we employ it at our peril.
- *
- */
-
-int
-VTCP_connect(int s, const struct sockaddr_storage *name, socklen_t namelen, int msec)
-{
-	int i, k;
-	socklen_t l;
-	struct pollfd fds[1];
-
-	assert(s >= 0);
-
-	/* Set the socket non-blocking */
-	if (msec > 0)
-		(void)VTCP_nonblocking(s);
-
-	/* Attempt the connect */
-	i = connect(s, (const void *)name, namelen);
-	if (i == 0 || errno != EINPROGRESS)
-		return (i);
-
-	assert(msec > 0);
-	/* Exercise our patience, polling for write */
-	fds[0].fd = s;
-	fds[0].events = POLLWRNORM;
-	fds[0].revents = 0;
-	i = poll(fds, 1, msec);
-
-	if (i == 0) {
-		/* Timeout, close and give up */
-		errno = ETIMEDOUT;
-		return (-1);
-	}
-
-	/* Find out if we got a connection */
-	l = sizeof k;
-	AZ(getsockopt(s, SOL_SOCKET, SO_ERROR, &k, &l));
-
-	/* An error means no connection established */
-	errno = k;
-	if (k)
-		return (-1);
-
-	(void)VTCP_blocking(s);
-	return (0);
-}
-
-/*--------------------------------------------------------------------
- * When closing a TCP connection, a couple of errno's are legit, we
- * can't be held responsible for the other end wanting to talk to us.
- */
-
-void
-VTCP_close(int *s)
-{
-	int i;
-
-	i = close(*s);
-
-	assert (VTCP_Check(i));
-	*s = -1;
-}
-
-void
-VTCP_set_read_timeout(int s, double seconds)
-{
-	struct timeval timeout;
-	timeout.tv_sec = (int)floor(seconds);
-	timeout.tv_usec = (int)(1e6 * (seconds - timeout.tv_sec));
-#ifdef SO_RCVTIMEO_WORKS
-	AZ(setsockopt(s, SOL_SOCKET, SO_RCVTIMEO, &timeout, sizeof timeout));
-#else
-	(void)s;
-#endif
-}
-
-/*--------------------------------------------------------------------
- * Set or reset SO_LINGER flag
- */
-
-int
-VTCP_linger(int sock, int linger)
-{
-	struct linger lin;
-	int i;
-
-	memset(&lin, 0, sizeof lin);
-	lin.l_onoff = linger;
-	i = setsockopt(sock, SOL_SOCKET, SO_LINGER, &lin, sizeof lin);
-	VTCP_Assert(i);
-	return (i);
-}
diff --git a/lib/libvarnish/vas.c b/lib/libvarnish/vas.c
new file mode 100644
index 0000000..82351ce
--- /dev/null
+++ b/lib/libvarnish/vas.c
@@ -0,0 +1,62 @@
+/*-
+ * Copyright (c) 2006 Verdens Gang AS
+ * Copyright (c) 2006-2011 Varnish Software AS
+ * All rights reserved.
+ *
+ * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * This is the default backend function for libvarnish' assert facilities.
+ */
+
+#include "config.h"
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include "libvarnish.h"
+
+static void
+VAS_Fail_default(const char *func, const char *file, int line,
+    const char *cond, int err, int xxx)
+{
+
+	if (xxx) {
+		fprintf(stderr,
+		    "Missing errorhandling code in %s(), %s line %d:\n"
+		    "  Condition(%s) not true.\n",
+		    func, file, line, cond);
+	} else {
+		fprintf(stderr,
+		    "Assert error in %s(), %s line %d:\n"
+		    "  Condition(%s) not true.\n",
+		    func, file, line, cond);
+	}
+	if (err)
+		fprintf(stderr,
+		    "  errno = %d (%s)\n", err, strerror(err));
+	abort();
+}
+
+vas_f *VAS_Fail = VAS_Fail_default;
diff --git a/lib/libvarnish/vav.c b/lib/libvarnish/vav.c
new file mode 100644
index 0000000..a0c57bb
--- /dev/null
+++ b/lib/libvarnish/vav.c
@@ -0,0 +1,264 @@
+/*-
+ * Copyright (c) 2006 Verdens Gang AS
+ * Copyright (c) 2006-2011 Varnish Software AS
+ * All rights reserved.
+ *
+ * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * const char **VAV_Parse(const char *s, int comment)
+ *	Parse a command like line into an argv[]
+ *	Index zero contains NULL or an error message
+ *	"double quotes" and backslash substitution is handled.
+ *
+ * void VAV_Free(const char **argv)
+ *	Free the result of VAV_Parse()
+ *
+ */
+
+#include "config.h"
+
+#include <ctype.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include <string.h>
+
+#include "vas.h"
+#include "vav.h"
+
+int
+VAV_BackSlash(const char *s, char *res)
+{
+	int r;
+	char c;
+	unsigned u;
+
+	assert(*s == '\\');
+	r = c = 0;
+	switch(s[1]) {
+	case 'n':
+		c = '\n';
+		r = 2;
+		break;
+	case 'r':
+		c = '\r';
+		r = 2;
+		break;
+	case 't':
+		c = '\t';
+		r = 2;
+		break;
+	case '"':
+		c = '"';
+		r = 2;
+		break;
+	case '\\':
+		c = '\\';
+		r = 2;
+		break;
+	case '0': case '1': case '2': case '3':
+	case '4': case '5': case '6': case '7':
+		for (r = 1; r < 4; r++) {
+			if (!isdigit(s[r]))
+				break;
+			if (s[r] - '0' > 7)
+				break;
+			c <<= 3;	/*lint !e701 signed left shift */
+			c |= s[r] - '0';
+		}
+		break;
+	case 'x':
+		if (1 == sscanf(s + 1, "x%02x", &u)) {
+			assert(!(u & ~0xff));
+			c = u;	/*lint !e734 loss of precision */
+			r = 4;
+		}
+		break;
+	default:
+		break;
+	}
+	if (res != NULL)
+		*res = c;
+	return (r);
+}
+
+char *
+VAV_BackSlashDecode(const char *s, const char *e)
+{
+	const char *q;
+	char *p, *r;
+	int i;
+
+	if (e == NULL)
+		e = strchr(s, '\0');
+	assert(e != NULL);
+	p = calloc((e - s) + 1, 1);
+	if (p == NULL)
+		return (p);
+	for (r = p, q = s; q < e; ) {
+		if (*q != '\\') {
+			*r++ = *q++;
+			continue;
+		}
+		i = VAV_BackSlash(q, r);
+		q += i;
+		r++;
+	}
+	*r = '\0';
+	return (p);
+}
+
+static char err_invalid_backslash[] = "Invalid backslash sequence";
+static char err_missing_quote[] = "Missing '\"'";
+
+char **
+VAV_Parse(const char *s, int *argc, int flag)
+{
+	char **argv;
+	const char *p;
+	int nargv, largv;
+	int i, quote;
+
+	assert(s != NULL);
+	nargv = 1;
+	largv = 16;
+	argv = calloc(sizeof *argv, largv);
+	if (argv == NULL)
+		return (NULL);
+
+	for (;;) {
+		if (*s == '\0')
+			break;
+		if (isspace(*s)) {
+			s++;
+			continue;
+		}
+		if ((flag & ARGV_COMMENT) && *s == '#')
+			break;
+		if (*s == '"' && !(flag & ARGV_NOESC)) {
+			p = ++s;
+			quote = 1;
+		} else {
+			p = s;
+			quote = 0;
+		}
+		while (1) {
+			if (*s == '\\' && !(flag & ARGV_NOESC)) {
+				i = VAV_BackSlash(s, NULL);
+				if (i == 0) {
+					argv[0] = err_invalid_backslash;
+					return (argv);
+				}
+				s += i;
+				continue;
+			}
+			if (!quote) {
+				if (*s == '\0' || isspace(*s))
+					break;
+				if ((flag & ARGV_COMMA) && *s == ',')
+					break;
+				s++;
+				continue;
+			}
+			if (*s == '"' && !(flag & ARGV_NOESC))
+				break;
+			if (*s == '\0') {
+				argv[0] = err_missing_quote;
+				return (argv);
+			}
+			s++;
+		}
+		if (nargv + 1 >= largv) {
+			argv = realloc(argv, sizeof (*argv) * (largv += largv));
+			assert(argv != NULL);
+		}
+		if (flag & ARGV_NOESC) {
+			argv[nargv] = malloc(1 + (s - p));
+			assert(argv[nargv] != NULL);
+			memcpy(argv[nargv], p, s - p);
+			argv[nargv][s - p] = '\0';
+			nargv++;
+		} else {
+			argv[nargv++] = VAV_BackSlashDecode(p, s);
+		}
+		if (*s != '\0')
+			s++;
+	}
+	argv[nargv] = NULL;
+	if (argc != NULL)
+		*argc = nargv;
+	return (argv);
+}
+
+void
+VAV_Free(char **argv)
+{
+	int i;
+
+	for (i = 1; argv[i] != NULL; i++)
+		free(argv[i]);
+	free(argv);
+}
+
+#ifdef TESTPROG
+
+#include <printf.h>
+
+static void
+VAV_Print(char **argv)
+{
+	int i;
+
+	printf("---- %p\n", argv);
+	if (argv[0] != NULL)
+		printf("err %V\n", argv[0]);
+	for (i = 1; argv[i] != NULL; i++)
+		printf("%3d %V\n", i, argv[i]);
+}
+
+static void
+Test(const char *str)
+{
+	char **av;
+
+	printf("Test: <%V>\n", str);
+	av = VAV_Parse(str, 0);
+	VAV_Print(av);
+}
+
+int
+main(int argc, char **argv)
+{
+	char buf[BUFSIZ];
+
+	(void)argc;
+	(void)argv;
+
+	register_printf_render_std("V");
+
+	while (fgets(buf, sizeof buf, stdin))
+		Test(buf);
+
+	return (0);
+}
+#endif
diff --git a/lib/libvarnish/vtcp.c b/lib/libvarnish/vtcp.c
new file mode 100644
index 0000000..e8932e8
--- /dev/null
+++ b/lib/libvarnish/vtcp.c
@@ -0,0 +1,290 @@
+/*-
+ * Copyright (c) 2006 Verdens Gang AS
+ * Copyright (c) 2006-2011 Varnish Software AS
+ * All rights reserved.
+ *
+ * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include "config.h"
+
+#include <sys/types.h>
+#include <sys/socket.h>
+
+#include <netinet/in.h>
+
+#ifdef __linux
+#include <netinet/tcp.h>
+#endif
+
+#include <errno.h>
+#include <sys/ioctl.h>
+#ifdef HAVE_SYS_FILIO_H
+#include <sys/filio.h>
+#endif
+#include <netdb.h>
+#include <poll.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+#include <math.h>
+
+#include "vtcp.h"
+#include "libvarnish.h"
+
+/*--------------------------------------------------------------------*/
+
+int
+VTCP_port(const struct sockaddr_storage *addr)
+{
+
+	if (addr->ss_family == AF_INET) {
+		const struct sockaddr_in *ain = (const void *)addr;
+		return ntohs((ain->sin_port));
+	}
+	if (addr->ss_family == AF_INET6) {
+		const struct sockaddr_in6 *ain = (const void *)addr;
+		return ntohs((ain->sin6_port));
+	}
+	return (-1);
+}
+
+
+/*--------------------------------------------------------------------*/
+
+void
+VTCP_name(const struct sockaddr_storage *addr, unsigned l,
+    char *abuf, unsigned alen, char *pbuf, unsigned plen)
+{
+	int i;
+
+	i = getnameinfo((const void *)addr, l, abuf, alen, pbuf, plen,
+	   NI_NUMERICHOST | NI_NUMERICSERV);
+	if (i) {
+		/*
+		 * XXX this printf is shitty, but we may not have space
+		 * for the gai_strerror in the bufffer :-(
+		 */
+		printf("getnameinfo = %d %s\n", i, gai_strerror(i));
+		(void)snprintf(abuf, alen, "Conversion");
+		(void)snprintf(pbuf, plen, "Failed");
+		return;
+	}
+	/* XXX dirty hack for v4-to-v6 mapped addresses */
+	if (strncmp(abuf, "::ffff:", 7) == 0) {
+		for (i = 0; abuf[i + 7]; ++i)
+			abuf[i] = abuf[i + 7];
+		abuf[i] = '\0';
+	}
+}
+
+/*--------------------------------------------------------------------*/
+
+void
+VTCP_myname(int sock, char *abuf, unsigned alen, char *pbuf, unsigned plen)
+{
+	struct sockaddr_storage addr_s;
+	socklen_t l;
+
+	l = sizeof addr_s;
+	AZ(getsockname(sock, (void *)&addr_s, &l));
+	VTCP_name(&addr_s, l, abuf, alen, pbuf, plen);
+}
+/*--------------------------------------------------------------------*/
+
+void
+VTCP_hisname(int sock, char *abuf, unsigned alen, char *pbuf, unsigned plen)
+{
+	struct sockaddr_storage addr_s;
+	socklen_t l;
+
+	l = sizeof addr_s;
+	if (!getpeername(sock, (void*)&addr_s, &l))
+		VTCP_name(&addr_s, l, abuf, alen, pbuf, plen);
+	else {
+		(void)snprintf(abuf, alen, "<none>");
+		(void)snprintf(pbuf, plen, "<none>");
+	}
+}
+
+/*--------------------------------------------------------------------*/
+
+int
+VTCP_filter_http(int sock)
+{
+#ifdef HAVE_ACCEPT_FILTERS
+	struct accept_filter_arg afa;
+	int i;
+
+	memset(&afa, 0, sizeof(afa));
+	strcpy(afa.af_name, "httpready");
+	errno = 0;
+	i = setsockopt(sock, SOL_SOCKET, SO_ACCEPTFILTER,
+	    &afa, sizeof(afa));
+	/* XXX ugly */
+	if (i)
+		printf("Acceptfilter(%d, httpready): %d %s\n",
+		    sock, i, strerror(errno));
+	return (i);
+#elif defined(__linux)
+	int defer = 1;
+	setsockopt(sock, SOL_TCP,TCP_DEFER_ACCEPT,(char *) &defer, sizeof(int));
+	return (0);
+#else
+	(void)sock;
+	return (0);
+#endif
+}
+
+/*--------------------------------------------------------------------
+ * Functions for controlling NONBLOCK mode.
+ *
+ * We use FIONBIO because it is cheaper than fcntl(2), which requires
+ * us to do two syscalls, one to get and one to set, the latter of
+ * which mucks about a bit before it ends up calling ioctl(FIONBIO),
+ * at least on FreeBSD.
+ */
+
+int
+VTCP_blocking(int sock)
+{
+	int i, j;
+
+	i = 0;
+	j = ioctl(sock, FIONBIO, &i);
+	VTCP_Assert(j);
+	return (j);
+}
+
+int
+VTCP_nonblocking(int sock)
+{
+	int i, j;
+
+	i = 1;
+	j = ioctl(sock, FIONBIO, &i);
+	VTCP_Assert(j);
+	return (j);
+}
+
+/*--------------------------------------------------------------------
+ * On TCP a connect(2) can block for a looong time, and we don't want that.
+ * Unfortunately, the SocketWizards back in those days were happy to wait
+ * any amount of time for a connection, so the connect(2) syscall does not
+ * take an argument for patience.
+ *
+ * There is a little used work-around, and we employ it at our peril.
+ *
+ */
+
+int
+VTCP_connect(int s, const struct sockaddr_storage *name, socklen_t namelen, int msec)
+{
+	int i, k;
+	socklen_t l;
+	struct pollfd fds[1];
+
+	assert(s >= 0);
+
+	/* Set the socket non-blocking */
+	if (msec > 0)
+		(void)VTCP_nonblocking(s);
+
+	/* Attempt the connect */
+	i = connect(s, (const void *)name, namelen);
+	if (i == 0 || errno != EINPROGRESS)
+		return (i);
+
+	assert(msec > 0);
+	/* Exercise our patience, polling for write */
+	fds[0].fd = s;
+	fds[0].events = POLLWRNORM;
+	fds[0].revents = 0;
+	i = poll(fds, 1, msec);
+
+	if (i == 0) {
+		/* Timeout, close and give up */
+		errno = ETIMEDOUT;
+		return (-1);
+	}
+
+	/* Find out if we got a connection */
+	l = sizeof k;
+	AZ(getsockopt(s, SOL_SOCKET, SO_ERROR, &k, &l));
+
+	/* An error means no connection established */
+	errno = k;
+	if (k)
+		return (-1);
+
+	(void)VTCP_blocking(s);
+	return (0);
+}
+
+/*--------------------------------------------------------------------
+ * When closing a TCP connection, a couple of errno's are legit, we
+ * can't be held responsible for the other end wanting to talk to us.
+ */
+
+void
+VTCP_close(int *s)
+{
+	int i;
+
+	i = close(*s);
+
+	assert (VTCP_Check(i));
+	*s = -1;
+}
+
+void
+VTCP_set_read_timeout(int s, double seconds)
+{
+	struct timeval timeout;
+	timeout.tv_sec = (int)floor(seconds);
+	timeout.tv_usec = (int)(1e6 * (seconds - timeout.tv_sec));
+#ifdef SO_RCVTIMEO_WORKS
+	AZ(setsockopt(s, SOL_SOCKET, SO_RCVTIMEO, &timeout, sizeof timeout));
+#else
+	(void)s;
+#endif
+}
+
+/*--------------------------------------------------------------------
+ * Set or reset SO_LINGER flag
+ */
+
+int
+VTCP_linger(int sock, int linger)
+{
+	struct linger lin;
+	int i;
+
+	memset(&lin, 0, sizeof lin);
+	lin.l_onoff = linger;
+	i = setsockopt(sock, SOL_SOCKET, SO_LINGER, &lin, sizeof lin);
+	VTCP_Assert(i);
+	return (i);
+}
diff --git a/lib/libvarnishapi/Makefile.am b/lib/libvarnishapi/Makefile.am
index f88d748..492b10d 100644
--- a/lib/libvarnishapi/Makefile.am
+++ b/lib/libvarnishapi/Makefile.am
@@ -10,8 +10,8 @@ libvarnishapi_la_SOURCES = \
 	vsm_api.h \
 	vsl_api.h \
 	\
-	../libvarnish/assert.c \
-	../libvarnish/argv.c \
+	../libvarnish/vas.c \
+	../libvarnish/vav.c \
 	../../include/vcs_version.h \
 	../libvarnish/version.c \
 	../libvarnish/cli_common.c \



From geoff at varnish-cache.org  Mon Jan  9 20:52:17 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:17 +0100
Subject: [experimental-ims] 905144b This is a table-generating #include
Message-ID: <E1RkMCX-0007du-8t@project.varnish-software.com>

commit 905144b25b68e6c940c71af5164ba2a04868659a
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Oct 10 10:51:18 2011 +0000

    This is a table-generating #include

diff --git a/include/Makefile.am b/include/Makefile.am
index 948cef7..935a6f9 100644
--- a/include/Makefile.am
+++ b/include/Makefile.am
@@ -9,6 +9,7 @@ pkginclude_HEADERS = \
 	tbl/http_response.h \
 	tbl/locks.h \
 	tbl/steps.h \
+	tbl/symbol_kind.h \
 	tbl/vcc_types.h \
 	tbl/vcl_returns.h \
 	tbl/vrt_stv_var.h \
diff --git a/include/tbl/symbol_kind.h b/include/tbl/symbol_kind.h
new file mode 100644
index 0000000..155d36e
--- /dev/null
+++ b/include/tbl/symbol_kind.h
@@ -0,0 +1,41 @@
+/*-
+ * Copyright (c) 2010 Varnish Software AS
+ * All rights reserved.
+ *
+ * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ */
+
+/*lint -save -e525 -e539 */
+VCC_SYMB(NONE,		none,		"undefined")
+VCC_SYMB(VAR,		var,		"variable")
+VCC_SYMB(FUNC,		func,		"function")	/* VMOD function */
+VCC_SYMB(PROC,		proc,		"procedure")	/* VMOD procedure */
+VCC_SYMB(VMOD,		vmod,		"vmod")
+VCC_SYMB(ACL,		acl,		"acl")
+VCC_SYMB(SUB,		sub,		"sub")		/* VCL subroutine */
+VCC_SYMB(BACKEND,	backend,	"backend")
+VCC_SYMB(PROBE,		probe,		"probe")
+VCC_SYMB(WILDCARD,	wildcard,	"wildcard")
+/*lint -restore */
diff --git a/lib/libvcl/Makefile.am b/lib/libvcl/Makefile.am
index 8406ed7..44e2957 100644
--- a/lib/libvcl/Makefile.am
+++ b/lib/libvcl/Makefile.am
@@ -9,7 +9,6 @@ libvcl_la_LDFLAGS = -avoid-version
 libvcl_la_SOURCES = \
 	vcc_compile.h \
 	vcc_token_defs.h \
-	symbol_kind.h \
 	\
 	vcc_acl.c \
 	vcc_action.c \
diff --git a/lib/libvcl/symbol_kind.h b/lib/libvcl/symbol_kind.h
deleted file mode 100644
index 155d36e..0000000
--- a/lib/libvcl/symbol_kind.h
+++ /dev/null
@@ -1,41 +0,0 @@
-/*-
- * Copyright (c) 2010 Varnish Software AS
- * All rights reserved.
- *
- * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- *
- */
-
-/*lint -save -e525 -e539 */
-VCC_SYMB(NONE,		none,		"undefined")
-VCC_SYMB(VAR,		var,		"variable")
-VCC_SYMB(FUNC,		func,		"function")	/* VMOD function */
-VCC_SYMB(PROC,		proc,		"procedure")	/* VMOD procedure */
-VCC_SYMB(VMOD,		vmod,		"vmod")
-VCC_SYMB(ACL,		acl,		"acl")
-VCC_SYMB(SUB,		sub,		"sub")		/* VCL subroutine */
-VCC_SYMB(BACKEND,	backend,	"backend")
-VCC_SYMB(PROBE,		probe,		"probe")
-VCC_SYMB(WILDCARD,	wildcard,	"wildcard")
-/*lint -restore */
diff --git a/lib/libvcl/vcc_compile.h b/lib/libvcl/vcc_compile.h
index c59c632..8088509 100644
--- a/lib/libvcl/vcc_compile.h
+++ b/lib/libvcl/vcc_compile.h
@@ -99,7 +99,7 @@ struct token {
 
 enum symkind {
 #define VCC_SYMB(uu, ll, dd)	SYM_##uu,
-#include "symbol_kind.h"
+#include "tbl/symbol_kind.h"
 #undef VCC_SYMB
 };
 
diff --git a/lib/libvcl/vcc_symb.c b/lib/libvcl/vcc_symb.c
index 0acb0cd..222680f 100644
--- a/lib/libvcl/vcc_symb.c
+++ b/lib/libvcl/vcc_symb.c
@@ -41,7 +41,7 @@ VCC_SymKind(struct vcc *tl, const struct symbol *s)
 {
 	switch(s->kind) {
 #define VCC_SYMB(uu, ll, dd)	case SYM_##uu: return(dd);
-#include "symbol_kind.h"
+#include "tbl/symbol_kind.h"
 #undef VCC_SYMB
 	default:
 		ErrInternal(tl);



From geoff at varnish-cache.org  Mon Jan  9 20:51:57 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:51:57 +0100
Subject: [experimental-ims] bc1d749 Shave acct_req down to a bodybytes counter,
	it's all we use it for.
Message-ID: <E1RkMCD-00072F-4v@project.varnish-software.com>

commit bc1d749a6618aecc2c075ce30e1366fa71c525da
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Sep 19 09:45:00 2011 +0000

    Shave acct_req down to a bodybytes counter, it's all we use it for.
    
    Make pipe mode use the acct_tmp like everybody else.

diff --git a/bin/varnishd/cache.h b/bin/varnishd/cache.h
index 4f18a08..6c80714 100644
--- a/bin/varnishd/cache.h
+++ b/bin/varnishd/cache.h
@@ -218,9 +218,9 @@ struct acct {
 
 /*--------------------------------------------------------------------*/
 
-#define L0(n)
-#define L1(n)			uint64_t n;
-#define VSC_F(n, t, l, f, e,d)	L##l(n)
+#define L0(t, n)
+#define L1(t, n)		t n;
+#define VSC_F(n, t, l, f, e,d)	L##l(t, n)
 #define VSC_DO_MAIN
 struct dstat {
 #include "vsc_fields.h"
@@ -615,7 +615,7 @@ struct sess {
 	struct sessmem		*mem;
 
 	VTAILQ_ENTRY(sess)	poollist;
-	struct acct		acct_req;
+	uint64_t		req_bodybytes;
 	struct acct		acct_ses;
 
 #if defined(HAVE_EPOLL_CTL)
diff --git a/bin/varnishd/cache_center.c b/bin/varnishd/cache_center.c
index 122bccf..32a4215 100644
--- a/bin/varnishd/cache_center.c
+++ b/bin/varnishd/cache_center.c
@@ -338,7 +338,8 @@ cnt_done(struct sess *sp)
 		/* XXX: Add StatReq == StatSess */
 		/* XXX: Workaround for pipe */
 		if (sp->fd >= 0) {
-			WSP(sp, SLT_Length, "%ju", (uintmax_t)sp->acct_req.bodybytes);
+			WSP(sp, SLT_Length, "%ju",
+			    (uintmax_t)sp->req_bodybytes);
 		}
 		WSL(sp->wrk, SLT_ReqEnd, sp->id, "%u %.9f %.9f %.9f %.9f %.9f",
 		    sp->xid, sp->t_req, sp->t_end, dh, dp, da);
@@ -352,7 +353,7 @@ cnt_done(struct sess *sp)
 	if (sp->esi_level > 0)
 		return (1);
 
-	memset(&sp->acct_req, 0, sizeof sp->acct_req);
+	sp->req_bodybytes = 0;
 
 	sp->t_req = NAN;
 	sp->hash_always_miss = 0;
diff --git a/bin/varnishd/cache_pipe.c b/bin/varnishd/cache_pipe.c
index ddb5c15..3968767 100644
--- a/bin/varnishd/cache_pipe.c
+++ b/bin/varnishd/cache_pipe.c
@@ -78,10 +78,10 @@ PipeSession(struct sess *sp)
 	(void)VTCP_blocking(vc->fd);
 
 	WRW_Reserve(w, &vc->fd);
-	sp->acct_req.hdrbytes += http_Write(w, sp->wrk->bereq, 0);
+	sp->wrk->acct_tmp.hdrbytes += http_Write(w, sp->wrk->bereq, 0);
 
 	if (sp->htc->pipeline.b != NULL)
-		sp->acct_req.bodybytes +=
+		sp->wrk->acct_tmp.bodybytes +=
 		    WRW_Write(w, sp->htc->pipeline.b, Tlen(sp->htc->pipeline));
 
 	i = WRW_FlushRelease(w);
diff --git a/bin/varnishd/cache_session.c b/bin/varnishd/cache_session.c
index e8352f6..bbc5706 100644
--- a/bin/varnishd/cache_session.c
+++ b/bin/varnishd/cache_session.c
@@ -80,9 +80,10 @@ SES_Charge(struct sess *sp)
 {
 	struct acct *a = &sp->wrk->acct_tmp;
 
+	sp->req_bodybytes += a->bodybytes;
+
 #define ACCT(foo)				\
 	sp->wrk->stats.s_##foo += a->foo;	\
-	sp->acct_req.foo += a->foo;		\
 	sp->acct_ses.foo += a->foo;		\
 	a->foo = 0;
 #include "acct_fields.h"



From geoff at varnish-cache.org  Mon Jan  9 20:52:03 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:03 +0100
Subject: [experimental-ims] 880b0f8 strange wording. typo
Message-ID: <E1RkMCJ-0007Go-Cb@project.varnish-software.com>

commit 880b0f857f0e19358d81561ae346cdc7a1024cfd
Author: Per Buer <perbu at varnish-software.com>
Date:   Thu Oct 6 11:03:43 2011 +0200

    strange wording. typo

diff --git a/doc/sphinx/index.rst b/doc/sphinx/index.rst
index 6ba912a..019fd3b 100644
--- a/doc/sphinx/index.rst
+++ b/doc/sphinx/index.rst
@@ -5,7 +5,7 @@ Welcome to Varnish's documentation!
 ===================================
 
 Varnish is a state of the art web accelerator. Its mission is to sit
-in front of a web server an cache the content. It makes your web site
+in front of a web server and to cache the content. It makes your web site
 go fast.
 
 We suggest you start by reading the installation guide



From geoff at varnish-cache.org  Mon Jan  9 20:52:27 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:27 +0100
Subject: [experimental-ims] cf66e08 Typographical corrections, spelling
Message-ID: <E1RkMCh-0007y5-EO@project.varnish-software.com>

commit cf66e086e6cfe8c6a530ac16ea5a7bc69bf67954
Author: Tollef Fog Heen <tfheen at varnish-software.com>
Date:   Mon Oct 31 09:59:45 2011 +0100

    Typographical corrections, spelling

diff --git a/doc/sphinx/phk/backends.rst b/doc/sphinx/phk/backends.rst
index 2b32353..9ab6123 100644
--- a/doc/sphinx/phk/backends.rst
+++ b/doc/sphinx/phk/backends.rst
@@ -9,10 +9,10 @@ question answered conclusively long time ago, but once you try to
 be efficient, things get hairy fast.
 
 One of the features of Varnish we are very fundamental about, is the
-ability to have multiple VCL's loaded at the same time, and to switch
+ability to have multiple VCLs loaded at the same time, and to switch
 between them instantly and seamlessly.
 
-So Imagine you have 1000 backends in your VCL, not an unreasonable
+So imagine you have 1000 backends in your VCL, not an unreasonable
 number, each configured with health-polling.
 
 Now you fiddle your vcl_recv{} a bit and load the VCL again, but
@@ -25,7 +25,7 @@ be up to date the instant we switch to the other VCL.
 This basically means that either all VCLs poll all their backends,
 or they must share, somehow.
 
-We can dismiss the all VCL's poll all their backends scenario,
+We can dismiss the all VCLs poll all their backends scenario,
 because it scales truly horribly, and would pummel backends with
 probes if people forget to vcl.discard their old dusty VCLs.
 
@@ -39,7 +39,7 @@ It would be truly stupid to close 100 ready and usable connections to
 a backend, and open 100 other, just because we switch to a different
 VCL that has an identical backend definition.
 
-But what is an identical backend definition in this context ?
+But what is an identical backend definition in this context?
 
 It is important to remember that we are not talking physical
 backends:  For instance, there is nothing preventing a VCL for
@@ -48,7 +48,7 @@ backends.
 
 The most obvious thing to do, is to use the VCL name of the backend
 as identifier, but that is not enough.  We can have two different
-VCL's where backend "b1" points at two different physical machines,
+VCLs where backend "b1" points at two different physical machines,
 for instance when we migrate or upgrade the backend.
 
 The identity of the state than can be shared is therefore the triplet:
@@ -78,11 +78,11 @@ is a wildcard-ish scheme, where you can write things like::
 
 	b1()				# All backends named b1
 
-	b1(127.0.0.1)			# All b1's on Ipv4 lookback
+	b1(127.0.0.1)			# All b1s on IPv4 lookback
 
-	b1(:8080)			# All b1's on port 8080, (IPv4 or IPv6)
+	b1(:8080)			# All b1s on port 8080, (IPv4 or IPv6)
 
-	b1(192.168.60.1,192.168.60.2)	# All b1's on one of those addresses.
+	b1(192.168.60.1,192.168.60.2)	# All b1s on one of those addresses.
 
 (Input very much welcome)
 
@@ -112,7 +112,7 @@ the backend gets retired anyway.
 We should either park a thread on each backend, or have a poller thread
 which throws jobs into the work-pool as the backends needs polled.
 
-The patternmatching is confined to CLI and possibly libvarnishapi
+The pattern matching is confined to CLI and possibly libvarnishapi
 
 I think this will work,
 



From geoff at varnish-cache.org  Mon Jan  9 20:52:26 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:26 +0100
Subject: [experimental-ims] 5068897 Fix indentation typo
Message-ID: <E1RkMCg-0007wf-PM@project.varnish-software.com>

commit 5068897dbb6cb902fdb95b8050cfb66fc6a30ae8
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Oct 25 10:00:32 2011 +0000

    Fix indentation typo

diff --git a/bin/varnishd/cache_gzip.c b/bin/varnishd/cache_gzip.c
index 08d298e..01ccebc 100644
--- a/bin/varnishd/cache_gzip.c
+++ b/bin/varnishd/cache_gzip.c
@@ -577,7 +577,7 @@ vfp_gzip_end(struct worker *w)
 			if (VGZ_ObufStorage(w, vg))
 				return (-1);
 			i = VGZ_Gzip(vg, &dp, &dl, VGZ_FINISH);
-		w->fetch_obj->len += dl;
+			w->fetch_obj->len += dl;
 		} while (i != Z_STREAM_END);
 		if (w->do_stream)
 			RES_StreamPoll(w);



From geoff at varnish-cache.org  Mon Jan  9 20:51:49 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:51:49 +0100
Subject: [experimental-ims] b52b3a6 Start separating out the worker thread
	from the worker pool stuff.
Message-ID: <E1RkMC5-0006pl-0S@project.varnish-software.com>

commit b52b3a688e2d13c57034a53bdd09565209e81c78
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sat Sep 17 09:32:18 2011 +0000

    Start separating out the worker thread from the worker pool stuff.

diff --git a/bin/varnishd/Makefile.am b/bin/varnishd/Makefile.am
index 0e0bc83..288f1f1 100644
--- a/bin/varnishd/Makefile.am
+++ b/bin/varnishd/Makefile.am
@@ -48,6 +48,7 @@ varnishd_SOURCES = \
 	cache_vrt_re.c \
 	cache_vrt_var.c \
 	cache_vrt_vmod.c \
+	cache_wrk.c \
 	cache_wrw.c \
 	cache_ws.c \
 	hash_classic.c \
diff --git a/bin/varnishd/cache.h b/bin/varnishd/cache.h
index e2faf30..1abd6db 100644
--- a/bin/varnishd/cache.h
+++ b/bin/varnishd/cache.h
@@ -840,8 +840,11 @@ void PipeSession(struct sess *sp);
 
 /* cache_pool.c */
 void WRK_Init(void);
+void WRK2_Init(void);
 int WRK_QueueSession(struct sess *sp);
 void WRK_SumStat(struct worker *w);
+int WRK_TrySumStat(struct worker *w);
+void WRK_thread_real(void *priv, struct worker *w);
 
 #define WRW_IsReleased(w)	((w)->wrw.wfd == NULL)
 int WRW_Error(const struct worker *w);
@@ -932,6 +935,10 @@ void ESI_DeliverChild(const struct sess *);
 /* cache_vrt_vmod.c */
 void VMOD_Init(void);
 
+/* cache_wrk.c */
+
+void *WRK_thread(void *priv);
+
 /* cache_ws.c */
 
 void WS_Init(struct ws *ws, const char *id, void *space, unsigned len);
diff --git a/bin/varnishd/cache_pool.c b/bin/varnishd/cache_pool.c
index 0875e07..8286162 100644
--- a/bin/varnishd/cache_pool.c
+++ b/bin/varnishd/cache_pool.c
@@ -82,73 +82,16 @@ static unsigned			nthr_max;
 
 static pthread_cond_t		herder_cond;
 static struct lock		herder_mtx;
-static struct lock		wstat_mtx;
 
 /*--------------------------------------------------------------------*/
 
-static void
-wrk_sumstat(struct worker *w)
-{
-
-	Lck_AssertHeld(&wstat_mtx);
-#define L0(n)
-#define L1(n) (VSC_C_main->n += w->stats.n)
-#define VSC_DO_MAIN
-#define VSC_F(n, t, l, f, d) L##l(n);
-#include "vsc_fields.h"
-#undef VSC_F
-#undef VSC_DO_MAIN
-#undef L0
-#undef L1
-	memset(&w->stats, 0, sizeof w->stats);
-}
-
 void
-WRK_SumStat(struct worker *w)
-{
-
-	Lck_Lock(&wstat_mtx);
-	wrk_sumstat(w);
-	Lck_Unlock(&wstat_mtx);
-}
-
-/*--------------------------------------------------------------------*/
-
-static void *
-wrk_thread_real(struct wq *qp, unsigned shm_workspace, unsigned sess_workspace,
-    uint16_t nhttp, unsigned http_space, unsigned siov)
+WRK_thread_real(void *priv, struct worker *w)
 {
-	struct worker *w, ww;
-	uint32_t wlog[shm_workspace / 4];
-	/* XXX: can we trust these to be properly aligned ? */
-	unsigned char ws[sess_workspace];
-	unsigned char http0[http_space];
-	unsigned char http1[http_space];
-	unsigned char http2[http_space];
-	struct iovec iov[siov];
-	struct SHA256Context sha256;
+	struct wq *qp;
 	int stats_clean;
 
-	THR_SetName("cache-worker");
-	w = &ww;
-	memset(w, 0, sizeof *w);
-	w->magic = WORKER_MAGIC;
-	w->lastused = NAN;
-	w->wlb = w->wlp = wlog;
-	w->wle = wlog + (sizeof wlog) / 4;
-	w->sha256ctx = &sha256;
-	w->bereq = HTTP_create(http0, nhttp);
-	w->beresp = HTTP_create(http1, nhttp);
-	w->resp = HTTP_create(http2, nhttp);
-	w->wrw.iov = iov;
-	w->wrw.siov = siov;
-	w->wrw.ciov = siov;
-	AZ(pthread_cond_init(&w->cond, NULL));
-
-	WS_Init(w->ws, "wrk", ws, sess_workspace);
-
-	VSL(SLT_WorkThread, 0, "%p start", w);
-
+	CAST_OBJ_NOTNULL(qp, priv, WQ_MAGIC);
 	Lck_Lock(&qp->mtx);
 	qp->nthr++;
 	stats_clean = 1;
@@ -195,43 +138,11 @@ wrk_thread_real(struct wq *qp, unsigned shm_workspace, unsigned sess_workspace,
 			if (w->vcl != NULL)
 				VCL_Rel(&w->vcl);
 		}
-		if (!Lck_Trylock(&wstat_mtx)) {
-			wrk_sumstat(w);
-			Lck_Unlock(&wstat_mtx);
-			stats_clean = 1;
-		}
+		stats_clean = WRK_TrySumStat(w);
 		Lck_Lock(&qp->mtx);
 	}
 	qp->nthr--;
 	Lck_Unlock(&qp->mtx);
-
-	VSL(SLT_WorkThread, 0, "%p end", w);
-	if (w->vcl != NULL)
-		VCL_Rel(&w->vcl);
-	AZ(pthread_cond_destroy(&w->cond));
-	HSH_Cleanup(w);
-	WRK_SumStat(w);
-	return (NULL);
-}
-
-static void *
-wrk_thread(void *priv)
-{
-	struct wq *qp;
-	uint16_t nhttp;
-	unsigned siov;
-
-	CAST_OBJ_NOTNULL(qp, priv, WQ_MAGIC);
-	assert(params->http_max_hdr <= 65535);
-	/* We need to snapshot these two for consistency */
-	nhttp = (uint16_t)params->http_max_hdr;
-	siov = nhttp * 2;
-	if (siov > IOV_MAX)
-		siov = IOV_MAX;
-	return (wrk_thread_real(qp,
-	    params->shm_workspace,
-	    params->wthread_workspace,
-	    nhttp, HTTP_estimate(nhttp), siov));
 }
 
 /*--------------------------------------------------------------------
@@ -481,7 +392,7 @@ wrk_breed_flock(struct wq *qp, const pthread_attr_t *tp_attr)
 	    qp->lqueue > qp->last_lqueue)) {	/* not getting better since last */
 		if (qp->nthr >= nthr_max) {
 			VSC_C_main->n_wrk_max++;
-		} else if (pthread_create(&tp, tp_attr, wrk_thread, qp)) {
+		} else if (pthread_create(&tp, tp_attr, WRK_thread, qp)) {
 			VSL(SLT_Debug, 0, "Create worker thread failed %d %s",
 			    errno, strerror(errno));
 			VSC_C_main->n_wrk_failed++;
@@ -600,13 +511,12 @@ WRK_BgThread(pthread_t *thr, const char *name, bgthread_t *func, void *priv)
 /*--------------------------------------------------------------------*/
 
 void
-WRK_Init(void)
+WRK2_Init(void)
 {
 	pthread_t tp;
 
 	AZ(pthread_cond_init(&herder_cond, NULL));
 	Lck_New(&herder_mtx, lck_herder);
-	Lck_New(&wstat_mtx, lck_wstat);
 
 	wrk_addpools(params->wthread_pools);
 	AZ(pthread_create(&tp, NULL, wrk_herdtimer_thread, NULL));
diff --git a/bin/varnishd/cache_wrk.c b/bin/varnishd/cache_wrk.c
new file mode 100644
index 0000000..5b78c19
--- /dev/null
+++ b/bin/varnishd/cache_wrk.c
@@ -0,0 +1,160 @@
+/*-
+ * Copyright (c) 2006 Verdens Gang AS
+ * Copyright (c) 2006-2011 Varnish Software AS
+ * All rights reserved.
+ *
+ * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * Worker thread stuff unrealted to the worker thread pools.
+ */
+
+#include "config.h"
+
+#include <sys/types.h>
+
+#include <errno.h>
+#include <stdio.h>
+#include <math.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+
+#include "vcl.h"
+#include "cli_priv.h"
+#include "cache.h"
+#include "stevedore.h"
+#include "hash_slinger.h"
+#include "vsha256.h"
+
+static struct lock		wstat_mtx;
+
+/*--------------------------------------------------------------------*/
+
+static void
+wrk_sumstat(struct worker *w)
+{
+
+	Lck_AssertHeld(&wstat_mtx);
+#define L0(n)
+#define L1(n) (VSC_C_main->n += w->stats.n)
+#define VSC_DO_MAIN
+#define VSC_F(n, t, l, f, d) L##l(n);
+#include "vsc_fields.h"
+#undef VSC_F
+#undef VSC_DO_MAIN
+#undef L0
+#undef L1
+	memset(&w->stats, 0, sizeof w->stats);
+}
+
+void
+WRK_SumStat(struct worker *w)
+{
+
+	Lck_Lock(&wstat_mtx);
+	wrk_sumstat(w);
+	Lck_Unlock(&wstat_mtx);
+}
+
+int
+WRK_TrySumStat(struct worker *w)
+{
+	if (Lck_Trylock(&wstat_mtx)) 
+		return (0);
+	wrk_sumstat(w);
+	Lck_Unlock(&wstat_mtx);
+	return (1);
+}
+
+/*--------------------------------------------------------------------*/
+
+static void *
+wrk_thread_real(void *priv, unsigned shm_workspace, unsigned sess_workspace,
+    uint16_t nhttp, unsigned http_space, unsigned siov)
+{
+	struct worker *w, ww;
+	uint32_t wlog[shm_workspace / 4];
+	/* XXX: can we trust these to be properly aligned ? */
+	unsigned char ws[sess_workspace];
+	unsigned char http0[http_space];
+	unsigned char http1[http_space];
+	unsigned char http2[http_space];
+	struct iovec iov[siov];
+	struct SHA256Context sha256;
+
+	THR_SetName("cache-worker");
+	w = &ww;
+	memset(w, 0, sizeof *w);
+	w->magic = WORKER_MAGIC;
+	w->lastused = NAN;
+	w->wlb = w->wlp = wlog;
+	w->wle = wlog + (sizeof wlog) / 4;
+	w->sha256ctx = &sha256;
+	w->bereq = HTTP_create(http0, nhttp);
+	w->beresp = HTTP_create(http1, nhttp);
+	w->resp = HTTP_create(http2, nhttp);
+	w->wrw.iov = iov;
+	w->wrw.siov = siov;
+	w->wrw.ciov = siov;
+	AZ(pthread_cond_init(&w->cond, NULL));
+
+	WS_Init(w->ws, "wrk", ws, sess_workspace);
+
+	VSL(SLT_WorkThread, 0, "%p start", w);
+
+	WRK_thread_real(priv, w);
+
+	VSL(SLT_WorkThread, 0, "%p end", w);
+	if (w->vcl != NULL)
+		VCL_Rel(&w->vcl);
+	AZ(pthread_cond_destroy(&w->cond));
+	HSH_Cleanup(w);
+	WRK_SumStat(w);
+	return (NULL);
+}
+
+void *
+WRK_thread(void *priv)
+{
+	uint16_t nhttp;
+	unsigned siov;
+
+	assert(params->http_max_hdr <= 65535);
+	/* We need to snapshot these two for consistency */
+	nhttp = (uint16_t)params->http_max_hdr;
+	siov = nhttp * 2;
+	if (siov > IOV_MAX)
+		siov = IOV_MAX;
+	return (wrk_thread_real(priv,
+	    params->shm_workspace,
+	    params->wthread_workspace,
+	    nhttp, HTTP_estimate(nhttp), siov));
+}
+
+void
+WRK_Init(void)
+{
+	Lck_New(&wstat_mtx, lck_wstat);
+	WRK2_Init();
+}



From geoff at varnish-cache.org  Mon Jan  9 20:52:03 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:03 +0100
Subject: [experimental-ims] 27ccc76 Merge branch 'master' of
	ssh://git.varnish-cache.org/git/varnish-cache
Message-ID: <E1RkMCJ-0007Hv-RM@project.varnish-software.com>

commit 27ccc764ba89b5c919abc9fc487054f57f5903c2
Merge: 0887a4f 5c2edfc
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu Oct 6 11:02:45 2011 +0000

    Merge branch 'master' of ssh://git.varnish-cache.org/git/varnish-cache



From geoff at varnish-cache.org  Mon Jan  9 20:51:55 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:51:55 +0100
Subject: [experimental-ims] 5d5e873 Prepare the acceptor code to become
	multithreaded by moving the pacing under a dedicated lock.
Message-ID: <E1RkMCB-0006z4-Or@project.varnish-software.com>

commit 5d5e8732ad589a0b417e8f78585e4c252ea07c15
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sat Sep 17 17:36:06 2011 +0000

    Prepare the acceptor code to become multithreaded by moving the
    pacing under a dedicated lock.

diff --git a/bin/varnishd/cache.h b/bin/varnishd/cache.h
index 65d1886..6c0c525 100644
--- a/bin/varnishd/cache.h
+++ b/bin/varnishd/cache.h
@@ -299,8 +299,13 @@ struct worker {
 	void			*nhashpriv;
 	struct dstat		stats;
 
+	/* Pool stuff */
 	double			lastused;
 
+	/* Accept stuff */
+	struct sockaddr_storage	acceptaddr;
+	int			acceptsock;
+
 	struct wrw		wrw;
 
 	pthread_cond_t		cond;
@@ -634,6 +639,7 @@ struct vbc {
 void VCA_Prep(struct sess *sp);
 void VCA_Init(void);
 void VCA_Shutdown(void);
+int VCA_Accept(int sock, socklen_t *slp, struct sockaddr_storage *sap);
 extern pthread_t VCA_thread;
 
 /* cache_backend.c */
diff --git a/bin/varnishd/cache_acceptor.c b/bin/varnishd/cache_acceptor.c
index 23752f3..0022037 100644
--- a/bin/varnishd/cache_acceptor.c
+++ b/bin/varnishd/cache_acceptor.c
@@ -153,6 +153,85 @@ VCA_Prep(struct sess *sp)
 #endif
 }
 
+/*--------------------------------------------------------------------
+ * If accept(2)'ing fails, we pace ourselves to relive any resource
+ * shortage if possible.
+ */
+
+static double vca_pace = 0.0;
+static struct lock pace_mtx;
+
+static void
+vca_pace_check(void)
+{
+	double p;
+
+	if (vca_pace == 0.0) 
+		return;
+	Lck_Lock(&pace_mtx);
+	p = vca_pace;
+	Lck_Unlock(&pace_mtx);
+	if (p > 0.0)
+		TIM_sleep(p);
+}
+
+static void
+vca_pace_bad(void)
+{
+	Lck_Lock(&pace_mtx);
+	vca_pace += params->acceptor_sleep_incr;
+	if (vca_pace > params->acceptor_sleep_max)
+		vca_pace = params->acceptor_sleep_max;
+	Lck_Unlock(&pace_mtx);
+}
+
+static void
+vca_pace_good(void)
+{
+
+	if (vca_pace == 0.0) 
+		return;
+	Lck_Lock(&pace_mtx);
+	vca_pace *= params->acceptor_sleep_decay;
+	if (vca_pace < params->acceptor_sleep_incr)
+		vca_pace = 0.0;
+	Lck_Unlock(&pace_mtx);
+}
+
+/*--------------------------------------------------------------------
+ * Accept on a listen socket, and handle error returns.
+ */
+
+int
+VCA_Accept(int sock, socklen_t *slp, struct sockaddr_storage *sap)
+{
+	int i;
+
+	vca_pace_check();
+
+	*slp = sizeof *sap;
+	i = accept(sock, (void*)sap, slp);
+
+	if (i < 0) {
+		VSC_C_main->accept_fail++;
+		switch (errno) {
+		case EAGAIN:
+		case ECONNABORTED:
+			break;
+		case EMFILE:
+			VSL(SLT_Debug, sock, "Too many open files");
+			vca_pace_bad();
+			break;
+		default:
+			VSL(SLT_Debug, sock, "Accept failed: %s",
+			    strerror(errno));
+			vca_pace_bad();
+			break;
+		}
+	}
+	return (i);
+}
+
 /*--------------------------------------------------------------------*/
 
 static void *
@@ -172,7 +251,7 @@ vca_acct(void *arg)
 	struct pollfd *pfd;
 	struct listen_sock *ls;
 	unsigned u;
-	double t0, now, pace;
+	double t0, now;
 
 	THR_SetName("cache-acceptor");
 	(void)arg;
@@ -192,7 +271,6 @@ vca_acct(void *arg)
 	}
 
 	need_test = 1;
-	pace = 0;
 	t0 = TIM_real();
 	while (1) {
 #ifdef SO_SNDTIMEO_WORKS
@@ -223,13 +301,6 @@ vca_acct(void *arg)
 			}
 		}
 #endif
-		/* Bound the pacing delay by parameter */
-		if (pace > params->acceptor_sleep_max)
-			pace = params->acceptor_sleep_max;
-		if (pace < params->acceptor_sleep_incr)
-			pace = 0.0;
-		if (pace > 0.0)
-			TIM_sleep(pace);
 		i = poll(pfd, heritage.nsocks, 1000);
 		now = TIM_real();
 		VSC_C_main->uptime = (uint64_t)(now - t0);
@@ -242,33 +313,14 @@ vca_acct(void *arg)
 			VSC_C_main->client_conn++;
 			l = sizeof addr_s;
 			addr = (void*)&addr_s;
-			i = accept(ls->sock, addr, &l);
-			if (i < 0) {
-				VSC_C_main->accept_fail++;
-				switch (errno) {
-				case EAGAIN:
-				case ECONNABORTED:
-					break;
-				case EMFILE:
-					VSL(SLT_Debug, ls->sock,
-					    "Too many open files "
-					    "when accept(2)ing. Sleeping.");
-					pace += params->acceptor_sleep_incr;
-					break;
-				default:
-					VSL(SLT_Debug, ls->sock,
-					    "Accept failed: %s",
-					    strerror(errno));
-					pace += params->acceptor_sleep_incr;
-					break;
-				}
+			i = VCA_Accept(ls->sock, &l, &addr_s);
+			if (i < 0) 
 				continue;
-			}
 			sp = SES_New();
 			if (sp == NULL) {
 				AZ(close(i));
 				VSC_C_main->client_drop++;
-				pace += params->acceptor_sleep_incr;
+				vca_pace_bad();
 				continue;
 			}
 			sp->fd = i;
@@ -283,9 +335,9 @@ vca_acct(void *arg)
 			sp->step = STP_FIRST;
 			if (Pool_QueueSession(sp)) {
 				VSC_C_main->client_drop++;
-				pace += params->acceptor_sleep_incr;
+				vca_pace_bad();
 			} else {
-				pace *= params->acceptor_sleep_decay;
+				vca_pace_good();
 			}
 		}
 	}
@@ -341,6 +393,7 @@ VCA_Init(void)
 {
 
 	CLI_AddFuncs(vca_cmds);
+	Lck_New(&pace_mtx, lck_vcapace);
 }
 
 void
diff --git a/bin/varnishd/locks.h b/bin/varnishd/locks.h
index e0a6bd5..6cbf91f 100644
--- a/bin/varnishd/locks.h
+++ b/bin/varnishd/locks.h
@@ -49,4 +49,5 @@ LOCK(ban)
 LOCK(vbp)
 LOCK(vbe)
 LOCK(backend)
+LOCK(vcapace)
 /*lint -restore */



From geoff at varnish-cache.org  Mon Jan  9 20:52:44 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:44 +0100
Subject: [experimental-ims] 84628ca Be more precise about default behaviour in
	the presence of cookie headers
Message-ID: <E1RkMCy-0008V7-Fr@project.varnish-software.com>

commit 84628cad04d8cc2ae131e42b6973fda4140ba2fa
Author: Andreas Plesner Jacobsen <apj at mutt.dk>
Date:   Tue Nov 29 20:16:03 2011 +0100

    Be more precise about default behaviour in the presence of cookie headers

diff --git a/doc/sphinx/tutorial/cookies.rst b/doc/sphinx/tutorial/cookies.rst
index 1d6f291..2ee14cd 100644
--- a/doc/sphinx/tutorial/cookies.rst
+++ b/doc/sphinx/tutorial/cookies.rst
@@ -3,9 +3,10 @@
 Cookies
 -------
 
-Varnish will not cache a object coming from the backend with a
-Set-Cookie header present. Also, if the client sends a Cookie header,
-Varnish will bypass the cache and go directly to the backend.
+Varnish will, in the default configuration, not cache a object coming
+from the backend with a Set-Cookie header present. Also, if the client
+sends a Cookie header, Varnish will bypass the cache and go directly to
+the backend.
 
 This can be overly conservative. A lot of sites use Google Analytics
 (GA) to analyze their traffic. GA sets a cookie to track you. This



From geoff at varnish-cache.org  Mon Jan  9 20:51:45 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:51:45 +0100
Subject: [experimental-ims] d56069e Merge branch 'master' of
	ssh://git.varnish-cache.org/git/varnish-cache
Message-ID: <E1RkMC1-0006kt-Vl@project.varnish-software.com>

commit d56069e8ef221310d75455feb9b03483c9caf63b
Merge: b09f55e 4633d00
Author: Andreas Plesner Jacobsen <apj at mutt.dk>
Date:   Tue Sep 6 19:33:47 2011 +0200

    Merge branch 'master' of ssh://git.varnish-cache.org/git/varnish-cache



From geoff at varnish-cache.org  Mon Jan  9 20:52:04 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:04 +0100
Subject: [experimental-ims] 2766a70 Think about and document how we encode
	bans into bytestrings.
Message-ID: <E1RkMCK-0007Iu-8c@project.varnish-software.com>

commit 2766a700cae5bbeef7149598eff3666c66c3333a
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Fri Oct 7 14:04:47 2011 +0000

    Think about and document how we encode bans into bytestrings.

diff --git a/bin/varnishd/cache_ban.c b/bin/varnishd/cache_ban.c
index fed958c..97d961d 100644
--- a/bin/varnishd/cache_ban.c
+++ b/bin/varnishd/cache_ban.c
@@ -38,6 +38,26 @@
  * We make the "&&" mandatory from the start, leaving the syntax space
  * for latter handling of "||" as well.
  *
+ * Bans are compiled into bytestrings as follows:
+ *	8 bytes	- double: timestamp		XXX: Byteorder ?
+ *	4 bytes - be32: length
+ *	1 byte - flags: 0x01: BAN_F_REQ
+ *	N tests
+ * A test have this form:
+ *	1 byte - arg (see ban_vars.h col 3 "BAN_ARG_XXX")
+ *	(n bytes) - http header name, canonical encoding
+ *	lump - comparison arg
+ *	1 byte - operation (BAN_OPER_)
+ *	(lump) - compiled regexp
+ * A lump is:
+ *	4 bytes - be32: length
+ * 	n bytes - content
+ *
+ * In a perfect world, we should vector through VRE to get to PCRE,
+ * but since we rely on PCRE's ability to encode the regexp into a
+ * byte string, that would be a little bit artificial, so this is
+ * the exception that confirmes the rule.
+ *
  */
 
 #include "config.h"
@@ -286,8 +306,6 @@ ban_parse_http(const struct ban *b, const char *a1)
 
 /*--------------------------------------------------------------------
  * Parse and add a ban test specification
- *
- * XXX: This should vector through VRE.
  */
 
 static int



From geoff at varnish-cache.org  Mon Jan  9 20:51:59 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:51:59 +0100
Subject: [experimental-ims] 8b925e3 Make -M help text better
Message-ID: <E1RkMCF-00075y-0H@project.varnish-software.com>

commit 8b925e389329874f4cca79718f5cbac7491fc0ae
Author: Andreas Plesner Jacobsen <apj at mutt.dk>
Date:   Tue Sep 20 12:24:06 2011 +0200

    Make -M help text better

diff --git a/bin/varnishd/varnishd.c b/bin/varnishd/varnishd.c
index 5fb054a..49e0f81 100644
--- a/bin/varnishd/varnishd.c
+++ b/bin/varnishd/varnishd.c
@@ -145,7 +145,7 @@ usage(void)
 	fprintf(stderr, FMT, "", "  shl: space for SHL records [80m]");
 	fprintf(stderr, FMT, "", "  free: space for other allocations [1m]");
 	fprintf(stderr, FMT, "", "  fill: prefill new file [+]");
-	fprintf(stderr, FMT, "-M address:port", "CLI-master to connect to.");
+	fprintf(stderr, FMT, "-M address:port", "Reverse CLI destination.");
 	fprintf(stderr, FMT, "-n dir", "varnishd working directory");
 	fprintf(stderr, FMT, "-P file", "PID file");
 	fprintf(stderr, FMT, "-p param=value", "set parameter");



From geoff at varnish-cache.org  Mon Jan  9 20:52:00 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:00 +0100
Subject: [experimental-ims] ff6f655 Increase varnishtest's HTTP rx timeout to
	15 seconds,
	a lot of test failures seems to be timeouts on stressed machines.
Message-ID: <E1RkMCG-00078t-Ct@project.varnish-software.com>

commit ff6f655bfd040ec05000b8be39849c5a3c437f59
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Sep 20 19:49:04 2011 +0000

    Increase varnishtest's HTTP rx timeout to 15 seconds, a lot of
    test failures seems to be timeouts on stressed machines.
    
    Teport timeouts more clearly.

diff --git a/bin/varnishtest/vtc_http.c b/bin/varnishtest/vtc_http.c
index d01240b..7dad4ed 100644
--- a/bin/varnishtest/vtc_http.c
+++ b/bin/varnishtest/vtc_http.c
@@ -343,7 +343,10 @@ http_rxchar_eof(struct http *hp, int n)
 		pfd[0].events = POLLIN;
 		pfd[0].revents = 0;
 		i = poll(pfd, 1, hp->timeout);
-		if (i <= 0)
+		if (i < 0)
+			vtc_log(hp->vl, 0, "HTTP rx timeout (%u ms)",
+			    hp->timeout);
+		if (i < 0)
 			vtc_log(hp->vl, 0, "HTTP rx failed (poll: %s)",
 			    strerror(errno));
 		assert(i > 0);
@@ -1143,7 +1146,7 @@ http_process(struct vtclog *vl, const char *spec, int sock, int sfd)
 	ALLOC_OBJ(hp, HTTP_MAGIC);
 	AN(hp);
 	hp->fd = sock;
-	hp->timeout = 5000;
+	hp->timeout = 15000;
 	hp->nrxbuf = 2048*1024;
 	hp->vsb = VSB_new_auto();
 	hp->rxbuf = malloc(hp->nrxbuf);		/* XXX */



From geoff at varnish-cache.org  Mon Jan  9 20:52:29 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:29 +0100
Subject: [experimental-ims] eab857a Test more of the backend matching code.
Message-ID: <E1RkMCj-00083V-Ii@project.varnish-software.com>

commit eab857aca974ea2546171d5a12b5c21f64fa9d2b
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Nov 7 12:27:24 2011 +0000

    Test more of the backend matching code.

diff --git a/bin/varnishtest/tests/c00048.vtc b/bin/varnishtest/tests/c00048.vtc
index bf1c3c9..4ddb888 100644
--- a/bin/varnishtest/tests/c00048.vtc
+++ b/bin/varnishtest/tests/c00048.vtc
@@ -59,4 +59,6 @@ varnish v1 -clierr 106 "backend.set_health s1 foo"
 varnish v1 -clierr 106 "backend.set_health s2 foo"
 varnish v1 -clierr 106 "backend.set_health s2 auto"
 varnish v1 -cliok "backend.list (foo)"
+varnish v1 -clierr 300 "backend.list ("
+varnish v1 -clierr 300 {backend.list " ( : ) -"}
 



From geoff at varnish-cache.org  Mon Jan  9 20:52:44 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:44 +0100
Subject: [experimental-ims] c1dac64 Move vef_priv from worker to busyobj
Message-ID: <E1RkMCy-0008Uy-CU@project.varnish-software.com>

commit c1dac64284b5f9298d2fb42ab4f822f16f4dc1a4
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Nov 29 19:09:39 2011 +0000

    Move vef_priv from worker to busyobj

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index c4d6f08..a0e2821 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -332,7 +332,6 @@ struct worker {
 	struct http		*beresp;
 	struct vbc		*vbc;
 
-	struct vef_priv		*vef_priv;
 	unsigned		do_stream;
 	unsigned		do_esi;
 	unsigned		do_gzip;
@@ -507,6 +506,7 @@ struct busyobj {
 	struct http_conn	htc;
 
 	enum body_status	body_status;
+	struct vef_priv		*vef_priv;
 };
 
 /* Object structure --------------------------------------------------*/
diff --git a/bin/varnishd/cache/cache_esi_fetch.c b/bin/varnishd/cache/cache_esi_fetch.c
index 5795849..e0765b6 100644
--- a/bin/varnishd/cache/cache_esi_fetch.c
+++ b/bin/varnishd/cache/cache_esi_fetch.c
@@ -150,7 +150,8 @@ vfp_vep_callback(struct worker *w, ssize_t l, enum vgz_flag flg)
 	int i;
 
 	CHECK_OBJ_NOTNULL(w, WORKER_MAGIC);
-	vef = w->vef_priv;
+	CHECK_OBJ_NOTNULL(w->busyobj, BUSYOBJ_MAGIC);
+	vef = w->busyobj->vef_priv;
 	CHECK_OBJ_NOTNULL(vef, VEF_MAGIC);
 	assert(l >= 0);
 
@@ -211,7 +212,8 @@ vfp_esi_bytes_ug(struct worker *w, struct http_conn *htc, ssize_t bytes)
 	struct vef_priv *vef;
 
 	CHECK_OBJ_NOTNULL(w, WORKER_MAGIC);
-	vef = w->vef_priv;
+	CHECK_OBJ_NOTNULL(w->busyobj, BUSYOBJ_MAGIC);
+	vef = w->busyobj->vef_priv;
 	CHECK_OBJ_NOTNULL(vef, VEF_MAGIC);
 
 	while (bytes > 0) {
@@ -252,7 +254,8 @@ vfp_esi_bytes_gg(struct worker *w, struct http_conn *htc, size_t bytes)
 	int i;
 
 	CHECK_OBJ_NOTNULL(w, WORKER_MAGIC);
-	vef = w->vef_priv;
+	CHECK_OBJ_NOTNULL(w->busyobj, BUSYOBJ_MAGIC);
+	vef = w->busyobj->vef_priv;
 	CHECK_OBJ_NOTNULL(vef, VEF_MAGIC);
 	assert(sizeof ibuf >= 1024);
 	ibuf2[0] = 0; /* For Flexelint */
@@ -308,19 +311,19 @@ vfp_esi_begin(struct worker *w, size_t estimate)
 		ALLOC_OBJ(vef, VEF_MAGIC);
 		AN(vef);
 		vef->vgz = VGZ_NewGzip(w, "G F E");
-		AZ(w->vef_priv);
-		w->vef_priv = vef;
+		AZ(w->busyobj->vef_priv);
+		w->busyobj->vef_priv = vef;
 		VEP_Init(w, vfp_vep_callback);
 	} else if (w->busyobj->is_gzip) {
 		w->busyobj->vgz_rx = VGZ_NewUngzip(w, "U F E");
 		ALLOC_OBJ(vef, VEF_MAGIC);
 		AN(vef);
 		vef->vgz = VGZ_NewGzip(w, "G F E");
-		AZ(w->vef_priv);
-		w->vef_priv = vef;
+		AZ(w->busyobj->vef_priv);
+		w->busyobj->vef_priv = vef;
 		VEP_Init(w, vfp_vep_callback);
 	} else {
-		AZ(w->vef_priv);
+		AZ(w->busyobj->vef_priv);
 		VEP_Init(w, NULL);
 	}
 
@@ -358,6 +361,7 @@ vfp_esi_end(struct worker *w)
 	int retval;
 
 	CHECK_OBJ_NOTNULL(w, WORKER_MAGIC);
+	CHECK_OBJ_NOTNULL(w->busyobj, BUSYOBJ_MAGIC);
 	AN(w->busyobj->vep);
 
 	retval = w->busyobj->fetch_failed;
@@ -387,10 +391,10 @@ vfp_esi_end(struct worker *w)
 		VSB_delete(vsb);
 	}
 
-	if (w->vef_priv != NULL) {
-		vef = w->vef_priv;
+	vef = w->busyobj->vef_priv;
+	if (vef != NULL) {
 		CHECK_OBJ_NOTNULL(vef, VEF_MAGIC);
-		w->vef_priv = NULL;
+		w->busyobj->vef_priv = NULL;
 		VGZ_UpdateObj(vef->vgz, w->busyobj->fetch_obj);
 		if (VGZ_Destroy(&vef->vgz,  -1) != VGZ_END)
 			retval = FetchError(w,



From geoff at varnish-cache.org  Mon Jan  9 20:51:55 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:51:55 +0100
Subject: [experimental-ims] 761d0c0 Add body_status.h to includes
Message-ID: <E1RkMCB-0006zM-RK@project.varnish-software.com>

commit 761d0c0d2bd78a6b094d4283064851436fbfbd19
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sat Sep 17 18:41:41 2011 +0000

    Add body_status.h to includes

diff --git a/bin/varnishd/Makefile.am b/bin/varnishd/Makefile.am
index e4ff980..c13f4e5 100644
--- a/bin/varnishd/Makefile.am
+++ b/bin/varnishd/Makefile.am
@@ -78,6 +78,7 @@ varnishd_SOURCES = \
 
 noinst_HEADERS = \
 	acct_fields.h \
+	body_status.h \
 	cache.h \
 	cache_backend.h \
 	cache_backend_poll.h \



From geoff at varnish-cache.org  Mon Jan  9 20:52:28 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:28 +0100
Subject: [experimental-ims] f6536dac Use scalbn(3) rather than exp2(3),
	it should be faster and more portable.
Message-ID: <E1RkMCi-00080Q-C2@project.varnish-software.com>

commit f6536dac9a6c9fdda4236e0ad92cfc4cdd6dd34a
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Nov 1 09:41:00 2011 +0000

    Use scalbn(3) rather than exp2(3), it should be faster and more portable.
    
    Fixes	#1046

diff --git a/bin/varnishd/cache_dir_random.c b/bin/varnishd/cache_dir_random.c
index f18c7d7..d6570ed 100644
--- a/bin/varnishd/cache_dir_random.c
+++ b/bin/varnishd/cache_dir_random.c
@@ -91,7 +91,7 @@ vdi_random_sha(const char *input, ssize_t len)
 	SHA256_Init(&ctx);
 	SHA256_Update(&ctx, input, len);
 	SHA256_Final(sign, &ctx);
-	return (vle32dec(sign) / exp2(32));
+	return (scalbn(vle32dec(sign), -32));
 }
 
 /*
@@ -113,11 +113,11 @@ vdi_random_init_seed(const struct vdi_random *vs, const struct sess *sp)
 		break;
 	case c_hash:
 		AN(sp->digest);
-		retval = vle32dec(sp->digest) / exp2(32);
+		retval = scalbn(vle32dec(sp->digest), -32);
 		break;
 	case c_random:
 	default:
-		retval = random() / exp2(31);
+		retval = scalbn(random(), -31);
 		break;
 	}
 	return (retval);



From geoff at varnish-cache.org  Mon Jan  9 20:52:30 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:30 +0100
Subject: [experimental-ims] b4aef74 A minor preemptive cleanup before the
	ban-lurker gets remodelled.
Message-ID: <E1RkMCk-000850-4j@project.varnish-software.com>

commit b4aef74f44a1b712ab5a0fb016c1ba8d9a5565f9
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Nov 8 10:03:43 2011 +0000

    A minor preemptive cleanup before the ban-lurker gets remodelled.

diff --git a/bin/varnishd/cache.h b/bin/varnishd/cache.h
index 1875ece..31db957 100644
--- a/bin/varnishd/cache.h
+++ b/bin/varnishd/cache.h
@@ -445,6 +445,7 @@ struct objcore {
 #define OC_F_PASS		(1<<2)
 #define OC_F_LRUDONTMOVE	(1<<4)
 #define OC_F_PRIV		(1<<5)		/* Stevedore private flag */
+#define OC_F_LURK		(3<<6)		/* Ban-lurker-color */
 	unsigned		timer_idx;
 	VTAILQ_ENTRY(objcore)	list;
 	VTAILQ_ENTRY(objcore)	lru_list;
diff --git a/bin/varnishd/cache_ban.c b/bin/varnishd/cache_ban.c
index 5b7d457..e7478a0 100644
--- a/bin/varnishd/cache_ban.c
+++ b/bin/varnishd/cache_ban.c
@@ -81,6 +81,7 @@ struct ban {
 	unsigned		flags;
 #define BAN_F_GONE		(1 << 0)
 #define BAN_F_REQ		(1 << 2)
+#define BAN_F_LURK		(3 << 3)	/* ban-lurker-color */
 	VTAILQ_HEAD(,objcore)	objcore;
 	struct vsb		*vsb;
 	uint8_t			*spec;
@@ -164,23 +165,6 @@ BAN_Free(struct ban *b)
 	FREE_OBJ(b);
 }
 
-static struct ban *
-ban_CheckLast(void)
-{
-	struct ban *b;
-
-	Lck_AssertHeld(&ban_mtx);
-	b = VTAILQ_LAST(&ban_head, banhead_s);
-	if (b != VTAILQ_FIRST(&ban_head) && b->refcount == 0) {
-		VSC_C_main->n_ban--;
-		VSC_C_main->n_ban_retire++;
-		VTAILQ_REMOVE(&ban_head, b, list);
-	} else {
-		b = NULL;
-	}
-	return (b);
-}
-
 /*--------------------------------------------------------------------
  * Get & Release a tail reference, used to hold the list stable for
  * traversals etc.
@@ -473,7 +457,6 @@ BAN_NewObjCore(struct objcore *oc)
 void
 BAN_DestroyObj(struct objcore *oc)
 {
-	struct ban *b;
 
 	CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
 	if (oc->ban == NULL)
@@ -484,12 +467,7 @@ BAN_DestroyObj(struct objcore *oc)
 	oc->ban->refcount--;
 	VTAILQ_REMOVE(&oc->ban->objcore, oc, ban_list);
 	oc->ban = NULL;
-
-	/* Attempt to purge last ban entry */
-	b = ban_CheckLast();
 	Lck_Unlock(&ban_mtx);
-	if (b != NULL)
-		BAN_Free(b);
 }
 
 /*--------------------------------------------------------------------
@@ -750,6 +728,23 @@ BAN_CheckObject(struct object *o, const struct sess *sp)
 	return (ban_check_object(o, sp, 1));
 }
 
+static struct ban *
+ban_CheckLast(void)
+{
+	struct ban *b;
+
+	Lck_AssertHeld(&ban_mtx);
+	b = VTAILQ_LAST(&ban_head, banhead_s);
+	if (b != VTAILQ_FIRST(&ban_head) && b->refcount == 0) {
+		VSC_C_main->n_ban--;
+		VSC_C_main->n_ban_retire++;
+		VTAILQ_REMOVE(&ban_head, b, list);
+	} else {
+		b = NULL;
+	}
+	return (b);
+}
+
 /*--------------------------------------------------------------------
  * Ban tail lurker thread
  */
@@ -763,9 +758,6 @@ ban_lurker_work(const struct sess *sp)
 	struct object *o;
 	int i;
 
-	WSL_Flush(sp->wrk, 0);
-	WRK_SumStat(sp->wrk);
-
 	Lck_Lock(&ban_mtx);
 
 	/* First try to route the last ban */
@@ -831,21 +823,34 @@ ban_lurker_work(const struct sess *sp)
 static void * __match_proto__(bgthread_t)
 ban_lurker(struct sess *sp, void *priv)
 {
+	struct ban *bf;
+
 	int i = 0;
 	(void)priv;
 	while (1) {
-		if (params->ban_lurker_sleep == 0.0) {
-			/* Lurker is disabled.  */
-			VTIM_sleep(1.0);
-			continue;
+
+		while (params->ban_lurker_sleep == 0.0) {
+			/*
+			 * Ban-lurker is disabled:
+			 * Clean the last ban, if possible, and sleep
+			 */
+			Lck_Lock(&ban_mtx);
+			bf = ban_CheckLast();
+			Lck_Unlock(&ban_mtx);
+			if (bf != NULL)
+				BAN_Free(bf);
+			else
+				VTIM_sleep(1.0);
 		}
+
+		i = ban_lurker_work(sp);
+		WSL_Flush(sp->wrk, 0);
+		WRK_SumStat(sp->wrk);
+
 		if (i != 0)
 			VTIM_sleep(params->ban_lurker_sleep);
 		else
 			VTIM_sleep(1.0);
-		i = ban_lurker_work(sp);
-		WSL_Flush(sp->wrk, 0);
-		WRK_SumStat(sp->wrk);
 	}
 	NEEDLESS_RETURN(NULL);
 }
@@ -957,10 +962,11 @@ ccf_ban_list(struct cli *cli, const char * const *av, void *priv)
 	/* Get a reference so we are safe to traverse the list */
 	bl = BAN_TailRef();
 
+	VCLI_Out(cli, "Present bans:\n");
 	VTAILQ_FOREACH(b, &ban_head, list) {
 		if (b == bl)
 			break;
-		VCLI_Out(cli, "%p %10.6f %5u%s\t", b, ban_time(b->spec),
+		VCLI_Out(cli, "%10.6f %5u%s\t", ban_time(b->spec),
 		    bl == b ? b->refcount - 1 : b->refcount,
 		    b->flags & BAN_F_GONE ? "G" : " ");
 		ban_render(cli, b->spec);



From geoff at varnish-cache.org  Mon Jan  9 20:52:01 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:01 +0100
Subject: [experimental-ims] d8b670d Merge branch 'master' of
	ssh://git.varnish-cache.org/git/varnish-cache
Message-ID: <E1RkMCH-0007B6-4D@project.varnish-software.com>

commit d8b670d15ac31cebd1d58030468791a0868678a3
Merge: 82344cb 6084802
Author: Per Buer <perbu at varnish-software.com>
Date:   Wed Sep 21 15:08:01 2011 +0200

    Merge branch 'master' of ssh://git.varnish-cache.org/git/varnish-cache



From geoff at varnish-cache.org  Mon Jan  9 20:51:58 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:51:58 +0100
Subject: [experimental-ims] d5adf88 Fix poll waiter,
	so that we don't terminate the search for poll'ed fd's early in the
	case of a timeout.
Message-ID: <E1RkMCE-00075E-JZ@project.varnish-software.com>

commit d5adf8877389d51455e2754d5fb21e24c9772e0e
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Sep 20 09:33:58 2011 +0000

    Fix poll waiter, so that we don't terminate the search for poll'ed
    fd's early in the case of a timeout.
    
    Fixes	#1023

diff --git a/bin/varnishd/cache_waiter_poll.c b/bin/varnishd/cache_waiter_poll.c
index 3a211bd..4678255 100644
--- a/bin/varnishd/cache_waiter_poll.c
+++ b/bin/varnishd/cache_waiter_poll.c
@@ -122,7 +122,7 @@ vwp_unpoll(struct vwp *vwp, int fd)
 static void *
 vwp_main(void *priv)
 {
-	int v;
+	int v, v2;
 	struct vwp *vwp;
 	struct sess *ss[NEEV], *sp, *sp2;
 	double deadline;
@@ -143,8 +143,9 @@ vwp_main(void *priv)
 		v = poll(vwp->pollfd, vwp->hpoll + 1, 100);
 		assert(v >= 0);
 		deadline = TIM_real() - params->sess_timeout;
+		v2 = v;
 		VTAILQ_FOREACH_SAFE(sp, &vwp->sesshead, list, sp2) {
-			if (v == 0)
+			if (v != 0 && v2 == 0)
 				break;
 			CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 			fd = sp->fd;
@@ -153,7 +154,7 @@ vwp_main(void *priv)
 			assert(fd < vwp->npoll);
 			assert(vwp->pollfd[fd].fd == fd);
 			if (vwp->pollfd[fd].revents) {
-				v--;
+				v2--;
 				i = HTC_Rx(sp->htc);
 				if (vwp->pollfd[fd].revents != POLLIN)
 					VSL(SLT_Debug, fd, "Poll: %x / %d",
@@ -174,14 +175,14 @@ vwp_main(void *priv)
 				SES_Delete(sp, "timeout");
 			}
 		}
-		if (v && vwp->pollfd[vwp->pipes[0]].revents) {
+		if (v2 && vwp->pollfd[vwp->pipes[0]].revents) {
 
 			if (vwp->pollfd[vwp->pipes[0]].revents != POLLIN)
 				VSL(SLT_Debug, 0, "pipe.revents= 0x%x",
 				    vwp->pollfd[vwp->pipes[0]].revents);
 			assert(vwp->pollfd[vwp->pipes[0]].revents == POLLIN);
 			vwp->pollfd[vwp->pipes[0]].revents = 0;
-			v--;
+			v2--;
 			i = read(vwp->pipes[0], ss, sizeof ss);
 			assert(i >= 0);
 			assert(((unsigned)i % sizeof ss[0]) == 0);
@@ -192,7 +193,7 @@ vwp_main(void *priv)
 				vwp_poll(vwp, ss[j]->fd);
 			}
 		}
-		assert(v == 0);
+		assert(v2 == 0);
 	}
 	NEEDLESS_RETURN(NULL);
 }



From geoff at varnish-cache.org  Mon Jan  9 20:52:31 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:31 +0100
Subject: [experimental-ims] 9b8598d Align error message with new parameter name
Message-ID: <E1RkMCl-00087c-Jd@project.varnish-software.com>

commit 9b8598d31d0218e5f8da8ff686a699970be8209e
Author: Andreas Plesner Jacobsen <apj at mutt.dk>
Date:   Thu Nov 10 11:13:52 2011 +0100

    Align error message with new parameter name

diff --git a/lib/libvarnishapi/vsl_arg.c b/lib/libvarnishapi/vsl_arg.c
index 56f93e2..4bcd44a 100644
--- a/lib/libvarnishapi/vsl_arg.c
+++ b/lib/libvarnishapi/vsl_arg.c
@@ -182,7 +182,7 @@ vsl_m_arg(const struct VSM_data *vd, const char *opt)
 	CHECK_OBJ_NOTNULL(vd, VSM_MAGIC);
 
 	if (!strchr(opt, ':')) {
-		fprintf(stderr, "No : found in -o option %s\n", opt);
+		fprintf(stderr, "No : found in -m option %s\n", opt);
 		return (-1);
 	}
 



From geoff at varnish-cache.org  Mon Jan  9 20:52:28 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:28 +0100
Subject: [experimental-ims] 2672b7a document the compression behaviour in
	Varnish 3.0
Message-ID: <E1RkMCi-00081k-Ta@project.varnish-software.com>

commit 2672b7ab7b11fa4bf4484759062ec3c15f46d2ef
Author: Per Buer <perbu at varnish-software.com>
Date:   Thu Nov 3 10:22:13 2011 +0100

    document the compression behaviour in Varnish 3.0

diff --git a/doc/sphinx/tutorial/compression.rst b/doc/sphinx/tutorial/compression.rst
new file mode 100644
index 0000000..0b8d1e8
--- /dev/null
+++ b/doc/sphinx/tutorial/compression.rst
@@ -0,0 +1,75 @@
+.. _tutorial-compression:
+
+Compression
+~~~~~~~~~~~
+
+New in Varnish 3.0 was native support for compression, using gzip
+encoding. *Before* 3.0, Varnish would never compress objects. 
+
+In Varnish 3.0 compression defaults to "on", meaning that it tries to
+be smart and do the sensible thing.
+
+If you don't want Varnish tampering with the encoding you can disable
+compression all together by setting the parameter http_gzip_support to
+*false*. Please see man :ref:`ref-varnishd` for details.
+
+
+Default behaviour
+~~~~~~~~~~~~~~~~~
+
+The default for Varnish is to check if the client supports our
+compression scheme (gzip) and if it does it will override the
+Accept-Encoding header and set it to "gzip".
+
+When Varnish then issues a backend request the Accept-Encoding will
+then only consist of "gzip". If the server responds with gzip'ed
+content it will be stored in memory in its compressed form. If the
+backend sends content in clear text it will be stored like that.
+
+You can make Varnish compress content before storing it in cache in
+vcl_fetch by setting do_gzip to true, like this::
+
+   sub vcl_fetch {
+        if (beresp.http.content-type ~ "text") {
+                set beresp.do_gzip = true;
+        }
+  }
+
+Please make sure that you don't try to compress content that is
+incompressable, like jpgs, gifs and mp3. You'll only waste CPU
+cycles. You can also uncompress objects before storing it in memory by
+setting do_gunzip to *true* but I have no idea why anybody would want
+to do that.
+
+Generally, Varnish doesn't use much CPU so it might make more sense to
+have Varnish spend CPU cycles compressing content than doing it in
+your web- or application servers, which are more likely to be
+CPU-bound.
+
+GZIP and ESI
+~~~~~~~~~~~~
+
+If you are using Edge Side Includes you'll be happy to note that ESI
+and GZIP work together really well. Varnish will magically decompress
+the content to do the ESI-processing, then recompress it for efficient
+storage and delivery. 
+
+
+Clients that don't support gzip
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+If the client does not support gzip the Accept-Encoding header is left
+alone and we'll end up serving whatever we get from the backend
+server. Remember that the Backend might tell Varnish to *Vary* on the
+Accept-Encoding.
+
+If the client does not support gzip but we've already got a compressed
+version of the page in memory Varnish will automatically decompress
+the page while delivering it.
+
+
+A random outburst
+~~~~~~~~~~~~~~~~~
+
+Poul has written :ref:`phk_gzip` which talks abit more about how the
+implementation works. 
diff --git a/doc/sphinx/tutorial/index.rst b/doc/sphinx/tutorial/index.rst
index 59fd94e..32c2264 100644
--- a/doc/sphinx/tutorial/index.rst
+++ b/doc/sphinx/tutorial/index.rst
@@ -25,6 +25,7 @@ separate topic. Good luck.
 	cookies
 	vary
 	purging
+	compression
 	esi
 	virtualised
 	advanced_backend_servers



From geoff at varnish-cache.org  Mon Jan  9 20:52:27 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:27 +0100
Subject: [experimental-ims] 56d17a6 Add new format %{VCL_Log:foo}x which
	output key:value from std.log() in VCL
Message-ID: <E1RkMCh-0007yG-HB@project.varnish-software.com>

commit 56d17a6dc795a8f61821e9edcaa04a57a3d13f00
Author: Lasse Karstensen <lasse at varnish-software.com>
Date:   Mon Oct 31 14:28:10 2011 +0100

    Add new format %{VCL_Log:foo}x which output key:value from std.log() in VCL

diff --git a/bin/varnishncsa/varnishncsa.c b/bin/varnishncsa/varnishncsa.c
index 47e5e2a..f699d9b 100644
--- a/bin/varnishncsa/varnishncsa.c
+++ b/bin/varnishncsa/varnishncsa.c
@@ -108,6 +108,7 @@ static struct logline {
 	uint64_t bitmap;		/* Bitmap for regex matches */
 	VTAILQ_HEAD(, hdr) req_headers; /* Request headers */
 	VTAILQ_HEAD(, hdr) resp_headers; /* Response headers */
+	VTAILQ_HEAD(, hdr) vcl_log;     /* VLC_Log entries */
 } **ll;
 
 struct VSM_data *vd;
@@ -219,6 +220,19 @@ resp_header(struct logline *l, const char *name)
 	return NULL;
 }
 
+static char *
+vcl_log(struct logline *l, const char *name)
+{
+	struct hdr *h;
+	VTAILQ_FOREACH(h, &l->vcl_log, list) {
+		if (strcasecmp(h->key, name) == 0) {
+			return h->value;
+			break;
+		}
+	}
+	return NULL;
+}
+
 static void
 clean_logline(struct logline *lp)
 {
@@ -245,6 +259,12 @@ clean_logline(struct logline *lp)
 		freez(h->value);
 		freez(h);
 	}
+	VTAILQ_FOREACH_SAFE(h, &lp->vcl_log, list, h2) {
+		VTAILQ_REMOVE(&lp->vcl_log, h, list);
+		freez(h->key);
+		freez(h->value);
+		freez(h);
+	}
 #undef freez
 	memset(lp, 0, sizeof *lp);
 }
@@ -465,6 +485,25 @@ collect_client(struct logline *lp, enum VSL_tag_e tag, unsigned spec,
 		}
 		break;
 
+	case SLT_VCL_Log:
+		if(!lp->active)
+			break;
+
+		split = strchr(ptr, ':');
+		if (split == NULL)
+			break;
+
+		struct hdr *h;
+		h = malloc(sizeof(struct hdr));
+		AN(h);
+		AN(split);
+
+		h->key = trimline(ptr, split);
+		h->value = trimline(split+1, end);
+
+		VTAILQ_INSERT_HEAD(&lp->vcl_log, h, list);
+		break;
+
 	case SLT_VCL_call:
 		if(!lp->active)
 			break;
@@ -718,6 +757,14 @@ h_ncsa(void *priv, enum VSL_tag_e tag, unsigned fd,
 					VSB_cat(os, (lp->df_handling ? lp->df_handling : "-"));
 					p = tmp;
 					break;
+				} else if (strncmp(fname, "VCL_Log:", 8) == 0) {
+					// support pulling entries logged with std.log() into output.
+					// Format: %{VCL_Log:keyname}x
+					// Logging: std.log("keyname:value")
+					h = vcl_log(lp, fname+8);
+					VSB_cat(os, h ? h : "-");
+					p = tmp;
+					break;
 				}
 			default:
 				fprintf(stderr, "Unknown format starting at: %s\n", --p);
diff --git a/doc/sphinx/reference/varnishncsa.rst b/doc/sphinx/reference/varnishncsa.rst
index e654e37..ddb6538 100644
--- a/doc/sphinx/reference/varnishncsa.rst
+++ b/doc/sphinx/reference/varnishncsa.rst
@@ -116,6 +116,10 @@ The following options are available:
 		   Varnish:handling
 		     How the request was handled, whether it was a
 		     cache hit, miss, pass, pipe or error.
+	
+		   VCL_Log:key
+		     Output value set by std.log("key=value") in VCL.
+		     
 
 -m tag:regex only list records where tag matches regex. Multiple
             -m options are AND-ed together.



From geoff at varnish-cache.org  Mon Jan  9 20:51:55 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:51:55 +0100
Subject: [experimental-ims] 0841b65 Move vca_return_session() to Pool_Wait()
Message-ID: <E1RkMCB-0006yP-Es@project.varnish-software.com>

commit 0841b65418d21eed601be56abefb4f56b462d750
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sat Sep 17 14:44:53 2011 +0000

    Move vca_return_session() to Pool_Wait()

diff --git a/bin/varnishd/cache.h b/bin/varnishd/cache.h
index 5d44092..4fd7086 100644
--- a/bin/varnishd/cache.h
+++ b/bin/varnishd/cache.h
@@ -645,7 +645,6 @@ struct vbc {
 /* Prototypes etc ----------------------------------------------------*/
 
 /* cache_acceptor.c */
-void vca_return_session(struct sess *sp);
 void VCA_Prep(struct sess *sp);
 void VCA_Init(void);
 void VCA_Shutdown(void);
@@ -840,6 +839,7 @@ void PipeSession(struct sess *sp);
 void Pool_Init(void);
 int Pool_QueueSession(struct sess *sp);
 void Pool_Work_Thread(void *priv, struct worker *w);
+void Pool_Wait(struct sess *sp);
 
 #define WRW_IsReleased(w)	((w)->wrw.wfd == NULL)
 int WRW_Error(const struct worker *w);
diff --git a/bin/varnishd/cache_acceptor.c b/bin/varnishd/cache_acceptor.c
index 3781f47..23752f3 100644
--- a/bin/varnishd/cache_acceptor.c
+++ b/bin/varnishd/cache_acceptor.c
@@ -40,9 +40,6 @@
 #include "vcli.h"
 #include "cli_priv.h"
 #include "cache.h"
-#include "cache_waiter.h"
-
-static void *waiter_priv;
 
 pthread_t		VCA_thread;
 static struct timeval	tv_sndtimeo;
@@ -298,25 +295,6 @@ vca_acct(void *arg)
 
 /*--------------------------------------------------------------------*/
 
-void
-vca_return_session(struct sess *sp)
-{
-
-	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
-	AZ(sp->obj);
-	AZ(sp->vcl);
-	assert(sp->fd >= 0);
-	/*
-	 * Set nonblocking in the worker-thread, before passing to the
-	 * acceptor thread, to reduce syscall density of the latter.
-	 */
-	if (VTCP_nonblocking(sp->fd))
-		SES_Close(sp, "remote closed");
-	waiter->pass(waiter_priv, sp);
-}
-
-/*--------------------------------------------------------------------*/
-
 static void
 ccf_start(struct cli *cli, const char * const *av, void *priv)
 {
@@ -325,14 +303,7 @@ ccf_start(struct cli *cli, const char * const *av, void *priv)
 	(void)av;
 	(void)priv;
 
-	AN(waiter);
-	AN(waiter->name);
-	AN(waiter->init);
-	AN(waiter->pass);
-
-	waiter_priv = waiter->init();
 	AZ(pthread_create(&VCA_thread, NULL, vca_acct, NULL));
-	VSL(SLT_Debug, 0, "Acceptor is %s", waiter->name);
 }
 
 /*--------------------------------------------------------------------*/
diff --git a/bin/varnishd/cache_center.c b/bin/varnishd/cache_center.c
index 24b0ddc..b561c84 100644
--- a/bin/varnishd/cache_center.c
+++ b/bin/varnishd/cache_center.c
@@ -108,7 +108,7 @@ cnt_wait(struct sess *sp)
 		sp->wrk->stats.sess_herd++;
 		SES_Charge(sp);
 		sp->wrk = NULL;
-		vca_return_session(sp);
+		Pool_Wait(sp);
 		return (1);
 	}
 	if (i == 1) {
@@ -398,7 +398,7 @@ cnt_done(struct sess *sp)
 	}
 	sp->wrk->stats.sess_herd++;
 	sp->wrk = NULL;
-	vca_return_session(sp);
+	Pool_Wait(sp);
 	return (1);
 }
 
diff --git a/bin/varnishd/cache_pool.c b/bin/varnishd/cache_pool.c
index 92e81b1..0bbb76d 100644
--- a/bin/varnishd/cache_pool.c
+++ b/bin/varnishd/cache_pool.c
@@ -54,9 +54,12 @@
 #include "vcl.h"
 #include "cli_priv.h"
 #include "cache.h"
+#include "cache_waiter.h"
 #include "stevedore.h"
 #include "hash_slinger.h"
 
+static void *waiter_priv;
+
 VTAILQ_HEAD(workerhead, worker);
 
 /* Number of work requests queued in excess of worker threads available */
@@ -454,6 +457,27 @@ wrk_herder_thread(void *priv)
 	NEEDLESS_RETURN(NULL);
 }
 
+/*--------------------------------------------------------------------
+ * Wait for another request
+ */
+
+void
+Pool_Wait(struct sess *sp)
+{
+
+	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+	AZ(sp->obj);
+	AZ(sp->vcl);
+	assert(sp->fd >= 0);
+	/*
+	 * Set nonblocking in the worker-thread, before passing to the
+	 * acceptor thread, to reduce syscall density of the latter.
+	 */
+	if (VTCP_nonblocking(sp->fd))
+		SES_Close(sp, "remote closed");
+	waiter->pass(waiter_priv, sp);
+}
+
 /*--------------------------------------------------------------------*/
 
 void
@@ -464,6 +488,8 @@ Pool_Init(void)
 	AZ(pthread_cond_init(&herder_cond, NULL));
 	Lck_New(&herder_mtx, lck_herder);
 
+	waiter_priv = waiter->init();
+
 	wrk_addpools(params->wthread_pools);
 	AZ(pthread_create(&tp, NULL, wrk_herdtimer_thread, NULL));
 	AZ(pthread_detach(tp));
diff --git a/bin/varnishd/cache_waiter.h b/bin/varnishd/cache_waiter.h
index 4377f82..28bc39d 100644
--- a/bin/varnishd/cache_waiter.h
+++ b/bin/varnishd/cache_waiter.h
@@ -53,11 +53,12 @@ extern const struct waiter waiter_kqueue;
 extern const struct waiter waiter_ports;
 #endif
 
-extern const struct waiter waiter_poll;
 
 /* cache_session.c */
 void SES_Handle(struct sess *sp, int status);
 
+/* cache_waiter.c */
+extern const struct waiter waiter_poll;
 const char *WAIT_GetName(void);
 void WAIT_tweak_waiter(struct cli *cli, const char *arg);
 void WAIT_Init(void);



From geoff at varnish-cache.org  Mon Jan  9 20:52:28 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:28 +0100
Subject: [experimental-ims] 76ca3a1 Remove a bit of stale debugging.
Message-ID: <E1RkMCi-0007zu-51@project.varnish-software.com>

commit 76ca3a10089adb2993d817ff1967e0079cbbed4a
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Oct 31 22:59:25 2011 +0000

    Remove a bit of stale debugging.

diff --git a/bin/varnishd/cache_fetch.c b/bin/varnishd/cache_fetch.c
index 516517f..ed1e5c8 100644
--- a/bin/varnishd/cache_fetch.c
+++ b/bin/varnishd/cache_fetch.c
@@ -91,10 +91,6 @@ static void __match_proto__()
 vfp_nop_begin(struct worker *w, size_t estimate)
 {
 
-	if (fetchfrag > 0) {
-		estimate = fetchfrag;
-		WSLB(w, SLT_Debug, "Fetch %d byte segments:", fetchfrag);
-	}
 	if (estimate > 0)
 		(void)FetchStorage(w, estimate);
 }



From geoff at varnish-cache.org  Mon Jan  9 20:52:42 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:42 +0100
Subject: [experimental-ims] d13c674 Add stats counters for VSM usage/overflows.
Message-ID: <E1RkMCw-0008Q0-4r@project.varnish-software.com>

commit d13c674d347663ec812c846f637b31ec48867ab5
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu Nov 24 10:01:23 2011 +0000

    Add stats counters for VSM usage/overflows.
    
    Various minor polish to VSM area.

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index 190e945..5e813ca 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -850,7 +850,7 @@ int SES_Schedule(struct sess *sp);
 
 /* cache_shmlog.c */
 extern struct VSC_C_main *VSC_C_main;
-void VSL_Init(void);
+void VSM_Init(void);
 void *VSM_Alloc(unsigned size, const char *class, const char *type,
     const char *ident);
 void VSM_Free(void *ptr);
diff --git a/bin/varnishd/cache/cache_main.c b/bin/varnishd/cache/cache_main.c
index 1f876cd..fd31de4 100644
--- a/bin/varnishd/cache/cache_main.c
+++ b/bin/varnishd/cache/cache_main.c
@@ -103,7 +103,7 @@ child_main(void)
 
 	THR_SetName("cache-main");
 
-	VSL_Init();	/* First, LCK needs it. */
+	VSM_Init();	/* First, LCK needs it. */
 
 	LCK_Init();	/* Second, locking */
 
diff --git a/bin/varnishd/cache/cache_shmlog.c b/bin/varnishd/cache/cache_shmlog.c
index 01f6bce..893525b 100644
--- a/bin/varnishd/cache/cache_shmlog.c
+++ b/bin/varnishd/cache/cache_shmlog.c
@@ -38,6 +38,7 @@
 #include "cache_backend.h"	// For w->vbc
 
 #include "vmb.h"
+#include "vtim.h"
 
 /* These cannot be struct lock, which depends on vsm/vsl working */
 static pthread_mutex_t vsl_mtx;
@@ -299,10 +300,26 @@ WSLB(struct worker *w, enum VSL_tag_e tag, const char *fmt, ...)
 
 /*--------------------------------------------------------------------*/
 
+static void *
+vsm_cleaner(void *priv)
+{
+	(void)priv;
+	THR_SetName("vsm_cleaner");
+	while (1) {
+		AZ(pthread_mutex_lock(&vsm_mtx));
+		VSM_common_cleaner(heritage.vsm, VSC_C_main);
+		AZ(pthread_mutex_unlock(&vsm_mtx));
+		VTIM_sleep(1.1);
+	}
+}
+
+/*--------------------------------------------------------------------*/
+
 void
-VSL_Init(void)
+VSM_Init(void)
 {
 	uint32_t *vsl_log_start;
+	pthread_t tp;
 
 	AZ(pthread_mutex_init(&vsl_mtx, NULL));
 	AZ(pthread_mutex_init(&vsm_mtx, NULL));
@@ -328,6 +345,8 @@ VSL_Init(void)
 	// VSM_head->starttime = (intmax_t)VTIM_real();
 	memset(VSC_C_main, 0, sizeof *VSC_C_main);
 	// VSM_head->child_pid = getpid();
+
+	AZ(pthread_create(&tp, NULL, vsm_cleaner, NULL));
 }
 
 /*--------------------------------------------------------------------*/
diff --git a/bin/varnishd/common/common.h b/bin/varnishd/common/common.h
index 0f25d31..2ff5623 100644
--- a/bin/varnishd/common/common.h
+++ b/bin/varnishd/common/common.h
@@ -69,12 +69,14 @@ void mgt_child_inherit(int fd, const char *what);
 
 /* vsm.c */
 struct vsm_sc;
+struct VSC_C_main;
 struct vsm_sc *VSM_common_new(void *ptr, ssize_t len);
 void *VSM_common_alloc(struct vsm_sc *sc, ssize_t size,
     const char *class, const char *type, const char *ident);
 void VSM_common_free(struct vsm_sc *sc, void *ptr);
 void VSM_common_delete(struct vsm_sc **sc);
 void VSM_common_copy(struct vsm_sc *to, const struct vsm_sc *from);
+void VSM_common_cleaner(struct vsm_sc *sc, struct VSC_C_main *stats);
 
 /*---------------------------------------------------------------------
  * Generic power-2 rounding macros
diff --git a/bin/varnishd/common/common_vsm.c b/bin/varnishd/common/common_vsm.c
index 3853ed8..e53972c 100644
--- a/bin/varnishd/common/common_vsm.c
+++ b/bin/varnishd/common/common_vsm.c
@@ -43,6 +43,7 @@
 #include "common.h"
 
 #include "vapi/vsm_int.h"
+#include "vapi/vsc_int.h"
 #include "vmb.h"
 #include "vtim.h"
 
@@ -69,6 +70,11 @@ struct vsm_sc {
 	VTAILQ_HEAD(,vsm_range)		r_cooling;
 	VTAILQ_HEAD(,vsm_range)		r_free;
 	VTAILQ_HEAD(,vsm_range)		r_bogus;
+	uint64_t			g_free;
+	uint64_t			g_used;
+	uint64_t			g_cooling;
+	uint64_t			g_overflow;
+	uint64_t			c_overflow;
 };
 
 /*--------------------------------------------------------------------
@@ -148,10 +154,37 @@ VSM_common_new(void *p, ssize_t l)
 	vr->off = RUP2(sizeof(*sc->head), 16);
 	vr->len = RDN2(l - vr->off, 16);
 	VTAILQ_INSERT_TAIL(&sc->r_free, vr, list);
+	sc->g_free = vr->len;
 	return (sc);
 }
 
 /*--------------------------------------------------------------------
+ * Move from cooling list to free list
+ */
+
+void
+VSM_common_cleaner(struct vsm_sc *sc, struct VSC_C_main *stats)
+{
+	double now = VTIM_real();
+	struct vsm_range *vr, *vr2;
+
+	CHECK_OBJ_NOTNULL(sc, VSM_SC_MAGIC);
+
+	/* Move cooled off stuff to free list */
+	VTAILQ_FOREACH_SAFE(vr, &sc->r_cooling, list, vr2) {
+		if (vr->cool > now)
+			break;
+		VTAILQ_REMOVE(&sc->r_cooling, vr, list);
+		vsm_common_insert_free(sc, vr);
+	}
+	stats->vsm_free = sc->g_free;
+	stats->vsm_used = sc->g_used;
+	stats->vsm_cooling = sc->g_cooling;
+	stats->vsm_overflow = sc->g_overflow;
+	stats->vsm_overflowed = sc->c_overflow;
+}
+
+/*--------------------------------------------------------------------
  * Allocate a chunk from VSM
  */
 
@@ -160,7 +193,6 @@ VSM_common_alloc(struct vsm_sc *sc, ssize_t size,
     const char *class, const char *type, const char *ident)
 {
 	struct vsm_range *vr, *vr2, *vr3;
-	double now = VTIM_real();
 	unsigned l1, l2;
 
 	CHECK_OBJ_NOTNULL(sc, VSM_SC_MAGIC);
@@ -174,14 +206,6 @@ VSM_common_alloc(struct vsm_sc *sc, ssize_t size,
 	AN(ident);
 	assert(strlen(ident) < sizeof(vr->chunk->ident));
 
-	/* Move cooled off stuff to free list */
-	VTAILQ_FOREACH_SAFE(vr, &sc->r_cooling, list, vr2) {
-		if (vr->cool > now)
-			break;
-		VTAILQ_REMOVE(&sc->r_cooling, vr, list);
-		vsm_common_insert_free(sc, vr);
-	}
-
 	l1 = RUP2(size + sizeof(struct VSM_chunk), 16);
 	l2 = RUP2(size + 2 * sizeof(struct VSM_chunk), 16);
 
@@ -213,12 +237,15 @@ VSM_common_alloc(struct vsm_sc *sc, ssize_t size,
 		AN(vr);
 		vr->ptr = malloc(size);
 		AN(vr->ptr);
+		vr->len = size;
 		VTAILQ_INSERT_TAIL(&sc->r_bogus, vr, list);
-		/* XXX: log + stats */
+		sc->g_overflow += vr->len;
+		sc->c_overflow += vr->len;
 		return (vr->ptr);
 	}
 
-	/* XXX: stats ? */
+	sc->g_free -= vr->len;
+	sc->g_used += vr->len;
 
 	/* Zero the entire allocation, to avoid garbage confusing readers */
 	memset(sc->b + vr->off, 0, vr->len);
@@ -263,7 +290,10 @@ VSM_common_free(struct vsm_sc *sc, void *ptr)
 	VTAILQ_FOREACH(vr, &sc->r_used, list) {
 		if (vr->ptr != ptr)
 			continue;
-		/* XXX: stats ? */
+
+		sc->g_used -= vr->len;
+		sc->g_cooling += vr->len;
+
 		vr2 = VTAILQ_NEXT(vr, list);
 		VTAILQ_REMOVE(&sc->r_used, vr, list);
 		VTAILQ_INSERT_TAIL(&sc->r_cooling, vr, list);
@@ -278,15 +308,18 @@ VSM_common_free(struct vsm_sc *sc, void *ptr)
 		VWMB();
 		return;
 	}
+
 	/* Look in bogus list, free */
 	VTAILQ_FOREACH(vr, &sc->r_bogus, list) {
-		if (vr->ptr == ptr) {
-			VTAILQ_REMOVE(&sc->r_bogus, vr, list);
-			FREE_OBJ(vr);
-			/* XXX: stats ? */
-			free(ptr);
-			return;
-		}
+		if (vr->ptr != ptr)
+			continue;
+
+		sc->g_overflow -= vr->len;
+
+		VTAILQ_REMOVE(&sc->r_bogus, vr, list);
+		FREE_OBJ(vr);
+		free(ptr);
+		return;
 	}
 	/* Panic */
 	assert(ptr == NULL);
@@ -326,7 +359,7 @@ VSM_common_delete(struct vsm_sc **scp)
 }
 
 /*--------------------------------------------------------------------
- * Copy one VSM to another
+ * Copy all chunks in one VSM segment to another VSM segment
  */
 
 void
diff --git a/include/tbl/vsc_f_main.h b/include/tbl/vsc_f_main.h
index 72d8842..a04767d 100644
--- a/include/tbl/vsc_f_main.h
+++ b/include/tbl/vsc_f_main.h
@@ -359,3 +359,38 @@ VSC_F(vmods,		uint64_t, 0, 'i', "Loaded VMODs", "")
 
 VSC_F(n_gzip,			uint64_t, 0, 'a', "Gzip operations", "")
 VSC_F(n_gunzip,			uint64_t, 0, 'a', "Gunzip operations", "")
+
+/**********************************************************************/
+
+VSC_F(vsm_free,			uint64_t, 0, 'g',
+    "Free VSM space",
+	"Number of bytes free in the shared memory used to communicate"
+	" with tools like varnishstat, varnishlog etc."
+)
+
+VSC_F(vsm_used,			uint64_t, 0, 'g',
+    "Used VSM space",
+	"Number of bytes used in the shared memory used to communicate"
+	" with tools like varnishstat, varnishlog etc."
+)
+
+VSC_F(vsm_cooling,		uint64_t, 0, 'g',
+    "Cooling VSM space",
+	"Number of bytes which will soon (max 1 minute) be freed"
+	" in the shared memory used to communicate"
+	" with tools like varnishstat, varnishlog etc."
+)
+
+VSC_F(vsm_overflow,		uint64_t, 0, 'g',
+    "Overflow VSM space",
+	"Number of bytes which does not fit"
+	" in the shared memory used to communicate"
+	" with tools like varnishstat, varnishlog etc."
+)
+
+VSC_F(vsm_overflowed,		uint64_t, 0, 'c',
+    "Overflowed VSM space",
+	"Total number of bytes which did not fit"
+	" in the shared memory used to communicate"
+	" with tools like varnishstat, varnishlog etc."
+)
diff --git a/include/vapi/vsm_int.h b/include/vapi/vsm_int.h
index 237fabd..4272aef 100644
--- a/include/vapi/vsm_int.h
+++ b/include/vapi/vsm_int.h
@@ -37,7 +37,7 @@
  * In particular we want the readers to seamlessly jump from one VSM instance
  * to another when the child restarts.
  *
- * The VSM life-cycle there is:
+ * The VSM segment life-cycle is:
  *
  *	Manager creates VSM file under temp name
  *
@@ -54,7 +54,7 @@
  *	it will zero the alloc_seq in it, before replacing the file.
  *
  * Subscribers will have to monitor two things to make sure they have
- * the current VSM instance:  The alloc_seq field and the inode number
+ * the current VSM instance:  The alloc_seq field and the dev+inode
  * of the path-name.  The former check is by far the cheaper and the
  * latter check should only be employed when lack of activity in the
  * VSM segment raises suspicion that something has happened.



From geoff at varnish-cache.org  Mon Jan  9 20:52:31 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:31 +0100
Subject: [experimental-ims] e4fa4c7 Rename the childs copy of params to
	cache_param so it is clear that it belongs there.
Message-ID: <E1RkMCl-00086w-5M@project.varnish-software.com>

commit e4fa4c7ab40a7514395645cbe6e1227d580d4882
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu Nov 10 09:05:18 2011 +0000

    Rename the childs copy of params to cache_param so it is clear that
    it belongs there.

diff --git a/bin/varnishd/cache.h b/bin/varnishd/cache.h
index 31db957..5941fcc 100644
--- a/bin/varnishd/cache.h
+++ b/bin/varnishd/cache.h
@@ -873,7 +873,7 @@ void WSL_Flush(struct worker *w, int overflow);
 
 #define DSL(flag, tag, id, ...)					\
 	do {							\
-		if (params->diag_bitmap & (flag))		\
+		if (cache_param->diag_bitmap & (flag))		\
 			VSL((tag), (id), __VA_ARGS__);		\
 	} while (0)
 
diff --git a/bin/varnishd/cache_acceptor.c b/bin/varnishd/cache_acceptor.c
index 03a1a34..550a53f 100644
--- a/bin/varnishd/cache_acceptor.c
+++ b/bin/varnishd/cache_acceptor.c
@@ -121,7 +121,7 @@ VCA_Prep(struct sess *sp)
 	    addr, sizeof addr, port, sizeof port);
 	sp->addr = WS_Dup(sp->ws, addr);
 	sp->port = WS_Dup(sp->ws, port);
-	if (params->log_local_addr) {
+	if (cache_param->log_local_addr) {
 		AZ(getsockname(sp->fd, (void*)&sp->mysockaddr, &sp->mysockaddrlen));
 		VTCP_name(&sp->mysockaddr, sp->mysockaddrlen,
 		    addr, sizeof addr, port, sizeof port);
@@ -176,9 +176,9 @@ vca_pace_bad(void)
 {
 
 	Lck_Lock(&pace_mtx);
-	vca_pace += params->acceptor_sleep_incr;
-	if (vca_pace > params->acceptor_sleep_max)
-		vca_pace = params->acceptor_sleep_max;
+	vca_pace += cache_param->acceptor_sleep_incr;
+	if (vca_pace > cache_param->acceptor_sleep_max)
+		vca_pace = cache_param->acceptor_sleep_max;
 	Lck_Unlock(&pace_mtx);
 }
 
@@ -189,8 +189,8 @@ vca_pace_good(void)
 	if (vca_pace == 0.0)
 		return;
 	Lck_Lock(&pace_mtx);
-	vca_pace *= params->acceptor_sleep_decay;
-	if (vca_pace < params->acceptor_sleep_incr)
+	vca_pace *= cache_param->acceptor_sleep_decay;
+	if (vca_pace < cache_param->acceptor_sleep_incr)
 		vca_pace = 0.0;
 	Lck_Unlock(&pace_mtx);
 }
@@ -306,7 +306,7 @@ vca_acct(void *arg)
 	VTAILQ_FOREACH(ls, &heritage.socks, list) {
 		if (ls->sock < 0)
 			continue;
-		AZ(listen(ls->sock, params->listen_depth));
+		AZ(listen(ls->sock, cache_param->listen_depth));
 		AZ(setsockopt(ls->sock, SOL_SOCKET, SO_LINGER,
 		    &linger, sizeof linger));
 	}
@@ -318,9 +318,9 @@ vca_acct(void *arg)
 	while (1) {
 		(void)sleep(1);
 #ifdef SO_SNDTIMEO_WORKS
-		if (params->idle_send_timeout != send_timeout) {
+		if (cache_param->idle_send_timeout != send_timeout) {
 			need_test = 1;
-			send_timeout = params->idle_send_timeout;
+			send_timeout = cache_param->idle_send_timeout;
 			tv_sndtimeo = VTIM_timeval(send_timeout);
 			VTAILQ_FOREACH(ls, &heritage.socks, list) {
 				if (ls->sock < 0)
@@ -332,9 +332,9 @@ vca_acct(void *arg)
 		}
 #endif
 #ifdef SO_RCVTIMEO_WORKS
-		if (params->sess_timeout != sess_timeout) {
+		if (cache_param->sess_timeout != sess_timeout) {
 			need_test = 1;
-			sess_timeout = params->sess_timeout;
+			sess_timeout = cache_param->sess_timeout;
 			tv_rcvtimeo = VTIM_timeval(sess_timeout);
 			VTAILQ_FOREACH(ls, &heritage.socks, list) {
 				if (ls->sock < 0)
diff --git a/bin/varnishd/cache_backend.c b/bin/varnishd/cache_backend.c
index c94d819..e9d0323 100644
--- a/bin/varnishd/cache_backend.c
+++ b/bin/varnishd/cache_backend.c
@@ -96,7 +96,7 @@ VBE_ReleaseConn(struct vbc *vc)
 		if (dst == 0.0)			\
 			dst = be->tmx;		\
 		if (dst == 0.0)			\
-			dst = params->tmx;	\
+			dst = cache_param->tmx;	\
 	} while (0)
 
 /*--------------------------------------------------------------------
@@ -158,7 +158,7 @@ bes_conn_try(const struct sess *sp, struct vbc *vc, const struct vdi_simple *vs)
 
 	/* release lock during stuff that can take a long time */
 
-	if (params->prefer_ipv6 && bp->ipv6 != NULL) {
+	if (cache_param->prefer_ipv6 && bp->ipv6 != NULL) {
 		s = vbe_TryConnect(sp, PF_INET6, bp->ipv6, bp->ipv6len, vs);
 		vc->addr = bp->ipv6;
 		vc->addrlen = bp->ipv6len;
@@ -168,7 +168,7 @@ bes_conn_try(const struct sess *sp, struct vbc *vc, const struct vdi_simple *vs)
 		vc->addr = bp->ipv4;
 		vc->addrlen = bp->ipv4len;
 	}
-	if (s == -1 && !params->prefer_ipv6 && bp->ipv6 != NULL) {
+	if (s == -1 && !cache_param->prefer_ipv6 && bp->ipv6 != NULL) {
 		s = vbe_TryConnect(sp, PF_INET6, bp->ipv6, bp->ipv6len, vs);
 		vc->addr = bp->ipv6;
 		vc->addrlen = bp->ipv6len;
@@ -232,7 +232,7 @@ vbe_NewConn(void)
  * It evaluates if a backend is healthy _for_a_specific_object_.
  * That means that it relies on sp->objcore->objhead. This is mainly for
  * saint-mode, but also takes backend->healthy into account. If
- * params->saintmode_threshold is 0, this is basically just a test of
+ * cache_param->saintmode_threshold is 0, this is basically just a test of
  * backend->healthy.
  *
  * The threshold has to be evaluated _after_ the timeout check, otherwise
@@ -266,7 +266,7 @@ vbe_Healthy(const struct vdi_simple *vs, const struct sess *sp)
 	 * specified by VCL (thus use param).
 	 */
 	if (vs->vrt->saintmode_threshold == UINT_MAX)
-		threshold = params->saintmode_threshold;
+		threshold = cache_param->saintmode_threshold;
 	else
 		threshold = vs->vrt->saintmode_threshold;
 
diff --git a/bin/varnishd/cache_backend_poll.c b/bin/varnishd/cache_backend_poll.c
index eb6cc45..efd64cb 100644
--- a/bin/varnishd/cache_backend_poll.c
+++ b/bin/varnishd/cache_backend_poll.c
@@ -138,7 +138,7 @@ vbp_poke(struct vbp_target *vt)
 	tmo = (int)round((t_end - t_now) * 1e3);
 
 	s = -1;
-	if (params->prefer_ipv6 && bp->ipv6 != NULL) {
+	if (cache_param->prefer_ipv6 && bp->ipv6 != NULL) {
 		s = vbp_connect(PF_INET6, bp->ipv6, bp->ipv6len, tmo);
 		t_now = VTIM_real();
 		tmo = (int)round((t_end - t_now) * 1e3);
diff --git a/bin/varnishd/cache_ban.c b/bin/varnishd/cache_ban.c
index ac45144..768e631 100644
--- a/bin/varnishd/cache_ban.c
+++ b/bin/varnishd/cache_ban.c
@@ -407,7 +407,7 @@ BAN_Insert(struct ban *b)
 		VSC_C_main->bans_req++;
 
 	be = VTAILQ_LAST(&ban_head, banhead_s);
-	if (params->ban_dups && be != b)
+	if (cache_param->ban_dups && be != b)
 		be->refcount++;
 	else
 		be = NULL;
@@ -825,13 +825,13 @@ ban_lurker_work(const struct sess *sp, unsigned pass)
 		b->flags &= ~BAN_F_LURK;
 		b->flags |= pass;
 	}
-	if (params->diag_bitmap & 0x80000)
+	if (cache_param->diag_bitmap & 0x80000)
 		VSL(SLT_Debug, 0, "lurker: %d actionable bans", i);
 	if (i == 0)
 		return (0);
 
 	VTAILQ_FOREACH_REVERSE(b, &ban_head, banhead_s, list) {
-		if (params->diag_bitmap & 0x80000)
+		if (cache_param->diag_bitmap & 0x80000)
 			VSL(SLT_Debug, 0, "lurker doing %f %d",
 			    ban_time(b->spec), b->refcount);
 		while (1) {
@@ -840,7 +840,7 @@ ban_lurker_work(const struct sess *sp, unsigned pass)
 			if (oc == NULL)
 				break;
 			CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
-			if (params->diag_bitmap & 0x80000)
+			if (cache_param->diag_bitmap & 0x80000)
 				VSL(SLT_Debug, 0, "test: %p %d %d",
 				    oc, oc->flags & OC_F_LURK, pass);
 			if ((oc->flags & OC_F_LURK) == pass)
@@ -849,7 +849,7 @@ ban_lurker_work(const struct sess *sp, unsigned pass)
 			CHECK_OBJ_NOTNULL(oh, OBJHEAD_MAGIC);
 			if (Lck_Trylock(&oh->mtx)) {
 				Lck_Unlock(&ban_mtx);
-				VTIM_sleep(params->ban_lurker_sleep);
+				VTIM_sleep(cache_param->ban_lurker_sleep);
 				continue;
 			}
 			/*
@@ -863,7 +863,7 @@ ban_lurker_work(const struct sess *sp, unsigned pass)
 			if (oc2 == NULL) {
 				Lck_Unlock(&oh->mtx);
 				Lck_Unlock(&ban_mtx);
-				VTIM_sleep(params->ban_lurker_sleep);
+				VTIM_sleep(cache_param->ban_lurker_sleep);
 				continue;
 			}
 			/*
@@ -879,7 +879,7 @@ ban_lurker_work(const struct sess *sp, unsigned pass)
 			 */
 			o = oc_getobj(sp->wrk, oc);
 			i = ban_check_object(o, sp, 0);
-			if (params->diag_bitmap & 0x80000)
+			if (cache_param->diag_bitmap & 0x80000)
 				VSL(SLT_Debug, 0, "lurker got: %p %d",
 				    oc, i);
 			if (i == -1) {
@@ -891,11 +891,11 @@ ban_lurker_work(const struct sess *sp, unsigned pass)
 				Lck_Unlock(&ban_mtx);
 			}
 			Lck_Unlock(&oh->mtx);
-			if (params->diag_bitmap & 0x80000)
+			if (cache_param->diag_bitmap & 0x80000)
 				VSL(SLT_Debug, 0, "lurker done: %p %d %d",
 				    oc, oc->flags & OC_F_LURK, pass);
 			(void)HSH_Deref(sp->wrk, NULL, &o);
-			VTIM_sleep(params->ban_lurker_sleep);
+			VTIM_sleep(cache_param->ban_lurker_sleep);
 		}
 		Lck_AssertHeld(&ban_mtx);
 		if (!(b->flags & BAN_F_REQ)) {
@@ -903,12 +903,12 @@ ban_lurker_work(const struct sess *sp, unsigned pass)
 				b->flags |= BAN_F_GONE;
 				VSC_C_main->bans_gone++;
 			}
-			if (params->diag_bitmap & 0x80000)
+			if (cache_param->diag_bitmap & 0x80000)
 				VSL(SLT_Debug, 0, "lurker BAN %f now gone",
 				    ban_time(b->spec));
 		}
 		Lck_Unlock(&ban_mtx);
-		VTIM_sleep(params->ban_lurker_sleep);
+		VTIM_sleep(cache_param->ban_lurker_sleep);
 		if (b == b0)
 			break;
 	}
@@ -925,7 +925,7 @@ ban_lurker(struct sess *sp, void *priv)
 	(void)priv;
 	while (1) {
 
-		while (params->ban_lurker_sleep == 0.0) {
+		while (cache_param->ban_lurker_sleep == 0.0) {
 			/*
 			 * Ban-lurker is disabled:
 			 * Clean the last ban, if possible, and sleep
@@ -947,7 +947,7 @@ ban_lurker(struct sess *sp, void *priv)
 			pass &= BAN_F_LURK;
 			if (pass == 0)
 				pass += (1 << LURK_SHIFT);
-			VTIM_sleep(params->ban_lurker_sleep);
+			VTIM_sleep(cache_param->ban_lurker_sleep);
 		} else {
 			VTIM_sleep(1.0);
 		}
@@ -1064,14 +1064,14 @@ ccf_ban_list(struct cli *cli, const char * const *av, void *priv)
 
 	VCLI_Out(cli, "Present bans:\n");
 	VTAILQ_FOREACH(b, &ban_head, list) {
-		if (b == bl && !(params->diag_bitmap & 0x80000))
+		if (b == bl && !(cache_param->diag_bitmap & 0x80000))
 			break;
 		VCLI_Out(cli, "%10.6f %5u%s\t", ban_time(b->spec),
 		    bl == b ? b->refcount - 1 : b->refcount,
 		    b->flags & BAN_F_GONE ? "G" : " ");
 		ban_render(cli, b->spec);
 		VCLI_Out(cli, "\n");
-		if (params->diag_bitmap & 0x80000) {
+		if (cache_param->diag_bitmap & 0x80000) {
 			Lck_Lock(&ban_mtx);
 			struct objcore *oc;
 			VTAILQ_FOREACH(oc, &b->objcore, ban_list)
diff --git a/bin/varnishd/cache_center.c b/bin/varnishd/cache_center.c
index 37c4b7b..e42fac8 100644
--- a/bin/varnishd/cache_center.c
+++ b/bin/varnishd/cache_center.c
@@ -94,11 +94,11 @@ cnt_wait(struct sess *sp)
 	assert(sp->xid == 0);
 
 	i = HTC_Complete(sp->htc);
-	if (i == 0 && params->session_linger > 0) {
+	if (i == 0 && cache_param->session_linger > 0) {
 		pfd[0].fd = sp->fd;
 		pfd[0].events = POLLIN;
 		pfd[0].revents = 0;
-		i = poll(pfd, 1, params->session_linger);
+		i = poll(pfd, 1, cache_param->session_linger);
 		if (i)
 			i = HTC_Rx(sp->htc);
 	}
@@ -183,7 +183,7 @@ cnt_prepresp(struct sess *sp)
 		sp->wrk->res_mode |= RES_ESI_CHILD;
 	}
 
-	if (params->http_gzip_support && sp->obj->gziped &&
+	if (cache_param->http_gzip_support && sp->obj->gziped &&
 	    !RFC2616_Req_Gzip(sp)) {
 		/*
 		 * We don't know what it uncompresses to
@@ -212,7 +212,7 @@ cnt_prepresp(struct sess *sp)
 
 	sp->t_resp = VTIM_real();
 	if (sp->obj->objcore != NULL) {
-		if ((sp->t_resp - sp->obj->last_lru) > params->lru_timeout &&
+		if ((sp->t_resp - sp->obj->last_lru) > cache_param->lru_timeout &&
 		    EXP_Touch(sp->obj->objcore))
 			sp->obj->last_lru = sp->t_resp;
 		sp->obj->last_use = sp->t_resp;	/* XXX: locking ? */
@@ -224,7 +224,7 @@ cnt_prepresp(struct sess *sp)
 	case VCL_RET_DELIVER:
 		break;
 	case VCL_RET_RESTART:
-		if (sp->restarts >= params->max_restarts)
+		if (sp->restarts >= cache_param->max_restarts)
 			break;
 		if (sp->wrk->do_stream) {
 			VDI_CloseFd(sp->wrk);
@@ -373,7 +373,7 @@ cnt_done(struct sess *sp)
 		return (1);
 	}
 
-	if (sp->wrk->stats.client_req >= params->wthread_stats_rate)
+	if (sp->wrk->stats.client_req >= cache_param->wthread_stats_rate)
 		WRK_SumStat(sp->wrk);
 	/* Reset the workspace to the session-watermark */
 	WS_Reset(sp->ws, sp->ws_ses);
@@ -390,7 +390,7 @@ cnt_done(struct sess *sp)
 		sp->step = STP_WAIT;
 		return (0);
 	}
-	if (params->session_linger > 0) {
+	if (cache_param->session_linger > 0) {
 		sp->wrk->stats.sess_linger++;
 		sp->step = STP_WAIT;
 		return (0);
@@ -436,12 +436,12 @@ cnt_error(struct sess *sp)
 	if (sp->obj == NULL) {
 		HSH_Prealloc(sp);
 		EXP_Clr(&w->exp);
-		sp->obj = STV_NewObject(sp, NULL, params->http_resp_size,
-		     &w->exp, (uint16_t)params->http_max_hdr);
+		sp->obj = STV_NewObject(sp, NULL, cache_param->http_resp_size,
+		     &w->exp, (uint16_t)cache_param->http_max_hdr);
 		if (sp->obj == NULL)
 			sp->obj = STV_NewObject(sp, TRANSIENT_STORAGE,
-			    params->http_resp_size, &w->exp,
-			    (uint16_t)params->http_max_hdr);
+			    cache_param->http_resp_size, &w->exp,
+			    (uint16_t)cache_param->http_max_hdr);
 		if (sp->obj == NULL) {
 			sp->doclose = "Out of objects";
 			sp->director = NULL;
@@ -477,7 +477,7 @@ cnt_error(struct sess *sp)
 	VCL_error_method(sp);
 
 	if (sp->handling == VCL_RET_RESTART &&
-	    sp->restarts <  params->max_restarts) {
+	    sp->restarts <  cache_param->max_restarts) {
 		HSH_Drop(sp);
 		sp->director = NULL;
 		sp->restarts++;
@@ -701,7 +701,7 @@ cnt_fetchbody(struct sess *sp)
 	AZ(sp->wrk->vfp);
 
 	/* We do nothing unless the param is set */
-	if (!params->http_gzip_support)
+	if (!cache_param->http_gzip_support)
 		sp->wrk->do_gzip = sp->wrk->do_gunzip = 0;
 
 	sp->wrk->is_gzip =
@@ -768,7 +768,7 @@ cnt_fetchbody(struct sess *sp)
 	 */
 	l += strlen("Content-Length: XxxXxxXxxXxxXxxXxx") + sizeof(void *);
 
-	if (sp->wrk->exp.ttl < params->shortlived || sp->objcore == NULL)
+	if (sp->wrk->exp.ttl < cache_param->shortlived || sp->objcore == NULL)
 		sp->wrk->storage_hint = TRANSIENT_STORAGE;
 
 	sp->obj = STV_NewObject(sp, sp->wrk->storage_hint, l,
@@ -780,8 +780,8 @@ cnt_fetchbody(struct sess *sp)
 		 */
 		sp->obj = STV_NewObject(sp, TRANSIENT_STORAGE, l,
 		    &sp->wrk->exp, nhttp);
-		if (sp->wrk->exp.ttl > params->shortlived)
-			sp->wrk->exp.ttl = params->shortlived;
+		if (sp->wrk->exp.ttl > cache_param->shortlived)
+			sp->wrk->exp.ttl = cache_param->shortlived;
 		sp->wrk->exp.grace = 0.0;
 		sp->wrk->exp.keep = 0.0;
 	}
@@ -893,7 +893,7 @@ cnt_streambody(struct sess *sp)
 	int i;
 	struct stream_ctx sctx;
 	uint8_t obuf[sp->wrk->res_mode & RES_GUNZIP ?
-	    params->gzip_stack_buffer : 1];
+	    cache_param->gzip_stack_buffer : 1];
 
 	memset(&sctx, 0, sizeof sctx);
 	sctx.magic = STREAM_CTX_MAGIC;
@@ -967,8 +967,8 @@ cnt_first(struct sess *sp)
 	sp->ws_ses = WS_Snapshot(sp->ws);
 
 	/* Receive a HTTP protocol request */
-	HTC_Init(sp->htc, sp->ws, sp->fd, sp->vsl_id, params->http_req_size,
-	    params->http_req_hdr_len);
+	HTC_Init(sp->htc, sp->ws, sp->fd, sp->vsl_id, cache_param->http_req_size,
+	    cache_param->http_req_hdr_len);
 	sp->wrk->lastused = sp->t_open;
 	sp->wrk->acct_tmp.sess++;
 
@@ -1186,7 +1186,7 @@ cnt_miss(struct sess *sp)
 	http_Setup(sp->wrk->bereq, sp->wrk->ws);
 	http_FilterHeader(sp, HTTPH_R_FETCH);
 	http_ForceGet(sp->wrk->bereq);
-	if (params->http_gzip_support) {
+	if (cache_param->http_gzip_support) {
 		/*
 		 * We always ask the backend for gzip, even if the
 		 * client doesn't grok it.  We will uncompress for
@@ -1377,7 +1377,7 @@ cnt_recv(struct sess *sp)
 	VCL_recv_method(sp);
 	recv_handling = sp->handling;
 
-	if (sp->restarts >= params->max_restarts) {
+	if (sp->restarts >= cache_param->max_restarts) {
 		if (sp->err_code == 0)
 			sp->err_code = 503;
 		sp->step = STP_ERROR;
@@ -1392,7 +1392,7 @@ cnt_recv(struct sess *sp)
 	sp->wrk->do_gunzip = 0;
 	sp->wrk->do_stream = 0;
 
-	if (params->http_gzip_support &&
+	if (cache_param->http_gzip_support &&
 	     (recv_handling != VCL_RET_PIPE) &&
 	     (recv_handling != VCL_RET_PASS)) {
 		if (RFC2616_Req_Gzip(sp)) {
@@ -1607,7 +1607,7 @@ CNT_Session(struct sess *sp)
 		switch (sp->step) {
 #define STEP(l,u) \
 		    case STP_##u: \
-			if (params->diag_bitmap & 0x01) \
+			if (cache_param->diag_bitmap & 0x01) \
 				cnt_diag(sp, #u); \
 			done = cnt_##l(sp); \
 		        break;
diff --git a/bin/varnishd/cache_cli.c b/bin/varnishd/cache_cli.c
index 30a12e3..ac2afc1 100644
--- a/bin/varnishd/cache_cli.c
+++ b/bin/varnishd/cache_cli.c
@@ -235,7 +235,7 @@ CLI_Init(void)
 	Lck_New(&cli_mtx, lck_cli);
 	cli_thread = pthread_self();
 
-	cls = VCLS_New(cli_cb_before, cli_cb_after, params->cli_buffer);
+	cls = VCLS_New(cli_cb_before, cli_cb_after, cache_param->cli_buffer);
 	AN(cls);
 
 	CLI_AddFuncs(master_cmds);
diff --git a/bin/varnishd/cache_esi_deliver.c b/bin/varnishd/cache_esi_deliver.c
index 2c62bb7..4051027 100644
--- a/bin/varnishd/cache_esi_deliver.c
+++ b/bin/varnishd/cache_esi_deliver.c
@@ -52,7 +52,7 @@ ved_include(struct sess *sp, const char *src, const char *host)
 
 	w = sp->wrk;
 
-	if (sp->esi_level >= params->max_esi_depth)
+	if (sp->esi_level >= cache_param->max_esi_depth)
 		return;
 	sp->esi_level++;
 
@@ -231,7 +231,7 @@ ESI_Deliver(struct sess *sp)
 	uint8_t tailbuf[8 + 5];
 	int isgzip;
 	struct vgz *vgz = NULL;
-	char obuf[params->gzip_stack_buffer];
+	char obuf[cache_param->gzip_stack_buffer];
 	ssize_t obufl = 0;
 	size_t dl;
 	const void *dp;
diff --git a/bin/varnishd/cache_esi_fetch.c b/bin/varnishd/cache_esi_fetch.c
index f414e7b..5ec8f6b 100644
--- a/bin/varnishd/cache_esi_fetch.c
+++ b/bin/varnishd/cache_esi_fetch.c
@@ -51,7 +51,7 @@ vef_read(struct worker *w, struct http_conn *htc, void *buf, ssize_t buflen,
 
 	if (buflen < bytes)
 		bytes = buflen;
-	if (params->esi_syntax & 0x8) {
+	if (cache_param->esi_syntax & 0x8) {
 		d = (random() & 3) + 1;
 		if (d < bytes)
 			bytes = d;
@@ -96,7 +96,7 @@ vfp_esi_bytes_gu(struct worker *w, struct http_conn *htc, ssize_t bytes)
 {
 	struct vgz *vg;
 	ssize_t wl;
-	uint8_t	ibuf[params->gzip_stack_buffer];
+	uint8_t	ibuf[cache_param->gzip_stack_buffer];
 	int i;
 	size_t dl;
 	const void *dp;
@@ -207,7 +207,7 @@ static int
 vfp_esi_bytes_ug(struct worker *w, struct http_conn *htc, ssize_t bytes)
 {
 	ssize_t wl;
-	char ibuf[params->gzip_stack_buffer];
+	char ibuf[cache_param->gzip_stack_buffer];
 	struct vef_priv *vef;
 
 	CHECK_OBJ_NOTNULL(w, WORKER_MAGIC);
@@ -244,8 +244,8 @@ static int
 vfp_esi_bytes_gg(struct worker *w, struct http_conn *htc, size_t bytes)
 {
 	ssize_t wl;
-	char ibuf[params->gzip_stack_buffer];
-	char ibuf2[params->gzip_stack_buffer];
+	char ibuf[cache_param->gzip_stack_buffer];
+	char ibuf2[cache_param->gzip_stack_buffer];
 	struct vef_priv *vef;
 	size_t dl;
 	const void *dp;
diff --git a/bin/varnishd/cache_esi_parse.c b/bin/varnishd/cache_esi_parse.c
index f6eb367..9e2b4f6 100644
--- a/bin/varnishd/cache_esi_parse.c
+++ b/bin/varnishd/cache_esi_parse.c
@@ -269,7 +269,7 @@ static void
 vep_emit_skip(const struct vep_state *vep, ssize_t l)
 {
 
-	if (params->esi_syntax & 0x20) {
+	if (cache_param->esi_syntax & 0x20) {
 		Debug("---> SKIP(%jd)\n", (intmax_t)l);
 	}
 	vep_emit_len(vep, l, VEC_S1, VEC_S2, VEC_S8);
@@ -280,7 +280,7 @@ vep_emit_verbatim(const struct vep_state *vep, ssize_t l, ssize_t l_crc)
 {
 	uint8_t buf[4];
 
-	if (params->esi_syntax & 0x20) {
+	if (cache_param->esi_syntax & 0x20) {
 		Debug("---> VERBATIM(%jd)\n", (intmax_t)l);
 	}
 	vep_emit_len(vep, l, VEC_V1, VEC_V2, VEC_V8);
@@ -585,7 +585,7 @@ VEP_Parse(const struct worker *w, const char *p, size_t l)
 		 */
 
 		if (vep->state == VEP_START) {
-			if (params->esi_syntax & 0x1)
+			if (cache_param->esi_syntax & 0x1)
 				vep->state = VEP_NEXTTAG;
 			else
 				vep->state = VEP_TESTXML;
@@ -618,7 +618,7 @@ VEP_Parse(const struct worker *w, const char *p, size_t l)
 		 */
 
 		} else if (vep->state == VEP_NOTMYTAG) {
-			if (params->esi_syntax & 0x2) {
+			if (cache_param->esi_syntax & 0x2) {
 				p++;
 				vep->state = VEP_NEXTTAG;
 			} else {
diff --git a/bin/varnishd/cache_expire.c b/bin/varnishd/cache_expire.c
index f7f779d..23e3fc6 100644
--- a/bin/varnishd/cache_expire.c
+++ b/bin/varnishd/cache_expire.c
@@ -113,7 +113,7 @@ EXP_Keep(const struct sess *sp, const struct object *o)
 {
 	double r;
 
-	r = (double)params->default_keep;
+	r = (double)cache_param->default_keep;
 	if (o->exp.keep > 0.)
 		r = o->exp.keep;
 	if (sp != NULL && sp->exp.keep > 0. && sp->exp.keep < r)
@@ -126,7 +126,7 @@ EXP_Grace(const struct sess *sp, const struct object *o)
 {
 	double r;
 
-	r = (double)params->default_grace;
+	r = (double)cache_param->default_grace;
 	if (o->exp.grace >= 0.)
 		r = o->exp.grace;
 	if (sp != NULL && sp->exp.grace > 0. && sp->exp.grace < r)
@@ -344,7 +344,7 @@ exp_timer(struct sess *sp, void *priv)
 		if (oc == NULL) {
 			WSL_Flush(sp->wrk, 0);
 			WRK_SumStat(sp->wrk);
-			VTIM_sleep(params->expiry_sleep);
+			VTIM_sleep(cache_param->expiry_sleep);
 			t = VTIM_real();
 		}
 
diff --git a/bin/varnishd/cache_fetch.c b/bin/varnishd/cache_fetch.c
index 67bf970..a678dcc 100644
--- a/bin/varnishd/cache_fetch.c
+++ b/bin/varnishd/cache_fetch.c
@@ -188,7 +188,7 @@ FetchStorage(struct worker *w, ssize_t sz)
 	if (l == 0)
 		l = sz;
 	if (l == 0)
-		l = params->fetch_chunksize * 1024LL;
+		l = cache_param->fetch_chunksize * 1024LL;
 	st = STV_alloc(w, l);
 	if (st == NULL) {
 		(void)FetchError(w, "Could not get storage");
@@ -437,8 +437,8 @@ FetchHdr(struct sess *sp)
 
 	/* Receive response */
 
-	HTC_Init(w->htc, w->ws, vc->fd, vc->vsl_id, params->http_resp_size,
-	    params->http_resp_hdr_len);
+	HTC_Init(w->htc, w->ws, vc->fd, vc->vsl_id, cache_param->http_resp_size,
+	    cache_param->http_resp_hdr_len);
 
 	VTCP_set_read_timeout(vc->fd, vc->first_byte_timeout);
 
diff --git a/bin/varnishd/cache_gzip.c b/bin/varnishd/cache_gzip.c
index 6da1595..32a7413 100644
--- a/bin/varnishd/cache_gzip.c
+++ b/bin/varnishd/cache_gzip.c
@@ -131,7 +131,7 @@ vgz_alloc_vgz(struct worker *wrk, const char *id)
 	vg->wrk = wrk;
 	vg->id = id;
 
-	switch (params->gzip_tmp_space) {
+	switch (cache_param->gzip_tmp_space) {
 	case 0:
 	case 1:
 		/* malloc, the default */
@@ -196,10 +196,10 @@ VGZ_NewGzip(struct worker *wrk, const char *id)
 	 * XXX: too many worker threads grow the stacks.
 	 */
 	i = deflateInit2(&vg->vz,
-	    params->gzip_level,		/* Level */
+	    cache_param->gzip_level,		/* Level */
 	    Z_DEFLATED,			/* Method */
-	    16 + params->gzip_window,	/* Window bits (16=gzip + 15) */
-	    params->gzip_memlevel,	/* memLevel */
+	    16 + cache_param->gzip_window,	/* Window bits (16=gzip + 15) */
+	    cache_param->gzip_memlevel,	/* memLevel */
 	    Z_DEFAULT_STRATEGY);
 	assert(Z_OK == i);
 	return (vg);
@@ -467,7 +467,7 @@ vfp_gunzip_bytes(struct worker *w, struct http_conn *htc, ssize_t bytes)
 	struct vgz *vg;
 	ssize_t l, wl;
 	int i = -100;
-	uint8_t	ibuf[params->gzip_stack_buffer];
+	uint8_t	ibuf[cache_param->gzip_stack_buffer];
 	size_t dl;
 	const void *dp;
 
@@ -545,7 +545,7 @@ vfp_gzip_bytes(struct worker *w, struct http_conn *htc, ssize_t bytes)
 	struct vgz *vg;
 	ssize_t l, wl;
 	int i = -100;
-	uint8_t ibuf[params->gzip_stack_buffer];
+	uint8_t ibuf[cache_param->gzip_stack_buffer];
 	size_t dl;
 	const void *dp;
 
@@ -632,7 +632,7 @@ vfp_testgzip_bytes(struct worker *w, struct http_conn *htc, ssize_t bytes)
 	struct vgz *vg;
 	ssize_t l, wl;
 	int i = -100;
-	uint8_t	obuf[params->gzip_stack_buffer];
+	uint8_t	obuf[cache_param->gzip_stack_buffer];
 	size_t dl;
 	const void *dp;
 	struct storage *st;
diff --git a/bin/varnishd/cache_hash.c b/bin/varnishd/cache_hash.c
index ecce40d..669fd40 100644
--- a/bin/varnishd/cache_hash.c
+++ b/bin/varnishd/cache_hash.c
@@ -169,7 +169,7 @@ HSH_AddString(const struct sess *sp, const char *str)
 	SHA256_Update(sp->wrk->sha256ctx, str, l);
 	SHA256_Update(sp->wrk->sha256ctx, "#", 1);
 
-	if (params->log_hash)
+	if (cache_param->log_hash)
 		WSP(sp, SLT_Hash, "%s", str);
 }
 
@@ -267,7 +267,7 @@ HSH_Insert(const struct sess *sp)
 	w = sp->wrk;
 
 	HSH_Prealloc(sp);
-	if (params->diag_bitmap & 0x80000000)
+	if (cache_param->diag_bitmap & 0x80000000)
 		hsh_testmagic(sp->wrk->nobjhead->digest);
 
 	AZ(sp->hash_objhead);
@@ -316,7 +316,7 @@ HSH_Lookup(struct sess *sp, struct objhead **poh)
 
 	HSH_Prealloc(sp);
 	memcpy(sp->wrk->nobjhead->digest, sp->digest, sizeof sp->digest);
-	if (params->diag_bitmap & 0x80000000)
+	if (cache_param->diag_bitmap & 0x80000000)
 		hsh_testmagic(sp->wrk->nobjhead->digest);
 
 	if (sp->hash_objhead != NULL) {
@@ -436,7 +436,7 @@ HSH_Lookup(struct sess *sp, struct objhead **poh)
 			}
 			VTAILQ_INSERT_TAIL(&oh->waitinglist->list, sp, list);
 		}
-		if (params->diag_bitmap & 0x20)
+		if (cache_param->diag_bitmap & 0x20)
 			WSP(sp, SLT_Debug,
 				"on waiting list <%p>", oh);
 		SES_Charge(sp);
@@ -492,7 +492,7 @@ hsh_rush(struct objhead *oh)
 	Lck_AssertHeld(&oh->mtx);
 	wl = oh->waitinglist;
 	CHECK_OBJ_NOTNULL(wl, WAITINGLIST_MAGIC);
-	for (u = 0; u < params->rush_exponent; u++) {
+	for (u = 0; u < cache_param->rush_exponent; u++) {
 		sp = VTAILQ_FIRST(&wl->list);
 		if (sp == NULL)
 			break;
@@ -616,7 +616,7 @@ HSH_Unbusy(const struct sess *sp)
 	assert(oh->refcnt > 0);
 	if (o->ws_o->overflow)
 		sp->wrk->stats.n_objoverflow++;
-	if (params->diag_bitmap & 0x40)
+	if (cache_param->diag_bitmap & 0x40)
 		WSP(sp, SLT_Debug,
 		    "Object %u workspace free %u", o->xid, WS_Free(o->ws_o));
 
diff --git a/bin/varnishd/cache_lck.c b/bin/varnishd/cache_lck.c
index ee93b9f..2aef6dc 100644
--- a/bin/varnishd/cache_lck.c
+++ b/bin/varnishd/cache_lck.c
@@ -64,7 +64,7 @@ Lck__Lock(struct lock *lck, const char *p, const char *f, int l)
 	int r;
 
 	CAST_OBJ_NOTNULL(ilck, lck->priv, ILCK_MAGIC);
-	if (!(params->diag_bitmap & 0x18)) {
+	if (!(cache_param->diag_bitmap & 0x18)) {
 		AZ(pthread_mutex_lock(&ilck->mtx));
 		AZ(ilck->held);
 		ilck->stat->locks++;
@@ -76,11 +76,11 @@ Lck__Lock(struct lock *lck, const char *p, const char *f, int l)
 	assert(r == 0 || r == EBUSY);
 	if (r) {
 		ilck->stat->colls++;
-		if (params->diag_bitmap & 0x8)
+		if (cache_param->diag_bitmap & 0x8)
 			VSL(SLT_Debug, 0, "MTX_CONTEST(%s,%s,%d,%s)",
 			    p, f, l, ilck->w);
 		AZ(pthread_mutex_lock(&ilck->mtx));
-	} else if (params->diag_bitmap & 0x8) {
+	} else if (cache_param->diag_bitmap & 0x8) {
 		VSL(SLT_Debug, 0, "MTX_LOCK(%s,%s,%d,%s)", p, f, l, ilck->w);
 	}
 	AZ(ilck->held);
@@ -99,7 +99,7 @@ Lck__Unlock(struct lock *lck, const char *p, const char *f, int l)
 	AN(ilck->held);
 	ilck->held = 0;
 	AZ(pthread_mutex_unlock(&ilck->mtx));
-	if (params->diag_bitmap & 0x8)
+	if (cache_param->diag_bitmap & 0x8)
 		VSL(SLT_Debug, 0, "MTX_UNLOCK(%s,%s,%d,%s)", p, f, l, ilck->w);
 }
 
@@ -112,7 +112,7 @@ Lck__Trylock(struct lock *lck, const char *p, const char *f, int l)
 	CAST_OBJ_NOTNULL(ilck, lck->priv, ILCK_MAGIC);
 	r = pthread_mutex_trylock(&ilck->mtx);
 	assert(r == 0 || r == EBUSY);
-	if (params->diag_bitmap & 0x8)
+	if (cache_param->diag_bitmap & 0x8)
 		VSL(SLT_Debug, 0,
 		    "MTX_TRYLOCK(%s,%s,%d,%s) = %d", p, f, l, ilck->w);
 	if (r == 0) {
diff --git a/bin/varnishd/cache_main.c b/bin/varnishd/cache_main.c
index c610bc3..5dc2d00 100644
--- a/bin/varnishd/cache_main.c
+++ b/bin/varnishd/cache_main.c
@@ -37,7 +37,7 @@
 #include "waiter/cache_waiter.h"
 #include "hash/hash_slinger.h"
 
-volatile struct params	*params;
+volatile struct params	*cache_param;
 
 /*--------------------------------------------------------------------
  * Per thread storage for the session currently being processed by
@@ -134,7 +134,7 @@ child_main(void)
 	BAN_Compile();
 
 	/* Wait for persistent storage to load if asked to */
-	if (params->diag_bitmap & 0x00020000)
+	if (cache_param->diag_bitmap & 0x00020000)
 		SMP_Ready();
 
 	CLI_Run();
diff --git a/bin/varnishd/cache_panic.c b/bin/varnishd/cache_panic.c
index 4076d26..f69e90b 100644
--- a/bin/varnishd/cache_panic.c
+++ b/bin/varnishd/cache_panic.c
@@ -346,7 +346,7 @@ pan_ic(const char *func, const char *file, int line, const char *cond,
 
 	pan_backtrace();
 
-	if (!(params->diag_bitmap & 0x2000)) {
+	if (!(cache_param->diag_bitmap & 0x2000)) {
 		sp = THR_GetSession();
 		if (sp != NULL)
 			pan_sess(sp);
@@ -354,11 +354,11 @@ pan_ic(const char *func, const char *file, int line, const char *cond,
 	VSB_printf(vsp, "\n");
 	VSB_bcat(vsp, "", 1);	/* NUL termination */
 
-	if (params->diag_bitmap & 0x4000)
+	if (cache_param->diag_bitmap & 0x4000)
 		(void)fputs(VSM_head->panicstr, stderr);
 
 #ifdef HAVE_ABORT2
-	if (params->diag_bitmap & 0x8000) {
+	if (cache_param->diag_bitmap & 0x8000) {
 		void *arg[1];
 		char *p;
 
@@ -369,7 +369,7 @@ pan_ic(const char *func, const char *file, int line, const char *cond,
 		abort2(VSM_head->panicstr, 1, arg);
 	}
 #endif
-	if (params->diag_bitmap & 0x1000)
+	if (cache_param->diag_bitmap & 0x1000)
 		exit(4);
 	else
 		abort();
diff --git a/bin/varnishd/cache_pipe.c b/bin/varnishd/cache_pipe.c
index be17ac9..4180d39 100644
--- a/bin/varnishd/cache_pipe.c
+++ b/bin/varnishd/cache_pipe.c
@@ -108,7 +108,7 @@ PipeSession(struct sess *sp)
 	while (fds[0].fd > -1 || fds[1].fd > -1) {
 		fds[0].revents = 0;
 		fds[1].revents = 0;
-		i = poll(fds, 2, params->pipe_timeout * 1000);
+		i = poll(fds, 2, cache_param->pipe_timeout * 1000);
 		if (i < 1)
 			break;
 		if (fds[0].revents && rdf(vc->fd, sp->fd)) {
diff --git a/bin/varnishd/cache_pool.c b/bin/varnishd/cache_pool.c
index aed596e..62d58aa 100644
--- a/bin/varnishd/cache_pool.c
+++ b/bin/varnishd/cache_pool.c
@@ -272,7 +272,7 @@ Pool_Work_Thread(void *priv, struct worker *w)
 			AZ(w->wrw.wfd);
 			AZ(w->storage_hint);
 			assert(w->wlp == w->wlb);
-			if (params->diag_bitmap & 0x00040000) {
+			if (cache_param->diag_bitmap & 0x00040000) {
 				if (w->vcl != NULL)
 					VCL_Rel(&w->vcl);
 			}
@@ -308,7 +308,7 @@ pool_queue(struct pool *pp, struct sess *sp)
 	}
 
 	/* If we have too much in the queue already, refuse. */
-	if (pp->lqueue > (params->queue_max * pp->nthr) / 100) {
+	if (pp->lqueue > (cache_param->queue_max * pp->nthr) / 100) {
 		pp->ndropped++;
 		Lck_Unlock(&pp->mtx);
 		return (-1);
@@ -385,10 +385,10 @@ pool_breed(struct pool *qp, const pthread_attr_t *tp_attr)
 	 * If we need more threads, and have space, create
 	 * one more thread.
 	 */
-	if (qp->nthr < params->wthread_min ||	/* Not enough threads yet */
-	    (qp->lqueue > params->wthread_add_threshold && /* more needed */
+	if (qp->nthr < cache_param->wthread_min ||	/* Not enough threads yet */
+	    (qp->lqueue > cache_param->wthread_add_threshold && /* more needed */
 	    qp->lqueue > qp->last_lqueue)) {	/* not getting better since last */
-		if (qp->nthr > params->wthread_max) {
+		if (qp->nthr > cache_param->wthread_max) {
 			Lck_Lock(&pool_mtx);
 			VSC_C_main->threads_limited++;
 			Lck_Unlock(&pool_mtx);
@@ -398,10 +398,10 @@ pool_breed(struct pool *qp, const pthread_attr_t *tp_attr)
 			Lck_Lock(&pool_mtx);
 			VSC_C_main->threads_limited++;
 			Lck_Unlock(&pool_mtx);
-			VTIM_sleep(params->wthread_fail_delay * 1e-3);
+			VTIM_sleep(cache_param->wthread_fail_delay * 1e-3);
 		} else {
 			AZ(pthread_detach(tp));
-			VTIM_sleep(params->wthread_add_delay * 1e-3);
+			VTIM_sleep(cache_param->wthread_add_delay * 1e-3);
 			qp->nthr++;
 			Lck_Lock(&pool_mtx);
 			VSC_C_main->threads++;
@@ -442,9 +442,9 @@ pool_herder(void *priv)
 
 	while (1) {
 		/* Set the stacksize for worker threads we create */
-		if (params->wthread_stacksize != UINT_MAX)
+		if (cache_param->wthread_stacksize != UINT_MAX)
 			AZ(pthread_attr_setstacksize(&tp_attr,
-			    params->wthread_stacksize));
+			    cache_param->wthread_stacksize));
 		else {
 			AZ(pthread_attr_destroy(&tp_attr));
 			AZ(pthread_attr_init(&tp_attr));
@@ -452,13 +452,13 @@ pool_herder(void *priv)
 
 		pool_breed(pp, &tp_attr);
 
-		if (pp->nthr < params->wthread_min)
+		if (pp->nthr < cache_param->wthread_min)
 			continue;
 
 		AZ(clock_gettime(CLOCK_MONOTONIC, &ts));
-		ts.tv_sec += params->wthread_purge_delay / 1000;
+		ts.tv_sec += cache_param->wthread_purge_delay / 1000;
 		ts.tv_nsec +=
-		    (params->wthread_purge_delay % 1000) * 1000000;
+		    (cache_param->wthread_purge_delay % 1000) * 1000000;
 		if (ts.tv_nsec >= 1000000000) {
 			ts.tv_sec++;
 			ts.tv_nsec -= 1000000000;
@@ -470,10 +470,10 @@ pool_herder(void *priv)
 		if (!i)
 			continue;
 
-		if (pp->nthr <= params->wthread_min)
+		if (pp->nthr <= cache_param->wthread_min)
 			continue;
 
-		t_idle = VTIM_real() - params->wthread_timeout;
+		t_idle = VTIM_real() - cache_param->wthread_timeout;
 
 		Lck_Lock(&pp->mtx);
 		VSC_C_main->sess_queued += pp->nqueued;
@@ -481,7 +481,7 @@ pool_herder(void *priv)
 		pp->nqueued = pp->ndropped = 0;
 		w = VTAILQ_LAST(&pp->idle, workerhead);
 		if (w != NULL &&
-		    (w->lastused < t_idle || pp->nthr > params->wthread_max)) {
+		    (w->lastused < t_idle || pp->nthr > cache_param->wthread_max)) {
 			VTAILQ_REMOVE(&pp->idle, w, list);
 		} else
 			w = NULL;
@@ -560,7 +560,7 @@ pool_poolherder(void *priv)
 
 	nwq = 0;
 	while (1) {
-		if (nwq < params->wthread_pools) {
+		if (nwq < cache_param->wthread_pools) {
 			pp = pool_mkpool();
 			if (pp != NULL) {
 				VTAILQ_INSERT_TAIL(&pools, pp, list);
diff --git a/bin/varnishd/cache_response.c b/bin/varnishd/cache_response.c
index 87b0919..487a514 100644
--- a/bin/varnishd/cache_response.c
+++ b/bin/varnishd/cache_response.c
@@ -118,7 +118,7 @@ RES_BuildHttp(const struct sess *sp)
 
 	if (!(sp->wrk->res_mode & RES_LEN)) {
 		http_Unset(sp->wrk->resp, H_Content_Length);
-	} else if (params->http_range_support) {
+	} else if (cache_param->http_range_support) {
 		/* We only accept ranges if we know the length */
 		http_SetHeader(sp->wrk, sp->vsl_id, sp->wrk->resp,
 		    "Accept-Ranges: bytes");
@@ -156,7 +156,7 @@ res_WriteGunzipObj(const struct sess *sp)
 	struct storage *st;
 	unsigned u = 0;
 	struct vgz *vg;
-	char obuf[params->gzip_stack_buffer];
+	char obuf[cache_param->gzip_stack_buffer];
 	ssize_t obufl = 0;
 	int i;
 
@@ -230,7 +230,7 @@ res_WriteDirObj(const struct sess *sp, ssize_t low, ssize_t high)
 		 * XXX: Should use getpagesize() ?
 		 */
 		if (st->fd >= 0 &&
-		    st->len >= params->sendfile_threshold) {
+		    st->len >= cache_param->sendfile_threshold) {
 			VSC_C_main->n_objsendfile++;
 			WRW_Sendfile(sp->wrk, st->fd, st->where + off, len);
 			continue;
@@ -275,7 +275,7 @@ RES_WriteObj(struct sess *sp)
 	    sp->wantbody &&
 	    (sp->wrk->res_mode & RES_LEN) &&
 	    !(sp->wrk->res_mode & (RES_ESI|RES_ESI_CHILD|RES_GUNZIP)) &&
-	    params->http_range_support &&
+	    cache_param->http_range_support &&
 	    sp->obj->response == 200 &&
 	    http_GetHdr(sp->http, H_Range, &r))
 		res_dorange(sp, r, &low, &high);
diff --git a/bin/varnishd/cache_session.c b/bin/varnishd/cache_session.c
index 5db418c..7befbcc 100644
--- a/bin/varnishd/cache_session.c
+++ b/bin/varnishd/cache_session.c
@@ -107,8 +107,8 @@ ses_sm_alloc(void)
 	 * cache them locally, to make sure we get a consistent
 	 * view of the value.
 	 */
-	nws = params->sess_workspace;
-	nhttp = (uint16_t)params->http_max_hdr;
+	nws = cache_param->sess_workspace;
+	nhttp = (uint16_t)cache_param->http_max_hdr;
 
 	hl = HTTP_estimate(nhttp);
 	l = sizeof *sm + nws + 2 * hl;
@@ -192,7 +192,7 @@ SES_New(struct worker *wrk, struct sesspool *pp)
 	sm = VTAILQ_FIRST(&pp->freelist);
 	if (sm != NULL) {
 		VTAILQ_REMOVE(&pp->freelist, sm, list);
-	} else if (pp->nsess < params->max_sess) {
+	} else if (pp->nsess < cache_param->max_sess) {
 		pp->nsess++;
 		do_alloc = 1;
 	}
@@ -355,9 +355,9 @@ SES_Delete(struct sess *sp, const char *reason)
 	    b->sess, b->req, b->pipe, b->pass,
 	    b->fetch, b->hdrbytes, b->bodybytes);
 
-	if (sm->workspace != params->sess_workspace ||
-	    sm->nhttp != (uint16_t)params->http_max_hdr ||
-	    pp->nsess > params->max_sess) {
+	if (sm->workspace != cache_param->sess_workspace ||
+	    sm->nhttp != (uint16_t)cache_param->http_max_hdr ||
+	    pp->nsess > cache_param->max_sess) {
 		free(sm);
 		Lck_Lock(&pp->mtx);
 		if (wrk != NULL)
diff --git a/bin/varnishd/cache_shmlog.c b/bin/varnishd/cache_shmlog.c
index 5601463..1252fa3 100644
--- a/bin/varnishd/cache_shmlog.c
+++ b/bin/varnishd/cache_shmlog.c
@@ -135,7 +135,7 @@ VSLR(enum VSL_tag_e tag, int id, const char *b, unsigned len)
 	uint32_t *p;
 	unsigned mlen;
 
-	mlen = params->shm_reclen;
+	mlen = cache_param->shm_reclen;
 
 	/* Truncate */
 	if (len > mlen)
@@ -153,7 +153,7 @@ void
 VSL(enum VSL_tag_e tag, int id, const char *fmt, ...)
 {
 	va_list ap;
-	unsigned n, mlen = params->shm_reclen;
+	unsigned n, mlen = cache_param->shm_reclen;
 	char buf[mlen];
 
 	/*
@@ -205,7 +205,7 @@ WSLR(struct worker *w, enum VSL_tag_e tag, int id, txt t)
 	unsigned l, mlen;
 
 	Tcheck(t);
-	mlen = params->shm_reclen;
+	mlen = cache_param->shm_reclen;
 
 	/* Truncate */
 	l = Tlen(t);
@@ -225,7 +225,7 @@ WSLR(struct worker *w, enum VSL_tag_e tag, int id, txt t)
 	w->wlp = VSL_END(w->wlp, l);
 	assert(w->wlp < w->wle);
 	w->wlr++;
-	if (params->diag_bitmap & 0x10000)
+	if (cache_param->diag_bitmap & 0x10000)
 		WSL_Flush(w, 0);
 }
 
@@ -239,7 +239,7 @@ wsl(struct worker *w, enum VSL_tag_e tag, int id, const char *fmt, va_list ap)
 	txt t;
 
 	AN(fmt);
-	mlen = params->shm_reclen;
+	mlen = cache_param->shm_reclen;
 
 	if (strchr(fmt, '%') == NULL) {
 		t.b = TRUST_ME(fmt);
@@ -261,7 +261,7 @@ wsl(struct worker *w, enum VSL_tag_e tag, int id, const char *fmt, va_list ap)
 		assert(w->wlp < w->wle);
 		w->wlr++;
 	}
-	if (params->diag_bitmap & 0x10000)
+	if (cache_param->diag_bitmap & 0x10000)
 		WSL_Flush(w, 0);
 }
 
diff --git a/bin/varnishd/cache_vary.c b/bin/varnishd/cache_vary.c
index c528fb8..026f937 100644
--- a/bin/varnishd/cache_vary.c
+++ b/bin/varnishd/cache_vary.c
@@ -155,7 +155,7 @@ vry_cmp(const uint8_t * const *v1, uint8_t * const *v2)
 	} else if (memcmp((*v1) + 2, (*v2) + 2, (*v1)[2] + 2)) {
 		/* Different header */
 		retval = 1;
-	} else if (params->http_gzip_support &&
+	} else if (cache_param->http_gzip_support &&
 	    !strcasecmp(H_Accept_Encoding, (const char*)((*v1)+2))) {
 		/*
 		 * If we do gzip processing, we do not vary on Accept-Encoding,
diff --git a/bin/varnishd/cache_vrt.c b/bin/varnishd/cache_vrt.c
index 16258ec..3851de7 100644
--- a/bin/varnishd/cache_vrt.c
+++ b/bin/varnishd/cache_vrt.c
@@ -73,7 +73,7 @@ VRT_count(const struct sess *sp, unsigned u)
 	if (sp == NULL)
 		return;
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
-	if (params->vcl_trace)
+	if (cache_param->vcl_trace)
 		WSP(sp, SLT_VCL_trace, "%u %d.%d", u,
 		    sp->vcl->ref[u].line, sp->vcl->ref[u].pos);
 }
diff --git a/bin/varnishd/cache_vrt_re.c b/bin/varnishd/cache_vrt_re.c
index d6dc5d5..7759d0a 100644
--- a/bin/varnishd/cache_vrt_re.c
+++ b/bin/varnishd/cache_vrt_re.c
@@ -71,7 +71,7 @@ VRT_re_match(const struct sess *sp, const char *s, void *re)
 		s = "";
 	AN(re);
 	t = re;
-	i = VRE_exec(t, s, strlen(s), 0, 0, NULL, 0, &params->vre_limits);
+	i = VRE_exec(t, s, strlen(s), 0, 0, NULL, 0, &cache_param->vre_limits);
 	if (i >= 0)
 		return (1);
 	if (i < VRE_ERROR_NOMATCH )
@@ -100,7 +100,7 @@ VRT_regsub(const struct sess *sp, int all, const char *str, void *re,
 	memset(ovector, 0, sizeof(ovector));
 	len = strlen(str);
 	i = VRE_exec(t, str, len, 0, options, ovector, 30,
-	    &params->vre_limits);
+	    &cache_param->vre_limits);
 
 	/* If it didn't match, we can return the original string */
 	if (i == VRE_ERROR_NOMATCH)
@@ -141,7 +141,7 @@ VRT_regsub(const struct sess *sp, int all, const char *str, void *re,
 		memset(&ovector, 0, sizeof(ovector));
 		options |= VRE_NOTEMPTY_ATSTART;
 		i = VRE_exec(t, str, len, 0, options, ovector, 30,
-		    &params->vre_limits);
+		    &cache_param->vre_limits);
 		if (i < VRE_ERROR_NOMATCH ) {
 			WS_Release(sp->http->ws, 0);
 			WSP(sp, SLT_VCL_Error,
diff --git a/bin/varnishd/cache_wrk.c b/bin/varnishd/cache_wrk.c
index e416eb9..bfee84c 100644
--- a/bin/varnishd/cache_wrk.c
+++ b/bin/varnishd/cache_wrk.c
@@ -185,15 +185,15 @@ WRK_thread(void *priv)
 	uint16_t nhttp;
 	unsigned siov;
 
-	assert(params->http_max_hdr <= 65535);
+	assert(cache_param->http_max_hdr <= 65535);
 	/* We need to snapshot these two for consistency */
-	nhttp = (uint16_t)params->http_max_hdr;
+	nhttp = (uint16_t)cache_param->http_max_hdr;
 	siov = nhttp * 2;
 	if (siov > IOV_MAX)
 		siov = IOV_MAX;
 	return (wrk_thread_real(priv,
-	    params->shm_workspace,
-	    params->wthread_workspace,
+	    cache_param->shm_workspace,
+	    cache_param->wthread_workspace,
 	    nhttp, HTTP_estimate(nhttp), siov));
 }
 
diff --git a/bin/varnishd/cache_wrw.c b/bin/varnishd/cache_wrw.c
index 3e2de4a..2160f69 100644
--- a/bin/varnishd/cache_wrw.c
+++ b/bin/varnishd/cache_wrw.c
@@ -134,7 +134,7 @@ WRW_Flush(struct worker *w)
 			*/
 			size_t used = 0;
 
-			if (VTIM_real() - w->sp->t_resp > params->send_timeout) {
+			if (VTIM_real() - w->sp->t_resp > cache_param->send_timeout) {
 				WSL(w, SLT_Debug, *wrw->wfd,
 				    "Hit total send timeout, wrote = %ld/%ld; not retrying",
 				    i, wrw->liov);
@@ -309,7 +309,7 @@ WRW_Sendfile(struct worker *w, int fd, off_t off, unsigned len)
 	} while (0);
 #elif defined(__sun) && defined(HAVE_SENDFILEV)
 	do {
-		sendfilevec_t svvec[params->http_headers * 2 + 1];
+		sendfilevec_t svvec[cache_param->http_headers * 2 + 1];
 		size_t xferred = 0, expected = 0;
 		int i;
 		for (i = 0; i < wrw->niov; i++) {
diff --git a/bin/varnishd/hash/hash_critbit.c b/bin/varnishd/hash/hash_critbit.c
index 56cc5c0..b1c9cf3 100644
--- a/bin/varnishd/hash/hash_critbit.c
+++ b/bin/varnishd/hash/hash_critbit.c
@@ -369,7 +369,7 @@ hcb_cleaner(void *priv)
 		VTAILQ_CONCAT(&dead_h, &cool_h, hoh_list);
 		Lck_Unlock(&hcb_mtx);
 		WRK_SumStat(&ww);
-		VTIM_sleep(params->critbit_cooloff);
+		VTIM_sleep(cache_param->critbit_cooloff);
 	}
 	NEEDLESS_RETURN(NULL);
 }
diff --git a/bin/varnishd/heritage.h b/bin/varnishd/heritage.h
index 3732f18..39fc03e 100644
--- a/bin/varnishd/heritage.h
+++ b/bin/varnishd/heritage.h
@@ -222,7 +222,7 @@ struct params {
  * We declare this a volatile pointer, so that reads of parameters
  * become atomic, leaving the CLI thread lattitude to change the values
  */
-extern volatile struct params * params;
+extern volatile struct params * cache_param;
 extern struct heritage heritage;
 
 void child_main(void);
diff --git a/bin/varnishd/mgt/mgt_child.c b/bin/varnishd/mgt/mgt_child.c
index 9c6d37d..9503e5b 100644
--- a/bin/varnishd/mgt/mgt_child.c
+++ b/bin/varnishd/mgt/mgt_child.c
@@ -315,7 +315,7 @@ start_child(struct cli *cli)
 	heritage.std_fd = cp[1];
 	child_output = cp[0];
 
-	AN(params);
+	AN(cache_param);
 	if ((pid = fork()) < 0) {
 		perror("Could not fork child");
 		exit(1);
diff --git a/bin/varnishd/mgt/mgt_param.c b/bin/varnishd/mgt/mgt_param.c
index 20456d7..dc6b708 100644
--- a/bin/varnishd/mgt/mgt_param.c
+++ b/bin/varnishd/mgt/mgt_param.c
@@ -1074,8 +1074,8 @@ MCF_ParamSet(struct cli *cli, const char *param, const char *val)
 	}
 	pp->func(cli, pp, val);
 
-	if (cli->result == CLIS_OK && params != NULL)
-		*params = mgt_param;
+	if (cli->result == CLIS_OK && cache_param != NULL)
+		*cache_param = mgt_param;
 
 	if (cli->result != CLIS_OK) {
 		VCLI_Out(cli, "(attempting to set param %s to %s)\n",
diff --git a/bin/varnishd/mgt/mgt_shmem.c b/bin/varnishd/mgt/mgt_shmem.c
index 37083df..3953bdb 100644
--- a/bin/varnishd/mgt/mgt_shmem.c
+++ b/bin/varnishd/mgt/mgt_shmem.c
@@ -321,9 +321,9 @@ mgt_SHM_Init(const char *l_arg)
 	AN(VSC_C_main);
 
 	/* XXX: We need to zero params if we dealloc/clean/wash */
-	params = VSM_Alloc(sizeof *params, VSM_CLASS_PARAM, "", "");
-	AN(params);
-	*params = mgt_param;
+	cache_param = VSM_Alloc(sizeof *cache_param, VSM_CLASS_PARAM, "", "");
+	AN(cache_param);
+	*cache_param = mgt_param;
 
 	vsl_log_start = VSM_Alloc(s1, VSL_CLASS, "", "");
 	AN(vsl_log_start);
diff --git a/bin/varnishd/rfc2616.c b/bin/varnishd/rfc2616.c
index d4d5a05..4041f45 100644
--- a/bin/varnishd/rfc2616.c
+++ b/bin/varnishd/rfc2616.c
@@ -74,7 +74,7 @@ RFC2616_Ttl(const struct sess *sp)
 
 	assert(sp->wrk->exp.entered != 0.0 && !isnan(sp->wrk->exp.entered));
 	/* If all else fails, cache using default ttl */
-	sp->wrk->exp.ttl = params->default_ttl;
+	sp->wrk->exp.ttl = cache_param->default_ttl;
 
 	max_age = age = 0;
 	h_expires = 0;
@@ -140,7 +140,7 @@ RFC2616_Ttl(const struct sess *sp)
 		}
 
 		if (h_date == 0 ||
-		    fabs(h_date - sp->wrk->exp.entered) < params->clock_skew) {
+		    fabs(h_date - sp->wrk->exp.entered) < cache_param->clock_skew) {
 			/*
 			 * If we have no Date: header or if it is
 			 * sufficiently close to our clock we will
diff --git a/bin/varnishd/storage/stevedore.c b/bin/varnishd/storage/stevedore.c
index 4048836..b119c06 100644
--- a/bin/varnishd/storage/stevedore.c
+++ b/bin/varnishd/storage/stevedore.c
@@ -169,8 +169,8 @@ stv_alloc(struct worker *w, const struct object *obj, size_t size)
 	stv = obj->objstore->stevedore;
 	CHECK_OBJ_NOTNULL(stv, STEVEDORE_MAGIC);
 
-	if (size > (size_t)(params->fetch_maxchunksize) << 10)
-		size = (size_t)(params->fetch_maxchunksize) << 10;
+	if (size > (size_t)(cache_param->fetch_maxchunksize) << 10)
+		size = (size_t)(cache_param->fetch_maxchunksize) << 10;
 
 	for (;;) {
 		/* try to allocate from it */
@@ -179,7 +179,7 @@ stv_alloc(struct worker *w, const struct object *obj, size_t size)
 		if (st != NULL)
 			break;
 
-		if (size > params->fetch_chunksize * 1024LL) {
+		if (size > cache_param->fetch_chunksize * 1024LL) {
 			size >>= 1;
 			continue;
 		}
@@ -189,7 +189,7 @@ stv_alloc(struct worker *w, const struct object *obj, size_t size)
 			break;
 
 		/* Enough is enough: try another if we have one */
-		if (++fail >= params->nuke_limit)
+		if (++fail >= cache_param->nuke_limit)
 			break;
 	}
 	if (st != NULL)
@@ -336,7 +336,7 @@ STV_NewObject(struct sess *sp, const char *hint, unsigned wsl, struct exp *ep,
 	}
 	if (o == NULL) {
 		/* no luck; try to free some space and keep trying */
-		for (i = 0; o == NULL && i < params->nuke_limit; i++) {
+		for (i = 0; o == NULL && i < cache_param->nuke_limit; i++) {
 			if (EXP_NukeOne(sp->wrk, stv->lru) == -1)
 				break;
 			o = stv->allocobj(stv, sp, ltot, &soc);
diff --git a/bin/varnishd/waiter/cache_waiter_epoll.c b/bin/varnishd/waiter/cache_waiter_epoll.c
index e700676..b5068a6 100644
--- a/bin/varnishd/waiter/cache_waiter_epoll.c
+++ b/bin/varnishd/waiter/cache_waiter_epoll.c
@@ -188,7 +188,7 @@ vwe_thread(void *priv)
 			continue;
 
 		/* check for timeouts */
-		deadline = VTIM_real() - params->sess_timeout;
+		deadline = VTIM_real() - cache_param->sess_timeout;
 		for (;;) {
 			sp = VTAILQ_FIRST(&vwe->sesshead);
 			if (sp == NULL)
diff --git a/bin/varnishd/waiter/cache_waiter_kqueue.c b/bin/varnishd/waiter/cache_waiter_kqueue.c
index b300fae..a631606 100644
--- a/bin/varnishd/waiter/cache_waiter_kqueue.c
+++ b/bin/varnishd/waiter/cache_waiter_kqueue.c
@@ -186,7 +186,7 @@ vwk_thread(void *priv)
 		 * would not know we meant "the old fd of this number".
 		 */
 		vwk_kq_flush(vwk);
-		deadline = VTIM_real() - params->sess_timeout;
+		deadline = VTIM_real() - cache_param->sess_timeout;
 		for (;;) {
 			sp = VTAILQ_FIRST(&vwk->sesshead);
 			if (sp == NULL)
diff --git a/bin/varnishd/waiter/cache_waiter_poll.c b/bin/varnishd/waiter/cache_waiter_poll.c
index 2617365..5f8dbd7 100644
--- a/bin/varnishd/waiter/cache_waiter_poll.c
+++ b/bin/varnishd/waiter/cache_waiter_poll.c
@@ -140,7 +140,7 @@ vwp_main(void *priv)
 		assert(vwp->pollfd[vwp->pipes[1]].fd == -1);
 		v = poll(vwp->pollfd, vwp->hpoll + 1, 100);
 		assert(v >= 0);
-		deadline = VTIM_real() - params->sess_timeout;
+		deadline = VTIM_real() - cache_param->sess_timeout;
 		v2 = v;
 		VTAILQ_FOREACH_SAFE(sp, &vwp->sesshead, list, sp2) {
 			if (v != 0 && v2 == 0)
diff --git a/bin/varnishd/waiter/cache_waiter_ports.c b/bin/varnishd/waiter/cache_waiter_ports.c
index 022131b..aa9f86d 100644
--- a/bin/varnishd/waiter/cache_waiter_ports.c
+++ b/bin/varnishd/waiter/cache_waiter_ports.c
@@ -197,7 +197,7 @@ vws_thread(void *priv)
 
 		/* check for timeouts */
 		now = VTIM_real();
-		deadline = now - params->sess_timeout;
+		deadline = now - cache_param->sess_timeout;
 
 		/*
 		 * This loop assumes that the oldest sessions are always at the
@@ -225,7 +225,7 @@ vws_thread(void *priv)
 		 */
 
 		if (sp) {
-			double tmo = (sp->t_open + params->sess_timeout) - now;
+			double tmo = (sp->t_open + cache_param->sess_timeout) - now;
 
 			/* we should have removed all sps whose timeout has passed */
 			assert(tmo > 0.0);



From geoff at varnish-cache.org  Mon Jan  9 20:51:47 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:51:47 +0100
Subject: [experimental-ims] 76a050f Add .expected_response to probe
	documentation. Fix error message for .expected_response.
Message-ID: <E1RkMC3-0006ms-9w@project.varnish-software.com>

commit 76a050fd10b6ab423be673cce5ee12fda049356f
Author: Andreas Plesner Jacobsen <apj at marvin.nerd.dk>
Date:   Fri Sep 9 14:54:24 2011 +0200

    Add .expected_response to probe documentation.
    Fix error message for .expected_response.
    
    Fixes #997

diff --git a/doc/sphinx/reference/vcl.rst b/doc/sphinx/reference/vcl.rst
index 7359e6f..0e39187 100644
--- a/doc/sphinx/reference/vcl.rst
+++ b/doc/sphinx/reference/vcl.rst
@@ -284,7 +284,8 @@ req.backend.healthy .window is how many of the latest polls we
 examine, while .threshold is how many of those must have succeeded for
 us to consider the backend healthy.  .initial is how many of the
 probes are considered good when Varnish starts - defaults to the same
-amount as the threshold.
+amount as the threshold. .expected_response is the expected backend
+HTTP response code.
 
 A backend with a probe can be defined like this, together with the
 backend or director:::
@@ -310,6 +311,7 @@ Or it can be defined separately and then referenced:::
      .window = 8;
      .threshold = 3;
      .initial = 3;
+     .expected_response = 200;
   }	
 
   backend www {
diff --git a/lib/libvcl/vcc_backend.c b/lib/libvcl/vcc_backend.c
index 1b8f8a0..f79447f 100644
--- a/lib/libvcl/vcc_backend.c
+++ b/lib/libvcl/vcc_backend.c
@@ -305,8 +305,9 @@ vcc_ParseProbeSpec(struct vcc *tl)
 			status = vcc_UintVal(tl);
 			if (status < 100 || status > 999) {
 				VSB_printf(tl->sb,
-				    "Must specify .status with exactly three "
-				    " digits (100 <= x <= 999)\n");
+				    "Must specify .expected_response with "
+				    "exactly three digits "
+				    "(100 <= x <= 999)\n");
 				vcc_ErrWhere(tl, tl->t);
 				return;
 			}



From geoff at varnish-cache.org  Mon Jan  9 20:52:49 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:49 +0100
Subject: [experimental-ims] 0bcfeee Move busyobj to their own source file,
	they are not really backend related and will have somewhat complex
	semantics.
Message-ID: <E1RkMD3-0000EL-Qw@project.varnish-software.com>

commit 0bcfeee0ae14213f4277b199dadaaab481b7273f
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu Dec 8 06:17:22 2011 +0000

    Move busyobj to their own source file, they are not really
    backend related and will have somewhat complex semantics.

diff --git a/bin/varnishd/Makefile.am b/bin/varnishd/Makefile.am
index 2116701..a4bae7f 100644
--- a/bin/varnishd/Makefile.am
+++ b/bin/varnishd/Makefile.am
@@ -16,6 +16,7 @@ varnishd_SOURCES = \
 	cache/cache_backend_cfg.c \
 	cache/cache_backend_poll.c \
 	cache/cache_ban.c \
+	cache/cache_busyobj.c \
 	cache/cache_center.c \
 	cache/cache_cli.c \
 	cache/cache_dir.c \
diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index 24c1dc9..0ee855b 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -673,12 +673,6 @@ void VDI_RecycleFd(struct worker *wrk, struct vbc **vbp);
 void VDI_AddHostHeader(struct worker *wrk, const struct vbc *vbc);
 void VBE_Poll(void);
 
-/* cache_backend.c */
-void VBE_Init(void);
-struct busyobj *VBE_GetBusyObj(struct worker *wrk);
-struct busyobj *VBE_RefBusyObj(struct busyobj *busyobj);
-void VBE_DerefBusyObj(struct worker *wrk, struct busyobj **busyobj);
-
 /* cache_backend_cfg.c */
 void VBE_InitCfg(void);
 struct backend *VBE_AddBackend(struct cli *cli, const struct vrt_backend *vb);
@@ -703,6 +697,12 @@ struct ban *BAN_RefBan(struct objcore *oc, double t0, const struct ban *tail);
 void BAN_TailDeref(struct ban **ban);
 double BAN_Time(const struct ban *ban);
 
+/* cache_busyobj.c */
+void VBO_Init(void);
+struct busyobj *VBO_GetBusyObj(struct worker *wrk);
+struct busyobj *VBO_RefBusyObj(struct busyobj *busyobj);
+void VBO_DerefBusyObj(struct worker *wrk, struct busyobj **busyobj);
+
 /* cache_center.c [CNT] */
 void CNT_Session(struct sess *sp);
 void CNT_Init(void);
diff --git a/bin/varnishd/cache/cache_backend.c b/bin/varnishd/cache/cache_backend.c
index cf9d60f..4d2f18a 100644
--- a/bin/varnishd/cache/cache_backend.c
+++ b/bin/varnishd/cache/cache_backend.c
@@ -43,105 +43,6 @@
 #include "vrt.h"
 #include "vtcp.h"
 
-static struct lock nbusyobj_mtx;
-static struct busyobj *nbusyobj;
-
-void
-VBE_Init(void)
-{
-	Lck_New(&nbusyobj_mtx, lck_nbusyobj);
-	nbusyobj = NULL;
-}
-
-/*--------------------------------------------------------------------
- * BusyObj handling
- */
-
-static struct busyobj *
-vbe_NewBusyObj(void)
-{
-	struct busyobj *busyobj;
-
-	ALLOC_OBJ(busyobj, BUSYOBJ_MAGIC);
-	AN(busyobj);
-	Lck_New(&busyobj->mtx, lck_busyobj);
-	return (busyobj);
-}
-
-static void
-vbe_FreeBusyObj(struct busyobj *busyobj)
-{
-	CHECK_OBJ_NOTNULL(busyobj, BUSYOBJ_MAGIC);
-	AZ(busyobj->refcount);
-	Lck_Delete(&busyobj->mtx);
-	FREE_OBJ(busyobj);
-}
-
-struct busyobj *
-VBE_GetBusyObj(struct worker *wrk)
-{
-	struct busyobj *busyobj = NULL;
-
-	(void)wrk;
-	Lck_Lock(&nbusyobj_mtx);
-	if (nbusyobj != NULL) {
-		CHECK_OBJ_NOTNULL(nbusyobj, BUSYOBJ_MAGIC);
-		busyobj = nbusyobj;
-		nbusyobj = NULL;
-		memset((char *)busyobj + offsetof(struct busyobj, refcount), 0,
-		       sizeof *busyobj - offsetof(struct busyobj, refcount));
-	}
-	Lck_Unlock(&nbusyobj_mtx);
-	if (busyobj == NULL)
-		busyobj = vbe_NewBusyObj();
-	AN(busyobj);
-	busyobj->refcount = 1;
-	busyobj->beresp = wrk->x_beresp;
-	busyobj->bereq = wrk->x_bereq;
-	return (busyobj);
-}
-
-struct busyobj *
-VBE_RefBusyObj(struct busyobj *busyobj)
-{
-	CHECK_OBJ_NOTNULL(busyobj, BUSYOBJ_MAGIC);
-	Lck_Lock(&busyobj->mtx);
-	assert(busyobj->refcount > 0);
-	busyobj->refcount++;
-	Lck_Unlock(&busyobj->mtx);
-	return (busyobj);
-}
-
-void
-VBE_DerefBusyObj(struct worker *wrk, struct busyobj **pbo)
-{
-	struct busyobj *busyobj;
-
-	(void)wrk;
-	busyobj = *pbo;
-	CHECK_OBJ_NOTNULL(busyobj, BUSYOBJ_MAGIC);
-	Lck_Lock(&busyobj->mtx);
-	assert(busyobj->refcount > 0);
-	busyobj->refcount--;
-	*pbo = NULL;
-	if (busyobj->refcount > 0) {
-		Lck_Unlock(&busyobj->mtx);
-		return;
-	}
-	Lck_Unlock(&busyobj->mtx);
-
-	/* XXX Sanity checks e.g. AZ(busyobj->vbc) */
-
-	Lck_Lock(&nbusyobj_mtx);
-	if (nbusyobj == NULL) {
-		nbusyobj = busyobj;
-		busyobj = NULL;
-	}
-	Lck_Unlock(&nbusyobj_mtx);
-	if (busyobj != NULL)
-		vbe_FreeBusyObj(busyobj);
-}
-
 /*--------------------------------------------------------------------
  * The "simple" director really isn't, since thats where all the actual
  * connections happen.  Nontheless, pretend it is simple by sequestering
diff --git a/bin/varnishd/cache/cache_busyobj.c b/bin/varnishd/cache/cache_busyobj.c
new file mode 100644
index 0000000..32bad7e
--- /dev/null
+++ b/bin/varnishd/cache/cache_busyobj.c
@@ -0,0 +1,138 @@
+/*-
+ * Copyright (c) 2006 Verdens Gang AS
+ * Copyright (c) 2006-2011 Varnish Software AS
+ * All rights reserved.
+ *
+ * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * Handle backend connections and backend request structures.
+ *
+ */
+
+#include "config.h"
+
+#include <stdlib.h>
+#include <stddef.h>
+#include <stdio.h>
+
+#include "cache.h"
+
+static struct lock nbusyobj_mtx;
+static struct busyobj *nbusyobj;
+
+void
+VBO_Init(void)
+{
+	Lck_New(&nbusyobj_mtx, lck_nbusyobj);
+	nbusyobj = NULL;
+}
+
+/*--------------------------------------------------------------------
+ * BusyObj handling
+ */
+
+static struct busyobj *
+vbo_NewBusyObj(void)
+{
+	struct busyobj *busyobj;
+
+	ALLOC_OBJ(busyobj, BUSYOBJ_MAGIC);
+	AN(busyobj);
+	Lck_New(&busyobj->mtx, lck_busyobj);
+	return (busyobj);
+}
+
+static void
+vbe_FreeBusyObj(struct busyobj *busyobj)
+{
+	CHECK_OBJ_NOTNULL(busyobj, BUSYOBJ_MAGIC);
+	AZ(busyobj->refcount);
+	Lck_Delete(&busyobj->mtx);
+	FREE_OBJ(busyobj);
+}
+
+struct busyobj *
+VBO_GetBusyObj(struct worker *wrk)
+{
+	struct busyobj *busyobj = NULL;
+
+	(void)wrk;
+	Lck_Lock(&nbusyobj_mtx);
+	if (nbusyobj != NULL) {
+		CHECK_OBJ_NOTNULL(nbusyobj, BUSYOBJ_MAGIC);
+		busyobj = nbusyobj;
+		nbusyobj = NULL;
+		memset((char *)busyobj + offsetof(struct busyobj, refcount), 0,
+		       sizeof *busyobj - offsetof(struct busyobj, refcount));
+	}
+	Lck_Unlock(&nbusyobj_mtx);
+	if (busyobj == NULL)
+		busyobj = vbo_NewBusyObj();
+	AN(busyobj);
+	busyobj->refcount = 1;
+	busyobj->beresp = wrk->x_beresp;
+	busyobj->bereq = wrk->x_bereq;
+	return (busyobj);
+}
+
+struct busyobj *
+VBO_RefBusyObj(struct busyobj *busyobj)
+{
+	CHECK_OBJ_NOTNULL(busyobj, BUSYOBJ_MAGIC);
+	Lck_Lock(&busyobj->mtx);
+	assert(busyobj->refcount > 0);
+	busyobj->refcount++;
+	Lck_Unlock(&busyobj->mtx);
+	return (busyobj);
+}
+
+void
+VBO_DerefBusyObj(struct worker *wrk, struct busyobj **pbo)
+{
+	struct busyobj *busyobj;
+
+	(void)wrk;
+	busyobj = *pbo;
+	CHECK_OBJ_NOTNULL(busyobj, BUSYOBJ_MAGIC);
+	Lck_Lock(&busyobj->mtx);
+	assert(busyobj->refcount > 0);
+	busyobj->refcount--;
+	*pbo = NULL;
+	if (busyobj->refcount > 0) {
+		Lck_Unlock(&busyobj->mtx);
+		return;
+	}
+	Lck_Unlock(&busyobj->mtx);
+
+	/* XXX Sanity checks e.g. AZ(busyobj->vbc) */
+
+	Lck_Lock(&nbusyobj_mtx);
+	if (nbusyobj == NULL) {
+		nbusyobj = busyobj;
+		busyobj = NULL;
+	}
+	Lck_Unlock(&nbusyobj_mtx);
+	if (busyobj != NULL)
+		vbe_FreeBusyObj(busyobj);
+}
diff --git a/bin/varnishd/cache/cache_center.c b/bin/varnishd/cache/cache_center.c
index 442e2bb..8826ae2 100644
--- a/bin/varnishd/cache/cache_center.c
+++ b/bin/varnishd/cache/cache_center.c
@@ -255,7 +255,7 @@ cnt_prepresp(struct sess *sp)
 			AN(wrk->busyobj->do_stream);
 			VDI_CloseFd(wrk, &wrk->busyobj->vbc);
 			HSH_Drop(wrk);
-			VBE_DerefBusyObj(wrk, &wrk->busyobj);
+			VBO_DerefBusyObj(wrk, &wrk->busyobj);
 		} else {
 			(void)HSH_Deref(wrk, NULL, &wrk->obj);
 		}
@@ -464,7 +464,7 @@ cnt_error(struct sess *sp)
 	if (wrk->obj == NULL) {
 		HSH_Prealloc(sp);
 		AZ(wrk->busyobj);
-		wrk->busyobj = VBE_GetBusyObj(wrk);
+		wrk->busyobj = VBO_GetBusyObj(wrk);
 		wrk->obj = STV_NewObject(wrk, NULL, cache_param->http_resp_size,
 		     (uint16_t)cache_param->http_max_hdr);
 		if (wrk->obj == NULL)
@@ -508,7 +508,7 @@ cnt_error(struct sess *sp)
 	if (sp->handling == VCL_RET_RESTART &&
 	    sp->restarts <  cache_param->max_restarts) {
 		HSH_Drop(wrk);
-		VBE_DerefBusyObj(wrk, &wrk->busyobj);
+		VBO_DerefBusyObj(wrk, &wrk->busyobj);
 		sp->director = NULL;
 		sp->restarts++;
 		sp->step = STP_RECV;
@@ -525,7 +525,7 @@ cnt_error(struct sess *sp)
 	sp->err_code = 0;
 	sp->err_reason = NULL;
 	http_Setup(wrk->busyobj->bereq, NULL);
-	VBE_DerefBusyObj(wrk, &wrk->busyobj);
+	VBO_DerefBusyObj(wrk, &wrk->busyobj);
 	sp->step = STP_PREPRESP;
 	return (0);
 }
@@ -654,7 +654,7 @@ cnt_fetch(struct sess *sp)
 		AZ(HSH_Deref(wrk, wrk->objcore, NULL));
 		wrk->objcore = NULL;
 	}
-	VBE_DerefBusyObj(wrk, &wrk->busyobj);
+	VBO_DerefBusyObj(wrk, &wrk->busyobj);
 	sp->director = NULL;
 	wrk->storage_hint = NULL;
 
@@ -827,7 +827,7 @@ cnt_fetchbody(struct sess *sp)
 		sp->err_code = 503;
 		sp->step = STP_ERROR;
 		VDI_CloseFd(wrk, &wrk->busyobj->vbc);
-		VBE_DerefBusyObj(wrk, &wrk->busyobj);
+		VBO_DerefBusyObj(wrk, &wrk->busyobj);
 		return (0);
 	}
 	CHECK_OBJ_NOTNULL(wrk->obj, OBJECT_MAGIC);
@@ -896,7 +896,7 @@ cnt_fetchbody(struct sess *sp)
 
 	if (i) {
 		HSH_Drop(wrk);
-		VBE_DerefBusyObj(wrk, &wrk->busyobj);
+		VBO_DerefBusyObj(wrk, &wrk->busyobj);
 		AZ(wrk->obj);
 		sp->err_code = 503;
 		sp->step = STP_ERROR;
@@ -909,7 +909,7 @@ cnt_fetchbody(struct sess *sp)
 		AN(wrk->obj->objcore->ban);
 		HSH_Unbusy(wrk);
 	}
-	VBE_DerefBusyObj(wrk, &wrk->busyobj);
+	VBO_DerefBusyObj(wrk, &wrk->busyobj);
 	wrk->acct_tmp.fetch++;
 	sp->step = STP_PREPRESP;
 	return (0);
@@ -983,7 +983,7 @@ cnt_streambody(struct sess *sp)
 	assert(WRW_IsReleased(wrk));
 	assert(wrk->wrw.ciov == wrk->wrw.siov);
 	(void)HSH_Deref(wrk, NULL, &wrk->obj);
-	VBE_DerefBusyObj(wrk, &wrk->busyobj);
+	VBO_DerefBusyObj(wrk, &wrk->busyobj);
 	http_Setup(wrk->resp, NULL);
 	sp->step = STP_DONE;
 	return (0);
@@ -1253,7 +1253,7 @@ cnt_miss(struct sess *sp)
 	AN(wrk->objcore);
 	CHECK_OBJ_NOTNULL(wrk->busyobj, BUSYOBJ_MAGIC);
 	WS_Reset(wrk->ws, NULL);
-	wrk->busyobj = VBE_GetBusyObj(wrk);
+	wrk->busyobj = VBO_GetBusyObj(wrk);
 	http_Setup(wrk->busyobj->bereq, wrk->ws);
 	http_FilterHeader(sp, HTTPH_R_FETCH);
 	http_ForceGet(wrk->busyobj->bereq);
@@ -1278,13 +1278,13 @@ cnt_miss(struct sess *sp)
 		AZ(HSH_Deref(wrk, wrk->objcore, NULL));
 		wrk->objcore = NULL;
 		http_Setup(wrk->busyobj->bereq, NULL);
-		VBE_DerefBusyObj(wrk, &wrk->busyobj);
+		VBO_DerefBusyObj(wrk, &wrk->busyobj);
 		sp->step = STP_ERROR;
 		return (0);
 	case VCL_RET_PASS:
 		AZ(HSH_Deref(wrk, wrk->objcore, NULL));
 		wrk->objcore = NULL;
-		VBE_DerefBusyObj(wrk, &wrk->busyobj);
+		VBO_DerefBusyObj(wrk, &wrk->busyobj);
 		sp->step = STP_PASS;
 		return (0);
 	case VCL_RET_FETCH:
@@ -1294,7 +1294,7 @@ cnt_miss(struct sess *sp)
 	case VCL_RET_RESTART:
 		AZ(HSH_Deref(wrk, wrk->objcore, NULL));
 		wrk->objcore = NULL;
-		VBE_DerefBusyObj(wrk, &wrk->busyobj);
+		VBO_DerefBusyObj(wrk, &wrk->busyobj);
 		INCOMPL();
 	default:
 		WRONG("Illegal action in vcl_miss{}");
@@ -1345,9 +1345,9 @@ cnt_pass(struct sess *sp)
 	AZ(wrk->obj);
 	AZ(wrk->busyobj);
 
-	wrk->busyobj = VBE_GetBusyObj(wrk);
+	wrk->busyobj = VBO_GetBusyObj(wrk);
 	WS_Reset(wrk->ws, NULL);
-	wrk->busyobj = VBE_GetBusyObj(wrk);
+	wrk->busyobj = VBO_GetBusyObj(wrk);
 	http_Setup(wrk->busyobj->bereq, wrk->ws);
 	http_FilterHeader(sp, HTTPH_R_PASS);
 
@@ -1357,7 +1357,7 @@ cnt_pass(struct sess *sp)
 	VCL_pass_method(sp);
 	if (sp->handling == VCL_RET_ERROR) {
 		http_Setup(wrk->busyobj->bereq, NULL);
-		VBE_DerefBusyObj(wrk, &wrk->busyobj);
+		VBO_DerefBusyObj(wrk, &wrk->busyobj);
 		sp->step = STP_ERROR;
 		return (0);
 	}
@@ -1405,9 +1405,9 @@ cnt_pipe(struct sess *sp)
 	AZ(wrk->busyobj);
 
 	wrk->acct_tmp.pipe++;
-	wrk->busyobj = VBE_GetBusyObj(wrk);
+	wrk->busyobj = VBO_GetBusyObj(wrk);
 	WS_Reset(wrk->ws, NULL);
-	wrk->busyobj = VBE_GetBusyObj(wrk);
+	wrk->busyobj = VBO_GetBusyObj(wrk);
 	http_Setup(wrk->busyobj->bereq, wrk->ws);
 	http_FilterHeader(sp, HTTPH_R_PIPE);
 
@@ -1420,7 +1420,7 @@ cnt_pipe(struct sess *sp)
 	PipeSession(sp);
 	assert(WRW_IsReleased(wrk));
 	http_Setup(wrk->busyobj->bereq, NULL);
-	VBE_DerefBusyObj(wrk, &wrk->busyobj);
+	VBO_DerefBusyObj(wrk, &wrk->busyobj);
 	sp->step = STP_DONE;
 	return (0);
 }
diff --git a/bin/varnishd/cache/cache_hash.c b/bin/varnishd/cache/cache_hash.c
index 8a73f6c..675afea 100644
--- a/bin/varnishd/cache/cache_hash.c
+++ b/bin/varnishd/cache/cache_hash.c
@@ -452,7 +452,7 @@ HSH_Lookup(struct sess *sp, struct objhead **poh)
 	oc->refcnt = 1;
 
 	AZ(wrk->busyobj);
-	wrk->busyobj = VBE_GetBusyObj(wrk);
+	wrk->busyobj = VBO_GetBusyObj(wrk);
 
 	VRY_Validate(sp->vary_b);
 	if (sp->vary_l != NULL)
diff --git a/bin/varnishd/cache/cache_main.c b/bin/varnishd/cache/cache_main.c
index fbe7624..a10b0fe 100644
--- a/bin/varnishd/cache/cache_main.c
+++ b/bin/varnishd/cache/cache_main.c
@@ -117,7 +117,7 @@ child_main(void)
 
 	HTTP_Init();
 
-	VBE_Init();
+	VBO_Init();
 	VBE_InitCfg();
 	VBP_Init();
 	WRK_Init();



From geoff at varnish-cache.org  Mon Jan  9 20:52:43 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:43 +0100
Subject: [experimental-ims] 64e06ad Move vep from worker to busyobj
Message-ID: <E1RkMCx-0008To-PY@project.varnish-software.com>

commit 64e06add5bacfb6b4a9ba855617f704e0691fdf6
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Nov 29 17:44:41 2011 +0000

    Move vep from worker to busyobj

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index 21382d7..634d9aa 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -348,7 +348,6 @@ struct worker {
 	struct stream_ctx	*sctx;
 
 	/* ESI stuff */
-	struct vep_state	*vep;
 	int			gzip_resp;
 	ssize_t			l_crc;
 	uint32_t		crc;
@@ -503,6 +502,7 @@ struct busyobj {
 	unsigned		is_gunzip;
 
 	struct vfp		*vfp;
+	struct vep_state	*vep;
 };
 
 /* Object structure --------------------------------------------------*/
diff --git a/bin/varnishd/cache/cache_esi.h b/bin/varnishd/cache/cache_esi.h
index ff84d41..4867614 100644
--- a/bin/varnishd/cache/cache_esi.h
+++ b/bin/varnishd/cache/cache_esi.h
@@ -43,6 +43,6 @@ typedef ssize_t vep_callback_t(struct worker *w, ssize_t l, enum vgz_flag flg);
 
 void VEP_Init(struct worker *w, vep_callback_t *cb);
 void VEP_Parse(const struct worker *w, const char *p, size_t l);
-struct vsb *VEP_Finish(struct worker *w);
+struct vsb *VEP_Finish(const struct worker *w);
 
 
diff --git a/bin/varnishd/cache/cache_esi_fetch.c b/bin/varnishd/cache/cache_esi_fetch.c
index 35166e7..f5748a3 100644
--- a/bin/varnishd/cache/cache_esi_fetch.c
+++ b/bin/varnishd/cache/cache_esi_fetch.c
@@ -325,7 +325,7 @@ vfp_esi_begin(struct worker *w, size_t estimate)
 	}
 
 	(void)estimate;
-	AN(w->vep);
+	AN(w->busyobj->vep);
 }
 
 static int __match_proto__()
@@ -335,7 +335,7 @@ vfp_esi_bytes(struct worker *w, struct http_conn *htc, ssize_t bytes)
 
 	CHECK_OBJ_NOTNULL(w, WORKER_MAGIC);
 	AZ(w->fetch_failed);
-	AN(w->vep);
+	AN(w->busyobj->vep);
 	assert(w->htc == htc);
 	if (w->busyobj->is_gzip && w->do_gunzip)
 		i = vfp_esi_bytes_gu(w, htc, bytes);
@@ -345,7 +345,7 @@ vfp_esi_bytes(struct worker *w, struct http_conn *htc, ssize_t bytes)
 		i = vfp_esi_bytes_gg(w, htc, bytes);
 	else
 		i = vfp_esi_bytes_uu(w, htc, bytes);
-	AN(w->vep);
+	AN(w->busyobj->vep);
 	return (i);
 }
 
@@ -358,7 +358,7 @@ vfp_esi_end(struct worker *w)
 	int retval;
 
 	CHECK_OBJ_NOTNULL(w, WORKER_MAGIC);
-	AN(w->vep);
+	AN(w->busyobj->vep);
 
 	retval = w->fetch_failed;
 
diff --git a/bin/varnishd/cache/cache_esi_parse.c b/bin/varnishd/cache/cache_esi_parse.c
index 9e2b4f6..71ec16d 100644
--- a/bin/varnishd/cache/cache_esi_parse.c
+++ b/bin/varnishd/cache/cache_esi_parse.c
@@ -556,7 +556,8 @@ VEP_Parse(const struct worker *w, const char *p, size_t l)
 	int i;
 
 	CHECK_OBJ_NOTNULL(w, WORKER_MAGIC);
-	vep = w->vep;
+	CHECK_OBJ_NOTNULL(w->busyobj, BUSYOBJ_MAGIC);
+	vep = w->busyobj->vep;
 	CHECK_OBJ_NOTNULL(vep, VEP_MAGIC);
 	assert(l > 0);
 
@@ -981,12 +982,15 @@ VEP_Parse(const struct worker *w, const char *p, size_t l)
 static ssize_t __match_proto__()
 vep_default_cb(struct worker *w, ssize_t l, enum vgz_flag flg)
 {
+	struct vep_state *vep;
 
+	CHECK_OBJ_NOTNULL(w, WORKER_MAGIC);
+	CHECK_OBJ_NOTNULL(w->busyobj, BUSYOBJ_MAGIC);
+	vep = w->busyobj->vep;
+	CHECK_OBJ_NOTNULL(vep, VEP_MAGIC);
 	(void)flg;
-	AN(w->vep);
-	w->vep->cb_x += l;
-Debug("CB(%jd,%d) = %jd\n", (intmax_t)l, flg, (intmax_t)w->vep->cb_x);
-	return (w->vep->cb_x);
+	vep->cb_x += l;
+	return (vep->cb_x);
 }
 
 /*---------------------------------------------------------------------
@@ -998,16 +1002,17 @@ VEP_Init(struct worker *w, vep_callback_t *cb)
 	struct vep_state *vep;
 
 	CHECK_OBJ_NOTNULL(w, WORKER_MAGIC);
-	AZ(w->vep);
-	w->vep = (void*)WS_Alloc(w->ws, sizeof *vep);
-	AN(w->vep);
+	CHECK_OBJ_NOTNULL(w->busyobj, BUSYOBJ_MAGIC);
+	AZ(w->busyobj->vep);
+	vep = (void*)WS_Alloc(w->ws, sizeof *vep);
+	AN(vep);
 
-	vep = w->vep;
 	memset(vep, 0, sizeof *vep);
 	vep->magic = VEP_MAGIC;
 	vep->wrk = w;
 	vep->vsb = VSB_new_auto();
 	AN(vep->vsb);
+	w->busyobj->vep = vep;
 
 	if (cb != NULL) {
 		vep->dogzip = 1;
@@ -1038,13 +1043,14 @@ VEP_Init(struct worker *w, vep_callback_t *cb)
  */
 
 struct vsb *
-VEP_Finish(struct worker *w)
+VEP_Finish(const struct worker *w)
 {
 	struct vep_state *vep;
 	ssize_t l, lcb;
 
 	CHECK_OBJ_NOTNULL(w, WORKER_MAGIC);
-	vep = w->vep;
+	CHECK_OBJ_NOTNULL(w->busyobj, BUSYOBJ_MAGIC);
+	vep = w->busyobj->vep;
 	CHECK_OBJ_NOTNULL(vep, VEP_MAGIC);
 
 	if (vep->o_pending)
@@ -1055,7 +1061,7 @@ VEP_Finish(struct worker *w)
 	}
 	(void)vep->cb(vep->wrk, 0, VGZ_FINISH);
 
-	w->vep = NULL;
+	w->busyobj->vep = NULL;
 
 	AZ(VSB_finish(vep->vsb));
 	l = VSB_len(vep->vsb);



From geoff at varnish-cache.org  Mon Jan  9 20:52:40 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:40 +0100
Subject: [experimental-ims] 2d1ca36 Add a "vsm" instance for the managers
	static VSM allocations (-S,
	-T args) and copy them to the "real vsm" when we create it.
Message-ID: <E1RkMCu-0008MV-QE@project.varnish-software.com>

commit 2d1ca36652afc1196189f60e7aa11b3c9cb948b7
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sun Nov 20 20:58:29 2011 +0000

    Add a "vsm" instance for the managers static VSM allocations (-S, -T args)
    and copy them to the "real vsm" when we create it.

diff --git a/bin/varnishd/common/common.h b/bin/varnishd/common/common.h
index d6de47d..0f25d31 100644
--- a/bin/varnishd/common/common.h
+++ b/bin/varnishd/common/common.h
@@ -48,9 +48,6 @@ struct cli;
 extern pid_t mgt_pid;
 #define ASSERT_MGT() do { assert(getpid() == mgt_pid);} while (0)
 
-/* mgt_shmem.c */
-#define PAN_CLASS "Panic"
-
 /* varnishd.c */
 extern struct vsb *vident;		// XXX: -> heritage ?
 int Symbol_Lookup(struct vsb *vsb, void *ptr);
@@ -77,6 +74,7 @@ void *VSM_common_alloc(struct vsm_sc *sc, ssize_t size,
     const char *class, const char *type, const char *ident);
 void VSM_common_free(struct vsm_sc *sc, void *ptr);
 void VSM_common_delete(struct vsm_sc **sc);
+void VSM_common_copy(struct vsm_sc *to, const struct vsm_sc *from);
 
 /*---------------------------------------------------------------------
  * Generic power-2 rounding macros
diff --git a/bin/varnishd/common/common_vsm.c b/bin/varnishd/common/common_vsm.c
index a2a4fe4..b67e721 100644
--- a/bin/varnishd/common/common_vsm.c
+++ b/bin/varnishd/common/common_vsm.c
@@ -321,3 +321,23 @@ VSM_common_delete(struct vsm_sc **scp)
 	VWMB();
 	FREE_OBJ(sc);
 }
+
+/*--------------------------------------------------------------------
+ * Copy one VSM to another
+ */
+
+void
+VSM_common_copy(struct vsm_sc *to, const struct vsm_sc *from)
+{
+	struct vsm_range *vr;
+	void *p;
+
+	CHECK_OBJ_NOTNULL(to, VSM_SC_MAGIC);
+	CHECK_OBJ_NOTNULL(from, VSM_SC_MAGIC);
+	VTAILQ_FOREACH(vr, &from->r_used, list) {
+		p = VSM_common_alloc(to, vr->chunk->len,
+		    vr->chunk->class, vr->chunk->type, vr->chunk->ident);
+		AN(p);
+		memcpy(p, vr->chunk + 1, vr->chunk->len);
+	}
+}
diff --git a/bin/varnishd/mgt/mgt.h b/bin/varnishd/mgt/mgt.h
index 775fe2c..1b5a872 100644
--- a/bin/varnishd/mgt/mgt.h
+++ b/bin/varnishd/mgt/mgt.h
@@ -82,6 +82,8 @@ void mgt_sandbox_solaris_privsep(void);
 
 /* mgt_shmem.c */
 void mgt_SHM_Init(void);
+void mgt_SHM_static_alloc(const void *, ssize_t size,
+    const char *class, const char *type, const char *ident);
 
 /* stevedore_mgt.c */
 void STV_Config(const char *spec);
@@ -109,9 +111,6 @@ extern unsigned mgt_vcc_err_unref;
 		syslog(pri, fmt, __VA_ARGS__);		\
 	} while (0)
 
-#define VSM_Alloc(a, b, c, d)	VSM_common_alloc(heritage.vsm, a,b,c,d)
-#define VSM_Free(a)		VSM_common_free(heritage.vsm, a)
-
 #if defined(PTHREAD_CANCELED) || defined(PTHREAD_MUTEX_DEFAULT)
 #error "Keep pthreads out of in manager process"
 #endif
diff --git a/bin/varnishd/mgt/mgt_cli.c b/bin/varnishd/mgt/mgt_cli.c
index d3b7396..b141f5b 100644
--- a/bin/varnishd/mgt/mgt_cli.c
+++ b/bin/varnishd/mgt/mgt_cli.c
@@ -43,7 +43,6 @@
 #include <unistd.h>
 
 #include "mgt/mgt.h"
-#include "common/heritage.h"
 #include "common/params.h"
 
 #include "vcli.h"
@@ -493,13 +492,9 @@ mgt_cli_secret(const char *S_arg)
 {
 	int i, fd;
 	char buf[BUFSIZ];
-	char *p;
 
 	/* Save in shmem */
-	i = strlen(S_arg);
-	p = VSM_Alloc(i + 1L, "Arg", "-S", "");
-	AN(p);
-	memcpy(p, S_arg, i + 1L);
+	mgt_SHM_static_alloc(S_arg, strlen(S_arg) + 1L, "Arg", "-S", "");
 
 	srandomdev();			/* XXX: why here ??? */
 	fd = open(S_arg, O_RDONLY);
@@ -527,7 +522,6 @@ mgt_cli_telnet(const char *T_arg)
 	struct vss_addr **ta;
 	int i, n, sock, good;
 	struct telnet *tn;
-	char *p;
 	struct vsb *vsb;
 	char abuf[VTCP_ADDRBUFSIZE];
 	char pbuf[VTCP_PORTBUFSIZE];
@@ -564,9 +558,7 @@ mgt_cli_telnet(const char *T_arg)
 	}
 	AZ(VSB_finish(vsb));
 	/* Save in shmem */
-	p = VSM_Alloc(VSB_len(vsb) + 1, "Arg", "-T", "");
-	AN(p);
-	memcpy(p, VSB_data(vsb), VSB_len(vsb) + 1);
+	mgt_SHM_static_alloc(VSB_data(vsb), VSB_len(vsb) + 1, "Arg", "-T", "");
 	VSB_delete(vsb);
 }
 
diff --git a/bin/varnishd/mgt/mgt_shmem.c b/bin/varnishd/mgt/mgt_shmem.c
index a9a2a7e..075b838 100644
--- a/bin/varnishd/mgt/mgt_shmem.c
+++ b/bin/varnishd/mgt/mgt_shmem.c
@@ -57,7 +57,31 @@
 #define MAP_NOSYNC 0 /* XXX Linux */
 #endif
 
-static int vsm_fd = -1;
+#define PAN_CLASS "Panic"
+
+/*--------------------------------------------------------------------
+ * Use a bogo-VSM to hold master-copies of the VSM chunks the master
+ * publishes, such as -S & -T arguments.
+ */
+
+static struct vsm_sc *static_vsm;
+static char static_vsm_buf[1024];
+
+void
+mgt_SHM_static_alloc(const void *ptr, ssize_t size,
+    const char *class, const char *type, const char *ident)
+{
+	void *p;
+
+	p = VSM_common_alloc(static_vsm, size, class, type, ident);
+	AN(p);
+	memcpy(p, ptr, size);
+	if (heritage.vsm != NULL) {
+		p = VSM_common_alloc(heritage.vsm, size, class, type, ident);
+		AN(p);
+		memcpy(p, ptr, size);
+	}
+}
 
 /*--------------------------------------------------------------------
  * Check that we are not started with the same -n argument as an already
@@ -150,36 +174,21 @@ vsm_zerofile(const char *fn, ssize_t size)
 }
 
 /*--------------------------------------------------------------------
- * Exit handler that clears the owning pid from the SHMLOG
  */
 
-static
-void
-mgt_shm_atexit(void)
+static void
+mgt_SHM_Setup(void)
 {
-
-	if (heritage.vsm != NULL)
-		VSM_common_delete(&heritage.vsm);
-}
-
-void
-mgt_SHM_Init(void)
-{
-	int i;
 	uintmax_t size, ps;
 	void *p;
 	char fnbuf[64];
+	int vsm_fd;
 
 	size = mgt_param.vsl_space + mgt_param.vsm_space;
 	ps = getpagesize();
 	size += ps - 1;
 	size &= ~(ps - 1);
 
-	/* Collision check with already running varnishd */
-	i = vsm_n_check();
-	if (i)
-		exit(i);
-
 	bprintf(fnbuf, "%s.%jd", VSM_FILENAME, (intmax_t)getpid());
 
 	vsm_fd = vsm_zerofile(fnbuf, size);
@@ -191,6 +200,8 @@ mgt_SHM_Init(void)
 	    MAP_HASSEMAPHORE | MAP_NOSYNC | MAP_SHARED,
 	    vsm_fd, 0);
 
+	AZ(close(vsm_fd));
+
 	if (p == MAP_FAILED) {
 		fprintf(stderr, "Mmap error %s: %s\n", fnbuf, strerror(errno));
 		exit (-1);
@@ -207,16 +218,46 @@ mgt_SHM_Init(void)
 		(void)unlink(fnbuf);
 		exit (-1);
 	}
+}
+
+/*--------------------------------------------------------------------
+ * Exit handler that clears the owning pid from the SHMLOG
+ */
+
+static
+void
+mgt_shm_atexit(void)
+{
+
+	if (heritage.vsm != NULL)
+		VSM_common_delete(&heritage.vsm);
+}
+
+void
+mgt_SHM_Init(void)
+{
+	int i;
+
+	/* Collision check with already running varnishd */
+	i = vsm_n_check();
+	if (i)
+		exit(i);
+
+	static_vsm = VSM_common_new(static_vsm_buf, sizeof static_vsm_buf);
+
+	mgt_SHM_Setup();
 
 	AZ(atexit(mgt_shm_atexit));
 
-	heritage.param =
-	    VSM_Alloc(sizeof *heritage.param, VSM_CLASS_PARAM, "", "");
+	VSM_common_copy(heritage.vsm, static_vsm);
+
+	heritage.param = VSM_common_alloc(heritage.vsm,
+	    sizeof *heritage.param, VSM_CLASS_PARAM, "", "");
 	AN(heritage.param);
 	*heritage.param = mgt_param;
 
 	heritage.panic_str_len = 64 * 1024;
-	heritage.panic_str =
-	    VSM_Alloc(heritage.panic_str_len, PAN_CLASS, "", "");
+	heritage.panic_str = VSM_common_alloc(heritage.vsm,
+	    heritage.panic_str_len, PAN_CLASS, "", "");
 	AN(heritage.panic_str);
 }



From geoff at varnish-cache.org  Mon Jan  9 20:51:56 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:51:56 +0100
Subject: [experimental-ims] 4571689 Shave 16 bytes of the session memory
	requirement by putting the sockaddr's directly into struct sess.
Message-ID: <E1RkMCC-00070c-9e@project.varnish-software.com>

commit 4571689fdc7a7fff5f1b13ace0979bc0085181f3
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sun Sep 18 08:14:22 2011 +0000

    Shave 16 bytes of the session memory requirement by putting the
    sockaddr's directly into struct sess.

diff --git a/bin/varnishd/cache.h b/bin/varnishd/cache.h
index 5db137a..122ab34 100644
--- a/bin/varnishd/cache.h
+++ b/bin/varnishd/cache.h
@@ -548,8 +548,8 @@ struct sess {
 
 	socklen_t		sockaddrlen;
 	socklen_t		mysockaddrlen;
-	struct sockaddr_storage	*sockaddr;
-	struct sockaddr_storage	*mysockaddr;
+	struct sockaddr_storage	sockaddr;
+	struct sockaddr_storage	mysockaddr;
 	struct listen_sock	*mylsock;
 
 	/* formatted ascii client address */
diff --git a/bin/varnishd/cache_acceptor.c b/bin/varnishd/cache_acceptor.c
index b2a8015..50cc824 100644
--- a/bin/varnishd/cache_acceptor.c
+++ b/bin/varnishd/cache_acceptor.c
@@ -122,13 +122,13 @@ VCA_Prep(struct sess *sp)
 	char addr[VTCP_ADDRBUFSIZE];
 	char port[VTCP_PORTBUFSIZE];
 
-	VTCP_name(sp->sockaddr, sp->sockaddrlen,
+	VTCP_name(&sp->sockaddr, sp->sockaddrlen,
 	    addr, sizeof addr, port, sizeof port);
 	sp->addr = WS_Dup(sp->ws, addr);
 	sp->port = WS_Dup(sp->ws, port);
 	if (params->log_local_addr) {
-		AZ(getsockname(sp->fd, (void*)sp->mysockaddr, &sp->mysockaddrlen));
-		VTCP_name(sp->mysockaddr, sp->mysockaddrlen,
+		AZ(getsockname(sp->fd, (void*)&sp->mysockaddr, &sp->mysockaddrlen));
+		VTCP_name(&sp->mysockaddr, sp->mysockaddrlen,
 		    addr, sizeof addr, port, sizeof port);
 		VSL(SLT_SessionOpen, sp->fd, "%s %s %s %s",
 		    sp->addr, sp->port, addr, port);
@@ -264,7 +264,7 @@ VCA_SetupSess(struct worker *w)
 	sp->t_end = sp->t_end;
 	sp->mylsock = w->acceptlsock;
 	assert(w->acceptaddrlen <= sp->sockaddrlen);
-	memcpy(sp->sockaddr, &w->acceptaddr, w->acceptaddrlen);
+	memcpy(&sp->sockaddr, &w->acceptaddr, w->acceptaddrlen);
 	sp->sockaddrlen = w->acceptaddrlen;
 	sp->step = STP_FIRST;
 	vca_pace_good();
diff --git a/bin/varnishd/cache_session.c b/bin/varnishd/cache_session.c
index 80e8da5..7012263 100644
--- a/bin/varnishd/cache_session.c
+++ b/bin/varnishd/cache_session.c
@@ -56,7 +56,6 @@ struct sessmem {
 	void			*wsp;
 	struct http		*http[2];
 	VTAILQ_ENTRY(sessmem)	list;
-	struct sockaddr_storage	sockaddr[2];
 };
 
 struct sesspool {
@@ -152,11 +151,9 @@ ses_setup(struct sessmem *sm)
 
 	sp->magic = SESS_MAGIC;
 	sp->mem = sm;
-	sp->sockaddr = (void*)(&sm->sockaddr[0]);
-	sp->sockaddrlen = sizeof(sm->sockaddr[0]);
-	sp->mysockaddr = (void*)(&sm->sockaddr[1]);
-	sp->mysockaddrlen = sizeof(sm->sockaddr[1]);
-	sp->sockaddr->ss_family = sp->mysockaddr->ss_family = PF_UNSPEC;
+	sp->sockaddrlen = sizeof(sp->sockaddr);
+	sp->mysockaddrlen = sizeof(sp->mysockaddr);
+	sp->sockaddr.ss_family = sp->mysockaddr.ss_family = PF_UNSPEC;
 	sp->t_open = NAN;
 	sp->t_req = NAN;
 	sp->t_resp = NAN;
diff --git a/bin/varnishd/cache_vrt_var.c b/bin/varnishd/cache_vrt_var.c
index 2054953..e26b433 100644
--- a/bin/varnishd/cache_vrt_var.c
+++ b/bin/varnishd/cache_vrt_var.c
@@ -457,10 +457,10 @@ REQ_BOOL(hash_always_miss)
 /*--------------------------------------------------------------------*/
 
 struct sockaddr_storage *
-VRT_r_client_ip(const struct sess *sp)
+VRT_r_client_ip(struct sess *sp)
 {
 
-	return (sp->sockaddr);
+	return (&sp->sockaddr);
 }
 
 struct sockaddr_storage *
@@ -468,12 +468,13 @@ VRT_r_server_ip(struct sess *sp)
 {
 	int i;
 
-	if (sp->mysockaddr->ss_family == AF_UNSPEC) {
-		i = getsockname(sp->fd, (void*)sp->mysockaddr, &sp->mysockaddrlen);
+	if (sp->mysockaddr.ss_family == AF_UNSPEC) {
+		i = getsockname(sp->fd,
+		    (void*)&sp->mysockaddr, &sp->mysockaddrlen);
 		assert(VTCP_Check(i));
 	}
 
-	return (sp->mysockaddr);
+	return (&sp->mysockaddr);
 }
 
 const char*
@@ -506,10 +507,14 @@ VRT_r_server_hostname(struct sess *sp)
 int
 VRT_r_server_port(struct sess *sp)
 {
+	int i;
 
-	if (sp->mysockaddr->ss_family == AF_UNSPEC)
-		AZ(getsockname(sp->fd, (void*)sp->mysockaddr, &sp->mysockaddrlen));
-	return (VTCP_port(sp->mysockaddr));
+	if (sp->mysockaddr.ss_family == AF_UNSPEC) {
+		i = getsockname(sp->fd,
+		    (void*)&sp->mysockaddr, &sp->mysockaddrlen);
+		assert(VTCP_Check(i));
+	}
+	return (VTCP_port(&sp->mysockaddr));
 }
 
 /*--------------------------------------------------------------------*/
diff --git a/lib/libvcl/generate.py b/lib/libvcl/generate.py
index fde8bb2..779091e 100755
--- a/lib/libvcl/generate.py
+++ b/lib/libvcl/generate.py
@@ -107,7 +107,7 @@ sp_variables = (
 		'IP',
 		( 'proc',),
 		( ),
-		'const struct sess *'
+		'struct sess *'
 	),
 	('client.identity',
 		'STRING',



From geoff at varnish-cache.org  Mon Jan  9 20:52:03 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:03 +0100
Subject: [experimental-ims] 5c2edfc Only build vsc2rst if we have rst2man,
	fixes build failure
Message-ID: <E1RkMCJ-0007HK-Jj@project.varnish-software.com>

commit 5c2edfc9537e60ea5d61a39601a02a963ab89384
Author: Tollef Fog Heen <tfheen at varnish-software.com>
Date:   Thu Oct 6 10:59:07 2011 +0200

    Only build vsc2rst if we have rst2man, fixes build failure

diff --git a/man/Makefile.am b/man/Makefile.am
index da95e8c..9317896 100644
--- a/man/Makefile.am
+++ b/man/Makefile.am
@@ -1,6 +1,8 @@
 #
 
+if HAVE_RST2MAN
 noinst_PROGRAMS = vsc2rst
+endif
 vsc2rst_SOURCES = vsc2rst.c \
 	$(top_srcdir)/include/vsc_fields.h
 
@@ -30,10 +32,11 @@ else
 	@false
 endif
 
-varnish-counters.7: vsc2rst
 if HAVE_RST2MAN
+varnish-counters.7: vsc2rst
 	./vsc2rst | ${RST2MAN} - $@
 else
+varnish-counters.7: $(top_srcdir)/include/vsc_fields.h
 	@echo "========================================"
 	@echo "You need rst2man installed to make dist"
 	@echo "========================================"



From geoff at varnish-cache.org  Mon Jan  9 20:52:45 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:45 +0100
Subject: [experimental-ims] 0c18d87 Move do_close from worker to busyobj and
	rename it should_close to distinguish it from the electables
	(do_esi, do_gzip etc).
Message-ID: <E1RkMCz-0008WN-41@project.varnish-software.com>

commit 0c18d87646410a2468e97fb231c10a524c3586ef
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Nov 30 07:33:47 2011 +0000

    Move do_close from worker to busyobj and rename it should_close to
    distinguish it from the electables (do_esi, do_gzip etc).

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index 8491150..819e701 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -332,7 +332,6 @@ struct worker {
 	struct http		*beresp;
 	struct vbc		*vbc;
 
-	unsigned		do_close;
 	char			*h_content_length;
 
 	/* Stream state */
@@ -504,6 +503,8 @@ struct busyobj {
 	enum body_status	body_status;
 	struct vef_priv		*vef_priv;
 
+	unsigned		should_close;
+
 	unsigned		do_esi;
 	unsigned		do_gzip;
 	unsigned		do_gunzip;
diff --git a/bin/varnishd/cache/cache_center.c b/bin/varnishd/cache/cache_center.c
index a8138da..6bda81f 100644
--- a/bin/varnishd/cache/cache_center.c
+++ b/bin/varnishd/cache/cache_center.c
@@ -568,7 +568,7 @@ cnt_fetch(struct sess *sp)
 	AN(sp->director);
 	AZ(wrk->vbc);
 	AZ(wrk->h_content_length);
-	AZ(wrk->do_close);
+	AZ(wrk->busyobj->should_close);
 	AZ(wrk->storage_hint);
 
 	http_Setup(wrk->beresp, wrk->ws);
diff --git a/bin/varnishd/cache/cache_dir.c b/bin/varnishd/cache/cache_dir.c
index d4794f9..eba6183 100644
--- a/bin/varnishd/cache/cache_dir.c
+++ b/bin/varnishd/cache/cache_dir.c
@@ -61,7 +61,6 @@ VDI_CloseFd(struct worker *wrk)
 	wrk->vbc->backend = NULL;
 	VBE_ReleaseConn(wrk->vbc);
 	wrk->vbc = NULL;
-	wrk->do_close = 0;
 }
 
 /* Recycle a connection ----------------------------------------------*/
@@ -74,7 +73,6 @@ VDI_RecycleFd(struct worker *wrk)
 	CHECK_OBJ_NOTNULL(wrk->vbc, VBC_MAGIC);
 	CHECK_OBJ_NOTNULL(wrk->vbc->backend, BACKEND_MAGIC);
 	assert(wrk->vbc->fd >= 0);
-	AZ(wrk->do_close);
 
 	bp = wrk->vbc->backend;
 
diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c
index a198650..16509f0 100644
--- a/bin/varnishd/cache/cache_fetch.c
+++ b/bin/varnishd/cache/cache_fetch.c
@@ -589,7 +589,7 @@ FetchBody(struct worker *w, struct object *obj)
 	}
 	AZ(w->busyobj->fetch_failed);
 
-	if (cls == 0 && w->do_close)
+	if (cls == 0 && w->busyobj->should_close)
 		cls = 1;
 
 	WSLB(w, SLT_Length, "%u", obj->len);
diff --git a/bin/varnishd/cache/cache_panic.c b/bin/varnishd/cache/cache_panic.c
index 58fb8e4..0908b74 100644
--- a/bin/varnishd/cache/cache_panic.c
+++ b/bin/varnishd/cache/cache_panic.c
@@ -217,6 +217,7 @@ pan_busyobj(const struct busyobj *bo)
 	if (bo->do_gunzip)	VSB_printf(pan_vsp, "    do_gunzip\n");
 	if (bo->do_esi)		VSB_printf(pan_vsp, "    do_esi\n");
 	if (bo->do_stream)	VSB_printf(pan_vsp, "    do_stream\n");
+	if (bo->should_close)	VSB_printf(pan_vsp, "    should_close\n");
 	VSB_printf(pan_vsp, "    bodystatus = %d,\n", bo->body_status);
 	VSB_printf(pan_vsp, "    },\n");
 }
@@ -258,9 +259,6 @@ pan_sess(const struct sess *sp)
 	VSB_printf(pan_vsp, "  restarts = %d, esi_level = %d\n",
 	    sp->restarts, sp->esi_level);
 
-	VSB_printf(pan_vsp, "  flags = ");
-	if (sp->wrk->do_close)	VSB_printf(pan_vsp, " do_close");
-	VSB_printf(pan_vsp, "\n");
 	pan_busyobj(sp->wrk->busyobj);
 
 	pan_ws(sp->ws, 2);
diff --git a/bin/varnishd/cache/cache_rfc2616.c b/bin/varnishd/cache/cache_rfc2616.c
index ae9614d..c2a795a 100644
--- a/bin/varnishd/cache/cache_rfc2616.c
+++ b/bin/varnishd/cache/cache_rfc2616.c
@@ -186,11 +186,11 @@ RFC2616_Body(const struct sess *sp)
 	hp = sp->wrk->beresp;
 
 	if (hp->protover < 11 && !http_HdrIs(hp, H_Connection, "keep-alive"))
-		sp->wrk->do_close = 1;
+		sp->wrk->busyobj->should_close = 1;
 	else if (http_HdrIs(hp, H_Connection, "close"))
-		sp->wrk->do_close = 1;
+		sp->wrk->busyobj->should_close = 1;
 	else
-		sp->wrk->do_close = 0;
+		sp->wrk->busyobj->should_close = 0;
 
 	if (!strcasecmp(http_GetReq(sp->wrk->bereq), "head")) {
 		/*



From geoff at varnish-cache.org  Mon Jan  9 20:52:14 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:14 +0100
Subject: [experimental-ims] 272aaa3 Exempt all tbl/* includes from multiple
	include guards (451)
Message-ID: <E1RkMCU-0007YI-Uq@project.varnish-software.com>

commit 272aaa35e1f8fa1a56bff758ac92a0ff4a94869b
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sat Oct 8 15:21:00 2011 +0000

    Exempt all tbl/* includes from multiple include guards (451)

diff --git a/bin/varnishd/flint.lnt b/bin/varnishd/flint.lnt
index f2d2a6b..ec0d955 100644
--- a/bin/varnishd/flint.lnt
+++ b/bin/varnishd/flint.lnt
@@ -85,17 +85,10 @@
 //////////////
 -efile(451, "sys/\*.h")	// No include guard
 -efile(451, "machine/\*.h")	// No include guard
--efile(451, "vcl_returns.h")	// No include guard
--efile(451, "body_status.h")	// No include guard
--efile(451, "locks.h")	// No include guard
--efile(451, "cache_backend_poll.h")	// No include guard
--efile(451, "steps.h")	// No include guard
--efile(451, "http_headers.h")	// No include guard
--efile(451, "acct_fields.h")	// No include guard
+-efile(451, "tbl/*.h")	// No include guard
 -efile(451, "vcc_types.h")	// No include guard
 -efile(451, "symbol_kind.h")	// No include guard
 -efile(451, "config.h")	// No include guard
--efile(451, "vrt_stv_var.h")	// No include guard
 //////////////
 // -e458			// unprotected access
 // -e456			// merged locking paths



From geoff at varnish-cache.org  Mon Jan  9 20:52:00 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:00 +0100
Subject: [experimental-ims] c2e5ccb Look for ncurses/curses.h before curses.h
Message-ID: <E1RkMCG-0007AU-Un@project.varnish-software.com>

commit c2e5ccb1325922975eeb2ee1dadaeb6c069e790e
Author: Tollef Fog Heen <tfheen at varnish-software.com>
Date:   Wed Sep 21 13:41:58 2011 +0200

    Look for ncurses/curses.h before curses.h
    
    Solaris puts the ncurses header in /usr/include/ncurses, and we need
    that to compile with -Werror.
    
    Fixes: #889

diff --git a/bin/varnishstat/varnishstat_curses.c b/bin/varnishstat/varnishstat_curses.c
index d0db9f0..22d21fc 100644
--- a/bin/varnishstat/varnishstat_curses.c
+++ b/bin/varnishstat/varnishstat_curses.c
@@ -34,7 +34,11 @@
 
 #include <sys/time.h>
 
+#ifdef HAVE_NCURSES_CURSES_H
+#include <ncurses/curses.h>
+#elif HAVE_CURSES_H
 #include <curses.h>
+#endif
 #include <errno.h>
 #include <signal.h>
 #include <stdio.h>
diff --git a/configure.ac b/configure.ac
index c8b61e5..ef7d76c 100644
--- a/configure.ac
+++ b/configure.ac
@@ -87,6 +87,7 @@ AC_SUBST(CURSES_LIBS)
 if test "$have_curses" = no; then
 	AC_MSG_WARN([curses not found; some tools will not be built])
 fi
+AC_CHECK_HEADERS([ncurses/curses.h curses.h])
 AM_CONDITIONAL([HAVE_CURSES], [test x$have_curses = xyes])
 
 save_LIBS="${LIBS}"



From geoff at varnish-cache.org  Mon Jan  9 20:52:00 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:00 +0100
Subject: [experimental-ims] 2da6772 Remember to sleep and terminate,
	rather than loop for ever on a zero VSL record. Still not quite sure
	how they happen, but we shouldn't hang no matter what.
Message-ID: <E1RkMCG-000782-3W@project.varnish-software.com>

commit 2da67725c55a9aa1c7719048b5bb9642129f0dcf
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Sep 20 14:31:44 2011 +0000

    Remember to sleep and terminate, rather than loop for ever on a zero
    VSL record.  Still not quite sure how they happen, but we shouldn't hang
    no matter what.

diff --git a/lib/libvarnishapi/vsl.c b/lib/libvarnishapi/vsl.c
index 848fe65..1bc46dd 100644
--- a/lib/libvarnishapi/vsl.c
+++ b/lib/libvarnishapi/vsl.c
@@ -175,6 +175,8 @@ vsl_nextlog(struct vsl *vsl, uint32_t **pp)
 
 		if (t == 0) {
 			/* Zero-initialized VSL */
+			w += SLEEP_USEC;
+			assert(usleep(SLEEP_USEC) == 0 || errno == EINTR);
 			VRMB();
 			continue;
 		}
@@ -197,6 +199,7 @@ vsl_nextlog(struct vsl *vsl, uint32_t **pp)
 				return (-1);
 			w += SLEEP_USEC;
 			assert(usleep(SLEEP_USEC) == 0 || errno == EINTR);
+			VRMB();
 			continue;
 		}
 		if (vsl->log_ptr == vsl->log_start + 1)



From geoff at varnish-cache.org  Mon Jan  9 20:52:41 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:41 +0100
Subject: [experimental-ims] 6cc6f23 Merge branch 'master' of
	ssh://git.varnish-cache.org/git/varnish-cache
Message-ID: <E1RkMCv-0008Os-My@project.varnish-software.com>

commit 6cc6f23fe4294dd0e33841a1dafbbe92cecf9874
Merge: 5a7a850 aed74d6
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Nov 23 08:27:05 2011 +0000

    Merge branch 'master' of ssh://git.varnish-cache.org/git/varnish-cache



From geoff at varnish-cache.org  Mon Jan  9 20:52:04 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:04 +0100
Subject: [experimental-ims] bc19f22 Merge branch 'master' of
	ssh://git.varnish-cache.org/git/varnish-cache
Message-ID: <E1RkMCK-0007IN-2H@project.varnish-software.com>

commit bc19f22d2b42914adff665fb447802dedf25ecbe
Merge: 880b0f8 1c9bf69
Author: Per Buer <perbu at varnish-software.com>
Date:   Thu Oct 6 17:33:53 2011 +0200

    Merge branch 'master' of ssh://git.varnish-cache.org/git/varnish-cache



From geoff at varnish-cache.org  Mon Jan  9 20:52:43 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:43 +0100
Subject: [experimental-ims] 0f0eae3 Move fetch_failed from worker to busyobj
Message-ID: <E1RkMCx-0008Ty-ST@project.varnish-software.com>

commit 0f0eae3726097ec9e77ef6dd15471a7b8b077856
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Nov 29 17:50:53 2011 +0000

    Move fetch_failed from worker to busyobj

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index 634d9aa..8cffdef 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -336,7 +336,6 @@ struct worker {
 	enum body_status	body_status;
 	struct vgz		*vgz_rx;
 	struct vef_priv		*vef_priv;
-	unsigned		fetch_failed;
 	unsigned		do_stream;
 	unsigned		do_esi;
 	unsigned		do_gzip;
@@ -347,7 +346,7 @@ struct worker {
 	/* Stream state */
 	struct stream_ctx	*sctx;
 
-	/* ESI stuff */
+	/* ESI delivery stuff */
 	int			gzip_resp;
 	ssize_t			l_crc;
 	uint32_t		crc;
@@ -503,6 +502,7 @@ struct busyobj {
 
 	struct vfp		*vfp;
 	struct vep_state	*vep;
+	unsigned		fetch_failed;
 };
 
 /* Object structure --------------------------------------------------*/
diff --git a/bin/varnishd/cache/cache_esi_fetch.c b/bin/varnishd/cache/cache_esi_fetch.c
index f5748a3..b23538c 100644
--- a/bin/varnishd/cache/cache_esi_fetch.c
+++ b/bin/varnishd/cache/cache_esi_fetch.c
@@ -334,7 +334,7 @@ vfp_esi_bytes(struct worker *w, struct http_conn *htc, ssize_t bytes)
 	int i;
 
 	CHECK_OBJ_NOTNULL(w, WORKER_MAGIC);
-	AZ(w->fetch_failed);
+	AZ(w->busyobj->fetch_failed);
 	AN(w->busyobj->vep);
 	assert(w->htc == htc);
 	if (w->busyobj->is_gzip && w->do_gunzip)
@@ -360,7 +360,7 @@ vfp_esi_end(struct worker *w)
 	CHECK_OBJ_NOTNULL(w, WORKER_MAGIC);
 	AN(w->busyobj->vep);
 
-	retval = w->fetch_failed;
+	retval = w->busyobj->fetch_failed;
 
 	if (w->vgz_rx != NULL && VGZ_Destroy(&w->vgz_rx, -1) != VGZ_END)
 		retval = FetchError(w,
diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c
index 9413300..895a5cb 100644
--- a/bin/varnishd/cache/cache_fetch.c
+++ b/bin/varnishd/cache/cache_fetch.c
@@ -46,7 +46,7 @@ static unsigned fetchfrag;
  * We want to issue the first error we encounter on fetching and
  * supress the rest.  This function does that.
  *
- * Other code is allowed to look at w->fetch_failed to bail out
+ * Other code is allowed to look at w->busyobj->fetch_failed to bail out
  *
  * For convenience, always return -1
  */
@@ -56,13 +56,13 @@ FetchError2(struct worker *w, const char *error, const char *more)
 {
 
 	CHECK_OBJ_NOTNULL(w, WORKER_MAGIC);
-	if (!w->fetch_failed) {
+	if (!w->busyobj->fetch_failed) {
 		if (more == NULL)
 			WSLB(w, SLT_FetchError, "%s", error);
 		else
 			WSLB(w, SLT_FetchError, "%s: %s", error, more);
 	}
-	w->fetch_failed = 1;
+	w->busyobj->fetch_failed = 1;
 	return (-1);
 }
 
@@ -112,7 +112,7 @@ vfp_nop_bytes(struct worker *w, struct http_conn *htc, ssize_t bytes)
 	ssize_t l, wl;
 	struct storage *st;
 
-	AZ(w->fetch_failed);
+	AZ(w->busyobj->fetch_failed);
 	while (bytes > 0) {
 		st = FetchStorage(w, 0);
 		if (st == NULL)
@@ -503,7 +503,7 @@ FetchBody(struct worker *w, struct object *obj)
 	AZ(VTAILQ_FIRST(&obj->store));
 
 	w->fetch_obj = obj;
-	w->fetch_failed = 0;
+	w->busyobj->fetch_failed = 0;
 
 	/* XXX: pick up estimate from objdr ? */
 	cl = 0;
@@ -579,7 +579,7 @@ FetchBody(struct worker *w, struct object *obj)
 		obj->len = 0;
 		return (__LINE__);
 	}
-	AZ(w->fetch_failed);
+	AZ(w->busyobj->fetch_failed);
 
 	if (cls == 0 && w->do_close)
 		cls = 1;
diff --git a/bin/varnishd/cache/cache_gzip.c b/bin/varnishd/cache/cache_gzip.c
index 57b25a5..a175fdf 100644
--- a/bin/varnishd/cache/cache_gzip.c
+++ b/bin/varnishd/cache/cache_gzip.c
@@ -471,7 +471,7 @@ vfp_gunzip_bytes(struct worker *w, struct http_conn *htc, ssize_t bytes)
 	size_t dl;
 	const void *dp;
 
-	AZ(w->fetch_failed);
+	AZ(w->busyobj->fetch_failed);
 	vg = w->vgz_rx;
 	CHECK_OBJ_NOTNULL(vg, VGZ_MAGIC);
 	AZ(vg->vz.avail_in);
@@ -508,7 +508,7 @@ vfp_gunzip_end(struct worker *w)
 	vg = w->vgz_rx;
 	w->vgz_rx = NULL;
 	CHECK_OBJ_NOTNULL(vg, VGZ_MAGIC);
-	if (w->fetch_failed) {
+	if (w->busyobj->fetch_failed) {
 		(void)VGZ_Destroy(&vg, -1);
 		return(0);
 	}
@@ -549,7 +549,7 @@ vfp_gzip_bytes(struct worker *w, struct http_conn *htc, ssize_t bytes)
 	size_t dl;
 	const void *dp;
 
-	AZ(w->fetch_failed);
+	AZ(w->busyobj->fetch_failed);
 	vg = w->vgz_rx;
 	CHECK_OBJ_NOTNULL(vg, VGZ_MAGIC);
 	AZ(vg->vz.avail_in);
@@ -586,7 +586,7 @@ vfp_gzip_end(struct worker *w)
 	vg = w->vgz_rx;
 	CHECK_OBJ_NOTNULL(vg, VGZ_MAGIC);
 	w->vgz_rx = NULL;
-	if (w->fetch_failed) {
+	if (w->busyobj->fetch_failed) {
 		(void)VGZ_Destroy(&vg, -1);
 		return(0);
 	}
@@ -637,7 +637,7 @@ vfp_testgzip_bytes(struct worker *w, struct http_conn *htc, ssize_t bytes)
 	const void *dp;
 	struct storage *st;
 
-	AZ(w->fetch_failed);
+	AZ(w->busyobj->fetch_failed);
 	vg = w->vgz_rx;
 	CHECK_OBJ_NOTNULL(vg, VGZ_MAGIC);
 	AZ(vg->vz.avail_in);
@@ -680,7 +680,7 @@ vfp_testgzip_end(struct worker *w)
 	vg = w->vgz_rx;
 	w->vgz_rx = NULL;
 	CHECK_OBJ_NOTNULL(vg, VGZ_MAGIC);
-	if (w->fetch_failed) {
+	if (w->busyobj->fetch_failed) {
 		(void)VGZ_Destroy(&vg, -1);
 		return(0);
 	}



From geoff at varnish-cache.org  Mon Jan  9 20:52:31 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:31 +0100
Subject: [experimental-ims] f72bf05 Merge branch 'master' of
	ssh://git.varnish-cache.org/git/varnish-cache
Message-ID: <E1RkMCl-00087n-Mr@project.varnish-software.com>

commit f72bf057df68e6b984f6c5aff87ebcc0eaf6f750
Merge: 4805cac 3bf99f6
Author: Lasse Karstensen <lasse at varnish-software.com>
Date:   Thu Nov 10 11:15:45 2011 +0100

    Merge branch 'master' of ssh://git.varnish-cache.org/git/varnish-cache



From geoff at varnish-cache.org  Mon Jan  9 20:52:01 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:01 +0100
Subject: [experimental-ims] 009eb1c Give VSL records a backend and client tag
	(2^30 fd's is enough for everybody!) and tag them at generation,
	rather than having varnishapi try to deduce each VSL's relationship.
Message-ID: <E1RkMCH-0007D7-Ty@project.varnish-software.com>

commit 009eb1c50e5dc3f6f1419636621327348ce22eb2
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Fri Sep 30 15:09:35 2011 +0000

    Give VSL records a backend and client tag (2^30 fd's is enough for
    everybody!) and tag them at generation, rather than having varnishapi
    try to deduce each VSL's relationship.
    
    Please yell if you have records in your varnishlog output which need
    tagging or which are tagged wrong.

diff --git a/bin/varnishd/cache.h b/bin/varnishd/cache.h
index 6c80714..5c793e9 100644
--- a/bin/varnishd/cache.h
+++ b/bin/varnishd/cache.h
@@ -199,6 +199,7 @@ struct http_conn {
 #define HTTP_CONN_MAGIC		0x3e19edd1
 
 	int			fd;
+	unsigned		vsl_id;
 	unsigned		maxbytes;
 	unsigned		maxhdr;
 	struct ws		*ws;
@@ -542,7 +543,7 @@ struct sess {
 	unsigned		magic;
 #define SESS_MAGIC		0x2c2f9c5a
 	int			fd;
-	int			id;
+	unsigned		vsl_id;
 	unsigned		xid;
 
 	int			restarts;
@@ -632,6 +633,7 @@ struct vbc {
 	VTAILQ_ENTRY(vbc)	list;
 	struct backend		*backend;
 	struct vdi_simple	*vdis;
+	unsigned		vsl_id;
 	int			fd;
 
 	struct sockaddr_storage	*addr;
@@ -756,7 +758,8 @@ const char *http_StatusMessage(unsigned);
 unsigned http_EstimateWS(const struct http *fm, unsigned how, uint16_t *nhd);
 void HTTP_Init(void);
 void http_ClrHeader(struct http *to);
-unsigned http_Write(struct worker *w, const struct http *hp, int resp);
+unsigned http_Write(struct worker *w, unsigned vsl_id, const struct http *hp,
+    int resp);
 void http_CopyResp(struct http *to, const struct http *fm);
 void http_SetResp(struct http *to, const char *proto, uint16_t status,
     const char *response);
@@ -792,8 +795,8 @@ void http_Unset(struct http *hp, const char *hdr);
 void http_CollectHdr(struct http *hp, const char *hdr);
 
 /* cache_httpconn.c */
-void HTC_Init(struct http_conn *htc, struct ws *ws, int fd, unsigned maxbytes,
-    unsigned maxhdr);
+void HTC_Init(struct http_conn *htc, struct ws *ws, int fd, unsigned vsl_id,
+    unsigned maxbytes, unsigned maxhdr);
 int HTC_Reinit(struct http_conn *htc);
 int HTC_Rx(struct http_conn *htc);
 ssize_t HTC_Read(struct http_conn *htc, void *d, size_t len);
@@ -887,10 +890,10 @@ void WSL_Flush(struct worker *w, int overflow);
 	} while (0)
 
 #define WSP(sess, tag, ...)					\
-	WSL((sess)->wrk, tag, (sess)->fd, __VA_ARGS__)
+	WSL((sess)->wrk, tag, (sess)->vsl_id, __VA_ARGS__)
 
 #define WSPR(sess, tag, txt)					\
-	WSLR((sess)->wrk, tag, (sess)->fd, txt)
+	WSLR((sess)->wrk, tag, (sess)->vsl_id, txt)
 
 #define INCOMPL() do {							\
 	VSL(SLT_Debug, 0, "INCOMPLETE AT: %s(%d)", __func__, __LINE__); \
diff --git a/bin/varnishd/cache_acceptor.c b/bin/varnishd/cache_acceptor.c
index 1d73a1a..ba28a39 100644
--- a/bin/varnishd/cache_acceptor.c
+++ b/bin/varnishd/cache_acceptor.c
@@ -277,7 +277,7 @@ VCA_SetupSess(struct worker *w)
 	sp = w->sp;
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 	sp->fd = wa->acceptsock;
-	sp->id = wa->acceptsock;
+	sp->vsl_id = wa->acceptsock | VSL_CLIENTMARKER ;
 	wa->acceptsock = -1;
 	sp->t_open = TIM_real();
 	sp->t_end = sp->t_open;
diff --git a/bin/varnishd/cache_backend.c b/bin/varnishd/cache_backend.c
index 3d6a405..d2bc860 100644
--- a/bin/varnishd/cache_backend.c
+++ b/bin/varnishd/cache_backend.c
@@ -118,8 +118,6 @@ vbe_TryConnect(const struct sess *sp, int pf, const struct sockaddr_storage *sa,
 {
 	int s, i, tmo;
 	double tmod;
-	char abuf1[VTCP_ADDRBUFSIZE], abuf2[VTCP_ADDRBUFSIZE];
-	char pbuf1[VTCP_PORTBUFSIZE], pbuf2[VTCP_PORTBUFSIZE];
 
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 	CHECK_OBJ_NOTNULL(vs, VDI_SIMPLE_MAGIC);
@@ -139,11 +137,6 @@ vbe_TryConnect(const struct sess *sp, int pf, const struct sockaddr_storage *sa,
 		return (-1);
 	}
 
-	VTCP_myname(s, abuf1, sizeof abuf1, pbuf1, sizeof pbuf1);
-	VTCP_name(sa, salen, abuf2, sizeof abuf2, pbuf2, sizeof pbuf2);
-	WSL(sp->wrk, SLT_BackendOpen, s, "%s %s %s %s %s",
-	    vs->backend->vcl_name, abuf1, pbuf1, abuf2, pbuf2);
-
 	return (s);
 }
 
@@ -154,6 +147,8 @@ bes_conn_try(const struct sess *sp, struct vbc *vc, const struct vdi_simple *vs)
 {
 	int s;
 	struct backend *bp = vs->backend;
+	char abuf1[VTCP_ADDRBUFSIZE], abuf2[VTCP_ADDRBUFSIZE];
+	char pbuf1[VTCP_PORTBUFSIZE], pbuf2[VTCP_PORTBUFSIZE];
 
 	CHECK_OBJ_NOTNULL(vs, VDI_SIMPLE_MAGIC);
 
@@ -191,7 +186,15 @@ bes_conn_try(const struct sess *sp, struct vbc *vc, const struct vdi_simple *vs)
 		Lck_Unlock(&bp->mtx);
 		vc->addr = NULL;
 		vc->addrlen = 0;
+	} else {
+		vc->vsl_id = s | VSL_BACKENDMARKER;
+		VTCP_myname(s, abuf1, sizeof abuf1, pbuf1, sizeof pbuf1);
+		VTCP_name(vc->addr, vc->addrlen,
+		    abuf2, sizeof abuf2, pbuf2, sizeof pbuf2);
+		WSL(sp->wrk, SLT_BackendOpen, vc->vsl_id, "%s %s %s %s %s",
+		    vs->backend->vcl_name, abuf1, pbuf1, abuf2, pbuf2);
 	}
+
 }
 
 /*--------------------------------------------------------------------
diff --git a/bin/varnishd/cache_center.c b/bin/varnishd/cache_center.c
index 32a4215..38ec4f2 100644
--- a/bin/varnishd/cache_center.c
+++ b/bin/varnishd/cache_center.c
@@ -341,7 +341,7 @@ cnt_done(struct sess *sp)
 			WSP(sp, SLT_Length, "%ju",
 			    (uintmax_t)sp->req_bodybytes);
 		}
-		WSL(sp->wrk, SLT_ReqEnd, sp->id, "%u %.9f %.9f %.9f %.9f %.9f",
+		WSP(sp, SLT_ReqEnd, "%u %.9f %.9f %.9f %.9f %.9f",
 		    sp->xid, sp->t_req, sp->t_end, dh, dp, da);
 	}
 	sp->xid = 0;
@@ -818,9 +818,9 @@ cnt_fetchbody(struct sess *sp)
 
 	hp2->logtag = HTTP_Obj;
 	http_CopyResp(hp2, hp);
-	http_FilterFields(sp->wrk, sp->fd, hp2, hp,
+	http_FilterFields(sp->wrk, sp->vsl_id, hp2, hp,
 	    pass ? HTTPH_R_PASS : HTTPH_A_INS);
-	http_CopyHome(sp->wrk, sp->fd, hp2);
+	http_CopyHome(sp->wrk, sp->vsl_id, hp2);
 
 	if (http_GetHdr(hp, H_Last_Modified, &b))
 		sp->obj->last_modified = TIM_parse(b);
@@ -968,7 +968,7 @@ cnt_first(struct sess *sp)
 	sp->ws_ses = WS_Snapshot(sp->ws);
 
 	/* Receive a HTTP protocol request */
-	HTC_Init(sp->htc, sp->ws, sp->fd, params->http_req_size,
+	HTC_Init(sp->htc, sp->ws, sp->fd, sp->vsl_id, params->http_req_size,
 	    params->http_req_hdr_len);
 	sp->wrk->lastused = sp->t_open;
 	sp->wrk->acct_tmp.sess++;
@@ -1537,12 +1537,11 @@ static void
 cnt_diag(struct sess *sp, const char *state)
 {
 	if (sp->wrk != NULL) {
-		WSL(sp->wrk, SLT_Debug, sp->id,
-		    "thr %p STP_%s sp %p obj %p vcl %p",
+		WSP(sp, SLT_Debug, "thr %p STP_%s sp %p obj %p vcl %p",
 		    pthread_self(), state, sp, sp->obj, sp->vcl);
 		WSL_Flush(sp->wrk, 0);
 	} else {
-		VSL(SLT_Debug, sp->id,
+		VSL(SLT_Debug, sp->vsl_id,
 		    "thr %p STP_%s sp %p obj %p vcl %p",
 		    pthread_self(), state, sp, sp->obj, sp->vcl);
 	}
diff --git a/bin/varnishd/cache_esi_deliver.c b/bin/varnishd/cache_esi_deliver.c
index ea4951f..8b2edba 100644
--- a/bin/varnishd/cache_esi_deliver.c
+++ b/bin/varnishd/cache_esi_deliver.c
@@ -105,7 +105,7 @@ ved_include(struct sess *sp, const char *src, const char *host)
 			break;
 		AZ(sp->wrk);
 		WSL_Flush(w, 0);
-		DSL(0x20, SLT_Debug, sp->id, "loop waiting for ESI");
+		DSL(0x20, SLT_Debug, sp->vsl_id, "loop waiting for ESI");
 		(void)usleep(10000);
 	}
 	sp->xid = sxid;
diff --git a/bin/varnishd/cache_fetch.c b/bin/varnishd/cache_fetch.c
index 73edaa1..6451f10 100644
--- a/bin/varnishd/cache_fetch.c
+++ b/bin/varnishd/cache_fetch.c
@@ -414,7 +414,7 @@ FetchHdr(struct sess *sp)
 
 	(void)VTCP_blocking(vc->fd);	/* XXX: we should timeout instead */
 	WRW_Reserve(w, &vc->fd);
-	(void)http_Write(w, hp, 0);	/* XXX: stats ? */
+	(void)http_Write(w, vc->vsl_id, hp, 0);	/* XXX: stats ? */
 
 	/* Deal with any message-body the request might have */
 	i = FetchReqBody(sp);
@@ -434,7 +434,7 @@ FetchHdr(struct sess *sp)
 
 	/* Receive response */
 
-	HTC_Init(w->htc, w->ws, vc->fd, params->http_resp_size,
+	HTC_Init(w->htc, w->ws, vc->fd, vc->vsl_id, params->http_resp_size,
 	    params->http_resp_hdr_len);
 
 	VTCP_set_read_timeout(vc->fd, vc->first_byte_timeout);
diff --git a/bin/varnishd/cache_hash.c b/bin/varnishd/cache_hash.c
index faf867a..58c3b6a 100644
--- a/bin/varnishd/cache_hash.c
+++ b/bin/varnishd/cache_hash.c
@@ -503,7 +503,7 @@ hsh_rush(struct objhead *oh)
 		CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 		AZ(sp->wrk);
 		VTAILQ_REMOVE(&wl->list, sp, list);
-		DSL(0x20, SLT_Debug, sp->id, "off waiting list");
+		DSL(0x20, SLT_Debug, sp->vsl_id, "off waiting list");
 		if (SES_Schedule(sp)) {
 			/*
 			 * We could not schedule the session, leave the
diff --git a/bin/varnishd/cache_http.c b/bin/varnishd/cache_http.c
index f77ab74..587b6bb 100644
--- a/bin/varnishd/cache_http.c
+++ b/bin/varnishd/cache_http.c
@@ -77,10 +77,11 @@ http2shmlog(const struct http *hp, int t)
 }
 
 static void
-WSLH(struct worker *w, int fd, const struct http *hp, unsigned hdr)
+WSLH(struct worker *w, int vsl_id, const struct http *hp, unsigned hdr)
 {
 
-	WSLR(w, http2shmlog(hp, hdr), fd, hp->hd[hdr]);
+	AN(vsl_id & (VSL_CLIENTMARKER|VSL_BACKENDMARKER));
+	WSLR(w, http2shmlog(hp, hdr), vsl_id, hp->hd[hdr]);
 }
 
 /*--------------------------------------------------------------------*/
@@ -485,7 +486,7 @@ http_GetReq(const struct http *hp)
  */
 
 static uint16_t
-http_dissect_hdrs(struct worker *w, struct http *hp, int fd, char *p,
+http_dissect_hdrs(struct worker *w, struct http *hp, int vsl_id, char *p,
     const struct http_conn *htc)
 {
 	char *q, *r;
@@ -522,7 +523,7 @@ http_dissect_hdrs(struct worker *w, struct http *hp, int fd, char *p,
 
 		if (q - p > htc->maxhdr) {
 			VSC_C_main->losthdr++;
-			WSL(w, SLT_LostHeader, fd, "%.*s",
+			WSL(w, SLT_LostHeader, vsl_id, "%.*s",
 			    q - p > 20 ? 20 : q - p, p);
 			return (413);
 		}
@@ -544,11 +545,11 @@ http_dissect_hdrs(struct worker *w, struct http *hp, int fd, char *p,
 			hp->hdf[hp->nhd] = 0;
 			hp->hd[hp->nhd].b = p;
 			hp->hd[hp->nhd].e = q;
-			WSLH(w, fd, hp, hp->nhd);
+			WSLH(w, vsl_id, hp, hp->nhd);
 			hp->nhd++;
 		} else {
 			VSC_C_main->losthdr++;
-			WSL(w, SLT_LostHeader, fd, "%.*s",
+			WSL(w, SLT_LostHeader, vsl_id, "%.*s",
 			    q - p > 20 ? 20 : q - p, p);
 			return (413);
 		}
@@ -561,7 +562,7 @@ http_dissect_hdrs(struct worker *w, struct http *hp, int fd, char *p,
  */
 
 static uint16_t
-http_splitline(struct worker *w, int fd, struct http *hp,
+http_splitline(struct worker *w, int vsl_id, struct http *hp,
     const struct http_conn *htc, int h1, int h2, int h3)
 {
 	char *p, *q;
@@ -623,17 +624,17 @@ http_splitline(struct worker *w, int fd, struct http *hp,
 	p += vct_skipcrlf(p);
 
 	*hp->hd[h1].e = '\0';
-	WSLH(w, fd, hp, h1);
+	WSLH(w, vsl_id, hp, h1);
 
 	*hp->hd[h2].e = '\0';
-	WSLH(w, fd, hp, h2);
+	WSLH(w, vsl_id, hp, h2);
 
 	if (hp->hd[h3].e != NULL) {
 		*hp->hd[h3].e = '\0';
-		WSLH(w, fd, hp, h3);
+		WSLH(w, vsl_id, hp, h3);
 	}
 
-	return (http_dissect_hdrs(w, hp, fd, p, htc));
+	return (http_dissect_hdrs(w, hp, vsl_id, p, htc));
 }
 
 /*--------------------------------------------------------------------*/
@@ -668,7 +669,7 @@ http_DissectRequest(struct sess *sp)
 
 	hp->logtag = HTTP_Rx;
 
-	retval = http_splitline(sp->wrk, sp->fd, hp, htc,
+	retval = http_splitline(sp->wrk, sp->vsl_id, hp, htc,
 	    HTTP_HDR_REQ, HTTP_HDR_URL, HTTP_HDR_PROTO);
 	if (retval != 0) {
 		WSPR(sp, SLT_HttpGarbage, htc->rxbuf);
@@ -693,7 +694,7 @@ http_DissectResponse(struct worker *w, const struct http_conn *htc,
 	CHECK_OBJ_NOTNULL(hp, HTTP_MAGIC);
 	hp->logtag = HTTP_Rx;
 
-	if (http_splitline(w, htc->fd, hp, htc,
+	if (http_splitline(w, htc->vsl_id, hp, htc,
 	    HTTP_HDR_PROTO, HTTP_HDR_STATUS, HTTP_HDR_RESPONSE))
 		retval = 503;
 
@@ -719,7 +720,7 @@ http_DissectResponse(struct worker *w, const struct http_conn *htc,
 	}
 
 	if (retval != 0) {
-		WSLR(w, SLT_HttpGarbage, htc->fd, htc->rxbuf);
+		WSLR(w, SLT_HttpGarbage, htc->vsl_id, htc->rxbuf);
 		assert(retval >= 100 && retval <= 999);
 		hp->status = retval;
 	} else {
@@ -791,7 +792,7 @@ http_SetResp(struct http *to, const char *proto, uint16_t status,
 }
 
 static void
-http_copyheader(struct worker *w, int fd, struct http *to,
+http_copyheader(struct worker *w, int vsl_id, struct http *to,
     const struct http *fm, unsigned n)
 {
 
@@ -805,7 +806,7 @@ http_copyheader(struct worker *w, int fd, struct http *to,
 		to->nhd++;
 	} else  {
 		VSC_C_main->losthdr++;
-		WSLR(w, SLT_LostHeader, fd, fm->hd[n]);
+		WSLR(w, SLT_LostHeader, vsl_id, fm->hd[n]);
 	}
 }
 
@@ -842,7 +843,7 @@ http_EstimateWS(const struct http *fm, unsigned how, uint16_t *nhd)
 /*--------------------------------------------------------------------*/
 
 void
-http_FilterFields(struct worker *w, int fd, struct http *to,
+http_FilterFields(struct worker *w, int vsl_id, struct http *to,
     const struct http *fm, unsigned how)
 {
 	unsigned u;
@@ -861,7 +862,7 @@ http_FilterFields(struct worker *w, int fd, struct http *to,
 			continue;
 #include "http_headers.h"
 #undef HTTPH
-		http_copyheader(w, fd, to, fm, u);
+		http_copyheader(w, vsl_id, to, fm, u);
 	}
 }
 
@@ -882,8 +883,8 @@ http_FilterHeader(const struct sess *sp, unsigned how)
 		http_SetH(hp, HTTP_HDR_PROTO, "HTTP/1.1");
 	else
 		http_copyh(hp, sp->http, HTTP_HDR_PROTO);
-	http_FilterFields(sp->wrk, sp->fd, hp, sp->http, how);
-	http_PrintfHeader(sp->wrk, sp->fd, hp, "X-Varnish: %u", sp->xid);
+	http_FilterFields(sp->wrk, sp->vsl_id, hp, sp->http, how);
+	http_PrintfHeader(sp->wrk, sp->vsl_id, hp, "X-Varnish: %u", sp->xid);
 }
 
 /*--------------------------------------------------------------------
@@ -892,7 +893,7 @@ http_FilterHeader(const struct sess *sp, unsigned how)
  */
 
 void
-http_CopyHome(struct worker *w, int fd, const struct http *hp)
+http_CopyHome(struct worker *w, int vsl_id, const struct http *hp)
 {
 	unsigned u, l;
 	char *p;
@@ -901,20 +902,20 @@ http_CopyHome(struct worker *w, int fd, const struct http *hp)
 		if (hp->hd[u].b == NULL)
 			continue;
 		if (hp->hd[u].b >= hp->ws->s && hp->hd[u].e <= hp->ws->e) {
-			WSLH(w, fd, hp, u);
+			WSLH(w, vsl_id, hp, u);
 			continue;
 		}
 		l = Tlen(hp->hd[u]);
 		p = WS_Alloc(hp->ws, l + 1);
 		if (p != NULL) {
-			WSLH(w, fd, hp, u);
+			WSLH(w, vsl_id, hp, u);
 			memcpy(p, hp->hd[u].b, l + 1L);
 			hp->hd[u].b = p;
 			hp->hd[u].e = p + l;
 		} else {
 			/* XXX This leaves a slot empty */
 			VSC_C_main->losthdr++;
-			WSLR(w, SLT_LostHeader, fd, hp->hd[u]);
+			WSLR(w, SLT_LostHeader, vsl_id, hp->hd[u]);
 			hp->hd[u].b = NULL;
 			hp->hd[u].e = NULL;
 		}
@@ -938,13 +939,13 @@ http_ClrHeader(struct http *to)
 /*--------------------------------------------------------------------*/
 
 void
-http_SetHeader(struct worker *w, int fd, struct http *to, const char *hdr)
+http_SetHeader(struct worker *w, int vsl_id, struct http *to, const char *hdr)
 {
 
 	CHECK_OBJ_NOTNULL(to, HTTP_MAGIC);
 	if (to->nhd >= to->shd) {
 		VSC_C_main->losthdr++;
-		WSL(w, SLT_LostHeader, fd, "%s", hdr);
+		WSL(w, SLT_LostHeader, vsl_id, "%s", hdr);
 		return;
 	}
 	http_SetH(to, to->nhd++, hdr);
@@ -953,7 +954,7 @@ http_SetHeader(struct worker *w, int fd, struct http *to, const char *hdr)
 /*--------------------------------------------------------------------*/
 
 static void
-http_PutField(struct worker *w, int fd, const struct http *to, int field,
+http_PutField(struct worker *w, int vsl_id, const struct http *to, int field,
     const char *string)
 {
 	char *p;
@@ -963,7 +964,7 @@ http_PutField(struct worker *w, int fd, const struct http *to, int field,
 	l = strlen(string);
 	p = WS_Alloc(to->ws, l + 1);
 	if (p == NULL) {
-		WSL(w, SLT_LostHeader, fd, "%s", string);
+		WSL(w, SLT_LostHeader, vsl_id, "%s", string);
 		to->hd[field].b = NULL;
 		to->hd[field].e = NULL;
 		to->hdf[field] = 0;
@@ -976,11 +977,11 @@ http_PutField(struct worker *w, int fd, const struct http *to, int field,
 }
 
 void
-http_PutProtocol(struct worker *w, int fd, const struct http *to,
+http_PutProtocol(struct worker *w, int vsl_id, const struct http *to,
     const char *protocol)
 {
 
-	http_PutField(w, fd, to, HTTP_HDR_PROTO, protocol);
+	http_PutField(w, vsl_id, to, HTTP_HDR_PROTO, protocol);
 }
 
 void
@@ -992,15 +993,15 @@ http_PutStatus(struct http *to, uint16_t status)
 }
 
 void
-http_PutResponse(struct worker *w, int fd, const struct http *to,
+http_PutResponse(struct worker *w, int vsl_id, const struct http *to,
     const char *response)
 {
 
-	http_PutField(w, fd, to, HTTP_HDR_RESPONSE, response);
+	http_PutField(w, vsl_id, to, HTTP_HDR_RESPONSE, response);
 }
 
 void
-http_PrintfHeader(struct worker *w, int fd, struct http *to,
+http_PrintfHeader(struct worker *w, int vsl_id, struct http *to,
     const char *fmt, ...)
 {
 	va_list ap;
@@ -1013,7 +1014,7 @@ http_PrintfHeader(struct worker *w, int fd, struct http *to,
 	va_end(ap);
 	if (n + 1 >= l || to->nhd >= to->shd) {
 		VSC_C_main->losthdr++;
-		WSL(w, SLT_LostHeader, fd, "%s", to->ws->f);
+		WSL(w, SLT_LostHeader, vsl_id, "%s", to->ws->f);
 		WS_Release(to->ws, 0);
 	} else {
 		to->hd[to->nhd].b = to->ws->f;
@@ -1063,14 +1064,13 @@ HTTP_Copy(struct http *to, const struct http * const fm)
 /*--------------------------------------------------------------------*/
 
 unsigned
-http_Write(struct worker *w, const struct http *hp, int resp)
+http_Write(struct worker *w, unsigned vsl_id, const struct http *hp, int resp)
 {
 	unsigned u, l;
-	int fd = *(w->wrw.wfd);
 
 	if (resp) {
 		l = WRW_WriteH(w, &hp->hd[HTTP_HDR_PROTO], " ");
-		WSLH(w, fd, hp, HTTP_HDR_PROTO);
+		WSLH(w, vsl_id, hp, HTTP_HDR_PROTO);
 
 		hp->hd[HTTP_HDR_STATUS].b = WS_Alloc(w->ws, 4);
 		AN(hp->hd[HTTP_HDR_STATUS].b);
@@ -1079,18 +1079,18 @@ http_Write(struct worker *w, const struct http *hp, int resp)
 		hp->hd[HTTP_HDR_STATUS].e = hp->hd[HTTP_HDR_STATUS].b + 3;
 
 		l += WRW_WriteH(w, &hp->hd[HTTP_HDR_STATUS], " ");
-		WSLH(w, fd, hp, HTTP_HDR_STATUS);
+		WSLH(w, vsl_id, hp, HTTP_HDR_STATUS);
 
 		l += WRW_WriteH(w, &hp->hd[HTTP_HDR_RESPONSE], "\r\n");
-		WSLH(w, fd, hp, HTTP_HDR_RESPONSE);
+		WSLH(w, vsl_id, hp, HTTP_HDR_RESPONSE);
 	} else {
 		AN(hp->hd[HTTP_HDR_URL].b);
 		l = WRW_WriteH(w, &hp->hd[HTTP_HDR_REQ], " ");
-		WSLH(w, fd, hp, HTTP_HDR_REQ);
+		WSLH(w, vsl_id, hp, HTTP_HDR_REQ);
 		l += WRW_WriteH(w, &hp->hd[HTTP_HDR_URL], " ");
-		WSLH(w, fd, hp, HTTP_HDR_URL);
+		WSLH(w, vsl_id, hp, HTTP_HDR_URL);
 		l += WRW_WriteH(w, &hp->hd[HTTP_HDR_PROTO], "\r\n");
-		WSLH(w, fd, hp, HTTP_HDR_PROTO);
+		WSLH(w, vsl_id, hp, HTTP_HDR_PROTO);
 	}
 	for (u = HTTP_HDR_FIRST; u < hp->nhd; u++) {
 		if (hp->hd[u].b == NULL)
@@ -1098,7 +1098,7 @@ http_Write(struct worker *w, const struct http *hp, int resp)
 		AN(hp->hd[u].b);
 		AN(hp->hd[u].e);
 		l += WRW_WriteH(w, &hp->hd[u], "\r\n");
-		WSLH(w, fd, hp, u);
+		WSLH(w, vsl_id, hp, u);
 	}
 	l += WRW_Write(w, "\r\n", -1);
 	return (l);
diff --git a/bin/varnishd/cache_httpconn.c b/bin/varnishd/cache_httpconn.c
index 9d6d926..3ba5d7f 100644
--- a/bin/varnishd/cache_httpconn.c
+++ b/bin/varnishd/cache_httpconn.c
@@ -79,13 +79,14 @@ htc_header_complete(txt *t)
 /*--------------------------------------------------------------------*/
 
 void
-HTC_Init(struct http_conn *htc, struct ws *ws, int fd, unsigned maxbytes,
-    unsigned maxhdr)
+HTC_Init(struct http_conn *htc, struct ws *ws, int fd, unsigned vsl_id,
+    unsigned maxbytes, unsigned maxhdr)
 {
 
 	htc->magic = HTTP_CONN_MAGIC;
 	htc->ws = ws;
 	htc->fd = fd;
+	htc->vsl_id = vsl_id;
 	htc->maxbytes = maxbytes;
 	htc->maxhdr = maxhdr;
 	htc->error = "No error recorded";
diff --git a/bin/varnishd/cache_panic.c b/bin/varnishd/cache_panic.c
index b243832..8ca1b4f 100644
--- a/bin/varnishd/cache_panic.c
+++ b/bin/varnishd/cache_panic.c
@@ -216,7 +216,8 @@ pan_sess(const struct sess *sp)
 
 	VSB_printf(vsp, "sp = %p {\n", sp);
 	VSB_printf(vsp,
-	    "  fd = %d, id = %d, xid = %u,\n", sp->fd, sp->id, sp->xid);
+	    "  fd = %d, id = %d, xid = %u,\n",
+	    sp->fd, sp->vsl_id & VSL_IDENTMASK, sp->xid);
 	VSB_printf(vsp, "  client = %s %s,\n",
 	    sp->addr ? sp->addr : "?.?.?.?",
 	    sp->port ? sp->port : "?");
diff --git a/bin/varnishd/cache_pipe.c b/bin/varnishd/cache_pipe.c
index 3968767..ea5189d 100644
--- a/bin/varnishd/cache_pipe.c
+++ b/bin/varnishd/cache_pipe.c
@@ -78,7 +78,8 @@ PipeSession(struct sess *sp)
 	(void)VTCP_blocking(vc->fd);
 
 	WRW_Reserve(w, &vc->fd);
-	sp->wrk->acct_tmp.hdrbytes += http_Write(w, sp->wrk->bereq, 0);
+	sp->wrk->acct_tmp.hdrbytes +=
+	    http_Write(w, sp->vsl_id, sp->wrk->bereq, 0);
 
 	if (sp->htc->pipeline.b != NULL)
 		sp->wrk->acct_tmp.bodybytes +=
diff --git a/bin/varnishd/cache_response.c b/bin/varnishd/cache_response.c
index eb2e62d..e490bbf 100644
--- a/bin/varnishd/cache_response.c
+++ b/bin/varnishd/cache_response.c
@@ -296,7 +296,7 @@ RES_WriteObj(struct sess *sp)
 	 */
 	if (!(sp->wrk->res_mode & RES_ESI_CHILD))
 		sp->wrk->acct_tmp.hdrbytes +=
-		    http_Write(sp->wrk, sp->wrk->resp, 1);
+		    http_Write(sp->wrk, sp->vsl_id, sp->wrk->resp, 1);
 
 	if (!sp->wantbody)
 		sp->wrk->res_mode &= ~RES_CHUNKED;
@@ -352,7 +352,7 @@ RES_StreamStart(struct sess *sp)
 		    "Content-Length: %s", sp->wrk->h_content_length);
 
 	sp->wrk->acct_tmp.hdrbytes +=
-	    http_Write(sp->wrk, sp->wrk->resp, 1);
+	    http_Write(sp->wrk, sp->vsl_id, sp->wrk->resp, 1);
 
 	if (sp->wrk->res_mode & RES_CHUNKED)
 		WRW_Chunked(sp->wrk);
diff --git a/bin/varnishd/cache_session.c b/bin/varnishd/cache_session.c
index bbc5706..4956154 100644
--- a/bin/varnishd/cache_session.c
+++ b/bin/varnishd/cache_session.c
@@ -303,7 +303,7 @@ SES_Close(struct sess *sp, const char *reason)
 	int i;
 
 	assert(sp->fd >= 0);
-	VSL(SLT_SessionClose, sp->id, "%s", reason);
+	VSL(SLT_SessionClose, sp->vsl_id, "%s", reason);
 	i = close(sp->fd);
 	assert(i == 0 || errno != EBADF); /* XXX EINVAL seen */
 	sp->fd = -1;
@@ -352,7 +352,7 @@ SES_Delete(struct sess *sp, const char *reason)
 	assert(!isnan(b->first));
 	assert(!isnan(sp->t_end));
 
-	VSL(SLT_StatSess, sp->id, "%s %s %.0f %ju %ju %ju %ju %ju %ju %ju",
+	VSL(SLT_StatSess, sp->vsl_id, "%s %s %.0f %ju %ju %ju %ju %ju %ju %ju",
 	    sp->addr, sp->port, sp->t_end - b->first,
 	    b->sess, b->req, b->pipe, b->pass,
 	    b->fetch, b->hdrbytes, b->bodybytes);
diff --git a/bin/varnishd/cache_waiter_kqueue.c b/bin/varnishd/cache_waiter_kqueue.c
index 6e2cb05..9575dc8 100644
--- a/bin/varnishd/cache_waiter_kqueue.c
+++ b/bin/varnishd/cache_waiter_kqueue.c
@@ -80,7 +80,7 @@ vwk_kq_sess(struct vwk *vwk, struct sess *sp, short arm)
 
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 	assert(sp->fd >= 0);
-	DSL(0x04, SLT_Debug, sp->fd, "KQ: EV_SET sp %p arm %x", sp, arm);
+	DSL(0x04, SLT_Debug, sp->vsl_id, "KQ: EV_SET sp %p arm %x", sp, arm);
 	EV_SET(&vwk->ki[vwk->nki], sp->fd, EVFILT_READ, arm, 0, 0, sp);
 	if (++vwk->nki == NKEV)
 		vwk_kq_flush(vwk);
@@ -112,12 +112,12 @@ vwk_kev(struct vwk *vwk, const struct kevent *kp)
 		return;
 	}
 	CAST_OBJ_NOTNULL(sp, kp->udata, SESS_MAGIC);
-	DSL(0x04, SLT_Debug, sp->id, "KQ: sp %p kev data %lu flags 0x%x%s",
+	DSL(0x04, SLT_Debug, sp->vsl_id, "KQ: sp %p kev data %lu flags 0x%x%s",
 	    sp, (unsigned long)kp->data, kp->flags,
 	    (kp->flags & EV_EOF) ? " EOF" : "");
 
-	assert(sp->id == kp->ident);
-	assert(sp->fd == sp->id);
+	assert((sp->vsl_id & VSL_IDENTMASK) == kp->ident);
+	assert((sp->vsl_id & VSL_IDENTMASK) == sp->fd);
 	if (kp->data > 0) {
 		i = HTC_Rx(sp->htc);
 		if (i == 0) {
@@ -132,7 +132,7 @@ vwk_kev(struct vwk *vwk, const struct kevent *kp)
 		SES_Delete(sp, "EOF");
 		return;
 	} else {
-		VSL(SLT_Debug, sp->id, "KQ: sp %p kev data %lu flags 0x%x%s",
+		VSL(SLT_Debug, sp->vsl_id, "KQ: sp %p kev data %lu flags 0x%x%s",
 		    sp, (unsigned long)kp->data, kp->flags,
 		    (kp->flags & EV_EOF) ? " EOF" : "");
 	}
diff --git a/include/vsl.h b/include/vsl.h
index 45406c0..7f771ce 100644
--- a/include/vsl.h
+++ b/include/vsl.h
@@ -50,12 +50,18 @@
  *	[n + 2] ... [m]	= content
  */
 
+#define VSL_CLIENTMARKER	(1U<<30)
+#define VSL_BACKENDMARKER	(1U<<31)
+#define VSL_IDENTMASK		(~(3U<<30))
+
 #define VSL_WORDS(len)		(((len) + 3) / 4)
 #define VSL_END(ptr, len)	((ptr) + 2 + VSL_WORDS(len))
 #define VSL_NEXT(ptr)		VSL_END(ptr, VSL_LEN(ptr))
 #define VSL_LEN(ptr)		((ptr)[0] & 0xffff)
 #define VSL_TAG(ptr)		((ptr)[0] >> 24)
-#define VSL_ID(ptr)		((ptr)[1])
+#define VSL_ID(ptr)		(((ptr)[1]) & VSL_IDENTMASK)
+#define VSL_CLIENT(ptr)		(((ptr)[1]) & VSL_CLIENTMARKER)
+#define VSL_BACKEND(ptr)	(((ptr)[1]) & VSL_BACKENDMARKER)
 #define VSL_DATA(ptr)		((char*)((ptr)+2))
 
 #define VSL_ENDMARKER	(((uint32_t)SLT_Reserved << 24) | 0x454545) /* "EEE" */
diff --git a/lib/libvarnishapi/vsl.c b/lib/libvarnishapi/vsl.c
index 8473d31..a34ea1e 100644
--- a/lib/libvarnishapi/vsl.c
+++ b/lib/libvarnishapi/vsl.c
@@ -75,8 +75,6 @@ VSL_Setup(struct VSM_data *vd)
 	vsl->regflags = 0;
 
 	/* XXX: Allocate only if log access */
-	vsl->vbm_client = vbit_init(4096);
-	vsl->vbm_backend = vbit_init(4096);
 	vsl->vbm_supress = vbit_init(256);
 	vsl->vbm_select = vbit_init(256);
 
@@ -102,8 +100,6 @@ VSL_Delete(struct VSM_data *vd)
 	vd->vsl = NULL;
 	CHECK_OBJ_NOTNULL(vsl, VSL_MAGIC);
 
-	vbit_destroy(vsl->vbm_client);
-	vbit_destroy(vsl->vbm_backend);
 	vbit_destroy(vsl->vbm_supress);
 	vbit_destroy(vsl->vbm_select);
 	free(vsl->rbuf);
@@ -232,20 +228,6 @@ VSL_NextLog(const struct VSM_data *vd, uint32_t **pp, uint64_t *bits)
 			return (i);
 		u = VSL_ID(p);
 		t = VSL_TAG(p);
-		switch(t) {
-		case SLT_SessionOpen:
-		case SLT_ReqStart:
-			vbit_set(vsl->vbm_client, u);
-			vbit_clr(vsl->vbm_backend, u);
-			break;
-		case SLT_BackendOpen:
-		case SLT_BackendXID:
-			vbit_clr(vsl->vbm_client, u);
-			vbit_set(vsl->vbm_backend, u);
-			break;
-		default:
-			break;
-		}
 		if (vsl->skip) {
 			--vsl->skip;
 			continue;
@@ -260,9 +242,9 @@ VSL_NextLog(const struct VSM_data *vd, uint32_t **pp, uint64_t *bits)
 		}
 		if (vbit_test(vsl->vbm_supress, t))
 			continue;
-		if (vsl->b_opt && !vbit_test(vsl->vbm_backend, u))
+		if (vsl->b_opt && !VSL_BACKEND(p))
 			continue;
-		if (vsl->c_opt && !vbit_test(vsl->vbm_client, u))
+		if (vsl->c_opt && !VSL_CLIENT(p))
 			continue;
 		if (vsl->regincl != NULL) {
 			i = VRE_exec(vsl->regincl, VSL_DATA(p), VSL_LEN(p),
@@ -319,10 +301,10 @@ VSL_Dispatch(struct VSM_data *vd, VSL_handler_f *func, void *priv)
 		u = VSL_ID(p);
 		l = VSL_LEN(p);
 		s = 0;
-		if (vbit_test(vsl->vbm_backend, u))
-			s |= VSL_S_BACKEND;
-		if (vbit_test(vsl->vbm_client, u))
+		if (VSL_CLIENT(p))
 			s |= VSL_S_CLIENT;
+		if (VSL_BACKEND(p))
+			s |= VSL_S_BACKEND;
 		if (func(priv, VSL_TAG(p), u, l, s, VSL_DATA(p), bitmap))
 			return (1);
 	}
diff --git a/lib/libvarnishapi/vsl_api.h b/lib/libvarnishapi/vsl_api.h
index e1684fd..aa3d0f4 100644
--- a/lib/libvarnishapi/vsl_api.h
+++ b/lib/libvarnishapi/vsl_api.h
@@ -64,13 +64,6 @@ struct vsl {
 #define F_NON_BLOCKING		(1 << 1)
 
 	/*
-	 * These two bitmaps mark fd's as belonging to client or backend
-	 * transactions respectively.
-	 */
-	struct vbitmap		*vbm_client;
-	struct vbitmap		*vbm_backend;
-
-	/*
 	 * Bit map of programatically selected tags, that cannot be suppressed.
 	 * This way programs can make sure they will see certain tags, even
 	 * if the user tries to supress them with -x/-X



From geoff at varnish-cache.org  Mon Jan  9 20:52:41 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:41 +0100
Subject: [experimental-ims] 3262173 Copyright year update
Message-ID: <E1RkMCv-0008Ni-8r@project.varnish-software.com>

commit 3262173468003a8c9ee7576620251da59a480cf4
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Nov 21 08:44:06 2011 +0000

    Copyright year update

diff --git a/bin/varnishd/cache/cache_dir_dns.c b/bin/varnishd/cache/cache_dir_dns.c
index ae96dbb..f735ebf 100644
--- a/bin/varnishd/cache/cache_dir_dns.c
+++ b/bin/varnishd/cache/cache_dir_dns.c
@@ -1,5 +1,5 @@
 /*-
- * Copyright (c) 2009-2010 Varnish Software AS
+ * Copyright (c) 2009-2011 Varnish Software AS
  * All rights reserved.
  *
  * Author: Kristian Lyngstol <kristian at redpill-linpro.com>
diff --git a/bin/varnishd/cache/cache_lck.c b/bin/varnishd/cache/cache_lck.c
index 2aef6dc..d74e31a 100644
--- a/bin/varnishd/cache/cache_lck.c
+++ b/bin/varnishd/cache/cache_lck.c
@@ -1,5 +1,5 @@
 /*-
- * Copyright (c) 2008-2010 Varnish Software AS
+ * Copyright (c) 2008-2011 Varnish Software AS
  * All rights reserved.
  *
  * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
diff --git a/bin/varnishd/cache/cache_main.c b/bin/varnishd/cache/cache_main.c
index 568e5e5..1f876cd 100644
--- a/bin/varnishd/cache/cache_main.c
+++ b/bin/varnishd/cache/cache_main.c
@@ -1,6 +1,6 @@
 /*-
  * Copyright (c) 2006 Verdens Gang AS
- * Copyright (c) 2006-2009 Varnish Software AS
+ * Copyright (c) 2006-2011 Varnish Software AS
  * All rights reserved.
  *
  * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
diff --git a/bin/varnishd/cache/cache_session.c b/bin/varnishd/cache/cache_session.c
index 7befbcc..eee8e9a 100644
--- a/bin/varnishd/cache/cache_session.c
+++ b/bin/varnishd/cache/cache_session.c
@@ -1,6 +1,6 @@
 /*-
  * Copyright (c) 2006 Verdens Gang AS
- * Copyright (c) 2006-2010 Varnish Software AS
+ * Copyright (c) 2006-2011 Varnish Software AS
  * All rights reserved.
  *
  * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
diff --git a/bin/varnishd/cache/cache_vrt.c b/bin/varnishd/cache/cache_vrt.c
index c20b552..5e19ccc 100644
--- a/bin/varnishd/cache/cache_vrt.c
+++ b/bin/varnishd/cache/cache_vrt.c
@@ -1,6 +1,6 @@
 /*-
  * Copyright (c) 2006 Verdens Gang AS
- * Copyright (c) 2006-2010 Varnish Software AS
+ * Copyright (c) 2006-2011 Varnish Software AS
  * All rights reserved.
  *
  * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
diff --git a/bin/varnishd/cache/cache_vrt_re.c b/bin/varnishd/cache/cache_vrt_re.c
index 9b4a2e6..e9ac9de 100644
--- a/bin/varnishd/cache/cache_vrt_re.c
+++ b/bin/varnishd/cache/cache_vrt_re.c
@@ -1,6 +1,6 @@
 /*-
  * Copyright (c) 2006 Verdens Gang AS
- * Copyright (c) 2006-2009 Varnish Software AS
+ * Copyright (c) 2006-2011 Varnish Software AS
  * All rights reserved.
  *
  * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
diff --git a/bin/varnishd/cache/cache_ws.c b/bin/varnishd/cache/cache_ws.c
index 9fca215..01937b3 100644
--- a/bin/varnishd/cache/cache_ws.c
+++ b/bin/varnishd/cache/cache_ws.c
@@ -1,6 +1,6 @@
 /*-
  * Copyright (c) 2006 Verdens Gang AS
- * Copyright (c) 2006-2010 Varnish Software AS
+ * Copyright (c) 2006-2011 Varnish Software AS
  * All rights reserved.
  *
  * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
diff --git a/bin/varnishd/hash/hash_classic.c b/bin/varnishd/hash/hash_classic.c
index 9ea27de..b530f25 100644
--- a/bin/varnishd/hash/hash_classic.c
+++ b/bin/varnishd/hash/hash_classic.c
@@ -1,6 +1,6 @@
 /*-
  * Copyright (c) 2006 Verdens Gang AS
- * Copyright (c) 2006-2009 Varnish Software AS
+ * Copyright (c) 2006-2011 Varnish Software AS
  * All rights reserved.
  *
  * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
diff --git a/bin/varnishd/hash/hash_critbit.c b/bin/varnishd/hash/hash_critbit.c
index ecb7b62..0b3a7b1 100644
--- a/bin/varnishd/hash/hash_critbit.c
+++ b/bin/varnishd/hash/hash_critbit.c
@@ -1,5 +1,5 @@
 /*-
- * Copyright (c) 2008-2010 Varnish Software AS
+ * Copyright (c) 2008-2011 Varnish Software AS
  * All rights reserved.
  *
  * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
diff --git a/bin/varnishd/hash/hash_simple_list.c b/bin/varnishd/hash/hash_simple_list.c
index 4bdae02..c1cc9c8 100644
--- a/bin/varnishd/hash/hash_simple_list.c
+++ b/bin/varnishd/hash/hash_simple_list.c
@@ -1,6 +1,6 @@
 /*-
  * Copyright (c) 2006 Verdens Gang AS
- * Copyright (c) 2006-2009 Varnish Software AS
+ * Copyright (c) 2006-2011 Varnish Software AS
  * All rights reserved.
  *
  * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
diff --git a/bin/varnishd/hash/hash_slinger.h b/bin/varnishd/hash/hash_slinger.h
index 23b892d..9e3b13f 100644
--- a/bin/varnishd/hash/hash_slinger.h
+++ b/bin/varnishd/hash/hash_slinger.h
@@ -1,6 +1,6 @@
 /*-
  * Copyright (c) 2006 Verdens Gang AS
- * Copyright (c) 2006-2010 Varnish Software AS
+ * Copyright (c) 2006-2011 Varnish Software AS
  * All rights reserved.
  *
  * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
diff --git a/bin/varnishd/mgt/mgt_cli.h b/bin/varnishd/mgt/mgt_cli.h
index 8c49abb..5d3ce8b 100644
--- a/bin/varnishd/mgt/mgt_cli.h
+++ b/bin/varnishd/mgt/mgt_cli.h
@@ -1,6 +1,6 @@
 /*-
  * Copyright (c) 2006 Verdens Gang AS
- * Copyright (c) 2006-2009 Varnish Software AS
+ * Copyright (c) 2006-2011 Varnish Software AS
  * All rights reserved.
  *
  * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
diff --git a/bin/varnishd/storage/stevedore_utils.c b/bin/varnishd/storage/stevedore_utils.c
index 7e9c7d5..89c6027 100644
--- a/bin/varnishd/storage/stevedore_utils.c
+++ b/bin/varnishd/storage/stevedore_utils.c
@@ -1,6 +1,6 @@
 /*-
  * Copyright (c) 2006 Verdens Gang AS
- * Copyright (c) 2006-2010 Varnish Software AS
+ * Copyright (c) 2006-2011 Varnish Software AS
  * All rights reserved.
  *
  * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
diff --git a/bin/varnishd/storage/storage_file.c b/bin/varnishd/storage/storage_file.c
index 2d9ccf4..9eb44d9 100644
--- a/bin/varnishd/storage/storage_file.c
+++ b/bin/varnishd/storage/storage_file.c
@@ -1,6 +1,6 @@
 /*-
  * Copyright (c) 2006 Verdens Gang AS
- * Copyright (c) 2006-2010 Varnish Software AS
+ * Copyright (c) 2006-2011 Varnish Software AS
  * All rights reserved.
  *
  * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
diff --git a/bin/varnishd/waiter/cache_waiter.h b/bin/varnishd/waiter/cache_waiter.h
index 28bc39d..1d6e679 100644
--- a/bin/varnishd/waiter/cache_waiter.h
+++ b/bin/varnishd/waiter/cache_waiter.h
@@ -1,6 +1,6 @@
 /*-
  * Copyright (c) 2006 Verdens Gang AS
- * Copyright (c) 2006-2009 Varnish Software AS
+ * Copyright (c) 2006-2011 Varnish Software AS
  * All rights reserved.
  *
  * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
diff --git a/bin/varnishd/waiter/cache_waiter_epoll.c b/bin/varnishd/waiter/cache_waiter_epoll.c
index b4b40f5..4d45909 100644
--- a/bin/varnishd/waiter/cache_waiter_epoll.c
+++ b/bin/varnishd/waiter/cache_waiter_epoll.c
@@ -1,6 +1,6 @@
 /*-
  * Copyright (c) 2006 Verdens Gang AS
- * Copyright (c) 2006-2010 Varnish Software AS
+ * Copyright (c) 2006-2011 Varnish Software AS
  * All rights reserved.
  *
  * Author: Rogerio Carvalho Schneider <stockrt at gmail.com>
diff --git a/bin/varnishd/waiter/cache_waiter_kqueue.c b/bin/varnishd/waiter/cache_waiter_kqueue.c
index c6a03de..b331a31 100644
--- a/bin/varnishd/waiter/cache_waiter_kqueue.c
+++ b/bin/varnishd/waiter/cache_waiter_kqueue.c
@@ -1,6 +1,6 @@
 /*-
  * Copyright (c) 2006 Verdens Gang AS
- * Copyright (c) 2006-2009 Varnish Software AS
+ * Copyright (c) 2006-2011 Varnish Software AS
  * All rights reserved.
  *
  * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
diff --git a/bin/varnishd/waiter/cache_waiter_poll.c b/bin/varnishd/waiter/cache_waiter_poll.c
index 3f5cea3..b4863a8 100644
--- a/bin/varnishd/waiter/cache_waiter_poll.c
+++ b/bin/varnishd/waiter/cache_waiter_poll.c
@@ -1,6 +1,6 @@
 /*-
  * Copyright (c) 2006 Verdens Gang AS
- * Copyright (c) 2006-2010 Varnish Software AS
+ * Copyright (c) 2006-2011 Varnish Software AS
  * All rights reserved.
  *
  * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>



From geoff at varnish-cache.org  Mon Jan  9 20:52:02 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:02 +0100
Subject: [experimental-ims] d4dbc04 This test is much to brittle due to
	dependence on random timing. Tighten it up with use of
	semaphores and collapse s1/s2/s3 using "accept" keyword.
Message-ID: <E1RkMCI-0007Ee-CS@project.varnish-software.com>

commit d4dbc04e9664a72baf6d41742617df259c6b1502
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Fri Sep 30 18:24:34 2011 +0000

    This test is much to brittle due to dependence on random timing.
    Tighten it up with use of semaphores and collapse s1/s2/s3 using
    "accept" keyword.

diff --git a/bin/varnishtest/tests/v00036.vtc b/bin/varnishtest/tests/v00036.vtc
index ff2ecd3..46bcca8 100644
--- a/bin/varnishtest/tests/v00036.vtc
+++ b/bin/varnishtest/tests/v00036.vtc
@@ -4,12 +4,46 @@ server s1 {
 	rxreq
 	expect req.url == "/"
 	txresp -body "slash"
+	accept
+
+	sema r1 sync 3
+
+	rxreq
+	expect req.url == "/"
+	txresp -body "slash"
+	accept
+
+	rxreq
+	expect req.url == "/"
+	txresp -body "slash"
+
+	sema r3 sync 2
+	accept
+
+	rxreq
+	expect req.url == "/foo"
+	txresp -hdr "Foo: 1" -body "foobar"
+
 } -start
 
 server s2 {
 	rxreq
 	expect req.url == "/"
 	txresp -body "slash"
+	accept
+
+	sema r1 sync 3
+
+	rxreq
+	expect req.url == "/"
+	txresp -body "slash"
+
+	sema r2 sync 2
+	accept
+
+	rxreq
+	expect req.url == "/foo"
+	txresp -hdr "Foo: 2" -body "foobar"
 } -start
 
 server s3 {
@@ -65,66 +99,23 @@ varnish v1 -vcl {
 	}
 } -start
 
-# s1 & s2 have both had 1 probe, so both are unhealthy
 
 client c1 {
+	# s1 & s2 are both sick, expect response from s3
 	txreq -url "/foo"
 	rxresp
 	expect resp.http.foo == "3"
-} -run
-
-# setup for probe #2
-
-server s1 {
-	rxreq
-	expect req.url == "/"
-	txresp -body "slash"
-} -start
-
-server s2 {
-	rxreq
-	expect req.url == "/"
-	txresp -body "slash"
-} -start
 
-# if we muck with a running server, the test will wait until it's done,
-# which will be after probe #2 completes. b2 will then be healthy.
+	sema r1 sync 3
 
-server s2 {
-	rxreq
-	expect req.url == "/foo"
-	txresp -hdr "Foo: 2" -body "foobar"
-} -start
-
-client c1 {
+	# wait for s2 to become healthy
+	sema r2 sync 2
 	txreq -url "/foo"
 	rxresp
 	expect resp.http.foo == "2"
-} -run
 
-# setup for probe #3
-
-server s1 {
-        rxreq
-        expect req.url == "/"
-        txresp -body "slash"
-} -start
-
-server s2 {
-        rxreq
-        expect req.url == "/"
-        txresp -body "slash"
-} -start
-
-# after probe #3 b1 should be healthy.
-
-server s1 {
-	rxreq
-	expect req.url == "/foo"
-	txresp -hdr "Foo: 1" -body "foobar"
-} -start
-
-client c1 {
+	# wait for s1 to become healthy
+	sema r3 sync 2
 	txreq -url "/foo"
 	rxresp
 	expect resp.http.foo == "1"



From geoff at varnish-cache.org  Mon Jan  9 20:52:18 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:18 +0100
Subject: [experimental-ims] ebd1601 Reduce #include pollution
Message-ID: <E1RkMCY-0007gG-HQ@project.varnish-software.com>

commit ebd1601675c542b9004ae8d89b4e65b80779c668
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Oct 12 15:25:45 2011 +0000

    Reduce #include pollution

diff --git a/bin/varnishd/cache_hash.c b/bin/varnishd/cache_hash.c
index ebbe281..14c5e7c 100644
--- a/bin/varnishd/cache_hash.c
+++ b/bin/varnishd/cache_hash.c
@@ -58,7 +58,6 @@
 
 #include "cache.h"
 
-#include "cache_backend.h"
 #include "hash_slinger.h"
 #include "vav.h"
 #include "vsha256.h"
@@ -311,7 +310,7 @@ HSH_Lookup(struct sess *sp, struct objhead **poh)
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 	CHECK_OBJ_NOTNULL(sp->wrk, WORKER_MAGIC);
 	CHECK_OBJ_NOTNULL(sp->http, HTTP_MAGIC);
-	CHECK_OBJ_NOTNULL(sp->director, DIRECTOR_MAGIC);
+	AN(sp->director);
 	AN(hash);
 	w = sp->wrk;
 



From geoff at varnish-cache.org  Mon Jan  9 20:52:15 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:15 +0100
Subject: [experimental-ims] 23baabd Get closer to distcheck working
Message-ID: <E1RkMCV-0007Yi-5V@project.varnish-software.com>

commit 23baabde796dea577d09fa22c82e497f1af6624b
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sat Oct 8 16:23:44 2011 +0000

    Get closer to distcheck working

diff --git a/.gitignore b/.gitignore
index f35e990..eb2db8c 100644
--- a/.gitignore
+++ b/.gitignore
@@ -43,9 +43,9 @@ TAGS
 
 # Various auto-generated code snippets
 /include/vcl.h
-/include/vcl_returns.h
+/include/tbl/vcl_returns.h
 /include/vrt_obj.h
-/include/vrt_stv_var.h
+/include/tbl/vrt_stv_var.h
 /include/vcs_version.h
 /lib/libvcl/vcc_fixed_token.c
 /lib/libvcl/vcc_obj.c
diff --git a/bin/flint.lnt b/bin/flint.lnt
index d2f7c0e..4ad63d1 100644
--- a/bin/flint.lnt
+++ b/bin/flint.lnt
@@ -9,8 +9,7 @@
 +libh ../../config.h
 -efile(451, ../../config.h)	// No include guard
 
--efile(451, vsl_tags.h)		// No include guard
--efile(451, vsc_fields.h)	// No include guard
+-efile(451, "tbl/*.h")		// No include guard
 
 ///////////////////////////////////////////////////////////////////////
 // assert() support, common to libvarnish and libvarnishapi
diff --git a/include/Makefile.am b/include/Makefile.am
index 652689b..9f317f4 100644
--- a/include/Makefile.am
+++ b/include/Makefile.am
@@ -16,7 +16,6 @@ pkginclude_HEADERS = \
 	vsm.h
 
 nobase_noinst_HEADERS = \
-	tbl/ban_vars.h \
 	binary_heap.h \
 	cli_common.h \
 	cli_priv.h \
@@ -25,36 +24,37 @@ nobase_noinst_HEADERS = \
 	compat/execinfo.h \
 	compat/srandomdev.h \
 	flopen.h \
-	tbl/http_headers.h \
-	tbl/http_response.h \
 	libvarnish.h \
 	libvcl.h \
 	miniobj.h \
 	persistent.h \
+	tbl/ban_vars.h \
+	tbl/http_headers.h \
+	tbl/http_response.h \
+	tbl/vcl_returns.h \
+	tbl/vrt_stv_var.h \
 	vas.h \
 	vav.h \
-	vsha256.h \
-	vqueue.h \
-	vpf.h \
-	vsb.h \
+	vbm.h \
 	vcl.h \
-	vcl_returns.h \
 	vcs_version.h \
 	vct.h \
 	vend.h \
 	vev.h \
 	vin.h \
 	vlu.h \
-	vbm.h \
 	vmb.h \
 	vmod_abi.h \
+	vpf.h \
+	vqueue.h \
 	vre.h \
 	vrt.h \
 	vrt_obj.h \
-	vrt_stv_var.h \
+	vsb.h \
+	vsha256.h \
 	vss.h
 
-vrt_stv_var.h vcl_returns.h vcl.h vrt_obj.h: $(top_srcdir)/lib/libvcl/generate.py $(top_srcdir)/include/vrt.h
+tbl/vrt_stv_var.h tbl/vcl_returns.h vcl.h vrt_obj.h: $(top_srcdir)/lib/libvcl/generate.py $(top_srcdir)/include/vrt.h
 	@PYTHON@ $(top_srcdir)/lib/libvcl/generate.py $(top_srcdir) $(top_builddir)
 
 BUILT_SOURCES = vcs_version.h vmod_abi.h
diff --git a/man/Makefile.am b/man/Makefile.am
index 9317896..9118c34 100644
--- a/man/Makefile.am
+++ b/man/Makefile.am
@@ -4,7 +4,7 @@ if HAVE_RST2MAN
 noinst_PROGRAMS = vsc2rst
 endif
 vsc2rst_SOURCES = vsc2rst.c \
-	$(top_srcdir)/include/vsc_fields.h
+	$(top_srcdir)/include/tbl/vsc_fields.h
 
 INCLUDES = -I$(top_srcdir)/include
 
@@ -36,7 +36,7 @@ if HAVE_RST2MAN
 varnish-counters.7: vsc2rst
 	./vsc2rst | ${RST2MAN} - $@
 else
-varnish-counters.7: $(top_srcdir)/include/vsc_fields.h
+varnish-counters.7: $(top_srcdir)/include/tbl/vsc_fields.h
 	@echo "========================================"
 	@echo "You need rst2man installed to make dist"
 	@echo "========================================"



From geoff at varnish-cache.org  Mon Jan  9 20:51:57 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:51:57 +0100
Subject: [experimental-ims] 319d601 More OSX compat hackery, this time for LLVM
Message-ID: <E1RkMCD-000730-Ga@project.varnish-software.com>

commit 319d60136f8bfbe52bda54b2a1c0e0c97a048a35
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Sep 19 20:38:39 2011 +0000

    More OSX compat hackery, this time for LLVM

diff --git a/bin/varnishd/cache_pool.c b/bin/varnishd/cache_pool.c
index 7b79873..4dbf8bd 100644
--- a/bin/varnishd/cache_pool.c
+++ b/bin/varnishd/cache_pool.c
@@ -71,6 +71,7 @@ clock_gettime(int foo, struct timespec *ts)
 {
 	struct timeval tv;
 
+	(void)foo;
 	gettimeofday(&tv, NULL);
 	ts->tv_sec = tv.tv_sec;
 	ts->tv_nsec = tv.tv_usec * 1000;



From geoff at varnish-cache.org  Mon Jan  9 20:51:59 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:51:59 +0100
Subject: [experimental-ims] d4455ed Correctly handle listen socket shutdown:
	Delete the poolsock instance, pretend nothing else happened.
Message-ID: <E1RkMCF-00076L-7m@project.varnish-software.com>

commit d4455ed4774c7433a9c70453bd54202daf299992
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Sep 20 12:10:42 2011 +0000

    Correctly handle listen socket shutdown:  Delete the poolsock instance,
    pretend nothing else happened.

diff --git a/bin/varnishd/cache_acceptor.c b/bin/varnishd/cache_acceptor.c
index 0808010..1d73a1a 100644
--- a/bin/varnishd/cache_acceptor.c
+++ b/bin/varnishd/cache_acceptor.c
@@ -212,7 +212,6 @@ VCA_Accept(struct listen_sock *ls, struct wrk_accept *wa)
 	int i;
 
 	CHECK_OBJ_NOTNULL(ls, LISTEN_SOCK_MAGIC);
-	assert(ls->sock >= 0);
 	vca_pace_check();
 
 	while(!hack_ready)
diff --git a/bin/varnishd/cache_pool.c b/bin/varnishd/cache_pool.c
index 4dbf8bd..5ff635b 100644
--- a/bin/varnishd/cache_pool.c
+++ b/bin/varnishd/cache_pool.c
@@ -137,7 +137,7 @@ static pthread_t		thr_pool_herder;
  * for a brief moment and it takes up around 144 bytes.
  */
 
-static void
+static int
 pool_accept(struct pool *pp, struct worker *w, const struct poolsock *ps)
 {
 	struct worker *w2;
@@ -156,6 +156,11 @@ pool_accept(struct pool *pp, struct worker *w, const struct poolsock *ps)
 		memset(wa, 0, sizeof *wa);
 		wa->magic = WRK_ACCEPT_MAGIC;
 
+		if (ps->lsock->sock < 0) {
+			/* Socket Shutdown */
+			Lck_Lock(&pp->mtx);
+			return (-1);
+		}
 		if (VCA_Accept(ps->lsock, wa) < 0) {
 			w->stats.sess_fail++;
 			/* We're going to pace in vca anyway... */
@@ -165,7 +170,7 @@ pool_accept(struct pool *pp, struct worker *w, const struct poolsock *ps)
 
 		Lck_Lock(&pp->mtx);
 		if (VTAILQ_EMPTY(&pp->idle))
-			return;
+			return (0);
 		w2 = VTAILQ_FIRST(&pp->idle);
 		VTAILQ_REMOVE(&pp->idle, w2, list);
 		Lck_Unlock(&pp->mtx);
@@ -184,7 +189,7 @@ void
 Pool_Work_Thread(void *priv, struct worker *w)
 {
 	struct pool *pp;
-	int stats_clean;
+	int stats_clean, i;
 	struct poolsock *ps;
 
 	CAST_OBJ_NOTNULL(pp, priv, POOL_MAGIC);
@@ -212,8 +217,13 @@ Pool_Work_Thread(void *priv, struct worker *w)
 			/* Accept on a socket */
 			ps = VTAILQ_FIRST(&pp->socks);
 			VTAILQ_REMOVE(&pp->socks, ps, list);
-			pool_accept(pp, w, ps);
+			i = pool_accept(pp, w, ps);
 			Lck_AssertHeld(&pp->mtx);
+			if (i < 0) {
+				/* Socket Shutdown */
+				FREE_OBJ(ps);
+				continue;
+			}
 			VTAILQ_INSERT_TAIL(&pp->socks, ps, list);
 		} else if (VTAILQ_EMPTY(&pp->socks)) {
 			/* Nothing to do: To sleep, perchance to dream ... */



From geoff at varnish-cache.org  Mon Jan  9 20:52:18 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:18 +0100
Subject: [experimental-ims] 18edda6 Move the hashing implementations into a
	subdirectory where VMODs won't see them.
Message-ID: <E1RkMCY-0007gw-OR@project.varnish-software.com>

commit 18edda6b2925d86115bd5e2488190f0f91a39d41
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Oct 12 17:08:56 2011 +0000

    Move the hashing implementations into a subdirectory where VMODs
    won't see them.

diff --git a/bin/varnishd/Makefile.am b/bin/varnishd/Makefile.am
index cdec084..499cb6c 100644
--- a/bin/varnishd/Makefile.am
+++ b/bin/varnishd/Makefile.am
@@ -52,9 +52,9 @@ varnishd_SOURCES = \
 	cache_wrk.c \
 	cache_wrw.c \
 	cache_ws.c \
-	hash_classic.c \
-	hash_critbit.c \
-	hash_simple_list.c \
+	hash/hash_classic.c \
+	hash/hash_critbit.c \
+	hash/hash_simple_list.c \
 	mgt_child.c \
 	mgt_cli.c \
 	mgt_param.c \
@@ -84,7 +84,7 @@ noinst_HEADERS = \
 	waiter/cache_waiter.h \
 	common.h \
 	default_vcl.h \
-	hash_slinger.h \
+	hash/hash_slinger.h \
 	heritage.h \
 	mgt.h \
 	mgt_cli.h \
diff --git a/bin/varnishd/cache_ban.c b/bin/varnishd/cache_ban.c
index d556ace..736cbf7 100644
--- a/bin/varnishd/cache_ban.c
+++ b/bin/varnishd/cache_ban.c
@@ -67,7 +67,7 @@
 
 #include "cache.h"
 
-#include "hash_slinger.h"
+#include "hash/hash_slinger.h"
 #include "vcli.h"
 #include "vcli_priv.h"
 #include "vend.h"
diff --git a/bin/varnishd/cache_center.c b/bin/varnishd/cache_center.c
index 1eb249e..0a81925 100644
--- a/bin/varnishd/cache_center.c
+++ b/bin/varnishd/cache_center.c
@@ -64,7 +64,7 @@ DOT acceptor -> start [style=bold,color=green]
 
 #include "cache.h"
 
-#include "hash_slinger.h"
+#include "hash/hash_slinger.h"
 #include "vcl.h"
 #include "vcli_priv.h"
 #include "vsha256.h"
diff --git a/bin/varnishd/cache_cli.c b/bin/varnishd/cache_cli.c
index 01da3f2..30a12e3 100644
--- a/bin/varnishd/cache_cli.c
+++ b/bin/varnishd/cache_cli.c
@@ -41,7 +41,7 @@
 #include "cache.h"
 
 #include "cache_backend.h"		// struct vbc
-#include "hash_slinger.h"		// objhead
+#include "hash/hash_slinger.h"		// struct objhead
 #include "vcli.h"
 #include "vcli_common.h"
 #include "vcli_priv.h"
diff --git a/bin/varnishd/cache_expire.c b/bin/varnishd/cache_expire.c
index f9dcdd1..601025c 100644
--- a/bin/varnishd/cache_expire.c
+++ b/bin/varnishd/cache_expire.c
@@ -56,7 +56,7 @@
 #include "cache.h"
 
 #include "binary_heap.h"
-#include "hash_slinger.h"
+#include "hash/hash_slinger.h"
 #include "vtim.h"
 
 static pthread_t exp_thread;
diff --git a/bin/varnishd/cache_hash.c b/bin/varnishd/cache_hash.c
index 14c5e7c..ecce40d 100644
--- a/bin/varnishd/cache_hash.c
+++ b/bin/varnishd/cache_hash.c
@@ -58,7 +58,7 @@
 
 #include "cache.h"
 
-#include "hash_slinger.h"
+#include "hash/hash_slinger.h"
 #include "vav.h"
 #include "vsha256.h"
 
diff --git a/bin/varnishd/cache_main.c b/bin/varnishd/cache_main.c
index 2d00ee2..8293d38 100644
--- a/bin/varnishd/cache_main.c
+++ b/bin/varnishd/cache_main.c
@@ -35,7 +35,7 @@
 #include "cache.h"
 
 #include "waiter/cache_waiter.h"
-#include "hash_slinger.h"
+#include "hash/hash_slinger.h"
 
 /*--------------------------------------------------------------------
  * Per thread storage for the session currently being processed by
diff --git a/bin/varnishd/cache_pool.c b/bin/varnishd/cache_pool.c
index 55ad333..141a0c7 100644
--- a/bin/varnishd/cache_pool.c
+++ b/bin/varnishd/cache_pool.c
@@ -48,7 +48,6 @@
 #include "cache.h"
 
 #include "waiter/cache_waiter.h"
-#include "hash_slinger.h"
 #include "vtcp.h"
 #include "vtim.h"
 
@@ -262,9 +261,7 @@ Pool_Work_Thread(void *priv, struct worker *w)
 			AZ(w->sp->wrk);
 			THR_SetSession(w->sp);
 			w->sp->wrk = w;
-			CHECK_OBJ_ORNULL(w->nobjhead, OBJHEAD_MAGIC);
 			CNT_Session(w->sp);
-			CHECK_OBJ_ORNULL(w->nobjhead, OBJHEAD_MAGIC);
 			THR_SetSession(NULL);
 			w->sp = NULL;
 
diff --git a/bin/varnishd/cache_vrt.c b/bin/varnishd/cache_vrt.c
index 2302b86..16258ec 100644
--- a/bin/varnishd/cache_vrt.c
+++ b/bin/varnishd/cache_vrt.c
@@ -40,7 +40,7 @@
 #include "cache.h"
 
 #include "cache_backend.h"
-#include "hash_slinger.h"
+#include "hash/hash_slinger.h"
 #include "vav.h"
 #include "vcl.h"
 #include "vrt.h"
diff --git a/bin/varnishd/cache_wrk.c b/bin/varnishd/cache_wrk.c
index 1032691..e416eb9 100644
--- a/bin/varnishd/cache_wrk.c
+++ b/bin/varnishd/cache_wrk.c
@@ -36,7 +36,7 @@
 
 #include "cache.h"
 
-#include "hash_slinger.h"
+#include "hash/hash_slinger.h"
 #include "vsha256.h"
 
 static struct lock		wstat_mtx;
diff --git a/bin/varnishd/hash/hash_classic.c b/bin/varnishd/hash/hash_classic.c
new file mode 100644
index 0000000..6b1d589
--- /dev/null
+++ b/bin/varnishd/hash/hash_classic.c
@@ -0,0 +1,184 @@
+/*-
+ * Copyright (c) 2006 Verdens Gang AS
+ * Copyright (c) 2006-2009 Varnish Software AS
+ * All rights reserved.
+ *
+ * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * A classic bucketed hash
+ */
+
+#include "config.h"
+
+#include <stdio.h>
+#include <stdlib.h>
+
+#include "cache.h"
+
+#include "hash/hash_slinger.h"
+
+/*--------------------------------------------------------------------*/
+
+struct hcl_hd {
+	unsigned		magic;
+#define HCL_HEAD_MAGIC		0x0f327016
+	VTAILQ_HEAD(, objhead)	head;
+	struct lock		mtx;
+};
+
+static unsigned			hcl_nhash = 16383;
+static struct hcl_hd		*hcl_head;
+
+/*--------------------------------------------------------------------
+ * The ->init method allows the management process to pass arguments
+ */
+
+static void
+hcl_init(int ac, char * const *av)
+{
+	int i;
+	unsigned u;
+
+	if (ac == 0)
+		return;
+	if (ac > 1)
+		ARGV_ERR("(-hclassic) too many arguments\n");
+	i = sscanf(av[0], "%u", &u);
+	if (i <= 0 || u == 0)
+		return;
+	if (u > 2 && !(u & (u - 1))) {
+		fprintf(stderr,
+		    "NOTE:\n"
+		    "\tA power of two number of hash buckets is "
+		    "marginally less efficient\n"
+		    "\twith systematic URLs.  Reducing by one"
+		    " hash bucket.\n");
+		u--;
+	}
+	hcl_nhash = u;
+	fprintf(stderr, "Classic hash: %u buckets\n", hcl_nhash);
+	return;
+}
+
+/*--------------------------------------------------------------------
+ * The ->start method is called during cache process start and allows
+ * initialization to happen before the first lookup.
+ */
+
+static void
+hcl_start(void)
+{
+	unsigned u;
+
+	hcl_head = calloc(sizeof *hcl_head, hcl_nhash);
+	XXXAN(hcl_head);
+
+	for (u = 0; u < hcl_nhash; u++) {
+		VTAILQ_INIT(&hcl_head[u].head);
+		Lck_New(&hcl_head[u].mtx, lck_hcl);
+		hcl_head[u].magic = HCL_HEAD_MAGIC;
+	}
+}
+
+/*--------------------------------------------------------------------
+ * Lookup and possibly insert element.
+ * If nobj != NULL and the lookup does not find key, nobj is inserted.
+ * If nobj == NULL and the lookup does not find key, NULL is returned.
+ * A reference to the returned object is held.
+ * We use a two-pass algorithm to handle inserts as they are quite
+ * rare and collisions even rarer.
+ */
+
+static struct objhead *
+hcl_lookup(const struct sess *sp, struct objhead *noh)
+{
+	struct objhead *oh;
+	struct hcl_hd *hp;
+	unsigned u1, digest;
+	int i;
+
+	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+	CHECK_OBJ_NOTNULL(noh, OBJHEAD_MAGIC);
+
+	assert(sizeof noh->digest > sizeof digest);
+	memcpy(&digest, noh->digest, sizeof digest);
+	u1 = digest % hcl_nhash;
+	hp = &hcl_head[u1];
+
+	Lck_Lock(&hp->mtx);
+	VTAILQ_FOREACH(oh, &hp->head, hoh_list) {
+		i = memcmp(oh->digest, noh->digest, sizeof oh->digest);
+		if (i < 0)
+			continue;
+		if (i > 0)
+			break;
+		oh->refcnt++;
+		Lck_Unlock(&hp->mtx);
+		return (oh);
+	}
+
+	if (oh != NULL)
+		VTAILQ_INSERT_BEFORE(oh, noh, hoh_list);
+	else
+		VTAILQ_INSERT_TAIL(&hp->head, noh, hoh_list);
+
+	noh->hoh_head = hp;
+
+	Lck_Unlock(&hp->mtx);
+	return (noh);
+}
+
+/*--------------------------------------------------------------------
+ * Dereference and if no references are left, free.
+ */
+
+static int
+hcl_deref(struct objhead *oh)
+{
+	struct hcl_hd *hp;
+	int ret;
+
+	CHECK_OBJ_NOTNULL(oh, OBJHEAD_MAGIC);
+	CAST_OBJ_NOTNULL(hp, oh->hoh_head, HCL_HEAD_MAGIC);
+	assert(oh->refcnt > 0);
+	Lck_Lock(&hp->mtx);
+	if (--oh->refcnt == 0) {
+		VTAILQ_REMOVE(&hp->head, oh, hoh_list);
+		ret = 0;
+	} else
+		ret = 1;
+	Lck_Unlock(&hp->mtx);
+	return (ret);
+}
+
+/*--------------------------------------------------------------------*/
+
+const struct hash_slinger hcl_slinger = {
+	.magic	=	SLINGER_MAGIC,
+	.name	=	"classic",
+	.init	=	hcl_init,
+	.start	=	hcl_start,
+	.lookup =	hcl_lookup,
+	.deref	=	hcl_deref,
+};
diff --git a/bin/varnishd/hash/hash_critbit.c b/bin/varnishd/hash/hash_critbit.c
new file mode 100644
index 0000000..56cc5c0
--- /dev/null
+++ b/bin/varnishd/hash/hash_critbit.c
@@ -0,0 +1,492 @@
+/*-
+ * Copyright (c) 2008-2010 Varnish Software AS
+ * All rights reserved.
+ *
+ * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * A Crit Bit tree based hash
+ */
+
+// #define PHK
+
+#include "config.h"
+
+#include <stdlib.h>
+
+#include "cache.h"
+
+#include "hash/hash_slinger.h"
+#include "vcli_priv.h"
+#include "vmb.h"
+#include "vtim.h"
+
+static struct lock hcb_mtx;
+
+/*---------------------------------------------------------------------
+ * Table for finding out how many bits two bytes have in common,
+ * counting from the MSB towards the LSB.
+ * ie:
+ *	hcb_bittbl[0x01 ^ 0x22] == 2
+ *	hcb_bittbl[0x10 ^ 0x0b] == 3
+ *
+ */
+
+static unsigned char hcb_bittbl[256];
+
+static unsigned char
+hcb_bits(unsigned char x, unsigned char y)
+{
+	return hcb_bittbl[x ^ y];
+}
+
+static void
+hcb_build_bittbl(void)
+{
+	unsigned char x;
+	unsigned y;
+
+	y = 0;
+	for (x = 0; x < 8; x++)
+		for (; y < (1U << x); y++)
+			hcb_bittbl[y] = 8 - x;
+
+	/* Quick asserts for sanity check */
+	assert(hcb_bits(0x34, 0x34) == 8);
+	assert(hcb_bits(0xaa, 0x55) == 0);
+	assert(hcb_bits(0x01, 0x22) == 2);
+	assert(hcb_bits(0x10, 0x0b) == 3);
+}
+
+/*---------------------------------------------------------------------
+ * For space reasons we overload the two pointers with two different
+ * kinds of of pointers.  We cast them to uintptr_t's and abuse the
+ * low two bits to tell them apart, assuming that Varnish will never
+ * run on machines with less than 32bit alignment.
+ *
+ * Asserts will explode if these assumptions are not met.
+ */
+
+struct hcb_y {
+	unsigned		magic;
+#define HCB_Y_MAGIC		0x125c4bd2
+	unsigned short		critbit;
+	unsigned char		ptr;
+	unsigned char		bitmask;
+	volatile uintptr_t	leaf[2];
+	VSTAILQ_ENTRY(hcb_y)	list;
+};
+
+#define HCB_BIT_NODE		(1<<0)
+#define HCB_BIT_Y		(1<<1)
+
+struct hcb_root {
+	volatile uintptr_t	origo;
+};
+
+static struct hcb_root	hcb_root;
+
+static VSTAILQ_HEAD(, hcb_y)	cool_y = VSTAILQ_HEAD_INITIALIZER(cool_y);
+static VSTAILQ_HEAD(, hcb_y)	dead_y = VSTAILQ_HEAD_INITIALIZER(dead_y);
+static VTAILQ_HEAD(, objhead)	cool_h = VTAILQ_HEAD_INITIALIZER(cool_h);
+static VTAILQ_HEAD(, objhead)	dead_h = VTAILQ_HEAD_INITIALIZER(dead_h);
+
+/*---------------------------------------------------------------------
+ * Pointer accessor functions
+ */
+static int
+hcb_is_node(uintptr_t u)
+{
+
+	return (u & HCB_BIT_NODE);
+}
+
+static int
+hcb_is_y(uintptr_t u)
+{
+
+	return (u & HCB_BIT_Y);
+}
+
+static uintptr_t
+hcb_r_node(struct objhead *n)
+{
+
+	assert(!((uintptr_t)n & (HCB_BIT_NODE|HCB_BIT_Y)));
+	return (HCB_BIT_NODE | (uintptr_t)n);
+}
+
+static struct objhead *
+hcb_l_node(uintptr_t u)
+{
+
+	assert(u & HCB_BIT_NODE);
+	assert(!(u & HCB_BIT_Y));
+	return ((struct objhead *)(u & ~HCB_BIT_NODE));
+}
+
+static uintptr_t
+hcb_r_y(struct hcb_y *y)
+{
+
+	CHECK_OBJ_NOTNULL(y, HCB_Y_MAGIC);
+	assert(!((uintptr_t)y & (HCB_BIT_NODE|HCB_BIT_Y)));
+	return (HCB_BIT_Y | (uintptr_t)y);
+}
+
+static struct hcb_y *
+hcb_l_y(uintptr_t u)
+{
+
+	assert(!(u & HCB_BIT_NODE));
+	assert(u & HCB_BIT_Y);
+	return ((struct hcb_y *)(u & ~HCB_BIT_Y));
+}
+
+/*---------------------------------------------------------------------
+ * Find the "critical" bit that separates these two digests
+ */
+
+static unsigned
+hcb_crit_bit(const struct objhead *oh1, const struct objhead *oh2,
+    struct hcb_y *y)
+{
+	unsigned char u, r;
+
+	CHECK_OBJ_NOTNULL(y, HCB_Y_MAGIC);
+	for (u = 0; u < DIGEST_LEN && oh1->digest[u] == oh2->digest[u]; u++)
+		;
+	assert(u < DIGEST_LEN);
+	r = hcb_bits(oh1->digest[u], oh2->digest[u]);
+	y->ptr = u;
+	y->bitmask = 0x80 >> r;
+	y->critbit = u * 8 + r;
+	return (y->critbit);
+}
+
+/*---------------------------------------------------------------------
+ * Unless we have the lock, we need to be very careful about pointer
+ * references into the tree, we cannot trust things to be the same
+ * in two consequtive memory accesses.
+ */
+
+static struct objhead *
+hcb_insert(struct worker *wrk, struct hcb_root *root, struct objhead *oh, int has_lock)
+{
+	volatile uintptr_t *p;
+	uintptr_t pp;
+	struct hcb_y *y, *y2;
+	struct objhead *oh2;
+	unsigned s, s2;
+
+	p = &root->origo;
+	pp = *p;
+	if (pp == 0) {
+		if (!has_lock)
+			return (NULL);
+		*p = hcb_r_node(oh);
+		return (oh);
+	}
+
+	while(hcb_is_y(pp)) {
+		y = hcb_l_y(pp);
+		CHECK_OBJ_NOTNULL(y, HCB_Y_MAGIC);
+		assert(y->ptr < DIGEST_LEN);
+		s = (oh->digest[y->ptr] & y->bitmask) != 0;
+		assert(s < 2);
+		p = &y->leaf[s];
+		pp = *p;
+	}
+
+	if (pp == 0) {
+		/* We raced hcb_delete and got a NULL pointer */
+		assert(!has_lock);
+		return (NULL);
+	}
+
+	assert(hcb_is_node(pp));
+
+	/* We found a node, does it match ? */
+	oh2 = hcb_l_node(pp);
+	CHECK_OBJ_NOTNULL(oh2, OBJHEAD_MAGIC);
+	if (!memcmp(oh2->digest, oh->digest, DIGEST_LEN))
+		return (oh2);
+
+	if (!has_lock)
+		return (NULL);
+
+	/* Insert */
+
+	CAST_OBJ_NOTNULL(y2, wrk->nhashpriv, HCB_Y_MAGIC);
+	wrk->nhashpriv = NULL;
+	(void)hcb_crit_bit(oh, oh2, y2);
+	s2 = (oh->digest[y2->ptr] & y2->bitmask) != 0;
+	assert(s2 < 2);
+	y2->leaf[s2] = hcb_r_node(oh);
+	s2 = 1-s2;
+
+	p = &root->origo;
+	assert(*p != 0);
+
+	while(hcb_is_y(*p)) {
+		y = hcb_l_y(*p);
+		CHECK_OBJ_NOTNULL(y, HCB_Y_MAGIC);
+		assert(y->critbit != y2->critbit);
+		if (y->critbit > y2->critbit)
+			break;
+		assert(y->ptr < DIGEST_LEN);
+		s = (oh->digest[y->ptr] & y->bitmask) != 0;
+		assert(s < 2);
+		p = &y->leaf[s];
+	}
+	y2->leaf[s2] = *p;
+	VWMB();
+	*p = hcb_r_y(y2);
+	return(oh);
+}
+
+/*--------------------------------------------------------------------*/
+
+static void
+hcb_delete(struct hcb_root *r, struct objhead *oh)
+{
+	struct hcb_y *y;
+	volatile uintptr_t *p;
+	unsigned s;
+
+	if (r->origo == hcb_r_node(oh)) {
+		r->origo = 0;
+		return;
+	}
+	p = &r->origo;
+	assert(hcb_is_y(*p));
+
+	y = NULL;
+	while(1) {
+		assert(hcb_is_y(*p));
+		y = hcb_l_y(*p);
+		assert(y->ptr < DIGEST_LEN);
+		s = (oh->digest[y->ptr] & y->bitmask) != 0;
+		assert(s < 2);
+		if (y->leaf[s] == hcb_r_node(oh)) {
+			*p = y->leaf[1 - s];
+			VSTAILQ_INSERT_TAIL(&cool_y, y, list);
+			return;
+		}
+		p = &y->leaf[s];
+	}
+}
+
+/*--------------------------------------------------------------------*/
+
+static void
+dumptree(struct cli *cli, uintptr_t p, int indent)
+{
+	int i;
+	const struct objhead *oh;
+	const struct hcb_y *y;
+
+	if (p == 0)
+		return;
+	if (hcb_is_node(p)) {
+		oh = hcb_l_node(p);
+		VCLI_Out(cli, "%*.*sN %d r%u <%02x%02x%02x...>\n",
+		    indent, indent, "", indent / 2, oh->refcnt,
+		    oh->digest[0], oh->digest[1], oh->digest[2]);
+		return;
+	}
+	assert(hcb_is_y(p));
+	y = hcb_l_y(p);
+	VCLI_Out(cli, "%*.*sY c %u p %u b %02x i %d\n",
+	    indent, indent, "",
+	    y->critbit, y->ptr, y->bitmask, indent / 2);
+	indent += 2;
+	for (i = 0; i < 2; i++)
+		dumptree(cli, y->leaf[i], indent);
+}
+
+static void
+hcb_dump(struct cli *cli, const char * const *av, void *priv)
+{
+
+	(void)priv;
+	(void)av;
+	VCLI_Out(cli, "HCB dump:\n");
+	dumptree(cli, hcb_root.origo, 0);
+	VCLI_Out(cli, "Coollist:\n");
+}
+
+static struct cli_proto hcb_cmds[] = {
+	{ "hcb.dump", "hcb.dump", "dump HCB tree\n", 0, 0, "d", hcb_dump },
+	{ NULL }
+};
+
+/*--------------------------------------------------------------------*/
+
+static void *
+hcb_cleaner(void *priv)
+{
+	struct hcb_y *y, *y2;
+	struct worker ww;
+	struct objhead *oh, *oh2;
+
+	memset(&ww, 0, sizeof ww);
+	ww.magic = WORKER_MAGIC;
+
+	THR_SetName("hcb_cleaner");
+	(void)priv;
+	while (1) {
+		VSTAILQ_FOREACH_SAFE(y, &dead_y, list, y2) {
+			VSTAILQ_REMOVE_HEAD(&dead_y, list);
+			FREE_OBJ(y);
+		}
+		VTAILQ_FOREACH_SAFE(oh, &dead_h, hoh_list, oh2) {
+			VTAILQ_REMOVE(&dead_h, oh, hoh_list);
+			HSH_DeleteObjHead(&ww, oh);
+		}
+		Lck_Lock(&hcb_mtx);
+		VSTAILQ_CONCAT(&dead_y, &cool_y);
+		VTAILQ_CONCAT(&dead_h, &cool_h, hoh_list);
+		Lck_Unlock(&hcb_mtx);
+		WRK_SumStat(&ww);
+		VTIM_sleep(params->critbit_cooloff);
+	}
+	NEEDLESS_RETURN(NULL);
+}
+
+/*--------------------------------------------------------------------*/
+
+static void
+hcb_start(void)
+{
+	struct objhead *oh = NULL;
+	pthread_t tp;
+
+	(void)oh;
+	CLI_AddFuncs(hcb_cmds);
+	Lck_New(&hcb_mtx, lck_hcb);
+	AZ(pthread_create(&tp, NULL, hcb_cleaner, NULL));
+	memset(&hcb_root, 0, sizeof hcb_root);
+	hcb_build_bittbl();
+}
+
+static int
+hcb_deref(struct objhead *oh)
+{
+	int r;
+
+	r = 1;
+	CHECK_OBJ_NOTNULL(oh, OBJHEAD_MAGIC);
+	Lck_Lock(&oh->mtx);
+	assert(oh->refcnt > 0);
+	oh->refcnt--;
+	if (oh->refcnt == 0) {
+		Lck_Lock(&hcb_mtx);
+		hcb_delete(&hcb_root, oh);
+		VTAILQ_INSERT_TAIL(&cool_h, oh, hoh_list);
+		Lck_Unlock(&hcb_mtx);
+		assert(VTAILQ_EMPTY(&oh->objcs));
+		AZ(oh->waitinglist);
+	}
+	Lck_Unlock(&oh->mtx);
+#ifdef PHK
+	fprintf(stderr, "hcb_defef %d %d <%s>\n", __LINE__, r, oh->hash);
+#endif
+	return (r);
+}
+
+static struct objhead *
+hcb_lookup(const struct sess *sp, struct objhead *noh)
+{
+	struct objhead *oh;
+	struct hcb_y *y;
+	unsigned u;
+	unsigned with_lock;
+
+	(void)sp;
+
+	with_lock = 0;
+	while (1) {
+		if (with_lock) {
+			CAST_OBJ_NOTNULL(y, sp->wrk->nhashpriv, HCB_Y_MAGIC);
+			Lck_Lock(&hcb_mtx);
+			VSC_C_main->hcb_lock++;
+			assert(noh->refcnt == 1);
+			oh = hcb_insert(sp->wrk, &hcb_root, noh, 1);
+			Lck_Unlock(&hcb_mtx);
+		} else {
+			VSC_C_main->hcb_nolock++;
+			oh = hcb_insert(sp->wrk, &hcb_root, noh, 0);
+		}
+
+		if (oh != NULL && oh == noh) {
+			/* Assert that we only muck with the tree with a lock */
+			assert(with_lock);
+			VSC_C_main->hcb_insert++;
+			assert(oh->refcnt > 0);
+			return (oh);
+		}
+
+		if (oh == NULL) {
+			assert(!with_lock);
+			/* Try again, with lock */
+			with_lock = 1;
+			continue;
+		}
+
+		CHECK_OBJ_NOTNULL(noh, OBJHEAD_MAGIC);
+		Lck_Lock(&oh->mtx);
+		/*
+		 * A refcount of zero indicates that the tree changed
+		 * under us, so fall through and try with the lock held.
+		 */
+		u = oh->refcnt;
+		if (u > 0)
+			oh->refcnt++;
+		else
+			with_lock = 1;
+		Lck_Unlock(&oh->mtx);
+		if (u > 0)
+			return (oh);
+	}
+}
+
+static void
+hcb_prep(const struct sess *sp)
+{
+	struct hcb_y *y;
+
+	if (sp->wrk->nhashpriv == NULL) {
+		ALLOC_OBJ(y, HCB_Y_MAGIC);
+		sp->wrk->nhashpriv = y;
+	}
+}
+
+const struct hash_slinger hcb_slinger = {
+	.magic  =	SLINGER_MAGIC,
+	.name   =	"critbit",
+	.start  =	hcb_start,
+	.lookup =	hcb_lookup,
+	.prep =		hcb_prep,
+	.deref  =	hcb_deref,
+};
diff --git a/bin/varnishd/hash/hash_simple_list.c b/bin/varnishd/hash/hash_simple_list.c
new file mode 100644
index 0000000..1a0cae5
--- /dev/null
+++ b/bin/varnishd/hash/hash_simple_list.c
@@ -0,0 +1,118 @@
+/*-
+ * Copyright (c) 2006 Verdens Gang AS
+ * Copyright (c) 2006-2009 Varnish Software AS
+ * All rights reserved.
+ *
+ * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * This is the reference hash(/lookup) implementation
+ */
+
+#include "config.h"
+
+#include "cache.h"
+
+#include "hash/hash_slinger.h"
+
+/*--------------------------------------------------------------------*/
+
+static VTAILQ_HEAD(, objhead)	hsl_head = VTAILQ_HEAD_INITIALIZER(hsl_head);
+static struct lock hsl_mtx;
+
+/*--------------------------------------------------------------------
+ * The ->init method is called during process start and allows
+ * initialization to happen before the first lookup.
+ */
+
+static void
+hsl_start(void)
+{
+
+	Lck_New(&hsl_mtx, lck_hsl);
+}
+
+/*--------------------------------------------------------------------
+ * Lookup and possibly insert element.
+ * If nobj != NULL and the lookup does not find key, nobj is inserted.
+ * If nobj == NULL and the lookup does not find key, NULL is returned.
+ * A reference to the returned object is held.
+ */
+
+static struct objhead *
+hsl_lookup(const struct sess *sp, struct objhead *noh)
+{
+	struct objhead *oh;
+	int i;
+
+	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+	CHECK_OBJ_NOTNULL(noh, OBJHEAD_MAGIC);
+	Lck_Lock(&hsl_mtx);
+	VTAILQ_FOREACH(oh, &hsl_head, hoh_list) {
+		i = memcmp(oh->digest, noh->digest, sizeof oh->digest);
+		if (i < 0)
+			continue;
+		if (i > 0)
+			break;
+		oh->refcnt++;
+		Lck_Unlock(&hsl_mtx);
+		return (oh);
+	}
+
+	if (oh != NULL)
+		VTAILQ_INSERT_BEFORE(oh, noh, hoh_list);
+	else
+		VTAILQ_INSERT_TAIL(&hsl_head, noh, hoh_list);
+
+	Lck_Unlock(&hsl_mtx);
+	return (noh);
+}
+
+/*--------------------------------------------------------------------
+ * Dereference and if no references are left, free.
+ */
+
+static int
+hsl_deref(struct objhead *oh)
+{
+	int ret;
+
+	Lck_Lock(&hsl_mtx);
+	if (--oh->refcnt == 0) {
+		VTAILQ_REMOVE(&hsl_head, oh, hoh_list);
+		ret = 0;
+	} else
+		ret = 1;
+	Lck_Unlock(&hsl_mtx);
+	return (ret);
+}
+
+/*--------------------------------------------------------------------*/
+
+const struct hash_slinger hsl_slinger = {
+	.magic	=	SLINGER_MAGIC,
+	.name	=	"simple",
+	.start	=	hsl_start,
+	.lookup =	hsl_lookup,
+	.deref	=	hsl_deref,
+};
diff --git a/bin/varnishd/hash/hash_slinger.h b/bin/varnishd/hash/hash_slinger.h
new file mode 100644
index 0000000..2c142cf
--- /dev/null
+++ b/bin/varnishd/hash/hash_slinger.h
@@ -0,0 +1,104 @@
+/*-
+ * Copyright (c) 2006 Verdens Gang AS
+ * Copyright (c) 2006-2010 Varnish Software AS
+ * All rights reserved.
+ *
+ * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ */
+
+struct sess;
+struct worker;
+struct object;
+
+typedef void hash_init_f(int ac, char * const *av);
+typedef void hash_start_f(void);
+typedef void hash_prep_f(const struct sess *sp);
+typedef struct objhead *
+    hash_lookup_f(const struct sess *sp, struct objhead *nobj);
+typedef int hash_deref_f(struct objhead *obj);
+
+struct hash_slinger {
+	unsigned		magic;
+#define SLINGER_MAGIC		0x1b720cba
+	const char		*name;
+	hash_init_f		*init;
+	hash_start_f		*start;
+	hash_prep_f		*prep;
+	hash_lookup_f		*lookup;
+	hash_deref_f		*deref;
+};
+
+/* cache_hash.c */
+void HSH_Prealloc(const struct sess *sp);
+void HSH_Cleanup(struct worker *w);
+struct objcore *HSH_Lookup(struct sess *sp, struct objhead **poh);
+void HSH_Unbusy(const struct sess *sp);
+void HSH_Ref(struct objcore *o);
+void HSH_Drop(struct sess *sp);
+void HSH_Init(void);
+void HSH_AddString(const struct sess *sp, const char *str);
+struct objcore *HSH_Insert(const struct sess *sp);
+void HSH_Purge(const struct sess *, struct objhead *, double ttl, double grace);
+void HSH_config(const char *h_arg);
+
+#ifdef VARNISH_CACHE_CHILD
+
+struct waitinglist {
+	unsigned		magic;
+#define WAITINGLIST_MAGIC	0x063a477a
+	VTAILQ_HEAD(, sess)	list;
+};
+
+struct objhead {
+	unsigned		magic;
+#define OBJHEAD_MAGIC		0x1b96615d
+
+	int			refcnt;
+	struct lock		mtx;
+	VTAILQ_HEAD(,objcore)	objcs;
+	unsigned char		digest[DIGEST_LEN];
+	struct waitinglist	*waitinglist;
+
+	/*----------------------------------------------------
+	 * The fields below are for the sole private use of
+	 * the hash implementation(s).
+	 */
+	union {
+		struct {
+			VTAILQ_ENTRY(objhead)	u_n_hoh_list;
+			void			*u_n_hoh_head;
+		} n;
+	} _u;
+#define hoh_list _u.n.u_n_hoh_list
+#define hoh_head _u.n.u_n_hoh_head
+};
+
+void HSH_DeleteObjHead(struct worker *w, struct objhead *oh);
+int HSH_Deref(struct worker *w, struct objcore *oc, struct object **o);
+#endif /* VARNISH_CACHE_CHILD */
+
+extern const struct hash_slinger hsl_slinger;
+extern const struct hash_slinger hcl_slinger;
+extern const struct hash_slinger hcb_slinger;
diff --git a/bin/varnishd/hash_classic.c b/bin/varnishd/hash_classic.c
deleted file mode 100644
index fc88757..0000000
--- a/bin/varnishd/hash_classic.c
+++ /dev/null
@@ -1,184 +0,0 @@
-/*-
- * Copyright (c) 2006 Verdens Gang AS
- * Copyright (c) 2006-2009 Varnish Software AS
- * All rights reserved.
- *
- * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- *
- * A classic bucketed hash
- */
-
-#include "config.h"
-
-#include <stdio.h>
-#include <stdlib.h>
-
-#include "cache.h"
-
-#include "hash_slinger.h"
-
-/*--------------------------------------------------------------------*/
-
-struct hcl_hd {
-	unsigned		magic;
-#define HCL_HEAD_MAGIC		0x0f327016
-	VTAILQ_HEAD(, objhead)	head;
-	struct lock		mtx;
-};
-
-static unsigned			hcl_nhash = 16383;
-static struct hcl_hd		*hcl_head;
-
-/*--------------------------------------------------------------------
- * The ->init method allows the management process to pass arguments
- */
-
-static void
-hcl_init(int ac, char * const *av)
-{
-	int i;
-	unsigned u;
-
-	if (ac == 0)
-		return;
-	if (ac > 1)
-		ARGV_ERR("(-hclassic) too many arguments\n");
-	i = sscanf(av[0], "%u", &u);
-	if (i <= 0 || u == 0)
-		return;
-	if (u > 2 && !(u & (u - 1))) {
-		fprintf(stderr,
-		    "NOTE:\n"
-		    "\tA power of two number of hash buckets is "
-		    "marginally less efficient\n"
-		    "\twith systematic URLs.  Reducing by one"
-		    " hash bucket.\n");
-		u--;
-	}
-	hcl_nhash = u;
-	fprintf(stderr, "Classic hash: %u buckets\n", hcl_nhash);
-	return;
-}
-
-/*--------------------------------------------------------------------
- * The ->start method is called during cache process start and allows
- * initialization to happen before the first lookup.
- */
-
-static void
-hcl_start(void)
-{
-	unsigned u;
-
-	hcl_head = calloc(sizeof *hcl_head, hcl_nhash);
-	XXXAN(hcl_head);
-
-	for (u = 0; u < hcl_nhash; u++) {
-		VTAILQ_INIT(&hcl_head[u].head);
-		Lck_New(&hcl_head[u].mtx, lck_hcl);
-		hcl_head[u].magic = HCL_HEAD_MAGIC;
-	}
-}
-
-/*--------------------------------------------------------------------
- * Lookup and possibly insert element.
- * If nobj != NULL and the lookup does not find key, nobj is inserted.
- * If nobj == NULL and the lookup does not find key, NULL is returned.
- * A reference to the returned object is held.
- * We use a two-pass algorithm to handle inserts as they are quite
- * rare and collisions even rarer.
- */
-
-static struct objhead *
-hcl_lookup(const struct sess *sp, struct objhead *noh)
-{
-	struct objhead *oh;
-	struct hcl_hd *hp;
-	unsigned u1, digest;
-	int i;
-
-	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
-	CHECK_OBJ_NOTNULL(noh, OBJHEAD_MAGIC);
-
-	assert(sizeof noh->digest > sizeof digest);
-	memcpy(&digest, noh->digest, sizeof digest);
-	u1 = digest % hcl_nhash;
-	hp = &hcl_head[u1];
-
-	Lck_Lock(&hp->mtx);
-	VTAILQ_FOREACH(oh, &hp->head, hoh_list) {
-		i = memcmp(oh->digest, noh->digest, sizeof oh->digest);
-		if (i < 0)
-			continue;
-		if (i > 0)
-			break;
-		oh->refcnt++;
-		Lck_Unlock(&hp->mtx);
-		return (oh);
-	}
-
-	if (oh != NULL)
-		VTAILQ_INSERT_BEFORE(oh, noh, hoh_list);
-	else
-		VTAILQ_INSERT_TAIL(&hp->head, noh, hoh_list);
-
-	noh->hoh_head = hp;
-
-	Lck_Unlock(&hp->mtx);
-	return (noh);
-}
-
-/*--------------------------------------------------------------------
- * Dereference and if no references are left, free.
- */
-
-static int
-hcl_deref(struct objhead *oh)
-{
-	struct hcl_hd *hp;
-	int ret;
-
-	CHECK_OBJ_NOTNULL(oh, OBJHEAD_MAGIC);
-	CAST_OBJ_NOTNULL(hp, oh->hoh_head, HCL_HEAD_MAGIC);
-	assert(oh->refcnt > 0);
-	Lck_Lock(&hp->mtx);
-	if (--oh->refcnt == 0) {
-		VTAILQ_REMOVE(&hp->head, oh, hoh_list);
-		ret = 0;
-	} else
-		ret = 1;
-	Lck_Unlock(&hp->mtx);
-	return (ret);
-}
-
-/*--------------------------------------------------------------------*/
-
-const struct hash_slinger hcl_slinger = {
-	.magic	=	SLINGER_MAGIC,
-	.name	=	"classic",
-	.init	=	hcl_init,
-	.start	=	hcl_start,
-	.lookup =	hcl_lookup,
-	.deref	=	hcl_deref,
-};
diff --git a/bin/varnishd/hash_critbit.c b/bin/varnishd/hash_critbit.c
deleted file mode 100644
index 78bda7a..0000000
--- a/bin/varnishd/hash_critbit.c
+++ /dev/null
@@ -1,492 +0,0 @@
-/*-
- * Copyright (c) 2008-2010 Varnish Software AS
- * All rights reserved.
- *
- * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- *
- * A Crit Bit tree based hash
- */
-
-// #define PHK
-
-#include "config.h"
-
-#include <stdlib.h>
-
-#include "cache.h"
-
-#include "hash_slinger.h"
-#include "vcli_priv.h"
-#include "vmb.h"
-#include "vtim.h"
-
-static struct lock hcb_mtx;
-
-/*---------------------------------------------------------------------
- * Table for finding out how many bits two bytes have in common,
- * counting from the MSB towards the LSB.
- * ie:
- *	hcb_bittbl[0x01 ^ 0x22] == 2
- *	hcb_bittbl[0x10 ^ 0x0b] == 3
- *
- */
-
-static unsigned char hcb_bittbl[256];
-
-static unsigned char
-hcb_bits(unsigned char x, unsigned char y)
-{
-	return hcb_bittbl[x ^ y];
-}
-
-static void
-hcb_build_bittbl(void)
-{
-	unsigned char x;
-	unsigned y;
-
-	y = 0;
-	for (x = 0; x < 8; x++)
-		for (; y < (1U << x); y++)
-			hcb_bittbl[y] = 8 - x;
-
-	/* Quick asserts for sanity check */
-	assert(hcb_bits(0x34, 0x34) == 8);
-	assert(hcb_bits(0xaa, 0x55) == 0);
-	assert(hcb_bits(0x01, 0x22) == 2);
-	assert(hcb_bits(0x10, 0x0b) == 3);
-}
-
-/*---------------------------------------------------------------------
- * For space reasons we overload the two pointers with two different
- * kinds of of pointers.  We cast them to uintptr_t's and abuse the
- * low two bits to tell them apart, assuming that Varnish will never
- * run on machines with less than 32bit alignment.
- *
- * Asserts will explode if these assumptions are not met.
- */
-
-struct hcb_y {
-	unsigned		magic;
-#define HCB_Y_MAGIC		0x125c4bd2
-	unsigned short		critbit;
-	unsigned char		ptr;
-	unsigned char		bitmask;
-	volatile uintptr_t	leaf[2];
-	VSTAILQ_ENTRY(hcb_y)	list;
-};
-
-#define HCB_BIT_NODE		(1<<0)
-#define HCB_BIT_Y		(1<<1)
-
-struct hcb_root {
-	volatile uintptr_t	origo;
-};
-
-static struct hcb_root	hcb_root;
-
-static VSTAILQ_HEAD(, hcb_y)	cool_y = VSTAILQ_HEAD_INITIALIZER(cool_y);
-static VSTAILQ_HEAD(, hcb_y)	dead_y = VSTAILQ_HEAD_INITIALIZER(dead_y);
-static VTAILQ_HEAD(, objhead)	cool_h = VTAILQ_HEAD_INITIALIZER(cool_h);
-static VTAILQ_HEAD(, objhead)	dead_h = VTAILQ_HEAD_INITIALIZER(dead_h);
-
-/*---------------------------------------------------------------------
- * Pointer accessor functions
- */
-static int
-hcb_is_node(uintptr_t u)
-{
-
-	return (u & HCB_BIT_NODE);
-}
-
-static int
-hcb_is_y(uintptr_t u)
-{
-
-	return (u & HCB_BIT_Y);
-}
-
-static uintptr_t
-hcb_r_node(struct objhead *n)
-{
-
-	assert(!((uintptr_t)n & (HCB_BIT_NODE|HCB_BIT_Y)));
-	return (HCB_BIT_NODE | (uintptr_t)n);
-}
-
-static struct objhead *
-hcb_l_node(uintptr_t u)
-{
-
-	assert(u & HCB_BIT_NODE);
-	assert(!(u & HCB_BIT_Y));
-	return ((struct objhead *)(u & ~HCB_BIT_NODE));
-}
-
-static uintptr_t
-hcb_r_y(struct hcb_y *y)
-{
-
-	CHECK_OBJ_NOTNULL(y, HCB_Y_MAGIC);
-	assert(!((uintptr_t)y & (HCB_BIT_NODE|HCB_BIT_Y)));
-	return (HCB_BIT_Y | (uintptr_t)y);
-}
-
-static struct hcb_y *
-hcb_l_y(uintptr_t u)
-{
-
-	assert(!(u & HCB_BIT_NODE));
-	assert(u & HCB_BIT_Y);
-	return ((struct hcb_y *)(u & ~HCB_BIT_Y));
-}
-
-/*---------------------------------------------------------------------
- * Find the "critical" bit that separates these two digests
- */
-
-static unsigned
-hcb_crit_bit(const struct objhead *oh1, const struct objhead *oh2,
-    struct hcb_y *y)
-{
-	unsigned char u, r;
-
-	CHECK_OBJ_NOTNULL(y, HCB_Y_MAGIC);
-	for (u = 0; u < DIGEST_LEN && oh1->digest[u] == oh2->digest[u]; u++)
-		;
-	assert(u < DIGEST_LEN);
-	r = hcb_bits(oh1->digest[u], oh2->digest[u]);
-	y->ptr = u;
-	y->bitmask = 0x80 >> r;
-	y->critbit = u * 8 + r;
-	return (y->critbit);
-}
-
-/*---------------------------------------------------------------------
- * Unless we have the lock, we need to be very careful about pointer
- * references into the tree, we cannot trust things to be the same
- * in two consequtive memory accesses.
- */
-
-static struct objhead *
-hcb_insert(struct worker *wrk, struct hcb_root *root, struct objhead *oh, int has_lock)
-{
-	volatile uintptr_t *p;
-	uintptr_t pp;
-	struct hcb_y *y, *y2;
-	struct objhead *oh2;
-	unsigned s, s2;
-
-	p = &root->origo;
-	pp = *p;
-	if (pp == 0) {
-		if (!has_lock)
-			return (NULL);
-		*p = hcb_r_node(oh);
-		return (oh);
-	}
-
-	while(hcb_is_y(pp)) {
-		y = hcb_l_y(pp);
-		CHECK_OBJ_NOTNULL(y, HCB_Y_MAGIC);
-		assert(y->ptr < DIGEST_LEN);
-		s = (oh->digest[y->ptr] & y->bitmask) != 0;
-		assert(s < 2);
-		p = &y->leaf[s];
-		pp = *p;
-	}
-
-	if (pp == 0) {
-		/* We raced hcb_delete and got a NULL pointer */
-		assert(!has_lock);
-		return (NULL);
-	}
-
-	assert(hcb_is_node(pp));
-
-	/* We found a node, does it match ? */
-	oh2 = hcb_l_node(pp);
-	CHECK_OBJ_NOTNULL(oh2, OBJHEAD_MAGIC);
-	if (!memcmp(oh2->digest, oh->digest, DIGEST_LEN))
-		return (oh2);
-
-	if (!has_lock)
-		return (NULL);
-
-	/* Insert */
-
-	CAST_OBJ_NOTNULL(y2, wrk->nhashpriv, HCB_Y_MAGIC);
-	wrk->nhashpriv = NULL;
-	(void)hcb_crit_bit(oh, oh2, y2);
-	s2 = (oh->digest[y2->ptr] & y2->bitmask) != 0;
-	assert(s2 < 2);
-	y2->leaf[s2] = hcb_r_node(oh);
-	s2 = 1-s2;
-
-	p = &root->origo;
-	assert(*p != 0);
-
-	while(hcb_is_y(*p)) {
-		y = hcb_l_y(*p);
-		CHECK_OBJ_NOTNULL(y, HCB_Y_MAGIC);
-		assert(y->critbit != y2->critbit);
-		if (y->critbit > y2->critbit)
-			break;
-		assert(y->ptr < DIGEST_LEN);
-		s = (oh->digest[y->ptr] & y->bitmask) != 0;
-		assert(s < 2);
-		p = &y->leaf[s];
-	}
-	y2->leaf[s2] = *p;
-	VWMB();
-	*p = hcb_r_y(y2);
-	return(oh);
-}
-
-/*--------------------------------------------------------------------*/
-
-static void
-hcb_delete(struct hcb_root *r, struct objhead *oh)
-{
-	struct hcb_y *y;
-	volatile uintptr_t *p;
-	unsigned s;
-
-	if (r->origo == hcb_r_node(oh)) {
-		r->origo = 0;
-		return;
-	}
-	p = &r->origo;
-	assert(hcb_is_y(*p));
-
-	y = NULL;
-	while(1) {
-		assert(hcb_is_y(*p));
-		y = hcb_l_y(*p);
-		assert(y->ptr < DIGEST_LEN);
-		s = (oh->digest[y->ptr] & y->bitmask) != 0;
-		assert(s < 2);
-		if (y->leaf[s] == hcb_r_node(oh)) {
-			*p = y->leaf[1 - s];
-			VSTAILQ_INSERT_TAIL(&cool_y, y, list);
-			return;
-		}
-		p = &y->leaf[s];
-	}
-}
-
-/*--------------------------------------------------------------------*/
-
-static void
-dumptree(struct cli *cli, uintptr_t p, int indent)
-{
-	int i;
-	const struct objhead *oh;
-	const struct hcb_y *y;
-
-	if (p == 0)
-		return;
-	if (hcb_is_node(p)) {
-		oh = hcb_l_node(p);
-		VCLI_Out(cli, "%*.*sN %d r%u <%02x%02x%02x...>\n",
-		    indent, indent, "", indent / 2, oh->refcnt,
-		    oh->digest[0], oh->digest[1], oh->digest[2]);
-		return;
-	}
-	assert(hcb_is_y(p));
-	y = hcb_l_y(p);
-	VCLI_Out(cli, "%*.*sY c %u p %u b %02x i %d\n",
-	    indent, indent, "",
-	    y->critbit, y->ptr, y->bitmask, indent / 2);
-	indent += 2;
-	for (i = 0; i < 2; i++)
-		dumptree(cli, y->leaf[i], indent);
-}
-
-static void
-hcb_dump(struct cli *cli, const char * const *av, void *priv)
-{
-
-	(void)priv;
-	(void)av;
-	VCLI_Out(cli, "HCB dump:\n");
-	dumptree(cli, hcb_root.origo, 0);
-	VCLI_Out(cli, "Coollist:\n");
-}
-
-static struct cli_proto hcb_cmds[] = {
-	{ "hcb.dump", "hcb.dump", "dump HCB tree\n", 0, 0, "d", hcb_dump },
-	{ NULL }
-};
-
-/*--------------------------------------------------------------------*/
-
-static void *
-hcb_cleaner(void *priv)
-{
-	struct hcb_y *y, *y2;
-	struct worker ww;
-	struct objhead *oh, *oh2;
-
-	memset(&ww, 0, sizeof ww);
-	ww.magic = WORKER_MAGIC;
-
-	THR_SetName("hcb_cleaner");
-	(void)priv;
-	while (1) {
-		VSTAILQ_FOREACH_SAFE(y, &dead_y, list, y2) {
-			VSTAILQ_REMOVE_HEAD(&dead_y, list);
-			FREE_OBJ(y);
-		}
-		VTAILQ_FOREACH_SAFE(oh, &dead_h, hoh_list, oh2) {
-			VTAILQ_REMOVE(&dead_h, oh, hoh_list);
-			HSH_DeleteObjHead(&ww, oh);
-		}
-		Lck_Lock(&hcb_mtx);
-		VSTAILQ_CONCAT(&dead_y, &cool_y);
-		VTAILQ_CONCAT(&dead_h, &cool_h, hoh_list);
-		Lck_Unlock(&hcb_mtx);
-		WRK_SumStat(&ww);
-		VTIM_sleep(params->critbit_cooloff);
-	}
-	NEEDLESS_RETURN(NULL);
-}
-
-/*--------------------------------------------------------------------*/
-
-static void
-hcb_start(void)
-{
-	struct objhead *oh = NULL;
-	pthread_t tp;
-
-	(void)oh;
-	CLI_AddFuncs(hcb_cmds);
-	Lck_New(&hcb_mtx, lck_hcb);
-	AZ(pthread_create(&tp, NULL, hcb_cleaner, NULL));
-	memset(&hcb_root, 0, sizeof hcb_root);
-	hcb_build_bittbl();
-}
-
-static int
-hcb_deref(struct objhead *oh)
-{
-	int r;
-
-	r = 1;
-	CHECK_OBJ_NOTNULL(oh, OBJHEAD_MAGIC);
-	Lck_Lock(&oh->mtx);
-	assert(oh->refcnt > 0);
-	oh->refcnt--;
-	if (oh->refcnt == 0) {
-		Lck_Lock(&hcb_mtx);
-		hcb_delete(&hcb_root, oh);
-		VTAILQ_INSERT_TAIL(&cool_h, oh, hoh_list);
-		Lck_Unlock(&hcb_mtx);
-		assert(VTAILQ_EMPTY(&oh->objcs));
-		AZ(oh->waitinglist);
-	}
-	Lck_Unlock(&oh->mtx);
-#ifdef PHK
-	fprintf(stderr, "hcb_defef %d %d <%s>\n", __LINE__, r, oh->hash);
-#endif
-	return (r);
-}
-
-static struct objhead *
-hcb_lookup(const struct sess *sp, struct objhead *noh)
-{
-	struct objhead *oh;
-	struct hcb_y *y;
-	unsigned u;
-	unsigned with_lock;
-
-	(void)sp;
-
-	with_lock = 0;
-	while (1) {
-		if (with_lock) {
-			CAST_OBJ_NOTNULL(y, sp->wrk->nhashpriv, HCB_Y_MAGIC);
-			Lck_Lock(&hcb_mtx);
-			VSC_C_main->hcb_lock++;
-			assert(noh->refcnt == 1);
-			oh = hcb_insert(sp->wrk, &hcb_root, noh, 1);
-			Lck_Unlock(&hcb_mtx);
-		} else {
-			VSC_C_main->hcb_nolock++;
-			oh = hcb_insert(sp->wrk, &hcb_root, noh, 0);
-		}
-
-		if (oh != NULL && oh == noh) {
-			/* Assert that we only muck with the tree with a lock */
-			assert(with_lock);
-			VSC_C_main->hcb_insert++;
-			assert(oh->refcnt > 0);
-			return (oh);
-		}
-
-		if (oh == NULL) {
-			assert(!with_lock);
-			/* Try again, with lock */
-			with_lock = 1;
-			continue;
-		}
-
-		CHECK_OBJ_NOTNULL(noh, OBJHEAD_MAGIC);
-		Lck_Lock(&oh->mtx);
-		/*
-		 * A refcount of zero indicates that the tree changed
-		 * under us, so fall through and try with the lock held.
-		 */
-		u = oh->refcnt;
-		if (u > 0)
-			oh->refcnt++;
-		else
-			with_lock = 1;
-		Lck_Unlock(&oh->mtx);
-		if (u > 0)
-			return (oh);
-	}
-}
-
-static void
-hcb_prep(const struct sess *sp)
-{
-	struct hcb_y *y;
-
-	if (sp->wrk->nhashpriv == NULL) {
-		ALLOC_OBJ(y, HCB_Y_MAGIC);
-		sp->wrk->nhashpriv = y;
-	}
-}
-
-const struct hash_slinger hcb_slinger = {
-	.magic  =	SLINGER_MAGIC,
-	.name   =	"critbit",
-	.start  =	hcb_start,
-	.lookup =	hcb_lookup,
-	.prep =		hcb_prep,
-	.deref  =	hcb_deref,
-};
diff --git a/bin/varnishd/hash_simple_list.c b/bin/varnishd/hash_simple_list.c
deleted file mode 100644
index c1c17da..0000000
--- a/bin/varnishd/hash_simple_list.c
+++ /dev/null
@@ -1,118 +0,0 @@
-/*-
- * Copyright (c) 2006 Verdens Gang AS
- * Copyright (c) 2006-2009 Varnish Software AS
- * All rights reserved.
- *
- * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- *
- * This is the reference hash(/lookup) implementation
- */
-
-#include "config.h"
-
-#include "cache.h"
-
-#include "hash_slinger.h"
-
-/*--------------------------------------------------------------------*/
-
-static VTAILQ_HEAD(, objhead)	hsl_head = VTAILQ_HEAD_INITIALIZER(hsl_head);
-static struct lock hsl_mtx;
-
-/*--------------------------------------------------------------------
- * The ->init method is called during process start and allows
- * initialization to happen before the first lookup.
- */
-
-static void
-hsl_start(void)
-{
-
-	Lck_New(&hsl_mtx, lck_hsl);
-}
-
-/*--------------------------------------------------------------------
- * Lookup and possibly insert element.
- * If nobj != NULL and the lookup does not find key, nobj is inserted.
- * If nobj == NULL and the lookup does not find key, NULL is returned.
- * A reference to the returned object is held.
- */
-
-static struct objhead *
-hsl_lookup(const struct sess *sp, struct objhead *noh)
-{
-	struct objhead *oh;
-	int i;
-
-	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
-	CHECK_OBJ_NOTNULL(noh, OBJHEAD_MAGIC);
-	Lck_Lock(&hsl_mtx);
-	VTAILQ_FOREACH(oh, &hsl_head, hoh_list) {
-		i = memcmp(oh->digest, noh->digest, sizeof oh->digest);
-		if (i < 0)
-			continue;
-		if (i > 0)
-			break;
-		oh->refcnt++;
-		Lck_Unlock(&hsl_mtx);
-		return (oh);
-	}
-
-	if (oh != NULL)
-		VTAILQ_INSERT_BEFORE(oh, noh, hoh_list);
-	else
-		VTAILQ_INSERT_TAIL(&hsl_head, noh, hoh_list);
-
-	Lck_Unlock(&hsl_mtx);
-	return (noh);
-}
-
-/*--------------------------------------------------------------------
- * Dereference and if no references are left, free.
- */
-
-static int
-hsl_deref(struct objhead *oh)
-{
-	int ret;
-
-	Lck_Lock(&hsl_mtx);
-	if (--oh->refcnt == 0) {
-		VTAILQ_REMOVE(&hsl_head, oh, hoh_list);
-		ret = 0;
-	} else
-		ret = 1;
-	Lck_Unlock(&hsl_mtx);
-	return (ret);
-}
-
-/*--------------------------------------------------------------------*/
-
-const struct hash_slinger hsl_slinger = {
-	.magic	=	SLINGER_MAGIC,
-	.name	=	"simple",
-	.start	=	hsl_start,
-	.lookup =	hsl_lookup,
-	.deref	=	hsl_deref,
-};
diff --git a/bin/varnishd/hash_slinger.h b/bin/varnishd/hash_slinger.h
deleted file mode 100644
index 2c142cf..0000000
--- a/bin/varnishd/hash_slinger.h
+++ /dev/null
@@ -1,104 +0,0 @@
-/*-
- * Copyright (c) 2006 Verdens Gang AS
- * Copyright (c) 2006-2010 Varnish Software AS
- * All rights reserved.
- *
- * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- *
- */
-
-struct sess;
-struct worker;
-struct object;
-
-typedef void hash_init_f(int ac, char * const *av);
-typedef void hash_start_f(void);
-typedef void hash_prep_f(const struct sess *sp);
-typedef struct objhead *
-    hash_lookup_f(const struct sess *sp, struct objhead *nobj);
-typedef int hash_deref_f(struct objhead *obj);
-
-struct hash_slinger {
-	unsigned		magic;
-#define SLINGER_MAGIC		0x1b720cba
-	const char		*name;
-	hash_init_f		*init;
-	hash_start_f		*start;
-	hash_prep_f		*prep;
-	hash_lookup_f		*lookup;
-	hash_deref_f		*deref;
-};
-
-/* cache_hash.c */
-void HSH_Prealloc(const struct sess *sp);
-void HSH_Cleanup(struct worker *w);
-struct objcore *HSH_Lookup(struct sess *sp, struct objhead **poh);
-void HSH_Unbusy(const struct sess *sp);
-void HSH_Ref(struct objcore *o);
-void HSH_Drop(struct sess *sp);
-void HSH_Init(void);
-void HSH_AddString(const struct sess *sp, const char *str);
-struct objcore *HSH_Insert(const struct sess *sp);
-void HSH_Purge(const struct sess *, struct objhead *, double ttl, double grace);
-void HSH_config(const char *h_arg);
-
-#ifdef VARNISH_CACHE_CHILD
-
-struct waitinglist {
-	unsigned		magic;
-#define WAITINGLIST_MAGIC	0x063a477a
-	VTAILQ_HEAD(, sess)	list;
-};
-
-struct objhead {
-	unsigned		magic;
-#define OBJHEAD_MAGIC		0x1b96615d
-
-	int			refcnt;
-	struct lock		mtx;
-	VTAILQ_HEAD(,objcore)	objcs;
-	unsigned char		digest[DIGEST_LEN];
-	struct waitinglist	*waitinglist;
-
-	/*----------------------------------------------------
-	 * The fields below are for the sole private use of
-	 * the hash implementation(s).
-	 */
-	union {
-		struct {
-			VTAILQ_ENTRY(objhead)	u_n_hoh_list;
-			void			*u_n_hoh_head;
-		} n;
-	} _u;
-#define hoh_list _u.n.u_n_hoh_list
-#define hoh_head _u.n.u_n_hoh_head
-};
-
-void HSH_DeleteObjHead(struct worker *w, struct objhead *oh);
-int HSH_Deref(struct worker *w, struct objcore *oc, struct object **o);
-#endif /* VARNISH_CACHE_CHILD */
-
-extern const struct hash_slinger hsl_slinger;
-extern const struct hash_slinger hcl_slinger;
-extern const struct hash_slinger hcb_slinger;
diff --git a/bin/varnishd/storage/storage_persistent.c b/bin/varnishd/storage/storage_persistent.c
index 84aef5c..02d244e 100644
--- a/bin/varnishd/storage/storage_persistent.c
+++ b/bin/varnishd/storage/storage_persistent.c
@@ -46,7 +46,7 @@
 #include "cache.h"
 #include "storage/storage.h"
 
-#include "hash_slinger.h"
+#include "hash/hash_slinger.h"
 #include "vcli.h"
 #include "vcli_priv.h"
 #include "vend.h"
diff --git a/bin/varnishd/storage/storage_persistent_silo.c b/bin/varnishd/storage/storage_persistent_silo.c
index 5393546..27f0c98 100644
--- a/bin/varnishd/storage/storage_persistent_silo.c
+++ b/bin/varnishd/storage/storage_persistent_silo.c
@@ -40,7 +40,7 @@
 #include "cache.h"
 #include "storage/storage.h"
 
-#include "hash_slinger.h"
+#include "hash/hash_slinger.h"
 #include "vsha256.h"
 #include "vtim.h"
 
diff --git a/bin/varnishd/varnishd.c b/bin/varnishd/varnishd.c
index 7fe72aa..e0654a3 100644
--- a/bin/varnishd/varnishd.c
+++ b/bin/varnishd/varnishd.c
@@ -45,7 +45,7 @@
 
 #include "mgt.h"
 
-#include "hash_slinger.h"
+#include "hash/hash_slinger.h"
 #include "heritage.h"
 #include "vav.h"
 #include "vcli.h"



From geoff at varnish-cache.org  Mon Jan  9 20:52:28 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:28 +0100
Subject: [experimental-ims] 8d2badd Polish the chunked body fetcher. It still
	irks me that it does one byte reads for the hex-length headers.
Message-ID: <E1RkMCi-0007zf-1K@project.varnish-software.com>

commit 8d2badd80efbf33cd17aa6b48d33e33e55e53bbf
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Oct 31 22:19:45 2011 +0000

    Polish the chunked body fetcher.  It still irks me that it does
    one byte reads for the hex-length headers.

diff --git a/bin/varnishd/cache_fetch.c b/bin/varnishd/cache_fetch.c
index fb2ce41..516517f 100644
--- a/bin/varnishd/cache_fetch.c
+++ b/bin/varnishd/cache_fetch.c
@@ -248,9 +248,9 @@ fetch_straight(struct worker *w, struct http_conn *htc, ssize_t cl)
 
 /*--------------------------------------------------------------------
  * Read a chunked HTTP object.
+ *
  * XXX: Reading one byte at a time is pretty pessimal.
  */
- 
 
 static int
 fetch_chunked(struct worker *w, struct http_conn *htc)
@@ -264,17 +264,18 @@ fetch_chunked(struct worker *w, struct http_conn *htc)
 	do {
 		/* Skip leading whitespace */
 		do {
-			i = HTC_Read(w, htc, buf, 1);
-			if (i <= 0)
-				return (i);
+			if (HTC_Read(w, htc, buf, 1) <= 0)
+				return (-1);
 		} while (vct_islws(buf[0]));
 
+		if (!vct_ishex(buf[0]))
+			return (FetchError(w,"chunked header non-hex"));
+
 		/* Collect hex digits, skipping leading zeros */
 		for (u = 1; u < sizeof buf; u++) {
 			do {
-				i = HTC_Read(w, htc, buf + u, 1);
-				if (i <= 0)
-					return (i);
+				if (HTC_Read(w, htc, buf + u, 1) <= 0)
+					return (-1);
 			} while (u == 1 && buf[0] == '0' && buf[u] == '0');
 			if (!vct_ishex(buf[u]))
 				break;
@@ -284,40 +285,32 @@ fetch_chunked(struct worker *w, struct http_conn *htc)
 			return (FetchError(w,"chunked header too long"));
 
 		/* Skip trailing white space */
-		while(vct_islws(buf[u]) && buf[u] != '\n') {
-			i = HTC_Read(w, htc, buf + u, 1);
-			if (i <= 0)
-				return (i);
-		}
+		while(vct_islws(buf[u]) && buf[u] != '\n')
+			if (HTC_Read(w, htc, buf + u, 1) <= 0)
+				return (-1);
 
 		if (buf[u] != '\n') 
 			return (FetchError(w,"chunked header no NL"));
-		buf[u] = '\0';
 
+		buf[u] = '\0';
 		cl = fetch_number(buf, 16);
-		if (cl < 0) {
+		if (cl < 0)
 			return (FetchError(w,"chunked header number syntax"));
-		} else if (cl > 0) {
-			i = w->vfp->bytes(w, htc, cl);
-			if (i <= 0)
-				return (-1);
-		}
+
+		if (cl > 0 && w->vfp->bytes(w, htc, cl) <= 0)
+			return (-1);
+
 		i = HTC_Read(w, htc, buf, 1);
 		if (i <= 0)
 			return (-1);
-		if (buf[0] == '\r') {
-			i = HTC_Read(w, htc, buf, 1);
-			if (i <= 0)
-				return (-1);
-		}
+		if (buf[0] == '\r' && HTC_Read(w, htc, buf, 1) <= 0)
+			return (-1);
 		if (buf[0] != '\n')
 			return (FetchError(w,"chunked tail no NL"));
 	} while (cl > 0);
 	return (0);
 }
 
-#undef CERR
-
 /*--------------------------------------------------------------------*/
 
 static int



From geoff at varnish-cache.org  Mon Jan  9 20:52:15 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:15 +0100
Subject: [experimental-ims] b3d6add Make TIM_parse() return double like all
	other TIM_ functions.
Message-ID: <E1RkMCV-0007aI-Qb@project.varnish-software.com>

commit b3d6addcbabd1885f7150c6fb45800a2bcbb339b
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sun Oct 9 17:46:57 2011 +0000

    Make TIM_parse() return double like all other TIM_ functions.
    
    Eliminate libvarnish.h embedded #includes.

diff --git a/bin/varnishd/cache_pool.c b/bin/varnishd/cache_pool.c
index c5183a4..0380b55 100644
--- a/bin/varnishd/cache_pool.c
+++ b/bin/varnishd/cache_pool.c
@@ -58,6 +58,8 @@
 #ifndef CLOCK_MONOTONIC
 #define CLOCK_MONOTONIC	0
 
+#include <sys/time.h>
+
 static int
 clock_gettime(int foo, struct timespec *ts)
 {
diff --git a/bin/varnishd/mgt_sandbox_solaris.c b/bin/varnishd/mgt_sandbox_solaris.c
index 2c975d9..3b6182f 100644
--- a/bin/varnishd/mgt_sandbox_solaris.c
+++ b/bin/varnishd/mgt_sandbox_solaris.c
@@ -36,6 +36,7 @@
 
 #include <stdio.h>
 #include <syslog.h>
+#include <strin.h>
 
 #ifdef HAVE_PRIV_H
 #include <priv.h>
diff --git a/bin/varnishncsa/varnishncsa.c b/bin/varnishncsa/varnishncsa.c
index bbaa366..d010a73 100644
--- a/bin/varnishncsa/varnishncsa.c
+++ b/bin/varnishncsa/varnishncsa.c
@@ -69,6 +69,7 @@
 #include <string.h>
 #include <strings.h>
 #include <unistd.h>
+#include <time.h>
 
 #include "compat/daemon.h"
 
diff --git a/include/libvarnish.h b/include/libvarnish.h
index 157a64e..b016848 100644
--- a/include/libvarnish.h
+++ b/include/libvarnish.h
@@ -28,10 +28,7 @@
  *
  */
 
-#include <errno.h>
-#include <time.h>
 #include <stdint.h>
-#include <sys/types.h>
 
 #include "vas.h"
 
@@ -88,7 +85,7 @@ void VTCP_set_read_timeout(int s, double seconds);
 /* from libvarnish/time.c */
 #define TIM_FORMAT_SIZE 30
 void TIM_format(double t, char *p);
-time_t TIM_parse(const char *p);
+double TIM_parse(const char *p);
 double TIM_mono(void);
 double TIM_real(void);
 void TIM_sleep(double t);
diff --git a/lib/libvarnish/cli_serve.c b/lib/libvarnish/cli_serve.c
index 854c2ff..b7c52f4 100644
--- a/lib/libvarnish/cli_serve.c
+++ b/lib/libvarnish/cli_serve.c
@@ -36,6 +36,7 @@
 #include <string.h>
 #include <stdlib.h>
 #include <stdio.h>
+#include <time.h>
 #include <unistd.h>
 
 #include "vqueue.h"
diff --git a/lib/libvarnish/time.c b/lib/libvarnish/time.c
index 8242853..31fc284 100644
--- a/lib/libvarnish/time.c
+++ b/lib/libvarnish/time.c
@@ -116,10 +116,10 @@ static const char *fmts[] = {
 	NULL
 };
 
-time_t
+double
 TIM_parse(const char *p)
 {
-	time_t t;
+	double t;
 	struct tm tm;
 	const char **r;
 
diff --git a/lib/libvarnish/vmb.c b/lib/libvarnish/vmb.c
index 7464bbe..2c004ce 100644
--- a/lib/libvarnish/vmb.c
+++ b/lib/libvarnish/vmb.c
@@ -27,6 +27,7 @@
  */
 
 #include <pthread.h>
+#include <unistd.h>
 #include "libvarnish.h"
 #include "vmb.h"
 



From geoff at varnish-cache.org  Mon Jan  9 20:52:42 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:42 +0100
Subject: [experimental-ims] 0034032 Fix VSL no longer working
Message-ID: <E1RkMCw-0008Re-QC@project.varnish-software.com>

commit 0034032ba77f9e58895fbe0a9a8bfed0f3b5df8c
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Nov 28 10:56:22 2011 +0000

    Fix VSL no longer working

diff --git a/bin/varnishtest/vtc_varnish.c b/bin/varnishtest/vtc_varnish.c
index 2f7514a..880f966 100644
--- a/bin/varnishtest/vtc_varnish.c
+++ b/bin/varnishtest/vtc_varnish.c
@@ -201,7 +201,7 @@ varnishlog_thread(void *priv)
 	(void)VSL_Arg(vsl, 'n', v->workdir);
 	while (v->pid) {
 		if (VSL_Dispatch(vsl, h_addlog, v) <= 0)
-			break;
+			usleep(100000);
 	}
 	VSM_Delete(vsl);
 	return (NULL);



From geoff at varnish-cache.org  Mon Jan  9 20:52:00 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:00 +0100
Subject: [experimental-ims] 223ff71 Do the zero check last,
	so a varnishd restart has a chance to recover it.
Message-ID: <E1RkMCG-00078H-5H@project.varnish-software.com>

commit 223ff71a0ce19a7d33d3e06f800fa6e7b8a713e4
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Sep 20 15:25:44 2011 +0000

    Do the zero check last, so a varnishd restart has a chance to recover it.

diff --git a/lib/libvarnishapi/vsl.c b/lib/libvarnishapi/vsl.c
index 1bc46dd..58a454d 100644
--- a/lib/libvarnishapi/vsl.c
+++ b/lib/libvarnishapi/vsl.c
@@ -173,13 +173,6 @@ vsl_nextlog(struct vsl *vsl, uint32_t **pp)
 	for (w = 0; w < TIMEOUT_USEC;) {
 		t = *vsl->log_ptr;
 
-		if (t == 0) {
-			/* Zero-initialized VSL */
-			w += SLEEP_USEC;
-			assert(usleep(SLEEP_USEC) == 0 || errno == EINTR);
-			VRMB();
-			continue;
-		}
 		if (t == VSL_WRAPMARKER) {
 			/* Wrap around not possible at front */
 			assert(vsl->log_ptr != vsl->log_start + 1);
@@ -202,6 +195,13 @@ vsl_nextlog(struct vsl *vsl, uint32_t **pp)
 			VRMB();
 			continue;
 		}
+		if (t == 0) {
+			/* Zero-initialized VSL */
+			w += SLEEP_USEC;
+			assert(usleep(SLEEP_USEC) == 0 || errno == EINTR);
+			VRMB();
+			continue;
+		}
 		if (vsl->log_ptr == vsl->log_start + 1)
 			vsl->last_seq = vsl->log_start[0];
 



From geoff at varnish-cache.org  Mon Jan  9 20:51:48 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:51:48 +0100
Subject: [experimental-ims] 87ae60f Merge branch 'master' of
	ssh://git.varnish-cache.org/git/varnish-cache
Message-ID: <E1RkMC4-0006oG-9g@project.varnish-software.com>

commit 87ae60f887d7b487be548898b64ac8183b797187
Merge: c717971 e13b98c
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Sep 12 07:36:51 2011 +0000

    Merge branch 'master' of ssh://git.varnish-cache.org/git/varnish-cache



From geoff at varnish-cache.org  Mon Jan  9 20:52:42 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:42 +0100
Subject: [experimental-ims] 699607a Explain more details about VSM
Message-ID: <E1RkMCw-0008QJ-8z@project.varnish-software.com>

commit 699607a0bcc7b7aed43cf83ef8746771c8a7d754
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu Nov 24 10:50:36 2011 +0000

    Explain more details about VSM

diff --git a/doc/sphinx/reference/vsm.rst b/doc/sphinx/reference/vsm.rst
index 26c5525..cdae8e7 100644
--- a/doc/sphinx/reference/vsm.rst
+++ b/doc/sphinx/reference/vsm.rst
@@ -1,10 +1,36 @@
-%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-Shared Memory Logging and Statistics
-%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+VSM: Shared Memory Logging and Statistics
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 
-Varnish uses shared memory for logging and statistics, because it
-is faster and much more efficient.  But it is also tricky in ways
-a regular logfile is not.
+Varnish uses shared memory to export parameters, logging and
+statistics, because it is faster and much more efficient than
+regular files.
+
+"Varnish Shared Memory" or VSM, is the overall mechanism, which
+manages a number of allocated "chunks" inside the same shared
+memory file.
+
+Each Chunk is just a slap of memory, which has
+a three-part name (class, type, ident) and a length.
+
+The Class indicates what type of data is stored in the chunk,
+for instance "Arg" for command line arguments useful for
+establishing an CLI connection to the varnishd, "Stat" for
+statistics counters (VSC) and "Log" for log records (VSL).
+
+The type and ident name parts are mostly used with stats
+counters, where they identify dynamic counters, such as:
+
+	SMA.Transient.c_bytes
+
+The size of the VSM is a parameter, but changes only take
+effect when the child process is restarted.
+
+Shared memory trickery
+----------------------
+
+Shared memory is faster than regular files, but it is also slightly
+tricky in ways a regular logfile is not.
 
 When you open a file in "append" mode, the operating system guarantees
 that whatever you write will not overwrite existing data in the file.
@@ -12,12 +38,14 @@ The neat result of this is that multiple procesess or threads writing
 to the same file does not even need to know about each other, it all
 works just as you would expect.
 
-With a shared memory log, we get no help from the kernel, the writers
-need to make sure they do not stomp on each other, and they need to
-make it possible and safe for the readers to access the data.
+With a shared memory log, we get no such help from the kernel, the
+writers need to make sure they do not stomp on each other, and they
+need to make it possible and safe for the readers to access the
+data.
 
-The "CS101" way, is to introduce locks, and much time is spent examining
-the relative merits of the many kinds of locks available.
+The "CS101" way to deal with that, is to introduce locks, and much
+time is spent examining the relative merits of the many kinds of
+locks available.
 
 Inside the varnishd (worker) process, we use mutexes to guarantee
 consistency, both with respect to allocations, log entries and stats
@@ -36,30 +64,41 @@ stuff, such as when a backend is taken out of the configuration,
 we need to give the readers a chance to discover this, a "cooling
 off" period.
 
-When Varnishd starts, if it finds an existing shared memory file,
-and it can safely read the master_pid field, it will check if that
-process is running, and if so, fail with an error message, indicating
-that -n arguments collide.
+The Varnish way:
+----------------
+
+If Varnishd starts, and finds a locked shared memory file, it will
+exit with a message about using different -n arguments if you want
+multiple instances of varnishd.
+
+Otherwise, it will create a new shared memory file each time it
+starts a child process, since that marks a clean break in operation
+anyway.
 
-In all other cases, it will delete and create a new shmlog file,
-in order to provide running readers a cooling off period, where
-they can discover that there is a new shmlog file, by doing a
-stat(2) call and checking the st_dev & st_inode fields.
+To the extent possible, old shared memory files are marked as
+abandoned by setting the alloc_seq field to zero, which should be
+monitored by all readers of the VSM.
 
-Allocations
------------
+Processes subscribing to VSM files for a long time, should notice
+if the VSM file goes "silent" and check that the file has not been
+renamed due to a child restart.
 
-Sections inside the shared memory file are allocated dynamically,
-for instance when a new backend is added.
+Chunks inside the shared memory file form a linked list, and whenever
+that list changes, the alloc_seq field changes.
 
-While changes happen to the linked list of allocations, the "alloc_seq"
-header field is zero, and after the change, it gets a value different
-from what it had before.
+The linked list and other metadata in the VSM file, works with
+offsets relative to the start address of where the VSM file is
+memory mapped, so it need not be mapped at any particular address.
 
-Deallocations
--------------
+When new chunks are allocated, for instance when a new backend is
+added, they are appended to the list, no matter where they are
+located in the VSM file.
 
-When a section is freed, its class will change to "Cool" for at
-least 10 seconds, giving programs using it time to detect the 
-change in alloc_seq header field and/or the change of class.
+When a chunk is freed, it will be taken out of the linked list of
+allocations, its length will be set to zero and alloc_seq will be
+changed to indicate a change of layout.  For the next 60 seconds
+the chunk will not be touched or reused, giving other subscribers
+a chance to discover the deallocation.
 
+The include file <vapi/vsm.h> provides the supported API for accessing
+VSM files.



From geoff at varnish-cache.org  Mon Jan  9 20:52:17 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:17 +0100
Subject: [experimental-ims] 92f239c Get the vcli includes into line
Message-ID: <E1RkMCX-0007eG-Jt@project.varnish-software.com>

commit 92f239c0fa236362ec0e6538efccbc71c11bfa91
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Oct 10 11:33:19 2011 +0000

    Get the vcli includes into line

diff --git a/bin/varnishd/cache_acceptor.c b/bin/varnishd/cache_acceptor.c
index 80d7b81..e6ad6e3 100644
--- a/bin/varnishd/cache_acceptor.c
+++ b/bin/varnishd/cache_acceptor.c
@@ -32,8 +32,8 @@
 
 #include "cache.h"
 
-#include "cli_priv.h"
 #include "vcli.h"
+#include "vcli_priv.h"
 #include "vtcp.h"
 #include "vtim.h"
 
diff --git a/bin/varnishd/cache_backend_cfg.c b/bin/varnishd/cache_backend_cfg.c
index 2e6d83c..b10512d 100644
--- a/bin/varnishd/cache_backend_cfg.c
+++ b/bin/varnishd/cache_backend_cfg.c
@@ -38,7 +38,7 @@
 #include "cache.h"
 
 #include "cache_backend.h"
-#include "cli_priv.h"
+#include "vcli_priv.h"
 #include "vrt.h"
 
 struct lock VBE_mtx;
diff --git a/bin/varnishd/cache_backend_poll.c b/bin/varnishd/cache_backend_poll.c
index a0ad62b..d79a806 100644
--- a/bin/varnishd/cache_backend_poll.c
+++ b/bin/varnishd/cache_backend_poll.c
@@ -45,7 +45,7 @@
 #include "cache.h"
 
 #include "cache_backend.h"
-#include "cli_priv.h"
+#include "vcli_priv.h"
 #include "vrt.h"
 #include "vtcp.h"
 #include "vtim.h"
diff --git a/bin/varnishd/cache_ban.c b/bin/varnishd/cache_ban.c
index bec8d85..d556ace 100644
--- a/bin/varnishd/cache_ban.c
+++ b/bin/varnishd/cache_ban.c
@@ -67,9 +67,9 @@
 
 #include "cache.h"
 
-#include "cli_priv.h"
 #include "hash_slinger.h"
 #include "vcli.h"
+#include "vcli_priv.h"
 #include "vend.h"
 #include "vtim.h"
 
diff --git a/bin/varnishd/cache_center.c b/bin/varnishd/cache_center.c
index ba61e9c..463a05b 100644
--- a/bin/varnishd/cache_center.c
+++ b/bin/varnishd/cache_center.c
@@ -65,10 +65,10 @@ DOT acceptor -> start [style=bold,color=green]
 
 #include "cache.h"
 
-#include "cli_priv.h"
 #include "hash_slinger.h"
 #include "stevedore.h"
 #include "vcl.h"
+#include "vcli_priv.h"
 #include "vsha256.h"
 #include "vtcp.h"
 #include "vtim.h"
diff --git a/bin/varnishd/cache_cli.c b/bin/varnishd/cache_cli.c
index a278e84..ea4d367 100644
--- a/bin/varnishd/cache_cli.c
+++ b/bin/varnishd/cache_cli.c
@@ -40,10 +40,10 @@
 
 #include "cache.h"
 
-#include "vcli.h"
-#include "cli_common.h"
-#include "cli_priv.h"
 #include "hash_slinger.h"		// objhead
+#include "vcli.h"
+#include "vcli_common.h"
+#include "vcli_priv.h"
 #include "vcli_serve.h"
 
 pthread_t		cli_thread;
diff --git a/bin/varnishd/cache_fetch.c b/bin/varnishd/cache_fetch.c
index 4bbc272..75319d8 100644
--- a/bin/varnishd/cache_fetch.c
+++ b/bin/varnishd/cache_fetch.c
@@ -36,8 +36,8 @@
 
 #include "cache.h"
 
-#include "cli_priv.h"
 #include "stevedore.h"
+#include "vcli_priv.h"
 #include "vct.h"
 #include "vtcp.h"
 
diff --git a/bin/varnishd/cache_vcl.c b/bin/varnishd/cache_vcl.c
index 48568ff..c77ab08 100644
--- a/bin/varnishd/cache_vcl.c
+++ b/bin/varnishd/cache_vcl.c
@@ -39,10 +39,10 @@
 
 #include "cache.h"
 
-#include "cli_priv.h"
 #include "libvcl.h"
 #include "vcl.h"
 #include "vcli.h"
+#include "vcli_priv.h"
 
 struct vcls {
 	unsigned		magic;
diff --git a/bin/varnishd/cache_vrt_vmod.c b/bin/varnishd/cache_vrt_vmod.c
index bb4eeb8..6b3b846 100644
--- a/bin/varnishd/cache_vrt_vmod.c
+++ b/bin/varnishd/cache_vrt_vmod.c
@@ -36,7 +36,7 @@
 
 #include "cache.h"
 
-#include "cli_priv.h"
+#include "vcli_priv.h"
 #include "vrt.h"
 
 /*--------------------------------------------------------------------
diff --git a/bin/varnishd/cache_waiter.c b/bin/varnishd/cache_waiter.c
index 08c5809..9c30edf 100644
--- a/bin/varnishd/cache_waiter.c
+++ b/bin/varnishd/cache_waiter.c
@@ -33,8 +33,8 @@
 #include "cache.h"
 
 #include "cache_waiter.h"
-#include "cli_priv.h"
 #include "vcli.h"
+#include "vcli_priv.h"
 
 static const struct waiter * const vca_waiters[] = {
     #if defined(HAVE_KQUEUE)
diff --git a/bin/varnishd/hash_critbit.c b/bin/varnishd/hash_critbit.c
index add2397..78bda7a 100644
--- a/bin/varnishd/hash_critbit.c
+++ b/bin/varnishd/hash_critbit.c
@@ -36,8 +36,8 @@
 
 #include "cache.h"
 
-#include "cli_priv.h"
 #include "hash_slinger.h"
+#include "vcli_priv.h"
 #include "vmb.h"
 #include "vtim.h"
 
diff --git a/bin/varnishd/mgt_child.c b/bin/varnishd/mgt_child.c
index 7338e6c..5af26fe 100644
--- a/bin/varnishd/mgt_child.c
+++ b/bin/varnishd/mgt_child.c
@@ -46,18 +46,19 @@
 
 #include "mgt.h"
 
-#include "cli_priv.h"
 #include "heritage.h"
-#include "mgt_cli.h"
 #include "vapi/vsm_int.h"
 #include "vbm.h"
 #include "vcli.h"
+#include "vcli_priv.h"
 #include "vev.h"
 #include "vlu.h"
 #include "vss.h"
 #include "vtcp.h"
 #include "vtim.h"
 
+#include "mgt_cli.h"
+
 pid_t		child_pid = -1;
 
 
diff --git a/bin/varnishd/mgt_cli.c b/bin/varnishd/mgt_cli.c
index f1bb2d8..dea5e7f 100644
--- a/bin/varnishd/mgt_cli.c
+++ b/bin/varnishd/mgt_cli.c
@@ -44,17 +44,18 @@
 
 #include "mgt.h"
 
-#include "vcli.h"
-#include "cli_common.h"
-#include "cli_priv.h"
 #include "heritage.h"
-#include "mgt_cli.h"
+#include "vcli.h"
+#include "vcli_common.h"
+#include "vcli_priv.h"
 #include "vcli_serve.h"
 #include "vev.h"
 #include "vlu.h"
 #include "vss.h"
 #include "vtcp.h"
 
+#include "mgt_cli.h"
+
 #ifndef HAVE_SRANDOMDEV
 #include "compat/srandomdev.h"
 #endif
diff --git a/bin/varnishd/mgt_param.c b/bin/varnishd/mgt_param.c
index 4bb1adf..9ded195 100644
--- a/bin/varnishd/mgt_param.c
+++ b/bin/varnishd/mgt_param.c
@@ -39,16 +39,17 @@
 
 #include "mgt.h"
 
-#include "vcli.h"
 #include "cache_waiter.h"
-#include "cli_common.h"
-#include "cli_priv.h"
 #include "heritage.h"
-#include "mgt_cli.h"
 #include "vav.h"
+#include "vcli.h"
+#include "vcli_common.h"
+#include "vcli_priv.h"
 #include "vparam.h"
 #include "vss.h"
 
+#include "mgt_cli.h"
+
 #define MAGIC_INIT_STRING	"\001"
 struct params master;
 static int nparspec;
diff --git a/bin/varnishd/mgt_vcc.c b/bin/varnishd/mgt_vcc.c
index 78a29dd..5901408 100644
--- a/bin/varnishd/mgt_vcc.c
+++ b/bin/varnishd/mgt_vcc.c
@@ -40,14 +40,15 @@
 
 #include "mgt.h"
 
-#include "cli_priv.h"
 #include "libvcl.h"
-#include "mgt_cli.h"
 #include "vcl.h"
 #include "vcli.h"
+#include "vcli_priv.h"
 #include "vfil.h"
 #include "vsub.h"
 
+#include "mgt_cli.h"
+
 struct vclprog {
 	VTAILQ_ENTRY(vclprog)	list;
 	char			*name;
diff --git a/bin/varnishd/stevedore.c b/bin/varnishd/stevedore.c
index 538ae27..ff1bdf5 100644
--- a/bin/varnishd/stevedore.c
+++ b/bin/varnishd/stevedore.c
@@ -38,9 +38,9 @@
 
 #include "cache.h"
 
-#include "cli_priv.h"
 #include "stevedore.h"
 #include "vav.h"
+#include "vcli_priv.h"
 #include "vrt.h"
 #include "vrt_obj.h"
 
diff --git a/bin/varnishd/storage_persistent.c b/bin/varnishd/storage_persistent.c
index d3ab02e..d379c4a 100644
--- a/bin/varnishd/storage_persistent.c
+++ b/bin/varnishd/storage_persistent.c
@@ -45,10 +45,10 @@
 
 #include "cache.h"
 
-#include "cli_priv.h"
 #include "hash_slinger.h"
 #include "stevedore.h"
 #include "vcli.h"
+#include "vcli_priv.h"
 #include "vend.h"
 #include "vsha256.h"
 
diff --git a/bin/varnishd/varnishd.c b/bin/varnishd/varnishd.c
index 7cecfe4..722c10e 100644
--- a/bin/varnishd/varnishd.c
+++ b/bin/varnishd/varnishd.c
@@ -46,12 +46,12 @@
 
 #include "mgt.h"
 
-#include "vcli.h"
-#include "cli_common.h"
 #include "hash_slinger.h"
 #include "heritage.h"
 #include "stevedore.h"
 #include "vav.h"
+#include "vcli.h"
+#include "vcli_common.h"
 #include "vev.h"
 #include "vfil.h"
 #include "vin.h"
diff --git a/include/Makefile.am b/include/Makefile.am
index 9c92b1e..62292fc 100644
--- a/include/Makefile.am
+++ b/include/Makefile.am
@@ -26,8 +26,6 @@ pkginclude_HEADERS = \
 
 nobase_noinst_HEADERS = \
 	binary_heap.h \
-	cli_common.h \
-	cli_priv.h \
 	compat/daemon.h \
 	compat/execinfo.h \
 	compat/srandomdev.h \
@@ -39,6 +37,8 @@ nobase_noinst_HEADERS = \
 	vav.h \
 	vbm.h \
 	vcl.h \
+	vcli_common.h \
+	vcli_priv.h \
 	vcli_serve.h \
 	vcs_version.h \
 	vcs.h \
diff --git a/include/cli_common.h b/include/cli_common.h
deleted file mode 100644
index 80d680f..0000000
--- a/include/cli_common.h
+++ /dev/null
@@ -1,45 +0,0 @@
-/*-
- * Copyright (c) 2006 Verdens Gang AS
- * Copyright (c) 2006-2011 Varnish Software AS
- * All rights reserved.
- *
- * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- *
- */
-
-struct vlu;
-struct VCLS;
-
-struct cli {
-	unsigned		magic;
-#define CLI_MAGIC		0x4038d570
-	struct vsb		*sb;
-	enum VCLI_status_e	result;
-	char			*cmd;
-	unsigned		auth;
-	char			challenge[34];
-	char			*ident;
-	struct vlu		*vlu;
-	struct VCLS		*cls;
-};
diff --git a/include/cli_priv.h b/include/cli_priv.h
deleted file mode 100644
index 60d6d69..0000000
--- a/include/cli_priv.h
+++ /dev/null
@@ -1,57 +0,0 @@
-/*-
- * Copyright (c) 2006 Verdens Gang AS
- * Copyright (c) 2006-2011 Varnish Software AS
- * All rights reserved.
- *
- * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- *
- * Varnish process internal CLI stuff.
- *
- * XXX: at a latter date we may want to move some to cli.h/libvarnishapi
- */
-
-#define CLI_PRIV_H
-
-struct cli;	/* NB: struct cli is opaque at this level.  */
-
-typedef void cli_func_t(struct cli*, const char * const *av, void *priv);
-
-struct cli_proto {
-	/* These must match the CLI_* macros in cli.h */
-	const char		*request;
-	const char		*syntax;
-	const char		*help;
-	unsigned		minarg;
-	unsigned		maxarg;
-	char			flags[4];
-
-	/* Dispatch information */
-	cli_func_t		*func;
-	void			*priv;
-};
-
-/* The implementation must provide these functions */
-void VCLI_Out(struct cli *cli, const char *fmt, ...);
-void VCLI_Quote(struct cli *cli, const char *str);
-void VCLI_SetResult(struct cli *cli, unsigned r);
diff --git a/include/vcli_common.h b/include/vcli_common.h
new file mode 100644
index 0000000..80d680f
--- /dev/null
+++ b/include/vcli_common.h
@@ -0,0 +1,45 @@
+/*-
+ * Copyright (c) 2006 Verdens Gang AS
+ * Copyright (c) 2006-2011 Varnish Software AS
+ * All rights reserved.
+ *
+ * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ */
+
+struct vlu;
+struct VCLS;
+
+struct cli {
+	unsigned		magic;
+#define CLI_MAGIC		0x4038d570
+	struct vsb		*sb;
+	enum VCLI_status_e	result;
+	char			*cmd;
+	unsigned		auth;
+	char			challenge[34];
+	char			*ident;
+	struct vlu		*vlu;
+	struct VCLS		*cls;
+};
diff --git a/include/vcli_priv.h b/include/vcli_priv.h
new file mode 100644
index 0000000..60d6d69
--- /dev/null
+++ b/include/vcli_priv.h
@@ -0,0 +1,57 @@
+/*-
+ * Copyright (c) 2006 Verdens Gang AS
+ * Copyright (c) 2006-2011 Varnish Software AS
+ * All rights reserved.
+ *
+ * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * Varnish process internal CLI stuff.
+ *
+ * XXX: at a latter date we may want to move some to cli.h/libvarnishapi
+ */
+
+#define CLI_PRIV_H
+
+struct cli;	/* NB: struct cli is opaque at this level.  */
+
+typedef void cli_func_t(struct cli*, const char * const *av, void *priv);
+
+struct cli_proto {
+	/* These must match the CLI_* macros in cli.h */
+	const char		*request;
+	const char		*syntax;
+	const char		*help;
+	unsigned		minarg;
+	unsigned		maxarg;
+	char			flags[4];
+
+	/* Dispatch information */
+	cli_func_t		*func;
+	void			*priv;
+};
+
+/* The implementation must provide these functions */
+void VCLI_Out(struct cli *cli, const char *fmt, ...);
+void VCLI_Quote(struct cli *cli, const char *str);
+void VCLI_SetResult(struct cli *cli, unsigned r);
diff --git a/lib/libvarnish/cli_common.c b/lib/libvarnish/cli_common.c
index cb4e8af..43b91c3 100644
--- a/lib/libvarnish/cli_common.c
+++ b/lib/libvarnish/cli_common.c
@@ -43,10 +43,10 @@
 #include <time.h>
 #include <unistd.h>
 
-#include "vcli.h"
-#include "cli_common.h"
-#include "cli_priv.h"
 #include "vas.h"
+#include "vcli.h"
+#include "vcli_common.h"
+#include "vcli_priv.h"
 #include "vsb.h"
 
 /*lint -e{818} cli could be const */
diff --git a/lib/libvarnish/cli_serve.c b/lib/libvarnish/cli_serve.c
index 9a9daed..f3def96 100644
--- a/lib/libvarnish/cli_serve.c
+++ b/lib/libvarnish/cli_serve.c
@@ -43,10 +43,10 @@
 #include "miniobj.h"
 #include "vas.h"
 
-#include "vcli.h"
-#include "cli_common.h"
-#include "cli_priv.h"
 #include "vav.h"
+#include "vcli.h"
+#include "vcli_common.h"
+#include "vcli_priv.h"
 #include "vcli_serve.h"
 #include "vlu.h"
 #include "vqueue.h"



From geoff at varnish-cache.org  Mon Jan  9 20:52:32 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:32 +0100
Subject: [experimental-ims] 4f9e61d Explicitly document that concatenation is
	only supported for the builtins.
Message-ID: <E1RkMCm-00088e-47@project.varnish-software.com>

commit 4f9e61d652c42f9a7b8650fd4c2ecbb204db031a
Author: Andreas Plesner Jacobsen <apj at mutt.dk>
Date:   Thu Nov 10 22:49:57 2011 +0100

    Explicitly document that concatenation is only supported for the builtins.
    
    Fixes #1042

diff --git a/doc/sphinx/reference/vcl.rst b/doc/sphinx/reference/vcl.rst
index 743d5f2..016e84f 100644
--- a/doc/sphinx/reference/vcl.rst
+++ b/doc/sphinx/reference/vcl.rst
@@ -640,8 +640,11 @@ default code.
 
 Multiple subroutines
 ~~~~~~~~~~~~~~~~~~~~
-If multiple subroutines with the same name are defined, they are
-concatenated in the order in which the appear in the source.
+If multiple subroutines with the the name of one of the builtin
+ones are defined, they are concatenated in the order in which they
+appear in the source.
+The default versions distributed with Varnish will be implicitly
+concatenated as a last resort at the end.
 
 Example:::
 
@@ -667,8 +670,6 @@ Example:::
 	  }
 	}
 
-The builtin default subroutines are implicitly appended in this way.
-
 Variables
 ~~~~~~~~~
 



From geoff at varnish-cache.org  Mon Jan  9 20:52:32 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:32 +0100
Subject: [experimental-ims] 26cfc94 Move cache/mgt common stuff under,
	you guessed it: common/
Message-ID: <E1RkMCm-00089z-Sl@project.varnish-software.com>

commit 26cfc94ebbff9946dd9dbdabc83382abb9d77e86
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sun Nov 13 10:38:37 2011 +0000

    Move cache/mgt common stuff under, you guessed it: common/

diff --git a/bin/varnishd/Makefile.am b/bin/varnishd/Makefile.am
index 0460303..245bc6b 100644
--- a/bin/varnishd/Makefile.am
+++ b/bin/varnishd/Makefile.am
@@ -45,14 +45,10 @@ varnishd_SOURCES = \
 	cache_vrt_re.c \
 	cache_vrt_var.c \
 	cache_vrt_vmod.c \
-	waiter/cache_waiter.c \
-	waiter/cache_waiter_epoll.c \
-	waiter/cache_waiter_kqueue.c \
-	waiter/cache_waiter_poll.c \
-	waiter/cache_waiter_ports.c \
 	cache_wrk.c \
 	cache_wrw.c \
 	cache_ws.c \
+	common/common_vsm.c \
 	hash/hash_classic.c \
 	hash/hash_critbit.c \
 	hash/hash_mgt.c \
@@ -76,23 +72,27 @@ varnishd_SOURCES = \
 	storage/storage_persistent_subr.c \
 	storage/storage_synth.c \
 	storage/storage_umem.c \
-	vsm.c
+	waiter/cache_waiter.c \
+	waiter/cache_waiter_epoll.c \
+	waiter/cache_waiter_kqueue.c \
+	waiter/cache_waiter_poll.c \
+	waiter/cache_waiter_ports.c 
 
 noinst_HEADERS = \
 	cache.h \
 	cache_backend.h \
 	cache_esi.h \
-	waiter/cache_waiter.h \
-	common.h \
+	common/common.h \
+	common/heritage.h \
+	common/params.h \
 	default_vcl.h \
 	hash/hash_slinger.h \
-	heritage.h \
 	mgt/mgt.h \
 	mgt/mgt_cli.h \
 	mgt/mgt_param.h \
-	params.h \
 	storage/storage.h \
-	storage/storage_persistent.h
+	storage/storage_persistent.h \
+	waiter/cache_waiter.h
 
 varnishd_CFLAGS = \
 	@PCRE_CFLAGS@ \
diff --git a/bin/varnishd/cache.h b/bin/varnishd/cache.h
index b21f2d4..653fe77 100644
--- a/bin/varnishd/cache.h
+++ b/bin/varnishd/cache.h
@@ -34,7 +34,7 @@
  */
 #define VARNISH_CACHE_CHILD	1
 
-#include "common.h"
+#include "common/common.h"
 
 #include "vapi/vsc_int.h"
 #include "vapi/vsl_int.h"
@@ -55,7 +55,7 @@
 #endif
 
 
-#include "params.h"
+#include "common/params.h"
 
 enum body_status {
 #define BODYSTATUS(U,l)	BS_##U,
diff --git a/bin/varnishd/cache_acceptor.c b/bin/varnishd/cache_acceptor.c
index 5161b09..8c83121 100644
--- a/bin/varnishd/cache_acceptor.c
+++ b/bin/varnishd/cache_acceptor.c
@@ -31,7 +31,7 @@
 #include "config.h"
 
 #include "cache.h"
-#include "heritage.h"
+#include "common/heritage.h"
 
 #include "vcli.h"
 #include "vcli_priv.h"
diff --git a/bin/varnishd/cache_cli.c b/bin/varnishd/cache_cli.c
index 0bbf94b..f29f86a 100644
--- a/bin/varnishd/cache_cli.c
+++ b/bin/varnishd/cache_cli.c
@@ -39,7 +39,7 @@
 #include <stddef.h>			// offsetof
 
 #include "cache.h"
-#include "heritage.h"
+#include "common/heritage.h"
 
 #include "cache_backend.h"		// struct vbc
 #include "hash/hash_slinger.h"		// struct objhead
diff --git a/bin/varnishd/cache_main.c b/bin/varnishd/cache_main.c
index 7a3bcd9..eb3fa1d 100644
--- a/bin/varnishd/cache_main.c
+++ b/bin/varnishd/cache_main.c
@@ -33,7 +33,7 @@
 #include <stdlib.h>
 
 #include "cache.h"
-#include "heritage.h"
+#include "common/heritage.h"
 
 #include "waiter/cache_waiter.h"
 #include "hash/hash_slinger.h"
diff --git a/bin/varnishd/cache_pool.c b/bin/varnishd/cache_pool.c
index be49548..79a5fcd 100644
--- a/bin/varnishd/cache_pool.c
+++ b/bin/varnishd/cache_pool.c
@@ -46,7 +46,7 @@
 #include <stdlib.h>
 
 #include "cache.h"
-#include "heritage.h"
+#include "common/heritage.h"
 
 #include "waiter/cache_waiter.h"
 #include "vtcp.h"
diff --git a/bin/varnishd/cache_vrt_var.c b/bin/varnishd/cache_vrt_var.c
index a9bf87f..860c7aa 100644
--- a/bin/varnishd/cache_vrt_var.c
+++ b/bin/varnishd/cache_vrt_var.c
@@ -34,7 +34,7 @@
 #include <stdlib.h>
 
 #include "cache.h"
-#include "heritage.h"
+#include "common/heritage.h"
 
 #include "cache_backend.h"
 #include "vrt_obj.h"
diff --git a/bin/varnishd/common.h b/bin/varnishd/common.h
deleted file mode 100644
index 4e56a40..0000000
--- a/bin/varnishd/common.h
+++ /dev/null
@@ -1,113 +0,0 @@
-/*-
- * Copyright (c) 2006 Verdens Gang AS
- * Copyright (c) 2006-2011 Varnish Software AS
- * All rights reserved.
- *
- * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- *
- */
-
-#include <errno.h>
-#include <stdint.h>
-
-#include <sys/types.h>
-
-#include "miniobj.h"
-#include "vas.h"
-#include "vcs.h"
-#include "vdef.h"
-#include "vqueue.h"
-#include "vsb.h"
-
-struct cli;
-
-extern pid_t mgt_pid;
-#define ASSERT_MGT() do { assert(getpid() == mgt_pid);} while (0)
-
-/* mgt_shmem.c */
-extern struct VSC_C_main *VSC_C_main;
-
-/* varnishd.c */
-struct vsb;
-extern struct vsb *vident;
-int Symbol_Lookup(struct vsb *vsb, void *ptr);
-
-#define TRUST_ME(ptr)	((void*)(uintptr_t)(ptr))
-
-
-/* Help shut up FlexeLint */
-#define __match_proto__(xxx) /*lint -e{818} */
-
-/* Really belongs in mgt.h, but storage_file chokes on both */
-void mgt_child_inherit(int fd, const char *what);
-
-#define ARGV_ERR(...)						\
-	do {							\
-		fprintf(stderr, "Error: " __VA_ARGS__);		\
-		exit(2);					\
-	} while (0);
-
-/* A tiny helper for choosing hash/storage modules */
-struct choice {
-	const char      *name;
-	const void	*ptr;
-};
-const void *pick(const struct choice *cp, const char *which, const char *kind);
-
-#define NEEDLESS_RETURN(foo)	return (foo)
-
-/* stevedore.c */
-void STV_Config(const char *spec);
-void STV_Config_Transient(void);
-
-/* vsm.c */
-// extern struct VSM_head		*VSM_head;
-// extern const struct VSM_chunk	*vsm_end;
-
-/*
- * These three should not be called directly, but only through
- * proper vectors in mgt.h/cache.h, hence the __
- */
-void *VSM__Alloc(unsigned size, const char *class, const char *type,
-    const char *ident);
-void VSM__Free(const void *ptr);
-void VSM__Clean(void);
-
-/* These classes are opaque to other programs, so we define the here */
-#define VSM_CLASS_FREE	"Free"
-#define VSM_CLASS_COOL	"Cool"
-#define VSM_CLASS_PARAM	"Params"
-#define VSM_CLASS_MARK	"MgrCld"
-#define VSM_COOL_TIME	5
-
-/* cache_lck.c */
-struct lock { void *priv; };		// Opaque
-
-/*---------------------------------------------------------------------
- * Generic power-2 rounding macros
- */
-
-#define PWR2(x)     ((((x)-1)&(x))==0)		/* Is a power of two */
-#define RDN2(x, y)  ((x)&(~((y)-1)))		/* if y is powers of two */
-#define RUP2(x, y)  (((x)+((y)-1))&(~((y)-1)))	/* if y is powers of two */
diff --git a/bin/varnishd/common/common.h b/bin/varnishd/common/common.h
new file mode 100644
index 0000000..4e56a40
--- /dev/null
+++ b/bin/varnishd/common/common.h
@@ -0,0 +1,113 @@
+/*-
+ * Copyright (c) 2006 Verdens Gang AS
+ * Copyright (c) 2006-2011 Varnish Software AS
+ * All rights reserved.
+ *
+ * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ */
+
+#include <errno.h>
+#include <stdint.h>
+
+#include <sys/types.h>
+
+#include "miniobj.h"
+#include "vas.h"
+#include "vcs.h"
+#include "vdef.h"
+#include "vqueue.h"
+#include "vsb.h"
+
+struct cli;
+
+extern pid_t mgt_pid;
+#define ASSERT_MGT() do { assert(getpid() == mgt_pid);} while (0)
+
+/* mgt_shmem.c */
+extern struct VSC_C_main *VSC_C_main;
+
+/* varnishd.c */
+struct vsb;
+extern struct vsb *vident;
+int Symbol_Lookup(struct vsb *vsb, void *ptr);
+
+#define TRUST_ME(ptr)	((void*)(uintptr_t)(ptr))
+
+
+/* Help shut up FlexeLint */
+#define __match_proto__(xxx) /*lint -e{818} */
+
+/* Really belongs in mgt.h, but storage_file chokes on both */
+void mgt_child_inherit(int fd, const char *what);
+
+#define ARGV_ERR(...)						\
+	do {							\
+		fprintf(stderr, "Error: " __VA_ARGS__);		\
+		exit(2);					\
+	} while (0);
+
+/* A tiny helper for choosing hash/storage modules */
+struct choice {
+	const char      *name;
+	const void	*ptr;
+};
+const void *pick(const struct choice *cp, const char *which, const char *kind);
+
+#define NEEDLESS_RETURN(foo)	return (foo)
+
+/* stevedore.c */
+void STV_Config(const char *spec);
+void STV_Config_Transient(void);
+
+/* vsm.c */
+// extern struct VSM_head		*VSM_head;
+// extern const struct VSM_chunk	*vsm_end;
+
+/*
+ * These three should not be called directly, but only through
+ * proper vectors in mgt.h/cache.h, hence the __
+ */
+void *VSM__Alloc(unsigned size, const char *class, const char *type,
+    const char *ident);
+void VSM__Free(const void *ptr);
+void VSM__Clean(void);
+
+/* These classes are opaque to other programs, so we define the here */
+#define VSM_CLASS_FREE	"Free"
+#define VSM_CLASS_COOL	"Cool"
+#define VSM_CLASS_PARAM	"Params"
+#define VSM_CLASS_MARK	"MgrCld"
+#define VSM_COOL_TIME	5
+
+/* cache_lck.c */
+struct lock { void *priv; };		// Opaque
+
+/*---------------------------------------------------------------------
+ * Generic power-2 rounding macros
+ */
+
+#define PWR2(x)     ((((x)-1)&(x))==0)		/* Is a power of two */
+#define RDN2(x, y)  ((x)&(~((y)-1)))		/* if y is powers of two */
+#define RUP2(x, y)  (((x)+((y)-1))&(~((y)-1)))	/* if y is powers of two */
diff --git a/bin/varnishd/common/common_vsm.c b/bin/varnishd/common/common_vsm.c
new file mode 100644
index 0000000..5298381
--- /dev/null
+++ b/bin/varnishd/common/common_vsm.c
@@ -0,0 +1,238 @@
+/*-
+ * Copyright (c) 2010-2011 Varnish Software AS
+ * All rights reserved.
+ *
+ * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * VSM stuff common to manager and child.
+ *
+ * We have three potential conflicts we need to lock against here:
+ *
+ * VSM-studying programs (varnishstat...) vs. everybody else
+ *	The VSM studying programs only have read-only access to the VSM
+ *	so everybody else must use memory barriers, stable storage and
+ *	similar tricks to keep the VSM image in sync (long enough) for
+ *	the studying programs.
+ *
+ * Manager process vs child process.
+ *	Will only muck about in VSM when child process is not running
+ *	Responsible for cleaning up any mess left behind by dying child.
+ *
+ * Child process threads
+ *	Pthread locking necessary.
+ *
+ * XXX: not all of this is in place yet.
+ */
+
+#include "config.h"
+
+#include <stdint.h>
+#include <stdio.h>
+#include <string.h>
+#include <unistd.h>
+
+#include "common.h"
+
+#include "vapi/vsm_int.h"
+#include "vmb.h"
+#include "vtim.h"
+
+/* These two come from beyond (mgt_shmem.c actually) */
+struct VSM_head		*VSM_head;
+const struct VSM_chunk	*vsm_end;
+
+static unsigned
+vsm_mark(void)
+{
+	unsigned seq;
+
+	seq = VSM_head->alloc_seq;
+	VSM_head->alloc_seq = 0;
+	VWMB();
+	return (seq);
+}
+
+static void
+vsm_release(unsigned seq)
+{
+
+	if (seq == 0)
+		return;
+	VWMB();
+	do
+		VSM_head->alloc_seq = ++seq;
+	while (VSM_head->alloc_seq == 0);
+}
+
+/*--------------------------------------------------------------------*/
+
+static void
+vsm_cleanup(void)
+{
+	unsigned now = (unsigned)VTIM_mono();
+	struct VSM_chunk *sha, *sha2;
+	unsigned seq;
+
+	CHECK_OBJ_NOTNULL(VSM_head, VSM_HEAD_MAGIC);
+	VSM_ITER(sha) {
+		if (strcmp(sha->class, VSM_CLASS_COOL))
+			continue;
+		if (sha->state + VSM_COOL_TIME < now)
+			break;
+	}
+	if (sha == NULL)
+		return;
+	seq = vsm_mark();
+	/* First pass, free, and collapse with next if applicable */
+	VSM_ITER(sha) {
+		if (strcmp(sha->class, VSM_CLASS_COOL))
+			continue;
+		if (sha->state + VSM_COOL_TIME >= now)
+			continue;
+
+		bprintf(sha->class, "%s", VSM_CLASS_FREE);
+		bprintf(sha->type, "%s", "");
+		bprintf(sha->ident, "%s", "");
+		sha2 = VSM_NEXT(sha);
+		assert(sha2 <= vsm_end);
+		if (sha2 == vsm_end)
+			break;
+		CHECK_OBJ_NOTNULL(sha2, VSM_CHUNK_MAGIC);
+		if (!strcmp(sha2->class, VSM_CLASS_FREE)) {
+			sha->len += sha2->len;
+			memset(sha2, 0, sizeof *sha2);
+		}
+		sha->state = 0;
+	}
+	/* Second pass, collaps with prev if applicable */
+	VSM_ITER(sha) {
+		if (strcmp(sha->class, VSM_CLASS_FREE))
+			continue;
+		sha2 = VSM_NEXT(sha);
+		assert(sha2 <= vsm_end);
+		if (sha2 == vsm_end)
+			break;
+		CHECK_OBJ_NOTNULL(sha2, VSM_CHUNK_MAGIC);
+		if (!strcmp(sha2->class, VSM_CLASS_FREE)) {
+			sha->len += sha2->len;
+			memset(sha2, 0, sizeof *sha2);
+		}
+	}
+	vsm_release(seq);
+}
+
+/*--------------------------------------------------------------------*/
+
+void *
+VSM__Alloc(unsigned size, const char *class, const char *type, const char *ident)
+{
+	struct VSM_chunk *sha, *sha2;
+	unsigned seq;
+
+	CHECK_OBJ_NOTNULL(VSM_head, VSM_HEAD_MAGIC);
+
+	vsm_cleanup();
+
+	/* Round up to pointersize */
+	size = RUP2(size, sizeof(void*));
+
+	size += sizeof *sha;		/* Make space for the header */
+
+	VSM_ITER(sha) {
+		CHECK_OBJ_NOTNULL(sha, VSM_CHUNK_MAGIC);
+
+		if (strcmp(sha->class, VSM_CLASS_FREE))
+			continue;
+
+		if (size > sha->len)
+			continue;
+
+		/* Mark as inconsistent while we write string fields */
+		seq = vsm_mark();
+
+		if (size + sizeof (*sha) < sha->len) {
+			sha2 = (void*)((uintptr_t)sha + size);
+
+			memset(sha2, 0, sizeof *sha2);
+			sha2->magic = VSM_CHUNK_MAGIC;
+			sha2->len = sha->len - size;
+			bprintf(sha2->class, "%s", VSM_CLASS_FREE);
+			sha->len = size;
+		}
+
+		bprintf(sha->class, "%s", class);
+		bprintf(sha->type, "%s", type);
+		bprintf(sha->ident, "%s", ident);
+
+		vsm_release(seq);
+		return (VSM_PTR(sha));
+	}
+	return (NULL);
+}
+
+/*--------------------------------------------------------------------*/
+
+void
+VSM__Free(const void *ptr)
+{
+	struct VSM_chunk *sha;
+	unsigned seq;
+
+	CHECK_OBJ_NOTNULL(VSM_head, VSM_HEAD_MAGIC);
+	VSM_ITER(sha)
+		if (VSM_PTR(sha) == ptr)
+			break;
+	AN(sha);
+	seq = vsm_mark();
+	bprintf(sha->class, "%s", VSM_CLASS_COOL);
+	sha->state = (unsigned)VTIM_mono();
+	vsm_release(seq);
+}
+
+/*--------------------------------------------------------------------
+ * Free all allocations after the mark (ie: allocated by child).
+ */
+
+void
+VSM__Clean(void)
+{
+	struct VSM_chunk *sha;
+	unsigned f, seq;
+
+	CHECK_OBJ_NOTNULL(VSM_head, VSM_HEAD_MAGIC);
+	f = 0;
+	seq = vsm_mark();
+	VSM_ITER(sha) {
+		if (f == 0 && !strcmp(sha->class, VSM_CLASS_MARK)) {
+			f = 1;
+			continue;
+		}
+		if (f == 0)
+			continue;
+		if (strcmp(sha->class, VSM_CLASS_FREE) &&
+		    strcmp(sha->class, VSM_CLASS_COOL))
+			VSM__Free(VSM_PTR(sha));
+	}
+	vsm_release(seq);
+}
diff --git a/bin/varnishd/common/heritage.h b/bin/varnishd/common/heritage.h
new file mode 100644
index 0000000..36433bb
--- /dev/null
+++ b/bin/varnishd/common/heritage.h
@@ -0,0 +1,65 @@
+/*-
+ * Copyright (c) 2006 Verdens Gang AS
+ * Copyright (c) 2006-2011 Varnish Software AS
+ * All rights reserved.
+ *
+ * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * This file contains the heritage passed when mgt forks cache
+ */
+
+struct listen_sock {
+	unsigned			magic;
+#define LISTEN_SOCK_MAGIC		0x999e4b57
+	VTAILQ_ENTRY(listen_sock)	list;
+	int				sock;
+	char				*name;
+	struct vss_addr			*addr;
+};
+
+VTAILQ_HEAD(listen_sock_head, listen_sock);
+
+struct heritage {
+
+	/* Two pipe(2)'s for CLI connection between cache and mgt.  */
+	int				cli_in;
+	int				cli_out;
+
+	/* File descriptor for stdout/stderr */
+	int				std_fd;
+
+	/* Sockets from which to accept connections */
+	struct listen_sock_head		socks;
+	unsigned			nsocks;
+
+	/* Hash method */
+	const struct hash_slinger	*hash;
+
+	char				*name;
+	char                            identity[1024];
+};
+
+extern struct heritage heritage;
+
+void child_main(void);
diff --git a/bin/varnishd/common/params.h b/bin/varnishd/common/params.h
new file mode 100644
index 0000000..fb71e33
--- /dev/null
+++ b/bin/varnishd/common/params.h
@@ -0,0 +1,194 @@
+/*-
+ * Copyright (c) 2006 Verdens Gang AS
+ * Copyright (c) 2006-2011 Varnish Software AS
+ * All rights reserved.
+ *
+ * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * This file contains the heritage passed when mgt forks cache
+ */
+
+#include "vre.h"
+
+struct params {
+
+	/* Unprivileged user / group */
+	char			*user;
+	uid_t			uid;
+	char			*group;
+	gid_t			gid;
+
+	/* TTL used for lack of anything better */
+	double			default_ttl;
+
+	/* Default grace period */
+	double			default_grace;
+
+	/* Default keep period */
+	double			default_keep;
+
+	/* Maximum concurrent sessions */
+	unsigned		max_sess;
+
+	/* Worker threads and pool */
+	unsigned		wthread_min;
+	unsigned		wthread_max;
+	unsigned		wthread_timeout;
+	unsigned		wthread_pools;
+	unsigned		wthread_add_threshold;
+	unsigned		wthread_add_delay;
+	unsigned		wthread_fail_delay;
+	unsigned		wthread_purge_delay;
+	unsigned		wthread_stats_rate;
+	unsigned		wthread_stacksize;
+	unsigned		wthread_workspace;
+
+	unsigned		queue_max;
+
+	/* Memory allocation hints */
+	unsigned		sess_workspace;
+	unsigned		shm_workspace;
+	unsigned		http_req_size;
+	unsigned		http_req_hdr_len;
+	unsigned		http_resp_size;
+	unsigned		http_resp_hdr_len;
+	unsigned		http_max_hdr;
+
+	unsigned		shm_reclen;
+
+	/* Acceptor hints */
+	unsigned		sess_timeout;
+	unsigned		pipe_timeout;
+	unsigned		send_timeout;
+	unsigned		idle_send_timeout;
+
+	/* Management hints */
+	unsigned		auto_restart;
+
+	/* Fetcher hints */
+	unsigned		fetch_chunksize;
+	unsigned		fetch_maxchunksize;
+	unsigned		nuke_limit;
+
+#ifdef SENDFILE_WORKS
+	/* Sendfile object minimum size */
+	unsigned		sendfile_threshold;
+#endif
+
+	/* VCL traces */
+	unsigned		vcl_trace;
+
+	/* Listen address */
+	char			*listen_address;
+
+	/* Listen depth */
+	unsigned		listen_depth;
+
+	/* CLI related */
+	unsigned		cli_timeout;
+	unsigned		ping_interval;
+
+	/* LRU list ordering interval */
+	unsigned		lru_timeout;
+
+	/* Maximum restarts allowed */
+	unsigned		max_restarts;
+
+	/* Maximum esi:include depth allowed */
+	unsigned		max_esi_depth;
+
+	/* ESI parser hints */
+	unsigned		esi_syntax;
+
+	/* Rush exponent */
+	unsigned		rush_exponent;
+
+	/* Default connection_timeout */
+	double			connect_timeout;
+
+	/* Read timeouts for backend */
+	double			first_byte_timeout;
+	double			between_bytes_timeout;
+
+	/* How long to linger on sessions */
+	unsigned		session_linger;
+
+	/* CLI buffer size */
+	unsigned		cli_buffer;
+
+	/* Control diagnostic code */
+	unsigned		diag_bitmap;
+
+	/* Log hash string to shm */
+	unsigned		log_hash;
+
+	/* Log local socket address to shm */
+	unsigned		log_local_addr;
+
+	/* Prefer IPv6 connections to backend*/
+	unsigned		prefer_ipv6;
+
+	/* Acceptable clockskew with backends */
+	unsigned		clock_skew;
+
+	/* Expiry pacer parameters */
+	double			expiry_sleep;
+
+	/* Acceptor pacer parameters */
+	double			acceptor_sleep_max;
+	double			acceptor_sleep_incr;
+	double			acceptor_sleep_decay;
+
+	/* Get rid of duplicate bans */
+	unsigned		ban_dups;
+
+	/* How long time does the ban lurker sleep */
+	double			ban_lurker_sleep;
+
+	/* Max size of the saintmode list. 0 == no saint mode. */
+	unsigned		saintmode_threshold;
+
+	unsigned		syslog_cli_traffic;
+
+	unsigned		http_range_support;
+
+	unsigned		http_gzip_support;
+	unsigned		gzip_stack_buffer;
+	unsigned		gzip_tmp_space;
+	unsigned		gzip_level;
+	unsigned		gzip_window;
+	unsigned		gzip_memlevel;
+
+	double			critbit_cooloff;
+
+	double			shortlived;
+
+	struct vre_limits	vre_limits;
+};
+
+/*
+ * We declare this a volatile pointer, so that reads of parameters
+ * become atomic, leaving the CLI thread lattitude to change the values
+ */
+extern volatile struct params * cache_param;
diff --git a/bin/varnishd/flint.sh b/bin/varnishd/flint.sh
index 4beb9d5..756f65a 100755
--- a/bin/varnishd/flint.sh
+++ b/bin/varnishd/flint.sh
@@ -16,6 +16,7 @@ flexelint \
 	-I/usr/local/include \
 	-DVARNISH_STATE_DIR=\"foo\" \
 	*.c \
+	common/*.c \
 	storage/*.c \
 	waiter/*.c \
 	hash/*.c \
diff --git a/bin/varnishd/hash/hash_mgt.c b/bin/varnishd/hash/hash_mgt.c
index 6226749..ffbba29 100644
--- a/bin/varnishd/hash/hash_mgt.c
+++ b/bin/varnishd/hash/hash_mgt.c
@@ -36,7 +36,7 @@
 #include <unistd.h>
 
 #include "mgt/mgt.h"
-#include "heritage.h"
+#include "common/heritage.h"
 
 #include "hash/hash_slinger.h"
 #include "vav.h"
diff --git a/bin/varnishd/heritage.h b/bin/varnishd/heritage.h
deleted file mode 100644
index 36433bb..0000000
--- a/bin/varnishd/heritage.h
+++ /dev/null
@@ -1,65 +0,0 @@
-/*-
- * Copyright (c) 2006 Verdens Gang AS
- * Copyright (c) 2006-2011 Varnish Software AS
- * All rights reserved.
- *
- * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- *
- * This file contains the heritage passed when mgt forks cache
- */
-
-struct listen_sock {
-	unsigned			magic;
-#define LISTEN_SOCK_MAGIC		0x999e4b57
-	VTAILQ_ENTRY(listen_sock)	list;
-	int				sock;
-	char				*name;
-	struct vss_addr			*addr;
-};
-
-VTAILQ_HEAD(listen_sock_head, listen_sock);
-
-struct heritage {
-
-	/* Two pipe(2)'s for CLI connection between cache and mgt.  */
-	int				cli_in;
-	int				cli_out;
-
-	/* File descriptor for stdout/stderr */
-	int				std_fd;
-
-	/* Sockets from which to accept connections */
-	struct listen_sock_head		socks;
-	unsigned			nsocks;
-
-	/* Hash method */
-	const struct hash_slinger	*hash;
-
-	char				*name;
-	char                            identity[1024];
-};
-
-extern struct heritage heritage;
-
-void child_main(void);
diff --git a/bin/varnishd/mgt/mgt.h b/bin/varnishd/mgt/mgt.h
index 61493b3..54352a0 100644
--- a/bin/varnishd/mgt/mgt.h
+++ b/bin/varnishd/mgt/mgt.h
@@ -30,7 +30,7 @@
 
 #include <stdint.h>
 
-#include "common.h"
+#include "common/common.h"
 
 struct cli;
 
diff --git a/bin/varnishd/mgt/mgt_child.c b/bin/varnishd/mgt/mgt_child.c
index d2e0abd..a2c0b58 100644
--- a/bin/varnishd/mgt/mgt_child.c
+++ b/bin/varnishd/mgt/mgt_child.c
@@ -44,8 +44,8 @@
 #include <unistd.h>
 
 #include "mgt/mgt.h"
-#include "heritage.h"
-#include "params.h"
+#include "common/heritage.h"
+#include "common/params.h"
 
 #include "vapi/vsm_int.h"
 #include "vbm.h"
diff --git a/bin/varnishd/mgt/mgt_cli.c b/bin/varnishd/mgt/mgt_cli.c
index 3fdb432..3a6f365 100644
--- a/bin/varnishd/mgt/mgt_cli.c
+++ b/bin/varnishd/mgt/mgt_cli.c
@@ -43,8 +43,8 @@
 #include <unistd.h>
 
 #include "mgt/mgt.h"
-#include "heritage.h"
-#include "params.h"
+#include "common/heritage.h"
+#include "common/params.h"
 
 #include "vcli.h"
 #include "vcli_common.h"
diff --git a/bin/varnishd/mgt/mgt_main.c b/bin/varnishd/mgt/mgt_main.c
index ccda130..981841b 100644
--- a/bin/varnishd/mgt/mgt_main.c
+++ b/bin/varnishd/mgt/mgt_main.c
@@ -44,8 +44,8 @@
 #include <unistd.h>
 
 #include "mgt/mgt.h"
-#include "heritage.h"
-#include "params.h"
+#include "common/heritage.h"
+#include "common/params.h"
 
 #include "hash/hash_slinger.h"
 #include "vav.h"
diff --git a/bin/varnishd/mgt/mgt_param.c b/bin/varnishd/mgt/mgt_param.c
index 8fff352..8050d08 100644
--- a/bin/varnishd/mgt/mgt_param.c
+++ b/bin/varnishd/mgt/mgt_param.c
@@ -38,8 +38,8 @@
 #include <unistd.h>
 
 #include "mgt/mgt.h"
-#include "heritage.h"
-#include "params.h"
+#include "common/heritage.h"
+#include "common/params.h"
 
 #include "mgt/mgt_param.h"
 #include "waiter/cache_waiter.h"
diff --git a/bin/varnishd/mgt/mgt_pool.c b/bin/varnishd/mgt/mgt_pool.c
index 6f61553..e8c4f27 100644
--- a/bin/varnishd/mgt/mgt_pool.c
+++ b/bin/varnishd/mgt/mgt_pool.c
@@ -48,8 +48,8 @@
 #include <unistd.h>
 
 #include "mgt/mgt.h"
-#include "heritage.h"
-#include "params.h"
+#include "common/heritage.h"
+#include "common/params.h"
 
 #include "mgt/mgt_param.h"
 
diff --git a/bin/varnishd/mgt/mgt_sandbox.c b/bin/varnishd/mgt/mgt_sandbox.c
index a9fce93..cdba825 100644
--- a/bin/varnishd/mgt/mgt_sandbox.c
+++ b/bin/varnishd/mgt/mgt_sandbox.c
@@ -53,8 +53,8 @@
 #include <unistd.h>
 
 #include "mgt/mgt.h"
-#include "heritage.h"
-#include "params.h"
+#include "common/heritage.h"
+#include "common/params.h"
 
 /*--------------------------------------------------------------------*/
 
diff --git a/bin/varnishd/mgt/mgt_sandbox_solaris.c b/bin/varnishd/mgt/mgt_sandbox_solaris.c
index 8c26d69..79f6650 100644
--- a/bin/varnishd/mgt/mgt_sandbox_solaris.c
+++ b/bin/varnishd/mgt/mgt_sandbox_solaris.c
@@ -44,7 +44,7 @@
 
 #include "mgt/mgt.h"
 
-#include "heritage.h"
+#include "common/heritage.h"
 
 /*--------------------------------------------------------------------
  * SOLARIS PRIVILEGES: Note on use of symbolic PRIV_* constants
diff --git a/bin/varnishd/mgt/mgt_shmem.c b/bin/varnishd/mgt/mgt_shmem.c
index bb87a8f..d4346df 100644
--- a/bin/varnishd/mgt/mgt_shmem.c
+++ b/bin/varnishd/mgt/mgt_shmem.c
@@ -98,8 +98,8 @@
 #include <unistd.h>
 
 #include "mgt/mgt.h"
-#include "heritage.h"
-#include "params.h"
+#include "common/heritage.h"
+#include "common/params.h"
 
 #include "flopen.h"
 #include "vapi/vsc_int.h"
diff --git a/bin/varnishd/params.h b/bin/varnishd/params.h
deleted file mode 100644
index fb71e33..0000000
--- a/bin/varnishd/params.h
+++ /dev/null
@@ -1,194 +0,0 @@
-/*-
- * Copyright (c) 2006 Verdens Gang AS
- * Copyright (c) 2006-2011 Varnish Software AS
- * All rights reserved.
- *
- * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- *
- * This file contains the heritage passed when mgt forks cache
- */
-
-#include "vre.h"
-
-struct params {
-
-	/* Unprivileged user / group */
-	char			*user;
-	uid_t			uid;
-	char			*group;
-	gid_t			gid;
-
-	/* TTL used for lack of anything better */
-	double			default_ttl;
-
-	/* Default grace period */
-	double			default_grace;
-
-	/* Default keep period */
-	double			default_keep;
-
-	/* Maximum concurrent sessions */
-	unsigned		max_sess;
-
-	/* Worker threads and pool */
-	unsigned		wthread_min;
-	unsigned		wthread_max;
-	unsigned		wthread_timeout;
-	unsigned		wthread_pools;
-	unsigned		wthread_add_threshold;
-	unsigned		wthread_add_delay;
-	unsigned		wthread_fail_delay;
-	unsigned		wthread_purge_delay;
-	unsigned		wthread_stats_rate;
-	unsigned		wthread_stacksize;
-	unsigned		wthread_workspace;
-
-	unsigned		queue_max;
-
-	/* Memory allocation hints */
-	unsigned		sess_workspace;
-	unsigned		shm_workspace;
-	unsigned		http_req_size;
-	unsigned		http_req_hdr_len;
-	unsigned		http_resp_size;
-	unsigned		http_resp_hdr_len;
-	unsigned		http_max_hdr;
-
-	unsigned		shm_reclen;
-
-	/* Acceptor hints */
-	unsigned		sess_timeout;
-	unsigned		pipe_timeout;
-	unsigned		send_timeout;
-	unsigned		idle_send_timeout;
-
-	/* Management hints */
-	unsigned		auto_restart;
-
-	/* Fetcher hints */
-	unsigned		fetch_chunksize;
-	unsigned		fetch_maxchunksize;
-	unsigned		nuke_limit;
-
-#ifdef SENDFILE_WORKS
-	/* Sendfile object minimum size */
-	unsigned		sendfile_threshold;
-#endif
-
-	/* VCL traces */
-	unsigned		vcl_trace;
-
-	/* Listen address */
-	char			*listen_address;
-
-	/* Listen depth */
-	unsigned		listen_depth;
-
-	/* CLI related */
-	unsigned		cli_timeout;
-	unsigned		ping_interval;
-
-	/* LRU list ordering interval */
-	unsigned		lru_timeout;
-
-	/* Maximum restarts allowed */
-	unsigned		max_restarts;
-
-	/* Maximum esi:include depth allowed */
-	unsigned		max_esi_depth;
-
-	/* ESI parser hints */
-	unsigned		esi_syntax;
-
-	/* Rush exponent */
-	unsigned		rush_exponent;
-
-	/* Default connection_timeout */
-	double			connect_timeout;
-
-	/* Read timeouts for backend */
-	double			first_byte_timeout;
-	double			between_bytes_timeout;
-
-	/* How long to linger on sessions */
-	unsigned		session_linger;
-
-	/* CLI buffer size */
-	unsigned		cli_buffer;
-
-	/* Control diagnostic code */
-	unsigned		diag_bitmap;
-
-	/* Log hash string to shm */
-	unsigned		log_hash;
-
-	/* Log local socket address to shm */
-	unsigned		log_local_addr;
-
-	/* Prefer IPv6 connections to backend*/
-	unsigned		prefer_ipv6;
-
-	/* Acceptable clockskew with backends */
-	unsigned		clock_skew;
-
-	/* Expiry pacer parameters */
-	double			expiry_sleep;
-
-	/* Acceptor pacer parameters */
-	double			acceptor_sleep_max;
-	double			acceptor_sleep_incr;
-	double			acceptor_sleep_decay;
-
-	/* Get rid of duplicate bans */
-	unsigned		ban_dups;
-
-	/* How long time does the ban lurker sleep */
-	double			ban_lurker_sleep;
-
-	/* Max size of the saintmode list. 0 == no saint mode. */
-	unsigned		saintmode_threshold;
-
-	unsigned		syslog_cli_traffic;
-
-	unsigned		http_range_support;
-
-	unsigned		http_gzip_support;
-	unsigned		gzip_stack_buffer;
-	unsigned		gzip_tmp_space;
-	unsigned		gzip_level;
-	unsigned		gzip_window;
-	unsigned		gzip_memlevel;
-
-	double			critbit_cooloff;
-
-	double			shortlived;
-
-	struct vre_limits	vre_limits;
-};
-
-/*
- * We declare this a volatile pointer, so that reads of parameters
- * become atomic, leaving the CLI thread lattitude to change the values
- */
-extern volatile struct params * cache_param;
diff --git a/bin/varnishd/vsm.c b/bin/varnishd/vsm.c
deleted file mode 100644
index 5298381..0000000
--- a/bin/varnishd/vsm.c
+++ /dev/null
@@ -1,238 +0,0 @@
-/*-
- * Copyright (c) 2010-2011 Varnish Software AS
- * All rights reserved.
- *
- * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- *
- * VSM stuff common to manager and child.
- *
- * We have three potential conflicts we need to lock against here:
- *
- * VSM-studying programs (varnishstat...) vs. everybody else
- *	The VSM studying programs only have read-only access to the VSM
- *	so everybody else must use memory barriers, stable storage and
- *	similar tricks to keep the VSM image in sync (long enough) for
- *	the studying programs.
- *
- * Manager process vs child process.
- *	Will only muck about in VSM when child process is not running
- *	Responsible for cleaning up any mess left behind by dying child.
- *
- * Child process threads
- *	Pthread locking necessary.
- *
- * XXX: not all of this is in place yet.
- */
-
-#include "config.h"
-
-#include <stdint.h>
-#include <stdio.h>
-#include <string.h>
-#include <unistd.h>
-
-#include "common.h"
-
-#include "vapi/vsm_int.h"
-#include "vmb.h"
-#include "vtim.h"
-
-/* These two come from beyond (mgt_shmem.c actually) */
-struct VSM_head		*VSM_head;
-const struct VSM_chunk	*vsm_end;
-
-static unsigned
-vsm_mark(void)
-{
-	unsigned seq;
-
-	seq = VSM_head->alloc_seq;
-	VSM_head->alloc_seq = 0;
-	VWMB();
-	return (seq);
-}
-
-static void
-vsm_release(unsigned seq)
-{
-
-	if (seq == 0)
-		return;
-	VWMB();
-	do
-		VSM_head->alloc_seq = ++seq;
-	while (VSM_head->alloc_seq == 0);
-}
-
-/*--------------------------------------------------------------------*/
-
-static void
-vsm_cleanup(void)
-{
-	unsigned now = (unsigned)VTIM_mono();
-	struct VSM_chunk *sha, *sha2;
-	unsigned seq;
-
-	CHECK_OBJ_NOTNULL(VSM_head, VSM_HEAD_MAGIC);
-	VSM_ITER(sha) {
-		if (strcmp(sha->class, VSM_CLASS_COOL))
-			continue;
-		if (sha->state + VSM_COOL_TIME < now)
-			break;
-	}
-	if (sha == NULL)
-		return;
-	seq = vsm_mark();
-	/* First pass, free, and collapse with next if applicable */
-	VSM_ITER(sha) {
-		if (strcmp(sha->class, VSM_CLASS_COOL))
-			continue;
-		if (sha->state + VSM_COOL_TIME >= now)
-			continue;
-
-		bprintf(sha->class, "%s", VSM_CLASS_FREE);
-		bprintf(sha->type, "%s", "");
-		bprintf(sha->ident, "%s", "");
-		sha2 = VSM_NEXT(sha);
-		assert(sha2 <= vsm_end);
-		if (sha2 == vsm_end)
-			break;
-		CHECK_OBJ_NOTNULL(sha2, VSM_CHUNK_MAGIC);
-		if (!strcmp(sha2->class, VSM_CLASS_FREE)) {
-			sha->len += sha2->len;
-			memset(sha2, 0, sizeof *sha2);
-		}
-		sha->state = 0;
-	}
-	/* Second pass, collaps with prev if applicable */
-	VSM_ITER(sha) {
-		if (strcmp(sha->class, VSM_CLASS_FREE))
-			continue;
-		sha2 = VSM_NEXT(sha);
-		assert(sha2 <= vsm_end);
-		if (sha2 == vsm_end)
-			break;
-		CHECK_OBJ_NOTNULL(sha2, VSM_CHUNK_MAGIC);
-		if (!strcmp(sha2->class, VSM_CLASS_FREE)) {
-			sha->len += sha2->len;
-			memset(sha2, 0, sizeof *sha2);
-		}
-	}
-	vsm_release(seq);
-}
-
-/*--------------------------------------------------------------------*/
-
-void *
-VSM__Alloc(unsigned size, const char *class, const char *type, const char *ident)
-{
-	struct VSM_chunk *sha, *sha2;
-	unsigned seq;
-
-	CHECK_OBJ_NOTNULL(VSM_head, VSM_HEAD_MAGIC);
-
-	vsm_cleanup();
-
-	/* Round up to pointersize */
-	size = RUP2(size, sizeof(void*));
-
-	size += sizeof *sha;		/* Make space for the header */
-
-	VSM_ITER(sha) {
-		CHECK_OBJ_NOTNULL(sha, VSM_CHUNK_MAGIC);
-
-		if (strcmp(sha->class, VSM_CLASS_FREE))
-			continue;
-
-		if (size > sha->len)
-			continue;
-
-		/* Mark as inconsistent while we write string fields */
-		seq = vsm_mark();
-
-		if (size + sizeof (*sha) < sha->len) {
-			sha2 = (void*)((uintptr_t)sha + size);
-
-			memset(sha2, 0, sizeof *sha2);
-			sha2->magic = VSM_CHUNK_MAGIC;
-			sha2->len = sha->len - size;
-			bprintf(sha2->class, "%s", VSM_CLASS_FREE);
-			sha->len = size;
-		}
-
-		bprintf(sha->class, "%s", class);
-		bprintf(sha->type, "%s", type);
-		bprintf(sha->ident, "%s", ident);
-
-		vsm_release(seq);
-		return (VSM_PTR(sha));
-	}
-	return (NULL);
-}
-
-/*--------------------------------------------------------------------*/
-
-void
-VSM__Free(const void *ptr)
-{
-	struct VSM_chunk *sha;
-	unsigned seq;
-
-	CHECK_OBJ_NOTNULL(VSM_head, VSM_HEAD_MAGIC);
-	VSM_ITER(sha)
-		if (VSM_PTR(sha) == ptr)
-			break;
-	AN(sha);
-	seq = vsm_mark();
-	bprintf(sha->class, "%s", VSM_CLASS_COOL);
-	sha->state = (unsigned)VTIM_mono();
-	vsm_release(seq);
-}
-
-/*--------------------------------------------------------------------
- * Free all allocations after the mark (ie: allocated by child).
- */
-
-void
-VSM__Clean(void)
-{
-	struct VSM_chunk *sha;
-	unsigned f, seq;
-
-	CHECK_OBJ_NOTNULL(VSM_head, VSM_HEAD_MAGIC);
-	f = 0;
-	seq = vsm_mark();
-	VSM_ITER(sha) {
-		if (f == 0 && !strcmp(sha->class, VSM_CLASS_MARK)) {
-			f = 1;
-			continue;
-		}
-		if (f == 0)
-			continue;
-		if (strcmp(sha->class, VSM_CLASS_FREE) &&
-		    strcmp(sha->class, VSM_CLASS_COOL))
-			VSM__Free(VSM_PTR(sha));
-	}
-	vsm_release(seq);
-}



From geoff at varnish-cache.org  Mon Jan  9 20:52:15 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:15 +0100
Subject: [experimental-ims] f971cdf Even more #include cleanup
Message-ID: <E1RkMCV-0007ZB-Bc@project.varnish-software.com>

commit f971cdfa6ce63ab7b4fbd2c1191f6b044dd2f488
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sat Oct 8 21:58:01 2011 +0000

    Even more #include cleanup

diff --git a/bin/varnishd/cache.h b/bin/varnishd/cache.h
index c9b3ac2..d3228a5 100644
--- a/bin/varnishd/cache.h
+++ b/bin/varnishd/cache.h
@@ -34,8 +34,6 @@
  */
 #define VARNISH_CACHE_CHILD	1
 
-#include <sys/time.h>
-#include <sys/uio.h>
 #include <sys/socket.h>
 
 #include <pthread.h>
@@ -52,15 +50,9 @@
 #include <sys/epoll.h>
 #endif
 
-#include "vqueue.h"
-
-#include "vsb.h"
-
-#include "libvarnish.h"
-
 #include "common.h"
+
 #include "heritage.h"
-#include "miniobj.h"
 
 #include "vsc.h"
 #include "vsl.h"
@@ -99,6 +91,7 @@ enum {
 	HTTP_HDR_FIRST,
 };
 
+struct iovec;
 struct cli;
 struct vsb;
 struct sess;
diff --git a/bin/varnishd/cache_backend.c b/bin/varnishd/cache_backend.c
index 1061612..a724923 100644
--- a/bin/varnishd/cache_backend.c
+++ b/bin/varnishd/cache_backend.c
@@ -32,16 +32,11 @@
 
 #include "config.h"
 
-#include <limits.h>
-#include <stdio.h>
 #include <stdlib.h>
-#include <string.h>
-#include <unistd.h>
 #include <poll.h>
 
-#include <sys/socket.h>
-
 #include "cache.h"
+
 #include "cache_backend.h"
 #include "vrt.h"
 
diff --git a/bin/varnishd/cache_ban.c b/bin/varnishd/cache_ban.c
index 09e68cc..f346b4d 100644
--- a/bin/varnishd/cache_ban.c
+++ b/bin/varnishd/cache_ban.c
@@ -63,7 +63,6 @@
 #include "config.h"
 
 #include <stdio.h>
-#include <stdint.h>
 
 #include <pcre.h>
 
diff --git a/bin/varnishd/cache_dir_dns.c b/bin/varnishd/cache_dir_dns.c
index d3635a9..3fd2aa1 100644
--- a/bin/varnishd/cache_dir_dns.c
+++ b/bin/varnishd/cache_dir_dns.c
@@ -29,14 +29,9 @@
 
 #include "config.h"
 
-#include <sys/types.h>
-#include <sys/socket.h>
-
 #include <errno.h>
 #include <stdlib.h>
-#include <string.h>
 #include <netdb.h>
-#include <unistd.h>
 
 #include <stdio.h>
 #include <netinet/in.h>
diff --git a/bin/varnishd/cache_fetch.c b/bin/varnishd/cache_fetch.c
index 7c6e2e5..df56c7a 100644
--- a/bin/varnishd/cache_fetch.c
+++ b/bin/varnishd/cache_fetch.c
@@ -31,7 +31,6 @@
 
 #include <stdio.h>
 #include <inttypes.h>
-#include <string.h>
 #include <stdlib.h>
 #include <fcntl.h>
 
diff --git a/bin/varnishd/cache_http.c b/bin/varnishd/cache_http.c
index 3703b00..dd55990 100644
--- a/bin/varnishd/cache_http.c
+++ b/bin/varnishd/cache_http.c
@@ -31,9 +31,7 @@
 
 #include "config.h"
 
-#include <stdarg.h>
 #include <stdio.h>
-#include <string.h>
 
 #include "cache.h"
 
diff --git a/bin/varnishd/cache_httpconn.c b/bin/varnishd/cache_httpconn.c
index d1ceb6e..ba881c9 100644
--- a/bin/varnishd/cache_httpconn.c
+++ b/bin/varnishd/cache_httpconn.c
@@ -31,9 +31,6 @@
 
 #include "config.h"
 
-#include <stdlib.h>
-#include <string.h>
-
 #include "cache.h"
 
 #include "vct.h"
diff --git a/bin/varnishd/cache_lck.c b/bin/varnishd/cache_lck.c
index 18d1b41..ee93b9f 100644
--- a/bin/varnishd/cache_lck.c
+++ b/bin/varnishd/cache_lck.c
@@ -35,8 +35,6 @@
 
 #include "config.h"
 
-#include <stdio.h>
-
 #include <stdlib.h>
 
 #include "cache.h"
diff --git a/bin/varnishd/cache_main.c b/bin/varnishd/cache_main.c
index 5660893..b4ee185 100644
--- a/bin/varnishd/cache_main.c
+++ b/bin/varnishd/cache_main.c
@@ -30,11 +30,10 @@
 #include "config.h"
 
 #include <stdio.h>
-#include <unistd.h>
 #include <stdlib.h>
-#include <signal.h>
 
 #include "cache.h"
+
 #include "stevedore.h"
 #include "hash_slinger.h"
 #include "cache_waiter.h"
diff --git a/bin/varnishd/cache_panic.c b/bin/varnishd/cache_panic.c
index a9c5c15..464924a 100644
--- a/bin/varnishd/cache_panic.c
+++ b/bin/varnishd/cache_panic.c
@@ -29,19 +29,17 @@
 
 #include "config.h"
 
-#include <signal.h>
-#include <stdarg.h>
 #include <stdio.h>
 #include <stdlib.h>
-#include <unistd.h>
-#include <pthread.h>
 
 #ifndef HAVE_EXECINFO_H
 #include "compat/execinfo.h"
 #else
 #include <execinfo.h>
 #endif
+
 #include "cache.h"
+
 #include "vsm.h"
 #include "cache_backend.h"
 #include "cache_waiter.h"
diff --git a/bin/varnishd/cache_shmlog.c b/bin/varnishd/cache_shmlog.c
index 023c20a..8c06897 100644
--- a/bin/varnishd/cache_shmlog.c
+++ b/bin/varnishd/cache_shmlog.c
@@ -30,8 +30,6 @@
 #include "config.h"
 
 #include <stdio.h>
-#include <unistd.h>
-#include <stdarg.h>
 
 #include "cache.h"
 #include "vmb.h"
diff --git a/bin/varnishd/cache_vary.c b/bin/varnishd/cache_vary.c
index b7a6b01..4440d55 100644
--- a/bin/varnishd/cache_vary.c
+++ b/bin/varnishd/cache_vary.c
@@ -54,10 +54,8 @@
 
 #include "config.h"
 
-#include <string.h>
-#include <stdlib.h>
-
 #include "cache.h"
+
 #include "vend.h"
 #include "vct.h"
 
diff --git a/bin/varnishd/cache_vcl.c b/bin/varnishd/cache_vcl.c
index 8848ddd..99364b5 100644
--- a/bin/varnishd/cache_vcl.c
+++ b/bin/varnishd/cache_vcl.c
@@ -34,15 +34,13 @@
 #include "config.h"
 
 #include <stdio.h>
-#include <unistd.h>
-#include <string.h>
 #include <stdlib.h>
 #include <dlfcn.h>
 
+#include "cache.h"
 #include "vcli.h"
 #include "cli_priv.h"
 #include "vcl.h"
-#include "cache.h"
 #include "libvcl.h"
 
 struct vcls {
diff --git a/bin/varnishd/cache_vrt.c b/bin/varnishd/cache_vrt.c
index 02d2a42..f0407cc 100644
--- a/bin/varnishd/cache_vrt.c
+++ b/bin/varnishd/cache_vrt.c
@@ -35,9 +35,7 @@
 #include <arpa/inet.h>
 
 #include <stdio.h>
-#include <string.h>
 #include <stdlib.h>
-#include <stdarg.h>
 
 #include "cache.h"
 
diff --git a/bin/varnishd/cache_vrt_var.c b/bin/varnishd/cache_vrt_var.c
index e26b433..ae65058 100644
--- a/bin/varnishd/cache_vrt_var.c
+++ b/bin/varnishd/cache_vrt_var.c
@@ -31,12 +31,12 @@
 #include "config.h"
 
 #include <stdio.h>
-#include <stdarg.h>
 #include <stdlib.h>
 
+#include "cache.h"
+
 #include "vrt.h"
 #include "vrt_obj.h"
-#include "cache.h"
 #include "cache_backend.h"
 #include "hash_slinger.h"
 
diff --git a/bin/varnishd/cache_waiter_epoll.c b/bin/varnishd/cache_waiter_epoll.c
index 28776cd..d4168db 100644
--- a/bin/varnishd/cache_waiter_epoll.c
+++ b/bin/varnishd/cache_waiter_epoll.c
@@ -35,22 +35,18 @@
 
 #if defined(HAVE_EPOLL_CTL)
 
-#include <stdio.h>
-#include <string.h>
 #include <errno.h>
 #include <fcntl.h>
 #include <stdlib.h>
-#include <unistd.h>
-
 #include <sys/epoll.h>
 
+#include "cache.h"
+#include "cache_waiter.h"
+
 #ifndef EPOLLRDHUP
 #  define EPOLLRDHUP 0
 #endif
 
-#include "cache.h"
-#include "cache_waiter.h"
-
 #define NEEV	100
 
 struct vwe {
diff --git a/bin/varnishd/cache_waiter_kqueue.c b/bin/varnishd/cache_waiter_kqueue.c
index 9575dc8..f023c3f 100644
--- a/bin/varnishd/cache_waiter_kqueue.c
+++ b/bin/varnishd/cache_waiter_kqueue.c
@@ -35,13 +35,10 @@
 
 #if defined(HAVE_KQUEUE)
 
-#include <stdio.h>
-#include <string.h>
 #include <errno.h>
 #include <fcntl.h>
 #include <stdlib.h>
 #include <unistd.h>
-
 #include <sys/event.h>
 
 #include "cache.h"
diff --git a/bin/varnishd/cache_waiter_ports.c b/bin/varnishd/cache_waiter_ports.c
index 695e9c0..04c6e92 100644
--- a/bin/varnishd/cache_waiter_ports.c
+++ b/bin/varnishd/cache_waiter_ports.c
@@ -35,9 +35,7 @@
 
 #include <stdio.h>
 #include <errno.h>
-#include <string.h>
 #include <stdlib.h>
-#include <unistd.h>
 #include <math.h>
 
 #include <port.h>
diff --git a/bin/varnishd/cache_wrw.c b/bin/varnishd/cache_wrw.c
index 61aa2e5..5f9c29b 100644
--- a/bin/varnishd/cache_wrw.c
+++ b/bin/varnishd/cache_wrw.c
@@ -40,13 +40,13 @@
 
 #ifdef SENDFILE_WORKS
 #if defined(__FreeBSD__) || defined(__DragonFly__)
-#include <sys/socket.h>
+   // We're fine
 #elif defined(__linux__)
-#include <sys/sendfile.h>
+#  include <sys/sendfile.h>
 #elif defined(__sun)
-#include <sys/sendfile.h>
+#  include <sys/sendfile.h>
 #else
-#error Unknown sendfile() implementation
+#  error Unknown sendfile() implementation
 #endif
 #endif /* SENDFILE_WORKS */
 
diff --git a/bin/varnishd/common.h b/bin/varnishd/common.h
index 3708b45..f755b97 100644
--- a/bin/varnishd/common.h
+++ b/bin/varnishd/common.h
@@ -28,6 +28,11 @@
  *
  */
 
+#include "libvarnish.h"
+#include "vsb.h"
+#include "vqueue.h"
+#include "miniobj.h"
+
 struct cli;
 
 extern pid_t mgt_pid;
diff --git a/bin/varnishd/mgt.h b/bin/varnishd/mgt.h
index 8958ab5..9d12752 100644
--- a/bin/varnishd/mgt.h
+++ b/bin/varnishd/mgt.h
@@ -30,13 +30,7 @@
 
 #include <stdint.h>
 
-#include "vqueue.h"
-
 #include "common.h"
-#include "vsb.h"
-#include "miniobj.h"
-
-#include "libvarnish.h"
 
 struct cli;
 
diff --git a/bin/varnishd/mgt_pool.c b/bin/varnishd/mgt_pool.c
index 1238f5e..4b3fe92 100644
--- a/bin/varnishd/mgt_pool.c
+++ b/bin/varnishd/mgt_pool.c
@@ -46,9 +46,7 @@
 #include <string.h>
 #include <limits.h>
 #include <unistd.h>
-#include <sys/types.h>
 
-#include "cli_priv.h"
 #include "mgt.h"
 
 #include "vparam.h"
diff --git a/bin/varnishd/mgt_sandbox_solaris.c b/bin/varnishd/mgt_sandbox_solaris.c
index 15b7c95..2c975d9 100644
--- a/bin/varnishd/mgt_sandbox_solaris.c
+++ b/bin/varnishd/mgt_sandbox_solaris.c
@@ -36,8 +36,6 @@
 
 #include <stdio.h>
 #include <syslog.h>
-#include <unistd.h>
-#include <string.h>
 
 #ifdef HAVE_PRIV_H
 #include <priv.h>
diff --git a/bin/varnishd/mgt_vcc.c b/bin/varnishd/mgt_vcc.c
index 6c100bb..f8e81ca 100644
--- a/bin/varnishd/mgt_vcc.c
+++ b/bin/varnishd/mgt_vcc.c
@@ -31,9 +31,6 @@
 
 #include "config.h"
 
-#include <sys/types.h>
-#include <sys/wait.h>
-
 #include <dlfcn.h>
 #include <fcntl.h>
 #include <stdio.h>
@@ -45,13 +42,9 @@
 
 #include "libvcl.h"
 #include "vcli.h"
+#include "vcl.h"
 #include "cli_priv.h"
-#include "cli_common.h"
-
 #include "mgt_cli.h"
-#include "heritage.h"
-
-#include "vcl.h"
 
 struct vclprog {
 	VTAILQ_ENTRY(vclprog)	list;
diff --git a/bin/varnishd/stevedore_utils.c b/bin/varnishd/stevedore_utils.c
index 1dcfd63..af7fc7f 100644
--- a/bin/varnishd/stevedore_utils.c
+++ b/bin/varnishd/stevedore_utils.c
@@ -33,7 +33,6 @@
 
 #include <fcntl.h>
 #include <stdio.h>
-#include <stdint.h>
 #include <stdlib.h>
 #include <string.h>
 #include <unistd.h>
diff --git a/bin/varnishd/storage_persistent.h b/bin/varnishd/storage_persistent.h
index 2a5bd77..84f3d21 100644
--- a/bin/varnishd/storage_persistent.h
+++ b/bin/varnishd/storage_persistent.h
@@ -33,14 +33,6 @@
  * XXX: Do we ever free the LRU-lists ?
  */
 
-#ifndef MAP_NOCORE
-#define MAP_NOCORE 0 /* XXX Linux */
-#endif
-
-#ifndef MAP_NOSYNC
-#define MAP_NOSYNC 0 /* XXX Linux */
-#endif
-
 #define ASSERT_SILO_THREAD(sc) \
     do {assert(pthread_self() == (sc)->thread);} while (0)
 
diff --git a/bin/varnishd/storage_persistent_mgt.c b/bin/varnishd/storage_persistent_mgt.c
index 3bd5809..c91fb43 100644
--- a/bin/varnishd/storage_persistent_mgt.c
+++ b/bin/varnishd/storage_persistent_mgt.c
@@ -47,6 +47,14 @@
 #include "persistent.h"
 #include "storage_persistent.h"
 
+#ifndef MAP_NOCORE
+#define MAP_NOCORE 0 /* XXX Linux */
+#endif
+
+#ifndef MAP_NOSYNC
+#define MAP_NOSYNC 0 /* XXX Linux */
+#endif
+
 /*--------------------------------------------------------------------
  * Calculate cleaner metrics from silo dimensions
  */
diff --git a/bin/varnishd/vsm.c b/bin/varnishd/vsm.c
index 62047ff..28f98e5 100644
--- a/bin/varnishd/vsm.c
+++ b/bin/varnishd/vsm.c
@@ -51,8 +51,6 @@
 #include <string.h>
 #include <stdio.h>
 
-#include "miniobj.h"
-#include "libvarnish.h"
 #include "common.h"
 #include "vsm.h"
 #include "vmb.h"



From geoff at varnish-cache.org  Mon Jan  9 20:52:16 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:16 +0100
Subject: [experimental-ims] 1c2a1b3 Add vapi/vsm_int.h to complete the picture.
Message-ID: <E1RkMCW-0007cc-N3@project.varnish-software.com>

commit 1c2a1b3437df7b39ca94e2cea6d9032ad0fc01b6
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sun Oct 9 20:49:59 2011 +0000

    Add vapi/vsm_int.h to complete the picture.

diff --git a/include/vapi/vsm_int.h b/include/vapi/vsm_int.h
new file mode 100644
index 0000000..98e8610
--- /dev/null
+++ b/include/vapi/vsm_int.h
@@ -0,0 +1,112 @@
+/*-
+ * Copyright (c) 2006 Verdens Gang AS
+ * Copyright (c) 2006-2011 Varnish Software AS
+ * All rights reserved.
+ *
+ * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * Define the layout of the shared memory log segment.
+ *
+ * NB: THIS IS NOT A PUBLIC API TO VARNISH!
+ */
+
+#ifndef VSM_H_INCLUDED
+#define VSM_H_INCLUDED
+
+#define VSM_FILENAME		"_.vsm"
+
+#include <time.h>
+#include <sys/types.h>
+
+/*
+ * This structure describes each allocation from the shmlog
+ */
+
+struct VSM_chunk {
+#define VSM_CHUNK_MAGIC		0x43907b6e	/* From /dev/random */
+	unsigned		magic;
+	unsigned		len;
+	unsigned		state;
+	char			class[8];
+	char			type[8];
+	char			ident[64];
+};
+
+#define VSM_NEXT(sha)		((void*)((uintptr_t)(sha) + (sha)->len))
+#define VSM_PTR(sha)		((void*)((uintptr_t)((sha) + 1)))
+
+struct VSM_head {
+#define VSM_HEAD_MAGIC		4185512502U	/* From /dev/random */
+	unsigned		magic;
+
+	unsigned		hdrsize;
+
+	time_t			starttime;
+	pid_t			master_pid;
+	pid_t			child_pid;
+
+	unsigned		shm_size;
+
+	/* Panic message buffer */
+	char			panicstr[64 * 1024];
+
+	unsigned		alloc_seq;
+	/* Must be last element */
+	struct VSM_chunk	head;
+};
+
+/*
+ * You must include "miniobj.h" and have an assert function to be
+ * able to use the VSM_ITER() macro.
+ */
+#ifdef CHECK_OBJ_NOTNULL
+
+static inline struct VSM_chunk *
+vsm_iter_0(void)
+{
+
+	CHECK_OBJ_NOTNULL(VSM_head, VSM_HEAD_MAGIC);
+	CHECK_OBJ_NOTNULL(&VSM_head->head, VSM_CHUNK_MAGIC);
+	return (&VSM_head->head);
+}
+
+static inline void
+vsm_iter_n(struct VSM_chunk **pp)
+{
+
+	CHECK_OBJ_NOTNULL(VSM_head, VSM_HEAD_MAGIC);
+	CHECK_OBJ_NOTNULL(*pp, VSM_CHUNK_MAGIC);
+	*pp = VSM_NEXT(*pp);
+	if (*pp >= vsm_end) {
+		*pp = NULL;
+		return;
+	}
+	CHECK_OBJ_NOTNULL(*pp, VSM_CHUNK_MAGIC);
+}
+
+#define VSM_ITER(vd) for ((vd) = vsm_iter_0(); (vd) != NULL; vsm_iter_n(&vd))
+
+#endif /* CHECK_OBJ_NOTNULL */
+
+#endif
diff --git a/include/vsm.h b/include/vsm.h
deleted file mode 100644
index 98e8610..0000000
--- a/include/vsm.h
+++ /dev/null
@@ -1,112 +0,0 @@
-/*-
- * Copyright (c) 2006 Verdens Gang AS
- * Copyright (c) 2006-2011 Varnish Software AS
- * All rights reserved.
- *
- * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- *
- * Define the layout of the shared memory log segment.
- *
- * NB: THIS IS NOT A PUBLIC API TO VARNISH!
- */
-
-#ifndef VSM_H_INCLUDED
-#define VSM_H_INCLUDED
-
-#define VSM_FILENAME		"_.vsm"
-
-#include <time.h>
-#include <sys/types.h>
-
-/*
- * This structure describes each allocation from the shmlog
- */
-
-struct VSM_chunk {
-#define VSM_CHUNK_MAGIC		0x43907b6e	/* From /dev/random */
-	unsigned		magic;
-	unsigned		len;
-	unsigned		state;
-	char			class[8];
-	char			type[8];
-	char			ident[64];
-};
-
-#define VSM_NEXT(sha)		((void*)((uintptr_t)(sha) + (sha)->len))
-#define VSM_PTR(sha)		((void*)((uintptr_t)((sha) + 1)))
-
-struct VSM_head {
-#define VSM_HEAD_MAGIC		4185512502U	/* From /dev/random */
-	unsigned		magic;
-
-	unsigned		hdrsize;
-
-	time_t			starttime;
-	pid_t			master_pid;
-	pid_t			child_pid;
-
-	unsigned		shm_size;
-
-	/* Panic message buffer */
-	char			panicstr[64 * 1024];
-
-	unsigned		alloc_seq;
-	/* Must be last element */
-	struct VSM_chunk	head;
-};
-
-/*
- * You must include "miniobj.h" and have an assert function to be
- * able to use the VSM_ITER() macro.
- */
-#ifdef CHECK_OBJ_NOTNULL
-
-static inline struct VSM_chunk *
-vsm_iter_0(void)
-{
-
-	CHECK_OBJ_NOTNULL(VSM_head, VSM_HEAD_MAGIC);
-	CHECK_OBJ_NOTNULL(&VSM_head->head, VSM_CHUNK_MAGIC);
-	return (&VSM_head->head);
-}
-
-static inline void
-vsm_iter_n(struct VSM_chunk **pp)
-{
-
-	CHECK_OBJ_NOTNULL(VSM_head, VSM_HEAD_MAGIC);
-	CHECK_OBJ_NOTNULL(*pp, VSM_CHUNK_MAGIC);
-	*pp = VSM_NEXT(*pp);
-	if (*pp >= vsm_end) {
-		*pp = NULL;
-		return;
-	}
-	CHECK_OBJ_NOTNULL(*pp, VSM_CHUNK_MAGIC);
-}
-
-#define VSM_ITER(vd) for ((vd) = vsm_iter_0(); (vd) != NULL; vsm_iter_n(&vd))
-
-#endif /* CHECK_OBJ_NOTNULL */
-
-#endif



From geoff at varnish-cache.org  Mon Jan  9 20:52:41 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:41 +0100
Subject: [experimental-ims] f394f0a The new VSM api:
Message-ID: <E1RkMCv-0008OS-IU@project.varnish-software.com>

commit f394f0adba53e7a60a516c85968a0f4c95095423
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Nov 22 08:51:07 2011 +0000

    The new VSM api:
    
    Instead of the diag() callback, use a vsb to collect diagnostics
    and return it with VSM_Error().
    
    Simpler logic around open/close.
    
    More useful Abandonned() and StillValid() functions.
    
    Still not happy with __itern(), will revisit later.

diff --git a/include/vapi/vsm.h b/include/vapi/vsm.h
index 48cbe70..277a670 100644
--- a/include/vapi/vsm.h
+++ b/include/vapi/vsm.h
@@ -26,14 +26,15 @@
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
  *
- * This is the public API for the VSM/VSC/VSL access.
+ * This is the public API for the VSM access.
+ *
+ * The VSM "class" acts as parent class for the VSL and VSC subclasses.
  *
  */
 
 #ifndef VAPI_VSM_H_INCLUDED
 #define VAPI_VSM_H_INCLUDED
 
-struct VSM_head;
 struct VSM_chunk;
 struct VSM_data;
 
@@ -45,7 +46,7 @@ struct VSM_fantom {
 	struct VSM_chunk	*chunk;
 	void			*b;		/* first byte of payload */
 	void			*e;		/* first byte past payload */
-	uintptr_t		priv;
+	uintptr_t		priv;		/* VSM private */
 };
 
 /*---------------------------------------------------------------------
@@ -56,20 +57,22 @@ struct VSM_data *VSM_New(void);
 	/*
 	 * Allocate and initialize a VSL_data handle structure.
 	 * This is the first thing you will have to do, always.
-	 * You can have multiple active VSL_data handles at the same time
+	 * You can have multiple active VSM_data handles at the same time
 	 * referencing the same or different shared memory files.
 	 * Returns:
 	 *	Pointer to usable VSL_data handle.
 	 *	NULL: malloc failed.
 	 */
 
-typedef void VSM_diag_f(void *priv, const char *fmt, ...);
+void VSM_Delete(struct VSM_data *vd);
+	/*
+	 * Close and deallocate all storage and mappings.
+	 * (including any VSC and VSL "sub-classes")
+	 */
 
-void VSM_Diag(struct VSM_data *vd, VSM_diag_f *func, void *priv);
+const char *VSM_Error(const struct VSM_data *vd);
 	/*
-	 * Set the diagnostics reporting function.
-	 * Default is fprintf(stderr, ...)
-	 * If func is NULL, diagnostics are disabled.
+	 * Return the latest error message.
 	 */
 
 #define VSM_n_USAGE	"[-n varnish_name]"
@@ -77,11 +80,10 @@ void VSM_Diag(struct VSM_data *vd, VSM_diag_f *func, void *priv);
 int VSM_n_Arg(struct VSM_data *vd, const char *n_arg);
 	/*
 	 * Configure which varnishd instance to access.
-	 * Can also be, and normally is done through the VSL_Log_arg()
-	 * and VSC_Arg() functions.
+	 * Can also be, and normally is done through VSC_Arg()/VSL_Arg().
 	 * Returns:
 	 *	 1 on success
-	 *	 -1 on failure, with diagnostic.
+	 *	 <0 on failure, use VSM_Error() to get diagnostics.
 	 */
 
 const char *VSM_Name(const struct VSM_data *vd);
@@ -89,42 +91,32 @@ const char *VSM_Name(const struct VSM_data *vd);
 	 * Return the instance name.
 	 */
 
-void VSM_Delete(struct VSM_data *vd);
+int VSM_Open(struct VSM_data *vd);
 	/*
-	 * Close and deallocate all storage and mappings.
-	 */
-
-/* XXX: extension:  Patience argument for sleeps */
-
-int VSM_Open(struct VSM_data *vd, int diag);
-	/*
-	 * Attempt to open and map the shared memory file.
+	 * Attempt to open and map the VSM file.
 	 * If diag is non-zero, diagnostics are emitted.
 	 * Returns:
 	 *	0 on success
-	 *	!= 0 on failure
+	 *	<0 on failure, use VSM_Error() to get diagnostics.
 	 */
 
-int VSM_ReOpen(struct VSM_data *vd, int diag);
+int VSM_Abandonned(const struct VSM_data *vd);
 	/*
-	 * Check if shared memory segment needs to be reopened/remapped
-	 * typically when the varnishd master process restarts.
-	 * diag is passed to VSM_Open()
+	 * Find out if the VSM file has been abandonned or closed and should
+	 * be reopened.  This function calls stat(2) and should only be
+	 * used when lack of activity or invalidation of fantoms indicate
+	 * abandonment.
+	 *
 	 * Returns:
 	 *	0  No reopen needed.
-	 *	1  shared memory reopened/remapped.
-	 *	-1 failure to reopen.
+	 *	1  VSM abandonned.
 	 */
 
-unsigned VSM_Seq(const struct VSM_data *vd);
+void VSM_Close(struct VSM_data *vd);
 	/*
-	 * Return the allocation sequence number
+	 * Close and unmap shared memory, if open.
 	 */
 
-struct VSM_head *VSM_Head(const struct VSM_data *vd);
-	/*
-	 * Return the head of the VSM.
-	 */
 
 void VSM__iter0(const struct VSM_data *vd, struct VSM_fantom *vf);
 int VSM__itern(const struct VSM_data *vd, struct VSM_fantom *vf);
@@ -139,8 +131,12 @@ int VSM__itern(const struct VSM_data *vd, struct VSM_fantom *vf);
 
 int VSM_StillValid(const struct VSM_data *vd, struct VSM_fantom *vf);
 	/*
+	 * This is a cheap syscall-less check to see if the fantom is still
+	 * valid.  Further checking with VSM_Abandonned() may be a good
+	 * idea.
+	 *
 	 * Return:
-	 *	0: fantom is invalid now.
+	 *	0: fantom is not valid any more.
 	 *	1: fantom is still the same.
 	 *	2: a fantom with same dimensions exist, check class/type/ident
 	 */
@@ -153,41 +149,4 @@ int VSM_Get(const struct VSM_data *vd, struct VSM_fantom *vf,
 	 * class is mandatory, type and ident optional.
 	 */
 
-void VSM_Close(struct VSM_data *vd);
-	/*
-	 * Unmap shared memory
-	 * Deallocate all storage (including VSC and VSL allocations)
-	 */
-
-/**********************************************************************
- * These are old API functions which are less safe because there is
- * fantom protecting the chunks worked on.
- * They will g
- */
-
-/* OBSOLETE: Will disappear from Varnish 4.x */
-void *VSM_Find_Chunk(const struct VSM_data *vd, const char *class,
-    const char *type, const char *ident, unsigned *lenp);
-	/*
-	 * Find a given chunk in the shared memory.
-	 * Returns pointer or NULL.
-	 * Lenp, if non-NULL, is set to length of chunk.
-	 */
-
-/* OBSOLETE: Will disappear from Varnish 4.x */
-struct VSM_chunk *VSM_iter0(struct VSM_data *vd);
-
-/* OBSOLETE: Will disappear from Varnish 4.x */
-void VSM_itern(struct VSM_data *vd, struct VSM_chunk **pp);
-
-/* OBSOLETE: Will disappear from Varnish 4.x */
-#define VSM_FOREACH(var, vd) \
-    for((var) = VSM_iter0((vd)); (var) != NULL; VSM_itern((vd), &(var)))
-
-	/*
-	 * Iterate over all chunks in shared memory
-	 * var = "struct VSM_chunk *"
-	 * vd = "struct VSM_data"
-	 */
-
 #endif /* VAPI_VSM_H_INCLUDED */
diff --git a/lib/libvarnishapi/vsm.c b/lib/libvarnishapi/vsm.c
index 6c78baa..d136c98 100644
--- a/lib/libvarnishapi/vsm.c
+++ b/lib/libvarnishapi/vsm.c
@@ -35,6 +35,7 @@
 
 #include <errno.h>
 #include <fcntl.h>
+#include <stdarg.h>
 #include <stdint.h>
 #include <stdio.h>
 #include <stdlib.h>
@@ -47,12 +48,14 @@
 #include "vapi/vsm.h"
 #include "vapi/vsm_int.h"
 #include "vin.h"
+#include "vsb.h"
 #include "vsm_api.h"
 
 #ifndef MAP_HASSEMAPHORE
 #define MAP_HASSEMAPHORE 0 /* XXX Linux */
 #endif
 
+
 /*--------------------------------------------------------------------*/
 
 struct VSM_data *
@@ -64,9 +67,6 @@ VSM_New(void)
 	if (vd == NULL)
 		return (vd);
 
-	vd->diag = (VSM_diag_f*)fprintf;
-	vd->priv = stderr;
-
 	vd->vsm_fd = -1;
 
 	CHECK_OBJ_NOTNULL(vd, VSM_MAGIC);
@@ -75,16 +75,35 @@ VSM_New(void)
 
 /*--------------------------------------------------------------------*/
 
-void
-VSM_Diag(struct VSM_data *vd, VSM_diag_f *func, void *priv)
+static int
+vsm_diag(struct VSM_data *vd, const char *fmt, ...)
+{
+	va_list ap;
+
+	CHECK_OBJ_NOTNULL(vd, VSM_MAGIC);
+	AN(fmt);
+
+	if (vd->diag == NULL)
+		vd->diag = VSB_new_auto();
+	AN(vd->diag);
+	va_start(ap, fmt);
+	VSB_vprintf(vd->diag, fmt, ap);
+	va_end(ap);
+	AZ(VSB_finish(vd->diag));
+	return (-1);
+}
+/*--------------------------------------------------------------------*/
+
+const char *
+VSM_Error(const struct VSM_data *vd)
 {
 
 	CHECK_OBJ_NOTNULL(vd, VSM_MAGIC);
-	if (func == NULL)
-		vd->diag = (VSM_diag_f*)getpid;
+
+	if (vd->diag == NULL)
+		return (NULL);
 	else
-		vd->diag = func;
-	vd->priv = priv;
+		return (VSB_data(vd->diag));
 }
 
 /*--------------------------------------------------------------------*/
@@ -94,13 +113,12 @@ VSM_n_Arg(struct VSM_data *vd, const char *opt)
 {
 
 	CHECK_OBJ_NOTNULL(vd, VSM_MAGIC);
-	REPLACE(vd->n_opt, opt);
 	AN(vd->n_opt);
-	if (VIN_N_Arg(vd->n_opt, NULL, NULL, &vd->fname)) {
-		vd->diag(vd->priv, "Invalid instance name: %s\n",
-		    strerror(errno));
-		return (-1);
-	}
+
+	REPLACE(vd->n_opt, opt);
+	if (VIN_N_Arg(vd->n_opt, NULL, NULL, &vd->fname))
+		return (vsm_diag(vd, "Invalid instance name: %s\n",
+		    strerror(errno)));
 	return (1);
 }
 
@@ -111,6 +129,7 @@ VSM_Name(const struct VSM_data *vd)
 {
 
 	CHECK_OBJ_NOTNULL(vd, VSM_MAGIC);
+
 	return (vd->n_opt);
 }
 
@@ -123,15 +142,12 @@ VSM_Delete(struct VSM_data *vd)
 	CHECK_OBJ_NOTNULL(vd, VSM_MAGIC);
 
 	VSM_Close(vd);
-
 	free(vd->n_opt);
 	free(vd->fname);
-
 	if (vd->vsc != NULL)
 		VSC_Delete(vd);
 	if (vd->vsl != NULL)
 		VSL_Delete(vd);
-
 	FREE_OBJ(vd);
 }
 
@@ -144,64 +160,59 @@ VSM_Delete(struct VSM_data *vd)
  *
  */
 
-static int
-vsm_open(struct VSM_data *vd, int diag)
+/*--------------------------------------------------------------------*/
+
+int
+VSM_Open(struct VSM_data *vd)
 {
 	int i;
 	struct VSM_head slh;
 	void *v;
 
 	CHECK_OBJ_NOTNULL(vd, VSM_MAGIC);
+
+	AZ(vd->head);
+	if (!vd->n_opt)
+		(void)VSM_n_Arg(vd, "");
+
 	AZ(vd->head);
 	AN(vd->fname);
 
 	vd->vsm_fd = open(vd->fname, O_RDONLY);
-	if (vd->vsm_fd < 0) {
-		if (diag)
-			vd->diag(vd->priv, "Cannot open %s: %s\n",
-			    vd->fname, strerror(errno));
-		return (-1);
-	}
+	if (vd->vsm_fd < 0)
+		return (vsm_diag(vd, "Cannot open %s: %s\n",
+		    vd->fname, strerror(errno)));
 
 	AZ(fstat(vd->vsm_fd, &vd->fstat));
 	if (!S_ISREG(vd->fstat.st_mode)) {
-		if (diag)
-			vd->diag(vd->priv, "%s is not a regular file\n",
-			    vd->fname);
 		AZ(close(vd->vsm_fd));
 		vd->vsm_fd = -1;
-		return (-1);
+		return (vsm_diag(vd, "%s is not a regular file\n",
+		    vd->fname));
 	}
 
 	i = read(vd->vsm_fd, &slh, sizeof slh);
 	if (i != sizeof slh) {
-		if (diag)
-			vd->diag(vd->priv, "Cannot read %s: %s\n",
-			    vd->fname, strerror(errno));
 		AZ(close(vd->vsm_fd));
 		vd->vsm_fd = -1;
-		return (-1);
+		return(vsm_diag(vd, "Cannot read %s: %s\n",
+		    vd->fname, strerror(errno)));
 	}
 
 	if (memcmp(slh.marker, VSM_HEAD_MARKER, sizeof slh.marker) ||
 	    slh.alloc_seq == 0) {
-		if (diag)
-			vd->diag(vd->priv, "Not a VSM file %s\n",
-			    vd->fname);
 		AZ(close(vd->vsm_fd));
 		vd->vsm_fd = -1;
-		return (-1);
+		return (vsm_diag(vd, "Not a VSM file %s\n", vd->fname));
 	}
 
 	v = mmap(NULL, slh.shm_size,
 	    PROT_READ, MAP_SHARED|MAP_HASSEMAPHORE, vd->vsm_fd, 0);
 	if (v == MAP_FAILED) {
-		if (diag)
-			vd->diag(vd->priv, "Cannot mmap %s: %s\n",
-			    vd->fname, strerror(errno));
 		AZ(close(vd->vsm_fd));
 		vd->vsm_fd = -1;
-		return (-1);
+		return (vsm_diag(vd, "Cannot mmap %s: %s\n",
+		    vd->fname, strerror(errno)));
 	}
 	vd->head = v;
 	vd->b = v;
@@ -212,25 +223,12 @@ vsm_open(struct VSM_data *vd, int diag)
 
 /*--------------------------------------------------------------------*/
 
-int
-VSM_Open(struct VSM_data *vd, int diag)
-
-{
-
-	CHECK_OBJ_NOTNULL(vd, VSM_MAGIC);
-	AZ(vd->head);
-	if (!vd->n_opt)
-		(void)VSM_n_Arg(vd, "");
-	return (vsm_open(vd, diag));
-}
-
-/*--------------------------------------------------------------------*/
-
 void
 VSM_Close(struct VSM_data *vd)
 {
 
 	CHECK_OBJ_NOTNULL(vd, VSM_MAGIC);
+
 	if (vd->head == NULL)
 		return;
 
@@ -246,42 +244,24 @@ VSM_Close(struct VSM_data *vd)
 /*--------------------------------------------------------------------*/
 
 int
-VSM_ReOpen(struct VSM_data *vd, int diag)
+VSM_Abandonned(const struct VSM_data *vd)
 {
 	struct stat st;
 
 	CHECK_OBJ_NOTNULL(vd, VSM_MAGIC);
-	AN(vd->head);
-
-	if (vd->head->alloc_seq &&
-	    !stat(vd->fname, &st) &&
-	    st.st_dev == vd->fstat.st_dev &&
-	    st.st_ino == vd->fstat.st_ino)
-		return (0);
-
-	VSM_Close(vd);
-	return (vsm_open(vd, diag));
-}
-
-/*--------------------------------------------------------------------*/
-
-unsigned
-VSM_Seq(const struct VSM_data *vd)
-{
-
-	CHECK_OBJ_NOTNULL(vd, VSM_MAGIC);
-	return (vd->head->alloc_seq);
-}
-
-/*--------------------------------------------------------------------*/
 
-struct VSM_head *
-VSM_Head(const struct VSM_data *vd)
-{
+	if (vd->head == NULL)
+		return (1);
 
-	CHECK_OBJ_NOTNULL(vd, VSM_MAGIC);
-	AN(vd->head);
-	return(vd->head);
+	if (!vd->head->alloc_seq)
+		return (1);
+	if (!stat(vd->fname, &st))
+		return (1);
+	if (st.st_dev != vd->fstat.st_dev)
+		return (1);
+	if (st.st_ino != vd->fstat.st_ino)
+		return (1);
+	return (0);
 }
 
 /*--------------------------------------------------------------------*/
@@ -291,15 +271,20 @@ VSM__iter0(const struct VSM_data *vd, struct VSM_fantom *vf)
 {
 
 	CHECK_OBJ_NOTNULL(vd, VSM_MAGIC);
+	AN(vf);
+
 	memset(vf, 0, sizeof *vf);
 }
 
+/* XXX: revisit, logic is unclear */
 int
 VSM__itern(const struct VSM_data *vd, struct VSM_fantom *vf)
 {
 	void *p;
 
 	CHECK_OBJ_NOTNULL(vd, VSM_MAGIC);
+	AN(vf);
+
 	if (vd->head->alloc_seq == 0)
 		return (0);	/* abandonned VSM */
 	else if (vf->priv != 0) {
@@ -325,7 +310,7 @@ VSM__itern(const struct VSM_data *vd, struct VSM_fantom *vf)
 	vf->b = (void*)(vf->chunk + 1);
 	vf->e = (char*)vf->b + vf->chunk->len;
 
-	if (vd->priv == 0)
+	if (vf->priv == 0)
 		return (0);	/* abandonned VSM */
 	if (vf->b == vf->e)
 		return (0);	/* freed chunk */
@@ -342,14 +327,13 @@ VSM_StillValid(const struct VSM_data *vd, struct VSM_fantom *vf)
 	struct VSM_fantom f2;
 
 	CHECK_OBJ_NOTNULL(vd, VSM_MAGIC);
-	if (vf == NULL)
-		return (vd->head->alloc_seq ? 1 : 0);
+	AN(vf);
+	if (!vd->head->alloc_seq)
+		return (0);
 	if (vf->priv == vd->head->alloc_seq)
 		return (1);
 	VSM_FOREACH_SAFE(&f2, vd) {
-		if (f2.chunk == vf->chunk &&
-		   f2.b == vf->b &&
-		   f2.e == vf->e) {
+		if (f2.chunk == vf->chunk && f2.b == vf->b && f2.e == vf->e) {
 			vf->priv = vd->head->alloc_seq;
 			return (2);
 		}
@@ -375,45 +359,3 @@ VSM_Get(const struct VSM_data *vd, struct VSM_fantom *vf,
 	memset(vf, 0, sizeof *vf);
 	return (0);
 }
-
-/*--------------------------------------------------------------------*/
-
-void *
-VSM_Find_Chunk(const struct VSM_data *vd, const char *class,
-    const char *type, const char *ident, unsigned *lenp)
-{
-	struct VSM_fantom vf;
-
-	if (VSM_Get(vd, &vf, class, type, ident)) {
-		if (lenp != NULL)
-			*lenp = (char*)vf.e - (char*)vf.b;
-		return (vf.chunk);
-	}
-	if (lenp != NULL)
-		*lenp = 0;
-	return (NULL);
-}
-
-/*--------------------------------------------------------------------*/
-
-struct VSM_chunk *
-VSM_iter0(struct VSM_data *vd)
-{
-
-	CHECK_OBJ_NOTNULL(vd, VSM_MAGIC);
-	VSM__iter0(vd, &vd->compat_vf);
-	if (VSM__itern(vd, &vd->compat_vf))
-		return(vd->compat_vf.chunk);
-	return (NULL);
-}
-
-void
-VSM_itern(struct VSM_data *vd, struct VSM_chunk **pp)
-{
-
-	CHECK_OBJ_NOTNULL(vd, VSM_MAGIC);
-	if (VSM__itern(vd, &vd->compat_vf))
-		*pp = vd->compat_vf.chunk;
-	else
-		*pp = NULL;
-}
diff --git a/lib/libvarnishapi/vsm_api.h b/lib/libvarnishapi/vsm_api.h
index 3355ee4..6690ba6 100644
--- a/lib/libvarnishapi/vsm_api.h
+++ b/lib/libvarnishapi/vsm_api.h
@@ -29,13 +29,13 @@
  */
 
 struct vsc;
+struct vsb;
 
 struct VSM_data {
 	unsigned		magic;
 #define VSM_MAGIC		0x6e3bd69b
 
-	VSM_diag_f		*diag;
-	void			*priv;
+	struct vsb		*diag;
 
 	char			*n_opt;
 	char			*fname;



From geoff at varnish-cache.org  Mon Jan  9 20:52:19 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:19 +0100
Subject: [experimental-ims] b06654b Drop explicit dependeny on Makefile for
	default_vcl.h to avoid spurious rebuilds
Message-ID: <E1RkMCZ-0007jF-HO@project.varnish-software.com>

commit b06654b978c59181f53c19a66b1468b0f11acc0f
Author: Tollef Fog Heen <tfheen at varnish-software.com>
Date:   Fri Oct 14 12:39:57 2011 +0200

    Drop explicit dependeny on Makefile for default_vcl.h to avoid spurious rebuilds

diff --git a/bin/varnishd/Makefile.am b/bin/varnishd/Makefile.am
index f2e9d04..c95b09a 100644
--- a/bin/varnishd/Makefile.am
+++ b/bin/varnishd/Makefile.am
@@ -115,7 +115,7 @@ DISTCLEANFILES = default_vcl.h
 #
 # Turn the default.vcl file into a C-string we can include in the program.
 #
-default_vcl.h:	default.vcl Makefile
+default_vcl.h:	default.vcl
 	echo '/*' > $@
 	echo ' * NB:  This file is machine generated, DO NOT EDIT!' >> $@
 	echo ' *' >> $@



From geoff at varnish-cache.org  Mon Jan  9 20:51:49 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:51:49 +0100
Subject: [experimental-ims] c9d9539 Further Pool/Wrk separation.
Message-ID: <E1RkMC5-0006qK-9Y@project.varnish-software.com>

commit c9d9539defa9769e688a5af0864fadcdf63ae7e9
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sat Sep 17 09:54:05 2011 +0000

    Further Pool/Wrk separation.

diff --git a/bin/varnishd/cache.h b/bin/varnishd/cache.h
index 5b4b575..a6b1786 100644
--- a/bin/varnishd/cache.h
+++ b/bin/varnishd/cache.h
@@ -840,10 +840,8 @@ void PipeSession(struct sess *sp);
 
 /* cache_pool.c */
 void Pool_Init(void);
-int WRK_QueueSession(struct sess *sp);
-void WRK_SumStat(struct worker *w);
-int WRK_TrySumStat(struct worker *w);
-void WRK_thread_real(void *priv, struct worker *w);
+int Pool_QueueSession(struct sess *sp);
+void Pool_Work_Thread(void *priv, struct worker *w);
 
 #define WRW_IsReleased(w)	((w)->wrw.wfd == NULL)
 int WRW_Error(const struct worker *w);
@@ -858,9 +856,6 @@ unsigned WRW_WriteH(struct worker *w, const txt *hh, const char *suf);
 void WRW_Sendfile(struct worker *w, int fd, off_t off, unsigned len);
 #endif  /* SENDFILE_WORKS */
 
-typedef void *bgthread_t(struct sess *, void *priv);
-void WRK_BgThread(pthread_t *thr, const char *name, bgthread_t *func,
-    void *priv);
 
 /* cache_session.c [SES] */
 void SES_Init(void);
@@ -937,7 +932,12 @@ void VMOD_Init(void);
 /* cache_wrk.c */
 
 void WRK_Init(void);
+int WRK_TrySumStat(struct worker *w);
+void WRK_SumStat(struct worker *w);
 void *WRK_thread(void *priv);
+typedef void *bgthread_t(struct sess *, void *priv);
+void WRK_BgThread(pthread_t *thr, const char *name, bgthread_t *func,
+    void *priv);
 
 /* cache_ws.c */
 
diff --git a/bin/varnishd/cache_acceptor.c b/bin/varnishd/cache_acceptor.c
index 2b3f87e..0692d46 100644
--- a/bin/varnishd/cache_acceptor.c
+++ b/bin/varnishd/cache_acceptor.c
@@ -319,7 +319,7 @@ vca_acct(void *arg)
 			sp->sockaddrlen = l;
 
 			sp->step = STP_FIRST;
-			if (WRK_QueueSession(sp)) {
+			if (Pool_QueueSession(sp)) {
 				VSC_C_main->client_drop++;
 				pace += params->acceptor_sleep_incr;
 			} else {
@@ -347,7 +347,7 @@ vca_handover(struct sess *sp, int status)
 		break;
 	case 1:
 		sp->step = STP_START;
-		if (WRK_QueueSession(sp))
+		if (Pool_QueueSession(sp))
 			VSC_C_main->client_drop_late++;
 		break;
 	default:
diff --git a/bin/varnishd/cache_hash.c b/bin/varnishd/cache_hash.c
index 328964a..16e5cf0 100644
--- a/bin/varnishd/cache_hash.c
+++ b/bin/varnishd/cache_hash.c
@@ -504,7 +504,7 @@ hsh_rush(struct objhead *oh)
 		AZ(sp->wrk);
 		VTAILQ_REMOVE(&wl->list, sp, list);
 		DSL(0x20, SLT_Debug, sp->id, "off waiting list");
-		if (WRK_QueueSession(sp)) {
+		if (Pool_QueueSession(sp)) {
 			/*
 			 * We could not schedule the session, leave the
 			 * rest on the busy list.
diff --git a/bin/varnishd/cache_pool.c b/bin/varnishd/cache_pool.c
index b7f1f41..a80cd9b 100644
--- a/bin/varnishd/cache_pool.c
+++ b/bin/varnishd/cache_pool.c
@@ -56,7 +56,6 @@
 #include "cache.h"
 #include "stevedore.h"
 #include "hash_slinger.h"
-#include "vsha256.h"
 
 VTAILQ_HEAD(workerhead, worker);
 
@@ -86,7 +85,7 @@ static struct lock		herder_mtx;
 /*--------------------------------------------------------------------*/
 
 void
-WRK_thread_real(void *priv, struct worker *w)
+Pool_Work_Thread(void *priv, struct worker *w)
 {
 	struct wq *qp;
 	int stats_clean;
@@ -217,7 +216,7 @@ wrk_do_cnt_sess(struct worker *w, void *priv)
 /*--------------------------------------------------------------------*/
 
 int
-WRK_QueueSession(struct sess *sp)
+Pool_QueueSession(struct sess *sp)
 {
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 	AZ(sp->wrk);
@@ -456,58 +455,6 @@ wrk_herder_thread(void *priv)
 	NEEDLESS_RETURN(NULL);
 }
 
-/*--------------------------------------------------------------------
- * Create and starte a back-ground thread which as its own worker and
- * session data structures;
- */
-
-struct bgthread {
-	unsigned	magic;
-#define BGTHREAD_MAGIC	0x23b5152b
-	const char	*name;
-	bgthread_t	*func;
-	void		*priv;
-};
-
-static void *
-wrk_bgthread(void *arg)
-{
-	struct bgthread *bt;
-	struct worker ww;
-	struct sess *sp;
-	uint32_t logbuf[1024];	/* XXX:  size ? */
-
-	CAST_OBJ_NOTNULL(bt, arg, BGTHREAD_MAGIC);
-	THR_SetName(bt->name);
-	sp = SES_Alloc();
-	XXXAN(sp);
-	memset(&ww, 0, sizeof ww);
-	sp->wrk = &ww;
-	ww.magic = WORKER_MAGIC;
-	ww.wlp = ww.wlb = logbuf;
-	ww.wle = logbuf + (sizeof logbuf) / 4;
-
-	(void)bt->func(sp, bt->priv);
-
-	WRONG("BgThread terminated");
-
-	NEEDLESS_RETURN(NULL);
-}
-
-void
-WRK_BgThread(pthread_t *thr, const char *name, bgthread_t *func, void *priv)
-{
-	struct bgthread *bt;
-
-	ALLOC_OBJ(bt, BGTHREAD_MAGIC);
-	AN(bt);
-
-	bt->name = name;
-	bt->func = func;
-	bt->priv = priv;
-	AZ(pthread_create(thr, NULL, wrk_bgthread, bt));
-}
-
 /*--------------------------------------------------------------------*/
 
 void
diff --git a/bin/varnishd/cache_wrk.c b/bin/varnishd/cache_wrk.c
index 9b3bc2b..66ba04b 100644
--- a/bin/varnishd/cache_wrk.c
+++ b/bin/varnishd/cache_wrk.c
@@ -43,7 +43,6 @@
 #include "vcl.h"
 #include "cli_priv.h"
 #include "cache.h"
-#include "stevedore.h"
 #include "hash_slinger.h"
 #include "vsha256.h"
 
@@ -87,6 +86,58 @@ WRK_TrySumStat(struct worker *w)
 	return (1);
 }
 
+/*--------------------------------------------------------------------
+ * Create and starte a back-ground thread which as its own worker and
+ * session data structures;
+ */
+
+struct bgthread {
+	unsigned	magic;
+#define BGTHREAD_MAGIC	0x23b5152b
+	const char	*name;
+	bgthread_t	*func;
+	void		*priv;
+};
+
+static void *
+wrk_bgthread(void *arg)
+{
+	struct bgthread *bt;
+	struct worker ww;
+	struct sess *sp;
+	uint32_t logbuf[1024];	/* XXX:  size ? */
+
+	CAST_OBJ_NOTNULL(bt, arg, BGTHREAD_MAGIC);
+	THR_SetName(bt->name);
+	sp = SES_Alloc();
+	XXXAN(sp);
+	memset(&ww, 0, sizeof ww);
+	sp->wrk = &ww;
+	ww.magic = WORKER_MAGIC;
+	ww.wlp = ww.wlb = logbuf;
+	ww.wle = logbuf + (sizeof logbuf) / 4;
+
+	(void)bt->func(sp, bt->priv);
+
+	WRONG("BgThread terminated");
+
+	NEEDLESS_RETURN(NULL);
+}
+
+void
+WRK_BgThread(pthread_t *thr, const char *name, bgthread_t *func, void *priv)
+{
+	struct bgthread *bt;
+
+	ALLOC_OBJ(bt, BGTHREAD_MAGIC);
+	AN(bt);
+
+	bt->name = name;
+	bt->func = func;
+	bt->priv = priv;
+	AZ(pthread_create(thr, NULL, wrk_bgthread, bt));
+}
+
 /*--------------------------------------------------------------------*/
 
 static void *
@@ -123,7 +174,7 @@ wrk_thread_real(void *priv, unsigned shm_workspace, unsigned sess_workspace,
 
 	VSL(SLT_WorkThread, 0, "%p start", w);
 
-	WRK_thread_real(priv, w);
+	Pool_Work_Thread(priv, w);
 
 	VSL(SLT_WorkThread, 0, "%p end", w);
 	if (w->vcl != NULL)



From geoff at varnish-cache.org  Mon Jan  9 20:52:16 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:16 +0100
Subject: [experimental-ims] 4766dba Don't include config.h in .h files
Message-ID: <E1RkMCW-0007cx-R1@project.varnish-software.com>

commit 4766dba945e95c5e110f6d625a10bd811836afa3
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sun Oct 9 21:02:54 2011 +0000

    Don't include config.h in .h files

diff --git a/include/vbm.h b/include/vbm.h
index 8443ac7..13d2d5f 100644
--- a/include/vbm.h
+++ b/include/vbm.h
@@ -29,8 +29,6 @@
  * Self-sizeing bitmap operations
  */
 
-#include "config.h"
-
 /**********************************************************************
  * Generic bitmap functions, may be generalized at some point.
  */



From geoff at varnish-cache.org  Mon Jan  9 20:52:41 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:41 +0100
Subject: [experimental-ims] 30a37c0 Compilers these days have no sense of fun!
Message-ID: <E1RkMCv-0008NR-6J@project.varnish-software.com>

commit 30a37c06d190338089b4e0e819b063298e34d37d
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Nov 21 08:15:21 2011 +0000

    Compilers these days have no sense of fun!

diff --git a/bin/varnishd/common/common_vsm.c b/bin/varnishd/common/common_vsm.c
index 4eb4195..3853ed8 100644
--- a/bin/varnishd/common/common_vsm.c
+++ b/bin/varnishd/common/common_vsm.c
@@ -289,7 +289,7 @@ VSM_common_free(struct vsm_sc *sc, void *ptr)
 		}
 	}
 	/* Panic */
-	assert(ptr == "Bogus pointer freed");
+	assert(ptr == NULL);
 }
 
 /*--------------------------------------------------------------------



From geoff at varnish-cache.org  Mon Jan  9 20:52:40 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:40 +0100
Subject: [experimental-ims] eb18d9b Move VSC_C_main to cache namespace only
Message-ID: <E1RkMCu-0008M2-Js@project.varnish-software.com>

commit eb18d9be13c042ee95b4c102641edb7693003c9f
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sun Nov 20 19:42:14 2011 +0000

    Move VSC_C_main to cache namespace only

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index 432e22b..e08cc32 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -850,6 +850,7 @@ int SES_Schedule(struct sess *sp);
 
 
 /* cache_shmlog.c */
+extern struct VSC_C_main *VSC_C_main;
 void VSL_Init(void);
 void *VSM_Alloc(unsigned size, const char *class, const char *type,
     const char *ident);
diff --git a/bin/varnishd/cache/cache_shmlog.c b/bin/varnishd/cache/cache_shmlog.c
index 950f57f..01f6bce 100644
--- a/bin/varnishd/cache/cache_shmlog.c
+++ b/bin/varnishd/cache/cache_shmlog.c
@@ -47,6 +47,8 @@ static uint32_t			*vsl_start;
 static const uint32_t		*vsl_end;
 static uint32_t			*vsl_ptr;
 
+struct VSC_C_main       *VSC_C_main;
+
 static inline uint32_t
 vsl_w0(uint32_t type, uint32_t length)
 {
diff --git a/bin/varnishd/common/common.h b/bin/varnishd/common/common.h
index a5d7633..157e5f7 100644
--- a/bin/varnishd/common/common.h
+++ b/bin/varnishd/common/common.h
@@ -49,7 +49,6 @@ extern pid_t mgt_pid;
 #define ASSERT_MGT() do { assert(getpid() == mgt_pid);} while (0)
 
 /* mgt_shmem.c */
-extern struct VSC_C_main *VSC_C_main;
 #define PAN_CLASS "Panic"
 extern char *PAN_panicstr;
 extern unsigned PAN_panicstr_len;
diff --git a/bin/varnishd/mgt/mgt_shmem.c b/bin/varnishd/mgt/mgt_shmem.c
index 40f0c24..5915b19 100644
--- a/bin/varnishd/mgt/mgt_shmem.c
+++ b/bin/varnishd/mgt/mgt_shmem.c
@@ -57,8 +57,6 @@
 #define MAP_NOSYNC 0 /* XXX Linux */
 #endif
 
-struct VSC_C_main	*VSC_C_main;
-
 static int vsm_fd = -1;
 
 /*--------------------------------------------------------------------



From geoff at varnish-cache.org  Mon Jan  9 20:51:59 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:51:59 +0100
Subject: [experimental-ims] fb5477b Don't ignore the return value of the
	read(2) call on a random dev.
Message-ID: <E1RkMCF-00077K-PL@project.varnish-software.com>

commit fb5477bec8dad639f3db0bd2ac6900b3010a61a6
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Sep 20 13:44:59 2011 +0000

    Don't ignore the return value of the read(2) call on a random dev.

diff --git a/lib/libvarnishcompat/srandomdev.c b/lib/libvarnishcompat/srandomdev.c
index 79f774d..d0332c9 100644
--- a/lib/libvarnishcompat/srandomdev.c
+++ b/lib/libvarnishcompat/srandomdev.c
@@ -40,6 +40,22 @@
 
 #include "compat/srandomdev.h"
 
+static int
+trydev(const char *fn, unsigned long *seed)
+{
+	int fd;
+	ssize_t sz;
+
+	fd = open(fn, O_RDONLY);
+	if (fd < 0)
+		return (-1);
+	sz = read(fd, seed, sizeof *seed);
+	(void)close(fd);
+	if (sz != sizeof *seed)
+		return (-1);
+	return (0);
+}
+
 void
 srandomdev(void)
 {
@@ -47,13 +63,11 @@ srandomdev(void)
 	unsigned long seed;
 	int fd;
 
-	if ((fd = open("/dev/urandom", O_RDONLY)) >= 0 ||
-	    (fd = open("/dev/random", O_RDONLY)) >= 0) {
-		read(fd, &seed, sizeof seed);
-		close(fd);
-	} else {
-		gettimeofday(&tv, NULL);
-		seed = (getpid() << 16) ^ tv.tv_sec ^ tv.tv_usec;
+	if (trydev("/dev/urandom", &seed)) {
+		if (trydev("/dev/random", &seed)) {
+			gettimeofday(&tv, NULL);
+			seed = (getpid() << 16) ^ tv.tv_sec ^ tv.tv_usec;
+		}
 	}
 	srandom(seed);
 }



From geoff at varnish-cache.org  Mon Jan  9 20:52:26 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:26 +0100
Subject: [experimental-ims] 0b3ec32 Update to 3.0 syntax
Message-ID: <E1RkMCg-0007vZ-9x@project.varnish-software.com>

commit 0b3ec32c049af7102c6d8dfbb38e1fcd06f300a9
Author: Andreas Plesner Jacobsen <apj at mutt.dk>
Date:   Mon Oct 24 22:38:48 2011 +0200

    Update to 3.0 syntax

diff --git a/doc/sphinx/tutorial/purging.rst b/doc/sphinx/tutorial/purging.rst
index e5da595..cea1ccd 100644
--- a/doc/sphinx/tutorial/purging.rst
+++ b/doc/sphinx/tutorial/purging.rst
@@ -134,7 +134,7 @@ object is not available in the ban lurker thread.
 You can use the following template to write ban lurker friendly bans::
 
   sub vcl_fetch {
-    set obj.http.x-url = req.url;
+    set beresp.http.x-url = req.url;
   }
 
   sub vcl_deliver {



From geoff at varnish-cache.org  Mon Jan  9 20:52:16 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:16 +0100
Subject: [experimental-ims] 080f259 Push "vas.h", <stdint.h>,
	and NULL out of libvarnish.h
Message-ID: <E1RkMCW-0007cA-Gi@project.varnish-software.com>

commit 080f2595d2d5f71a6795445cc6f688d221c48f08
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sun Oct 9 19:51:08 2011 +0000

    Push "vas.h", <stdint.h>, and NULL out of libvarnish.h

diff --git a/bin/varnishadm/varnishadm.c b/bin/varnishadm/varnishadm.c
index d371b9f..b857f8a 100644
--- a/bin/varnishadm/varnishadm.c
+++ b/bin/varnishadm/varnishadm.c
@@ -46,7 +46,7 @@
 
 #include "vcli.h"
 #include "cli_common.h"
-#include "libvarnish.h"
+#include "vas.h"
 #include "vss.h"
 
 #ifdef HAVE_LIBEDIT
diff --git a/bin/varnishd/common.h b/bin/varnishd/common.h
index f755b97..0e6d95d 100644
--- a/bin/varnishd/common.h
+++ b/bin/varnishd/common.h
@@ -29,6 +29,7 @@
  */
 
 #include "libvarnish.h"
+#include "vas.h"
 #include "vsb.h"
 #include "vqueue.h"
 #include "miniobj.h"
diff --git a/bin/varnishhist/varnishhist.c b/bin/varnishhist/varnishhist.c
index 9485df7..92d17b3 100644
--- a/bin/varnishhist/varnishhist.c
+++ b/bin/varnishhist/varnishhist.c
@@ -48,6 +48,7 @@
 #include <string.h>
 #include <unistd.h>
 
+#include "vas.h"
 #include "libvarnish.h"
 
 #define HIST_N 2000 /* how far back we remember */
diff --git a/bin/varnishlog/varnishlog.c b/bin/varnishlog/varnishlog.c
index c67432d..3142d1b 100644
--- a/bin/varnishlog/varnishlog.c
+++ b/bin/varnishlog/varnishlog.c
@@ -47,6 +47,7 @@
 #include "vsb.h"
 #include "vpf.h"
 
+#include "vas.h"
 #include "libvarnish.h"
 
 static int	b_flag, c_flag;
diff --git a/bin/varnishncsa/varnishncsa.c b/bin/varnishncsa/varnishncsa.c
index d010a73..4805d81 100644
--- a/bin/varnishncsa/varnishncsa.c
+++ b/bin/varnishncsa/varnishncsa.c
@@ -73,6 +73,7 @@
 
 #include "compat/daemon.h"
 
+#include "vas.h"
 #include "vsb.h"
 #include "vpf.h"
 #include "vqueue.h"
diff --git a/bin/varnishreplay/varnishreplay.c b/bin/varnishreplay/varnishreplay.c
index dbb6f51..ef4118a 100644
--- a/bin/varnishreplay/varnishreplay.c
+++ b/bin/varnishreplay/varnishreplay.c
@@ -48,6 +48,7 @@
 
 #include "vqueue.h"
 
+#include "vas.h"
 #include "libvarnish.h"
 #include "vss.h"
 
diff --git a/bin/varnishsizes/varnishsizes.c b/bin/varnishsizes/varnishsizes.c
index ac9dbf8..70fcad1 100644
--- a/bin/varnishsizes/varnishsizes.c
+++ b/bin/varnishsizes/varnishsizes.c
@@ -48,6 +48,7 @@
 #include <string.h>
 #include <unistd.h>
 
+#include "vas.h"
 #include "libvarnish.h"
 
 #define HIST_N 2000 /* how far back we remember */
diff --git a/bin/varnishstat/varnishstat.c b/bin/varnishstat/varnishstat.c
index 76a2e03..b160a26 100644
--- a/bin/varnishstat/varnishstat.c
+++ b/bin/varnishstat/varnishstat.c
@@ -44,7 +44,6 @@
 #include <time.h>
 #include <unistd.h>
 
-#include "libvarnish.h"
 #include "varnishstat.h"
 
 /*--------------------------------------------------------------------*/
diff --git a/bin/varnishstat/varnishstat.h b/bin/varnishstat/varnishstat.h
index 245af35..322ebc4 100644
--- a/bin/varnishstat/varnishstat.h
+++ b/bin/varnishstat/varnishstat.h
@@ -27,4 +27,7 @@
  *
  */
 
+#include "vas.h"
+#include "libvarnish.h"
+
 void do_curses(struct VSM_data *vd, const struct VSC_C_main *VSC_C_main, int delay);
diff --git a/bin/varnishstat/varnishstat_curses.c b/bin/varnishstat/varnishstat_curses.c
index fc948e9..1e93155 100644
--- a/bin/varnishstat/varnishstat_curses.c
+++ b/bin/varnishstat/varnishstat_curses.c
@@ -50,7 +50,6 @@
 #include <time.h>
 #include <unistd.h>
 
-#include "libvarnish.h"
 #include "vqueue.h"
 #include "varnishstat.h"
 
diff --git a/bin/varnishtest/vtc.h b/bin/varnishtest/vtc.h
index e96940d..3621dc5 100644
--- a/bin/varnishtest/vtc.h
+++ b/bin/varnishtest/vtc.h
@@ -34,11 +34,11 @@
 #include <pthread_np.h>
 #endif
 
+#include "miniobj.h"
+#include "vas.h"
 #include "vqueue.h"
 #include "vsb.h"
-#include "miniobj.h"
 
-struct vsb;
 struct vtclog;
 struct cmds;
 
diff --git a/bin/varnishtop/varnishtop.c b/bin/varnishtop/varnishtop.c
index de3789f..1022e32 100644
--- a/bin/varnishtop/varnishtop.c
+++ b/bin/varnishtop/varnishtop.c
@@ -50,6 +50,7 @@
 
 #include "vsb.h"
 
+#include "vas.h"
 #include "libvarnish.h"
 
 #if 0
diff --git a/include/libvarnish.h b/include/libvarnish.h
index 65287b5..4cbb18d 100644
--- a/include/libvarnish.h
+++ b/include/libvarnish.h
@@ -28,16 +28,6 @@
  *
  */
 
-#include <stdint.h>
-
-#include "vas.h"
-
-#ifndef NULL
-#define NULL ((void*)0)
-#endif
-
-struct vsb;
-
 /* from libvarnish/version.c */
 void VCS_Message(const char *);
 
@@ -54,3 +44,4 @@ void VCS_Message(const char *);
 		assert(vsnprintf(buf, sizeof buf, fmt, ap)		\
 		    < sizeof buf);					\
 	} while (0)
+
diff --git a/include/vnum.h b/include/vnum.h
new file mode 100644
index 0000000..a821355
--- /dev/null
+++ b/include/vnum.h
@@ -0,0 +1,34 @@
+/*-
+ * Copyright (c) 2006 Verdens Gang AS
+ * Copyright (c) 2006-2011 Varnish Software AS
+ * All rights reserved.
+ *
+ * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ */
+
+#include <stdint.h>
+
+/* from libvarnish/vnum.c */
+const char *VNUM_2bytes(const char *p, uintmax_t *r, uintmax_t rel);
diff --git a/lib/libvarnish/binary_heap.c b/lib/libvarnish/binary_heap.c
index f49c736..4b0edc3 100644
--- a/lib/libvarnish/binary_heap.c
+++ b/lib/libvarnish/binary_heap.c
@@ -35,12 +35,13 @@
 
 #include "config.h"
 
-#include <unistd.h>
-#include <stdlib.h>
 #include <limits.h>
+#include <stdint.h>
+#include <stdlib.h>
+#include <unistd.h>
 
 #include "binary_heap.h"
-#include "libvarnish.h"
+#include "vas.h"
 
 /* Parameters --------------------------------------------------------*/
 
diff --git a/lib/libvarnish/cli_auth.c b/lib/libvarnish/cli_auth.c
index fc66fa7..da0a241 100644
--- a/lib/libvarnish/cli_auth.c
+++ b/lib/libvarnish/cli_auth.c
@@ -32,11 +32,10 @@
 #include <stdio.h>
 #include <unistd.h>
 
+#include "vas.h"
 #include "vcli.h"
-#include "libvarnish.h"
 #include "vsha256.h"
 
-
 void
 VCLI_AuthResponse(int S_fd, const char *challenge,
     char response[CLI_AUTH_RESPONSE_LEN + 1])
diff --git a/lib/libvarnish/cli_common.c b/lib/libvarnish/cli_common.c
index 994bd9a..e423da4 100644
--- a/lib/libvarnish/cli_common.c
+++ b/lib/libvarnish/cli_common.c
@@ -36,6 +36,7 @@
 #include <poll.h>
 #include <signal.h>
 #include <stdarg.h>
+#include <stdint.h>
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
@@ -44,7 +45,7 @@
 
 #include "vsb.h"
 
-#include "libvarnish.h"
+#include "vas.h"
 
 #include "vcli.h"
 #include "cli_priv.h"
diff --git a/lib/libvarnish/cli_serve.c b/lib/libvarnish/cli_serve.c
index 03cf1d4..13b9663 100644
--- a/lib/libvarnish/cli_serve.c
+++ b/lib/libvarnish/cli_serve.c
@@ -39,6 +39,7 @@
 #include <time.h>
 #include <unistd.h>
 
+#include "vas.h"
 #include "vqueue.h"
 #include "vsb.h"
 #include "vav.h"
@@ -47,7 +48,6 @@
 #include "cli_priv.h"
 #include "cli_common.h"
 #include "cli_serve.h"
-#include "libvarnish.h"
 #include "miniobj.h"
 
 struct VCLS_func {
diff --git a/lib/libvarnish/vas.c b/lib/libvarnish/vas.c
index 82351ce..6f69928 100644
--- a/lib/libvarnish/vas.c
+++ b/lib/libvarnish/vas.c
@@ -35,7 +35,7 @@
 #include <stdlib.h>
 #include <string.h>
 
-#include "libvarnish.h"
+#include "vas.h"
 
 static void
 VAS_Fail_default(const char *func, const char *file, int line,
diff --git a/lib/libvarnish/vev.c b/lib/libvarnish/vev.c
index c55ebfc..004ce08 100644
--- a/lib/libvarnish/vev.c
+++ b/lib/libvarnish/vev.c
@@ -38,7 +38,7 @@
 #include <stdlib.h>
 #include <pthread.h>
 
-#include "libvarnish.h"
+#include "vas.h"
 #include "vtim.h"
 #include "vev.h"
 #include "miniobj.h"
diff --git a/lib/libvarnish/vin.c b/lib/libvarnish/vin.c
index 0f1b680..ccd58a9 100644
--- a/lib/libvarnish/vin.c
+++ b/lib/libvarnish/vin.c
@@ -36,9 +36,10 @@
 #include <string.h>
 #include <unistd.h>
 
-#include "libvarnish.h"
+#include "vas.h"
 #include "vsm.h"
 #include "vin.h"
+#include "libvarnish.h"
 
 int
 VIN_N_Arg(const char *n_arg, char **name, char **dir, char **vsl)
diff --git a/lib/libvarnish/vlu.c b/lib/libvarnish/vlu.c
index db107a1..01d21de 100644
--- a/lib/libvarnish/vlu.c
+++ b/lib/libvarnish/vlu.c
@@ -34,6 +34,7 @@
 #include <stdlib.h>
 #include <unistd.h>
 #include "libvarnish.h"
+#include "vas.h"
 #include "vlu.h"
 #include "miniobj.h"
 
diff --git a/lib/libvarnish/vpf.c b/lib/libvarnish/vpf.c
index c35b3cd..f6543c6 100644
--- a/lib/libvarnish/vpf.c
+++ b/lib/libvarnish/vpf.c
@@ -34,6 +34,7 @@
 
 #include <errno.h>
 #include <fcntl.h>
+#include <stdint.h>
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
@@ -41,6 +42,7 @@
 
 #include "libvarnish.h"		/* XXX: for assert() */
 #include "flopen.h"
+#include "vas.h"
 #include "vpf.h"
 
 struct vpf_fh {
diff --git a/lib/libvarnish/vre.c b/lib/libvarnish/vre.c
index 68beaf8..e330e99 100644
--- a/lib/libvarnish/vre.c
+++ b/lib/libvarnish/vre.c
@@ -31,6 +31,7 @@
 
 #include "libvarnish.h"
 #include "miniobj.h"
+#include "vas.h"
 #include "vre.h"
 
 struct vre {
diff --git a/lib/libvarnish/vsb.c b/lib/libvarnish/vsb.c
index 75414f7..4930be2 100644
--- a/lib/libvarnish/vsb.c
+++ b/lib/libvarnish/vsb.c
@@ -35,7 +35,7 @@ __FBSDID("$FreeBSD: head/sys/kern/subr_vsb.c 222004 2011-05-17 06:36:32Z phk $")
 #include <stdlib.h>
 #include <string.h>
 
-#include "libvarnish.h"
+#include "vas.h"
 #include "vsb.h"
 
 #define	KASSERT(e, m)		assert(e)
diff --git a/lib/libvarnish/vsha256.c b/lib/libvarnish/vsha256.c
index 83dd6a1..942a186 100644
--- a/lib/libvarnish/vsha256.c
+++ b/lib/libvarnish/vsha256.c
@@ -42,7 +42,7 @@
 #define VBIG_ENDIAN	_BIG_ENDIAN
 #endif
 
-#include "libvarnish.h"
+#include "vas.h"
 #include "vsha256.h"
 #include "vend.h"
 
diff --git a/lib/libvarnish/vss.c b/lib/libvarnish/vss.c
index ff19ac7..297df11 100644
--- a/lib/libvarnish/vss.c
+++ b/lib/libvarnish/vss.c
@@ -43,7 +43,7 @@
 #include <string.h>
 #include <unistd.h>
 
-#include "libvarnish.h"
+#include "vas.h"
 #include "vtcp.h"
 #include "vss.h"
 
diff --git a/lib/libvarnish/vtcp.c b/lib/libvarnish/vtcp.c
index e8932e8..c955e65 100644
--- a/lib/libvarnish/vtcp.c
+++ b/lib/libvarnish/vtcp.c
@@ -51,8 +51,8 @@
 #include <unistd.h>
 #include <math.h>
 
+#include "vas.h"
 #include "vtcp.h"
-#include "libvarnish.h"
 
 /*--------------------------------------------------------------------*/
 
diff --git a/lib/libvcl/vcc_compile.h b/lib/libvcl/vcc_compile.h
index 141f4f1..ebbba86 100644
--- a/lib/libvcl/vcc_compile.h
+++ b/lib/libvcl/vcc_compile.h
@@ -28,15 +28,22 @@
  *
  */
 
-#include "vqueue.h"
+#include <stdint.h>
+
+#include "libvarnish.h"
+#include "vas.h"
 
 #include "miniobj.h"
-#include "vsb.h"
 #include "vcl.h"
-#include "libvarnish.h"
+#include "vqueue.h"
+#include "vsb.h"
 
 #include "vcc_token_defs.h"
 
+#ifndef NULL
+#define NULL ((void*)0)
+#endif
+
 struct vsb;
 
 #define isident1(c) (isalpha(c))



From geoff at varnish-cache.org  Mon Jan  9 20:52:31 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:31 +0100
Subject: [experimental-ims] 6c290ee Merge branch 'master' of
	ssh://git.varnish-cache.org/git/varnish-cache
Message-ID: <E1RkMCl-00088J-Up@project.varnish-software.com>

commit 6c290ee38a280d5c9726ce1b7e10006122429e22
Merge: dd74a96 3ef111c
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu Nov 10 13:53:04 2011 +0000

    Merge branch 'master' of ssh://git.varnish-cache.org/git/varnish-cache



From geoff at varnish-cache.org  Mon Jan  9 20:52:27 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:27 +0100
Subject: [experimental-ims] a0db44f Don't abbreviate panic message output.
Message-ID: <E1RkMCh-0007xe-8Z@project.varnish-software.com>

commit a0db44fdc2ea7b361dce9f17e54a3233ab1886ca
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Oct 31 08:09:01 2011 +0000

    Don't abbreviate panic message output.

diff --git a/bin/varnishtest/vtc_log.c b/bin/varnishtest/vtc_log.c
index 651bd89..57fa27a 100644
--- a/bin/varnishtest/vtc_log.c
+++ b/bin/varnishtest/vtc_log.c
@@ -164,7 +164,7 @@ vtc_log(struct vtclog *vl, int lvl, const char *fmt, ...)
 void
 vtc_dump(struct vtclog *vl, int lvl, const char *pfx, const char *str, int len)
 {
-	int nl = 1;
+	int nl = 1, olen;
 	unsigned l;
 	double tx;
 
@@ -181,10 +181,11 @@ vtc_dump(struct vtclog *vl, int lvl, const char *pfx, const char *str, int len)
 		VSB_printf(vl->vsb, "%s %-4s %4.1f %s(null)\n",
 		    lead[lvl], vl->id, tx, pfx);
 	else {
-		if (len == -1)
+		olen = len;
+		if (len < 0)
 			len = strlen(str);
 		for (l = 0; l < len; l++, str++) {
-			if (l > 512) {
+			if (l > 512 && olen != -2) {
 				VSB_printf(vl->vsb, "...");
 				break;
 			}
diff --git a/bin/varnishtest/vtc_varnish.c b/bin/varnishtest/vtc_varnish.c
index 87b9423..cf1742d 100644
--- a/bin/varnishtest/vtc_varnish.c
+++ b/bin/varnishtest/vtc_varnish.c
@@ -316,7 +316,7 @@ varnish_thread(void *priv)
 		if (i <= 0)
 			break;
 		buf[i] = '\0';
-		vtc_dump(v->vl, 3, "debug", buf, -1);
+		vtc_dump(v->vl, 3, "debug", buf, -2);
 	}
 	return (NULL);
 }



From geoff at varnish-cache.org  Mon Jan  9 20:52:00 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:00 +0100
Subject: [experimental-ims] 63af998 Unrace this testcase a little bit more.
Message-ID: <E1RkMCG-00079o-NW@project.varnish-software.com>

commit 63af9987adee7bca0bd008ce2334835295ec3c90
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Sep 21 09:55:09 2011 +0000

    Unrace this testcase a little bit more.

diff --git a/bin/varnishtest/tests/s00002.vtc b/bin/varnishtest/tests/s00002.vtc
index 939d0ee..3cb56cf 100644
--- a/bin/varnishtest/tests/s00002.vtc
+++ b/bin/varnishtest/tests/s00002.vtc
@@ -53,9 +53,9 @@ varnish v1 -vcl {
 	}
 } -start
 
+sema r1 sync 2
 
 client c1 {
-	sema r1 sync 2
 	txreq -url "/"
 	rxresp
 	expect resp.http.foo == "bar"



From geoff at varnish-cache.org  Mon Jan  9 20:52:09 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:09 +0100
Subject: [experimental-ims] e37a675 Cleanup some #inclues
Message-ID: <E1RkMCP-0007JL-GT@project.varnish-software.com>

commit e37a675a170c69bcad462d4af12da1cbc07425b7
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sat Oct 8 09:20:10 2011 +0000

    Cleanup some #inclues

diff --git a/bin/varnishd/cache_cli.c b/bin/varnishd/cache_cli.c
index dd26303..79ac6f6 100644
--- a/bin/varnishd/cache_cli.c
+++ b/bin/varnishd/cache_cli.c
@@ -49,7 +49,6 @@
 #include "cli_common.h"
 #include "cli_serve.h"
 #include "cache.h"
-#include "vsb.h"
 #include "hash_slinger.h"
 
 pthread_t		cli_thread;
diff --git a/bin/varnishd/mgt.h b/bin/varnishd/mgt.h
index a21ca1c..8958ab5 100644
--- a/bin/varnishd/mgt.h
+++ b/bin/varnishd/mgt.h
@@ -33,6 +33,7 @@
 #include "vqueue.h"
 
 #include "common.h"
+#include "vsb.h"
 #include "miniobj.h"
 
 #include "libvarnish.h"
diff --git a/bin/varnishd/mgt_child.c b/bin/varnishd/mgt_child.c
index 81e1c8d..3d6fa28 100644
--- a/bin/varnishd/mgt_child.c
+++ b/bin/varnishd/mgt_child.c
@@ -51,7 +51,6 @@
 #include "mgt_cli.h"
 #include "vev.h"
 #include "vlu.h"
-#include "vsb.h"
 #include "vss.h"
 #include "vbm.h"
 
diff --git a/bin/varnishd/mgt_cli.c b/bin/varnishd/mgt_cli.c
index e31f3f8..45f08af 100644
--- a/bin/varnishd/mgt_cli.c
+++ b/bin/varnishd/mgt_cli.c
@@ -48,7 +48,6 @@
 
 #include "cli_priv.h"
 #include "vcli.h"
-#include "vsb.h"
 #include "cli_common.h"
 #include "cli_serve.h"
 #include "vev.h"
diff --git a/bin/varnishd/mgt_vcc.c b/bin/varnishd/mgt_vcc.c
index 9e204bc..6c100bb 100644
--- a/bin/varnishd/mgt_vcc.c
+++ b/bin/varnishd/mgt_vcc.c
@@ -41,14 +41,13 @@
 #include <string.h>
 #include <unistd.h>
 
-#include "vsb.h"
+#include "mgt.h"
 
 #include "libvcl.h"
 #include "vcli.h"
 #include "cli_priv.h"
 #include "cli_common.h"
 
-#include "mgt.h"
 #include "mgt_cli.h"
 #include "heritage.h"
 
diff --git a/bin/varnishd/storage_synth.c b/bin/varnishd/storage_synth.c
index 24479e3..1c16b44 100644
--- a/bin/varnishd/storage_synth.c
+++ b/bin/varnishd/storage_synth.c
@@ -36,7 +36,6 @@
 #include <stdlib.h>
 
 #include "cache.h"
-#include "vsb.h"
 #include "stevedore.h"
 #include "hash_slinger.h"
 
diff --git a/bin/varnishd/varnishd.c b/bin/varnishd/varnishd.c
index fd91d08..2962e74 100644
--- a/bin/varnishd/varnishd.c
+++ b/bin/varnishd/varnishd.c
@@ -50,7 +50,6 @@
 
 #include "compat/daemon.h"
 
-#include "vsb.h"
 #include "vev.h"
 #include "vpf.h"
 #include "vsha256.h"
diff --git a/bin/varnishtest/vtc.c b/bin/varnishtest/vtc.c
index 3bee42a..f39b628 100644
--- a/bin/varnishtest/vtc.c
+++ b/bin/varnishtest/vtc.c
@@ -43,9 +43,6 @@
 
 #include "libvarnish.h"
 #include "vev.h"
-#include "vsb.h"
-#include "vqueue.h"
-#include "miniobj.h"
 
 #include "vtc.h"
 
diff --git a/bin/varnishtest/vtc.h b/bin/varnishtest/vtc.h
index 5e5cce7..e96940d 100644
--- a/bin/varnishtest/vtc.h
+++ b/bin/varnishtest/vtc.h
@@ -33,7 +33,10 @@
 #ifdef HAVE_PTHREAD_NP_H
 #include <pthread_np.h>
 #endif
+
 #include "vqueue.h"
+#include "vsb.h"
+#include "miniobj.h"
 
 struct vsb;
 struct vtclog;
diff --git a/bin/varnishtest/vtc_client.c b/bin/varnishtest/vtc_client.c
index 3bd430b..d06b478 100644
--- a/bin/varnishtest/vtc_client.c
+++ b/bin/varnishtest/vtc_client.c
@@ -38,9 +38,6 @@
 
 #include "vtc.h"
 
-#include "vsb.h"
-#include "vqueue.h"
-#include "miniobj.h"
 #include "vss.h"
 #include "libvarnish.h"
 
diff --git a/bin/varnishtest/vtc_http.c b/bin/varnishtest/vtc_http.c
index fddcd26..a0ee01c 100644
--- a/bin/varnishtest/vtc_http.c
+++ b/bin/varnishtest/vtc_http.c
@@ -41,8 +41,6 @@
 
 #include "libvarnish.h"
 #include "vct.h"
-#include "miniobj.h"
-#include "vsb.h"
 
 #include "vtc.h"
 
diff --git a/bin/varnishtest/vtc_log.c b/bin/varnishtest/vtc_log.c
index 18f295a..6b36aa5 100644
--- a/bin/varnishtest/vtc_log.c
+++ b/bin/varnishtest/vtc_log.c
@@ -36,8 +36,6 @@
 #include <stdarg.h>
 
 #include "libvarnish.h"
-#include "vsb.h"
-#include "miniobj.h"
 #include "vas.h"
 
 #include "vtc.h"
diff --git a/bin/varnishtest/vtc_main.c b/bin/varnishtest/vtc_main.c
index 01a5fe2..d80aa6a 100644
--- a/bin/varnishtest/vtc_main.c
+++ b/bin/varnishtest/vtc_main.c
@@ -42,9 +42,6 @@
 
 #include "libvarnish.h"
 #include "vev.h"
-#include "vsb.h"
-#include "vqueue.h"
-#include "miniobj.h"
 
 #include "vtc.h"
 
diff --git a/bin/varnishtest/vtc_sema.c b/bin/varnishtest/vtc_sema.c
index 071b4c0..4df1979 100644
--- a/bin/varnishtest/vtc_sema.c
+++ b/bin/varnishtest/vtc_sema.c
@@ -35,7 +35,6 @@
 
 #include "vtc.h"
 
-#include "vqueue.h"
 #include "miniobj.h"
 #include "libvarnish.h"
 
diff --git a/bin/varnishtest/vtc_server.c b/bin/varnishtest/vtc_server.c
index d1787b1..9390e5b 100644
--- a/bin/varnishtest/vtc_server.c
+++ b/bin/varnishtest/vtc_server.c
@@ -39,9 +39,6 @@
 
 #include "vtc.h"
 
-#include "vqueue.h"
-#include "miniobj.h"
-#include "vsb.h"
 #include "vss.h"
 #include "libvarnish.h"
 
diff --git a/bin/varnishtest/vtc_varnish.c b/bin/varnishtest/vtc_varnish.c
index 9690a23..ce6a45a 100644
--- a/bin/varnishtest/vtc_varnish.c
+++ b/bin/varnishtest/vtc_varnish.c
@@ -42,13 +42,10 @@
 #include <sys/wait.h>
 #include <sys/socket.h>
 
-#include "vqueue.h"
-#include "miniobj.h"
 #include "libvarnish.h"
 #include "varnishapi.h"
 #include "vcli.h"
 #include "vss.h"
-#include "vsb.h"
 
 #include "vtc.h"
 
diff --git a/lib/libvcl/vcc_acl.c b/lib/libvcl/vcc_acl.c
index 30c9eea..6eb451c 100644
--- a/lib/libvcl/vcc_acl.c
+++ b/lib/libvcl/vcc_acl.c
@@ -38,7 +38,6 @@
 #include <netdb.h>
 #include <netinet/in.h>
 
-#include "vsb.h"
 #include "vrt.h"
 
 #include "vcc_priv.h"
diff --git a/lib/libvcl/vcc_action.c b/lib/libvcl/vcc_action.c
index 353a40b..8c538ef 100644
--- a/lib/libvcl/vcc_action.c
+++ b/lib/libvcl/vcc_action.c
@@ -35,8 +35,6 @@
 #include <stdio.h>
 #include <string.h>
 
-#include "vsb.h"
-
 #include "vcc_priv.h"
 #include "vcc_compile.h"
 #include "libvarnish.h"
diff --git a/lib/libvcl/vcc_backend.c b/lib/libvcl/vcc_backend.c
index f79447f..dfe58d2 100644
--- a/lib/libvcl/vcc_backend.c
+++ b/lib/libvcl/vcc_backend.c
@@ -60,7 +60,6 @@
 #include <stdlib.h>
 #include <string.h>
 
-#include "vsb.h"
 #include "vss.h"
 
 #include "vcc_priv.h"
diff --git a/lib/libvcl/vcc_backend_util.c b/lib/libvcl/vcc_backend_util.c
index 968ec4e..8d2cc1a 100644
--- a/lib/libvcl/vcc_backend_util.c
+++ b/lib/libvcl/vcc_backend_util.c
@@ -34,8 +34,6 @@
 #include <stdarg.h>
 #include <string.h>
 
-#include "vsb.h"
-
 #include "vcc_priv.h"
 #include "vcc_compile.h"
 #include "libvarnish.h"
diff --git a/lib/libvcl/vcc_compile.c b/lib/libvcl/vcc_compile.c
index f4a85a0..36efa4b 100644
--- a/lib/libvcl/vcc_compile.c
+++ b/lib/libvcl/vcc_compile.c
@@ -62,10 +62,6 @@
 #include <string.h>
 #include <unistd.h>
 
-#include "vqueue.h"
-
-#include "vsb.h"
-
 #include "vcc_priv.h"
 #include "vcc_compile.h"
 
diff --git a/lib/libvcl/vcc_compile.h b/lib/libvcl/vcc_compile.h
index ec7caed..a3aa800 100644
--- a/lib/libvcl/vcc_compile.h
+++ b/lib/libvcl/vcc_compile.h
@@ -31,6 +31,7 @@
 #include "vqueue.h"
 
 #include "miniobj.h"
+#include "vsb.h"
 #include "vcl.h"
 
 #define INDENT		2
diff --git a/lib/libvcl/vcc_dir_dns.c b/lib/libvcl/vcc_dir_dns.c
index 8168a26..a8cb24b 100644
--- a/lib/libvcl/vcc_dir_dns.c
+++ b/lib/libvcl/vcc_dir_dns.c
@@ -36,8 +36,6 @@
 #include <string.h>
 #include <limits.h>
 
-#include "vsb.h"
-
 #include "vcc_priv.h"
 #include "vcc_compile.h"
 #include "libvarnish.h"
diff --git a/lib/libvcl/vcc_dir_random.c b/lib/libvcl/vcc_dir_random.c
index 125e7d1..aa362f9 100644
--- a/lib/libvcl/vcc_dir_random.c
+++ b/lib/libvcl/vcc_dir_random.c
@@ -36,8 +36,6 @@
 #include <stdarg.h>
 #include <string.h>
 
-#include "vsb.h"
-
 #include "vcc_priv.h"
 #include "vcc_compile.h"
 #include "libvarnish.h"
diff --git a/lib/libvcl/vcc_dir_round_robin.c b/lib/libvcl/vcc_dir_round_robin.c
index 6229c86..67fb118 100644
--- a/lib/libvcl/vcc_dir_round_robin.c
+++ b/lib/libvcl/vcc_dir_round_robin.c
@@ -35,8 +35,6 @@
 #include <stdarg.h>
 #include <string.h>
 
-#include "vsb.h"
-
 #include "vcc_priv.h"
 #include "vcc_compile.h"
 #include "libvarnish.h"
diff --git a/lib/libvcl/vcc_expr.c b/lib/libvcl/vcc_expr.c
index 1f1c274..b7b6ba6 100644
--- a/lib/libvcl/vcc_expr.c
+++ b/lib/libvcl/vcc_expr.c
@@ -37,8 +37,6 @@
 #include <string.h>
 #include <math.h>
 
-#include "vsb.h"
-
 #include "vcc_priv.h"
 #include "vcc_compile.h"
 #include "libvarnish.h"
diff --git a/lib/libvcl/vcc_parse.c b/lib/libvcl/vcc_parse.c
index c616c3c..23c9bf7 100644
--- a/lib/libvcl/vcc_parse.c
+++ b/lib/libvcl/vcc_parse.c
@@ -32,8 +32,6 @@
 #include <stdio.h>
 #include <string.h>
 
-#include "vsb.h"
-
 #include "vcc_priv.h"
 #include "vcc_compile.h"
 #include "libvarnish.h"
diff --git a/lib/libvcl/vcc_storage.c b/lib/libvcl/vcc_storage.c
index 1acb2ed..77c657c 100644
--- a/lib/libvcl/vcc_storage.c
+++ b/lib/libvcl/vcc_storage.c
@@ -59,8 +59,6 @@
 #include <stdlib.h>
 #include <string.h>
 
-#include "vsb.h"
-
 #include "vcc_priv.h"
 #include "vcc_compile.h"
 #include "libvarnish.h"
diff --git a/lib/libvcl/vcc_string.c b/lib/libvcl/vcc_string.c
index f643a4b..c6353d7 100644
--- a/lib/libvcl/vcc_string.c
+++ b/lib/libvcl/vcc_string.c
@@ -32,8 +32,6 @@
 #include <stdio.h>
 #include <string.h>
 
-#include "vsb.h"
-
 #include "vcc_priv.h"
 #include "vcc_compile.h"
 #include "libvarnish.h"
diff --git a/lib/libvcl/vcc_symb.c b/lib/libvcl/vcc_symb.c
index d473a50..4975c12 100644
--- a/lib/libvcl/vcc_symb.c
+++ b/lib/libvcl/vcc_symb.c
@@ -32,8 +32,6 @@
 #include <stdlib.h>
 #include <string.h>
 
-#include "vsb.h"
-
 #include "vcc_priv.h"
 #include "vcc_compile.h"
 #include "libvarnish.h"
diff --git a/lib/libvcl/vcc_token.c b/lib/libvcl/vcc_token.c
index c5766c0..5a79732 100644
--- a/lib/libvcl/vcc_token.c
+++ b/lib/libvcl/vcc_token.c
@@ -34,10 +34,6 @@
 #include <stdlib.h>
 #include <string.h>
 
-#include "vqueue.h"
-
-#include "vsb.h"
-
 #include "libvarnish.h"
 #include "vcc_priv.h"
 #include "vcc_compile.h"
diff --git a/lib/libvcl/vcc_var.c b/lib/libvcl/vcc_var.c
index 3871dbc..b6e1ea7 100644
--- a/lib/libvcl/vcc_var.c
+++ b/lib/libvcl/vcc_var.c
@@ -32,8 +32,6 @@
 #include <stdio.h>
 #include <string.h>
 
-#include "vsb.h"
-
 #include "vcc_priv.h"
 #include "vcc_compile.h"
 #include "libvarnish.h"
diff --git a/lib/libvcl/vcc_vmod.c b/lib/libvcl/vcc_vmod.c
index f74f874..37a98fd 100644
--- a/lib/libvcl/vcc_vmod.c
+++ b/lib/libvcl/vcc_vmod.c
@@ -32,8 +32,6 @@
 #include <dlfcn.h>
 #include <string.h>
 
-#include "vsb.h"
-
 #include "vcc_priv.h"
 #include "vcc_compile.h"
 #include "libvarnish.h"
diff --git a/lib/libvcl/vcc_xref.c b/lib/libvcl/vcc_xref.c
index 2188ef5..560fe78 100644
--- a/lib/libvcl/vcc_xref.c
+++ b/lib/libvcl/vcc_xref.c
@@ -41,8 +41,6 @@
 
 #include <stdio.h>
 
-#include "vsb.h"
-
 #include "libvarnish.h"
 #include "vcc_priv.h"
 #include "vcc_compile.h"



From geoff at varnish-cache.org  Mon Jan  9 20:52:41 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:41 +0100
Subject: [experimental-ims] aed74d6 Enable regression test for bug 971,
	since 1060 was a dup of it
Message-ID: <E1RkMCv-0008OH-Gu@project.varnish-software.com>

commit aed74d6c93f932abcd111c2496c543d16f2745d5
Author: Rogier 'DocWilco' Mulhuijzen <github at bsdchicks.com>
Date:   Mon Nov 21 21:35:22 2011 +0100

    Enable regression test for bug 971, since 1060 was a dup of it
    
    1060 was fixed with bdbb1d59513cba8b268ed1dbe2d948619ef4ae07
    
    Fixes	#971

diff --git a/bin/varnishtest/tests.disabled/r00971.vtc b/bin/varnishtest/tests.disabled/r00971.vtc
deleted file mode 100644
index f8b57f0..0000000
--- a/bin/varnishtest/tests.disabled/r00971.vtc
+++ /dev/null
@@ -1,25 +0,0 @@
-
-varnishtest "Test DNS director order"
-
-varnish v1 -vcl+backend {
-	
-	backend test {
-		.host = "192.168.0.1";
-	}
-
-	director foo dns {
-		{ .backend = { .host = "127.0.0.1";} }
-		.list = {
-			"192.168.0.0"/24;
-		}
-	}
-
-	sub vcl_recv {
-		set req.backend = foo;
-		if (req.http.x-aa) {
-			set req.backend = test;
-		}
-	}
-
-} -start
-
diff --git a/bin/varnishtest/tests/r00971.vtc b/bin/varnishtest/tests/r00971.vtc
new file mode 100644
index 0000000..f8b57f0
--- /dev/null
+++ b/bin/varnishtest/tests/r00971.vtc
@@ -0,0 +1,25 @@
+
+varnishtest "Test DNS director order"
+
+varnish v1 -vcl+backend {
+	
+	backend test {
+		.host = "192.168.0.1";
+	}
+
+	director foo dns {
+		{ .backend = { .host = "127.0.0.1";} }
+		.list = {
+			"192.168.0.0"/24;
+		}
+	}
+
+	sub vcl_recv {
+		set req.backend = foo;
+		if (req.http.x-aa) {
+			set req.backend = test;
+		}
+	}
+
+} -start
+



From geoff at varnish-cache.org  Mon Jan  9 20:52:00 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:00 +0100
Subject: [experimental-ims] 08b8b27 More detail on write errors.
Message-ID: <E1RkMCG-0007A9-R6@project.varnish-software.com>

commit 08b8b27eedd5d52ccd9ca608cda762332d944ebe
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Sep 21 10:40:05 2011 +0000

    More detail on write errors.

diff --git a/bin/varnishtest/vtc_http.c b/bin/varnishtest/vtc_http.c
index e6aaf33..024e376 100644
--- a/bin/varnishtest/vtc_http.c
+++ b/bin/varnishtest/vtc_http.c
@@ -143,7 +143,8 @@ http_write(const struct http *hp, int lvl, const char *pfx)
 	vtc_dump(hp->vl, lvl, pfx, VSB_data(hp->vsb), VSB_len(hp->vsb));
 	l = write(hp->fd, VSB_data(hp->vsb), VSB_len(hp->vsb));
 	if (l != VSB_len(hp->vsb))
-		vtc_log(hp->vl, 0, "Write failed: %s", strerror(errno));
+		vtc_log(hp->vl, 0, "Write failed: (%d vs %d) %s",
+		    l, VSB_len(hp->vsb), strerror(errno));
 }
 
 /**********************************************************************



From geoff at varnish-cache.org  Mon Jan  9 20:52:40 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:40 +0100
Subject: [experimental-ims] 5cbd8ca Make Varnish API aware of the new
	structure and make everything compile again. Much still outstanding.
Message-ID: <E1RkMCu-0008Ld-FV@project.varnish-software.com>

commit 5cbd8caab0650ab4178a5b387667fa8fc0672744
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sun Nov 20 16:11:50 2011 +0000

    Make Varnish API aware of the new structure and make everything compile
    again.  Much still outstanding.

diff --git a/bin/varnishstat/varnishstat.c b/bin/varnishstat/varnishstat.c
index d0ef409..2fe7ffb 100644
--- a/bin/varnishstat/varnishstat.c
+++ b/bin/varnishstat/varnishstat.c
@@ -289,6 +289,7 @@ main(int argc, char * const *argv)
 		exit(1);
 
 	VSC_C_main = VSC_Main(vd);
+	AN(VSC_C_main);
 
 	if (!(xml || json || once)) {
 		do_curses(vd, VSC_C_main, delay);
diff --git a/include/vapi/vsc.h b/include/vapi/vsc.h
index bc52d65..ac711ea 100644
--- a/include/vapi/vsc.h
+++ b/include/vapi/vsc.h
@@ -42,11 +42,14 @@ struct VSM_data;
 void VSC_Setup(struct VSM_data *vd);
 	/*
 	 * Setup vd for use with VSC functions.
+	 * Must be called once before any other VSC function is called
 	 */
 
 #define VSC_ARGS	"f:n:"
 #define VSC_n_USAGE	VSM_n_USAGE
-#define VSC_USAGE	VSC_N_USAGE
+#define VSC_f_USAGE	"[-f field_name,...]"
+#define VSC_USAGE	VSC_n_USAGE \
+			VSC_f_USAGE
 
 int VSC_Arg(struct VSM_data *vd, int arg, const char *opt);
 	/*
@@ -66,6 +69,7 @@ int VSC_Open(struct VSM_data *vd, int diag);
 struct VSC_C_main *VSC_Main(struct VSM_data *vd);
 	/*
 	 * return Main stats structure
+	 * returns NULL until child has been started.
 	 */
 
 struct VSC_point {
diff --git a/include/vapi/vsl.h b/include/vapi/vsl.h
index 9c9a4c9..fb02051 100644
--- a/include/vapi/vsl.h
+++ b/include/vapi/vsl.h
@@ -42,11 +42,12 @@ struct VSM_data;
 void VSL_Setup(struct VSM_data *vd);
 	/*
 	 * Setup vd for use with VSL functions.
+	 * Must be called once before any other VSL function is called.
 	 */
 
 int VSL_Open(struct VSM_data *vd, int diag);
 	/*
-	 * Attempt to open and map the shared memory file.
+	 * Attempt to open the VSM (unless -r given)
 	 * If diag is non-zero, diagnostics are emitted.
 	 * Returns:
 	 *	0 on success
diff --git a/include/vapi/vsm.h b/include/vapi/vsm.h
index 275f6d9..50a7dde 100644
--- a/include/vapi/vsm.h
+++ b/include/vapi/vsm.h
@@ -26,6 +26,8 @@
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
  *
+ * This is the public API for the VSM/VSC/VSL access.
+ *
  */
 
 #ifndef VAPI_VSM_H_INCLUDED
@@ -35,10 +37,14 @@ struct VSM_head;
 struct VSM_chunk;
 struct VSM_data;
 
+/*
+ * This structure is used to reference a VSM chunk
+ */
+
 struct VSM_fantom {
 	struct VSM_chunk	*chunk;
-	void			*b;
-	void			*e;
+	void			*b;		/* first byte of payload */
+	void			*e;		/* first byte past payload */
 	uintptr_t		priv;
 };
 
@@ -120,12 +126,31 @@ struct VSM_head *VSM_Head(const struct VSM_data *vd);
 	 * Return the head of the VSM.
 	 */
 
-void *VSM_Find_Chunk(struct VSM_data *vd, const char *class,
-    const char *type, const char *ident, unsigned *lenp);
+void VSM__iter0(struct VSM_data *vd, struct VSM_fantom *vf);
+int VSM__itern(struct VSM_data *vd, struct VSM_fantom *vf);
+
+#define VSM_FOREACH_SAFE(vf, vd) \
+    for(VSM__iter0((vd), (vf)); VSM__itern((vd), (vf));)
 	/*
-	 * Find a given chunk in the shared memory.
-	 * Returns pointer or NULL.
-	 * Lenp, if non-NULL, is set to length of chunk.
+	 * Iterate over all chunks in shared memory
+	 * vf = "struct VSM_fantom *"
+	 * vd = "struct VSM_data *"
+	 */
+
+int VSM_StillValid(struct VSM_data *vd, struct VSM_fantom *vf);
+	/*
+	 * Return:
+	 *	0: fantom is invalid now.
+	 *	1: fantom is still the same.
+	 *	2: a fantom with same dimensions exist, check class/type/ident
+	 */
+
+int VSM_Get(struct VSM_data *vd, struct VSM_fantom *vf, const char *class,
+    const char *type, const char *ident);
+	/*
+	 * Find a chunk, produce fantom for it.
+	 * Returns zero on failure.
+	 * class is mandatory, type and ident optional.
 	 */
 
 void VSM_Close(struct VSM_data *vd);
@@ -134,9 +159,28 @@ void VSM_Close(struct VSM_data *vd);
 	 * Deallocate all storage (including VSC and VSL allocations)
 	 */
 
+/**********************************************************************
+ * These are old API functions which are less safe because there is
+ * fantom protecting the chunks worked on.
+ * They will g
+ */
+
+/* OBSOLETE: Will disappear from Varnish 4.x */
+void *VSM_Find_Chunk(struct VSM_data *vd, const char *class,
+    const char *type, const char *ident, unsigned *lenp);
+	/*
+	 * Find a given chunk in the shared memory.
+	 * Returns pointer or NULL.
+	 * Lenp, if non-NULL, is set to length of chunk.
+	 */
+
+/* OBSOLETE: Will disappear from Varnish 4.x */
 struct VSM_chunk *VSM_iter0(struct VSM_data *vd);
-void VSM_itern(const struct VSM_data *vd, struct VSM_chunk **pp);
 
+/* OBSOLETE: Will disappear from Varnish 4.x */
+void VSM_itern(struct VSM_data *vd, struct VSM_chunk **pp);
+
+/* OBSOLETE: Will disappear from Varnish 4.x */
 #define VSM_FOREACH(var, vd) \
     for((var) = VSM_iter0((vd)); (var) != NULL; VSM_itern((vd), &(var)))
 
diff --git a/lib/libvarnishapi/vsc.c b/lib/libvarnishapi/vsc.c
index eea7f5c..1e2206b 100644
--- a/lib/libvarnishapi/vsc.c
+++ b/lib/libvarnishapi/vsc.c
@@ -79,7 +79,6 @@ VSC_Setup(struct VSM_data *vd)
 
 	CHECK_OBJ_NOTNULL(vd, VSM_MAGIC);
 	AZ(vd->vsc);
-	AZ(vd->vsl);
 	ALLOC_OBJ(vd->vsc, VSC_MAGIC);
 	AN(vd->vsc);
 	VTAILQ_INIT(&vd->vsc->sf_list);
@@ -225,14 +224,14 @@ VSC_Open(struct VSM_data *vd, int diag)
 struct VSC_C_main *
 VSC_Main(struct VSM_data *vd)
 {
-	struct VSM_chunk *sha;
+	struct VSM_fantom vf;
 
 	CHECK_OBJ_NOTNULL(vd, VSM_MAGIC);
 	CHECK_OBJ_NOTNULL(vd->vsc, VSC_MAGIC);
 
-	sha = VSM_find_alloc(vd, VSC_CLASS, "", "");
-	assert(sha != NULL);
-	return (VSM_PTR(sha));
+	if (!VSM_Get(vd, &vf, VSC_CLASS, "", ""))
+		return (NULL);
+	return ((void*)vf.b);
 }
 
 /*--------------------------------------------------------------------
@@ -285,7 +284,7 @@ iter_call(const struct vsc *vsc, VSC_iter_f *func, void *priv,
 
 #define VSC_DO(U,l,t)							\
 	static int							\
-	iter_##l(const struct vsc *vsc, struct VSM_chunk *sha,		\
+	iter_##l(const struct vsc *vsc, struct VSM_fantom *vf,		\
 	    VSC_iter_f *func, void *priv)				\
 	{								\
 		struct VSC_C_##l *st;					\
@@ -293,10 +292,9 @@ iter_call(const struct vsc *vsc, VSC_iter_f *func, void *priv,
 		int i;							\
 									\
 		CHECK_OBJ_NOTNULL(vsc, VSC_MAGIC);			\
-		CHECK_OBJ_NOTNULL(sha, VSM_CHUNK_MAGIC);		\
-		st = VSM_PTR(sha);					\
+		st = vf->b; 						\
 		sp.class = t;						\
-		sp.ident = sha->ident;
+		sp.ident = vf->chunk->ident;
 
 #define VSC_F(nn,tt,ll,ff,dd,ee)					\
 		sp.name = #nn;						\
@@ -321,24 +319,22 @@ int
 VSC_Iter(struct VSM_data *vd, VSC_iter_f *func, void *priv)
 {
 	struct vsc *vsc;
-	struct VSM_chunk *sha;
+	struct VSM_fantom vf;
 	int i;
 
 	CHECK_OBJ_NOTNULL(vd, VSM_MAGIC);
 	vsc = vd->vsc;
 	CHECK_OBJ_NOTNULL(vsc, VSC_MAGIC);
-
 	i = 0;
-	VSM_FOREACH(sha, vd) {
-		CHECK_OBJ_NOTNULL(sha, VSM_CHUNK_MAGIC);
-		if (strcmp(sha->class, VSC_CLASS))
+	VSM_FOREACH_SAFE(&vf, vd) {
+		if (strcmp(vf.chunk->class, VSC_CLASS))
 			continue;
 		/*lint -save -e525 -e539 */
 #define VSC_F(n,t,l,f,d,e)
 #define VSC_DONE(a,b,c)
 #define VSC_DO(U,l,t)						\
-		if (!strcmp(sha->type, t)) {			\
-			i = iter_##l(vsc, sha, func, priv);	\
+		if (!strcmp(vf.chunk->type, t)) {		\
+			i = iter_##l(vsc, &vf, func, priv);	\
 			if (!i)					\
 				continue;			\
 		}
diff --git a/lib/libvarnishapi/vsl.c b/lib/libvarnishapi/vsl.c
index b17cd36..d896301 100644
--- a/lib/libvarnishapi/vsl.c
+++ b/lib/libvarnishapi/vsl.c
@@ -67,25 +67,14 @@ VSL_Setup(struct VSM_data *vd)
 	struct vsl *vsl;
 
 	CHECK_OBJ_NOTNULL(vd, VSM_MAGIC);
-	AZ(vd->vsc);
 	AZ(vd->vsl);
-	ALLOC_OBJ(vsl, VSL_MAGIC);
-	AN(vsl);
-
-	vd->vsl = vsl;
-
+	ALLOC_OBJ(vd->vsl, VSL_MAGIC);
+	AN(vd->vsl);
+	vsl = vd->vsl;
 	vsl->regflags = 0;
-
-	/* XXX: Allocate only if log access */
 	vsl->vbm_supress = vbit_init(256);
 	vsl->vbm_select = vbit_init(256);
-
 	vsl->r_fd = -1;
-	/* XXX: Allocate only if -r option given ? */
-	vsl->rbuflen = 256;      /* XXX ?? */
-	vsl->rbuf = malloc(vsl->rbuflen * 4L);
-	assert(vsl->rbuf != NULL);
-
 	vsl->num_matchers = 0;
 	VTAILQ_INIT(&vsl->matchers);
 }
@@ -343,28 +332,6 @@ VSL_H_Print(void *priv, enum VSL_tag_e tag, unsigned fd, unsigned len,
 
 /*--------------------------------------------------------------------*/
 
-void
-VSL_Open_CallBack(struct VSM_data *vd)
-{
-	struct vsl *vsl;
-	struct VSM_chunk *sha;
-
-	CHECK_OBJ_NOTNULL(vd, VSM_MAGIC);
-	vsl = vd->vsl;
-	CHECK_OBJ_NOTNULL(vsl, VSL_MAGIC);
-	sha = VSM_find_alloc(vd, VSL_CLASS, "", "");
-	assert(sha != NULL);
-
-	vsl->log_start = VSM_PTR(sha);
-	vsl->log_end = VSM_NEXT(sha);
-	vsl->log_ptr = vsl->log_start + 1;
-
-	vsl->last_seq = vsl->log_start[0];
-	VRMB();
-}
-
-/*--------------------------------------------------------------------*/
-
 int
 VSL_Open(struct VSM_data *vd, int diag)
 {
@@ -379,11 +346,21 @@ VSL_Open(struct VSM_data *vd, int diag)
 		i = VSM_Open(vd, diag);
 		if (i)
 			return (i);
-	}
-
-	if (!vsl->d_opt && vsl->r_fd == -1) {
-		while (*vsl->log_ptr != VSL_ENDMARKER)
-			vsl->log_ptr = VSL_NEXT(vsl->log_ptr);
+		if (!VSM_Get(vd, &vsl->vf, VSL_CLASS, NULL, NULL)) {
+			VSM_Close(vd);
+			if (diag)
+				vd->diag(vd->priv,
+				    "No VSL chunk found "
+				    " (child not started ?)\n");
+			return (1);
+		}
+		vsl->log_start = vsl->vf.b;
+		vsl->log_end = vsl->vf.e;
+		vsl->log_ptr = vsl->log_start + 1;
+		if (!vsl->d_opt) {
+			while (*vsl->log_ptr != VSL_ENDMARKER)
+				vsl->log_ptr = VSL_NEXT(vsl->log_ptr);
+		}
 	}
 	return (0);
 }
diff --git a/lib/libvarnishapi/vsl_api.h b/lib/libvarnishapi/vsl_api.h
index aa3d0f4..013d1ae 100644
--- a/lib/libvarnishapi/vsl_api.h
+++ b/lib/libvarnishapi/vsl_api.h
@@ -30,6 +30,10 @@
 
 #include "vqueue.h"
 
+#define	SLEEP_USEC			(50*1000)
+#define	TIMEOUT_USEC			(5*1000*1000)
+
+
 struct vsl_re_match {
 	unsigned			magic;
 #define VSL_RE_MATCH_MAGIC		0x4013151e
@@ -42,6 +46,8 @@ struct vsl {
 	unsigned		magic;
 #define VSL_MAGIC		0x7a31db38
 
+	struct VSM_fantom	vf;
+
 	/* Stuff relating the log records below here */
 
 	volatile uint32_t	*log_start;
diff --git a/lib/libvarnishapi/vsm.c b/lib/libvarnishapi/vsm.c
index 97e3847..1669278 100644
--- a/lib/libvarnishapi/vsm.c
+++ b/lib/libvarnishapi/vsm.c
@@ -141,7 +141,7 @@ VSM_Delete(struct VSM_data *vd)
  *
  * Return:
  *	0 = sucess
- * 	1 = failure
+ * 	<0 = failure
  *
  */
 
@@ -150,9 +150,10 @@ vsm_open(struct VSM_data *vd, int diag)
 {
 	int i;
 	struct VSM_head slh;
+	void *v;
 
 	CHECK_OBJ_NOTNULL(vd, VSM_MAGIC);
-	AZ(vd->VSM_head);
+	AZ(vd->head);
 	AN(vd->fname);
 
 	vd->vsm_fd = open(vd->fname, O_RDONLY);
@@ -160,7 +161,7 @@ vsm_open(struct VSM_data *vd, int diag)
 		if (diag)
 			vd->diag(vd->priv, "Cannot open %s: %s\n",
 			    vd->fname, strerror(errno));
-		return (1);
+		return (-1);
 	}
 
 	AZ(fstat(vd->vsm_fd, &vd->fstat));
@@ -170,7 +171,7 @@ vsm_open(struct VSM_data *vd, int diag)
 			    vd->fname);
 		AZ(close(vd->vsm_fd));
 		vd->vsm_fd = -1;
-		return (1);
+		return (-1);
 	}
 
 	i = read(vd->vsm_fd, &slh, sizeof slh);
@@ -180,33 +181,33 @@ vsm_open(struct VSM_data *vd, int diag)
 			    vd->fname, strerror(errno));
 		AZ(close(vd->vsm_fd));
 		vd->vsm_fd = -1;
-		return (1);
+		return (-1);
 	}
-	if (slh.magic != VSM_HEAD_MAGIC || slh.alloc_seq == 0) {
+
+	if (memcmp(slh.marker, VSM_HEAD_MARKER, sizeof slh.marker) ||
+	    slh.alloc_seq == 0) {
 		if (diag)
-			vd->diag(vd->priv, "Not a ready VSM file %s\n",
+			vd->diag(vd->priv, "Not a VSM file %s\n",
 			    vd->fname);
 		AZ(close(vd->vsm_fd));
 		vd->vsm_fd = -1;
-		return (1);
+		return (-1);
 	}
 
-	vd->VSM_head = mmap(NULL, slh.shm_size,
+	v = mmap(NULL, slh.shm_size,
 	    PROT_READ, MAP_SHARED|MAP_HASSEMAPHORE, vd->vsm_fd, 0);
-	if (vd->VSM_head == MAP_FAILED) {
+	if (v == MAP_FAILED) {
 		if (diag)
 			vd->diag(vd->priv, "Cannot mmap %s: %s\n",
 			    vd->fname, strerror(errno));
 		AZ(close(vd->vsm_fd));
 		vd->vsm_fd = -1;
-		vd->VSM_head = NULL;
-		return (1);
+		return (-1);
 	}
-	vd->vsm_end = (uint8_t *)vd->VSM_head + slh.shm_size;
-	vd->my_alloc_seq = vd->VSM_head->alloc_seq;
+	vd->head = v;
+	vd->b = v;
+	vd->e = vd->b + slh.shm_size;
 
-	if (vd->vsl != NULL)
-		VSL_Open_CallBack(vd);
 	return (0);
 }
 
@@ -218,7 +219,7 @@ VSM_Open(struct VSM_data *vd, int diag)
 {
 
 	CHECK_OBJ_NOTNULL(vd, VSM_MAGIC);
-	AZ(vd->VSM_head);
+	AZ(vd->head);
 	if (!vd->n_opt)
 		(void)VSM_n_Arg(vd, "");
 	return (vsm_open(vd, diag));
@@ -231,11 +232,14 @@ VSM_Close(struct VSM_data *vd)
 {
 
 	CHECK_OBJ_NOTNULL(vd, VSM_MAGIC);
-	if (vd->VSM_head == NULL)
+	if (vd->head == NULL)
 		return;
-	AZ(munmap((void*)vd->VSM_head, vd->VSM_head->shm_size));
-	vd->VSM_head = NULL;
+
 	assert(vd->vsm_fd >= 0);
+	AZ(munmap((void*)vd->b, vd->e - vd->b));
+	vd->b = NULL;
+	vd->e = NULL;
+	vd->head = NULL;
 	AZ(close(vd->vsm_fd));
 	vd->vsm_fd = -1;
 }
@@ -246,25 +250,28 @@ int
 VSM_ReOpen(struct VSM_data *vd, int diag)
 {
 	struct stat st;
-	int i;
 
 	CHECK_OBJ_NOTNULL(vd, VSM_MAGIC);
-	AN(vd->VSM_head);
-
-	if (stat(vd->fname, &st))
-		return (0);
+	AN(vd->head);
 
-	if (st.st_dev == vd->fstat.st_dev && st.st_ino == vd->fstat.st_ino)
+	if (vd->head->alloc_seq &&
+	    !stat(vd->fname, &st) &&
+	    st.st_dev == vd->fstat.st_dev &&
+	    st.st_ino == vd->fstat.st_ino)
 		return (0);
 
 	VSM_Close(vd);
-	for (i = 0; i < 5; i++) {		/* XXX param */
-		if (!vsm_open(vd, 0))
-			return (1);
-	}
-	if (vsm_open(vd, diag))
-		return (-1);
-	return (1);
+	return (vsm_open(vd, diag));
+}
+
+/*--------------------------------------------------------------------*/
+
+unsigned
+VSM_Seq(const struct VSM_data *vd)
+{
+
+	CHECK_OBJ_NOTNULL(vd, VSM_MAGIC);
+	return (vd->head->alloc_seq);
 }
 
 /*--------------------------------------------------------------------*/
@@ -274,31 +281,87 @@ VSM_Head(const struct VSM_data *vd)
 {
 
 	CHECK_OBJ_NOTNULL(vd, VSM_MAGIC);
-	AN(vd->VSM_head);
-	return(vd->VSM_head);
+	AN(vd->head);
+	return(vd->head);
+}
+
+/*--------------------------------------------------------------------*/
+
+void
+VSM__iter0(struct VSM_data *vd, struct VSM_fantom *vf)
+{
+
+	CHECK_OBJ_NOTNULL(vd, VSM_MAGIC);
+	memset(vf, 0, sizeof *vf);
 }
 
+int
+VSM__itern(struct VSM_data *vd, struct VSM_fantom *vf)
+{
+
+	CHECK_OBJ_NOTNULL(vd, VSM_MAGIC);
+	if (vf->priv != 0) {
+		if (vf->priv != vd->head->alloc_seq)
+			return (0);
+		if (vf->chunk->len == 0)
+			return (0);
+		if (vf->chunk->next == 0)
+			return (0);
+		vf->chunk = (void*)(vd->b + vf->chunk->next);
+	} else if (vd->head->first == 0) {
+		return (0);
+	} else {
+		vf->chunk = (void*)(vd->b + vd->head->first);
+	}
+	if (memcmp(vf->chunk->marker, VSM_CHUNK_MARKER,
+	    sizeof vf->chunk->marker))
+		return (0);
+	vf->priv = vd->head->alloc_seq;
+	vf->b = (void*)(vf->chunk + 1);
+	vf->e = (char*)vf->b + vf->chunk->len;
+	if (vf->b == vf->e)
+		return (0);
+	return (1);
+}
 
 /*--------------------------------------------------------------------*/
 
-struct VSM_chunk *
-VSM_find_alloc(struct VSM_data *vd, const char *class, const char *type, const char *ident)
+int
+VSM_StillValid(struct VSM_data *vd, struct VSM_fantom *vf)
+{
+	struct VSM_fantom f2;
+
+	CHECK_OBJ_NOTNULL(vd, VSM_MAGIC);
+	if (vf->priv == vd->head->alloc_seq)
+		return (1);
+	VSM_FOREACH_SAFE(&f2, vd) {
+		if (f2.chunk == vf->chunk &&
+		   f2.b == vf->b &&
+		   f2.e == vf->e) {
+			vf->priv = vd->head->alloc_seq;
+			return (2);
+		}
+	}
+	return (0);
+}
+
+int
+VSM_Get(struct VSM_data *vd, struct VSM_fantom *vf, const char *class,
+    const char *type, const char *ident)
 {
-	struct VSM_chunk *sha;
 
 	CHECK_OBJ_NOTNULL(vd, VSM_MAGIC);
-	AN(vd->VSM_head);
-	VSM_FOREACH(sha, vd) {
-		CHECK_OBJ_NOTNULL(sha, VSM_CHUNK_MAGIC);
-		if (strcmp(sha->class, class))
+	VSM_FOREACH_SAFE(vf, vd) {
+		if (strcmp(vf->chunk->class, class))
 			continue;
-		if (type != NULL && strcmp(sha->type, type))
+		if (type != NULL && strcmp(vf->chunk->type, type))
 			continue;
-		if (ident != NULL && strcmp(sha->ident, ident))
+		if (ident != NULL && strcmp(vf->chunk->ident, ident))
 			continue;
-		return (sha);
+		return (1);
 	}
-	return (NULL);
+	memset(vf, 0, sizeof *vf);
+	return (0);
 }
 
 /*--------------------------------------------------------------------*/
@@ -307,15 +370,16 @@ void *
 VSM_Find_Chunk(struct VSM_data *vd, const char *class, const char *type,
     const char *ident, unsigned *lenp)
 {
-	struct VSM_chunk *sha;
+	struct VSM_fantom vf;
 
-	CHECK_OBJ_NOTNULL(vd, VSM_MAGIC);
-	sha = VSM_find_alloc(vd, class, type, ident);
-	if (sha == NULL)
-		return (NULL);
+	if (VSM_Get(vd, &vf, class, type, ident)) {
+		if (lenp != NULL)
+			*lenp = (char*)vf.e - (char*)vf.b;
+		return (vf.chunk);
+	}
 	if (lenp != NULL)
-		*lenp = sha->len - sizeof *sha;
-	return (VSM_PTR(sha));
+		*lenp = 0;
+	return (NULL);
 }
 
 /*--------------------------------------------------------------------*/
@@ -325,38 +389,19 @@ VSM_iter0(struct VSM_data *vd)
 {
 
 	CHECK_OBJ_NOTNULL(vd, VSM_MAGIC);
-	vd->my_alloc_seq = vd->VSM_head->alloc_seq;
-	while (vd->my_alloc_seq == 0) {
-		(void)usleep(50000);
-		vd->my_alloc_seq = vd->VSM_head->alloc_seq;
-	}
-	CHECK_OBJ_NOTNULL(&vd->VSM_head->head, VSM_CHUNK_MAGIC);
-	return (&vd->VSM_head->head);
+	VSM__iter0(vd, &vd->compat_vf);
+	if (VSM__itern(vd, &vd->compat_vf))
+		return(vd->compat_vf.chunk);
+	return (NULL);
 }
 
 void
-VSM_itern(const struct VSM_data *vd, struct VSM_chunk **pp)
+VSM_itern(struct VSM_data *vd, struct VSM_chunk **pp)
 {
 
 	CHECK_OBJ_NOTNULL(vd, VSM_MAGIC);
-	if (vd->my_alloc_seq != vd->VSM_head->alloc_seq) {
-		*pp = NULL;
-		return;
-	}
-	CHECK_OBJ_NOTNULL(*pp, VSM_CHUNK_MAGIC);
-	*pp = VSM_NEXT(*pp);
-	if ((void*)(*pp) >= vd->vsm_end) {
+	if (VSM__itern(vd, &vd->compat_vf))
+		*pp = vd->compat_vf.chunk;
+	else
 		*pp = NULL;
-		return;
-	}
-	CHECK_OBJ_NOTNULL(*pp, VSM_CHUNK_MAGIC);
-}
-
-/*--------------------------------------------------------------------*/
-unsigned
-VSM_Seq(const struct VSM_data *vd)
-{
-
-	CHECK_OBJ_NOTNULL(vd, VSM_MAGIC);
-	return (vd->VSM_head->alloc_seq);
 }
diff --git a/lib/libvarnishapi/vsm_api.h b/lib/libvarnishapi/vsm_api.h
index da5222e..84ea523 100644
--- a/lib/libvarnishapi/vsm_api.h
+++ b/lib/libvarnishapi/vsm_api.h
@@ -28,10 +28,6 @@
  *
  */
 
-/* Parameters */
-#define			SLEEP_USEC	(50*1000)
-#define			TIMEOUT_USEC	(5*1000*1000)
-
 struct vsc;
 
 struct VSM_data {
@@ -47,9 +43,12 @@ struct VSM_data {
 	struct stat		fstat;
 
 	int			vsm_fd;
-	struct VSM_head		*VSM_head;
-	void			*vsm_end;
-	unsigned		my_alloc_seq;
+	struct VSM_head		*head;
+	char			*b;
+	char			*e;
+
+	/* Stuff for backwards compat */
+	struct VSM_fantom	compat_vf;
 
 	/* Stuff relating the stats fields start here */
 
@@ -62,4 +61,3 @@ struct VSM_chunk *VSM_find_alloc(struct VSM_data *vd, const char *class,
 
 void VSC_Delete(struct VSM_data *vd);
 void VSL_Delete(struct VSM_data *vd);
-void VSL_Open_CallBack(struct VSM_data *vd);



From geoff at varnish-cache.org  Mon Jan  9 20:52:49 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:49 +0100
Subject: [experimental-ims] 7cdeb0d Merge branch 'master' of
	ssh://git.varnish-cache.org/git/varnish-cache
Message-ID: <E1RkMD3-0000Cg-1m@project.varnish-software.com>

commit 7cdeb0d5653806875fe920a4fdfb6b3d50f51e14
Merge: 02fd3c7 3d723fd
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Dec 6 11:04:23 2011 +0000

    Merge branch 'master' of ssh://git.varnish-cache.org/git/varnish-cache



From geoff at varnish-cache.org  Mon Jan  9 20:52:27 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:27 +0100
Subject: [experimental-ims] 2fceda5 Make the VFP calls symmetric and pair-wise
	visible, allow ->end() to fail, and act accordingly.
Message-ID: <E1RkMCh-0007xr-Bp@project.varnish-software.com>

commit 2fceda5de01b6adee39436b7631fbadb5d80efac
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Oct 31 09:03:04 2011 +0000

    Make the VFP calls symmetric and pair-wise visible, allow ->end()
    to fail, and act accordingly.

diff --git a/bin/varnishd/cache_fetch.c b/bin/varnishd/cache_fetch.c
index acef94b..6e92963 100644
--- a/bin/varnishd/cache_fetch.c
+++ b/bin/varnishd/cache_fetch.c
@@ -198,15 +198,12 @@ fetch_number(const char *nbr, int radix)
 /*--------------------------------------------------------------------*/
 
 static int
-fetch_straight(struct worker *w, struct http_conn *htc, const char *b)
+fetch_straight(struct worker *w, struct http_conn *htc, ssize_t cl)
 {
 	int i;
-	ssize_t cl;
 
 	assert(w->body_status == BS_LENGTH);
 
-	cl = fetch_number(b, 10);
-	w->vfp->begin(w, cl > 0 ? cl : 0);
 	if (cl < 0) {
 		WSLB(w, SLT_FetchError, "straight length field bogus");
 		return (-1);
@@ -242,7 +239,6 @@ fetch_chunked(struct worker *w, struct http_conn *htc)
 	unsigned u;
 	ssize_t cl;
 
-	w->vfp->begin(w, 0);
 	assert(w->body_status == BS_CHUNKED);
 	do {
 		/* Skip leading whitespace */
@@ -310,7 +306,6 @@ fetch_eof(struct worker *w, struct http_conn *htc)
 	int i;
 
 	assert(w->body_status == BS_EOF);
-	w->vfp->begin(w, 0);
 	i = w->vfp->bytes(w, htc, SSIZE_MAX);
 	if (i < 0) {
 		WSLB(w, SLT_FetchError, "eof read_error: %d (%s)",
@@ -485,6 +480,7 @@ FetchBody(struct worker *w, struct object *obj)
 	int cls;
 	struct storage *st;
 	int mklen;
+	ssize_t cl;
 
 	CHECK_OBJ_NOTNULL(w, WORKER_MAGIC);
 	AZ(w->fetch_obj);
@@ -502,6 +498,8 @@ FetchBody(struct worker *w, struct object *obj)
 
 	w->fetch_obj = obj;
 
+	/* XXX: pick up estimate from objdr ? */
+	cl = 0;
 	switch (w->body_status) {
 	case BS_NONE:
 		cls = 0;
@@ -512,20 +510,26 @@ FetchBody(struct worker *w, struct object *obj)
 		mklen = 1;
 		break;
 	case BS_LENGTH:
-		cls = fetch_straight(w, w->htc,
-		    w->h_content_length);
+		cl = fetch_number( w->h_content_length, 10);
+		w->vfp->begin(w, cl > 0 ? cl : 0);
+		cls = fetch_straight(w, w->htc, cl);
 		mklen = 1;
-		XXXAZ(w->vfp->end(w));
+		if (w->vfp->end(w))
+			cls = -1;
 		break;
 	case BS_CHUNKED:
+		w->vfp->begin(w, cl);
 		cls = fetch_chunked(w, w->htc);
 		mklen = 1;
-		XXXAZ(w->vfp->end(w));
+		if (w->vfp->end(w))
+			cls = -1;
 		break;
 	case BS_EOF:
+		w->vfp->begin(w, cl);
 		cls = fetch_eof(w, w->htc);
 		mklen = 1;
-		XXXAZ(w->vfp->end(w));
+		if (w->vfp->end(w))
+			cls = -1;
 		break;
 	case BS_ERROR:
 		cls = 1;



From geoff at varnish-cache.org  Mon Jan  9 20:52:29 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:29 +0100
Subject: [experimental-ims] 513c4a2 Remove more traces of varnishsizes
Message-ID: <E1RkMCj-00083n-Lm@project.varnish-software.com>

commit 513c4a28655e9f38035bf0a1bb82f1efc3fd7786
Author: Tollef Fog Heen <tfheen at varnish-software.com>
Date:   Mon Nov 7 13:34:51 2011 +0100

    Remove more traces of varnishsizes

diff --git a/.gitignore b/.gitignore
index eb2db8c..14c5393 100644
--- a/.gitignore
+++ b/.gitignore
@@ -70,8 +70,6 @@ TAGS
 /bin/varnishncsa/varnishncsa.1
 /bin/varnishreplay/varnishreplay
 /bin/varnishreplay/varnishreplay.1
-/bin/varnishsizes/varnishsizes
-/bin/varnishsizes/varnishsizes.1
 /bin/varnishstat/varnishstat
 /bin/varnishstat/varnishstat.1
 /bin/varnishtest/varnishtest
diff --git a/configure.ac b/configure.ac
index ebea42b..bbe44b5 100644
--- a/configure.ac
+++ b/configure.ac
@@ -569,7 +569,6 @@ AC_CONFIG_FILES([
     bin/varnishncsa/Makefile
     bin/varnishreplay/Makefile
     bin/varnishstat/Makefile
-    bin/varnishsizes/Makefile
     bin/varnishtest/Makefile
     bin/varnishtop/Makefile
     doc/Makefile



From geoff at varnish-cache.org  Mon Jan  9 20:52:17 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:17 +0100
Subject: [experimental-ims] f4f1e8a Add short segment on VMs / OpenVZ
Message-ID: <E1RkMCX-0007f7-UA@project.varnish-software.com>

commit f4f1e8ad2e9ffff37a10c2bf673c2da65a54ece9
Author: Lasse Karstensen <lasse at varnish-software.com>
Date:   Tue Oct 11 09:59:00 2011 +0200

    Add short segment on VMs / OpenVZ

diff --git a/doc/sphinx/tutorial/index.rst b/doc/sphinx/tutorial/index.rst
index e0e5023..59fd94e 100644
--- a/doc/sphinx/tutorial/index.rst
+++ b/doc/sphinx/tutorial/index.rst
@@ -26,6 +26,7 @@ separate topic. Good luck.
 	vary
 	purging
 	esi
+	virtualised
 	advanced_backend_servers
         handling_misbehaving_servers
         advanced_topics
diff --git a/doc/sphinx/tutorial/virtualised.rst b/doc/sphinx/tutorial/virtualised.rst
new file mode 100644
index 0000000..ce4b9ad
--- /dev/null
+++ b/doc/sphinx/tutorial/virtualised.rst
@@ -0,0 +1,20 @@
+
+Running inside a virtual machine (VM)
+-------------------------------------
+
+It is possible, but not recommended for high performance, to run Varnish on virtualised 
+hardware.
+
+OpenVZ
+''''''
+
+If you are running on 64bit OpenVZ (or Parallels VPS), you must reduce the 
+maximum stack size before starting Varnish. The default allocates to much memory per thread, 
+which will make varnish fail as soon as the number of threads (==traffic) increases.
+
+Reduce the maximum stack size by running::
+
+    ulimit -s 256
+
+in the startup script.
+



From geoff at varnish-cache.org  Mon Jan  9 20:52:00 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:00 +0100
Subject: [experimental-ims] 6d76ccd Don't rely on the test executing fast
	enough and repeatably enough that two separate timestamps just
	happen to be the same, just make them the same.
Message-ID: <E1RkMCG-00078S-7J@project.varnish-software.com>

commit 6d76ccd9fbd3b5022e5d55ae1779b7ea33f0365d
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Sep 20 19:15:11 2011 +0000

    Don't rely on the test executing fast enough and repeatably enough that
    two separate timestamps just happen to be the same, just make them
    the same.

diff --git a/bin/varnishtest/tests/r00907.vtc b/bin/varnishtest/tests/r00907.vtc
index 8bbcd2e..b64a160 100644
--- a/bin/varnishtest/tests/r00907.vtc
+++ b/bin/varnishtest/tests/r00907.vtc
@@ -3,7 +3,8 @@ varnishtest "Ticket #907 200/304 handling with Etags + Last-Modified"
 server s1 {
 	rxreq 
 	txresp \
-	       -hdr "ETag: saengei1Ohshicich4iteesu"
+		-hdr "ETag: saengei1Ohshicich4iteesu" \
+		-hdr "Last-Modified: Tue, 20 Sep 2011 18:55:00 GMT"
 } -start
 
 varnish v1 -vcl+backend {
@@ -16,8 +17,15 @@ client c1 {
 	txreq -hdr "If-None-Match: saengei1Ohshicich4iteesu"
 	rxresp
 	expect resp.status == 304
+
 	txreq -hdr "If-None-Match: saengei1Ohshicich4iteesu" \
-	      -hdr "If-Modified-Since: ${date}"
+	      -hdr "If-Modified-Since: Tue, 20 Sep 2011 18:54:59 GMT"
+	rxresp
+	expect resp.status == 200
+
+	txreq -hdr "If-None-Match: saengei1Ohshicich4iteesu" \
+	      -hdr "If-Modified-Since: Tue, 20 Sep 2011 18:55:00 GMT"
 	rxresp
 	expect resp.status == 304
+
 } -run



From geoff at varnish-cache.org  Mon Jan  9 20:52:31 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:31 +0100
Subject: [experimental-ims] dd74a96 OCD patch:  return (x)
Message-ID: <E1RkMCl-000887-Sf@project.varnish-software.com>

commit dd74a963e2a1ea84275ff67aef905c33cc6edc39
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu Nov 10 10:54:49 2011 +0000

    OCD patch:  return (x)

diff --git a/bin/varnishd/cache_backend.c b/bin/varnishd/cache_backend.c
index e9d0323..1f290c3 100644
--- a/bin/varnishd/cache_backend.c
+++ b/bin/varnishd/cache_backend.c
@@ -412,7 +412,7 @@ vdi_get_backend_if_simple(const struct director *d)
 
 	vs2 = d->priv;
 	if (vs2->magic != VDI_SIMPLE_MAGIC)
-		return NULL;
+		return (NULL);
 	CAST_OBJ_NOTNULL(vs, d->priv, VDI_SIMPLE_MAGIC);
 	return (vs->backend);
 }
diff --git a/bin/varnishd/cache_vrt.c b/bin/varnishd/cache_vrt.c
index 3851de7..c20b552 100644
--- a/bin/varnishd/cache_vrt.c
+++ b/bin/varnishd/cache_vrt.c
@@ -342,7 +342,7 @@ VRT_time_string(const struct sess *sp, double t)
 
 	AN(p = WS_Alloc(sp->http->ws, VTIM_FORMAT_SIZE));
 	VTIM_format(t, p);
-	return p;
+	return (p);
 }
 
 const char *
diff --git a/bin/varnishd/cache_vrt_var.c b/bin/varnishd/cache_vrt_var.c
index 92d8451..7a5aed0 100644
--- a/bin/varnishd/cache_vrt_var.c
+++ b/bin/varnishd/cache_vrt_var.c
@@ -485,9 +485,9 @@ VRT_r_server_identity(struct sess *sp)
 	(void)sp;
 
 	if (heritage.identity[0] != '\0')
-		return heritage.identity;
+		return (heritage.identity);
 	else
-		return heritage.name;
+		return (heritage.name);
 }
 
 
diff --git a/bin/varnishd/hash/hash_critbit.c b/bin/varnishd/hash/hash_critbit.c
index b1c9cf3..fd50b26 100644
--- a/bin/varnishd/hash/hash_critbit.c
+++ b/bin/varnishd/hash/hash_critbit.c
@@ -57,7 +57,7 @@ static unsigned char hcb_bittbl[256];
 static unsigned char
 hcb_bits(unsigned char x, unsigned char y)
 {
-	return hcb_bittbl[x ^ y];
+	return (hcb_bittbl[x ^ y]);
 }
 
 static void
diff --git a/bin/varnishd/waiter/cache_waiter_epoll.c b/bin/varnishd/waiter/cache_waiter_epoll.c
index b5068a6..46cb288 100644
--- a/bin/varnishd/waiter/cache_waiter_epoll.c
+++ b/bin/varnishd/waiter/cache_waiter_epoll.c
@@ -200,7 +200,7 @@ vwe_thread(void *priv)
 			SES_Delete(sp, "timeout");
 		}
 	}
-	return NULL;
+	return (NULL);
 }
 
 /*--------------------------------------------------------------------*/
@@ -219,7 +219,7 @@ vwe_sess_timeout_ticker(void *priv)
 		assert(write(vwe->timer_pipes[1], &ticker, 1));
 		VTIM_sleep(100 * 1e-3);
 	}
-	return NULL;
+	return (NULL);
 }
 
 /*--------------------------------------------------------------------*/
diff --git a/bin/varnishncsa/varnishncsa.c b/bin/varnishncsa/varnishncsa.c
index 8d969eb..9765796 100644
--- a/bin/varnishncsa/varnishncsa.c
+++ b/bin/varnishncsa/varnishncsa.c
@@ -200,11 +200,11 @@ req_header(struct logline *l, const char *name)
 	struct hdr *h;
 	VTAILQ_FOREACH(h, &l->req_headers, list) {
 		if (strcasecmp(h->key, name) == 0) {
-			return h->value;
+			return (h->value);
 			break;
 		}
 	}
-	return NULL;
+	return (NULL);
 }
 
 static char *
@@ -213,11 +213,11 @@ resp_header(struct logline *l, const char *name)
 	struct hdr *h;
 	VTAILQ_FOREACH(h, &l->resp_headers, list) {
 		if (strcasecmp(h->key, name) == 0) {
-			return h->value;
+			return (h->value);
 			break;
 		}
 	}
-	return NULL;
+	return (NULL);
 }
 
 static char *
@@ -226,11 +226,11 @@ vcl_log(struct logline *l, const char *name)
 	struct hdr *h;
 	VTAILQ_FOREACH(h, &l->vcl_log, list) {
 		if (strcasecmp(h->key, name) == 0) {
-			return h->value;
+			return (h->value);
 			break;
 		}
 	}
-	return NULL;
+	return (NULL);
 }
 
 static void
diff --git a/lib/libvarnish/vtcp.c b/lib/libvarnish/vtcp.c
index 21de7f2..d15f516 100644
--- a/lib/libvarnish/vtcp.c
+++ b/lib/libvarnish/vtcp.c
@@ -61,11 +61,11 @@ VTCP_port(const struct sockaddr_storage *addr)
 
 	if (addr->ss_family == AF_INET) {
 		const struct sockaddr_in *ain = (const void *)addr;
-		return ntohs((ain->sin_port));
+		return (ntohs((ain->sin_port)));
 	}
 	if (addr->ss_family == AF_INET6) {
 		const struct sockaddr_in6 *ain = (const void *)addr;
-		return ntohs((ain->sin6_port));
+		return (ntohs((ain->sin6_port)));
 	}
 	return (-1);
 }
diff --git a/man/vsc2rst.c b/man/vsc2rst.c
index bf4091f..18250d8 100644
--- a/man/vsc2rst.c
+++ b/man/vsc2rst.c
@@ -64,6 +64,6 @@ int main(int argc, char **argv)
 #include "tbl/vsc_fields.h"
 #undef VSC_DO_VBE
 
-	return 0;
+	return (0);
 }
 



From geoff at varnish-cache.org  Mon Jan  9 20:52:26 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:26 +0100
Subject: [experimental-ims] 4133b27 Push sess out of the stream-polling
	operation.
Message-ID: <E1RkMCg-0007ur-0s@project.varnish-software.com>

commit 4133b274b11c980b302f4c79716b8083efaf3d83
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Oct 24 14:01:25 2011 +0000

    Push sess out of the stream-polling operation.

diff --git a/bin/varnishd/cache.h b/bin/varnishd/cache.h
index 298f820..08ba997 100644
--- a/bin/varnishd/cache.h
+++ b/bin/varnishd/cache.h
@@ -722,7 +722,7 @@ int VGZ_Gzip(struct vgz *, const void **, size_t *len, enum vgz_flag);
 int VGZ_Gunzip(struct vgz *, const void **, size_t *len);
 void VGZ_Destroy(struct vgz **);
 void VGZ_UpdateObj(const struct vgz*, struct object *);
-int VGZ_WrwGunzip(const struct sess *, struct vgz *, const void *ibuf,
+int VGZ_WrwGunzip(struct worker *w, struct vgz *, const void *ibuf,
     ssize_t ibufl, char *obuf, ssize_t obufl, ssize_t *obufp);
 
 /* Return values */
@@ -898,7 +898,7 @@ void RES_BuildHttp(const struct sess *sp);
 void RES_WriteObj(struct sess *sp);
 void RES_StreamStart(struct sess *sp);
 void RES_StreamEnd(struct sess *sp);
-void RES_StreamPoll(const struct sess *sp);
+void RES_StreamPoll(struct worker *);
 
 /* cache_vary.c */
 struct vsb *VRY_Create(const struct sess *sp, const struct http *hp);
diff --git a/bin/varnishd/cache_esi_deliver.c b/bin/varnishd/cache_esi_deliver.c
index aaf5f7a..c79da64 100644
--- a/bin/varnishd/cache_esi_deliver.c
+++ b/bin/varnishd/cache_esi_deliver.c
@@ -330,7 +330,7 @@ ESI_Deliver(struct sess *sp)
 					 * response
 					 */
 					AN(vgz);
-					i = VGZ_WrwGunzip(sp, vgz,
+					i = VGZ_WrwGunzip(sp->wrk, vgz,
 						st->ptr + off, l2,
 						obuf, sizeof obuf, &obufl);
 					if (WRW_Error(sp->wrk)) {
diff --git a/bin/varnishd/cache_fetch.c b/bin/varnishd/cache_fetch.c
index d71b2ec..08dd99c 100644
--- a/bin/varnishd/cache_fetch.c
+++ b/bin/varnishd/cache_fetch.c
@@ -101,7 +101,7 @@ vfp_nop_bytes(struct sess *sp, struct http_conn *htc, ssize_t bytes)
 		sp->obj->len += w;
 		bytes -= w;
 		if (sp->wrk->do_stream)
-			RES_StreamPoll(sp);
+			RES_StreamPoll(sp->wrk);
 	}
 	return (1);
 }
diff --git a/bin/varnishd/cache_gzip.c b/bin/varnishd/cache_gzip.c
index 68dc04b..81c2475 100644
--- a/bin/varnishd/cache_gzip.c
+++ b/bin/varnishd/cache_gzip.c
@@ -350,7 +350,7 @@ VGZ_Gzip(struct vgz *vg, const void **pptr, size_t *plen, enum vgz_flag flags)
  */
 
 int
-VGZ_WrwGunzip(const struct sess *sp, struct vgz *vg, const void *ibuf,
+VGZ_WrwGunzip(struct worker *w, struct vgz *vg, const void *ibuf,
     ssize_t ibufl, char *obuf, ssize_t obufl, ssize_t *obufp)
 {
 	int i;
@@ -375,9 +375,9 @@ VGZ_WrwGunzip(const struct sess *sp, struct vgz *vg, const void *ibuf,
 			return (-1);
 		}
 		if (obufl == *obufp || i == VGZ_STUCK) {
-			sp->wrk->acct_tmp.bodybytes += *obufp;
-			(void)WRW_Write(sp->wrk, obuf, *obufp);
-			(void)WRW_Flush(sp->wrk);
+			w->acct_tmp.bodybytes += *obufp;
+			(void)WRW_Write(w, obuf, *obufp);
+			(void)WRW_Flush(w);
 			*obufp = 0;
 			VGZ_Obuf(vg, obuf + *obufp, obufl - *obufp);
 		}
@@ -474,7 +474,7 @@ vfp_gunzip_bytes(struct sess *sp, struct http_conn *htc, ssize_t bytes)
 		assert(i == VGZ_OK || i == VGZ_END);
 		sp->obj->len += dl;
 		if (sp->wrk->do_stream)
-			RES_StreamPoll(sp);
+			RES_StreamPoll(sp->wrk);
 	}
 	if (i == Z_OK || i == Z_STREAM_END)
 		return (1);
@@ -549,7 +549,7 @@ vfp_gzip_bytes(struct sess *sp, struct http_conn *htc, ssize_t bytes)
 		assert(i == Z_OK);
 		sp->obj->len += dl;
 		if (sp->wrk->do_stream)
-			RES_StreamPoll(sp);
+			RES_StreamPoll(sp->wrk);
 	}
 	return (1);
 }
@@ -573,7 +573,7 @@ vfp_gzip_end(struct sess *sp)
 		sp->obj->len += dl;
 	} while (i != Z_STREAM_END);
 	if (sp->wrk->do_stream)
-		RES_StreamPoll(sp);
+		RES_StreamPoll(sp->wrk);
 	VGZ_UpdateObj(vg, sp->obj);
 	VGZ_Destroy(&vg);
 	return (0);
@@ -631,7 +631,7 @@ vfp_testgzip_bytes(struct sess *sp, struct http_conn *htc, ssize_t bytes)
 		st->len += w;
 		sp->obj->len += w;
 		if (sp->wrk->do_stream)
-			RES_StreamPoll(sp);
+			RES_StreamPoll(sp->wrk);
 
 		while (!VGZ_IbufEmpty(vg)) {
 			VGZ_Obuf(vg, obuf, sizeof obuf);
diff --git a/bin/varnishd/cache_response.c b/bin/varnishd/cache_response.c
index 5521bc0..ecc9839 100644
--- a/bin/varnishd/cache_response.c
+++ b/bin/varnishd/cache_response.c
@@ -172,7 +172,7 @@ res_WriteGunzipObj(const struct sess *sp)
 
 		VSC_C_main->n_objwrite++;
 
-		i = VGZ_WrwGunzip(sp, vg,
+		i = VGZ_WrwGunzip(sp->wrk, vg,
 		    st->ptr, st->len,
 		    obuf, sizeof obuf, &obufl);
 		/* XXX: error check */
@@ -354,50 +354,52 @@ RES_StreamStart(struct sess *sp)
 }
 
 void
-RES_StreamPoll(const struct sess *sp)
+RES_StreamPoll(struct worker *w)
 {
 	struct stream_ctx *sctx;
 	struct storage *st;
 	ssize_t l, l2;
 	void *ptr;
 
-	sctx = sp->wrk->sctx;
+	CHECK_OBJ_NOTNULL(w, WORKER_MAGIC);
+	CHECK_OBJ_NOTNULL(w->fetch_obj, OBJECT_MAGIC);
+	sctx = w->sctx;
 	CHECK_OBJ_NOTNULL(sctx, STREAM_CTX_MAGIC);
-	if (sp->obj->len == sctx->stream_next)
+	if (w->fetch_obj->len == sctx->stream_next)
 		return;
-	assert(sp->obj->len > sctx->stream_next);
+	assert(w->fetch_obj->len > sctx->stream_next);
 	l = sctx->stream_front;
-	VTAILQ_FOREACH(st, &sp->obj->store, list) {
+	VTAILQ_FOREACH(st, &w->fetch_obj->store, list) {
 		if (st->len + l <= sctx->stream_next) {
 			l += st->len;
 			continue;
 		}
 		l2 = st->len + l - sctx->stream_next;
 		ptr = st->ptr + (sctx->stream_next - l);
-		if (sp->wrk->res_mode & RES_GUNZIP) {
-			(void)VGZ_WrwGunzip(sp, sctx->vgz, ptr, l2,
+		if (w->res_mode & RES_GUNZIP) {
+			(void)VGZ_WrwGunzip(w, sctx->vgz, ptr, l2,
 			    sctx->obuf, sctx->obuf_len, &sctx->obuf_ptr);
 		} else {
-			(void)WRW_Write(sp->wrk, ptr, l2);
+			(void)WRW_Write(w, ptr, l2);
 		}
 		l += st->len;
 		sctx->stream_next += l2;
 	}
-	if (!(sp->wrk->res_mode & RES_GUNZIP))
-		(void)WRW_Flush(sp->wrk);
+	if (!(w->res_mode & RES_GUNZIP))
+		(void)WRW_Flush(w);
 
-	if (sp->obj->objcore == NULL ||
-	    (sp->obj->objcore->flags & OC_F_PASS)) {
+	if (w->fetch_obj->objcore == NULL ||
+	    (w->fetch_obj->objcore->flags & OC_F_PASS)) {
 		/*
 		 * This is a pass object, release storage as soon as we
 		 * have delivered it.
 		 */
 		while (1) {
-			st = VTAILQ_FIRST(&sp->obj->store);
+			st = VTAILQ_FIRST(&w->fetch_obj->store);
 			if (st == NULL ||
 			    sctx->stream_front + st->len > sctx->stream_next)
 				break;
-			VTAILQ_REMOVE(&sp->obj->store, st, list);
+			VTAILQ_REMOVE(&w->fetch_obj->store, st, list);
 			sctx->stream_front += st->len;
 			STV_free(st);
 		}



From geoff at varnish-cache.org  Mon Jan  9 20:51:56 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:51:56 +0100
Subject: [experimental-ims] 12bf085 Get rid of the single acceptor thread.
Message-ID: <E1RkMCC-0006zk-1Q@project.varnish-software.com>

commit 12bf085b48f787b4744be61197136cd6ee983b5d
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sun Sep 18 07:19:44 2011 +0000

    Get rid of the single acceptor thread.
    
    Instead of a single thread which accepts all sockets with a poll/accept
    sequence, each thread-pool will have a thread accepting on each socket.
    
    If no threads are available, the sockets will not be accepted on.
    
    CAVEATS:
    
    This commit undoubtedly leaves a number of minor issues dangling,
    they will be cleaned up as we find them.
    
    Please notice that there are changes to stats counters (some don't
    work right now a nd more changes will be coming)
    
    Changing certain acceptor-related params on the fly may not work.
    
    Testing would be very welcome.

diff --git a/bin/varnishd/cache.h b/bin/varnishd/cache.h
index 71cf6b3..49ca3a4 100644
--- a/bin/varnishd/cache.h
+++ b/bin/varnishd/cache.h
@@ -305,7 +305,9 @@ struct worker {
 
 	/* Accept stuff */
 	struct sockaddr_storage	acceptaddr;
+	socklen_t		acceptaddrlen;
 	int			acceptsock;
+	struct listen_sock	*acceptlsock;
 
 	struct wrw		wrw;
 
@@ -641,7 +643,7 @@ void VCA_Prep(struct sess *sp);
 void VCA_Init(void);
 void VCA_Shutdown(void);
 int VCA_Accept(int sock, socklen_t *slp, struct sockaddr_storage *sap);
-extern pthread_t VCA_thread;
+void VCA_SetupSess(struct worker *w);
 
 /* cache_backend.c */
 void VBE_UseHealth(const struct director *vdi);
@@ -849,12 +851,13 @@ void WRW_Sendfile(struct worker *w, int fd, off_t off, unsigned len);
 
 
 /* cache_session.c [SES] */
-void SES_Init(void);
-struct sess *SES_New(struct sesspool *pp);
+struct sess *SES_New(struct worker *wrk, struct sesspool *pp);
 struct sess *SES_Alloc(void);
 void SES_Close(struct sess *sp, const char *reason);
 void SES_Delete(struct sess *sp, const char *reason);
 void SES_Charge(struct sess *sp);
+struct sesspool *SES_NewPool(void);
+
 
 /* cache_shmlog.c */
 void VSL_Init(void);
diff --git a/bin/varnishd/cache_acceptor.c b/bin/varnishd/cache_acceptor.c
index 37cc857..b2a8015 100644
--- a/bin/varnishd/cache_acceptor.c
+++ b/bin/varnishd/cache_acceptor.c
@@ -33,6 +33,7 @@
 #include <errno.h>
 #include <poll.h>
 #include <stdlib.h>
+#include <stdio.h>
 
 #include <sys/types.h>
 #include <sys/socket.h>
@@ -41,7 +42,7 @@
 #include "cli_priv.h"
 #include "cache.h"
 
-pthread_t		VCA_thread;
+static pthread_t	VCA_thread;
 static struct timeval	tv_sndtimeo;
 static struct timeval	tv_rcvtimeo;
 
@@ -178,6 +179,7 @@ vca_pace_check(void)
 static void
 vca_pace_bad(void)
 {
+
 	Lck_Lock(&pace_mtx);
 	vca_pace += params->acceptor_sleep_incr;
 	if (vca_pace > params->acceptor_sleep_max)
@@ -202,13 +204,19 @@ vca_pace_good(void)
  * Accept on a listen socket, and handle error returns.
  */
 
+static int hack_ready;
+
 int
 VCA_Accept(int sock, socklen_t *slp, struct sockaddr_storage *sap)
 {
 	int i;
 
+	assert(sock >= 0);
 	vca_pace_check();
 
+	while(!hack_ready)
+		(void)usleep(100*1000);
+
 	*slp = sizeof *sap;
 	i = accept(sock, (void*)sap, slp);
 
@@ -234,45 +242,67 @@ VCA_Accept(int sock, socklen_t *slp, struct sockaddr_storage *sap)
 
 /*--------------------------------------------------------------------*/
 
+void
+VCA_SetupSess(struct worker *w)
+{
+	struct sess *sp;
+
+	sp = w->sp;
+	if (sp == NULL) {
+		AZ(close(w->acceptsock));
+		w->acceptsock = -1;
+		VSC_C_main->client_drop++;
+		/* XXX: 50x Reply ? */
+		vca_pace_bad();
+		INCOMPL();
+	}
+	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+	sp->fd = w->acceptsock;
+	sp->id = w->acceptsock;
+	w->acceptsock = -1;
+	sp->t_open = TIM_real();
+	sp->t_end = sp->t_end;
+	sp->mylsock = w->acceptlsock;
+	assert(w->acceptaddrlen <= sp->sockaddrlen);
+	memcpy(sp->sockaddr, &w->acceptaddr, w->acceptaddrlen);
+	sp->sockaddrlen = w->acceptaddrlen;
+	sp->step = STP_FIRST;
+	vca_pace_good();
+	w->sp = sp;
+	w->stats.client_conn++;
+}
+
+/*--------------------------------------------------------------------*/
+
 static void *
 vca_acct(void *arg)
 {
-	struct sess *sp;
-	socklen_t l;
-	struct sockaddr_storage addr_s;
-	struct sockaddr *addr;
 #ifdef SO_RCVTIMEO_WORKS
 	double sess_timeout = 0;
 #endif
 #ifdef SO_SNDTIMEO_WORKS
 	double send_timeout = 0;
 #endif
-	int i;
-	struct pollfd *pfd;
 	struct listen_sock *ls;
-	unsigned u;
 	double t0, now;
 
 	THR_SetName("cache-acceptor");
 	(void)arg;
 
-	/* Set up the poll argument */
-	pfd = calloc(sizeof *pfd, heritage.nsocks);
-	AN(pfd);
-	i = 0;
 	VTAILQ_FOREACH(ls, &heritage.socks, list) {
 		if (ls->sock < 0)
 			continue;
 		AZ(listen(ls->sock, params->listen_depth));
 		AZ(setsockopt(ls->sock, SOL_SOCKET, SO_LINGER,
 		    &linger, sizeof linger));
-		pfd[i].events = POLLIN;
-		pfd[i++].fd = ls->sock;
 	}
 
+	hack_ready = 1;
+
 	need_test = 1;
 	t0 = TIM_real();
 	while (1) {
+		(void)sleep(1);
 #ifdef SO_SNDTIMEO_WORKS
 		if (params->send_timeout != send_timeout) {
 			need_test = 1;
@@ -301,45 +331,8 @@ vca_acct(void *arg)
 			}
 		}
 #endif
-		i = poll(pfd, heritage.nsocks, 1000);
 		now = TIM_real();
 		VSC_C_main->uptime = (uint64_t)(now - t0);
-		u = 0;
-		VTAILQ_FOREACH(ls, &heritage.socks, list) {
-			if (ls->sock < 0)
-				continue;
-			if (pfd[u++].revents == 0)
-				continue;
-			VSC_C_main->client_conn++;
-			l = sizeof addr_s;
-			addr = (void*)&addr_s;
-			i = VCA_Accept(ls->sock, &l, &addr_s);
-			if (i < 0) 
-				continue;
-			sp = SES_New(NULL);
-			if (sp == NULL) {
-				AZ(close(i));
-				VSC_C_main->client_drop++;
-				vca_pace_bad();
-				continue;
-			}
-			sp->fd = i;
-			sp->id = i;
-			sp->t_open = now;
-			sp->t_end = now;
-			sp->mylsock = ls;
-			assert(l < sp->sockaddrlen);
-			memcpy(sp->sockaddr, addr, l);
-			sp->sockaddrlen = l;
-
-			sp->step = STP_FIRST;
-			if (Pool_QueueSession(sp)) {
-				VSC_C_main->client_drop++;
-				vca_pace_bad();
-			} else {
-				vca_pace_good();
-			}
-		}
 	}
 	NEEDLESS_RETURN(NULL);
 }
diff --git a/bin/varnishd/cache_center.c b/bin/varnishd/cache_center.c
index b561c84..db51927 100644
--- a/bin/varnishd/cache_center.c
+++ b/bin/varnishd/cache_center.c
@@ -369,7 +369,6 @@ cnt_done(struct sess *sp)
 
 	if (sp->fd < 0) {
 		sp->wrk->stats.sess_closed++;
-		sp->wrk = NULL;
 		SES_Delete(sp, NULL);
 		return (1);
 	}
diff --git a/bin/varnishd/cache_hash.c b/bin/varnishd/cache_hash.c
index 16e5cf0..8e0fd4e 100644
--- a/bin/varnishd/cache_hash.c
+++ b/bin/varnishd/cache_hash.c
@@ -509,7 +509,6 @@ hsh_rush(struct objhead *oh)
 			 * We could not schedule the session, leave the
 			 * rest on the busy list.
 			 */
-			VSC_C_main->client_drop_late++;
 			break;
 		}
 	}
diff --git a/bin/varnishd/cache_main.c b/bin/varnishd/cache_main.c
index 3489530..5660893 100644
--- a/bin/varnishd/cache_main.c
+++ b/bin/varnishd/cache_main.c
@@ -113,7 +113,6 @@ child_main(void)
 	VCL_Init();
 
 	HTTP_Init();
-	SES_Init();
 
 	VBE_Init();
 	VBP_Init();
diff --git a/bin/varnishd/cache_pool.c b/bin/varnishd/cache_pool.c
index 05ce529..1bfc6b4 100644
--- a/bin/varnishd/cache_pool.c
+++ b/bin/varnishd/cache_pool.c
@@ -66,6 +66,7 @@ struct poolsock {
 	unsigned			magic;
 #define POOLSOCK_MAGIC			0x1b0a2d38
 	VTAILQ_ENTRY(poolsock)		list;
+	struct listen_sock		*lsock;
 	int				sock;
 };
 
@@ -83,6 +84,7 @@ struct pool {
 	unsigned		last_lqueue;
 	uintmax_t		ndrop;
 	uintmax_t		nqueue;
+	struct sesspool		*sesspool;
 };
 
 static struct pool		**wq;
@@ -95,11 +97,46 @@ static struct lock		herder_mtx;
 
 /*--------------------------------------------------------------------*/
 
+static void
+pool_accept(struct pool *pp, struct worker *w, const struct poolsock *ps)
+{
+	struct worker *w2;
+
+	CHECK_OBJ_NOTNULL(pp, POOL_MAGIC);
+	CHECK_OBJ_NOTNULL(w, WORKER_MAGIC);
+	CHECK_OBJ_NOTNULL(ps, POOLSOCK_MAGIC);
+
+	assert(ps->sock >= 0);
+	Lck_AssertHeld(&pp->mtx);
+	Lck_Unlock(&pp->mtx);
+	while (1) {
+		w->acceptsock =
+		    VCA_Accept(ps->sock, &w->acceptaddrlen, &w->acceptaddr);
+		if (w->acceptsock == -1)
+			continue;
+		w->acceptlsock = ps->lsock;
+		Lck_Lock(&pp->mtx);
+		if (VTAILQ_EMPTY(&pp->idle))
+			return;
+		w2 = VTAILQ_FIRST(&pp->idle);
+		VTAILQ_REMOVE(&pp->idle, w2, list);
+		Lck_Unlock(&pp->mtx);
+		w2->acceptaddr = w->acceptaddr;
+		w2->acceptaddrlen = w->acceptaddrlen;
+		w2->acceptsock = w->acceptsock;
+		w2->acceptlsock = w->acceptlsock;
+		AZ(pthread_cond_signal(&w2->cond));
+	}
+}
+
+/*--------------------------------------------------------------------*/
+
 void
 Pool_Work_Thread(void *priv, struct worker *w)
 {
 	struct pool *qp;
 	int stats_clean;
+	struct poolsock *ps;
 
 	CAST_OBJ_NOTNULL(qp, priv, POOL_MAGIC);
 	w->pool = qp;
@@ -107,6 +144,9 @@ Pool_Work_Thread(void *priv, struct worker *w)
 	qp->nthr++;
 	stats_clean = 1;
 	while (1) {
+
+		Lck_AssertHeld(&qp->mtx);
+
 		CHECK_OBJ_NOTNULL(w->bereq, HTTP_MAGIC);
 		CHECK_OBJ_NOTNULL(w->beresp, HTTP_MAGIC);
 		CHECK_OBJ_NOTNULL(w->resp, HTTP_MAGIC);
@@ -117,17 +157,29 @@ Pool_Work_Thread(void *priv, struct worker *w)
 		if (w->sp != NULL) {
 			VTAILQ_REMOVE(&qp->queue, w->sp, poollist);
 			qp->lqueue--;
-		} else {
+		} else if (VTAILQ_EMPTY(&qp->socks)) {
 			if (isnan(w->lastused))
 				w->lastused = TIM_real();
 			VTAILQ_INSERT_HEAD(&qp->idle, w, list);
 			if (!stats_clean)
 				WRK_SumStat(w);
 			Lck_CondWait(&w->cond, &qp->mtx);
+		} else {
+			ps = VTAILQ_FIRST(&qp->socks);
+			VTAILQ_REMOVE(&qp->socks, ps, list);
+			pool_accept(qp, w, ps);
+			Lck_AssertHeld(&qp->mtx);
+			VTAILQ_INSERT_TAIL(&qp->socks, ps, list);
 		}
-		if (w->sp == NULL)
+		if (w->sp == NULL && w->acceptsock == -1)
 			break;
 		Lck_Unlock(&qp->mtx);
+		if (w->sp == NULL) {
+			w->sp = SES_New(w, qp->sesspool);
+			VCA_SetupSess(w);
+		}
+		AN(w->sp);
+		assert(w->acceptsock == -1);
 		stats_clean = 0;
 		w->lastused = NAN;
 		WS_Reset(w->ws, NULL);
@@ -221,7 +273,9 @@ Pool_QueueSession(struct sess *sp)
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 	AZ(sp->wrk);
 	if (WRK_Queue(sp) == 0)
-		return (0);
+		return(0);
+
+	VSC_C_main->client_drop_late++;
 
 	/*
 	 * Couldn't queue it -- kill it.
@@ -258,11 +312,16 @@ pool_mkpool(void)
 	VTAILQ_INIT(&pp->queue);
 	VTAILQ_INIT(&pp->idle);
 	VTAILQ_INIT(&pp->socks);
+	pp->sesspool = SES_NewPool();
+	AN(pp->sesspool);
 
 	VTAILQ_FOREACH(ls, &heritage.socks, list) {
+		if (ls->sock < 0)
+			continue;
 		ALLOC_OBJ(ps, POOLSOCK_MAGIC);
 		XXXAN(ps);
 		ps->sock = ls->sock;
+		ps->lsock = ls;
 		VTAILQ_INSERT_TAIL(&pp->socks, ps, list);
 	}
 	return (pp);
diff --git a/bin/varnishd/cache_session.c b/bin/varnishd/cache_session.c
index a930c07..4c01f34 100644
--- a/bin/varnishd/cache_session.c
+++ b/bin/varnishd/cache_session.c
@@ -28,6 +28,9 @@
  *
  * Session management
  *
+ * This is a little bit of a mixed back, containing both memory management
+ * and various state-change functions.
+ *
  */
 
 #include "config.h"
@@ -39,7 +42,6 @@
 #include <sys/socket.h>
 
 #include "cache.h"
-#include "cache_backend.h"
 #include "cache_waiter.h"
 
 /*--------------------------------------------------------------------*/
@@ -63,16 +65,12 @@ struct sesspool {
 	VTAILQ_HEAD(,sessmem)	freelist;
 	struct lock		mtx;
 	unsigned		nsess;
-	unsigned		maxsess;
+	unsigned		dly_free_cnt;
 };
 
-static struct sesspool *sesspool;
-
-/*--------------------------------------------------------------------*/
-
-static struct lock		stat_mtx;
-
-/*--------------------------------------------------------------------*/
+/*--------------------------------------------------------------------
+ * Charge statistics from worker to request and session.
+ */
 
 void
 SES_Charge(struct sess *sp)
@@ -112,16 +110,12 @@ ses_sm_alloc(void)
 
 	hl = HTTP_estimate(nhttp);
 	l = sizeof *sm + nws + 2 * hl;
+	VSC_C_main->g_sessmem_size = l;
 	p = malloc(l);
 	if (p == NULL)
 		return (NULL);
 	q = p + l;
 
-	/* XXX Stats */
-	Lck_Lock(&stat_mtx);
-	VSC_C_main->n_sess_mem++;
-	Lck_Unlock(&stat_mtx);
-
 	/* Don't waste time zeroing the workspace */
 	memset(p, 0, l - nws);
 
@@ -179,38 +173,42 @@ ses_setup(struct sessmem *sm)
  */
 
 struct sess *
-SES_New(struct sesspool *pp)
+SES_New(struct worker *wrk, struct sesspool *pp)
 {
 	struct sessmem *sm;
 	struct sess *sp;
-	int do_alloc = 0;
+	int do_alloc;
 
-	if (pp == NULL)
-		pp = sesspool;
 	CHECK_OBJ_NOTNULL(pp, SESSPOOL_MAGIC);
 
+	do_alloc = 0;
 	Lck_Lock(&pp->mtx);
 	sm = VTAILQ_FIRST(&pp->freelist);
 	if (sm != NULL) {
 		VTAILQ_REMOVE(&pp->freelist, sm, list);
-	} else if (pp->nsess < pp->maxsess) {
+	} else if (pp->nsess < params->max_sess) {
 		pp->nsess++;
 		do_alloc = 1;
 	}
+	wrk->stats.c_sessmem_free += pp->dly_free_cnt;
+	pp->dly_free_cnt = 0;
 	Lck_Unlock(&pp->mtx);
 	if (do_alloc) {
 		sm = ses_sm_alloc();
 		if (sm != NULL) {
+			wrk->stats.c_sessmem_alloc++;
 			sm->pool = pp;
 			ses_setup(sm);
+		} else {
+			wrk->stats.c_sessmem_fail++;
 		}
+	} else if (sm == NULL) {
+		wrk->stats.c_sessmem_limit++;
 	}
 	if (sm == NULL)
 		return (NULL);
 	sp = &sm->sess;
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
-	/* XXX Stats */
-	VSC_C_main->n_sess++;		/* XXX: locking  ? */
 	return (sp);
 }
 
@@ -251,8 +249,7 @@ SES_Handle(struct sess *sp, int status)
 		break;
 	case 1:
 		sp->step = STP_START;
-		if (Pool_QueueSession(sp))
-			VSC_C_main->client_drop_late++;
+		(void)Pool_QueueSession(sp);
 		break;
 	default:
 		WRONG("Unexpected return from HTC_Rx()");
@@ -276,16 +273,21 @@ SES_Close(struct sess *sp, const char *reason)
 }
 
 /*--------------------------------------------------------------------
- * (Close &) Recycle a session.  If the workspace has changed, deleted it,
- * otherwise wash it, and put it up for adoption.
+ * (Close &) Free or Recycle a session.
+ *
+ * If the workspace has changed, deleted it, otherwise wash it, and put
+ * it up for adoption.
+ *
+ * XXX: We should also check nhttp
  */
 
 void
 SES_Delete(struct sess *sp, const char *reason)
 {
-	struct acct *b = &sp->acct_ses;
+	struct acct *b;
 	struct sessmem *sm;
 	static char noaddr[] = "-";
+	struct worker *wrk;
 	struct sesspool *pp;
 
 
@@ -294,6 +296,9 @@ SES_Delete(struct sess *sp, const char *reason)
 	CHECK_OBJ_NOTNULL(sm, SESSMEM_MAGIC);
 	pp = sm->pool;
 	CHECK_OBJ_NOTNULL(pp, SESSPOOL_MAGIC);
+	wrk = sp->wrk;
+	CHECK_OBJ_ORNULL(wrk, WORKER_MAGIC);
+
 
 	if (reason != NULL)
 		SES_Close(sp, reason);
@@ -301,54 +306,55 @@ SES_Delete(struct sess *sp, const char *reason)
 
 	AZ(sp->obj);
 	AZ(sp->vcl);
-	VSC_C_main->n_sess--;			/* XXX: locking ? */
-	assert(!isnan(b->first));
-	assert(!isnan(sp->t_end));
 	if (sp->addr == NULL)
 		sp->addr = noaddr;
 	if (sp->port == NULL)
 		sp->port = noaddr;
+
+	b = &sp->acct_ses;
+	assert(!isnan(b->first));
+	assert(!isnan(sp->t_end));
+
 	VSL(SLT_StatSess, sp->id, "%s %s %.0f %ju %ju %ju %ju %ju %ju %ju",
 	    sp->addr, sp->port, sp->t_end - b->first,
 	    b->sess, b->req, b->pipe, b->pass,
 	    b->fetch, b->hdrbytes, b->bodybytes);
 
-	if (sm->workspace != params->sess_workspace) {
-		Lck_Lock(&stat_mtx);
-		VSC_C_main->n_sess_mem--;
-		Lck_Unlock(&stat_mtx);
+	if (sm->workspace != params->sess_workspace ||
+	    pp->nsess > params->max_sess) {
 		free(sm);
 		Lck_Lock(&pp->mtx);
+		if (wrk != NULL)
+			wrk->stats.c_sessmem_free++;
+		else
+			pp->dly_free_cnt++;
 		pp->nsess--;
 		Lck_Unlock(&pp->mtx);
 	} else {
 		/* Clean and prepare for reuse */
 		ses_setup(sm);
 		Lck_Lock(&pp->mtx);
+		if (wrk != NULL) {
+			wrk->stats.c_sessmem_free += pp->dly_free_cnt;
+			pp->dly_free_cnt = 0;
+		}
 		VTAILQ_INSERT_HEAD(&pp->freelist, sm, list);
 		Lck_Unlock(&pp->mtx);
 	}
 }
 
-/*--------------------------------------------------------------------*/
+/*--------------------------------------------------------------------
+ * Create a new pool to allocate from
+ */
 
-static struct sesspool *
-SES_NewPool(unsigned maxsess)
+struct sesspool *
+SES_NewPool(void)
 {
 	struct sesspool *sp;
 
 	ALLOC_OBJ(sp, SESSPOOL_MAGIC);
+	AN(sp);
 	VTAILQ_INIT(&sp->freelist);
 	Lck_New(&sp->mtx, lck_sessmem);
-	sp->maxsess = maxsess;
 	return (sp);
 }
-
-
-void
-SES_Init()
-{
-
-	sesspool = SES_NewPool(params->max_sess);
-	Lck_New(&stat_mtx, lck_stat);
-}
diff --git a/bin/varnishd/cache_wrk.c b/bin/varnishd/cache_wrk.c
index 7b731ba..631342f 100644
--- a/bin/varnishd/cache_wrk.c
+++ b/bin/varnishd/cache_wrk.c
@@ -165,6 +165,7 @@ wrk_thread_real(void *priv, unsigned shm_workspace, unsigned sess_workspace,
 	w->bereq = HTTP_create(http0, nhttp);
 	w->beresp = HTTP_create(http1, nhttp);
 	w->resp = HTTP_create(http2, nhttp);
+	w->acceptsock = -1;
 	w->wrw.iov = iov;
 	w->wrw.siov = siov;
 	w->wrw.ciov = siov;
diff --git a/bin/varnishd/locks.h b/bin/varnishd/locks.h
index 6cbf91f..f1b634b 100644
--- a/bin/varnishd/locks.h
+++ b/bin/varnishd/locks.h
@@ -36,7 +36,6 @@ LOCK(hsl)
 LOCK(hcb)
 LOCK(hcl)
 LOCK(vcl)
-LOCK(stat)
 LOCK(sessmem)
 LOCK(wstat)
 LOCK(herder)
diff --git a/bin/varnishd/mgt_child.c b/bin/varnishd/mgt_child.c
index 9d6966a..4181a43 100644
--- a/bin/varnishd/mgt_child.c
+++ b/bin/varnishd/mgt_child.c
@@ -241,7 +241,6 @@ open_sockets(void)
 		 * closes before we call accept(2) and nobody else are in
 		 * the listen queue to release us.
 		 */
-		(void)VTCP_nonblocking(ls->sock);
 		(void)VTCP_filter_http(ls->sock);
 		good++;
 	}
diff --git a/include/vsc_fields.h b/include/vsc_fields.h
index c35acec..e337321 100644
--- a/include/vsc_fields.h
+++ b/include/vsc_fields.h
@@ -41,7 +41,7 @@
 
 #ifdef VSC_DO_MAIN
 
-VSC_F(client_conn,		uint64_t, 0, 'a', "Client connections accepted")
+VSC_F(client_conn,		uint64_t, 1, 'a', "Client connections accepted")
 VSC_F(client_drop,		uint64_t, 0, 'a',
 					"Connection dropped, no sess/wrk")
 VSC_F(client_req,		uint64_t, 1, 'a', "Client requests received")
@@ -72,6 +72,12 @@ VSC_F(fetch_1xx,		uint64_t, 1, 'a', "Fetch no body (1xx)")
 VSC_F(fetch_204,		uint64_t, 1, 'a', "Fetch no body (204)")
 VSC_F(fetch_304,		uint64_t, 1, 'a', "Fetch no body (304)")
 
+/* Sessmem cache_session.c */
+VSC_F(g_sessmem_size,		uint64_t, 1, 'i', "Session mem size")
+VSC_F(c_sessmem_alloc,		uint64_t, 1, 'a', "Session mem allocated")
+VSC_F(c_sessmem_free,		uint64_t, 1, 'a', "Session mem freed")
+VSC_F(c_sessmem_fail,		uint64_t, 1, 'a', "Session mem alloc failed")
+VSC_F(c_sessmem_limit,		uint64_t, 1, 'a', "Session mem alloc limited")
 
 VSC_F(n_sess_mem,		uint64_t, 0, 'i', "N struct sess_mem")
 VSC_F(n_sess,			uint64_t, 0, 'i', "N struct sess")



From geoff at varnish-cache.org  Mon Jan  9 20:52:18 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:18 +0100
Subject: [experimental-ims] 5a385b5 Add a necessary #include
Message-ID: <E1RkMCY-0007fZ-4L@project.varnish-software.com>

commit 5a385b581d41002e7bdd19776f262bd35b222c41
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Oct 12 08:44:46 2011 +0000

    Add a necessary #include

diff --git a/lib/libvarnish/vmb.c b/lib/libvarnish/vmb.c
index 3f5c12d..8f130f1 100644
--- a/lib/libvarnish/vmb.c
+++ b/lib/libvarnish/vmb.c
@@ -28,6 +28,7 @@
 
 #include "config.h"
 
+#include <errno.h>
 #include <pthread.h>
 #include <unistd.h>
 



From geoff at varnish-cache.org  Mon Jan  9 20:52:47 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:47 +0100
Subject: [experimental-ims] e4c002c Make VDI_RecycleFd() take a pointer to the
	vbc to recycle.
Message-ID: <E1RkMD1-0000AH-VX@project.varnish-software.com>

commit e4c002c41096468083fddef3a47e6b5c49577744
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Dec 5 13:28:20 2011 +0000

    Make VDI_RecycleFd() take a pointer to the vbc to recycle.

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index 05aaf39..4cc356e 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -651,7 +651,7 @@ void VBE_UseHealth(const struct director *vdi);
 struct vbc *VDI_GetFd(const struct director *, struct sess *sp);
 int VDI_Healthy(const struct director *, const struct sess *sp);
 void VDI_CloseFd(struct worker *wrk, struct vbc **vbp);
-void VDI_RecycleFd(struct worker *wrk);
+void VDI_RecycleFd(struct worker *wrk, struct vbc **vbp);
 void VDI_AddHostHeader(const struct sess *sp);
 void VBE_Poll(void);
 
diff --git a/bin/varnishd/cache/cache_dir.c b/bin/varnishd/cache/cache_dir.c
index 434277e..5902ce8 100644
--- a/bin/varnishd/cache/cache_dir.c
+++ b/bin/varnishd/cache/cache_dir.c
@@ -70,17 +70,22 @@ VDI_CloseFd(struct worker *wrk, struct vbc **vbp)
 /* Recycle a connection ----------------------------------------------*/
 
 void
-VDI_RecycleFd(struct worker *wrk)
+VDI_RecycleFd(struct worker *wrk, struct vbc **vbp)
 {
 	struct backend *bp;
+	struct vbc *vc;
 
-	CHECK_OBJ_NOTNULL(wrk->vbc, VBC_MAGIC);
-	CHECK_OBJ_NOTNULL(wrk->vbc->backend, BACKEND_MAGIC);
-	assert(wrk->vbc->fd >= 0);
+	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
+	AN(vbp);
+	vc = *vbp;
+	*vbp = NULL;
+	CHECK_OBJ_NOTNULL(vc, VBC_MAGIC);
+	CHECK_OBJ_NOTNULL(vc->backend, BACKEND_MAGIC);
+	assert(vc->fd >= 0);
 
-	bp = wrk->vbc->backend;
+	bp = vc->backend;
 
-	WSL(wrk, SLT_BackendReuse, wrk->vbc->vsl_id, "%s", bp->display_name);
+	WSL(wrk, SLT_BackendReuse, vc->vsl_id, "%s", bp->display_name);
 	/*
 	 * Flush the shmlog, so that another session reusing this backend
 	 * will log chronologically later than our use of it.
@@ -88,8 +93,7 @@ VDI_RecycleFd(struct worker *wrk)
 	WSL_Flush(wrk, 0);
 	Lck_Lock(&bp->mtx);
 	VSC_C_main->backend_recycle++;
-	VTAILQ_INSERT_HEAD(&bp->connlist, wrk->vbc, list);
-	wrk->vbc = NULL;
+	VTAILQ_INSERT_HEAD(&bp->connlist, vc, list);
 	VBE_DropRefLocked(bp);
 }
 
diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c
index 7d8763f..d321d7a 100644
--- a/bin/varnishd/cache/cache_fetch.c
+++ b/bin/varnishd/cache/cache_fetch.c
@@ -617,7 +617,7 @@ FetchBody(struct worker *w, struct object *obj)
 	if (cls)
 		VDI_CloseFd(w, &w->vbc);
 	else
-		VDI_RecycleFd(w);
+		VDI_RecycleFd(w, &w->vbc);
 
 	return (0);
 }



From geoff at varnish-cache.org  Mon Jan  9 20:51:55 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:51:55 +0100
Subject: [experimental-ims] c105995 Move waiter related code from
	cache_acceptor.c to its own file.
Message-ID: <E1RkMCB-0006y6-BC@project.varnish-software.com>

commit c10599526e8f43d9c999a470d530f643752d2564
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sat Sep 17 14:08:17 2011 +0000

    Move waiter related code from cache_acceptor.c to its own file.
    
    (Sort Makefile.am::SOURCES which had become unsorted)

diff --git a/bin/varnishd/Makefile.am b/bin/varnishd/Makefile.am
index 288f1f1..e4ff980 100644
--- a/bin/varnishd/Makefile.am
+++ b/bin/varnishd/Makefile.am
@@ -11,10 +11,6 @@ dist_man_MANS = varnishd.1
 
 varnishd_SOURCES = \
 	cache_acceptor.c \
-	cache_waiter_epoll.c \
-	cache_waiter_kqueue.c \
-	cache_waiter_poll.c \
-	cache_waiter_ports.c \
 	cache_backend.c \
 	cache_backend_cfg.c \
 	cache_backend_poll.c \
@@ -22,20 +18,20 @@ varnishd_SOURCES = \
 	cache_center.c \
 	cache_cli.c \
 	cache_dir.c \
-	cache_dir_random.c \
 	cache_dir_dns.c \
+	cache_dir_random.c \
 	cache_dir_round_robin.c \
+	cache_esi_deliver.c \
 	cache_esi_fetch.c \
 	cache_esi_parse.c \
-	cache_esi_deliver.c \
 	cache_expire.c \
 	cache_fetch.c \
 	cache_gzip.c \
 	cache_hash.c \
 	cache_http.c \
 	cache_httpconn.c \
-	cache_main.c \
 	cache_lck.c \
+	cache_main.c \
 	cache_panic.c \
 	cache_pipe.c \
 	cache_pool.c \
@@ -48,6 +44,11 @@ varnishd_SOURCES = \
 	cache_vrt_re.c \
 	cache_vrt_var.c \
 	cache_vrt_vmod.c \
+	cache_waiter.c \
+	cache_waiter_epoll.c \
+	cache_waiter_kqueue.c \
+	cache_waiter_poll.c \
+	cache_waiter_ports.c \
 	cache_wrk.c \
 	cache_wrw.c \
 	cache_ws.c \
@@ -63,6 +64,7 @@ varnishd_SOURCES = \
 	mgt_vcc.c \
 	rfc2616.c \
 	stevedore.c \
+	stevedore_utils.c \
 	storage_file.c \
 	storage_malloc.c \
 	storage_persistent.c \
@@ -71,7 +73,6 @@ varnishd_SOURCES = \
 	storage_persistent_subr.c \
 	storage_synth.c \
 	storage_umem.c \
-	stevedore_utils.c \
 	varnishd.c \
 	vsm.c
 
diff --git a/bin/varnishd/cache.h b/bin/varnishd/cache.h
index 2f43ff8..5d44092 100644
--- a/bin/varnishd/cache.h
+++ b/bin/varnishd/cache.h
@@ -649,7 +649,6 @@ void vca_return_session(struct sess *sp);
 void VCA_Prep(struct sess *sp);
 void VCA_Init(void);
 void VCA_Shutdown(void);
-const char *VCA_waiter_name(void);
 extern pthread_t VCA_thread;
 
 /* cache_backend.c */
diff --git a/bin/varnishd/cache_acceptor.c b/bin/varnishd/cache_acceptor.c
index 46f955b..3781f47 100644
--- a/bin/varnishd/cache_acceptor.c
+++ b/bin/varnishd/cache_acceptor.c
@@ -30,14 +30,10 @@
 
 #include "config.h"
 
-#include <stdio.h>
 #include <errno.h>
 #include <poll.h>
-#include <string.h>
 #include <stdlib.h>
-#include <unistd.h>
 
-#include <sys/uio.h>
 #include <sys/types.h>
 #include <sys/socket.h>
 
@@ -46,22 +42,6 @@
 #include "cache.h"
 #include "cache_waiter.h"
 
-static struct waiter * const vca_waiters[] = {
-#if defined(HAVE_KQUEUE)
-	&waiter_kqueue,
-#endif
-#if defined(HAVE_EPOLL_CTL)
-	&waiter_epoll,
-#endif
-#if defined(HAVE_PORT_CREATE)
-	&waiter_ports,
-#endif
-	&waiter_poll,
-	NULL,
-};
-
-static struct waiter const *vca_act;
-
 static void *waiter_priv;
 
 pthread_t		VCA_thread;
@@ -69,21 +49,6 @@ static struct timeval	tv_sndtimeo;
 static struct timeval	tv_rcvtimeo;
 
 /*--------------------------------------------------------------------
- * Report waiter name to panics
- */
-
-const char *
-VCA_waiter_name(void)
-{
-
-	if (vca_act != NULL)
-		return (vca_act->name);
-	else
-		return ("no_waiter");
-}
-
-
-/*--------------------------------------------------------------------
  * We want to get out of any kind of trouble-hit TCP connections as fast
  * as absolutely possible, so we set them LINGER enabled with zero timeout,
  * so that even if there are outstanding write data on the socket, a close(2)
@@ -347,7 +312,7 @@ vca_return_session(struct sess *sp)
 	 */
 	if (VTCP_nonblocking(sp->fd))
 		SES_Close(sp, "remote closed");
-	vca_act->pass(waiter_priv, sp);
+	waiter->pass(waiter_priv, sp);
 }
 
 /*--------------------------------------------------------------------*/
@@ -360,17 +325,14 @@ ccf_start(struct cli *cli, const char * const *av, void *priv)
 	(void)av;
 	(void)priv;
 
-	if (vca_act == NULL)
-		vca_act = vca_waiters[0];
-
-	AN(vca_act);
-	AN(vca_act->name);
-	AN(vca_act->init);
-	AN(vca_act->pass);
+	AN(waiter);
+	AN(waiter->name);
+	AN(waiter->init);
+	AN(waiter->pass);
 
-	waiter_priv = vca_act->init();
+	waiter_priv = waiter->init();
 	AZ(pthread_create(&VCA_thread, NULL, vca_acct, NULL));
-	VSL(SLT_Debug, 0, "Acceptor is %s", vca_act->name);
+	VSL(SLT_Debug, 0, "Acceptor is %s", waiter->name);
 }
 
 /*--------------------------------------------------------------------*/
@@ -424,37 +386,3 @@ VCA_Shutdown(void)
 		(void)close(i);
 	}
 }
-
-void
-VCA_tweak_waiter(struct cli *cli, const char *arg)
-{
-	int i;
-
-	 ASSERT_MGT();
-
-	if (arg == NULL) {
-		if (vca_act == NULL)
-			VCLI_Out(cli, "default");
-		else
-			VCLI_Out(cli, "%s", vca_act->name);
-
-		VCLI_Out(cli, " (");
-		for (i = 0; vca_waiters[i] != NULL; i++)
-			VCLI_Out(cli, "%s%s", i == 0 ? "" : ", ",
-			    vca_waiters[i]->name);
-		VCLI_Out(cli, ")");
-		return;
-	}
-	if (!strcmp(arg, "default")) {
-		vca_act = NULL;
-		return;
-	}
-	for (i = 0; vca_waiters[i]; i++) {
-		if (!strcmp(arg, vca_waiters[i]->name)) {
-			vca_act = vca_waiters[i];
-			return;
-		}
-	}
-	VCLI_Out(cli, "Unknown waiter");
-	VCLI_SetResult(cli, CLIS_PARAM);
-}
diff --git a/bin/varnishd/cache_main.c b/bin/varnishd/cache_main.c
index 4312393..3489530 100644
--- a/bin/varnishd/cache_main.c
+++ b/bin/varnishd/cache_main.c
@@ -37,6 +37,7 @@
 #include "cache.h"
 #include "stevedore.h"
 #include "hash_slinger.h"
+#include "cache_waiter.h"
 
 /*--------------------------------------------------------------------
  * Per thread storage for the session currently being processed by
@@ -103,6 +104,7 @@ child_main(void)
 
 	LCK_Init();	/* Second, locking */
 
+	WAIT_Init();
 	PAN_Init();
 	CLI_Init();
 	Fetch_Init();
diff --git a/bin/varnishd/cache_panic.c b/bin/varnishd/cache_panic.c
index 2f0921d..b243832 100644
--- a/bin/varnishd/cache_panic.c
+++ b/bin/varnishd/cache_panic.c
@@ -44,6 +44,7 @@
 #include "cache.h"
 #include "vsm.h"
 #include "cache_backend.h"
+#include "cache_waiter.h"
 #include "vcl.h"
 #include "libvcl.h"
 
@@ -341,7 +342,7 @@ pan_ic(const char *func, const char *file, int line, const char *cond,
 		VSB_printf(vsp, "thread = (%s)\n", q);
 
 	VSB_printf(vsp, "ident = %s,%s\n",
-	    VSB_data(vident) + 1, VCA_waiter_name());
+	    VSB_data(vident) + 1, WAIT_GetName());
 
 	pan_backtrace();
 
diff --git a/bin/varnishd/cache_response.c b/bin/varnishd/cache_response.c
index 982d8fe..eb2e62d 100644
--- a/bin/varnishd/cache_response.c
+++ b/bin/varnishd/cache_response.c
@@ -322,7 +322,7 @@ RES_WriteObj(struct sess *sp)
 	    !(sp->wrk->res_mode & RES_ESI_CHILD))
 		WRW_EndChunk(sp->wrk);
 
-	if (WRW_FlushRelease(sp->wrk))
+	if (WRW_FlushRelease(sp->wrk) && sp->fd >= 0)
 		SES_Close(sp, "remote closed");
 }
 
diff --git a/bin/varnishd/cache_waiter.c b/bin/varnishd/cache_waiter.c
new file mode 100644
index 0000000..cd3e007
--- /dev/null
+++ b/bin/varnishd/cache_waiter.c
@@ -0,0 +1,109 @@
+/*-
+ * Copyright (c) 2006 Verdens Gang AS
+ * Copyright (c) 2006-2011 Varnish Software AS
+ * All rights reserved.
+ *
+ * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ */
+
+#include "config.h"
+
+#include "vcli.h"
+#include "cli_priv.h"
+#include "cache.h"
+#include "cache_waiter.h"
+
+static const struct waiter * const vca_waiters[] = {
+    #if defined(HAVE_KQUEUE)
+	&waiter_kqueue,
+    #endif
+    #if defined(HAVE_EPOLL_CTL)
+	&waiter_epoll,
+    #endif
+    #if defined(HAVE_PORT_CREATE)
+	&waiter_ports,
+    #endif
+	&waiter_poll,
+	NULL,
+};
+
+struct waiter const * waiter;
+
+const char *
+WAIT_GetName(void)
+{
+
+	if (waiter != NULL)
+		return (waiter->name);
+	else
+		return ("no_waiter");
+}
+
+void
+WAIT_tweak_waiter(struct cli *cli, const char *arg)
+{
+	int i;
+
+	ASSERT_MGT();
+
+	if (arg == NULL) {
+		if (waiter == NULL)
+			VCLI_Out(cli, "default");
+		else
+			VCLI_Out(cli, "%s", waiter->name);
+
+		VCLI_Out(cli, " (");
+		for (i = 0; vca_waiters[i] != NULL; i++)
+			VCLI_Out(cli, "%s%s", i == 0 ? "" : ", ",
+			    vca_waiters[i]->name);
+		VCLI_Out(cli, ")");
+		return;
+	}
+	if (!strcmp(arg, "default")) {
+		waiter = NULL;
+		return;
+	}
+	for (i = 0; vca_waiters[i]; i++) {
+		if (!strcmp(arg, vca_waiters[i]->name)) {
+			waiter = vca_waiters[i];
+			return;
+		}
+	}
+	VCLI_Out(cli, "Unknown waiter");
+	VCLI_SetResult(cli, CLIS_PARAM);
+}
+
+void
+WAIT_Init(void)
+{
+
+	if (waiter == NULL)
+		waiter = vca_waiters[0];
+
+	AN(waiter);
+	AN(waiter->name);
+	AN(waiter->init);
+	AN(waiter->pass);
+}
diff --git a/bin/varnishd/cache_waiter.h b/bin/varnishd/cache_waiter.h
index 86eb2cb..4377f82 100644
--- a/bin/varnishd/cache_waiter.h
+++ b/bin/varnishd/cache_waiter.h
@@ -39,20 +39,25 @@ struct waiter {
 	waiter_pass_f		*pass;
 };
 
+extern struct waiter const * waiter;
+
 #if defined(HAVE_EPOLL_CTL)
-extern struct waiter waiter_epoll;
+extern const struct waiter waiter_epoll;
 #endif
 
 #if defined(HAVE_KQUEUE)
-extern struct waiter waiter_kqueue;
+extern const struct waiter waiter_kqueue;
 #endif
 
-extern struct waiter waiter_poll;
-
 #if defined(HAVE_PORT_CREATE)
-extern struct waiter waiter_ports;
+extern const struct waiter waiter_ports;
 #endif
 
+extern const struct waiter waiter_poll;
+
 /* cache_session.c */
 void SES_Handle(struct sess *sp, int status);
 
+const char *WAIT_GetName(void);
+void WAIT_tweak_waiter(struct cli *cli, const char *arg);
+void WAIT_Init(void);
diff --git a/bin/varnishd/cache_waiter_epoll.c b/bin/varnishd/cache_waiter_epoll.c
index acb8e13..28776cd 100644
--- a/bin/varnishd/cache_waiter_epoll.c
+++ b/bin/varnishd/cache_waiter_epoll.c
@@ -269,7 +269,7 @@ vwe_init(void)
 
 /*--------------------------------------------------------------------*/
 
-struct waiter waiter_epoll = {
+const struct waiter waiter_epoll = {
 	.name =		"epoll",
 	.init =		vwe_init,
 	.pass =		vwe_pass,
diff --git a/bin/varnishd/cache_waiter_kqueue.c b/bin/varnishd/cache_waiter_kqueue.c
index 7e6b10f..7decfea 100644
--- a/bin/varnishd/cache_waiter_kqueue.c
+++ b/bin/varnishd/cache_waiter_kqueue.c
@@ -237,7 +237,7 @@ vwk_init(void)
 
 /*--------------------------------------------------------------------*/
 
-struct waiter waiter_kqueue = {
+const struct waiter waiter_kqueue = {
 	.name =		"kqueue",
 	.init =		vwk_init,
 	.pass =		vwk_pass,
diff --git a/bin/varnishd/cache_waiter_poll.c b/bin/varnishd/cache_waiter_poll.c
index f44daa9..3a211bd 100644
--- a/bin/varnishd/cache_waiter_poll.c
+++ b/bin/varnishd/cache_waiter_poll.c
@@ -227,7 +227,7 @@ vwp_poll_init(void)
 
 /*--------------------------------------------------------------------*/
 
-struct waiter waiter_poll = {
+const struct waiter waiter_poll = {
 	.name =		"poll",
 	.init =		vwp_poll_init,
 	.pass =		vwp_poll_pass,
diff --git a/bin/varnishd/cache_waiter_ports.c b/bin/varnishd/cache_waiter_ports.c
index 329ee23..695e9c0 100644
--- a/bin/varnishd/cache_waiter_ports.c
+++ b/bin/varnishd/cache_waiter_ports.c
@@ -275,7 +275,7 @@ vws_init(void)
 
 /*--------------------------------------------------------------------*/
 
-struct waiter waiter_ports = {
+const struct waiter waiter_ports = {
 	.name =		"ports",
 	.init =		vws_init,
 	.pass =		vws_pass
diff --git a/bin/varnishd/common.h b/bin/varnishd/common.h
index 1a64cf3..3708b45 100644
--- a/bin/varnishd/common.h
+++ b/bin/varnishd/common.h
@@ -33,9 +33,6 @@ struct cli;
 extern pid_t mgt_pid;
 #define ASSERT_MGT() do { assert(getpid() == mgt_pid);} while (0)
 
-/* cache_acceptor.c */
-void VCA_tweak_waiter(struct cli *cli, const char *arg);
-
 /* mgt_shmem.c */
 extern struct VSC_C_main *VSC_C_main;
 
diff --git a/bin/varnishd/mgt_param.c b/bin/varnishd/mgt_param.c
index 33101bb..b9067e7 100644
--- a/bin/varnishd/mgt_param.c
+++ b/bin/varnishd/mgt_param.c
@@ -48,6 +48,7 @@
 
 #include "heritage.h"
 #include "vparam.h"
+#include "cache_waiter.h"
 
 #include "vss.h"
 
@@ -438,7 +439,7 @@ tweak_waiter(struct cli *cli, const struct parspec *par, const char *arg)
 
 	/* XXX should have tweak_generic_string */
 	(void)par;
-	VCA_tweak_waiter(cli, arg);
+	WAIT_tweak_waiter(cli, arg);
 }
 
 /*--------------------------------------------------------------------*/



From geoff at varnish-cache.org  Mon Jan  9 20:52:01 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:01 +0100
Subject: [experimental-ims] 194d664 Merge branch 'master' of
	ssh://git.varnish-cache.org/git/varnish-cache
Message-ID: <E1RkMCH-0007CG-Ie@project.varnish-software.com>

commit 194d664e2308d3caca6f465d99aef0638fa0ce36
Merge: 8a385bf 701e5c3
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu Sep 22 14:57:11 2011 +0000

    Merge branch 'master' of ssh://git.varnish-cache.org/git/varnish-cache



From geoff at varnish-cache.org  Mon Jan  9 20:52:19 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:19 +0100
Subject: [experimental-ims] bebf42d Just small typofixes
Message-ID: <E1RkMCZ-0007it-DL@project.varnish-software.com>

commit bebf42d62aac3e9013c70ba5e489bcd4b6e8c232
Author: Ingvar Hagelund <ingvar at redpill-linpro.com>
Date:   Fri Oct 14 00:30:02 2011 +0200

    Just small typofixes

diff --git a/redhat/varnish.spec b/redhat/varnish.spec
index e8ea8c7..0e3f1e6 100644
--- a/redhat/varnish.spec
+++ b/redhat/varnish.spec
@@ -373,7 +373,7 @@ fi
 - Whitespace changes to make rpmlint more happy
 
 * Fri Sep 12 2008 Ingvar Hagelund <ingvar at linpro.no> - 2.0-0.9.20080912svn3184
-- Added varnisnsca init script (Colin Hill)
+- Added varnisncsa init script (Colin Hill)
 - Corrected varnishlog init script (Colin Hill)
 
 * Tue Sep 09 2008 Ingvar Hagelund <ingvar at linpro.no> - 2.0-0.8.beta1
diff --git a/redhat/varnishncsa.initrc b/redhat/varnishncsa.initrc
index ee26376..b23fa48 100644
--- a/redhat/varnishncsa.initrc
+++ b/redhat/varnishncsa.initrc
@@ -1,6 +1,6 @@
 #! /bin/sh
 #
-# varnishncsa Control the Varnish NSCA logging daemon
+# varnishncsa Control the Varnish NCSA logging daemon
 #
 # chkconfig: - 90 10
 # description: Varnish Cache logging daemon
@@ -15,7 +15,7 @@
 # Default-Start:
 # Default-Stop:
 # Short-Description: start and stop varnishncsa
-# Description: Varnish Cache NSCA logging daemon
+# Description: Varnish Cache NCSA logging daemon
 ### END INIT INFO
 
 # Source function library.



From geoff at varnish-cache.org  Mon Jan  9 20:52:26 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:26 +0100
Subject: [experimental-ims] 71fe293 Push struct session entirely out of
	FetchBody().
Message-ID: <E1RkMCg-0007vL-6u@project.varnish-software.com>

commit 71fe2939a4435a1c5765de17708ecb22b3ddd66e
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Oct 24 14:39:45 2011 +0000

    Push struct session entirely out of FetchBody().
    
    This should make it possible to get a different worker thread to pull
    the body in from the backend, without cloning the session structure,
    with all the trouble that would cause.

diff --git a/bin/varnishd/cache.h b/bin/varnishd/cache.h
index 85f5d5f..b92fd60 100644
--- a/bin/varnishd/cache.h
+++ b/bin/varnishd/cache.h
@@ -704,7 +704,7 @@ int EXP_NukeOne(struct worker *w, struct lru *lru);
 /* cache_fetch.c */
 struct storage *FetchStorage(struct worker *w, ssize_t sz);
 int FetchHdr(struct sess *sp);
-int FetchBody(const struct sess *sp, struct object *obj);
+int FetchBody(struct worker *w, struct object *obj);
 int FetchReqBody(struct sess *sp);
 void Fetch_Init(void);
 
diff --git a/bin/varnishd/cache_center.c b/bin/varnishd/cache_center.c
index 13d8083..418fbe9 100644
--- a/bin/varnishd/cache_center.c
+++ b/bin/varnishd/cache_center.c
@@ -846,7 +846,7 @@ cnt_fetchbody(struct sess *sp)
 	}
 
 	/* Use unmodified headers*/
-	i = FetchBody(sp, sp->obj);
+	i = FetchBody(sp->wrk, sp->obj);
 
 	sp->wrk->h_content_length = NULL;
 
@@ -910,7 +910,7 @@ cnt_streambody(struct sess *sp)
 
 	AssertObjCorePassOrBusy(sp->obj->objcore);
 
-	i = FetchBody(sp, sp->obj);
+	i = FetchBody(sp->wrk, sp->obj);
 
 	sp->wrk->h_content_length = NULL;
 
diff --git a/bin/varnishd/cache_fetch.c b/bin/varnishd/cache_fetch.c
index 1769d3a..acef94b 100644
--- a/bin/varnishd/cache_fetch.c
+++ b/bin/varnishd/cache_fetch.c
@@ -198,24 +198,24 @@ fetch_number(const char *nbr, int radix)
 /*--------------------------------------------------------------------*/
 
 static int
-fetch_straight(const struct sess *sp, struct http_conn *htc, const char *b)
+fetch_straight(struct worker *w, struct http_conn *htc, const char *b)
 {
 	int i;
 	ssize_t cl;
 
-	assert(sp->wrk->body_status == BS_LENGTH);
+	assert(w->body_status == BS_LENGTH);
 
 	cl = fetch_number(b, 10);
-	sp->wrk->vfp->begin(sp->wrk, cl > 0 ? cl : 0);
+	w->vfp->begin(w, cl > 0 ? cl : 0);
 	if (cl < 0) {
-		WSP(sp, SLT_FetchError, "straight length field bogus");
+		WSLB(w, SLT_FetchError, "straight length field bogus");
 		return (-1);
 	} else if (cl == 0)
 		return (0);
 
-	i = sp->wrk->vfp->bytes(sp->wrk, htc, cl);
+	i = w->vfp->bytes(w, htc, cl);
 	if (i <= 0) {
-		WSP(sp, SLT_FetchError, "straight read_error: %d %d (%s)",
+		WSLB(w, SLT_FetchError, "straight read_error: %d %d (%s)",
 		    i, errno, htc->error);
 		return (-1);
 	}
@@ -227,7 +227,7 @@ fetch_straight(const struct sess *sp, struct http_conn *htc, const char *b)
 
 #define CERR() do {						\
 		if (i != 1) {					\
-			WSP(sp, SLT_FetchError,			\
+			WSLB(w, SLT_FetchError,			\
 			    "chunked read_error: %d (%s)",	\
 			    errno, htc->error);			\
 			return (-1);				\
@@ -235,15 +235,15 @@ fetch_straight(const struct sess *sp, struct http_conn *htc, const char *b)
 	} while (0)
 
 static int
-fetch_chunked(const struct sess *sp, struct http_conn *htc)
+fetch_chunked(struct worker *w, struct http_conn *htc)
 {
 	int i;
 	char buf[20];		/* XXX: 20 is arbitrary */
 	unsigned u;
 	ssize_t cl;
 
-	sp->wrk->vfp->begin(sp->wrk, 0);
-	assert(sp->wrk->body_status == BS_CHUNKED);
+	w->vfp->begin(w, 0);
+	assert(w->body_status == BS_CHUNKED);
 	do {
 		/* Skip leading whitespace */
 		do {
@@ -262,7 +262,7 @@ fetch_chunked(const struct sess *sp, struct http_conn *htc)
 		}
 
 		if (u >= sizeof buf) {
-			WSP(sp, SLT_FetchError,	"chunked header too long");
+			WSLB(w, SLT_FetchError,	"chunked header too long");
 			return (-1);
 		}
 
@@ -273,17 +273,17 @@ fetch_chunked(const struct sess *sp, struct http_conn *htc)
 		}
 
 		if (buf[u] != '\n') {
-			WSP(sp, SLT_FetchError,	"chunked header char syntax");
+			WSLB(w, SLT_FetchError,	"chunked header char syntax");
 			return (-1);
 		}
 		buf[u] = '\0';
 
 		cl = fetch_number(buf, 16);
 		if (cl < 0) {
-			WSP(sp, SLT_FetchError,	"chunked header nbr syntax");
+			WSLB(w, SLT_FetchError,	"chunked header nbr syntax");
 			return (-1);
 		} else if (cl > 0) {
-			i = sp->wrk->vfp->bytes(sp->wrk, htc, cl);
+			i = w->vfp->bytes(w, htc, cl);
 			CERR();
 		}
 		i = HTC_Read(htc, buf, 1);
@@ -293,7 +293,7 @@ fetch_chunked(const struct sess *sp, struct http_conn *htc)
 			CERR();
 		}
 		if (buf[0] != '\n') {
-			WSP(sp, SLT_FetchError,	"chunked tail syntax");
+			WSLB(w, SLT_FetchError,	"chunked tail syntax");
 			return (-1);
 		}
 	} while (cl > 0);
@@ -305,15 +305,15 @@ fetch_chunked(const struct sess *sp, struct http_conn *htc)
 /*--------------------------------------------------------------------*/
 
 static int
-fetch_eof(const struct sess *sp, struct http_conn *htc)
+fetch_eof(struct worker *w, struct http_conn *htc)
 {
 	int i;
 
-	assert(sp->wrk->body_status == BS_EOF);
-	sp->wrk->vfp->begin(sp->wrk, 0);
-	i = sp->wrk->vfp->bytes(sp->wrk, htc, SSIZE_MAX);
+	assert(w->body_status == BS_EOF);
+	w->vfp->begin(w, 0);
+	i = w->vfp->bytes(w, htc, SSIZE_MAX);
 	if (i < 0) {
-		WSP(sp, SLT_FetchError, "eof read_error: %d (%s)",
+		WSLB(w, SLT_FetchError, "eof read_error: %d (%s)",
 		    errno, htc->error);
 		return (-1);
 	}
@@ -480,24 +480,21 @@ FetchHdr(struct sess *sp)
 /*--------------------------------------------------------------------*/
 
 int
-FetchBody(const struct sess *sp, struct object *obj)
+FetchBody(struct worker *w, struct object *obj)
 {
 	int cls;
 	struct storage *st;
-	struct worker *w;
 	int mklen;
 
-	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
-	CHECK_OBJ_NOTNULL(sp->wrk, WORKER_MAGIC);
-	w = sp->wrk;
+	CHECK_OBJ_NOTNULL(w, WORKER_MAGIC);
 	AZ(w->fetch_obj);
+	CHECK_OBJ_NOTNULL(w->vbc, VBC_MAGIC);
 	CHECK_OBJ_NOTNULL(obj, OBJECT_MAGIC);
 	CHECK_OBJ_NOTNULL(obj->http, HTTP_MAGIC);
 
 	if (w->vfp == NULL)
 		w->vfp = &vfp_nop;
 
-	AN(sp->director);
 	AssertObjCorePassOrBusy(obj->objcore);
 
 	AZ(w->vgz_rx);
@@ -515,20 +512,20 @@ FetchBody(const struct sess *sp, struct object *obj)
 		mklen = 1;
 		break;
 	case BS_LENGTH:
-		cls = fetch_straight(sp, w->htc,
+		cls = fetch_straight(w, w->htc,
 		    w->h_content_length);
 		mklen = 1;
-		XXXAZ(w->vfp->end(sp->wrk));
+		XXXAZ(w->vfp->end(w));
 		break;
 	case BS_CHUNKED:
-		cls = fetch_chunked(sp, w->htc);
+		cls = fetch_chunked(w, w->htc);
 		mklen = 1;
-		XXXAZ(w->vfp->end(sp->wrk));
+		XXXAZ(w->vfp->end(w));
 		break;
 	case BS_EOF:
-		cls = fetch_eof(sp, w->htc);
+		cls = fetch_eof(w, w->htc);
 		mklen = 1;
-		XXXAZ(w->vfp->end(sp->wrk));
+		XXXAZ(w->vfp->end(w));
 		break;
 	case BS_ERROR:
 		cls = 1;
@@ -546,11 +543,11 @@ FetchBody(const struct sess *sp, struct object *obj)
 	 * sitting on w->storage, we will always call vfp_nop_end()
 	 * to get it trimmed or thrown out if empty.
 	 */
-	AZ(vfp_nop_end(sp->wrk));
+	AZ(vfp_nop_end(w));
 
 	w->fetch_obj = NULL;
 
-	WSL(w, SLT_Fetch_Body, w->vbc->vsl_id, "%u(%s) cls %d mklen %u",
+	WSLB(w, SLT_Fetch_Body, "%u(%s) cls %d mklen %u",
 	    w->body_status, body_status(w->body_status),
 	    cls, mklen);
 
@@ -575,7 +572,7 @@ FetchBody(const struct sess *sp, struct object *obj)
 	if (cls == 0 && w->do_close)
 		cls = 1;
 
-	WSL(w, SLT_Length, w->vbc->vsl_id, "%u", obj->len);
+	WSLB(w, SLT_Length, "%u", obj->len);
 
 	{
 	/* Sanity check fetch methods accounting */
@@ -584,7 +581,7 @@ FetchBody(const struct sess *sp, struct object *obj)
 		uu = 0;
 		VTAILQ_FOREACH(st, &obj->store, list)
 			uu += st->len;
-		if (sp->objcore == NULL || (sp->objcore->flags & OC_F_PASS))
+		if (w->do_stream)
 			/* Streaming might have started freeing stuff */
 			assert (uu <= obj->len);
 		else
@@ -593,7 +590,7 @@ FetchBody(const struct sess *sp, struct object *obj)
 
 	if (mklen > 0) {
 		http_Unset(obj->http, H_Content_Length);
-		http_PrintfHeader(w, sp->vsl_id, obj->http,
+		http_PrintfHeader(w, w->vbc->vsl_id, obj->http,
 		    "Content-Length: %jd", (intmax_t)obj->len);
 	}
 



From geoff at varnish-cache.org  Mon Jan  9 20:51:56 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:51:56 +0100
Subject: [experimental-ims] 4f610d3 I've spent some time trying to think more
	clearly about stats counters and it's all a mess.
Message-ID: <E1RkMCC-00070M-7A@project.varnish-software.com>

commit 4f610d3a28939cc2bbc74d4c7100a6613a3d9555
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sun Sep 18 07:58:27 2011 +0000

    I've spent some time trying to think more clearly about stats counters
    and it's all a mess.
    
    Try to get it right, and document in the head-comment what I mean with
    "right".
    
    Goal:  Cleaned up for 3.1

diff --git a/bin/varnishd/cache_session.c b/bin/varnishd/cache_session.c
index 4c01f34..80e8da5 100644
--- a/bin/varnishd/cache_session.c
+++ b/bin/varnishd/cache_session.c
@@ -110,7 +110,7 @@ ses_sm_alloc(void)
 
 	hl = HTTP_estimate(nhttp);
 	l = sizeof *sm + nws + 2 * hl;
-	VSC_C_main->g_sessmem_size = l;
+	VSC_C_main->sessmem_size = l;
 	p = malloc(l);
 	if (p == NULL)
 		return (NULL);
@@ -190,20 +190,20 @@ SES_New(struct worker *wrk, struct sesspool *pp)
 		pp->nsess++;
 		do_alloc = 1;
 	}
-	wrk->stats.c_sessmem_free += pp->dly_free_cnt;
+	wrk->stats.sessmem_free += pp->dly_free_cnt;
 	pp->dly_free_cnt = 0;
 	Lck_Unlock(&pp->mtx);
 	if (do_alloc) {
 		sm = ses_sm_alloc();
 		if (sm != NULL) {
-			wrk->stats.c_sessmem_alloc++;
+			wrk->stats.sessmem_alloc++;
 			sm->pool = pp;
 			ses_setup(sm);
 		} else {
-			wrk->stats.c_sessmem_fail++;
+			wrk->stats.sessmem_fail++;
 		}
 	} else if (sm == NULL) {
-		wrk->stats.c_sessmem_limit++;
+		wrk->stats.sessmem_limit++;
 	}
 	if (sm == NULL)
 		return (NULL);
@@ -325,7 +325,7 @@ SES_Delete(struct sess *sp, const char *reason)
 		free(sm);
 		Lck_Lock(&pp->mtx);
 		if (wrk != NULL)
-			wrk->stats.c_sessmem_free++;
+			wrk->stats.sessmem_free++;
 		else
 			pp->dly_free_cnt++;
 		pp->nsess--;
@@ -335,7 +335,7 @@ SES_Delete(struct sess *sp, const char *reason)
 		ses_setup(sm);
 		Lck_Lock(&pp->mtx);
 		if (wrk != NULL) {
-			wrk->stats.c_sessmem_free += pp->dly_free_cnt;
+			wrk->stats.sessmem_free += pp->dly_free_cnt;
 			pp->dly_free_cnt = 0;
 		}
 		VTAILQ_INSERT_HEAD(&pp->freelist, sm, list);
diff --git a/bin/varnishstat/varnishstat.c b/bin/varnishstat/varnishstat.c
index b5b1e6f..a8b1d5f 100644
--- a/bin/varnishstat/varnishstat.c
+++ b/bin/varnishstat/varnishstat.c
@@ -110,7 +110,7 @@ do_once_cb(void *priv, const struct VSC_point * const pt)
 	if (i > op->pad)
 		op->pad = i + 1;
 	printf("%*.*s", op->pad - i, op->pad - i, "");
-	if (pt->flag == 'a')
+	if (pt->flag == 'a' || pt->flag == 'c')
 		printf("%12ju %12.2f %s\n", val, val / op->up, pt->desc);
 	else
 		printf("%12ju %12s %s\n", val, ".  ", pt->desc);
diff --git a/bin/varnishstat/varnishstat_curses.c b/bin/varnishstat/varnishstat_curses.c
index d0eb001..d0db9f0 100644
--- a/bin/varnishstat/varnishstat_curses.c
+++ b/bin/varnishstat/varnishstat_curses.c
@@ -59,7 +59,7 @@ struct pt {
 	VTAILQ_ENTRY(pt)	next;
 	const volatile uint64_t	*ptr;
 	uint64_t		ref;
-	int			type;
+	int			flag;
 	char			seen;
 	char			*name;
 };
@@ -81,7 +81,7 @@ do_curses_cb(void *priv, const struct VSC_point * const sp)
 
 	pt->ptr = sp->ptr;
 	pt->ref = *pt->ptr;
-	pt->type = sp->flag;
+	pt->flag = sp->flag;
 
 	*buf = '\0';
 	if (strcmp(sp->class, "")) {
@@ -210,13 +210,13 @@ do_curses(struct VSM_data *vd, const struct VSC_C_main *VSC_C_main,
 				line++;
 				if (line >= LINES)
 					break;
-				if (pt->type == 'a') {
+				if (pt->flag == 'a' || pt->flag == 'c') {
 					AC(mvprintw(line, 0,
 					    "%12ju %12.2f %12.2f %s\n",
 					    ju, (ju - (intmax_t)pt->ref)/lt,
 					    ju / up, pt->name));
 					pt->ref = ju;
-				} else if (pt->type == 'b') {
+				} else if (pt->flag == 'b') {
 					AC(mvprintw(line, 0, "  %010.10jx <",
 					    (ju >> 24) & 0xffffffffffLL));
 					for (ch = 0x800000; ch; ch >>= 1)
diff --git a/include/vsc_fields.h b/include/vsc_fields.h
index 79ad315..6f910de 100644
--- a/include/vsc_fields.h
+++ b/include/vsc_fields.h
@@ -26,15 +26,31 @@
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
  *
- * 3rd argument marks fields for inclusion in the per worker-thread
- * stats structure.
+ * Definition of all shared memory statistics below.
+ *
+ * Fields (n, t, l, f, e, d):
+ *    n - Name:		Field name, in C-source and stats programs
+ *    t - Type:		C-type, uint64_t, unless marked in 'f'
+ *    l - Local:	Local counter in worker thread.
+ *    f - Format: 	Semantics of the value in this field
+ *				'a' - Accumulator (deprecated, use 'c')
+ *				'b' - Bitmap
+ *				'c' - Counter, never decreases.
+ *				'g' - Gauge, goes up and down
+ *				'i' - Integer (deprecated, use 'g')
+ *    e - Explantion:	Short explanation of field (for screen use)
+ *    d - Description:	Long explanation of field (for doc use)
+ *	
+ * -----------------------
+ * NB: Cleanup in progress
+ * -----------------------
+ *
+ * Insufficient attention has caused this to become a swamp of conflicting
+ * conventions, shorthands and general mumbo-jumbo.  I'm trying to clean
+ * it up as I go over the code in other business.
+ *
+ * Please see the sessmem section for how it should look.
  *
- * XXX: We need a much more consistent naming of these fields, this has
- * XXX: turned into a major mess, causing trouble already for backends.
- * XXX:
- * XXX: Please converge on:
- * XXX:		c_* counter	(total bytes ever allocated from sma, "")
- * XXX:		g_* gauge	(presently allocated bytes from sma, "")
  */
 
 /**********************************************************************/
@@ -77,27 +93,27 @@ VSC_F(fetch_304,		uint64_t, 1, 'a', "Fetch no body (304)", "")
  *    see: cache_session.c
  */
 
-VSC_F(g_sessmem_size,		uint64_t, 1, 'i',
+VSC_F(sessmem_size,		uint64_t, 1, 'g',
     "Session mem size",
 	"Bytes of memory allocated for last allocated session."
 )
 
-VSC_F(c_sessmem_alloc,		uint64_t, 1, 'a',
+VSC_F(sessmem_alloc,		uint64_t, 1, 'c',
     "Session mem allocated",
 	"Count of all allocations of session memory."
 )
 
-VSC_F(c_sessmem_free,		uint64_t, 1, 'a',
+VSC_F(sessmem_free,		uint64_t, 1, 'c',
     "Session mem freed",
 	"Count of all frees of session memory."
 )
 
-VSC_F(c_sessmem_fail,		uint64_t, 1, 'a',
+VSC_F(sessmem_fail,		uint64_t, 1, 'c',
     "Session mem alloc failed",
 	"Count of session memory allocation failures."
 )
 
-VSC_F(c_sessmem_limit,		uint64_t, 1, 'a',
+VSC_F(sessmem_limit,		uint64_t, 1, 'c',
     "Session mem alloc limited",
 	"Count of session memory allocations blocked by limit (max_sess)."
 )



From geoff at varnish-cache.org  Mon Jan  9 20:52:41 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:41 +0100
Subject: [experimental-ims] 1f3fbd8 Make VSC/varnishstat sorta-work again.
Message-ID: <E1RkMCv-0008P1-Pk@project.varnish-software.com>

commit 1f3fbd8eb67062fe211e731cccaf7251b74d2840
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Nov 23 08:57:48 2011 +0000

    Make VSC/varnishstat sorta-work again.

diff --git a/bin/varnishstat/varnishstat.c b/bin/varnishstat/varnishstat.c
index 2fe7ffb..5d9226f 100644
--- a/bin/varnishstat/varnishstat.c
+++ b/bin/varnishstat/varnishstat.c
@@ -52,7 +52,9 @@ do_xml_cb(void *priv, const struct VSC_point * const pt)
 	uint64_t val;
 
 	(void)priv;
-	assert(!strcmp(pt->fmt, "uint64_t"));
+	if (pt == NULL)
+		return (0);
+	assert(!strcmp(pt->desc->fmt, "uint64_t"));
 	val = *(const volatile uint64_t*)pt->ptr;
 
 	printf("\t<stat>\n");
@@ -60,10 +62,10 @@ do_xml_cb(void *priv, const struct VSC_point * const pt)
 		printf("\t\t<type>%s</type>\n", pt->class);
 	if (strcmp(pt->ident, ""))
 		printf("\t\t<ident>%s</ident>\n", pt->ident);
-	printf("\t\t<name>%s</name>\n", pt->name);
+	printf("\t\t<name>%s</name>\n", pt->desc->name);
 	printf("\t\t<value>%ju</value>\n", val);
-	printf("\t\t<flag>%c</flag>\n", pt->flag);
-	printf("\t\t<description>%s</description>\n", pt->desc);
+	printf("\t\t<flag>%c</flag>\n", pt->desc->flag);
+	printf("\t\t<description>%s</description>\n", pt->desc->sdesc);
 	printf("\t</stat>\n");
 	return (0);
 }
@@ -91,9 +93,11 @@ do_json_cb(void *priv, const struct VSC_point * const pt)
 	uint64_t val;
 	int *jp;
 
-	jp = priv;
+	if (pt == NULL)
+		return (0);
 
-	assert(!strcmp(pt->fmt, "uint64_t"));
+	jp = priv;
+	assert(!strcmp(pt->desc->fmt, "uint64_t"));
 	val = *(const volatile uint64_t*)pt->ptr;
 
 	if (*jp) *jp = 0; else printf(",\n");
@@ -104,15 +108,15 @@ do_json_cb(void *priv, const struct VSC_point * const pt)
 		printf("%s.", pt->class);
 	if (pt->ident[0])
 		printf("%s.", pt->ident);
-	printf("%s\": {", pt->name);
+	printf("%s\": {", pt->desc->name);
 
 	if (strcmp(pt->class, "")) printf("\"type\": \"%s\", ",  pt->class);
 	if (strcmp(pt->ident, "")) printf("\"ident\": \"%s\", ", pt->ident);
 
 	printf("\"value\": %ju, ", val);
 
-	printf("\"flag\": \"%c\", ", pt->flag);
-	printf("\"description\": \"%s\"", pt->desc);
+	printf("\"flag\": \"%c\", ", pt->desc->flag);
+	printf("\"description\": \"%s\"", pt->desc->sdesc);
 	printf("}");
 
 	if (*jp) printf("\n");
@@ -153,22 +157,24 @@ do_once_cb(void *priv, const struct VSC_point * const pt)
 	uint64_t val;
 	int i;
 
+	if (pt == NULL)
+		return (0);
 	op = priv;
-	assert(!strcmp(pt->fmt, "uint64_t"));
+	assert(!strcmp(pt->desc->fmt, "uint64_t"));
 	val = *(const volatile uint64_t*)pt->ptr;
 	i = 0;
 	if (strcmp(pt->class, ""))
 		i += printf("%s.", pt->class);
 	if (strcmp(pt->ident, ""))
 		i += printf("%s.", pt->ident);
-	i += printf("%s", pt->name);
+	i += printf("%s", pt->desc->name);
 	if (i > op->pad)
 		op->pad = i + 1;
 	printf("%*.*s", op->pad - i, op->pad - i, "");
-	if (pt->flag == 'a' || pt->flag == 'c')
-		printf("%12ju %12.2f %s\n", val, val / op->up, pt->desc);
+	if (pt->desc->flag == 'a' || pt->desc->flag == 'c')
+		printf("%12ju %12.2f %s\n", val, val / op->up, pt->desc->sdesc);
 	else
-		printf("%12ju %12s %s\n", val, ".  ", pt->desc);
+		printf("%12ju %12s %s\n", val, ".  ", pt->desc->sdesc);
 	return (0);
 }
 
@@ -197,10 +203,10 @@ do_list_cb(void *priv, const struct VSC_point * const pt)
 		i += fprintf(stderr, "%s.", pt->class);
 	if (strcmp(pt->ident, ""))
 		i += fprintf(stderr, "%s.", pt->ident);
-	i += fprintf(stderr, "%s", pt->name);
+	i += fprintf(stderr, "%s", pt->desc->name);
 	if (i < 30)
 		fprintf(stderr, "%*s", i - 30, "");
-	fprintf(stderr, " %s\n", pt->desc);
+	fprintf(stderr, " %s\n", pt->desc->sdesc);
 	return (0);
 }
 
@@ -253,7 +259,6 @@ main(int argc, char * const *argv)
 	int delay = 1, once = 0, xml = 0, json = 0, do_repeat = 0;
 
 	vd = VSM_New();
-	VSC_Setup(vd);
 
 	while ((c = getopt(argc, argv, VSC_ARGS "1f:lVw:xjt:")) != -1) {
 		switch (c) {
@@ -261,8 +266,6 @@ main(int argc, char * const *argv)
 			once = 1;
 			break;
 		case 'l':
-			if (VSC_Open(vd, 1))
-				exit(1);
 			list_fields(vd);
 			exit(0);
 		case 'V':
@@ -285,9 +288,6 @@ main(int argc, char * const *argv)
 		}
 	}
 
-	if (VSC_Open(vd, 1))
-		exit(1);
-
 	VSC_C_main = VSC_Main(vd);
 	AN(VSC_C_main);
 
diff --git a/bin/varnishstat/varnishstat_curses.c b/bin/varnishstat/varnishstat_curses.c
index 9fa6b82..d4e714c 100644
--- a/bin/varnishstat/varnishstat_curses.c
+++ b/bin/varnishstat/varnishstat_curses.c
@@ -81,7 +81,9 @@ do_curses_cb(void *priv, const struct VSC_point * const sp)
 	char buf[128];
 
 	(void)priv;
-	assert(!strcmp(sp->fmt, "uint64_t"));
+	if (sp == NULL)
+		return (0);
+	assert(!strcmp(sp->desc->fmt, "uint64_t"));
 
 	pt = calloc(sizeof *pt, 1);
 	AN(pt);
@@ -89,7 +91,7 @@ do_curses_cb(void *priv, const struct VSC_point * const sp)
 
 	pt->ptr = sp->ptr;
 	pt->ref = *pt->ptr;
-	pt->flag = sp->flag;
+	pt->flag = sp->desc->flag;
 
 	*buf = '\0';
 	if (strcmp(sp->class, "")) {
@@ -100,9 +102,9 @@ do_curses_cb(void *priv, const struct VSC_point * const sp)
 		strcat(buf, sp->ident);
 		strcat(buf, ".");
 	}
-	strcat(buf, sp->name);
+	strcat(buf, sp->desc->name);
 	strcat(buf, " - ");
-	strcat(buf, sp->desc);
+	strcat(buf, sp->desc->sdesc);
 	pt->name = strdup(buf);
 	AN(pt->name);
 	return (0);
@@ -144,7 +146,6 @@ do_curses(struct VSM_data *vd, const struct VSC_C_main *VSC_C_main,
 	int ch, line;
 	struct pt *pt;
 	double act, lact;
-	unsigned seq;
 
 	(void)initscr();
 	AC(raw());
@@ -157,7 +158,6 @@ do_curses(struct VSM_data *vd, const struct VSC_C_main *VSC_C_main,
 		/*
 		 * Initialization goes in outher loop
 		 */
-		seq = VSM_Seq(vd);
 		prep_pts(vd);
 		AC(erase());
 		AC(refresh());
@@ -170,15 +170,11 @@ do_curses(struct VSM_data *vd, const struct VSC_C_main *VSC_C_main,
 		lact = 0;
 
 		while (1) {
-			if (seq != VSM_Seq(vd))
-				break;
 			/*
 			 * Break to outher loop if we need to re-read file.
 			 * Only check if it looks like nothing is happening.
 			 */
 			act = VSC_C_main->cache_hit + VSC_C_main->cache_miss + 1;
-			if (act == lact && VSM_ReOpen(vd, 1))
-				break;
 			lact = act;
 
 			AZ(gettimeofday(&tv, NULL));
diff --git a/include/vapi/vsc.h b/include/vapi/vsc.h
index 3cf16c6..6ebab31 100644
--- a/include/vapi/vsc.h
+++ b/include/vapi/vsc.h
@@ -26,6 +26,10 @@
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
  *
+ * This is the public API for the VSC access.
+ *
+ * VSC is a "subclass" of VSM.
+ *
  */
 
 #ifndef VAPI_VSC_H_INCLUDED
@@ -34,17 +38,12 @@
 #include "vapi/vsc_int.h"
 
 struct VSM_data;
+struct VSM_fantom;
 
 /*---------------------------------------------------------------------
  * VSC level access functions
  */
 
-void VSC_Setup(struct VSM_data *vd);
-	/*
-	 * Setup vd for use with VSC functions.
-	 * Must be called once before any other VSC function is called
-	 */
-
 #define VSC_ARGS	"f:n:"
 #define VSC_n_USAGE	VSM_n_USAGE
 #define VSC_f_USAGE	"[-f field_name,...]"
@@ -55,39 +54,58 @@ int VSC_Arg(struct VSM_data *vd, int arg, const char *opt);
 	/*
 	 * Handle standard stat-presenter arguments
 	 * Return:
-	 *	-1 error
+	 *	-1 error, VSM_Error() returns diagnostic string
 	 *	 0 not handled
 	 *	 1 Handled.
 	 */
 
-int VSC_Open(struct VSM_data *vd, int diag);
-	/*
-	 * Open shared memory for VSC processing.
-	 * args and returns as VSM_Open()
-	 */
-
-struct VSC_C_main *VSC_Main(const struct VSM_data *vd);
+struct VSC_C_main *VSC_Main(struct VSM_data *vd);
 	/*
 	 * return Main stats structure
 	 * returns NULL until child has been started.
 	 */
 
+struct VSC_desc {
+	const char *name;		/* field name			*/
+	const char *fmt;		/* field format ("uint64_t")	*/
+	int flag;			/* 'c' = counter, 'g' = gauge	*/
+	const char *sdesc;		/* short description		*/
+	const char *ldesc;		/* long description		*/
+};
+
 struct VSC_point {
 	const char *class;		/* stat struct type		*/
 	const char *ident;		/* stat struct ident		*/
-	const char *name;		/* field name			*/
-	const char *fmt;		/* field format ("uint64_t")	*/
-	int flag;			/* 'a' = counter, 'i' = gauge	*/
-	const char *desc;		/* description			*/
+	const struct VSC_desc *desc;	/* point description		*/
 	const volatile void *ptr;	/* field value			*/
+	struct VSM_fantom *vf;
 };
 
 typedef int VSC_iter_f(void *priv, const struct VSC_point *const pt);
 
-int VSC_Iter(const struct VSM_data *vd, VSC_iter_f *func, void *priv);
+int VSC_Iter(struct VSM_data *vd, VSC_iter_f *func, void *priv);
 	/*
 	 * Iterate over all statistics counters, calling "func" for
 	 * each counter not suppressed by any "-f" arguments.
+	 *
+ 	 * Func is called with pt == NULL, whenever VSM allocations
+	 * change (child restart, allocations/deallocations)
+	 *
+	 * Returns:
+	 * 	!=0:	func returned non-zero
+	 *	-1:	No VSC's available
+	 *	0: 	Done
 	 */
 
+/**********************************************************************
+ * Precompiled VSC_desc's for all know VSCs.
+ */
+#define VSC_F(n,t,l,f,d,e)
+#define VSC_DO(U,l,t) extern const struct VSC_desc VSC_desc_##l[];
+#define VSC_DONE(U,l,t)
+#include "tbl/vsc_all.h"
+#undef VSC_F
+#undef VSC_DO
+#undef VSC_DONE
+
 #endif /* VAPI_VSC_H_INCLUDED */
diff --git a/lib/libvarnishapi/vsc.c b/lib/libvarnishapi/vsc.c
index c9facfd..54bf20f 100644
--- a/lib/libvarnishapi/vsc.c
+++ b/lib/libvarnishapi/vsc.c
@@ -47,15 +47,22 @@
 #include "vqueue.h"
 #include "vsm_api.h"
 
+struct vsc_pt {
+	unsigned		magic;
+#define VSC_PT_MAGIC		0xa4ff159a
+	struct VSC_point	point;
+	VTAILQ_ENTRY(vsc_pt)	list;
+};
+
 struct vsc_sf {
 	unsigned		magic;
-#define VSL_SF_MAGIC		0x558478dd
-	VTAILQ_ENTRY(vsc_sf)	next;
+#define VSC_SF_MAGIC		0x558478dd
+	VTAILQ_ENTRY(vsc_sf)	list;
 	int			flags;
-#define VSL_SF_EXCL		(1 << 0)
-#define VSL_SF_CL_WC		(1 << 1)
-#define VSL_SF_ID_WC		(1 << 2)
-#define VSL_SF_NM_WC		(1 << 3)
+#define VSC_SF_EXCL		(1 << 0)
+#define VSC_SF_CL_WC		(1 << 1)
+#define VSC_SF_ID_WC		(1 << 2)
+#define VSC_SF_NM_WC		(1 << 3)
 	char			*class;
 	char			*ident;
 	char			*name;
@@ -65,40 +72,56 @@ struct vsc {
 	unsigned		magic;
 #define VSC_MAGIC		0x3373554a
 
-	int			sf_init;
+	VTAILQ_HEAD(, vsc_pt)	pt_list;
 	VTAILQ_HEAD(, vsc_sf)	sf_list;
-
+	struct VSM_fantom	main_fantom;
+	struct VSM_fantom	iter_fantom;
 };
 
 
 /*--------------------------------------------------------------------*/
 
-void
-VSC_Setup(struct VSM_data *vd)
+static struct vsc *
+vsc_setup(struct VSM_data *vd)
 {
 
 	CHECK_OBJ_NOTNULL(vd, VSM_MAGIC);
-	AZ(vd->vsc);
-	ALLOC_OBJ(vd->vsc, VSC_MAGIC);
-	AN(vd->vsc);
-	VTAILQ_INIT(&vd->vsc->sf_list);
+	if (vd->vsc == NULL) {
+		ALLOC_OBJ(vd->vsc, VSC_MAGIC);
+		VTAILQ_INIT(&vd->vsc->sf_list);
+		VTAILQ_INIT(&vd->vsc->pt_list);
+	}
+	CHECK_OBJ_NOTNULL(vd->vsc, VSC_MAGIC);
+	return (vd->vsc);
 }
 
 /*--------------------------------------------------------------------*/
 
-void
-VSC_Delete(struct VSM_data *vd)
+static void
+vsc_delete_pts(struct vsc *vsc)
+{
+	struct vsc_pt *pt;
+	struct VSM_fantom *vf = NULL;
+
+	while(!VTAILQ_EMPTY(&vsc->pt_list)) {
+		pt = VTAILQ_FIRST(&vsc->pt_list);
+		VTAILQ_REMOVE(&vsc->pt_list, pt, list);
+		if (pt->point.vf != vf) {
+			vf = pt->point.vf;
+			free(vf);
+		}
+		FREE_OBJ(pt);
+	}
+}
+
+static void
+vsc_delete_sfs(struct vsc *vsc)
 {
 	struct vsc_sf *sf;
-	struct vsc *vsc;
 
-	CHECK_OBJ_NOTNULL(vd, VSM_MAGIC);
-	vsc = vd->vsc;
-	vd->vsc = NULL;
-	CHECK_OBJ_NOTNULL(vsc, VSC_MAGIC);
 	while(!VTAILQ_EMPTY(&vsc->sf_list)) {
 		sf = VTAILQ_FIRST(&vsc->sf_list);
-		VTAILQ_REMOVE(&vsc->sf_list, sf, next);
+		VTAILQ_REMOVE(&vsc->sf_list, sf, list);
 		free(sf->class);
 		free(sf->ident);
 		free(sf->name);
@@ -106,39 +129,42 @@ VSC_Delete(struct VSM_data *vd)
 	}
 }
 
-/*--------------------------------------------------------------------*/
-
-static int
-vsc_sf_arg(const struct VSM_data *vd, const char *opt)
+void
+VSC_Delete(struct VSM_data *vd)
 {
 	struct vsc *vsc;
-	struct vsc_sf *sf;
-	char **av, *q, *p;
-	int i;
 
 	CHECK_OBJ_NOTNULL(vd, VSM_MAGIC);
 	vsc = vd->vsc;
+	vd->vsc = NULL;
 	CHECK_OBJ_NOTNULL(vsc, VSC_MAGIC);
+	vsc_delete_sfs(vsc);
+	vsc_delete_pts(vsc);
+	FREE_OBJ(vsc);
+}
 
-	if (VTAILQ_EMPTY(&vsc->sf_list)) {
-		if (*opt == '^')
-			vsc->sf_init = 1;
-	}
+/*--------------------------------------------------------------------*/
+
+static int
+vsc_f_arg(struct VSM_data *vd, const char *opt)
+{
+	struct vsc *vsc = vsc_setup(vd);
+	struct vsc_sf *sf;
+	char **av, *q, *p;
+	int i;
 
 	av = VAV_Parse(opt, NULL, ARGV_COMMA);
 	AN(av);
-	if (av[0] != NULL) {
-		vd->diag(vd->priv, "Parse error: %s", av[0]);
-		return (-1);
-	}
+	if (av[0] != NULL)
+		return (vsm_diag(vd, "Parse error: %s", av[0]));
 	for (i = 1; av[i] != NULL; i++) {
-		ALLOC_OBJ(sf, VSL_SF_MAGIC);
+		ALLOC_OBJ(sf, VSC_SF_MAGIC);
 		AN(sf);
-		VTAILQ_INSERT_TAIL(&vsc->sf_list, sf, next);
+		VTAILQ_INSERT_TAIL(&vsc->sf_list, sf, list);
 
 		p = av[i];
 		if (*p == '^') {
-			sf->flags |= VSL_SF_EXCL;
+			sf->flags |= VSC_SF_EXCL;
 			p++;
 		}
 
@@ -167,21 +193,21 @@ vsc_sf_arg(const struct VSM_data *vd, const char *opt)
 			q = strchr(sf->class, '*');
 			if (q != NULL && q[1] == '\0') {
 				*q = '\0';
-				sf->flags |= VSL_SF_CL_WC;
+				sf->flags |= VSC_SF_CL_WC;
 			}
 		}
 		if (sf->ident != NULL) {
 			q = strchr(sf->ident, '*');
 			if (q != NULL && q[1] == '\0') {
 				*q = '\0';
-				sf->flags |= VSL_SF_ID_WC;
+				sf->flags |= VSC_SF_ID_WC;
 			}
 		}
 		if (sf->name != NULL) {
 			q = strchr(sf->name, '*');
 			if (q != NULL && q[1] == '\0') {
 				*q = '\0';
-				sf->flags |= VSL_SF_NM_WC;
+				sf->flags |= VSC_SF_NM_WC;
 			}
 		}
 	}
@@ -195,10 +221,8 @@ int
 VSC_Arg(struct VSM_data *vd, int arg, const char *opt)
 {
 
-	CHECK_OBJ_NOTNULL(vd, VSM_MAGIC);
-	AN(vd->vsc);
 	switch (arg) {
-	case 'f': return (vsc_sf_arg(vd, opt));
+	case 'f': return (vsc_f_arg(vd, opt));
 	case 'n': return (VSM_n_Arg(vd, opt));
 	default:
 		return (0);
@@ -207,33 +231,19 @@ VSC_Arg(struct VSM_data *vd, int arg, const char *opt)
 
 /*--------------------------------------------------------------------*/
 
-int
-VSC_Open(struct VSM_data *vd, int diag)
-{
-	int i;
-
-	CHECK_OBJ_NOTNULL(vd, VSM_MAGIC);
-	AN(vd->vsc);
-
-	i = VSM_Open(vd, diag);
-	return (i);
-}
-
-/*--------------------------------------------------------------------*/
-
 struct VSC_C_main *
-VSC_Main(const struct VSM_data *vd)
+VSC_Main(struct VSM_data *vd)
 {
-	struct VSM_fantom vf;
+	struct vsc *vsc = vsc_setup(vd);
 
-	CHECK_OBJ_NOTNULL(vd, VSM_MAGIC);
-	CHECK_OBJ_NOTNULL(vd->vsc, VSC_MAGIC);
-
-	if (!VSM_Get(vd, &vf, VSC_CLASS, "", ""))
+	if (!vd->head && VSM_Open(vd))
 		return (NULL);
-	return ((void*)vf.b);
+	if (!VSM_Get(vd, &vsc->main_fantom, VSC_CLASS, "", ""))
+		return (NULL);
+	return ((void*)vsc->main_fantom.b);
 }
 
+#if 0
 /*--------------------------------------------------------------------
  * -1 -> unknown stats encountered.
  */
@@ -256,23 +266,28 @@ iter_call(const struct vsc *vsc, VSC_iter_f *func, void *priv,
     const struct VSC_point *const sp)
 {
 	struct vsc_sf *sf;
+	struct vsc_pt *pt;
 	int good;
 
 	CHECK_OBJ_NOTNULL(vsc, VSC_MAGIC);
 
-	if (VTAILQ_EMPTY(&vsc->sf_list))
-		return (func(priv, sp));
+	ALLOC_OBJ(pt, VSC_PT_MAGIC);
+	AN(pt);
 
-	good = vsc->sf_init;
+	if (VTAILQ_EMPTY(&vsc->sf_list)) {
+		VTAILQ_INSERT_TAIL(&vsc->pt_list, pt, list);
+		return (func(priv, sp));
+	}
 
-	VTAILQ_FOREACH(sf, &vsc->sf_list, next) {
-		if (iter_test(sf->class, sp->class, sf->flags & VSL_SF_CL_WC))
+	good = 0;
+	VTAILQ_FOREACH(sf, &vsc->sf_list, list) {
+		if (iter_test(sf->class, sp->class, sf->flags & VSC_SF_CL_WC))
 			continue;
-		if (iter_test(sf->ident, sp->ident, sf->flags & VSL_SF_ID_WC))
+		if (iter_test(sf->ident, sp->ident, sf->flags & VSC_SF_ID_WC))
 			continue;
-		if (iter_test(sf->name, sp->name, sf->flags & VSL_SF_NM_WC))
+		if (iter_test(sf->name, sp->desc->name, sf->flags & VSC_SF_NM_WC))
 			continue;
-		if (sf->flags & VSL_SF_EXCL)
+		if (sf->flags & VSC_SF_EXCL)
 			good = 0;
 		else
 			good = 1;
@@ -283,26 +298,27 @@ iter_call(const struct vsc *vsc, VSC_iter_f *func, void *priv,
 }
 
 #define VSC_DO(U,l,t)							\
-	static int							\
+	static void							\
 	iter_##l(const struct vsc *vsc, struct VSM_fantom *vf,		\
-	    VSC_iter_f *func, void *priv)				\
+	    const struct VSC_desc *descs)				\
 	{								\
 		struct VSC_C_##l *st;					\
-		struct VSC_point sp;					\
+		struct VSM_fantom *vf2;					\
 		int i;							\
 									\
 		CHECK_OBJ_NOTNULL(vsc, VSC_MAGIC);			\
 		st = vf->b;						\
 		sp.class = t;						\
-		sp.ident = vf->chunk->ident;
+		sp.ident = vf->chunk->ident;				\
+		sp.desc = descs++;					\
+		vf2 = malloc(sizeof *vf2);				\
+		AN(vf2);						\
+		memcpy(vf2, vf, sizeof *vf2);
 
 #define VSC_F(nn,tt,ll,ff,dd,ee)					\
-		sp.name = #nn;						\
-		sp.fmt = #tt;						\
-		sp.flag = ff;						\
-		sp.desc = dd;						\
 		sp.ptr = &st->nn;					\
-		i = iter_call(vsc, func, priv, &sp);			\
+		sp.vf = vf2;						\
+		i = iter_call(vsc, &sp);				\
 		if (i)							\
 			return(i);
 
@@ -315,19 +331,14 @@ iter_call(const struct vsc *vsc, VSC_iter_f *func, void *priv,
 #undef VSC_F
 #undef VSC_DONE
 
-int
-VSC_Iter(const struct VSM_data *vd, VSC_iter_f *func, void *priv)
+static void
+vsc_build_pt_list(struct VSM_data *vd)
 {
-	struct vsc *vsc;
+	struct vsc *vsc = vsc_setup(vd);
 	struct VSM_fantom vf;
-	int i;
 
-	CHECK_OBJ_NOTNULL(vd, VSM_MAGIC);
-	vsc = vd->vsc;
-	CHECK_OBJ_NOTNULL(vsc, VSC_MAGIC);
-	i = 0;
-	if (!VSM_StillValid(vd, NULL))
-		return (-1);
+	vsc_delete_pts(vsc *vsc);
+
 	VSM_FOREACH_SAFE(&vf, vd) {
 		if (strcmp(vf.chunk->class, VSC_CLASS))
 			continue;
@@ -335,11 +346,8 @@ VSC_Iter(const struct VSM_data *vd, VSC_iter_f *func, void *priv)
 #define VSC_F(n,t,l,f,d,e)
 #define VSC_DONE(a,b,c)
 #define VSC_DO(U,l,t)						\
-		if (!strcmp(vf.chunk->type, t)) {		\
-			i = iter_##l(vsc, &vf, func, priv);	\
-			if (!i)					\
-				continue;			\
-		}
+		if (!strcmp(vf.chunk->type, t))			\
+			iter_##l(vsc, &vf, VSC_desc_##l);
 #include "tbl/vsc_all.h"
 #undef VSC_F
 #undef VSC_DO
@@ -349,3 +357,127 @@ VSC_Iter(const struct VSM_data *vd, VSC_iter_f *func, void *priv)
 	}
 	return (i);
 }
+#endif
+
+/*--------------------------------------------------------------------
+ */
+
+static void
+vsc_add_pt(struct vsc *vsc, const char *class, const char *ident,
+    const struct VSC_desc *desc, const volatile void *ptr,
+    struct VSM_fantom *vf)
+{
+	struct vsc_pt *pt;
+
+	ALLOC_OBJ(pt, VSC_PT_MAGIC);
+	AN(pt);
+	pt->point.class = class;
+	pt->point.ident = ident;
+	pt->point.desc = desc;
+	pt->point.ptr = ptr;
+	pt->point.vf = vf;
+	VTAILQ_INSERT_TAIL(&vsc->pt_list, pt, list);
+}
+
+#define VSC_DO(U,l,t)							\
+	static void							\
+	iter_##l(struct vsc *vsc, struct VSM_fantom *vf,		\
+	    const struct VSC_desc *descs)				\
+	{								\
+		struct VSC_C_##l *st;					\
+		struct VSM_fantom *vf2;					\
+		const char *class = t;					\
+									\
+		CHECK_OBJ_NOTNULL(vsc, VSC_MAGIC);			\
+		st = vf->b;						\
+		vf2 = malloc(sizeof *vf2);				\
+		AN(vf2);						\
+		memcpy(vf2, vf, sizeof *vf2);
+
+#define VSC_F(nn,tt,ll,ff,dd,ee)					\
+		vsc_add_pt(vsc, class, vf->chunk->ident, descs++,	\
+		    &st->nn, vf2);
+
+#define VSC_DONE(U,l,t)							\
+	}
+
+#include "tbl/vsc_all.h"
+#undef VSC_DO
+#undef VSC_F
+#undef VSC_DONE
+
+/*--------------------------------------------------------------------
+ */
+
+static void
+vsc_build_pt_list(struct VSM_data *vd)
+{
+	struct vsc *vsc = vsc_setup(vd);
+	struct VSM_fantom vf;
+
+	vsc_delete_pts(vsc);
+
+	VSM_FOREACH_SAFE(&vf, vd) {
+		if (strcmp(vf.chunk->class, VSC_CLASS))
+			continue;
+		/*lint -save -e525 -e539 */
+#define VSC_F(n,t,l,f,d,e)
+#define VSC_DONE(a,b,c)
+#define VSC_DO(U,l,t)						\
+		if (!strcmp(vf.chunk->type, t))			\
+			iter_##l(vsc, &vf, VSC_desc_##l);
+#include "tbl/vsc_all.h"
+#undef VSC_F
+#undef VSC_DO
+#undef VSC_DONE
+		/*lint -restore */
+	}
+
+	/* XXX: filter pt list */
+}
+
+/*--------------------------------------------------------------------
+ */
+
+int
+VSC_Iter(struct VSM_data *vd, VSC_iter_f *func, void *priv)
+{
+	struct vsc *vsc = vsc_setup(vd);
+	struct vsc_pt *pt;
+	int i;
+
+	if (1 != VSM_StillValid(vd, &vsc->iter_fantom)) {
+		if (!VSM_Get(vd, &vsc->iter_fantom, VSC_CLASS, "", "")) {
+			VSM_Close(vd);
+			if (!vd->head && VSM_Open(vd))
+				return (-1);
+			if (!VSM_Get(vd, &vsc->iter_fantom, VSC_CLASS, "", "")) {
+				return (-1);
+			}
+		}
+		AN(vd->head);
+		func(priv, NULL);
+		vsc_build_pt_list(vd);
+	}
+	AN(vd->head);
+	VTAILQ_FOREACH(pt, &vsc->pt_list, list) {
+		i = func(priv, &pt->point);
+		if (i)
+			return (i);
+	}
+	return (0);
+}
+
+/*--------------------------------------------------------------------
+ * Build the static point descriptions
+ */
+
+#define VSC_F(n,t,l,f,d,e)	{#n,#t,f,d,e},
+#define VSC_DO(U,l,t) const struct VSC_desc VSC_desc_##l[] = {
+#define VSC_DONE(U,l,t) };
+#include "tbl/vsc_all.h"
+#undef VSC_F
+#undef VSC_DO
+#undef VSC_DONE
+
+
diff --git a/lib/libvarnishapi/vsm.c b/lib/libvarnishapi/vsm.c
index 1d762e5..a9f176c 100644
--- a/lib/libvarnishapi/vsm.c
+++ b/lib/libvarnishapi/vsm.c
@@ -113,7 +113,7 @@ VSM_n_Arg(struct VSM_data *vd, const char *opt)
 {
 
 	CHECK_OBJ_NOTNULL(vd, VSM_MAGIC);
-	AN(vd->n_opt);
+	AN(opt);
 
 	REPLACE(vd->n_opt, opt);
 	if (VIN_N_Arg(vd->n_opt, NULL, NULL, &vd->fname))



From geoff at varnish-cache.org  Mon Jan  9 20:52:17 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:17 +0100
Subject: [experimental-ims] 81386d6 Add needed #include
Message-ID: <E1RkMCX-0007dY-4g@project.varnish-software.com>

commit 81386d693ac53fbbf7bde9c8134c84aa2674090c
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Oct 10 10:13:57 2011 +0000

    Add needed #include

diff --git a/lib/libvarnish/cli_auth.c b/lib/libvarnish/cli_auth.c
index 396a127..08bbddc 100644
--- a/lib/libvarnish/cli_auth.c
+++ b/lib/libvarnish/cli_auth.c
@@ -31,6 +31,7 @@
 #include <sys/types.h>
 
 #include <errno.h>
+#include <stdint.h>
 #include <stdio.h>
 #include <unistd.h>
 



From geoff at varnish-cache.org  Mon Jan  9 20:52:26 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:26 +0100
Subject: [experimental-ims] 1c2a646 Make it possible to disable checks for
	rst2man and rst2html
Message-ID: <E1RkMCg-0007x8-V5@project.varnish-software.com>

commit 1c2a6461fef2819d07bc2e557defbcb66e72e823
Author: Tollef Fog Heen <tfheen at varnish-software.com>
Date:   Mon Oct 24 09:24:36 2011 +0200

    Make it possible to disable checks for rst2man and rst2html

diff --git a/configure.ac b/configure.ac
index e0119f3..4591148 100644
--- a/configure.ac
+++ b/configure.ac
@@ -50,16 +50,24 @@ if test "x$XSLTPROC" = "xno"; then
    AC_MSG_WARN([xsltproc not found ? not building documentation])
 fi
 AM_CONDITIONAL(HAVE_XSLTPROC,[test "x$XSLTPROC" != "xno"])
-AC_CHECK_PROGS(RST2MAN, [rst2man rst2man.py], "no")
-if test "x$RST2MAN" = "xno"; then
-   AC_MSG_WARN([rst2man not found ? not building man pages])
-fi
+AC_ARG_WITH([rst2man],
+               AS_HELP_STRING([--with-rst2man=PATH],
+                              [Location of rst2man (auto)]),
+               [RST2MAN="$withval"],
+               [AC_CHECK_PROGS(RST2MAN, [rst2man rst2man.py], "no")
+	        if test "x$RST2MAN" = "xno"; then
+		   AC_MSG_WARN([rst2man not found ? not building man pages])
+		fi])
 AM_CONDITIONAL(HAVE_RST2MAN,[test "x$RST2MAN" != "xno"])
 
-AC_CHECK_PROGS(RST2HTML, [rst2html rst2html.py], "no")
-if test "x$RST2HTML" = "xno"; then
-   AC_MSG_WARN([rst2html not found ? not building changelog])
-fi
+AC_ARG_WITH([rst2html],
+               AS_HELP_STRING([--with-rst2html=PATH],
+                              [Location of rst2html (auto)]),
+               [RST2HTML="$withval"],
+               [AC_CHECK_PROGS(RST2HTML, [rst2html rst2html.py], "no")
+	        if test "x$RST2HTML" = "xno"; then
+		   AC_MSG_WARN([rst2html not found ? not building changelog])
+		fi])
 AM_CONDITIONAL(HAVE_RST2HTML,[test "x$RST2HTML" != "xno"])
 
 # Checks for libraries.



From geoff at varnish-cache.org  Mon Jan  9 20:52:29 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:29 +0100
Subject: [experimental-ims] c3fb33c Expose VSL_Name2Tag in libvarnishapi
Message-ID: <E1RkMCj-00082a-7d@project.varnish-software.com>

commit c3fb33c70e8a92ec2cd18e8314458d48a630a47f
Author: Tollef Fog Heen <tfheen at varnish-software.com>
Date:   Fri Nov 4 12:02:31 2011 +0100

    Expose VSL_Name2Tag in libvarnishapi

diff --git a/lib/libvarnishapi/libvarnishapi.map b/lib/libvarnishapi/libvarnishapi.map
index 4df8577..83f5169 100644
--- a/lib/libvarnishapi/libvarnishapi.map
+++ b/lib/libvarnishapi/libvarnishapi.map
@@ -68,3 +68,10 @@ LIBVARNISHAPI_1.0 {
   local:
 	*;
 };
+
+LIBVARNISHAPI_1.1 {
+  global:
+	# Functions:
+	VSL_Name2Tag;
+	# Variables:
+} LIBVARNISHAPI_1.0;



From geoff at varnish-cache.org  Mon Jan  9 20:52:02 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:02 +0100
Subject: [experimental-ims] 0d58f56 Second pass to tag VSL's with backend or
	client bits.
Message-ID: <E1RkMCI-0007Dj-3v@project.varnish-software.com>

commit 0d58f56e1bcc7ca1c17b23a370afb3da4e21b989
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Fri Sep 30 15:35:02 2011 +0000

    Second pass to tag VSL's with backend or client bits.

diff --git a/bin/varnishd/cache.h b/bin/varnishd/cache.h
index 5c793e9..5a91dd8 100644
--- a/bin/varnishd/cache.h
+++ b/bin/varnishd/cache.h
@@ -763,17 +763,18 @@ unsigned http_Write(struct worker *w, unsigned vsl_id, const struct http *hp,
 void http_CopyResp(struct http *to, const struct http *fm);
 void http_SetResp(struct http *to, const char *proto, uint16_t status,
     const char *response);
-void http_FilterFields(struct worker *w, int fd, struct http *to,
+void http_FilterFields(struct worker *w, unsigned vsl_id, struct http *to,
     const struct http *fm, unsigned how);
 void http_FilterHeader(const struct sess *sp, unsigned how);
-void http_PutProtocol(struct worker *w, int fd, const struct http *to,
+void http_PutProtocol(struct worker *w, unsigned vsl_id, const struct http *to,
     const char *protocol);
 void http_PutStatus(struct http *to, uint16_t status);
-void http_PutResponse(struct worker *w, int fd, const struct http *to,
+void http_PutResponse(struct worker *w, unsigned vsl_id, const struct http *to,
     const char *response);
-void http_PrintfHeader(struct worker *w, int fd, struct http *to,
+void http_PrintfHeader(struct worker *w, unsigned vsl_id, struct http *to,
     const char *fmt, ...);
-void http_SetHeader(struct worker *w, int fd, struct http *to, const char *hdr);
+void http_SetHeader(struct worker *w, unsigned vsl_id, struct http *to,
+    const char *hdr);
 void http_SetH(const struct http *to, unsigned n, const char *fm);
 void http_ForceGet(const struct http *to);
 void http_Setup(struct http *ht, struct ws *ws);
@@ -790,7 +791,7 @@ uint16_t http_DissectRequest(struct sess *sp);
 uint16_t http_DissectResponse(struct worker *w, const struct http_conn *htc,
     struct http *sp);
 const char *http_DoConnection(const struct http *hp);
-void http_CopyHome(struct worker *w, int fd, const struct http *hp);
+void http_CopyHome(struct worker *w, unsigned vsl_id, const struct http *hp);
 void http_Unset(struct http *hp, const char *hdr);
 void http_CollectHdr(struct http *hp, const char *hdr);
 
diff --git a/bin/varnishd/cache_acceptor.c b/bin/varnishd/cache_acceptor.c
index ba28a39..e2be560 100644
--- a/bin/varnishd/cache_acceptor.c
+++ b/bin/varnishd/cache_acceptor.c
@@ -130,10 +130,10 @@ VCA_Prep(struct sess *sp)
 		AZ(getsockname(sp->fd, (void*)&sp->mysockaddr, &sp->mysockaddrlen));
 		VTCP_name(&sp->mysockaddr, sp->mysockaddrlen,
 		    addr, sizeof addr, port, sizeof port);
-		VSL(SLT_SessionOpen, sp->fd, "%s %s %s %s",
+		WSP(sp, SLT_SessionOpen, "%s %s %s %s",
 		    sp->addr, sp->port, addr, port);
 	} else {
-		VSL(SLT_SessionOpen, sp->fd, "%s %s %s",
+		WSP(sp, SLT_SessionOpen, "%s %s %s",
 		    sp->addr, sp->port, sp->mylsock->name);
 	}
 	sp->acct_ses.first = sp->t_open;
diff --git a/bin/varnishd/cache_backend.c b/bin/varnishd/cache_backend.c
index d2bc860..1061612 100644
--- a/bin/varnishd/cache_backend.c
+++ b/bin/varnishd/cache_backend.c
@@ -70,7 +70,7 @@ VDI_AddHostHeader(const struct sess *sp)
 	CHECK_OBJ_NOTNULL(sp->wrk->bereq, HTTP_MAGIC);
 	CHECK_OBJ_NOTNULL(sp->vbc, VBC_MAGIC);
 	CHECK_OBJ_NOTNULL(sp->vbc->vdis, VDI_SIMPLE_MAGIC);
-	http_PrintfHeader(sp->wrk, sp->fd, sp->wrk->bereq,
+	http_PrintfHeader(sp->wrk, sp->vsl_id, sp->wrk->bereq,
 	    "Host: %s", sp->vbc->vdis->vrt->hosthdr);
 }
 
@@ -357,7 +357,7 @@ vbe_GetVbe(const struct sess *sp, struct vdi_simple *vs)
 			return (vc);
 		}
 		VSC_C_main->backend_toolate++;
-		WSL(sp->wrk, SLT_BackendClose, vc->fd, "%s", bp->vcl_name);
+		WSL(sp->wrk, SLT_BackendClose, vc->vsl_id, "%s", bp->vcl_name);
 
 		/* Checkpoint log to flush all info related to this connection
 		   before the OS reuses the FD */
diff --git a/bin/varnishd/cache_center.c b/bin/varnishd/cache_center.c
index 38ec4f2..b4a056d 100644
--- a/bin/varnishd/cache_center.c
+++ b/bin/varnishd/cache_center.c
@@ -104,7 +104,7 @@ cnt_wait(struct sess *sp)
 			i = HTC_Rx(sp->htc);
 	}
 	if (i == 0) {
-		WSL(sp->wrk, SLT_Debug, sp->fd, "herding");
+		WSP(sp, SLT_Debug, "herding");
 		sp->wrk->stats.sess_herd++;
 		SES_Charge(sp);
 		sp->wrk = NULL;
@@ -464,16 +464,16 @@ cnt_error(struct sess *sp)
 	if (sp->err_code < 100 || sp->err_code > 999)
 		sp->err_code = 501;
 
-	http_PutProtocol(w, sp->fd, h, "HTTP/1.1");
+	http_PutProtocol(w, sp->vsl_id, h, "HTTP/1.1");
 	http_PutStatus(h, sp->err_code);
 	TIM_format(TIM_real(), date);
-	http_PrintfHeader(w, sp->fd, h, "Date: %s", date);
-	http_PrintfHeader(w, sp->fd, h, "Server: Varnish");
+	http_PrintfHeader(w, sp->vsl_id, h, "Date: %s", date);
+	http_PrintfHeader(w, sp->vsl_id, h, "Server: Varnish");
 
 	if (sp->err_reason != NULL)
-		http_PutResponse(w, sp->fd, h, sp->err_reason);
+		http_PutResponse(w, sp->vsl_id, h, sp->err_reason);
 	else
-		http_PutResponse(w, sp->fd, h,
+		http_PutResponse(w, sp->vsl_id, h,
 		    http_StatusMessage(sp->err_code));
 	VCL_error_method(sp);
 
@@ -728,7 +728,7 @@ cnt_fetchbody(struct sess *sp)
 
 	/* If we do gzip, add the C-E header */
 	if (sp->wrk->do_gzip)
-		http_PrintfHeader(sp->wrk, sp->fd, sp->wrk->beresp,
+		http_PrintfHeader(sp->wrk, sp->vsl_id, sp->wrk->beresp,
 		    "Content-Encoding: %s", "gzip");
 
 	/* But we can't do both at the same time */
@@ -1194,7 +1194,7 @@ cnt_miss(struct sess *sp)
 		 * the minority of clients which don't.
 		 */
 		http_Unset(sp->wrk->bereq, H_Accept_Encoding);
-		http_PrintfHeader(sp->wrk, sp->fd, sp->wrk->bereq,
+		http_PrintfHeader(sp->wrk, sp->vsl_id, sp->wrk->bereq,
 		    "Accept-Encoding: gzip");
 	}
 	sp->wrk->connect_timeout = 0;
@@ -1398,7 +1398,7 @@ cnt_recv(struct sess *sp)
 	     (recv_handling != VCL_RET_PASS)) {
 		if (RFC2616_Req_Gzip(sp)) {
 			http_Unset(sp->http, H_Accept_Encoding);
-			http_PrintfHeader(sp->wrk, sp->fd, sp->http,
+			http_PrintfHeader(sp->wrk, sp->vsl_id, sp->http,
 			    "Accept-Encoding: gzip");
 		} else {
 			http_Unset(sp->http, H_Accept_Encoding);
diff --git a/bin/varnishd/cache_dir.c b/bin/varnishd/cache_dir.c
index 2463aa8..03fdf26 100644
--- a/bin/varnishd/cache_dir.c
+++ b/bin/varnishd/cache_dir.c
@@ -48,7 +48,7 @@ VDI_CloseFd(struct sess *sp)
 
 	bp = sp->vbc->backend;
 
-	WSL(sp->wrk, SLT_BackendClose, sp->vbc->fd, "%s", bp->vcl_name);
+	WSL(sp->wrk, SLT_BackendClose, sp->vbc->vsl_id, "%s", bp->vcl_name);
 
 	/* Checkpoint log to flush all info related to this connection
 	   before the OS reuses the FD */
@@ -76,7 +76,7 @@ VDI_RecycleFd(struct sess *sp)
 
 	bp = sp->vbc->backend;
 
-	WSL(sp->wrk, SLT_BackendReuse, sp->vbc->fd, "%s", bp->vcl_name);
+	WSL(sp->wrk, SLT_BackendReuse, sp->vbc->vsl_id, "%s", bp->vcl_name);
 	/*
 	 * Flush the shmlog, so that another session reusing this backend
 	 * will log chronologically later than our use of it.
diff --git a/bin/varnishd/cache_esi_deliver.c b/bin/varnishd/cache_esi_deliver.c
index 8b2edba..5bfa654 100644
--- a/bin/varnishd/cache_esi_deliver.c
+++ b/bin/varnishd/cache_esi_deliver.c
@@ -72,7 +72,7 @@ ved_include(struct sess *sp, const char *src, const char *host)
 	if (host != NULL && *host != '\0')  {
 		http_Unset(sp->http, H_Host);
 		http_Unset(sp->http, H_If_Modified_Since);
-		http_SetHeader(w, sp->fd, sp->http, host);
+		http_SetHeader(w, sp->vsl_id, sp->http, host);
 	}
 	/*
 	 * XXX: We should decide if we should cache the director
diff --git a/bin/varnishd/cache_fetch.c b/bin/varnishd/cache_fetch.c
index 6451f10..7c6e2e5 100644
--- a/bin/varnishd/cache_fetch.c
+++ b/bin/varnishd/cache_fetch.c
@@ -63,8 +63,7 @@ vfp_nop_begin(struct sess *sp, size_t estimate)
 
 	if (fetchfrag > 0) {
 		estimate = fetchfrag;
-		WSL(sp->wrk, SLT_Debug, sp->fd,
-		    "Fetch %d byte segments:", fetchfrag);
+		WSP(sp, SLT_Debug, "Fetch %d byte segments:", fetchfrag);
 	}
 	if (estimate > 0)
 		(void)FetchStorage(sp, estimate);
@@ -355,7 +354,7 @@ FetchReqBody(struct sess *sp)
 	}
 	if (http_GetHdr(sp->http, H_Transfer_Encoding, NULL)) {
 		/* XXX: Handle chunked encoding. */
-		WSL(sp->wrk, SLT_Debug, sp->fd, "Transfer-Encoding in request");
+		WSP(sp, SLT_Debug, "Transfer-Encoding in request");
 		return (1);
 	}
 	return (0);
@@ -542,7 +541,7 @@ FetchBody(struct sess *sp)
 	 */
 	AZ(vfp_nop_end(sp));
 
-	WSL(w, SLT_Fetch_Body, sp->vbc->fd, "%u(%s) cls %d mklen %u",
+	WSL(w, SLT_Fetch_Body, sp->vbc->vsl_id, "%u(%s) cls %d mklen %u",
 	    w->body_status, body_status(w->body_status),
 	    cls, mklen);
 
@@ -567,7 +566,7 @@ FetchBody(struct sess *sp)
 	if (cls == 0 && w->do_close)
 		cls = 1;
 
-	WSL(w, SLT_Length, sp->vbc->fd, "%u", sp->obj->len);
+	WSL(w, SLT_Length, sp->vbc->vsl_id, "%u", sp->obj->len);
 
 	{
 	/* Sanity check fetch methods accounting */
@@ -585,7 +584,7 @@ FetchBody(struct sess *sp)
 
 	if (mklen > 0) {
 		http_Unset(sp->obj->http, H_Content_Length);
-		http_PrintfHeader(w, sp->fd, sp->obj->http,
+		http_PrintfHeader(w, sp->vsl_id, sp->obj->http,
 		    "Content-Length: %jd", (intmax_t)sp->obj->len);
 	}
 
diff --git a/bin/varnishd/cache_http.c b/bin/varnishd/cache_http.c
index 587b6bb..c8a5415 100644
--- a/bin/varnishd/cache_http.c
+++ b/bin/varnishd/cache_http.c
@@ -77,7 +77,7 @@ http2shmlog(const struct http *hp, int t)
 }
 
 static void
-WSLH(struct worker *w, int vsl_id, const struct http *hp, unsigned hdr)
+WSLH(struct worker *w, unsigned vsl_id, const struct http *hp, unsigned hdr)
 {
 
 	AN(vsl_id & (VSL_CLIENTMARKER|VSL_BACKENDMARKER));
@@ -486,7 +486,7 @@ http_GetReq(const struct http *hp)
  */
 
 static uint16_t
-http_dissect_hdrs(struct worker *w, struct http *hp, int vsl_id, char *p,
+http_dissect_hdrs(struct worker *w, struct http *hp, unsigned vsl_id, char *p,
     const struct http_conn *htc)
 {
 	char *q, *r;
@@ -562,7 +562,7 @@ http_dissect_hdrs(struct worker *w, struct http *hp, int vsl_id, char *p,
  */
 
 static uint16_t
-http_splitline(struct worker *w, int vsl_id, struct http *hp,
+http_splitline(struct worker *w, unsigned vsl_id, struct http *hp,
     const struct http_conn *htc, int h1, int h2, int h3)
 {
 	char *p, *q;
@@ -792,7 +792,7 @@ http_SetResp(struct http *to, const char *proto, uint16_t status,
 }
 
 static void
-http_copyheader(struct worker *w, int vsl_id, struct http *to,
+http_copyheader(struct worker *w, unsigned vsl_id, struct http *to,
     const struct http *fm, unsigned n)
 {
 
@@ -843,7 +843,7 @@ http_EstimateWS(const struct http *fm, unsigned how, uint16_t *nhd)
 /*--------------------------------------------------------------------*/
 
 void
-http_FilterFields(struct worker *w, int vsl_id, struct http *to,
+http_FilterFields(struct worker *w, unsigned vsl_id, struct http *to,
     const struct http *fm, unsigned how)
 {
 	unsigned u;
@@ -893,7 +893,7 @@ http_FilterHeader(const struct sess *sp, unsigned how)
  */
 
 void
-http_CopyHome(struct worker *w, int vsl_id, const struct http *hp)
+http_CopyHome(struct worker *w, unsigned vsl_id, const struct http *hp)
 {
 	unsigned u, l;
 	char *p;
@@ -939,7 +939,8 @@ http_ClrHeader(struct http *to)
 /*--------------------------------------------------------------------*/
 
 void
-http_SetHeader(struct worker *w, int vsl_id, struct http *to, const char *hdr)
+http_SetHeader(struct worker *w, unsigned vsl_id, struct http *to,
+    const char *hdr)
 {
 
 	CHECK_OBJ_NOTNULL(to, HTTP_MAGIC);
@@ -954,8 +955,8 @@ http_SetHeader(struct worker *w, int vsl_id, struct http *to, const char *hdr)
 /*--------------------------------------------------------------------*/
 
 static void
-http_PutField(struct worker *w, int vsl_id, const struct http *to, int field,
-    const char *string)
+http_PutField(struct worker *w, unsigned vsl_id, const struct http *to,
+    int field, const char *string)
 {
 	char *p;
 	unsigned l;
@@ -977,7 +978,7 @@ http_PutField(struct worker *w, int vsl_id, const struct http *to, int field,
 }
 
 void
-http_PutProtocol(struct worker *w, int vsl_id, const struct http *to,
+http_PutProtocol(struct worker *w, unsigned vsl_id, const struct http *to,
     const char *protocol)
 {
 
@@ -993,7 +994,7 @@ http_PutStatus(struct http *to, uint16_t status)
 }
 
 void
-http_PutResponse(struct worker *w, int vsl_id, const struct http *to,
+http_PutResponse(struct worker *w, unsigned vsl_id, const struct http *to,
     const char *response)
 {
 
@@ -1001,7 +1002,7 @@ http_PutResponse(struct worker *w, int vsl_id, const struct http *to,
 }
 
 void
-http_PrintfHeader(struct worker *w, int vsl_id, struct http *to,
+http_PrintfHeader(struct worker *w, unsigned vsl_id, struct http *to,
     const char *fmt, ...)
 {
 	va_list ap;
diff --git a/bin/varnishd/cache_response.c b/bin/varnishd/cache_response.c
index e490bbf..4d7c88b 100644
--- a/bin/varnishd/cache_response.c
+++ b/bin/varnishd/cache_response.c
@@ -92,12 +92,12 @@ res_dorange(const struct sess *sp, const char *r, ssize_t *plow, ssize_t *phigh)
 	if (low > high)
 		return;
 
-	http_PrintfHeader(sp->wrk, sp->fd, sp->wrk->resp,
+	http_PrintfHeader(sp->wrk, sp->vsl_id, sp->wrk->resp,
 	    "Content-Range: bytes %jd-%jd/%jd",
 	    (intmax_t)low, (intmax_t)high, (intmax_t)sp->obj->len);
 	http_Unset(sp->wrk->resp, H_Content_Length);
 	assert(sp->wrk->res_mode & RES_LEN);
-	http_PrintfHeader(sp->wrk, sp->fd, sp->wrk->resp,
+	http_PrintfHeader(sp->wrk, sp->vsl_id, sp->wrk->resp,
 	    "Content-Length: %jd", (intmax_t)(1 + high - low));
 	http_SetResp(sp->wrk->resp, "HTTP/1.1", 206, "Partial Content");
 
@@ -118,34 +118,34 @@ RES_BuildHttp(const struct sess *sp)
 	http_ClrHeader(sp->wrk->resp);
 	sp->wrk->resp->logtag = HTTP_Tx;
 	http_CopyResp(sp->wrk->resp, sp->obj->http);
-	http_FilterFields(sp->wrk, sp->fd, sp->wrk->resp, sp->obj->http,
+	http_FilterFields(sp->wrk, sp->vsl_id, sp->wrk->resp, sp->obj->http,
 	    HTTPH_A_DELIVER);
 
 	if (!(sp->wrk->res_mode & RES_LEN)) {
 		http_Unset(sp->wrk->resp, H_Content_Length);
 	} else if (params->http_range_support) {
 		/* We only accept ranges if we know the length */
-		http_SetHeader(sp->wrk, sp->fd, sp->wrk->resp,
+		http_SetHeader(sp->wrk, sp->vsl_id, sp->wrk->resp,
 		    "Accept-Ranges: bytes");
 	}
 
 	if (sp->wrk->res_mode & RES_CHUNKED)
-		http_PrintfHeader(sp->wrk, sp->fd, sp->wrk->resp,
+		http_PrintfHeader(sp->wrk, sp->vsl_id, sp->wrk->resp,
 		    "Transfer-Encoding: chunked");
 
 	TIM_format(TIM_real(), time_str);
-	http_PrintfHeader(sp->wrk, sp->fd, sp->wrk->resp, "Date: %s", time_str);
+	http_PrintfHeader(sp->wrk, sp->vsl_id, sp->wrk->resp, "Date: %s", time_str);
 
 	if (sp->xid != sp->obj->xid)
-		http_PrintfHeader(sp->wrk, sp->fd, sp->wrk->resp,
+		http_PrintfHeader(sp->wrk, sp->vsl_id, sp->wrk->resp,
 		    "X-Varnish: %u %u", sp->xid, sp->obj->xid);
 	else
-		http_PrintfHeader(sp->wrk, sp->fd, sp->wrk->resp,
+		http_PrintfHeader(sp->wrk, sp->vsl_id, sp->wrk->resp,
 		    "X-Varnish: %u", sp->xid);
-	http_PrintfHeader(sp->wrk, sp->fd, sp->wrk->resp, "Age: %.0f",
+	http_PrintfHeader(sp->wrk, sp->vsl_id, sp->wrk->resp, "Age: %.0f",
 	    sp->obj->exp.age + sp->t_resp - sp->obj->exp.entered);
-	http_SetHeader(sp->wrk, sp->fd, sp->wrk->resp, "Via: 1.1 varnish");
-	http_PrintfHeader(sp->wrk, sp->fd, sp->wrk->resp, "Connection: %s",
+	http_SetHeader(sp->wrk, sp->vsl_id, sp->wrk->resp, "Via: 1.1 varnish");
+	http_PrintfHeader(sp->wrk, sp->vsl_id, sp->wrk->resp, "Connection: %s",
 	    sp->doclose ? "close" : "keep-alive");
 }
 
@@ -348,7 +348,7 @@ RES_StreamStart(struct sess *sp)
 
 	if (!(sp->wrk->res_mode & RES_CHUNKED) &&
 	    sp->wrk->h_content_length != NULL)
-		http_PrintfHeader(sp->wrk, sp->fd, sp->wrk->resp,
+		http_PrintfHeader(sp->wrk, sp->vsl_id, sp->wrk->resp,
 		    "Content-Length: %s", sp->wrk->h_content_length);
 
 	sp->wrk->acct_tmp.hdrbytes +=
diff --git a/bin/varnishd/cache_vrt.c b/bin/varnishd/cache_vrt.c
index c17c361..86cb10f 100644
--- a/bin/varnishd/cache_vrt.c
+++ b/bin/varnishd/cache_vrt.c
@@ -86,7 +86,7 @@ VRT_count(const struct sess *sp, unsigned u)
 void
 VRT_acl_log(const struct sess *sp, const char *msg)
 {
-	WSL(sp->wrk, SLT_VCL_acl, sp->fd, msg);
+	WSP(sp, SLT_VCL_acl, msg);
 }
 
 /*--------------------------------------------------------------------*/
@@ -232,7 +232,7 @@ VRT_SetHdr(const struct sess *sp , enum gethdr_e where, const char *hdr,
 			WSP(sp, SLT_LostHeader, "%s", hdr + 1);
 		} else {
 			http_Unset(hp, hdr);
-			http_SetHeader(sp->wrk, sp->fd, hp, b);
+			http_SetHeader(sp->wrk, sp->vsl_id, hp, b);
 		}
 	}
 	va_end(ap);
@@ -418,7 +418,7 @@ VRT_synth_page(const struct sess *sp, unsigned flags, const char *str, ...)
 	va_end(ap);
 	SMS_Finish(sp->obj);
 	http_Unset(sp->obj->http, H_Content_Length);
-	http_PrintfHeader(sp->wrk, sp->fd, sp->obj->http,
+	http_PrintfHeader(sp->wrk, sp->vsl_id, sp->obj->http,
 	    "Content-Length: %d", sp->obj->len);
 }
 



From geoff at varnish-cache.org  Mon Jan  9 20:52:49 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:49 +0100
Subject: [experimental-ims] 810428b Give VBE_GetBusyObj() and
	VBE_DerefBusyObj() a worker argument, we will need it shortly.
Message-ID: <E1RkMD3-0000DK-9i@project.varnish-software.com>

commit 810428b8d1f7cef243707e0efa178ae045cc6e0e
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Dec 7 10:13:43 2011 +0000

    Give VBE_GetBusyObj() and VBE_DerefBusyObj() a worker argument,
    we will need it shortly.
    
    Add more calls to them, they will also be needed shortly.

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index cf0a6ee..56f23fd 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -673,9 +673,9 @@ void VBE_Poll(void);
 
 /* cache_backend.c */
 void VBE_Init(void);
-struct busyobj *VBE_GetBusyObj(void);
+struct busyobj *VBE_GetBusyObj(struct worker *wrk);
 struct busyobj *VBE_RefBusyObj(struct busyobj *busyobj);
-void VBE_DerefBusyObj(struct busyobj **busyobj);
+void VBE_DerefBusyObj(struct worker *wrk, struct busyobj **busyobj);
 
 /* cache_backend_cfg.c */
 void VBE_InitCfg(void);
diff --git a/bin/varnishd/cache/cache_backend.c b/bin/varnishd/cache/cache_backend.c
index ef937ea..21c1a4c 100644
--- a/bin/varnishd/cache/cache_backend.c
+++ b/bin/varnishd/cache/cache_backend.c
@@ -78,10 +78,11 @@ vbe_FreeBusyObj(struct busyobj *busyobj)
 }
 
 struct busyobj *
-VBE_GetBusyObj(void)
+VBE_GetBusyObj(struct worker *wrk)
 {
 	struct busyobj *busyobj = NULL;
 
+	(void)wrk;
 	Lck_Lock(&nbusyobj_mtx);
 	if (nbusyobj != NULL) {
 		CHECK_OBJ_NOTNULL(nbusyobj, BUSYOBJ_MAGIC);
@@ -110,10 +111,11 @@ VBE_RefBusyObj(struct busyobj *busyobj)
 }
 
 void
-VBE_DerefBusyObj(struct busyobj **pbo)
+VBE_DerefBusyObj(struct worker *wrk, struct busyobj **pbo)
 {
 	struct busyobj *busyobj;
 
+	(void)wrk;
 	busyobj = *pbo;
 	CHECK_OBJ_NOTNULL(busyobj, BUSYOBJ_MAGIC);
 	Lck_Lock(&busyobj->mtx);
diff --git a/bin/varnishd/cache/cache_center.c b/bin/varnishd/cache/cache_center.c
index 995268a..79e7956 100644
--- a/bin/varnishd/cache/cache_center.c
+++ b/bin/varnishd/cache/cache_center.c
@@ -255,7 +255,7 @@ cnt_prepresp(struct sess *sp)
 			AN(wrk->busyobj->do_stream);
 			VDI_CloseFd(wrk, &wrk->busyobj->vbc);
 			HSH_Drop(wrk);
-			VBE_DerefBusyObj(&wrk->busyobj);
+			VBE_DerefBusyObj(wrk, &wrk->busyobj);
 		} else {
 			(void)HSH_Deref(wrk, NULL, &wrk->obj);
 		}
@@ -466,7 +466,7 @@ cnt_error(struct sess *sp)
 	if (wrk->obj == NULL) {
 		HSH_Prealloc(sp);
 		AZ(wrk->busyobj);
-		wrk->busyobj = VBE_GetBusyObj();
+		wrk->busyobj = VBE_GetBusyObj(wrk);
 		wrk->obj = STV_NewObject(wrk, NULL, cache_param->http_resp_size,
 		     (uint16_t)cache_param->http_max_hdr);
 		if (wrk->obj == NULL)
@@ -510,7 +510,7 @@ cnt_error(struct sess *sp)
 	if (sp->handling == VCL_RET_RESTART &&
 	    sp->restarts <  cache_param->max_restarts) {
 		HSH_Drop(wrk);
-		VBE_DerefBusyObj(&wrk->busyobj);
+		VBE_DerefBusyObj(wrk, &wrk->busyobj);
 		sp->director = NULL;
 		sp->restarts++;
 		sp->step = STP_RECV;
@@ -527,7 +527,7 @@ cnt_error(struct sess *sp)
 	sp->err_code = 0;
 	sp->err_reason = NULL;
 	http_Setup(wrk->bereq, NULL);
-	VBE_DerefBusyObj(&wrk->busyobj);
+	VBE_DerefBusyObj(wrk, &wrk->busyobj);
 	sp->step = STP_PREPRESP;
 	return (0);
 }
@@ -656,7 +656,7 @@ cnt_fetch(struct sess *sp)
 		AZ(HSH_Deref(wrk, wrk->objcore, NULL));
 		wrk->objcore = NULL;
 	}
-	VBE_DerefBusyObj(&wrk->busyobj);
+	VBE_DerefBusyObj(wrk, &wrk->busyobj);
 	http_Setup(wrk->bereq, NULL);
 	http_Setup(wrk->beresp, NULL);
 	sp->director = NULL;
@@ -831,7 +831,7 @@ cnt_fetchbody(struct sess *sp)
 		sp->err_code = 503;
 		sp->step = STP_ERROR;
 		VDI_CloseFd(wrk, &wrk->busyobj->vbc);
-		VBE_DerefBusyObj(&wrk->busyobj);
+		VBE_DerefBusyObj(wrk, &wrk->busyobj);
 		return (0);
 	}
 	CHECK_OBJ_NOTNULL(wrk->obj, OBJECT_MAGIC);
@@ -900,7 +900,7 @@ cnt_fetchbody(struct sess *sp)
 
 	if (i) {
 		HSH_Drop(wrk);
-		VBE_DerefBusyObj(&wrk->busyobj);
+		VBE_DerefBusyObj(wrk, &wrk->busyobj);
 		AZ(wrk->obj);
 		sp->err_code = 503;
 		sp->step = STP_ERROR;
@@ -913,7 +913,7 @@ cnt_fetchbody(struct sess *sp)
 		AN(wrk->obj->objcore->ban);
 		HSH_Unbusy(wrk);
 	}
-	VBE_DerefBusyObj(&wrk->busyobj);
+	VBE_DerefBusyObj(wrk, &wrk->busyobj);
 	wrk->acct_tmp.fetch++;
 	sp->step = STP_PREPRESP;
 	return (0);
@@ -987,7 +987,7 @@ cnt_streambody(struct sess *sp)
 	assert(WRW_IsReleased(wrk));
 	assert(wrk->wrw.ciov == wrk->wrw.siov);
 	(void)HSH_Deref(wrk, NULL, &wrk->obj);
-	VBE_DerefBusyObj(&wrk->busyobj);
+	VBE_DerefBusyObj(wrk, &wrk->busyobj);
 	http_Setup(wrk->resp, NULL);
 	sp->step = STP_DONE;
 	return (0);
@@ -1281,13 +1281,13 @@ cnt_miss(struct sess *sp)
 		AZ(HSH_Deref(wrk, wrk->objcore, NULL));
 		wrk->objcore = NULL;
 		http_Setup(wrk->bereq, NULL);
-		VBE_DerefBusyObj(&wrk->busyobj);
+		VBE_DerefBusyObj(wrk, &wrk->busyobj);
 		sp->step = STP_ERROR;
 		return (0);
 	case VCL_RET_PASS:
 		AZ(HSH_Deref(wrk, wrk->objcore, NULL));
 		wrk->objcore = NULL;
-		VBE_DerefBusyObj(&wrk->busyobj);
+		VBE_DerefBusyObj(wrk, &wrk->busyobj);
 		sp->step = STP_PASS;
 		return (0);
 	case VCL_RET_FETCH:
@@ -1297,7 +1297,7 @@ cnt_miss(struct sess *sp)
 	case VCL_RET_RESTART:
 		AZ(HSH_Deref(wrk, wrk->objcore, NULL));
 		wrk->objcore = NULL;
-		VBE_DerefBusyObj(&wrk->busyobj);
+		VBE_DerefBusyObj(wrk, &wrk->busyobj);
 		INCOMPL();
 	default:
 		WRONG("Illegal action in vcl_miss{}");
@@ -1348,6 +1348,7 @@ cnt_pass(struct sess *sp)
 	AZ(wrk->obj);
 	AZ(wrk->busyobj);
 
+	wrk->busyobj = VBE_GetBusyObj(wrk);
 	WS_Reset(wrk->ws, NULL);
 	http_Setup(wrk->bereq, wrk->ws);
 	http_FilterHeader(sp, HTTPH_R_PASS);
@@ -1358,6 +1359,7 @@ cnt_pass(struct sess *sp)
 	VCL_pass_method(sp);
 	if (sp->handling == VCL_RET_ERROR) {
 		http_Setup(wrk->bereq, NULL);
+		VBE_DerefBusyObj(wrk, &wrk->busyobj);
 		sp->step = STP_ERROR;
 		return (0);
 	}
@@ -1365,7 +1367,6 @@ cnt_pass(struct sess *sp)
 	wrk->acct_tmp.pass++;
 	sp->sendbody = 1;
 	sp->step = STP_FETCH;
-	wrk->busyobj = VBE_GetBusyObj();
 	return (0);
 }
 
@@ -1405,6 +1406,7 @@ cnt_pipe(struct sess *sp)
 	CHECK_OBJ_NOTNULL(sp->vcl, VCL_CONF_MAGIC);
 
 	wrk->acct_tmp.pipe++;
+	wrk->busyobj = VBE_GetBusyObj(wrk);
 	WS_Reset(wrk->ws, NULL);
 	http_Setup(wrk->bereq, wrk->ws);
 	http_FilterHeader(sp, HTTPH_R_PIPE);
@@ -1417,6 +1419,7 @@ cnt_pipe(struct sess *sp)
 
 	PipeSession(sp);
 	assert(WRW_IsReleased(wrk));
+	VBE_DerefBusyObj(wrk, &wrk->busyobj);
 	http_Setup(wrk->bereq, NULL);
 	sp->step = STP_DONE;
 	return (0);
diff --git a/bin/varnishd/cache/cache_hash.c b/bin/varnishd/cache/cache_hash.c
index e33cdbe..0bd002d 100644
--- a/bin/varnishd/cache/cache_hash.c
+++ b/bin/varnishd/cache/cache_hash.c
@@ -452,7 +452,7 @@ HSH_Lookup(struct sess *sp, struct objhead **poh)
 	oc->refcnt = 1;
 
 	AZ(w->busyobj);
-	w->busyobj = VBE_GetBusyObj();
+	w->busyobj = VBE_GetBusyObj(w);
 
 	VRY_Validate(sp->vary_b);
 	if (sp->vary_l != NULL)



From geoff at varnish-cache.org  Mon Jan  9 20:52:49 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:49 +0100
Subject: [experimental-ims] 67a4826 Move beresp and bereq to busyobj,
	but leave the memory allocation in worker for now.
Message-ID: <E1RkMD3-0000Dr-Jp@project.varnish-software.com>

commit 67a48263698c6cd5fb8c9e43768805345cc5bea6
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Dec 7 10:30:44 2011 +0000

    Move beresp and bereq to busyobj, but leave the memory allocation in
    worker for now.

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index 56f23fd..24c1dc9 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -328,8 +328,8 @@ struct worker {
 	const char		*storage_hint;
 
 	/* Fetch stuff.  Here because pipe has no busyobj */
-	struct http		*bereq;
-	struct http		*beresp;
+	struct http		*x_bereq;
+	struct http		*x_beresp;
 
 	/* Stream state */
 	struct stream_ctx	*sctx;
@@ -510,6 +510,8 @@ struct busyobj {
 	struct vgz		*vgz_rx;
 
 	struct vbc		*vbc;
+	struct http		*bereq;
+	struct http		*beresp;
 	struct object		*fetch_obj;
 	struct exp		exp;
 	struct http_conn	htc;
diff --git a/bin/varnishd/cache/cache_backend.c b/bin/varnishd/cache/cache_backend.c
index 21c1a4c..cf9d60f 100644
--- a/bin/varnishd/cache/cache_backend.c
+++ b/bin/varnishd/cache/cache_backend.c
@@ -96,6 +96,8 @@ VBE_GetBusyObj(struct worker *wrk)
 		busyobj = vbe_NewBusyObj();
 	AN(busyobj);
 	busyobj->refcount = 1;
+	busyobj->beresp = wrk->x_beresp;
+	busyobj->bereq = wrk->x_bereq;
 	return (busyobj);
 }
 
@@ -162,10 +164,10 @@ VDI_AddHostHeader(struct worker *wrk, const struct vbc *vbc)
 {
 
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
-	CHECK_OBJ_NOTNULL(wrk->bereq, HTTP_MAGIC);
+	CHECK_OBJ_NOTNULL(wrk->busyobj->bereq, HTTP_MAGIC);
 	CHECK_OBJ_NOTNULL(vbc, VBC_MAGIC);
 	CHECK_OBJ_NOTNULL(vbc->vdis, VDI_SIMPLE_MAGIC);
-	http_PrintfHeader(wrk, vbc->vsl_id, wrk->bereq,
+	http_PrintfHeader(wrk, vbc->vsl_id, wrk->busyobj->bereq,
 	    "Host: %s", vbc->vdis->vrt->hosthdr);
 }
 
diff --git a/bin/varnishd/cache/cache_center.c b/bin/varnishd/cache/cache_center.c
index 79e7956..442e2bb 100644
--- a/bin/varnishd/cache/cache_center.c
+++ b/bin/varnishd/cache/cache_center.c
@@ -262,8 +262,6 @@ cnt_prepresp(struct sess *sp)
 		AZ(wrk->obj);
 		sp->restarts++;
 		sp->director = NULL;
-		http_Setup(wrk->bereq, NULL);
-		http_Setup(wrk->beresp, NULL);
 		http_Setup(wrk->resp, NULL);
 		sp->step = STP_RECV;
 		return (0);
@@ -476,8 +474,8 @@ cnt_error(struct sess *sp)
 		if (wrk->obj == NULL) {
 			sp->doclose = "Out of objects";
 			sp->director = NULL;
-			http_Setup(wrk->beresp, NULL);
-			http_Setup(wrk->bereq, NULL);
+			http_Setup(wrk->busyobj->beresp, NULL);
+			http_Setup(wrk->busyobj->bereq, NULL);
 			sp->step = STP_DONE;
 			return(0);
 		}
@@ -526,7 +524,7 @@ cnt_error(struct sess *sp)
 	assert(sp->handling == VCL_RET_DELIVER);
 	sp->err_code = 0;
 	sp->err_reason = NULL;
-	http_Setup(wrk->bereq, NULL);
+	http_Setup(wrk->busyobj->bereq, NULL);
 	VBE_DerefBusyObj(wrk, &wrk->busyobj);
 	sp->step = STP_PREPRESP;
 	return (0);
@@ -579,9 +577,9 @@ cnt_fetch(struct sess *sp)
 	AZ(wrk->busyobj->should_close);
 	AZ(wrk->storage_hint);
 
-	http_Setup(wrk->beresp, wrk->ws);
+	http_Setup(wrk->busyobj->beresp, wrk->ws);
 
-	need_host_hdr = !http_GetHdr(wrk->bereq, H_Host, NULL);
+	need_host_hdr = !http_GetHdr(wrk->busyobj->bereq, H_Host, NULL);
 
 	i = FetchHdr(sp, need_host_hdr);
 	/*
@@ -603,8 +601,8 @@ cnt_fetch(struct sess *sp)
 		 * and we rely on their content outside of VCL, so collect them
 		 * into one line here.
 		 */
-		http_CollectHdr(wrk->beresp, H_Cache_Control);
-		http_CollectHdr(wrk->beresp, H_Vary);
+		http_CollectHdr(wrk->busyobj->beresp, H_Cache_Control);
+		http_CollectHdr(wrk->busyobj->beresp, H_Vary);
 
 		/*
 		 * Figure out how the fetch is supposed to happen, before the
@@ -613,7 +611,7 @@ cnt_fetch(struct sess *sp)
 		 */
 		wrk->busyobj->body_status = RFC2616_Body(sp);
 
-		sp->err_code = http_GetStatus(wrk->beresp);
+		sp->err_code = http_GetStatus(wrk->busyobj->beresp);
 
 		/*
 		 * What does RFC2616 think about TTL ?
@@ -657,8 +655,6 @@ cnt_fetch(struct sess *sp)
 		wrk->objcore = NULL;
 	}
 	VBE_DerefBusyObj(wrk, &wrk->busyobj);
-	http_Setup(wrk->bereq, NULL);
-	http_Setup(wrk->beresp, NULL);
 	sp->director = NULL;
 	wrk->storage_hint = NULL;
 
@@ -748,10 +744,10 @@ cnt_fetchbody(struct sess *sp)
 		wrk->busyobj->do_gzip = wrk->busyobj->do_gunzip = 0;
 
 	wrk->busyobj->is_gzip =
-	    http_HdrIs(wrk->beresp, H_Content_Encoding, "gzip");
+	    http_HdrIs(wrk->busyobj->beresp, H_Content_Encoding, "gzip");
 
 	wrk->busyobj->is_gunzip =
-	    !http_GetHdr(wrk->beresp, H_Content_Encoding, NULL);
+	    !http_GetHdr(wrk->busyobj->beresp, H_Content_Encoding, NULL);
 
 	/* It can't be both */
 	assert(wrk->busyobj->is_gzip == 0 || wrk->busyobj->is_gunzip == 0);
@@ -762,7 +758,7 @@ cnt_fetchbody(struct sess *sp)
 
 	/* If we do gunzip, remove the C-E header */
 	if (wrk->busyobj->do_gunzip)
-		http_Unset(wrk->beresp, H_Content_Encoding);
+		http_Unset(wrk->busyobj->beresp, H_Content_Encoding);
 
 	/* We wont gzip unless it is ungziped */
 	if (wrk->busyobj->do_gzip && !wrk->busyobj->is_gunzip)
@@ -770,7 +766,7 @@ cnt_fetchbody(struct sess *sp)
 
 	/* If we do gzip, add the C-E header */
 	if (wrk->busyobj->do_gzip)
-		http_SetHeader(wrk, sp->vsl_id, wrk->beresp,
+		http_SetHeader(wrk, sp->vsl_id, wrk->busyobj->beresp,
 		    "Content-Encoding: gzip");
 
 	/* But we can't do both at the same time */
@@ -791,13 +787,13 @@ cnt_fetchbody(struct sess *sp)
 	if (!sp->wantbody)
 		wrk->busyobj->do_stream = 0;
 
-	l = http_EstimateWS(wrk->beresp,
+	l = http_EstimateWS(wrk->busyobj->beresp,
 	    pass ? HTTPH_R_PASS : HTTPH_A_INS, &nhttp);
 
 	/* Create Vary instructions */
 	if (wrk->objcore != NULL) {
 		CHECK_OBJ_NOTNULL(wrk->objcore, OBJCORE_MAGIC);
-		vary = VRY_Create(sp, wrk->beresp);
+		vary = VRY_Create(sp, wrk->busyobj->beresp);
 		if (vary != NULL) {
 			varyl = VSB_len(vary);
 			assert(varyl > 0);
@@ -855,7 +851,7 @@ cnt_fetchbody(struct sess *sp)
 	WS_Assert(wrk->obj->ws_o);
 
 	/* Filter into object */
-	hp = wrk->beresp;
+	hp = wrk->busyobj->beresp;
 	hp2 = wrk->obj->http;
 
 	hp2->logtag = HTTP_Obj;
@@ -891,8 +887,8 @@ cnt_fetchbody(struct sess *sp)
 	/* Use unmodified headers*/
 	i = FetchBody(wrk, wrk->obj);
 
-	http_Setup(wrk->bereq, NULL);
-	http_Setup(wrk->beresp, NULL);
+	http_Setup(wrk->busyobj->bereq, NULL);
+	http_Setup(wrk->busyobj->beresp, NULL);
 	wrk->busyobj->vfp = NULL;
 	assert(WRW_IsReleased(wrk));
 	AZ(wrk->busyobj->vbc);
@@ -961,8 +957,8 @@ cnt_streambody(struct sess *sp)
 
 	i = FetchBody(wrk, wrk->obj);
 
-	http_Setup(wrk->bereq, NULL);
-	http_Setup(wrk->beresp, NULL);
+	http_Setup(wrk->busyobj->bereq, NULL);
+	http_Setup(wrk->busyobj->beresp, NULL);
 	wrk->busyobj->vfp = NULL;
 	AZ(wrk->busyobj->vbc);
 	AN(sp->director);
@@ -1075,8 +1071,8 @@ cnt_hit(struct sess *sp)
 	if (sp->handling == VCL_RET_DELIVER) {
 		/* Dispose of any body part of the request */
 		(void)FetchReqBody(sp);
-		AZ(wrk->bereq->ws);
-		AZ(wrk->beresp->ws);
+		//AZ(wrk->busyobj->bereq->ws);
+		//AZ(wrk->busyobj->beresp->ws);
 		sp->step = STP_PREPRESP;
 		return (0);
 	}
@@ -1257,17 +1253,18 @@ cnt_miss(struct sess *sp)
 	AN(wrk->objcore);
 	CHECK_OBJ_NOTNULL(wrk->busyobj, BUSYOBJ_MAGIC);
 	WS_Reset(wrk->ws, NULL);
-	http_Setup(wrk->bereq, wrk->ws);
+	wrk->busyobj = VBE_GetBusyObj(wrk);
+	http_Setup(wrk->busyobj->bereq, wrk->ws);
 	http_FilterHeader(sp, HTTPH_R_FETCH);
-	http_ForceGet(wrk->bereq);
+	http_ForceGet(wrk->busyobj->bereq);
 	if (cache_param->http_gzip_support) {
 		/*
 		 * We always ask the backend for gzip, even if the
 		 * client doesn't grok it.  We will uncompress for
 		 * the minority of clients which don't.
 		 */
-		http_Unset(wrk->bereq, H_Accept_Encoding);
-		http_SetHeader(wrk, sp->vsl_id, wrk->bereq,
+		http_Unset(wrk->busyobj->bereq, H_Accept_Encoding);
+		http_SetHeader(wrk, sp->vsl_id, wrk->busyobj->bereq,
 		    "Accept-Encoding: gzip");
 	}
 	wrk->connect_timeout = 0;
@@ -1280,7 +1277,7 @@ cnt_miss(struct sess *sp)
 	case VCL_RET_ERROR:
 		AZ(HSH_Deref(wrk, wrk->objcore, NULL));
 		wrk->objcore = NULL;
-		http_Setup(wrk->bereq, NULL);
+		http_Setup(wrk->busyobj->bereq, NULL);
 		VBE_DerefBusyObj(wrk, &wrk->busyobj);
 		sp->step = STP_ERROR;
 		return (0);
@@ -1350,7 +1347,8 @@ cnt_pass(struct sess *sp)
 
 	wrk->busyobj = VBE_GetBusyObj(wrk);
 	WS_Reset(wrk->ws, NULL);
-	http_Setup(wrk->bereq, wrk->ws);
+	wrk->busyobj = VBE_GetBusyObj(wrk);
+	http_Setup(wrk->busyobj->bereq, wrk->ws);
 	http_FilterHeader(sp, HTTPH_R_PASS);
 
 	wrk->connect_timeout = 0;
@@ -1358,7 +1356,7 @@ cnt_pass(struct sess *sp)
 	wrk->between_bytes_timeout = 0;
 	VCL_pass_method(sp);
 	if (sp->handling == VCL_RET_ERROR) {
-		http_Setup(wrk->bereq, NULL);
+		http_Setup(wrk->busyobj->bereq, NULL);
 		VBE_DerefBusyObj(wrk, &wrk->busyobj);
 		sp->step = STP_ERROR;
 		return (0);
@@ -1404,11 +1402,13 @@ cnt_pipe(struct sess *sp)
 	wrk = sp->wrk;
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 	CHECK_OBJ_NOTNULL(sp->vcl, VCL_CONF_MAGIC);
+	AZ(wrk->busyobj);
 
 	wrk->acct_tmp.pipe++;
 	wrk->busyobj = VBE_GetBusyObj(wrk);
 	WS_Reset(wrk->ws, NULL);
-	http_Setup(wrk->bereq, wrk->ws);
+	wrk->busyobj = VBE_GetBusyObj(wrk);
+	http_Setup(wrk->busyobj->bereq, wrk->ws);
 	http_FilterHeader(sp, HTTPH_R_PIPE);
 
 	VCL_pipe_method(sp);
@@ -1419,8 +1419,8 @@ cnt_pipe(struct sess *sp)
 
 	PipeSession(sp);
 	assert(WRW_IsReleased(wrk));
+	http_Setup(wrk->busyobj->bereq, NULL);
 	VBE_DerefBusyObj(wrk, &wrk->busyobj);
-	http_Setup(wrk->bereq, NULL);
 	sp->step = STP_DONE;
 	return (0);
 }
diff --git a/bin/varnishd/cache/cache_dir_dns.c b/bin/varnishd/cache/cache_dir_dns.c
index f735ebf..56bce48 100644
--- a/bin/varnishd/cache/cache_dir_dns.c
+++ b/bin/varnishd/cache/cache_dir_dns.c
@@ -347,8 +347,8 @@ vdi_dns_find_backend(const struct sess *sp, struct vdi_dns *vs)
 	/* bereq is only present after recv et. al, otherwise use req (ie:
 	 * use req for health checks in vcl_recv and such).
 	 */
-	if (sp->wrk->bereq)
-		hp = sp->wrk->bereq;
+	if (sp->wrk->busyobj != NULL && sp->wrk->busyobj->bereq)
+		hp = sp->wrk->busyobj->bereq;
 	else
 		hp = sp->http;
 
diff --git a/bin/varnishd/cache/cache_esi_parse.c b/bin/varnishd/cache/cache_esi_parse.c
index e431a8d..8352717 100644
--- a/bin/varnishd/cache/cache_esi_parse.c
+++ b/bin/varnishd/cache/cache_esi_parse.c
@@ -499,7 +499,7 @@ vep_do_include(struct vep_state *vep, enum dowhat what)
 		VSB_printf(vep->vsb, "%c", 0);
 	} else {
 		VSB_printf(vep->vsb, "%c", 0);
-		url = vep->wrk->bereq->hd[HTTP_HDR_URL];
+		url = vep->wrk->busyobj->bereq->hd[HTTP_HDR_URL];
 		/* Look for the last / before a '?' */
 		h = NULL;
 		for (q = url.b; q < url.e && *q != '?'; q++)
diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c
index 8a1ac5b..936dc30 100644
--- a/bin/varnishd/cache/cache_fetch.c
+++ b/bin/varnishd/cache/cache_fetch.c
@@ -398,7 +398,7 @@ FetchHdr(struct sess *sp, int need_host_hdr)
 		AN(sp->wrk->objcore->flags & OC_F_BUSY);
 	}
 
-	hp = wrk->bereq;
+	hp = wrk->busyobj->bereq;
 
 	sp->wrk->busyobj->vbc = VDI_GetFd(NULL, sp);
 	if (sp->wrk->busyobj->vbc == NULL) {
@@ -470,7 +470,7 @@ FetchHdr(struct sess *sp, int need_host_hdr)
 		}
 	}
 
-	hp = wrk->beresp;
+	hp = wrk->busyobj->beresp;
 
 	if (http_DissectResponse(wrk, htc, hp)) {
 		WSP(sp, SLT_FetchError, "http format error");
diff --git a/bin/varnishd/cache/cache_http.c b/bin/varnishd/cache/cache_http.c
index 267af00..4a6ba82 100644
--- a/bin/varnishd/cache/cache_http.c
+++ b/bin/varnishd/cache/cache_http.c
@@ -869,7 +869,7 @@ http_FilterHeader(const struct sess *sp, unsigned how)
 {
 	struct http *hp;
 
-	hp = sp->wrk->bereq;
+	hp = sp->wrk->busyobj->bereq;
 	CHECK_OBJ_NOTNULL(hp, HTTP_MAGIC);
 	hp->logtag = HTTP_Tx;
 
diff --git a/bin/varnishd/cache/cache_panic.c b/bin/varnishd/cache/cache_panic.c
index 9fa14e0..94a4c09 100644
--- a/bin/varnishd/cache/cache_panic.c
+++ b/bin/varnishd/cache/cache_panic.c
@@ -197,10 +197,10 @@ pan_wrk(const struct worker *wrk)
 
 	VSB_printf(pan_vsp, "  worker = %p {\n", wrk);
 	pan_ws(wrk->ws, 4);
-	if (wrk->bereq->ws != NULL)
-		pan_http("bereq", wrk->bereq, 4);
-	if (wrk->beresp->ws != NULL)
-		pan_http("beresp", wrk->beresp, 4);
+	if (wrk->busyobj != NULL && wrk->busyobj->bereq->ws != NULL)
+		pan_http("bereq", wrk->busyobj->bereq, 4);
+	if (wrk->busyobj != NULL && wrk->busyobj->beresp->ws != NULL)
+		pan_http("beresp", wrk->busyobj->beresp, 4);
 	if (wrk->resp->ws != NULL)
 		pan_http("resp", wrk->resp, 4);
 	VSB_printf(pan_vsp, "  },\n");
diff --git a/bin/varnishd/cache/cache_pipe.c b/bin/varnishd/cache/cache_pipe.c
index 5337eda..8963fb7 100644
--- a/bin/varnishd/cache/cache_pipe.c
+++ b/bin/varnishd/cache/cache_pipe.c
@@ -78,7 +78,7 @@ PipeSession(struct sess *sp)
 
 	WRW_Reserve(w, &vc->fd);
 	sp->wrk->acct_tmp.hdrbytes +=
-	    http_Write(w, sp->vsl_id, sp->wrk->bereq, 0);
+	    http_Write(w, sp->vsl_id, sp->wrk->busyobj->bereq, 0);
 
 	if (sp->htc->pipeline.b != NULL)
 		sp->wrk->acct_tmp.bodybytes +=
diff --git a/bin/varnishd/cache/cache_pool.c b/bin/varnishd/cache/cache_pool.c
index 582bb70..cf40e85 100644
--- a/bin/varnishd/cache/cache_pool.c
+++ b/bin/varnishd/cache/cache_pool.c
@@ -196,8 +196,8 @@ Pool_Work_Thread(void *priv, struct worker *w)
 		Lck_AssertHeld(&pp->mtx);
 
 		CHECK_OBJ_NOTNULL(w, WORKER_MAGIC);
-		CHECK_OBJ_NOTNULL(w->bereq, HTTP_MAGIC);
-		CHECK_OBJ_NOTNULL(w->beresp, HTTP_MAGIC);
+		CHECK_OBJ_NOTNULL(w->x_bereq, HTTP_MAGIC);
+		CHECK_OBJ_NOTNULL(w->x_beresp, HTTP_MAGIC);
 		CHECK_OBJ_NOTNULL(w->resp, HTTP_MAGIC);
 
 		WS_Reset(w->ws, NULL);
@@ -267,8 +267,7 @@ Pool_Work_Thread(void *priv, struct worker *w)
 			w->sp = NULL;
 
 			WS_Assert(w->ws);
-			AZ(w->bereq->ws);
-			AZ(w->beresp->ws);
+			AZ(w->busyobj);
 			AZ(w->resp->ws);
 			AZ(w->wrw.wfd);
 			AZ(w->storage_hint);
diff --git a/bin/varnishd/cache/cache_rfc2616.c b/bin/varnishd/cache/cache_rfc2616.c
index 599ffbb..6e7efea 100644
--- a/bin/varnishd/cache/cache_rfc2616.c
+++ b/bin/varnishd/cache/cache_rfc2616.c
@@ -73,7 +73,7 @@ RFC2616_Ttl(const struct sess *sp)
 
 	expp = &sp->wrk->busyobj->exp;
 
-	hp = sp->wrk->beresp;
+	hp = sp->wrk->busyobj->beresp;
 
 	assert(expp->entered != 0.0 && !isnan(expp->entered));
 	/* If all else fails, cache using default ttl */
@@ -183,7 +183,7 @@ RFC2616_Body(const struct sess *sp)
 	struct http *hp;
 	char *b;
 
-	hp = sp->wrk->beresp;
+	hp = sp->wrk->busyobj->beresp;
 
 	if (hp->protover < 11 && !http_HdrIs(hp, H_Connection, "keep-alive"))
 		sp->wrk->busyobj->should_close = 1;
@@ -192,7 +192,7 @@ RFC2616_Body(const struct sess *sp)
 	else
 		sp->wrk->busyobj->should_close = 0;
 
-	if (!strcasecmp(http_GetReq(sp->wrk->bereq), "head")) {
+	if (!strcasecmp(http_GetReq(sp->wrk->busyobj->bereq), "head")) {
 		/*
 		 * A HEAD request can never have a body in the reply,
 		 * no matter what the headers might say.
diff --git a/bin/varnishd/cache/cache_vrt.c b/bin/varnishd/cache/cache_vrt.c
index bbf93b5..a696f64 100644
--- a/bin/varnishd/cache/cache_vrt.c
+++ b/bin/varnishd/cache/cache_vrt.c
@@ -100,10 +100,10 @@ vrt_selecthttp(const struct sess *sp, enum gethdr_e where)
 		hp = sp->http;
 		break;
 	case HDR_BEREQ:
-		hp = sp->wrk->bereq;
+		hp = sp->wrk->busyobj->bereq;
 		break;
 	case HDR_BERESP:
-		hp = sp->wrk->beresp;
+		hp = sp->wrk->busyobj->beresp;
 		break;
 	case HDR_RESP:
 		hp = sp->wrk->resp;
diff --git a/bin/varnishd/cache/cache_vrt_var.c b/bin/varnishd/cache/cache_vrt_var.c
index cb54837..0662e26 100644
--- a/bin/varnishd/cache/cache_vrt_var.c
+++ b/bin/varnishd/cache/cache_vrt_var.c
@@ -85,15 +85,15 @@ VRT_r_##obj##_##hdr(const struct sess *sp)			\
 VRT_DO_HDR(req,   request,	sp->http,		HTTP_HDR_REQ)
 VRT_DO_HDR(req,   url,		sp->http,		HTTP_HDR_URL)
 VRT_DO_HDR(req,   proto,	sp->http,		HTTP_HDR_PROTO)
-VRT_DO_HDR(bereq, request,	sp->wrk->bereq,		HTTP_HDR_REQ)
-VRT_DO_HDR(bereq, url,		sp->wrk->bereq,		HTTP_HDR_URL)
-VRT_DO_HDR(bereq, proto,	sp->wrk->bereq,		HTTP_HDR_PROTO)
+VRT_DO_HDR(bereq, request,	sp->wrk->busyobj->bereq,	HTTP_HDR_REQ)
+VRT_DO_HDR(bereq, url,		sp->wrk->busyobj->bereq,	HTTP_HDR_URL)
+VRT_DO_HDR(bereq, proto,	sp->wrk->busyobj->bereq,	HTTP_HDR_PROTO)
 VRT_DO_HDR(obj,   proto,	sp->wrk->obj->http,		HTTP_HDR_PROTO)
 VRT_DO_HDR(obj,   response,	sp->wrk->obj->http,		HTTP_HDR_RESPONSE)
 VRT_DO_HDR(resp,  proto,	sp->wrk->resp,		HTTP_HDR_PROTO)
 VRT_DO_HDR(resp,  response,	sp->wrk->resp,		HTTP_HDR_RESPONSE)
-VRT_DO_HDR(beresp,  proto,	sp->wrk->beresp,	HTTP_HDR_PROTO)
-VRT_DO_HDR(beresp,  response,	sp->wrk->beresp,	HTTP_HDR_RESPONSE)
+VRT_DO_HDR(beresp,  proto,	sp->wrk->busyobj->beresp,	HTTP_HDR_PROTO)
+VRT_DO_HDR(beresp,  response,	sp->wrk->busyobj->beresp,	HTTP_HDR_RESPONSE)
 
 /*--------------------------------------------------------------------*/
 
@@ -115,7 +115,7 @@ VRT_r_##obj##_status(const struct sess *sp)			\
 }
 
 VRT_DO_STATUS(obj, sp->wrk->obj->http)
-VRT_DO_STATUS(beresp, sp->wrk->beresp)
+VRT_DO_STATUS(beresp, sp->wrk->busyobj->beresp)
 VRT_DO_STATUS(resp, sp->wrk->resp)
 
 /*--------------------------------------------------------------------*/
diff --git a/bin/varnishd/cache/cache_wrk.c b/bin/varnishd/cache/cache_wrk.c
index 1d80805..73eedb2 100644
--- a/bin/varnishd/cache/cache_wrk.c
+++ b/bin/varnishd/cache/cache_wrk.c
@@ -153,8 +153,8 @@ wrk_thread_real(void *priv, unsigned shm_workspace, unsigned sess_workspace,
 	w->wlb = w->wlp = wlog;
 	w->wle = wlog + (sizeof wlog) / 4;
 	w->sha256ctx = &sha256;
-	w->bereq = HTTP_create(http0, nhttp);
-	w->beresp = HTTP_create(http1, nhttp);
+	w->x_bereq = HTTP_create(http0, nhttp);
+	w->x_beresp = HTTP_create(http1, nhttp);
 	w->resp = HTTP_create(http2, nhttp);
 	w->wrw.iov = iov;
 	w->wrw.siov = siov;



From geoff at varnish-cache.org  Mon Jan  9 20:52:50 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:50 +0100
Subject: [experimental-ims] e541ffb Use the busyobj->vbc also for pipe,
	so that panic can see it.
Message-ID: <E1RkMD4-0000F6-5l@project.varnish-software.com>

commit e541ffbd0e75dfa163ea6dabfe426d905a5cb81a
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu Dec 8 08:01:08 2011 +0000

    Use the busyobj->vbc also for pipe, so that panic can see it.

diff --git a/bin/varnishd/cache/cache_pipe.c b/bin/varnishd/cache/cache_pipe.c
index 8963fb7..cc98fba 100644
--- a/bin/varnishd/cache/cache_pipe.c
+++ b/bin/varnishd/cache/cache_pipe.c
@@ -69,11 +69,13 @@ PipeSession(struct sess *sp)
 
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 	CHECK_OBJ_NOTNULL(sp->wrk, WORKER_MAGIC);
+	CHECK_OBJ_NOTNULL(sp->wrk->busyobj, BUSYOBJ_MAGIC);
 	w = sp->wrk;
 
 	vc = VDI_GetFd(NULL, sp);
 	if (vc == NULL)
 		return;
+	sp->wrk->busyobj->vbc = vc;		/* For panic dumping */
 	(void)VTCP_blocking(vc->fd);
 
 	WRW_Reserve(w, &vc->fd);
@@ -129,4 +131,5 @@ PipeSession(struct sess *sp)
 	}
 	SES_Close(sp, "pipe");
 	VDI_CloseFd(sp->wrk, &vc);
+	sp->wrk->busyobj->vbc = NULL;
 }



From geoff at varnish-cache.org  Mon Jan  9 20:52:14 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:14 +0100
Subject: [experimental-ims] 0e63559 First sweep to remove needless #includes.
Message-ID: <E1RkMCU-0007Re-MJ@project.varnish-software.com>

commit 0e63559a6cf53997d4dfa3ec601c472eb6889a23
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sat Oct 8 10:45:26 2011 +0000

    First sweep to remove needless #includes.
    
    This is based on output from some scripts and basic sanity testing.

diff --git a/bin/varnishd/cache_acceptor.c b/bin/varnishd/cache_acceptor.c
index e2be560..c02e170 100644
--- a/bin/varnishd/cache_acceptor.c
+++ b/bin/varnishd/cache_acceptor.c
@@ -31,12 +31,6 @@
 #include "config.h"
 
 #include <errno.h>
-#include <poll.h>
-#include <stdlib.h>
-#include <stdio.h>
-
-#include <sys/types.h>
-#include <sys/socket.h>
 
 #include "vcli.h"
 #include "cli_priv.h"
diff --git a/bin/varnishd/cache_backend_cfg.c b/bin/varnishd/cache_backend_cfg.c
index 0582f72..e08e09e 100644
--- a/bin/varnishd/cache_backend_cfg.c
+++ b/bin/varnishd/cache_backend_cfg.c
@@ -34,11 +34,6 @@
 
 #include <stdio.h>
 #include <stdlib.h>
-#include <string.h>
-#include <unistd.h>
-#include <poll.h>
-
-#include <sys/socket.h>
 
 #include "cache.h"
 #include "vrt.h"
diff --git a/bin/varnishd/cache_backend_poll.c b/bin/varnishd/cache_backend_poll.c
index 0854dfd..5fae557 100644
--- a/bin/varnishd/cache_backend_poll.c
+++ b/bin/varnishd/cache_backend_poll.c
@@ -40,12 +40,8 @@
 #include <stdio.h>
 #include <math.h>
 #include <stdlib.h>
-#include <string.h>
-#include <unistd.h>
 #include <poll.h>
 
-#include <sys/socket.h>
-
 #include "cli_priv.h"
 #include "cache.h"
 #include "vrt.h"
diff --git a/bin/varnishd/cache_cli.c b/bin/varnishd/cache_cli.c
index 79ac6f6..bfc1cbc 100644
--- a/bin/varnishd/cache_cli.c
+++ b/bin/varnishd/cache_cli.c
@@ -37,19 +37,14 @@
 #include "config.h"
 
 #include <stdio.h>
-#include <errno.h>
-#include <stddef.h>
-#include <stdlib.h>
-#include <string.h>
-#include <unistd.h>
-#include <poll.h>
+#include <stddef.h>			// offsetof
 
 #include "vcli.h"
 #include "cli_priv.h"
 #include "cli_common.h"
 #include "cli_serve.h"
 #include "cache.h"
-#include "hash_slinger.h"
+#include "hash_slinger.h"		// objhead
 
 pthread_t		cli_thread;
 static struct lock	cli_mtx;
diff --git a/bin/varnishd/cache_dir_random.c b/bin/varnishd/cache_dir_random.c
index c2b74dd..f05d8f4 100644
--- a/bin/varnishd/cache_dir_random.c
+++ b/bin/varnishd/cache_dir_random.c
@@ -46,15 +46,8 @@
 
 #include "config.h"
 
-#include <sys/types.h>
-#include <sys/socket.h>
-
-#include <stdio.h>
-#include <errno.h>
 #include <math.h>
 #include <stdlib.h>
-#include <string.h>
-#include <unistd.h>
 
 #include "cache.h"
 #include "cache_backend.h"
diff --git a/bin/varnishd/cache_dir_round_robin.c b/bin/varnishd/cache_dir_round_robin.c
index 8e47198..9d0a257 100644
--- a/bin/varnishd/cache_dir_round_robin.c
+++ b/bin/varnishd/cache_dir_round_robin.c
@@ -29,13 +29,7 @@
 
 #include "config.h"
 
-#include <sys/types.h>
-#include <sys/socket.h>
-
-#include <errno.h>
 #include <stdlib.h>
-#include <string.h>
-#include <unistd.h>
 
 #include "cache.h"
 #include "cache_backend.h"
diff --git a/bin/varnishd/cache_esi_deliver.c b/bin/varnishd/cache_esi_deliver.c
index 5bfa654..063464a 100644
--- a/bin/varnishd/cache_esi_deliver.c
+++ b/bin/varnishd/cache_esi_deliver.c
@@ -36,9 +36,7 @@
 #include "cache.h"
 #include "cache_esi.h"
 #include "vend.h"
-#include "vct.h"
 #include "vgz.h"
-#include "stevedore.h"
 
 /*--------------------------------------------------------------------*/
 
diff --git a/bin/varnishd/cache_esi_fetch.c b/bin/varnishd/cache_esi_fetch.c
index c8df1b1..cac2eb6 100644
--- a/bin/varnishd/cache_esi_fetch.c
+++ b/bin/varnishd/cache_esi_fetch.c
@@ -30,7 +30,6 @@
 
 #include "config.h"
 
-#include <stdio.h>
 #include <stdlib.h>
 
 #include "cache.h"
diff --git a/bin/varnishd/cache_esi_parse.c b/bin/varnishd/cache_esi_parse.c
index afef1fb..c659dd5 100644
--- a/bin/varnishd/cache_esi_parse.c
+++ b/bin/varnishd/cache_esi_parse.c
@@ -38,7 +38,6 @@
 #include "vend.h"
 #include "vct.h"
 #include "vgz.h"
-#include "stevedore.h"
 
 //#define Debug(fmt, ...) printf(fmt, __VA_ARGS__)
 #define Debug(fmt, ...) /**/
diff --git a/bin/varnishd/cache_expire.c b/bin/varnishd/cache_expire.c
index a07ab6d..35d0a1e 100644
--- a/bin/varnishd/cache_expire.c
+++ b/bin/varnishd/cache_expire.c
@@ -51,10 +51,6 @@
 
 #include "config.h"
 
-#include <unistd.h>
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
 #include <math.h>
 
 #include "binary_heap.h"
diff --git a/bin/varnishd/cache_gzip.c b/bin/varnishd/cache_gzip.c
index b32b5cf..28d61d9 100644
--- a/bin/varnishd/cache_gzip.c
+++ b/bin/varnishd/cache_gzip.c
@@ -68,9 +68,7 @@
 #include <stdio.h>
 #include <stdlib.h>
 
-#include "vsl.h"
 #include "cache.h"
-#include "stevedore.h"
 
 #include "vgz.h"
 
diff --git a/bin/varnishd/cache_hash.c b/bin/varnishd/cache_hash.c
index 58c3b6a..7e45b8a 100644
--- a/bin/varnishd/cache_hash.c
+++ b/bin/varnishd/cache_hash.c
@@ -56,9 +56,6 @@
 #include <stdlib.h>
 #include <math.h>
 #include <string.h>
-#include <limits.h>
-#include <sys/types.h>
-#include <fcntl.h>
 
 #include "cache.h"
 #include "stevedore.h"
diff --git a/bin/varnishd/cache_http.c b/bin/varnishd/cache_http.c
index c8a5415..25a6627 100644
--- a/bin/varnishd/cache_http.c
+++ b/bin/varnishd/cache_http.c
@@ -31,7 +31,6 @@
 
 #include "config.h"
 
-#include <errno.h>
 #include <unistd.h>
 #include <stdarg.h>
 #include <stdio.h>
diff --git a/bin/varnishd/cache_pipe.c b/bin/varnishd/cache_pipe.c
index ea5189d..e02da80 100644
--- a/bin/varnishd/cache_pipe.c
+++ b/bin/varnishd/cache_pipe.c
@@ -32,11 +32,7 @@
 #include "config.h"
 
 #include <stdio.h>
-#include <unistd.h>
-#include <string.h>
 #include <poll.h>
-#include <stdlib.h>
-#include <sys/socket.h>
 
 #include "cache.h"
 
diff --git a/bin/varnishd/cache_pool.c b/bin/varnishd/cache_pool.c
index 6be7613..1aaa2ac 100644
--- a/bin/varnishd/cache_pool.c
+++ b/bin/varnishd/cache_pool.c
@@ -49,11 +49,7 @@
 #include <math.h>
 #include <stdlib.h>
 #include <string.h>
-#include <time.h>
-#include <unistd.h>
 
-#include "vcl.h"
-#include "cli_priv.h"
 #include "cache.h"
 #include "cache_waiter.h"
 #include "stevedore.h"
diff --git a/bin/varnishd/cache_session.c b/bin/varnishd/cache_session.c
index 4956154..1f500ba 100644
--- a/bin/varnishd/cache_session.c
+++ b/bin/varnishd/cache_session.c
@@ -36,10 +36,7 @@
 #include "config.h"
 
 #include <stdlib.h>
-#include <string.h>
 #include <math.h>
-#include <sys/uio.h>
-#include <sys/socket.h>
 
 #include "cache.h"
 #include "cache_waiter.h"
diff --git a/bin/varnishd/cache_vrt_re.c b/bin/varnishd/cache_vrt_re.c
index b2afaa5..ef2376f 100644
--- a/bin/varnishd/cache_vrt_re.c
+++ b/bin/varnishd/cache_vrt_re.c
@@ -31,10 +31,6 @@
 
 #include "config.h"
 
-#include <sys/types.h>
-
-#include <stdio.h>
-#include <string.h>
 #include <ctype.h>
 #include <stdlib.h>
 
diff --git a/bin/varnishd/cache_wrk.c b/bin/varnishd/cache_wrk.c
index 0c3b18e..ba2e41a 100644
--- a/bin/varnishd/cache_wrk.c
+++ b/bin/varnishd/cache_wrk.c
@@ -31,17 +31,11 @@
 
 #include "config.h"
 
-#include <sys/types.h>
-
-#include <errno.h>
 #include <stdio.h>
 #include <math.h>
 #include <stdlib.h>
-#include <string.h>
-#include <unistd.h>
 
 #include "vcl.h"
-#include "cli_priv.h"
 #include "cache.h"
 #include "hash_slinger.h"
 #include "vsha256.h"
diff --git a/bin/varnishd/cache_ws.c b/bin/varnishd/cache_ws.c
index a803797..9fca215 100644
--- a/bin/varnishd/cache_ws.c
+++ b/bin/varnishd/cache_ws.c
@@ -30,17 +30,6 @@
 
 #include "config.h"
 
-#include <sys/types.h>
-#include <sys/uio.h>
-
-#include <errno.h>
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#include <unistd.h>
-
-#include "vcl.h"
-#include "cli_priv.h"
 #include "cache.h"
 
 void
diff --git a/bin/varnishd/hash_critbit.c b/bin/varnishd/hash_critbit.c
index 0a1ccca..7428097 100644
--- a/bin/varnishd/hash_critbit.c
+++ b/bin/varnishd/hash_critbit.c
@@ -32,7 +32,6 @@
 
 #include "config.h"
 
-#include <stdio.h>
 #include <stdlib.h>
 
 #include "cache.h"
diff --git a/bin/varnishd/hash_simple_list.c b/bin/varnishd/hash_simple_list.c
index 5cb1fc1..13cf0ee 100644
--- a/bin/varnishd/hash_simple_list.c
+++ b/bin/varnishd/hash_simple_list.c
@@ -31,10 +31,6 @@
 
 #include "config.h"
 
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-
 #include "cache.h"
 #include "hash_slinger.h"
 
diff --git a/bin/varnishd/mgt_cli.c b/bin/varnishd/mgt_cli.c
index 45f08af..eccf268 100644
--- a/bin/varnishd/mgt_cli.c
+++ b/bin/varnishd/mgt_cli.c
@@ -31,8 +31,6 @@
 
 #include "config.h"
 
-#include <sys/types.h>
-
 #include <stdarg.h>
 #include <stdio.h>
 #include <fcntl.h>
@@ -51,7 +49,6 @@
 #include "cli_common.h"
 #include "cli_serve.h"
 #include "vev.h"
-#include "vsc.h"
 #include "vlu.h"
 #include "vss.h"
 
diff --git a/bin/varnishd/rfc2616.c b/bin/varnishd/rfc2616.c
index 84e95f4..51d4c74 100644
--- a/bin/varnishd/rfc2616.c
+++ b/bin/varnishd/rfc2616.c
@@ -29,8 +29,6 @@
 
 #include "config.h"
 
-#include <sys/types.h>
-
 #include <stdio.h>
 #include <stdlib.h>
 #include <limits.h>
diff --git a/bin/varnishd/stevedore.c b/bin/varnishd/stevedore.c
index 624d2b1..c1e52cb 100644
--- a/bin/varnishd/stevedore.c
+++ b/bin/varnishd/stevedore.c
@@ -35,11 +35,9 @@
 
 #include <stdio.h>
 #include <stdlib.h>
-#include <string.h>
 
 #include "cache.h"
 #include "stevedore.h"
-#include "hash_slinger.h"
 #include "cli_priv.h"
 #include "vrt_obj.h"
 
diff --git a/bin/varnishd/stevedore_utils.c b/bin/varnishd/stevedore_utils.c
index bd4e368..1dcfd63 100644
--- a/bin/varnishd/stevedore_utils.c
+++ b/bin/varnishd/stevedore_utils.c
@@ -37,7 +37,7 @@
 #include <stdlib.h>
 #include <string.h>
 #include <unistd.h>
-#include <sys/param.h>
+#include <sys/types.h>
 #include <sys/stat.h>
 
 #ifdef HAVE_SYS_MOUNT_H
diff --git a/bin/varnishd/storage_file.c b/bin/varnishd/storage_file.c
index 30203f8..7388878 100644
--- a/bin/varnishd/storage_file.c
+++ b/bin/varnishd/storage_file.c
@@ -35,9 +35,6 @@
 #include <sys/mman.h>
 #include <sys/stat.h>
 
-#include <errno.h>
-#include <limits.h>
-#include <stdint.h>
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
diff --git a/bin/varnishd/storage_persistent.c b/bin/varnishd/storage_persistent.c
index a4ba598..3ba7061 100644
--- a/bin/varnishd/storage_persistent.c
+++ b/bin/varnishd/storage_persistent.c
@@ -35,8 +35,6 @@
 
 #include "config.h"
 
-#include <errno.h>
-#include <math.h>
 #include <stdio.h>
 #include <stdint.h>
 #include <stdlib.h>
diff --git a/bin/varnishd/storage_persistent_silo.c b/bin/varnishd/storage_persistent_silo.c
index a1eb98b..ba84b79 100644
--- a/bin/varnishd/storage_persistent_silo.c
+++ b/bin/varnishd/storage_persistent_silo.c
@@ -34,11 +34,8 @@
 
 #include "config.h"
 
-#include <math.h>
 #include <stdio.h>
-#include <stdint.h>
 #include <stdlib.h>
-#include <sys/mman.h>
 
 #include "cache.h"
 #include "stevedore.h"
diff --git a/bin/varnishd/varnishd.c b/bin/varnishd/varnishd.c
index 2962e74..8b35970 100644
--- a/bin/varnishd/varnishd.c
+++ b/bin/varnishd/varnishd.c
@@ -36,9 +36,6 @@
 #include <ctype.h>
 #include <errno.h>
 #include <fcntl.h>
-#include <poll.h>
-#include <signal.h>
-#include <stdarg.h>
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
@@ -64,11 +61,6 @@
 #include "hash_slinger.h"
 #include "stevedore.h"
 
-/* INFTIM indicates an infinite timeout for poll(2) */
-#ifndef INFTIM
-#define INFTIM -1
-#endif
-
 struct heritage		heritage;
 volatile struct params	*params;
 unsigned		d_flag = 0;



From geoff at varnish-cache.org  Mon Jan  9 20:52:44 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:44 +0100
Subject: [experimental-ims] 2c88943 Move fetch_object to busyobj
Message-ID: <E1RkMCy-0008Ue-75@project.varnish-software.com>

commit 2c88943245cc48241823b3848b40cfe56f8b7dcf
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Nov 29 18:30:02 2011 +0000

    Move fetch_object to busyobj

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index 125dd53..cb2a68e 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -329,7 +329,6 @@ struct worker {
 
 	/* Fetch stuff */
 	struct vbc		*vbc;
-	struct object		*fetch_obj;
 	enum body_status	body_status;
 	struct vef_priv		*vef_priv;
 	unsigned		do_stream;
@@ -501,6 +500,7 @@ struct busyobj {
 	unsigned		fetch_failed;
 	struct vgz		*vgz_rx;
 
+	struct object		*fetch_obj;
 	struct exp		exp;
 	struct http_conn	htc;
 };
diff --git a/bin/varnishd/cache/cache_esi_fetch.c b/bin/varnishd/cache/cache_esi_fetch.c
index 71dd815..5795849 100644
--- a/bin/varnishd/cache/cache_esi_fetch.c
+++ b/bin/varnishd/cache/cache_esi_fetch.c
@@ -81,7 +81,7 @@ vfp_esi_bytes_uu(struct worker *w, struct http_conn *htc, ssize_t bytes)
 			return (wl);
 		VEP_Parse(w, (const char *)st->ptr + st->len, wl);
 		st->len += wl;
-		w->fetch_obj->len += wl;
+		w->busyobj->fetch_obj->len += wl;
 		bytes -= wl;
 	}
 	return (1);
@@ -117,7 +117,7 @@ vfp_esi_bytes_gu(struct worker *w, struct http_conn *htc, ssize_t bytes)
 		i = VGZ_Gunzip(vg, &dp, &dl);
 		xxxassert(i == VGZ_OK || i == VGZ_END);
 		VEP_Parse(w, dp, dl);
-		w->fetch_obj->len += dl;
+		w->busyobj->fetch_obj->len += dl;
 	}
 	return (1);
 }
@@ -187,7 +187,7 @@ vfp_vep_callback(struct worker *w, ssize_t l, enum vgz_flag flg)
 			}
 			i = VGZ_Gzip(vef->vgz, &dp, &dl, flg);
 			vef->tot += dl;
-			w->fetch_obj->len += dl;
+			w->busyobj->fetch_obj->len += dl;
 		} while (!VGZ_IbufEmpty(vef->vgz) ||
 		    (flg != VGZ_NORMAL && VGZ_ObufFull(vef->vgz)));
 		if (px != 0) {
@@ -374,11 +374,11 @@ vfp_esi_end(struct worker *w)
 			l = VSB_len(vsb);
 			assert(l > 0);
 			/* XXX: This is a huge waste of storage... */
-			w->fetch_obj->esidata = STV_alloc(w, l);
-			if (w->fetch_obj->esidata != NULL) {
-				memcpy(w->fetch_obj->esidata->ptr,
+			w->busyobj->fetch_obj->esidata = STV_alloc(w, l);
+			if (w->busyobj->fetch_obj->esidata != NULL) {
+				memcpy(w->busyobj->fetch_obj->esidata->ptr,
 				    VSB_data(vsb), l);
-				w->fetch_obj->esidata->len = l;
+				w->busyobj->fetch_obj->esidata->len = l;
 			} else {
 				retval = FetchError(w,
 				    "Could not allocate storage for esidata");
@@ -391,7 +391,7 @@ vfp_esi_end(struct worker *w)
 		vef = w->vef_priv;
 		CHECK_OBJ_NOTNULL(vef, VEF_MAGIC);
 		w->vef_priv = NULL;
-		VGZ_UpdateObj(vef->vgz, w->fetch_obj);
+		VGZ_UpdateObj(vef->vgz, w->busyobj->fetch_obj);
 		if (VGZ_Destroy(&vef->vgz,  -1) != VGZ_END)
 			retval = FetchError(w,
 			    "ESI+Gzip Failed at the very end");
diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c
index 0b86821..7180a5b 100644
--- a/bin/varnishd/cache/cache_fetch.c
+++ b/bin/varnishd/cache/cache_fetch.c
@@ -124,7 +124,7 @@ vfp_nop_bytes(struct worker *w, struct http_conn *htc, ssize_t bytes)
 		if (wl <= 0)
 			return (wl);
 		st->len += wl;
-		w->fetch_obj->len += wl;
+		w->busyobj->fetch_obj->len += wl;
 		bytes -= wl;
 		if (w->do_stream)
 			RES_StreamPoll(w);
@@ -146,12 +146,12 @@ vfp_nop_end(struct worker *w)
 {
 	struct storage *st;
 
-	st = VTAILQ_LAST(&w->fetch_obj->store, storagehead);
+	st = VTAILQ_LAST(&w->busyobj->fetch_obj->store, storagehead);
 	if (st == NULL)
 		return (0);
 
 	if (st->len == 0) {
-		VTAILQ_REMOVE(&w->fetch_obj->store, st, list);
+		VTAILQ_REMOVE(&w->busyobj->fetch_obj->store, st, list);
 		STV_free(st);
 		return (0);
 	}
@@ -178,7 +178,7 @@ FetchStorage(struct worker *w, ssize_t sz)
 	struct storage *st;
 	struct object *obj;
 
-	obj = w->fetch_obj;
+	obj = w->busyobj->fetch_obj;
 	CHECK_OBJ_NOTNULL(obj, OBJECT_MAGIC);
 	st = VTAILQ_LAST(&obj->store, storagehead);
 	if (st != NULL && st->len < st->space)
@@ -495,7 +495,7 @@ FetchBody(struct worker *w, struct object *obj)
 
 	CHECK_OBJ_NOTNULL(w, WORKER_MAGIC);
 	CHECK_OBJ_NOTNULL(w->busyobj, BUSYOBJ_MAGIC);
-	AZ(w->fetch_obj);
+	AZ(w->busyobj->fetch_obj);
 	CHECK_OBJ_NOTNULL(w->vbc, VBC_MAGIC);
 	CHECK_OBJ_NOTNULL(obj, OBJECT_MAGIC);
 	CHECK_OBJ_NOTNULL(obj->http, HTTP_MAGIC);
@@ -510,7 +510,7 @@ FetchBody(struct worker *w, struct object *obj)
 	AZ(w->busyobj->vgz_rx);
 	AZ(VTAILQ_FIRST(&obj->store));
 
-	w->fetch_obj = obj;
+	w->busyobj->fetch_obj = obj;
 	w->busyobj->fetch_failed = 0;
 
 	/* XXX: pick up estimate from objdr ? */
@@ -564,7 +564,7 @@ FetchBody(struct worker *w, struct object *obj)
 	 */
 	AZ(vfp_nop_end(w));
 
-	w->fetch_obj = NULL;
+	w->busyobj->fetch_obj = NULL;
 
 	WSLB(w, SLT_Fetch_Body, "%u(%s) cls %d mklen %u",
 	    w->body_status, body_status(w->body_status),
diff --git a/bin/varnishd/cache/cache_gzip.c b/bin/varnishd/cache/cache_gzip.c
index f325292..4076081 100644
--- a/bin/varnishd/cache/cache_gzip.c
+++ b/bin/varnishd/cache/cache_gzip.c
@@ -492,7 +492,7 @@ vfp_gunzip_bytes(struct worker *w, struct http_conn *htc, ssize_t bytes)
 		i = VGZ_Gunzip(vg, &dp, &dl);
 		if (i != VGZ_OK && i != VGZ_END)
 			return(FetchError(w, "Gunzip data error"));
-		w->fetch_obj->len += dl;
+		w->busyobj->fetch_obj->len += dl;
 		if (w->do_stream)
 			RES_StreamPoll(w);
 	}
@@ -568,7 +568,7 @@ vfp_gzip_bytes(struct worker *w, struct http_conn *htc, ssize_t bytes)
 			return(-1);
 		i = VGZ_Gzip(vg, &dp, &dl, VGZ_NORMAL);
 		assert(i == Z_OK);
-		w->fetch_obj->len += dl;
+		w->busyobj->fetch_obj->len += dl;
 		if (w->do_stream)
 			RES_StreamPoll(w);
 	}
@@ -595,11 +595,11 @@ vfp_gzip_end(struct worker *w)
 		if (VGZ_ObufStorage(w, vg))
 			return(-1);
 		i = VGZ_Gzip(vg, &dp, &dl, VGZ_FINISH);
-		w->fetch_obj->len += dl;
+		w->busyobj->fetch_obj->len += dl;
 	} while (i != Z_STREAM_END);
 	if (w->do_stream)
 		RES_StreamPoll(w);
-	VGZ_UpdateObj(vg, w->fetch_obj);
+	VGZ_UpdateObj(vg, w->busyobj->fetch_obj);
 	if (VGZ_Destroy(&vg, -1) != VGZ_END)
 		return(FetchError(w, "Gzip error at the very end"));
 	return (0);
@@ -654,7 +654,7 @@ vfp_testgzip_bytes(struct worker *w, struct http_conn *htc, ssize_t bytes)
 		bytes -= wl;
 		VGZ_Ibuf(vg, st->ptr + st->len, wl);
 		st->len += wl;
-		w->fetch_obj->len += wl;
+		w->busyobj->fetch_obj->len += wl;
 		if (w->do_stream)
 			RES_StreamPoll(w);
 
@@ -684,7 +684,7 @@ vfp_testgzip_end(struct worker *w)
 		(void)VGZ_Destroy(&vg, -1);
 		return(0);
 	}
-	VGZ_UpdateObj(vg, w->fetch_obj);
+	VGZ_UpdateObj(vg, w->busyobj->fetch_obj);
 	if (VGZ_Destroy(&vg, -1) != VGZ_END)
 		return(FetchError(w, "TestGunzip error at the very end"));
 	return (0);
diff --git a/bin/varnishd/cache/cache_response.c b/bin/varnishd/cache/cache_response.c
index bdc0fca..39c0ee6 100644
--- a/bin/varnishd/cache/cache_response.c
+++ b/bin/varnishd/cache/cache_response.c
@@ -365,14 +365,14 @@ RES_StreamPoll(struct worker *w)
 	void *ptr;
 
 	CHECK_OBJ_NOTNULL(w, WORKER_MAGIC);
-	CHECK_OBJ_NOTNULL(w->fetch_obj, OBJECT_MAGIC);
+	CHECK_OBJ_NOTNULL(w->busyobj->fetch_obj, OBJECT_MAGIC);
 	sctx = w->sctx;
 	CHECK_OBJ_NOTNULL(sctx, STREAM_CTX_MAGIC);
-	if (w->fetch_obj->len == sctx->stream_next)
+	if (w->busyobj->fetch_obj->len == sctx->stream_next)
 		return;
-	assert(w->fetch_obj->len > sctx->stream_next);
+	assert(w->busyobj->fetch_obj->len > sctx->stream_next);
 	l = sctx->stream_front;
-	VTAILQ_FOREACH(st, &w->fetch_obj->store, list) {
+	VTAILQ_FOREACH(st, &w->busyobj->fetch_obj->store, list) {
 		if (st->len + l <= sctx->stream_next) {
 			l += st->len;
 			continue;
@@ -391,18 +391,18 @@ RES_StreamPoll(struct worker *w)
 	if (!(w->res_mode & RES_GUNZIP))
 		(void)WRW_Flush(w);
 
-	if (w->fetch_obj->objcore == NULL ||
-	    (w->fetch_obj->objcore->flags & OC_F_PASS)) {
+	if (w->busyobj->fetch_obj->objcore == NULL ||
+	    (w->busyobj->fetch_obj->objcore->flags & OC_F_PASS)) {
 		/*
 		 * This is a pass object, release storage as soon as we
 		 * have delivered it.
 		 */
 		while (1) {
-			st = VTAILQ_FIRST(&w->fetch_obj->store);
+			st = VTAILQ_FIRST(&w->busyobj->fetch_obj->store);
 			if (st == NULL ||
 			    sctx->stream_front + st->len > sctx->stream_next)
 				break;
-			VTAILQ_REMOVE(&w->fetch_obj->store, st, list);
+			VTAILQ_REMOVE(&w->busyobj->fetch_obj->store, st, list);
 			sctx->stream_front += st->len;
 			STV_free(st);
 		}
diff --git a/bin/varnishd/storage/stevedore.c b/bin/varnishd/storage/stevedore.c
index d0b4226..2748999 100644
--- a/bin/varnishd/storage/stevedore.c
+++ b/bin/varnishd/storage/stevedore.c
@@ -367,7 +367,7 @@ struct storage *
 STV_alloc(struct worker *w, size_t size)
 {
 
-	return (stv_alloc(w, w->fetch_obj, size));
+	return (stv_alloc(w, w->busyobj->fetch_obj, size));
 }
 
 void



From geoff at varnish-cache.org  Mon Jan  9 20:52:02 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:02 +0100
Subject: [experimental-ims] 558ee90 Add mising ;'s
Message-ID: <E1RkMCI-0007E9-7S@project.varnish-software.com>

commit 558ee90f00a157c6452b35ff9eda974f53d993c6
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Fri Sep 30 15:39:55 2011 +0000

    Add mising ;'s

diff --git a/bin/varnishd/mgt.h b/bin/varnishd/mgt.h
index b6dac01..24293c2 100644
--- a/bin/varnishd/mgt.h
+++ b/bin/varnishd/mgt.h
@@ -73,8 +73,8 @@ void mgt_sandbox(void);
 
 /* mgt_sandbox_solaris.c */
 #ifdef HAVE_SETPPRIV
-void mgt_sandbox_solaris_init(void)
-void mgt_sandbox_solaris_fini(void)
+void mgt_sandbox_solaris_init(void);
+void mgt_sandbox_solaris_fini(void);
 #endif
 
 /* mgt_shmem.c */



From geoff at varnish-cache.org  Mon Jan  9 20:52:42 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:42 +0100
Subject: [experimental-ims] e67da13 Rename shmem.rst to vsm.rst
Message-ID: <E1RkMCw-0008Q9-6t@project.varnish-software.com>

commit e67da13c4d8d35543ce5b6bfc7092a1efbc6c6de
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu Nov 24 10:49:45 2011 +0000

    Rename shmem.rst to vsm.rst

diff --git a/doc/sphinx/Makefile.am b/doc/sphinx/Makefile.am
index dc20abd..d2e5258 100644
--- a/doc/sphinx/Makefile.am
+++ b/doc/sphinx/Makefile.am
@@ -116,7 +116,6 @@ EXTRA_DIST = \
 	phk/three-zero.rst \
 	phk/vcl_expr.rst \
 	reference/index.rst \
-	reference/shmem.rst \
 	reference/params.rst \
 	reference/varnishadm.rst \
 	reference/varnish-cli.rst \
@@ -132,6 +131,7 @@ EXTRA_DIST = \
 	reference/vcl.rst \
 	reference/vmod.rst \
 	reference/vmod_std.rst \
+	reference/vsm.rst \
 	tutorial/advanced_backend_servers.rst \
 	tutorial/advanced_topics.rst \
 	tutorial/backend_servers.rst \
diff --git a/doc/sphinx/reference/index.rst b/doc/sphinx/reference/index.rst
index 84449da..3af59c9 100644
--- a/doc/sphinx/reference/index.rst
+++ b/doc/sphinx/reference/index.rst
@@ -18,7 +18,7 @@ The Varnish Reference Manual
 	varnishstat.rst
 	varnishtest.rst
 	varnishtop.rst
-	shmem.rst
+	vsm.rst
 	vmod.rst
 	vmod_std.rst
 	vsl.rst
diff --git a/doc/sphinx/reference/shmem.rst b/doc/sphinx/reference/shmem.rst
deleted file mode 100644
index 26c5525..0000000
--- a/doc/sphinx/reference/shmem.rst
+++ /dev/null
@@ -1,65 +0,0 @@
-%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-Shared Memory Logging and Statistics
-%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-
-Varnish uses shared memory for logging and statistics, because it
-is faster and much more efficient.  But it is also tricky in ways
-a regular logfile is not.
-
-When you open a file in "append" mode, the operating system guarantees
-that whatever you write will not overwrite existing data in the file.
-The neat result of this is that multiple procesess or threads writing
-to the same file does not even need to know about each other, it all
-works just as you would expect.
-
-With a shared memory log, we get no help from the kernel, the writers
-need to make sure they do not stomp on each other, and they need to
-make it possible and safe for the readers to access the data.
-
-The "CS101" way, is to introduce locks, and much time is spent examining
-the relative merits of the many kinds of locks available.
-
-Inside the varnishd (worker) process, we use mutexes to guarantee
-consistency, both with respect to allocations, log entries and stats
-counters.
-
-We do not want a varnishncsa trying to push data through a stalled
-ssh connection to stall the delivery of content, so readers like
-that are purely read-only, they do not get to affect the varnishd
-process and that means no locks for them.
-
-Instead we use "stable storage" concepts, to make sure the view
-seen by the readers is consistent at all times.
-
-As long as you only add stuff, that is trivial, but taking away
-stuff, such as when a backend is taken out of the configuration,
-we need to give the readers a chance to discover this, a "cooling
-off" period.
-
-When Varnishd starts, if it finds an existing shared memory file,
-and it can safely read the master_pid field, it will check if that
-process is running, and if so, fail with an error message, indicating
-that -n arguments collide.
-
-In all other cases, it will delete and create a new shmlog file,
-in order to provide running readers a cooling off period, where
-they can discover that there is a new shmlog file, by doing a
-stat(2) call and checking the st_dev & st_inode fields.
-
-Allocations
------------
-
-Sections inside the shared memory file are allocated dynamically,
-for instance when a new backend is added.
-
-While changes happen to the linked list of allocations, the "alloc_seq"
-header field is zero, and after the change, it gets a value different
-from what it had before.
-
-Deallocations
--------------
-
-When a section is freed, its class will change to "Cool" for at
-least 10 seconds, giving programs using it time to detect the 
-change in alloc_seq header field and/or the change of class.
-
diff --git a/doc/sphinx/reference/vsm.rst b/doc/sphinx/reference/vsm.rst
new file mode 100644
index 0000000..26c5525
--- /dev/null
+++ b/doc/sphinx/reference/vsm.rst
@@ -0,0 +1,65 @@
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+Shared Memory Logging and Statistics
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+
+Varnish uses shared memory for logging and statistics, because it
+is faster and much more efficient.  But it is also tricky in ways
+a regular logfile is not.
+
+When you open a file in "append" mode, the operating system guarantees
+that whatever you write will not overwrite existing data in the file.
+The neat result of this is that multiple procesess or threads writing
+to the same file does not even need to know about each other, it all
+works just as you would expect.
+
+With a shared memory log, we get no help from the kernel, the writers
+need to make sure they do not stomp on each other, and they need to
+make it possible and safe for the readers to access the data.
+
+The "CS101" way, is to introduce locks, and much time is spent examining
+the relative merits of the many kinds of locks available.
+
+Inside the varnishd (worker) process, we use mutexes to guarantee
+consistency, both with respect to allocations, log entries and stats
+counters.
+
+We do not want a varnishncsa trying to push data through a stalled
+ssh connection to stall the delivery of content, so readers like
+that are purely read-only, they do not get to affect the varnishd
+process and that means no locks for them.
+
+Instead we use "stable storage" concepts, to make sure the view
+seen by the readers is consistent at all times.
+
+As long as you only add stuff, that is trivial, but taking away
+stuff, such as when a backend is taken out of the configuration,
+we need to give the readers a chance to discover this, a "cooling
+off" period.
+
+When Varnishd starts, if it finds an existing shared memory file,
+and it can safely read the master_pid field, it will check if that
+process is running, and if so, fail with an error message, indicating
+that -n arguments collide.
+
+In all other cases, it will delete and create a new shmlog file,
+in order to provide running readers a cooling off period, where
+they can discover that there is a new shmlog file, by doing a
+stat(2) call and checking the st_dev & st_inode fields.
+
+Allocations
+-----------
+
+Sections inside the shared memory file are allocated dynamically,
+for instance when a new backend is added.
+
+While changes happen to the linked list of allocations, the "alloc_seq"
+header field is zero, and after the change, it gets a value different
+from what it had before.
+
+Deallocations
+-------------
+
+When a section is freed, its class will change to "Cool" for at
+least 10 seconds, giving programs using it time to detect the 
+change in alloc_seq header field and/or the change of class.
+



From geoff at varnish-cache.org  Mon Jan  9 20:52:01 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:01 +0100
Subject: [experimental-ims] f837fbc Split solaris sandboxing out to a separate
	source file,
	and apply patch received from Nils Goroll <nils.goroll@uplex.de>
Message-ID: <E1RkMCH-0007Co-O9@project.varnish-software.com>

commit f837fbca893cc09458482c5283456bf8990aeee6
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Fri Sep 30 13:30:43 2011 +0000

    Split solaris sandboxing out to a separate source file, and apply
    patch received from Nils Goroll <nils.goroll at uplex.de>
    
    - [e0ee2a2e69654a9df74aaf3dcadc9639659cf42b] adds the file_read
      privilege needed for onnv_140 and newer (see #912), but we also need
      the file_write privilege for stevedore access.
    
    - If available, keep sys_resource in the permitted/limited set to
      allow cache_waiter_ports to raise the process.max-port-events
      resource control (feature to be added later).
    
    - When starting varnish with euid 0 on Solaris, privilege seperation
      prohibited preserving additional privileges (in excess of the basic
      set) in the child, because, for a non privilege aware process,
      setuid() resets the effective, inheritable and permitted sets to the
      basic set.
    
      To achieve interoperability between solaris privileges and
      setuid()/setgid(), we now make the varnish child privilege aware
      before calling setuid() by trying to add all privileges we will need
      plus proc_setid.
    
    - On solaris, check for proc_setid rather than checking the euid as a
      prerequisite for changing the uid/gid and only change the uid/gid if
      we need to (for a privilege aware process, [ers]uid 0 loose their
      magic powers).
    
      Note that setuid() will always set SNOCD on Solaris, which will
      prevent core dumps from being written, unless setuid core dumps are
      explicitly enabled using coreadm(1M).
    
      To avoid setuid() (and the SNOCD flag, consequently), start varnish
      as the user you intend to run the child as, but with additional
      privileges, e.g. using
    
      ppriv -e -s A=basic,net_privaddr,sys_resource varnishd ...
    
    - setppriv(PRIV_SET, ...) failed when the privileges to be applied
      were not available in the permitted set.
    
      We change the logic to only clear the privileges which are not
      needed by inverting the sets and removing all unneeded privileges
      using setppriv(PRIV_OFF, ...).
    
      So the child might end up with less privileges than given initially,

diff --git a/bin/varnishd/Makefile.am b/bin/varnishd/Makefile.am
index c13f4e5..0dbc745 100644
--- a/bin/varnishd/Makefile.am
+++ b/bin/varnishd/Makefile.am
@@ -60,6 +60,7 @@ varnishd_SOURCES = \
 	mgt_param.c \
 	mgt_pool.c \
 	mgt_sandbox.c \
+	mgt_sandbox_solaris.c \
 	mgt_shmem.c \
 	mgt_vcc.c \
 	rfc2616.c \
diff --git a/bin/varnishd/mgt.h b/bin/varnishd/mgt.h
index 485de26..b6dac01 100644
--- a/bin/varnishd/mgt.h
+++ b/bin/varnishd/mgt.h
@@ -71,6 +71,12 @@ void MCF_DumpRst(void);
 /* mgt_sandbox.c */
 void mgt_sandbox(void);
 
+/* mgt_sandbox_solaris.c */
+#ifdef HAVE_SETPPRIV
+void mgt_sandbox_solaris_init(void)
+void mgt_sandbox_solaris_fini(void)
+#endif
+
 /* mgt_shmem.c */
 void mgt_SHM_Init(const char *arg);
 void mgt_SHM_Pid(void);
diff --git a/bin/varnishd/mgt_sandbox.c b/bin/varnishd/mgt_sandbox.c
index a5eee2f..94f9f4d 100644
--- a/bin/varnishd/mgt_sandbox.c
+++ b/bin/varnishd/mgt_sandbox.c
@@ -48,10 +48,6 @@
 #include <syslog.h>
 #include <unistd.h>
 
-#ifdef HAVE_PRIV_H
-#include <priv.h>
-#endif
-
 #ifdef __linux__
 #include <sys/prctl.h>
 #endif
@@ -67,6 +63,10 @@ void
 mgt_sandbox(void)
 {
 
+#ifdef HAVE_SETPPRIV
+	mgt_sandbox_solaris_init();
+#endif
+
 	if (geteuid() == 0) {
 		XXXAZ(setgid(params->gid));
 		XXXAZ(setuid(params->uid));
@@ -84,35 +84,7 @@ mgt_sandbox(void)
 #endif
 
 #ifdef HAVE_SETPPRIV
-	priv_set_t *empty, *minimal;
-
-	if (!(empty = priv_allocset()) ||
-	    !(minimal = priv_allocset())) {
-		REPORT0(LOG_ERR, "priv_allocset_failed");
-	} else {
-		priv_emptyset(empty);
-		priv_emptyset(minimal);
-
-		/*
-		 * new privilege,
-		 * silently ignore any errors if it doesn't exist
-		 */
-		priv_addset(minimal, "net_access");
-		priv_addset(minimal, "file_read");
-
-#define SETPPRIV(which, set)						\
-		if (setppriv(PRIV_SET, which, set))			\
-			REPORT0(LOG_ERR,				\
-			    "Waiving privileges failed on " #which)
-
-		/* need to set I after P to avoid SNOCD being set */
-		SETPPRIV(PRIV_LIMIT, minimal);
-		SETPPRIV(PRIV_PERMITTED, minimal); /* implies PRIV_EFFECTIVE */
-		SETPPRIV(PRIV_INHERITABLE, empty);
-
-		priv_freeset(empty);
-		priv_freeset(minimal);
-	}
+	mgt_sandbox_solaris_fini();
 #endif
 
 }
diff --git a/bin/varnishd/mgt_sandbox_solaris.c b/bin/varnishd/mgt_sandbox_solaris.c
new file mode 100644
index 0000000..15b7c95
--- /dev/null
+++ b/bin/varnishd/mgt_sandbox_solaris.c
@@ -0,0 +1,219 @@
+/*-
+ * Copyright (c) 2006-2011 Varnish Software AS
+ * All rights reserved.
+ *
+ * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
+ *	   Nils Goroll <nils.goroll at uplex.de>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * Sandboxing child processes on Solaris
+ *
+ */
+
+#include "config.h"
+
+#ifdef HAVE_SETPPRIV
+
+#include <stdio.h>
+#include <syslog.h>
+#include <unistd.h>
+#include <string.h>
+
+#ifdef HAVE_PRIV_H
+#include <priv.h>
+#endif
+
+#include "mgt.h"
+#include "heritage.h"
+
+/*--------------------------------------------------------------------
+ * SOLARIS PRIVILEGES: Note on use of symbolic PRIV_* constants
+ * 
+ * For privileges which existed in Solaris 10 FCS, we may use the constants from
+ * sys/priv_names.h
+ *
+ * For privileges which have been added later, we need to use strings in order
+ * not to break builds of varnish on these platforms. To remain binary
+ * compatible, we need to silently ignore errors from priv_addset when using
+ * these strings.
+ *
+ * For optimal build and binary forward comatibility, we could use subtractive
+ * set specs like
+ *
+ *       basic,!file_link_any,!proc_exec,!proc_fork,!proc_info,!proc_session
+ *
+ * but I (Nils) have a preference for making an informed decision about which
+ * privileges the varnish child should have and which it shouldn't.
+ *
+ * Newly introduced privileges should be annotated with their PSARC / commit ID
+ * (as long as Oracle reveils these :/ )
+ *
+ * SOLARIS PRIVILEGES: Note on accidentally setting the SNOCD flag
+ *
+ * When setting privileges, we need to take care not to accidentally set the
+ * SNOCD flag which will disable core dumps unnecessarily. (see
+ * https://www.varnish-cache.org/trac/ticket/671 )
+ *
+ * When changing the logic herein, always check with mdb -k. Replace _PID_ with
+ * the pid of your varnish child, the result should be 0, otherwise a regression
+ * has been introduced.
+ *
+ * > 0t_PID_::pid2proc | ::print proc_t p_flag | >a
+ * > (<a & 0x10000000)=X
+ *                 0
+ *
+ * (a value of 0x10000000 indicates that SNOCD is set)
+ *
+ * NOTE that on Solaris changing the uid will _always_ set SNOCD, so make sure
+ * you run this test with appropriate privileges, but without proc_setid, so
+ * varnish won't setuid(), e.g.
+ *
+ * pfexec ppriv -e -s A=basic,net_privaddr,sys_resource varnish ...
+ *
+ * SOLARIS COREDUMPS with setuid(): See coreadm(1M) - global-setid / proc-setid
+ *
+ */
+
+/* effective during runtime of the child */
+static inline void
+mgt_sandbox_solaris_add_effective(priv_set_t *pset)
+{
+	/* PSARC/2009/685 - 8eca52188202 - onnv_132 */
+	priv_addset(pset, "net_access");
+
+	/* PSARC/2009/378 - 63678502e95e - onnv_140 */
+	priv_addset(pset, "file_read");
+	priv_addset(pset, "file_write");
+}
+
+/* permitted during runtime of the child - for privilege bracketing */
+static inline void
+mgt_sandbox_solaris_add_permitted(priv_set_t *pset)
+{
+	/* for raising limits in cache_waiter_ports.c */
+	priv_addset(pset, PRIV_SYS_RESOURCE);
+}
+
+/* effective during mgt_sandbox */
+static inline void
+mgt_sandbox_solaris_add_initial(priv_set_t *pset)
+{
+	/* for setgid/setuid */
+	priv_addset(pset, PRIV_PROC_SETID);
+}
+
+/*
+ * if we are not yet privilege-aware already (ie we have been started
+ * not-privilege aware wird euid 0), we need to grab any additional privileges
+ * needed during mgt_standbox, until we reduce to least privileges in
+ * mgt_sandbox_waive, otherwise we would loose them with setuid()
+ */
+
+void
+mgt_sandbox_solaris_init(void)
+{
+	priv_set_t *priv_all;
+
+	if (! (priv_all = priv_allocset())) {
+		REPORT(LOG_ERR,
+		    "Child start warning: mgt_sandbox_init - priv_allocset failed: errno=%d (%s)",
+		    errno, strerror(errno));
+		return;
+	}
+	
+	priv_emptyset(priv_all);
+
+	mgt_sandbox_solaris_add_effective(priv_all);
+	mgt_sandbox_solaris_add_permitted(priv_all);
+	mgt_sandbox_solaris_add_initial(priv_all);
+
+	setppriv(PRIV_ON, PRIV_PERMITTED, priv_all);
+	setppriv(PRIV_ON, PRIV_EFFECTIVE, priv_all);
+	setppriv(PRIV_ON, PRIV_INHERITABLE, priv_all);
+
+	priv_freeset(priv_all);
+}
+
+/* 
+ * Waive most privileges in the child
+ *
+ * as of onnv_151a, we should end up with:
+ *
+ * > ppriv -v #pid of varnish child
+ * PID:  .../varnishd ...
+ * flags = PRIV_AWARE
+ *      E: file_read,file_write,net_access
+ *      I: none
+ *      P: file_read,file_write,net_access,sys_resource
+ *      L: file_read,file_write,net_access,sys_resource
+ *
+ * We should keep sys_resource in P in order to adjust our limits if we need to
+ */
+
+void
+mgt_sandbox_solaris_fini(void)
+{
+	priv_set_t *effective, *inheritable, *permitted;
+
+	if (!(effective = priv_allocset()) ||
+	    !(inheritable = priv_allocset()) ||
+	    !(permitted = priv_allocset())) {
+		REPORT(LOG_ERR,
+		    "Child start warning: mgt_sandbox_waive - priv_allocset failed: errno=%d (%s)",
+		    errno, strerror(errno));
+		return;
+	}
+
+	priv_emptyset(inheritable);
+
+	priv_emptyset(effective);
+	mgt_sandbox_solaris_add_effective(effective);
+
+	priv_copyset(effective, permitted);
+	mgt_sandbox_solaris_add_permitted(permitted);
+
+	/* 
+	 * invert the sets and clear privileges such that setppriv will always
+	 * succeed
+	 */
+	priv_inverse(inheritable);
+	priv_inverse(effective);
+	priv_inverse(permitted);
+
+#define SETPPRIV(which, set)						\
+	if (setppriv(PRIV_OFF, which, set))				\
+		REPORT(LOG_ERR,						\
+		    "Child start warning: Waiving privileges failed on %s: errno=%d (%s)", \
+		    #which, errno, strerror(errno));
+
+	SETPPRIV(PRIV_INHERITABLE, inheritable);
+	SETPPRIV(PRIV_EFFECTIVE, effective);
+	SETPPRIV(PRIV_PERMITTED, permitted);
+	SETPPRIV(PRIV_LIMIT, permitted);
+#undef SETPPRIV
+
+	priv_freeset(inheritable);
+	priv_freeset(effective);
+}
+
+#endif /* HAVE_SETPPRIV */



From geoff at varnish-cache.org  Mon Jan  9 20:52:30 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:30 +0100
Subject: [experimental-ims] 561be2e Make it possible to limit the total
	transfer time.
Message-ID: <E1RkMCk-00084u-2k@project.varnish-software.com>

commit 561be2ef8a694fc39b34cf577011a549ba17d417
Author: Tollef Fog Heen <tfheen at varnish-software.com>
Date:   Mon Nov 7 14:20:06 2011 +0100

    Make it possible to limit the total transfer time.
    
    Set the default total transfer time to 600s.

diff --git a/bin/varnishd/cache_acceptor.c b/bin/varnishd/cache_acceptor.c
index f92bace..03a1a34 100644
--- a/bin/varnishd/cache_acceptor.c
+++ b/bin/varnishd/cache_acceptor.c
@@ -318,9 +318,9 @@ vca_acct(void *arg)
 	while (1) {
 		(void)sleep(1);
 #ifdef SO_SNDTIMEO_WORKS
-		if (params->send_timeout != send_timeout) {
+		if (params->idle_send_timeout != send_timeout) {
 			need_test = 1;
-			send_timeout = params->send_timeout;
+			send_timeout = params->idle_send_timeout;
 			tv_sndtimeo = VTIM_timeval(send_timeout);
 			VTAILQ_FOREACH(ls, &heritage.socks, list) {
 				if (ls->sock < 0)
diff --git a/bin/varnishd/cache_wrw.c b/bin/varnishd/cache_wrw.c
index 5673b54..3e2de4a 100644
--- a/bin/varnishd/cache_wrw.c
+++ b/bin/varnishd/cache_wrw.c
@@ -51,6 +51,7 @@
 #include <stdio.h>
 
 #include "cache.h"
+#include "vtim.h"
 
 /*--------------------------------------------------------------------
  */
@@ -133,6 +134,14 @@ WRW_Flush(struct worker *w)
 			*/
 			size_t used = 0;
 
+			if (VTIM_real() - w->sp->t_resp > params->send_timeout) {
+				WSL(w, SLT_Debug, *wrw->wfd,
+				    "Hit total send timeout, wrote = %ld/%ld; not retrying",
+				    i, wrw->liov);
+				i = -1;
+				break;
+			}
+
 			WSL(w, SLT_Debug, *wrw->wfd,
 			    "Hit send timeout, wrote = %ld/%ld; retrying",
 			    i, wrw->liov);
diff --git a/bin/varnishd/heritage.h b/bin/varnishd/heritage.h
index bb684b3..3732f18 100644
--- a/bin/varnishd/heritage.h
+++ b/bin/varnishd/heritage.h
@@ -112,6 +112,7 @@ struct params {
 	unsigned		sess_timeout;
 	unsigned		pipe_timeout;
 	unsigned		send_timeout;
+	unsigned		idle_send_timeout;
 
 	/* Management hints */
 	unsigned		auto_restart;
diff --git a/bin/varnishd/mgt/mgt_param.c b/bin/varnishd/mgt/mgt_param.c
index 8665ff5..3169914 100644
--- a/bin/varnishd/mgt/mgt_param.c
+++ b/bin/varnishd/mgt/mgt_param.c
@@ -613,6 +613,13 @@ static const struct parspec input_parspec[] = {
                 "seconds the session is closed. \n"
 		"See setsockopt(2) under SO_SNDTIMEO for more information.",
 		DELAYED_EFFECT,
+		"600", "seconds" },
+	{ "idle_send_timeout", tweak_timeout, &master.idle_send_timeout, 0, 0,
+		"Time to wait with no data sent. "
+		"If no data has been transmitted in this many\n"
+                "seconds the session is closed. \n"
+		"See setsockopt(2) under SO_SNDTIMEO for more information.",
+		DELAYED_EFFECT,
 		"60", "seconds" },
 	{ "auto_restart", tweak_bool, &master.auto_restart, 0, 0,
 		"Restart child process automatically if it dies.\n",



From geoff at varnish-cache.org  Mon Jan  9 20:52:42 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:42 +0100
Subject: [experimental-ims] ace762d Merge with trunk as of 2011-11-22
Message-ID: <E1RkMCw-0008QZ-EO@project.varnish-software.com>

commit ace762d59b261f22d14849677e297e70e4f6b1d0
Merge: 71ee192 aed74d6
Author: Geoff Simmons <geoff at uplex.de>
Date:   Fri Nov 25 11:17:52 2011 +0100

    Merge with trunk as of 2011-11-22

diff --cc bin/varnishd/cache/cache.h
index 0000000,4b66309..4df2e3d
mode 000000,100644..100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@@ -1,0 -1,1032 +1,1040 @@@
+ /*-
+  * Copyright (c) 2006 Verdens Gang AS
+  * Copyright (c) 2006-2011 Varnish Software AS
+  * All rights reserved.
+  *
+  * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
+  *
+  * Redistribution and use in source and binary forms, with or without
+  * modification, are permitted provided that the following conditions
+  * are met:
+  * 1. Redistributions of source code must retain the above copyright
+  *    notice, this list of conditions and the following disclaimer.
+  * 2. Redistributions in binary form must reproduce the above copyright
+  *    notice, this list of conditions and the following disclaimer in the
+  *    documentation and/or other materials provided with the distribution.
+  *
+  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+  * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+  * SUCH DAMAGE.
+  *
+  */
+ 
+ /*
+  * This macro can be used in .h files to isolate bits that the manager
+  * should not (need to) see, such as pthread mutexes etc.
+  */
+ #define VARNISH_CACHE_CHILD	1
+ 
+ #include "common/common.h"
+ 
+ #include "vapi/vsc_int.h"
+ #include "vapi/vsl_int.h"
+ 
+ #include <sys/socket.h>
+ 
+ #include <pthread.h>
+ #ifdef HAVE_PTHREAD_NP_H
+ #include <pthread_np.h>
+ #endif
+ #include <stdarg.h>
+ #include <string.h>
+ #include <limits.h>
+ #include <unistd.h>
+ 
+ #if defined(HAVE_EPOLL_CTL)
+ #include <sys/epoll.h>
+ #endif
+ 
+ 
+ #include "common/params.h"
+ 
+ enum body_status {
+ #define BODYSTATUS(U,l)	BS_##U,
+ #include "tbl/body_status.h"
+ #undef BODYSTATUS
+ };
+ 
+ static inline const char *
+ body_status(enum body_status e)
+ {
+ 	switch(e) {
+ #define BODYSTATUS(U,l)	case BS_##U: return (#l);
+ #include "tbl/body_status.h"
+ #undef BODYSTATUS
+ 	default:
+ 		return ("?");
+ 	}
+ }
+ 
+ /*
+  * NB: HDR_STATUS is only used in cache_http.c, everybody else uses the
+  * http->status integer field.
+  */
+ 
+ enum {
+ 	/* Fields from the first line of HTTP proto */
+ 	HTTP_HDR_REQ,
+ 	HTTP_HDR_URL,
+ 	HTTP_HDR_PROTO,
+ 	HTTP_HDR_STATUS,
+ 	HTTP_HDR_RESPONSE,
+ 	/* HTTP header lines */
+ 	HTTP_HDR_FIRST,
+ };
+ 
+ struct SHA256Context;
+ struct VSC_C_lck;
+ struct ban;
+ struct busyobj;
+ struct cli;
+ struct cli_proto;
+ struct director;
+ struct iovec;
+ struct objcore;
+ struct object;
+ struct objhead;
+ struct pool;
+ struct sess;
+ struct sesspool;
+ struct vbc;
+ struct vef_priv;
+ struct vrt_backend;
+ struct vsb;
+ struct waitinglist;
+ struct worker;
+ 
+ #define DIGEST_LEN		32
+ 
+ /*--------------------------------------------------------------------*/
+ 
+ typedef struct {
+ 	char			*b;
+ 	char			*e;
+ } txt;
+ 
+ /*--------------------------------------------------------------------*/
+ 
+ enum step {
+ #define STEP(l, u)	STP_##u,
+ #include "tbl/steps.h"
+ #undef STEP
+ };
+ 
+ /*--------------------------------------------------------------------*/
+ struct lock { void *priv; };	// Opaque
+ 
+ /*--------------------------------------------------------------------
+  * Workspace structure for quick memory allocation.
+  */
+ 
+ struct ws {
+ 	unsigned		magic;
+ #define WS_MAGIC		0x35fac554
+ 	unsigned		overflow;	/* workspace overflowed */
+ 	const char		*id;		/* identity */
+ 	char			*s;		/* (S)tart of buffer */
+ 	char			*f;		/* (F)ree pointer */
+ 	char			*r;		/* (R)eserved length */
+ 	char			*e;		/* (E)nd of buffer */
+ };
+ 
+ /*--------------------------------------------------------------------
+  * HTTP Request/Response/Header handling structure.
+  */
+ 
+ enum httpwhence {
+ 	HTTP_Rx	 = 1,
+ 	HTTP_Tx  = 2,
+ 	HTTP_Obj = 3
+ };
+ 
+ /* NB: remember to update http_Copy() if you add fields */
+ struct http {
+ 	unsigned		magic;
+ #define HTTP_MAGIC		0x6428b5c9
+ 
+ 	enum httpwhence		logtag;
+ 
+ 	struct ws		*ws;
+ 	txt			*hd;
+ 	unsigned char		*hdf;
+ #define HDF_FILTER		(1 << 0)	/* Filtered by Connection */
+ 	uint16_t		shd;		/* Size of hd space */
+ 	uint16_t		nhd;		/* Next free hd */
+ 	uint16_t		status;
+ 	uint8_t			protover;
+ 	uint8_t			conds;		/* If-* headers present */
+ };
+ 
+ /*--------------------------------------------------------------------
+  * HTTP Protocol connection structure
+  */
+ 
+ struct http_conn {
+ 	unsigned		magic;
+ #define HTTP_CONN_MAGIC		0x3e19edd1
+ 
+ 	int			fd;
+ 	unsigned		vsl_id;
+ 	unsigned		maxbytes;
+ 	unsigned		maxhdr;
+ 	struct ws		*ws;
+ 	txt			rxbuf;
+ 	txt			pipeline;
+ };
+ 
+ /*--------------------------------------------------------------------*/
+ 
+ struct acct {
+ 	double			first;
+ #define ACCT(foo)	uint64_t	foo;
+ #include "tbl/acct_fields.h"
+ #undef ACCT
+ };
+ 
+ /*--------------------------------------------------------------------*/
+ 
+ #define L0(t, n)
+ #define L1(t, n)		t n;
+ #define VSC_F(n, t, l, f, e,d)	L##l(t, n)
+ #define VSC_DO_MAIN
+ struct dstat {
+ #include "tbl/vsc_fields.h"
+ };
+ #undef VSC_F
+ #undef VSC_DO_MAIN
+ #undef L0
+ #undef L1
+ 
+ /* Fetch processors --------------------------------------------------*/
+ 
+ typedef void vfp_begin_f(struct worker *, size_t );
+ typedef int vfp_bytes_f(struct worker *, struct http_conn *, ssize_t);
+ typedef int vfp_end_f(struct worker *);
+ 
+ struct vfp {
+ 	vfp_begin_f	*begin;
+ 	vfp_bytes_f	*bytes;
+ 	vfp_end_f	*end;
+ };
+ 
+ extern struct vfp vfp_gunzip;
+ extern struct vfp vfp_gzip;
+ extern struct vfp vfp_testgzip;
+ extern struct vfp vfp_esi;
+ 
+ /*--------------------------------------------------------------------*/
+ 
+ struct exp {
+ 	double			ttl;
+ 	double			grace;
+ 	double			keep;
+ 	double			age;
+ 	double			entered;
+ };
+ 
+ /*--------------------------------------------------------------------*/
+ 
+ struct wrw {
+ 	int			*wfd;
+ 	unsigned		werr;	/* valid after WRW_Flush() */
+ 	struct iovec		*iov;
+ 	unsigned		siov;
+ 	unsigned		niov;
+ 	ssize_t			liov;
+ 	ssize_t			cliov;
+ 	unsigned		ciov;	/* Chunked header marker */
+ };
+ 
+ /*--------------------------------------------------------------------*/
+ 
+ struct stream_ctx {
+ 	unsigned		magic;
+ #define STREAM_CTX_MAGIC	0x8213728b
+ 
+ 	struct vgz		*vgz;
+ 	void			*obuf;
+ 	ssize_t			obuf_len;
+ 	ssize_t			obuf_ptr;
+ 
+ 	/* Next byte we will take from storage */
+ 	ssize_t			stream_next;
+ 
+ 	/* First byte of storage if we free it as we go (pass) */
+ 	ssize_t			stream_front;
+ };
+ 
+ /*--------------------------------------------------------------------*/
+ 
+ struct wrk_accept {
+ 	unsigned		magic;
+ #define WRK_ACCEPT_MAGIC	0x8c4b4d59
+ 
+ 	/* Accept stuff */
+ 	struct sockaddr_storage	acceptaddr;
+ 	socklen_t		acceptaddrlen;
+ 	int			acceptsock;
+ 	struct listen_sock	*acceptlsock;
+ };
+ 
+ /*--------------------------------------------------------------------*/
+ 
+ struct worker {
+ 	unsigned		magic;
+ #define WORKER_MAGIC		0x6391adcf
+ 	struct pool		*pool;
+ 	struct objhead		*nobjhead;
+ 	struct objcore		*nobjcore;
+ 	struct waitinglist	*nwaitinglist;
+ 	struct busyobj		*nbusyobj;
+ 	void			*nhashpriv;
+ 	struct dstat		stats;
+ 
+ 	/* Pool stuff */
+ 	double			lastused;
+ 
+ 	struct wrw		wrw;
+ 
+ 	pthread_cond_t		cond;
+ 
+ 	VTAILQ_ENTRY(worker)	list;
+ 	struct sess		*sp;
+ 
+ 	struct VCL_conf		*vcl;
+ 
+ 	uint32_t		*wlb, *wlp, *wle;
+ 	unsigned		wlr;
+ 
+ 	/* Lookup stuff */
+ 	struct SHA256Context	*sha256ctx;
+ 
+ 	struct http_conn	htc[1];
+ 	struct ws		ws[1];
+ 	struct http		*bereq;
+ 	struct http		*beresp;
+ 	struct http		*resp;
+ 
+ 	struct exp		exp;
+ 
+ 	/* This is only here so VRT can find it */
+ 	const char		*storage_hint;
+ 
+ 	/* Fetch stuff */
+ 	struct vbc		*vbc;
+ 	struct object		*fetch_obj;
+ 	enum body_status	body_status;
+ 	struct vfp		*vfp;
+ 	struct vgz		*vgz_rx;
+ 	struct vef_priv		*vef_priv;
+ 	unsigned		fetch_failed;
+ 	unsigned		do_stream;
+ 	unsigned		do_esi;
+ 	unsigned		do_gzip;
+ 	unsigned		is_gzip;
+ 	unsigned		do_gunzip;
+ 	unsigned		is_gunzip;
+ 	unsigned		do_close;
+ 	char			*h_content_length;
+ 
+ 	/* Stream state */
+ 	struct stream_ctx	*sctx;
+ 
+ 	/* ESI stuff */
+ 	struct vep_state	*vep;
+ 	int			gzip_resp;
+ 	ssize_t			l_crc;
+ 	uint32_t		crc;
+ 
+ 	/* Timeouts */
+ 	double			connect_timeout;
+ 	double			first_byte_timeout;
+ 	double			between_bytes_timeout;
+ 
+ 	/* Delivery mode */
+ 	unsigned		res_mode;
+ #define RES_LEN			(1<<1)
+ #define RES_EOF			(1<<2)
+ #define RES_CHUNKED		(1<<3)
+ #define RES_ESI			(1<<4)
+ #define RES_ESI_CHILD		(1<<5)
+ #define RES_GUNZIP		(1<<6)
+ 
+ 	/* Temporary accounting */
+ 	struct acct		acct_tmp;
+ };
+ 
+ /* LRU ---------------------------------------------------------------*/
+ 
+ struct lru {
+ 	unsigned		magic;
+ #define LRU_MAGIC		0x3fec7bb0
+ 	VTAILQ_HEAD(,objcore)	lru_head;
+ 	struct lock		mtx;
+ };
+ 
+ /* Storage -----------------------------------------------------------*/
+ 
+ struct storage {
+ 	unsigned		magic;
+ #define STORAGE_MAGIC		0x1a4e51c0
+ 
+ #ifdef SENDFILE_WORKS
+ 	int			fd;
+ 	off_t			where;
+ #endif
+ 
+ 	VTAILQ_ENTRY(storage)	list;
+ 	struct stevedore	*stevedore;
+ 	void			*priv;
+ 
+ 	unsigned char		*ptr;
+ 	unsigned		len;
+ 	unsigned		space;
+ };
+ 
+ /* Object core structure ---------------------------------------------
+  * Objects have sideways references in the binary heap and the LRU list
+  * and we want to avoid paging in a lot of objects just to move them up
+  * or down the binheap or to move a unrelated object on the LRU list.
+  * To avoid this we use a proxy object, objcore, to hold the relevant
+  * housekeeping fields parts of an object.
+  */
+ 
+ typedef struct object *getobj_f(struct worker *wrk, struct objcore *oc);
+ typedef void updatemeta_f(struct objcore *oc);
+ typedef void freeobj_f(struct objcore *oc);
+ typedef struct lru *getlru_f(const struct objcore *oc);
+ 
+ struct objcore_methods {
+ 	getobj_f	*getobj;
+ 	updatemeta_f	*updatemeta;
+ 	freeobj_f	*freeobj;
+ 	getlru_f	*getlru;
+ };
+ 
+ struct objcore {
+ 	unsigned		magic;
+ #define OBJCORE_MAGIC		0x4d301302
+ 	unsigned		refcnt;
+ 	struct objcore_methods	*methods;
+ 	void			*priv;
+ 	unsigned		priv2;
+ 	struct objhead		*objhead;
+ 	struct busyobj		*busyobj;
+ 	double			timer_when;
+ 	unsigned		flags;
+ #define OC_F_BUSY		(1<<1)
+ #define OC_F_PASS		(1<<2)
+ #define OC_F_LRUDONTMOVE	(1<<4)
+ #define OC_F_PRIV		(1<<5)		/* Stevedore private flag */
+ #define OC_F_LURK		(3<<6)		/* Ban-lurker-color */
+ 	unsigned		timer_idx;
+ 	VTAILQ_ENTRY(objcore)	list;
+ 	VTAILQ_ENTRY(objcore)	lru_list;
+ 	VTAILQ_ENTRY(objcore)	ban_list;
+ 	struct ban		*ban;
+ };
+ 
+ static inline struct object *
+ oc_getobj(struct worker *wrk, struct objcore *oc)
+ {
+ 
+ 	CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
+ 	AZ(oc->flags & OC_F_BUSY);
+ 	AN(oc->methods);
+ 	AN(oc->methods->getobj);
+ 	return (oc->methods->getobj(wrk, oc));
+ }
+ 
+ static inline void
+ oc_updatemeta(struct objcore *oc)
+ {
+ 
+ 	CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
+ 	AN(oc->methods);
+ 	if (oc->methods->updatemeta != NULL)
+ 		oc->methods->updatemeta(oc);
+ }
+ 
+ static inline void
+ oc_freeobj(struct objcore *oc)
+ {
+ 
+ 	CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
+ 	AN(oc->methods);
+ 	AN(oc->methods->freeobj);
+ 	oc->methods->freeobj(oc);
+ }
+ 
+ static inline struct lru *
+ oc_getlru(const struct objcore *oc)
+ {
+ 
+ 	CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
+ 	AN(oc->methods);
+ 	AN(oc->methods->getlru);
+ 	return (oc->methods->getlru(oc));
+ }
+ 
+ /* Busy Object structure ---------------------------------------------*/
+ 
+ struct busyobj {
+ 	unsigned		magic;
+ #define BUSYOBJ_MAGIC		0x23b95567
+ 	uint8_t			*vary;
+ };
+ 
+ /* Object structure --------------------------------------------------*/
+ 
+ VTAILQ_HEAD(storagehead, storage);
+ 
+ struct object {
+ 	unsigned		magic;
+ #define OBJECT_MAGIC		0x32851d42
+ 	unsigned		xid;
+ 	struct storage		*objstore;
+ 	struct objcore		*objcore;
+ 
+ 	struct ws		ws_o[1];
+ 
+ 	uint8_t			*vary;
+ 	unsigned		hits;
+ 	uint16_t		response;
+ 
+ 	/* XXX: make bitmap */
+ 	uint8_t			gziped;
+ 	/* Bit positions in the gzip stream */
+ 	ssize_t			gzip_start;
+ 	ssize_t			gzip_last;
+ 	ssize_t			gzip_stop;
+ 
+ 	ssize_t			len;
+ 
+ 	struct exp		exp;
+ 
+ 	double			last_modified;
+ 	double			last_lru;
+ 
+ 	struct http		*http;
+ 
+ 	struct storagehead	store;
+ 
+ 	struct storage		*esidata;
+ 
+ 	double			last_use;
+ 
+ };
+ 
+ /* -------------------------------------------------------------------*/
+ 
+ struct sess {
+ 	unsigned		magic;
+ #define SESS_MAGIC		0x2c2f9c5a
+ 	int			fd;
+ 	unsigned		vsl_id;
+ 	unsigned		xid;
+ 
+ 	int			restarts;
+ 	int			esi_level;
+ 	int			disable_esi;
+ 
+ 	uint8_t			hash_ignore_busy;
+ 	uint8_t			hash_always_miss;
+ 
+ 	struct worker		*wrk;
+ 
+ 	socklen_t		sockaddrlen;
+ 	socklen_t		mysockaddrlen;
+ 	struct sockaddr_storage	sockaddr;
+ 	struct sockaddr_storage	mysockaddr;
+ 	struct listen_sock	*mylsock;
+ 
+ 	/* formatted ascii client address */
+ 	char			*addr;
+ 	char			*port;
+ 	char			*client_identity;
+ 
+ 	/* HTTP request */
+ 	const char		*doclose;
+ 	struct http		*http;
+ 	struct http		*http0;
+ 
+ 	struct ws		ws[1];
+ 	char			*ws_ses;	/* WS above session data */
+ 	char			*ws_req;	/* WS above request data */
+ 
+ 	unsigned char		digest[DIGEST_LEN];
+ 
+ 	/* Built Vary string */
+ 	uint8_t			*vary_b;
+ 	uint8_t			*vary_l;
+ 	uint8_t			*vary_e;
+ 
+ 	struct http_conn	htc[1];
+ 
+ 	/* Timestamps, all on TIM_real() timescale */
+ 	double			t_open;
+ 	double			t_req;
+ 	double			t_resp;
+ 	double			t_end;
+ 
+ 	/* Acceptable grace period */
+ 	struct exp		exp;
+ 
+ 	enum step		step;
+ 	unsigned		cur_method;
+ 	unsigned		handling;
+ 	unsigned char		sendbody;
+ 	unsigned char		wantbody;
+ 	uint16_t		err_code;
+ 	const char		*err_reason;
+ 
+ 	VTAILQ_ENTRY(sess)	list;
+ 
+ 	struct director		*director;
+ 	struct object		*obj;
+ 	struct objcore		*objcore;
+ 	struct VCL_conf		*vcl;
+ 
++	struct object		*stale_obj;
+ 	/* The busy objhead we sleep on */
+ 	struct objhead		*hash_objhead;
+ 
+ 	/* Various internal stuff */
+ 	struct sessmem		*mem;
+ 
+ 	VTAILQ_ENTRY(sess)	poollist;
+ 	uint64_t		req_bodybytes;
+ 	struct acct		acct_ses;
+ 
+ #if defined(HAVE_EPOLL_CTL)
+ 	struct epoll_event ev;
+ #endif
+ };
+ 
+ /* Prototypes etc ----------------------------------------------------*/
+ 
+ /* cache_acceptor.c */
+ void VCA_Prep(struct sess *sp);
+ void VCA_Init(void);
+ void VCA_Shutdown(void);
+ int VCA_Accept(struct listen_sock *ls, struct wrk_accept *wa);
+ void VCA_SetupSess(struct worker *w);
+ void VCA_FailSess(struct worker *w);
+ 
+ /* cache_backend.c */
+ void VBE_UseHealth(const struct director *vdi);
+ 
+ struct vbc *VDI_GetFd(const struct director *, struct sess *sp);
+ int VDI_Healthy(const struct director *, const struct sess *sp);
+ void VDI_CloseFd(struct worker *wrk);
+ void VDI_RecycleFd(struct worker *wrk);
+ void VDI_AddHostHeader(const struct sess *sp);
+ void VBE_Poll(void);
+ 
+ /* cache_backend_cfg.c */
+ void VBE_Init(void);
+ struct backend *VBE_AddBackend(struct cli *cli, const struct vrt_backend *vb);
+ 
+ /* cache_backend_poll.c */
+ void VBP_Init(void);
+ 
+ /* cache_ban.c */
+ struct ban *BAN_New(void);
+ int BAN_AddTest(struct cli *, struct ban *, const char *, const char *,
+     const char *);
+ void BAN_Free(struct ban *b);
+ void BAN_Insert(struct ban *b);
+ void BAN_Init(void);
+ void BAN_NewObjCore(struct objcore *oc);
+ void BAN_DestroyObj(struct objcore *oc);
+ int BAN_CheckObject(struct object *o, const struct sess *sp);
+ void BAN_Reload(const uint8_t *ban, unsigned len);
+ struct ban *BAN_TailRef(void);
+ void BAN_Compile(void);
+ struct ban *BAN_RefBan(struct objcore *oc, double t0, const struct ban *tail);
+ void BAN_TailDeref(struct ban **ban);
+ double BAN_Time(const struct ban *ban);
+ 
+ /* cache_center.c [CNT] */
+ void CNT_Session(struct sess *sp);
+ void CNT_Init(void);
+ 
+ /* cache_cli.c [CLI] */
+ void CLI_Init(void);
+ void CLI_Run(void);
+ void CLI_AddFuncs(struct cli_proto *p);
+ extern pthread_t cli_thread;
+ #define ASSERT_CLI() do {assert(pthread_self() == cli_thread);} while (0)
+ 
+ /* cache_expiry.c */
+ void EXP_Clr(struct exp *e);
+ double EXP_Get_ttl(const struct exp *e);
+ double EXP_Get_grace(const struct exp *e);
+ double EXP_Get_keep(const struct exp *e);
+ void EXP_Set_ttl(struct exp *e, double v);
+ void EXP_Set_grace(struct exp *e, double v);
+ void EXP_Set_keep(struct exp *e, double v);
+ 
+ double EXP_Ttl(const struct sess *, const struct object*);
+ double EXP_Grace(const struct sess *, const struct object*);
++double EXP_Keep(const struct sess *, const struct object*);
+ void EXP_Insert(struct object *o);
+ void EXP_Inject(struct objcore *oc, struct lru *lru, double when);
+ void EXP_Init(void);
+ void EXP_Rearm(const struct object *o);
+ int EXP_Touch(struct objcore *oc);
+ int EXP_NukeOne(struct worker *w, struct lru *lru);
+ 
+ /* cache_fetch.c */
+ struct storage *FetchStorage(struct worker *w, ssize_t sz);
+ int FetchError(struct worker *w, const char *error);
+ int FetchError2(struct worker *w, const char *error, const char *more);
+ int FetchHdr(struct sess *sp);
+ int FetchBody(struct worker *w, struct object *obj);
+ int FetchReqBody(struct sess *sp);
+ void Fetch_Init(void);
+ 
+ /* cache_gzip.c */
+ struct vgz;
+ 
+ enum vgz_flag { VGZ_NORMAL, VGZ_ALIGN, VGZ_RESET, VGZ_FINISH };
+ struct vgz *VGZ_NewUngzip(struct worker *wrk, const char *id);
+ struct vgz *VGZ_NewGzip(struct worker *wrk, const char *id);
+ void VGZ_Ibuf(struct vgz *, const void *, ssize_t len);
+ int VGZ_IbufEmpty(const struct vgz *vg);
+ void VGZ_Obuf(struct vgz *, void *, ssize_t len);
+ int VGZ_ObufFull(const struct vgz *vg);
+ int VGZ_ObufStorage(struct worker *w, struct vgz *vg);
+ int VGZ_Gzip(struct vgz *, const void **, size_t *len, enum vgz_flag);
+ int VGZ_Gunzip(struct vgz *, const void **, size_t *len);
+ int VGZ_Destroy(struct vgz **, int vsl_id);
+ void VGZ_UpdateObj(const struct vgz*, struct object *);
+ int VGZ_WrwGunzip(struct worker *w, struct vgz *, const void *ibuf,
+     ssize_t ibufl, char *obuf, ssize_t obufl, ssize_t *obufp);
+ 
+ /* Return values */
+ #define VGZ_ERROR	-1
+ #define VGZ_OK		0
+ #define VGZ_END		1
+ #define VGZ_STUCK	2
+ 
+ /* cache_http.c */
+ unsigned HTTP_estimate(unsigned nhttp);
+ void HTTP_Copy(struct http *to, const struct http * const fm);
+ struct http *HTTP_create(void *p, uint16_t nhttp);
+ const char *http_StatusMessage(unsigned);
+ unsigned http_EstimateWS(const struct http *fm, unsigned how, uint16_t *nhd);
+ void HTTP_Init(void);
+ void http_ClrHeader(struct http *to);
+ unsigned http_Write(struct worker *w, unsigned vsl_id, const struct http *hp,
+     int resp);
+ void http_CopyResp(struct http *to, const struct http *fm);
+ void http_SetResp(struct http *to, const char *proto, uint16_t status,
+     const char *response);
+ void http_FilterFields(struct worker *w, unsigned vsl_id, struct http *to,
+     const struct http *fm, unsigned how);
+ void http_FilterHeader(const struct sess *sp, unsigned how);
++
++/* Check if a refresh should be done */
++void http_CheckRefresh(struct sess *sp);
++/* Check if we got 304 response */
++void http_Check304(struct sess *sp);
++
+ void http_PutProtocol(struct worker *w, unsigned vsl_id, const struct http *to,
+     const char *protocol);
+ void http_PutStatus(struct http *to, uint16_t status);
+ void http_PutResponse(struct worker *w, unsigned vsl_id, const struct http *to,
+     const char *response);
+ void http_PrintfHeader(struct worker *w, unsigned vsl_id, struct http *to,
+     const char *fmt, ...);
+ void http_SetHeader(struct worker *w, unsigned vsl_id, struct http *to,
+     const char *hdr);
+ void http_SetH(const struct http *to, unsigned n, const char *fm);
+ void http_ForceGet(const struct http *to);
+ void http_Setup(struct http *ht, struct ws *ws);
+ int http_GetHdr(const struct http *hp, const char *hdr, char **ptr);
+ int http_GetHdrData(const struct http *hp, const char *hdr,
+     const char *field, char **ptr);
+ int http_GetHdrField(const struct http *hp, const char *hdr,
+     const char *field, char **ptr);
+ double http_GetHdrQ(const struct http *hp, const char *hdr, const char *field);
+ uint16_t http_GetStatus(const struct http *hp);
+ const char *http_GetReq(const struct http *hp);
+ int http_HdrIs(const struct http *hp, const char *hdr, const char *val);
+ uint16_t http_DissectRequest(struct sess *sp);
+ uint16_t http_DissectResponse(struct worker *w, const struct http_conn *htc,
+     struct http *sp);
+ const char *http_DoConnection(const struct http *hp);
+ void http_CopyHome(struct worker *w, unsigned vsl_id, const struct http *hp);
+ void http_Unset(struct http *hp, const char *hdr);
+ void http_CollectHdr(struct http *hp, const char *hdr);
+ 
+ /* cache_httpconn.c */
+ void HTC_Init(struct http_conn *htc, struct ws *ws, int fd, unsigned vsl_id,
+     unsigned maxbytes, unsigned maxhdr);
+ int HTC_Reinit(struct http_conn *htc);
+ int HTC_Rx(struct http_conn *htc);
+ ssize_t HTC_Read(struct worker *w, struct http_conn *htc, void *d, size_t len);
+ int HTC_Complete(struct http_conn *htc);
+ 
+ #define HTTPH(a, b, c, d, e, f, g) extern char b[];
+ #include "tbl/http_headers.h"
+ #undef HTTPH
+ 
+ /* cache_main.c */
+ extern volatile struct params * cache_param;
+ void THR_SetName(const char *name);
+ const char* THR_GetName(void);
+ void THR_SetSession(const struct sess *sp);
+ const struct sess * THR_GetSession(void);
+ 
+ /* cache_lck.c */
+ 
+ /* Internal functions, call only through macros below */
+ void Lck__Lock(struct lock *lck, const char *p, const char *f, int l);
+ void Lck__Unlock(struct lock *lck, const char *p, const char *f, int l);
+ int Lck__Trylock(struct lock *lck, const char *p, const char *f, int l);
+ void Lck__New(struct lock *lck, struct VSC_C_lck *, const char *);
+ void Lck__Assert(const struct lock *lck, int held);
+ 
+ /* public interface: */
+ void LCK_Init(void);
+ void Lck_Delete(struct lock *lck);
+ int Lck_CondWait(pthread_cond_t *cond, struct lock *lck, struct timespec *ts);
+ 
+ #define Lck_New(a, b) Lck__New(a, b, #b)
+ #define Lck_Lock(a) Lck__Lock(a, __func__, __FILE__, __LINE__)
+ #define Lck_Unlock(a) Lck__Unlock(a, __func__, __FILE__, __LINE__)
+ #define Lck_Trylock(a) Lck__Trylock(a, __func__, __FILE__, __LINE__)
+ #define Lck_AssertHeld(a) Lck__Assert(a, 1)
+ 
+ #define LOCK(nam) extern struct VSC_C_lck *lck_##nam;
+ #include "tbl/locks.h"
+ #undef LOCK
+ 
+ /* cache_panic.c */
+ void PAN_Init(void);
+ 
+ /* cache_pipe.c */
+ void PipeSession(struct sess *sp);
+ 
+ /* cache_pool.c */
+ void Pool_Init(void);
+ void Pool_Work_Thread(void *priv, struct worker *w);
+ void Pool_Wait(struct sess *sp);
+ int Pool_Schedule(struct pool *pp, struct sess *sp);
+ 
+ #define WRW_IsReleased(w)	((w)->wrw.wfd == NULL)
+ int WRW_Error(const struct worker *w);
+ void WRW_Chunked(struct worker *w);
+ void WRW_EndChunk(struct worker *w);
+ void WRW_Reserve(struct worker *w, int *fd);
+ unsigned WRW_Flush(struct worker *w);
+ unsigned WRW_FlushRelease(struct worker *w);
+ unsigned WRW_Write(struct worker *w, const void *ptr, int len);
+ unsigned WRW_WriteH(struct worker *w, const txt *hh, const char *suf);
+ #ifdef SENDFILE_WORKS
+ void WRW_Sendfile(struct worker *w, int fd, off_t off, unsigned len);
+ #endif  /* SENDFILE_WORKS */
+ 
+ /* cache_session.c [SES] */
+ struct sess *SES_New(struct worker *wrk, struct sesspool *pp);
+ struct sess *SES_Alloc(void);
+ void SES_Close(struct sess *sp, const char *reason);
+ void SES_Delete(struct sess *sp, const char *reason);
+ void SES_Charge(struct sess *sp);
+ struct sesspool *SES_NewPool(struct pool *pp);
+ void SES_DeletePool(struct sesspool *sp, struct worker *wrk);
+ int SES_Schedule(struct sess *sp);
+ 
+ 
+ /* cache_shmlog.c */
+ extern struct VSC_C_main *VSC_C_main;
+ void VSL_Init(void);
+ void *VSM_Alloc(unsigned size, const char *class, const char *type,
+     const char *ident);
+ void VSM_Free(void *ptr);
+ #ifdef VSL_ENDMARKER
+ void VSL(enum VSL_tag_e tag, int id, const char *fmt, ...);
+ void WSLR(struct worker *w, enum VSL_tag_e tag, int id, txt t);
+ void WSL(struct worker *w, enum VSL_tag_e tag, int id, const char *fmt, ...);
+ void WSLB(struct worker *w, enum VSL_tag_e tag, const char *fmt, ...);
+ 
+ void WSL_Flush(struct worker *w, int overflow);
+ 
+ #define DSL(flag, tag, id, ...)					\
+ 	do {							\
+ 		if (cache_param->diag_bitmap & (flag))		\
+ 			VSL((tag), (id), __VA_ARGS__);		\
+ 	} while (0)
+ 
+ #define WSP(sess, tag, ...)					\
+ 	WSL((sess)->wrk, tag, (sess)->vsl_id, __VA_ARGS__)
+ 
+ #define WSPR(sess, tag, txt)					\
+ 	WSLR((sess)->wrk, tag, (sess)->vsl_id, txt)
+ 
+ #define INCOMPL() do {							\
+ 	VSL(SLT_Debug, 0, "INCOMPLETE AT: %s(%d)", __func__, __LINE__); \
+ 	fprintf(stderr,							\
+ 	    "INCOMPLETE AT: %s(%d)\n",					\
+ 	    (const char *)__func__, __LINE__);				\
+ 	abort();							\
+ 	} while (0)
+ #endif
+ 
+ /* cache_response.c */
+ void RES_BuildHttp(const struct sess *sp);
+ void RES_WriteObj(struct sess *sp);
+ void RES_StreamStart(struct sess *sp);
+ void RES_StreamEnd(struct sess *sp);
+ void RES_StreamPoll(struct worker *);
+ 
+ /* cache_vary.c */
+ struct vsb *VRY_Create(const struct sess *sp, const struct http *hp);
+ int VRY_Match(struct sess *sp, const uint8_t *vary);
+ void VRY_Validate(const uint8_t *vary);
+ 
+ /* cache_vcl.c */
+ void VCL_Init(void);
+ void VCL_Refresh(struct VCL_conf **vcc);
+ void VCL_Rel(struct VCL_conf **vcc);
+ void VCL_Poll(void);
+ const char *VCL_Return_Name(unsigned method);
+ 
+ #define VCL_MET_MAC(l,u,b) void VCL_##l##_method(struct sess *);
+ #include "tbl/vcl_returns.h"
+ #undef VCL_MET_MAC
+ 
+ /* cache_vrt.c */
+ 
+ char *VRT_String(struct ws *ws, const char *h, const char *p, va_list ap);
+ char *VRT_StringList(char *d, unsigned dl, const char *p, va_list ap);
+ 
+ void ESI_Deliver(struct sess *);
+ void ESI_DeliverChild(const struct sess *);
+ 
+ /* cache_vrt_vmod.c */
+ void VMOD_Init(void);
+ 
+ /* cache_wrk.c */
+ 
+ void WRK_Init(void);
+ int WRK_TrySumStat(struct worker *w);
+ void WRK_SumStat(struct worker *w);
+ void *WRK_thread(void *priv);
+ typedef void *bgthread_t(struct sess *, void *priv);
+ void WRK_BgThread(pthread_t *thr, const char *name, bgthread_t *func,
+     void *priv);
+ 
+ /* cache_ws.c */
+ 
+ void WS_Init(struct ws *ws, const char *id, void *space, unsigned len);
+ unsigned WS_Reserve(struct ws *ws, unsigned bytes);
+ void WS_Release(struct ws *ws, unsigned bytes);
+ void WS_ReleaseP(struct ws *ws, char *ptr);
+ void WS_Assert(const struct ws *ws);
+ void WS_Reset(struct ws *ws, char *p);
+ char *WS_Alloc(struct ws *ws, unsigned bytes);
+ char *WS_Dup(struct ws *ws, const char *);
+ char *WS_Snapshot(struct ws *ws);
+ unsigned WS_Free(const struct ws *ws);
+ 
+ /* rfc2616.c */
+ void RFC2616_Ttl(const struct sess *sp);
+ enum body_status RFC2616_Body(const struct sess *sp);
+ unsigned RFC2616_Req_Gzip(const struct sess *sp);
+ int RFC2616_Do_Cond(const struct sess *sp);
+ 
+ /* stevedore.c */
+ struct object *STV_NewObject(struct sess *sp, const char *hint, unsigned len,
+     struct exp *, uint16_t nhttp);
+ struct storage *STV_alloc(struct worker *w, size_t size);
+ void STV_trim(struct storage *st, size_t size);
+ void STV_free(struct storage *st);
+ void STV_open(void);
+ void STV_close(void);
+ void STV_Freestore(struct object *o);
+ 
+ /* storage_synth.c */
+ struct vsb *SMS_Makesynth(struct object *obj);
+ void SMS_Finish(struct object *obj);
+ void SMS_Init(void);
+ 
+ /* storage_persistent.c */
+ void SMP_Init(void);
+ void SMP_Ready(void);
+ void SMP_NewBan(const uint8_t *ban, unsigned len);
+ 
+ /*
+  * A normal pointer difference is signed, but we never want a negative value
+  * so this little tool will make sure we don't get that.
+  */
+ 
+ static inline unsigned
+ pdiff(const void *b, const void *e)
+ {
+ 
+ 	assert(b <= e);
+ 	return
+ 	    ((unsigned)((const unsigned char *)e - (const unsigned char *)b));
+ }
+ 
+ static inline void
+ Tcheck(const txt t)
+ {
+ 
+ 	AN(t.b);
+ 	AN(t.e);
+ 	assert(t.b <= t.e);
+ }
+ 
+ /*
+  * unsigned length of a txt
+  */
+ 
+ static inline unsigned
+ Tlen(const txt t)
+ {
+ 
+ 	Tcheck(t);
+ 	return ((unsigned)(t.e - t.b));
+ }
+ 
+ static inline void
+ Tadd(txt *t, const char *p, int l)
+ {
+ 	Tcheck(*t);
+ 
+ 	if (l <= 0) {
+ 	} if (t->b + l < t->e) {
+ 		memcpy(t->b, p, l);
+ 		t->b += l;
+ 	} else {
+ 		t->b = t->e;
+ 	}
+ }
+ 
+ static inline void
+ AssertObjBusy(const struct object *o)
+ {
+ 	AN(o->objcore);
+ 	AN (o->objcore->flags & OC_F_BUSY);
+ }
+ 
+ static inline void
+ AssertObjCorePassOrBusy(const struct objcore *oc)
+ {
+ 	if (oc != NULL)
+ 		AN (oc->flags & OC_F_BUSY);
+ }
diff --cc bin/varnishd/cache/cache_center.c
index 0000000,e42fac8..7b8dc8a
mode 000000,100644..100644
--- a/bin/varnishd/cache/cache_center.c
+++ b/bin/varnishd/cache/cache_center.c
@@@ -1,0 -1,1691 +1,1724 @@@
+ /*-
+  * Copyright (c) 2006 Verdens Gang AS
+  * Copyright (c) 2006-2011 Varnish Software AS
+  * All rights reserved.
+  *
+  * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
+  *
+  * Redistribution and use in source and binary forms, with or without
+  * modification, are permitted provided that the following conditions
+  * are met:
+  * 1. Redistributions of source code must retain the above copyright
+  *    notice, this list of conditions and the following disclaimer.
+  * 2. Redistributions in binary form must reproduce the above copyright
+  *    notice, this list of conditions and the following disclaimer in the
+  *    documentation and/or other materials provided with the distribution.
+  *
+  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+  * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+  * SUCH DAMAGE.
+  *
+  * This file contains the central state machine for pushing requests.
+  *
+  * We cannot just use direct calls because it is possible to kick a
+  * request back to the lookup stage (usually after a rewrite).  The
+  * state engine also allows us to break the processing up into some
+  * logical chunks which improves readability a little bit.
+  *
+  * Since the states are rather nasty in detail, I have decided to embedd
+  * a dot(1) graph in the source code comments.  So to see the big picture,
+  * extract the DOT lines and run though dot(1), for instance with the
+  * command:
+  *	sed -n '/^DOT/s///p' cache_center.c | dot -Tps > /tmp/_.ps
+  */
+ 
+ /*
+ DOT digraph vcl_center {
+ xDOT	page="8.2,11.5"
+ DOT	size="7.2,10.5"
+ DOT	margin="0.5"
+ DOT	center="1"
+ DOT acceptor [
+ DOT	shape=hexagon
+ DOT	label="Request received"
+ DOT ]
+ DOT ERROR [shape=plaintext]
+ DOT RESTART [shape=plaintext]
+ DOT acceptor -> start [style=bold,color=green]
+  */
+ 
+ #include "config.h"
+ 
+ #include <math.h>
+ #include <poll.h>
+ #include <stdio.h>
+ #include <stdlib.h>
+ 
+ #include "cache.h"
+ 
+ #include "hash/hash_slinger.h"
+ #include "vcl.h"
+ #include "vcli_priv.h"
+ #include "vsha256.h"
+ #include "vtcp.h"
+ #include "vtim.h"
+ 
+ #ifndef HAVE_SRANDOMDEV
+ #include "compat/srandomdev.h"
+ #endif
+ 
+ static unsigned xids;
+ 
+ /*--------------------------------------------------------------------
+  * WAIT
+  * Wait (briefly) until we have a full request in our htc.
+  */
+ 
+ static int
+ cnt_wait(struct sess *sp)
+ {
+ 	int i;
+ 	struct pollfd pfd[1];
+ 
+ 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+ 	AZ(sp->vcl);
+ 	AZ(sp->obj);
+ 	assert(sp->xid == 0);
+ 
+ 	i = HTC_Complete(sp->htc);
+ 	if (i == 0 && cache_param->session_linger > 0) {
+ 		pfd[0].fd = sp->fd;
+ 		pfd[0].events = POLLIN;
+ 		pfd[0].revents = 0;
+ 		i = poll(pfd, 1, cache_param->session_linger);
+ 		if (i)
+ 			i = HTC_Rx(sp->htc);
+ 	}
+ 	if (i == 0) {
+ 		WSP(sp, SLT_Debug, "herding");
+ 		sp->wrk->stats.sess_herd++;
+ 		SES_Charge(sp);
+ 		sp->wrk = NULL;
+ 		Pool_Wait(sp);
+ 		return (1);
+ 	}
+ 	if (i == 1) {
+ 		sp->step = STP_START;
+ 		return (0);
+ 	}
+ 	if (i == -2) {
+ 		SES_Close(sp, "overflow");
+ 		return (0);
+ 	}
+ 	if (i == -1 && Tlen(sp->htc->rxbuf) == 0 &&
+ 	    (errno == 0 || errno == ECONNRESET))
+ 		SES_Close(sp, "EOF");
+ 	else
+ 		SES_Close(sp, "error");
+ 	sp->step = STP_DONE;
+ 	return (0);
+ }
+ 
+ /*--------------------------------------------------------------------
+  * We have a refcounted object on the session, now deliver it.
+  *
+ DOT subgraph xcluster_prepresp {
+ DOT	prepresp [
+ DOT		shape=ellipse
+ DOT		label="Filter obj.->resp."
+ DOT	]
+ DOT	vcl_deliver [
+ DOT		shape=record
+ DOT		label="vcl_deliver()|resp."
+ DOT	]
+ DOT	prepresp -> vcl_deliver [style=bold,color=green]
+ DOT	prepresp -> vcl_deliver [style=bold,color=cyan]
+ DOT	prepresp -> vcl_deliver [style=bold,color=red]
+ DOT	prepresp -> vcl_deliver [style=bold,color=blue,]
+ DOT	vcl_deliver -> deliver [style=bold,color=green,label=deliver]
+ DOT	vcl_deliver -> deliver [style=bold,color=red]
+ DOT	vcl_deliver -> deliver [style=bold,color=blue]
+ DOT     vcl_deliver -> errdeliver [label="error"]
+ DOT     errdeliver [label="ERROR",shape=plaintext]
+ DOT     vcl_deliver -> rstdeliver [label="restart",color=purple]
+ DOT     rstdeliver [label="RESTART",shape=plaintext]
+ DOT     vcl_deliver -> streambody [style=bold,color=cyan,label="deliver"]
+ DOT }
+  *
+  */
+ 
+ static int
+ cnt_prepresp(struct sess *sp)
+ {
+ 
+ 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+ 	CHECK_OBJ_NOTNULL(sp->obj, OBJECT_MAGIC);
+ 	CHECK_OBJ_NOTNULL(sp->vcl, VCL_CONF_MAGIC);
+ 
+ 	if (sp->wrk->do_stream)
+ 		AssertObjCorePassOrBusy(sp->obj->objcore);
+ 
+ 	sp->wrk->res_mode = 0;
+ 
+ 	if ((sp->wrk->h_content_length != NULL || !sp->wrk->do_stream) &&
+ 	    !sp->wrk->do_gzip && !sp->wrk->do_gunzip)
+ 		sp->wrk->res_mode |= RES_LEN;
+ 
+ 	if (!sp->disable_esi && sp->obj->esidata != NULL) {
+ 		/* In ESI mode, we don't know the aggregate length */
+ 		sp->wrk->res_mode &= ~RES_LEN;
+ 		sp->wrk->res_mode |= RES_ESI;
+ 	}
+ 
+ 	if (sp->esi_level > 0) {
+ 		sp->wrk->res_mode &= ~RES_LEN;
+ 		sp->wrk->res_mode |= RES_ESI_CHILD;
+ 	}
+ 
+ 	if (cache_param->http_gzip_support && sp->obj->gziped &&
+ 	    !RFC2616_Req_Gzip(sp)) {
+ 		/*
+ 		 * We don't know what it uncompresses to
+ 		 * XXX: we could cache that
+ 		 */
+ 		sp->wrk->res_mode &= ~RES_LEN;
+ 		sp->wrk->res_mode |= RES_GUNZIP;
+ 	}
+ 
+ 	if (!(sp->wrk->res_mode & (RES_LEN|RES_CHUNKED|RES_EOF))) {
+ 		if (sp->obj->len == 0 && !sp->wrk->do_stream)
+ 			/*
+ 			 * If the object is empty, neither ESI nor GUNZIP
+ 			 * can make it any different size
+ 			 */
+ 			sp->wrk->res_mode |= RES_LEN;
+ 		else if (!sp->wantbody) {
+ 			/* Nothing */
+ 		} else if (sp->http->protover >= 11) {
+ 			sp->wrk->res_mode |= RES_CHUNKED;
+ 		} else {
+ 			sp->wrk->res_mode |= RES_EOF;
+ 			sp->doclose = "EOF mode";
+ 		}
+ 	}
+ 
+ 	sp->t_resp = VTIM_real();
+ 	if (sp->obj->objcore != NULL) {
+ 		if ((sp->t_resp - sp->obj->last_lru) > cache_param->lru_timeout &&
+ 		    EXP_Touch(sp->obj->objcore))
+ 			sp->obj->last_lru = sp->t_resp;
+ 		sp->obj->last_use = sp->t_resp;	/* XXX: locking ? */
+ 	}
+ 	http_Setup(sp->wrk->resp, sp->wrk->ws);
+ 	RES_BuildHttp(sp);
+ 	VCL_deliver_method(sp);
+ 	switch (sp->handling) {
+ 	case VCL_RET_DELIVER:
+ 		break;
+ 	case VCL_RET_RESTART:
+ 		if (sp->restarts >= cache_param->max_restarts)
+ 			break;
+ 		if (sp->wrk->do_stream) {
+ 			VDI_CloseFd(sp->wrk);
+ 			HSH_Drop(sp);
+ 		} else {
+ 			(void)HSH_Deref(sp->wrk, NULL, &sp->obj);
+ 		}
+ 		AZ(sp->obj);
+ 		sp->restarts++;
+ 		sp->director = NULL;
+ 		sp->wrk->h_content_length = NULL;
+ 		http_Setup(sp->wrk->bereq, NULL);
+ 		http_Setup(sp->wrk->beresp, NULL);
+ 		http_Setup(sp->wrk->resp, NULL);
+ 		sp->step = STP_RECV;
+ 		return (0);
+ 	default:
+ 		WRONG("Illegal action in vcl_deliver{}");
+ 	}
+ 	if (sp->wrk->do_stream) {
+ 		AssertObjCorePassOrBusy(sp->obj->objcore);
+ 		sp->step = STP_STREAMBODY;
+ 	} else {
+ 		sp->step = STP_DELIVER;
+ 	}
+ 	return (0);
+ }
+ 
+ /*--------------------------------------------------------------------
+  * Deliver an already stored object
+  *
+ DOT subgraph xcluster_deliver {
+ DOT	deliver [
+ DOT		shape=ellipse
+ DOT		label="Send body"
+ DOT	]
+ DOT }
+ DOT deliver -> DONE [style=bold,color=green]
+ DOT deliver -> DONE [style=bold,color=red]
+ DOT deliver -> DONE [style=bold,color=blue]
+  *
+  */
+ 
+ static int
+ cnt_deliver(struct sess *sp)
+ {
+ 
+ 	sp->director = NULL;
+ 	sp->restarts = 0;
+ 
+ 	RES_WriteObj(sp);
+ 
+ 	assert(WRW_IsReleased(sp->wrk));
+ 	assert(sp->wrk->wrw.ciov == sp->wrk->wrw.siov);
+ 	(void)HSH_Deref(sp->wrk, NULL, &sp->obj);
+ 	http_Setup(sp->wrk->resp, NULL);
+ 	sp->step = STP_DONE;
+ 	return (0);
+ }
+ 
+ /*--------------------------------------------------------------------
+  * This is the final state, figure out if we should close or recycle
+  * the client connection
+  *
+ DOT	DONE [
+ DOT		shape=hexagon
+ DOT		label="Request completed"
+ DOT	]
+  */
+ 
+ static int
+ cnt_done(struct sess *sp)
+ {
+ 	double dh, dp, da;
+ 	int i;
+ 
+ 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+ 	CHECK_OBJ_ORNULL(sp->vcl, VCL_CONF_MAGIC);
+ 
+ 	AZ(sp->obj);
+ 	AZ(sp->wrk->vbc);
+ 	sp->director = NULL;
+ 	sp->restarts = 0;
+ 
+ 	sp->wrk->do_esi = 0;
+ 	sp->wrk->do_gunzip = 0;
+ 	sp->wrk->do_gzip = 0;
+ 	sp->wrk->do_stream = 0;
+ 	sp->wrk->is_gunzip = 0;
+ 	sp->wrk->is_gzip = 0;
+ 
+ 	if (sp->vcl != NULL && sp->esi_level == 0) {
+ 		if (sp->wrk->vcl != NULL)
+ 			VCL_Rel(&sp->wrk->vcl);
+ 		sp->wrk->vcl = sp->vcl;
+ 		sp->vcl = NULL;
+ 	}
+ 
+ 	SES_Charge(sp);
+ 
+ 	sp->t_end = VTIM_real();
+ 	sp->wrk->lastused = sp->t_end;
+ 	if (sp->xid == 0) {
+ 		sp->t_req = sp->t_end;
+ 		sp->t_resp = sp->t_end;
+ 	} else if (sp->esi_level == 0) {
+ 		dp = sp->t_resp - sp->t_req;
+ 		da = sp->t_end - sp->t_resp;
+ 		dh = sp->t_req - sp->t_open;
+ 		/* XXX: Add StatReq == StatSess */
+ 		/* XXX: Workaround for pipe */
+ 		if (sp->fd >= 0) {
+ 			WSP(sp, SLT_Length, "%ju",
+ 			    (uintmax_t)sp->req_bodybytes);
+ 		}
+ 		WSP(sp, SLT_ReqEnd, "%u %.9f %.9f %.9f %.9f %.9f",
+ 		    sp->xid, sp->t_req, sp->t_end, dh, dp, da);
+ 	}
+ 	sp->xid = 0;
+ 	sp->t_open = sp->t_end;
+ 	sp->t_resp = NAN;
+ 	WSL_Flush(sp->wrk, 0);
+ 
+ 	/* If we did an ESI include, don't mess up our state */
+ 	if (sp->esi_level > 0)
+ 		return (1);
+ 
+ 	sp->req_bodybytes = 0;
+ 
+ 	sp->t_req = NAN;
+ 	sp->hash_always_miss = 0;
+ 	sp->hash_ignore_busy = 0;
+ 
+ 	if (sp->fd >= 0 && sp->doclose != NULL) {
+ 		/*
+ 		 * This is an orderly close of the connection; ditch nolinger
+ 		 * before we close, to get queued data transmitted.
+ 		 */
+ 		// XXX: not yet (void)VTCP_linger(sp->fd, 0);
+ 		SES_Close(sp, sp->doclose);
+ 	}
+ 
+ 	if (sp->fd < 0) {
+ 		sp->wrk->stats.sess_closed++;
+ 		SES_Delete(sp, NULL);
+ 		return (1);
+ 	}
+ 
+ 	if (sp->wrk->stats.client_req >= cache_param->wthread_stats_rate)
+ 		WRK_SumStat(sp->wrk);
+ 	/* Reset the workspace to the session-watermark */
+ 	WS_Reset(sp->ws, sp->ws_ses);
+ 	WS_Reset(sp->wrk->ws, NULL);
+ 
+ 	i = HTC_Reinit(sp->htc);
+ 	if (i == 1) {
+ 		sp->wrk->stats.sess_pipeline++;
+ 		sp->step = STP_START;
+ 		return (0);
+ 	}
+ 	if (Tlen(sp->htc->rxbuf)) {
+ 		sp->wrk->stats.sess_readahead++;
+ 		sp->step = STP_WAIT;
+ 		return (0);
+ 	}
+ 	if (cache_param->session_linger > 0) {
+ 		sp->wrk->stats.sess_linger++;
+ 		sp->step = STP_WAIT;
+ 		return (0);
+ 	}
+ 	sp->wrk->stats.sess_herd++;
+ 	sp->wrk = NULL;
+ 	Pool_Wait(sp);
+ 	return (1);
+ }
+ 
+ /*--------------------------------------------------------------------
+  * Emit an error
+  *
+ DOT subgraph xcluster_error {
+ DOT	vcl_error [
+ DOT		shape=record
+ DOT		label="vcl_error()|resp."
+ DOT	]
+ DOT	ERROR -> vcl_error
+ DOT	vcl_error-> prepresp [label=deliver]
+ DOT }
+ DOT vcl_error-> rsterr [label="restart",color=purple]
+ DOT rsterr [label="RESTART",shape=plaintext]
+  */
+ 
+ static int
+ cnt_error(struct sess *sp)
+ {
+ 	struct worker *w;
+ 	struct http *h;
+ 	char date[40];
+ 
+ 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+ 
+ 	sp->wrk->do_esi = 0;
+ 	sp->wrk->is_gzip = 0;
+ 	sp->wrk->is_gunzip = 0;
+ 	sp->wrk->do_gzip = 0;
+ 	sp->wrk->do_gunzip = 0;
+ 	sp->wrk->do_stream = 0;
+ 
+ 	w = sp->wrk;
+ 	if (sp->obj == NULL) {
+ 		HSH_Prealloc(sp);
+ 		EXP_Clr(&w->exp);
+ 		sp->obj = STV_NewObject(sp, NULL, cache_param->http_resp_size,
+ 		     &w->exp, (uint16_t)cache_param->http_max_hdr);
+ 		if (sp->obj == NULL)
+ 			sp->obj = STV_NewObject(sp, TRANSIENT_STORAGE,
+ 			    cache_param->http_resp_size, &w->exp,
+ 			    (uint16_t)cache_param->http_max_hdr);
+ 		if (sp->obj == NULL) {
+ 			sp->doclose = "Out of objects";
+ 			sp->director = NULL;
+ 			sp->wrk->h_content_length = NULL;
+ 			http_Setup(sp->wrk->beresp, NULL);
+ 			http_Setup(sp->wrk->bereq, NULL);
+ 			sp->step = STP_DONE;
+ 			return(0);
+ 		}
+ 		AN(sp->obj);
+ 		sp->obj->xid = sp->xid;
+ 		sp->obj->exp.entered = sp->t_req;
+ 	} else {
+ 		/* XXX: Null the headers ? */
+ 	}
+ 	CHECK_OBJ_NOTNULL(sp->obj, OBJECT_MAGIC);
+ 	h = sp->obj->http;
+ 
+ 	if (sp->err_code < 100 || sp->err_code > 999)
+ 		sp->err_code = 501;
+ 
+ 	http_PutProtocol(w, sp->vsl_id, h, "HTTP/1.1");
+ 	http_PutStatus(h, sp->err_code);
+ 	VTIM_format(VTIM_real(), date);
+ 	http_PrintfHeader(w, sp->vsl_id, h, "Date: %s", date);
+ 	http_SetHeader(w, sp->vsl_id, h, "Server: Varnish");
+ 
+ 	if (sp->err_reason != NULL)
+ 		http_PutResponse(w, sp->vsl_id, h, sp->err_reason);
+ 	else
+ 		http_PutResponse(w, sp->vsl_id, h,
+ 		    http_StatusMessage(sp->err_code));
+ 	VCL_error_method(sp);
+ 
+ 	if (sp->handling == VCL_RET_RESTART &&
+ 	    sp->restarts <  cache_param->max_restarts) {
+ 		HSH_Drop(sp);
+ 		sp->director = NULL;
+ 		sp->restarts++;
+ 		sp->step = STP_RECV;
+ 		return (0);
+ 	} else if (sp->handling == VCL_RET_RESTART)
+ 		sp->handling = VCL_RET_DELIVER;
+ 
+ 
+ 	/* We always close when we take this path */
+ 	sp->doclose = "error";
+ 	sp->wantbody = 1;
+ 
+ 	assert(sp->handling == VCL_RET_DELIVER);
+ 	sp->err_code = 0;
+ 	sp->err_reason = NULL;
+ 	http_Setup(sp->wrk->bereq, NULL);
+ 	sp->step = STP_PREPRESP;
+ 	return (0);
+ }
+ 
+ /*--------------------------------------------------------------------
+  * Fetch response headers from the backend
+  *
+ DOT subgraph xcluster_fetch {
+ DOT	fetch [
+ DOT		shape=ellipse
+ DOT		label="fetch hdr\nfrom backend\n(find obj.ttl)"
+ DOT	]
+ DOT	vcl_fetch [
+ DOT		shape=record
+ DOT		label="vcl_fetch()|req.\nbereq.\nberesp."
+ DOT	]
+ DOT	fetch -> vcl_fetch [style=bold,color=blue]
+ DOT	fetch -> vcl_fetch [style=bold,color=red]
+ DOT	fetch_pass [
+ DOT		shape=ellipse
+ DOT		label="obj.f.pass=true"
+ DOT	]
+ DOT	vcl_fetch -> fetch_pass [label="hit_for_pass",style=bold,color=red]
+ DOT }
+ DOT fetch_pass -> fetchbody [style=bold,color=red]
+ DOT vcl_fetch -> fetchbody [label="deliver",style=bold,color=blue]
+ DOT vcl_fetch -> rstfetch [label="restart",color=purple]
+ DOT rstfetch [label="RESTART",shape=plaintext]
+ DOT fetch -> errfetch
+ DOT vcl_fetch -> errfetch [label="error"]
+ DOT errfetch [label="ERROR",shape=plaintext]
+  */
+ 
+ static int
+ cnt_fetch(struct sess *sp)
+ {
+ 	int i;
+ 
+ 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+ 	CHECK_OBJ_NOTNULL(sp->vcl, VCL_CONF_MAGIC);
+ 
+ 	AN(sp->director);
+ 	AZ(sp->wrk->vbc);
+ 	AZ(sp->wrk->h_content_length);
+ 	AZ(sp->wrk->do_close);
+ 	AZ(sp->wrk->storage_hint);
+ 
+ 	http_Setup(sp->wrk->beresp, sp->wrk->ws);
+ 
+ 	i = FetchHdr(sp);
+ 	/*
+ 	 * If we recycle a backend connection, there is a finite chance
+ 	 * that the backend closed it before we get a request to it.
+ 	 * Do a single retry in that case.
+ 	 */
+ 	if (i == 1) {
+ 		VSC_C_main->backend_retry++;
+ 		i = FetchHdr(sp);
+ 	}
+ 
+ 	if (i) {
+ 		sp->handling = VCL_RET_ERROR;
+ 		sp->err_code = 503;
+ 	} else {
+ 		/*
+ 		 * These two headers can be spread over multiple actual headers
+ 		 * and we rely on their content outside of VCL, so collect them
+ 		 * into one line here.
+ 		 */
+ 		http_CollectHdr(sp->wrk->beresp, H_Cache_Control);
+ 		http_CollectHdr(sp->wrk->beresp, H_Vary);
+ 
+ 		/*
+ 		 * Figure out how the fetch is supposed to happen, before the
+ 		 * headers are adultered by VCL
+ 		 * NB: Also sets other sp->wrk variables
+ 		 */
+ 		sp->wrk->body_status = RFC2616_Body(sp);
+ 
+ 		sp->err_code = http_GetStatus(sp->wrk->beresp);
+ 
+ 		/*
+ 		 * What does RFC2616 think about TTL ?
+ 		 */
+ 		EXP_Clr(&sp->wrk->exp);
+ 		sp->wrk->exp.entered = VTIM_real();
+ 		RFC2616_Ttl(sp);
++		sp->wrk->exp.keep = cache_param->default_keep;
+ 
+ 		/* pass from vclrecv{} has negative TTL */
+ 		if (sp->objcore == NULL)
+ 			sp->wrk->exp.ttl = -1.;
+ 
+ 		AZ(sp->wrk->do_esi);
+ 
+ 		VCL_fetch_method(sp);
+ 
+ 		switch (sp->handling) {
+ 		case VCL_RET_HIT_FOR_PASS:
+ 			if (sp->objcore != NULL)
+ 				sp->objcore->flags |= OC_F_PASS;
+ 			sp->step = STP_FETCHBODY;
+ 			return (0);
+ 		case VCL_RET_DELIVER:
+ 			AssertObjCorePassOrBusy(sp->objcore);
+ 			sp->step = STP_FETCHBODY;
+ 			return (0);
+ 		default:
+ 			break;
+ 		}
+ 
+ 		/* We are not going to fetch the body, Close the connection */
+ 		VDI_CloseFd(sp->wrk);
+ 	}
+ 
+ 	/* Clean up partial fetch */
+ 	AZ(sp->wrk->vbc);
+ 
+ 	if (sp->objcore != NULL) {
+ 		CHECK_OBJ_NOTNULL(sp->objcore, OBJCORE_MAGIC);
+ 		AZ(HSH_Deref(sp->wrk, sp->objcore, NULL));
+ 		sp->objcore = NULL;
+ 	}
+ 	http_Setup(sp->wrk->bereq, NULL);
+ 	http_Setup(sp->wrk->beresp, NULL);
+ 	sp->wrk->h_content_length = NULL;
+ 	sp->director = NULL;
+ 	sp->wrk->storage_hint = NULL;
+ 
+ 	switch (sp->handling) {
+ 	case VCL_RET_RESTART:
+ 		sp->restarts++;
+ 		sp->step = STP_RECV;
+ 		return (0);
+ 	case VCL_RET_ERROR:
+ 		sp->step = STP_ERROR;
+ 		return (0);
+ 	default:
+ 		WRONG("Illegal action in vcl_fetch{}");
+ 	}
+ }
+ 
+ /*--------------------------------------------------------------------
+  * Fetch response body from the backend
+  *
+ DOT subgraph xcluster_body {
+ DOT	fetchbody [
+ DOT		shape=diamond
+ DOT		label="stream ?"
+ DOT	]
+ DOT	fetchbody2 [
+ DOT		shape=ellipse
+ DOT		label="fetch body\nfrom backend\n"
+ DOT	]
+ DOT }
+ DOT fetchbody -> fetchbody2 [label=no,style=bold,color=red]
+ DOT fetchbody -> fetchbody2 [style=bold,color=blue]
+ DOT fetchbody -> prepresp [label=yes,style=bold,color=cyan]
+ DOT fetchbody2 -> prepresp [style=bold,color=red]
+ DOT fetchbody2 -> prepresp [style=bold,color=blue]
+  */
+ 
+ 
+ static int
+ cnt_fetchbody(struct sess *sp)
+ {
+ 	int i;
+ 	struct http *hp, *hp2;
+ 	char *b;
 -	uint16_t nhttp;
++	uint16_t nhttp, stale_nhttp;
+ 	unsigned l;
+ 	struct vsb *vary = NULL;
+ 	int varyl = 0, pass;
+ 
+ 	assert(sp->handling == VCL_RET_HIT_FOR_PASS ||
+ 	    sp->handling == VCL_RET_DELIVER);
+ 
+ 	if (sp->objcore == NULL) {
+ 		/* This is a pass from vcl_recv */
+ 		pass = 1;
+ 		/* VCL may have fiddled this, but that doesn't help */
+ 		sp->wrk->exp.ttl = -1.;
+ 	} else if (sp->handling == VCL_RET_HIT_FOR_PASS) {
+ 		/* pass from vcl_fetch{} -> hit-for-pass */
+ 		/* XXX: the bereq was not filtered pass... */
+ 		pass = 1;
+ 	} else {
+ 		/* regular object */
+ 		pass = 0;
+ 	}
+ 
+ 	/*
+ 	 * The VCL variables beresp.do_g[un]zip tells us how we want the
+ 	 * object processed before it is stored.
+ 	 *
+ 	 * The backend Content-Encoding header tells us what we are going
+ 	 * to receive, which we classify in the following three classes:
+ 	 *
+ 	 *	"Content-Encoding: gzip"	--> object is gzip'ed.
+ 	 *	no Content-Encoding		--> object is not gzip'ed.
+ 	 *	anything else			--> do nothing wrt gzip
+ 	 *
+ 	 */
+ 
+ 	AZ(sp->wrk->vfp);
+ 
+ 	/* We do nothing unless the param is set */
+ 	if (!cache_param->http_gzip_support)
+ 		sp->wrk->do_gzip = sp->wrk->do_gunzip = 0;
+ 
+ 	sp->wrk->is_gzip =
+ 	    http_HdrIs(sp->wrk->beresp, H_Content_Encoding, "gzip");
+ 
+ 	sp->wrk->is_gunzip =
+ 	    !http_GetHdr(sp->wrk->beresp, H_Content_Encoding, NULL);
+ 
+ 	/* It can't be both */
+ 	assert(sp->wrk->is_gzip == 0 || sp->wrk->is_gunzip == 0);
+ 
+ 	/* We won't gunzip unless it is gzip'ed */
+ 	if (sp->wrk->do_gunzip && !sp->wrk->is_gzip)
+ 		sp->wrk->do_gunzip = 0;
+ 
+ 	/* If we do gunzip, remove the C-E header */
+ 	if (sp->wrk->do_gunzip)
+ 		http_Unset(sp->wrk->beresp, H_Content_Encoding);
+ 
+ 	/* We wont gzip unless it is ungziped */
+ 	if (sp->wrk->do_gzip && !sp->wrk->is_gunzip)
+ 		sp->wrk->do_gzip = 0;
+ 
+ 	/* If we do gzip, add the C-E header */
+ 	if (sp->wrk->do_gzip)
+ 		http_SetHeader(sp->wrk, sp->vsl_id, sp->wrk->beresp,
+ 		    "Content-Encoding: gzip");
+ 
+ 	/* But we can't do both at the same time */
+ 	assert(sp->wrk->do_gzip == 0 || sp->wrk->do_gunzip == 0);
+ 
+ 	/* ESI takes precedence and handles gzip/gunzip itself */
+ 	if (sp->wrk->do_esi)
+ 		sp->wrk->vfp = &vfp_esi;
+ 	else if (sp->wrk->do_gunzip)
+ 		sp->wrk->vfp = &vfp_gunzip;
+ 	else if (sp->wrk->do_gzip)
+ 		sp->wrk->vfp = &vfp_gzip;
+ 	else if (sp->wrk->is_gzip)
+ 		sp->wrk->vfp = &vfp_testgzip;
+ 
+ 	if (sp->wrk->do_esi || sp->esi_level > 0)
+ 		sp->wrk->do_stream = 0;
+ 	if (!sp->wantbody)
+ 		sp->wrk->do_stream = 0;
+ 
+ 	l = http_EstimateWS(sp->wrk->beresp,
+ 	    pass ? HTTPH_R_PASS : HTTPH_A_INS, &nhttp);
++        if (sp->stale_obj) {
++            l += http_EstimateWS(sp->stale_obj->http, 0, &stale_nhttp);
++            nhttp += stale_nhttp;
++        }
+ 
+ 	/* Create Vary instructions */
+ 	if (sp->objcore != NULL) {
+ 		CHECK_OBJ_NOTNULL(sp->objcore, OBJCORE_MAGIC);
+ 		vary = VRY_Create(sp, sp->wrk->beresp);
+ 		if (vary != NULL) {
+ 			varyl = VSB_len(vary);
+ 			assert(varyl > 0);
+ 			l += varyl;
+ 		}
+ 	}
+ 
+ 	/*
+ 	 * Space for producing a Content-Length: header including padding
+ 	 * A billion gigabytes is enough for anybody.
+ 	 */
+ 	l += strlen("Content-Length: XxxXxxXxxXxxXxxXxx") + sizeof(void *);
+ 
+ 	if (sp->wrk->exp.ttl < cache_param->shortlived || sp->objcore == NULL)
+ 		sp->wrk->storage_hint = TRANSIENT_STORAGE;
+ 
+ 	sp->obj = STV_NewObject(sp, sp->wrk->storage_hint, l,
+ 	    &sp->wrk->exp, nhttp);
+ 	if (sp->obj == NULL) {
+ 		/*
+ 		 * Try to salvage the transaction by allocating a
+ 		 * shortlived object on Transient storage.
+ 		 */
+ 		sp->obj = STV_NewObject(sp, TRANSIENT_STORAGE, l,
+ 		    &sp->wrk->exp, nhttp);
+ 		if (sp->wrk->exp.ttl > cache_param->shortlived)
+ 			sp->wrk->exp.ttl = cache_param->shortlived;
+ 		sp->wrk->exp.grace = 0.0;
+ 		sp->wrk->exp.keep = 0.0;
+ 	}
+ 	if (sp->obj == NULL) {
+ 		sp->err_code = 503;
+ 		sp->step = STP_ERROR;
+ 		VDI_CloseFd(sp->wrk);
+ 		return (0);
+ 	}
+ 	CHECK_OBJ_NOTNULL(sp->obj, OBJECT_MAGIC);
++        sp->obj->exp.keep = sp->wrk->exp.keep;
+ 
+ 	sp->wrk->storage_hint = NULL;
+ 
+ 	if (sp->wrk->do_gzip || (sp->wrk->is_gzip && !sp->wrk->do_gunzip))
+ 		sp->obj->gziped = 1;
+ 
+ 	if (vary != NULL) {
+ 		sp->obj->vary =
+ 		    (void *)WS_Alloc(sp->obj->http->ws, varyl);
+ 		AN(sp->obj->vary);
+ 		memcpy(sp->obj->vary, VSB_data(vary), varyl);
+ 		VRY_Validate(sp->obj->vary);
+ 		VSB_delete(vary);
+ 	}
+ 
+ 	sp->obj->xid = sp->xid;
+ 	sp->obj->response = sp->err_code;
+ 	WS_Assert(sp->obj->ws_o);
+ 
+ 	/* Filter into object */
+ 	hp = sp->wrk->beresp;
+ 	hp2 = sp->obj->http;
+ 
+ 	hp2->logtag = HTTP_Obj;
+ 	http_CopyResp(hp2, hp);
++        
+ 	http_FilterFields(sp->wrk, sp->vsl_id, hp2, hp,
+ 	    pass ? HTTPH_R_PASS : HTTPH_A_INS);
++
++        /*
++	 * If we found a candidate for conditional backend request, attempt it
++         * now. If backend responds with 304, http_Check304() merges stale_obj
++         * into sp->obj, any other response is handled as usual. In either case,
++         * the stale_obj is no longer needed in the cache, so discard it.
++         */
++        if (sp->stale_obj) {
++            http_Check304(sp);
++            if (sp->wrk->beresp->status == 304)
++                assert(sp->obj->http->status == 200);
++	    EXP_Clr(&sp->stale_obj->exp);
++	    EXP_Rearm(sp->stale_obj);
++	    HSH_Deref(sp->wrk, NULL, &sp->stale_obj);
++	    AZ(sp->stale_obj);
++        }
+ 	http_CopyHome(sp->wrk, sp->vsl_id, hp2);
+ 
 -	if (http_GetHdr(hp, H_Last_Modified, &b))
++	if (http_GetHdr(hp, H_Last_Modified, &b)
++            || http_GetHdr(sp->obj->http, H_Last_Modified, &b))
+ 		sp->obj->last_modified = VTIM_parse(b);
+ 	else
+ 		sp->obj->last_modified = floor(sp->wrk->exp.entered);
+ 
+ 	assert(WRW_IsReleased(sp->wrk));
+ 
+ 	/*
+ 	 * If we can deliver a 304 reply, we don't bother streaming.
+ 	 * Notice that vcl_deliver{} could still nuke the headers
+ 	 * that allow the 304, in which case we return 200 non-stream.
+ 	 */
+ 	if (sp->obj->response == 200 &&
+ 	    sp->http->conds &&
+ 	    RFC2616_Do_Cond(sp))
+ 		sp->wrk->do_stream = 0;
+ 
+ 	AssertObjCorePassOrBusy(sp->obj->objcore);
+ 
+ 	if (sp->wrk->do_stream) {
+ 		sp->step = STP_PREPRESP;
+ 		return (0);
+ 	}
+ 
+ 	/* Use unmodified headers*/
+ 	i = FetchBody(sp->wrk, sp->obj);
+ 
+ 	sp->wrk->h_content_length = NULL;
+ 
+ 	http_Setup(sp->wrk->bereq, NULL);
+ 	http_Setup(sp->wrk->beresp, NULL);
+ 	sp->wrk->vfp = NULL;
+ 	assert(WRW_IsReleased(sp->wrk));
+ 	AZ(sp->wrk->vbc);
+ 	AN(sp->director);
+ 
+ 	if (i) {
+ 		HSH_Drop(sp);
+ 		AZ(sp->obj);
+ 		sp->err_code = 503;
+ 		sp->step = STP_ERROR;
+ 		return (0);
+ 	}
+ 
+ 	if (sp->obj->objcore != NULL) {
+ 		EXP_Insert(sp->obj);
+ 		AN(sp->obj->objcore);
+ 		AN(sp->obj->objcore->ban);
+ 		HSH_Unbusy(sp);
+ 	}
+ 	sp->wrk->acct_tmp.fetch++;
+ 	sp->step = STP_PREPRESP;
+ 	return (0);
+ }
+ 
+ /*--------------------------------------------------------------------
+  * Stream the body as we fetch it
+ DOT subgraph xstreambody {
+ DOT	streambody [
+ DOT		shape=ellipse
+ DOT		label="streaming\nfetch/deliver"
+ DOT	]
+ DOT }
+ DOT streambody -> DONE [style=bold,color=cyan]
+  */
+ 
+ static int
+ cnt_streambody(struct sess *sp)
+ {
+ 	int i;
+ 	struct stream_ctx sctx;
+ 	uint8_t obuf[sp->wrk->res_mode & RES_GUNZIP ?
+ 	    cache_param->gzip_stack_buffer : 1];
+ 
+ 	memset(&sctx, 0, sizeof sctx);
+ 	sctx.magic = STREAM_CTX_MAGIC;
+ 	AZ(sp->wrk->sctx);
+ 	sp->wrk->sctx = &sctx;
+ 
+ 	if (sp->wrk->res_mode & RES_GUNZIP) {
+ 		sctx.vgz = VGZ_NewUngzip(sp->wrk, "U S -");
+ 		sctx.obuf = obuf;
+ 		sctx.obuf_len = sizeof (obuf);
+ 	}
+ 
+ 	RES_StreamStart(sp);
+ 
+ 	AssertObjCorePassOrBusy(sp->obj->objcore);
+ 
+ 	i = FetchBody(sp->wrk, sp->obj);
+ 
+ 	sp->wrk->h_content_length = NULL;
+ 
+ 	http_Setup(sp->wrk->bereq, NULL);
+ 	http_Setup(sp->wrk->beresp, NULL);
+ 	sp->wrk->vfp = NULL;
+ 	AZ(sp->wrk->vbc);
+ 	AN(sp->director);
+ 
+ 	if (!i && sp->obj->objcore != NULL) {
+ 		EXP_Insert(sp->obj);
+ 		AN(sp->obj->objcore);
+ 		AN(sp->obj->objcore->ban);
+ 		HSH_Unbusy(sp);
+ 	} else {
+ 		sp->doclose = "Stream error";
+ 	}
+ 	sp->wrk->acct_tmp.fetch++;
+ 	sp->director = NULL;
+ 	sp->restarts = 0;
+ 
+ 	RES_StreamEnd(sp);
+ 	if (sp->wrk->res_mode & RES_GUNZIP)
+ 		(void)VGZ_Destroy(&sctx.vgz, sp->vsl_id);
+ 
+ 	sp->wrk->sctx = NULL;
+ 	assert(WRW_IsReleased(sp->wrk));
+ 	assert(sp->wrk->wrw.ciov == sp->wrk->wrw.siov);
+ 	(void)HSH_Deref(sp->wrk, NULL, &sp->obj);
+ 	http_Setup(sp->wrk->resp, NULL);
+ 	sp->step = STP_DONE;
+ 	return (0);
+ }
+ 
+ /*--------------------------------------------------------------------
+  * The very first request
+  */
+ static int
+ cnt_first(struct sess *sp)
+ {
+ 
+ 	/*
+ 	 * XXX: If we don't have acceptfilters we are somewhat subject
+ 	 * XXX: to DoS'ing here.  One remedy would be to set a shorter
+ 	 * XXX: SO_RCVTIMEO and once we have received something here
+ 	 * XXX: increase it to the normal value.
+ 	 */
+ 
+ 	assert(sp->xid == 0);
+ 	assert(sp->restarts == 0);
+ 	VCA_Prep(sp);
+ 
+ 	/* Record the session watermark */
+ 	sp->ws_ses = WS_Snapshot(sp->ws);
+ 
+ 	/* Receive a HTTP protocol request */
+ 	HTC_Init(sp->htc, sp->ws, sp->fd, sp->vsl_id, cache_param->http_req_size,
+ 	    cache_param->http_req_hdr_len);
+ 	sp->wrk->lastused = sp->t_open;
+ 	sp->wrk->acct_tmp.sess++;
+ 
+ 	sp->step = STP_WAIT;
+ 	return (0);
+ }
+ 
+ /*--------------------------------------------------------------------
+  * HIT
+  * We had a cache hit.  Ask VCL, then march off as instructed.
+  *
+ DOT subgraph xcluster_hit {
+ DOT	hit [
+ DOT		shape=record
+ DOT		label="vcl_hit()|req.\nobj."
+ DOT	]
+ DOT }
+ DOT hit -> err_hit [label="error"]
+ DOT err_hit [label="ERROR",shape=plaintext]
+ DOT hit -> rst_hit [label="restart",color=purple]
+ DOT rst_hit [label="RESTART",shape=plaintext]
+ DOT hit -> pass [label=pass,style=bold,color=red]
+ DOT hit -> prepresp [label="deliver",style=bold,color=green]
+  */
+ 
+ static int
+ cnt_hit(struct sess *sp)
+ {
+ 
+ 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+ 	CHECK_OBJ_NOTNULL(sp->obj, OBJECT_MAGIC);
+ 	CHECK_OBJ_NOTNULL(sp->vcl, VCL_CONF_MAGIC);
+ 
+ 	assert(!(sp->obj->objcore->flags & OC_F_PASS));
+ 
+ 	AZ(sp->wrk->do_stream);
+ 
+ 	VCL_hit_method(sp);
+ 
+ 	if (sp->handling == VCL_RET_DELIVER) {
+ 		/* Dispose of any body part of the request */
+ 		(void)FetchReqBody(sp);
+ 		AZ(sp->wrk->bereq->ws);
+ 		AZ(sp->wrk->beresp->ws);
+ 		sp->step = STP_PREPRESP;
+ 		return (0);
+ 	}
+ 
+ 	/* Drop our object, we won't need it */
+ 	(void)HSH_Deref(sp->wrk, NULL, &sp->obj);
+ 	sp->objcore = NULL;
+ 
+ 	switch(sp->handling) {
+ 	case VCL_RET_PASS:
+ 		sp->step = STP_PASS;
+ 		return (0);
+ 	case VCL_RET_ERROR:
+ 		sp->step = STP_ERROR;
+ 		return (0);
+ 	case VCL_RET_RESTART:
+ 		sp->director = NULL;
+ 		sp->restarts++;
+ 		sp->step = STP_RECV;
+ 		return (0);
+ 	default:
+ 		WRONG("Illegal action in vcl_hit{}");
+ 	}
+ }
+ 
+ /*--------------------------------------------------------------------
+  * LOOKUP
+  * Hash things together and look object up in hash-table.
+  *
+  * LOOKUP consists of two substates so that we can reenter if we
+  * encounter a busy object.
+  *
+ DOT subgraph xcluster_lookup {
+ DOT	hash [
+ DOT		shape=record
+ DOT		label="vcl_hash()|req."
+ DOT	]
+ DOT	lookup [
+ DOT		shape=diamond
+ DOT		label="obj in cache ?\ncreate if not"
+ DOT	]
+ DOT	lookup2 [
+ DOT		shape=diamond
+ DOT		label="obj.f.pass ?"
+ DOT	]
+ DOT	hash -> lookup [label="hash",style=bold,color=green]
+ DOT	lookup -> lookup2 [label="yes",style=bold,color=green]
+ DOT }
+ DOT lookup2 -> hit [label="no", style=bold,color=green]
+ DOT lookup2 -> pass [label="yes",style=bold,color=red]
+ DOT lookup -> miss [label="no",style=bold,color=blue]
+  */
+ 
+ static int
+ cnt_lookup(struct sess *sp)
+ {
+ 	struct objcore *oc;
+ 	struct object *o;
+ 	struct objhead *oh;
+ 
+ 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+ 	CHECK_OBJ_NOTNULL(sp->vcl, VCL_CONF_MAGIC);
+ 
+ 	if (sp->hash_objhead == NULL) {
+ 		/* Not a waiting list return */
+ 		AZ(sp->vary_b);
+ 		AZ(sp->vary_l);
+ 		AZ(sp->vary_e);
+ 		(void)WS_Reserve(sp->ws, 0);
+ 	} else {
+ 		AN(sp->ws->r);
+ 	}
+ 	sp->vary_b = (void*)sp->ws->f;
+ 	sp->vary_e = (void*)sp->ws->r;
+ 	sp->vary_b[2] = '\0';
+ 
+ 	oc = HSH_Lookup(sp, &oh);
+ 
+ 	if (oc == NULL) {
+ 		/*
+ 		 * We lost the session to a busy object, disembark the
+ 		 * worker thread.   The hash code to restart the session,
+ 		 * still in STP_LOOKUP, later when the busy object isn't.
+ 		 * NB:  Do not access sp any more !
+ 		 */
+ 		return (1);
+ 	}
+ 
+ 
+ 	CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
+ 	CHECK_OBJ_NOTNULL(oh, OBJHEAD_MAGIC);
+ 
+ 	/* If we inserted a new object it's a miss */
+ 	if (oc->flags & OC_F_BUSY) {
+ 		sp->wrk->stats.cache_miss++;
+ 
+ 		if (sp->vary_l != NULL) {
+ 			assert(oc->busyobj->vary == sp->vary_b);
+ 			VRY_Validate(oc->busyobj->vary);
+ 			WS_ReleaseP(sp->ws, (void*)sp->vary_l);
+ 		} else {
+ 			AZ(oc->busyobj->vary);
+ 			WS_Release(sp->ws, 0);
+ 		}
+ 		sp->vary_b = NULL;
+ 		sp->vary_l = NULL;
+ 		sp->vary_e = NULL;
+ 
+ 		sp->objcore = oc;
+ 		sp->step = STP_MISS;
+ 		return (0);
+ 	}
+ 
+ 	o = oc_getobj(sp->wrk, oc);
+ 	CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC);
+ 	sp->obj = o;
+ 
+ 	WS_Release(sp->ws, 0);
+ 	sp->vary_b = NULL;
+ 	sp->vary_l = NULL;
+ 	sp->vary_e = NULL;
+ 
+ 	if (oc->flags & OC_F_PASS) {
+ 		sp->wrk->stats.cache_hitpass++;
+ 		WSP(sp, SLT_HitPass, "%u", sp->obj->xid);
+ 		(void)HSH_Deref(sp->wrk, NULL, &sp->obj);
++                if (sp->stale_obj != NULL)
++                    (void)HSH_Deref(sp->wrk, NULL, &sp->stale_obj);
+ 		sp->objcore = NULL;
+ 		sp->step = STP_PASS;
+ 		return (0);
+ 	}
+ 
+ 	sp->wrk->stats.cache_hit++;
+ 	WSP(sp, SLT_Hit, "%u", sp->obj->xid);
+ 	sp->step = STP_HIT;
+ 	return (0);
+ }
+ 
+ /*--------------------------------------------------------------------
+  * We had a miss, ask VCL, proceed as instructed
+  *
+ DOT subgraph xcluster_miss {
+ DOT	miss [
+ DOT		shape=ellipse
+ DOT		label="filter req.->bereq."
+ DOT	]
+ DOT	vcl_miss [
+ DOT		shape=record
+ DOT		label="vcl_miss()|req.\nbereq."
+ DOT	]
+ DOT	miss -> vcl_miss [style=bold,color=blue]
+ DOT }
+ DOT vcl_miss -> rst_miss [label="restart",color=purple]
+ DOT rst_miss [label="RESTART",shape=plaintext]
+ DOT vcl_miss -> err_miss [label="error"]
+ DOT err_miss [label="ERROR",shape=plaintext]
+ DOT vcl_miss -> fetch [label="fetch",style=bold,color=blue]
+ DOT vcl_miss -> pass [label="pass",style=bold,color=red]
+ DOT
+  */
+ 
+ static int
+ cnt_miss(struct sess *sp)
+ {
+ 
+ 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+ 	CHECK_OBJ_NOTNULL(sp->vcl, VCL_CONF_MAGIC);
+ 
+ 	AZ(sp->obj);
+ 	AN(sp->objcore);
+ 	WS_Reset(sp->wrk->ws, NULL);
+ 	http_Setup(sp->wrk->bereq, sp->wrk->ws);
+ 	http_FilterHeader(sp, HTTPH_R_FETCH);
+ 	http_ForceGet(sp->wrk->bereq);
+ 	if (cache_param->http_gzip_support) {
+ 		/*
+ 		 * We always ask the backend for gzip, even if the
+ 		 * client doesn't grok it.  We will uncompress for
+ 		 * the minority of clients which don't.
+ 		 */
+ 		http_Unset(sp->wrk->bereq, H_Accept_Encoding);
+ 		http_SetHeader(sp->wrk, sp->vsl_id, sp->wrk->bereq,
+ 		    "Accept-Encoding: gzip");
+ 	}
+ 	sp->wrk->connect_timeout = 0;
+ 	sp->wrk->first_byte_timeout = 0;
+ 	sp->wrk->between_bytes_timeout = 0;
++
++        /* If a candidate for a conditional backend request was found,
++         * add If-Modified-Since and/or If-None-Match to the bereq.
++         */
++        if (sp->stale_obj)
++                http_CheckRefresh(sp);
++
+ 	VCL_miss_method(sp);
+ 	switch(sp->handling) {
+ 	case VCL_RET_ERROR:
+ 		AZ(HSH_Deref(sp->wrk, sp->objcore, NULL));
+ 		sp->objcore = NULL;
+ 		http_Setup(sp->wrk->bereq, NULL);
+ 		sp->step = STP_ERROR;
+ 		return (0);
+ 	case VCL_RET_PASS:
+ 		AZ(HSH_Deref(sp->wrk, sp->objcore, NULL));
+ 		sp->objcore = NULL;
+ 		sp->step = STP_PASS;
+ 		return (0);
+ 	case VCL_RET_FETCH:
+ 		sp->step = STP_FETCH;
+ 		return (0);
+ 	case VCL_RET_RESTART:
+ 		AZ(HSH_Deref(sp->wrk, sp->objcore, NULL));
+ 		sp->objcore = NULL;
+ 		INCOMPL();
+ 	default:
+ 		WRONG("Illegal action in vcl_miss{}");
+ 	}
+ }
+ 
+ /*--------------------------------------------------------------------
+  * Start pass processing by getting headers from backend, then
+  * continue in passbody.
+  *
+ DOT subgraph xcluster_pass {
+ DOT	pass [
+ DOT		shape=ellipse
+ DOT		label="deref obj."
+ DOT	]
+ DOT	pass2 [
+ DOT		shape=ellipse
+ DOT		label="filter req.->bereq."
+ DOT	]
+ DOT	vcl_pass [
+ DOT		shape=record
+ DOT		label="vcl_pass()|req.\nbereq."
+ DOT	]
+ DOT	pass_do [
+ DOT		shape=ellipse
+ DOT		label="create anon object\n"
+ DOT	]
+ DOT	pass -> pass2 [style=bold, color=red]
+ DOT	pass2 -> vcl_pass [style=bold, color=red]
+ DOT	vcl_pass -> pass_do [label="pass"] [style=bold, color=red]
+ DOT }
+ DOT pass_do -> fetch [style=bold, color=red]
+ DOT vcl_pass -> rst_pass [label="restart",color=purple]
+ DOT rst_pass [label="RESTART",shape=plaintext]
+ DOT vcl_pass -> err_pass [label="error"]
+ DOT err_pass [label="ERROR",shape=plaintext]
+  */
+ 
+ static int
+ cnt_pass(struct sess *sp)
+ {
+ 
+ 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+ 	CHECK_OBJ_NOTNULL(sp->vcl, VCL_CONF_MAGIC);
+ 	AZ(sp->obj);
+ 
+ 	WS_Reset(sp->wrk->ws, NULL);
+ 	http_Setup(sp->wrk->bereq, sp->wrk->ws);
+ 	http_FilterHeader(sp, HTTPH_R_PASS);
+ 
+ 	sp->wrk->connect_timeout = 0;
+ 	sp->wrk->first_byte_timeout = 0;
+ 	sp->wrk->between_bytes_timeout = 0;
+ 	VCL_pass_method(sp);
+ 	if (sp->handling == VCL_RET_ERROR) {
+ 		http_Setup(sp->wrk->bereq, NULL);
+ 		sp->step = STP_ERROR;
+ 		return (0);
+ 	}
+ 	assert(sp->handling == VCL_RET_PASS);
+ 	sp->wrk->acct_tmp.pass++;
+ 	sp->sendbody = 1;
+ 	sp->step = STP_FETCH;
+ 	return (0);
+ }
+ 
+ /*--------------------------------------------------------------------
+  * Ship the request header to the backend unchanged, then pipe
+  * until one of the ends close the connection.
+  *
+ DOT subgraph xcluster_pipe {
+ DOT	pipe [
+ DOT		shape=ellipse
+ DOT		label="Filter req.->bereq."
+ DOT	]
+ DOT	vcl_pipe [
+ DOT		shape=record
+ DOT		label="vcl_pipe()|req.\nbereq\."
+ DOT	]
+ DOT	pipe_do [
+ DOT		shape=ellipse
+ DOT		label="send bereq.\npipe until close"
+ DOT	]
+ DOT	vcl_pipe -> pipe_do [label="pipe",style=bold,color=orange]
+ DOT	pipe -> vcl_pipe [style=bold,color=orange]
+ DOT }
+ DOT pipe_do -> DONE [style=bold,color=orange]
+ DOT vcl_pipe -> err_pipe [label="error"]
+ DOT err_pipe [label="ERROR",shape=plaintext]
+  */
+ 
+ static int
+ cnt_pipe(struct sess *sp)
+ {
+ 
+ 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+ 	CHECK_OBJ_NOTNULL(sp->vcl, VCL_CONF_MAGIC);
+ 
+ 	sp->wrk->acct_tmp.pipe++;
+ 	WS_Reset(sp->wrk->ws, NULL);
+ 	http_Setup(sp->wrk->bereq, sp->wrk->ws);
+ 	http_FilterHeader(sp, HTTPH_R_PIPE);
+ 
+ 	VCL_pipe_method(sp);
+ 
+ 	if (sp->handling == VCL_RET_ERROR)
+ 		INCOMPL();
+ 	assert(sp->handling == VCL_RET_PIPE);
+ 
+ 	PipeSession(sp);
+ 	assert(WRW_IsReleased(sp->wrk));
+ 	http_Setup(sp->wrk->bereq, NULL);
+ 	sp->step = STP_DONE;
+ 	return (0);
+ }
+ 
+ /*--------------------------------------------------------------------
+  * RECV
+  * We have a complete request, set everything up and start it.
+  *
+ DOT subgraph xcluster_recv {
+ DOT	recv [
+ DOT		shape=record
+ DOT		label="vcl_recv()|req."
+ DOT	]
+ DOT }
+ DOT RESTART -> recv
+ DOT recv -> pipe [label="pipe",style=bold,color=orange]
+ DOT recv -> pass2 [label="pass",style=bold,color=red]
+ DOT recv -> err_recv [label="error"]
+ DOT err_recv [label="ERROR",shape=plaintext]
+ DOT recv -> hash [label="lookup",style=bold,color=green]
+  */
+ 
+ static int
+ cnt_recv(struct sess *sp)
+ {
+ 	unsigned recv_handling;
+ 
+ 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+ 	CHECK_OBJ_NOTNULL(sp->vcl, VCL_CONF_MAGIC);
+ 	AZ(sp->obj);
+ 	assert(sp->wrk->wrw.ciov == sp->wrk->wrw.siov);
+ 
+ 	/* By default we use the first backend */
+ 	AZ(sp->director);
+ 	sp->director = sp->vcl->director[0];
+ 	AN(sp->director);
+ 
+ 	sp->disable_esi = 0;
+ 	sp->hash_always_miss = 0;
+ 	sp->hash_ignore_busy = 0;
+ 	sp->client_identity = NULL;
+ 
+ 	http_CollectHdr(sp->http, H_Cache_Control);
+ 
+ 	VCL_recv_method(sp);
+ 	recv_handling = sp->handling;
+ 
+ 	if (sp->restarts >= cache_param->max_restarts) {
+ 		if (sp->err_code == 0)
+ 			sp->err_code = 503;
+ 		sp->step = STP_ERROR;
+ 		return (0);
+ 	}
+ 
+ 	/* Zap these, in case we came here through restart */
+ 	sp->wrk->do_esi = 0;
+ 	sp->wrk->is_gzip = 0;
+ 	sp->wrk->is_gunzip = 0;
+ 	sp->wrk->do_gzip = 0;
+ 	sp->wrk->do_gunzip = 0;
+ 	sp->wrk->do_stream = 0;
+ 
+ 	if (cache_param->http_gzip_support &&
+ 	     (recv_handling != VCL_RET_PIPE) &&
+ 	     (recv_handling != VCL_RET_PASS)) {
+ 		if (RFC2616_Req_Gzip(sp)) {
+ 			http_Unset(sp->http, H_Accept_Encoding);
+ 			http_SetHeader(sp->wrk, sp->vsl_id, sp->http,
+ 			    "Accept-Encoding: gzip");
+ 		} else {
+ 			http_Unset(sp->http, H_Accept_Encoding);
+ 		}
+ 	}
+ 
+ 	SHA256_Init(sp->wrk->sha256ctx);
+ 	VCL_hash_method(sp);
+ 	assert(sp->handling == VCL_RET_HASH);
+ 	SHA256_Final(sp->digest, sp->wrk->sha256ctx);
+ 
+ 	if (!strcmp(sp->http->hd[HTTP_HDR_REQ].b, "HEAD"))
+ 		sp->wantbody = 0;
+ 	else
+ 		sp->wantbody = 1;
+ 
+ 	sp->sendbody = 0;
+ 	switch(recv_handling) {
+ 	case VCL_RET_LOOKUP:
+ 		/* XXX: discard req body, if any */
+ 		sp->step = STP_LOOKUP;
+ 		return (0);
+ 	case VCL_RET_PIPE:
+ 		if (sp->esi_level > 0) {
+ 			/* XXX: VSL something */
+ 			INCOMPL();
+ 			/* sp->step = STP_DONE; */
+ 			return (1);
+ 		}
+ 		sp->step = STP_PIPE;
+ 		return (0);
+ 	case VCL_RET_PASS:
+ 		sp->step = STP_PASS;
+ 		return (0);
+ 	case VCL_RET_ERROR:
+ 		/* XXX: discard req body, if any */
+ 		sp->step = STP_ERROR;
+ 		return (0);
+ 	default:
+ 		WRONG("Illegal action in vcl_recv{}");
+ 	}
+ }
+ 
+ /*--------------------------------------------------------------------
+  * START
+  * Handle a request, wherever it came from recv/restart.
+  *
+ DOT start [shape=box,label="Dissect request"]
+ DOT start -> recv [style=bold,color=green]
+  */
+ 
+ static int
+ cnt_start(struct sess *sp)
+ {
+ 	uint16_t done;
+ 	char *p;
+ 	const char *r = "HTTP/1.1 100 Continue\r\n\r\n";
+ 
+ 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+ 	AZ(sp->restarts);
+ 	AZ(sp->obj);
+ 	AZ(sp->vcl);
+ 
+ 	/* Update stats of various sorts */
+ 	sp->wrk->stats.client_req++;
+ 	sp->t_req = VTIM_real();
+ 	sp->wrk->lastused = sp->t_req;
+ 	sp->wrk->acct_tmp.req++;
+ 
+ 	/* Assign XID and log */
+ 	sp->xid = ++xids;				/* XXX not locked */
+ 	WSP(sp, SLT_ReqStart, "%s %s %u", sp->addr, sp->port,  sp->xid);
+ 
+ 	/* Borrow VCL reference from worker thread */
+ 	VCL_Refresh(&sp->wrk->vcl);
+ 	sp->vcl = sp->wrk->vcl;
+ 	sp->wrk->vcl = NULL;
+ 
+ 	http_Setup(sp->http, sp->ws);
+ 	done = http_DissectRequest(sp);
+ 
+ 	/* If we could not even parse the request, just close */
+ 	if (done == 400) {
+ 		sp->step = STP_DONE;
+ 		SES_Close(sp, "junk");
+ 		return (0);
+ 	}
+ 
+ 	/* Catch request snapshot */
+ 	sp->ws_req = WS_Snapshot(sp->ws);
+ 
+ 	/* Catch original request, before modification */
+ 	HTTP_Copy(sp->http0, sp->http);
+ 
+ 	if (done != 0) {
+ 		sp->err_code = done;
+ 		sp->step = STP_ERROR;
+ 		return (0);
+ 	}
+ 
+ 	sp->doclose = http_DoConnection(sp->http);
+ 
+ 	/* XXX: Handle TRACE & OPTIONS of Max-Forwards = 0 */
+ 
+ 	/*
+ 	 * Handle Expect headers
+ 	 */
+ 	if (http_GetHdr(sp->http, H_Expect, &p)) {
+ 		if (strcasecmp(p, "100-continue")) {
+ 			sp->err_code = 417;
+ 			sp->step = STP_ERROR;
+ 			return (0);
+ 		}
+ 
+ 		/* XXX: Don't bother with write failures for now */
+ 		(void)write(sp->fd, r, strlen(r));
+ 		/* XXX: When we do ESI includes, this is not removed
+ 		 * XXX: because we use http0 as our basis.  Believed
+ 		 * XXX: safe, but potentially confusing.
+ 		 */
+ 		http_Unset(sp->http, H_Expect);
+ 	}
+ 
+ 	sp->step = STP_RECV;
+ 	return (0);
+ }
+ 
+ /*--------------------------------------------------------------------
+  * Central state engine dispatcher.
+  *
+  * Kick the session around until it has had enough.
+  *
+  */
+ 
+ static void
+ cnt_diag(struct sess *sp, const char *state)
+ {
+ 	if (sp->wrk != NULL) {
+ 		WSP(sp, SLT_Debug, "thr %p STP_%s sp %p obj %p vcl %p",
+ 		    pthread_self(), state, sp, sp->obj, sp->vcl);
+ 		WSL_Flush(sp->wrk, 0);
+ 	} else {
+ 		VSL(SLT_Debug, sp->vsl_id,
+ 		    "thr %p STP_%s sp %p obj %p vcl %p",
+ 		    pthread_self(), state, sp, sp->obj, sp->vcl);
+ 	}
+ }
+ 
+ void
+ CNT_Session(struct sess *sp)
+ {
+ 	int done;
+ 	struct worker *w;
+ 
+ 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+ 	w = sp->wrk;
+ 	CHECK_OBJ_NOTNULL(w, WORKER_MAGIC);
+ 
+ 	/*
+ 	 * Possible entrance states
+ 	 */
+ 	assert(
+ 	    sp->step == STP_FIRST ||
+ 	    sp->step == STP_START ||
+ 	    sp->step == STP_LOOKUP ||
+ 	    sp->step == STP_RECV);
+ 
+ 	AZ(w->do_stream);
+ 	AZ(w->is_gzip);
+ 	AZ(w->do_gzip);
+ 	AZ(w->is_gunzip);
+ 	AZ(w->do_gunzip);
+ 	AZ(w->do_esi);
+ 
+ 	/*
+ 	 * Whenever we come in from the acceptor or waiter, we need to set
+ 	 * blocking mode, but there is no point in setting it when we come from
+ 	 * ESI or when a parked sessions returns.
+ 	 * It would be simpler to do this in the acceptor or waiter, but we'd
+ 	 * rather do the syscall in the worker thread.
+ 	 * On systems which return errors for ioctl, we close early
+ 	 */
+ 	if ((sp->step == STP_FIRST || sp->step == STP_START) &&
+ 	    VTCP_blocking(sp->fd)) {
+ 		if (errno == ECONNRESET)
+ 			SES_Close(sp, "remote closed");
+ 		else
+ 			SES_Close(sp, "error");
+ 		sp->step = STP_DONE;
+ 	}
+ 
+ 	/*
+ 	 * NB: Once done is set, we can no longer touch sp!
+ 	 */
+ 	for (done = 0; !done; ) {
+ 		assert(sp->wrk == w);
+ 		/*
+ 		 * This is a good place to be paranoid about the various
+ 		 * pointers still pointing to the things we expect.
+ 		 */
+ 		CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+ 		CHECK_OBJ_ORNULL(sp->obj, OBJECT_MAGIC);
+ 		CHECK_OBJ_NOTNULL(sp->wrk, WORKER_MAGIC);
+ 		CHECK_OBJ_ORNULL(w->nobjhead, OBJHEAD_MAGIC);
+ 		WS_Assert(w->ws);
+ 
+ 		switch (sp->step) {
+ #define STEP(l,u) \
+ 		    case STP_##u: \
+ 			if (cache_param->diag_bitmap & 0x01) \
+ 				cnt_diag(sp, #u); \
+ 			done = cnt_##l(sp); \
+ 		        break;
+ #include "tbl/steps.h"
+ #undef STEP
+ 		default:
+ 			WRONG("State engine misfire");
+ 		}
+ 		WS_Assert(w->ws);
+ 		CHECK_OBJ_ORNULL(w->nobjhead, OBJHEAD_MAGIC);
+ 	}
+ 	WSL_Flush(w, 0);
+ 	AZ(w->do_stream);
+ 	AZ(w->is_gzip);
+ 	AZ(w->do_gzip);
+ 	AZ(w->is_gunzip);
+ 	AZ(w->do_gunzip);
+ 	AZ(w->do_esi);
+ #define ACCT(foo)	AZ(w->acct_tmp.foo);
+ #include "tbl/acct_fields.h"
+ #undef ACCT
+ 	assert(WRW_IsReleased(w));
+ }
+ 
+ /*
+ DOT }
+ */
+ 
+ /*--------------------------------------------------------------------
+  * Debugging aids
+  */
+ 
+ static void
+ cli_debug_xid(struct cli *cli, const char * const *av, void *priv)
+ {
+ 	(void)priv;
+ 	if (av[2] != NULL)
+ 		xids = strtoul(av[2], NULL, 0);
+ 	VCLI_Out(cli, "XID is %u", xids);
+ }
+ 
+ /*
+  * Default to seed=1, this is the only seed value POSIXl guarantees will
+  * result in a reproducible random number sequence.
+  */
+ static void
+ cli_debug_srandom(struct cli *cli, const char * const *av, void *priv)
+ {
+ 	(void)priv;
+ 	unsigned seed = 1;
+ 
+ 	if (av[2] != NULL)
+ 		seed = strtoul(av[2], NULL, 0);
+ 	srandom(seed);
+ 	srand48(random());
+ 	VCLI_Out(cli, "Random(3) seeded with %lu", seed);
+ }
+ 
+ static struct cli_proto debug_cmds[] = {
+ 	{ "debug.xid", "debug.xid",
+ 		"\tExamine or set XID\n", 0, 1, "d", cli_debug_xid },
+ 	{ "debug.srandom", "debug.srandom",
+ 		"\tSeed the random(3) function\n", 0, 1, "d", cli_debug_srandom },
+ 	{ NULL }
+ };
+ 
+ /*--------------------------------------------------------------------
+  *
+  */
+ 
+ void
+ CNT_Init(void)
+ {
+ 
+ 	srandomdev();
+ 	srand48(random());
+ 	xids = random();
+ 	CLI_AddFuncs(debug_cmds);
+ }
+ 
+ 
diff --cc bin/varnishd/cache/cache_expire.c
index 0000000,23e3fc6..31fd41a
mode 000000,100644..100644
--- a/bin/varnishd/cache/cache_expire.c
+++ b/bin/varnishd/cache/cache_expire.c
@@@ -1,0 -1,490 +1,490 @@@
+ /*-
+  * Copyright (c) 2006 Verdens Gang AS
+  * Copyright (c) 2006-2011 Varnish Software AS
+  * All rights reserved.
+  *
+  * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
+  *
+  * Redistribution and use in source and binary forms, with or without
+  * modification, are permitted provided that the following conditions
+  * are met:
+  * 1. Redistributions of source code must retain the above copyright
+  *    notice, this list of conditions and the following disclaimer.
+  * 2. Redistributions in binary form must reproduce the above copyright
+  *    notice, this list of conditions and the following disclaimer in the
+  *    documentation and/or other materials provided with the distribution.
+  *
+  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+  * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+  * SUCH DAMAGE.
+  *
+  * LRU and object timer handling.
+  *
+  * We have two data structures, a LRU-list and a binary heap for the timers
+  * and two ways to kill objects: TTL-timeouts and LRU cleanups.
+  *
+  * Any object on the LRU is also on the binheap and vice versa.
+  *
+  * We hold a single object reference for both data structures.
+  *
+  * An attempted overview:
+  *
+  *	                        EXP_Ttl()      EXP_Grace()   EXP_Keep()
+  *				   |                |            |
+  *      entered                    v                v            |
+  *         |                       +--------------->+            |
+  *         v                       |      grace                  |
+  *         +---------------------->+                             |
+  *                  ttl            |                             v
+  *                                 +---------------------------->+
+  *                                     keep
+  *
+  */
+ 
+ #include "config.h"
+ 
+ #include <math.h>
+ 
+ #include "cache.h"
+ 
+ #include "binary_heap.h"
+ #include "hash/hash_slinger.h"
+ #include "vtim.h"
+ 
+ static pthread_t exp_thread;
+ static struct binheap *exp_heap;
+ static struct lock exp_mtx;
+ 
+ /*--------------------------------------------------------------------
+  * struct exp manipulations
+  *
+  * The Get/Set functions encapsulate the mutual magic between the
+  * fields in one single place.
+  */
+ 
+ void
+ EXP_Clr(struct exp *e)
+ {
+ 
+ 	e->ttl = -1;
+ 	e->grace = -1;
+ 	e->keep = -1;
+ 	e->age = 0;
+ 	e->entered = 0;
+ }
+ 
+ #define EXP_ACCESS(fld, low_val, extra)				\
+ 	double							\
+ 	EXP_Get_##fld(const struct exp *e)			\
+ 	{							\
+ 		return (e->fld > 0. ? e->fld : low_val);	\
+ 	}							\
+ 								\
+ 	void							\
+ 	EXP_Set_##fld(struct exp *e, double v)			\
+ 	{							\
+ 		if (v > 0.)					\
+ 			e->fld = v;				\
+ 		else {						\
+ 			e->fld = -1.;				\
+ 			extra;					\
+ 		}						\
+ 	}							\
+ 
+ EXP_ACCESS(ttl, -1., (e->grace = e->keep = -1.))
+ EXP_ACCESS(grace, 0., )
+ EXP_ACCESS(keep, 0.,)
+ 
+ /*--------------------------------------------------------------------
+  * Calculate an objects effective keep, grace or ttl time, suitably
+  * adjusted for defaults and by per-session limits.
+  */
+ 
 -static double
++double
+ EXP_Keep(const struct sess *sp, const struct object *o)
+ {
+ 	double r;
+ 
+ 	r = (double)cache_param->default_keep;
+ 	if (o->exp.keep > 0.)
+ 		r = o->exp.keep;
+ 	if (sp != NULL && sp->exp.keep > 0. && sp->exp.keep < r)
+ 		r = sp->exp.keep;
+ 	return (EXP_Ttl(sp, o) + r);
+ }
+ 
+ double
+ EXP_Grace(const struct sess *sp, const struct object *o)
+ {
+ 	double r;
+ 
+ 	r = (double)cache_param->default_grace;
+ 	if (o->exp.grace >= 0.)
+ 		r = o->exp.grace;
+ 	if (sp != NULL && sp->exp.grace > 0. && sp->exp.grace < r)
+ 		r = sp->exp.grace;
+ 	return (EXP_Ttl(sp, o) + r);
+ }
+ 
+ double
+ EXP_Ttl(const struct sess *sp, const struct object *o)
+ {
+ 	double r;
+ 
+ 	r = o->exp.ttl;
+ 	if (sp != NULL && sp->exp.ttl > 0. && sp->exp.ttl < r)
+ 		r = sp->exp.ttl;
+ 	return (o->exp.entered + r);
+ }
+ 
+ /*--------------------------------------------------------------------
+  * When & why does the timer fire for this object ?
+  */
+ 
+ static int
+ update_object_when(const struct object *o)
+ {
+ 	struct objcore *oc;
+ 	double when, w2;
+ 
+ 	CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC);
+ 	oc = o->objcore;
+ 	CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
+ 	Lck_AssertHeld(&exp_mtx);
+ 
+ 	when = EXP_Keep(NULL, o);
+ 	w2 = EXP_Grace(NULL, o);
+ 	if (w2 > when)
+ 		when = w2;
+ 	assert(!isnan(when));
+ 	if (when == oc->timer_when)
+ 		return (0);
+ 	oc->timer_when = when;
+ 	return (1);
+ }
+ 
+ /*--------------------------------------------------------------------*/
+ 
+ static void
+ exp_insert(struct objcore *oc, struct lru *lru)
+ {
+ 	CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
+ 	CHECK_OBJ_NOTNULL(lru, LRU_MAGIC);
+ 
+ 	Lck_AssertHeld(&lru->mtx);
+ 	Lck_AssertHeld(&exp_mtx);
+ 	assert(oc->timer_idx == BINHEAP_NOIDX);
+ 	binheap_insert(exp_heap, oc);
+ 	assert(oc->timer_idx != BINHEAP_NOIDX);
+ 	VTAILQ_INSERT_TAIL(&lru->lru_head, oc, lru_list);
+ }
+ 
+ /*--------------------------------------------------------------------
+  * Object has been added to cache, record in lru & binheap.
+  *
+  * The objcore comes with a reference, which we inherit.
+  */
+ 
+ void
+ EXP_Inject(struct objcore *oc, struct lru *lru, double when)
+ {
+ 
+ 	CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
+ 	CHECK_OBJ_NOTNULL(lru, LRU_MAGIC);
+ 
+ 	Lck_Lock(&lru->mtx);
+ 	Lck_Lock(&exp_mtx);
+ 	oc->timer_when = when;
+ 	exp_insert(oc, lru);
+ 	Lck_Unlock(&exp_mtx);
+ 	Lck_Unlock(&lru->mtx);
+ }
+ 
+ /*--------------------------------------------------------------------
+  * Object has been added to cache, record in lru & binheap.
+  *
+  * We grab a reference to the object, which will keep it around until
+  * we decide its time to let it go.
+  */
+ 
+ void
+ EXP_Insert(struct object *o)
+ {
+ 	struct objcore *oc;
+ 	struct lru *lru;
+ 
+ 	CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC);
+ 	oc = o->objcore;
+ 	CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
+ 	AssertObjBusy(o);
+ 	HSH_Ref(oc);
+ 
+ 	assert(o->exp.entered != 0 && !isnan(o->exp.entered));
+ 	o->last_lru = o->exp.entered;
+ 
+ 	lru = oc_getlru(oc);
+ 	CHECK_OBJ_NOTNULL(lru, LRU_MAGIC);
+ 	Lck_Lock(&lru->mtx);
+ 	Lck_Lock(&exp_mtx);
+ 	(void)update_object_when(o);
+ 	exp_insert(oc, lru);
+ 	Lck_Unlock(&exp_mtx);
+ 	Lck_Unlock(&lru->mtx);
+ 	oc_updatemeta(oc);
+ }
+ 
+ /*--------------------------------------------------------------------
+  * Object was used, move to tail of LRU list.
+  *
+  * To avoid the exp_mtx becoming a hotspot, we only attempt to move
+  * objects if they have not been moved recently and if the lock is available.
+  * This optimization obviously leaves the LRU list imperfectly sorted.
+  */
+ 
+ int
+ EXP_Touch(struct objcore *oc)
+ {
+ 	struct lru *lru;
+ 
+ 	CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
+ 
+ 	/*
+ 	 * For -spersistent we don't move objects on the lru list.  Each
+ 	 * segment has its own LRU list, and the order on it is not material
+ 	 * for anything.  The code below would move the objects to the
+ 	 * LRU list of the currently open segment, which would prevent
+ 	 * the cleaner from doing its job.
+ 	 */
+ 	if (oc->flags & OC_F_LRUDONTMOVE)
+ 		return (0);
+ 
+ 	lru = oc_getlru(oc);
+ 	CHECK_OBJ_NOTNULL(lru, LRU_MAGIC);
+ 
+ 	/*
+ 	 * We only need the LRU lock here.  The locking order is LRU->EXP
+ 	 * so we can trust the content of the oc->timer_idx without the
+ 	 * EXP lock.   Since each lru list has its own lock, this should
+ 	 * reduce contention a fair bit
+ 	 */
+ 	if (Lck_Trylock(&lru->mtx))
+ 		return (0);
+ 
+ 	if (oc->timer_idx != BINHEAP_NOIDX) {
+ 		VTAILQ_REMOVE(&lru->lru_head, oc, lru_list);
+ 		VTAILQ_INSERT_TAIL(&lru->lru_head, oc, lru_list);
+ 		VSC_C_main->n_lru_moved++;
+ 	}
+ 	Lck_Unlock(&lru->mtx);
+ 	return (1);
+ }
+ 
+ /*--------------------------------------------------------------------
+  * We have changed one or more of the object timers, shuffle it
+  * accordingly in the binheap
+  *
+  * The VCL code can send us here on a non-cached object, just return.
+  *
+  * XXX: special case check for ttl = 0 ?
+  */
+ 
+ void
+ EXP_Rearm(const struct object *o)
+ {
+ 	struct objcore *oc;
+ 	struct lru *lru;
+ 
+ 	CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC);
+ 	oc = o->objcore;
+ 	if (oc == NULL)
+ 		return;
+ 	CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
+ 	lru = oc_getlru(oc);
+ 	Lck_Lock(&lru->mtx);
+ 	Lck_Lock(&exp_mtx);
+ 	/*
+ 	 * The hang-man might have this object of the binheap while
+ 	 * tending to a timer.  If so, we do not muck with it here.
+ 	 */
+ 	if (oc->timer_idx != BINHEAP_NOIDX && update_object_when(o)) {
+ 		assert(oc->timer_idx != BINHEAP_NOIDX);
+ 		binheap_reorder(exp_heap, oc->timer_idx);
+ 		assert(oc->timer_idx != BINHEAP_NOIDX);
+ 	}
+ 	Lck_Unlock(&exp_mtx);
+ 	Lck_Unlock(&lru->mtx);
+ 	oc_updatemeta(oc);
+ }
+ 
+ /*--------------------------------------------------------------------
+  * This thread monitors the root of the binary heap and whenever an
+  * object expires, accounting also for graceability, it is killed.
+  */
+ 
+ static void * __match_proto__(void *start_routine(void *))
+ exp_timer(struct sess *sp, void *priv)
+ {
+ 	struct objcore *oc;
+ 	struct lru *lru;
+ 	double t;
+ 	struct object *o;
+ 
+ 	(void)priv;
+ 	t = VTIM_real();
+ 	oc = NULL;
+ 	while (1) {
+ 		if (oc == NULL) {
+ 			WSL_Flush(sp->wrk, 0);
+ 			WRK_SumStat(sp->wrk);
+ 			VTIM_sleep(cache_param->expiry_sleep);
+ 			t = VTIM_real();
+ 		}
+ 
+ 		Lck_Lock(&exp_mtx);
+ 		oc = binheap_root(exp_heap);
+ 		if (oc == NULL) {
+ 			Lck_Unlock(&exp_mtx);
+ 			continue;
+ 		}
+ 		CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
+ 
+ 		/*
+ 		 * We may have expired so many objects that our timestamp
+ 		 * got out of date, refresh it and check again.
+ 		 */
+ 		if (oc->timer_when > t)
+ 			t = VTIM_real();
+ 		if (oc->timer_when > t) {
+ 			Lck_Unlock(&exp_mtx);
+ 			oc = NULL;
+ 			continue;
+ 		}
+ 
+ 		/*
+ 		 * It's time...
+ 		 * Technically we should drop the exp_mtx, get the lru->mtx
+ 		 * get the exp_mtx again and then check that the oc is still
+ 		 * on the binheap.  We take the shorter route and try to
+ 		 * get the lru->mtx and punt if we fail.
+ 		 */
+ 
+ 		lru = oc_getlru(oc);
+ 		CHECK_OBJ_NOTNULL(lru, LRU_MAGIC);
+ 		if (Lck_Trylock(&lru->mtx)) {
+ 			Lck_Unlock(&exp_mtx);
+ 			oc = NULL;
+ 			continue;
+ 		}
+ 
+ 		/* Remove from binheap */
+ 		assert(oc->timer_idx != BINHEAP_NOIDX);
+ 		binheap_delete(exp_heap, oc->timer_idx);
+ 		assert(oc->timer_idx == BINHEAP_NOIDX);
+ 
+ 		/* And from LRU */
+ 		lru = oc_getlru(oc);
+ 		VTAILQ_REMOVE(&lru->lru_head, oc, lru_list);
+ 
+ 		Lck_Unlock(&exp_mtx);
+ 		Lck_Unlock(&lru->mtx);
+ 
+ 		VSC_C_main->n_expired++;
+ 
+ 		CHECK_OBJ_NOTNULL(oc->objhead, OBJHEAD_MAGIC);
+ 		o = oc_getobj(sp->wrk, oc);
+ 		WSL(sp->wrk, SLT_ExpKill, 0, "%u %.0f",
+ 		    o->xid, EXP_Ttl(NULL, o) - t);
+ 		(void)HSH_Deref(sp->wrk, oc, NULL);
+ 	}
+ 	NEEDLESS_RETURN(NULL);
+ }
+ 
+ /*--------------------------------------------------------------------
+  * Attempt to make space by nuking the oldest object on the LRU list
+  * which isn't in use.
+  * Returns: 1: did, 0: didn't, -1: can't
+  */
+ 
+ int
+ EXP_NukeOne(struct worker *w, struct lru *lru)
+ {
+ 	struct objcore *oc;
+ 	struct object *o;
+ 
+ 	/* Find the first currently unused object on the LRU.  */
+ 	Lck_Lock(&lru->mtx);
+ 	Lck_Lock(&exp_mtx);
+ 	VTAILQ_FOREACH(oc, &lru->lru_head, lru_list) {
+ 		CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
+ 		assert (oc->timer_idx != BINHEAP_NOIDX);
+ 		/*
+ 		 * It wont release any space if we cannot release the last
+ 		 * reference, besides, if somebody else has a reference,
+ 		 * it's a bad idea to nuke this object anyway.
+ 		 */
+ 		if (oc->refcnt == 1)
+ 			break;
+ 	}
+ 	if (oc != NULL) {
+ 		VTAILQ_REMOVE(&lru->lru_head, oc, lru_list);
+ 		binheap_delete(exp_heap, oc->timer_idx);
+ 		assert(oc->timer_idx == BINHEAP_NOIDX);
+ 		VSC_C_main->n_lru_nuked++;
+ 	}
+ 	Lck_Unlock(&exp_mtx);
+ 	Lck_Unlock(&lru->mtx);
+ 
+ 	if (oc == NULL)
+ 		return (-1);
+ 
+ 	/* XXX: bad idea for -spersistent */
+ 	o = oc_getobj(w, oc);
+ 	WSL(w, SLT_ExpKill, 0, "%u LRU", o->xid);
+ 	(void)HSH_Deref(w, NULL, &o);
+ 	return (1);
+ }
+ 
+ /*--------------------------------------------------------------------
+  * BinHeap helper functions for objcore.
+  */
+ 
+ static int
+ object_cmp(void *priv, void *a, void *b)
+ {
+ 	struct objcore *aa, *bb;
+ 
+ 	(void)priv;
+ 	CAST_OBJ_NOTNULL(aa, a, OBJCORE_MAGIC);
+ 	CAST_OBJ_NOTNULL(bb, b, OBJCORE_MAGIC);
+ 	return (aa->timer_when < bb->timer_when);
+ }
+ 
+ static void
+ object_update(void *priv, void *p, unsigned u)
+ {
+ 	struct objcore *oc;
+ 
+ 	(void)priv;
+ 	CAST_OBJ_NOTNULL(oc, p, OBJCORE_MAGIC);
+ 	oc->timer_idx = u;
+ }
+ 
+ /*--------------------------------------------------------------------*/
+ 
+ void
+ EXP_Init(void)
+ {
+ 
+ 	Lck_New(&exp_mtx, lck_exp);
+ 	exp_heap = binheap_new(NULL, object_cmp, object_update);
+ 	XXXAN(exp_heap);
+ 	WRK_BgThread(&exp_thread, "cache-timeout", exp_timer, NULL);
+ }
diff --cc bin/varnishd/cache/cache_fetch.c
index 0000000,a5c0323..cc46222
mode 000000,100644..100644
--- a/bin/varnishd/cache/cache_fetch.c
+++ b/bin/varnishd/cache/cache_fetch.c
@@@ -1,0 -1,645 +1,650 @@@
+ /*-
+  * Copyright (c) 2006 Verdens Gang AS
+  * Copyright (c) 2006-2011 Varnish Software AS
+  * All rights reserved.
+  *
+  * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
+  *
+  * Redistribution and use in source and binary forms, with or without
+  * modification, are permitted provided that the following conditions
+  * are met:
+  * 1. Redistributions of source code must retain the above copyright
+  *    notice, this list of conditions and the following disclaimer.
+  * 2. Redistributions in binary form must reproduce the above copyright
+  *    notice, this list of conditions and the following disclaimer in the
+  *    documentation and/or other materials provided with the distribution.
+  *
+  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+  * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+  * SUCH DAMAGE.
+  */
+ 
+ #include "config.h"
+ 
+ #include <inttypes.h>
+ #include <stdio.h>
+ #include <stdlib.h>
+ 
+ #include "cache.h"
+ 
+ #include "cache_backend.h"
+ #include "vcli_priv.h"
+ #include "vct.h"
+ #include "vtcp.h"
+ 
+ static unsigned fetchfrag;
+ 
+ /*--------------------------------------------------------------------
+  * We want to issue the first error we encounter on fetching and
+  * supress the rest.  This function does that.
+  *
+  * Other code is allowed to look at w->fetch_failed to bail out
+  *
+  * For convenience, always return -1
+  */
+ 
+ int
+ FetchError2(struct worker *w, const char *error, const char *more)
+ {
+ 
+ 	CHECK_OBJ_NOTNULL(w, WORKER_MAGIC);
+ 	if (!w->fetch_failed) {
+ 		if (more == NULL)
+ 			WSLB(w, SLT_FetchError, "%s", error);
+ 		else
+ 			WSLB(w, SLT_FetchError, "%s: %s", error, more);
+ 	}
+ 	w->fetch_failed = 1;
+ 	return (-1);
+ }
+ 
+ int
+ FetchError(struct worker *w, const char *error)
+ {
+ 	return(FetchError2(w, error, NULL));
+ }
+ 
+ /*--------------------------------------------------------------------
+  * VFP_NOP
+  *
+  * This fetch-processor does nothing but store the object.
+  * It also documents the API
+  */
+ 
+ /*--------------------------------------------------------------------
+  * VFP_BEGIN
+  *
+  * Called to set up stuff.
+  *
+  * 'estimate' is the estimate of the number of bytes we expect to receive,
+  * as seen on the socket, or zero if unknown.
+  */
+ static void __match_proto__()
+ vfp_nop_begin(struct worker *w, size_t estimate)
+ {
+ 
+ 	if (estimate > 0)
+ 		(void)FetchStorage(w, estimate);
+ }
+ 
+ /*--------------------------------------------------------------------
+  * VFP_BYTES
+  *
+  * Process (up to) 'bytes' from the socket.
+  *
+  * Return -1 on error, issue FetchError()
+  *	will not be called again, once error happens.
+  * Return 0 on EOF on socket even if bytes not reached.
+  * Return 1 when 'bytes' have been processed.
+  */
+ 
+ static int __match_proto__()
+ vfp_nop_bytes(struct worker *w, struct http_conn *htc, ssize_t bytes)
+ {
+ 	ssize_t l, wl;
+ 	struct storage *st;
+ 
+ 	AZ(w->fetch_failed);
+ 	while (bytes > 0) {
+ 		st = FetchStorage(w, 0);
+ 		if (st == NULL)
+ 			return(-1);
+ 		l = st->space - st->len;
+ 		if (l > bytes)
+ 			l = bytes;
+ 		wl = HTC_Read(w, htc, st->ptr + st->len, l);
+ 		if (wl <= 0)
+ 			return (wl);
+ 		st->len += wl;
+ 		w->fetch_obj->len += wl;
+ 		bytes -= wl;
+ 		if (w->do_stream)
+ 			RES_StreamPoll(w);
+ 	}
+ 	return (1);
+ }
+ 
+ /*--------------------------------------------------------------------
+  * VFP_END
+  *
+  * Finish & cleanup
+  *
+  * Return -1 for error
+  * Return 0 for OK
+  */
+ 
+ static int __match_proto__()
+ vfp_nop_end(struct worker *w)
+ {
+ 	struct storage *st;
+ 
+ 	st = VTAILQ_LAST(&w->fetch_obj->store, storagehead);
+ 	if (st == NULL)
+ 		return (0);
+ 
+ 	if (st->len == 0) {
+ 		VTAILQ_REMOVE(&w->fetch_obj->store, st, list);
+ 		STV_free(st);
+ 		return (0);
+ 	}
+ 	if (st->len < st->space)
+ 		STV_trim(st, st->len);
+ 	return (0);
+ }
+ 
+ static struct vfp vfp_nop = {
+ 	.begin	=	vfp_nop_begin,
+ 	.bytes	=	vfp_nop_bytes,
+ 	.end	=	vfp_nop_end,
+ };
+ 
+ /*--------------------------------------------------------------------
+  * Fetch Storage to put object into.
+  *
+  */
+ 
+ struct storage *
+ FetchStorage(struct worker *w, ssize_t sz)
+ {
+ 	ssize_t l;
+ 	struct storage *st;
+ 	struct object *obj;
+ 
+ 	obj = w->fetch_obj;
+ 	CHECK_OBJ_NOTNULL(obj, OBJECT_MAGIC);
+ 	st = VTAILQ_LAST(&obj->store, storagehead);
+ 	if (st != NULL && st->len < st->space)
+ 		return (st);
+ 
+ 	l = fetchfrag;
+ 	if (l == 0)
+ 		l = sz;
+ 	if (l == 0)
+ 		l = cache_param->fetch_chunksize;
+ 	st = STV_alloc(w, l);
+ 	if (st == NULL) {
+ 		(void)FetchError(w, "Could not get storage");
+ 		return (NULL);
+ 	}
+ 	AZ(st->len);
+ 	VTAILQ_INSERT_TAIL(&obj->store, st, list);
+ 	return (st);
+ }
+ 
+ /*--------------------------------------------------------------------
+  * Convert a string to a size_t safely
+  */
+ 
+ static ssize_t
+ fetch_number(const char *nbr, int radix)
+ {
+ 	uintmax_t cll;
+ 	ssize_t cl;
+ 	char *q;
+ 
+ 	if (*nbr == '\0')
+ 		return (-1);
+ 	cll = strtoumax(nbr, &q, radix);
+ 	if (q == NULL || *q != '\0')
+ 		return (-1);
+ 
+ 	cl = (ssize_t)cll;
+ 	if((uintmax_t)cl != cll) /* Protect against bogusly large values */
+ 		return (-1);
+ 	return (cl);
+ }
+ 
+ /*--------------------------------------------------------------------*/
+ 
+ static int
+ fetch_straight(struct worker *w, struct http_conn *htc, ssize_t cl)
+ {
+ 	int i;
+ 
+ 	assert(w->body_status == BS_LENGTH);
+ 
+ 	if (cl < 0) {
+ 		return (FetchError(w, "straight length field bogus"));
+ 	} else if (cl == 0)
+ 		return (0);
+ 
+ 	i = w->vfp->bytes(w, htc, cl);
+ 	if (i <= 0)
+ 		return (FetchError(w, "straight insufficient bytes"));
+ 	return (0);
+ }
+ 
+ /*--------------------------------------------------------------------
+  * Read a chunked HTTP object.
+  *
+  * XXX: Reading one byte at a time is pretty pessimal.
+  */
+ 
+ static int
+ fetch_chunked(struct worker *w, struct http_conn *htc)
+ {
+ 	int i;
+ 	char buf[20];		/* XXX: 20 is arbitrary */
+ 	unsigned u;
+ 	ssize_t cl;
+ 
+ 	assert(w->body_status == BS_CHUNKED);
+ 	do {
+ 		/* Skip leading whitespace */
+ 		do {
+ 			if (HTC_Read(w, htc, buf, 1) <= 0)
+ 				return (-1);
+ 		} while (vct_islws(buf[0]));
+ 
+ 		if (!vct_ishex(buf[0]))
+ 			return (FetchError(w,"chunked header non-hex"));
+ 
+ 		/* Collect hex digits, skipping leading zeros */
+ 		for (u = 1; u < sizeof buf; u++) {
+ 			do {
+ 				if (HTC_Read(w, htc, buf + u, 1) <= 0)
+ 					return (-1);
+ 			} while (u == 1 && buf[0] == '0' && buf[u] == '0');
+ 			if (!vct_ishex(buf[u]))
+ 				break;
+ 		}
+ 
+ 		if (u >= sizeof buf)
+ 			return (FetchError(w,"chunked header too long"));
+ 
+ 		/* Skip trailing white space */
+ 		while(vct_islws(buf[u]) && buf[u] != '\n')
+ 			if (HTC_Read(w, htc, buf + u, 1) <= 0)
+ 				return (-1);
+ 
+ 		if (buf[u] != '\n')
+ 			return (FetchError(w,"chunked header no NL"));
+ 
+ 		buf[u] = '\0';
+ 		cl = fetch_number(buf, 16);
+ 		if (cl < 0)
+ 			return (FetchError(w,"chunked header number syntax"));
+ 
+ 		if (cl > 0 && w->vfp->bytes(w, htc, cl) <= 0)
+ 			return (-1);
+ 
+ 		i = HTC_Read(w, htc, buf, 1);
+ 		if (i <= 0)
+ 			return (-1);
+ 		if (buf[0] == '\r' && HTC_Read(w, htc, buf, 1) <= 0)
+ 			return (-1);
+ 		if (buf[0] != '\n')
+ 			return (FetchError(w,"chunked tail no NL"));
+ 	} while (cl > 0);
+ 	return (0);
+ }
+ 
+ /*--------------------------------------------------------------------*/
+ 
+ static int
+ fetch_eof(struct worker *w, struct http_conn *htc)
+ {
+ 	int i;
+ 
+ 	assert(w->body_status == BS_EOF);
+ 	i = w->vfp->bytes(w, htc, SSIZE_MAX);
+ 	if (i < 0)
+ 		return (-1);
+ 	return (0);
+ }
+ 
+ /*--------------------------------------------------------------------
+  * Fetch any body attached to the incoming request, and either write it
+  * to the backend (if we pass) or discard it (anything else).
+  * This is mainly a separate function to isolate the stack buffer and
+  * to contain the complexity when we start handling chunked encoding.
+  */
+ 
+ int
+ FetchReqBody(struct sess *sp)
+ {
+ 	unsigned long content_length;
+ 	char buf[8192];
+ 	char *ptr, *endp;
+ 	int rdcnt;
+ 
+ 	if (http_GetHdr(sp->http, H_Content_Length, &ptr)) {
+ 
+ 		content_length = strtoul(ptr, &endp, 10);
+ 		/* XXX should check result of conversion */
+ 		while (content_length) {
+ 			if (content_length > sizeof buf)
+ 				rdcnt = sizeof buf;
+ 			else
+ 				rdcnt = content_length;
+ 			rdcnt = HTC_Read(sp->wrk, sp->htc, buf, rdcnt);
+ 			if (rdcnt <= 0)
+ 				return (1);
+ 			content_length -= rdcnt;
+ 			if (!sp->sendbody)
+ 				continue;
+ 			(void)WRW_Write(sp->wrk, buf, rdcnt); /* XXX: stats ? */
+ 			if (WRW_Flush(sp->wrk))
+ 				return (2);
+ 		}
+ 	}
+ 	if (http_GetHdr(sp->http, H_Transfer_Encoding, NULL)) {
+ 		/* XXX: Handle chunked encoding. */
+ 		WSP(sp, SLT_Debug, "Transfer-Encoding in request");
+ 		return (1);
+ 	}
+ 	return (0);
+ }
+ 
+ /*--------------------------------------------------------------------
+  * Send request, and receive the HTTP protocol response, but not the
+  * response body.
+  *
+  * Return value:
+  *	-1 failure, not retryable
+  *	 0 success
+  *	 1 failure which can be retried.
+  */
+ 
+ int
+ FetchHdr(struct sess *sp)
+ {
+ 	struct vbc *vc;
+ 	struct worker *w;
+ 	char *b;
+ 	struct http *hp;
+ 	int retry = -1;
+ 	int i;
+ 
+ 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+ 	CHECK_OBJ_NOTNULL(sp->wrk, WORKER_MAGIC);
+ 	w = sp->wrk;
+ 
+ 	AN(sp->director);
+ 	AZ(sp->obj);
+ 
+ 	if (sp->objcore != NULL) {		/* pass has no objcore */
+ 		CHECK_OBJ_NOTNULL(sp->objcore, OBJCORE_MAGIC);
+ 		AN(sp->objcore->flags & OC_F_BUSY);
+ 	}
+ 
+ 	hp = w->bereq;
+ 
+ 	sp->wrk->vbc = VDI_GetFd(NULL, sp);
+ 	if (sp->wrk->vbc == NULL) {
+ 		WSP(sp, SLT_FetchError, "no backend connection");
+ 		return (-1);
+ 	}
+ 	vc = sp->wrk->vbc;
+ 	if (vc->recycled)
+ 		retry = 1;
+ 
+ 	/*
+ 	 * Now that we know our backend, we can set a default Host:
+ 	 * header if one is necessary.  This cannot be done in the VCL
+ 	 * because the backend may be chosen by a director.
+ 	 */
+ 	if (!http_GetHdr(hp, H_Host, &b))
+ 		VDI_AddHostHeader(sp);
+ 
+ 	(void)VTCP_blocking(vc->fd);	/* XXX: we should timeout instead */
+ 	WRW_Reserve(w, &vc->fd);
+ 	(void)http_Write(w, vc->vsl_id, hp, 0);	/* XXX: stats ? */
+ 
+ 	/* Deal with any message-body the request might have */
+ 	i = FetchReqBody(sp);
+ 	if (WRW_FlushRelease(w) || i > 0) {
+ 		WSP(sp, SLT_FetchError, "backend write error: %d (%s)",
+ 		    errno, strerror(errno));
+ 		VDI_CloseFd(sp->wrk);
+ 		/* XXX: other cleanup ? */
+ 		return (retry);
+ 	}
+ 
+ 	/* Checkpoint the vsl.here */
+ 	WSL_Flush(w, 0);
+ 
+ 	/* XXX is this the right place? */
+ 	VSC_C_main->backend_req++;
+ 
+ 	/* Receive response */
+ 
+ 	HTC_Init(w->htc, w->ws, vc->fd, vc->vsl_id, cache_param->http_resp_size,
+ 	    cache_param->http_resp_hdr_len);
+ 
+ 	VTCP_set_read_timeout(vc->fd, vc->first_byte_timeout);
+ 
+ 	i = HTC_Rx(w->htc);
+ 
+ 	if (i < 0) {
+ 		WSP(sp, SLT_FetchError, "http first read error: %d %d (%s)",
+ 		    i, errno, strerror(errno));
+ 		VDI_CloseFd(sp->wrk);
+ 		/* XXX: other cleanup ? */
+ 		/* Retryable if we never received anything */
+ 		return (i == -1 ? retry : -1);
+ 	}
+ 
+ 	VTCP_set_read_timeout(vc->fd, vc->between_bytes_timeout);
+ 
+ 	while (i == 0) {
+ 		i = HTC_Rx(w->htc);
+ 		if (i < 0) {
+ 			WSP(sp, SLT_FetchError,
+ 			    "http first read error: %d %d (%s)",
+ 			    i, errno, strerror(errno));
+ 			VDI_CloseFd(sp->wrk);
+ 			/* XXX: other cleanup ? */
+ 			return (-1);
+ 		}
+ 	}
+ 
+ 	hp = w->beresp;
+ 
+ 	if (http_DissectResponse(w, w->htc, hp)) {
+ 		WSP(sp, SLT_FetchError, "http format error");
+ 		VDI_CloseFd(sp->wrk);
+ 		/* XXX: other cleanup ? */
+ 		return (-1);
+ 	}
+ 	return (0);
+ }
+ 
+ /*--------------------------------------------------------------------*/
+ 
+ int
+ FetchBody(struct worker *w, struct object *obj)
+ {
+ 	int cls;
+ 	struct storage *st;
+ 	int mklen;
+ 	ssize_t cl;
+ 
+ 	CHECK_OBJ_NOTNULL(w, WORKER_MAGIC);
+ 	AZ(w->fetch_obj);
+ 	CHECK_OBJ_NOTNULL(w->vbc, VBC_MAGIC);
+ 	CHECK_OBJ_NOTNULL(obj, OBJECT_MAGIC);
+ 	CHECK_OBJ_NOTNULL(obj->http, HTTP_MAGIC);
+ 
+ 	if (w->vfp == NULL)
+ 		w->vfp = &vfp_nop;
+ 
+ 	AssertObjCorePassOrBusy(obj->objcore);
+ 
+ 	AZ(w->vgz_rx);
 -	AZ(VTAILQ_FIRST(&obj->store));
++
++        /* If we've freshened from another object and got a "Not Modified"
++         * response, then we have already duped the other object's body.
++         */
++        if (w->beresp->status != 304)
++        	AZ(VTAILQ_FIRST(&obj->store));
+ 
+ 	w->fetch_obj = obj;
+ 	w->fetch_failed = 0;
+ 
+ 	/* XXX: pick up estimate from objdr ? */
+ 	cl = 0;
+ 	switch (w->body_status) {
+ 	case BS_NONE:
+ 		cls = 0;
+ 		mklen = 0;
+ 		break;
+ 	case BS_ZERO:
+ 		cls = 0;
+ 		mklen = 1;
+ 		break;
+ 	case BS_LENGTH:
+ 		cl = fetch_number( w->h_content_length, 10);
+ 		w->vfp->begin(w, cl > 0 ? cl : 0);
+ 		cls = fetch_straight(w, w->htc, cl);
+ 		mklen = 1;
+ 		if (w->vfp->end(w))
+ 			cls = -1;
+ 		break;
+ 	case BS_CHUNKED:
+ 		w->vfp->begin(w, cl);
+ 		cls = fetch_chunked(w, w->htc);
+ 		mklen = 1;
+ 		if (w->vfp->end(w))
+ 			cls = -1;
+ 		break;
+ 	case BS_EOF:
+ 		w->vfp->begin(w, cl);
+ 		cls = fetch_eof(w, w->htc);
+ 		mklen = 1;
+ 		if (w->vfp->end(w))
+ 			cls = -1;
+ 		break;
+ 	case BS_ERROR:
+ 		cls = 1;
+ 		mklen = 0;
+ 		break;
+ 	default:
+ 		cls = 0;
+ 		mklen = 0;
+ 		INCOMPL();
+ 	}
+ 	AZ(w->vgz_rx);
+ 
+ 	/*
+ 	 * It is OK for ->end to just leave the last storage segment
+ 	 * sitting on w->storage, we will always call vfp_nop_end()
+ 	 * to get it trimmed or thrown out if empty.
+ 	 */
+ 	AZ(vfp_nop_end(w));
+ 
+ 	w->fetch_obj = NULL;
+ 
+ 	WSLB(w, SLT_Fetch_Body, "%u(%s) cls %d mklen %u",
+ 	    w->body_status, body_status(w->body_status),
+ 	    cls, mklen);
+ 
+ 	if (w->body_status == BS_ERROR) {
+ 		VDI_CloseFd(w);
+ 		return (__LINE__);
+ 	}
+ 
+ 	if (cls < 0) {
+ 		w->stats.fetch_failed++;
+ 		/* XXX: Wouldn't this store automatically be released ? */
+ 		while (!VTAILQ_EMPTY(&obj->store)) {
+ 			st = VTAILQ_FIRST(&obj->store);
+ 			VTAILQ_REMOVE(&obj->store, st, list);
+ 			STV_free(st);
+ 		}
+ 		VDI_CloseFd(w);
+ 		obj->len = 0;
+ 		return (__LINE__);
+ 	}
+ 	AZ(w->fetch_failed);
+ 
+ 	if (cls == 0 && w->do_close)
+ 		cls = 1;
+ 
+ 	WSLB(w, SLT_Length, "%u", obj->len);
+ 
+ 	{
+ 	/* Sanity check fetch methods accounting */
+ 		ssize_t uu;
+ 
+ 		uu = 0;
+ 		VTAILQ_FOREACH(st, &obj->store, list)
+ 			uu += st->len;
+ 		if (w->do_stream)
+ 			/* Streaming might have started freeing stuff */
+ 			assert (uu <= obj->len);
+ 
+ 		else
+ 			assert(uu == obj->len);
+ 	}
+ 
+ 	if (mklen > 0) {
+ 		http_Unset(obj->http, H_Content_Length);
+ 		http_PrintfHeader(w, w->vbc->vsl_id, obj->http,
+ 		    "Content-Length: %jd", (intmax_t)obj->len);
+ 	}
+ 
+ 	if (cls)
+ 		VDI_CloseFd(w);
+ 	else
+ 		VDI_RecycleFd(w);
+ 
+ 	return (0);
+ }
+ 
+ /*--------------------------------------------------------------------
+  * Debugging aids
+  */
+ 
+ static void
+ debug_fragfetch(struct cli *cli, const char * const *av, void *priv)
+ {
+ 	(void)priv;
+ 	(void)cli;
+ 	fetchfrag = strtoul(av[2], NULL, 0);
+ }
+ 
+ static struct cli_proto debug_cmds[] = {
+ 	{ "debug.fragfetch", "debug.fragfetch",
+ 		"\tEnable fetch fragmentation\n", 1, 1, "d", debug_fragfetch },
+ 	{ NULL }
+ };
+ 
+ /*--------------------------------------------------------------------
+  *
+  */
+ 
+ void
+ Fetch_Init(void)
+ {
+ 
+ 	CLI_AddFuncs(debug_cmds);
+ }
diff --cc bin/varnishd/cache/cache_hash.c
index 0000000,db865de..5251f6d
mode 000000,100644..100644
--- a/bin/varnishd/cache/cache_hash.c
+++ b/bin/varnishd/cache/cache_hash.c
@@@ -1,0 -1,752 +1,789 @@@
+ /*-
+  * Copyright (c) 2006 Verdens Gang AS
+  * Copyright (c) 2006-2011 Varnish Software AS
+  * All rights reserved.
+  *
+  * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
+  *
+  * Redistribution and use in source and binary forms, with or without
+  * modification, are permitted provided that the following conditions
+  * are met:
+  * 1. Redistributions of source code must retain the above copyright
+  *    notice, this list of conditions and the following disclaimer.
+  * 2. Redistributions in binary form must reproduce the above copyright
+  *    notice, this list of conditions and the following disclaimer in the
+  *    documentation and/or other materials provided with the distribution.
+  *
+  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+  * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+  * SUCH DAMAGE.
+  *
+  * This is the central hash-table code, it relies on a chosen hash
+  * implementation only for the actual hashing, all the housekeeping
+  * happens here.
+  *
+  * We have two kinds of structures, objecthead and object.  An objecthead
+  * corresponds to a given (Host:, URL) tupple, and the objects hung from
+  * the objecthead may represent various variations (ie: Vary: header,
+  * different TTL etc) instances of that web-entity.
+  *
+  * Each objecthead has a mutex which locks both its own fields, the
+  * list of objects and fields in the objects.
+  *
+  * The hash implementation must supply a reference count facility on
+  * the objecthead, and return with a reference held after a lookup.
+  *
+  * Lookups in the hash implementation returns with a ref held and each
+  * object hung from the objhead holds a ref as well.
+  *
+  * Objects have refcounts which are locked by the objecthead mutex.
+  *
+  * New objects are always marked busy, and they can go from busy to
+  * not busy only once.
+  */
+ 
+ #include "config.h"
+ 
+ #include <math.h>
+ #include <stdio.h>
+ #include <stdlib.h>
+ 
+ #include "cache.h"
+ 
+ #include "hash/hash_slinger.h"
+ #include "vsha256.h"
+ 
+ static const struct hash_slinger *hash;
+ 
+ /*---------------------------------------------------------------------*/
+ /* Precreate an objhead and object for later use */
+ void
+ HSH_Prealloc(const struct sess *sp)
+ {
+ 	struct worker *w;
+ 	struct objhead *oh;
+ 	struct objcore *oc;
+ 	struct waitinglist *wl;
+ 
+ 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+ 	CHECK_OBJ_NOTNULL(sp->wrk, WORKER_MAGIC);
+ 	w = sp->wrk;
+ 
+ 	if (w->nobjcore == NULL) {
+ 		ALLOC_OBJ(oc, OBJCORE_MAGIC);
+ 		XXXAN(oc);
+ 		w->nobjcore = oc;
+ 		w->stats.n_objectcore++;
+ 		oc->flags |= OC_F_BUSY;
+ 	}
+ 	CHECK_OBJ_NOTNULL(w->nobjcore, OBJCORE_MAGIC);
+ 
+ 	if (w->nobjhead == NULL) {
+ 		ALLOC_OBJ(oh, OBJHEAD_MAGIC);
+ 		XXXAN(oh);
+ 		oh->refcnt = 1;
+ 		VTAILQ_INIT(&oh->objcs);
+ 		Lck_New(&oh->mtx, lck_objhdr);
+ 		w->nobjhead = oh;
+ 		w->stats.n_objecthead++;
+ 	}
+ 	CHECK_OBJ_NOTNULL(w->nobjhead, OBJHEAD_MAGIC);
+ 
+ 	if (w->nwaitinglist == NULL) {
+ 		ALLOC_OBJ(wl, WAITINGLIST_MAGIC);
+ 		XXXAN(wl);
+ 		VTAILQ_INIT(&wl->list);
+ 		w->nwaitinglist = wl;
+ 		w->stats.n_waitinglist++;
+ 	}
+ 	CHECK_OBJ_NOTNULL(w->nwaitinglist, WAITINGLIST_MAGIC);
+ 
+ 	if (w->nbusyobj == NULL) {
+ 		ALLOC_OBJ(w->nbusyobj, BUSYOBJ_MAGIC);
+ 		XXXAN(w->nbusyobj);
+ 	}
+ 
+ 	if (hash->prep != NULL)
+ 		hash->prep(sp);
+ }
+ 
+ void
+ HSH_Cleanup(struct worker *w)
+ {
+ 
+ 	if (w->nobjcore != NULL) {
+ 		FREE_OBJ(w->nobjcore);
+ 		w->stats.n_objectcore--;
+ 		w->nobjcore = NULL;
+ 	}
+ 	if (w->nobjhead != NULL) {
+ 		Lck_Delete(&w->nobjhead->mtx);
+ 		FREE_OBJ(w->nobjhead);
+ 		w->nobjhead = NULL;
+ 		w->stats.n_objecthead--;
+ 	}
+ 	if (w->nwaitinglist != NULL) {
+ 		FREE_OBJ(w->nwaitinglist);
+ 		w->nwaitinglist = NULL;
+ 	}
+ 	if (w->nhashpriv != NULL) {
+ 		/* XXX: If needed, add slinger method for this */
+ 		free(w->nhashpriv);
+ 		w->nhashpriv = NULL;
+ 	}
+ 	if (w->nbusyobj != NULL) {
+ 		FREE_OBJ(w->nbusyobj);
+ 		w->nbusyobj = NULL;
+ 	}
+ }
+ 
+ void
+ HSH_DeleteObjHead(struct worker *w, struct objhead *oh)
+ {
+ 
+ 	AZ(oh->refcnt);
+ 	assert(VTAILQ_EMPTY(&oh->objcs));
+ 	Lck_Delete(&oh->mtx);
+ 	w->stats.n_objecthead--;
+ 	FREE_OBJ(oh);
+ }
+ 
+ void
+ HSH_AddString(const struct sess *sp, const char *str)
+ {
+ 	int l;
+ 
+ 	if (str == NULL)
+ 		str = "";
+ 	l = strlen(str);
+ 
+ 	SHA256_Update(sp->wrk->sha256ctx, str, l);
+ 	SHA256_Update(sp->wrk->sha256ctx, "#", 1);
+ 
+ 	if (cache_param->log_hash)
+ 		WSP(sp, SLT_Hash, "%s", str);
+ }
+ 
+ /*---------------------------------------------------------------------
+  * This is a debugging hack to enable testing of boundary conditions
+  * in the hash algorithm.
+  * We trap the first 9 different digests and translate them to different
+  * digests with edge bit conditions
+  */
+ 
+ static struct hsh_magiclist {
+ 	unsigned char was[SHA256_LEN];
+ 	unsigned char now[SHA256_LEN];
+ } hsh_magiclist[] = {
+ 	{ .now = {	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 } },
+ 	{ .now = {	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01 } },
+ 	{ .now = {	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02 } },
+ 	{ .now = {	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40 } },
+ 	{ .now = {	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80 } },
+ 	{ .now = {	0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 } },
+ 	{ .now = {	0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 } },
+ 	{ .now = {	0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 } },
+ 	{ .now = {	0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 			0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 } },
+ };
+ 
+ #define HSH_NMAGIC (sizeof hsh_magiclist / sizeof hsh_magiclist[0])
+ 
+ static void
+ hsh_testmagic(void *result)
+ {
+ 	int i, j;
+ 	static int nused = 0;
+ 
+ 	for (i = 0; i < nused; i++)
+ 		if (!memcmp(hsh_magiclist[i].was, result, SHA256_LEN))
+ 			break;
+ 	if (i == nused && i < HSH_NMAGIC)
+ 		memcpy(hsh_magiclist[nused++].was, result, SHA256_LEN);
+ 	if (i == nused)
+ 		return;
+ 	assert(i < HSH_NMAGIC);
+ 	fprintf(stderr, "HASHMAGIC: <");
+ 	for (j = 0; j < SHA256_LEN; j++)
+ 		fprintf(stderr, "%02x", ((unsigned char*)result)[j]);
+ 	fprintf(stderr, "> -> <");
+ 	memcpy(result, hsh_magiclist[i].now, SHA256_LEN);
+ 	for (j = 0; j < SHA256_LEN; j++)
+ 		fprintf(stderr, "%02x", ((unsigned char*)result)[j]);
+ 	fprintf(stderr, ">\n");
+ }
+ 
+ /*---------------------------------------------------------------------
+  * Insert an object which magically appears out of nowhere or, more likely,
+  * comes off some persistent storage device.
+  * Return it with a reference held.
+  */
+ 
+ struct objcore *
+ HSH_Insert(const struct sess *sp)
+ {
+ 	struct worker *w;
+ 	struct objhead *oh;
+ 	struct objcore *oc;
+ 
+ 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+ 	CHECK_OBJ_NOTNULL(sp->wrk, WORKER_MAGIC);
+ 	AN(hash);
+ 	w = sp->wrk;
+ 
+ 	HSH_Prealloc(sp);
+ 	if (cache_param->diag_bitmap & 0x80000000)
+ 		hsh_testmagic(sp->wrk->nobjhead->digest);
+ 
+ 	AZ(sp->hash_objhead);
+ 	AN(w->nobjhead);
+ 	oh = hash->lookup(sp, w->nobjhead);
+ 	CHECK_OBJ_NOTNULL(oh, OBJHEAD_MAGIC);
+ 	if (oh == w->nobjhead)
+ 		w->nobjhead = NULL;
+ 	Lck_Lock(&oh->mtx);
+ 	assert(oh->refcnt > 0);
+ 
+ 	/* Insert (precreated) objcore in objecthead */
+ 	oc = w->nobjcore;
+ 	w->nobjcore = NULL;
+ 	oc->refcnt = 1;
+ 	CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
+ 	AZ(oc->flags & OC_F_BUSY);
+ 
+ 	VTAILQ_INSERT_HEAD(&oh->objcs, oc, list);
+ 	/* NB: do not deref objhead the new object inherits our reference */
+ 	oc->objhead = oh;
+ 	Lck_Unlock(&oh->mtx);
+ 	sp->wrk->stats.n_vampireobject++;
+ 	return (oc);
+ }
+ 
+ /*---------------------------------------------------------------------
+  */
+ 
+ struct objcore *
+ HSH_Lookup(struct sess *sp, struct objhead **poh)
+ {
+ 	struct worker *w;
+ 	struct objhead *oh;
+ 	struct objcore *oc;
+ 	struct objcore *busy_oc, *grace_oc;
+ 	struct object *o;
 -	double grace_ttl;
++	struct object *stale_o;       /* for freshness check */
++	double grace_ttl, stale_ttl;
++        char *p;
+ 
+ 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+ 	CHECK_OBJ_NOTNULL(sp->wrk, WORKER_MAGIC);
+ 	CHECK_OBJ_NOTNULL(sp->http, HTTP_MAGIC);
+ 	AN(sp->director);
+ 	AN(hash);
++        AZ(sp->stale_obj);
+ 	w = sp->wrk;
+ 
+ 	HSH_Prealloc(sp);
+ 	memcpy(sp->wrk->nobjhead->digest, sp->digest, sizeof sp->digest);
+ 	if (cache_param->diag_bitmap & 0x80000000)
+ 		hsh_testmagic(sp->wrk->nobjhead->digest);
+ 
+ 	if (sp->hash_objhead != NULL) {
+ 		/*
+ 		 * This sess came off the waiting list, and brings a
+ 		 * oh refcnt with it.
+ 		 */
+ 		CHECK_OBJ_NOTNULL(sp->hash_objhead, OBJHEAD_MAGIC);
+ 		oh = sp->hash_objhead;
+ 		sp->hash_objhead = NULL;
+ 	} else {
+ 		AN(w->nobjhead);
+ 		oh = hash->lookup(sp, w->nobjhead);
+ 		if (oh == w->nobjhead)
+ 			w->nobjhead = NULL;
+ 	}
+ 
+ 	CHECK_OBJ_NOTNULL(oh, OBJHEAD_MAGIC);
+ 	Lck_Lock(&oh->mtx);
+ 	assert(oh->refcnt > 0);
+ 	busy_oc = NULL;
+ 	grace_oc = NULL;
++	stale_o = NULL;       /* for freshness check */
+ 	grace_ttl = NAN;
++        stale_ttl = NAN;
+ 	VTAILQ_FOREACH(oc, &oh->objcs, list) {
+ 		/* Must be at least our own ref + the objcore we examine */
+ 		assert(oh->refcnt > 1);
+ 		CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
+ 		assert(oc->objhead == oh);
+ 
+ 		if (oc->flags & OC_F_BUSY) {
+ 			CHECK_OBJ_NOTNULL(oc->busyobj, BUSYOBJ_MAGIC);
+ 			if (sp->hash_ignore_busy)
+ 				continue;
+ 
+ 			if (oc->busyobj->vary != NULL &&
+ 			    !VRY_Match(sp, oc->busyobj->vary))
+ 				continue;
+ 
+ 			busy_oc = oc;
+ 			continue;
+ 		}
+ 
+ 		o = oc_getobj(sp->wrk, oc);
+ 		CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC);
+ 
 -		if (o->exp.ttl <= 0.)
++		if (o->exp.ttl <= 0. && o->exp.grace <= 0.
++		    && o->exp.keep <= 0.)
+ 			continue;
+ 		if (BAN_CheckObject(o, sp))
+ 			continue;
+ 		if (o->vary != NULL && !VRY_Match(sp, o->vary))
+ 			continue;
+ 
+ 		/* If still valid, use it */
+ 		if (EXP_Ttl(sp, o) >= sp->t_req)
+ 			break;
+ 
+ 		/*
+ 		 * Remember any matching objects inside their grace period
+ 		 * and if there are several, use the least expired one.
+ 		 */
+ 		if (EXP_Grace(sp, o) >= sp->t_req) {
+ 			if (grace_oc == NULL ||
+ 			    grace_ttl < o->exp.entered + o->exp.ttl) {
+ 				grace_oc = oc;
+ 				grace_ttl = o->exp.entered + o->exp.ttl;
+ 			}
+ 		}
++
++		/* At this point we know:
++		 * - o's TTL has elapsed
++		 * - o is not busy or banned,
++                 * - o is not a Vary match.
++                 * The object may be used for a conditional backend request if
++                 * - the keep time has not elapsed, and
++                 * - it has a Last-Modified and/or an ETag header.
++                 * If there are several, use the least expired one.
++                 */
++               if (EXP_Keep(sp, o) >= sp->t_req
++                   && (http_GetHdr(o->http, H_Last_Modified, &p)
++                       || http_GetHdr(o->http, H_ETag, &p)))
++                   if (stale_o == NULL ||
++		       stale_ttl < o->exp.entered + o->exp.ttl) {
++			   stale_o = o;
++			   stale_ttl = o->exp.entered + o->exp.ttl;
++                   }
++
+ 	}
+ 
+ 	/*
+ 	 * If we have seen a busy object or the backend is unhealthy, and
+ 	 * we have an object in grace, use it, if req.grace is also
+ 	 * satisified.
+ 	 * XXX: Interesting footnote:  The busy object might be for a
+ 	 * XXX: different "Vary:" than we sought.  We have no way of knowing
+ 	 * XXX: this until the object is unbusy'ed, so in practice we
+ 	 * XXX: serialize fetch of all Vary's if grace is possible.
+ 	 */
+ 
+ 	AZ(sp->objcore);
+ 	sp->objcore = grace_oc;		/* XXX: Hack-ish */
+ 	if (oc == NULL			/* We found no live object */
+ 	    && grace_oc != NULL		/* There is a grace candidate */
+ 	    && (busy_oc != NULL		/* Somebody else is already busy */
+ 	    || !VDI_Healthy(sp->director, sp))) {
+ 					/* Or it is impossible to fetch */
+ 		o = oc_getobj(sp->wrk, grace_oc);
+ 		CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC);
+ 		oc = grace_oc;
+ 	}
+ 	sp->objcore = NULL;
+ 
+ 	if (oc != NULL && !sp->hash_always_miss) {
+ 		o = oc_getobj(sp->wrk, oc);
+ 		CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC);
+ 		assert(oc->objhead == oh);
+ 
+ 		/* We found an object we like */
+ 		oc->refcnt++;
+ 		if (o->hits < INT_MAX)
+ 			o->hits++;
+ 		assert(oh->refcnt > 1);
+ 		Lck_Unlock(&oh->mtx);
+ 		assert(hash->deref(oh));
+ 		*poh = oh;
+ 		return (oc);
+ 	}
+ 
+ 	if (busy_oc != NULL) {
+ 		/* There are one or more busy objects, wait for them */
+ 		if (sp->esi_level == 0) {
+ 			CHECK_OBJ_NOTNULL(sp->wrk->nwaitinglist,
+ 			    WAITINGLIST_MAGIC);
+ 			if (oh->waitinglist == NULL) {
+ 				oh->waitinglist = sp->wrk->nwaitinglist;
+ 				sp->wrk->nwaitinglist = NULL;
+ 			}
+ 			VTAILQ_INSERT_TAIL(&oh->waitinglist->list, sp, list);
+ 		}
+ 		if (cache_param->diag_bitmap & 0x20)
+ 			WSP(sp, SLT_Debug,
+ 				"on waiting list <%p>", oh);
+ 		SES_Charge(sp);
+ 		/*
+ 		 * The objhead reference transfers to the sess, we get it
+ 		 * back when the sess comes off the waiting list and
+ 		 * calls us again
+ 		 */
+ 		sp->hash_objhead = oh;
+ 		sp->wrk = NULL;
+ 		Lck_Unlock(&oh->mtx);
+ 		return (NULL);
+ 	}
+ 
++        /* If we're not serving a valid or graced object and we saved stale_o,
++	 * it is a candidate for the conditional backend request. */
++        AZ(oc && !sp->hash_always_miss);
++        AZ(busy_oc);
++        if (stale_o != NULL) {
++                AZ(stale_o->objcore->flags & OC_F_BUSY);
++		CHECK_OBJ_NOTNULL(stale_o->objcore, OBJCORE_MAGIC);
++                Lck_AssertHeld(&oh->mtx);
++                stale_o->objcore->refcnt++;
++                sp->stale_obj = stale_o;
++        }
++
+ 	/* Insert (precreated) objcore in objecthead */
+ 	oc = w->nobjcore;
+ 	w->nobjcore = NULL;
+ 	AN(oc->flags & OC_F_BUSY);
+ 	oc->refcnt = 1;
+ 
+ 	/* XXX: clear w->nbusyobj before use */
+ 	VRY_Validate(sp->vary_b);
+ 	if (sp->vary_l != NULL)
+ 		w->nbusyobj->vary = sp->vary_b;
+ 	else
+ 		w->nbusyobj->vary = NULL;
+ 	oc->busyobj = w->nbusyobj;
+ 	w->nbusyobj = NULL;
+ 
+ 	/*
+ 	 * Busy objects go on the tail, so they will not trip up searches.
+ 	 * HSH_Unbusy() will move them to the front.
+ 	 */
+ 	VTAILQ_INSERT_TAIL(&oh->objcs, oc, list);
+ 	oc->objhead = oh;
+ 	/* NB: do not deref objhead the new object inherits our reference */
+ 	Lck_Unlock(&oh->mtx);
+ 	*poh = oh;
+ 	return (oc);
+ }
+ 
+ /*---------------------------------------------------------------------
+  */
+ 
+ static void
+ hsh_rush(struct objhead *oh)
+ {
+ 	unsigned u;
+ 	struct sess *sp;
+ 	struct waitinglist *wl;
+ 
+ 	CHECK_OBJ_NOTNULL(oh, OBJHEAD_MAGIC);
+ 	Lck_AssertHeld(&oh->mtx);
+ 	wl = oh->waitinglist;
+ 	CHECK_OBJ_NOTNULL(wl, WAITINGLIST_MAGIC);
+ 	for (u = 0; u < cache_param->rush_exponent; u++) {
+ 		sp = VTAILQ_FIRST(&wl->list);
+ 		if (sp == NULL)
+ 			break;
+ 		CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+ 		AZ(sp->wrk);
+ 		VTAILQ_REMOVE(&wl->list, sp, list);
+ 		DSL(0x20, SLT_Debug, sp->vsl_id, "off waiting list");
+ 		if (SES_Schedule(sp)) {
+ 			/*
+ 			 * We could not schedule the session, leave the
+ 			 * rest on the busy list.
+ 			 */
+ 			break;
+ 		}
+ 	}
+ 	if (VTAILQ_EMPTY(&wl->list)) {
+ 		oh->waitinglist = NULL;
+ 		FREE_OBJ(wl);
+ 	}
+ }
+ 
+ /*---------------------------------------------------------------------
+  * Purge an entire objhead
+  */
+ 
+ void
+ HSH_Purge(const struct sess *sp, struct objhead *oh, double ttl, double grace)
+ {
+ 	struct objcore *oc, **ocp;
+ 	unsigned spc, nobj, n;
+ 	struct object *o;
+ 
+ 	CHECK_OBJ_NOTNULL(oh, OBJHEAD_MAGIC);
+ 	spc = WS_Reserve(sp->wrk->ws, 0);
+ 	ocp = (void*)sp->wrk->ws->f;
+ 	Lck_Lock(&oh->mtx);
+ 	assert(oh->refcnt > 0);
+ 	nobj = 0;
+ 	VTAILQ_FOREACH(oc, &oh->objcs, list) {
+ 		CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
+ 		assert(oc->objhead == oh);
+ 		if (oc->flags & OC_F_BUSY) {
+ 			/*
+ 			 * We cannot purge busy objects here, because their
+ 			 * owners have special rights to them, and may nuke
+ 			 * them without concern for the refcount, which by
+ 			 * definition always must be one, so they don't check.
+ 			 */
+ 			continue;
+ 		}
+ 
+ 		(void)oc_getobj(sp->wrk, oc); /* XXX: still needed ? */
+ 
+ 		xxxassert(spc >= sizeof *ocp);
+ 		oc->refcnt++;
+ 		spc -= sizeof *ocp;
+ 		ocp[nobj++] = oc;
+ 	}
+ 	Lck_Unlock(&oh->mtx);
+ 
+ 	/* NB: inverse test to catch NAN also */
+ 	if (!(ttl > 0.))
+ 		ttl = -1.;
+ 	if (!(grace > 0.))
+ 		grace = -1.;
+ 	for (n = 0; n < nobj; n++) {
+ 		oc = ocp[n];
+ 		CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
+ 		o = oc_getobj(sp->wrk, oc);
+ 		if (o == NULL)
+ 			continue;
+ 		CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC);
+ 		o->exp.ttl = ttl;
+ 		o->exp.grace = grace;
+ 		EXP_Rearm(o);
+ 		(void)HSH_Deref(sp->wrk, NULL, &o);
+ 	}
+ 	WS_Release(sp->wrk->ws, 0);
+ }
+ 
+ 
+ /*---------------------------------------------------------------------
+  * Kill a busy object we don't need anyway.
+  * There may be sessions on the waiting list, so we cannot just blow
+  * it out of the water.
+  */
+ 
+ void
+ HSH_Drop(struct sess *sp)
+ {
+ 	struct object *o;
+ 
+ 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+ 	o = sp->obj;
+ 	CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC);
+ 	AssertObjCorePassOrBusy(o->objcore);
+ 	o->exp.ttl = -1.;
+ 	if (o->objcore != NULL)		/* Pass has no objcore */
+ 		HSH_Unbusy(sp);
+ 	(void)HSH_Deref(sp->wrk, NULL, &sp->obj);
+ }
+ 
+ void
+ HSH_Unbusy(const struct sess *sp)
+ {
+ 	struct object *o;
+ 	struct objhead *oh;
+ 	struct objcore *oc;
+ 
+ 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+ 	o = sp->obj;
+ 	CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC);
+ 	oc = o->objcore;
+ 	CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
+ 	oh = oc->objhead;
+ 	CHECK_OBJ(oh, OBJHEAD_MAGIC);
+ 
+ 	AssertObjBusy(o);
+ 	AN(oc->ban);
+ 	assert(oc->refcnt > 0);
+ 	assert(oh->refcnt > 0);
+ 	if (o->ws_o->overflow)
+ 		sp->wrk->stats.n_objoverflow++;
+ 	if (cache_param->diag_bitmap & 0x40)
+ 		WSP(sp, SLT_Debug,
+ 		    "Object %u workspace free %u", o->xid, WS_Free(o->ws_o));
+ 
+ 	/* XXX: pretouch neighbors on oh->objcs to prevent page-on under mtx */
+ 	Lck_Lock(&oh->mtx);
+ 	assert(oh->refcnt > 0);
+ 	/* XXX: strictly speaking, we should sort in Date: order. */
+ 	VTAILQ_REMOVE(&oh->objcs, oc, list);
+ 	VTAILQ_INSERT_HEAD(&oh->objcs, oc, list);
+ 	oc->flags &= ~OC_F_BUSY;
+ 	AZ(sp->wrk->nbusyobj);
+ 	sp->wrk->nbusyobj = oc->busyobj;
+ 	oc->busyobj = NULL;
+ 	if (oh->waitinglist != NULL)
+ 		hsh_rush(oh);
+ 	AN(oc->ban);
+ 	Lck_Unlock(&oh->mtx);
+ 	assert(oc_getobj(sp->wrk, oc) == o);
+ }
+ 
+ void
+ HSH_Ref(struct objcore *oc)
+ {
+ 	struct objhead *oh;
+ 
+ 	CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
+ 	oh = oc->objhead;
+ 	CHECK_OBJ_NOTNULL(oh, OBJHEAD_MAGIC);
+ 	Lck_Lock(&oh->mtx);
+ 	assert(oc->refcnt > 0);
+ 	oc->refcnt++;
+ 	Lck_Unlock(&oh->mtx);
+ }
+ 
+ /*--------------------------------------------------------------------
+  * Dereference objcore and or object
+  *
+  * Can deal with:
+  *	bare objcore (incomplete fetch)
+  *	bare object (pass)
+  *	object with objcore
+  *	XXX later:  objcore with object (?)
+  *
+  * But you can only supply one of the two arguments at a time.
+  *
+  * Returns zero if target was destroyed.
+  */
+ 
+ int
+ HSH_Deref(struct worker *w, struct objcore *oc, struct object **oo)
+ {
+ 	struct object *o = NULL;
+ 	struct objhead *oh;
+ 	unsigned r;
+ 
+ 	/* Only one arg at a time */
+ 	assert(oc == NULL || oo == NULL);
+ 
+ 	if (oo != NULL) {
+ 		o = *oo;
+ 		*oo = NULL;
+ 		CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC);
+ 		oc = o->objcore;
+ 	}
+ 
+ 	if (o != NULL && oc == NULL) {
+ 		/*
+ 		 * A pass object with neither objcore nor objhdr reference.
+ 		 * -> simply free the (Transient) storage
+ 		 */
+ 		STV_Freestore(o);
+ 		STV_free(o->objstore);
+ 		w->stats.n_object--;
+ 		return (0);
+ 	}
+ 
+ 	CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
+ 
+ 	oh = oc->objhead;
+ 	CHECK_OBJ_NOTNULL(oh, OBJHEAD_MAGIC);
+ 
+ 	Lck_Lock(&oh->mtx);
+ 	assert(oh->refcnt > 0);
+ 	assert(oc->refcnt > 0);
+ 	r = --oc->refcnt;
+ 	if (!r)
+ 		VTAILQ_REMOVE(&oh->objcs, oc, list);
+ 	else {
+ 		/* Must have an object */
+ 		AN(oc->methods);
+ 	}
+ 	if (oh->waitinglist != NULL)
+ 		hsh_rush(oh);
+ 	Lck_Unlock(&oh->mtx);
+ 	if (r != 0)
+ 		return (r);
+ 
+ 	BAN_DestroyObj(oc);
+ 	AZ(oc->ban);
+ 
+ 	if (oc->flags & OC_F_BUSY) {
+ 		CHECK_OBJ_NOTNULL(oc->busyobj, BUSYOBJ_MAGIC);
+ 		if (w->nbusyobj == NULL)
+ 			w->nbusyobj = oc->busyobj;
+ 		else
+ 			FREE_OBJ(oc->busyobj);
+ 		oc->busyobj = NULL;
+ 	}
+ 	AZ(oc->busyobj);
+ 
+ 	if (oc->methods != NULL) {
+ 		oc_freeobj(oc);
+ 		w->stats.n_object--;
+ 	}
+ 	FREE_OBJ(oc);
+ 
+ 	w->stats.n_objectcore--;
+ 	/* Drop our ref on the objhead */
+ 	assert(oh->refcnt > 0);
+ 	if (hash->deref(oh))
+ 		return (0);
+ 	HSH_DeleteObjHead(w, oh);
+ 	return (0);
+ }
+ 
+ void
+ HSH_Init(const struct hash_slinger *slinger)
+ {
+ 
+ 	assert(DIGEST_LEN == SHA256_LEN);	/* avoid #include pollution */
+ 	hash = slinger;
+ 	if (hash->start != NULL)
+ 		hash->start();
+ }
diff --cc bin/varnishd/cache/cache_http.c
index 0000000,784eb28..b937d64
mode 000000,100644..100644
--- a/bin/varnishd/cache/cache_http.c
+++ b/bin/varnishd/cache/cache_http.c
@@@ -1,0 -1,1119 +1,1236 @@@
+ /*-
+  * Copyright (c) 2006 Verdens Gang AS
+  * Copyright (c) 2006-2011 Varnish Software AS
+  * All rights reserved.
+  *
+  * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
+  *
+  * Redistribution and use in source and binary forms, with or without
+  * modification, are permitted provided that the following conditions
+  * are met:
+  * 1. Redistributions of source code must retain the above copyright
+  *    notice, this list of conditions and the following disclaimer.
+  * 2. Redistributions in binary form must reproduce the above copyright
+  *    notice, this list of conditions and the following disclaimer in the
+  *    documentation and/or other materials provided with the distribution.
+  *
+  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+  * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+  * SUCH DAMAGE.
+  *
+  * HTTP request storage and manipulation
+  */
+ 
+ #include "config.h"
+ 
+ #include <stdio.h>
+ 
+ #include "cache.h"
++#include "storage/storage.h"
+ 
+ #include "vct.h"
+ 
+ #define HTTPH(a, b, c, d, e, f, g) char b[] = "*" a ":";
+ #include "tbl/http_headers.h"
+ #undef HTTPH
+ 
+ /*lint -save -e773 not () */
+ #define LOGMTX2(ax, bx, cx)	[bx] = SLT_##ax##cx
+ 
+ #define LOGMTX1(ax) {					\
+ 	LOGMTX2(ax, HTTP_HDR_REQ,	Request),	\
+ 	LOGMTX2(ax, HTTP_HDR_RESPONSE,	Response),	\
+ 	LOGMTX2(ax, HTTP_HDR_STATUS,	Status),	\
+ 	LOGMTX2(ax, HTTP_HDR_URL,	URL),		\
+ 	LOGMTX2(ax, HTTP_HDR_PROTO,	Protocol),	\
+ 	LOGMTX2(ax, HTTP_HDR_FIRST,	Header),	\
+ 	}
+ 
+ static const enum VSL_tag_e logmtx[][HTTP_HDR_FIRST + 1] = {
+ 	[HTTP_Rx] = LOGMTX1(Rx),
+ 	[HTTP_Tx] = LOGMTX1(Tx),
+ 	[HTTP_Obj] = LOGMTX1(Obj)
+ };
+ /*lint -restore */
+ 
++void http_FilterMissingFields(struct worker *w, int fd, struct http *to,
++    const struct http *fm);
++
+ static enum VSL_tag_e
+ http2shmlog(const struct http *hp, int t)
+ {
+ 
+ 	CHECK_OBJ_NOTNULL(hp, HTTP_MAGIC);
+ 	if (t > HTTP_HDR_FIRST)
+ 		t = HTTP_HDR_FIRST;
+ 	assert(hp->logtag >= HTTP_Rx && hp->logtag <= HTTP_Obj); /*lint !e685*/
+ 	assert(t >= HTTP_HDR_REQ && t <= HTTP_HDR_FIRST);
+ 	return (logmtx[hp->logtag][t]);
+ }
+ 
+ static void
+ WSLH(struct worker *w, unsigned vsl_id, const struct http *hp, unsigned hdr)
+ {
+ 
+ 	AN(vsl_id & (VSL_CLIENTMARKER|VSL_BACKENDMARKER));
+ 	WSLR(w, http2shmlog(hp, hdr), vsl_id, hp->hd[hdr]);
+ }
+ 
+ /*--------------------------------------------------------------------*/
+ /* List of canonical HTTP response code names from RFC2616 */
+ 
+ static struct http_msg {
+ 	unsigned	nbr;
+ 	const char	*txt;
+ } http_msg[] = {
+ #define HTTP_RESP(n, t)	{ n, t},
+ #include "tbl/http_response.h"
+ 	{ 0, NULL }
+ };
+ 
+ const char *
+ http_StatusMessage(unsigned status)
+ {
+ 	struct http_msg *mp;
+ 
+ 	assert(status >= 100 && status <= 999);
+ 	for (mp = http_msg; mp->nbr != 0 && mp->nbr <= status; mp++)
+ 		if (mp->nbr == status)
+ 			return (mp->txt);
+ 	return ("Unknown Error");
+ }
+ 
+ /*--------------------------------------------------------------------*/
+ 
+ unsigned
+ HTTP_estimate(unsigned nhttp)
+ {
+ 
+ 	/* XXX: We trust the structs to size-aligned as necessary */
+ 	return (sizeof (struct http) + (sizeof (txt) + 1) * nhttp);
+ }
+ 
+ struct http *
+ HTTP_create(void *p, uint16_t nhttp)
+ {
+ 	struct http *hp;
+ 
+ 	hp = p;
+ 	hp->magic = HTTP_MAGIC;
+ 	hp->hd = (void*)(hp + 1);
+ 	hp->shd = nhttp;
+ 	hp->hdf = (void*)(hp->hd + nhttp);
+ 	return (hp);
+ }
+ 
+ /*--------------------------------------------------------------------*/
+ 
+ void
+ http_Setup(struct http *hp, struct ws *ws)
+ {
+ 	uint16_t shd;
+ 	txt *hd;
+ 	unsigned char *hdf;
+ 
+ 	/* XXX: This is not elegant, is it efficient ? */
+ 	shd = hp->shd;
+ 	hd = hp->hd;
+ 	hdf = hp->hdf;
+ 	memset(hp, 0, sizeof *hp);
+ 	memset(hd, 0, sizeof *hd * shd);
+ 	memset(hdf, 0, sizeof *hdf * shd);
+ 	hp->magic = HTTP_MAGIC;
+ 	hp->ws = ws;
+ 	hp->nhd = HTTP_HDR_FIRST;
+ 	hp->shd = shd;
+ 	hp->hd = hd;
+ 	hp->hdf = hdf;
+ }
+ 
+ /*--------------------------------------------------------------------*/
+ 
+ static int
+ http_IsHdr(const txt *hh, const char *hdr)
+ {
+ 	unsigned l;
+ 
+ 	Tcheck(*hh);
+ 	AN(hdr);
+ 	l = hdr[0];
+ 	assert(l == strlen(hdr + 1));
+ 	assert(hdr[l] == ':');
+ 	hdr++;
+ 	return (!strncasecmp(hdr, hh->b, l));
+ }
+ 
+ /*--------------------------------------------------------------------
+  * This function collapses multiple headerlines of the same name.
+  * The lines are joined with a comma, according to [rfc2616, 4.2bot, p32]
+  */
+ 
+ void
+ http_CollectHdr(struct http *hp, const char *hdr)
+ {
+ 	unsigned u, v, ml, f = 0, x;
+ 	char *b = NULL, *e = NULL;
+ 
+ 	for (u = HTTP_HDR_FIRST; u < hp->nhd; u++) {
+ 		while (u < hp->nhd && http_IsHdr(&hp->hd[u], hdr)) {
+ 			Tcheck(hp->hd[u]);
+ 			if (f == 0) {
+ 				/* Found first header, just record the fact */
+ 				f = u;
+ 				break;
+ 			}
+ 			if (b == NULL) {
+ 				/* Found second header, start our collection */
+ 				ml = WS_Reserve(hp->ws, 0);
+ 				b = hp->ws->f;
+ 				e = b + ml;
+ 				x = Tlen(hp->hd[f]);
+ 				if (b + x < e) {
+ 					memcpy(b, hp->hd[f].b, x);
+ 					b += x;
+ 				} else
+ 					b = e;
+ 			}
+ 
+ 			AN(b);
+ 			AN(e);
+ 
+ 			/* Append the Nth header we found */
+ 			if (b < e)
+ 				*b++ = ',';
+ 			x = Tlen(hp->hd[u]) - *hdr;
+ 			if (b + x < e) {
+ 				memcpy(b, hp->hd[u].b + *hdr, x);
+ 				b += x;
+ 			} else
+ 				b = e;
+ 
+ 			/* Shift remaining headers up one slot */
+ 			for (v = u; v < hp->nhd - 1; v++)
+ 				hp->hd[v] = hp->hd[v + 1];
+ 			hp->nhd--;
+ 		}
+ 
+ 	}
+ 	if (b == NULL)
+ 		return;
+ 	AN(e);
+ 	if (b >= e) {
+ 		WS_Release(hp->ws, 0);
+ 		return;
+ 	}
+ 	*b = '\0';
+ 	hp->hd[f].b = hp->ws->f;
+ 	hp->hd[f].e = b;
+ 	WS_ReleaseP(hp->ws, b + 1);
+ }
+ 
+ 
+ /*--------------------------------------------------------------------*/
+ 
+ static unsigned
+ http_findhdr(const struct http *hp, unsigned l, const char *hdr)
+ {
+ 	unsigned u;
+ 
+ 	for (u = HTTP_HDR_FIRST; u < hp->nhd; u++) {
+ 		Tcheck(hp->hd[u]);
+ 		if (hp->hd[u].e < hp->hd[u].b + l + 1)
+ 			continue;
+ 		if (hp->hd[u].b[l] != ':')
+ 			continue;
+ 		if (strncasecmp(hdr, hp->hd[u].b, l))
+ 			continue;
+ 		return (u);
+ 	}
+ 	return (0);
+ }
+ 
+ int
+ http_GetHdr(const struct http *hp, const char *hdr, char **ptr)
+ {
+ 	unsigned u, l;
+ 	char *p;
+ 
+ 	l = hdr[0];
+ 	diagnostic(l == strlen(hdr + 1));
+ 	assert(hdr[l] == ':');
+ 	hdr++;
+ 	u = http_findhdr(hp, l - 1, hdr);
+ 	if (u == 0) {
+ 		if (ptr != NULL)
+ 			*ptr = NULL;
+ 		return (0);
+ 	}
+ 	if (ptr != NULL) {
+ 		p = hp->hd[u].b + l;
+ 		while (vct_issp(*p))
+ 			p++;
+ 		*ptr = p;
+ 	}
+ 	return (1);
+ }
+ 
+ 
+ /*--------------------------------------------------------------------
+  * Find a given data element in a header according to RFC2616's #rule
+  * (section 2.1, p15)
+  */
+ 
+ int
+ http_GetHdrData(const struct http *hp, const char *hdr,
+     const char *field, char **ptr)
+ {
+ 	char *h, *e;
+ 	unsigned fl;
+ 
+ 	if (ptr != NULL)
+ 		*ptr = NULL;
+ 	if (!http_GetHdr(hp, hdr, &h))
+ 		return (0);
+ 	AN(h);
+ 	e = strchr(h, '\0');
+ 	fl = strlen(field);
+ 	while (h + fl <= e) {
+ 		/* Skip leading whitespace and commas */
+ 		if (vct_islws(*h) || *h == ',') {
+ 			h++;
+ 			continue;
+ 		}
+ 		/* Check for substrings before memcmp() */
+ 		if ((h + fl == e || vct_issepctl(h[fl])) &&
+ 		    !memcmp(h, field, fl)) {
+ 			if (ptr != NULL) {
+ 				h += fl;
+ 				while (vct_islws(*h))
+ 					h++;
+ 				*ptr = h;
+ 			}
+ 			return (1);
+ 		}
+ 		/* Skip until end of header or comma */
+ 		while (*h && *h != ',')
+ 			h++;
+ 	}
+ 	return (0);
+ }
+ 
+ /*--------------------------------------------------------------------
+  * Find a given headerfields Q value.
+  */
+ 
+ double
+ http_GetHdrQ(const struct http *hp, const char *hdr, const char *field)
+ {
+ 	char *h;
+ 	int i;
+ 	double a, b;
+ 
+ 	h = NULL;
+ 	i = http_GetHdrData(hp, hdr, field, &h);
+ 	if (!i)
+ 		return (0.);
+ 
+ 	if (h == NULL)
+ 		return (1.);
+ 	/* Skip whitespace, looking for '=' */
+ 	while (*h && vct_issp(*h))
+ 		h++;
+ 	if (*h++ != ';')
+ 		return (1.);
+ 	while (*h && vct_issp(*h))
+ 		h++;
+ 	if (*h++ != 'q')
+ 		return (1.);
+ 	while (*h && vct_issp(*h))
+ 		h++;
+ 	if (*h++ != '=')
+ 		return (1.);
+ 	while (*h && vct_issp(*h))
+ 		h++;
+ 	a = 0.;
+ 	while (vct_isdigit(*h)) {
+ 		a *= 10.;
+ 		a += *h - '0';
+ 		h++;
+ 	}
+ 	if (*h++ != '.')
+ 		return (a);
+ 	b = .1;
+ 	while (vct_isdigit(*h)) {
+ 		a += b * (*h - '0');
+ 		b *= .1;
+ 		h++;
+ 	}
+ 	return (a);
+ }
+ 
+ /*--------------------------------------------------------------------
+  * Find a given headerfields value.
+  */
+ 
+ int
+ http_GetHdrField(const struct http *hp, const char *hdr,
+     const char *field, char **ptr)
+ {
+ 	char *h;
+ 	int i;
+ 
+ 	if (ptr != NULL)
+ 		*ptr = NULL;
+ 
+ 	h = NULL;
+ 	i = http_GetHdrData(hp, hdr, field, &h);
+ 	if (!i)
+ 		return (i);
+ 
+ 	if (ptr != NULL && h != NULL) {
+ 		/* Skip whitespace, looking for '=' */
+ 		while (*h && vct_issp(*h))
+ 			h++;
+ 		if (*h == '=') {
+ 			h++;
+ 			while (*h && vct_issp(*h))
+ 				h++;
+ 			*ptr = h;
+ 		}
+ 	}
+ 	return (i);
+ }
+ 
+ /*--------------------------------------------------------------------
+  * XXX: redo with http_GetHdrField() ?
+  */
+ 
+ const char *
+ http_DoConnection(const struct http *hp)
+ {
+ 	char *p, *q;
+ 	const char *ret;
+ 	unsigned u;
+ 
+ 	if (!http_GetHdr(hp, H_Connection, &p)) {
+ 		if (hp->protover < 11)
+ 			return ("not HTTP/1.1");
+ 		return (NULL);
+ 	}
+ 	ret = NULL;
+ 	AN(p);
+ 	for (; *p; p++) {
+ 		if (vct_issp(*p))
+ 			continue;
+ 		if (*p == ',')
+ 			continue;
+ 		for (q = p + 1; *q; q++)
+ 			if (*q == ',' || vct_issp(*q))
+ 				break;
+ 		u = pdiff(p, q);
+ 		if (u == 5 && !strncasecmp(p, "close", u))
+ 			ret = "Connection: close";
+ 		u = http_findhdr(hp, u, p);
+ 		if (u != 0)
+ 			hp->hdf[u] |= HDF_FILTER;
+ 		if (!*q)
+ 			break;
+ 		p = q;
+ 	}
+ 	return (ret);
+ }
+ 
+ /*--------------------------------------------------------------------*/
+ 
+ int
+ http_HdrIs(const struct http *hp, const char *hdr, const char *val)
+ {
+ 	char *p;
+ 
+ 	if (!http_GetHdr(hp, hdr, &p))
+ 		return (0);
+ 	AN(p);
+ 	if (!strcasecmp(p, val))
+ 		return (1);
+ 	return (0);
+ }
+ 
+ /*--------------------------------------------------------------------*/
+ 
+ uint16_t
+ http_GetStatus(const struct http *hp)
+ {
+ 
+ 	return (hp->status);
+ }
+ 
+ const char *
+ http_GetReq(const struct http *hp)
+ {
+ 
+ 	Tcheck(hp->hd[HTTP_HDR_REQ]);
+ 	return (hp->hd[HTTP_HDR_REQ].b);
+ }
+ 
+ /*--------------------------------------------------------------------
+  * Dissect the headers of the HTTP protocol message.
+  * Detect conditionals (headers which start with '^[Ii][Ff]-')
+  */
+ 
+ static uint16_t
+ http_dissect_hdrs(struct worker *w, struct http *hp, unsigned vsl_id, char *p,
+     const struct http_conn *htc)
+ {
+ 	char *q, *r;
+ 	txt t = htc->rxbuf;
+ 
+ 	if (*p == '\r')
+ 		p++;
+ 
+ 	hp->nhd = HTTP_HDR_FIRST;
+ 	hp->conds = 0;
+ 	r = NULL;		/* For FlexeLint */
+ 	for (; p < t.e; p = r) {
+ 
+ 		/* Find end of next header */
+ 		q = r = p;
+ 		while (r < t.e) {
+ 			if (!vct_iscrlf(*r)) {
+ 				r++;
+ 				continue;
+ 			}
+ 			q = r;
+ 			assert(r < t.e);
+ 			r += vct_skipcrlf(r);
+ 			if (r >= t.e)
+ 				break;
+ 			/* If line does not continue: got it. */
+ 			if (!vct_issp(*r))
+ 				break;
+ 
+ 			/* Clear line continuation LWS to spaces */
+ 			while (vct_islws(*q))
+ 				*q++ = ' ';
+ 		}
+ 
+ 		if (q - p > htc->maxhdr) {
+ 			VSC_C_main->losthdr++;
+ 			WSL(w, SLT_LostHeader, vsl_id, "%.*s",
+ 			    q - p > 20 ? 20 : q - p, p);
+ 			return (413);
+ 		}
+ 
+ 		/* Empty header = end of headers */
+ 		if (p == q)
+ 			break;
+ 
+ 		if ((p[0] == 'i' || p[0] == 'I') &&
+ 		    (p[1] == 'f' || p[1] == 'F') &&
+ 		    p[2] == '-')
+ 			hp->conds = 1;
+ 
+ 		while (q > p && vct_issp(q[-1]))
+ 			q--;
+ 		*q = '\0';
+ 
+ 		if (hp->nhd < hp->shd) {
+ 			hp->hdf[hp->nhd] = 0;
+ 			hp->hd[hp->nhd].b = p;
+ 			hp->hd[hp->nhd].e = q;
+ 			WSLH(w, vsl_id, hp, hp->nhd);
+ 			hp->nhd++;
+ 		} else {
+ 			VSC_C_main->losthdr++;
+ 			WSL(w, SLT_LostHeader, vsl_id, "%.*s",
+ 			    q - p > 20 ? 20 : q - p, p);
+ 			return (413);
+ 		}
+ 	}
+ 	return (0);
+ }
+ 
+ /*--------------------------------------------------------------------
+  * Deal with first line of HTTP protocol message.
+  */
+ 
+ static uint16_t
+ http_splitline(struct worker *w, unsigned vsl_id, struct http *hp,
+     const struct http_conn *htc, int h1, int h2, int h3)
+ {
+ 	char *p, *q;
+ 
+ 	CHECK_OBJ_NOTNULL(htc, HTTP_CONN_MAGIC);
+ 	CHECK_OBJ_NOTNULL(hp, HTTP_MAGIC);
+ 
+ 	/* XXX: Assert a NUL at rx.e ? */
+ 	Tcheck(htc->rxbuf);
+ 
+ 	/* Skip leading LWS */
+ 	for (p = htc->rxbuf.b ; vct_islws(*p); p++)
+ 		continue;
+ 
+ 	/* First field cannot contain SP, CRLF or CTL */
+ 	q = p;
+ 	for (; !vct_issp(*p); p++) {
+ 		if (vct_isctl(*p))
+ 			return (400);
+ 	}
+ 	hp->hd[h1].b = q;
+ 	hp->hd[h1].e = p;
+ 
+ 	/* Skip SP */
+ 	for (; vct_issp(*p); p++) {
+ 		if (vct_isctl(*p))
+ 			return (400);
+ 	}
+ 
+ 	/* Second field cannot contain LWS or CTL */
+ 	q = p;
+ 	for (; !vct_islws(*p); p++) {
+ 		if (vct_isctl(*p))
+ 			return (400);
+ 	}
+ 	hp->hd[h2].b = q;
+ 	hp->hd[h2].e = p;
+ 
+ 	if (!Tlen(hp->hd[h2]))
+ 		return (413);
+ 
+ 	/* Skip SP */
+ 	for (; vct_issp(*p); p++) {
+ 		if (vct_isctl(*p))
+ 			return (400);
+ 	}
+ 
+ 	/* Third field is optional and cannot contain CTL */
+ 	q = p;
+ 	if (!vct_iscrlf(*p)) {
+ 		for (; !vct_iscrlf(*p); p++)
+ 			if (!vct_issep(*p) && vct_isctl(*p))
+ 				return (400);
+ 	}
+ 	hp->hd[h3].b = q;
+ 	hp->hd[h3].e = p;
+ 
+ 	/* Skip CRLF */
+ 	p += vct_skipcrlf(p);
+ 
+ 	*hp->hd[h1].e = '\0';
+ 	WSLH(w, vsl_id, hp, h1);
+ 
+ 	*hp->hd[h2].e = '\0';
+ 	WSLH(w, vsl_id, hp, h2);
+ 
+ 	if (hp->hd[h3].e != NULL) {
+ 		*hp->hd[h3].e = '\0';
+ 		WSLH(w, vsl_id, hp, h3);
+ 	}
+ 
+ 	return (http_dissect_hdrs(w, hp, vsl_id, p, htc));
+ }
+ 
+ /*--------------------------------------------------------------------*/
+ 
+ static void
+ http_ProtoVer(struct http *hp)
+ {
+ 
+ 	if (!strcasecmp(hp->hd[HTTP_HDR_PROTO].b, "HTTP/1.0"))
+ 		hp->protover = 10;
+ 	else if (!strcasecmp(hp->hd[HTTP_HDR_PROTO].b, "HTTP/1.1"))
+ 		hp->protover = 11;
+ 	else
+ 		hp->protover = 9;
+ }
+ 
+ 
+ /*--------------------------------------------------------------------*/
+ 
+ uint16_t
+ http_DissectRequest(struct sess *sp)
+ {
+ 	struct http_conn *htc;
+ 	struct http *hp;
+ 	uint16_t retval;
+ 
+ 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+ 	htc = sp->htc;
+ 	CHECK_OBJ_NOTNULL(htc, HTTP_CONN_MAGIC);
+ 	hp = sp->http;
+ 	CHECK_OBJ_NOTNULL(hp, HTTP_MAGIC);
+ 
+ 	hp->logtag = HTTP_Rx;
+ 
+ 	retval = http_splitline(sp->wrk, sp->vsl_id, hp, htc,
+ 	    HTTP_HDR_REQ, HTTP_HDR_URL, HTTP_HDR_PROTO);
+ 	if (retval != 0) {
+ 		WSPR(sp, SLT_HttpGarbage, htc->rxbuf);
+ 		return (retval);
+ 	}
+ 	http_ProtoVer(hp);
+ 	return (retval);
+ }
+ 
+ /*--------------------------------------------------------------------*/
+ 
+ uint16_t
+ http_DissectResponse(struct worker *w, const struct http_conn *htc,
+     struct http *hp)
+ {
+ 	int j;
+ 	uint16_t retval = 0;
+ 	char *p;
+ 
+ 
+ 	CHECK_OBJ_NOTNULL(htc, HTTP_CONN_MAGIC);
+ 	CHECK_OBJ_NOTNULL(hp, HTTP_MAGIC);
+ 	hp->logtag = HTTP_Rx;
+ 
+ 	if (http_splitline(w, htc->vsl_id, hp, htc,
+ 	    HTTP_HDR_PROTO, HTTP_HDR_STATUS, HTTP_HDR_RESPONSE))
+ 		retval = 503;
+ 
+ 	if (retval == 0 && memcmp(hp->hd[HTTP_HDR_PROTO].b, "HTTP/1.", 7))
+ 		retval = 503;
+ 
+ 	if (retval == 0 && Tlen(hp->hd[HTTP_HDR_STATUS]) != 3)
+ 		retval = 503;
+ 
+ 	if (retval == 0) {
+ 		hp->status = 0;
+ 		p = hp->hd[HTTP_HDR_STATUS].b;
+ 		for (j = 100; j != 0; j /= 10) {
+ 			if (!vct_isdigit(*p)) {
+ 				retval = 503;
+ 				break;
+ 			}
+ 			hp->status += (uint16_t)(j * (*p - '0'));
+ 			p++;
+ 		}
+ 		if (*p != '\0')
+ 			retval = 503;
+ 	}
+ 
+ 	if (retval != 0) {
+ 		WSLR(w, SLT_HttpGarbage, htc->vsl_id, htc->rxbuf);
+ 		assert(retval >= 100 && retval <= 999);
+ 		hp->status = retval;
+ 	} else {
+ 		http_ProtoVer(hp);
+ 	}
+ 
+ 	if (hp->hd[HTTP_HDR_RESPONSE].b == NULL ||
+ 	    !Tlen(hp->hd[HTTP_HDR_RESPONSE])) {
+ 		/* Backend didn't send a response string, use the standard */
+ 		hp->hd[HTTP_HDR_RESPONSE].b =
+ 		    TRUST_ME(http_StatusMessage(hp->status));
+ 		hp->hd[HTTP_HDR_RESPONSE].e =
+ 		    strchr(hp->hd[HTTP_HDR_RESPONSE].b, '\0');
+ 	}
+ 	return (retval);
+ }
+ 
+ /*--------------------------------------------------------------------*/
+ 
+ void
+ http_SetH(const struct http *to, unsigned n, const char *fm)
+ {
+ 
+ 	assert(n < to->shd);
+ 	AN(fm);
+ 	to->hd[n].b = TRUST_ME(fm);
+ 	to->hd[n].e = strchr(to->hd[n].b, '\0');
+ 	to->hdf[n] = 0;
+ }
+ 
+ static void
+ http_copyh(const struct http *to, const struct http *fm, unsigned n)
+ {
+ 
+ 	assert(n < HTTP_HDR_FIRST);
+ 	Tcheck(fm->hd[n]);
+ 	to->hd[n] = fm->hd[n];
+ 	to->hdf[n] = fm->hdf[n];
+ }
+ 
+ void
+ http_ForceGet(const struct http *to)
+ {
+ 	if (strcmp(http_GetReq(to), "GET"))
+ 		http_SetH(to, HTTP_HDR_REQ, "GET");
+ }
+ 
+ void
+ http_CopyResp(struct http *to, const struct http *fm)
+ {
+ 
+ 	CHECK_OBJ_NOTNULL(fm, HTTP_MAGIC);
+ 	CHECK_OBJ_NOTNULL(to, HTTP_MAGIC);
+ 	http_SetH(to, HTTP_HDR_PROTO, "HTTP/1.1");
+ 	to->status = fm->status;
+ 	http_copyh(to, fm, HTTP_HDR_RESPONSE);
+ }
+ 
+ void
+ http_SetResp(struct http *to, const char *proto, uint16_t status,
+     const char *response)
+ {
+ 
+ 	CHECK_OBJ_NOTNULL(to, HTTP_MAGIC);
+ 	http_SetH(to, HTTP_HDR_PROTO, proto);
+ 	assert(status >= 100 && status <= 999);
+ 	to->status = status;
+ 	http_SetH(to, HTTP_HDR_RESPONSE, response);
+ }
+ 
+ static void
+ http_copyheader(struct worker *w, unsigned vsl_id, struct http *to,
+     const struct http *fm, unsigned n)
+ {
+ 
+ 	CHECK_OBJ_NOTNULL(fm, HTTP_MAGIC);
+ 	CHECK_OBJ_NOTNULL(to, HTTP_MAGIC);
+ 	assert(n < fm->shd);
+ 	Tcheck(fm->hd[n]);
+ 	if (to->nhd < to->shd) {
+ 		to->hd[to->nhd] = fm->hd[n];
+ 		to->hdf[to->nhd] = 0;
+ 		to->nhd++;
+ 	} else  {
+ 		VSC_C_main->losthdr++;
+ 		WSLR(w, SLT_LostHeader, vsl_id, fm->hd[n]);
+ 	}
+ }
+ 
+ /*--------------------------------------------------------------------
+  * Estimate how much workspace we need to Filter this header according
+  * to 'how'.
+  */
+ 
+ unsigned
+ http_EstimateWS(const struct http *fm, unsigned how, uint16_t *nhd)
+ {
+ 	unsigned u, l;
+ 
+ 	l = 0;
+ 	*nhd = HTTP_HDR_FIRST;
+ 	CHECK_OBJ_NOTNULL(fm, HTTP_MAGIC);
+ 	for (u = 0; u < fm->nhd; u++) {
+ 		if (fm->hd[u].b == NULL)
+ 			continue;
+ 		if (fm->hdf[u] & HDF_FILTER)
+ 			continue;
+ #define HTTPH(a, b, c, d, e, f, g) \
+ 		if (((e) & how) && http_IsHdr(&fm->hd[u], (b))) \
+ 			continue;
+ #include "tbl/http_headers.h"
+ #undef HTTPH
+ 		l += PRNDUP(Tlen(fm->hd[u]) + 1);
+ 		(*nhd)++;
+ 		// fm->hdf[u] |= HDF_COPY;
+ 	}
+ 	return (l);
+ }
+ 
+ /*--------------------------------------------------------------------*/
+ 
+ void
+ http_FilterFields(struct worker *w, unsigned vsl_id, struct http *to,
+     const struct http *fm, unsigned how)
+ {
+ 	unsigned u;
+ 
+ 	CHECK_OBJ_NOTNULL(fm, HTTP_MAGIC);
+ 	CHECK_OBJ_NOTNULL(to, HTTP_MAGIC);
+ 	to->nhd = HTTP_HDR_FIRST;
+ 	to->status = fm->status;
+ 	for (u = HTTP_HDR_FIRST; u < fm->nhd; u++) {
+ 		if (fm->hd[u].b == NULL)
+ 			continue;
+ 		if (fm->hdf[u] & HDF_FILTER)
+ 			continue;
+ #define HTTPH(a, b, c, d, e, f, g) \
+ 		if (((e) & how) && http_IsHdr(&fm->hd[u], (b))) \
+ 			continue;
+ #include "tbl/http_headers.h"
+ #undef HTTPH
+ 		http_copyheader(w, vsl_id, to, fm, u);
+ 	}
+ }
+ 
++/*---------------------------------------------------------------------
++ * Same as http_FilterFields but keep any existing hdrs in fm.
++ * Furthermore, before copy, check if fm already has that hdr, and if so
++ * do not copy.  Used for 304 refresh processing.
++ */
++
++/* XXX: uplex/GS: Also, don't filter according to the "how" bitmap in
++ *      http_headers.h. We only use this to copy from one cached object to
++ *      another, so if a header made into the first object, we want it.
++ */
++
++void
++http_FilterMissingFields(struct worker *w, int fd, struct http *to,
++    const struct http *fm)
++{
++	unsigned u;
++	unsigned hdrlen;
++
++	CHECK_OBJ_NOTNULL(fm, HTTP_MAGIC);
++	CHECK_OBJ_NOTNULL(to, HTTP_MAGIC);
++	for (u = HTTP_HDR_FIRST; u < fm->nhd; u++) {
++		if (fm->hd[u].b == NULL)
++			continue;
++                hdrlen = strchr(fm->hd[u].b, ':') - fm->hd[u].b;
++                if (http_findhdr(to, hdrlen, fm->hd[u].b))
++                    continue;
++                http_copyheader(w, fd, to, fm, u);
++	}
++}
++
+ /*--------------------------------------------------------------------*/
+ 
+ void
+ http_FilterHeader(const struct sess *sp, unsigned how)
+ {
+ 	struct http *hp;
+ 
+ 	hp = sp->wrk->bereq;
+ 	CHECK_OBJ_NOTNULL(hp, HTTP_MAGIC);
+ 	hp->logtag = HTTP_Tx;
+ 
+ 	http_copyh(hp, sp->http, HTTP_HDR_REQ);
+ 	http_copyh(hp, sp->http, HTTP_HDR_URL);
+ 	if (how == HTTPH_R_FETCH)
+ 		http_SetH(hp, HTTP_HDR_PROTO, "HTTP/1.1");
+ 	else
+ 		http_copyh(hp, sp->http, HTTP_HDR_PROTO);
+ 	http_FilterFields(sp->wrk, sp->vsl_id, hp, sp->http, how);
+ 	http_PrintfHeader(sp->wrk, sp->vsl_id, hp, "X-Varnish: %u", sp->xid);
+ }
+ 
++/*-------------------------------------------------------------------
++ * This function checks for sp->freshen_obj.  If present, HSH_Lookup()
++ * found an expired object that qualifies for a refresh check,
++ * so add the appropriate headers.
++ */
++
++void
++http_CheckRefresh(struct sess *sp)
++{
++	struct object *freshen_obj;
++	struct http *obj_hp, *bereq_hp;
++	char *p;
++
++	freshen_obj = sp->stale_obj;
++	CHECK_OBJ_NOTNULL(freshen_obj, OBJECT_MAGIC);
++	bereq_hp = sp->wrk->bereq;
++	CHECK_OBJ_NOTNULL(bereq_hp, HTTP_MAGIC);
++	obj_hp = freshen_obj->http;
++	CHECK_OBJ_NOTNULL(obj_hp, HTTP_MAGIC);
++
++	if(http_GetHdr(obj_hp, H_ETag, &p))
++		http_PrintfHeader(sp->wrk, sp->fd, bereq_hp, "If-None-Match: %s", p);
++
++	if(http_GetHdr(obj_hp, H_Last_Modified, &p))
++		http_PrintfHeader(sp->wrk, sp->fd, bereq_hp, "If-Modified-Since: %s",p);
++}
++
++/*-------------------------------------------------------------------
++ * Called after fetch and sp->freshen_obj present.  Check
++ * response and handle as needed.
++ */
++
++void
++http_Check304(struct sess *sp)
++{
++	struct object *o, *o_stale;
++	char *p;
++
++	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
++	o_stale = sp->stale_obj;
++	CHECK_OBJ_NOTNULL(o_stale, OBJECT_MAGIC);
++	o = sp->obj;
++	CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC);
++
++	if (sp->wrk->beresp->status != 304) {
++	    /*
++	     * IMS/INM headers may have been removed in VCL, so only count a
++	     * non-validating response if they were present in the request.
++	     */
++	    if (http_GetHdr(sp->wrk->bereq, H_If_Modified_Since, &p)
++		|| http_GetHdr(sp->wrk->bereq, H_If_None_Match, &p))
++		sp->wrk->stats.cond_not_validated++;
++	    return;
++	}
++
++	/* 
++	 * Copy headers we need from the stale object into the 304 response
++	 */
++	http_FilterMissingFields(sp->wrk, sp->fd, sp->obj->http,
++				 sp->stale_obj->http);
++
++	/*
++	 * Dup the stale object's storage in to the new object
++	 * and reset Content-Length from the size of the storage.
++	 */
++	STV_dup(sp, o_stale, o);
++	http_Unset(o->http, H_Content_Length);
++	http_PrintfHeader(sp->wrk, sp->fd, o->http, "Content-Length: %u", o->len);
++
++	http_SetResp(o->http, "HTTP/1.1", 200, "Ok Not Modified");
++	http_SetH(o->http, HTTP_HDR_REQ, "GET");
++	http_copyh(o->http, sp->wrk->bereq, HTTP_HDR_URL);
++
++	/*
++	 * XXX: Are we copying all the necessary fields from stale_obj?
++	 *	Should we copy o_stale->hits into o->hits?
++	 */
++	o->response = 200;
++	o->gziped = o_stale->gziped;
++
++        AZ(o_stale->objcore->flags & OC_F_BUSY);
++}
++
+ /*--------------------------------------------------------------------
+  * This function copies any header fields which reference foreign
+  * storage into our own WS.
+  */
+ 
+ void
+ http_CopyHome(struct worker *w, unsigned vsl_id, const struct http *hp)
+ {
+ 	unsigned u, l;
+ 	char *p;
+ 
+ 	for (u = 0; u < hp->nhd; u++) {
+ 		if (hp->hd[u].b == NULL)
+ 			continue;
+ 		if (hp->hd[u].b >= hp->ws->s && hp->hd[u].e <= hp->ws->e) {
+ 			WSLH(w, vsl_id, hp, u);
+ 			continue;
+ 		}
+ 		l = Tlen(hp->hd[u]);
+ 		p = WS_Alloc(hp->ws, l + 1);
+ 		if (p != NULL) {
+ 			WSLH(w, vsl_id, hp, u);
+ 			memcpy(p, hp->hd[u].b, l + 1L);
+ 			hp->hd[u].b = p;
+ 			hp->hd[u].e = p + l;
+ 		} else {
+ 			/* XXX This leaves a slot empty */
+ 			VSC_C_main->losthdr++;
+ 			WSLR(w, SLT_LostHeader, vsl_id, hp->hd[u]);
+ 			hp->hd[u].b = NULL;
+ 			hp->hd[u].e = NULL;
+ 		}
+ 	}
+ }
+ 
+ /*--------------------------------------------------------------------*/
+ 
+ void
+ http_ClrHeader(struct http *to)
+ {
+ 
+ 	CHECK_OBJ_NOTNULL(to, HTTP_MAGIC);
+ 	to->nhd = HTTP_HDR_FIRST;
+ 	to->status = 0;
+ 	to->protover = 0;
+ 	to->conds = 0;
+ 	memset(to->hd, 0, sizeof *to->hd * to->shd);
+ }
+ 
+ /*--------------------------------------------------------------------*/
+ 
+ void
+ http_SetHeader(struct worker *w, unsigned vsl_id, struct http *to,
+     const char *hdr)
+ {
+ 
+ 	CHECK_OBJ_NOTNULL(to, HTTP_MAGIC);
+ 	if (to->nhd >= to->shd) {
+ 		VSC_C_main->losthdr++;
+ 		WSL(w, SLT_LostHeader, vsl_id, "%s", hdr);
+ 		return;
+ 	}
+ 	http_SetH(to, to->nhd++, hdr);
+ }
+ 
+ /*--------------------------------------------------------------------*/
+ 
+ static void
+ http_PutField(struct worker *w, unsigned vsl_id, const struct http *to,
+     int field, const char *string)
+ {
+ 	char *p;
+ 	unsigned l;
+ 
+ 	CHECK_OBJ_NOTNULL(to, HTTP_MAGIC);
+ 	l = strlen(string);
+ 	p = WS_Alloc(to->ws, l + 1);
+ 	if (p == NULL) {
+ 		WSL(w, SLT_LostHeader, vsl_id, "%s", string);
+ 		to->hd[field].b = NULL;
+ 		to->hd[field].e = NULL;
+ 		to->hdf[field] = 0;
+ 	} else {
+ 		memcpy(p, string, l + 1L);
+ 		to->hd[field].b = p;
+ 		to->hd[field].e = p + l;
+ 		to->hdf[field] = 0;
+ 	}
+ }
+ 
+ void
+ http_PutProtocol(struct worker *w, unsigned vsl_id, const struct http *to,
+     const char *protocol)
+ {
+ 
+ 	http_PutField(w, vsl_id, to, HTTP_HDR_PROTO, protocol);
+ 	if (to->hd[HTTP_HDR_PROTO].b == NULL)
+ 		http_SetH(to, HTTP_HDR_PROTO, "HTTP/1.1");
+ 	Tcheck(to->hd[HTTP_HDR_PROTO]);
+ }
+ 
+ void
+ http_PutStatus(struct http *to, uint16_t status)
+ {
+ 
+ 	assert(status >= 100 && status <= 999);
+ 	to->status = status;
+ }
+ 
+ void
+ http_PutResponse(struct worker *w, unsigned vsl_id, const struct http *to,
+     const char *response)
+ {
+ 
+ 	http_PutField(w, vsl_id, to, HTTP_HDR_RESPONSE, response);
+ 	if (to->hd[HTTP_HDR_RESPONSE].b == NULL)
+ 		http_SetH(to, HTTP_HDR_RESPONSE, "Lost Response");
+ 	Tcheck(to->hd[HTTP_HDR_RESPONSE]);
+ }
+ 
+ void
+ http_PrintfHeader(struct worker *w, unsigned vsl_id, struct http *to,
+     const char *fmt, ...)
+ {
+ 	va_list ap;
+ 	unsigned l, n;
+ 
+ 	CHECK_OBJ_NOTNULL(to, HTTP_MAGIC);
+ 	l = WS_Reserve(to->ws, 0);
+ 	va_start(ap, fmt);
+ 	n = vsnprintf(to->ws->f, l, fmt, ap);
+ 	va_end(ap);
+ 	if (n + 1 >= l || to->nhd >= to->shd) {
+ 		VSC_C_main->losthdr++;
+ 		WSL(w, SLT_LostHeader, vsl_id, "%s", to->ws->f);
+ 		WS_Release(to->ws, 0);
+ 	} else {
+ 		to->hd[to->nhd].b = to->ws->f;
+ 		to->hd[to->nhd].e = to->ws->f + n;
+ 		to->hdf[to->nhd] = 0;
+ 		WS_Release(to->ws, n + 1);
+ 		to->nhd++;
+ 	}
+ }
+ /*--------------------------------------------------------------------*/
+ 
+ void
+ http_Unset(struct http *hp, const char *hdr)
+ {
+ 	uint16_t u, v;
+ 
+ 	for (v = u = HTTP_HDR_FIRST; u < hp->nhd; u++) {
+ 		if (hp->hd[u].b == NULL)
+ 			continue;
+ 		if (http_IsHdr(&hp->hd[u], hdr))
+ 			continue;
+ 		if (v != u) {
+ 			memcpy(&hp->hd[v], &hp->hd[u], sizeof *hp->hd);
+ 			memcpy(&hp->hdf[v], &hp->hdf[u], sizeof *hp->hdf);
+ 		}
+ 		v++;
+ 	}
+ 	hp->nhd = v;
+ }
+ 
+ /*--------------------------------------------------------------------*/
+ 
+ void
+ HTTP_Copy(struct http *to, const struct http * const fm)
+ {
+ 
+ 	to->conds = fm->conds;
+ 	to->logtag = fm->logtag;
+ 	to->status = fm->status;
+ 	to->protover = fm->protover;
+ 	to->nhd = fm->nhd;
+ 	assert(fm->nhd <= to->shd);
+ 	memcpy(to->hd, fm->hd, fm->nhd * sizeof *to->hd);
+ 	memcpy(to->hdf, fm->hdf, fm->nhd * sizeof *to->hdf);
+ }
+ 
+ /*--------------------------------------------------------------------*/
+ 
+ unsigned
+ http_Write(struct worker *w, unsigned vsl_id, const struct http *hp, int resp)
+ {
+ 	unsigned u, l;
+ 
+ 	if (resp) {
+ 		l = WRW_WriteH(w, &hp->hd[HTTP_HDR_PROTO], " ");
+ 		WSLH(w, vsl_id, hp, HTTP_HDR_PROTO);
+ 
+ 		hp->hd[HTTP_HDR_STATUS].b = WS_Alloc(w->ws, 4);
+ 		AN(hp->hd[HTTP_HDR_STATUS].b);
+ 
+ 		sprintf(hp->hd[HTTP_HDR_STATUS].b, "%3d", hp->status);
+ 		hp->hd[HTTP_HDR_STATUS].e = hp->hd[HTTP_HDR_STATUS].b + 3;
+ 
+ 		l += WRW_WriteH(w, &hp->hd[HTTP_HDR_STATUS], " ");
+ 		WSLH(w, vsl_id, hp, HTTP_HDR_STATUS);
+ 
+ 		l += WRW_WriteH(w, &hp->hd[HTTP_HDR_RESPONSE], "\r\n");
+ 		WSLH(w, vsl_id, hp, HTTP_HDR_RESPONSE);
+ 	} else {
+ 		AN(hp->hd[HTTP_HDR_URL].b);
+ 		l = WRW_WriteH(w, &hp->hd[HTTP_HDR_REQ], " ");
+ 		WSLH(w, vsl_id, hp, HTTP_HDR_REQ);
+ 		l += WRW_WriteH(w, &hp->hd[HTTP_HDR_URL], " ");
+ 		WSLH(w, vsl_id, hp, HTTP_HDR_URL);
+ 		l += WRW_WriteH(w, &hp->hd[HTTP_HDR_PROTO], "\r\n");
+ 		WSLH(w, vsl_id, hp, HTTP_HDR_PROTO);
+ 	}
+ 	for (u = HTTP_HDR_FIRST; u < hp->nhd; u++) {
+ 		if (hp->hd[u].b == NULL)
+ 			continue;
+ 		AN(hp->hd[u].b);
+ 		AN(hp->hd[u].e);
+ 		l += WRW_WriteH(w, &hp->hd[u], "\r\n");
+ 		WSLH(w, vsl_id, hp, u);
+ 	}
+ 	l += WRW_Write(w, "\r\n", -1);
+ 	return (l);
+ }
+ 
+ /*--------------------------------------------------------------------*/
+ 
+ void
+ HTTP_Init(void)
+ {
+ 
+ #define HTTPH(a, b, c, d, e, f, g) b[0] = (char)strlen(b + 1);
+ #include "tbl/http_headers.h"
+ #undef HTTPH
+ }
diff --cc bin/varnishd/cache/cache_vrt.c
index 0000000,5e19ccc..27964dc
mode 000000,100644..100644
--- a/bin/varnishd/cache/cache_vrt.c
+++ b/bin/varnishd/cache/cache_vrt.c
@@@ -1,0 -1,535 +1,544 @@@
+ /*-
+  * Copyright (c) 2006 Verdens Gang AS
+  * Copyright (c) 2006-2011 Varnish Software AS
+  * All rights reserved.
+  *
+  * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
+  *
+  * Redistribution and use in source and binary forms, with or without
+  * modification, are permitted provided that the following conditions
+  * are met:
+  * 1. Redistributions of source code must retain the above copyright
+  *    notice, this list of conditions and the following disclaimer.
+  * 2. Redistributions in binary form must reproduce the above copyright
+  *    notice, this list of conditions and the following disclaimer in the
+  *    documentation and/or other materials provided with the distribution.
+  *
+  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+  * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+  * SUCH DAMAGE.
+  *
+  * Runtime support for compiled VCL programs
+  */
+ 
+ #include "config.h"
+ 
+ #include <netinet/in.h>
+ #include <arpa/inet.h>
+ 
+ #include <stdio.h>
+ #include <stdlib.h>
+ 
+ #include "cache.h"
+ 
+ #include "cache_backend.h"
+ #include "hash/hash_slinger.h"
+ #include "vav.h"
+ #include "vcl.h"
+ #include "vrt.h"
+ #include "vrt_obj.h"
+ #include "vtim.h"
+ 
+ const void * const vrt_magic_string_end = &vrt_magic_string_end;
+ 
+ /*--------------------------------------------------------------------*/
+ 
+ void
+ VRT_error(struct sess *sp, unsigned code, const char *reason)
+ {
+ 
+ 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+ 	WSL(sp->wrk, SLT_Debug, 0, "VCL_error(%u, %s)", code, reason ?
+ 	    reason : "(null)");
+ 	if (code < 100 || code > 999)
+ 		code = 503;
+ 	sp->err_code = (uint16_t)code;
+ 	sp->err_reason = reason ? reason : http_StatusMessage(sp->err_code);
+ }
+ 
+ /*--------------------------------------------------------------------*/
+ 
+ void
+ VRT_count(const struct sess *sp, unsigned u)
+ {
+ 
+ 	if (sp == NULL)
+ 		return;
+ 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+ 	if (cache_param->vcl_trace)
+ 		WSP(sp, SLT_VCL_trace, "%u %d.%d", u,
+ 		    sp->vcl->ref[u].line, sp->vcl->ref[u].pos);
+ }
+ 
+ /*--------------------------------------------------------------------*/
+ 
+ void
+ VRT_acl_log(const struct sess *sp, const char *msg)
+ {
+ 	WSP(sp, SLT_VCL_acl, msg);
+ }
+ 
+ /*--------------------------------------------------------------------*/
+ 
+ static struct http *
+ vrt_selecthttp(const struct sess *sp, enum gethdr_e where)
+ {
+ 	struct http *hp;
+ 
+ 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+ 	switch (where) {
+ 	case HDR_REQ:
+ 		hp = sp->http;
+ 		break;
+ 	case HDR_BEREQ:
+ 		hp = sp->wrk->bereq;
+ 		break;
+ 	case HDR_BERESP:
+ 		hp = sp->wrk->beresp;
+ 		break;
+ 	case HDR_RESP:
+ 		hp = sp->wrk->resp;
+ 		break;
+ 	case HDR_OBJ:
+ 		CHECK_OBJ_NOTNULL(sp->obj, OBJECT_MAGIC);
+ 		hp = sp->obj->http;
+ 		break;
++        case HDR_STALE_OBJ:
++		CHECK_OBJ_NOTNULL(sp->stale_obj, OBJECT_MAGIC);
++		hp = sp->stale_obj->http;
++		break;
+ 	default:
+ 		INCOMPL();
+ 	}
+ 	CHECK_OBJ_NOTNULL(hp, HTTP_MAGIC);
+ 	return (hp);
+ }
+ 
+ char *
+ VRT_GetHdr(const struct sess *sp, enum gethdr_e where, const char *n)
+ {
+ 	char *p;
+ 	struct http *hp;
+ 
+ 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
++        if (where == HDR_STALE_OBJ && sp->stale_obj == NULL) {
++	    WSP(sp, SLT_VCL_error,
++		"stale_obj does not exist (reading header %s)", n);
++            return NULL;
++	}
+ 	hp = vrt_selecthttp(sp, where);
+ 	if (!http_GetHdr(hp, n, &p))
+ 		return (NULL);
+ 	return (p);
+ }
+ 
+ /*--------------------------------------------------------------------
+  * XXX: Optimize the single element case ?
+  */
+ 
+ char *
+ VRT_StringList(char *d, unsigned dl, const char *p, va_list ap)
+ {
+ 	char *b, *e;
+ 	unsigned x;
+ 
+ 	b = d;
+ 	e = b + dl;
+ 	while (p != vrt_magic_string_end && b < e) {
+ 		if (p != NULL) {
+ 			x = strlen(p);
+ 			if (b + x < e)
+ 				memcpy(b, p, x);
+ 			b += x;
+ 		}
+ 		p = va_arg(ap, const char *);
+ 	}
+ 	if (b >= e)
+ 		return (NULL);
+ 	*b++ = '\0';
+ 	return (b);
+ }
+ 
+ /*--------------------------------------------------------------------
+  * XXX: Optimize the single element case ?
+  */
+ 
+ char *
+ VRT_String(struct ws *ws, const char *h, const char *p, va_list ap)
+ {
+ 	char *b, *e;
+ 	unsigned u, x;
+ 
+ 	u = WS_Reserve(ws, 0);
+ 	e = b = ws->f;
+ 	e += u;
+ 	if (h != NULL) {
+ 		x = strlen(h);
+ 		if (b + x < e)
+ 			memcpy(b, h, x);
+ 		b += x;
+ 		if (b < e)
+ 			*b = ' ';
+ 		b++;
+ 	}
+ 	b = VRT_StringList(b, e > b ? e - b : 0, p, ap);
+ 	if (b == NULL || b == e) {
+ 		WS_Release(ws, 0);
+ 		return (NULL);
+ 	}
+ 	e = b;
+ 	b = ws->f;
+ 	WS_Release(ws, e - b);
+ 	return (b);
+ }
+ 
+ /*--------------------------------------------------------------------
+  * Build a string on the worker threads workspace
+  */
+ 
+ const char *
+ VRT_WrkString(const struct sess *sp, const char *p, ...)
+ {
+ 	va_list ap;
+ 	char *b;
+ 
+ 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+ 	va_start(ap, p);
+ 	b = VRT_String(sp->wrk->ws, NULL, p, ap);
+ 	va_end(ap);
+ 	return (b);
+ }
+ 
+ /*--------------------------------------------------------------------*/
+ 
+ void
+ VRT_SetHdr(const struct sess *sp , enum gethdr_e where, const char *hdr,
+     const char *p, ...)
+ {
+ 	struct http *hp;
+ 	va_list ap;
+ 	char *b;
+ 
+ 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+ 	hp = vrt_selecthttp(sp, where);
+ 	va_start(ap, p);
+ 	if (p == NULL) {
+ 		http_Unset(hp, hdr);
+ 	} else {
+ 		b = VRT_String(hp->ws, hdr + 1, p, ap);
+ 		if (b == NULL) {
+ 			WSP(sp, SLT_LostHeader, "%s", hdr + 1);
+ 		} else {
+ 			http_Unset(hp, hdr);
+ 			http_SetHeader(sp->wrk, sp->vsl_id, hp, b);
+ 		}
+ 	}
+ 	va_end(ap);
+ }
+ 
+ /*--------------------------------------------------------------------*/
+ 
+ void
+ VRT_handling(struct sess *sp, unsigned hand)
+ {
+ 
+ 	if (sp == NULL) {
+ 		assert(hand == VCL_RET_OK);
+ 		return;
+ 	}
+ 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+ 	assert(hand < VCL_RET_MAX);
+ 	sp->handling = hand;
+ }
+ 
+ /*--------------------------------------------------------------------
+  * Add an element to the array/list of hash bits.
+  */
+ 
+ void
+ VRT_hashdata(const struct sess *sp, const char *str, ...)
+ {
+ 	va_list ap;
+ 	const char *p;
+ 
+ 	HSH_AddString(sp, str);
+ 	va_start(ap, str);
+ 	while (1) {
+ 		p = va_arg(ap, const char *);
+ 		if (p == vrt_magic_string_end)
+ 			break;
+ 		HSH_AddString(sp, p);
+ 	}
+ }
+ 
+ /*--------------------------------------------------------------------*/
+ 
+ double
+ VRT_r_now(const struct sess *sp)
+ {
+ 
+ 	(void)sp;
+ 	return (VTIM_real());
+ }
+ 
+ /*--------------------------------------------------------------------*/
+ 
+ char *
+ VRT_IP_string(const struct sess *sp, const struct sockaddr_storage *sa)
+ {
+ 	char *p;
+ 	const struct sockaddr_in *si4;
+ 	const struct sockaddr_in6 *si6;
+ 	const void *addr;
+ 	int len;
+ 
+ 	switch (sa->ss_family) {
+ 	case AF_INET:
+ 		len = INET_ADDRSTRLEN;
+ 		si4 = (const void *)sa;
+ 		addr = &(si4->sin_addr);
+ 		break;
+ 	case AF_INET6:
+ 		len = INET6_ADDRSTRLEN;
+ 		si6 = (const void *)sa;
+ 		addr = &(si6->sin6_addr);
+ 		break;
+ 	default:
+ 		INCOMPL();
+ 	}
+ 	XXXAN(len);
+ 	AN(p = WS_Alloc(sp->http->ws, len));
+ 	AN(inet_ntop(sa->ss_family, addr, p, len));
+ 	return (p);
+ }
+ 
+ char *
+ VRT_int_string(const struct sess *sp, int num)
+ {
+ 	char *p;
+ 	int size;
+ 
+ 	size = snprintf(NULL, 0, "%d", num) + 1;
+ 	AN(p = WS_Alloc(sp->http->ws, size));
+ 	assert(snprintf(p, size, "%d", num) < size);
+ 	return (p);
+ }
+ 
+ char *
+ VRT_double_string(const struct sess *sp, double num)
+ {
+ 	char *p;
+ 	int size;
+ 
+ 	size = snprintf(NULL, 0, "%.3f", num) + 1;
+ 	AN(p = WS_Alloc(sp->http->ws, size));
+ 	assert(snprintf(p, size, "%.3f", num) < size);
+ 	return (p);
+ }
+ 
+ char *
+ VRT_time_string(const struct sess *sp, double t)
+ {
+ 	char *p;
+ 
+ 	AN(p = WS_Alloc(sp->http->ws, VTIM_FORMAT_SIZE));
+ 	VTIM_format(t, p);
+ 	return (p);
+ }
+ 
+ const char *
+ VRT_backend_string(const struct sess *sp, const struct director *d)
+ {
+ 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+ 	if (d == NULL)
+ 		d = sp->director;
+ 	if (d == NULL)
+ 		return (NULL);
+ 	return (d->vcl_name);
+ }
+ 
+ const char *
+ VRT_bool_string(const struct sess *sp, unsigned val)
+ {
+ 
+ 	(void)sp;
+ 	return (val ? "true" : "false");
+ }
+ 
+ /*--------------------------------------------------------------------*/
+ 
+ void
+ VRT_Rollback(struct sess *sp)
+ {
+ 
+ 	HTTP_Copy(sp->http, sp->http0);
+ 	WS_Reset(sp->ws, sp->ws_req);
+ }
+ 
+ /*--------------------------------------------------------------------*/
+ 
+ void
+ VRT_panic(const struct sess *sp, const char *str, ...)
+ {
+ 	va_list ap;
+ 	char *b;
+ 
+ 	va_start(ap, str);
+ 	b = VRT_String(sp->http->ws, "PANIC: ", str, ap);
+ 	va_end(ap);
+ 	VAS_Fail("VCL", "", 0, b, 0, 2);
+ }
+ 
+ /*--------------------------------------------------------------------*/
+ 
+ void
+ VRT_synth_page(const struct sess *sp, unsigned flags, const char *str, ...)
+ {
+ 	va_list ap;
+ 	const char *p;
+ 	struct vsb *vsb;
+ 
+ 	(void)flags;
+ 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+ 	CHECK_OBJ_NOTNULL(sp->obj, OBJECT_MAGIC);
+ 	vsb = SMS_Makesynth(sp->obj);
+ 	AN(vsb);
+ 
+ 	VSB_cat(vsb, str);
+ 	va_start(ap, str);
+ 	p = va_arg(ap, const char *);
+ 	while (p != vrt_magic_string_end) {
+ 		if (p == NULL)
+ 			p = "(null)";
+ 		VSB_cat(vsb, p);
+ 		p = va_arg(ap, const char *);
+ 	}
+ 	va_end(ap);
+ 	SMS_Finish(sp->obj);
+ 	http_Unset(sp->obj->http, H_Content_Length);
+ 	http_PrintfHeader(sp->wrk, sp->vsl_id, sp->obj->http,
+ 	    "Content-Length: %d", sp->obj->len);
+ }
+ 
+ /*--------------------------------------------------------------------*/
+ 
+ void
+ VRT_ban(struct sess *sp, char *cmds, ...)
+ {
+ 	char *a1, *a2, *a3;
+ 	va_list ap;
+ 	struct ban *b;
+ 	int good;
+ 
+ 	(void)sp;
+ 	b = BAN_New();
+ 	va_start(ap, cmds);
+ 	a1 = cmds;
+ 	good = 0;
+ 	while (a1 != NULL) {
+ 		good = 0;
+ 		a2 = va_arg(ap, char *);
+ 		if (a2 == NULL)
+ 			break;
+ 		a3 = va_arg(ap, char *);
+ 		if (a3 == NULL)
+ 			break;
+ 		if (BAN_AddTest(NULL, b, a1, a2, a3))
+ 			break;
+ 		a1 = va_arg(ap, char *);
+ 		good = 1;
+ 	}
+ 	if (!good)
+ 		/* XXX: report error how ? */
+ 		BAN_Free(b);
+ 	else
+ 		BAN_Insert(b);
+ }
+ 
+ /*--------------------------------------------------------------------*/
+ 
+ void
+ VRT_ban_string(struct sess *sp, const char *str)
+ {
+ 	char *a1, *a2, *a3;
+ 	char **av;
+ 	struct ban *b;
+ 	int good;
+ 	int i;
+ 
+ 	(void)sp;
+ 	av = VAV_Parse(str, NULL, ARGV_NOESC);
+ 	if (av[0] != NULL) {
+ 		/* XXX: report error how ? */
+ 		VAV_Free(av);
+ 		return;
+ 	}
+ 	b = BAN_New();
+ 	good = 0;
+ 	for (i = 1; ;) {
+ 		a1 = av[i++];
+ 		if (a1 == NULL)
+ 			break;
+ 		good = 0;
+ 		a2 = av[i++];
+ 		if (a2 == NULL)
+ 			break;
+ 		a3 = av[i++];
+ 		if (a3 == NULL)
+ 			break;
+ 		if (BAN_AddTest(NULL, b, a1, a2, a3))
+ 			break;
+ 		good = 1;
+ 		if (av[i] == NULL)
+ 			break;
+ 		good = 0;
+ 		if (strcmp(av[i++], "&&"))
+ 			break;
+ 	}
+ 	if (!good)
+ 		/* XXX: report error how ? */
+ 		BAN_Free(b);
+ 	else
+ 		BAN_Insert(b);
+ 	VAV_Free(av);
+ }
+ 
+ /*--------------------------------------------------------------------
+  * "real" purges
+  */
+ 
+ void
+ VRT_purge(const struct sess *sp, double ttl, double grace)
+ {
+ 	if (sp->cur_method == VCL_MET_HIT)
+ 		HSH_Purge(sp, sp->obj->objcore->objhead, ttl, grace);
+ 	else if (sp->cur_method == VCL_MET_MISS)
+ 		HSH_Purge(sp, sp->objcore->objhead, ttl, grace);
+ }
+ 
+ /*--------------------------------------------------------------------
+  * Simple stuff
+  */
+ 
+ int
+ VRT_strcmp(const char *s1, const char *s2)
+ {
+ 	if (s1 == NULL || s2 == NULL)
+ 		return(1);
+ 	return (strcmp(s1, s2));
+ }
+ 
+ void
+ VRT_memmove(void *dst, const void *src, unsigned len)
+ {
+ 
+ 	(void)memmove(dst, src, len);
+ }
diff --cc bin/varnishd/cache/cache_vrt_var.c
index 0000000,860c7aa..407de1d
mode 000000,100644..100644
--- a/bin/varnishd/cache/cache_vrt_var.c
+++ b/bin/varnishd/cache/cache_vrt_var.c
@@@ -1,0 -1,550 +1,620 @@@
+ /*-
+  * Copyright (c) 2006 Verdens Gang AS
+  * Copyright (c) 2006-2011 Varnish Software AS
+  * All rights reserved.
+  *
+  * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
+  *
+  * Redistribution and use in source and binary forms, with or without
+  * modification, are permitted provided that the following conditions
+  * are met:
+  * 1. Redistributions of source code must retain the above copyright
+  *    notice, this list of conditions and the following disclaimer.
+  * 2. Redistributions in binary form must reproduce the above copyright
+  *    notice, this list of conditions and the following disclaimer in the
+  *    documentation and/or other materials provided with the distribution.
+  *
+  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+  * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+  * SUCH DAMAGE.
+  *
+  * Runtime support for compiled VCL programs
+  */
+ #include "config.h"
+ 
+ #include <stdio.h>
+ #include <stdlib.h>
+ 
+ #include "cache.h"
+ #include "common/heritage.h"
+ 
+ #include "cache_backend.h"
+ #include "vrt_obj.h"
+ #include "vtcp.h"
+ #include "vtim.h"
+ 
++#define ILLEGAL_R(sess, obj, field)                                         \
++WSP(sess, SLT_VCL_error, "%s does not exist (reading field %s)", obj, field)
++
+ static char vrt_hostname[255] = "";
+ 
+ /*--------------------------------------------------------------------*/
+ 
+ static void
+ vrt_do_string(struct worker *w, int fd, const struct http *hp, int fld,
+     const char *err, const char *p, va_list ap)
+ {
+ 	char *b;
+ 
+ 	// AN(p);
+ 	AN(hp);
+ 	b = VRT_String(hp->ws, NULL, p, ap);
+ 	if (b == NULL || *b == '\0') {
+ 		WSL(w, SLT_LostHeader, fd, err);
+ 	} else {
+ 		http_SetH(hp, fld, b);
+ 	}
+ 	va_end(ap);
+ }
+ 
 -#define VRT_DO_HDR(obj, hdr, http, fld)				\
++#define VRT_DO_HDR_l(obj, hdr, cont, http, fld)			\
+ void								\
+ VRT_l_##obj##_##hdr(const struct sess *sp, const char *p, ...)	\
+ {								\
+ 	va_list ap;						\
+ 								\
+ 	va_start(ap, p);					\
+ 	vrt_do_string(sp->wrk, sp->fd,				\
 -	    http, fld, #obj "." #hdr, p, ap);			\
++	    cont->http, fld, #obj "." #hdr, p, ap);		\
+ 	va_end(ap);						\
 -}								\
 -								\
++}
++
++#define VRT_DO_HDR_r(obj, hdr, cont, http, fld, nullable)	\
+ const char *							\
+ VRT_r_##obj##_##hdr(const struct sess *sp)			\
+ {								\
+ 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);			\
 -	CHECK_OBJ_NOTNULL(http, HTTP_MAGIC);			\
 -	return (http->hd[fld].b);				\
 -}
 -
 -VRT_DO_HDR(req,   request,	sp->http,		HTTP_HDR_REQ)
 -VRT_DO_HDR(req,   url,		sp->http,		HTTP_HDR_URL)
 -VRT_DO_HDR(req,   proto,	sp->http,		HTTP_HDR_PROTO)
 -VRT_DO_HDR(bereq, request,	sp->wrk->bereq,		HTTP_HDR_REQ)
 -VRT_DO_HDR(bereq, url,		sp->wrk->bereq,		HTTP_HDR_URL)
 -VRT_DO_HDR(bereq, proto,	sp->wrk->bereq,		HTTP_HDR_PROTO)
 -VRT_DO_HDR(obj,   proto,	sp->obj->http,		HTTP_HDR_PROTO)
 -VRT_DO_HDR(obj,   response,	sp->obj->http,		HTTP_HDR_RESPONSE)
 -VRT_DO_HDR(resp,  proto,	sp->wrk->resp,		HTTP_HDR_PROTO)
 -VRT_DO_HDR(resp,  response,	sp->wrk->resp,		HTTP_HDR_RESPONSE)
 -VRT_DO_HDR(beresp,  proto,	sp->wrk->beresp,	HTTP_HDR_PROTO)
 -VRT_DO_HDR(beresp,  response,	sp->wrk->beresp,	HTTP_HDR_RESPONSE)
++        if (!nullable || cont != NULL) {			\
++            CHECK_OBJ_NOTNULL(cont->http, HTTP_MAGIC);		\
++            return (cont->http->hd[fld].b);			\
++        }							\
++        ILLEGAL_R(sp, #obj, #hdr);				\
++        return(NULL);                                           \
++}                                                               \
++
++#define VRT_DO_HDR(obj, hdr, cont, http, fld, nullable)		\
++VRT_DO_HDR_l(obj, hdr, cont, http, fld)				\
++VRT_DO_HDR_r(obj, hdr, cont, http, fld, nullable)		\
++
++VRT_DO_HDR(req,         request,    sp,             http,	HTTP_HDR_REQ,	    0)
++VRT_DO_HDR(req,         url,        sp,             http,	HTTP_HDR_URL,	    0)
++VRT_DO_HDR(req,         proto,      sp,             http,	HTTP_HDR_PROTO,	    0)
++VRT_DO_HDR(bereq,       request,    sp->wrk,        bereq,	HTTP_HDR_REQ,	    0)
++VRT_DO_HDR(bereq,       url,        sp->wrk,        bereq,	HTTP_HDR_URL,	    0)
++VRT_DO_HDR(bereq,       proto,      sp->wrk,        bereq,	HTTP_HDR_PROTO,	    0)
++VRT_DO_HDR(obj,         proto,      sp->obj,        http,	HTTP_HDR_PROTO,	    0)
++VRT_DO_HDR(obj,         response,   sp->obj,        http,	HTTP_HDR_RESPONSE,  0)
++VRT_DO_HDR(resp,        proto,      sp->wrk,        resp,	HTTP_HDR_PROTO,	    0)
++VRT_DO_HDR(resp,        response,   sp->wrk,        resp,	HTTP_HDR_RESPONSE,  0)
++VRT_DO_HDR(beresp,      proto,      sp->wrk,        beresp,	HTTP_HDR_PROTO,	    0)
++VRT_DO_HDR(beresp,      response,   sp->wrk,        beresp,	HTTP_HDR_RESPONSE,  0)
++VRT_DO_HDR_r(stale_obj,   proto,      sp->stale_obj,  http,	HTTP_HDR_PROTO,	    1)
++VRT_DO_HDR_r(stale_obj,   response,   sp->stale_obj,  http,	HTTP_HDR_RESPONSE,  1)
+ 
+ /*--------------------------------------------------------------------*/
+ 
 -#define VRT_DO_STATUS(obj, http)				\
++#define VRT_DO_STATUS_l(obj, cont, http)			\
+ void								\
+ VRT_l_##obj##_status(const struct sess *sp, int num)		\
+ {								\
+ 								\
+ 	assert(num >= 100 && num <= 999);			\
 -	http->status = (uint16_t)num;				\
 -}								\
 -								\
++	cont->http->status = (uint16_t) num;			\
++}
++
++#define VRT_DO_STATUS_r(obj, cont, http, nullable)		\
+ int								\
+ VRT_r_##obj##_status(const struct sess *sp)			\
+ {								\
+ 								\
+ 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);			\
 -	return(http->status);					\
++        if (nullable && cont == NULL) {				\
++            ILLEGAL_R(sp, #obj, "status");			\
++            return (503);                                       \
++        }                                                       \
++	return(cont->http->status);				\
+ }
+ 
 -VRT_DO_STATUS(obj, sp->obj->http)
 -VRT_DO_STATUS(beresp, sp->wrk->beresp)
 -VRT_DO_STATUS(resp, sp->wrk->resp)
++#define VRT_DO_STATUS(obj, cont, http, nullable)		\
++VRT_DO_STATUS_l(obj, cont, http)				\
++VRT_DO_STATUS_r(obj, cont, http, nullable)			\
++
++VRT_DO_STATUS(obj,          sp->obj,        http,	0)
++VRT_DO_STATUS(beresp,       sp->wrk,        beresp,	0)
++VRT_DO_STATUS(resp,	    sp->wrk,        resp,	0)
++VRT_DO_STATUS_r(stale_obj,  sp->stale_obj,  http,	1)
+ 
+ /*--------------------------------------------------------------------*/
+ 
+ /* XXX: review this */
+ /* Add an objecthead to the saintmode list for the (hopefully) relevant
+  * backend. Some double-up asserting here to avoid assert-errors when there
+  * is no object.
+  */
+ void
+ VRT_l_beresp_saintmode(const struct sess *sp, double a)
+ {
+ 	struct trouble *new;
+ 	struct trouble *tr;
+ 	struct trouble *tr2;
+ 	struct worker *wrk;
+ 
+ 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+ 	wrk = sp->wrk;
+ 	if (!wrk->vbc)
+ 		return;
+ 	CHECK_OBJ_NOTNULL(wrk->vbc, VBC_MAGIC);
+ 	if (!wrk->vbc->backend)
+ 		return;
+ 	CHECK_OBJ_NOTNULL(wrk->vbc->backend, BACKEND_MAGIC);
+ 	if (!sp->objcore)
+ 		return;
+ 	CHECK_OBJ_NOTNULL(sp->objcore, OBJCORE_MAGIC);
+ 
+ 	/* Setting a negative holdoff period is a mistake. Detecting this
+ 	 * when compiling the VCL would be better.
+ 	 */
+ 	assert(a > 0);
+ 
+ 	ALLOC_OBJ(new, TROUBLE_MAGIC);
+ 	AN(new);
+ 	new->target = (uintptr_t)(sp->objcore->objhead);
+ 	new->timeout = sp->t_req + a;
+ 
+ 	/* Insert the new item on the list before the first item with a
+ 	 * timeout at a later date (ie: sort by which entry will time out
+ 	 * from the list
+ 	 */
+ 	Lck_Lock(&wrk->vbc->backend->mtx);
+ 	VTAILQ_FOREACH_SAFE(tr, &wrk->vbc->backend->troublelist, list, tr2) {
+ 		if (tr->timeout < new->timeout) {
+ 			VTAILQ_INSERT_BEFORE(tr, new, list);
+ 			new = NULL;
+ 			break;
+ 		}
+ 	}
+ 
+ 	/* Insert the item at the end if the list is empty or all other
+ 	 * items have a longer timeout.
+ 	 */
+ 	if (new)
+ 		VTAILQ_INSERT_TAIL(&wrk->vbc->backend->troublelist, new, list);
+ 
+ 	Lck_Unlock(&wrk->vbc->backend->mtx);
+ }
+ 
+ /*--------------------------------------------------------------------*/
+ 
+ #define VBERESP(dir, type, onm, field)					\
+ void									\
+ VRT_l_##dir##_##onm(const struct sess *sp, type a)			\
+ {									\
+ 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);				\
+ 	sp->wrk->field = a;						\
+ }									\
+ 									\
+ type									\
+ VRT_r_##dir##_##onm(const struct sess *sp)				\
+ {									\
+ 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);				\
+ 	return (sp->wrk->field);					\
+ }
+ 
+ VBERESP(beresp, unsigned, do_esi, do_esi)
+ VBERESP(beresp, unsigned, do_gzip, do_gzip)
+ VBERESP(beresp, unsigned, do_gunzip, do_gunzip)
+ VBERESP(beresp, unsigned, do_stream, do_stream)
+ 
+ /*--------------------------------------------------------------------*/
+ 
+ const char * __match_proto__()
+ VRT_r_client_identity(struct sess *sp)
+ {
+ 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+ 	if (sp->client_identity != NULL)
+ 		return (sp->client_identity);
+ 	else
+ 		return (sp->addr);
+ }
+ 
+ void
+ VRT_l_client_identity(struct sess *sp, const char *str, ...)
+ {
+ 	va_list ap;
+ 	char *b;
+ 
+ 	va_start(ap, str);
+ 	b = VRT_String(sp->http->ws, NULL, str, ap);
+ 	va_end(ap);
+ 	sp->client_identity = b;
+ }
+ 
+ /*--------------------------------------------------------------------*/
+ 
+ #define BEREQ_TIMEOUT(which)					\
+ void __match_proto__()						\
+ VRT_l_bereq_##which(struct sess *sp, double num)		\
+ {								\
+ 								\
+ 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);			\
+ 	sp->wrk->which = (num > 0.0 ? num : 0.0);		\
+ }								\
+ 								\
+ double __match_proto__()					\
+ VRT_r_bereq_##which(struct sess *sp)				\
+ {								\
+ 								\
+ 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);			\
+ 	return(sp->wrk->which);					\
+ }
+ 
+ BEREQ_TIMEOUT(connect_timeout)
+ BEREQ_TIMEOUT(first_byte_timeout)
+ BEREQ_TIMEOUT(between_bytes_timeout)
+ 
+ /*--------------------------------------------------------------------*/
+ 
+ const char *
+ VRT_r_beresp_backend_name(const struct sess *sp)
+ {
+ 
+ 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+ 	CHECK_OBJ_NOTNULL(sp->wrk->vbc, VBC_MAGIC);
+ 	return(sp->wrk->vbc->backend->vcl_name);
+ }
+ 
+ struct sockaddr_storage *
+ VRT_r_beresp_backend_ip(const struct sess *sp)
+ {
+ 
+ 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+ 	CHECK_OBJ_NOTNULL(sp->wrk->vbc, VBC_MAGIC);
+ 	return(sp->wrk->vbc->addr);
+ }
+ 
+ int
+ VRT_r_beresp_backend_port(const struct sess *sp)
+ {
+ 
+ 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+ 	CHECK_OBJ_NOTNULL(sp->wrk->vbc, VBC_MAGIC);
+ 	return (VTCP_port(sp->wrk->vbc->addr));
+ }
+ 
+ const char * __match_proto__()
+ VRT_r_beresp_storage(struct sess *sp)
+ {
+ 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+ 	if (sp->wrk->storage_hint != NULL)
+ 		return (sp->wrk->storage_hint);
+ 	else
+ 		return (NULL);
+ }
+ 
+ void __match_proto__()
+ VRT_l_beresp_storage(struct sess *sp, const char *str, ...)
+ {
+ 	va_list ap;
+ 	char *b;
+ 
+ 	va_start(ap, str);
+ 	b = VRT_String(sp->wrk->ws, NULL, str, ap);
+ 	va_end(ap);
+ 	sp->wrk->storage_hint = b;
+ }
+ 
+ /*--------------------------------------------------------------------*/
+ 
+ void
+ VRT_l_req_backend(struct sess *sp, struct director *be)
+ {
+ 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+ 	sp->director = be;
+ }
+ 
+ struct director * __match_proto__()
+ VRT_r_req_backend(struct sess *sp)
+ {
+ 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+ 	return (sp->director);
+ }
+ 
+ /*--------------------------------------------------------------------*/
+ 
+ void
+ VRT_l_req_esi(struct sess *sp, unsigned process_esi)
+ {
+ 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+ 	/*
+ 	 * Only allow you to turn of esi in the main request
+ 	 * else everything gets confused
+ 	 */
+ 	if(sp->esi_level == 0)
+ 		sp->disable_esi = !process_esi;
+ }
+ 
+ unsigned __match_proto__()
+ VRT_r_req_esi(struct sess *sp)
+ {
+ 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+ 	return (!sp->disable_esi);
+ }
+ 
+ int
+ VRT_r_req_esi_level(const struct sess *sp)
+ {
+ 
+ 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+ 	return(sp->esi_level);
+ }
+ 
+ /*--------------------------------------------------------------------*/
+ 
+ unsigned __match_proto__()
+ VRT_r_req_can_gzip(struct sess *sp)
+ {
+ 
+ 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+ 	return (RFC2616_Req_Gzip(sp));
+ }
+ 
+ 
+ /*--------------------------------------------------------------------*/
+ 
+ int
+ VRT_r_req_restarts(const struct sess *sp)
+ {
+ 
+ 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+ 	return (sp->restarts);
+ }
+ 
+ /*--------------------------------------------------------------------
+  * NB: TTL is relative to when object was created, whereas grace and
+  * keep are relative to ttl.
+  */
+ 
 -#define VRT_DO_EXP(which, exp, fld, offset, extra)		\
 -								\
 -void __match_proto__()						\
 -VRT_l_##which##_##fld(struct sess *sp, double a)		\
 -{								\
 -								\
 -	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);			\
 -	if (a > 0.)						\
 -		a += offset;					\
 -	EXP_Set_##fld(&exp, a);					\
 -	extra;							\
 -}								\
 -								\
 -double __match_proto__()					\
 -VRT_r_##which##_##fld(struct sess *sp)				\
 -{								\
 -								\
 -	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);			\
 -	return(EXP_Get_##fld(&exp) - offset);			\
 -}
++#define VRT_DO_EXP_l(which, cont, fld, offset, extra)		    \
++void __match_proto__()						    \
++VRT_l_##which##_##fld(struct sess *sp, double a)		    \
++{								    \
++								    \
++	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);			    \
++	if (a > 0.)						    \
++		a += offset;					    \
++	EXP_Set_##fld(&cont->exp, a);				    \
++	extra;							    \
++}
++
++#define VRT_DO_EXP_r(which, cont, fld, offset, nullable)	    \
++double __match_proto__()					    \
++VRT_r_##which##_##fld(struct sess *sp)				    \
++{								    \
++								    \
++	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);			    \
++	if (nullable && cont == NULL) {				    \
++	    ILLEGAL_R(sp, #which, #fld);			    \
++	    return (-1);					    \
++	}							    \
++	return(EXP_Get_##fld(&cont->exp) - offset);		    \
++}
++
++#define VRT_DO_EXP(which, cont, fld, offset, nullable, extra)	    \
++VRT_DO_EXP_l(which, cont, fld, offset, extra)  		            \
++VRT_DO_EXP_r(which, cont, fld, offset, nullable)
+ 
+ static void
+ vrt_wsp_exp(const struct sess *sp, unsigned xid, const struct exp *e)
+ {
+ 	WSP(sp, SLT_TTL, "%u VCL %.0f %.0f %.0f %.0f %.0f",
+ 	    xid, e->ttl - (sp->t_req - e->entered), e->grace, e->keep,
+ 	    sp->t_req, e->age + (sp->t_req - e->entered));
+ }
+ 
 -VRT_DO_EXP(req, sp->exp, ttl, 0, )
 -VRT_DO_EXP(req, sp->exp, grace, 0, )
 -VRT_DO_EXP(req, sp->exp, keep, 0, )
++VRT_DO_EXP(req, sp, ttl, 0, 0, )
++VRT_DO_EXP(req, sp, grace, 0, 0, )
++VRT_DO_EXP(req, sp, keep, 0, 0, )
+ 
 -VRT_DO_EXP(obj, sp->obj->exp, grace, 0,
++VRT_DO_EXP(obj, sp->obj, grace, 0, 0,
+    EXP_Rearm(sp->obj);
+    vrt_wsp_exp(sp, sp->obj->xid, &sp->obj->exp);)
 -VRT_DO_EXP(obj, sp->obj->exp, ttl, (sp->t_req - sp->obj->exp.entered),
++VRT_DO_EXP(obj, sp->obj, ttl, (sp->t_req - sp->obj->exp.entered), 0,
+    EXP_Rearm(sp->obj);
+    vrt_wsp_exp(sp, sp->obj->xid, &sp->obj->exp);)
 -VRT_DO_EXP(obj, sp->obj->exp, keep, 0,
++VRT_DO_EXP(obj, sp->obj, keep, 0, 0,
+    EXP_Rearm(sp->obj);
+    vrt_wsp_exp(sp, sp->obj->xid, &sp->obj->exp);)
+ 
 -VRT_DO_EXP(beresp, sp->wrk->exp, grace, 0,
++VRT_DO_EXP(beresp, sp->wrk, grace, 0, 0,
+    vrt_wsp_exp(sp, sp->xid, &sp->wrk->exp);)
 -VRT_DO_EXP(beresp, sp->wrk->exp, ttl, 0,
++VRT_DO_EXP(beresp, sp->wrk, ttl, 0, 0,
+    vrt_wsp_exp(sp, sp->xid, &sp->wrk->exp);)
 -VRT_DO_EXP(beresp, sp->wrk->exp, keep, 0,
++VRT_DO_EXP(beresp, sp->wrk, keep, 0, 0,
+    vrt_wsp_exp(sp, sp->xid, &sp->wrk->exp);)
++    
++VRT_DO_EXP_r(stale_obj, sp->stale_obj, grace, 0, 1)
++VRT_DO_EXP_r(stale_obj, sp->stale_obj, ttl, 0, 1)
++VRT_DO_EXP_r(stale_obj, sp->stale_obj, keep, 0, 1)
+ 
+ /*--------------------------------------------------------------------
+  * req.xid
+  */
+ 
+ const char * __match_proto__()
+ VRT_r_req_xid(struct sess *sp)
+ {
+ 	char *p;
+ 	int size;
+ 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+ 
+ 	size = snprintf(NULL, 0, "%u", sp->xid) + 1;
+ 	AN(p = WS_Alloc(sp->http->ws, size));
+ 	assert(snprintf(p, size, "%u", sp->xid) < size);
+ 	return (p);
+ }
+ 
+ /*--------------------------------------------------------------------*/
+ 
+ #define REQ_BOOL(which)						\
+ void __match_proto__()						\
+ VRT_l_req_##which(struct sess *sp, unsigned val)		\
+ {								\
+ 								\
+ 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);			\
+ 	sp->which = val ? 1 : 0;				\
+ }								\
+ 								\
+ unsigned __match_proto__()					\
+ VRT_r_req_##which(struct sess *sp)				\
+ {								\
+ 								\
+ 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);			\
+ 	return(sp->which);					\
+ }
+ 
+ REQ_BOOL(hash_ignore_busy)
+ REQ_BOOL(hash_always_miss)
+ 
+ /*--------------------------------------------------------------------*/
+ 
+ struct sockaddr_storage *
+ VRT_r_client_ip(struct sess *sp)
+ {
+ 
+ 	return (&sp->sockaddr);
+ }
+ 
+ struct sockaddr_storage *
+ VRT_r_server_ip(struct sess *sp)
+ {
+ 	int i;
+ 
+ 	if (sp->mysockaddr.ss_family == AF_UNSPEC) {
+ 		i = getsockname(sp->fd,
+ 		    (void*)&sp->mysockaddr, &sp->mysockaddrlen);
+ 		assert(VTCP_Check(i));
+ 	}
+ 
+ 	return (&sp->mysockaddr);
+ }
+ 
+ const char*
+ VRT_r_server_identity(struct sess *sp)
+ {
+ 	(void)sp;
+ 
+ 	if (heritage.identity[0] != '\0')
+ 		return (heritage.identity);
+ 	else
+ 		return (heritage.name);
+ }
+ 
+ 
+ const char*
+ VRT_r_server_hostname(struct sess *sp)
+ {
+ 	(void)sp;
+ 
+ 	if (vrt_hostname[0] == '\0')
+ 		AZ(gethostname(vrt_hostname, sizeof(vrt_hostname)));
+ 
+ 	return (vrt_hostname);
+ }
+ 
+ /*--------------------------------------------------------------------
+  * XXX: This is pessimistically silly
+  */
+ 
+ int
+ VRT_r_server_port(struct sess *sp)
+ {
+ 	int i;
+ 
+ 	if (sp->mysockaddr.ss_family == AF_UNSPEC) {
+ 		i = getsockname(sp->fd,
+ 		    (void*)&sp->mysockaddr, &sp->mysockaddrlen);
+ 		assert(VTCP_Check(i));
+ 	}
+ 	return (VTCP_port(&sp->mysockaddr));
+ }
+ 
+ /*--------------------------------------------------------------------*/
+ 
++/* XXX: uplex/GS: a nice macro would eliminate the repetition here ... */
++
+ int
+ VRT_r_obj_hits(const struct sess *sp)
+ {
+ 
+ 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+ 	CHECK_OBJ_NOTNULL(sp->obj, OBJECT_MAGIC);	/* XXX */
+ 	return (sp->obj->hits);
+ }
+ 
++int
++VRT_r_stale_obj_hits(const struct sess *sp)
++{
++
++	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
++        if (sp->stale_obj == NULL) {
++            ILLEGAL_R(sp, "stale_obj", "hits");
++            return (0);
++        }
++	CHECK_OBJ(sp->stale_obj, OBJECT_MAGIC);	/* XXX */
++	return (sp->stale_obj->hits);
++}
++
+ double
+ VRT_r_obj_lastuse(const struct sess *sp)
+ {
+ 
+ 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+ 	CHECK_OBJ_NOTNULL(sp->obj, OBJECT_MAGIC);	/* XXX */
+ 	return (VTIM_real() - sp->obj->last_use);
+ }
+ 
++double
++VRT_r_stale_obj_lastuse(const struct sess *sp)
++{
++
++	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
++        if (sp->stale_obj == NULL) {
++            ILLEGAL_R(sp, "stale_obj", "lastuse");
++            return (0);
++        }
++	CHECK_OBJ(sp->stale_obj, OBJECT_MAGIC);	/* XXX */
++	return (VTIM_real() - sp->stale_obj->last_use);
++}
++
+ unsigned
+ VRT_r_req_backend_healthy(const struct sess *sp)
+ {
+ 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+ 	CHECK_OBJ_NOTNULL(sp->director, DIRECTOR_MAGIC);
+ 	return (VDI_Healthy(sp->director, sp));
+ }
+ 
++unsigned
++VRT_r_stale_obj(const struct sess *sp)
++{
++        CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
++        return (sp->stale_obj != NULL);
++}
diff --cc bin/varnishd/mgt/mgt_param.c
index 0000000,e4be5d8..991793a
mode 000000,100644..100644
--- a/bin/varnishd/mgt/mgt_param.c
+++ b/bin/varnishd/mgt/mgt_param.c
@@@ -1,0 -1,1368 +1,1368 @@@
+ /*-
+  * Copyright (c) 2006 Verdens Gang AS
+  * Copyright (c) 2006-2011 Varnish Software AS
+  * All rights reserved.
+  *
+  * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
+  *
+  * Redistribution and use in source and binary forms, with or without
+  * modification, are permitted provided that the following conditions
+  * are met:
+  * 1. Redistributions of source code must retain the above copyright
+  *    notice, this list of conditions and the following disclaimer.
+  * 2. Redistributions in binary form must reproduce the above copyright
+  *    notice, this list of conditions and the following disclaimer in the
+  *    documentation and/or other materials provided with the distribution.
+  *
+  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+  * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+  * SUCH DAMAGE.
+  */
+ 
+ #include "config.h"
+ 
+ #include <grp.h>
+ #include <limits.h>
+ #include <math.h>
+ #include <pwd.h>
+ #include <stdio.h>
+ #include <stdlib.h>
+ #include <string.h>
+ #include <unistd.h>
+ 
+ #include "mgt/mgt.h"
+ #include "common/heritage.h"
+ #include "common/params.h"
+ 
+ #include "mgt/mgt_param.h"
+ #include "waiter/cache_waiter.h"
+ #include "vav.h"
+ #include "vcli.h"
+ #include "vcli_common.h"
+ #include "vcli_priv.h"
+ #include "vnum.h"
+ #include "vss.h"
+ 
+ #include "mgt_cli.h"
+ 
+ #define MAGIC_INIT_STRING	"\001"
+ struct params mgt_param;
+ static int nparspec;
+ static struct parspec const ** parspec;
+ static int margin;
+ 
+ /*--------------------------------------------------------------------*/
+ 
+ static const struct parspec *
+ mcf_findpar(const char *name)
+ {
+ 	int i;
+ 
+ 	for (i = 0; i < nparspec; i++)
+ 		if (!strcmp(parspec[i]->name, name))
+ 			return (parspec[i]);
+ 	return (NULL);
+ }
+ 
+ /*--------------------------------------------------------------------*/
+ 
+ static void
+ tweak_generic_timeout(struct cli *cli, volatile unsigned *dst, const char *arg)
+ {
+ 	unsigned u;
+ 
+ 	if (arg != NULL) {
+ 		u = strtoul(arg, NULL, 0);
+ 		if (u == 0) {
+ 			VCLI_Out(cli, "Timeout must be greater than zero\n");
+ 			VCLI_SetResult(cli, CLIS_PARAM);
+ 			return;
+ 		}
+ 		*dst = u;
+ 	} else
+ 		VCLI_Out(cli, "%u", *dst);
+ }
+ 
+ /*--------------------------------------------------------------------*/
+ 
+ void
+ tweak_timeout(struct cli *cli, const struct parspec *par, const char *arg)
+ {
+ 	volatile unsigned *dest;
+ 
+ 	dest = par->priv;
+ 	tweak_generic_timeout(cli, dest, arg);
+ }
+ 
+ static void
+ tweak_timeout_double(struct cli *cli, const struct parspec *par,
+     const char *arg)
+ {
+ 	volatile double *dest;
+ 	double u;
+ 
+ 	dest = par->priv;
+ 	if (arg != NULL) {
+ 		u = strtod(arg, NULL);
+ 		if (u < par->min) {
+ 			VCLI_Out(cli,
+ 			    "Timeout must be greater or equal to %.g\n",
+ 				 par->min);
+ 			VCLI_SetResult(cli, CLIS_PARAM);
+ 			return;
+ 		}
+ 		if (u > par->max) {
+ 			VCLI_Out(cli,
+ 			    "Timeout must be less than or equal to %.g\n",
+ 				 par->max);
+ 			VCLI_SetResult(cli, CLIS_PARAM);
+ 			return;
+ 		}
+ 		*dest = u;
+ 	} else
+ 		VCLI_Out(cli, "%.6f", *dest);
+ }
+ 
+ /*--------------------------------------------------------------------*/
+ 
+ static void
+ tweak_generic_double(struct cli *cli, const struct parspec *par,
+     const char *arg)
+ {
+ 	volatile double *dest;
+ 	double u;
+ 
+ 	dest = par->priv;
+ 	if (arg != NULL) {
+ 		u = strtod(arg, NULL);
+ 		if (u < par->min) {
+ 			VCLI_Out(cli,
+ 			    "Must be greater or equal to %.g\n",
+ 				 par->min);
+ 			VCLI_SetResult(cli, CLIS_PARAM);
+ 			return;
+ 		}
+ 		if (u > par->max) {
+ 			VCLI_Out(cli,
+ 			    "Must be less than or equal to %.g\n",
+ 				 par->max);
+ 			VCLI_SetResult(cli, CLIS_PARAM);
+ 			return;
+ 		}
+ 		*dest = u;
+ 	} else
+ 		VCLI_Out(cli, "%f", *dest);
+ }
+ 
+ /*--------------------------------------------------------------------*/
+ 
+ static void
+ tweak_generic_bool(struct cli *cli, volatile unsigned *dest, const char *arg)
+ {
+ 	if (arg != NULL) {
+ 		if (!strcasecmp(arg, "off"))
+ 			*dest = 0;
+ 		else if (!strcasecmp(arg, "disable"))
+ 			*dest = 0;
+ 		else if (!strcasecmp(arg, "no"))
+ 			*dest = 0;
+ 		else if (!strcasecmp(arg, "false"))
+ 			*dest = 0;
+ 		else if (!strcasecmp(arg, "on"))
+ 			*dest = 1;
+ 		else if (!strcasecmp(arg, "enable"))
+ 			*dest = 1;
+ 		else if (!strcasecmp(arg, "yes"))
+ 			*dest = 1;
+ 		else if (!strcasecmp(arg, "true"))
+ 			*dest = 1;
+ 		else {
+ 			VCLI_Out(cli, "use \"on\" or \"off\"\n");
+ 			VCLI_SetResult(cli, CLIS_PARAM);
+ 			return;
+ 		}
+ 	} else
+ 		VCLI_Out(cli, *dest ? "on" : "off");
+ }
+ 
+ /*--------------------------------------------------------------------*/
+ 
+ static void
+ tweak_bool(struct cli *cli, const struct parspec *par, const char *arg)
+ {
+ 	volatile unsigned *dest;
+ 
+ 	dest = par->priv;
+ 	tweak_generic_bool(cli, dest, arg);
+ }
+ 
+ /*--------------------------------------------------------------------*/
+ 
+ void
+ tweak_generic_uint(struct cli *cli, volatile unsigned *dest, const char *arg,
+     unsigned min, unsigned max)
+ {
+ 	unsigned u;
+ 
+ 	if (arg != NULL) {
+ 		if (!strcasecmp(arg, "unlimited"))
+ 			u = UINT_MAX;
+ 		else
+ 			u = strtoul(arg, NULL, 0);
+ 		if (u < min) {
+ 			VCLI_Out(cli, "Must be at least %u\n", min);
+ 			VCLI_SetResult(cli, CLIS_PARAM);
+ 			return;
+ 		}
+ 		if (u > max) {
+ 			VCLI_Out(cli, "Must be no more than %u\n", max);
+ 			VCLI_SetResult(cli, CLIS_PARAM);
+ 			return;
+ 		}
+ 		*dest = u;
+ 	} else if (*dest == UINT_MAX) {
+ 		VCLI_Out(cli, "unlimited", *dest);
+ 	} else {
+ 		VCLI_Out(cli, "%u", *dest);
+ 	}
+ }
+ 
+ /*--------------------------------------------------------------------*/
+ 
+ void
+ tweak_uint(struct cli *cli, const struct parspec *par, const char *arg)
+ {
+ 	volatile unsigned *dest;
+ 
+ 	dest = par->priv;
+ 	tweak_generic_uint(cli, dest, arg, (uint)par->min, (uint)par->max);
+ }
+ 
+ /*--------------------------------------------------------------------*/
+ 
+ static void
+ fmt_bytes(struct cli *cli, ssize_t t)
+ {
+ 	const char *p;
+ 
+ 	if (t & 0xff) {
+ 		VCLI_Out(cli, "%zub", t);
+ 		return;
+ 	}
+ 	for (p = "kMGTPEZY"; *p; p++) {
+ 		if (t & 0x300) {
+ 			VCLI_Out(cli, "%.2f%c", t / 1024.0, *p);
+ 			return;
+ 		}
+ 		t /= 1024;
+ 		if (t & 0x0ff) {
+ 			VCLI_Out(cli, "%zu%c", t, *p);
+ 			return;
+ 		}
+ 	}
+ 	VCLI_Out(cli, "(bogus number)");
+ }
+ 
+ static void
+ tweak_generic_bytes(struct cli *cli, volatile ssize_t *dest, const char *arg,
+     double min, double max)
+ {
+ 	uintmax_t r;
+ 	const char *p;
+ 
+ 	if (arg != NULL) {
+ 		p = VNUM_2bytes(arg, &r, 0);
+ 		if (p != NULL) {
+ 			VCLI_Out(cli, "Could not convert to bytes.\n");
+ 			VCLI_Out(cli, "%s\n", p);
+ 			VCLI_Out(cli,
+ 			    "  Try something like '80k' or '120M'\n");
+ 			VCLI_SetResult(cli, CLIS_PARAM);
+ 			return;
+ 		}
+ 		if ((uintmax_t)((ssize_t)r) != r || r > max) {
+ 			VCLI_Out(cli, "Must be no more than ");
+ 			fmt_bytes(cli, (ssize_t)max);
+ 			VCLI_Out(cli, "\n");
+ 			VCLI_SetResult(cli, CLIS_PARAM);
+ 			return;
+ 		}
+ 		if (r < min) {
+ 			VCLI_Out(cli, "Must be at least ");
+ 			fmt_bytes(cli, (ssize_t)min);
+ 			VCLI_Out(cli, "\n");
+ 			VCLI_SetResult(cli, CLIS_PARAM);
+ 			return;
+ 		}
+ 		*dest = r;
+ 	} else {
+ 		fmt_bytes(cli, *dest);
+ 	}
+ }
+ 
+ /*--------------------------------------------------------------------*/
+ 
+ static void
+ tweak_bytes(struct cli *cli, const struct parspec *par, const char *arg)
+ {
+ 	volatile ssize_t *dest;
+ 
+ 	assert(par->min >= 0);
+ 	dest = par->priv;
+ 	tweak_generic_bytes(cli, dest, arg, par->min, par->max);
+ }
+ 
+ 
+ /*--------------------------------------------------------------------*/
+ 
+ static void
+ tweak_bytes_u(struct cli *cli, const struct parspec *par, const char *arg)
+ {
+ 	volatile unsigned *d1;
+ 	volatile ssize_t dest;
+ 
+ 	assert(par->max <= UINT_MAX);
+ 	assert(par->min >= 0);
+ 	d1 = par->priv;
+ 	dest = *d1;
+ 	tweak_generic_bytes(cli, &dest, arg, par->min, par->max);
+ 	*d1 = dest;
+ }
+ 
+ /*--------------------------------------------------------------------
+  * XXX: slightly magic.  We want to initialize to "nobody" (XXX: shouldn't
+  * XXX: that be something autocrap found for us ?) but we don't want to
+  * XXX: fail initialization if that user doesn't exists, even though we
+  * XXX: do want to fail it, in subsequent sets.
+  * XXX: The magic init string is a hack for this.
+  */
+ 
+ static void
+ tweak_user(struct cli *cli, const struct parspec *par, const char *arg)
+ {
+ 	struct passwd *pw;
+ 	struct group *gr;
+ 
+ 	(void)par;
+ 	if (arg != NULL) {
+ 		if (!strcmp(arg, MAGIC_INIT_STRING)) {
+ 			pw = getpwnam("nobody");
+ 			if (pw == NULL) {
+ 				mgt_param.uid = getuid();
+ 				return;
+ 			}
+ 		} else
+ 			pw = getpwnam(arg);
+ 		if (pw == NULL) {
+ 			VCLI_Out(cli, "Unknown user");
+ 			VCLI_SetResult(cli, CLIS_PARAM);
+ 			return;
+ 		}
+ 		REPLACE(mgt_param.user, pw->pw_name);
+ 		mgt_param.uid = pw->pw_uid;
+ 		mgt_param.gid = pw->pw_gid;
+ 
+ 		/* set group to user's primary group */
+ 		if ((gr = getgrgid(pw->pw_gid)) != NULL &&
+ 		    (gr = getgrnam(gr->gr_name)) != NULL &&
+ 		    gr->gr_gid == pw->pw_gid)
+ 			REPLACE(mgt_param.group, gr->gr_name);
+ 	} else if (mgt_param.user) {
+ 		VCLI_Out(cli, "%s (%d)", mgt_param.user, (int)mgt_param.uid);
+ 	} else {
+ 		VCLI_Out(cli, "%d", (int)mgt_param.uid);
+ 	}
+ }
+ 
+ /*--------------------------------------------------------------------
+  * XXX: see comment for tweak_user, same thing here.
+  */
+ 
+ static void
+ tweak_group(struct cli *cli, const struct parspec *par, const char *arg)
+ {
+ 	struct group *gr;
+ 
+ 	(void)par;
+ 	if (arg != NULL) {
+ 		if (!strcmp(arg, MAGIC_INIT_STRING)) {
+ 			gr = getgrnam("nogroup");
+ 			if (gr == NULL) {
+ 				/* Only replace if tweak_user didn't */
+ 				if (mgt_param.gid == 0)
+ 					mgt_param.gid = getgid();
+ 				return;
+ 			}
+ 		} else
+ 			gr = getgrnam(arg);
+ 		if (gr == NULL) {
+ 			VCLI_Out(cli, "Unknown group");
+ 			VCLI_SetResult(cli, CLIS_PARAM);
+ 			return;
+ 		}
+ 		REPLACE(mgt_param.group, gr->gr_name);
+ 		mgt_param.gid = gr->gr_gid;
+ 	} else if (mgt_param.group) {
+ 		VCLI_Out(cli, "%s (%d)", mgt_param.group, (int)mgt_param.gid);
+ 	} else {
+ 		VCLI_Out(cli, "%d", (int)mgt_param.gid);
+ 	}
+ }
+ 
+ /*--------------------------------------------------------------------*/
+ 
+ static void
+ clean_listen_sock_head(struct listen_sock_head *lsh)
+ {
+ 	struct listen_sock *ls, *ls2;
+ 
+ 	VTAILQ_FOREACH_SAFE(ls, lsh, list, ls2) {
+ 		CHECK_OBJ_NOTNULL(ls, LISTEN_SOCK_MAGIC);
+ 		VTAILQ_REMOVE(lsh, ls, list);
+ 		free(ls->name);
+ 		free(ls->addr);
+ 		FREE_OBJ(ls);
+ 	}
+ }
+ 
+ static void
+ tweak_listen_address(struct cli *cli, const struct parspec *par,
+     const char *arg)
+ {
+ 	char **av;
+ 	int i;
+ 	struct listen_sock		*ls;
+ 	struct listen_sock_head		lsh;
+ 
+ 	(void)par;
+ 	if (arg == NULL) {
+ 		VCLI_Quote(cli, mgt_param.listen_address);
+ 		return;
+ 	}
+ 
+ 	av = VAV_Parse(arg, NULL, ARGV_COMMA);
+ 	if (av == NULL) {
+ 		VCLI_Out(cli, "Parse error: out of memory");
+ 		VCLI_SetResult(cli, CLIS_PARAM);
+ 		return;
+ 	}
+ 	if (av[0] != NULL) {
+ 		VCLI_Out(cli, "Parse error: %s", av[0]);
+ 		VCLI_SetResult(cli, CLIS_PARAM);
+ 		VAV_Free(av);
+ 		return;
+ 	}
+ 	if (av[1] == NULL) {
+ 		VCLI_Out(cli, "Empty listen address");
+ 		VCLI_SetResult(cli, CLIS_PARAM);
+ 		VAV_Free(av);
+ 		return;
+ 	}
+ 	VTAILQ_INIT(&lsh);
+ 	for (i = 1; av[i] != NULL; i++) {
+ 		struct vss_addr **ta;
+ 		int j, n;
+ 
+ 		n = VSS_resolve(av[i], "http", &ta);
+ 		if (n == 0) {
+ 			VCLI_Out(cli, "Invalid listen address ");
+ 			VCLI_Quote(cli, av[i]);
+ 			VCLI_SetResult(cli, CLIS_PARAM);
+ 			break;
+ 		}
+ 		for (j = 0; j < n; ++j) {
+ 			ALLOC_OBJ(ls, LISTEN_SOCK_MAGIC);
+ 			AN(ls);
+ 			ls->sock = -1;
+ 			ls->addr = ta[j];
+ 			ls->name = strdup(av[i]);
+ 			AN(ls->name);
+ 			VTAILQ_INSERT_TAIL(&lsh, ls, list);
+ 		}
+ 		free(ta);
+ 	}
+ 	VAV_Free(av);
+ 	if (cli != NULL && cli->result != CLIS_OK) {
+ 		clean_listen_sock_head(&lsh);
+ 		return;
+ 	}
+ 
+ 	REPLACE(mgt_param.listen_address, arg);
+ 
+ 	clean_listen_sock_head(&heritage.socks);
+ 	heritage.nsocks = 0;
+ 
+ 	while (!VTAILQ_EMPTY(&lsh)) {
+ 		ls = VTAILQ_FIRST(&lsh);
+ 		VTAILQ_REMOVE(&lsh, ls, list);
+ 		CHECK_OBJ_NOTNULL(ls, LISTEN_SOCK_MAGIC);
+ 		VTAILQ_INSERT_TAIL(&heritage.socks, ls, list);
+ 		heritage.nsocks++;
+ 	}
+ }
+ 
+ /*--------------------------------------------------------------------*/
+ 
+ static void
+ tweak_string(struct cli *cli, const struct parspec *par, const char *arg)
+ {
+ 	char **p = TRUST_ME(par->priv);
+ 
+ 	AN(p);
+ 	/* XXX should have tweak_generic_string */
+ 	if (arg == NULL) {
+ 		VCLI_Quote(cli, *p);
+ 	} else {
+ 		REPLACE(*p, arg);
+ 	}
+ }
+ 
+ /*--------------------------------------------------------------------*/
+ 
+ static void
+ tweak_waiter(struct cli *cli, const struct parspec *par, const char *arg)
+ {
+ 
+ 	/* XXX should have tweak_generic_string */
+ 	(void)par;
+ 	WAIT_tweak_waiter(cli, arg);
+ }
+ 
+ /*--------------------------------------------------------------------*/
+ 
+ static void
+ tweak_diag_bitmap(struct cli *cli, const struct parspec *par, const char *arg)
+ {
+ 	unsigned u;
+ 
+ 	(void)par;
+ 	if (arg != NULL) {
+ 		u = strtoul(arg, NULL, 0);
+ 		mgt_param.diag_bitmap = u;
+ 	} else {
+ 		VCLI_Out(cli, "0x%x", mgt_param.diag_bitmap);
+ 	}
+ }
+ 
+ /*--------------------------------------------------------------------*/
+ 
+ /*
+  * Make sure to end all lines with either a space or newline of the
+  * formatting will go haywire.
+  */
+ 
+ #define DELAYED_EFFECT_TEXT \
+ 	"\nNB: This parameter may take quite some time to take (full) effect."
+ 
+ #define MUST_RESTART_TEXT \
+ 	"\nNB: This parameter will not take any effect until the " \
+ 	"child process has been restarted."
+ 
+ #define MUST_RELOAD_TEXT \
+ 	"\nNB: This parameter will not take any effect until the " \
+ 	"VCL programs have been reloaded."
+ 
+ #define EXPERIMENTAL_TEXT \
+ 	"\nNB: We do not know yet if it is a good idea to change " \
+ 	"this parameter, or if the default value is even sensible.  " \
+ 	"Caution is advised, and feedback is most welcome."
+ 
+ #define WIZARD_TEXT \
+ 	"\nNB: Do not change this parameter, unless a developer tell " \
+ 	"you to do so."
+ 
+ /*
+  * Remember to update varnishd.1 whenever you add / remove a parameter or
+  * change its default value.
+  * XXX: we should generate the relevant section of varnishd.1 from here.
+  */
+ static const struct parspec input_parspec[] = {
+ 	{ "user", tweak_user, NULL, 0, 0,
+ 		"The unprivileged user to run as.  Setting this will "
+ 		"also set \"group\" to the specified user's primary group.",
+ 		MUST_RESTART,
+ 		MAGIC_INIT_STRING },
+ 	{ "group", tweak_group, NULL, 0, 0,
+ 		"The unprivileged group to run as.",
+ 		MUST_RESTART,
+ 		MAGIC_INIT_STRING },
+ 	{ "default_ttl", tweak_timeout_double, &mgt_param.default_ttl,
+ 		0, UINT_MAX,
+ 		"The TTL assigned to objects if neither the backend nor "
+ 		"the VCL code assigns one.\n"
+ 		"Objects already cached will not be affected by changes "
+ 		"made until they are fetched from the backend again.\n"
+ 		"To force an immediate effect at the expense of a total "
+ 		"flush of the cache use \"ban.url .\"",
+ 		0,
+ 		"120", "seconds" },
+ 	{ "sess_workspace",
+ 		tweak_bytes_u, &mgt_param.sess_workspace, 1024, UINT_MAX,
+ 		"Bytes of HTTP protocol workspace allocated for sessions. "
+ 		"This space must be big enough for the entire HTTP protocol "
+ 		"header and any edits done to it in the VCL code.\n"
+ 		"Minimum is 1024 bytes.",
+ 		DELAYED_EFFECT,
+ 		"64k", "bytes" },
+ 	{ "http_req_hdr_len",
+ 		tweak_bytes_u, &mgt_param.http_req_hdr_len,
+ 		40, UINT_MAX,
+ 		"Maximum length of any HTTP client request header we will "
+ 		"allow.  The limit is inclusive its continuation lines.\n",
+ 		0,
+ 		"8k", "bytes" },
+ 	{ "http_req_size",
+ 		tweak_bytes_u, &mgt_param.http_req_size,
+ 		256, UINT_MAX,
+ 		"Maximum number of bytes of HTTP client request we will deal "
+ 		"with.  This is a limit on all bytes up to the double blank "
+ 		"line which ends the HTTP request.\n"
+ 		"The memory for the request is allocated from the session "
+ 		"workspace (param: sess_workspace) and this parameter limits "
+ 		"how much of that the request is allowed to take up.",
+ 		0,
+ 		"32k", "bytes" },
+ 	{ "http_resp_hdr_len",
+ 		tweak_bytes_u, &mgt_param.http_resp_hdr_len,
+ 		40, UINT_MAX,
+ 		"Maximum length of any HTTP backend response header we will "
+ 		"allow.  The limit is inclusive its continuation lines.\n",
+ 		0,
+ 		"8k", "bytes" },
+ 	{ "http_resp_size",
+ 		tweak_bytes_u, &mgt_param.http_resp_size,
+ 		256, UINT_MAX,
+ 		"Maximum number of bytes of HTTP backend resonse we will deal "
+ 		"with.  This is a limit on all bytes up to the double blank "
+ 		"line which ends the HTTP request.\n"
+ 		"The memory for the request is allocated from the worker "
+ 		"workspace (param: sess_workspace) and this parameter limits "
+ 		"how much of that the request is allowed to take up.",
+ 		0,
+ 		"32k", "bytes" },
+ 	{ "http_max_hdr", tweak_uint, &mgt_param.http_max_hdr, 32, 65535,
+ 		"Maximum number of HTTP headers we will deal with in "
+ 		"client request or backend reponses.  "
+ 		"Note that the first line occupies five header fields.\n"
+ 		"This paramter does not influence storage consumption, "
+ 		"objects allocate exact space for the headers they store.\n",
+ 		0,
+ 		"64", "header lines" },
+ 	{ "shm_workspace",
+ 		tweak_bytes_u, &mgt_param.shm_workspace, 4096, UINT_MAX,
+ 		"Bytes of shmlog workspace allocated for worker threads. "
+ 		"If too big, it wastes some ram, if too small it causes "
+ 		"needless flushes of the SHM workspace.\n"
+ 		"These flushes show up in stats as "
+ 		"\"SHM flushes due to overflow\".\n"
+ 		"Minimum is 4096 bytes.",
+ 		DELAYED_EFFECT,
+ 		"8k", "bytes" },
+ 	{ "shm_reclen",
+ 		tweak_bytes_u, &mgt_param.shm_reclen, 16, 65535,
+ 		"Maximum number of bytes in SHM log record.\n"
+ 		"Maximum is 65535 bytes.",
+ 		0,
+ 		"255", "bytes" },
+ 	{ "default_grace", tweak_timeout_double, &mgt_param.default_grace,
+ 		0, UINT_MAX,
+ 		"Default grace period.  We will deliver an object "
+ 		"this long after it has expired, provided another thread "
+ 		"is attempting to get a new copy.\n"
+ 		"Objects already cached will not be affected by changes "
+ 		"made until they are fetched from the backend again.\n",
+ 		DELAYED_EFFECT,
+ 		"10", "seconds" },
+ 	{ "default_keep", tweak_timeout_double, &mgt_param.default_keep,
+ 		0, UINT_MAX,
 -		"Default keep period.  We will keep a useless object "
++		"Default keep period.  We will keep a stale object "
+ 		"around this long, making it available for conditional "
+ 		"backend fetches.  "
+ 		"That means that the object will be removed from the "
 -		"cache at the end of ttl+grace+keep.",
++		"cache at the end of ttl+max(grace,keep).",
+ 		DELAYED_EFFECT,
 -		"0", "seconds" },
++		"10", "seconds" },
+ 	{ "sess_timeout", tweak_timeout, &mgt_param.sess_timeout, 0, 0,
+ 		"Idle timeout for persistent sessions. "
+ 		"If a HTTP request has not been received in this many "
+ 		"seconds, the session is closed.",
+ 		0,
+ 		"5", "seconds" },
+ 	{ "expiry_sleep", tweak_timeout_double, &mgt_param.expiry_sleep, 0, 60,
+ 		"How long the expiry thread sleeps when there is nothing "
+ 		"for it to do.\n",
+ 		0,
+ 		"1", "seconds" },
+ 	{ "pipe_timeout", tweak_timeout, &mgt_param.pipe_timeout, 0, 0,
+ 		"Idle timeout for PIPE sessions. "
+ 		"If nothing have been received in either direction for "
+ 		"this many seconds, the session is closed.\n",
+ 		0,
+ 		"60", "seconds" },
+ 	{ "send_timeout", tweak_timeout, &mgt_param.send_timeout, 0, 0,
+ 		"Send timeout for client connections. "
+ 		"If the HTTP response hasn't been transmitted in this many\n"
+                 "seconds the session is closed. \n"
+ 		"See setsockopt(2) under SO_SNDTIMEO for more information.",
+ 		DELAYED_EFFECT,
+ 		"600", "seconds" },
+ 	{ "idle_send_timeout", tweak_timeout, &mgt_param.idle_send_timeout, 0, 0,
+ 		"Time to wait with no data sent. "
+ 		"If no data has been transmitted in this many\n"
+                 "seconds the session is closed. \n"
+ 		"See setsockopt(2) under SO_SNDTIMEO for more information.",
+ 		DELAYED_EFFECT,
+ 		"60", "seconds" },
+ 	{ "auto_restart", tweak_bool, &mgt_param.auto_restart, 0, 0,
+ 		"Restart child process automatically if it dies.\n",
+ 		0,
+ 		"on", "bool" },
+ 	{ "nuke_limit",
+ 		tweak_uint, &mgt_param.nuke_limit, 0, UINT_MAX,
+ 		"Maximum number of objects we attempt to nuke in order"
+ 		"to make space for a object body.",
+ 		EXPERIMENTAL,
+ 		"50", "allocations" },
+ 	{ "fetch_chunksize",
+ 		tweak_bytes_u,
+ 		    &mgt_param.fetch_chunksize, 4 * 1024, UINT_MAX,
+ 		"The default chunksize used by fetcher. "
+ 		"This should be bigger than the majority of objects with "
+ 		"short TTLs.\n"
+ 		"Internal limits in the storage_file module makes increases "
+ 		"above 128kb a dubious idea.",
+ 		EXPERIMENTAL,
+ 		"128k", "bytes" },
+ 	{ "fetch_maxchunksize",
+ 		tweak_bytes_u,
+ 		    &mgt_param.fetch_maxchunksize, 64 * 1024, UINT_MAX,
+ 		"The maximum chunksize we attempt to allocate from storage. "
+ 		"Making this too large may cause delays and storage "
+ 		"fragmentation.\n",
+ 		EXPERIMENTAL,
+ 		"256m", "bytes" },
+ #ifdef SENDFILE_WORKS
+ 	{ "sendfile_threshold",
+ 		tweak_bytes, &mgt_param.sendfile_threshold, 0, HUGE_VAL,
+ 		"The minimum size of objects transmitted with sendfile.",
+ 		EXPERIMENTAL,
+ 		"1E", "bytes" },
+ #endif /* SENDFILE_WORKS */
+ 	{ "vcl_trace", tweak_bool,  &mgt_param.vcl_trace, 0, 0,
+ 		"Trace VCL execution in the shmlog.\n"
+ 		"Enabling this will allow you to see the path each "
+ 		"request has taken through the VCL program.\n"
+ 		"This generates a lot of logrecords so it is off by "
+ 		"default.",
+ 		0,
+ 		"off", "bool" },
+ 	{ "listen_address", tweak_listen_address, NULL, 0, 0,
+ 		"Whitespace separated list of network endpoints where "
+ 		"Varnish will accept requests.\n"
+ 		"Possible formats: host, host:port, :port",
+ 		MUST_RESTART,
+ 		":80" },
+ 	{ "listen_depth", tweak_uint, &mgt_param.listen_depth, 0, UINT_MAX,
+ 		"Listen queue depth.",
+ 		MUST_RESTART,
+ 		"1024", "connections" },
+ 	{ "cli_buffer",
+ 		tweak_bytes_u, &mgt_param.cli_buffer, 4096, UINT_MAX,
+ 		"Size of buffer for CLI command input."
+ 		"\nYou may need to increase this if you have big VCL files "
+ 		"and use the vcl.inline CLI command.\n"
+ 		"NB: Must be specified with -p to have effect.\n",
+ 		0,
+ 		"8k", "bytes" },
+ 	{ "cli_limit",
+ 		tweak_bytes_u, &mgt_param.cli_limit, 128, 99999999,
+ 		"Maximum size of CLI response.  If the response exceeds"
+ 		" this limit, the reponse code will be 201 instead of"
+ 		" 200 and the last line will indicate the truncation.",
+ 		0,
+ 		"4k", "bytes" },
+ 	{ "cli_timeout", tweak_timeout, &mgt_param.cli_timeout, 0, 0,
+ 		"Timeout for the childs replies to CLI requests from "
+ 		"the mgt_param.",
+ 		0,
+ 		"10", "seconds" },
+ 	{ "ping_interval", tweak_uint, &mgt_param.ping_interval, 0, UINT_MAX,
+ 		"Interval between pings from parent to child.\n"
+ 		"Zero will disable pinging entirely, which makes "
+ 		"it possible to attach a debugger to the child.",
+ 		MUST_RESTART,
+ 		"3", "seconds" },
+ 	{ "lru_interval", tweak_timeout, &mgt_param.lru_timeout, 0, 0,
+ 		"Grace period before object moves on LRU list.\n"
+ 		"Objects are only moved to the front of the LRU "
+ 		"list if they have not been moved there already inside "
+ 		"this timeout period.  This reduces the amount of lock "
+ 		"operations necessary for LRU list access.",
+ 		EXPERIMENTAL,
+ 		"2", "seconds" },
+ 	{ "cc_command", tweak_string, &mgt_cc_cmd, 0, 0,
+ 		"Command used for compiling the C source code to a "
+ 		"dlopen(3) loadable object.  Any occurrence of %s in "
+ 		"the string will be replaced with the source file name, "
+ 		"and %o will be replaced with the output file name.",
+ 		MUST_RELOAD,
+ 		VCC_CC , NULL },
+ 	{ "max_restarts", tweak_uint, &mgt_param.max_restarts, 0, UINT_MAX,
+ 		"Upper limit on how many times a request can restart."
+ 		"\nBe aware that restarts are likely to cause a hit against "
+ 		"the backend, so don't increase thoughtlessly.\n",
+ 		0,
+ 		"4", "restarts" },
+ 	{ "esi_syntax",
+ 		tweak_uint, &mgt_param.esi_syntax, 0, UINT_MAX,
+ 		"Bitmap controlling ESI parsing code:\n"
+ 		"  0x00000001 - Don't check if it looks like XML\n"
+ 		"  0x00000002 - Ignore non-esi elements\n"
+ 		"  0x00000004 - Emit parsing debug records\n"
+ 		"  0x00000008 - Force-split parser input (debugging)\n"
+ 		"\n"
+ 		"Use 0x notation and do the bitor in your head :-)\n",
+ 		0,
+ 		"0", "bitmap" },
+ 	{ "max_esi_depth",
+ 		tweak_uint, &mgt_param.max_esi_depth, 0, UINT_MAX,
+ 		"Maximum depth of esi:include processing.\n",
+ 		0,
+ 		"5", "levels" },
+ 	{ "connect_timeout", tweak_timeout_double,
+ 		&mgt_param.connect_timeout,0, UINT_MAX,
+ 		"Default connection timeout for backend connections. "
+ 		"We only try to connect to the backend for this many "
+ 		"seconds before giving up. "
+ 		"VCL can override this default value for each backend and "
+ 		"backend request.",
+ 		0,
+ 		"0.7", "s" },
+ 	{ "first_byte_timeout", tweak_timeout_double,
+ 		&mgt_param.first_byte_timeout,0, UINT_MAX,
+ 		"Default timeout for receiving first byte from backend. "
+ 		"We only wait for this many seconds for the first "
+ 		"byte before giving up. A value of 0 means it will never time "
+ 		"out. "
+ 		"VCL can override this default value for each backend and "
+ 		"backend request. This parameter does not apply to pipe.",
+ 		0,
+ 		"60", "s" },
+ 	{ "between_bytes_timeout", tweak_timeout_double,
+ 		&mgt_param.between_bytes_timeout,0, UINT_MAX,
+ 		"Default timeout between bytes when receiving data from "
+ 		"backend. "
+ 		"We only wait for this many seconds between bytes "
+ 		"before giving up. A value of 0 means it will never time out. "
+ 		"VCL can override this default value for each backend request "
+ 		"and backend request. This parameter does not apply to pipe.",
+ 		0,
+ 		"60", "s" },
+ 	{ "acceptor_sleep_max", tweak_timeout_double,
+ 		&mgt_param.acceptor_sleep_max, 0,  10,
+ 		"If we run out of resources, such as file descriptors or "
+ 		"worker threads, the acceptor will sleep between accepts.\n"
+ 		"This parameter limits how long it can sleep between "
+ 		"attempts to accept new connections.",
+ 		EXPERIMENTAL,
+ 		"0.050", "s" },
+ 	{ "acceptor_sleep_incr", tweak_timeout_double,
+ 		&mgt_param.acceptor_sleep_incr, 0,  1,
+ 		"If we run out of resources, such as file descriptors or "
+ 		"worker threads, the acceptor will sleep between accepts.\n"
+ 		"This parameter control how much longer we sleep, each time "
+ 		"we fail to accept a new connection.",
+ 		EXPERIMENTAL,
+ 		"0.001", "s" },
+ 	{ "acceptor_sleep_decay", tweak_generic_double,
+ 		&mgt_param.acceptor_sleep_decay, 0,  1,
+ 		"If we run out of resources, such as file descriptors or "
+ 		"worker threads, the acceptor will sleep between accepts.\n"
+ 		"This parameter (multiplicatively) reduce the sleep duration "
+ 		"for each succesfull accept. (ie: 0.9 = reduce by 10%)",
+ 		EXPERIMENTAL,
+ 		"0.900", "" },
+ 	{ "clock_skew", tweak_uint, &mgt_param.clock_skew, 0, UINT_MAX,
+ 		"How much clockskew we are willing to accept between the "
+ 		"backend and our own clock.",
+ 		0,
+ 		"10", "s" },
+ 	{ "prefer_ipv6", tweak_bool, &mgt_param.prefer_ipv6, 0, 0,
+ 		"Prefer IPv6 address when connecting to backends which "
+ 		"have both IPv4 and IPv6 addresses.",
+ 		0,
+ 		"off", "bool" },
+ 	{ "session_max", tweak_uint,
+ 		&mgt_param.max_sess, 1000, UINT_MAX,
+ 		"Maximum number of sessions we will allocate from one pool "
+ 		"before just dropping connections.\n"
+ 		"This is mostly an anti-DoS measure, and setting it plenty "
+ 		"high should not hurt, as long as you have the memory for "
+ 		"it.\n",
+ 		0,
+ 		"100000", "sessions" },
+ 	{ "session_linger", tweak_uint,
+ 		&mgt_param.session_linger,0, UINT_MAX,
+ 		"How long time the workerthread lingers on the session "
+ 		"to see if a new request appears right away.\n"
+ 		"If sessions are reused, as much as half of all reuses "
+ 		"happen within the first 100 msec of the previous request "
+ 		"completing.\n"
+ 		"Setting this too high results in worker threads not doing "
+ 		"anything for their keep, setting it too low just means that "
+ 		"more sessions take a detour around the waiter.",
+ 		EXPERIMENTAL,
+ 		"50", "ms" },
+ 	{ "log_hashstring", tweak_bool, &mgt_param.log_hash, 0, 0,
+ 		"Log the hash string components to shared memory log.\n",
+ 		0,
+ 		"on", "bool" },
+ 	{ "log_local_address", tweak_bool, &mgt_param.log_local_addr, 0, 0,
+ 		"Log the local address on the TCP connection in the "
+ 		"SessionOpen shared memory record.\n",
+ 		0,
+ 		"off", "bool" },
+ 	{ "waiter", tweak_waiter, NULL, 0, 0,
+ 		"Select the waiter kernel interface.\n",
+ 		EXPERIMENTAL | MUST_RESTART,
+ 		"default", NULL },
+ 	{ "diag_bitmap", tweak_diag_bitmap, 0, 0, 0,
+ 		"Bitmap controlling diagnostics code:\n"
+ 		"  0x00000001 - CNT_Session states.\n"
+ 		"  0x00000002 - workspace debugging.\n"
+ 		"  0x00000004 - kqueue debugging.\n"
+ 		"  0x00000008 - mutex logging.\n"
+ 		"  0x00000010 - mutex contests.\n"
+ 		"  0x00000020 - waiting list.\n"
+ 		"  0x00000040 - object workspace.\n"
+ 		"  0x00001000 - do not core-dump child process.\n"
+ 		"  0x00002000 - only short panic message.\n"
+ 		"  0x00004000 - panic to stderr.\n"
+ 		"  0x00010000 - synchronize shmlog.\n"
+ 		"  0x00020000 - synchronous start of persistence.\n"
+ 		"  0x00040000 - release VCL early.\n"
+ 		"  0x00080000 - ban-lurker debugging.\n"
+ 		"  0x80000000 - do edge-detection on digest.\n"
+ 		"\n"
+ 		"Use 0x notation and do the bitor in your head :-)\n",
+ 		0,
+ 		"0", "bitmap" },
+ 	{ "ban_dups", tweak_bool, &mgt_param.ban_dups, 0, 0,
+ 		"Detect and eliminate duplicate bans.\n",
+ 		0,
+ 		"on", "bool" },
+ 	{ "syslog_cli_traffic", tweak_bool, &mgt_param.syslog_cli_traffic, 0, 0,
+ 		"Log all CLI traffic to syslog(LOG_INFO).\n",
+ 		0,
+ 		"on", "bool" },
+ 	{ "ban_lurker_sleep", tweak_timeout_double,
+ 		&mgt_param.ban_lurker_sleep, 0, UINT_MAX,
+ 		"How long time does the ban lurker thread sleeps between "
+ 		"successful attempts to push the last item up the ban "
+ 		" list.  It always sleeps a second when nothing can be done.\n"
+ 		"A value of zero disables the ban lurker.",
+ 		0,
+ 		"0.01", "s" },
+ 	{ "saintmode_threshold", tweak_uint,
+ 		&mgt_param.saintmode_threshold, 0, UINT_MAX,
+ 		"The maximum number of objects held off by saint mode before "
+ 		"no further will be made to the backend until one times out.  "
+ 		"A value of 0 disables saintmode.",
+ 		EXPERIMENTAL,
+ 		"10", "objects" },
+ 	{ "http_range_support", tweak_bool, &mgt_param.http_range_support, 0, 0,
+ 		"Enable support for HTTP Range headers.\n",
+ 		EXPERIMENTAL,
+ 		"on", "bool" },
+ 	{ "http_gzip_support", tweak_bool, &mgt_param.http_gzip_support, 0, 0,
+ 		"Enable gzip support. When enabled Varnish will compress "
+ 		"uncompressed objects before they are stored in the cache. "
+ 		"If a client does not support gzip encoding Varnish will "
+ 		"uncompress compressed objects on demand. Varnish will also "
+ 		"rewrite the Accept-Encoding header of clients indicating "
+ 		"support for gzip to:\n"
+ 		"  Accept-Encoding: gzip\n\n"
+ 		"Clients that do not support gzip will have their "
+ 		"Accept-Encoding header removed. For more information on how "
+ 		"gzip is implemented please see the chapter on gzip in the "
+ 		"Varnish reference.",
+ 		EXPERIMENTAL,
+ 		"on", "bool" },
+ 	{ "gzip_tmp_space", tweak_uint, &mgt_param.gzip_tmp_space, 0, 2,
+ 		"Where temporary space for gzip/gunzip is allocated:\n"
+ 		"  0 - malloc\n"
+ 		"  2 - thread workspace\n"
+ 		"\n"
+ 		"If you have much gzip/gunzip activity, it may be an"
+ 		" advantage to use workspace for these allocations to reduce"
+ 		" malloc activity.  Be aware that gzip needs 256+KB and gunzip"
+ 		" needs 32+KB of workspace (64+KB if ESI processing).",
+ 		EXPERIMENTAL,
+ 		"0", "" },
+ 	{ "gzip_level", tweak_uint, &mgt_param.gzip_level, 0, 9,
+ 		"Gzip compression level: 0=debug, 1=fast, 9=best",
+ 		0,
+ 		"6", ""},
+ 	{ "gzip_window", tweak_uint, &mgt_param.gzip_window, 8, 15,
+ 		"Gzip window size 8=least, 15=most compression.\n"
+ 		"Memory impact is 8=1k, 9=2k, ... 15=128k.",
+ 		0,
+ 		"15", ""},
+ 	{ "gzip_memlevel", tweak_uint, &mgt_param.gzip_memlevel, 1, 9,
+ 		"Gzip memory level 1=slow/least, 9=fast/most compression.\n"
+ 		"Memory impact is 1=1k, 2=2k, ... 9=256k.",
+ 		0,
+ 		"8", ""},
+ 	{ "gzip_stack_buffer",
+ 		tweak_bytes_u, &mgt_param.gzip_stack_buffer,
+ 	        2048, UINT_MAX,
+ 		"Size of stack buffer used for gzip processing.\n"
+ 		"The stack buffers are used for in-transit data,"
+ 		" for instance gunzip'ed data being sent to a client."
+ 		"Making this space to small results in more overhead,"
+ 		" writes to sockets etc, making it too big is probably"
+ 		" just a waste of memory.",
+ 		EXPERIMENTAL,
+ 		"32k", "bytes" },
+ 	{ "shortlived", tweak_timeout_double,
+ 		&mgt_param.shortlived, 0, UINT_MAX,
+ 		"Objects created with TTL shorter than this are always "
+ 		"put in transient storage.\n",
+ 		0,
+ 		"10.0", "s" },
+ 	{ "critbit_cooloff", tweak_timeout_double,
+ 		&mgt_param.critbit_cooloff, 60, 254,
+ 		"How long time the critbit hasher keeps deleted objheads "
+ 		"on the cooloff list.\n",
+ 		WIZARD,
+ 		"180.0", "s" },
+ 	{ "vcl_dir", tweak_string, &mgt_vcl_dir, 0, 0,
+ 		"Directory from which relative VCL filenames (vcl.load and "
+ 		"include) are opened.",
+ 		0,
+ #ifdef VARNISH_VCL_DIR
+ 		VARNISH_VCL_DIR,
+ #else
+ 		".",
+ #endif
+ 		NULL },
+ 	{ "vmod_dir", tweak_string, &mgt_vmod_dir, 0, 0,
+ 		"Directory where VCL modules are to be found.",
+ 		0,
+ #ifdef VARNISH_VMOD_DIR
+ 		VARNISH_VMOD_DIR,
+ #else
+ 		".",
+ #endif
+ 		NULL },
+ 	{ "vcc_err_unref", tweak_bool, &mgt_vcc_err_unref, 0, 0,
+ 		"Unreferenced VCL objects result in error.\n",
+ 		0,
+ 		"on", "bool" },
+ 
+ 
+ 	{ "pcre_match_limit", tweak_uint,
+ 		&mgt_param.vre_limits.match,
+ 		1, UINT_MAX,
+ 		"The limit for the  number of internal matching function"
+ 		" calls in a pcre_exec() execution.",
+ 		0,
+ 		"10000", ""},
+ 
+ 	{ "pcre_match_limit_recursion", tweak_uint,
+ 		&mgt_param.vre_limits.match_recursion,
+ 		1, UINT_MAX,
+ 		"The limit for the  number of internal matching function"
+ 		" recursions in a pcre_exec() execution.",
+ 		0,
+ 		"10000", ""},
+ 
+ 	{ "vsl_space", tweak_bytes,
+ 		&mgt_param.vsl_space, 1024*1024, HUGE_VAL,
+ 		"The amount of space to allocate for the VSL fifo buffer"
+ 		" in the VSM memory segment."
+ 		"  If you make this too small, varnish{ncsa|log} etc will"
+ 		" not be able to keep up."
+ 		"  Making it too large just costs memory resources.",
+ 		MUST_RESTART,
+ 		"80M", "bytes"},
+ 
+ 	{ "vsm_space", tweak_bytes,
+ 		&mgt_param.vsm_space, 1024*1024, HUGE_VAL,
+ 		"The amount of space to allocate for stats counters"
+ 		" in the VSM memory segment."
+ 		"  If you make this too small, some counters will be"
+ 		" invisible."
+ 		"  Making it too large just costs memory resources.",
+ 		MUST_RESTART,
+ 		"1M", "bytes"},
+ 
+ 	{ NULL, NULL, NULL }
+ };
+ 
+ /*--------------------------------------------------------------------*/
+ 
+ #define WIDTH 76
+ 
+ static void
+ mcf_wrap(struct cli *cli, const char *text)
+ {
+ 	const char *p, *q;
+ 
+ 	/* Format text to COLUMNS width */
+ 	for (p = text; *p != '\0'; ) {
+ 		q = strchr(p, '\n');
+ 		if (q == NULL)
+ 			q = strchr(p, '\0');
+ 		if (q > p + WIDTH - margin) {
+ 			q = p + WIDTH - margin;
+ 			while (q > p && *q != ' ')
+ 				q--;
+ 			AN(q);
+ 		}
+ 		VCLI_Out(cli, "%*s %.*s\n", margin, "", (int)(q - p), p);
+ 		p = q;
+ 		if (*p == ' ' || *p == '\n')
+ 			p++;
+ 	}
+ }
+ 
+ void
+ mcf_param_show(struct cli *cli, const char * const *av, void *priv)
+ {
+ 	int i;
+ 	const struct parspec *pp;
+ 	int lfmt;
+ 
+ 	(void)priv;
+ 	if (av[2] == NULL || strcmp(av[2], "-l"))
+ 		lfmt = 0;
+ 	else
+ 		lfmt = 1;
+ 	for (i = 0; i < nparspec; i++) {
+ 		pp = parspec[i];
+ 		if (av[2] != NULL && !lfmt && strcmp(pp->name, av[2]))
+ 			continue;
+ 		VCLI_Out(cli, "%-*s ", margin, pp->name);
+ 		if (pp->func == NULL) {
+ 			VCLI_Out(cli, "Not implemented.\n");
+ 			if (av[2] != NULL && !lfmt)
+ 				return;
+ 			else
+ 				continue;
+ 		}
+ 		pp->func(cli, pp, NULL);
+ 		if (pp->units != NULL)
+ 			VCLI_Out(cli, " [%s]\n", pp->units);
+ 		else
+ 			VCLI_Out(cli, "\n");
+ 		if (av[2] != NULL) {
+ 			VCLI_Out(cli, "%-*s Default is %s\n",
+ 			    margin, "", pp->def);
+ 			mcf_wrap(cli, pp->descr);
+ 			if (pp->flags & DELAYED_EFFECT)
+ 				mcf_wrap(cli, DELAYED_EFFECT_TEXT);
+ 			if (pp->flags & EXPERIMENTAL)
+ 				mcf_wrap(cli, EXPERIMENTAL_TEXT);
+ 			if (pp->flags & MUST_RELOAD)
+ 				mcf_wrap(cli, MUST_RELOAD_TEXT);
+ 			if (pp->flags & MUST_RESTART)
+ 				mcf_wrap(cli, MUST_RESTART_TEXT);
+ 			if (pp->flags & WIZARD)
+ 				mcf_wrap(cli, WIZARD_TEXT);
+ 			if (!lfmt)
+ 				return;
+ 			else
+ 				VCLI_Out(cli, "\n");
+ 		}
+ 	}
+ 	if (av[2] != NULL && !lfmt) {
+ 		VCLI_SetResult(cli, CLIS_PARAM);
+ 		VCLI_Out(cli, "Unknown parameter \"%s\".", av[2]);
+ 	}
+ }
+ 
+ /*--------------------------------------------------------------------*/
+ 
+ void
+ MCF_ParamSet(struct cli *cli, const char *param, const char *val)
+ {
+ 	const struct parspec *pp;
+ 
+ 	pp = mcf_findpar(param);
+ 	if (pp == NULL) {
+ 		VCLI_SetResult(cli, CLIS_PARAM);
+ 		VCLI_Out(cli, "Unknown parameter \"%s\".", param);
+ 		return;
+ 	}
+ 	pp->func(cli, pp, val);
+ 
+ 	if (cli->result == CLIS_OK && heritage.param != NULL)
+ 		*heritage.param = mgt_param;
+ 
+ 	if (cli->result != CLIS_OK) {
+ 		VCLI_Out(cli, "(attempting to set param %s to %s)\n",
+ 		    pp->name, val);
+ 	} else if (child_pid >= 0 && pp->flags & MUST_RESTART) {
+ 		VCLI_Out(cli, "Change will take effect"
+ 		    " when child is restarted");
+ 	} else if (pp->flags & MUST_RELOAD) {
+ 		VCLI_Out(cli, "Change will take effect"
+ 		    " when VCL script is reloaded");
+ 	}
+ }
+ 
+ 
+ /*--------------------------------------------------------------------*/
+ 
+ void
+ mcf_param_set(struct cli *cli, const char * const *av, void *priv)
+ {
+ 
+ 	(void)priv;
+ 	MCF_ParamSet(cli, av[2], av[3]);
+ }
+ 
+ /*--------------------------------------------------------------------
+  * Add a group of parameters to the global set and sort by name.
+  */
+ 
+ static int
+ parspec_cmp(const void *a, const void *b)
+ {
+ 	struct parspec * const * pa = a;
+ 	struct parspec * const * pb = b;
+ 	return (strcmp((*pa)->name, (*pb)->name));
+ }
+ 
+ static void
+ MCF_AddParams(const struct parspec *ps)
+ {
+ 	const struct parspec *pp;
+ 	int n;
+ 
+ 	n = 0;
+ 	for (pp = ps; pp->name != NULL; pp++) {
+ 		if (mcf_findpar(pp->name) != NULL)
+ 			fprintf(stderr, "Duplicate param: %s\n", pp->name);
+ 		if (strlen(pp->name) + 1 > margin)
+ 			margin = strlen(pp->name) + 1;
+ 		n++;
+ 	}
+ 	parspec = realloc(parspec, (1L + nparspec + n) * sizeof *parspec);
+ 	XXXAN(parspec);
+ 	for (pp = ps; pp->name != NULL; pp++)
+ 		parspec[nparspec++] = pp;
+ 	parspec[nparspec] = NULL;
+ 	qsort (parspec, nparspec, sizeof parspec[0], parspec_cmp);
+ }
+ 
+ /*--------------------------------------------------------------------
+  * Set defaults for all parameters
+  */
+ 
+ static void
+ MCF_SetDefaults(struct cli *cli)
+ {
+ 	const struct parspec *pp;
+ 	int i;
+ 
+ 	for (i = 0; i < nparspec; i++) {
+ 		pp = parspec[i];
+ 		if (cli != NULL)
+ 			VCLI_Out(cli,
+ 			    "Set Default for %s = %s\n", pp->name, pp->def);
+ 		pp->func(cli, pp, pp->def);
+ 		if (cli != NULL && cli->result != CLIS_OK)
+ 			return;
+ 	}
+ }
+ 
+ /*--------------------------------------------------------------------*/
+ 
+ void
+ MCF_ParamInit(struct cli *cli)
+ {
+ 
+ 	MCF_AddParams(input_parspec);
+ 	MCF_AddParams(WRK_parspec);
+ 
+ 	/* XXX: We do this twice, to get past any interdependencies */
+ 	MCF_SetDefaults(NULL);
+ 	MCF_SetDefaults(cli);
+ }
+ 
+ /*--------------------------------------------------------------------*/
+ 
+ void
+ MCF_DumpRst(void)
+ {
+ 	const struct parspec *pp;
+ 	const char *p, *q;
+ 	int i;
+ 
+ 	printf("\n.. The following is the autogenerated output from varnishd -x dumprst\n\n");
+ 	for (i = 0; i < nparspec; i++) {
+ 		pp = parspec[i];
+ 		printf("%s\n", pp->name);
+ 		if (pp->units != NULL && *pp->units != '\0')
+ 			printf("\t- Units: %s\n", pp->units);
+ 		printf("\t- Default: %s\n",
+ 		    strcmp(pp->def,MAGIC_INIT_STRING) == 0 ? "magic" : pp->def);
+ 		/*
+ 		 * XXX: we should mark the params with one/two flags
+ 		 * XXX: that say if ->min/->max are valid, so we
+ 		 * XXX: can emit those also in help texts.
+ 		 */
+ 		if (pp->flags) {
+ 			printf("\t- Flags: ");
+ 			q = "";
+ 			if (pp->flags & DELAYED_EFFECT) {
+ 				printf("%sdelayed", q);
+ 				q = ", ";
+ 			}
+ 			if (pp->flags & MUST_RESTART) {
+ 				printf("%smust_restart", q);
+ 				q = ", ";
+ 			}
+ 			if (pp->flags & MUST_RELOAD) {
+ 				printf("%smust_reload", q);
+ 				q = ", ";
+ 			}
+ 			if (pp->flags & EXPERIMENTAL) {
+ 				printf("%sexperimental", q);
+ 				q = ", ";
+ 			}
+ 			printf("\n");
+ 		}
+ 		printf("\n\t");
+ 		for (p = pp->descr; *p; p++) {
+ 			if (*p == '\n' && p[1] =='\0')
+ 				break;
+ 			if (*p == '\n' && p[1] =='\n') {
+ 				printf("\n\n\t");
+ 				p++;
+ 			} else if (*p == '\n') {
+ 				printf("\n\t");
+ 			} else if (*p == ':' && p[1] == '\n') {
+ 				/*
+ 				 * Start of definition list,
+ 				 * use RSTs code mode for this
+ 				 */
+ 				printf("::\n");
+ 			} else {
+ 				printf("%c", *p);
+ 			}
+ 		}
+ 		printf("\n\n");
+ 	}
+ 	printf("\n");
+ }
diff --cc bin/varnishd/mgt/mgt_sandbox_solaris.c
index 0000000,79f6650..114d6a4
mode 000000,100644..100644
--- a/bin/varnishd/mgt/mgt_sandbox_solaris.c
+++ b/bin/varnishd/mgt/mgt_sandbox_solaris.c
@@@ -1,0 -1,233 +1,234 @@@
+ /*-
+  * Copyright (c) 2006-2011 Varnish Software AS
+  * All rights reserved.
+  *
+  * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
+  *	   Nils Goroll <nils.goroll at uplex.de>
+  *
+  * Redistribution and use in source and binary forms, with or without
+  * modification, are permitted provided that the following conditions
+  * are met:
+  * 1. Redistributions of source code must retain the above copyright
+  *    notice, this list of conditions and the following disclaimer.
+  * 2. Redistributions in binary form must reproduce the above copyright
+  *    notice, this list of conditions and the following disclaimer in the
+  *    documentation and/or other materials provided with the distribution.
+  *
+  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+  * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+  * SUCH DAMAGE.
+  *
+  * Sandboxing child processes on Solaris
+  *
+  */
+ 
+ #include "config.h"
+ 
+ #ifdef HAVE_SETPPRIV
+ 
+ #ifdef HAVE_PRIV_H
+ #include <priv.h>
+ #endif
+ #include <stdio.h>
+ #include <string.h>
+ #include <syslog.h>
+ #include <unistd.h>
+ 
+ #include "mgt/mgt.h"
+ 
+ #include "common/heritage.h"
++#include "common/params.h"
+ 
+ /*--------------------------------------------------------------------
+  * SOLARIS PRIVILEGES: Note on use of symbolic PRIV_* constants
+  *
+  * For privileges which existed in Solaris 10 FCS, we may use the constants from
+  * sys/priv_names.h
+  *
+  * For privileges which have been added later, we need to use strings in order
+  * not to break builds of varnish on these platforms. To remain binary
+  * compatible, we need to silently ignore errors from priv_addset when using
+  * these strings.
+  *
+  * For optimal build and binary forward comatibility, we could use subtractive
+  * set specs like
+  *
+  *       basic,!file_link_any,!proc_exec,!proc_fork,!proc_info,!proc_session
+  *
+  * but I (Nils) have a preference for making an informed decision about which
+  * privileges the varnish child should have and which it shouldn't.
+  *
+  * Newly introduced privileges should be annotated with their PSARC / commit ID
+  * (as long as Oracle reveils these :/ )
+  *
+  * SOLARIS PRIVILEGES: Note on accidentally setting the SNOCD flag
+  *
+  * When setting privileges, we need to take care not to accidentally set the
+  * SNOCD flag which will disable core dumps unnecessarily. (see
+  * https://www.varnish-cache.org/trac/ticket/671 )
+  *
+  * When changing the logic herein, always check with mdb -k. Replace _PID_ with
+  * the pid of your varnish child, the result should be 0, otherwise a regression
+  * has been introduced.
+  *
+  * > 0t_PID_::pid2proc | ::print proc_t p_flag | >a
+  * > (<a & 0x10000000)=X
+  *                 0
+  *
+  * (a value of 0x10000000 indicates that SNOCD is set)
+  *
+  * NOTE that on Solaris changing the uid will _always_ set SNOCD, so make sure
+  * you run this test with appropriate privileges, but without proc_setid, so
+  * varnish won't setuid(), e.g.
+  *
+  * pfexec ppriv -e -s A=basic,net_privaddr,sys_resource varnish ...
+  *
+  * SOLARIS COREDUMPS with setuid(): See coreadm(1M) - global-setid / proc-setid
+  *
+  */
+ 
+ /* effective during runtime of the child */
+ static inline void
+ mgt_sandbox_solaris_add_effective(priv_set_t *pset)
+ {
+ 	/* PSARC/2009/685 - 8eca52188202 - onnv_132 */
+ 	priv_addset(pset, "net_access");
+ 
+ 	/* PSARC/2009/378 - 63678502e95e - onnv_140 */
+ 	priv_addset(pset, "file_read");
+ 	priv_addset(pset, "file_write");
+ }
+ 
+ /* permitted during runtime of the child - for privilege bracketing */
+ static inline void
+ mgt_sandbox_solaris_add_permitted(priv_set_t *pset)
+ {
+ 	/* for raising limits in cache_waiter_ports.c */
+ 	priv_addset(pset, PRIV_SYS_RESOURCE);
+ }
+ 
+ /* effective during mgt_sandbox */
+ static inline void
+ mgt_sandbox_solaris_add_initial(priv_set_t *pset)
+ {
+ 	/* for setgid/setuid */
+ 	priv_addset(pset, PRIV_PROC_SETID);
+ }
+ 
+ /*
+  * if we are not yet privilege-aware already (ie we have been started
+  * not-privilege aware wird euid 0), we need to grab any additional privileges
+  * needed during mgt_standbox, until we reduce to least privileges in
+  * mgt_sandbox_waive, otherwise we would loose them with setuid()
+  */
+ 
+ void
+ mgt_sandbox_solaris_init(void)
+ {
+ 	priv_set_t *priv_all;
+ 
+ 	if (! (priv_all = priv_allocset())) {
+ 		REPORT(LOG_ERR,
+ 		    "Child start warning: mgt_sandbox_init - priv_allocset failed: errno=%d (%s)",
+ 		    errno, strerror(errno));
+ 		return;
+ 	}
+ 
+ 	priv_emptyset(priv_all);
+ 
+ 	mgt_sandbox_solaris_add_effective(priv_all);
+ 	mgt_sandbox_solaris_add_permitted(priv_all);
+ 	mgt_sandbox_solaris_add_initial(priv_all);
+ 
+ 	setppriv(PRIV_ON, PRIV_PERMITTED, priv_all);
+ 	setppriv(PRIV_ON, PRIV_EFFECTIVE, priv_all);
+ 	setppriv(PRIV_ON, PRIV_INHERITABLE, priv_all);
+ 
+ 	priv_freeset(priv_all);
+ }
+ 
+ void
+ mgt_sandbox_solaris_privsep(void)
+ {
+ 	if (priv_ineffect(PRIV_PROC_SETID)) {
+                 if (getgid() != mgt_param.gid)
+                         XXXAZ(setgid(mgt_param.gid));
+                 if (getuid() != mgt_param.uid)
+                         XXXAZ(setuid(mgt_param.uid));
+         } else {
+                 REPORT(LOG_INFO, "Privilege %s missing, will not change uid/gid",
+ 		    PRIV_PROC_SETID);
+         }
+ }
+ 
+ /*
+  * Waive most privileges in the child
+  *
+  * as of onnv_151a, we should end up with:
+  *
+  * > ppriv -v #pid of varnish child
+  * PID:  .../varnishd ...
+  * flags = PRIV_AWARE
+  *      E: file_read,file_write,net_access
+  *      I: none
+  *      P: file_read,file_write,net_access,sys_resource
+  *      L: file_read,file_write,net_access,sys_resource
+  *
+  * We should keep sys_resource in P in order to adjust our limits if we need to
+  */
+ 
+ void
+ mgt_sandbox_solaris_fini(void)
+ {
+ 	priv_set_t *effective, *inheritable, *permitted;
+ 
+ 	if (!(effective = priv_allocset()) ||
+ 	    !(inheritable = priv_allocset()) ||
+ 	    !(permitted = priv_allocset())) {
+ 		REPORT(LOG_ERR,
+ 		    "Child start warning: mgt_sandbox_waive - priv_allocset failed: errno=%d (%s)",
+ 		    errno, strerror(errno));
+ 		return;
+ 	}
+ 
+ 	priv_emptyset(inheritable);
+ 
+ 	priv_emptyset(effective);
+ 	mgt_sandbox_solaris_add_effective(effective);
+ 
+ 	priv_copyset(effective, permitted);
+ 	mgt_sandbox_solaris_add_permitted(permitted);
+ 
+ 	/*
+ 	 * invert the sets and clear privileges such that setppriv will always
+ 	 * succeed
+ 	 */
+ 	priv_inverse(inheritable);
+ 	priv_inverse(effective);
+ 	priv_inverse(permitted);
+ 
+ #define SETPPRIV(which, set)						\
+ 	if (setppriv(PRIV_OFF, which, set))				\
+ 		REPORT(LOG_ERR,						\
+ 		    "Child start warning: Waiving privileges failed on %s: errno=%d (%s)", \
+ 		    #which, errno, strerror(errno));
+ 
+ 	SETPPRIV(PRIV_LIMIT, permitted);
+ 	SETPPRIV(PRIV_PERMITTED, permitted);
+ 	SETPPRIV(PRIV_EFFECTIVE, effective);
+ 	SETPPRIV(PRIV_INHERITABLE, inheritable);
+ #undef SETPPRIV
+ 
+ 	priv_freeset(inheritable);
+ 	priv_freeset(effective);
+ }
+ 
+ #endif /* HAVE_SETPPRIV */
diff --cc bin/varnishd/storage/stevedore.c
index 0000000,860604e..71241f5
mode 000000,100644..100644
--- a/bin/varnishd/storage/stevedore.c
+++ b/bin/varnishd/storage/stevedore.c
@@@ -1,0 -1,466 +1,514 @@@
+ /*-
+  * Copyright (c) 2007-2011 Varnish Software AS
+  * All rights reserved.
+  *
+  * Author: Dag-Erling Sm?rgav <des at des.no>
+  *
+  * Redistribution and use in source and binary forms, with or without
+  * modification, are permitted provided that the following conditions
+  * are met:
+  * 1. Redistributions of source code must retain the above copyright
+  *    notice, this list of conditions and the following disclaimer.
+  * 2. Redistributions in binary form must reproduce the above copyright
+  *    notice, this list of conditions and the following disclaimer in the
+  *    documentation and/or other materials provided with the distribution.
+  *
+  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+  * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+  * SUCH DAMAGE.
+  *
+  * STEVEDORE: one who works at or is responsible for loading and
+  * unloading ships in port.  Example: "on the wharves, stevedores were
+  * unloading cargo from the far corners of the world." Origin: Spanish
+  * estibador, from estibar to pack.  First Known Use: 1788
+  */
+ 
+ #include "config.h"
+ 
+ #include <stdio.h>
+ #include <stdlib.h>
+ 
+ #include "cache/cache.h"
+ 
+ #include "storage/storage.h"
+ #include "vrt.h"
+ #include "vrt_obj.h"
+ 
+ static const struct stevedore * volatile stv_next;
+ 
+ /*---------------------------------------------------------------------
+  * Default objcore methods
+  */
+ 
+ static struct object * __match_proto__(getobj_f)
+ default_oc_getobj(struct worker *wrk, struct objcore *oc)
+ {
+ 	struct object *o;
+ 
+ 	(void)wrk;
+ 	if (oc->priv == NULL)
+ 		return (NULL);
+ 	CAST_OBJ_NOTNULL(o, oc->priv, OBJECT_MAGIC);
+ 	return (o);
+ }
+ 
+ static void
+ default_oc_freeobj(struct objcore *oc)
+ {
+ 	struct object *o;
+ 
+ 	CAST_OBJ_NOTNULL(o, oc->priv, OBJECT_MAGIC);
+ 	oc->priv = NULL;
+ 	oc->methods = NULL;
+ 
+ 	STV_Freestore(o);
+ 	STV_free(o->objstore);
+ }
+ 
+ static struct lru *
+ default_oc_getlru(const struct objcore *oc)
+ {
+ 	struct object *o;
+ 
+ 	CAST_OBJ_NOTNULL(o, oc->priv, OBJECT_MAGIC);
+ 	return (o->objstore->stevedore->lru);
+ }
+ 
+ static struct objcore_methods default_oc_methods = {
+ 	.getobj = default_oc_getobj,
+ 	.freeobj = default_oc_freeobj,
+ 	.getlru = default_oc_getlru,
+ };
+ 
+ 
+ /*--------------------------------------------------------------------
+  */
+ 
+ struct lru *
+ LRU_Alloc(void)
+ {
+ 	struct lru *l;
+ 
+ 	ALLOC_OBJ(l, LRU_MAGIC);
+ 	AN(l);
+ 	VTAILQ_INIT(&l->lru_head);
+ 	Lck_New(&l->mtx, lck_lru);
+ 	return (l);
+ }
+ 
+ void
+ LRU_Free(struct lru *lru)
+ {
+ 	CHECK_OBJ_NOTNULL(lru, LRU_MAGIC);
+ 	Lck_Delete(&lru->mtx);
+ 	FREE_OBJ(lru);
+ }
+ 
+ /*--------------------------------------------------------------------
+  * XXX: trust pointer writes to be atomic
+  */
+ 
+ static struct stevedore *
+ stv_pick_stevedore(const struct sess *sp, const char **hint)
+ {
+ 	struct stevedore *stv;
+ 
+ 	AN(hint);
+ 	if (*hint != NULL && **hint != '\0') {
+ 		VTAILQ_FOREACH(stv, &stv_stevedores, list) {
+ 			if (!strcmp(stv->ident, *hint))
+ 				return (stv);
+ 		}
+ 		if (!strcmp(TRANSIENT_STORAGE, *hint))
+ 			return (stv_transient);
+ 
+ 		/* Hint was not valid, nuke it */
+ 		WSP(sp, SLT_Debug, "Storage hint not usable");
+ 		*hint = NULL;
+ 	}
+ 	/* pick a stevedore and bump the head along */
+ 	stv = VTAILQ_NEXT(stv_next, list);
+ 	if (stv == NULL)
+ 		stv = VTAILQ_FIRST(&stv_stevedores);
+ 	AN(stv);
+ 	AN(stv->name);
+ 	stv_next = stv;
+ 	return (stv);
+ }
+ 
+ /*-------------------------------------------------------------------*/
+ 
+ static struct storage *
+ stv_alloc(struct worker *w, const struct object *obj, size_t size)
+ {
+ 	struct storage *st;
+ 	struct stevedore *stv;
+ 	unsigned fail = 0;
+ 
+ 	/*
+ 	 * Always use the stevedore which allocated the object in order to
+ 	 * keep an object inside the same stevedore.
+ 	 */
+ 	CHECK_OBJ_NOTNULL(obj, OBJECT_MAGIC);
+ 	CHECK_OBJ_NOTNULL(w, WORKER_MAGIC);
+ 	stv = obj->objstore->stevedore;
+ 	CHECK_OBJ_NOTNULL(stv, STEVEDORE_MAGIC);
+ 
+ 	if (size > cache_param->fetch_maxchunksize)
+ 		size = cache_param->fetch_maxchunksize;
+ 
+ 	for (;;) {
+ 		/* try to allocate from it */
+ 		AN(stv->alloc);
+ 		st = stv->alloc(stv, size);
+ 		if (st != NULL)
+ 			break;
+ 
+ 		if (size > cache_param->fetch_chunksize) {
+ 			size >>= 1;
+ 			continue;
+ 		}
+ 
+ 		/* no luck; try to free some space and keep trying */
+ 		if (EXP_NukeOne(w, stv->lru) == -1)
+ 			break;
+ 
+ 		/* Enough is enough: try another if we have one */
+ 		if (++fail >= cache_param->nuke_limit)
+ 			break;
+ 	}
+ 	if (st != NULL)
+ 		CHECK_OBJ_NOTNULL(st, STORAGE_MAGIC);
+ 	return (st);
+ }
+ 
+ 
+ /*-------------------------------------------------------------------*
+  * Structure used to transport internal knowledge from STV_NewObject()
+  * to STV_MkObject().  Nobody else should mess with this struct.
+  */
+ 
+ struct stv_objsecrets {
+ 	unsigned	magic;
+ #define STV_OBJ_SECRETES_MAGIC	0x78c87247
+ 	uint16_t	nhttp;
+ 	unsigned	lhttp;
+ 	unsigned	wsl;
+ 	struct exp	*exp;
+ };
+ 
+ /*--------------------------------------------------------------------
+  * This function is called by stevedores ->allocobj() method, which
+  * very often will be stv_default_allocobj() below, to convert a slab
+  * of storage into object which the stevedore can then register in its
+  * internal state, before returning it to STV_NewObject().
+  * As you probably guessed: All this for persistence.
+  */
+ 
+ struct object *
+ STV_MkObject(struct sess *sp, void *ptr, unsigned ltot,
+     const struct stv_objsecrets *soc)
+ {
+ 	struct object *o;
+ 	unsigned l;
+ 
+ 	CHECK_OBJ_NOTNULL(soc, STV_OBJ_SECRETES_MAGIC);
+ 
+ 	assert(PAOK(ptr));
+ 	assert(PAOK(soc->wsl));
+ 	assert(PAOK(soc->lhttp));
+ 
+ 	assert(ltot >= sizeof *o + soc->lhttp + soc->wsl);
+ 
+ 	o = ptr;
+ 	memset(o, 0, sizeof *o);
+ 	o->magic = OBJECT_MAGIC;
+ 
+ 	l = PRNDDN(ltot - (sizeof *o + soc->lhttp));
+ 	assert(l >= soc->wsl);
+ 
+ 	o->http = HTTP_create(o + 1, soc->nhttp);
+ 	WS_Init(o->ws_o, "obj", (char *)(o + 1) + soc->lhttp, soc->wsl);
+ 	WS_Assert(o->ws_o);
+ 	assert(o->ws_o->e <= (char*)ptr + ltot);
+ 
+ 	http_Setup(o->http, o->ws_o);
+ 	o->http->magic = HTTP_MAGIC;
+ 	o->exp = *soc->exp;
+ 	VTAILQ_INIT(&o->store);
+ 	sp->wrk->stats.n_object++;
+ 
+ 	if (sp->objcore != NULL) {
+ 		CHECK_OBJ_NOTNULL(sp->objcore, OBJCORE_MAGIC);
+ 
+ 		o->objcore = sp->objcore;
+ 		sp->objcore = NULL;     /* refcnt follows pointer. */
+ 		BAN_NewObjCore(o->objcore);
+ 
+ 		o->objcore->methods = &default_oc_methods;
+ 		o->objcore->priv = o;
+ 	}
+ 	return (o);
+ }
+ 
+ /*--------------------------------------------------------------------
+  * This is the default ->allocobj() which all stevedores who do not
+  * implement persistent storage can rely on.
+  */
+ 
+ struct object *
+ stv_default_allocobj(struct stevedore *stv, struct sess *sp, unsigned ltot,
+     const struct stv_objsecrets *soc)
+ {
+ 	struct object *o;
+ 	struct storage *st;
+ 
+ 	CHECK_OBJ_NOTNULL(soc, STV_OBJ_SECRETES_MAGIC);
+ 	st = stv->alloc(stv, ltot);
+ 	if (st == NULL)
+ 		return (NULL);
+ 	if (st->space < ltot) {
+ 		stv->free(st);
+ 		return (NULL);
+ 	}
+ 	ltot = st->len = st->space;
+ 	o = STV_MkObject(sp, st->ptr, ltot, soc);
+ 	CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC);
+ 	o->objstore = st;
+ 	return (o);
+ }
+ 
+ /*-------------------------------------------------------------------
+  * Allocate storage for an object, based on the header information.
+  * XXX: If we know (a hint of) the length, we could allocate space
+  * XXX: for the body in the same allocation while we are at it.
+  */
+ 
+ struct object *
+ STV_NewObject(struct sess *sp, const char *hint, unsigned wsl, struct exp *ep,
+     uint16_t nhttp)
+ {
+ 	struct object *o;
+ 	struct stevedore *stv, *stv0;
+ 	unsigned lhttp, ltot;
+ 	struct stv_objsecrets soc;
+ 	int i;
+ 
+ 	assert(wsl > 0);
+ 	wsl = PRNDUP(wsl);
+ 
+ 	lhttp = HTTP_estimate(nhttp);
+ 	lhttp = PRNDUP(lhttp);
+ 
+ 	memset(&soc, 0, sizeof soc);
+ 	soc.magic = STV_OBJ_SECRETES_MAGIC;
+ 	soc.nhttp = nhttp;
+ 	soc.lhttp = lhttp;
+ 	soc.wsl = wsl;
+ 	soc.exp = ep;
+ 
+ 	ltot = sizeof *o + wsl + lhttp;
+ 
+ 	stv = stv0 = stv_pick_stevedore(sp, &hint);
+ 	AN(stv->allocobj);
+ 	o = stv->allocobj(stv, sp, ltot, &soc);
+ 	if (o == NULL && hint == NULL) {
+ 		do {
+ 			stv = stv_pick_stevedore(sp, &hint);
+ 			AN(stv->allocobj);
+ 			o = stv->allocobj(stv, sp, ltot, &soc);
+ 		} while (o == NULL && stv != stv0);
+ 	}
+ 	if (o == NULL) {
+ 		/* no luck; try to free some space and keep trying */
+ 		for (i = 0; o == NULL && i < cache_param->nuke_limit; i++) {
+ 			if (EXP_NukeOne(sp->wrk, stv->lru) == -1)
+ 				break;
+ 			o = stv->allocobj(stv, sp, ltot, &soc);
+ 		}
+ 	}
+ 
+ 	if (o == NULL)
+ 		return (NULL);
+ 	CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC);
+ 	CHECK_OBJ_NOTNULL(o->objstore, STORAGE_MAGIC);
+ 	return (o);
+ }
+ 
+ /*-------------------------------------------------------------------*/
+ 
+ void
+ STV_Freestore(struct object *o)
+ {
+ 	struct storage *st, *stn;
+ 
+ 	if (o->esidata != NULL) {
+ 		STV_free(o->esidata);
+ 		o->esidata = NULL;
+ 	}
+ 	VTAILQ_FOREACH_SAFE(st, &o->store, list, stn) {
+ 		CHECK_OBJ_NOTNULL(st, STORAGE_MAGIC);
+ 		VTAILQ_REMOVE(&o->store, st, list);
+ 		STV_free(st);
+ 	}
+ }
+ 
+ /*-------------------------------------------------------------------*/
+ 
+ struct storage *
+ STV_alloc(struct worker *w, size_t size)
+ {
++	struct object *obj = w->fetch_obj;
++	if (obj == NULL)
++		obj = w->sp->obj;
+ 
 -	return (stv_alloc(w, w->fetch_obj, size));
++	return (stv_alloc(w, obj, size));
+ }
+ 
+ void
+ STV_trim(struct storage *st, size_t size)
+ {
+ 
+ 	CHECK_OBJ_NOTNULL(st, STORAGE_MAGIC);
+ 	AN(st->stevedore);
+ 	if (st->stevedore->trim)
+ 		st->stevedore->trim(st, size);
+ }
+ 
++/*
++ * Duplicate the object storage (HTML body) from src into target, using a
++ * stevedore-specific dup method for src's stevedore.
++ *
++ * Currently, every method just copies storage from one object to the other,
++ * but this method of encapsulation opens the path to future techniques of
++ * sharing storage together with reference counting.
++ */
++void
++STV_dup(const struct sess *sp, struct object *src, struct object *target)
++{
++        struct stevedore *stv;
++
++        CHECK_OBJ_NOTNULL(src, OBJECT_MAGIC);
++        CHECK_OBJ_NOTNULL(target, OBJECT_MAGIC);
++        CHECK_OBJ_NOTNULL(src->objstore, STORAGE_MAGIC);
++        CHECK_OBJ_NOTNULL(src->objstore->stevedore, STEVEDORE_MAGIC);
++
++        stv = src->objstore->stevedore;
++        AN(stv->dup);
++        
++        stv->dup(sp, src, target);
++}
++
+ void
+ STV_free(struct storage *st)
+ {
+ 
+ 	CHECK_OBJ_NOTNULL(st, STORAGE_MAGIC);
+ 	AN(st->stevedore);
+ 	AN(st->stevedore->free);
+ 	st->stevedore->free(st);
+ }
+ 
+ void
+ STV_open(void)
+ {
+ 	struct stevedore *stv;
+ 
+ 	VTAILQ_FOREACH(stv, &stv_stevedores, list) {
+ 		stv->lru = LRU_Alloc();
+ 		if (stv->open != NULL)
+ 			stv->open(stv);
+ 	}
+ 	stv = stv_transient;
+ 	if (stv->open != NULL) {
+ 		stv->lru = LRU_Alloc();
+ 		stv->open(stv);
+ 	}
+ 	stv_next = VTAILQ_FIRST(&stv_stevedores);
+ }
+ 
+ void
+ STV_close(void)
+ {
+ 	struct stevedore *stv;
+ 
+ 	VTAILQ_FOREACH(stv, &stv_stevedores, list)
+ 		if (stv->close != NULL)
+ 			stv->close(stv);
+ 	stv = stv_transient;
+ 	if (stv->close != NULL)
+ 		stv->close(stv);
+ }
+ 
+ 
+ /*--------------------------------------------------------------------
+  * VRT functions for stevedores
+  */
+ 
+ static const struct stevedore *
+ stv_find(const char *nm)
+ {
+ 	const struct stevedore *stv;
+ 
+ 	VTAILQ_FOREACH(stv, &stv_stevedores, list)
+ 		if (!strcmp(stv->ident, nm))
+ 			return (stv);
+ 	if (!strcmp(TRANSIENT_STORAGE, nm))
+ 		return (stv_transient);
+ 	return (NULL);
+ }
+ 
+ int
+ VRT_Stv(const char *nm)
+ {
+ 
+ 	if (stv_find(nm) != NULL)
+ 		return (1);
+ 	return (0);
+ }
+ 
+ #define VRTSTVVAR(nm, vtype, ctype, dval)	\
+ ctype						\
+ VRT_Stv_##nm(const char *nm)			\
+ {						\
+ 	const struct stevedore *stv;		\
+ 						\
+ 	stv = stv_find(nm);			\
+ 	if (stv == NULL)			\
+ 		return (dval);			\
+ 	if (stv->var_##nm == NULL)		\
+ 		return (dval);			\
+ 	return (stv->var_##nm(stv));		\
+ }
+ 
+ #include "tbl/vrt_stv_var.h"
+ #undef VRTSTVVAR
++
++/*
++ * Default object store dup just copies the storage.
++ */
++void
++default_dup(const struct sess *sp, struct object *src, struct object *target)
++{
++        struct storage *st, *st2;
++        unsigned cl;
++
++        VTAILQ_FOREACH(st2, &src->store, list) {
++		cl = st2->len;
++		st = STV_alloc(sp->wrk, cl);
++		XXXAN(st);
++                assert(st->space >= cl);
++		VTAILQ_INSERT_TAIL(&target->store, st, list);
++		st->len = cl;
++		target->len += cl;
++		memcpy(st->ptr, st2->ptr, cl);
++	}
++}
diff --cc bin/varnishd/storage/storage.h
index 0000000,a813a36..879a6fb
mode 000000,100644..100644
--- a/bin/varnishd/storage/storage.h
+++ b/bin/varnishd/storage/storage.h
@@@ -1,0 -1,104 +1,110 @@@
+ /*-
+  * Copyright (c) 2006 Verdens Gang AS
+  * Copyright (c) 2006-2011 Varnish Software AS
+  * All rights reserved.
+  *
+  * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
+  *
+  * Redistribution and use in source and binary forms, with or without
+  * modification, are permitted provided that the following conditions
+  * are met:
+  * 1. Redistributions of source code must retain the above copyright
+  *    notice, this list of conditions and the following disclaimer.
+  * 2. Redistributions in binary form must reproduce the above copyright
+  *    notice, this list of conditions and the following disclaimer in the
+  *    documentation and/or other materials provided with the distribution.
+  *
+  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+  * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+  * SUCH DAMAGE.
+  *
+  * This defines the backend interface between the stevedore and the
+  * pluggable storage implementations.
+  *
+  */
+ 
+ struct stv_objsecrets;
+ struct stevedore;
+ struct sess;
+ struct lru;
+ 
+ typedef void storage_init_f(struct stevedore *, int ac, char * const *av);
+ typedef void storage_open_f(const struct stevedore *);
+ typedef struct storage *storage_alloc_f(struct stevedore *, size_t size);
++typedef void storage_dup_f(const struct sess *sp, struct object *src, struct object *target);
+ typedef void storage_trim_f(struct storage *, size_t size);
+ typedef void storage_free_f(struct storage *);
+ typedef struct object *storage_allocobj_f(struct stevedore *, struct sess *sp,
+     unsigned ltot, const struct stv_objsecrets *);
+ typedef void storage_close_f(const struct stevedore *);
+ 
+ /* Prototypes for VCL variable responders */
+ #define VRTSTVTYPE(ct) typedef ct storage_var_##ct(const struct stevedore *);
+ #include "tbl/vrt_stv_var.h"
+ #undef VRTSTVTYPE
+ 
+ extern storage_allocobj_f stv_default_allocobj;
+ 
+ /*--------------------------------------------------------------------*/
+ 
+ struct stevedore {
+ 	unsigned		magic;
+ #define STEVEDORE_MAGIC		0x4baf43db
+ 	const char		*name;
+ 	unsigned		transient;
+ 	storage_init_f		*init;		/* called by mgt process */
+ 	storage_open_f		*open;		/* called by cache process */
+ 	storage_alloc_f		*alloc;		/* --//-- */
+ 	storage_trim_f		*trim;		/* --//-- */
++        storage_dup_f           *dup;           /* --//-- */
+ 	storage_free_f		*free;		/* --//-- */
+ 	storage_close_f		*close;		/* --//-- */
+ 	storage_allocobj_f	*allocobj;	/* --//-- */
+ 
+ 	struct lru		*lru;
+ 
+ #define VRTSTVVAR(nm, vtype, ctype, dval) storage_var_##ctype *var_##nm;
+ #include "tbl/vrt_stv_var.h"
+ #undef VRTSTVVAR
+ 
+ 	/* private fields */
+ 	void			*priv;
+ 
+ 	VTAILQ_ENTRY(stevedore)	list;
+ 	char			ident[16];	/* XXX: match VSM_chunk.ident */
+ };
+ 
+ VTAILQ_HEAD(stevedore_head, stevedore);
+ 
+ extern struct stevedore_head stv_stevedores;
+ extern struct stevedore *stv_transient;
+ 
+ /*--------------------------------------------------------------------*/
+ int STV_GetFile(const char *fn, int *fdp, const char **fnp, const char *ctx);
+ uintmax_t STV_FileSize(int fd, const char *size, unsigned *granularity,
+     const char *ctx);
+ struct object *STV_MkObject(struct sess *sp, void *ptr, unsigned ltot,
+     const struct stv_objsecrets *soc);
+ 
+ struct lru *LRU_Alloc(void);
+ void LRU_Free(struct lru *lru);
+ 
+ /*--------------------------------------------------------------------*/
+ extern const struct stevedore sma_stevedore;
+ extern const struct stevedore smf_stevedore;
+ extern const struct stevedore smp_stevedore;
+ #ifdef HAVE_LIBUMEM
+ extern const struct stevedore smu_stevedore;
+ #endif
++
++/* Default dup method */
++void STV_dup(const struct sess *sp, struct object *src, struct object *target);
++void default_dup(const struct sess *sp, struct object *src, struct object *target);
diff --cc bin/varnishd/storage/storage_file.c
index 0000000,9eb44d9..028563d
mode 000000,100644..100644
--- a/bin/varnishd/storage/storage_file.c
+++ b/bin/varnishd/storage/storage_file.c
@@@ -1,0 -1,616 +1,617 @@@
+ /*-
+  * Copyright (c) 2006 Verdens Gang AS
+  * Copyright (c) 2006-2011 Varnish Software AS
+  * All rights reserved.
+  *
+  * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
+  *
+  * Redistribution and use in source and binary forms, with or without
+  * modification, are permitted provided that the following conditions
+  * are met:
+  * 1. Redistributions of source code must retain the above copyright
+  *    notice, this list of conditions and the following disclaimer.
+  * 2. Redistributions in binary form must reproduce the above copyright
+  *    notice, this list of conditions and the following disclaimer in the
+  *    documentation and/or other materials provided with the distribution.
+  *
+  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+  * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+  * SUCH DAMAGE.
+  *
+  * Storage method based on mmap'ed file
+  */
+ 
+ #include "config.h"
+ 
+ #include <sys/mman.h>
+ 
+ #include <stdio.h>
+ #include <stdlib.h>
+ 
+ #include "cache/cache.h"
+ #include "storage/storage.h"
+ 
+ #include "vnum.h"
+ 
+ #ifndef MAP_NOCORE
+ #define MAP_NOCORE 0 /* XXX Linux */
+ #endif
+ 
+ #ifndef MAP_NOSYNC
+ #define MAP_NOSYNC 0 /* XXX Linux */
+ #endif
+ 
+ #define MINPAGES		128
+ 
+ /*
+  * Number of buckets on free-list.
+  *
+  * Last bucket is "larger than" so choose number so that the second
+  * to last bucket matches the 128k CHUNKSIZE in cache_fetch.c when
+  * using the a 4K minimal page size
+  */
+ #define NBUCKET			(128 / 4 + 1)
+ 
+ /*--------------------------------------------------------------------*/
+ 
+ VTAILQ_HEAD(smfhead, smf);
+ 
+ struct smf {
+ 	unsigned		magic;
+ #define SMF_MAGIC		0x0927a8a0
+ 	struct storage		s;
+ 	struct smf_sc		*sc;
+ 
+ 	int			alloc;
+ 
+ 	off_t			size;
+ 	off_t			offset;
+ 	unsigned char		*ptr;
+ 
+ 	VTAILQ_ENTRY(smf)	order;
+ 	VTAILQ_ENTRY(smf)	status;
+ 	struct smfhead		*flist;
+ };
+ 
+ struct smf_sc {
+ 	unsigned		magic;
+ #define SMF_SC_MAGIC		0x52962ee7
+ 	struct lock		mtx;
+ 	struct VSC_C_smf	*stats;
+ 
+ 	const char		*filename;
+ 	int			fd;
+ 	unsigned		pagesize;
+ 	uintmax_t		filesize;
+ 	struct smfhead		order;
+ 	struct smfhead		free[NBUCKET];
+ 	struct smfhead		used;
+ };
+ 
+ /*--------------------------------------------------------------------*/
+ 
+ static void
+ smf_initfile(struct smf_sc *sc, const char *size)
+ {
+ 	sc->filesize = STV_FileSize(sc->fd, size, &sc->pagesize, "-sfile");
+ 
+ 	AZ(ftruncate(sc->fd, (off_t)sc->filesize));
+ 
+ 	/* XXX: force block allocation here or in open ? */
+ }
+ 
+ static const char default_size[] = "100M";
+ static const char default_filename[] = ".";
+ 
+ static void
+ smf_init(struct stevedore *parent, int ac, char * const *av)
+ {
+ 	const char *size, *fn, *r;
+ 	struct smf_sc *sc;
+ 	unsigned u;
+ 	uintmax_t page_size;
+ 
+ 	AZ(av[ac]);
+ 
+ 	fn = default_filename;
+ 	size = default_size;
+ 	page_size = getpagesize();
+ 
+ 	if (ac > 3)
+ 		ARGV_ERR("(-sfile) too many arguments\n");
+ 	if (ac > 0 && *av[0] != '\0')
+ 		fn = av[0];
+ 	if (ac > 1 && *av[1] != '\0')
+ 		size = av[1];
+ 	if (ac > 2 && *av[2] != '\0') {
+ 
+ 		r = VNUM_2bytes(av[2], &page_size, 0);
+ 		if (r != NULL)
+ 			ARGV_ERR("(-sfile) granularity \"%s\": %s\n", av[2], r);
+ 	}
+ 
+ 	AN(fn);
+ 	AN(size);
+ 
+ 	ALLOC_OBJ(sc, SMF_SC_MAGIC);
+ 	XXXAN(sc);
+ 	VTAILQ_INIT(&sc->order);
+ 	for (u = 0; u < NBUCKET; u++)
+ 		VTAILQ_INIT(&sc->free[u]);
+ 	VTAILQ_INIT(&sc->used);
+ 	sc->pagesize = page_size;
+ 
+ 	parent->priv = sc;
+ 
+ 	(void)STV_GetFile(fn, &sc->fd, &sc->filename, "-sfile");
+ 
+ 	mgt_child_inherit(sc->fd, "storage_file");
+ 	smf_initfile(sc, size);
+ }
+ 
+ /*--------------------------------------------------------------------
+  * Insert/Remove from correct freelist
+  */
+ 
+ static void
+ insfree(struct smf_sc *sc, struct smf *sp)
+ {
+ 	size_t b;
+ 	struct smf *sp2;
+ 	size_t ns;
+ 
+ 	assert(sp->alloc == 0);
+ 	assert(sp->flist == NULL);
+ 	Lck_AssertHeld(&sc->mtx);
+ 	b = sp->size / sc->pagesize;
+ 	if (b >= NBUCKET) {
+ 		b = NBUCKET - 1;
+ 		sc->stats->g_smf_large++;
+ 	} else {
+ 		sc->stats->g_smf_frag++;
+ 	}
+ 	sp->flist = &sc->free[b];
+ 	ns = b * sc->pagesize;
+ 	VTAILQ_FOREACH(sp2, sp->flist, status) {
+ 		assert(sp2->size >= ns);
+ 		assert(sp2->alloc == 0);
+ 		assert(sp2->flist == sp->flist);
+ 		if (sp->offset < sp2->offset)
+ 			break;
+ 	}
+ 	if (sp2 == NULL)
+ 		VTAILQ_INSERT_TAIL(sp->flist, sp, status);
+ 	else
+ 		VTAILQ_INSERT_BEFORE(sp2, sp, status);
+ }
+ 
+ static void
+ remfree(const struct smf_sc *sc, struct smf *sp)
+ {
+ 	size_t b;
+ 
+ 	assert(sp->alloc == 0);
+ 	assert(sp->flist != NULL);
+ 	Lck_AssertHeld(&sc->mtx);
+ 	b = sp->size / sc->pagesize;
+ 	if (b >= NBUCKET) {
+ 		b = NBUCKET - 1;
+ 		sc->stats->g_smf_large--;
+ 	} else {
+ 		sc->stats->g_smf_frag--;
+ 	}
+ 	assert(sp->flist == &sc->free[b]);
+ 	VTAILQ_REMOVE(sp->flist, sp, status);
+ 	sp->flist = NULL;
+ }
+ 
+ /*--------------------------------------------------------------------
+  * Allocate a range from the first free range that is large enough.
+  */
+ 
+ static struct smf *
+ alloc_smf(struct smf_sc *sc, size_t bytes)
+ {
+ 	struct smf *sp, *sp2;
+ 	size_t b;
+ 
+ 	assert(!(bytes % sc->pagesize));
+ 	b = bytes / sc->pagesize;
+ 	if (b >= NBUCKET)
+ 		b = NBUCKET - 1;
+ 	sp = NULL;
+ 	for (; b < NBUCKET - 1; b++) {
+ 		sp = VTAILQ_FIRST(&sc->free[b]);
+ 		if (sp != NULL)
+ 			break;
+ 	}
+ 	if (sp == NULL) {
+ 		VTAILQ_FOREACH(sp, &sc->free[NBUCKET -1], status)
+ 			if (sp->size >= bytes)
+ 				break;
+ 	}
+ 	if (sp == NULL)
+ 		return (sp);
+ 
+ 	assert(sp->size >= bytes);
+ 	remfree(sc, sp);
+ 
+ 	if (sp->size == bytes) {
+ 		sp->alloc = 1;
+ 		VTAILQ_INSERT_TAIL(&sc->used, sp, status);
+ 		return (sp);
+ 	}
+ 
+ 	/* Split from front */
+ 	sp2 = malloc(sizeof *sp2);
+ 	XXXAN(sp2);
+ 	sc->stats->g_smf++;
+ 	*sp2 = *sp;
+ 
+ 	sp->offset += bytes;
+ 	sp->ptr += bytes;
+ 	sp->size -= bytes;
+ 
+ 	sp2->size = bytes;
+ 	sp2->alloc = 1;
+ 	VTAILQ_INSERT_BEFORE(sp, sp2, order);
+ 	VTAILQ_INSERT_TAIL(&sc->used, sp2, status);
+ 	insfree(sc, sp);
+ 	return (sp2);
+ }
+ 
+ /*--------------------------------------------------------------------
+  * Free a range.  Attempt merge forward and backward, then sort into
+  * free list according to age.
+  */
+ 
+ static void
+ free_smf(struct smf *sp)
+ {
+ 	struct smf *sp2;
+ 	struct smf_sc *sc = sp->sc;
+ 
+ 	CHECK_OBJ_NOTNULL(sp, SMF_MAGIC);
+ 	assert(sp->alloc != 0);
+ 	assert(sp->size > 0);
+ 	assert(!(sp->size % sc->pagesize));
+ 	VTAILQ_REMOVE(&sc->used, sp, status);
+ 	sp->alloc = 0;
+ 
+ 	sp2 = VTAILQ_NEXT(sp, order);
+ 	if (sp2 != NULL &&
+ 	    sp2->alloc == 0 &&
+ 	    (sp2->ptr == sp->ptr + sp->size) &&
+ 	    (sp2->offset == sp->offset + sp->size)) {
+ 		sp->size += sp2->size;
+ 		VTAILQ_REMOVE(&sc->order, sp2, order);
+ 		remfree(sc, sp2);
+ 		free(sp2);
+ 		sc->stats->g_smf--;
+ 	}
+ 
+ 	sp2 = VTAILQ_PREV(sp, smfhead, order);
+ 	if (sp2 != NULL &&
+ 	    sp2->alloc == 0 &&
+ 	    (sp->ptr == sp2->ptr + sp2->size) &&
+ 	    (sp->offset == sp2->offset + sp2->size)) {
+ 		remfree(sc, sp2);
+ 		sp2->size += sp->size;
+ 		VTAILQ_REMOVE(&sc->order, sp, order);
+ 		free(sp);
+ 		sc->stats->g_smf--;
+ 		sp = sp2;
+ 	}
+ 
+ 	insfree(sc, sp);
+ }
+ 
+ /*--------------------------------------------------------------------
+  * Trim the tail of a range.
+  */
+ 
+ static void
+ trim_smf(struct smf *sp, size_t bytes)
+ {
+ 	struct smf *sp2;
+ 	struct smf_sc *sc = sp->sc;
+ 
+ 	assert(sp->alloc != 0);
+ 	assert(bytes > 0);
+ 	assert(bytes < sp->size);
+ 	assert(!(bytes % sc->pagesize));
+ 	assert(!(sp->size % sc->pagesize));
+ 	CHECK_OBJ_NOTNULL(sp, SMF_MAGIC);
+ 	sp2 = malloc(sizeof *sp2);
+ 	XXXAN(sp2);
+ 	sc->stats->g_smf++;
+ 	*sp2 = *sp;
+ 
+ 	sp2->size -= bytes;
+ 	sp->size = bytes;
+ 	sp2->ptr += bytes;
+ 	sp2->offset += bytes;
+ 	VTAILQ_INSERT_AFTER(&sc->order, sp, sp2, order);
+ 	VTAILQ_INSERT_TAIL(&sc->used, sp2, status);
+ 	free_smf(sp2);
+ }
+ 
+ /*--------------------------------------------------------------------
+  * Insert a newly created range as busy, then free it to do any collapses
+  */
+ 
+ static void
+ new_smf(struct smf_sc *sc, unsigned char *ptr, off_t off, size_t len)
+ {
+ 	struct smf *sp, *sp2;
+ 
+ 	assert(!(len % sc->pagesize));
+ 	sp = calloc(sizeof *sp, 1);
+ 	XXXAN(sp);
+ 	sp->magic = SMF_MAGIC;
+ 	sp->s.magic = STORAGE_MAGIC;
+ 	sc->stats->g_smf++;
+ 
+ 	sp->sc = sc;
+ 	sp->size = len;
+ 	sp->ptr = ptr;
+ 	sp->offset = off;
+ 	sp->alloc = 1;
+ 
+ 	VTAILQ_FOREACH(sp2, &sc->order, order) {
+ 		if (sp->ptr < sp2->ptr) {
+ 			VTAILQ_INSERT_BEFORE(sp2, sp, order);
+ 			break;
+ 		}
+ 	}
+ 	if (sp2 == NULL)
+ 		VTAILQ_INSERT_TAIL(&sc->order, sp, order);
+ 
+ 	VTAILQ_INSERT_HEAD(&sc->used, sp, status);
+ 
+ 	free_smf(sp);
+ }
+ 
+ /*--------------------------------------------------------------------*/
+ 
+ /*
+  * XXX: This may be too aggressive and soak up too much address room.
+  * XXX: On the other hand, the user, directly or implicitly asked us to
+  * XXX: use this much storage, so we should make a decent effort.
+  * XXX: worst case (I think), malloc will fail.
+  */
+ 
+ static void
+ smf_open_chunk(struct smf_sc *sc, off_t sz, off_t off, off_t *fail, off_t *sum)
+ {
+ 	void *p;
+ 	off_t h;
+ 
+ 	assert(sz != 0);
+ 	assert(!(sz % sc->pagesize));
+ 
+ 	if (*fail < (uintmax_t)sc->pagesize * MINPAGES)
+ 		return;
+ 
+ 	if (sz > 0 && sz < *fail && sz < SSIZE_MAX) {
+ 		p = mmap(NULL, sz, PROT_READ|PROT_WRITE,
+ 		    MAP_NOCORE | MAP_NOSYNC | MAP_SHARED, sc->fd, off);
+ 		if (p != MAP_FAILED) {
+ 			(void) madvise(p, sz, MADV_RANDOM);
+ 			(*sum) += sz;
+ 			new_smf(sc, p, off, sz);
+ 			return;
+ 		}
+ 	}
+ 
+ 	if (sz < *fail)
+ 		*fail = sz;
+ 
+ 	h = sz / 2;
+ 	if (h > SSIZE_MAX)
+ 		h = SSIZE_MAX;
+ 	h -= (h % sc->pagesize);
+ 
+ 	smf_open_chunk(sc, h, off, fail, sum);
+ 	smf_open_chunk(sc, sz - h, off + h, fail, sum);
+ }
+ 
+ static void
+ smf_open(const struct stevedore *st)
+ {
+ 	struct smf_sc *sc;
+ 	off_t fail = 1 << 30;	/* XXX: where is OFF_T_MAX ? */
+ 	off_t sum = 0;
+ 
+ 	CAST_OBJ_NOTNULL(sc, st->priv, SMF_SC_MAGIC);
+ 	sc->stats = VSM_Alloc(sizeof *sc->stats,
+ 	    VSC_CLASS, VSC_TYPE_SMF, st->ident);
+ 	Lck_New(&sc->mtx, lck_smf);
+ 	Lck_Lock(&sc->mtx);
+ 	smf_open_chunk(sc, sc->filesize, 0, &fail, &sum);
+ 	Lck_Unlock(&sc->mtx);
+ 	printf("SMF.%s mmap'ed %ju bytes of %ju\n",
+ 	    st->ident, (uintmax_t)sum, sc->filesize);
+ 
+ 	/* XXX */
+ 	if (sum < MINPAGES * (off_t)getpagesize())
+ 		exit (2);
+ 
+ 	sc->stats->g_space += sc->filesize;
+ }
+ 
+ /*--------------------------------------------------------------------*/
+ 
+ static struct storage *
+ smf_alloc(struct stevedore *st, size_t size)
+ {
+ 	struct smf *smf;
+ 	struct smf_sc *sc;
+ 
+ 	CAST_OBJ_NOTNULL(sc, st->priv, SMF_SC_MAGIC);
+ 	assert(size > 0);
+ 	size += (sc->pagesize - 1);
+ 	size &= ~(sc->pagesize - 1);
+ 	Lck_Lock(&sc->mtx);
+ 	sc->stats->c_req++;
+ 	smf = alloc_smf(sc, size);
+ 	if (smf == NULL) {
+ 		sc->stats->c_fail++;
+ 		Lck_Unlock(&sc->mtx);
+ 		return (NULL);
+ 	}
+ 	CHECK_OBJ_NOTNULL(smf, SMF_MAGIC);
+ 	sc->stats->g_alloc++;
+ 	sc->stats->c_bytes += smf->size;
+ 	sc->stats->g_bytes += smf->size;
+ 	sc->stats->g_space -= smf->size;
+ 	Lck_Unlock(&sc->mtx);
+ 	CHECK_OBJ_NOTNULL(&smf->s, STORAGE_MAGIC);	/*lint !e774 */
+ 	XXXAN(smf);
+ 	assert(smf->size == size);
+ 	smf->s.space = size;
+ 	smf->s.priv = smf;
+ 	smf->s.ptr = smf->ptr;
+ 	smf->s.len = 0;
+ 	smf->s.stevedore = st;
+ #ifdef SENDFILE_WORKS
+ 	smf->s.fd = smf->sc->fd;
+ 	smf->s.where = smf->offset;
+ #endif
+ 	return (&smf->s);
+ }
+ 
+ /*--------------------------------------------------------------------*/
+ 
+ static void
+ smf_trim(struct storage *s, size_t size)
+ {
+ 	struct smf *smf;
+ 	struct smf_sc *sc;
+ 
+ 	CHECK_OBJ_NOTNULL(s, STORAGE_MAGIC);
+ 	assert(size > 0);
+ 	assert(size <= s->space);
+ 	xxxassert(size > 0);	/* XXX: seen */
+ 	CAST_OBJ_NOTNULL(smf, s->priv, SMF_MAGIC);
+ 	assert(size <= smf->size);
+ 	sc = smf->sc;
+ 	size += (sc->pagesize - 1);
+ 	size &= ~(sc->pagesize - 1);
+ 	if (smf->size > size) {
+ 		Lck_Lock(&sc->mtx);
+ 		sc->stats->c_freed += (smf->size - size);
+ 		sc->stats->g_bytes -= (smf->size - size);
+ 		sc->stats->g_space += (smf->size - size);
+ 		trim_smf(smf, size);
+ 		assert(smf->size == size);
+ 		Lck_Unlock(&sc->mtx);
+ 		s->space = size;
+ 	}
+ }
+ 
+ /*--------------------------------------------------------------------*/
+ 
+ static void __match_proto__(storage_free_f)
+ smf_free(struct storage *s)
+ {
+ 	struct smf *smf;
+ 	struct smf_sc *sc;
+ 
+ 	CHECK_OBJ_NOTNULL(s, STORAGE_MAGIC);
+ 	CAST_OBJ_NOTNULL(smf, s->priv, SMF_MAGIC);
+ 	sc = smf->sc;
+ 	Lck_Lock(&sc->mtx);
+ 	sc->stats->g_alloc--;
+ 	sc->stats->c_freed += smf->size;
+ 	sc->stats->g_bytes -= smf->size;
+ 	sc->stats->g_space += smf->size;
+ 	free_smf(smf);
+ 	Lck_Unlock(&sc->mtx);
+ }
+ 
+ /*--------------------------------------------------------------------*/
+ 
+ const struct stevedore smf_stevedore = {
+ 	.magic	=	STEVEDORE_MAGIC,
+ 	.name	=	"file",
+ 	.init	=	smf_init,
+ 	.open	=	smf_open,
+ 	.alloc	=	smf_alloc,
+ 	.trim	=	smf_trim,
+ 	.free	=	smf_free,
++        .dup    =       default_dup,
+ };
+ 
+ #ifdef INCLUDE_TEST_DRIVER
+ 
+ void vca_flush(struct sess *sp) {}
+ 
+ #define N	100
+ #define M	(128*1024)
+ 
+ struct storage *s[N];
+ 
+ static void
+ dumpit(void)
+ {
+ 	struct smf_sc *sc = smf_stevedore.priv;
+ 	struct smf *s;
+ 
+ 	return (0);
+ 	printf("----------------\n");
+ 	printf("Order:\n");
+ 	VTAILQ_FOREACH(s, &sc->order, order) {
+ 		printf("%10p %12ju %12ju %12ju\n",
+ 		    s, s->offset, s->size, s->offset + s->size);
+ 	}
+ 	printf("Used:\n");
+ 	VTAILQ_FOREACH(s, &sc->used, status) {
+ 		printf("%10p %12ju %12ju %12ju\n",
+ 		    s, s->offset, s->size, s->offset + s->size);
+ 	}
+ 	printf("Free:\n");
+ 	VTAILQ_FOREACH(s, &sc->free, status) {
+ 		printf("%10p %12ju %12ju %12ju\n",
+ 		    s, s->offset, s->size, s->offset + s->size);
+ 	}
+ 	printf("================\n");
+ }
+ 
+ int
+ main(int argc, char **argv)
+ {
+ 	int i, j;
+ 
+ 	setbuf(stdout, NULL);
+ 	smf_init(&smf_stevedore, "");
+ 	smf_open(&smf_stevedore);
+ 	while (1) {
+ 		dumpit();
+ 		i = random() % N;
+ 		do
+ 			j = random() % M;
+ 		while (j == 0);
+ 		if (s[i] == NULL) {
+ 			s[i] = smf_alloc(&smf_stevedore, j);
+ 			printf("A %10p %12d\n", s[i], j);
+ 		} else if (j < s[i]->space) {
+ 			smf_trim(s[i], j);
+ 			printf("T %10p %12d\n", s[i], j);
+ 		} else {
+ 			smf_free(s[i]);
+ 			printf("D %10p\n", s[i]);
+ 			s[i] = NULL;
+ 		}
+ 	}
+ }
+ 
+ #endif /* INCLUDE_TEST_DRIVER */
diff --cc bin/varnishd/storage/storage_malloc.c
index 0000000,156c832..79fefb6
mode 000000,100644..100644
--- a/bin/varnishd/storage/storage_malloc.c
+++ b/bin/varnishd/storage/storage_malloc.c
@@@ -1,0 -1,256 +1,257 @@@
+ /*-
+  * Copyright (c) 2006 Verdens Gang AS
+  * Copyright (c) 2006-2011 Varnish Software AS
+  * All rights reserved.
+  *
+  * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
+  *
+  * Redistribution and use in source and binary forms, with or without
+  * modification, are permitted provided that the following conditions
+  * are met:
+  * 1. Redistributions of source code must retain the above copyright
+  *    notice, this list of conditions and the following disclaimer.
+  * 2. Redistributions in binary form must reproduce the above copyright
+  *    notice, this list of conditions and the following disclaimer in the
+  *    documentation and/or other materials provided with the distribution.
+  *
+  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+  * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+  * SUCH DAMAGE.
+  *
+  * Storage method based on malloc(3)
+  */
+ 
+ #include "config.h"
+ 
+ #include <stdio.h>
+ #include <stdlib.h>
+ 
+ #include "cache/cache.h"
+ #include "storage/storage.h"
+ 
+ #include "vnum.h"
+ 
+ struct sma_sc {
+ 	unsigned		magic;
+ #define SMA_SC_MAGIC		0x1ac8a345
+ 	struct lock		sma_mtx;
+ 	size_t			sma_max;
+ 	size_t			sma_alloc;
+ 	struct VSC_C_sma	*stats;
+ };
+ 
+ struct sma {
+ 	unsigned		magic;
+ #define SMA_MAGIC		0x69ae9bb9
+ 	struct storage		s;
+ 	size_t			sz;
+ 	struct sma_sc		*sc;
+ };
+ 
+ static struct storage *
+ sma_alloc(struct stevedore *st, size_t size)
+ {
+ 	struct sma_sc *sma_sc;
+ 	struct sma *sma = NULL;
+ 	void *p;
+ 
+ 	CAST_OBJ_NOTNULL(sma_sc, st->priv, SMA_SC_MAGIC);
+ 	Lck_Lock(&sma_sc->sma_mtx);
+ 	sma_sc->stats->c_req++;
+ 	if (sma_sc->sma_alloc + size > sma_sc->sma_max) {
+ 		sma_sc->stats->c_fail += size;
+ 		size = 0;
+ 	} else {
+ 		sma_sc->sma_alloc += size;
+ 		sma_sc->stats->c_bytes += size;
+ 		sma_sc->stats->g_alloc++;
+ 		sma_sc->stats->g_bytes += size;
+ 		if (sma_sc->sma_max != SIZE_MAX)
+ 			sma_sc->stats->g_space -= size;
+ 	}
+ 	Lck_Unlock(&sma_sc->sma_mtx);
+ 
+ 	if (size == 0)
+ 		return (NULL);
+ 
+ 	/*
+ 	 * Do not collaps the sma allocation with sma->s.ptr: it is not
+ 	 * a good idea.  Not only would it make ->trim impossible,
+ 	 * performance-wise it would be a catastropy with chunksized
+ 	 * allocations growing another full page, just to accomodate the sma.
+ 	 */
+ 
+ 	p = malloc(size);
+ 	if (p != NULL) {
+ 		ALLOC_OBJ(sma, SMA_MAGIC);
+ 		if (sma != NULL)
+ 			sma->s.ptr = p;
+ 		else
+ 			free(p);
+ 	}
+ 	if (sma == NULL) {
+ 		Lck_Lock(&sma_sc->sma_mtx);
+ 		/*
+ 		 * XXX: Not nice to have counters go backwards, but we do
+ 		 * XXX: Not want to pick up the lock twice just for stats.
+ 		 */
+ 		sma_sc->stats->c_fail++;
+ 		sma_sc->stats->c_bytes -= size;
+ 		sma_sc->stats->g_alloc--;
+ 		sma_sc->stats->g_bytes -= size;
+ 		if (sma_sc->sma_max != SIZE_MAX)
+ 			sma_sc->stats->g_space += size;
+ 		Lck_Unlock(&sma_sc->sma_mtx);
+ 		return (NULL);
+ 	}
+ 	sma->sc = sma_sc;
+ 	sma->sz = size;
+ 	sma->s.priv = sma;
+ 	sma->s.len = 0;
+ 	sma->s.space = size;
+ #ifdef SENDFILE_WORKS
+ 	sma->s.fd = -1;
+ #endif
+ 	sma->s.stevedore = st;
+ 	sma->s.magic = STORAGE_MAGIC;
+ 	return (&sma->s);
+ }
+ 
+ static void __match_proto__(storage_free_f)
+ sma_free(struct storage *s)
+ {
+ 	struct sma_sc *sma_sc;
+ 	struct sma *sma;
+ 
+ 	CHECK_OBJ_NOTNULL(s, STORAGE_MAGIC);
+ 	CAST_OBJ_NOTNULL(sma, s->priv, SMA_MAGIC);
+ 	sma_sc = sma->sc;
+ 	assert(sma->sz == sma->s.space);
+ 	Lck_Lock(&sma_sc->sma_mtx);
+ 	sma_sc->sma_alloc -= sma->sz;
+ 	sma_sc->stats->g_alloc--;
+ 	sma_sc->stats->g_bytes -= sma->sz;
+ 	sma_sc->stats->c_freed += sma->sz;
+ 	if (sma_sc->sma_max != SIZE_MAX)
+ 		sma_sc->stats->g_space += sma->sz;
+ 	Lck_Unlock(&sma_sc->sma_mtx);
+ 	free(sma->s.ptr);
+ 	free(sma);
+ }
+ 
+ static void
+ sma_trim(struct storage *s, size_t size)
+ {
+ 	struct sma_sc *sma_sc;
+ 	struct sma *sma;
+ 	void *p;
+ 	size_t delta;
+ 
+ 	CHECK_OBJ_NOTNULL(s, STORAGE_MAGIC);
+ 	CAST_OBJ_NOTNULL(sma, s->priv, SMA_MAGIC);
+ 	sma_sc = sma->sc;
+ 
+ 	assert(sma->sz == sma->s.space);
+ 	assert(size < sma->sz);
+ 	delta = sma->sz - size;
+ 	if (delta < 256)
+ 		return;
+ 	if ((p = realloc(sma->s.ptr, size)) != NULL) {
+ 		Lck_Lock(&sma_sc->sma_mtx);
+ 		sma_sc->sma_alloc -= delta;
+ 		sma_sc->stats->g_bytes -= delta;
+ 		sma_sc->stats->c_freed += delta;
+ 		if (sma_sc->sma_max != SIZE_MAX)
+ 			sma_sc->stats->g_space += delta;
+ 		sma->sz = size;
+ 		Lck_Unlock(&sma_sc->sma_mtx);
+ 		sma->s.ptr = p;
+ 		s->space = size;
+ 	}
+ }
+ 
+ static double
+ sma_used_space(const struct stevedore *st)
+ {
+ 	struct sma_sc *sma_sc;
+ 
+ 	CAST_OBJ_NOTNULL(sma_sc, st->priv, SMA_SC_MAGIC);
+ 	return (sma_sc->sma_alloc);
+ }
+ 
+ static double
+ sma_free_space(const struct stevedore *st)
+ {
+ 	struct sma_sc *sma_sc;
+ 
+ 	CAST_OBJ_NOTNULL(sma_sc, st->priv, SMA_SC_MAGIC);
+ 	return (sma_sc->sma_max - sma_sc->sma_alloc);
+ }
+ 
+ static void
+ sma_init(struct stevedore *parent, int ac, char * const *av)
+ {
+ 	const char *e;
+ 	uintmax_t u;
+ 	struct sma_sc *sc;
+ 
+ 	ASSERT_MGT();
+ 	ALLOC_OBJ(sc, SMA_SC_MAGIC);
+ 	AN(sc);
+ 	sc->sma_max = SIZE_MAX;
+ 	assert(sc->sma_max == SIZE_MAX);
+ 	parent->priv = sc;
+ 
+ 	AZ(av[ac]);
+ 	if (ac > 1)
+ 		ARGV_ERR("(-smalloc) too many arguments\n");
+ 
+ 	if (ac == 0 || *av[0] == '\0')
+ 		 return;
+ 
+ 	e = VNUM_2bytes(av[0], &u, 0);
+ 	if (e != NULL)
+ 		ARGV_ERR("(-smalloc) size \"%s\": %s\n", av[0], e);
+ 	if ((u != (uintmax_t)(size_t)u))
+ 		ARGV_ERR("(-smalloc) size \"%s\": too big\n", av[0]);
+ 	if (u < 1024*1024)
+ 		ARGV_ERR("(-smalloc) size \"%s\": too small, "
+ 			 "did you forget to specify M or G?\n", av[0]);
+ 
+ 	sc->sma_max = u;
+ }
+ 
+ static void
+ sma_open(const struct stevedore *st)
+ {
+ 	struct sma_sc *sma_sc;
+ 
+ 	CAST_OBJ_NOTNULL(sma_sc, st->priv, SMA_SC_MAGIC);
+ 	Lck_New(&sma_sc->sma_mtx, lck_sma);
+ 	sma_sc->stats = VSM_Alloc(sizeof *sma_sc->stats,
+ 	    VSC_CLASS, VSC_TYPE_SMA, st->ident);
+ 	memset(sma_sc->stats, 0, sizeof *sma_sc->stats);
+ 	if (sma_sc->sma_max != SIZE_MAX)
+ 		sma_sc->stats->g_space = sma_sc->sma_max;
+ }
+ 
+ const struct stevedore sma_stevedore = {
+ 	.magic	=	STEVEDORE_MAGIC,
+ 	.name	=	"malloc",
+ 	.init	=	sma_init,
+ 	.open	=	sma_open,
+ 	.alloc	=	sma_alloc,
+ 	.free	=	sma_free,
+ 	.trim	=	sma_trim,
+ 	.var_free_space =	sma_free_space,
+ 	.var_used_space =	sma_used_space,
++        .dup    =       default_dup,
+ };
diff --cc bin/varnishd/storage/storage_persistent.c
index 0000000,ded638b..095fcf2
mode 000000,100644..100644
--- a/bin/varnishd/storage/storage_persistent.c
+++ b/bin/varnishd/storage/storage_persistent.c
@@@ -1,0 -1,678 +1,679 @@@
+ /*-
+  * Copyright (c) 2008-2011 Varnish Software AS
+  * All rights reserved.
+  *
+  * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
+  *
+  * Redistribution and use in source and binary forms, with or without
+  * modification, are permitted provided that the following conditions
+  * are met:
+  * 1. Redistributions of source code must retain the above copyright
+  *    notice, this list of conditions and the following disclaimer.
+  * 2. Redistributions in binary form must reproduce the above copyright
+  *    notice, this list of conditions and the following disclaimer in the
+  *    documentation and/or other materials provided with the distribution.
+  *
+  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+  * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+  * SUCH DAMAGE.
+  *
+  * Persistent storage method
+  *
+  * XXX: Before we start the client or maybe after it stops, we should give the
+  * XXX: stevedores a chance to examine their storage for consistency.
+  *
+  * XXX: Do we ever free the LRU-lists ?
+  */
+ 
+ #include "config.h"
+ 
+ #include <sys/param.h>
+ #include <sys/mman.h>
+ 
+ #include <stdint.h>
+ #include <stdio.h>
+ #include <stdlib.h>
+ #include <string.h>
+ 
+ #include "cache/cache.h"
+ #include "storage/storage.h"
+ 
+ #include "hash/hash_slinger.h"
+ #include "vcli.h"
+ #include "vcli_priv.h"
+ #include "vend.h"
+ #include "vsha256.h"
+ 
+ #include "persistent.h"
+ #include "storage/storage_persistent.h"
+ 
+ /*--------------------------------------------------------------------*/
+ 
+ /*
+  * silos is unlocked, it only changes during startup when we are
+  * single-threaded
+  */
+ static VTAILQ_HEAD(,smp_sc)	silos = VTAILQ_HEAD_INITIALIZER(silos);
+ 
+ /*--------------------------------------------------------------------
+  * Add bans to silos
+  */
+ 
+ static void
+ smp_appendban(struct smp_sc *sc, struct smp_signctx *ctx,
+     uint32_t len, const uint8_t *ban)
+ {
+ 	uint8_t *ptr, *ptr2;
+ 
+ 	(void)sc;
+ 	ptr = ptr2 = SIGN_END(ctx);
+ 
+ 	memcpy(ptr, "BAN", 4);
+ 	ptr += 4;
+ 
+ 	vbe32enc(ptr, len);
+ 	ptr += 4;
+ 
+ 	memcpy(ptr, ban, len);
+ 	ptr += len;
+ 
+ 	smp_append_sign(ctx, ptr2, ptr - ptr2);
+ }
+ 
+ /* Trust that cache_ban.c takes care of locking */
+ 
+ void
+ SMP_NewBan(const uint8_t *ban, unsigned ln)
+ {
+ 	struct smp_sc *sc;
+ 
+ 	VTAILQ_FOREACH(sc, &silos, list) {
+ 		smp_appendban(sc, &sc->ban1, ln, ban);
+ 		smp_appendban(sc, &sc->ban2, ln, ban);
+ 	}
+ }
+ 
+ /*--------------------------------------------------------------------
+  * Attempt to open and read in a ban list
+  */
+ 
+ static int
+ smp_open_bans(struct smp_sc *sc, struct smp_signctx *ctx)
+ {
+ 	uint8_t *ptr, *pe;
+ 	uint32_t length;
+ 	int i, retval = 0;
+ 
+ 	ASSERT_CLI();
+ 	(void)sc;
+ 	i = smp_chk_sign(ctx);
+ 	if (i)
+ 		return (i);
+ 	ptr = SIGN_DATA(ctx);
+ 	pe = ptr + ctx->ss->length;
+ 
+ 	while (ptr < pe) {
+ 		if (memcmp(ptr, "BAN", 4)) {
+ 			retval = 1001;
+ 			break;
+ 		}
+ 		ptr += 4;
+ 
+ 		length = vbe32dec(ptr);
+ 		ptr += 4;
+ 
+ 		if (ptr + length > pe) {
+ 			retval = 1003;
+ 			break;
+ 		}
+ 
+ 		BAN_Reload(ptr, length);
+ 
+ 		ptr += length;
+ 	}
+ 	assert(ptr <= pe);
+ 	return (retval);
+ }
+ 
+ /*--------------------------------------------------------------------
+  * Attempt to open and read in a segment list
+  */
+ 
+ static int
+ smp_open_segs(struct smp_sc *sc, struct smp_signctx *ctx)
+ {
+ 	uint64_t length, l;
+ 	struct smp_segptr *ss, *se;
+ 	struct smp_seg *sg, *sg1, *sg2;
+ 	int i, n = 0;
+ 
+ 	ASSERT_CLI();
+ 	i = smp_chk_sign(ctx);
+ 	if (i)
+ 		return (i);
+ 
+ 	ss = SIGN_DATA(ctx);
+ 	length = ctx->ss->length;
+ 
+ 	if (length == 0) {
+ 		/* No segments */
+ 		sc->free_offset = sc->ident->stuff[SMP_SPC_STUFF];
+ 		return (0);
+ 	}
+ 	se = ss + length / sizeof *ss;
+ 	se--;
+ 	assert(ss <= se);
+ 
+ 	/*
+ 	 * Locate the free reserve, there are only two basic cases,
+ 	 * but once we start dropping segments, things gets more complicated.
+ 	 */
+ 
+ 	sc->free_offset = se->offset + se->length;
+ 	l = sc->mediasize - sc->free_offset;
+ 	if (se->offset > ss->offset && l >= sc->free_reserve) {
+ 		/*
+ 		 * [__xxxxyyyyzzzz___]
+ 		 * Plenty of space at tail, do nothing.
+ 		 */
+ 	} else if (ss->offset > se->offset) {
+ 		/*
+ 		 * [zzzz____xxxxyyyy_]
+ 		 * (make) space between ends
+ 		 * We might nuke the entire tail end without getting
+ 		 * enough space, in which case we fall through to the
+ 		 * last check.
+ 		 */
+ 		while (ss < se && ss->offset > se->offset) {
+ 			l = ss->offset - (se->offset + se->length);
+ 			if (l > sc->free_reserve)
+ 				break;
+ 			ss++;
+ 			n++;
+ 		}
+ 	}
+ 
+ 	if (l < sc->free_reserve) {
+ 		/*
+ 		 * [__xxxxyyyyzzzz___]
+ 		 * (make) space at front
+ 		 */
+ 		sc->free_offset = sc->ident->stuff[SMP_SPC_STUFF];
+ 		while (ss < se) {
+ 			l = ss->offset - sc->free_offset;
+ 			if (l > sc->free_reserve)
+ 				break;
+ 			ss++;
+ 			n++;
+ 		}
+ 	}
+ 
+ 	assert (l >= sc->free_reserve);
+ 
+ 
+ 	sg1 = NULL;
+ 	sg2 = NULL;
+ 	for(; ss <= se; ss++) {
+ 		ALLOC_OBJ(sg, SMP_SEG_MAGIC);
+ 		AN(sg);
+ 		sg->lru = LRU_Alloc();
+ 		CHECK_OBJ_NOTNULL(sg->lru, LRU_MAGIC);
+ 		sg->p = *ss;
+ 
+ 		sg->flags |= SMP_SEG_MUSTLOAD;
+ 
+ 		/*
+ 		 * HACK: prevent save_segs from nuking segment until we have
+ 		 * HACK: loaded it.
+ 		 */
+ 		sg->nobj = 1;
+ 		if (sg1 != NULL) {
+ 			assert(sg1->p.offset != sg->p.offset);
+ 			if (sg1->p.offset < sg->p.offset)
+ 				assert(smp_segend(sg1) <= sg->p.offset);
+ 			else
+ 				assert(smp_segend(sg) <= sg1->p.offset);
+ 		}
+ 		if (sg2 != NULL) {
+ 			assert(sg2->p.offset != sg->p.offset);
+ 			if (sg2->p.offset < sg->p.offset)
+ 				assert(smp_segend(sg2) <= sg->p.offset);
+ 			else
+ 				assert(smp_segend(sg) <= sg2->p.offset);
+ 		}
+ 
+ 		/* XXX: check that they are inside silo */
+ 		/* XXX: check that they don't overlap */
+ 		/* XXX: check that they are serial */
+ 		sg->sc = sc;
+ 		VTAILQ_INSERT_TAIL(&sc->segments, sg, list);
+ 		sg2 = sg;
+ 		if (sg1 == NULL)
+ 			sg1 = sg;
+ 	}
+ 	printf("Dropped %d segments to make free_reserve\n", n);
+ 	return (0);
+ }
+ 
+ /*--------------------------------------------------------------------
+  * Silo worker thread
+  */
+ 
+ static void *
+ smp_thread(struct sess *sp, void *priv)
+ {
+ 	struct smp_sc	*sc;
+ 	struct smp_seg *sg;
+ 
+ 	(void)sp;
+ 	CAST_OBJ_NOTNULL(sc, priv, SMP_SC_MAGIC);
+ 
+ 	/* First, load all the objects from all segments */
+ 	VTAILQ_FOREACH(sg, &sc->segments, list)
+ 		if (sg->flags & SMP_SEG_MUSTLOAD)
+ 			smp_load_seg(sp, sc, sg);
+ 
+ 	sc->flags |= SMP_SC_LOADED;
+ 	BAN_TailDeref(&sc->tailban);
+ 	AZ(sc->tailban);
+ 	printf("Silo completely loaded\n");
+ 	while (1) {
+ 		(void)sleep (1);
+ 		sg = VTAILQ_FIRST(&sc->segments);
+ 		if (sg != NULL && sg -> sc->cur_seg &&
+ 		    sg->nobj == 0) {
+ 			Lck_Lock(&sc->mtx);
+ 			smp_save_segs(sc);
+ 			Lck_Unlock(&sc->mtx);
+ 		}
+ 	}
+ 	NEEDLESS_RETURN(NULL);
+ }
+ 
+ /*--------------------------------------------------------------------
+  * Open a silo in the worker process
+  */
+ 
+ static void
+ smp_open(const struct stevedore *st)
+ {
+ 	struct smp_sc	*sc;
+ 
+ 	ASSERT_CLI();
+ 
+ 	CAST_OBJ_NOTNULL(sc, st->priv, SMP_SC_MAGIC);
+ 
+ 	Lck_New(&sc->mtx, lck_smp);
+ 	Lck_Lock(&sc->mtx);
+ 
+ 	sc->stevedore = st;
+ 
+ 	/* We trust the parent to give us a valid silo, for good measure: */
+ 	AZ(smp_valid_silo(sc));
+ 
+ 	AZ(mprotect(sc->base, 4096, PROT_READ));
+ 
+ 	sc->ident = SIGN_DATA(&sc->idn);
+ 
+ 	/* We attempt ban1 first, and if that fails, try ban2 */
+ 	if (smp_open_bans(sc, &sc->ban1))
+ 		AZ(smp_open_bans(sc, &sc->ban2));
+ 
+ 	/* We attempt seg1 first, and if that fails, try seg2 */
+ 	if (smp_open_segs(sc, &sc->seg1))
+ 		AZ(smp_open_segs(sc, &sc->seg2));
+ 
+ 	/*
+ 	 * Grap a reference to the tail of the ban list, until the thread
+ 	 * has loaded all objects, so we can be sure that all of our
+ 	 * proto-bans survive until then.
+ 	 */
+ 	sc->tailban = BAN_TailRef();
+ 	AN(sc->tailban);
+ 
+ 	/* XXX: save segments to ensure consistency between seg1 & seg2 ? */
+ 
+ 	/* XXX: abandon early segments to make sure we have free space ? */
+ 
+ 	/* Open a new segment, so we are ready to write */
+ 	smp_new_seg(sc);
+ 
+ 	/* Start the worker silo worker thread, it will load the objects */
+ 	WRK_BgThread(&sc->thread, "persistence", smp_thread, sc);
+ 
+ 	VTAILQ_INSERT_TAIL(&silos, sc, list);
+ 	Lck_Unlock(&sc->mtx);
+ }
+ 
+ /*--------------------------------------------------------------------
+  * Close a silo
+  */
+ 
+ static void
+ smp_close(const struct stevedore *st)
+ {
+ 	struct smp_sc	*sc;
+ 
+ 	ASSERT_CLI();
+ 
+ 	CAST_OBJ_NOTNULL(sc, st->priv, SMP_SC_MAGIC);
+ 	Lck_Lock(&sc->mtx);
+ 	smp_close_seg(sc, sc->cur_seg);
+ 	Lck_Unlock(&sc->mtx);
+ 
+ 	/* XXX: reap thread */
+ }
+ 
+ /*--------------------------------------------------------------------
+  * Allocate a bite.
+  *
+  * Allocate [min_size...max_size] space from the bottom of the segment,
+  * as is convenient.
+  *
+  * If 'so' + 'idx' is given, also allocate a smp_object from the top
+  * of the segment.
+  *
+  * Return the segment in 'ssg' if given.
+  */
+ 
+ static struct storage *
+ smp_allocx(struct stevedore *st, size_t min_size, size_t max_size,
+     struct smp_object **so, unsigned *idx, struct smp_seg **ssg)
+ {
+ 	struct smp_sc *sc;
+ 	struct storage *ss;
+ 	struct smp_seg *sg;
+ 	unsigned tries;
+ 	uint64_t left, extra;
+ 
+ 	CAST_OBJ_NOTNULL(sc, st->priv, SMP_SC_MAGIC);
+ 	assert(min_size <= max_size);
+ 
+ 	max_size = IRNUP(sc, max_size);
+ 	min_size = IRNUP(sc, min_size);
+ 
+ 	extra = IRNUP(sc, sizeof(*ss));
+ 	if (so != NULL) {
+ 		extra += sizeof(**so);
+ 		AN(idx);
+ 	}
+ 
+ 	Lck_Lock(&sc->mtx);
+ 	sg = NULL;
+ 	ss = NULL;
+ 	for (tries = 0; tries < 3; tries++) {
+ 		left = smp_spaceleft(sc, sc->cur_seg);
+ 		if (left >= extra + min_size)
+ 			break;
+ 		smp_close_seg(sc, sc->cur_seg);
+ 		smp_new_seg(sc);
+ 	}
+ 	if (left >= extra + min_size)  {
+ 		if (left < extra + max_size)
+ 			max_size = IRNDN(sc, left - extra);
+ 
+ 		sg = sc->cur_seg;
+ 		ss = (void*)(sc->base + sc->next_bot);
+ 		sc->next_bot += max_size + IRNUP(sc, sizeof(*ss));
+ 		sg->nalloc++;
+ 		if (so != NULL) {
+ 			sc->next_top -= sizeof(**so);
+ 			*so = (void*)(sc->base + sc->next_top);
+ 			/* Render this smp_object mostly harmless */
+ 			(*so)->ttl = 0.;
+ 			(*so)->ban = 0.;
+ 			(*so)->ptr = 0;;
+ 			sg->objs = *so;
+ 			*idx = ++sg->p.lobjlist;
+ 		}
+ 		(void)smp_spaceleft(sc, sg);	/* for the assert */
+ 	}
+ 	Lck_Unlock(&sc->mtx);
+ 
+ 	if (ss == NULL)
+ 		return (ss);
+ 	AN(sg);
+ 	assert(max_size >= min_size);
+ 
+ 	/* Fill the storage structure */
+ 	memset(ss, 0, sizeof *ss);
+ 	ss->magic = STORAGE_MAGIC;
+ 	ss->ptr = PRNUP(sc, ss + 1);
+ 	ss->space = max_size;
+ 	ss->priv = sc;
+ 	ss->stevedore = st;
+ #ifdef SENDFILE_WORKS
+ 	ss->fd = sc->fd;
+ #endif
+ 	if (ssg != NULL)
+ 		*ssg = sg;
+ 	return (ss);
+ }
+ 
+ /*--------------------------------------------------------------------
+  * Allocate an object
+  */
+ 
+ static struct object *
+ smp_allocobj(struct stevedore *stv, struct sess *sp, unsigned ltot,
+     const struct stv_objsecrets *soc)
+ {
+ 	struct object *o;
+ 	struct storage *st;
+ 	struct smp_sc	*sc;
+ 	struct smp_seg *sg;
+ 	struct smp_object *so;
+ 	struct objcore *oc;
+ 	unsigned objidx;
+ 
+ 	if (sp->objcore == NULL)
+ 		return (NULL);		/* from cnt_error */
+ 	CAST_OBJ_NOTNULL(sc, stv->priv, SMP_SC_MAGIC);
+ 	AN(sp->objcore);
+ 	AN(sp->wrk->exp.ttl > 0.);
+ 
+ 	ltot = IRNUP(sc, ltot);
+ 
+ 	st = smp_allocx(stv, ltot, ltot, &so, &objidx, &sg);
+ 	if (st == NULL)
+ 		return (NULL);
+ 
+ 	assert(st->space >= ltot);
+ 	ltot = st->len = st->space;
+ 
+ 	o = STV_MkObject(sp, st->ptr, ltot, soc);
+ 	CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC);
+ 	o->objstore = st;
+ 
+ 	oc = o->objcore;
+ 	CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
+ 	oc->flags |= OC_F_LRUDONTMOVE;
+ 
+ 	Lck_Lock(&sc->mtx);
+ 	sg->nfixed++;
+ 	sg->nobj++;
+ 
+ 	/* We have to do this somewhere, might as well be here... */
+ 	assert(sizeof so->hash == DIGEST_LEN);
+ 	memcpy(so->hash, oc->objhead->digest, DIGEST_LEN);
+ 	so->ttl = EXP_Grace(NULL, o);
+ 	so->ptr = (uint8_t*)o - sc->base;
+ 	so->ban = BAN_Time(oc->ban);
+ 
+ 	smp_init_oc(oc, sg, objidx);
+ 
+ 	Lck_Unlock(&sc->mtx);
+ 	return (o);
+ }
+ 
+ /*--------------------------------------------------------------------
+  * Allocate a bite
+  */
+ 
+ static struct storage *
+ smp_alloc(struct stevedore *st, size_t size)
+ {
+ 
+ 	return (smp_allocx(st,
+ 	    size > 4096 ? 4096 : size, size, NULL, NULL, NULL));
+ }
+ 
+ /*--------------------------------------------------------------------
+  * Trim a bite
+  * XXX: We could trim the last allocation.
+  */
+ 
+ static void
+ smp_trim(struct storage *ss, size_t size)
+ {
+ 
+ 	(void)ss;
+ 	(void)size;
+ }
+ 
+ /*--------------------------------------------------------------------
+  * We don't track frees of storage, we track the objects which own the
+  * storage and when there are no more objects in in the first segment,
+  * it can be reclaimed.
+  * XXX: We could free the last allocation, but does that happen ?
+  */
+ 
+ static void __match_proto__(storage_free_f)
+ smp_free(struct storage *st)
+ {
+ 
+ 	/* XXX */
+ 	(void)st;
+ }
+ 
+ 
+ /*--------------------------------------------------------------------*/
+ 
+ const struct stevedore smp_stevedore = {
+ 	.magic	=	STEVEDORE_MAGIC,
+ 	.name	=	"persistent",
+ 	.init	=	smp_mgt_init,
+ 	.open	=	smp_open,
+ 	.close	=	smp_close,
+ 	.alloc	=	smp_alloc,
+ 	.allocobj =	smp_allocobj,
+ 	.free	=	smp_free,
+ 	.trim	=	smp_trim,
++        .dup    =       default_dup,
+ };
+ 
+ /*--------------------------------------------------------------------
+  * Persistence is a bear to test unadultered, so we cheat by adding
+  * a cli command we can use to make it do tricks for us.
+  */
+ 
+ static void
+ debug_report_silo(struct cli *cli, const struct smp_sc *sc, int objs)
+ {
+ 	struct smp_seg *sg;
+ 	struct objcore *oc;
+ 
+ 	VCLI_Out(cli, "Silo: %s (%s)\n",
+ 	    sc->stevedore->ident, sc->filename);
+ 	VTAILQ_FOREACH(sg, &sc->segments, list) {
+ 		VCLI_Out(cli, "  Seg: [0x%jx ... +0x%jx]\n",
+ 		   (uintmax_t)sg->p.offset, (uintmax_t)sg->p.length);
+ 		if (sg == sc->cur_seg)
+ 			VCLI_Out(cli,
+ 			   "    Alloc: [0x%jx ... 0x%jx] = 0x%jx free\n",
+ 			   (uintmax_t)(sc->next_bot),
+ 			   (uintmax_t)(sc->next_top),
+ 			   (uintmax_t)(sc->next_top - sc->next_bot));
+ 		VCLI_Out(cli, "    %u nobj, %u alloc, %u lobjlist, %u fixed\n",
+ 		    sg->nobj, sg->nalloc, sg->p.lobjlist, sg->nfixed);
+ 		if (objs) {
+ 			VTAILQ_FOREACH(oc, &sg->lru->lru_head, lru_list)
+ 				VCLI_Out(cli, "      OC %p\n", oc);
+ 		}
+ 	}
+ }
+ 
+ static void
+ debug_persistent(struct cli *cli, const char * const * av, void *priv)
+ {
+ 	struct smp_sc *sc;
+ 
+ 	(void)priv;
+ 
+ 	if (av[2] == NULL) {
+ 		VTAILQ_FOREACH(sc, &silos, list)
+ 			debug_report_silo(cli, sc, 0);
+ 		return;
+ 	}
+ 	VTAILQ_FOREACH(sc, &silos, list)
+ 		if (!strcmp(av[2], sc->stevedore->ident))
+ 			break;
+ 	if (sc == NULL) {
+ 		VCLI_Out(cli, "Silo <%s> not found\n", av[2]);
+ 		VCLI_SetResult(cli, CLIS_PARAM);
+ 		return;
+ 	}
+ 	if (av[3] == NULL) {
+ 		debug_report_silo(cli, sc, 0);
+ 		return;
+ 	}
+ 	Lck_Lock(&sc->mtx);
+ 	if (!strcmp(av[3], "sync")) {
+ 		smp_close_seg(sc, sc->cur_seg);
+ 		smp_new_seg(sc);
+ 	} else if (!strcmp(av[3], "dump")) {
+ 		debug_report_silo(cli, sc, 1);
+ 	} else {
+ 		VCLI_Out(cli, "Unknown operation\n");
+ 		VCLI_SetResult(cli, CLIS_PARAM);
+ 	}
+ 	Lck_Unlock(&sc->mtx);
+ }
+ 
+ static struct cli_proto debug_cmds[] = {
+         { "debug.persistent", "debug.persistent",
+                 "Persistent debugging magic:\n"
+ 		"\tdebug.persistent [stevedore [cmd]]\n"
+ 		"With no cmd arg, a summary of the silo is returned.\n"
+ 		"Possible commands:\n"
+ 		"\tsync\tClose current segment, open a new one\n"
+ 		"\tdump\tinclude objcores in silo summary\n"
+ 		"",
+ 		0, 2, "d", debug_persistent },
+         { NULL }
+ };
+ 
+ /*--------------------------------------------------------------------*/
+ 
+ void
+ SMP_Init(void)
+ {
+ 	CLI_AddFuncs(debug_cmds);
+ }
+ 
+ /*--------------------------------------------------------------------
+  * Pause until all silos have loaded.
+  */
+ 
+ void
+ SMP_Ready(void)
+ {
+ 	struct smp_sc *sc;
+ 
+ 	ASSERT_CLI();
+ 	do {
+ 		VTAILQ_FOREACH(sc, &silos, list)
+ 			if (!(sc->flags & SMP_SC_LOADED))
+ 				break;
+ 		if (sc != NULL)
+ 			(void)sleep(1);
+ 	} while (sc != NULL);
+ }
diff --cc bin/varnishd/storage/storage_synth.c
index 0000000,e9e9b2f..062cfba
mode 000000,100644..100644
--- a/bin/varnishd/storage/storage_synth.c
+++ b/bin/varnishd/storage/storage_synth.c
@@@ -1,0 -1,120 +1,121 @@@
+ /*-
+  * Copyright (c) 2008-2011 Varnish Software AS
+  * All rights reserved.
+  *
+  * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
+  *
+  * Redistribution and use in source and binary forms, with or without
+  * modification, are permitted provided that the following conditions
+  * are met:
+  * 1. Redistributions of source code must retain the above copyright
+  *    notice, this list of conditions and the following disclaimer.
+  * 2. Redistributions in binary form must reproduce the above copyright
+  *    notice, this list of conditions and the following disclaimer in the
+  *    documentation and/or other materials provided with the distribution.
+  *
+  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+  * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+  * SUCH DAMAGE.
+  *
+  * Storage method for synthetic content, based on vsb.
+  */
+ 
+ #include "config.h"
+ 
+ #include <stdlib.h>
+ 
+ #include "cache/cache.h"
+ #include "storage/storage.h"
+ 
+ 
+ static struct lock		sms_mtx;
+ 
+ static void
+ sms_free(struct storage *sto)
+ {
+ 
+ 	CHECK_OBJ_NOTNULL(sto, STORAGE_MAGIC);
+ 	Lck_Lock(&sms_mtx);
+ 	VSC_C_main->sms_nobj--;
+ 	VSC_C_main->sms_nbytes -= sto->len;
+ 	VSC_C_main->sms_bfree += sto->len;
+ 	Lck_Unlock(&sms_mtx);
+ 	VSB_delete(sto->priv);
+ 	free(sto);
+ }
+ 
+ void
+ SMS_Init(void)
+ {
+ 
+ 	Lck_New(&sms_mtx, lck_sms);
+ }
+ 
+ static struct stevedore sms_stevedore = {
+ 	.magic	=	STEVEDORE_MAGIC,
+ 	.name	=	"synth",
+ 	.free	=	sms_free,
++        .dup    =       default_dup,
+ };
+ 
+ struct vsb *
+ SMS_Makesynth(struct object *obj)
+ {
+ 	struct storage *sto;
+ 	struct vsb *vsb;
+ 
+ 	CHECK_OBJ_NOTNULL(obj, OBJECT_MAGIC);
+ 	STV_Freestore(obj);
+ 	obj->len = 0;
+ 
+ 	Lck_Lock(&sms_mtx);
+ 	VSC_C_main->sms_nreq++;
+ 	VSC_C_main->sms_nobj++;
+ 	Lck_Unlock(&sms_mtx);
+ 
+ 	sto = calloc(sizeof *sto, 1);
+ 	XXXAN(sto);
+ 	vsb = VSB_new_auto();
+ 	XXXAN(vsb);
+ 	sto->priv = vsb;
+ 	sto->len = 0;
+ 	sto->space = 0;
+ #ifdef SENDFILE_WORKS
+ 	sto->fd = -1;
+ #endif
+ 	sto->stevedore = &sms_stevedore;
+ 	sto->magic = STORAGE_MAGIC;
+ 
+ 	VTAILQ_INSERT_TAIL(&obj->store, sto, list);
+ 	return (vsb);
+ }
+ 
+ void
+ SMS_Finish(struct object *obj)
+ {
+ 	struct storage *sto;
+ 	struct vsb *vsb;
+ 
+ 	CHECK_OBJ_NOTNULL(obj, OBJECT_MAGIC);
+ 	sto = VTAILQ_FIRST(&obj->store);
+ 	assert(sto->stevedore == &sms_stevedore);
+ 	vsb = sto->priv;
+ 	AZ(VSB_finish(vsb));
+ 
+ 	sto->ptr = (void*)VSB_data(vsb);
+ 	sto->len = VSB_len(vsb);
+ 	sto->space = VSB_len(vsb);
+ 	obj->len = sto->len;
+ 	Lck_Lock(&sms_mtx);
+ 	VSC_C_main->sms_nbytes += sto->len;
+ 	VSC_C_main->sms_balloc += sto->len;
+ 	Lck_Unlock(&sms_mtx);
+ }
diff --cc bin/varnishd/storage/storage_umem.c
index 0000000,52d238d..add6bd7
mode 000000,100644..100644
--- a/bin/varnishd/storage/storage_umem.c
+++ b/bin/varnishd/storage/storage_umem.c
@@@ -1,0 -1,166 +1,167 @@@
+ /*-
+  * Copyright (c) 2006 Verdens Gang AS
+  * Copyright (c) 2006-2011 Varnish Software AS
+  * All rights reserved.
+  *
+  * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
+  *
+  * Redistribution and use in source and binary forms, with or without
+  * modification, are permitted provided that the following conditions
+  * are met:
+  * 1. Redistributions of source code must retain the above copyright
+  *    notice, this list of conditions and the following disclaimer.
+  * 2. Redistributions in binary form must reproduce the above copyright
+  *    notice, this list of conditions and the following disclaimer in the
+  *    documentation and/or other materials provided with the distribution.
+  *
+  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+  * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+  * SUCH DAMAGE.
+  *
+  * Storage method based on umem_alloc(3MALLOC)
+  */
+ 
+ #include "config.h"
+ 
+ #ifdef HAVE_LIBUMEM
+ 
+ #include <sys/types.h>
+ 
+ #include <stdio.h>
+ #include <stdlib.h>
+ #include <umem.h>
+ 
+ #include "cache/cache.h"
+ #include "storage/storage.h"
+ 
+ static size_t			smu_max = SIZE_MAX;
+ static MTX			smu_mtx;
+ 
+ struct smu {
+ 	struct storage		s;
+ 	size_t			sz;
+ };
+ 
+ static struct storage *
+ smu_alloc(struct stevedore *st, size_t size)
+ {
+ 	struct smu *smu;
+ 
+ 	Lck_Lock(&smu_mtx);
+ 	VSC_C_main->sma_nreq++;
+ 	if (VSC_C_main->sma_nbytes + size > smu_max)
+ 		size = 0;
+ 	else {
+ 		VSC_C_main->sma_nobj++;
+ 		VSC_C_main->sma_nbytes += size;
+ 		VSC_C_main->sma_balloc += size;
+ 	}
+ 	Lck_Unlock(&smu_mtx);
+ 
+ 	if (size == 0)
+ 		return (NULL);
+ 
+ 	smu = umem_zalloc(sizeof *smu, UMEM_DEFAULT);
+ 	if (smu == NULL)
+ 		return (NULL);
+ 	smu->sz = size;
+ 	smu->s.priv = smu;
+ 	smu->s.ptr = umem_alloc(size, UMEM_DEFAULT);
+ 	XXXAN(smu->s.ptr);
+ 	smu->s.len = 0;
+ 	smu->s.space = size;
+ 	smu->s.fd = -1;
+ 	smu->s.stevedore = st;
+ 	smu->s.magic = STORAGE_MAGIC;
+ 	return (&smu->s);
+ }
+ 
+ static void
+ smu_free(struct storage *s)
+ {
+ 	struct smu *smu;
+ 
+ 	CHECK_OBJ_NOTNULL(s, STORAGE_MAGIC);
+ 	smu = s->priv;
+ 	assert(smu->sz == smu->s.space);
+ 	Lck_Lock(&smu_mtx);
+ 	VSC_C_main->sma_nobj--;
+ 	VSC_C_main->sma_nbytes -= smu->sz;
+ 	VSC_C_main->sma_bfree += smu->sz;
+ 	Lck_Unlock(&smu_mtx);
+ 	umem_free(smu->s.ptr, smu->s.space);
+ 	umem_free(smu, sizeof *smu);
+ }
+ 
+ static void
+ smu_trim(const struct storage *s, size_t size)
+ {
+ 	struct smu *smu;
+ 	void *p;
+ 
+ 	CHECK_OBJ_NOTNULL(s, STORAGE_MAGIC);
+ 	smu = s->priv;
+ 	assert(smu->sz == smu->s.space);
+ 	if ((p = umem_alloc(size, UMEM_DEFAULT)) != NULL) {
+ 		memcpy(p, smu->s.ptr, size);
+ 		umem_free(smu->s.ptr, smu->s.space);
+ 		Lck_Lock(&smu_mtx);
+ 		VSC_C_main->sma_nbytes -= (smu->sz - size);
+ 		VSC_C_main->sma_bfree += smu->sz - size;
+ 		smu->sz = size;
+ 		Lck_Unlock(&smu_mtx);
+ 		smu->s.ptr = p;
+ 		smu->s.space = size;
+ 	}
+ }
+ 
+ static void
+ smu_init(struct stevedore *parent, int ac, char * const *av)
+ {
+ 	const char *e;
+ 	uintmax_t u;
+ 
+ 	(void)parent;
+ 
+ 	AZ(av[ac]);
+ 	if (ac > 1)
+ 		ARGV_ERR("(-sumem) too many arguments\n");
+ 
+ 	if (ac == 0 || *av[0] == '\0')
+ 		 return;
+ 
+ 	e = VNUM_2bytes(av[0], &u, 0);
+ 	if (e != NULL)
+ 		ARGV_ERR("(-sumem) size \"%s\": %s\n", av[0], e);
+ 	if ((u != (uintmax_t)(size_t)u))
+ 		ARGV_ERR("(-sumem) size \"%s\": too big\n", av[0]);
+ 	smu_max = u;
+ }
+ 
+ static void
+ smu_open(const struct stevedore *st)
+ {
+ 	(void)st;
+ 	AZ(pthread_mutex_init(&smu_mtx, NULL));
+ }
+ 
+ const struct stevedore smu_stevedore = {
+ 	.magic	=	STEVEDORE_MAGIC,
+ 	.name	=	"umem",
+ 	.init	=	smu_init,
+ 	.open	=	smu_open,
+ 	.alloc	=	smu_alloc,
+ 	.free	=	smu_free,
+ 	.trim	=	smu_trim,
++        .dup    =       default_dup,
+ };
+ 
+ #endif /* HAVE_UMEM_H */
diff --cc include/tbl/vsc_fields.h
index 0000000,738703c..824ba6d
mode 000000,100644..100644
--- a/include/tbl/vsc_fields.h
+++ b/include/tbl/vsc_fields.h
@@@ -1,0 -1,417 +1,419 @@@
+ /*-
+  * Copyright (c) 2006 Verdens Gang AS
+  * Copyright (c) 2006-2011 Varnish Software AS
+  * All rights reserved.
+  *
+  * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
+  *
+  * Redistribution and use in source and binary forms, with or without
+  * modification, are permitted provided that the following conditions
+  * are met:
+  * 1. Redistributions of source code must retain the above copyright
+  *    notice, this list of conditions and the following disclaimer.
+  * 2. Redistributions in binary form must reproduce the above copyright
+  *    notice, this list of conditions and the following disclaimer in the
+  *    documentation and/or other materials provided with the distribution.
+  *
+  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+  * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION, "")
+  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+  * SUCH DAMAGE.
+  *
+  * Definition of all shared memory statistics below.
+  *
+  * Fields (n, t, l, f, e, d):
+  *    n - Name:		Field name, in C-source and stats programs
+  *    t - Type:		C-type, uint64_t, unless marked in 'f'
+  *    l - Local:	Local counter in worker thread.
+  *    f - Format:	Semantics of the value in this field
+  *				'a' - Accumulator (deprecated, use 'c')
+  *				'b' - Bitmap
+  *				'c' - Counter, never decreases.
+  *				'g' - Gauge, goes up and down
+  *				'i' - Integer (deprecated, use 'g')
+  *    e - Explantion:	Short explanation of field (for screen use)
+  *    d - Description:	Long explanation of field (for doc use)
+  *
+  * Please describe Gauge variables as "Number of..." to indicate that
+  * this is a snapshot, and Counter variables as "Count of" to indicate
+  * accumulative count.
+  *
+  * -----------------------
+  * NB: Cleanup in progress
+  * -----------------------
+  *
+  * Insufficient attention has caused this to become a swamp of conflicting
+  * conventions, shorthands and general mumbo-jumbo.  I'm trying to clean
+  * it up as I go over the code in other business.
+  *
+  * Please see the sessmem section for how it should look.
+  *
+  */
+ 
+ /**********************************************************************/
+ 
+ #ifdef VSC_DO_MAIN
+ 
+ /*---------------------------------------------------------------------
+  * Sessions
+  *    see: cache_acceptor.c and cache_pool.c
+  */
+ 
+ VSC_F(sess_conn,		uint64_t, 1, 'c',
+     "Sessions accepted",
+ 	"Count of sessions succesfully accepted"
+ )
+ VSC_F(sess_drop,		uint64_t, 1, 'c',
+     "Sessions dropped",
+ 	"Count of sessions silently dropped due to lack of session memory."
+ 	"  See parameter 'max_sess'."
+ )
+ 
+ VSC_F(sess_fail,		uint64_t, 1, 'c',
+     "Session accept failures",
+ 	"Count of failures to accept TCP connection."
+ 	"  Either the client changed its mind, or the kernel ran out of"
+ 	" some resource like filedescriptors."
+ )
+ 
+ /*---------------------------------------------------------------------*/
+ 
+ VSC_F(client_req,		uint64_t, 1, 'a',
+       "Client requests received",
+       "")
+ 
+ VSC_F(cache_hit,		uint64_t, 1, 'a',
+       "Cache hits",
+       "Count of cache hits. "
+       "  A cache hit indicates that an object has been delivered to a"
+       "  client without fetching it from a backend server."
+ )
+ 
+ VSC_F(cache_hitpass,	uint64_t, 1, 'a',
+       "Cache hits for pass",
+       "Count of hits for pass"
+       "  A cache hit for pass indicates that Varnish is going to"
+       "  pass the request to the backend and this decision has been "
+       "  cached in it self. This counts how many times the cached "
+       "  decision is being used."
+ )
+ VSC_F(cache_miss,		uint64_t, 1, 'a',
+       "Cache misses",
+       "Count of misses"
+       "  A cache miss indicates the object was fetched from the"
+       "  backend before delivering it to the backend.")
+ 
+ VSC_F(backend_conn,	uint64_t, 0, 'a',
+       "Backend conn. success",
+       "")
+ 
+ VSC_F(backend_unhealthy,	uint64_t, 0, 'a',
+       "Backend conn. not attempted",
+       ""
+ )
+ VSC_F(backend_busy,	uint64_t, 0, 'a', "Backend conn. too many", "")
+ VSC_F(backend_fail,	uint64_t, 0, 'a', "Backend conn. failures", "")
+ VSC_F(backend_reuse,	uint64_t, 0, 'a',
+       "Backend conn. reuses",
+       "Count of backend connection reuses"
+       "  This counter is increased whenever we reuse a recycled connection.")
+ VSC_F(backend_toolate,	uint64_t, 0, 'a', "Backend conn. was closed", "")
+ VSC_F(backend_recycle,	uint64_t, 0, 'a',
+       "Backend conn. recycles",
+       "Count of backend connection recycles"
+       "  This counter is increased whenever we have a keep-alive"
+       "  connection that is put back into the pool of connections."
+       "  It has not yet been used, but it might be, unless the backend"
+       "  closes it.")
+ VSC_F(backend_retry,	uint64_t, 0, 'a', "Backend conn. retry", "")
+ 
+ VSC_F(fetch_head,		uint64_t, 1, 'a', "Fetch head", "")
+ VSC_F(fetch_length,		uint64_t, 1, 'a', "Fetch with Length", "")
+ VSC_F(fetch_chunked,		uint64_t, 1, 'a', "Fetch chunked", "")
+ VSC_F(fetch_eof,		uint64_t, 1, 'a', "Fetch EOF", "")
+ VSC_F(fetch_bad,		uint64_t, 1, 'a', "Fetch had bad headers", "")
+ VSC_F(fetch_close,		uint64_t, 1, 'a', "Fetch wanted close", "")
+ VSC_F(fetch_oldhttp,		uint64_t, 1, 'a', "Fetch pre HTTP/1.1 closed", "")
+ VSC_F(fetch_zero,		uint64_t, 1, 'a', "Fetch zero len", "")
+ VSC_F(fetch_failed,		uint64_t, 1, 'a', "Fetch failed", "")
+ VSC_F(fetch_1xx,		uint64_t, 1, 'a', "Fetch no body (1xx)", "")
+ VSC_F(fetch_204,		uint64_t, 1, 'a', "Fetch no body (204)", "")
+ VSC_F(fetch_304,		uint64_t, 1, 'a', "Fetch no body (304)", "")
+ 
+ /*---------------------------------------------------------------------
+  * Session Memory
+  *    see: cache_session.c
+  */
+ 
+ VSC_F(sessmem_size,		uint64_t, 1, 'g',
+     "Session mem size",
+ 	"Bytes of memory allocated for last allocated session."
+ )
+ 
+ VSC_F(sessmem_alloc,		uint64_t, 1, 'c',
+     "Session mem allocated",
+ 	"Count of all allocations of session memory."
+ )
+ 
+ VSC_F(sessmem_free,		uint64_t, 1, 'c',
+     "Session mem freed",
+ 	"Count of all frees of session memory."
+ )
+ 
+ VSC_F(sessmem_fail,		uint64_t, 1, 'c',
+     "Session mem alloc failed",
+ 	"Count of session memory allocation failures."
+ )
+ 
+ VSC_F(sessmem_limit,		uint64_t, 1, 'c',
+     "Session mem alloc limited",
+ 	"Count of session memory allocations blocked by limit (max_sess)."
+ )
+ 
+ /*---------------------------------------------------------------------
+  * Pools, threads, and sessions
+  *    see: cache_pool.c
+  *
+  */
+ 
+ VSC_F(pools,			uint64_t, 1, 'g',
+     "Number of thread pools",
+ 	"Number of thread pools.  See also param wthread_pools."
+ 	"  NB: Presently pools cannot be removed once created."
+ )
+ 
+ VSC_F(threads,			uint64_t, 1, 'g',
+     "Total number of threads",
+ 	"Number of threads in all pools."
+ 	"  See also params thread_pools, thread_pool_min & thread_pool_max."
+ )
+ 
+ VSC_F(threads_limited,		uint64_t, 1, 'c',
+     "Threads hit max",
+ 	"Number of times more threads were needed, but limit was reached"
+ 	" in a thread pool."
+ 	"  See also param thread_pool_max."
+ )
+ 
+ VSC_F(threads_created,		uint64_t, 1, 'c',
+     "Threads created",
+ 	"Total number of threads created in all pools."
+ )
+ 
+ VSC_F(threads_destroyed,	uint64_t, 1, 'c',
+     "Threads destoryed",
+ 	"Total number of threads destroyed in all pools."
+ )
+ 
+ VSC_F(threads_failed,		uint64_t, 1, 'c',
+     "Thread creation failed",
+ 	"Number of times creating a thread failed."
+ 	"  See VSL::Debug for diagnostics."
+ 	"  See also param thread_fail_delay."
+ )
+ 
+ VSC_F(thread_queue_len,		uint64_t, 1, 'g',
+     "Length of session queue",
+ 	"Length of session queue waiting for threads."
+ 	"  NB: Only updates once per second."
+ 	"  See also param queue_max."
+ )
+ 
+ VSC_F(sess_queued,		uint64_t, 1, 'c',
+     "Sessions queued for thread",
+ 	"Number of times session was queued waiting for a thread."
+ 	"  See also param queue_max."
+ )
+ 
+ VSC_F(sess_dropped,		uint64_t, 1, 'c',
+     "Sessions dropped for thread",
+ 	"Number of times session was dropped because the queue were too"
+ 	" long already."
+ 	"  See also param queue_max."
+ )
+ 
+ /*---------------------------------------------------------------------*/
+ 
+ VSC_F(n_sess_mem,		uint64_t, 0, 'i', "N struct sess_mem", "")
+ VSC_F(n_sess,			uint64_t, 0, 'i', "N struct sess", "")
+ VSC_F(n_object,			uint64_t, 1, 'i', "N struct object", "")
+ VSC_F(n_vampireobject,		uint64_t, 1, 'i', "N unresurrected objects", "")
+ VSC_F(n_objectcore,		uint64_t, 1, 'i', "N struct objectcore", "")
+ VSC_F(n_objecthead,		uint64_t, 1, 'i', "N struct objecthead", "")
+ VSC_F(n_waitinglist,		uint64_t, 1, 'i', "N struct waitinglist", "")
+ 
+ VSC_F(n_vbc,		uint64_t, 0, 'i', "N struct vbc", "")
+ 
+ VSC_F(n_backend,		uint64_t, 0, 'i', "N backends", "")
+ 
+ VSC_F(n_expired,		uint64_t, 0, 'i', "N expired objects", "")
+ VSC_F(n_lru_nuked,		uint64_t, 0, 'i', "N LRU nuked objects", "")
+ VSC_F(n_lru_moved,		uint64_t, 0, 'i', "N LRU moved objects", "")
+ 
+ VSC_F(losthdr,		uint64_t, 0, 'a', "HTTP header overflows", "")
+ 
+ VSC_F(n_objsendfile,	uint64_t, 0, 'a', "Objects sent with sendfile",
+       "The number of objects sent with the sendfile system call. If enabled "
+       "sendfile will be used on object larger than a certain size.")
+ VSC_F(n_objwrite,		uint64_t, 0, 'a', "Objects sent with write",
+       "The number of objects sent with regular write calls."
+       "Writes are used when the objects are too small for sendfile "
+       "or if the sendfile call has been disabled")
+ VSC_F(n_objoverflow,	uint64_t, 1, 'a',
+ 					"Objects overflowing workspace", "")
+ 
+ VSC_F(s_sess,		uint64_t, 1, 'a', "Total Sessions", "")
+ VSC_F(s_req,		uint64_t, 1, 'a', "Total Requests", "")
+ VSC_F(s_pipe,		uint64_t, 1, 'a', "Total pipe", "")
+ VSC_F(s_pass,		uint64_t, 1, 'a', "Total pass", "")
+ VSC_F(s_fetch,		uint64_t, 1, 'a', "Total fetch", "")
+ VSC_F(s_hdrbytes,		uint64_t, 1, 'a', "Total header bytes", "")
+ VSC_F(s_bodybytes,		uint64_t, 1, 'a', "Total body bytes", "")
+ 
+ VSC_F(sess_closed,		uint64_t, 1, 'a', "Session Closed", "")
+ VSC_F(sess_pipeline,	uint64_t, 1, 'a', "Session Pipeline", "")
+ VSC_F(sess_readahead,	uint64_t, 1, 'a', "Session Read Ahead", "")
+ VSC_F(sess_linger,		uint64_t, 1, 'a', "Session Linger", "")
+ VSC_F(sess_herd,		uint64_t, 1, 'a', "Session herd", "")
+ 
+ VSC_F(shm_records,		uint64_t, 0, 'a', "SHM records", "")
+ VSC_F(shm_writes,		uint64_t, 0, 'a', "SHM writes", "")
+ VSC_F(shm_flushes,		uint64_t, 0, 'a', "SHM flushes due to overflow", "")
+ VSC_F(shm_cont,		uint64_t, 0, 'a', "SHM MTX contention", "")
+ VSC_F(shm_cycles,		uint64_t, 0, 'a', "SHM cycles through buffer", "")
+ 
+ VSC_F(sms_nreq,		uint64_t, 0, 'a', "SMS allocator requests", "")
+ VSC_F(sms_nobj,		uint64_t, 0, 'i', "SMS outstanding allocations", "")
+ VSC_F(sms_nbytes,		uint64_t, 0, 'i', "SMS outstanding bytes", "")
+ VSC_F(sms_balloc,		uint64_t, 0, 'i', "SMS bytes allocated", "")
+ VSC_F(sms_bfree,		uint64_t, 0, 'i', "SMS bytes freed", "")
+ 
+ VSC_F(backend_req,		uint64_t, 0, 'a', "Backend requests made", "")
+ 
+ VSC_F(n_vcl,		uint64_t, 0, 'a', "N vcl total", "")
+ VSC_F(n_vcl_avail,		uint64_t, 0, 'a', "N vcl available", "")
+ VSC_F(n_vcl_discard,	uint64_t, 0, 'a', "N vcl discarded", "")
+ 
+ /**********************************************************************/
+ 
+ VSC_F(bans,			uint64_t, 0, 'g',
+    "Count of bans",
+ 	"Number of all bans in system, including bans superseded"
+ 	" by newer bans and bans already checked by the ban-lurker."
+ )
+ VSC_F(bans_gone,		uint64_t, 0, 'g',
+     "Number of bans marked 'gone'",
+ 	"Number of bans which are no longer active, either because they"
+ 	" got checked by the ban-lurker or superseded by newer identical bans."
+ )
+ VSC_F(bans_req,			uint64_t, 0, 'g',
+     "Number of bans using req.*",
+ 	"Number of bans which use req.* variables.  These bans can not"
+ 	" be washed by the ban-lurker."
+ )
+ VSC_F(bans_added,		uint64_t, 0, 'c',
+     "Bans added",
+ 	"Counter of bans added to ban list."
+ )
+ VSC_F(bans_deleted,		uint64_t, 0, 'c',
+     "Bans deleted",
+ 	"Counter of bans deleted from ban list."
+ )
+ 
+ VSC_F(bans_tested,		uint64_t, 0, 'c',
+     "Bans tested against objects",
+ 	"Count of how many bans and objects have been tested against"
+ 	" each other."
+ )
+ VSC_F(bans_tests_tested,	uint64_t, 0, 'c',
+     "Ban tests tested against objects",
+ 	"Count of how many tests and objects have been tested against"
+ 	" each other.  'ban req.url == foo && req.http.host == bar'"
+ 	" counts as one in 'bans_tested' and as two in 'bans_tests_tested'"
+ )
+ VSC_F(bans_dups,		uint64_t, 0, 'c',
+     "Bans superseded by other bans",
+ 	"Count of bans replaced by later identical bans."
+ )
+ 
+ /**********************************************************************/
+ 
+ VSC_F(hcb_nolock,		uint64_t, 0, 'a', "HCB Lookups without lock", "")
+ VSC_F(hcb_lock,		uint64_t, 0, 'a', "HCB Lookups with lock", "")
+ VSC_F(hcb_insert,		uint64_t, 0, 'a', "HCB Inserts", "")
+ 
+ VSC_F(esi_errors,		uint64_t, 0, 'a', "ESI parse errors (unlock)", "")
+ VSC_F(esi_warnings,		uint64_t, 0, 'a', "ESI parse warnings (unlock)", "")
+ VSC_F(client_drop_late,	uint64_t, 0, 'a', "Connection dropped late", "")
+ VSC_F(uptime,		uint64_t, 0, 'a', "Client uptime", "")
+ 
+ VSC_F(dir_dns_lookups,	uint64_t, 0, 'a', "DNS director lookups", "")
+ VSC_F(dir_dns_failed,	uint64_t, 0, 'a', "DNS director failed lookups", "")
+ VSC_F(dir_dns_hit,		uint64_t, 0, 'a', "DNS director cached lookups hit", "")
+ VSC_F(dir_dns_cache_full,	uint64_t, 0, 'a', "DNS director full dnscache", "")
+ 
+ VSC_F(vmods,		uint64_t, 0, 'i', "Loaded VMODs", "")
+ 
+ VSC_F(n_gzip,			uint64_t, 0, 'a', "Gzip operations", "")
+ VSC_F(n_gunzip,			uint64_t, 0, 'a', "Gunzip operations", "")
+ 
++VSC_F(cond_not_validated,	uint64_t, 1, 'c', "Non-validating responses",
++      "Count of backend responses to conditional requests with status != 304")
+ #endif
+ 
+ /**********************************************************************/
+ 
+ #ifdef VSC_DO_LCK
+ 
+ VSC_F(creat,		uint64_t, 0, 'a', "Created locks", "")
+ VSC_F(destroy,		uint64_t, 0, 'a', "Destroyed locks", "")
+ VSC_F(locks,		uint64_t, 0, 'a', "Lock Operations", "")
+ VSC_F(colls,		uint64_t, 0, 'a', "Collisions", "")
+ 
+ #endif
+ 
+ /**********************************************************************
+  * All Stevedores support these counters
+  */
+ 
+ #if defined(VSC_DO_SMA) || defined (VSC_DO_SMF)
+ VSC_F(c_req,		uint64_t, 0, 'a', "Allocator requests", "")
+ VSC_F(c_fail,		uint64_t, 0, 'a', "Allocator failures", "")
+ VSC_F(c_bytes,		uint64_t, 0, 'a', "Bytes allocated", "")
+ VSC_F(c_freed,		uint64_t, 0, 'a', "Bytes freed", "")
+ VSC_F(g_alloc,		uint64_t, 0, 'i', "Allocations outstanding", "")
+ VSC_F(g_bytes,		uint64_t, 0, 'i', "Bytes outstanding", "")
+ VSC_F(g_space,		uint64_t, 0, 'i', "Bytes available", "")
+ #endif
+ 
+ 
+ /**********************************************************************/
+ 
+ #ifdef VSC_DO_SMA
+ /* No SMA specific counters */
+ #endif
+ 
+ /**********************************************************************/
+ 
+ #ifdef VSC_DO_SMF
+ VSC_F(g_smf,			uint64_t, 0, 'i', "N struct smf", "")
+ VSC_F(g_smf_frag,		uint64_t, 0, 'i', "N small free smf", "")
+ VSC_F(g_smf_large,		uint64_t, 0, 'i', "N large free smf", "")
+ #endif
+ 
+ /**********************************************************************/
+ 
+ #ifdef VSC_DO_VBE
+ 
+ VSC_F(vcls,			uint64_t, 0, 'i', "VCL references", "")
+ VSC_F(happy,		uint64_t, 0, 'b', "Happy health probes", "")
+ 
+ #endif
+ 



From geoff at varnish-cache.org  Mon Jan  9 20:52:26 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:26 +0100
Subject: [experimental-ims] 8e5b998 Use a better way of checking we have
	pkg-config
Message-ID: <E1RkMCg-0007wx-Sw@project.varnish-software.com>

commit 8e5b998b249c830a14bf76e05d506bd128de3ea9
Author: Tollef Fog Heen <tfheen at varnish-software.com>
Date:   Tue Oct 18 20:29:53 2011 +0200

    Use a better way of checking we have pkg-config

diff --git a/configure.ac b/configure.ac
index 8295fbb..e0119f3 100644
--- a/configure.ac
+++ b/configure.ac
@@ -108,7 +108,7 @@ AC_SUBST(NET_LIBS)
 AC_CHECK_LIBM
 AC_SUBST(LIBM)
 
-m4_pattern_forbid([^_?PKG_[A-Z_]+$],[pkg.m4 missing, please install pkg-config])
+m4_ifndef([PKG_PROG_PKG_CONFIG], [m4_fatal([pkg.m4 missing, please install pkg-config])])
 PKG_PROG_PKG_CONFIG
 if test -n $PKG_CONFIG; then
    PKG_CHECK_MODULES([PCRE], [libpcre])



From geoff at varnish-cache.org  Mon Jan  9 20:51:58 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:51:58 +0100
Subject: [experimental-ims] c43c726 Missing newline
Message-ID: <E1RkMCE-00075e-RP@project.varnish-software.com>

commit c43c7265d4b931ddca971743dc6c75a6d56824b7
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Sep 20 09:41:28 2011 +0000

    Missing newline

diff --git a/bin/varnishtest/vtc_main.c b/bin/varnishtest/vtc_main.c
index b189d1e..01a5fe2 100644
--- a/bin/varnishtest/vtc_main.c
+++ b/bin/varnishtest/vtc_main.c
@@ -228,7 +228,7 @@ tst_cb(const struct vev *ve, int what)
 			    jp->tst->filename, t);
 			if (WIFSIGNALED(stx))
 				printf(" signal=%d", WTERMSIG(stx));
-			printf(" exit=%d", WEXITSTATUS(stx));
+			printf(" exit=%d\n", WEXITSTATUS(stx));
 			if (!vtc_continue) {
 				/* XXX kill -9 other jobs ? */
 				exit(2);



From geoff at varnish-cache.org  Mon Jan  9 20:52:29 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:29 +0100
Subject: [experimental-ims] ac387cf varnishstat: Add json output and continous
	mode
Message-ID: <E1RkMCj-000842-Nn@project.varnish-software.com>

commit ac387cf066aead9e0c0aae63dc0802417d1ca306
Author: Lasse Karstensen <lasse at varnish-software.com>
Date:   Mon Nov 7 13:14:35 2011 +0100

    varnishstat: Add json output and continous mode

diff --git a/bin/varnishstat/varnishstat.c b/bin/varnishstat/varnishstat.c
index 3c6c4b5..c20b87f 100644
--- a/bin/varnishstat/varnishstat.c
+++ b/bin/varnishstat/varnishstat.c
@@ -82,6 +82,69 @@ do_xml(struct VSM_data *vd)
 	printf("</varnishstat>\n");
 }
 
+
+/*--------------------------------------------------------------------*/
+
+static int
+do_json_cb(void *priv, const struct VSC_point * const pt)
+{
+	uint64_t val;
+	int *jp;
+	char jsonkey[255];
+	char jsontmp[255];
+
+	jp = priv;
+
+	assert(!strcmp(pt->fmt, "uint64_t"));
+	val = *(const volatile uint64_t*)pt->ptr;
+
+	if (*jp) *jp = 0; else printf(",\n");
+	jsonkey[0] = '\0';
+
+	/* build the JSON key name.  */
+	if (strcmp(pt->ident, "")  && strcmp(pt->class, ""))  sprintf(jsonkey, "%s.%s", pt->class, pt->ident);
+	if (strcmp(pt->ident, "")  && !strcmp(pt->class, "")) sprintf(jsonkey, "%s", pt->ident);
+	if (!strcmp(pt->ident, "") && strcmp(pt->class, ""))  sprintf(jsonkey, "%s", pt->class);
+
+	strcpy(jsontmp, jsonkey);
+	if (strcmp(jsonkey, "")) sprintf(jsonkey, "%s.%s", jsontmp, pt->name);
+	else strcpy(jsonkey, pt->name);
+
+	printf("\t\"%s\": {", jsonkey);
+
+	if (strcmp(pt->class, "")) printf("\"type\": \"%s\", ",  pt->class);
+	if (strcmp(pt->ident, "")) printf("\"ident\": \"%s\", ", pt->ident);
+
+	printf("\"value\": %ju, ", val);
+
+	printf("\"flag\": \"%c\", ", pt->flag);
+	printf("\"description\": \"%s\"", pt->desc);
+	printf("}");
+
+	if (*jp) printf("\n");
+	return (0);
+}
+
+static void
+do_json(struct VSM_data *vd)
+{
+	char time_stamp[20];
+	time_t now;
+	int jp;
+
+	jp = 1;
+
+	printf("{\n");
+	now = time(NULL);
+
+	(void)strftime(time_stamp, 20, "%Y-%m-%dT%H:%M:%S", localtime(&now));
+	printf("\t\"timestamp\": \"%s\",\n", time_stamp);
+	(void)VSC_Iter(vd, do_json_cb, &jp);
+	printf("\n}\n");
+	fflush(stdout);
+}
+
+
 /*--------------------------------------------------------------------*/
 
 struct once_priv {
@@ -167,7 +230,7 @@ usage(void)
 	    "[-1lV] [-f field_list] "
 	    VSC_n_USAGE " "
 	    "[-w delay]\n");
-	fprintf(stderr, FMT, "-1", "Print the statistics once and exit");
+	fprintf(stderr, FMT, "-1", "Print the statistics to stdout.");
 	fprintf(stderr, FMT, "-f field_list",
 	    "Comma separated list of fields to display. ");
 	fprintf(stderr, FMT, "",
@@ -178,9 +241,11 @@ usage(void)
 	    "The varnishd instance to get logs from");
 	fprintf(stderr, FMT, "-V", "Display the version number and exit");
 	fprintf(stderr, FMT, "-w delay",
-	    "Wait delay seconds between updates.  The default is 1.");
+	    "Wait delay seconds between updates.  Default is 1 second. Can also be be used with -1, -x or -j for repeated output.");
 	fprintf(stderr, FMT, "-x",
-	    "Print statistics once as XML and exit.");
+	    "Print statistics to stdout as XML.");
+	fprintf(stderr, FMT, "-j",
+	    "Print statistics to stdout as JSON.");
 #undef FMT
 	exit(1);
 }
@@ -191,12 +256,12 @@ main(int argc, char * const *argv)
 	int c;
 	struct VSM_data *vd;
 	const struct VSC_C_main *VSC_C_main;
-	int delay = 1, once = 0, xml = 0;
+	int delay = 1, once = 0, xml = 0, json = 0, do_repeat = 0;
 
 	vd = VSM_New();
 	VSC_Setup(vd);
 
-	while ((c = getopt(argc, argv, VSC_ARGS "1f:lVw:x")) != -1) {
+	while ((c = getopt(argc, argv, VSC_ARGS "1f:lVw:xjt:")) != -1) {
 		switch (c) {
 		case '1':
 			once = 1;
@@ -210,11 +275,15 @@ main(int argc, char * const *argv)
 			VCS_Message("varnishstat");
 			exit(0);
 		case 'w':
+			do_repeat = 1;
 			delay = atoi(optarg);
 			break;
 		case 'x':
 			xml = 1;
 			break;
+		case 'j':
+			json = 1;
+			break;
 		default:
 			if (VSC_Arg(vd, c, optarg) > 0)
 				break;
@@ -227,12 +296,27 @@ main(int argc, char * const *argv)
 
 	VSC_C_main = VSC_Main(vd);
 
-	if (xml)
-		do_xml(vd);
-	else if (once)
-		do_once(vd, VSC_C_main);
-	else
+	if (!(xml || json || once)) {
 		do_curses(vd, VSC_C_main, delay);
+		exit(0);
+	}
+
+	while (1) {
+		if (xml)
+			do_xml(vd);
+		else if (json)
+			do_json(vd);
+		else if (once)
+			do_once(vd, VSC_C_main);
+		else {
+			assert(0);
+		}
+		if (!do_repeat) break;
+
+		// end of output block marker.
+		printf("\n");
 
+		sleep(delay);
+	} 
 	exit(0);
 }
diff --git a/doc/sphinx/reference/varnishstat.rst b/doc/sphinx/reference/varnishstat.rst
index 09c4989..3c5d541 100644
--- a/doc/sphinx/reference/varnishstat.rst
+++ b/doc/sphinx/reference/varnishstat.rst
@@ -10,15 +10,16 @@ Varnish Cache statistics
 
 :Author: Dag-Erling Sm?rgrav
 :Author: Per Buer
-:Date:   2010-06-1
-:Version: 1.0
+:Author: Lasse Karstensen
+:Date:   2011-11-07
+:Version: 1.1
 :Manual section: 1
 
 
 SYNOPSIS
 ========
 
-varnishstat [-1] [-x] [-f field_list] [-l] [-n varnish_name] [-V] [-w delay]
+varnishstat [-1] [-x] [-j] [-f field_list] [-l] [-n varnish_name] [-V] [-w delay]
 
 DESCRIPTION
 ===========
@@ -27,7 +28,7 @@ The varnishstat utility displays statistics from a running varnishd(1) instance.
 
 The following options are available:
 
--1          Instead of presenting of a continuously updated display, print the statistics once and exit.
+-1          Instead of presenting of a continuously updated display, print the statistics to stdout.
 
 -f          A comma separated list of the fields to display.  If it starts with '^' it is used as an exclusion
 	    list.
@@ -39,9 +40,11 @@ The following options are available:
 
 -V          Display the version number and exit.
 
--w delay    Wait delay seconds between updates.  The default is 1.
+-w delay    Wait delay seconds between updates.  The default is 1. Can also be used with -1, -x or -j for repeated output.
 
--x          Displays the result as XML once.
+-x          Displays the result as XML.
+
+-j          Displays the result as JSON.
 
 The columns in the main display are, from left to right:
 
@@ -65,6 +68,29 @@ When using the -x option, the output is::
     <description>FIELD DESCRIPTION</description> 
   </stat> 
 
+With -j the output format is::
+
+  {
+    "timestamp": "YYYY-MM-DDTHH:mm:SS",
+    "client_conn": {
+	"value": 0, "flag": "a", 
+	"description": "Client connections accepted"
+    },
+    "client_drop": {
+	"value": 0, "flag": "a", 
+	"description": "Connection dropped, no sess/wrk"
+    },
+    "LCK.backend.creat": {
+	"type": "LCK", "ident": "backend", "value": 1, 
+	"flag": "a", "description": "Created locks"
+    },
+    [..]
+  }
+
+Timestamp is the time when the report was generated by varnishstat.
+
+Repeated output with -1, -x or -j will have a single empty line (\\n) between each block of output.
+
 
 SEE ALSO
 ========
@@ -91,4 +117,4 @@ This document is licensed under the same licence as Varnish
 itself. See LICENCE for details.
 
 * Copyright (c) 2006 Verdens Gang AS
-* Copyright (c) 2006-2008 Varnish Software AS
+* Copyright (c) 2006-2011 Varnish Software AS



From geoff at varnish-cache.org  Mon Jan  9 20:52:16 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:16 +0100
Subject: [experimental-ims] 9d0ec2a Add necessary vas.h include
Message-ID: <E1RkMCW-0007dH-VL@project.varnish-software.com>

commit 9d0ec2a1e58fff1fedcffc98f7955ca2110c0c2f
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sun Oct 9 21:58:33 2011 +0000

    Add necessary vas.h include

diff --git a/lib/libvarnish/vmb.c b/lib/libvarnish/vmb.c
index aa069a7..e6b0f25 100644
--- a/lib/libvarnish/vmb.c
+++ b/lib/libvarnish/vmb.c
@@ -28,6 +28,7 @@
 
 #include <pthread.h>
 #include <unistd.h>
+#include "vas.h"
 #include "vmb.h"
 
 #ifdef VMB_NEEDS_PTHREAD_WORKAROUND_THIS_IS_BAD_FOR_PERFORMANCE



From geoff at varnish-cache.org  Mon Jan  9 20:52:42 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:42 +0100
Subject: [experimental-ims] 40c0fef Split the main stats counters into their
	own file.
Message-ID: <E1RkMCw-0008Pk-21@project.varnish-software.com>

commit 40c0fef63a2e90bab9ea3a5418a3670815fb00ca
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu Nov 24 09:46:47 2011 +0000

    Split the main stats counters into their own file.

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index 4b66309..190e945 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -206,12 +206,10 @@ struct acct {
 #define L0(t, n)
 #define L1(t, n)		t n;
 #define VSC_F(n, t, l, f, e,d)	L##l(t, n)
-#define VSC_DO_MAIN
 struct dstat {
-#include "tbl/vsc_fields.h"
+#include "tbl/vsc_f_main.h"
 };
 #undef VSC_F
-#undef VSC_DO_MAIN
 #undef L0
 #undef L1
 
diff --git a/bin/varnishd/cache/cache_wrk.c b/bin/varnishd/cache/cache_wrk.c
index bfee84c..1d80805 100644
--- a/bin/varnishd/cache/cache_wrk.c
+++ b/bin/varnishd/cache/cache_wrk.c
@@ -50,11 +50,9 @@ wrk_sumstat(struct worker *w)
 	Lck_AssertHeld(&wstat_mtx);
 #define L0(n)
 #define L1(n) (VSC_C_main->n += w->stats.n)
-#define VSC_DO_MAIN
 #define VSC_F(n, t, l, f, d, e) L##l(n);
-#include "tbl/vsc_fields.h"
+#include "tbl/vsc_f_main.h"
 #undef VSC_F
-#undef VSC_DO_MAIN
 #undef L0
 #undef L1
 	memset(&w->stats, 0, sizeof w->stats);
diff --git a/include/Makefile.am b/include/Makefile.am
index 62292fc..bb00ce6 100644
--- a/include/Makefile.am
+++ b/include/Makefile.am
@@ -15,6 +15,7 @@ pkginclude_HEADERS = \
 	tbl/vrt_stv_var.h \
 	tbl/vsc_all.h \
 	tbl/vsc_fields.h \
+	tbl/vsc_f_main.h \
 	tbl/vsl_tags.h \
 	vapi/vsm.h \
 	vapi/vsm_int.h \
diff --git a/include/tbl/vsc_all.h b/include/tbl/vsc_all.h
index e12bd2f..4d7b927 100644
--- a/include/tbl/vsc_all.h
+++ b/include/tbl/vsc_all.h
@@ -36,9 +36,7 @@ VSC_DO(LCK, lck, VSC_TYPE_LCK)
 VSC_DONE(LCK, lck, VSC_TYPE_LCK)
 
 VSC_DO(MAIN, main, VSC_TYPE_MAIN)
-#define VSC_DO_MAIN
-#include "tbl/vsc_fields.h"
-#undef VSC_DO_MAIN
+#include "tbl/vsc_f_main.h"
 VSC_DONE(MAIN, main, VSC_TYPE_MAIN)
 
 VSC_DO(SMA, sma, VSC_TYPE_SMA)
diff --git a/include/tbl/vsc_f_main.h b/include/tbl/vsc_f_main.h
new file mode 100644
index 0000000..72d8842
--- /dev/null
+++ b/include/tbl/vsc_f_main.h
@@ -0,0 +1,361 @@
+/*-
+ * Copyright (c) 2006 Verdens Gang AS
+ * Copyright (c) 2006-2011 Varnish Software AS
+ * All rights reserved.
+ *
+ * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION, "")
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * Definition of all shared memory statistics below.
+ *
+ * Fields (n, t, l, f, e, d):
+ *    n - Name:		Field name, in C-source and stats programs
+ *    t - Type:		C-type, uint64_t, unless marked in 'f'
+ *    l - Local:	Local counter in worker thread.
+ *    f - Format:	Semantics of the value in this field
+ *				'a' - Accumulator (deprecated, use 'c')
+ *				'b' - Bitmap
+ *				'c' - Counter, never decreases.
+ *				'g' - Gauge, goes up and down
+ *				'i' - Integer (deprecated, use 'g')
+ *    e - Explantion:	Short explanation of field (for screen use)
+ *    d - Description:	Long explanation of field (for doc use)
+ *
+ * Please describe Gauge variables as "Number of..." to indicate that
+ * this is a snapshot, and Counter variables as "Count of" to indicate
+ * accumulative count.
+ *
+ * -----------------------
+ * NB: Cleanup in progress
+ * -----------------------
+ *
+ * Insufficient attention has caused this to become a swamp of conflicting
+ * conventions, shorthands and general mumbo-jumbo.  I'm trying to clean
+ * it up as I go over the code in other business.
+ *
+ * Please see the sessmem section for how it should look.
+ *
+ */
+
+/*---------------------------------------------------------------------
+ * Sessions
+ *    see: cache_acceptor.c and cache_pool.c
+ */
+
+VSC_F(sess_conn,		uint64_t, 1, 'c',
+    "Sessions accepted",
+	"Count of sessions succesfully accepted"
+)
+VSC_F(sess_drop,		uint64_t, 1, 'c',
+    "Sessions dropped",
+	"Count of sessions silently dropped due to lack of session memory."
+	"  See parameter 'max_sess'."
+)
+
+VSC_F(sess_fail,		uint64_t, 1, 'c',
+    "Session accept failures",
+	"Count of failures to accept TCP connection."
+	"  Either the client changed its mind, or the kernel ran out of"
+	" some resource like filedescriptors."
+)
+
+/*---------------------------------------------------------------------*/
+
+VSC_F(client_req,		uint64_t, 1, 'a',
+      "Client requests received",
+      "")
+
+VSC_F(cache_hit,		uint64_t, 1, 'a',
+      "Cache hits",
+      "Count of cache hits. "
+      "  A cache hit indicates that an object has been delivered to a"
+      "  client without fetching it from a backend server."
+)
+
+VSC_F(cache_hitpass,	uint64_t, 1, 'a',
+      "Cache hits for pass",
+      "Count of hits for pass"
+      "  A cache hit for pass indicates that Varnish is going to"
+      "  pass the request to the backend and this decision has been "
+      "  cached in it self. This counts how many times the cached "
+      "  decision is being used."
+)
+VSC_F(cache_miss,		uint64_t, 1, 'a',
+      "Cache misses",
+      "Count of misses"
+      "  A cache miss indicates the object was fetched from the"
+      "  backend before delivering it to the backend.")
+
+VSC_F(backend_conn,	uint64_t, 0, 'a',
+      "Backend conn. success",
+      "")
+
+VSC_F(backend_unhealthy,	uint64_t, 0, 'a',
+      "Backend conn. not attempted",
+      ""
+)
+VSC_F(backend_busy,	uint64_t, 0, 'a', "Backend conn. too many", "")
+VSC_F(backend_fail,	uint64_t, 0, 'a', "Backend conn. failures", "")
+VSC_F(backend_reuse,	uint64_t, 0, 'a',
+      "Backend conn. reuses",
+      "Count of backend connection reuses"
+      "  This counter is increased whenever we reuse a recycled connection.")
+VSC_F(backend_toolate,	uint64_t, 0, 'a', "Backend conn. was closed", "")
+VSC_F(backend_recycle,	uint64_t, 0, 'a',
+      "Backend conn. recycles",
+      "Count of backend connection recycles"
+      "  This counter is increased whenever we have a keep-alive"
+      "  connection that is put back into the pool of connections."
+      "  It has not yet been used, but it might be, unless the backend"
+      "  closes it.")
+VSC_F(backend_retry,	uint64_t, 0, 'a', "Backend conn. retry", "")
+
+VSC_F(fetch_head,		uint64_t, 1, 'a', "Fetch head", "")
+VSC_F(fetch_length,		uint64_t, 1, 'a', "Fetch with Length", "")
+VSC_F(fetch_chunked,		uint64_t, 1, 'a', "Fetch chunked", "")
+VSC_F(fetch_eof,		uint64_t, 1, 'a', "Fetch EOF", "")
+VSC_F(fetch_bad,		uint64_t, 1, 'a', "Fetch had bad headers", "")
+VSC_F(fetch_close,		uint64_t, 1, 'a', "Fetch wanted close", "")
+VSC_F(fetch_oldhttp,		uint64_t, 1, 'a', "Fetch pre HTTP/1.1 closed", "")
+VSC_F(fetch_zero,		uint64_t, 1, 'a', "Fetch zero len", "")
+VSC_F(fetch_failed,		uint64_t, 1, 'a', "Fetch failed", "")
+VSC_F(fetch_1xx,		uint64_t, 1, 'a', "Fetch no body (1xx)", "")
+VSC_F(fetch_204,		uint64_t, 1, 'a', "Fetch no body (204)", "")
+VSC_F(fetch_304,		uint64_t, 1, 'a', "Fetch no body (304)", "")
+
+/*---------------------------------------------------------------------
+ * Session Memory
+ *    see: cache_session.c
+ */
+
+VSC_F(sessmem_size,		uint64_t, 1, 'g',
+    "Session mem size",
+	"Bytes of memory allocated for last allocated session."
+)
+
+VSC_F(sessmem_alloc,		uint64_t, 1, 'c',
+    "Session mem allocated",
+	"Count of all allocations of session memory."
+)
+
+VSC_F(sessmem_free,		uint64_t, 1, 'c',
+    "Session mem freed",
+	"Count of all frees of session memory."
+)
+
+VSC_F(sessmem_fail,		uint64_t, 1, 'c',
+    "Session mem alloc failed",
+	"Count of session memory allocation failures."
+)
+
+VSC_F(sessmem_limit,		uint64_t, 1, 'c',
+    "Session mem alloc limited",
+	"Count of session memory allocations blocked by limit (max_sess)."
+)
+
+/*---------------------------------------------------------------------
+ * Pools, threads, and sessions
+ *    see: cache_pool.c
+ *
+ */
+
+VSC_F(pools,			uint64_t, 1, 'g',
+    "Number of thread pools",
+	"Number of thread pools.  See also param wthread_pools."
+	"  NB: Presently pools cannot be removed once created."
+)
+
+VSC_F(threads,			uint64_t, 1, 'g',
+    "Total number of threads",
+	"Number of threads in all pools."
+	"  See also params thread_pools, thread_pool_min & thread_pool_max."
+)
+
+VSC_F(threads_limited,		uint64_t, 1, 'c',
+    "Threads hit max",
+	"Number of times more threads were needed, but limit was reached"
+	" in a thread pool."
+	"  See also param thread_pool_max."
+)
+
+VSC_F(threads_created,		uint64_t, 1, 'c',
+    "Threads created",
+	"Total number of threads created in all pools."
+)
+
+VSC_F(threads_destroyed,	uint64_t, 1, 'c',
+    "Threads destoryed",
+	"Total number of threads destroyed in all pools."
+)
+
+VSC_F(threads_failed,		uint64_t, 1, 'c',
+    "Thread creation failed",
+	"Number of times creating a thread failed."
+	"  See VSL::Debug for diagnostics."
+	"  See also param thread_fail_delay."
+)
+
+VSC_F(thread_queue_len,		uint64_t, 1, 'g',
+    "Length of session queue",
+	"Length of session queue waiting for threads."
+	"  NB: Only updates once per second."
+	"  See also param queue_max."
+)
+
+VSC_F(sess_queued,		uint64_t, 1, 'c',
+    "Sessions queued for thread",
+	"Number of times session was queued waiting for a thread."
+	"  See also param queue_max."
+)
+
+VSC_F(sess_dropped,		uint64_t, 1, 'c',
+    "Sessions dropped for thread",
+	"Number of times session was dropped because the queue were too"
+	" long already."
+	"  See also param queue_max."
+)
+
+/*---------------------------------------------------------------------*/
+
+VSC_F(n_sess_mem,		uint64_t, 0, 'i', "N struct sess_mem", "")
+VSC_F(n_sess,			uint64_t, 0, 'i', "N struct sess", "")
+VSC_F(n_object,			uint64_t, 1, 'i', "N struct object", "")
+VSC_F(n_vampireobject,		uint64_t, 1, 'i', "N unresurrected objects", "")
+VSC_F(n_objectcore,		uint64_t, 1, 'i', "N struct objectcore", "")
+VSC_F(n_objecthead,		uint64_t, 1, 'i', "N struct objecthead", "")
+VSC_F(n_waitinglist,		uint64_t, 1, 'i', "N struct waitinglist", "")
+
+VSC_F(n_vbc,		uint64_t, 0, 'i', "N struct vbc", "")
+
+VSC_F(n_backend,		uint64_t, 0, 'i', "N backends", "")
+
+VSC_F(n_expired,		uint64_t, 0, 'i', "N expired objects", "")
+VSC_F(n_lru_nuked,		uint64_t, 0, 'i', "N LRU nuked objects", "")
+VSC_F(n_lru_moved,		uint64_t, 0, 'i', "N LRU moved objects", "")
+
+VSC_F(losthdr,		uint64_t, 0, 'a', "HTTP header overflows", "")
+
+VSC_F(n_objsendfile,	uint64_t, 0, 'a', "Objects sent with sendfile",
+      "The number of objects sent with the sendfile system call. If enabled "
+      "sendfile will be used on object larger than a certain size.")
+VSC_F(n_objwrite,		uint64_t, 0, 'a', "Objects sent with write",
+      "The number of objects sent with regular write calls."
+      "Writes are used when the objects are too small for sendfile "
+      "or if the sendfile call has been disabled")
+VSC_F(n_objoverflow,	uint64_t, 1, 'a',
+					"Objects overflowing workspace", "")
+
+VSC_F(s_sess,		uint64_t, 1, 'a', "Total Sessions", "")
+VSC_F(s_req,		uint64_t, 1, 'a', "Total Requests", "")
+VSC_F(s_pipe,		uint64_t, 1, 'a', "Total pipe", "")
+VSC_F(s_pass,		uint64_t, 1, 'a', "Total pass", "")
+VSC_F(s_fetch,		uint64_t, 1, 'a', "Total fetch", "")
+VSC_F(s_hdrbytes,		uint64_t, 1, 'a', "Total header bytes", "")
+VSC_F(s_bodybytes,		uint64_t, 1, 'a', "Total body bytes", "")
+
+VSC_F(sess_closed,		uint64_t, 1, 'a', "Session Closed", "")
+VSC_F(sess_pipeline,	uint64_t, 1, 'a', "Session Pipeline", "")
+VSC_F(sess_readahead,	uint64_t, 1, 'a', "Session Read Ahead", "")
+VSC_F(sess_linger,		uint64_t, 1, 'a', "Session Linger", "")
+VSC_F(sess_herd,		uint64_t, 1, 'a', "Session herd", "")
+
+VSC_F(shm_records,		uint64_t, 0, 'a', "SHM records", "")
+VSC_F(shm_writes,		uint64_t, 0, 'a', "SHM writes", "")
+VSC_F(shm_flushes,		uint64_t, 0, 'a', "SHM flushes due to overflow", "")
+VSC_F(shm_cont,		uint64_t, 0, 'a', "SHM MTX contention", "")
+VSC_F(shm_cycles,		uint64_t, 0, 'a', "SHM cycles through buffer", "")
+
+VSC_F(sms_nreq,		uint64_t, 0, 'a', "SMS allocator requests", "")
+VSC_F(sms_nobj,		uint64_t, 0, 'i', "SMS outstanding allocations", "")
+VSC_F(sms_nbytes,		uint64_t, 0, 'i', "SMS outstanding bytes", "")
+VSC_F(sms_balloc,		uint64_t, 0, 'i', "SMS bytes allocated", "")
+VSC_F(sms_bfree,		uint64_t, 0, 'i', "SMS bytes freed", "")
+
+VSC_F(backend_req,		uint64_t, 0, 'a', "Backend requests made", "")
+
+VSC_F(n_vcl,		uint64_t, 0, 'a', "N vcl total", "")
+VSC_F(n_vcl_avail,		uint64_t, 0, 'a', "N vcl available", "")
+VSC_F(n_vcl_discard,	uint64_t, 0, 'a', "N vcl discarded", "")
+
+/**********************************************************************/
+
+VSC_F(bans,			uint64_t, 0, 'g',
+   "Count of bans",
+	"Number of all bans in system, including bans superseded"
+	" by newer bans and bans already checked by the ban-lurker."
+)
+VSC_F(bans_gone,		uint64_t, 0, 'g',
+    "Number of bans marked 'gone'",
+	"Number of bans which are no longer active, either because they"
+	" got checked by the ban-lurker or superseded by newer identical bans."
+)
+VSC_F(bans_req,			uint64_t, 0, 'g',
+    "Number of bans using req.*",
+	"Number of bans which use req.* variables.  These bans can not"
+	" be washed by the ban-lurker."
+)
+VSC_F(bans_added,		uint64_t, 0, 'c',
+    "Bans added",
+	"Counter of bans added to ban list."
+)
+VSC_F(bans_deleted,		uint64_t, 0, 'c',
+    "Bans deleted",
+	"Counter of bans deleted from ban list."
+)
+
+VSC_F(bans_tested,		uint64_t, 0, 'c',
+    "Bans tested against objects",
+	"Count of how many bans and objects have been tested against"
+	" each other."
+)
+VSC_F(bans_tests_tested,	uint64_t, 0, 'c',
+    "Ban tests tested against objects",
+	"Count of how many tests and objects have been tested against"
+	" each other.  'ban req.url == foo && req.http.host == bar'"
+	" counts as one in 'bans_tested' and as two in 'bans_tests_tested'"
+)
+VSC_F(bans_dups,		uint64_t, 0, 'c',
+    "Bans superseded by other bans",
+	"Count of bans replaced by later identical bans."
+)
+
+/**********************************************************************/
+
+VSC_F(hcb_nolock,		uint64_t, 0, 'a', "HCB Lookups without lock", "")
+VSC_F(hcb_lock,		uint64_t, 0, 'a', "HCB Lookups with lock", "")
+VSC_F(hcb_insert,		uint64_t, 0, 'a', "HCB Inserts", "")
+
+VSC_F(esi_errors,		uint64_t, 0, 'a', "ESI parse errors (unlock)", "")
+VSC_F(esi_warnings,		uint64_t, 0, 'a', "ESI parse warnings (unlock)", "")
+VSC_F(client_drop_late,	uint64_t, 0, 'a', "Connection dropped late", "")
+VSC_F(uptime,		uint64_t, 0, 'a', "Client uptime", "")
+
+VSC_F(dir_dns_lookups,	uint64_t, 0, 'a', "DNS director lookups", "")
+VSC_F(dir_dns_failed,	uint64_t, 0, 'a', "DNS director failed lookups", "")
+VSC_F(dir_dns_hit,		uint64_t, 0, 'a', "DNS director cached lookups hit", "")
+VSC_F(dir_dns_cache_full,	uint64_t, 0, 'a', "DNS director full dnscache", "")
+
+VSC_F(vmods,		uint64_t, 0, 'i', "Loaded VMODs", "")
+
+VSC_F(n_gzip,			uint64_t, 0, 'a', "Gzip operations", "")
+VSC_F(n_gunzip,			uint64_t, 0, 'a', "Gunzip operations", "")
diff --git a/include/tbl/vsc_fields.h b/include/tbl/vsc_fields.h
index 738703c..60cde5d 100644
--- a/include/tbl/vsc_fields.h
+++ b/include/tbl/vsc_fields.h
@@ -59,315 +59,6 @@
 
 /**********************************************************************/
 
-#ifdef VSC_DO_MAIN
-
-/*---------------------------------------------------------------------
- * Sessions
- *    see: cache_acceptor.c and cache_pool.c
- */
-
-VSC_F(sess_conn,		uint64_t, 1, 'c',
-    "Sessions accepted",
-	"Count of sessions succesfully accepted"
-)
-VSC_F(sess_drop,		uint64_t, 1, 'c',
-    "Sessions dropped",
-	"Count of sessions silently dropped due to lack of session memory."
-	"  See parameter 'max_sess'."
-)
-
-VSC_F(sess_fail,		uint64_t, 1, 'c',
-    "Session accept failures",
-	"Count of failures to accept TCP connection."
-	"  Either the client changed its mind, or the kernel ran out of"
-	" some resource like filedescriptors."
-)
-
-/*---------------------------------------------------------------------*/
-
-VSC_F(client_req,		uint64_t, 1, 'a',
-      "Client requests received",
-      "")
-
-VSC_F(cache_hit,		uint64_t, 1, 'a',
-      "Cache hits",
-      "Count of cache hits. "
-      "  A cache hit indicates that an object has been delivered to a"
-      "  client without fetching it from a backend server."
-)
-
-VSC_F(cache_hitpass,	uint64_t, 1, 'a',
-      "Cache hits for pass",
-      "Count of hits for pass"
-      "  A cache hit for pass indicates that Varnish is going to"
-      "  pass the request to the backend and this decision has been "
-      "  cached in it self. This counts how many times the cached "
-      "  decision is being used."
-)
-VSC_F(cache_miss,		uint64_t, 1, 'a',
-      "Cache misses",
-      "Count of misses"
-      "  A cache miss indicates the object was fetched from the"
-      "  backend before delivering it to the backend.")
-
-VSC_F(backend_conn,	uint64_t, 0, 'a',
-      "Backend conn. success",
-      "")
-
-VSC_F(backend_unhealthy,	uint64_t, 0, 'a',
-      "Backend conn. not attempted",
-      ""
-)
-VSC_F(backend_busy,	uint64_t, 0, 'a', "Backend conn. too many", "")
-VSC_F(backend_fail,	uint64_t, 0, 'a', "Backend conn. failures", "")
-VSC_F(backend_reuse,	uint64_t, 0, 'a',
-      "Backend conn. reuses",
-      "Count of backend connection reuses"
-      "  This counter is increased whenever we reuse a recycled connection.")
-VSC_F(backend_toolate,	uint64_t, 0, 'a', "Backend conn. was closed", "")
-VSC_F(backend_recycle,	uint64_t, 0, 'a',
-      "Backend conn. recycles",
-      "Count of backend connection recycles"
-      "  This counter is increased whenever we have a keep-alive"
-      "  connection that is put back into the pool of connections."
-      "  It has not yet been used, but it might be, unless the backend"
-      "  closes it.")
-VSC_F(backend_retry,	uint64_t, 0, 'a', "Backend conn. retry", "")
-
-VSC_F(fetch_head,		uint64_t, 1, 'a', "Fetch head", "")
-VSC_F(fetch_length,		uint64_t, 1, 'a', "Fetch with Length", "")
-VSC_F(fetch_chunked,		uint64_t, 1, 'a', "Fetch chunked", "")
-VSC_F(fetch_eof,		uint64_t, 1, 'a', "Fetch EOF", "")
-VSC_F(fetch_bad,		uint64_t, 1, 'a', "Fetch had bad headers", "")
-VSC_F(fetch_close,		uint64_t, 1, 'a', "Fetch wanted close", "")
-VSC_F(fetch_oldhttp,		uint64_t, 1, 'a', "Fetch pre HTTP/1.1 closed", "")
-VSC_F(fetch_zero,		uint64_t, 1, 'a', "Fetch zero len", "")
-VSC_F(fetch_failed,		uint64_t, 1, 'a', "Fetch failed", "")
-VSC_F(fetch_1xx,		uint64_t, 1, 'a', "Fetch no body (1xx)", "")
-VSC_F(fetch_204,		uint64_t, 1, 'a', "Fetch no body (204)", "")
-VSC_F(fetch_304,		uint64_t, 1, 'a', "Fetch no body (304)", "")
-
-/*---------------------------------------------------------------------
- * Session Memory
- *    see: cache_session.c
- */
-
-VSC_F(sessmem_size,		uint64_t, 1, 'g',
-    "Session mem size",
-	"Bytes of memory allocated for last allocated session."
-)
-
-VSC_F(sessmem_alloc,		uint64_t, 1, 'c',
-    "Session mem allocated",
-	"Count of all allocations of session memory."
-)
-
-VSC_F(sessmem_free,		uint64_t, 1, 'c',
-    "Session mem freed",
-	"Count of all frees of session memory."
-)
-
-VSC_F(sessmem_fail,		uint64_t, 1, 'c',
-    "Session mem alloc failed",
-	"Count of session memory allocation failures."
-)
-
-VSC_F(sessmem_limit,		uint64_t, 1, 'c',
-    "Session mem alloc limited",
-	"Count of session memory allocations blocked by limit (max_sess)."
-)
-
-/*---------------------------------------------------------------------
- * Pools, threads, and sessions
- *    see: cache_pool.c
- *
- */
-
-VSC_F(pools,			uint64_t, 1, 'g',
-    "Number of thread pools",
-	"Number of thread pools.  See also param wthread_pools."
-	"  NB: Presently pools cannot be removed once created."
-)
-
-VSC_F(threads,			uint64_t, 1, 'g',
-    "Total number of threads",
-	"Number of threads in all pools."
-	"  See also params thread_pools, thread_pool_min & thread_pool_max."
-)
-
-VSC_F(threads_limited,		uint64_t, 1, 'c',
-    "Threads hit max",
-	"Number of times more threads were needed, but limit was reached"
-	" in a thread pool."
-	"  See also param thread_pool_max."
-)
-
-VSC_F(threads_created,		uint64_t, 1, 'c',
-    "Threads created",
-	"Total number of threads created in all pools."
-)
-
-VSC_F(threads_destroyed,	uint64_t, 1, 'c',
-    "Threads destoryed",
-	"Total number of threads destroyed in all pools."
-)
-
-VSC_F(threads_failed,		uint64_t, 1, 'c',
-    "Thread creation failed",
-	"Number of times creating a thread failed."
-	"  See VSL::Debug for diagnostics."
-	"  See also param thread_fail_delay."
-)
-
-VSC_F(thread_queue_len,		uint64_t, 1, 'g',
-    "Length of session queue",
-	"Length of session queue waiting for threads."
-	"  NB: Only updates once per second."
-	"  See also param queue_max."
-)
-
-VSC_F(sess_queued,		uint64_t, 1, 'c',
-    "Sessions queued for thread",
-	"Number of times session was queued waiting for a thread."
-	"  See also param queue_max."
-)
-
-VSC_F(sess_dropped,		uint64_t, 1, 'c',
-    "Sessions dropped for thread",
-	"Number of times session was dropped because the queue were too"
-	" long already."
-	"  See also param queue_max."
-)
-
-/*---------------------------------------------------------------------*/
-
-VSC_F(n_sess_mem,		uint64_t, 0, 'i', "N struct sess_mem", "")
-VSC_F(n_sess,			uint64_t, 0, 'i', "N struct sess", "")
-VSC_F(n_object,			uint64_t, 1, 'i', "N struct object", "")
-VSC_F(n_vampireobject,		uint64_t, 1, 'i', "N unresurrected objects", "")
-VSC_F(n_objectcore,		uint64_t, 1, 'i', "N struct objectcore", "")
-VSC_F(n_objecthead,		uint64_t, 1, 'i', "N struct objecthead", "")
-VSC_F(n_waitinglist,		uint64_t, 1, 'i', "N struct waitinglist", "")
-
-VSC_F(n_vbc,		uint64_t, 0, 'i', "N struct vbc", "")
-
-VSC_F(n_backend,		uint64_t, 0, 'i', "N backends", "")
-
-VSC_F(n_expired,		uint64_t, 0, 'i', "N expired objects", "")
-VSC_F(n_lru_nuked,		uint64_t, 0, 'i', "N LRU nuked objects", "")
-VSC_F(n_lru_moved,		uint64_t, 0, 'i', "N LRU moved objects", "")
-
-VSC_F(losthdr,		uint64_t, 0, 'a', "HTTP header overflows", "")
-
-VSC_F(n_objsendfile,	uint64_t, 0, 'a', "Objects sent with sendfile",
-      "The number of objects sent with the sendfile system call. If enabled "
-      "sendfile will be used on object larger than a certain size.")
-VSC_F(n_objwrite,		uint64_t, 0, 'a', "Objects sent with write",
-      "The number of objects sent with regular write calls."
-      "Writes are used when the objects are too small for sendfile "
-      "or if the sendfile call has been disabled")
-VSC_F(n_objoverflow,	uint64_t, 1, 'a',
-					"Objects overflowing workspace", "")
-
-VSC_F(s_sess,		uint64_t, 1, 'a', "Total Sessions", "")
-VSC_F(s_req,		uint64_t, 1, 'a', "Total Requests", "")
-VSC_F(s_pipe,		uint64_t, 1, 'a', "Total pipe", "")
-VSC_F(s_pass,		uint64_t, 1, 'a', "Total pass", "")
-VSC_F(s_fetch,		uint64_t, 1, 'a', "Total fetch", "")
-VSC_F(s_hdrbytes,		uint64_t, 1, 'a', "Total header bytes", "")
-VSC_F(s_bodybytes,		uint64_t, 1, 'a', "Total body bytes", "")
-
-VSC_F(sess_closed,		uint64_t, 1, 'a', "Session Closed", "")
-VSC_F(sess_pipeline,	uint64_t, 1, 'a', "Session Pipeline", "")
-VSC_F(sess_readahead,	uint64_t, 1, 'a', "Session Read Ahead", "")
-VSC_F(sess_linger,		uint64_t, 1, 'a', "Session Linger", "")
-VSC_F(sess_herd,		uint64_t, 1, 'a', "Session herd", "")
-
-VSC_F(shm_records,		uint64_t, 0, 'a', "SHM records", "")
-VSC_F(shm_writes,		uint64_t, 0, 'a', "SHM writes", "")
-VSC_F(shm_flushes,		uint64_t, 0, 'a', "SHM flushes due to overflow", "")
-VSC_F(shm_cont,		uint64_t, 0, 'a', "SHM MTX contention", "")
-VSC_F(shm_cycles,		uint64_t, 0, 'a', "SHM cycles through buffer", "")
-
-VSC_F(sms_nreq,		uint64_t, 0, 'a', "SMS allocator requests", "")
-VSC_F(sms_nobj,		uint64_t, 0, 'i', "SMS outstanding allocations", "")
-VSC_F(sms_nbytes,		uint64_t, 0, 'i', "SMS outstanding bytes", "")
-VSC_F(sms_balloc,		uint64_t, 0, 'i', "SMS bytes allocated", "")
-VSC_F(sms_bfree,		uint64_t, 0, 'i', "SMS bytes freed", "")
-
-VSC_F(backend_req,		uint64_t, 0, 'a', "Backend requests made", "")
-
-VSC_F(n_vcl,		uint64_t, 0, 'a', "N vcl total", "")
-VSC_F(n_vcl_avail,		uint64_t, 0, 'a', "N vcl available", "")
-VSC_F(n_vcl_discard,	uint64_t, 0, 'a', "N vcl discarded", "")
-
-/**********************************************************************/
-
-VSC_F(bans,			uint64_t, 0, 'g',
-   "Count of bans",
-	"Number of all bans in system, including bans superseded"
-	" by newer bans and bans already checked by the ban-lurker."
-)
-VSC_F(bans_gone,		uint64_t, 0, 'g',
-    "Number of bans marked 'gone'",
-	"Number of bans which are no longer active, either because they"
-	" got checked by the ban-lurker or superseded by newer identical bans."
-)
-VSC_F(bans_req,			uint64_t, 0, 'g',
-    "Number of bans using req.*",
-	"Number of bans which use req.* variables.  These bans can not"
-	" be washed by the ban-lurker."
-)
-VSC_F(bans_added,		uint64_t, 0, 'c',
-    "Bans added",
-	"Counter of bans added to ban list."
-)
-VSC_F(bans_deleted,		uint64_t, 0, 'c',
-    "Bans deleted",
-	"Counter of bans deleted from ban list."
-)
-
-VSC_F(bans_tested,		uint64_t, 0, 'c',
-    "Bans tested against objects",
-	"Count of how many bans and objects have been tested against"
-	" each other."
-)
-VSC_F(bans_tests_tested,	uint64_t, 0, 'c',
-    "Ban tests tested against objects",
-	"Count of how many tests and objects have been tested against"
-	" each other.  'ban req.url == foo && req.http.host == bar'"
-	" counts as one in 'bans_tested' and as two in 'bans_tests_tested'"
-)
-VSC_F(bans_dups,		uint64_t, 0, 'c',
-    "Bans superseded by other bans",
-	"Count of bans replaced by later identical bans."
-)
-
-/**********************************************************************/
-
-VSC_F(hcb_nolock,		uint64_t, 0, 'a', "HCB Lookups without lock", "")
-VSC_F(hcb_lock,		uint64_t, 0, 'a', "HCB Lookups with lock", "")
-VSC_F(hcb_insert,		uint64_t, 0, 'a', "HCB Inserts", "")
-
-VSC_F(esi_errors,		uint64_t, 0, 'a', "ESI parse errors (unlock)", "")
-VSC_F(esi_warnings,		uint64_t, 0, 'a', "ESI parse warnings (unlock)", "")
-VSC_F(client_drop_late,	uint64_t, 0, 'a', "Connection dropped late", "")
-VSC_F(uptime,		uint64_t, 0, 'a', "Client uptime", "")
-
-VSC_F(dir_dns_lookups,	uint64_t, 0, 'a', "DNS director lookups", "")
-VSC_F(dir_dns_failed,	uint64_t, 0, 'a', "DNS director failed lookups", "")
-VSC_F(dir_dns_hit,		uint64_t, 0, 'a', "DNS director cached lookups hit", "")
-VSC_F(dir_dns_cache_full,	uint64_t, 0, 'a', "DNS director full dnscache", "")
-
-VSC_F(vmods,		uint64_t, 0, 'i', "Loaded VMODs", "")
-
-VSC_F(n_gzip,			uint64_t, 0, 'a', "Gzip operations", "")
-VSC_F(n_gunzip,			uint64_t, 0, 'a', "Gunzip operations", "")
-
-#endif
-
-/**********************************************************************/
-
 #ifdef VSC_DO_LCK
 
 VSC_F(creat,		uint64_t, 0, 'a', "Created locks", "")
diff --git a/man/vsc2rst.c b/man/vsc2rst.c
index 18250d8..4ea94be 100644
--- a/man/vsc2rst.c
+++ b/man/vsc2rst.c
@@ -28,9 +28,7 @@ int main(int argc, char **argv)
 	P("MAIN COUNTERS");
 	P("=============");
 	P("");
-#define VSC_DO_MAIN
-#include "tbl/vsc_fields.h"
-#undef VSC_DO_MAIN
+#include "tbl/vsc_f_main.h"
 
 	P("");
 	P("LOCK COUNTERS");



From geoff at varnish-cache.org  Mon Jan  9 20:51:47 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:51:47 +0100
Subject: [experimental-ims] c717971 Always call fetch-processor begin,
	also if C-L header is bogus.
Message-ID: <E1RkMC3-0006o1-Ox@project.varnish-software.com>

commit c71797102fae2ea0775ceabba96431990394939f
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Sep 12 07:36:12 2011 +0000

    Always call fetch-processor begin, also if C-L header is bogus.
    
    Fixes	#1014

diff --git a/bin/varnishtest/tests/r01014.vtc b/bin/varnishtest/tests/r01014.vtc
new file mode 100644
index 0000000..53a4473
--- /dev/null
+++ b/bin/varnishtest/tests/r01014.vtc
@@ -0,0 +1,13 @@
+varnishtest "bug 1014, Invalid C-L header with gzip"
+
+server s1 {
+	rxreq
+	txresp -nolen -hdr "Content-Encoding: gzip" -hdr "Content-Length:"
+} -start
+
+varnish v1 -vcl+backend { } -start
+
+client c1 {
+	txreq
+	rxresp
+} -run



From geoff at varnish-cache.org  Mon Jan  9 20:52:30 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:30 +0100
Subject: [experimental-ims] 78e7b84 Add a new stats counter for "gone" marked
	bans
Message-ID: <E1RkMCk-00085S-Bf@project.varnish-software.com>

commit 78e7b84010fc8db7a1b9ad35ff76b80f8323d1c1
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Nov 8 13:10:24 2011 +0000

    Add a new stats counter for "gone" marked bans

diff --git a/include/tbl/vsc_fields.h b/include/tbl/vsc_fields.h
index 0fd7e46..904a9e2 100644
--- a/include/tbl/vsc_fields.h
+++ b/include/tbl/vsc_fields.h
@@ -297,12 +297,13 @@ VSC_F(n_vcl,		uint64_t, 0, 'a', "N vcl total", "")
 VSC_F(n_vcl_avail,		uint64_t, 0, 'a', "N vcl available", "")
 VSC_F(n_vcl_discard,	uint64_t, 0, 'a', "N vcl discarded", "")
 
-VSC_F(n_ban,		uint64_t, 0, 'i', "N total active bans", "")
+VSC_F(n_ban,			uint64_t, 0, 'i', "N total active bans", "")
+VSC_F(n_ban_gone,		uint64_t, 0, 'i', "N total gone bans", "")
 VSC_F(n_ban_add,		uint64_t, 0, 'a', "N new bans added", "")
-VSC_F(n_ban_retire,	uint64_t, 0, 'a', "N old bans deleted", "")
-VSC_F(n_ban_obj_test,	uint64_t, 0, 'a', "N objects tested", "")
-VSC_F(n_ban_re_test,	uint64_t, 0, 'a', "N regexps tested against", "")
-VSC_F(n_ban_dups,	uint64_t, 0, 'a', "N duplicate bans removed", "")
+VSC_F(n_ban_retire,		uint64_t, 0, 'a', "N old bans deleted", "")
+VSC_F(n_ban_obj_test,		uint64_t, 0, 'a', "N objects tested", "")
+VSC_F(n_ban_re_test,		uint64_t, 0, 'a', "N regexps tested against", "")
+VSC_F(n_ban_dups,		uint64_t, 0, 'a', "N duplicate bans removed", "")
 
 VSC_F(hcb_nolock,		uint64_t, 0, 'a', "HCB Lookups without lock", "")
 VSC_F(hcb_lock,		uint64_t, 0, 'a', "HCB Lookups with lock", "")



From geoff at varnish-cache.org  Mon Jan  9 20:52:41 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:41 +0100
Subject: [experimental-ims] 900920c Terminate when -r reached EOF
Message-ID: <E1RkMCv-0008O6-EB@project.varnish-software.com>

commit 900920c301449ac631518f0da3e1830401603c56
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Nov 21 09:13:08 2011 +0000

    Terminate when -r reached EOF

diff --git a/lib/libvarnishapi/vsl.c b/lib/libvarnishapi/vsl.c
index 3e7eb40..e1ace33 100644
--- a/lib/libvarnishapi/vsl.c
+++ b/lib/libvarnishapi/vsl.c
@@ -153,7 +153,7 @@ vsl_nextlog(struct vsl *vsl, uint32_t **pp)
 		assert(vsl->rbuflen >= 8);
 		i = read(vsl->r_fd, vsl->rbuf, 8);
 		if (i == 0)
-			return (0);
+			return (-1);
 		if (i != 8)
 			return (-1);
 		l = 2 + VSL_WORDS(VSL_LEN(vsl->rbuf));



From geoff at varnish-cache.org  Mon Jan  9 20:51:46 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:51:46 +0100
Subject: [experimental-ims] bac89d8 Improve the FetchError VSL messages.
Message-ID: <E1RkMC2-0006lJ-6C@project.varnish-software.com>

commit bac89d865448f9525e2a82a9ace0c8d708e0c144
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Sep 7 12:08:02 2011 +0000

    Improve the FetchError VSL messages.
    
    Fixes	#1005

diff --git a/bin/varnishd/cache.h b/bin/varnishd/cache.h
index 09c94f3..1c1952d 100644
--- a/bin/varnishd/cache.h
+++ b/bin/varnishd/cache.h
@@ -203,6 +203,7 @@ struct http_conn {
 	struct ws		*ws;
 	txt			rxbuf;
 	txt			pipeline;
+	const char		*error;
 };
 
 /*--------------------------------------------------------------------*/
diff --git a/bin/varnishd/cache_fetch.c b/bin/varnishd/cache_fetch.c
index 24463f5..36df887 100644
--- a/bin/varnishd/cache_fetch.c
+++ b/bin/varnishd/cache_fetch.c
@@ -88,8 +88,10 @@ vfp_nop_bytes(struct sess *sp, struct http_conn *htc, ssize_t bytes)
 
 	while (bytes > 0) {
 		st = FetchStorage(sp, 0);
-		if (st == NULL)
+		if (st == NULL) {
+			htc->error = "Could not get storage";
 			return (-1);
+		}
 		l = st->space - st->len;
 		if (l > bytes)
 			l = bytes;
@@ -202,7 +204,7 @@ fetch_straight(struct sess *sp, struct http_conn *htc, const char *b)
 	assert(sp->wrk->body_status == BS_LENGTH);
 	cl = fetch_number(b, 10);
 	if (cl < 0) {
-		WSP(sp, SLT_FetchError, "straight length syntax");
+		WSP(sp, SLT_FetchError, "straight length field bogus");
 		return (-1);
 	}
 	/*
@@ -217,7 +219,7 @@ fetch_straight(struct sess *sp, struct http_conn *htc, const char *b)
 	i = sp->wrk->vfp->bytes(sp, htc, cl);
 	if (i <= 0) {
 		WSP(sp, SLT_FetchError, "straight read_error: %d %d (%s)",
-		    i, errno, strerror(errno));
+		    i, errno, htc->error);
 		return (-1);
 	}
 	return (0);
@@ -230,7 +232,7 @@ fetch_straight(struct sess *sp, struct http_conn *htc, const char *b)
 		if (i != 1) {					\
 			WSP(sp, SLT_FetchError,			\
 			    "chunked read_error: %d (%s)",	\
-			    errno, strerror(errno));		\
+			    errno, htc->error);			\
 			return (-1);				\
 		}						\
 	} while (0)
@@ -315,7 +317,7 @@ fetch_eof(struct sess *sp, struct http_conn *htc)
 	i = sp->wrk->vfp->bytes(sp, htc, SSIZE_MAX);
 	if (i < 0) {
 		WSP(sp, SLT_FetchError, "eof read_error: %d (%s)",
-		    errno, strerror(errno));
+		    errno, htc->error);
 		return (-1);
 	}
 	return (0);
@@ -396,7 +398,7 @@ FetchHdr(struct sess *sp)
 		AN(sp->objcore->flags & OC_F_BUSY);
 	}
 
-	hp = sp->wrk->bereq;
+	hp = w->bereq;
 
 	sp->vbc = VDI_GetFd(NULL, sp);
 	if (sp->vbc == NULL) {
@@ -437,16 +439,16 @@ FetchHdr(struct sess *sp)
 
 	/* Receive response */
 
-	HTC_Init(sp->wrk->htc, sp->wrk->ws, vc->fd, params->http_resp_size,
+	HTC_Init(w->htc, w->ws, vc->fd, params->http_resp_size,
 	    params->http_resp_hdr_len);
 
 	VTCP_set_read_timeout(vc->fd, vc->first_byte_timeout);
 
-	i = HTC_Rx(sp->wrk->htc);
+	i = HTC_Rx(w->htc);
 
 	if (i < 0) {
 		WSP(sp, SLT_FetchError, "http first read error: %d %d (%s)",
-		    i, errno, strerror(errno));
+		    i, errno, w->htc->error);
 		VDI_CloseFd(sp);
 		/* XXX: other cleanup ? */
 		/* Retryable if we never received anything */
@@ -456,20 +458,20 @@ FetchHdr(struct sess *sp)
 	VTCP_set_read_timeout(vc->fd, vc->between_bytes_timeout);
 
 	while (i == 0) {
-		i = HTC_Rx(sp->wrk->htc);
+		i = HTC_Rx(w->htc);
 		if (i < 0) {
 			WSP(sp, SLT_FetchError,
 			    "http first read error: %d %d (%s)",
-			    i, errno, strerror(errno));
+			    i, errno, w->htc->error);
 			VDI_CloseFd(sp);
 			/* XXX: other cleanup ? */
 			return (-1);
 		}
 	}
 
-	hp = sp->wrk->beresp;
+	hp = w->beresp;
 
-	if (http_DissectResponse(sp->wrk, sp->wrk->htc, hp)) {
+	if (http_DissectResponse(w, w->htc, hp)) {
 		WSP(sp, SLT_FetchError, "http format error");
 		VDI_CloseFd(sp);
 		/* XXX: other cleanup ? */
@@ -485,22 +487,24 @@ FetchBody(struct sess *sp)
 {
 	int cls;
 	struct storage *st;
+	struct worker *w;
 	int mklen;
 
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 	CHECK_OBJ_NOTNULL(sp->wrk, WORKER_MAGIC);
+	w = sp->wrk;
 	CHECK_OBJ_NOTNULL(sp->obj, OBJECT_MAGIC);
 	CHECK_OBJ_NOTNULL(sp->obj->http, HTTP_MAGIC);
 
-	if (sp->wrk->vfp == NULL)
-		sp->wrk->vfp = &vfp_nop;
+	if (w->vfp == NULL)
+		w->vfp = &vfp_nop;
 
 	AN(sp->director);
 	AssertObjCorePassOrBusy(sp->obj->objcore);
 
-	AZ(sp->wrk->vgz_rx);
+	AZ(w->vgz_rx);
 	AZ(VTAILQ_FIRST(&sp->obj->store));
-	switch (sp->wrk->body_status) {
+	switch (w->body_status) {
 	case BS_NONE:
 		cls = 0;
 		mklen = 0;
@@ -510,20 +514,20 @@ FetchBody(struct sess *sp)
 		mklen = 1;
 		break;
 	case BS_LENGTH:
-		cls = fetch_straight(sp, sp->wrk->htc,
-		    sp->wrk->h_content_length);
+		cls = fetch_straight(sp, w->htc,
+		    w->h_content_length);
 		mklen = 1;
-		XXXAZ(sp->wrk->vfp->end(sp));
+		XXXAZ(w->vfp->end(sp));
 		break;
 	case BS_CHUNKED:
-		cls = fetch_chunked(sp, sp->wrk->htc);
+		cls = fetch_chunked(sp, w->htc);
 		mklen = 1;
-		XXXAZ(sp->wrk->vfp->end(sp));
+		XXXAZ(w->vfp->end(sp));
 		break;
 	case BS_EOF:
-		cls = fetch_eof(sp, sp->wrk->htc);
+		cls = fetch_eof(sp, w->htc);
 		mklen = 1;
-		XXXAZ(sp->wrk->vfp->end(sp));
+		XXXAZ(w->vfp->end(sp));
 		break;
 	case BS_ERROR:
 		cls = 1;
@@ -534,26 +538,26 @@ FetchBody(struct sess *sp)
 		mklen = 0;
 		INCOMPL();
 	}
-	AZ(sp->wrk->vgz_rx);
+	AZ(w->vgz_rx);
 
 	/*
 	 * It is OK for ->end to just leave the last storage segment
-	 * sitting on sp->wrk->storage, we will always call vfp_nop_end()
+	 * sitting on w->storage, we will always call vfp_nop_end()
 	 * to get it trimmed or thrown out if empty.
 	 */
 	AZ(vfp_nop_end(sp));
 
-	WSL(sp->wrk, SLT_Fetch_Body, sp->vbc->fd, "%u(%s) %d %u",
-	    sp->wrk->body_status, body_status(sp->wrk->body_status),
+	WSL(w, SLT_Fetch_Body, sp->vbc->fd, "%u(%s) cls %d mklen %u",
+	    w->body_status, body_status(w->body_status),
 	    cls, mklen);
 
-	if (sp->wrk->body_status == BS_ERROR) {
+	if (w->body_status == BS_ERROR) {
 		VDI_CloseFd(sp);
 		return (__LINE__);
 	}
 
 	if (cls < 0) {
-		sp->wrk->stats.fetch_failed++;
+		w->stats.fetch_failed++;
 		/* XXX: Wouldn't this store automatically be released ? */
 		while (!VTAILQ_EMPTY(&sp->obj->store)) {
 			st = VTAILQ_FIRST(&sp->obj->store);
@@ -565,10 +569,10 @@ FetchBody(struct sess *sp)
 		return (__LINE__);
 	}
 
-	if (cls == 0 && sp->wrk->do_close)
+	if (cls == 0 && w->do_close)
 		cls = 1;
 
-	WSL(sp->wrk, SLT_Length, sp->vbc->fd, "%u", sp->obj->len);
+	WSL(w, SLT_Length, sp->vbc->fd, "%u", sp->obj->len);
 
 	{
 	/* Sanity check fetch methods accounting */
@@ -586,7 +590,7 @@ FetchBody(struct sess *sp)
 
 	if (mklen > 0) {
 		http_Unset(sp->obj->http, H_Content_Length);
-		http_PrintfHeader(sp->wrk, sp->fd, sp->obj->http,
+		http_PrintfHeader(w, sp->fd, sp->obj->http,
 		    "Content-Length: %jd", (intmax_t)sp->obj->len);
 	}
 
diff --git a/bin/varnishd/cache_gzip.c b/bin/varnishd/cache_gzip.c
index bed3104..675718a 100644
--- a/bin/varnishd/cache_gzip.c
+++ b/bin/varnishd/cache_gzip.c
@@ -469,8 +469,10 @@ vfp_gunzip_bytes(struct sess *sp, struct http_conn *htc, ssize_t bytes)
 			bytes -= w;
 		}
 
-		if (VGZ_ObufStorage(sp, vg))
+		if (VGZ_ObufStorage(sp, vg)) {
+			htc->error = "Could not get storage";
 			return (-1);
+		}
 		i = VGZ_Gunzip(vg, &dp, &dl);
 		assert(i == VGZ_OK || i == VGZ_END);
 		sp->obj->len += dl;
@@ -479,6 +481,8 @@ vfp_gunzip_bytes(struct sess *sp, struct http_conn *htc, ssize_t bytes)
 	}
 	if (i == Z_OK || i == Z_STREAM_END)
 		return (1);
+	htc->error = "See other message";
+	WSP(sp, SLT_FetchError, "Gunzip trouble (%d)", i);
 	return (-1);
 }
 
@@ -540,8 +544,10 @@ vfp_gzip_bytes(struct sess *sp, struct http_conn *htc, ssize_t bytes)
 			VGZ_Ibuf(vg, ibuf, w);
 			bytes -= w;
 		}
-		if (VGZ_ObufStorage(sp, vg))
+		if (VGZ_ObufStorage(sp, vg)) {
+			htc->error = "Could not get storage";
 			return (-1);
+		}
 		i = VGZ_Gzip(vg, &dp, &dl, VGZ_NORMAL);
 		assert(i == Z_OK);
 		sp->obj->len += dl;
@@ -612,8 +618,10 @@ vfp_testgzip_bytes(struct sess *sp, struct http_conn *htc, ssize_t bytes)
 	AZ(vg->vz.avail_in);
 	while (bytes > 0) {
 		st = FetchStorage(sp, 0);
-		if (st == NULL)
+		if (st == NULL) {
+			htc->error = "Could not get storage";
 			return (-1);
+		}
 		l = st->space - st->len;
 		if (l > bytes)
 			l = bytes;
@@ -631,10 +639,11 @@ vfp_testgzip_bytes(struct sess *sp, struct http_conn *htc, ssize_t bytes)
 			VGZ_Obuf(vg, obuf, sizeof obuf);
 			i = VGZ_Gunzip(vg, &dp, &dl);
 			if (i == VGZ_END && !VGZ_IbufEmpty(vg)) {
-				WSP(sp, SLT_FetchError, "Junk after gzip data");
+				htc->error = "Junk after gzip data";
 				return (-1);
 			}
 			if (i != VGZ_OK && i != VGZ_END) {
+				htc->error = "See other message";
 				WSP(sp, SLT_FetchError,
 				    "Invalid Gzip data: %s", vg->vz.msg);
 				return (-1);
@@ -643,6 +652,7 @@ vfp_testgzip_bytes(struct sess *sp, struct http_conn *htc, ssize_t bytes)
 	}
 	if (i == VGZ_OK || i == VGZ_END)
 		return (1);
+	htc->error = "See other message";
 	WSP(sp, SLT_FetchError, "Gunzip trouble (%d)", i);
 	return (-1);
 }
@@ -665,5 +675,3 @@ struct vfp vfp_testgzip = {
         .bytes  =       vfp_testgzip_bytes,
         .end    =       vfp_testgzip_end,
 };
-
-
diff --git a/bin/varnishd/cache_httpconn.c b/bin/varnishd/cache_httpconn.c
index 3840ed6..9d6d926 100644
--- a/bin/varnishd/cache_httpconn.c
+++ b/bin/varnishd/cache_httpconn.c
@@ -88,6 +88,7 @@ HTC_Init(struct http_conn *htc, struct ws *ws, int fd, unsigned maxbytes,
 	htc->fd = fd;
 	htc->maxbytes = maxbytes;
 	htc->maxhdr = maxhdr;
+	htc->error = "No error recorded";
 
 	(void)WS_Reserve(htc->ws, htc->maxbytes);
 	htc->rxbuf.b = ws->f;
@@ -109,6 +110,7 @@ HTC_Reinit(struct http_conn *htc)
 	unsigned l;
 
 	CHECK_OBJ_NOTNULL(htc, HTTP_CONN_MAGIC);
+	htc->error = "No error recorded";
 	(void)WS_Reserve(htc->ws, htc->maxbytes);
 	htc->rxbuf.b = htc->ws->f;
 	htc->rxbuf.e = htc->ws->f;
@@ -201,7 +203,10 @@ HTC_Read(struct http_conn *htc, void *d, size_t len)
 	if (len == 0)
 		return (l);
 	i = read(htc->fd, p, len);
-	if (i < 0)
+	if (i < 0) {
+		htc->error = strerror(errno);
 		return (i);
+	} else if (i == 0)
+		htc->error = "Remote closed connection";
 	return (i + l);
 }



From geoff at varnish-cache.org  Mon Jan  9 20:52:42 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:42 +0100
Subject: [experimental-ims] a68844e Clarify various details while trying to
	figure out req-timing reporting
Message-ID: <E1RkMCw-0008Ru-TK@project.varnish-software.com>

commit a68844e29db79e880390331f189e3a07f1b83af8
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Nov 29 10:54:47 2011 +0000

    Clarify various details while trying to figure out req-timing reporting

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index 5e813ca..1475d6d 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -1028,3 +1028,9 @@ AssertObjCorePassOrBusy(const struct objcore *oc)
 	if (oc != NULL)
 		AN (oc->flags & OC_F_BUSY);
 }
+
+/*
+ * We want to cache the most recent timestamp in wrk->lastused to avoid
+ * extra timestamps in cache_pool.c.  Hide this detail with a macro
+ */
+#define W_TIM_real(w) ((w)->lastused = VTIM_real())
diff --git a/bin/varnishd/cache/cache_center.c b/bin/varnishd/cache/cache_center.c
index e42fac8..89886c2 100644
--- a/bin/varnishd/cache/cache_center.c
+++ b/bin/varnishd/cache/cache_center.c
@@ -52,7 +52,7 @@ DOT	label="Request received"
 DOT ]
 DOT ERROR [shape=plaintext]
 DOT RESTART [shape=plaintext]
-DOT acceptor -> start [style=bold,color=green]
+DOT acceptor -> first [style=bold,color=green]
  */
 
 #include "config.h"
@@ -80,6 +80,17 @@ static unsigned xids;
 /*--------------------------------------------------------------------
  * WAIT
  * Wait (briefly) until we have a full request in our htc.
+ *
+DOT subgraph xcluster_wait {
+DOT	wait [
+DOT		shape=box
+DOT		label="wait for\nrequest"
+DOT	]
+DOT	herding [shape=hexagon]
+DOT	wait -> start [label="got req"]
+DOT	wait -> DONE [label="errors"]
+DOT	wait -> herding [label="timeout"]
+DOT }
  */
 
 static int
@@ -91,6 +102,7 @@ cnt_wait(struct sess *sp)
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 	AZ(sp->vcl);
 	AZ(sp->obj);
+	AZ(sp->esi_level);
 	assert(sp->xid == 0);
 
 	i = HTC_Complete(sp->htc);
@@ -116,9 +128,7 @@ cnt_wait(struct sess *sp)
 	}
 	if (i == -2) {
 		SES_Close(sp, "overflow");
-		return (0);
-	}
-	if (i == -1 && Tlen(sp->htc->rxbuf) == 0 &&
+	} else if (i == -1 && Tlen(sp->htc->rxbuf) == 0 &&
 	    (errno == 0 || errno == ECONNRESET))
 		SES_Close(sp, "EOF");
 	else
@@ -210,7 +220,7 @@ cnt_prepresp(struct sess *sp)
 		}
 	}
 
-	sp->t_resp = VTIM_real();
+	sp->t_resp = W_TIM_real(sp->wrk);
 	if (sp->obj->objcore != NULL) {
 		if ((sp->t_resp - sp->obj->last_lru) > cache_param->lru_timeout &&
 		    EXP_Touch(sp->obj->objcore))
@@ -293,6 +303,10 @@ DOT	DONE [
 DOT		shape=hexagon
 DOT		label="Request completed"
 DOT	]
+DOT	ESI_RESP [ shape=hexagon ]
+DOT	DONE -> start
+DOT	DONE -> wait
+DOT	DONE -> ESI_RESP
  */
 
 static int
@@ -325,10 +339,11 @@ cnt_done(struct sess *sp)
 
 	SES_Charge(sp);
 
-	sp->t_end = VTIM_real();
-	sp->wrk->lastused = sp->t_end;
+	sp->t_end = W_TIM_real(sp->wrk);
+WSP(sp, SLT_Debug, "PHK req %.9f resp %.9f end %.9f open %.9f",
+    sp->t_req, sp->t_resp, sp->t_end,  sp->t_open);
 	if (sp->xid == 0) {
-		sp->t_req = sp->t_end;
+		// sp->t_req = sp->t_end;
 		sp->t_resp = sp->t_end;
 	} else if (sp->esi_level == 0) {
 		dp = sp->t_resp - sp->t_req;
@@ -344,14 +359,15 @@ cnt_done(struct sess *sp)
 		    sp->xid, sp->t_req, sp->t_end, dh, dp, da);
 	}
 	sp->xid = 0;
-	sp->t_open = sp->t_end;
-	sp->t_resp = NAN;
 	WSL_Flush(sp->wrk, 0);
 
 	/* If we did an ESI include, don't mess up our state */
 	if (sp->esi_level > 0)
 		return (1);
 
+	sp->t_open = sp->t_end;
+	sp->t_resp = NAN;
+
 	sp->req_bodybytes = 0;
 
 	sp->t_req = NAN;
@@ -465,7 +481,7 @@ cnt_error(struct sess *sp)
 
 	http_PutProtocol(w, sp->vsl_id, h, "HTTP/1.1");
 	http_PutStatus(h, sp->err_code);
-	VTIM_format(VTIM_real(), date);
+	VTIM_format(W_TIM_real(sp->wrk), date);
 	http_PrintfHeader(w, sp->vsl_id, h, "Date: %s", date);
 	http_SetHeader(w, sp->vsl_id, h, "Server: Varnish");
 
@@ -580,7 +596,7 @@ cnt_fetch(struct sess *sp)
 		 * What does RFC2616 think about TTL ?
 		 */
 		EXP_Clr(&sp->wrk->exp);
-		sp->wrk->exp.entered = VTIM_real();
+		sp->wrk->exp.entered = W_TIM_real(sp->wrk);
 		RFC2616_Ttl(sp);
 
 		/* pass from vclrecv{} has negative TTL */
@@ -947,7 +963,15 @@ cnt_streambody(struct sess *sp)
 
 /*--------------------------------------------------------------------
  * The very first request
+DOT subgraph xcluster_first {
+DOT	first [
+DOT		shape=box
+DOT		label="first\nConfigure data structures"
+DOT	]
+DOT }
+DOT first -> wait 
  */
+
 static int
 cnt_first(struct sess *sp)
 {
@@ -961,6 +985,7 @@ cnt_first(struct sess *sp)
 
 	assert(sp->xid == 0);
 	assert(sp->restarts == 0);
+	AZ(sp->esi_level);
 	VCA_Prep(sp);
 
 	/* Record the session watermark */
@@ -969,7 +994,6 @@ cnt_first(struct sess *sp)
 	/* Receive a HTTP protocol request */
 	HTC_Init(sp->htc, sp->ws, sp->fd, sp->vsl_id, cache_param->http_req_size,
 	    cache_param->http_req_hdr_len);
-	sp->wrk->lastused = sp->t_open;
 	sp->wrk->acct_tmp.sess++;
 
 	sp->step = STP_WAIT;
@@ -1344,7 +1368,9 @@ DOT		shape=record
 DOT		label="vcl_recv()|req."
 DOT	]
 DOT }
+DOT ESI_REQ [ shape=hexagon ]
 DOT RESTART -> recv
+DOT ESI_REQ -> recv
 DOT recv -> pipe [label="pipe",style=bold,color=orange]
 DOT recv -> pass2 [label="pass",style=bold,color=red]
 DOT recv -> err_recv [label="error"]
@@ -1445,8 +1471,12 @@ cnt_recv(struct sess *sp)
  * START
  * Handle a request, wherever it came from recv/restart.
  *
-DOT start [shape=box,label="Dissect request"]
+DOT start [
+DOT	shape=box
+DOT	label="Dissect request\nHandle expect"
+DOT ]
 DOT start -> recv [style=bold,color=green]
+DOT start -> DONE [label=errors]
  */
 
 static int
@@ -1460,11 +1490,11 @@ cnt_start(struct sess *sp)
 	AZ(sp->restarts);
 	AZ(sp->obj);
 	AZ(sp->vcl);
+	AZ(sp->esi_level);
 
 	/* Update stats of various sorts */
 	sp->wrk->stats.client_req++;
-	sp->t_req = VTIM_real();
-	sp->wrk->lastused = sp->t_req;
+	sp->t_req = W_TIM_real(sp->wrk);
 	sp->wrk->acct_tmp.req++;
 
 	/* Assign XID and log */



From geoff at varnish-cache.org  Mon Jan  9 20:52:33 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:33 +0100
Subject: [experimental-ims] 6298611 Update to match new layout
Message-ID: <E1RkMCn-0008AN-4R@project.varnish-software.com>

commit 629861111eeb9260428573c3ef9eceff52a67cf0
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sun Nov 13 11:05:45 2011 +0000

    Update to match new layout

diff --git a/bin/varnishd/flint.sh b/bin/varnishd/flint.sh
index 756f65a..9067263 100755
--- a/bin/varnishd/flint.sh
+++ b/bin/varnishd/flint.sh
@@ -15,7 +15,7 @@ flexelint \
 	-I../.. \
 	-I/usr/local/include \
 	-DVARNISH_STATE_DIR=\"foo\" \
-	*.c \
+	cache/*.c \
 	common/*.c \
 	storage/*.c \
 	waiter/*.c \



From geoff at varnish-cache.org  Mon Jan  9 20:52:28 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:28 +0100
Subject: [experimental-ims] 374c3a2 Add a missing case: ESI parent document
	gunzip'ed but included document gzip'ed.
Message-ID: <E1RkMCi-00081B-NA@project.varnish-software.com>

commit 374c3a235fec63ee4f213696fc43731fd329ac2e
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Nov 1 10:49:09 2011 +0000

    Add a missing case:  ESI parent document gunzip'ed but included
    document gzip'ed.
    
    Fixes	#1029

diff --git a/bin/varnishd/cache_esi_parse.c b/bin/varnishd/cache_esi_parse.c
index 663894a..f6eb367 100644
--- a/bin/varnishd/cache_esi_parse.c
+++ b/bin/varnishd/cache_esi_parse.c
@@ -801,7 +801,7 @@ VEP_Parse(const struct worker *w, const char *p, size_t l)
 				vep->state = VEP_ATTR;
 			} else if (p < e) {
 				vep_error(vep,
-				    "XML 1.0 Illegal attribute tart char");
+				    "XML 1.0 Illegal attribute start char");
 				vep->state = VEP_TAGERROR;
 			}
 		} else if (vep->state == VEP_TAGERROR) {
diff --git a/bin/varnishd/cache_response.c b/bin/varnishd/cache_response.c
index 04d2da5..87b0919 100644
--- a/bin/varnishd/cache_response.c
+++ b/bin/varnishd/cache_response.c
@@ -307,6 +307,9 @@ RES_WriteObj(struct sess *sp)
 		ESI_Deliver(sp);
 	} else if (sp->wrk->res_mode & RES_ESI_CHILD && sp->wrk->gzip_resp) {
 		ESI_DeliverChild(sp);
+	} else if (sp->wrk->res_mode & RES_ESI_CHILD &&
+	    !sp->wrk->gzip_resp && sp->obj->gziped) {
+		res_WriteGunzipObj(sp);
 	} else if (sp->wrk->res_mode & RES_GUNZIP) {
 		res_WriteGunzipObj(sp);
 	} else {
diff --git a/bin/varnishtest/tests/r01029.vtc b/bin/varnishtest/tests/r01029.vtc
new file mode 100644
index 0000000..2817bba
--- /dev/null
+++ b/bin/varnishtest/tests/r01029.vtc
@@ -0,0 +1,34 @@
+varnishtest "#1029"
+
+server s1 {
+	rxreq
+	expect req.url == "/bar"
+	txresp -gzipbody {[bar]}
+
+	rxreq
+	expect req.url == "/foo"
+	txresp -body {<h1>FOO<esi:include src="/bar"/>BARF</h1>}
+
+} -start
+
+varnish v1 -vcl+backend {
+	sub vcl_fetch {
+		set beresp.do_esi = true;
+		if (req.url == "/foo") {
+			set beresp.ttl = 0s;
+		} else {
+			set beresp.ttl = 10m;
+		}
+	}
+} -start
+
+client c1 {
+	txreq -url "/bar" -hdr "Accept-Encoding: gzip"
+	rxresp
+	gunzip
+	expect resp.bodylen == 5
+
+	txreq -url "/foo" -hdr "Accept-Encoding: gzip"
+	rxresp
+	expect resp.bodylen == 21
+} -run



From geoff at varnish-cache.org  Mon Jan  9 20:52:47 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:47 +0100
Subject: [experimental-ims] e91fe76 Fix a buglet where we could send one
	backends Host: header to another backend under very rare
	circumstances.
Message-ID: <E1RkMD1-00009v-QM@project.varnish-software.com>

commit e91fe766effc59fc42bbde821b69552c086c60f3
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Dec 5 13:06:28 2011 +0000

    Fix a buglet where we could send one backends Host: header to another
    backend under very rare circumstances.

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index 4cfad78..05aaf39 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -712,7 +712,7 @@ int EXP_NukeOne(struct worker *w, struct lru *lru);
 struct storage *FetchStorage(struct worker *w, ssize_t sz);
 int FetchError(struct worker *w, const char *error);
 int FetchError2(struct worker *w, const char *error, const char *more);
-int FetchHdr(struct sess *sp);
+int FetchHdr(struct sess *sp, int need_host_hdr);
 int FetchBody(struct worker *w, struct object *obj);
 int FetchReqBody(struct sess *sp);
 void Fetch_Init(void);
diff --git a/bin/varnishd/cache/cache_center.c b/bin/varnishd/cache/cache_center.c
index 997ecac..d2126ba 100644
--- a/bin/varnishd/cache/cache_center.c
+++ b/bin/varnishd/cache/cache_center.c
@@ -554,7 +554,7 @@ DOT errfetch [label="ERROR",shape=plaintext]
 static int
 cnt_fetch(struct sess *sp)
 {
-	int i;
+	int i, need_host_hdr;
 	struct worker *wrk;
 
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
@@ -571,7 +571,9 @@ cnt_fetch(struct sess *sp)
 
 	http_Setup(wrk->beresp, wrk->ws);
 
-	i = FetchHdr(sp);
+	need_host_hdr = !http_GetHdr(wrk->bereq, H_Host, NULL);
+
+	i = FetchHdr(sp, need_host_hdr);
 	/*
 	 * If we recycle a backend connection, there is a finite chance
 	 * that the backend closed it before we get a request to it.
@@ -579,7 +581,7 @@ cnt_fetch(struct sess *sp)
 	 */
 	if (i == 1) {
 		VSC_C_main->backend_retry++;
-		i = FetchHdr(sp);
+		i = FetchHdr(sp, need_host_hdr);
 	}
 
 	if (i) {
diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c
index 1d4eff3..7d8763f 100644
--- a/bin/varnishd/cache/cache_fetch.c
+++ b/bin/varnishd/cache/cache_fetch.c
@@ -375,11 +375,10 @@ FetchReqBody(struct sess *sp)
  */
 
 int
-FetchHdr(struct sess *sp)
+FetchHdr(struct sess *sp, int need_host_hdr)
 {
 	struct vbc *vc;
 	struct worker *w;
-	char *b;
 	struct http *hp;
 	int retry = -1;
 	int i;
@@ -415,7 +414,7 @@ FetchHdr(struct sess *sp)
 	 * header if one is necessary.  This cannot be done in the VCL
 	 * because the backend may be chosen by a director.
 	 */
-	if (!http_GetHdr(hp, H_Host, &b))
+	if (need_host_hdr)
 		VDI_AddHostHeader(sp);
 
 	(void)VTCP_blocking(vc->fd);	/* XXX: we should timeout instead */



From geoff at varnish-cache.org  Mon Jan  9 20:51:46 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:51:46 +0100
Subject: [experimental-ims] 20732d8 Add return(restart) in vcl_error to the
	dot-graph
Message-ID: <E1RkMC2-0006m3-Ua@project.varnish-software.com>

commit 20732d87437fd0daa32d5abbbc85c38f08a8991a
Author: Kristian Lyngstol <kristian at bohemians.org>
Date:   Thu Sep 8 09:48:50 2011 +0200

    Add return(restart) in vcl_error to the dot-graph
    
    Thanks to scoof for pointing this out.

diff --git a/bin/varnishd/cache_center.c b/bin/varnishd/cache_center.c
index 6bb8324..39a2104 100644
--- a/bin/varnishd/cache_center.c
+++ b/bin/varnishd/cache_center.c
@@ -413,6 +413,8 @@ DOT	]
 DOT	ERROR -> vcl_error
 DOT	vcl_error-> prepresp [label=deliver]
 DOT }
+DOT vcl_error-> rsterr [label="restart",color=purple]
+DOT rsterr [label="RESTART",shape=plaintext]
  */
 
 static int



From geoff at varnish-cache.org  Mon Jan  9 20:52:33 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:33 +0100
Subject: [experimental-ims] 10bfefc Rewrite VCLI_ReadResult() to be much more
	robust.
Message-ID: <E1RkMCn-0008AV-7F@project.varnish-software.com>

commit 10bfefcaf40f4fd3302bc29237e263b99f590d6e
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sun Nov 13 21:58:52 2011 +0000

    Rewrite VCLI_ReadResult() to be much more robust.

diff --git a/lib/libvarnish/cli_common.c b/lib/libvarnish/cli_common.c
index 43b91c3..3a44e9f 100644
--- a/lib/libvarnish/cli_common.c
+++ b/lib/libvarnish/cli_common.c
@@ -121,7 +121,7 @@ read_tmo(int fd, char *ptr, unsigned len, double tmo)
 	struct pollfd pfd;
 
 	if (tmo > 0)
-		to = tmo * 1e3;
+		to = (int)(tmo * 1e3);
 	else
 		to = -1;
 	pfd.fd = fd;
@@ -150,44 +150,56 @@ VCLI_ReadResult(int fd, unsigned *status, char **ptr, double tmo)
 	char res[CLI_LINE0_LEN];	/* For NUL */
 	int i, j;
 	unsigned u, v, s;
-	char *p;
+	char *p = NULL;
+	const char *err = "CLI communication error (hdr)";
 
 	if (status == NULL)
 		status = &s;
 	if (ptr != NULL)
 		*ptr = NULL;
-	i = read_tmo(fd, res, CLI_LINE0_LEN, tmo);
-	if (i != CLI_LINE0_LEN) {
-		*status = CLIS_COMMS;
-		if (ptr != NULL)
-			*ptr = strdup("CLI communication error (hdr)");
-		if (i != 0)
-			return (i);
-		return (*status);
-	}
-	assert(i == CLI_LINE0_LEN);
-	assert(res[3] == ' ');
-	assert(res[CLI_LINE0_LEN - 1] == '\n');
-	res[CLI_LINE0_LEN - 1] = '\0';
-	j = sscanf(res, "%u %u\n", &u, &v);
-	assert(j == 2);
-	*status = u;
-	p = malloc(v + 1L);
-	assert(p != NULL);
-	i = read_tmo(fd, p, v + 1, tmo);
-	if (i < 0) {
-		*status = CLIS_COMMS;
-		free(p);
-		if (ptr != NULL)
-			*ptr = strdup("CLI communication error (body)");
-		return (i);
-	}
-	assert(i == v + 1);
-	assert(p[v] == '\n');
-	p[v] = '\0';
-	if (ptr == NULL)
+	do {
+		i = read_tmo(fd, res, CLI_LINE0_LEN, tmo);
+		if (i != CLI_LINE0_LEN)
+			break;
+
+		if (res[3] != ' ')
+			break;
+
+		if (res[CLI_LINE0_LEN - 1] != '\n')
+			break;
+
+		res[CLI_LINE0_LEN - 1] = '\0';
+		j = sscanf(res, "%u %u\n", &u, &v);
+		if (j != 2)
+			break;
+
+		err = "CLI communication error (body)";
+
+		*status = u;
+		p = malloc(v + 1L);
+		if (p == NULL)
+			break;
+
+		i = read_tmo(fd, p, v + 1, tmo);
+		if (i < 0)
+			break;
+		if (i != v + 1)
+			break;
+		if (p[v] != '\n')
+			break;
+		
+		p[v] = '\0';
+		if (ptr == NULL)
+			free(p);
+		else
+			*ptr = p;
+		return (0);
+	} while(0);
+
+	if (p != NULL)
 		free(p);
-	else
-		*ptr = p;
-	return (0);
+	*status = CLIS_COMMS;
+	if (ptr != NULL)
+		*ptr = strdup(err);
+	return (*status);
 }



From geoff at varnish-cache.org  Mon Jan  9 20:52:00 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:00 +0100
Subject: [experimental-ims] 4a976ec Remove never effected test
Message-ID: <E1RkMCG-00078f-9V@project.varnish-software.com>

commit 4a976ecab7fdcbd58c79c6a1d5b589e3b9605e69
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Sep 20 19:48:49 2011 +0000

    Remove never effected test

diff --git a/bin/varnishtest/tests/g00002.vtc b/bin/varnishtest/tests/g00002.vtc
index 574f1f2..1a0ab35 100644
--- a/bin/varnishtest/tests/g00002.vtc
+++ b/bin/varnishtest/tests/g00002.vtc
@@ -47,11 +47,4 @@ client c1 {
 	rxresp
 	expect resp.http.content-encoding == "resp.http.content-encoding"
 	expect resp.bodylen == 4109
-
-	# See varnish can deliver gzip'ed ESI (NOTYET)
-	#txreq -url /bar -hdr "Accept-Encoding: gzip"
-	#rxresp
-	# expect resp.http.content-encoding == "gzip"
-	#gunzip
-	#expect resp.bodylen == 4109
 } -run



From geoff at varnish-cache.org  Mon Jan  9 20:51:46 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:51:46 +0100
Subject: [experimental-ims] 3302916 Quench a pointless debug printf
Message-ID: <E1RkMC2-0006lg-Dl@project.varnish-software.com>

commit 3302916f320dd64ad5a866b03687306ceb441936
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Sep 7 20:13:07 2011 +0000

    Quench a pointless debug printf

diff --git a/bin/varnishd/storage_file.c b/bin/varnishd/storage_file.c
index 7687179..45dfac4 100644
--- a/bin/varnishd/storage_file.c
+++ b/bin/varnishd/storage_file.c
@@ -104,13 +104,10 @@ struct smf_sc {
 /*--------------------------------------------------------------------*/
 
 static void
-smf_initfile(struct stevedore *st, struct smf_sc *sc, const char *size)
+smf_initfile(struct smf_sc *sc, const char *size)
 {
 	sc->filesize = STV_FileSize(sc->fd, size, &sc->pagesize, "-sfile");
 
-	printf("SMF.%s: filename: %s size %ju MB.\n",
-	      st->ident, sc->filename, sc->filesize / (1024 * 1024));
-
 	AZ(ftruncate(sc->fd, (off_t)sc->filesize));
 
 	/* XXX: force block allocation here or in open ? */
@@ -162,7 +159,7 @@ smf_init(struct stevedore *parent, int ac, char * const *av)
 	(void)STV_GetFile(fn, &sc->fd, &sc->filename, "-sfile");
 
 	mgt_child_inherit(sc->fd, "storage_file");
-	smf_initfile(parent, sc, size);
+	smf_initfile(sc, size);
 }
 
 /*--------------------------------------------------------------------



From geoff at varnish-cache.org  Mon Jan  9 20:52:00 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:00 +0100
Subject: [experimental-ims] 4351233 Fix a long-standing bug in varnishtest:
Message-ID: <E1RkMCG-000793-FH@project.varnish-software.com>

commit 43512331ff01fad7f5078cda5339ba169bd1d81f
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Sep 20 20:22:09 2011 +0000

    Fix a long-standing bug in varnishtest:
    
    It used to be that we'd call http processing on a filedescriptor and
    then that was that.  Then we added keywords to do new accepts in the
    server and suddenly the calling code closes a wrong filedesc which
    belongs to somebody else and things get really inconvenient fast.
    
    This made all test-cases which use the "accept" server directive flakey.

diff --git a/bin/varnishtest/vtc.h b/bin/varnishtest/vtc.h
index eb9a02a..5e5cce7 100644
--- a/bin/varnishtest/vtc.h
+++ b/bin/varnishtest/vtc.h
@@ -64,7 +64,7 @@ extern pthread_t	vtc_thread;
 
 void init_sema(void);
 
-void http_process(struct vtclog *vl, const char *spec, int sock, int sfd);
+int http_process(struct vtclog *vl, const char *spec, int sock, int *sfd);
 
 void cmd_server_genvcl(struct vsb *vsb);
 
diff --git a/bin/varnishtest/vtc_client.c b/bin/varnishtest/vtc_client.c
index 20b152b..3bd430b 100644
--- a/bin/varnishtest/vtc_client.c
+++ b/bin/varnishtest/vtc_client.c
@@ -103,7 +103,7 @@ client_thread(void *priv)
 		VTCP_myname(fd, mabuf, sizeof mabuf, mpbuf, sizeof mpbuf);
 		vtc_log(vl, 3, "connected fd %d from %s %s to %s",
 		    fd, mabuf, mpbuf, VSB_data(vsb));
-		http_process(vl, c->spec, fd, -1);
+		fd = http_process(vl, c->spec, fd, NULL);
 		vtc_log(vl, 3, "closing fd %d", fd);
 		VTCP_close(&fd);
 	}
diff --git a/bin/varnishtest/vtc_http.c b/bin/varnishtest/vtc_http.c
index 7dad4ed..32c7e0c 100644
--- a/bin/varnishtest/vtc_http.c
+++ b/bin/varnishtest/vtc_http.c
@@ -54,7 +54,7 @@ struct http {
 	unsigned		magic;
 #define HTTP_MAGIC		0x2f02169c
 	int			fd;
-	int			sfd;
+	int			*sfd;
 	int			timeout;
 	struct vtclog		*vl;
 
@@ -77,14 +77,14 @@ struct http {
 
 #define ONLY_CLIENT(hp, av)						\
 	do {								\
-		if (hp->sfd >= 0)					\
+		if (hp->sfd != NULL)					\
 			vtc_log(hp->vl, 0,				\
 			    "\"%s\" only possible in client", av[0]);	\
 	} while (0)
 
 #define ONLY_SERVER(hp, av)						\
 	do {								\
-		if (hp->sfd < 0)					\
+		if (hp->sfd == NULL)					\
 			vtc_log(hp->vl, 0,				\
 			    "\"%s\" only possible in server", av[0]);	\
 	} while (0)
@@ -344,19 +344,22 @@ http_rxchar_eof(struct http *hp, int n)
 		pfd[0].revents = 0;
 		i = poll(pfd, 1, hp->timeout);
 		if (i < 0)
-			vtc_log(hp->vl, 0, "HTTP rx timeout (%u ms)",
-			    hp->timeout);
+			vtc_log(hp->vl, 0, "HTTP rx timeout (fd:%d %u ms)",
+			    hp->fd, hp->timeout);
 		if (i < 0)
-			vtc_log(hp->vl, 0, "HTTP rx failed (poll: %s)",
-			    strerror(errno));
+			vtc_log(hp->vl, 0, "HTTP rx failed (fd:%d poll: %s)",
+			    hp->fd, strerror(errno));
 		assert(i > 0);
+		if (pfd[0].revents & ~POLLIN)
+			vtc_log(hp->vl, 4, "HTTP rx poll (fd:%d revents: %x)",
+			    hp->fd, pfd[0].revents);
 		assert(hp->prxbuf + n < hp->nrxbuf);
 		i = read(hp->fd, hp->rxbuf + hp->prxbuf, n);
 		if (i == 0)
 			return (i);
 		if (i <= 0)
-			vtc_log(hp->vl, 0, "HTTP rx failed (read: %s)",
-			    strerror(errno));
+			vtc_log(hp->vl, 0, "HTTP rx failed (fd:%d read: %s)",
+			    hp->fd, strerror(errno));
 		hp->prxbuf += i;
 		hp->rxbuf[hp->prxbuf] = '\0';
 		n -= i;
@@ -1036,7 +1039,7 @@ cmd_http_expect_close(CMD_ARGS)
 	(void)vl;
 	CAST_OBJ_NOTNULL(hp, priv, HTTP_MAGIC);
 	AZ(av[1]);
-	assert(hp->sfd >= 0);
+	assert(hp->sfd != NULL);
 
 	vtc_log(vl, 4, "Expecting close (fd = %d)", hp->fd);
 	while (1) {
@@ -1074,10 +1077,11 @@ cmd_http_accept(CMD_ARGS)
 	(void)vl;
 	CAST_OBJ_NOTNULL(hp, priv, HTTP_MAGIC);
 	AZ(av[1]);
+	assert(hp->sfd != NULL);
 	assert(hp->sfd >= 0);
 	VTCP_close(&hp->fd);
 	vtc_log(vl, 4, "Accepting");
-	hp->fd = accept(hp->sfd, NULL, NULL);
+	hp->fd = accept(*hp->sfd, NULL, NULL);
 	if (hp->fd < 0)
 		vtc_log(vl, 0, "Accepted failed: %s", strerror(errno));
 	vtc_log(vl, 3, "Accepted socket fd is %d", hp->fd);
@@ -1136,11 +1140,12 @@ static const struct cmds http_cmds[] = {
 	{ NULL,			NULL }
 };
 
-void
-http_process(struct vtclog *vl, const char *spec, int sock, int sfd)
+int
+http_process(struct vtclog *vl, const char *spec, int sock, int *sfd)
 {
 	struct http *hp;
 	char *s, *q;
+	int retval;
 
 	(void)sfd;
 	ALLOC_OBJ(hp, HTTP_MAGIC);
@@ -1162,9 +1167,11 @@ http_process(struct vtclog *vl, const char *spec, int sock, int sfd)
 	assert(q > s);
 	AN(s);
 	parse_string(s, http_cmds, hp, vl);
+	retval = hp->fd;
 	VSB_delete(hp->vsb);
 	free(hp->rxbuf);
 	free(hp);
+	return (retval);
 }
 
 /**********************************************************************
diff --git a/bin/varnishtest/vtc_server.c b/bin/varnishtest/vtc_server.c
index 542a266..d1787b1 100644
--- a/bin/varnishtest/vtc_server.c
+++ b/bin/varnishtest/vtc_server.c
@@ -100,7 +100,7 @@ server_thread(void *priv)
 		if (fd < 0)
 			vtc_log(vl, 0, "Accepted failed: %s", strerror(errno));
 		vtc_log(vl, 3, "accepted fd %d", fd);
-		http_process(vl, s->spec, fd, s->sock);
+		fd = http_process(vl, s->spec, fd, &s->sock);
 		vtc_log(vl, 3, "shutting fd %d", fd);
 		j = shutdown(fd, SHUT_WR);
 		if (!VTCP_Check(j))



From geoff at varnish-cache.org  Mon Jan  9 20:52:50 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:50 +0100
Subject: [experimental-ims] 9e5fe24 Merge experimental-ims with trunk as of
	2011-11-22
Message-ID: <E1RkMD4-0000FZ-L7@project.varnish-software.com>

commit 9e5fe240f7c393aa7f6cae56795b5e9cbc336a25
Merge: ace762d dcd622d
Author: Geoff Simmons <geoff at uplex.de>
Date:   Mon Jan 9 21:28:45 2012 +0100

    Merge experimental-ims with trunk as of 2011-11-22

diff --cc bin/varnishd/cache/cache.h
index 4df2e3d,ed6a3fd..521786b
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@@ -601,11 -631,8 +631,9 @@@ struct sess 
  	VTAILQ_ENTRY(sess)	list;
  
  	struct director		*director;
- 	struct object		*obj;
- 	struct objcore		*objcore;
  	struct VCL_conf		*vcl;
  
 +	struct object		*stale_obj;
  	/* The busy objhead we sleep on */
  	struct objhead		*hash_objhead;
  
diff --cc bin/varnishd/cache/cache_center.c
index 7b8dc8a,8826ae2..dd4e1bd
--- a/bin/varnishd/cache/cache_center.c
+++ b/bin/varnishd/cache/cache_center.c
@@@ -579,16 -616,15 +616,16 @@@ cnt_fetch(struct sess *sp
  		/*
  		 * What does RFC2616 think about TTL ?
  		 */
- 		EXP_Clr(&sp->wrk->exp);
- 		sp->wrk->exp.entered = VTIM_real();
+ 		EXP_Clr(&wrk->busyobj->exp);
+ 		wrk->busyobj->exp.entered = W_TIM_real(wrk);
  		RFC2616_Ttl(sp);
- 		sp->wrk->exp.keep = cache_param->default_keep;
++		wrk->busyobj->exp.keep = cache_param->default_keep;
  
  		/* pass from vclrecv{} has negative TTL */
- 		if (sp->objcore == NULL)
- 			sp->wrk->exp.ttl = -1.;
+ 		if (wrk->objcore == NULL)
+ 			wrk->busyobj->exp.ttl = -1.;
  
- 		AZ(sp->wrk->do_esi);
+ 		AZ(wrk->busyobj->do_esi);
  
  		VCL_fetch_method(sp);
  
@@@ -732,34 -770,30 +771,34 @@@ cnt_fetchbody(struct sess *sp
  		    "Content-Encoding: gzip");
  
  	/* But we can't do both at the same time */
- 	assert(sp->wrk->do_gzip == 0 || sp->wrk->do_gunzip == 0);
+ 	assert(wrk->busyobj->do_gzip == 0 || wrk->busyobj->do_gunzip == 0);
  
  	/* ESI takes precedence and handles gzip/gunzip itself */
- 	if (sp->wrk->do_esi)
- 		sp->wrk->vfp = &vfp_esi;
- 	else if (sp->wrk->do_gunzip)
- 		sp->wrk->vfp = &vfp_gunzip;
- 	else if (sp->wrk->do_gzip)
- 		sp->wrk->vfp = &vfp_gzip;
- 	else if (sp->wrk->is_gzip)
- 		sp->wrk->vfp = &vfp_testgzip;
- 
- 	if (sp->wrk->do_esi || sp->esi_level > 0)
- 		sp->wrk->do_stream = 0;
+ 	if (wrk->busyobj->do_esi)
+ 		wrk->busyobj->vfp = &vfp_esi;
+ 	else if (wrk->busyobj->do_gunzip)
+ 		wrk->busyobj->vfp = &vfp_gunzip;
+ 	else if (wrk->busyobj->do_gzip)
+ 		wrk->busyobj->vfp = &vfp_gzip;
+ 	else if (wrk->busyobj->is_gzip)
+ 		wrk->busyobj->vfp = &vfp_testgzip;
+ 
+ 	if (wrk->busyobj->do_esi || sp->esi_level > 0)
+ 		wrk->busyobj->do_stream = 0;
  	if (!sp->wantbody)
- 		sp->wrk->do_stream = 0;
+ 		wrk->busyobj->do_stream = 0;
  
- 	l = http_EstimateWS(sp->wrk->beresp,
+ 	l = http_EstimateWS(wrk->busyobj->beresp,
  	    pass ? HTTPH_R_PASS : HTTPH_A_INS, &nhttp);
 +        if (sp->stale_obj) {
 +            l += http_EstimateWS(sp->stale_obj->http, 0, &stale_nhttp);
 +            nhttp += stale_nhttp;
 +        }
  
  	/* Create Vary instructions */
- 	if (sp->objcore != NULL) {
- 		CHECK_OBJ_NOTNULL(sp->objcore, OBJCORE_MAGIC);
- 		vary = VRY_Create(sp, sp->wrk->beresp);
+ 	if (wrk->objcore != NULL) {
+ 		CHECK_OBJ_NOTNULL(wrk->objcore, OBJCORE_MAGIC);
+ 		vary = VRY_Create(sp, wrk->busyobj->beresp);
  		if (vary != NULL) {
  			varyl = VSB_len(vary);
  			assert(varyl > 0);
@@@ -783,33 -817,32 +822,33 @@@
  		 * Try to salvage the transaction by allocating a
  		 * shortlived object on Transient storage.
  		 */
- 		sp->obj = STV_NewObject(sp, TRANSIENT_STORAGE, l,
- 		    &sp->wrk->exp, nhttp);
- 		if (sp->wrk->exp.ttl > cache_param->shortlived)
- 			sp->wrk->exp.ttl = cache_param->shortlived;
- 		sp->wrk->exp.grace = 0.0;
- 		sp->wrk->exp.keep = 0.0;
+ 		wrk->obj = STV_NewObject(wrk, TRANSIENT_STORAGE, l, nhttp);
+ 		if (wrk->busyobj->exp.ttl > cache_param->shortlived)
+ 			wrk->busyobj->exp.ttl = cache_param->shortlived;
+ 		wrk->busyobj->exp.grace = 0.0;
+ 		wrk->busyobj->exp.keep = 0.0;
  	}
- 	if (sp->obj == NULL) {
+ 	if (wrk->obj == NULL) {
  		sp->err_code = 503;
  		sp->step = STP_ERROR;
- 		VDI_CloseFd(sp->wrk);
+ 		VDI_CloseFd(wrk, &wrk->busyobj->vbc);
+ 		VBO_DerefBusyObj(wrk, &wrk->busyobj);
  		return (0);
  	}
- 	CHECK_OBJ_NOTNULL(sp->obj, OBJECT_MAGIC);
-         sp->obj->exp.keep = sp->wrk->exp.keep;
+ 	CHECK_OBJ_NOTNULL(wrk->obj, OBJECT_MAGIC);
++	wrk->busyobj->exp.keep = sp->exp.keep;
  
- 	sp->wrk->storage_hint = NULL;
+ 	wrk->storage_hint = NULL;
  
- 	if (sp->wrk->do_gzip || (sp->wrk->is_gzip && !sp->wrk->do_gunzip))
- 		sp->obj->gziped = 1;
+ 	if (wrk->busyobj->do_gzip ||
+ 	    (wrk->busyobj->is_gzip && !wrk->busyobj->do_gunzip))
+ 		wrk->obj->gziped = 1;
  
  	if (vary != NULL) {
- 		sp->obj->vary =
- 		    (void *)WS_Alloc(sp->obj->http->ws, varyl);
- 		AN(sp->obj->vary);
- 		memcpy(sp->obj->vary, VSB_data(vary), varyl);
- 		VRY_Validate(sp->obj->vary);
+ 		wrk->obj->vary = (void *)WS_Alloc(wrk->obj->http->ws, varyl);
+ 		AN(wrk->obj->vary);
+ 		memcpy(wrk->obj->vary, VSB_data(vary), varyl);
+ 		VRY_Validate(wrk->obj->vary);
  		VSB_delete(vary);
  	}
  
@@@ -823,34 -856,16 +862,34 @@@
  
  	hp2->logtag = HTTP_Obj;
  	http_CopyResp(hp2, hp);
 +        
- 	http_FilterFields(sp->wrk, sp->vsl_id, hp2, hp,
+ 	http_FilterFields(wrk, sp->vsl_id, hp2, hp,
  	    pass ? HTTPH_R_PASS : HTTPH_A_INS);
 +
 +        /*
 +	 * If we found a candidate for conditional backend request, attempt it
 +         * now. If backend responds with 304, http_Check304() merges stale_obj
-          * into sp->obj, any other response is handled as usual. In either case,
++         * into wrk->obj, any other response is handled as usual. In either case,
 +         * the stale_obj is no longer needed in the cache, so discard it.
 +         */
 +        if (sp->stale_obj) {
 +            http_Check304(sp);
-             if (sp->wrk->beresp->status == 304)
-                 assert(sp->obj->http->status == 200);
++            if (wrk->busyobj->beresp->status == 304)
++                assert(wrk->obj->http->status == 200);
 +	    EXP_Clr(&sp->stale_obj->exp);
 +	    EXP_Rearm(sp->stale_obj);
 +	    HSH_Deref(sp->wrk, NULL, &sp->stale_obj);
 +	    AZ(sp->stale_obj);
 +        }
- 	http_CopyHome(sp->wrk, sp->vsl_id, hp2);
+ 	http_CopyHome(wrk, sp->vsl_id, hp2);
  
 -	if (http_GetHdr(hp, H_Last_Modified, &b))
 +	if (http_GetHdr(hp, H_Last_Modified, &b)
-             || http_GetHdr(sp->obj->http, H_Last_Modified, &b))
- 		sp->obj->last_modified = VTIM_parse(b);
++            || http_GetHdr(wrk->obj->http, H_Last_Modified, &b))
+ 		wrk->obj->last_modified = VTIM_parse(b);
  	else
- 		sp->obj->last_modified = floor(sp->wrk->exp.entered);
+ 		wrk->obj->last_modified = floor(wrk->busyobj->exp.entered);
  
- 	assert(WRW_IsReleased(sp->wrk));
+ 	assert(WRW_IsReleased(wrk));
  
  	/*
  	 * If we can deliver a 304 reply, we don't bother streaming.
@@@ -1160,12 -1202,10 +1226,12 @@@ cnt_lookup(struct sess *sp
  	sp->vary_e = NULL;
  
  	if (oc->flags & OC_F_PASS) {
- 		sp->wrk->stats.cache_hitpass++;
- 		WSP(sp, SLT_HitPass, "%u", sp->obj->xid);
- 		(void)HSH_Deref(sp->wrk, NULL, &sp->obj);
+ 		wrk->stats.cache_hitpass++;
+ 		WSP(sp, SLT_HitPass, "%u", wrk->obj->xid);
+ 		(void)HSH_Deref(wrk, NULL, &wrk->obj);
 +                if (sp->stale_obj != NULL)
-                     (void)HSH_Deref(sp->wrk, NULL, &sp->stale_obj);
- 		sp->objcore = NULL;
++                    (void)HSH_Deref(wrk, NULL, &sp->stale_obj);
+ 		wrk->objcore = NULL;
  		sp->step = STP_PASS;
  		return (0);
  	}
@@@ -1218,26 -1263,22 +1289,28 @@@ cnt_miss(struct sess *sp
  		 * client doesn't grok it.  We will uncompress for
  		 * the minority of clients which don't.
  		 */
- 		http_Unset(sp->wrk->bereq, H_Accept_Encoding);
- 		http_SetHeader(sp->wrk, sp->vsl_id, sp->wrk->bereq,
+ 		http_Unset(wrk->busyobj->bereq, H_Accept_Encoding);
+ 		http_SetHeader(wrk, sp->vsl_id, wrk->busyobj->bereq,
  		    "Accept-Encoding: gzip");
  	}
- 	sp->wrk->connect_timeout = 0;
- 	sp->wrk->first_byte_timeout = 0;
- 	sp->wrk->between_bytes_timeout = 0;
+ 	wrk->connect_timeout = 0;
+ 	wrk->first_byte_timeout = 0;
+ 	wrk->between_bytes_timeout = 0;
  
 +        /* If a candidate for a conditional backend request was found,
 +         * add If-Modified-Since and/or If-None-Match to the bereq.
 +         */
 +        if (sp->stale_obj)
 +                http_CheckRefresh(sp);
 +
  	VCL_miss_method(sp);
+ 
  	switch(sp->handling) {
  	case VCL_RET_ERROR:
- 		AZ(HSH_Deref(sp->wrk, sp->objcore, NULL));
- 		sp->objcore = NULL;
- 		http_Setup(sp->wrk->bereq, NULL);
+ 		AZ(HSH_Deref(wrk, wrk->objcore, NULL));
+ 		wrk->objcore = NULL;
+ 		http_Setup(wrk->busyobj->bereq, NULL);
+ 		VBO_DerefBusyObj(wrk, &wrk->busyobj);
  		sp->step = STP_ERROR;
  		return (0);
  	case VCL_RET_PASS:
diff --cc bin/varnishd/cache/cache_fetch.c
index cc46222,936dc30..6291ac0
--- a/bin/varnishd/cache/cache_fetch.c
+++ b/bin/varnishd/cache/cache_fetch.c
@@@ -499,16 -508,11 +508,16 @@@ FetchBody(struct worker *wrk, struct ob
  
  	AssertObjCorePassOrBusy(obj->objcore);
  
- 	AZ(w->vgz_rx);
+ 	AZ(bo->vgz_rx);
 -	AZ(VTAILQ_FIRST(&obj->store));
 +
 +        /* If we've freshened from another object and got a "Not Modified"
 +         * response, then we have already duped the other object's body.
 +         */
-         if (w->beresp->status != 304)
++        if (bo->beresp->status != 304)
 +        	AZ(VTAILQ_FIRST(&obj->store));
  
- 	w->fetch_obj = obj;
- 	w->fetch_failed = 0;
+ 	bo->fetch_obj = obj;
+ 	bo->fetch_failed = 0;
  
  	/* XXX: pick up estimate from objdr ? */
  	cl = 0;
diff --cc bin/varnishd/cache/cache_hash.c
index 5251f6d,6f6a85f..c56a2d5
--- a/bin/varnishd/cache/cache_hash.c
+++ b/bin/varnishd/cache/cache_hash.c
@@@ -313,8 -304,7 +306,8 @@@ HSH_Lookup(struct sess *sp, struct objh
  	CHECK_OBJ_NOTNULL(sp->http, HTTP_MAGIC);
  	AN(sp->director);
  	AN(hash);
 +        AZ(sp->stale_obj);
- 	w = sp->wrk;
+ 	wrk = sp->wrk;
  
  	HSH_Prealloc(sp);
  	memcpy(sp->wrk->nobjhead->digest, sp->digest, sizeof sp->digest);
@@@ -475,21 -443,9 +468,21 @@@
  		return (NULL);
  	}
  
 +        /* If we're not serving a valid or graced object and we saved stale_o,
 +	 * it is a candidate for the conditional backend request. */
 +        AZ(oc && !sp->hash_always_miss);
 +        AZ(busy_oc);
 +        if (stale_o != NULL) {
 +                AZ(stale_o->objcore->flags & OC_F_BUSY);
 +		CHECK_OBJ_NOTNULL(stale_o->objcore, OBJCORE_MAGIC);
 +                Lck_AssertHeld(&oh->mtx);
 +                stale_o->objcore->refcnt++;
 +                sp->stale_obj = stale_o;
 +        }
 +
  	/* Insert (precreated) objcore in objecthead */
- 	oc = w->nobjcore;
- 	w->nobjcore = NULL;
+ 	oc = wrk->nobjcore;
+ 	wrk->nobjcore = NULL;
  	AN(oc->flags & OC_F_BUSY);
  	oc->refcnt = 1;
  
diff --cc bin/varnishd/cache/cache_http.c
index b937d64,4a6ba82..b314b55
--- a/bin/varnishd/cache/cache_http.c
+++ b/bin/varnishd/cache/cache_http.c
@@@ -917,89 -883,6 +917,88 @@@ http_FilterHeader(const struct sess *sp
  	http_PrintfHeader(sp->wrk, sp->vsl_id, hp, "X-Varnish: %u", sp->xid);
  }
  
 +/*-------------------------------------------------------------------
 + * This function checks for sp->freshen_obj.  If present, HSH_Lookup()
 + * found an expired object that qualifies for a refresh check,
 + * so add the appropriate headers.
 + */
 +
 +void
 +http_CheckRefresh(struct sess *sp)
 +{
 +	struct object *freshen_obj;
 +	struct http *obj_hp, *bereq_hp;
 +	char *p;
 +
 +	freshen_obj = sp->stale_obj;
 +	CHECK_OBJ_NOTNULL(freshen_obj, OBJECT_MAGIC);
- 	bereq_hp = sp->wrk->bereq;
++	bereq_hp = sp->wrk->busyobj->bereq;
 +	CHECK_OBJ_NOTNULL(bereq_hp, HTTP_MAGIC);
 +	obj_hp = freshen_obj->http;
 +	CHECK_OBJ_NOTNULL(obj_hp, HTTP_MAGIC);
 +
 +	if(http_GetHdr(obj_hp, H_ETag, &p))
 +		http_PrintfHeader(sp->wrk, sp->fd, bereq_hp, "If-None-Match: %s", p);
 +
 +	if(http_GetHdr(obj_hp, H_Last_Modified, &p))
 +		http_PrintfHeader(sp->wrk, sp->fd, bereq_hp, "If-Modified-Since: %s",p);
 +}
 +
 +/*-------------------------------------------------------------------
 + * Called after fetch and sp->freshen_obj present.  Check
 + * response and handle as needed.
 + */
 +
 +void
 +http_Check304(struct sess *sp)
 +{
 +	struct object *o, *o_stale;
 +	char *p;
 +
 +	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 +	o_stale = sp->stale_obj;
 +	CHECK_OBJ_NOTNULL(o_stale, OBJECT_MAGIC);
- 	o = sp->obj;
++	o = sp->wrk->obj;
 +	CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC);
 +
- 	if (sp->wrk->beresp->status != 304) {
++	if (sp->wrk->busyobj->beresp->status != 304) {
 +	    /*
 +	     * IMS/INM headers may have been removed in VCL, so only count a
 +	     * non-validating response if they were present in the request.
 +	     */
- 	    if (http_GetHdr(sp->wrk->bereq, H_If_Modified_Since, &p)
- 		|| http_GetHdr(sp->wrk->bereq, H_If_None_Match, &p))
- 		sp->wrk->stats.cond_not_validated++;
++	    if (http_GetHdr(sp->wrk->busyobj->bereq, H_If_Modified_Since, &p)
++		|| http_GetHdr(sp->wrk->busyobj->bereq, H_If_None_Match, &p))
++		sp->wrk->stats.fetch_not_validated++;
 +	    return;
 +	}
 +
 +	/* 
 +	 * Copy headers we need from the stale object into the 304 response
 +	 */
- 	http_FilterMissingFields(sp->wrk, sp->fd, sp->obj->http,
- 				 sp->stale_obj->http);
++	http_FilterMissingFields(sp->wrk, sp->fd, o->http, o_stale->http);
 +
 +	/*
 +	 * Dup the stale object's storage in to the new object
 +	 * and reset Content-Length from the size of the storage.
 +	 */
 +	STV_dup(sp, o_stale, o);
 +	http_Unset(o->http, H_Content_Length);
 +	http_PrintfHeader(sp->wrk, sp->fd, o->http, "Content-Length: %u", o->len);
 +
 +	http_SetResp(o->http, "HTTP/1.1", 200, "Ok Not Modified");
 +	http_SetH(o->http, HTTP_HDR_REQ, "GET");
- 	http_copyh(o->http, sp->wrk->bereq, HTTP_HDR_URL);
++	http_copyh(o->http, sp->wrk->busyobj->bereq, HTTP_HDR_URL);
 +
 +	/*
 +	 * XXX: Are we copying all the necessary fields from stale_obj?
 +	 *	Should we copy o_stale->hits into o->hits?
 +	 */
 +	o->response = 200;
 +	o->gziped = o_stale->gziped;
 +
 +        AZ(o_stale->objcore->flags & OC_F_BUSY);
 +}
 +
  /*--------------------------------------------------------------------
   * This function copies any header fields which reference foreign
   * storage into our own WS.
diff --cc bin/varnishd/cache/cache_vrt.c
index 27964dc,a696f64..290b6bf
--- a/bin/varnishd/cache/cache_vrt.c
+++ b/bin/varnishd/cache/cache_vrt.c
@@@ -108,13 -109,9 +109,13 @@@ vrt_selecthttp(const struct sess *sp, e
  		hp = sp->wrk->resp;
  		break;
  	case HDR_OBJ:
- 		CHECK_OBJ_NOTNULL(sp->obj, OBJECT_MAGIC);
- 		hp = sp->obj->http;
+ 		CHECK_OBJ_NOTNULL(sp->wrk->obj, OBJECT_MAGIC);
+ 		hp = sp->wrk->obj->http;
  		break;
 +        case HDR_STALE_OBJ:
 +		CHECK_OBJ_NOTNULL(sp->stale_obj, OBJECT_MAGIC);
 +		hp = sp->stale_obj->http;
 +		break;
  	default:
  		INCOMPL();
  	}
diff --cc bin/varnishd/cache/cache_vrt_var.c
index 407de1d,0662e26..b5a5bac
--- a/bin/varnishd/cache/cache_vrt_var.c
+++ b/bin/varnishd/cache/cache_vrt_var.c
@@@ -82,32 -78,22 +82,32 @@@ const char *							
  VRT_r_##obj##_##hdr(const struct sess *sp)			\
  {								\
  	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);			\
 -	CHECK_OBJ_NOTNULL(http, HTTP_MAGIC);			\
 -	return (http->hd[fld].b);				\
 -}
 -
 -VRT_DO_HDR(req,   request,	sp->http,		HTTP_HDR_REQ)
 -VRT_DO_HDR(req,   url,		sp->http,		HTTP_HDR_URL)
 -VRT_DO_HDR(req,   proto,	sp->http,		HTTP_HDR_PROTO)
 -VRT_DO_HDR(bereq, request,	sp->wrk->busyobj->bereq,	HTTP_HDR_REQ)
 -VRT_DO_HDR(bereq, url,		sp->wrk->busyobj->bereq,	HTTP_HDR_URL)
 -VRT_DO_HDR(bereq, proto,	sp->wrk->busyobj->bereq,	HTTP_HDR_PROTO)
 -VRT_DO_HDR(obj,   proto,	sp->wrk->obj->http,		HTTP_HDR_PROTO)
 -VRT_DO_HDR(obj,   response,	sp->wrk->obj->http,		HTTP_HDR_RESPONSE)
 -VRT_DO_HDR(resp,  proto,	sp->wrk->resp,		HTTP_HDR_PROTO)
 -VRT_DO_HDR(resp,  response,	sp->wrk->resp,		HTTP_HDR_RESPONSE)
 -VRT_DO_HDR(beresp,  proto,	sp->wrk->busyobj->beresp,	HTTP_HDR_PROTO)
 -VRT_DO_HDR(beresp,  response,	sp->wrk->busyobj->beresp,	HTTP_HDR_RESPONSE)
 +        if (!nullable || cont != NULL) {			\
 +            CHECK_OBJ_NOTNULL(cont->http, HTTP_MAGIC);		\
 +            return (cont->http->hd[fld].b);			\
 +        }							\
 +        ILLEGAL_R(sp, #obj, #hdr);				\
 +        return(NULL);                                           \
 +}                                                               \
 +
 +#define VRT_DO_HDR(obj, hdr, cont, http, fld, nullable)		\
 +VRT_DO_HDR_l(obj, hdr, cont, http, fld)				\
 +VRT_DO_HDR_r(obj, hdr, cont, http, fld, nullable)		\
 +
- VRT_DO_HDR(req,         request,    sp,             http,	HTTP_HDR_REQ,	    0)
- VRT_DO_HDR(req,         url,        sp,             http,	HTTP_HDR_URL,	    0)
- VRT_DO_HDR(req,         proto,      sp,             http,	HTTP_HDR_PROTO,	    0)
- VRT_DO_HDR(bereq,       request,    sp->wrk,        bereq,	HTTP_HDR_REQ,	    0)
- VRT_DO_HDR(bereq,       url,        sp->wrk,        bereq,	HTTP_HDR_URL,	    0)
- VRT_DO_HDR(bereq,       proto,      sp->wrk,        bereq,	HTTP_HDR_PROTO,	    0)
- VRT_DO_HDR(obj,         proto,      sp->obj,        http,	HTTP_HDR_PROTO,	    0)
- VRT_DO_HDR(obj,         response,   sp->obj,        http,	HTTP_HDR_RESPONSE,  0)
- VRT_DO_HDR(resp,        proto,      sp->wrk,        resp,	HTTP_HDR_PROTO,	    0)
- VRT_DO_HDR(resp,        response,   sp->wrk,        resp,	HTTP_HDR_RESPONSE,  0)
- VRT_DO_HDR(beresp,      proto,      sp->wrk,        beresp,	HTTP_HDR_PROTO,	    0)
- VRT_DO_HDR(beresp,      response,   sp->wrk,        beresp,	HTTP_HDR_RESPONSE,  0)
- VRT_DO_HDR_r(stale_obj,   proto,      sp->stale_obj,  http,	HTTP_HDR_PROTO,	    1)
- VRT_DO_HDR_r(stale_obj,   response,   sp->stale_obj,  http,	HTTP_HDR_RESPONSE,  1)
++VRT_DO_HDR(req,		request,	sp,			http,	HTTP_HDR_REQ,		0)
++VRT_DO_HDR(req,		url,		sp,			http,	HTTP_HDR_URL,		0)
++VRT_DO_HDR(req,		proto,		sp,			http,	HTTP_HDR_PROTO,		0)
++VRT_DO_HDR(bereq,	request,	sp->wrk->busyobj,	bereq,	HTTP_HDR_REQ,		0)
++VRT_DO_HDR(bereq,	url,		sp->wrk->busyobj,	bereq,	HTTP_HDR_URL,		0)
++VRT_DO_HDR(bereq,	proto,		sp->wrk->busyobj,	bereq,	HTTP_HDR_PROTO,		0)
++VRT_DO_HDR(obj,		proto,		sp->wrk->obj,		http,	HTTP_HDR_PROTO,		0)
++VRT_DO_HDR(obj,		response,	sp->wrk->obj,		http,	HTTP_HDR_RESPONSE,	0)
++VRT_DO_HDR(resp,	proto,		sp->wrk,		resp,	HTTP_HDR_PROTO,		0)
++VRT_DO_HDR(resp,	response,	sp->wrk,		resp,	HTTP_HDR_RESPONSE,	0)
++VRT_DO_HDR(beresp,	proto,		sp->wrk->busyobj,	beresp,	HTTP_HDR_PROTO,		0)
++VRT_DO_HDR(beresp,	response,	sp->wrk->busyobj,	beresp,	HTTP_HDR_RESPONSE,	0)
++VRT_DO_HDR_r(stale_obj,	proto,		sp->stale_obj,		http,	HTTP_HDR_PROTO,		1)
++VRT_DO_HDR_r(stale_obj,	response,	sp->stale_obj,		http,	HTTP_HDR_RESPONSE,	1)
  
  /*--------------------------------------------------------------------*/
  
@@@ -126,21 -111,12 +126,21 @@@ VRT_r_##obj##_status(const struct sess 
  {								\
  								\
  	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);			\
 -	return(http->status);					\
 +        if (nullable && cont == NULL) {				\
 +            ILLEGAL_R(sp, #obj, "status");			\
 +            return (503);                                       \
 +        }                                                       \
 +	return(cont->http->status);				\
  }
  
 -VRT_DO_STATUS(obj, sp->wrk->obj->http)
 -VRT_DO_STATUS(beresp, sp->wrk->busyobj->beresp)
 -VRT_DO_STATUS(resp, sp->wrk->resp)
 +#define VRT_DO_STATUS(obj, cont, http, nullable)		\
 +VRT_DO_STATUS_l(obj, cont, http)				\
 +VRT_DO_STATUS_r(obj, cont, http, nullable)			\
 +
- VRT_DO_STATUS(obj,          sp->obj,        http,	0)
- VRT_DO_STATUS(beresp,       sp->wrk,        beresp,	0)
- VRT_DO_STATUS(resp,	    sp->wrk,        resp,	0)
- VRT_DO_STATUS_r(stale_obj,  sp->stale_obj,  http,	1)
++VRT_DO_STATUS(obj,		sp->wrk->obj,		http,	0)
++VRT_DO_STATUS(beresp,		sp->wrk->busyobj,	beresp,	0)
++VRT_DO_STATUS(resp,		sp->wrk,		resp,	0)
++VRT_DO_STATUS_r(stale_obj,	sp->stale_obj,		http,	1)
  
  /*--------------------------------------------------------------------*/
  
@@@ -429,30 -401,26 +433,30 @@@ vrt_wsp_exp(const struct sess *sp, unsi
  	    sp->t_req, e->age + (sp->t_req - e->entered));
  }
  
 -VRT_DO_EXP(req, sp->exp, ttl, 0, )
 -VRT_DO_EXP(req, sp->exp, grace, 0, )
 -VRT_DO_EXP(req, sp->exp, keep, 0, )
 +VRT_DO_EXP(req, sp, ttl, 0, 0, )
 +VRT_DO_EXP(req, sp, grace, 0, 0, )
 +VRT_DO_EXP(req, sp, keep, 0, 0, )
  
- VRT_DO_EXP(obj, sp->obj, grace, 0, 0,
-    EXP_Rearm(sp->obj);
-    vrt_wsp_exp(sp, sp->obj->xid, &sp->obj->exp);)
- VRT_DO_EXP(obj, sp->obj, ttl, (sp->t_req - sp->obj->exp.entered), 0,
-    EXP_Rearm(sp->obj);
-    vrt_wsp_exp(sp, sp->obj->xid, &sp->obj->exp);)
- VRT_DO_EXP(obj, sp->obj, keep, 0, 0,
-    EXP_Rearm(sp->obj);
-    vrt_wsp_exp(sp, sp->obj->xid, &sp->obj->exp);)
- 
- VRT_DO_EXP(beresp, sp->wrk, grace, 0, 0,
-    vrt_wsp_exp(sp, sp->xid, &sp->wrk->exp);)
- VRT_DO_EXP(beresp, sp->wrk, ttl, 0, 0,
-    vrt_wsp_exp(sp, sp->xid, &sp->wrk->exp);)
- VRT_DO_EXP(beresp, sp->wrk, keep, 0, 0,
-    vrt_wsp_exp(sp, sp->xid, &sp->wrk->exp);)
 -VRT_DO_EXP(obj, sp->wrk->obj->exp, grace, 0,
++VRT_DO_EXP(obj, sp->wrk->obj, grace, 0, 0,
+    EXP_Rearm(sp->wrk->obj);
+    vrt_wsp_exp(sp, sp->wrk->obj->xid, &sp->wrk->obj->exp);)
 -VRT_DO_EXP(obj, sp->wrk->obj->exp, ttl, (sp->t_req - sp->wrk->obj->exp.entered),
++VRT_DO_EXP(obj, sp->wrk->obj, ttl, (sp->t_req - sp->wrk->obj->exp.entered), 0,
+    EXP_Rearm(sp->wrk->obj);
+    vrt_wsp_exp(sp, sp->wrk->obj->xid, &sp->wrk->obj->exp);)
 -VRT_DO_EXP(obj, sp->wrk->obj->exp, keep, 0,
++VRT_DO_EXP(obj, sp->wrk->obj, keep, 0, 0,
+    EXP_Rearm(sp->wrk->obj);
+    vrt_wsp_exp(sp, sp->wrk->obj->xid, &sp->wrk->obj->exp);)
+ 
 -VRT_DO_EXP(beresp, sp->wrk->busyobj->exp, grace, 0,
++VRT_DO_EXP(beresp, sp->wrk->busyobj, grace, 0, 0,
+    vrt_wsp_exp(sp, sp->xid, &sp->wrk->busyobj->exp);)
 -VRT_DO_EXP(beresp, sp->wrk->busyobj->exp, ttl, 0,
++VRT_DO_EXP(beresp, sp->wrk->busyobj, ttl, 0, 0,
+    vrt_wsp_exp(sp, sp->xid, &sp->wrk->busyobj->exp);)
 -VRT_DO_EXP(beresp, sp->wrk->busyobj->exp, keep, 0,
++VRT_DO_EXP(beresp, sp->wrk->busyobj, keep, 0, 0,
+    vrt_wsp_exp(sp, sp->xid, &sp->wrk->busyobj->exp);)
 +    
 +VRT_DO_EXP_r(stale_obj, sp->stale_obj, grace, 0, 1)
 +VRT_DO_EXP_r(stale_obj, sp->stale_obj, ttl, 0, 1)
 +VRT_DO_EXP_r(stale_obj, sp->stale_obj, keep, 0, 1)
  
  /*--------------------------------------------------------------------
   * req.xid
@@@ -565,45 -531,19 +569,45 @@@ VRT_r_obj_hits(const struct sess *sp
  {
  
  	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
- 	CHECK_OBJ_NOTNULL(sp->obj, OBJECT_MAGIC);	/* XXX */
- 	return (sp->obj->hits);
+ 	CHECK_OBJ_NOTNULL(sp->wrk->obj, OBJECT_MAGIC);	/* XXX */
+ 	return (sp->wrk->obj->hits);
  }
  
 +int
 +VRT_r_stale_obj_hits(const struct sess *sp)
 +{
 +
 +	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 +        if (sp->stale_obj == NULL) {
 +            ILLEGAL_R(sp, "stale_obj", "hits");
 +            return (0);
 +        }
 +	CHECK_OBJ(sp->stale_obj, OBJECT_MAGIC);	/* XXX */
 +	return (sp->stale_obj->hits);
 +}
 +
  double
  VRT_r_obj_lastuse(const struct sess *sp)
  {
  
  	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
- 	CHECK_OBJ_NOTNULL(sp->obj, OBJECT_MAGIC);	/* XXX */
- 	return (VTIM_real() - sp->obj->last_use);
+ 	CHECK_OBJ_NOTNULL(sp->wrk->obj, OBJECT_MAGIC);	/* XXX */
+ 	return (VTIM_real() - sp->wrk->obj->last_use);
  }
  
 +double
 +VRT_r_stale_obj_lastuse(const struct sess *sp)
 +{
 +
 +	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 +        if (sp->stale_obj == NULL) {
 +            ILLEGAL_R(sp, "stale_obj", "lastuse");
 +            return (0);
 +        }
 +	CHECK_OBJ(sp->stale_obj, OBJECT_MAGIC);	/* XXX */
 +	return (VTIM_real() - sp->stale_obj->last_use);
 +}
 +
  unsigned
  VRT_r_req_backend_healthy(const struct sess *sp)
  {
diff --cc bin/varnishd/storage/stevedore.c
index 71241f5,a6156d4..037d77c
--- a/bin/varnishd/storage/stevedore.c
+++ b/bin/varnishd/storage/stevedore.c
@@@ -366,11 -376,8 +376,11 @@@ STV_Freestore(struct object *o
  struct storage *
  STV_alloc(struct worker *w, size_t size)
  {
- 	struct object *obj = w->fetch_obj;
++	struct object *obj = w->busyobj->fetch_obj;
 +	if (obj == NULL)
- 		obj = w->sp->obj;
++		obj = w->obj;
  
 -	return (stv_alloc(w, w->busyobj->fetch_obj, size));
 +	return (stv_alloc(w, obj, size));
  }
  
  void
diff --cc bin/varnishd/storage/storage.h
index 879a6fb,c8c3689..70468be
--- a/bin/varnishd/storage/storage.h
+++ b/bin/varnishd/storage/storage.h
@@@ -39,11 -40,10 +40,11 @@@ struct lru
  typedef void storage_init_f(struct stevedore *, int ac, char * const *av);
  typedef void storage_open_f(const struct stevedore *);
  typedef struct storage *storage_alloc_f(struct stevedore *, size_t size);
 +typedef void storage_dup_f(const struct sess *sp, struct object *src, struct object *target);
  typedef void storage_trim_f(struct storage *, size_t size);
  typedef void storage_free_f(struct storage *);
- typedef struct object *storage_allocobj_f(struct stevedore *, struct sess *sp,
-     unsigned ltot, const struct stv_objsecrets *);
+ typedef struct object *storage_allocobj_f(struct stevedore *,
+     struct worker *wrk, unsigned ltot, const struct stv_objsecrets *);
  typedef void storage_close_f(const struct stevedore *);
  
  /* Prototypes for VCL variable responders */
diff --cc bin/varnishtest/tests/i00001.vtc
index 306cc57,0000000..6383a1f
mode 100644,000000..100644
--- a/bin/varnishtest/tests/i00001.vtc
+++ b/bin/varnishtest/tests/i00001.vtc
@@@ -1,271 -1,0 +1,271 @@@
 +# $Id$
 +
 +varnishtest "Test basic conditional requests to backends"
 +
 +## By default (default_keep = default_grace), cond. requests happen during grace
 +
 +server s1 {
 +        rxreq
 +        expect req.url == "/foo"
 +        txresp -status 200 \
 +    	    -hdr "Last-Modified: Thu, 26 Jun 2008 12:00:01 GMT" \
 +            -hdr "ETag: foo" \
 +            -body "123456"
 +
 +        rxreq
 +        expect req.url == "/foo"
 +        expect req.http.If-Modified-Since == "Thu, 26 Jun 2008 12:00:01 GMT"
 +        expect req.http.If-None-Match == "foo"
 +        txresp -status 304
 +} -start
 +
 +varnish v1 -vcl+backend {
 +        sub vcl_fetch {
 +            set beresp.ttl = 0.5s;
 +        }
 +} -start
 +
 +client c1 {
 +        txreq -url "/foo"
 +        rxresp
 +        expect resp.status == 200
 +        expect resp.http.content-length == 6
 +        expect resp.bodylen == 6
 +        expect resp.http.Age == 0
 +} -run
 +
 +delay 0.6
 +
 +client c1 -run
 +
 +varnish v1 -expect fetch_304 == 1
- varnish v1 -expect cond_not_validated == 0
++varnish v1 -expect fetch_not_validated == 0
 +
 +server s1 -wait
 +varnish v1 -stop
 +
 +## With keep > 0, using both If-Modified-Since and If-None-Match
 +
 +server s1 {
 +        rxreq
 +        expect req.url == "/foo"
 +        txresp -status 200 \
 +    	    -hdr "Last-Modified: Thu, 26 Jun 2008 12:00:01 GMT" \
 +            -hdr "ETag: foo" \
 +            -body "123456"
 +
 +        rxreq
 +        expect req.url == "/foo"
 +        expect req.http.If-Modified-Since == "Thu, 26 Jun 2008 12:00:01 GMT"
 +        expect req.http.If-None-Match == "foo"
 +        txresp -status 304
 +
 +        rxreq
 +        expect req.url == "/foo"
 +        expect req.http.If-Modified-Since == "Thu, 26 Jun 2008 12:00:01 GMT"
 +        expect req.http.If-None-Match == "foo"
 +        txresp -status 200 \
 +            -hdr "Last-Modified: Wed, 27 Jun 2008 12:00:01 GMT" \
 +            -hdr "ETag: foo" \
 +            -body "ABCDEF"
 +
 +        rxreq
 +        expect req.url == "/foo"
 +        expect req.http.If-Modified-Since == "Wed, 27 Jun 2008 12:00:01 GMT"
 +        expect req.http.If-None-Match == "foo"
 +        txresp -status 304
 +
 +} -start
 +
 +varnish v1 -vcl {
 +        backend s1 {
 +                .host = "${s1_addr}"; .port = "${s1_port}";
 +        }
 +
 +        sub vcl_fetch {
 +            set beresp.http.X-Original-Keep = beresp.keep;
 +            set beresp.grace = 0s;
 +            set beresp.ttl = 1s;
 +            set beresp.keep = 1m;
 +        }
 +} -start
 +
 +client c1 {
 +        txreq -url "/foo"
 +        rxresp
 +        expect resp.status == 200
 +        expect resp.http.ETag == "foo"
 +        expect resp.http.content-length == 6
 +        expect resp.bodylen == 6
 +        expect resp.http.Age == 0
 +        expect resp.http.X-Original-Keep == 10.000
 +}
 +
 +client c1 -run
 +
 +delay 1.1
 +
 +client c1 -run
 +
 +delay 1.1
 +
 +client c1 -run
 +
 +delay 1.1
 +
 +client c1 -run
 +
 +varnish v1 -expect fetch_304 == 2
- varnish v1 -expect cond_not_validated == 1
++varnish v1 -expect fetch_not_validated == 1
 +
 +server s1 -wait
 +varnish v1 -stop
 +
 +## Just If-Modified-Since
 +
 +server s1 {
 +        rxreq
 +        expect req.url == "/foo"
 +        txresp -status 200 \
 +    	    -hdr "Last-Modified: Thu, 26 Jun 2008 12:00:01 GMT" \
 +            -body "123456"
 +
 +        rxreq
 +        expect req.url == "/foo"
 +        expect req.http.If-Modified-Since == "Thu, 26 Jun 2008 12:00:01 GMT"
 +        txresp -status 304
 +
 +        rxreq
 +        expect req.url == "/foo"
 +        expect req.http.If-Modified-Since == "Thu, 26 Jun 2008 12:00:01 GMT"
 +        txresp -status 200 \
 +            -hdr "Last-Modified: Wed, 27 Jun 2008 12:00:01 GMT" \
 +            -body "ABCDEF"
 +
 +        rxreq
 +        expect req.url == "/foo"
 +        expect req.http.If-Modified-Since == "Wed, 27 Jun 2008 12:00:01 GMT"
 +        txresp -status 304
 +
 +} -start
 +
 +varnish v1 -vcl {
 +        backend s1 {
 +                .host = "${s1_addr}"; .port = "${s1_port}";
 +        }
 +        sub vcl_fetch {
 +            set beresp.http.X-Original-Keep = beresp.keep;
 +            set beresp.grace = 0s;
 +            set beresp.ttl = 1s;
 +            set beresp.keep = 1m;
 +        }
 +} -start
 +
 +client c1 {
 +        txreq -url "/foo"
 +        rxresp
 +        expect resp.status == 200
 +        expect resp.http.content-length == 6
 +        expect resp.bodylen == 6
 +        expect resp.http.Age == 0
 +        expect resp.http.X-Original-Keep == 10.000
 +}
 +
 +client c1 -run
 +
 +delay 1.1
 +
 +client c1 -run
 +
 +delay 1.1
 +
 +client c1 -run
 +
 +delay 1.1
 +
 +client c1 -run
 +
 +varnish v1 -expect fetch_304 == 2
- varnish v1 -expect cond_not_validated == 1
++varnish v1 -expect fetch_not_validated == 1
 +
 +server s1 -wait
 +varnish v1 -stop
 +
 +## Just If-None-Match
 +
 +server s1 {
 +        rxreq
 +        expect req.url == "/foo"
 +        txresp -status 200 \
 +    	    -hdr "ETag: foo" \
 +            -body "123456"
 +
 +        rxreq
 +        expect req.url == "/foo"
 +        expect req.http.If-None-Match == "foo"
 +        txresp -status 304
 +
 +        rxreq
 +        expect req.url == "/foo"
 +        expect req.http.If-None-Match == "foo"
 +        txresp -status 200 \
 +            -hdr "ETag: bar" \
 +            -body "ABCDEF"
 +
 +        rxreq
 +        expect req.url == "/foo"
 +        expect req.http.If-None-Match == "bar"
 +        txresp -status 304
 +
 +} -start
 +
 +varnish v1 -vcl {
 +        backend s1 {
 +                .host = "${s1_addr}"; .port = "${s1_port}";
 +        }
 +        sub vcl_fetch {
 +            set beresp.http.X-Original-Keep = beresp.keep;
 +            set beresp.grace = 0s;
 +            set beresp.ttl = 1s;
 +            set beresp.keep = 1m;
 +        }
 +} -start
 +
 +client c1 {
 +        txreq -url "/foo"
 +        rxresp
 +        expect resp.status == 200
 +        expect resp.http.ETag == "foo"
 +        expect resp.http.content-length == 6
 +        expect resp.bodylen == 6
 +        expect resp.http.Age == 0
 +        expect resp.http.X-Original-Keep == 10.000
 +}
 +
 +client c2 {
 +        txreq -url "/foo"
 +        rxresp
 +        expect resp.status == 200
 +        expect resp.http.ETag == "bar"
 +        expect resp.http.content-length == 6
 +        expect resp.bodylen == 6
 +        expect resp.http.Age == 0
 +        expect resp.http.X-Original-Keep == 10.000
 +}
 +
 +client c1 -run
 +
 +delay 1.1
 +
 +client c1 -run
 +
 +delay 1.1
 +
 +client c2 -run
 +
 +delay 1.1
 +
 +client c2 -run
 +
 +varnish v1 -expect fetch_304 == 2
- varnish v1 -expect cond_not_validated == 1
++varnish v1 -expect fetch_not_validated == 1
diff --cc bin/varnishtest/tests/i00003.vtc
index 293e843,0000000..fb66266
mode 100644,000000..100644
--- a/bin/varnishtest/tests/i00003.vtc
+++ b/bin/varnishtest/tests/i00003.vtc
@@@ -1,146 -1,0 +1,146 @@@
 +# $Id$
 +
 +varnishtest "Test some anticipated use cases for conditional backend requests"
 +
 +## In vcl_miss(), it is possible to veto a conditional request by removing any
 +## If-Modified-Since or If-None-Match header.
 +
 +server s1 {
 +        rxreq
 +        expect req.url == "/foo"
 +        txresp -status 200 \
 +    	    -hdr "Last-Modified: Thu, 26 Jun 2008 12:00:01 GMT" \
 +            -hdr "ETag: foo" \
 +            -body "abcdefghijklmonpqrstuvwxyz"
 +
 +        rxreq
 +        expect req.url == "/foo"
 +        expect req.http.If-Modified-Since != "Thu, 26 Jun 2008 12:00:01 GMT"
 +        expect req.http.If-None-Match != "foo"
 +        txresp -status 200 -body "abcdefghijklmonpqrstuvwxyz"
 +} -start
 +
 +varnish v1 -vcl+backend {
 +        sub vcl_fetch {
 +            set beresp.grace = 0s;
 +            set beresp.ttl = 1s;
 +            set beresp.keep = 1m;
 +        }
 +
 +        sub vcl_miss {
 +            unset bereq.http.If-Modified-Since;
 +            unset bereq.http.If-None-Match;
 +        }
 +} -start
 +
 +client c1 {
 +        txreq -url "/foo"
 +        rxresp
 +        expect resp.status == 200
 +        expect resp.http.content-length == 26
 +}
 +
 +client c1 -run
 +
 +delay 1.1
 +
 +client c1 -run
 +
 +varnish v1 -expect fetch_304 == 0
- varnish v1 -expect cond_not_validated == 0
++varnish v1 -expect fetch_not_validated == 0
 +
 +server s1 -wait
 +varnish v1 -stop
 +
 +## Verify that if a client sends a conditional request to Varnish, then Varnish
 +## can return a 304 response to the client after it got 304 from the backend
 +
 +server s2 {
 +        rxreq
 +        expect req.url == "/foo"
 +        txresp -status 200 \
 +    	    -hdr "Last-Modified: Thu, 26 Jun 2008 12:00:01 GMT" \
 +            -body "ABCDEFGHIJKLMNOPQRSTUVWXYZ"
 +
 +        rxreq
 +        expect req.url == "/foo"
 +        expect req.http.If-Modified-Since == "Thu, 26 Jun 2008 12:00:01 GMT"
 +        txresp -status 304
 +} -start
 +
 +varnish v1 -vcl {
 +        backend s2 {
 +                .host = "${s2_addr}"; .port = "${s2_port}";
 +        }
 +
 +        sub vcl_fetch {
 +            set beresp.grace = 0s;
 +            set beresp.ttl = 1s;
 +            set beresp.keep = 1m;
 +        }
 +} -start
 +
 +client c2 {
 +        txreq -url "/foo" \
 +            -hdr "If-Modified-Since: Thu, 26 Jun 2008 12:00:01 GMT"
 +        rxresp
 +        expect resp.status == 304
 +}
 +
 +client c1 -run
 +
 +delay 1.1
 +
 +client c2 -run
 +
 +varnish v1 -expect fetch_304 == 1
- varnish v1 -expect cond_not_validated == 0
++varnish v1 -expect fetch_not_validated == 0
 +
 +server s2 -wait
 +varnish v1 -stop
 +
 +##
 +## If stale_obj has a gzipped body, make sure it interacts correctly with clients
 +##
 +
 +server s2 {
 +        rxreq
 +        expect req.http.accept-encoding == "gzip"
 +        txresp -hdr "Last-Modified: Thu, 26 Jun 2008 12:00:01 GMT" -gzipbody FOO
 +
 +        rxreq
 +        expect req.http.If-Modified-Since == "Thu, 26 Jun 2008 12:00:01 GMT"
 +        txresp -status 304
 +} -start
 +
 +varnish v1 -cliok "param.set http_gzip_support true" -vcl {
 +        backend s2 {
 +                .host = "${s2_addr}"; .port = "${s2_port}";
 +        }
 +
 +        sub vcl_fetch {
 +            set beresp.grace = 0s;
 +            set beresp.ttl = 1s;
 +            set beresp.keep = 1m;
 +        }
 +} -start
 +
 +client c1 {
 +        txreq -hdr "Accept-encoding: gzip;q=0.1"
 +        rxresp
 +        expect resp.http.content-encoding == "gzip"
 +        gunzip
 +        expect resp.bodylen == "3"
 +} -run
 +
 +delay 1.1
 +
 +client c2 {
 +        txreq
 +        rxresp
 +        expect resp.bodylen == "3"
 +        expect resp.http.content-encoding != "gzip"
 +} -run
 +
 +varnish v1 -expect fetch_304 == 1
- varnish v1 -expect cond_not_validated == 0
++varnish v1 -expect fetch_not_validated == 0
diff --cc bin/varnishtest/tests/i00004.vtc
index bb70176,0000000..77fb9a6
mode 100644,000000..100644
--- a/bin/varnishtest/tests/i00004.vtc
+++ b/bin/varnishtest/tests/i00004.vtc
@@@ -1,201 -1,0 +1,201 @@@
 +# $Id$
 +
 +varnishtest "Verify the semantics of keep (timeout for conditional requests)"
 +
 +server s1 {
 +        rxreq
 +        expect req.url == "/foo"
 +        txresp -status 200 \
 +    	    -hdr "Last-Modified: Thu, 26 Jun 2008 12:00:01 GMT" \
 +            -body "foo bar baz quux"
 +
 +        rxreq
 +        expect req.url == "/foo"
 +        expect req.http.If-Modified-Since == "Thu, 26 Jun 2008 12:00:01 GMT"
 +        txresp -status 304
 +
 +        rxreq
 +        expect req.url == "/foo"
 +        expect req.http.If-Modified-Since != "Thu, 26 Jun 2008 12:00:01 GMT"
 +        txresp -status 200 -body "foo bar baz quux"
 +} -start
 +
 +varnish v1 -vcl+backend {
 +        sub vcl_fetch {
 +            set beresp.grace = 0s;
 +            set beresp.ttl = 1s;
 +        }
 +} -start
 +
 +client c1 {
 +        txreq -url "/foo"
 +        rxresp
 +        expect resp.status == 200
 +        expect resp.http.content-length == 16
 +        expect resp.http.Age == 0
 +}
 +
 +varnish v1 -cli "param.set default_keep 1"
 +
 +client c1 -run
 +
 +delay 1.1
 +
 +client c1 -run
 +
 +delay 2.1
 +
 +client c1 -run
 +
 +varnish v1 -expect fetch_304 == 1
- varnish v1 -expect cond_not_validated == 0
++varnish v1 -expect fetch_not_validated == 0
 +
 +server s1 -wait
 +varnish v1 -stop
 +
 +## Start a new varnish (now using the default default_keep),
 +## and manipulate beresp.keep in VCL
 +
 +server s1 -start
 +
 +varnish v1 -vcl {
 +    backend s1 {
 +            .host = "${s1_addr}"; .port = "${s1_port}";
 +    }
 +
 +    sub vcl_fetch {
 +        set beresp.grace = 0s;
 +        set beresp.ttl = 1s;
 +        set beresp.keep = 1s;
 +    }
 +} -start
 +
 +client c1 -run
 +
 +delay 1.1
 +
 +client c1 -run
 +
 +delay 2.1
 +
 +client c1 -run
 +
 +varnish v1 -expect fetch_304 == 1
- varnish v1 -expect cond_not_validated == 0
++varnish v1 -expect fetch_not_validated == 0
 +
 +server s1 -wait
 +varnish v1 -stop
 +
 +## Manipulate obj.keep in vcl_hit()
 +
 +server s1 -start
 +
 +varnish v1 -vcl {
 +    backend s1 {
 +            .host = "${s1_addr}"; .port = "${s1_port}";
 +    }
 +
 +    sub vcl_fetch {
 +        set beresp.grace = 0s;
 +        set beresp.ttl = 1s;
 +    }
 +
 +    sub vcl_hit {
 +        set obj.keep = 1s;
 +    }
 +} -start
 +
 +client c1 -run
 +
 +delay 1.1
 +
 +client c1 -run
 +
 +delay 2.1
 +
 +client c1 -run
 +
 +varnish v1 -expect fetch_304 == 1
- varnish v1 -expect cond_not_validated == 0
++varnish v1 -expect fetch_not_validated == 0
 +
 +server s1 -wait
 +varnish v1 -stop
 +
 +## Manipulate obj.keep in vcl_error()
 +
 +server s1 -start
 +
 +varnish v1 -vcl {
 +    backend s1 {
 +            .host = "${s1_addr}"; .port = "${s1_port}";
 +    }
 +
 +    sub vcl_fetch {
 +        set beresp.grace = 0s;
 +        set beresp.ttl = 1s;
 +    }
 +
 +    sub vcl_hit {
 +        error 200 "Ok";
 +    }
 +
 +    sub vcl_error {
 +        set obj.keep = 1s;
 +        return(deliver);
 +    }
 +} -start
 +
 +client c1 {
 +        txreq -url "/foo"
 +        rxresp
 +        expect resp.status == 200
 +}
 +
 +client c1 -run
 +
 +delay 1.1
 +
 +client c1 -run
 +
 +delay 2.1
 +
 +client c1 -run
 +
 +varnish v1 -expect fetch_304 == 1
- varnish v1 -expect cond_not_validated == 0
++varnish v1 -expect fetch_not_validated == 0
 +
 +server s1 -wait
 +varnish v1 -stop
 +
 +## req.keep sets an upper bound for all *.keeps in a session
 +
 +server s1 -start
 +
 +varnish v1 -vcl {
 +    backend s1 {
 +            .host = "${s1_addr}"; .port = "${s1_port}";
 +    }
 +
 +    sub vcl_recv {
 +        set req.keep = 0.5s;
 +    }
 +
 +    sub vcl_fetch {
 +        set beresp.grace = 0s;
 +        set beresp.ttl = 1s;
 +        set beresp.keep = 1s;
 +    }
 +} -start
 +
 +client c1 -run
 +
 +delay 1.1
 +
 +client c1 -run
 +
 +delay 1.6
 +
 +client c1 -run
 +
 +varnish v1 -expect fetch_304 == 1
- varnish v1 -expect cond_not_validated == 0
++varnish v1 -expect fetch_not_validated == 0
diff --cc bin/varnishtest/tests/i00005.vtc
index 983afb2,0000000..7f16892
mode 100644,000000..100644
--- a/bin/varnishtest/tests/i00005.vtc
+++ b/bin/varnishtest/tests/i00005.vtc
@@@ -1,147 -1,0 +1,147 @@@
 +# $Id$
 +
 +varnishtest "Verify interactions of ttl, keep, grace and bans"
 +
 +## Banned objects are not used for conditional requests
 +
 +server s1 {
 +        rxreq
 +        expect req.url == "/foo"
 +        txresp -status 200 \
 +    	    -hdr "Last-Modified: Thu, 26 Jun 2008 12:00:01 GMT" \
 +            -body "The quick brown fox jumps over the lazy dog."
 +
 +        rxreq
 +        expect req.url == "/foo"
 +        expect req.http.If-Modified-Since != "Thu, 26 Jun 2008 12:00:01 GMT"
 +        txresp -status 200 -body "The quick brown fox jumps over the lazy dog."
 +} -start
 +
 +varnish v1 -vcl+backend {
 +        sub vcl_fetch {
 +        set beresp.grace = 0s;
 +        set beresp.ttl = 1s;
 +        set beresp.keep = 1m;
 +        }
 +} -start
 +
 +client c1 {
 +        txreq -url "/foo"
 +        rxresp
 +        expect resp.status == 200
 +        expect resp.bodylen == 44
 +}
 +
 +client c1 -run
 +
 +delay 1.1
 +
 +varnish v1 -cli "ban.url \"/foo\""
 +
 +client c1 -run
 +
 +varnish v1 -expect fetch_304 == 0
- varnish v1 -expect cond_not_validated == 0
++varnish v1 -expect fetch_not_validated == 0
 +
 +server s1 -wait
 +varnish v1 -stop
 +
 +## If keep is set to non-zero, but then ttl is set to 0,
 +## then the object is not used for conditional requests
 +
 +server s1 -start
 +
 +varnish v1 -vcl {
 +    backend s2 {
 +            .host = "${s1_addr}"; .port = "${s1_port}";
 +    }
 +
 +    sub vcl_fetch {
 +        set beresp.keep = 1m;
 +        set beresp.ttl = 0s;
 +    }
 +} -start
 +
 +client c1 -run
 +
 +client c1 -run
 +
 +varnish v1 -expect fetch_304 == 0
- varnish v1 -expect cond_not_validated == 0
++varnish v1 -expect fetch_not_validated == 0
 +
 +server s1 -wait
 +varnish v1 -stop
 +
 +## If ttl is set to 0 but keep is then set to non-zero,
 +## then the object is used for conditional requests
 +## First test using beresp in vcl_fetch()
 +
 +server s2 {
 +        rxreq
 +        expect req.url == "/foo"
 +        txresp -status 200 \
 +    	    -hdr "Last-Modified: Thu, 26 Jun 2008 12:00:01 GMT" \
 +            -body "The quick brown fox jumps over the lazy dog."
 +
 +        rxreq
 +        expect req.url == "/foo"
 +        expect req.http.If-Modified-Since == "Thu, 26 Jun 2008 12:00:01 GMT"
 +        txresp -status 304
 +} -start
 +
 +varnish v1 -vcl {
 +    backend s2 {
 +            .host = "${s2_addr}"; .port = "${s2_port}";
 +    }
 +
 +    # NB: return(deliver) necessary here, because default vcl_fetch() returns
 +    # hit_for_pass if beresp.ttl <= 0
 +    sub vcl_fetch {
 +        set beresp.ttl = 0s;
 +        set beresp.keep = 1m;
 +        return (deliver);
 +    }
 +} -start
 +
 +client c2 {
 +        txreq -url "/foo"
 +        rxresp
 +        expect resp.status == 200
 +        expect resp.bodylen == 44
 +} -run
 +
 +delay 0.5
 +
 +client c2 -run
 +
 +varnish v1 -expect fetch_304 == 1
- varnish v1 -expect cond_not_validated == 0
++varnish v1 -expect fetch_not_validated == 0
 +
 +server s2 -wait
 +varnish v1 -stop
 +
 +## Verify using obj.ttl in vcl_hit()
 +
 +server s2 -start
 +
 +varnish v1 -vcl {
 +    backend s2 {
 +            .host = "${s2_addr}"; .port = "${s2_port}";
 +    }
 +
 +    sub vcl_hit {
 +        set obj.ttl = 0s;
 +        set obj.keep = 1m;
 +    }
 +} -start
 +
 +client c1 -run
 +
 +client c1 -run
 +
 +delay 0.5
 +
 +client c1 -run
 +
 +varnish v1 -expect fetch_304 == 1
- varnish v1 -expect cond_not_validated == 0
++varnish v1 -expect fetch_not_validated == 0
diff --cc bin/varnishtest/tests/i00006.vtc
index db15078,0000000..5059a17
mode 100644,000000..100644
--- a/bin/varnishtest/tests/i00006.vtc
+++ b/bin/varnishtest/tests/i00006.vtc
@@@ -1,156 -1,0 +1,156 @@@
 +# $Id$
 +
 +varnishtest "Verify effects of ttl, keep and grace on expiration"
 +
 +## Verify that an object's lifetime in the cache is
 +## obj.ttl + max(obj.grace, obj.keep)
 +## First with grace < keep
 +
 +server s1 {
 +        rxreq
 +        expect req.url == "/foo"
 +        txresp -status 200 \
 +    	    -hdr "Last-Modified: Thu, 26 Jun 2008 12:00:01 GMT" \
 +            -body "fred garply waldo xyzzy"
 +
 +        rxreq
 +        expect req.url == "/foo"
 +        expect req.http.If-Modified-Since == "Thu, 26 Jun 2008 12:00:01 GMT"
 +        txresp -status 304
 +
 +        rxreq
 +        expect req.url == "/foo"
 +        expect req.http.If-Modified-Since != "Thu, 26 Jun 2008 12:00:01 GMT"
 +        txresp -status 200 -body "fred garply waldo xyzzy"
 +} -start
 +
 +varnish v1 -vcl+backend {
 +        sub vcl_fetch {
 +            set beresp.ttl = 0.5s;
 +            set beresp.grace = 0.5s;
 +            set beresp.keep = 1s;
 +        }
 +} -start
 +
 +client c1 {
 +        txreq -url "/foo"
 +        rxresp
 +        expect resp.status == 200
 +        expect resp.bodylen == 23
 +} -run
 +
 +delay 1.1
 +
 +client c2 {
 +        txreq -url "/foo"
 +        rxresp
 +        expect resp.status == 200
 +        expect resp.bodylen == 23
 +        expect resp.msg == "Ok Not Modified"
 +} -run
 +
 +delay 1.6
 +
 +client c3 {
 +        txreq -url "/foo"
 +        rxresp
 +        expect resp.msg == "Ok"
 +} -run
 +
 +varnish v1 -expect fetch_304 == 1
- varnish v1 -expect cond_not_validated == 0
++varnish v1 -expect fetch_not_validated == 0
 +
 +server s1 -wait
 +varnish v1 -stop
 +
 +## Now with keep < grace
 +
 +server s1 {
 +        rxreq
 +        expect req.url == "/foo"
 +        txresp -status 200 \
 +    	    -hdr "Last-Modified: Thu, 26 Jun 2008 12:00:01 GMT" \
 +            -body "fred garply waldo xyzzy"
 +
 +        rxreq
 +        expect req.url == "/foo"
 +        expect req.http.If-Modified-Since == "Thu, 26 Jun 2008 12:00:01 GMT"
 +        txresp -status 304
 +
 +        rxreq
 +        expect req.url == "/foo"
 +        expect req.http.If-Modified-Since != "Thu, 26 Jun 2008 12:00:01 GMT"
 +        txresp -status 200 -body "fred garply waldo xyzzy"
 +} -start
 +
 +varnish v1 -vcl+backend {
 +        sub vcl_fetch {
 +            set beresp.ttl = 1s;
 +            set beresp.grace = 1s;
 +            set beresp.keep = 0.5s;
 +        }
 +} -start
 +
 +client c1 {
 +        txreq -url "/foo"
 +        rxresp
 +        expect resp.status == 200
 +        expect resp.bodylen == 23
 +} -run
 +
 +delay 1.1
 +
 +client c2 {
 +        txreq -url "/foo"
 +        rxresp
 +        expect resp.status == 200
 +        expect resp.bodylen == 23
 +        expect resp.msg == "Ok Not Modified"
 +} -run
 +
 +delay 1.6
 +
 +client c3 {
 +        txreq -url "/foo"
 +        rxresp
 +        expect resp.msg == "Ok"
 +} -run
 +
 +varnish v1 -expect fetch_304 == 1
- varnish v1 -expect cond_not_validated == 0
++varnish v1 -expect fetch_not_validated == 0
 +
 +server s1 -wait
 +varnish v1 -stop
 +
 +## Verify that req.keep sets an upper bound in the session
 +
 +server s1 -start
 +
 +varnish v1 -vcl {
 +    backend s1 {
 +            .host = "${s1_addr}"; .port = "${s1_port}";
 +    }
 +
 +        sub vcl_recv {
 +            set req.keep = 1s;
 +        }
 +
 +        sub vcl_fetch {
 +            set beresp.ttl = 1s;
 +            set beresp.grace = 0s;
 +            set beresp.keep = 2s;
 +        }
 +} -start
 +
 +client c1 -run
 +
 +delay 1.1
 +
 +client c2 -run
 +
 +delay 2.1
 +
 +client c3 -run
 +
 +varnish v1 -expect fetch_304 == 1
- varnish v1 -expect cond_not_validated == 0
++varnish v1 -expect fetch_not_validated == 0
diff --cc bin/varnishtest/tests/i00007.vtc
index 4a9b8c0,0000000..74bb596
mode 100644,000000..100644
--- a/bin/varnishtest/tests/i00007.vtc
+++ b/bin/varnishtest/tests/i00007.vtc
@@@ -1,129 -1,0 +1,129 @@@
 +# $Id$
 +
 +varnishtest "Passes through responses to backend conditionals to the client if status != 304 or 200"
 +
 +# Testing a sample from each of the Nxx status codes
 +
 +server s1 {
 +        rxreq
 +        expect req.url == "/foo"
 +        txresp -status 200 \
 +    	    -hdr "Last-Modified: Thu, 26 Jun 2008 12:00:01 GMT" \
 +            -body "Varnish has poked you"
 +
 +        rxreq
 +        expect req.url == "/foo"
 +        expect req.http.If-Modified-Since == "Thu, 26 Jun 2008 12:00:01 GMT"
 +        txresp -status 304
 +
 +        rxreq
 +        expect req.url == "/foo"
 +        expect req.http.If-Modified-Since == "Thu, 26 Jun 2008 12:00:01 GMT"
 +        txresp -status 201 -msg "Created" \
 +    	    -hdr "Last-Modified: Thu, 26 Jun 2008 12:00:01 GMT" \
 +            -body "Varnish has poked you"
 +
 +        # 3xx responses typically do not include Last-Modified or a body
 +        rxreq
 +        expect req.url == "/foo"
 +        expect req.http.If-Modified-Since == "Thu, 26 Jun 2008 12:00:01 GMT"
 +        txresp -status 301 -msg "Moved Permanently"
 +
 +        # Restore a cached object with Last-Modified
 +        rxreq
 +        expect req.url == "/foo"
 +        expect req.http.If-Modified-Since != "Thu, 26 Jun 2008 12:00:01 GMT"
 +        txresp -status 200 \
 +    	    -hdr "Last-Modified: Thu, 26 Jun 2008 12:00:01 GMT" \
 +            -body "Varnish has poked you"
 +
 +        rxreq
 +        expect req.url == "/foo"
 +        expect req.http.If-Modified-Since == "Thu, 26 Jun 2008 12:00:01 GMT"
 +        txresp -status 400 -msg "Bad Request" \
 +            -body "Varnish has poked you"
 +
 +        # Restore a cached object with Last-Modified
 +        rxreq
 +        expect req.url == "/foo"
 +        expect req.http.If-Modified-Since != "Thu, 26 Jun 2008 12:00:01 GMT"
 +        txresp -status 200 \
 +    	    -hdr "Last-Modified: Thu, 26 Jun 2008 12:00:01 GMT" \
 +            -body "Varnish has poked you"
 +
 +        rxreq
 +        expect req.url == "/foo"
 +        expect req.http.If-Modified-Since == "Thu, 26 Jun 2008 12:00:01 GMT"
 +        txresp -status 500 -msg "Internal Server Error" \
 +            -body "Varnish has poked you"
 +} -start
 +
 +varnish v1 -vcl+backend {
 +        sub vcl_fetch {
 +            set beresp.ttl = 1s;
 +            set beresp.grace = 0s;
 +            set beresp.keep = 1s;
 +        }
 +} -start
 +
 +client c1 {
 +        txreq -url "/foo"
 +        rxresp
 +        expect resp.status == 200
 +        expect resp.bodylen == 21
 +} -run
 +
 +delay 1.1
 +
 +client c1 -run
 +
 +delay 1.1
 +
 +client c2 {
 +        txreq -url "/foo"
 +        rxresp
 +        expect resp.status == 201
 +        expect resp.msg == "Created"
 +        expect resp.http.Last-Modified == "Thu, 26 Jun 2008 12:00:01 GMT"
 +        expect resp.bodylen == 21
 +} -run
 +
 +delay 1.1
 +
 +client c2 {
 +        txreq -url "/foo"
 +        rxresp
 +        expect resp.status == 301
 +        expect resp.msg == "Moved Permanently"
 +}  -run
 +
 +delay 1.1
 +
 +client c1 -run
 +
 +delay 1.1
 +
 +client c2 {
 +        txreq -url "/foo"
 +        rxresp
 +        expect resp.status == 400
 +        expect resp.msg == "Bad Request"
 +        expect resp.bodylen == 21
 +}  -run
 +
 +delay 1.1
 +
 +client c1 -run
 +
 +delay 1.1
 +
 +client c2 {
 +        txreq -url "/foo"
 +        rxresp
 +        expect resp.status == 500
 +        expect resp.msg == "Internal Server Error"
 +        expect resp.bodylen == 21
 +}  -run
 +
 +varnish v1 -expect fetch_304 == 1
- varnish v1 -expect cond_not_validated == 4
++varnish v1 -expect fetch_not_validated == 4
diff --cc include/tbl/vsc_f_main.h
index 0000000,ab8415c..5b9abf6
mode 000000,100644..100644
--- a/include/tbl/vsc_f_main.h
+++ b/include/tbl/vsc_f_main.h
@@@ -1,0 -1,410 +1,412 @@@
+ /*-
+  * Copyright (c) 2006 Verdens Gang AS
+  * Copyright (c) 2006-2011 Varnish Software AS
+  * All rights reserved.
+  *
+  * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
+  *
+  * Redistribution and use in source and binary forms, with or without
+  * modification, are permitted provided that the following conditions
+  * are met:
+  * 1. Redistributions of source code must retain the above copyright
+  *    notice, this list of conditions and the following disclaimer.
+  * 2. Redistributions in binary form must reproduce the above copyright
+  *    notice, this list of conditions and the following disclaimer in the
+  *    documentation and/or other materials provided with the distribution.
+  *
+  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+  * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION, "")
+  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+  * SUCH DAMAGE.
+  *
+  * Definition of all shared memory statistics below.
+  *
+  * Fields (n, t, l, f, e, d):
+  *    n - Name:		Field name, in C-source and stats programs
+  *    t - Type:		C-type, uint64_t, unless marked in 'f'
+  *    l - Local:	Local counter in worker thread.
+  *    f - Format:	Semantics of the value in this field
+  *				'a' - Accumulator (deprecated, use 'c')
+  *				'b' - Bitmap
+  *				'c' - Counter, never decreases.
+  *				'g' - Gauge, goes up and down
+  *				'i' - Integer (deprecated, use 'g')
+  *    e - Explantion:	Short explanation of field (for screen use)
+  *    d - Description:	Long explanation of field (for doc use)
+  *
+  * Please describe Gauge variables as "Number of..." to indicate that
+  * this is a snapshot, and Counter variables as "Count of" to indicate
+  * accumulative count.
+  *
+  * -----------------------
+  * NB: Cleanup in progress
+  * -----------------------
+  *
+  * Insufficient attention has caused this to become a swamp of conflicting
+  * conventions, shorthands and general mumbo-jumbo.  I'm trying to clean
+  * it up as I go over the code in other business.
+  *
+  * Please see the sessmem section for how it should look.
+  *
+  */
+ 
+ /*---------------------------------------------------------------------
+  * Sessions
+  *    see: cache_acceptor.c and cache_pool.c
+  */
+ 
+ VSC_F(sess_conn,		uint64_t, 1, 'c',
+     "Sessions accepted",
+ 	"Count of sessions succesfully accepted"
+ )
+ VSC_F(sess_drop,		uint64_t, 1, 'c',
+     "Sessions dropped",
+ 	"Count of sessions silently dropped due to lack of session memory."
+ 	"  See parameter 'max_sess'."
+ )
+ 
+ VSC_F(sess_fail,		uint64_t, 1, 'c',
+     "Session accept failures",
+ 	"Count of failures to accept TCP connection."
+ 	"  Either the client changed its mind, or the kernel ran out of"
+ 	" some resource like filedescriptors."
+ )
+ 
+ /*---------------------------------------------------------------------*/
+ 
+ VSC_F(client_req,		uint64_t, 1, 'a',
+       "Client requests received",
+       "")
+ 
+ VSC_F(cache_hit,		uint64_t, 1, 'a',
+       "Cache hits",
+       "Count of cache hits. "
+       "  A cache hit indicates that an object has been delivered to a"
+       "  client without fetching it from a backend server."
+ )
+ 
+ VSC_F(cache_hitpass,	uint64_t, 1, 'a',
+       "Cache hits for pass",
+       "Count of hits for pass"
+       "  A cache hit for pass indicates that Varnish is going to"
+       "  pass the request to the backend and this decision has been "
+       "  cached in it self. This counts how many times the cached "
+       "  decision is being used."
+ )
+ VSC_F(cache_miss,		uint64_t, 1, 'a',
+       "Cache misses",
+       "Count of misses"
+       "  A cache miss indicates the object was fetched from the"
+       "  backend before delivering it to the backend.")
+ 
+ VSC_F(backend_conn,	uint64_t, 0, 'a',
+       "Backend conn. success",
+       "")
+ 
+ VSC_F(backend_unhealthy,	uint64_t, 0, 'a',
+       "Backend conn. not attempted",
+       ""
+ )
+ VSC_F(backend_busy,	uint64_t, 0, 'a', "Backend conn. too many", "")
+ VSC_F(backend_fail,	uint64_t, 0, 'a', "Backend conn. failures", "")
+ VSC_F(backend_reuse,	uint64_t, 0, 'a',
+       "Backend conn. reuses",
+       "Count of backend connection reuses"
+       "  This counter is increased whenever we reuse a recycled connection.")
+ VSC_F(backend_toolate,	uint64_t, 0, 'a', "Backend conn. was closed", "")
+ VSC_F(backend_recycle,	uint64_t, 0, 'a',
+       "Backend conn. recycles",
+       "Count of backend connection recycles"
+       "  This counter is increased whenever we have a keep-alive"
+       "  connection that is put back into the pool of connections."
+       "  It has not yet been used, but it might be, unless the backend"
+       "  closes it.")
+ VSC_F(backend_retry,	uint64_t, 0, 'a', "Backend conn. retry", "")
+ 
+ VSC_F(fetch_head,		uint64_t, 1, 'a', "Fetch head", "")
+ VSC_F(fetch_length,		uint64_t, 1, 'a', "Fetch with Length", "")
+ VSC_F(fetch_chunked,		uint64_t, 1, 'a', "Fetch chunked", "")
+ VSC_F(fetch_eof,		uint64_t, 1, 'a', "Fetch EOF", "")
+ VSC_F(fetch_bad,		uint64_t, 1, 'a', "Fetch had bad headers", "")
+ VSC_F(fetch_close,		uint64_t, 1, 'a', "Fetch wanted close", "")
+ VSC_F(fetch_oldhttp,		uint64_t, 1, 'a', "Fetch pre HTTP/1.1 closed", "")
+ VSC_F(fetch_zero,		uint64_t, 1, 'a', "Fetch zero len", "")
+ VSC_F(fetch_failed,		uint64_t, 1, 'a', "Fetch failed", "")
+ VSC_F(fetch_1xx,		uint64_t, 1, 'a', "Fetch no body (1xx)", "")
+ VSC_F(fetch_204,		uint64_t, 1, 'a', "Fetch no body (204)", "")
+ VSC_F(fetch_304,		uint64_t, 1, 'a', "Fetch no body (304)", "")
++VSC_F(fetch_not_validated,	uint64_t, 1, 'c', "Non-validating responses",
++      "Count of backend responses to conditional requests with status != 304")
+ 
+ /*---------------------------------------------------------------------
+  * Session Memory
+  *    see: cache_session.c
+  */
+ 
+ VSC_F(sessmem_size,		uint64_t, 1, 'g',
+     "Session mem size",
+ 	"Bytes of memory allocated for last allocated session."
+ )
+ 
+ VSC_F(sessmem_alloc,		uint64_t, 1, 'c',
+     "Session mem allocated",
+ 	"Count of all allocations of session memory."
+ )
+ 
+ VSC_F(sessmem_free,		uint64_t, 1, 'c',
+     "Session mem freed",
+ 	"Count of all frees of session memory."
+ )
+ 
+ VSC_F(sessmem_fail,		uint64_t, 1, 'c',
+     "Session mem alloc failed",
+ 	"Count of session memory allocation failures."
+ )
+ 
+ VSC_F(sessmem_limit,		uint64_t, 1, 'c',
+     "Session mem alloc limited",
+ 	"Count of session memory allocations blocked by limit (max_sess)."
+ )
+ 
+ /*---------------------------------------------------------------------
+  * Pools, threads, and sessions
+  *    see: cache_pool.c
+  *
+  */
+ 
+ VSC_F(pools,			uint64_t, 1, 'g',
+     "Number of thread pools",
+ 	"Number of thread pools.  See also param wthread_pools."
+ 	"  NB: Presently pools cannot be removed once created."
+ )
+ 
+ VSC_F(threads,			uint64_t, 1, 'g',
+     "Total number of threads",
+ 	"Number of threads in all pools."
+ 	"  See also params thread_pools, thread_pool_min & thread_pool_max."
+ )
+ 
+ VSC_F(threads_limited,		uint64_t, 1, 'c',
+     "Threads hit max",
+ 	"Number of times more threads were needed, but limit was reached"
+ 	" in a thread pool."
+ 	"  See also param thread_pool_max."
+ )
+ 
+ VSC_F(threads_created,		uint64_t, 1, 'c',
+     "Threads created",
+ 	"Total number of threads created in all pools."
+ )
+ 
+ VSC_F(threads_destroyed,	uint64_t, 1, 'c',
+     "Threads destoryed",
+ 	"Total number of threads destroyed in all pools."
+ )
+ 
+ VSC_F(threads_failed,		uint64_t, 1, 'c',
+     "Thread creation failed",
+ 	"Number of times creating a thread failed."
+ 	"  See VSL::Debug for diagnostics."
+ 	"  See also param thread_fail_delay."
+ )
+ 
+ VSC_F(thread_queue_len,		uint64_t, 1, 'g',
+     "Length of session queue",
+ 	"Length of session queue waiting for threads."
+ 	"  NB: Only updates once per second."
+ 	"  See also param queue_max."
+ )
+ 
+ VSC_F(sess_queued,		uint64_t, 1, 'c',
+     "Sessions queued for thread",
+ 	"Number of times session was queued waiting for a thread."
+ 	"  See also param queue_max."
+ )
+ 
+ VSC_F(sess_dropped,		uint64_t, 1, 'c',
+     "Sessions dropped for thread",
+ 	"Number of times session was dropped because the queue were too"
+ 	" long already."
+ 	"  See also param queue_max."
+ )
+ 
+ /*---------------------------------------------------------------------
+  * BusyObj
+  */
+ 
+ VSC_F(busyobj_alloc,		uint64_t, 1, 'c',
+     "Busyobj allocations",
+ 	"Number of busyobj structures allocated."
+ )
+ 
+ VSC_F(busyobj_free,		uint64_t, 1, 'c',
+     "Busyobj freed",
+ 	"Number of busyobj structures freed."
+ )
+ 
+ /*---------------------------------------------------------------------*/
+ 
+ VSC_F(n_sess_mem,		uint64_t, 0, 'i', "N struct sess_mem", "")
+ VSC_F(n_sess,			uint64_t, 0, 'i', "N struct sess", "")
+ VSC_F(n_object,			uint64_t, 1, 'i', "N struct object", "")
+ VSC_F(n_vampireobject,		uint64_t, 1, 'i', "N unresurrected objects", "")
+ VSC_F(n_objectcore,		uint64_t, 1, 'i', "N struct objectcore", "")
+ VSC_F(n_objecthead,		uint64_t, 1, 'i', "N struct objecthead", "")
+ VSC_F(n_waitinglist,		uint64_t, 1, 'i', "N struct waitinglist", "")
+ 
+ VSC_F(n_vbc,		uint64_t, 0, 'i', "N struct vbc", "")
+ 
+ VSC_F(n_backend,		uint64_t, 0, 'i', "N backends", "")
+ 
+ VSC_F(n_expired,		uint64_t, 0, 'i', "N expired objects", "")
+ VSC_F(n_lru_nuked,		uint64_t, 0, 'i', "N LRU nuked objects", "")
+ VSC_F(n_lru_moved,		uint64_t, 0, 'i', "N LRU moved objects", "")
+ 
+ VSC_F(losthdr,		uint64_t, 0, 'a', "HTTP header overflows", "")
+ 
+ VSC_F(n_objsendfile,	uint64_t, 0, 'a', "Objects sent with sendfile",
+       "The number of objects sent with the sendfile system call. If enabled "
+       "sendfile will be used on object larger than a certain size.")
+ VSC_F(n_objwrite,		uint64_t, 0, 'a', "Objects sent with write",
+       "The number of objects sent with regular write calls."
+       "Writes are used when the objects are too small for sendfile "
+       "or if the sendfile call has been disabled")
+ VSC_F(n_objoverflow,	uint64_t, 1, 'a',
+ 					"Objects overflowing workspace", "")
+ 
+ VSC_F(s_sess,		uint64_t, 1, 'a', "Total Sessions", "")
+ VSC_F(s_req,		uint64_t, 1, 'a', "Total Requests", "")
+ VSC_F(s_pipe,		uint64_t, 1, 'a', "Total pipe", "")
+ VSC_F(s_pass,		uint64_t, 1, 'a', "Total pass", "")
+ VSC_F(s_fetch,		uint64_t, 1, 'a', "Total fetch", "")
+ VSC_F(s_hdrbytes,		uint64_t, 1, 'a', "Total header bytes", "")
+ VSC_F(s_bodybytes,		uint64_t, 1, 'a', "Total body bytes", "")
+ 
+ VSC_F(sess_closed,		uint64_t, 1, 'a', "Session Closed", "")
+ VSC_F(sess_pipeline,	uint64_t, 1, 'a', "Session Pipeline", "")
+ VSC_F(sess_readahead,	uint64_t, 1, 'a', "Session Read Ahead", "")
+ VSC_F(sess_linger,		uint64_t, 1, 'a', "Session Linger", "")
+ VSC_F(sess_herd,		uint64_t, 1, 'a', "Session herd", "")
+ 
+ VSC_F(shm_records,		uint64_t, 0, 'a', "SHM records", "")
+ VSC_F(shm_writes,		uint64_t, 0, 'a', "SHM writes", "")
+ VSC_F(shm_flushes,		uint64_t, 0, 'a', "SHM flushes due to overflow", "")
+ VSC_F(shm_cont,		uint64_t, 0, 'a', "SHM MTX contention", "")
+ VSC_F(shm_cycles,		uint64_t, 0, 'a', "SHM cycles through buffer", "")
+ 
+ VSC_F(sms_nreq,		uint64_t, 0, 'a', "SMS allocator requests", "")
+ VSC_F(sms_nobj,		uint64_t, 0, 'i', "SMS outstanding allocations", "")
+ VSC_F(sms_nbytes,		uint64_t, 0, 'i', "SMS outstanding bytes", "")
+ VSC_F(sms_balloc,		uint64_t, 0, 'i', "SMS bytes allocated", "")
+ VSC_F(sms_bfree,		uint64_t, 0, 'i', "SMS bytes freed", "")
+ 
+ VSC_F(backend_req,		uint64_t, 0, 'a', "Backend requests made", "")
+ 
+ VSC_F(n_vcl,		uint64_t, 0, 'a', "N vcl total", "")
+ VSC_F(n_vcl_avail,		uint64_t, 0, 'a', "N vcl available", "")
+ VSC_F(n_vcl_discard,	uint64_t, 0, 'a', "N vcl discarded", "")
+ 
+ /**********************************************************************/
+ 
+ VSC_F(bans,			uint64_t, 0, 'g',
+    "Count of bans",
+ 	"Number of all bans in system, including bans superseded"
+ 	" by newer bans and bans already checked by the ban-lurker."
+ )
+ VSC_F(bans_gone,		uint64_t, 0, 'g',
+     "Number of bans marked 'gone'",
+ 	"Number of bans which are no longer active, either because they"
+ 	" got checked by the ban-lurker or superseded by newer identical bans."
+ )
+ VSC_F(bans_req,			uint64_t, 0, 'g',
+     "Number of bans using req.*",
+ 	"Number of bans which use req.* variables.  These bans can not"
+ 	" be washed by the ban-lurker."
+ )
+ VSC_F(bans_added,		uint64_t, 0, 'c',
+     "Bans added",
+ 	"Counter of bans added to ban list."
+ )
+ VSC_F(bans_deleted,		uint64_t, 0, 'c',
+     "Bans deleted",
+ 	"Counter of bans deleted from ban list."
+ )
+ 
+ VSC_F(bans_tested,		uint64_t, 0, 'c',
+     "Bans tested against objects",
+ 	"Count of how many bans and objects have been tested against"
+ 	" each other."
+ )
+ VSC_F(bans_tests_tested,	uint64_t, 0, 'c',
+     "Ban tests tested against objects",
+ 	"Count of how many tests and objects have been tested against"
+ 	" each other.  'ban req.url == foo && req.http.host == bar'"
+ 	" counts as one in 'bans_tested' and as two in 'bans_tests_tested'"
+ )
+ VSC_F(bans_dups,		uint64_t, 0, 'c',
+     "Bans superseded by other bans",
+ 	"Count of bans replaced by later identical bans."
+ )
+ 
+ /**********************************************************************/
+ 
+ VSC_F(hcb_nolock,		uint64_t, 0, 'a', "HCB Lookups without lock", "")
+ VSC_F(hcb_lock,		uint64_t, 0, 'a', "HCB Lookups with lock", "")
+ VSC_F(hcb_insert,		uint64_t, 0, 'a', "HCB Inserts", "")
+ 
+ VSC_F(esi_errors,		uint64_t, 0, 'a', "ESI parse errors (unlock)", "")
+ VSC_F(esi_warnings,		uint64_t, 0, 'a', "ESI parse warnings (unlock)", "")
+ VSC_F(client_drop_late,	uint64_t, 0, 'a', "Connection dropped late", "")
+ VSC_F(uptime,		uint64_t, 0, 'a', "Client uptime", "")
+ 
+ VSC_F(dir_dns_lookups,	uint64_t, 0, 'a', "DNS director lookups", "")
+ VSC_F(dir_dns_failed,	uint64_t, 0, 'a', "DNS director failed lookups", "")
+ VSC_F(dir_dns_hit,		uint64_t, 0, 'a', "DNS director cached lookups hit", "")
+ VSC_F(dir_dns_cache_full,	uint64_t, 0, 'a', "DNS director full dnscache", "")
+ 
+ VSC_F(vmods,		uint64_t, 0, 'i', "Loaded VMODs", "")
+ 
+ VSC_F(n_gzip,			uint64_t, 0, 'a', "Gzip operations", "")
+ VSC_F(n_gunzip,			uint64_t, 0, 'a', "Gunzip operations", "")
+ 
+ /**********************************************************************/
+ 
+ VSC_F(vsm_free,			uint64_t, 0, 'g',
+     "Free VSM space",
+ 	"Number of bytes free in the shared memory used to communicate"
+ 	" with tools like varnishstat, varnishlog etc."
+ )
+ 
+ VSC_F(vsm_used,			uint64_t, 0, 'g',
+     "Used VSM space",
+ 	"Number of bytes used in the shared memory used to communicate"
+ 	" with tools like varnishstat, varnishlog etc."
+ )
+ 
+ VSC_F(vsm_cooling,		uint64_t, 0, 'g',
+     "Cooling VSM space",
+ 	"Number of bytes which will soon (max 1 minute) be freed"
+ 	" in the shared memory used to communicate"
+ 	" with tools like varnishstat, varnishlog etc."
+ )
+ 
+ VSC_F(vsm_overflow,		uint64_t, 0, 'g',
+     "Overflow VSM space",
+ 	"Number of bytes which does not fit"
+ 	" in the shared memory used to communicate"
+ 	" with tools like varnishstat, varnishlog etc."
+ )
+ 
+ VSC_F(vsm_overflowed,		uint64_t, 0, 'c',
+     "Overflowed VSM space",
+ 	"Total number of bytes which did not fit"
+ 	" in the shared memory used to communicate"
+ 	" with tools like varnishstat, varnishlog etc."
+ )



From geoff at varnish-cache.org  Mon Jan  9 20:51:56 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:51:56 +0100
Subject: [experimental-ims] eafb66a Add a "long description field" to the
	VSC_F() macro,
	the intent is that they should be mined and dumped into the
	documentation.
Message-ID: <E1RkMCC-000700-3u@project.varnish-software.com>

commit eafb66a243531c88b82ecae1b5ee0c34d1d95e16
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sun Sep 18 07:39:30 2011 +0000

    Add a "long description field" to the VSC_F() macro, the intent is
    that they should be mined and dumped into the documentation.
    
    Feel free to add (or send patches with descriptions, preferably
    only one line or paragraph for each.

diff --git a/bin/varnishd/cache.h b/bin/varnishd/cache.h
index 49ca3a4..5db137a 100644
--- a/bin/varnishd/cache.h
+++ b/bin/varnishd/cache.h
@@ -220,7 +220,7 @@ struct acct {
 
 #define L0(n)
 #define L1(n)			uint64_t n;
-#define VSC_F(n, t, l, f, e)	L##l(n)
+#define VSC_F(n, t, l, f, e,d)	L##l(n)
 #define VSC_DO_MAIN
 struct dstat {
 #include "vsc_fields.h"
diff --git a/bin/varnishd/cache_wrk.c b/bin/varnishd/cache_wrk.c
index 631342f..3cfaba0 100644
--- a/bin/varnishd/cache_wrk.c
+++ b/bin/varnishd/cache_wrk.c
@@ -58,7 +58,7 @@ wrk_sumstat(struct worker *w)
 #define L0(n)
 #define L1(n) (VSC_C_main->n += w->stats.n)
 #define VSC_DO_MAIN
-#define VSC_F(n, t, l, f, d) L##l(n);
+#define VSC_F(n, t, l, f, d, e) L##l(n);
 #include "vsc_fields.h"
 #undef VSC_F
 #undef VSC_DO_MAIN
diff --git a/include/vsc.h b/include/vsc.h
index 1256dae..7cb3235 100644
--- a/include/vsc.h
+++ b/include/vsc.h
@@ -38,7 +38,7 @@
 #define VSC_TYPE_VBE	"VBE"
 #define VSC_TYPE_LCK	"LCK"
 
-#define VSC_F(n, t, l, f, e)	t n;
+#define VSC_F(n, t, l, f, e, d)	t n;
 
 #define VSC_DO(u,l,t) struct VSC_C_##l {
 #define VSC_DONE(u,l,t) };
diff --git a/include/vsc_fields.h b/include/vsc_fields.h
index e337321..79ad315 100644
--- a/include/vsc_fields.h
+++ b/include/vsc_fields.h
@@ -20,7 +20,7 @@
  * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION, "")
  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
@@ -33,140 +33,165 @@
  * XXX: turned into a major mess, causing trouble already for backends.
  * XXX:
  * XXX: Please converge on:
- * XXX:		c_* counter	(total bytes ever allocated from sma)
- * XXX:		g_* gauge	(presently allocated bytes from sma)
+ * XXX:		c_* counter	(total bytes ever allocated from sma, "")
+ * XXX:		g_* gauge	(presently allocated bytes from sma, "")
  */
 
 /**********************************************************************/
 
 #ifdef VSC_DO_MAIN
 
-VSC_F(client_conn,		uint64_t, 1, 'a', "Client connections accepted")
+VSC_F(client_conn,		uint64_t, 1, 'a', "Client connections accepted", "")
 VSC_F(client_drop,		uint64_t, 0, 'a',
-					"Connection dropped, no sess/wrk")
-VSC_F(client_req,		uint64_t, 1, 'a', "Client requests received")
-
-VSC_F(cache_hit,		uint64_t, 1, 'a', "Cache hits")
-VSC_F(cache_hitpass,	uint64_t, 1, 'a', "Cache hits for pass")
-VSC_F(cache_miss,		uint64_t, 1, 'a', "Cache misses")
-
-VSC_F(backend_conn,	uint64_t, 0, 'a', "Backend conn. success")
-VSC_F(backend_unhealthy,	uint64_t, 0, 'a', "Backend conn. not attempted")
-VSC_F(backend_busy,	uint64_t, 0, 'a', "Backend conn. too many")
-VSC_F(backend_fail,	uint64_t, 0, 'a', "Backend conn. failures")
-VSC_F(backend_reuse,	uint64_t, 0, 'a', "Backend conn. reuses")
-VSC_F(backend_toolate,	uint64_t, 0, 'a', "Backend conn. was closed")
-VSC_F(backend_recycle,	uint64_t, 0, 'a', "Backend conn. recycles")
-VSC_F(backend_retry,	uint64_t, 0, 'a', "Backend conn. retry")
-
-VSC_F(fetch_head,		uint64_t, 1, 'a', "Fetch head")
-VSC_F(fetch_length,		uint64_t, 1, 'a', "Fetch with Length")
-VSC_F(fetch_chunked,		uint64_t, 1, 'a', "Fetch chunked")
-VSC_F(fetch_eof,		uint64_t, 1, 'a', "Fetch EOF")
-VSC_F(fetch_bad,		uint64_t, 1, 'a', "Fetch had bad headers")
-VSC_F(fetch_close,		uint64_t, 1, 'a', "Fetch wanted close")
-VSC_F(fetch_oldhttp,		uint64_t, 1, 'a', "Fetch pre HTTP/1.1 closed")
-VSC_F(fetch_zero,		uint64_t, 1, 'a', "Fetch zero len")
-VSC_F(fetch_failed,		uint64_t, 1, 'a', "Fetch failed")
-VSC_F(fetch_1xx,		uint64_t, 1, 'a', "Fetch no body (1xx)")
-VSC_F(fetch_204,		uint64_t, 1, 'a', "Fetch no body (204)")
-VSC_F(fetch_304,		uint64_t, 1, 'a', "Fetch no body (304)")
-
-/* Sessmem cache_session.c */
-VSC_F(g_sessmem_size,		uint64_t, 1, 'i', "Session mem size")
-VSC_F(c_sessmem_alloc,		uint64_t, 1, 'a', "Session mem allocated")
-VSC_F(c_sessmem_free,		uint64_t, 1, 'a', "Session mem freed")
-VSC_F(c_sessmem_fail,		uint64_t, 1, 'a', "Session mem alloc failed")
-VSC_F(c_sessmem_limit,		uint64_t, 1, 'a', "Session mem alloc limited")
-
-VSC_F(n_sess_mem,		uint64_t, 0, 'i', "N struct sess_mem")
-VSC_F(n_sess,			uint64_t, 0, 'i', "N struct sess")
-VSC_F(n_object,			uint64_t, 1, 'i', "N struct object")
-VSC_F(n_vampireobject,		uint64_t, 1, 'i', "N unresurrected objects")
-VSC_F(n_objectcore,		uint64_t, 1, 'i', "N struct objectcore")
-VSC_F(n_objecthead,		uint64_t, 1, 'i', "N struct objecthead")
-VSC_F(n_waitinglist,		uint64_t, 1, 'i', "N struct waitinglist")
-
-VSC_F(n_vbc,		uint64_t, 0, 'i', "N struct vbc")
-VSC_F(n_wrk,		uint64_t, 0, 'i', "N worker threads")
-VSC_F(n_wrk_create,	uint64_t, 0, 'a', "N worker threads created")
+					"Connection dropped, no sess/wrk", "")
+VSC_F(client_req,		uint64_t, 1, 'a', "Client requests received", "")
+
+VSC_F(cache_hit,		uint64_t, 1, 'a', "Cache hits", "")
+VSC_F(cache_hitpass,	uint64_t, 1, 'a', "Cache hits for pass", "")
+VSC_F(cache_miss,		uint64_t, 1, 'a', "Cache misses", "")
+
+VSC_F(backend_conn,	uint64_t, 0, 'a', "Backend conn. success", "")
+VSC_F(backend_unhealthy,	uint64_t, 0, 'a', "Backend conn. not attempted", "")
+VSC_F(backend_busy,	uint64_t, 0, 'a', "Backend conn. too many", "")
+VSC_F(backend_fail,	uint64_t, 0, 'a', "Backend conn. failures", "")
+VSC_F(backend_reuse,	uint64_t, 0, 'a', "Backend conn. reuses", "")
+VSC_F(backend_toolate,	uint64_t, 0, 'a', "Backend conn. was closed", "")
+VSC_F(backend_recycle,	uint64_t, 0, 'a', "Backend conn. recycles", "")
+VSC_F(backend_retry,	uint64_t, 0, 'a', "Backend conn. retry", "")
+
+VSC_F(fetch_head,		uint64_t, 1, 'a', "Fetch head", "")
+VSC_F(fetch_length,		uint64_t, 1, 'a', "Fetch with Length", "")
+VSC_F(fetch_chunked,		uint64_t, 1, 'a', "Fetch chunked", "")
+VSC_F(fetch_eof,		uint64_t, 1, 'a', "Fetch EOF", "")
+VSC_F(fetch_bad,		uint64_t, 1, 'a', "Fetch had bad headers", "")
+VSC_F(fetch_close,		uint64_t, 1, 'a', "Fetch wanted close", "")
+VSC_F(fetch_oldhttp,		uint64_t, 1, 'a', "Fetch pre HTTP/1.1 closed", "")
+VSC_F(fetch_zero,		uint64_t, 1, 'a', "Fetch zero len", "")
+VSC_F(fetch_failed,		uint64_t, 1, 'a', "Fetch failed", "")
+VSC_F(fetch_1xx,		uint64_t, 1, 'a', "Fetch no body (1xx)", "")
+VSC_F(fetch_204,		uint64_t, 1, 'a', "Fetch no body (204)", "")
+VSC_F(fetch_304,		uint64_t, 1, 'a', "Fetch no body (304)", "")
+
+/*---------------------------------------------------------------------
+ * Session Memory
+ *    see: cache_session.c
+ */
+
+VSC_F(g_sessmem_size,		uint64_t, 1, 'i',
+    "Session mem size",
+	"Bytes of memory allocated for last allocated session."
+)
+
+VSC_F(c_sessmem_alloc,		uint64_t, 1, 'a',
+    "Session mem allocated",
+	"Count of all allocations of session memory."
+)
+
+VSC_F(c_sessmem_free,		uint64_t, 1, 'a',
+    "Session mem freed",
+	"Count of all frees of session memory."
+)
+
+VSC_F(c_sessmem_fail,		uint64_t, 1, 'a',
+    "Session mem alloc failed",
+	"Count of session memory allocation failures."
+)
+
+VSC_F(c_sessmem_limit,		uint64_t, 1, 'a',
+    "Session mem alloc limited",
+	"Count of session memory allocations blocked by limit (max_sess)."
+)
+
+/*---------------------------------------------------------------------*/
+
+VSC_F(n_sess_mem,		uint64_t, 0, 'i', "N struct sess_mem", "")
+VSC_F(n_sess,			uint64_t, 0, 'i', "N struct sess", "")
+VSC_F(n_object,			uint64_t, 1, 'i', "N struct object", "")
+VSC_F(n_vampireobject,		uint64_t, 1, 'i', "N unresurrected objects", "")
+VSC_F(n_objectcore,		uint64_t, 1, 'i', "N struct objectcore", "")
+VSC_F(n_objecthead,		uint64_t, 1, 'i', "N struct objecthead", "")
+VSC_F(n_waitinglist,		uint64_t, 1, 'i', "N struct waitinglist", "")
+
+VSC_F(n_vbc,		uint64_t, 0, 'i', "N struct vbc", "")
+VSC_F(n_wrk,		uint64_t, 0, 'i', "N worker threads", "")
+VSC_F(n_wrk_create,	uint64_t, 0, 'a', "N worker threads created", "")
 VSC_F(n_wrk_failed,	uint64_t, 0, 'a',
-					"N worker threads not created")
-VSC_F(n_wrk_max,		uint64_t, 0, 'a', "N worker threads limited")
-VSC_F(n_wrk_lqueue,		uint64_t, 0, 'a', "work request queue length")
-VSC_F(n_wrk_queued,		uint64_t, 0, 'a', "N queued work requests")
-VSC_F(n_wrk_drop,		uint64_t, 0, 'a', "N dropped work requests")
-VSC_F(n_backend,		uint64_t, 0, 'i', "N backends")
+					"N worker threads not created", "")
+VSC_F(n_wrk_max,		uint64_t, 0, 'a', "N worker threads limited", "")
+VSC_F(n_wrk_lqueue,		uint64_t, 0, 'a', "work request queue length", "")
+VSC_F(n_wrk_queued,		uint64_t, 0, 'a', "N queued work requests", "")
+VSC_F(n_wrk_drop,		uint64_t, 0, 'a', "N dropped work requests", "")
+VSC_F(n_backend,		uint64_t, 0, 'i', "N backends", "")
 
-VSC_F(n_expired,		uint64_t, 0, 'i', "N expired objects")
-VSC_F(n_lru_nuked,		uint64_t, 0, 'i', "N LRU nuked objects")
-VSC_F(n_lru_moved,		uint64_t, 0, 'i', "N LRU moved objects")
+VSC_F(n_expired,		uint64_t, 0, 'i', "N expired objects", "")
+VSC_F(n_lru_nuked,		uint64_t, 0, 'i', "N LRU nuked objects", "")
+VSC_F(n_lru_moved,		uint64_t, 0, 'i', "N LRU moved objects", "")
 
-VSC_F(losthdr,		uint64_t, 0, 'a', "HTTP header overflows")
+VSC_F(losthdr,		uint64_t, 0, 'a', "HTTP header overflows", "")
 
-VSC_F(n_objsendfile,	uint64_t, 0, 'a', "Objects sent with sendfile")
-VSC_F(n_objwrite,		uint64_t, 0, 'a', "Objects sent with write")
+VSC_F(n_objsendfile,	uint64_t, 0, 'a', "Objects sent with sendfile", "")
+VSC_F(n_objwrite,		uint64_t, 0, 'a', "Objects sent with write", "")
 VSC_F(n_objoverflow,	uint64_t, 1, 'a',
-					"Objects overflowing workspace")
-
-VSC_F(s_sess,		uint64_t, 1, 'a', "Total Sessions")
-VSC_F(s_req,		uint64_t, 1, 'a', "Total Requests")
-VSC_F(s_pipe,		uint64_t, 1, 'a', "Total pipe")
-VSC_F(s_pass,		uint64_t, 1, 'a', "Total pass")
-VSC_F(s_fetch,		uint64_t, 1, 'a', "Total fetch")
-VSC_F(s_hdrbytes,		uint64_t, 1, 'a', "Total header bytes")
-VSC_F(s_bodybytes,		uint64_t, 1, 'a', "Total body bytes")
-
-VSC_F(sess_closed,		uint64_t, 1, 'a', "Session Closed")
-VSC_F(sess_pipeline,	uint64_t, 1, 'a', "Session Pipeline")
-VSC_F(sess_readahead,	uint64_t, 1, 'a', "Session Read Ahead")
-VSC_F(sess_linger,		uint64_t, 1, 'a', "Session Linger")
-VSC_F(sess_herd,		uint64_t, 1, 'a', "Session herd")
-
-VSC_F(shm_records,		uint64_t, 0, 'a', "SHM records")
-VSC_F(shm_writes,		uint64_t, 0, 'a', "SHM writes")
-VSC_F(shm_flushes,		uint64_t, 0, 'a', "SHM flushes due to overflow")
-VSC_F(shm_cont,		uint64_t, 0, 'a', "SHM MTX contention")
-VSC_F(shm_cycles,		uint64_t, 0, 'a', "SHM cycles through buffer")
-
-VSC_F(sms_nreq,		uint64_t, 0, 'a', "SMS allocator requests")
-VSC_F(sms_nobj,		uint64_t, 0, 'i', "SMS outstanding allocations")
-VSC_F(sms_nbytes,		uint64_t, 0, 'i', "SMS outstanding bytes")
-VSC_F(sms_balloc,		uint64_t, 0, 'i', "SMS bytes allocated")
-VSC_F(sms_bfree,		uint64_t, 0, 'i', "SMS bytes freed")
-
-VSC_F(backend_req,		uint64_t, 0, 'a', "Backend requests made")
-
-VSC_F(n_vcl,		uint64_t, 0, 'a', "N vcl total")
-VSC_F(n_vcl_avail,		uint64_t, 0, 'a', "N vcl available")
-VSC_F(n_vcl_discard,	uint64_t, 0, 'a', "N vcl discarded")
-
-VSC_F(n_ban,		uint64_t, 0, 'i', "N total active bans")
-VSC_F(n_ban_add,		uint64_t, 0, 'a', "N new bans added")
-VSC_F(n_ban_retire,	uint64_t, 0, 'a', "N old bans deleted")
-VSC_F(n_ban_obj_test,	uint64_t, 0, 'a', "N objects tested")
-VSC_F(n_ban_re_test,	uint64_t, 0, 'a', "N regexps tested against")
-VSC_F(n_ban_dups,	uint64_t, 0, 'a', "N duplicate bans removed")
-
-VSC_F(hcb_nolock,		uint64_t, 0, 'a', "HCB Lookups without lock")
-VSC_F(hcb_lock,		uint64_t, 0, 'a', "HCB Lookups with lock")
-VSC_F(hcb_insert,		uint64_t, 0, 'a', "HCB Inserts")
-
-VSC_F(esi_errors,		uint64_t, 0, 'a', "ESI parse errors (unlock)")
-VSC_F(esi_warnings,		uint64_t, 0, 'a', "ESI parse warnings (unlock)")
-VSC_F(accept_fail,		uint64_t, 0, 'a', "Accept failures")
-VSC_F(client_drop_late,	uint64_t, 0, 'a', "Connection dropped late")
-VSC_F(uptime,		uint64_t, 0, 'a', "Client uptime")
-
-VSC_F(dir_dns_lookups,	uint64_t, 0, 'a', "DNS director lookups")
-VSC_F(dir_dns_failed,	uint64_t, 0, 'a', "DNS director failed lookups")
-VSC_F(dir_dns_hit,		uint64_t, 0, 'a', "DNS director cached lookups hit")
-VSC_F(dir_dns_cache_full,	uint64_t, 0, 'a', "DNS director full dnscache")
-
-VSC_F(vmods,		uint64_t, 0, 'i', "Loaded VMODs")
-
-VSC_F(n_gzip,			uint64_t, 0, 'a', "Gzip operations")
-VSC_F(n_gunzip,			uint64_t, 0, 'a', "Gunzip operations")
+					"Objects overflowing workspace", "")
+
+VSC_F(s_sess,		uint64_t, 1, 'a', "Total Sessions", "")
+VSC_F(s_req,		uint64_t, 1, 'a', "Total Requests", "")
+VSC_F(s_pipe,		uint64_t, 1, 'a', "Total pipe", "")
+VSC_F(s_pass,		uint64_t, 1, 'a', "Total pass", "")
+VSC_F(s_fetch,		uint64_t, 1, 'a', "Total fetch", "")
+VSC_F(s_hdrbytes,		uint64_t, 1, 'a', "Total header bytes", "")
+VSC_F(s_bodybytes,		uint64_t, 1, 'a', "Total body bytes", "")
+
+VSC_F(sess_closed,		uint64_t, 1, 'a', "Session Closed", "")
+VSC_F(sess_pipeline,	uint64_t, 1, 'a', "Session Pipeline", "")
+VSC_F(sess_readahead,	uint64_t, 1, 'a', "Session Read Ahead", "")
+VSC_F(sess_linger,		uint64_t, 1, 'a', "Session Linger", "")
+VSC_F(sess_herd,		uint64_t, 1, 'a', "Session herd", "")
+
+VSC_F(shm_records,		uint64_t, 0, 'a', "SHM records", "")
+VSC_F(shm_writes,		uint64_t, 0, 'a', "SHM writes", "")
+VSC_F(shm_flushes,		uint64_t, 0, 'a', "SHM flushes due to overflow", "")
+VSC_F(shm_cont,		uint64_t, 0, 'a', "SHM MTX contention", "")
+VSC_F(shm_cycles,		uint64_t, 0, 'a', "SHM cycles through buffer", "")
+
+VSC_F(sms_nreq,		uint64_t, 0, 'a', "SMS allocator requests", "")
+VSC_F(sms_nobj,		uint64_t, 0, 'i', "SMS outstanding allocations", "")
+VSC_F(sms_nbytes,		uint64_t, 0, 'i', "SMS outstanding bytes", "")
+VSC_F(sms_balloc,		uint64_t, 0, 'i', "SMS bytes allocated", "")
+VSC_F(sms_bfree,		uint64_t, 0, 'i', "SMS bytes freed", "")
+
+VSC_F(backend_req,		uint64_t, 0, 'a', "Backend requests made", "")
+
+VSC_F(n_vcl,		uint64_t, 0, 'a', "N vcl total", "")
+VSC_F(n_vcl_avail,		uint64_t, 0, 'a', "N vcl available", "")
+VSC_F(n_vcl_discard,	uint64_t, 0, 'a', "N vcl discarded", "")
+
+VSC_F(n_ban,		uint64_t, 0, 'i', "N total active bans", "")
+VSC_F(n_ban_add,		uint64_t, 0, 'a', "N new bans added", "")
+VSC_F(n_ban_retire,	uint64_t, 0, 'a', "N old bans deleted", "")
+VSC_F(n_ban_obj_test,	uint64_t, 0, 'a', "N objects tested", "")
+VSC_F(n_ban_re_test,	uint64_t, 0, 'a', "N regexps tested against", "")
+VSC_F(n_ban_dups,	uint64_t, 0, 'a', "N duplicate bans removed", "")
+
+VSC_F(hcb_nolock,		uint64_t, 0, 'a', "HCB Lookups without lock", "")
+VSC_F(hcb_lock,		uint64_t, 0, 'a', "HCB Lookups with lock", "")
+VSC_F(hcb_insert,		uint64_t, 0, 'a', "HCB Inserts", "")
+
+VSC_F(esi_errors,		uint64_t, 0, 'a', "ESI parse errors (unlock)", "")
+VSC_F(esi_warnings,		uint64_t, 0, 'a', "ESI parse warnings (unlock)", "")
+VSC_F(accept_fail,		uint64_t, 0, 'a', "Accept failures", "")
+VSC_F(client_drop_late,	uint64_t, 0, 'a', "Connection dropped late", "")
+VSC_F(uptime,		uint64_t, 0, 'a', "Client uptime", "")
+
+VSC_F(dir_dns_lookups,	uint64_t, 0, 'a', "DNS director lookups", "")
+VSC_F(dir_dns_failed,	uint64_t, 0, 'a', "DNS director failed lookups", "")
+VSC_F(dir_dns_hit,		uint64_t, 0, 'a', "DNS director cached lookups hit", "")
+VSC_F(dir_dns_cache_full,	uint64_t, 0, 'a', "DNS director full dnscache", "")
+
+VSC_F(vmods,		uint64_t, 0, 'i', "Loaded VMODs", "")
+
+VSC_F(n_gzip,			uint64_t, 0, 'a', "Gzip operations", "")
+VSC_F(n_gunzip,			uint64_t, 0, 'a', "Gunzip operations", "")
 
 #endif
 
@@ -174,10 +199,10 @@ VSC_F(n_gunzip,			uint64_t, 0, 'a', "Gunzip operations")
 
 #ifdef VSC_DO_LCK
 
-VSC_F(creat,		uint64_t, 0, 'a', "Created locks")
-VSC_F(destroy,		uint64_t, 0, 'a', "Destroyed locks")
-VSC_F(locks,		uint64_t, 0, 'a', "Lock Operations")
-VSC_F(colls,		uint64_t, 0, 'a', "Collisions")
+VSC_F(creat,		uint64_t, 0, 'a', "Created locks", "")
+VSC_F(destroy,		uint64_t, 0, 'a', "Destroyed locks", "")
+VSC_F(locks,		uint64_t, 0, 'a', "Lock Operations", "")
+VSC_F(colls,		uint64_t, 0, 'a', "Collisions", "")
 
 #endif
 
@@ -186,13 +211,13 @@ VSC_F(colls,		uint64_t, 0, 'a', "Collisions")
  */
 
 #if defined(VSC_DO_SMA) || defined (VSC_DO_SMF)
-VSC_F(c_req,		uint64_t, 0, 'a', "Allocator requests")
-VSC_F(c_fail,		uint64_t, 0, 'a', "Allocator failures")
-VSC_F(c_bytes,		uint64_t, 0, 'a', "Bytes allocated")
-VSC_F(c_freed,		uint64_t, 0, 'a', "Bytes freed")
-VSC_F(g_alloc,		uint64_t, 0, 'i', "Allocations outstanding")
-VSC_F(g_bytes,		uint64_t, 0, 'i', "Bytes outstanding")
-VSC_F(g_space,		uint64_t, 0, 'i', "Bytes available")
+VSC_F(c_req,		uint64_t, 0, 'a', "Allocator requests", "")
+VSC_F(c_fail,		uint64_t, 0, 'a', "Allocator failures", "")
+VSC_F(c_bytes,		uint64_t, 0, 'a', "Bytes allocated", "")
+VSC_F(c_freed,		uint64_t, 0, 'a', "Bytes freed", "")
+VSC_F(g_alloc,		uint64_t, 0, 'i', "Allocations outstanding", "")
+VSC_F(g_bytes,		uint64_t, 0, 'i', "Bytes outstanding", "")
+VSC_F(g_space,		uint64_t, 0, 'i', "Bytes available", "")
 #endif
 
 
@@ -205,17 +230,17 @@ VSC_F(g_space,		uint64_t, 0, 'i', "Bytes available")
 /**********************************************************************/
 
 #ifdef VSC_DO_SMF
-VSC_F(g_smf,			uint64_t, 0, 'i', "N struct smf")
-VSC_F(g_smf_frag,		uint64_t, 0, 'i', "N small free smf")
-VSC_F(g_smf_large,		uint64_t, 0, 'i', "N large free smf")
+VSC_F(g_smf,			uint64_t, 0, 'i', "N struct smf", "")
+VSC_F(g_smf_frag,		uint64_t, 0, 'i', "N small free smf", "")
+VSC_F(g_smf_large,		uint64_t, 0, 'i', "N large free smf", "")
 #endif
 
 /**********************************************************************/
 
 #ifdef VSC_DO_VBE
 
-VSC_F(vcls,			uint64_t, 0, 'i', "VCL references")
-VSC_F(happy,		uint64_t, 0, 'b', "Happy health probes")
+VSC_F(vcls,			uint64_t, 0, 'i', "VCL references", "")
+VSC_F(happy,		uint64_t, 0, 'b', "Happy health probes", "")
 
 #endif
 
diff --git a/lib/libvarnishapi/vsc.c b/lib/libvarnishapi/vsc.c
index fc18cb3..25a1948 100644
--- a/lib/libvarnishapi/vsc.c
+++ b/lib/libvarnishapi/vsc.c
@@ -295,7 +295,7 @@ iter_call(const struct vsc *vsc, VSC_iter_f *func, void *priv,
 		sp.class = t;						\
 		sp.ident = sha->ident;
 
-#define VSC_F(nn,tt,ll,ff,dd)						\
+#define VSC_F(nn,tt,ll,ff,dd,ee)					\
 		sp.name = #nn;						\
 		sp.fmt = #tt;						\
 		sp.flag = ff;						\
@@ -331,7 +331,7 @@ VSC_Iter(struct VSM_data *vd, VSC_iter_f *func, void *priv)
 		if (strcmp(sha->class, VSC_CLASS))
 			continue;
 		/*lint -save -e525 -e539 */
-#define VSC_F(a,b,c,d,e)
+#define VSC_F(n,t,l,f,d,e)
 #define VSC_DONE(a,b,c)
 #define VSC_DO(U,l,t)						\
 		if (!strcmp(sha->type, t)) {			\



From geoff at varnish-cache.org  Mon Jan  9 20:52:18 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:18 +0100
Subject: [experimental-ims] 552f71c Move the waiters into their own
	subdirectory where VMODs won't see them.
Message-ID: <E1RkMCY-0007ge-M5@project.varnish-software.com>

commit 552f71c17ee8fb8d745869d1fce65e8f9521b629
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Oct 12 16:03:18 2011 +0000

    Move the waiters into their own subdirectory where VMODs won't see them.

diff --git a/bin/varnishd/Makefile.am b/bin/varnishd/Makefile.am
index 5260427..cdec084 100644
--- a/bin/varnishd/Makefile.am
+++ b/bin/varnishd/Makefile.am
@@ -44,11 +44,11 @@ varnishd_SOURCES = \
 	cache_vrt_re.c \
 	cache_vrt_var.c \
 	cache_vrt_vmod.c \
-	cache_waiter.c \
-	cache_waiter_epoll.c \
-	cache_waiter_kqueue.c \
-	cache_waiter_poll.c \
-	cache_waiter_ports.c \
+	waiter/cache_waiter.c \
+	waiter/cache_waiter_epoll.c \
+	waiter/cache_waiter_kqueue.c \
+	waiter/cache_waiter_poll.c \
+	waiter/cache_waiter_ports.c \
 	cache_wrk.c \
 	cache_wrw.c \
 	cache_ws.c \
@@ -81,7 +81,7 @@ noinst_HEADERS = \
 	cache.h \
 	cache_backend.h \
 	cache_esi.h \
-	cache_waiter.h \
+	waiter/cache_waiter.h \
 	common.h \
 	default_vcl.h \
 	hash_slinger.h \
diff --git a/bin/varnishd/cache_main.c b/bin/varnishd/cache_main.c
index 6e62fd2..2d00ee2 100644
--- a/bin/varnishd/cache_main.c
+++ b/bin/varnishd/cache_main.c
@@ -34,7 +34,7 @@
 
 #include "cache.h"
 
-#include "cache_waiter.h"
+#include "waiter/cache_waiter.h"
 #include "hash_slinger.h"
 
 /*--------------------------------------------------------------------
diff --git a/bin/varnishd/cache_panic.c b/bin/varnishd/cache_panic.c
index 482708d..76ea866 100644
--- a/bin/varnishd/cache_panic.c
+++ b/bin/varnishd/cache_panic.c
@@ -43,7 +43,7 @@
 #include "vapi/vsm_int.h"
 
 #include "cache_backend.h"
-#include "cache_waiter.h"
+#include "waiter/cache_waiter.h"
 #include "libvcl.h"
 #include "vcl.h"
 
diff --git a/bin/varnishd/cache_pool.c b/bin/varnishd/cache_pool.c
index 00eaef5..55ad333 100644
--- a/bin/varnishd/cache_pool.c
+++ b/bin/varnishd/cache_pool.c
@@ -47,7 +47,7 @@
 
 #include "cache.h"
 
-#include "cache_waiter.h"
+#include "waiter/cache_waiter.h"
 #include "hash_slinger.h"
 #include "vtcp.h"
 #include "vtim.h"
diff --git a/bin/varnishd/cache_session.c b/bin/varnishd/cache_session.c
index d55bae0..5db418c 100644
--- a/bin/varnishd/cache_session.c
+++ b/bin/varnishd/cache_session.c
@@ -40,7 +40,7 @@
 
 #include "cache.h"
 
-#include "cache_waiter.h"
+#include "waiter/cache_waiter.h"
 
 /*--------------------------------------------------------------------*/
 
diff --git a/bin/varnishd/cache_waiter.c b/bin/varnishd/cache_waiter.c
deleted file mode 100644
index 9c30edf..0000000
--- a/bin/varnishd/cache_waiter.c
+++ /dev/null
@@ -1,110 +0,0 @@
-/*-
- * Copyright (c) 2006 Verdens Gang AS
- * Copyright (c) 2006-2011 Varnish Software AS
- * All rights reserved.
- *
- * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- *
- */
-
-#include "config.h"
-
-#include "cache.h"
-
-#include "cache_waiter.h"
-#include "vcli.h"
-#include "vcli_priv.h"
-
-static const struct waiter * const vca_waiters[] = {
-    #if defined(HAVE_KQUEUE)
-	&waiter_kqueue,
-    #endif
-    #if defined(HAVE_EPOLL_CTL)
-	&waiter_epoll,
-    #endif
-    #if defined(HAVE_PORT_CREATE)
-	&waiter_ports,
-    #endif
-	&waiter_poll,
-	NULL,
-};
-
-struct waiter const * waiter;
-
-const char *
-WAIT_GetName(void)
-{
-
-	if (waiter != NULL)
-		return (waiter->name);
-	else
-		return ("no_waiter");
-}
-
-void
-WAIT_tweak_waiter(struct cli *cli, const char *arg)
-{
-	int i;
-
-	ASSERT_MGT();
-
-	if (arg == NULL) {
-		if (waiter == NULL)
-			VCLI_Out(cli, "default");
-		else
-			VCLI_Out(cli, "%s", waiter->name);
-
-		VCLI_Out(cli, " (");
-		for (i = 0; vca_waiters[i] != NULL; i++)
-			VCLI_Out(cli, "%s%s", i == 0 ? "" : ", ",
-			    vca_waiters[i]->name);
-		VCLI_Out(cli, ")");
-		return;
-	}
-	if (!strcmp(arg, "default")) {
-		waiter = NULL;
-		return;
-	}
-	for (i = 0; vca_waiters[i]; i++) {
-		if (!strcmp(arg, vca_waiters[i]->name)) {
-			waiter = vca_waiters[i];
-			return;
-		}
-	}
-	VCLI_Out(cli, "Unknown waiter");
-	VCLI_SetResult(cli, CLIS_PARAM);
-}
-
-void
-WAIT_Init(void)
-{
-
-	if (waiter == NULL)
-		waiter = vca_waiters[0];
-
-	AN(waiter);
-	AN(waiter->name);
-	AN(waiter->init);
-	AN(waiter->pass);
-}
diff --git a/bin/varnishd/cache_waiter.h b/bin/varnishd/cache_waiter.h
deleted file mode 100644
index 28bc39d..0000000
--- a/bin/varnishd/cache_waiter.h
+++ /dev/null
@@ -1,64 +0,0 @@
-/*-
- * Copyright (c) 2006 Verdens Gang AS
- * Copyright (c) 2006-2009 Varnish Software AS
- * All rights reserved.
- *
- * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- *
- */
-
-struct sess;
-
-typedef void* waiter_init_f(void);
-typedef void waiter_pass_f(void *priv, const struct sess *);
-
-struct waiter {
-	const char		*name;
-	waiter_init_f		*init;
-	waiter_pass_f		*pass;
-};
-
-extern struct waiter const * waiter;
-
-#if defined(HAVE_EPOLL_CTL)
-extern const struct waiter waiter_epoll;
-#endif
-
-#if defined(HAVE_KQUEUE)
-extern const struct waiter waiter_kqueue;
-#endif
-
-#if defined(HAVE_PORT_CREATE)
-extern const struct waiter waiter_ports;
-#endif
-
-
-/* cache_session.c */
-void SES_Handle(struct sess *sp, int status);
-
-/* cache_waiter.c */
-extern const struct waiter waiter_poll;
-const char *WAIT_GetName(void);
-void WAIT_tweak_waiter(struct cli *cli, const char *arg);
-void WAIT_Init(void);
diff --git a/bin/varnishd/cache_waiter_epoll.c b/bin/varnishd/cache_waiter_epoll.c
deleted file mode 100644
index 301fcf2..0000000
--- a/bin/varnishd/cache_waiter_epoll.c
+++ /dev/null
@@ -1,276 +0,0 @@
-/*-
- * Copyright (c) 2006 Verdens Gang AS
- * Copyright (c) 2006-2010 Varnish Software AS
- * All rights reserved.
- *
- * Author: Rogerio Carvalho Schneider <stockrt at gmail.com>
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- *
- * XXX: We need to pass sessions back into the event engine when they are
- * reused.  Not sure what the most efficient way is for that.  For now
- * write the session pointer to a pipe which the event engine monitors.
- */
-
-#include "config.h"
-
-#if defined(HAVE_EPOLL_CTL)
-
-#include <sys/epoll.h>
-
-#include <fcntl.h>
-#include <stdlib.h>
-
-#include "cache.h"
-
-#include "cache_waiter.h"
-#include "vtim.h"
-
-#ifndef EPOLLRDHUP
-#  define EPOLLRDHUP 0
-#endif
-
-#define NEEV	100
-
-struct vwe {
-	unsigned		magic;
-#define VWE_MAGIC		0x6bd73424
-
-	pthread_t		epoll_thread;
-	pthread_t		timer_thread;
-	int			epfd;
-
-	VTAILQ_HEAD(,sess)	sesshead;
-	int			pipes[2];
-	int			timer_pipes[2];
-};
-
-static void
-vwe_modadd(struct vwe *vwe, int fd, void *data, short arm)
-{
-
-	/* XXX: EPOLLET (edge triggered) can cause rather Bad Things to
-	 * XXX: happen: If NEEV+1 threads get stuck in write(), all threads
-	 * XXX: will hang. See #644.
-	 */
-	assert(fd >= 0);
-	if (data == vwe->pipes || data == vwe->timer_pipes) {
-		struct epoll_event ev = {
-		    EPOLLIN | EPOLLPRI , { data }
-		};
-		AZ(epoll_ctl(vwe->epfd, arm, fd, &ev));
-	} else {
-		struct sess *sp = (struct sess *)data;
-		CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
-		sp->ev.data.ptr = data;
-		sp->ev.events = EPOLLIN | EPOLLPRI | EPOLLONESHOT | EPOLLRDHUP;
-		AZ(epoll_ctl(vwe->epfd, arm, fd, &sp->ev));
-	}
-}
-
-static void
-vwe_cond_modadd(struct vwe *vwe, int fd, void *data)
-{
-	struct sess *sp = (struct sess *)data;
-
-	assert(fd >= 0);
-	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
-	if (sp->ev.data.ptr)
-		AZ(epoll_ctl(vwe->epfd, EPOLL_CTL_MOD, fd, &sp->ev));
-	else {
-		sp->ev.data.ptr = data;
-		sp->ev.events = EPOLLIN | EPOLLPRI | EPOLLONESHOT | EPOLLRDHUP;
-		AZ(epoll_ctl(vwe->epfd, EPOLL_CTL_ADD, fd, &sp->ev));
-	}
-}
-
-static void
-vwe_eev(struct vwe *vwe, const struct epoll_event *ep)
-{
-	struct sess *ss[NEEV], *sp;
-	int i, j;
-
-	AN(ep->data.ptr);
-	if (ep->data.ptr == vwe->pipes) {
-		if (ep->events & EPOLLIN || ep->events & EPOLLPRI) {
-			j = 0;
-			i = read(vwe->pipes[0], ss, sizeof ss);
-			if (i == -1 && errno == EAGAIN)
-				return;
-			while (i >= sizeof ss[0]) {
-				CHECK_OBJ_NOTNULL(ss[j], SESS_MAGIC);
-				assert(ss[j]->fd >= 0);
-				AZ(ss[j]->obj);
-				VTAILQ_INSERT_TAIL(&vwe->sesshead, ss[j], list);
-				vwe_cond_modadd(vwe, ss[j]->fd, ss[j]);
-				j++;
-				i -= sizeof ss[0];
-			}
-			assert(i == 0);
-		}
-	} else {
-		CAST_OBJ_NOTNULL(sp, ep->data.ptr, SESS_MAGIC);
-		if (ep->events & EPOLLIN || ep->events & EPOLLPRI) {
-			i = HTC_Rx(sp->htc);
-			if (i == 0) {
-				vwe_modadd(vwe, sp->fd, sp, EPOLL_CTL_MOD);
-				return;	/* more needed */
-			}
-			VTAILQ_REMOVE(&vwe->sesshead, sp, list);
-			SES_Handle(sp, i);
-		} else if (ep->events & EPOLLERR) {
-			VTAILQ_REMOVE(&vwe->sesshead, sp, list);
-			SES_Delete(sp, "ERR");
-		} else if (ep->events & EPOLLHUP) {
-			VTAILQ_REMOVE(&vwe->sesshead, sp, list);
-			SES_Delete(sp, "HUP");
-		} else if (ep->events & EPOLLRDHUP) {
-			VTAILQ_REMOVE(&vwe->sesshead, sp, list);
-			SES_Delete(sp, "RHUP");
-		}
-	}
-}
-
-/*--------------------------------------------------------------------*/
-
-static void *
-vwe_thread(void *priv)
-{
-	struct epoll_event ev[NEEV], *ep;
-	struct sess *sp;
-	char junk;
-	double deadline;
-	int dotimer, i, n;
-	struct vwe *vwe;
-
-	CAST_OBJ_NOTNULL(vwe, priv, VWE_MAGIC);
-
-	THR_SetName("cache-epoll");
-
-	vwe->epfd = epoll_create(1);
-	assert(vwe->epfd >= 0);
-
-	vwe_modadd(vwe, vwe->pipes[0], vwe->pipes, EPOLL_CTL_ADD);
-	vwe_modadd(vwe, vwe->timer_pipes[0], vwe->timer_pipes, EPOLL_CTL_ADD);
-
-	while (1) {
-		dotimer = 0;
-		n = epoll_wait(vwe->epfd, ev, NEEV, -1);
-		for (ep = ev, i = 0; i < n; i++, ep++) {
-			if (ep->data.ptr == vwe->timer_pipes &&
-			    (ep->events == EPOLLIN || ep->events == EPOLLPRI))
-			{
-				assert(read(vwe->timer_pipes[0], &junk, 1));
-				dotimer = 1;
-			} else
-				vwe_eev(vwe, ep);
-		}
-		if (!dotimer)
-			continue;
-
-		/* check for timeouts */
-		deadline = VTIM_real() - params->sess_timeout;
-		for (;;) {
-			sp = VTAILQ_FIRST(&vwe->sesshead);
-			if (sp == NULL)
-				break;
-			if (sp->t_open > deadline)
-				break;
-			VTAILQ_REMOVE(&vwe->sesshead, sp, list);
-			// XXX: not yet VTCP_linger(sp->fd, 0);
-			SES_Delete(sp, "timeout");
-		}
-	}
-	return NULL;
-}
-
-/*--------------------------------------------------------------------*/
-
-static void *
-vwe_sess_timeout_ticker(void *priv)
-{
-	char ticker = 'R';
-	struct vwe *vwe;
-
-	CAST_OBJ_NOTNULL(vwe, priv, VWE_MAGIC);
-	THR_SetName("cache-epoll-sess_timeout_ticker");
-
-	while (1) {
-		/* ticking */
-		assert(write(vwe->timer_pipes[1], &ticker, 1));
-		VTIM_sleep(100 * 1e-3);
-	}
-	return NULL;
-}
-
-/*--------------------------------------------------------------------*/
-
-static void
-vwe_pass(void *priv, const struct sess *sp)
-{
-	struct vwe *vwe;
-
-	CAST_OBJ_NOTNULL(vwe, priv, VWE_MAGIC);
-	assert(sizeof sp == write(vwe->pipes[1], &sp, sizeof sp));
-}
-
-/*--------------------------------------------------------------------*/
-
-static void *
-vwe_init(void)
-{
-	int i;
-	struct vwe *vwe;
-
-	ALLOC_OBJ(vwe, VWE_MAGIC);
-	AN(vwe);
-	VTAILQ_INIT(&vwe->sesshead);
-	AZ(pipe(vwe->pipes));
-	AZ(pipe(vwe->timer_pipes));
-
-	i = fcntl(vwe->pipes[0], F_GETFL);
-	assert(i != -1);
-	i |= O_NONBLOCK;
-	i = fcntl(vwe->pipes[0], F_SETFL, i);
-	assert(i != -1);
-
-	i = fcntl(vwe->timer_pipes[0], F_GETFL);
-	assert(i != -1);
-	i |= O_NONBLOCK;
-	i = fcntl(vwe->timer_pipes[0], F_SETFL, i);
-	assert(i != -1);
-
-	AZ(pthread_create(&vwe->timer_thread,
-	    NULL, vwe_sess_timeout_ticker, vwe));
-	AZ(pthread_create(&vwe->epoll_thread, NULL, vwe_thread, vwe));
-	return(vwe);
-}
-
-/*--------------------------------------------------------------------*/
-
-const struct waiter waiter_epoll = {
-	.name =		"epoll",
-	.init =		vwe_init,
-	.pass =		vwe_pass,
-};
-
-#endif /* defined(HAVE_EPOLL_CTL) */
diff --git a/bin/varnishd/cache_waiter_kqueue.c b/bin/varnishd/cache_waiter_kqueue.c
deleted file mode 100644
index 8bee1c9..0000000
--- a/bin/varnishd/cache_waiter_kqueue.c
+++ /dev/null
@@ -1,246 +0,0 @@
-/*-
- * Copyright (c) 2006 Verdens Gang AS
- * Copyright (c) 2006-2009 Varnish Software AS
- * All rights reserved.
- *
- * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- *
- * XXX: We need to pass sessions back into the event engine when they are
- * reused.  Not sure what the most efficient way is for that.  For now
- * write the session pointer to a pipe which the event engine monitors.
- */
-
-#include "config.h"
-
-#if defined(HAVE_KQUEUE)
-
-#include <sys/types.h>
-#include <sys/event.h>
-
-#include <fcntl.h>
-#include <stdlib.h>
-#include <unistd.h>
-
-#include "cache.h"
-
-#include "cache_waiter.h"
-#include "vtim.h"
-
-#define NKEV	100
-
-struct vwk {
-	unsigned		magic;
-#define VWK_MAGIC		0x1cc2acc2
-	pthread_t		thread;
-	int			pipes[2];
-	int			kq;
-	struct kevent		ki[NKEV];
-	unsigned		nki;
-	VTAILQ_HEAD(,sess)	sesshead;
-};
-
-/*--------------------------------------------------------------------*/
-
-static void
-vwk_kq_flush(struct vwk *vwk)
-{
-	int i;
-
-	if (vwk->nki == 0)
-		return;
-	i = kevent(vwk->kq, vwk->ki, vwk->nki, NULL, 0, NULL);
-	assert(i == 0);
-	vwk->nki = 0;
-}
-
-static void
-vwk_kq_sess(struct vwk *vwk, struct sess *sp, short arm)
-{
-
-	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
-	assert(sp->fd >= 0);
-	DSL(0x04, SLT_Debug, sp->vsl_id, "KQ: EV_SET sp %p arm %x", sp, arm);
-	EV_SET(&vwk->ki[vwk->nki], sp->fd, EVFILT_READ, arm, 0, 0, sp);
-	if (++vwk->nki == NKEV)
-		vwk_kq_flush(vwk);
-}
-
-static void
-vwk_kev(struct vwk *vwk, const struct kevent *kp)
-{
-	int i, j;
-	struct sess *sp;
-	struct sess *ss[NKEV];
-
-	AN(kp->udata);
-	if (kp->udata == vwk->pipes) {
-		j = 0;
-		i = read(vwk->pipes[0], ss, sizeof ss);
-		if (i == -1 && errno == EAGAIN)
-			return;
-		while (i >= sizeof ss[0]) {
-			CHECK_OBJ_NOTNULL(ss[j], SESS_MAGIC);
-			assert(ss[j]->fd >= 0);
-			AZ(ss[j]->obj);
-			VTAILQ_INSERT_TAIL(&vwk->sesshead, ss[j], list);
-			vwk_kq_sess(vwk, ss[j], EV_ADD | EV_ONESHOT);
-			j++;
-			i -= sizeof ss[0];
-		}
-		assert(i == 0);
-		return;
-	}
-	CAST_OBJ_NOTNULL(sp, kp->udata, SESS_MAGIC);
-	DSL(0x04, SLT_Debug, sp->vsl_id, "KQ: sp %p kev data %lu flags 0x%x%s",
-	    sp, (unsigned long)kp->data, kp->flags,
-	    (kp->flags & EV_EOF) ? " EOF" : "");
-
-	assert((sp->vsl_id & VSL_IDENTMASK) == kp->ident);
-	assert((sp->vsl_id & VSL_IDENTMASK) == sp->fd);
-	if (kp->data > 0) {
-		i = HTC_Rx(sp->htc);
-		if (i == 0) {
-			vwk_kq_sess(vwk, sp, EV_ADD | EV_ONESHOT);
-			return;	/* more needed */
-		}
-		VTAILQ_REMOVE(&vwk->sesshead, sp, list);
-		SES_Handle(sp, i);
-		return;
-	} else if (kp->flags & EV_EOF) {
-		VTAILQ_REMOVE(&vwk->sesshead, sp, list);
-		SES_Delete(sp, "EOF");
-		return;
-	} else {
-		VSL(SLT_Debug, sp->vsl_id, "KQ: sp %p kev data %lu flags 0x%x%s",
-		    sp, (unsigned long)kp->data, kp->flags,
-		    (kp->flags & EV_EOF) ? " EOF" : "");
-	}
-}
-
-/*--------------------------------------------------------------------*/
-
-static void *
-vwk_thread(void *priv)
-{
-	struct vwk *vwk;
-	struct kevent ke[NKEV], *kp;
-	int j, n, dotimer;
-	double deadline;
-	struct sess *sp;
-
-	CAST_OBJ_NOTNULL(vwk, priv, VWK_MAGIC);
-	THR_SetName("cache-kqueue");
-
-	vwk->kq = kqueue();
-	assert(vwk->kq >= 0);
-
-	j = 0;
-	EV_SET(&ke[j], 0, EVFILT_TIMER, EV_ADD, 0, 100, NULL);
-	j++;
-	EV_SET(&ke[j], vwk->pipes[0], EVFILT_READ, EV_ADD, 0, 0, vwk->pipes);
-	j++;
-	AZ(kevent(vwk->kq, ke, j, NULL, 0, NULL));
-
-	vwk->nki = 0;
-	while (1) {
-		dotimer = 0;
-		n = kevent(vwk->kq, vwk->ki, vwk->nki, ke, NKEV, NULL);
-		assert(n >= 1 && n <= NKEV);
-		vwk->nki = 0;
-		for (kp = ke, j = 0; j < n; j++, kp++) {
-			if (kp->filter == EVFILT_TIMER) {
-				dotimer = 1;
-				continue;
-			}
-			assert(kp->filter == EVFILT_READ);
-			vwk_kev(vwk, kp);
-		}
-		if (!dotimer)
-			continue;
-		/*
-		 * Make sure we have no pending changes for the fd's
-		 * we are about to close, in case the accept(2) in the
-		 * other thread creates new fd's betwen our close and
-		 * the kevent(2) at the top of this loop, the kernel
-		 * would not know we meant "the old fd of this number".
-		 */
-		vwk_kq_flush(vwk);
-		deadline = VTIM_real() - params->sess_timeout;
-		for (;;) {
-			sp = VTAILQ_FIRST(&vwk->sesshead);
-			if (sp == NULL)
-				break;
-			if (sp->t_open > deadline)
-				break;
-			VTAILQ_REMOVE(&vwk->sesshead, sp, list);
-			// XXX: not yet (void)VTCP_linger(sp->fd, 0);
-			SES_Delete(sp, "timeout");
-		}
-	}
-}
-
-/*--------------------------------------------------------------------*/
-
-static void
-vwk_pass(void *priv, const struct sess *sp)
-{
-	struct vwk *vwk;
-
-	CAST_OBJ_NOTNULL(vwk, priv, VWK_MAGIC);
-	assert(sizeof sp == write(vwk->pipes[1], &sp, sizeof sp));
-}
-
-/*--------------------------------------------------------------------*/
-
-static void *
-vwk_init(void)
-{
-	int i;
-	struct vwk *vwk;
-
-	ALLOC_OBJ(vwk, VWK_MAGIC);
-	AN(vwk);
-
-	VTAILQ_INIT(&vwk->sesshead);
-	AZ(pipe(vwk->pipes));
-
-	i = fcntl(vwk->pipes[0], F_GETFL);
-	assert(i != -1);
-	i |= O_NONBLOCK;
-	i = fcntl(vwk->pipes[0], F_SETFL, i);
-	assert(i != -1);
-
-	AZ(pthread_create(&vwk->thread, NULL, vwk_thread, vwk));
-	return (vwk);
-}
-
-/*--------------------------------------------------------------------*/
-
-const struct waiter waiter_kqueue = {
-	.name =		"kqueue",
-	.init =		vwk_init,
-	.pass =		vwk_pass,
-};
-
-#endif /* defined(HAVE_KQUEUE) */
diff --git a/bin/varnishd/cache_waiter_poll.c b/bin/varnishd/cache_waiter_poll.c
deleted file mode 100644
index eb48870..0000000
--- a/bin/varnishd/cache_waiter_poll.c
+++ /dev/null
@@ -1,233 +0,0 @@
-/*-
- * Copyright (c) 2006 Verdens Gang AS
- * Copyright (c) 2006-2010 Varnish Software AS
- * All rights reserved.
- *
- * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- *
- */
-
-#include "config.h"
-
-#include <poll.h>
-#include <stdlib.h>
-
-#include "cache.h"
-
-#include "cache_waiter.h"
-#include "vtim.h"
-
-#define NEEV	128
-
-struct vwp {
-	unsigned		magic;
-#define VWP_MAGIC		0x4b2cc735
-	int			pipes[2];
-	pthread_t		poll_thread;
-	struct pollfd		*pollfd;
-	unsigned		npoll;
-	unsigned		hpoll;
-
-	VTAILQ_HEAD(,sess)	sesshead;
-};
-
-/*--------------------------------------------------------------------*/
-
-static void
-vwp_pollspace(struct vwp *vwp, unsigned fd)
-{
-	struct pollfd *newpollfd = vwp->pollfd;
-	unsigned newnpoll;
-
-	if (fd < vwp->npoll)
-		return;
-	newnpoll = vwp->npoll;
-	if (newnpoll == 0)
-		newnpoll = 1;
-	while (fd >= newnpoll)
-		newnpoll = newnpoll * 2;
-	VSL(SLT_Debug, 0, "Acceptor poll space increased to %u", newnpoll);
-	newpollfd = realloc(newpollfd, newnpoll * sizeof *newpollfd);
-	XXXAN(newpollfd);
-	memset(newpollfd + vwp->npoll, 0,
-	    (newnpoll - vwp->npoll) * sizeof *newpollfd);
-	vwp->pollfd = newpollfd;
-	while (vwp->npoll < newnpoll)
-		vwp->pollfd[vwp->npoll++].fd = -1;
-	assert(fd < vwp->npoll);
-}
-
-/*--------------------------------------------------------------------*/
-
-static void
-vwp_poll(struct vwp *vwp, int fd)
-{
-
-	assert(fd >= 0);
-	vwp_pollspace(vwp, (unsigned)fd);
-	assert(fd < vwp->npoll);
-
-	if (vwp->hpoll < fd)
-		vwp->hpoll = fd;
-
-	assert(vwp->pollfd[fd].fd == -1);
-	assert(vwp->pollfd[fd].events == 0);
-	assert(vwp->pollfd[fd].revents == 0);
-
-	vwp->pollfd[fd].fd = fd;
-	vwp->pollfd[fd].events = POLLIN;
-}
-
-static void
-vwp_unpoll(struct vwp *vwp, int fd)
-{
-
-	assert(fd >= 0);
-	assert(fd < vwp->npoll);
-	vwp_pollspace(vwp, (unsigned)fd);
-
-	assert(vwp->pollfd[fd].fd == fd);
-	assert(vwp->pollfd[fd].events == POLLIN);
-	assert(vwp->pollfd[fd].revents == 0);
-
-	vwp->pollfd[fd].fd = -1;
-	vwp->pollfd[fd].events = 0;
-}
-
-/*--------------------------------------------------------------------*/
-
-static void *
-vwp_main(void *priv)
-{
-	int v, v2;
-	struct vwp *vwp;
-	struct sess *ss[NEEV], *sp, *sp2;
-	double deadline;
-	int i, j, fd;
-
-	CAST_OBJ_NOTNULL(vwp, priv, VWP_MAGIC);
-	THR_SetName("cache-poll");
-
-	vwp_poll(vwp, vwp->pipes[0]);
-
-	while (1) {
-		assert(vwp->hpoll < vwp->npoll);
-		while (vwp->hpoll > 0 && vwp->pollfd[vwp->hpoll].fd == -1)
-			vwp->hpoll--;
-		assert(vwp->pipes[0] <= vwp->hpoll);
-		assert(vwp->pollfd[vwp->pipes[0]].fd == vwp->pipes[0]);
-		assert(vwp->pollfd[vwp->pipes[1]].fd == -1);
-		v = poll(vwp->pollfd, vwp->hpoll + 1, 100);
-		assert(v >= 0);
-		deadline = VTIM_real() - params->sess_timeout;
-		v2 = v;
-		VTAILQ_FOREACH_SAFE(sp, &vwp->sesshead, list, sp2) {
-			if (v != 0 && v2 == 0)
-				break;
-			CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
-			fd = sp->fd;
-			assert(fd >= 0);
-			assert(fd <= vwp->hpoll);
-			assert(fd < vwp->npoll);
-			assert(vwp->pollfd[fd].fd == fd);
-			if (vwp->pollfd[fd].revents) {
-				v2--;
-				i = HTC_Rx(sp->htc);
-				if (vwp->pollfd[fd].revents != POLLIN)
-					VSL(SLT_Debug, fd, "Poll: %x / %d",
-					    vwp->pollfd[fd].revents, i);
-				vwp->pollfd[fd].revents = 0;
-				VTAILQ_REMOVE(&vwp->sesshead, sp, list);
-				if (i == 0) {
-					/* Mov to front of list for speed */
-					VTAILQ_INSERT_HEAD(&vwp->sesshead, sp, list);
-				} else {
-					vwp_unpoll(vwp, fd);
-					SES_Handle(sp, i);
-				}
-			} else if (sp->t_open <= deadline) {
-				VTAILQ_REMOVE(&vwp->sesshead, sp, list);
-				vwp_unpoll(vwp, fd);
-				// XXX: not yet (void)VTCP_linger(sp->fd, 0);
-				SES_Delete(sp, "timeout");
-			}
-		}
-		if (v2 && vwp->pollfd[vwp->pipes[0]].revents) {
-
-			if (vwp->pollfd[vwp->pipes[0]].revents != POLLIN)
-				VSL(SLT_Debug, 0, "pipe.revents= 0x%x",
-				    vwp->pollfd[vwp->pipes[0]].revents);
-			assert(vwp->pollfd[vwp->pipes[0]].revents == POLLIN);
-			vwp->pollfd[vwp->pipes[0]].revents = 0;
-			v2--;
-			i = read(vwp->pipes[0], ss, sizeof ss);
-			assert(i >= 0);
-			assert(((unsigned)i % sizeof ss[0]) == 0);
-			for (j = 0; j * sizeof ss[0] < i; j++) {
-				CHECK_OBJ_NOTNULL(ss[j], SESS_MAGIC);
-				assert(ss[j]->fd >= 0);
-				VTAILQ_INSERT_TAIL(&vwp->sesshead, ss[j], list);
-				vwp_poll(vwp, ss[j]->fd);
-			}
-		}
-		assert(v2 == 0);
-	}
-	NEEDLESS_RETURN(NULL);
-}
-
-/*--------------------------------------------------------------------*/
-
-static void
-vwp_poll_pass(void *priv, const struct sess *sp)
-{
-	struct vwp *vwp;
-
-	CAST_OBJ_NOTNULL(vwp, priv, VWP_MAGIC);
-
-	assert(sizeof sp == write(vwp->pipes[1], &sp, sizeof sp));
-}
-
-/*--------------------------------------------------------------------*/
-
-static void *
-vwp_poll_init(void)
-{
-	struct vwp *vwp;
-
-	ALLOC_OBJ(vwp, VWP_MAGIC);
-	AN(vwp);
-	VTAILQ_INIT(&vwp->sesshead);
-	AZ(pipe(vwp->pipes));
-	vwp_pollspace(vwp, 256);
-	AZ(pthread_create(&vwp->poll_thread, NULL, vwp_main, vwp));
-	return (vwp);
-}
-
-/*--------------------------------------------------------------------*/
-
-const struct waiter waiter_poll = {
-	.name =		"poll",
-	.init =		vwp_poll_init,
-	.pass =		vwp_poll_pass,
-};
diff --git a/bin/varnishd/cache_waiter_ports.c b/bin/varnishd/cache_waiter_ports.c
deleted file mode 100644
index 821ebc3..0000000
--- a/bin/varnishd/cache_waiter_ports.c
+++ /dev/null
@@ -1,283 +0,0 @@
-/*-
- * Copyright (c) 2006 Verdens Gang AS
- * Copyright (c) 2006 Varnish Software AS
- * Copyright (c) 2007 OmniTI Computer Consulting, Inc.
- * Copyright (c) 2007 Theo Schlossnagle
- * Copyright (c) 2010 UPLEX, Nils Goroll
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- *
- */
-
-#include "config.h"
-
-#if defined(HAVE_PORT_CREATE)
-
-#include <sys/time.h>
-
-#include <math.h>
-#include <port.h>
-#include <stdio.h>
-#include <stdlib.h>
-
-#include "cache.h"
-
-#include "cache_waiter.h"
-#include "vtim.h"
-
-#define MAX_EVENTS 256
-
-struct vws {
-	unsigned		magic;
-#define VWS_MAGIC		0x0b771473
-	pthread_t		ports_thread;
-	int			dport;
-	VTAILQ_HEAD(,sess)	sesshead;
-};
-
-static inline void
-vws_add(struct vws *vws, int fd, void *data)
-{
-	/*
-	 * POLLIN should be all we need here
-	 *
-	 */
-	AZ(port_associate(vws->dport, PORT_SOURCE_FD, fd, POLLIN, data));
-}
-
-static inline void
-vws_del(struct vws *vws, int fd)
-{
-	port_dissociate(vws->dport, PORT_SOURCE_FD, fd);
-}
-
-static inline void
-vws_port_ev(struct vws *vws, port_event_t *ev) {
-	struct sess *sp;
-	if(ev->portev_source == PORT_SOURCE_USER) {
-		CAST_OBJ_NOTNULL(sp, ev->portev_user, SESS_MAGIC);
-		assert(sp->fd >= 0);
-		AZ(sp->obj);
-		VTAILQ_INSERT_TAIL(&vws->sesshead, sp, list);
-		vws_add(vws, sp->fd, sp);
-	} else {
-		int i;
-		assert(ev->portev_source == PORT_SOURCE_FD);
-		CAST_OBJ_NOTNULL(sp, ev->portev_user, SESS_MAGIC);
-		assert(sp->fd >= 0);
-		if(ev->portev_events & POLLERR) {
-			vws_del(vws, sp->fd);
-			VTAILQ_REMOVE(&vws->sesshead, sp, list);
-			SES_Delete(sp, "EOF");
-			return;
-		}
-		i = HTC_Rx(sp->htc);
-
-		if (i == 0) {
-			/* incomplete header, wait for more data */
-			vws_add(vws, sp->fd, sp);
-			return;
-		}
-
-		/*
-		 * note: the original man page for port_associate(3C) states:
-		 *
-		 *    When an event for a PORT_SOURCE_FD object is retrieved,
-		 *    the object no longer has an association with the port.
-		 *
-		 * This can be read along the lines of sparing the
-		 * port_dissociate after port_getn(), but in fact,
-		 * port_dissociate should be used
-		 *
-		 * Ref: http://opensolaris.org/jive/thread.jspa?threadID=129476&tstart=0
-		 */
-		vws_del(vws, sp->fd);
-		VTAILQ_REMOVE(&vws->sesshead, sp, list);
-
-		/* SES_Handle will also handle errors */
-		SES_Handle(sp, i);
-	}
-	return;
-}
-
-static void *
-vws_thread(void *priv)
-{
-	struct sess *sp;
-	struct vws *vws;
-
-	CAST_OBJ_NOTNULL(vws, priv, VWS_MAGIC);
-	/*
-	 * timeouts:
-	 *
-	 * min_ts : Minimum timeout for port_getn
-	 * min_t  : ^ equivalent in floating point representation
-	 *
-	 * max_ts : Maximum timeout for port_getn
-	 * max_t  : ^ equivalent in floating point representation
-	 *
-	 * with (nevents == 1), we should always choose the correct port_getn
-	 * timeout to check session timeouts, so max is just a safety measure
-	 * (if this implementation is correct, it could be set to an "infinte"
-	 *  value)
-	 *
-	 * with (nevents > 1), min and max define the acceptable range for
-	 * - additional latency of keep-alive connections and
-	 * - additional tolerance for handling session timeouts
-	 *
-	 */
-	static struct timespec min_ts = {0L,    100L /*ms*/  * 1000L /*us*/  * 1000L /*ns*/};
-	static double          min_t  = 0.1; /* 100    ms*/
-	static struct timespec max_ts = {1L, 0L};		/* 1 second */
-	static double	       max_t  = 1.0;			/* 1 second */
-
-	/* XXX: These should probably go in vws ? */
-	struct timespec ts;
-	struct timespec *timeout;
-
-	vws->dport = port_create();
-	assert(vws->dport >= 0);
-
-	timeout = &max_ts;
-
-	while (1) {
-		port_event_t ev[MAX_EVENTS];
-		int nevents, ei, ret;
-		double now, deadline;
-
-		/*
-		 * XXX Do we want to scale this up dynamically to increase
-		 *     efficiency in high throughput situations? - would need to
-		 *     start with one to keep latency low at any rate
-		 *
-		 *     Note: when increasing nevents, we must lower min_ts
-		 *	     and max_ts
-		 */
-		nevents = 1;
-
-		/*
-		 * see disucssion in
-		 * - https://issues.apache.org/bugzilla/show_bug.cgi?id=47645
-		 * - http://mail.opensolaris.org/pipermail/networking-discuss/2009-August/011979.html
-		 *
-		 * comment from apr/poll/unix/port.c :
-		 *
-		 * This confusing API can return an event at the same time
-		 * that it reports EINTR or ETIME.
-		 *
-		 */
-
-		ret = port_getn(vws->dport, ev, MAX_EVENTS, &nevents, timeout);
-
-		if (ret < 0)
-			assert((errno == EINTR) || (errno == ETIME));
-
-		for (ei = 0; ei < nevents; ei++)
-			vws_port_ev(vws, ev + ei);
-
-		/* check for timeouts */
-		now = VTIM_real();
-		deadline = now - params->sess_timeout;
-
-		/*
-		 * This loop assumes that the oldest sessions are always at the
-		 * beginning of the list (which is the case if we guarantee to
-		 * enqueue at the tail only
-		 *
-		 */
-
-		for (;;) {
-			sp = VTAILQ_FIRST(&vws->sesshead);
-			if (sp == NULL)
-				break;
-			if (sp->t_open > deadline) {
-				break;
-			}
-			VTAILQ_REMOVE(&vws->sesshead, sp, list);
-			if(sp->fd != -1) {
-				vws_del(vws, sp->fd);
-			}
-			SES_Delete(sp, "timeout");
-		}
-
-		/*
-		 * Calculate the timeout for the next get_portn
-		 */
-
-		if (sp) {
-			double tmo = (sp->t_open + params->sess_timeout) - now;
-
-			/* we should have removed all sps whose timeout has passed */
-			assert(tmo > 0.0);
-
-			if (tmo < min_t) {
-				timeout = &min_ts;
-			} else if (tmo > max_t) {
-				timeout = &max_ts;
-			} else {
-				ts = VTIM_timespec(tmo);
-				timeout = &ts;
-			}
-		} else {
-			timeout = &max_ts;
-		}
-	}
-}
-
-/*--------------------------------------------------------------------*/
-
-static void
-vws_pass(void *priv, const struct sess *sp)
-{
-	int r;
-	struct vws *vws;
-
-	CAST_OBJ_NOTNULL(vws, priv, VWS_MAGIC);
-	while((r = port_send(vws->dport, 0, TRUST_ME(sp))) == -1 &&
-		errno == EAGAIN);
-	AZ(r);
-}
-
-/*--------------------------------------------------------------------*/
-
-static void *
-vws_init(void)
-{
-	struct vws *vws;
-
-	ALLOC_OBJ(vws, VWS_MAGIC);
-	AN(vws);
-	VTAILQ_INIT(&vws->sesshead);
-	AZ(pthread_create(&vws->ports_thread, NULL, vws_thread, vws));
-	return (vws);
-}
-
-/*--------------------------------------------------------------------*/
-
-const struct waiter waiter_ports = {
-	.name =		"ports",
-	.init =		vws_init,
-	.pass =		vws_pass
-};
-
-#endif /* defined(HAVE_PORT_CREATE) */
diff --git a/bin/varnishd/flint.sh b/bin/varnishd/flint.sh
index 86d4ad0..357d280 100755
--- a/bin/varnishd/flint.sh
+++ b/bin/varnishd/flint.sh
@@ -17,6 +17,7 @@ flexelint \
 	-DVARNISH_STATE_DIR=\"foo\" \
 	*.c \
 	storage/*.c \
+	waiter/*.c \
 	../../lib/libvarnish/*.c \
 	../../lib/libvarnishcompat/execinfo.c \
 	../../lib/libvcl/*.c \
diff --git a/bin/varnishd/mgt_param.c b/bin/varnishd/mgt_param.c
index 9ded195..43749b6 100644
--- a/bin/varnishd/mgt_param.c
+++ b/bin/varnishd/mgt_param.c
@@ -39,7 +39,7 @@
 
 #include "mgt.h"
 
-#include "cache_waiter.h"
+#include "waiter/cache_waiter.h"
 #include "heritage.h"
 #include "vav.h"
 #include "vcli.h"
diff --git a/bin/varnishd/waiter/cache_waiter.c b/bin/varnishd/waiter/cache_waiter.c
new file mode 100644
index 0000000..e6654fd
--- /dev/null
+++ b/bin/varnishd/waiter/cache_waiter.c
@@ -0,0 +1,110 @@
+/*-
+ * Copyright (c) 2006 Verdens Gang AS
+ * Copyright (c) 2006-2011 Varnish Software AS
+ * All rights reserved.
+ *
+ * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ */
+
+#include "config.h"
+
+#include "cache.h"
+
+#include "waiter/cache_waiter.h"
+#include "vcli.h"
+#include "vcli_priv.h"
+
+static const struct waiter * const vca_waiters[] = {
+    #if defined(HAVE_KQUEUE)
+	&waiter_kqueue,
+    #endif
+    #if defined(HAVE_EPOLL_CTL)
+	&waiter_epoll,
+    #endif
+    #if defined(HAVE_PORT_CREATE)
+	&waiter_ports,
+    #endif
+	&waiter_poll,
+	NULL,
+};
+
+struct waiter const * waiter;
+
+const char *
+WAIT_GetName(void)
+{
+
+	if (waiter != NULL)
+		return (waiter->name);
+	else
+		return ("no_waiter");
+}
+
+void
+WAIT_tweak_waiter(struct cli *cli, const char *arg)
+{
+	int i;
+
+	ASSERT_MGT();
+
+	if (arg == NULL) {
+		if (waiter == NULL)
+			VCLI_Out(cli, "default");
+		else
+			VCLI_Out(cli, "%s", waiter->name);
+
+		VCLI_Out(cli, " (");
+		for (i = 0; vca_waiters[i] != NULL; i++)
+			VCLI_Out(cli, "%s%s", i == 0 ? "" : ", ",
+			    vca_waiters[i]->name);
+		VCLI_Out(cli, ")");
+		return;
+	}
+	if (!strcmp(arg, "default")) {
+		waiter = NULL;
+		return;
+	}
+	for (i = 0; vca_waiters[i]; i++) {
+		if (!strcmp(arg, vca_waiters[i]->name)) {
+			waiter = vca_waiters[i];
+			return;
+		}
+	}
+	VCLI_Out(cli, "Unknown waiter");
+	VCLI_SetResult(cli, CLIS_PARAM);
+}
+
+void
+WAIT_Init(void)
+{
+
+	if (waiter == NULL)
+		waiter = vca_waiters[0];
+
+	AN(waiter);
+	AN(waiter->name);
+	AN(waiter->init);
+	AN(waiter->pass);
+}
diff --git a/bin/varnishd/waiter/cache_waiter.h b/bin/varnishd/waiter/cache_waiter.h
new file mode 100644
index 0000000..28bc39d
--- /dev/null
+++ b/bin/varnishd/waiter/cache_waiter.h
@@ -0,0 +1,64 @@
+/*-
+ * Copyright (c) 2006 Verdens Gang AS
+ * Copyright (c) 2006-2009 Varnish Software AS
+ * All rights reserved.
+ *
+ * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ */
+
+struct sess;
+
+typedef void* waiter_init_f(void);
+typedef void waiter_pass_f(void *priv, const struct sess *);
+
+struct waiter {
+	const char		*name;
+	waiter_init_f		*init;
+	waiter_pass_f		*pass;
+};
+
+extern struct waiter const * waiter;
+
+#if defined(HAVE_EPOLL_CTL)
+extern const struct waiter waiter_epoll;
+#endif
+
+#if defined(HAVE_KQUEUE)
+extern const struct waiter waiter_kqueue;
+#endif
+
+#if defined(HAVE_PORT_CREATE)
+extern const struct waiter waiter_ports;
+#endif
+
+
+/* cache_session.c */
+void SES_Handle(struct sess *sp, int status);
+
+/* cache_waiter.c */
+extern const struct waiter waiter_poll;
+const char *WAIT_GetName(void);
+void WAIT_tweak_waiter(struct cli *cli, const char *arg);
+void WAIT_Init(void);
diff --git a/bin/varnishd/waiter/cache_waiter_epoll.c b/bin/varnishd/waiter/cache_waiter_epoll.c
new file mode 100644
index 0000000..e700676
--- /dev/null
+++ b/bin/varnishd/waiter/cache_waiter_epoll.c
@@ -0,0 +1,276 @@
+/*-
+ * Copyright (c) 2006 Verdens Gang AS
+ * Copyright (c) 2006-2010 Varnish Software AS
+ * All rights reserved.
+ *
+ * Author: Rogerio Carvalho Schneider <stockrt at gmail.com>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * XXX: We need to pass sessions back into the event engine when they are
+ * reused.  Not sure what the most efficient way is for that.  For now
+ * write the session pointer to a pipe which the event engine monitors.
+ */
+
+#include "config.h"
+
+#if defined(HAVE_EPOLL_CTL)
+
+#include <sys/epoll.h>
+
+#include <fcntl.h>
+#include <stdlib.h>
+
+#include "cache.h"
+
+#include "waiter/cache_waiter.h"
+#include "vtim.h"
+
+#ifndef EPOLLRDHUP
+#  define EPOLLRDHUP 0
+#endif
+
+#define NEEV	100
+
+struct vwe {
+	unsigned		magic;
+#define VWE_MAGIC		0x6bd73424
+
+	pthread_t		epoll_thread;
+	pthread_t		timer_thread;
+	int			epfd;
+
+	VTAILQ_HEAD(,sess)	sesshead;
+	int			pipes[2];
+	int			timer_pipes[2];
+};
+
+static void
+vwe_modadd(struct vwe *vwe, int fd, void *data, short arm)
+{
+
+	/* XXX: EPOLLET (edge triggered) can cause rather Bad Things to
+	 * XXX: happen: If NEEV+1 threads get stuck in write(), all threads
+	 * XXX: will hang. See #644.
+	 */
+	assert(fd >= 0);
+	if (data == vwe->pipes || data == vwe->timer_pipes) {
+		struct epoll_event ev = {
+		    EPOLLIN | EPOLLPRI , { data }
+		};
+		AZ(epoll_ctl(vwe->epfd, arm, fd, &ev));
+	} else {
+		struct sess *sp = (struct sess *)data;
+		CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+		sp->ev.data.ptr = data;
+		sp->ev.events = EPOLLIN | EPOLLPRI | EPOLLONESHOT | EPOLLRDHUP;
+		AZ(epoll_ctl(vwe->epfd, arm, fd, &sp->ev));
+	}
+}
+
+static void
+vwe_cond_modadd(struct vwe *vwe, int fd, void *data)
+{
+	struct sess *sp = (struct sess *)data;
+
+	assert(fd >= 0);
+	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+	if (sp->ev.data.ptr)
+		AZ(epoll_ctl(vwe->epfd, EPOLL_CTL_MOD, fd, &sp->ev));
+	else {
+		sp->ev.data.ptr = data;
+		sp->ev.events = EPOLLIN | EPOLLPRI | EPOLLONESHOT | EPOLLRDHUP;
+		AZ(epoll_ctl(vwe->epfd, EPOLL_CTL_ADD, fd, &sp->ev));
+	}
+}
+
+static void
+vwe_eev(struct vwe *vwe, const struct epoll_event *ep)
+{
+	struct sess *ss[NEEV], *sp;
+	int i, j;
+
+	AN(ep->data.ptr);
+	if (ep->data.ptr == vwe->pipes) {
+		if (ep->events & EPOLLIN || ep->events & EPOLLPRI) {
+			j = 0;
+			i = read(vwe->pipes[0], ss, sizeof ss);
+			if (i == -1 && errno == EAGAIN)
+				return;
+			while (i >= sizeof ss[0]) {
+				CHECK_OBJ_NOTNULL(ss[j], SESS_MAGIC);
+				assert(ss[j]->fd >= 0);
+				AZ(ss[j]->obj);
+				VTAILQ_INSERT_TAIL(&vwe->sesshead, ss[j], list);
+				vwe_cond_modadd(vwe, ss[j]->fd, ss[j]);
+				j++;
+				i -= sizeof ss[0];
+			}
+			assert(i == 0);
+		}
+	} else {
+		CAST_OBJ_NOTNULL(sp, ep->data.ptr, SESS_MAGIC);
+		if (ep->events & EPOLLIN || ep->events & EPOLLPRI) {
+			i = HTC_Rx(sp->htc);
+			if (i == 0) {
+				vwe_modadd(vwe, sp->fd, sp, EPOLL_CTL_MOD);
+				return;	/* more needed */
+			}
+			VTAILQ_REMOVE(&vwe->sesshead, sp, list);
+			SES_Handle(sp, i);
+		} else if (ep->events & EPOLLERR) {
+			VTAILQ_REMOVE(&vwe->sesshead, sp, list);
+			SES_Delete(sp, "ERR");
+		} else if (ep->events & EPOLLHUP) {
+			VTAILQ_REMOVE(&vwe->sesshead, sp, list);
+			SES_Delete(sp, "HUP");
+		} else if (ep->events & EPOLLRDHUP) {
+			VTAILQ_REMOVE(&vwe->sesshead, sp, list);
+			SES_Delete(sp, "RHUP");
+		}
+	}
+}
+
+/*--------------------------------------------------------------------*/
+
+static void *
+vwe_thread(void *priv)
+{
+	struct epoll_event ev[NEEV], *ep;
+	struct sess *sp;
+	char junk;
+	double deadline;
+	int dotimer, i, n;
+	struct vwe *vwe;
+
+	CAST_OBJ_NOTNULL(vwe, priv, VWE_MAGIC);
+
+	THR_SetName("cache-epoll");
+
+	vwe->epfd = epoll_create(1);
+	assert(vwe->epfd >= 0);
+
+	vwe_modadd(vwe, vwe->pipes[0], vwe->pipes, EPOLL_CTL_ADD);
+	vwe_modadd(vwe, vwe->timer_pipes[0], vwe->timer_pipes, EPOLL_CTL_ADD);
+
+	while (1) {
+		dotimer = 0;
+		n = epoll_wait(vwe->epfd, ev, NEEV, -1);
+		for (ep = ev, i = 0; i < n; i++, ep++) {
+			if (ep->data.ptr == vwe->timer_pipes &&
+			    (ep->events == EPOLLIN || ep->events == EPOLLPRI))
+			{
+				assert(read(vwe->timer_pipes[0], &junk, 1));
+				dotimer = 1;
+			} else
+				vwe_eev(vwe, ep);
+		}
+		if (!dotimer)
+			continue;
+
+		/* check for timeouts */
+		deadline = VTIM_real() - params->sess_timeout;
+		for (;;) {
+			sp = VTAILQ_FIRST(&vwe->sesshead);
+			if (sp == NULL)
+				break;
+			if (sp->t_open > deadline)
+				break;
+			VTAILQ_REMOVE(&vwe->sesshead, sp, list);
+			// XXX: not yet VTCP_linger(sp->fd, 0);
+			SES_Delete(sp, "timeout");
+		}
+	}
+	return NULL;
+}
+
+/*--------------------------------------------------------------------*/
+
+static void *
+vwe_sess_timeout_ticker(void *priv)
+{
+	char ticker = 'R';
+	struct vwe *vwe;
+
+	CAST_OBJ_NOTNULL(vwe, priv, VWE_MAGIC);
+	THR_SetName("cache-epoll-sess_timeout_ticker");
+
+	while (1) {
+		/* ticking */
+		assert(write(vwe->timer_pipes[1], &ticker, 1));
+		VTIM_sleep(100 * 1e-3);
+	}
+	return NULL;
+}
+
+/*--------------------------------------------------------------------*/
+
+static void
+vwe_pass(void *priv, const struct sess *sp)
+{
+	struct vwe *vwe;
+
+	CAST_OBJ_NOTNULL(vwe, priv, VWE_MAGIC);
+	assert(sizeof sp == write(vwe->pipes[1], &sp, sizeof sp));
+}
+
+/*--------------------------------------------------------------------*/
+
+static void *
+vwe_init(void)
+{
+	int i;
+	struct vwe *vwe;
+
+	ALLOC_OBJ(vwe, VWE_MAGIC);
+	AN(vwe);
+	VTAILQ_INIT(&vwe->sesshead);
+	AZ(pipe(vwe->pipes));
+	AZ(pipe(vwe->timer_pipes));
+
+	i = fcntl(vwe->pipes[0], F_GETFL);
+	assert(i != -1);
+	i |= O_NONBLOCK;
+	i = fcntl(vwe->pipes[0], F_SETFL, i);
+	assert(i != -1);
+
+	i = fcntl(vwe->timer_pipes[0], F_GETFL);
+	assert(i != -1);
+	i |= O_NONBLOCK;
+	i = fcntl(vwe->timer_pipes[0], F_SETFL, i);
+	assert(i != -1);
+
+	AZ(pthread_create(&vwe->timer_thread,
+	    NULL, vwe_sess_timeout_ticker, vwe));
+	AZ(pthread_create(&vwe->epoll_thread, NULL, vwe_thread, vwe));
+	return(vwe);
+}
+
+/*--------------------------------------------------------------------*/
+
+const struct waiter waiter_epoll = {
+	.name =		"epoll",
+	.init =		vwe_init,
+	.pass =		vwe_pass,
+};
+
+#endif /* defined(HAVE_EPOLL_CTL) */
diff --git a/bin/varnishd/waiter/cache_waiter_kqueue.c b/bin/varnishd/waiter/cache_waiter_kqueue.c
new file mode 100644
index 0000000..b300fae
--- /dev/null
+++ b/bin/varnishd/waiter/cache_waiter_kqueue.c
@@ -0,0 +1,246 @@
+/*-
+ * Copyright (c) 2006 Verdens Gang AS
+ * Copyright (c) 2006-2009 Varnish Software AS
+ * All rights reserved.
+ *
+ * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * XXX: We need to pass sessions back into the event engine when they are
+ * reused.  Not sure what the most efficient way is for that.  For now
+ * write the session pointer to a pipe which the event engine monitors.
+ */
+
+#include "config.h"
+
+#if defined(HAVE_KQUEUE)
+
+#include <sys/types.h>
+#include <sys/event.h>
+
+#include <fcntl.h>
+#include <stdlib.h>
+#include <unistd.h>
+
+#include "cache.h"
+
+#include "waiter/cache_waiter.h"
+#include "vtim.h"
+
+#define NKEV	100
+
+struct vwk {
+	unsigned		magic;
+#define VWK_MAGIC		0x1cc2acc2
+	pthread_t		thread;
+	int			pipes[2];
+	int			kq;
+	struct kevent		ki[NKEV];
+	unsigned		nki;
+	VTAILQ_HEAD(,sess)	sesshead;
+};
+
+/*--------------------------------------------------------------------*/
+
+static void
+vwk_kq_flush(struct vwk *vwk)
+{
+	int i;
+
+	if (vwk->nki == 0)
+		return;
+	i = kevent(vwk->kq, vwk->ki, vwk->nki, NULL, 0, NULL);
+	assert(i == 0);
+	vwk->nki = 0;
+}
+
+static void
+vwk_kq_sess(struct vwk *vwk, struct sess *sp, short arm)
+{
+
+	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+	assert(sp->fd >= 0);
+	DSL(0x04, SLT_Debug, sp->vsl_id, "KQ: EV_SET sp %p arm %x", sp, arm);
+	EV_SET(&vwk->ki[vwk->nki], sp->fd, EVFILT_READ, arm, 0, 0, sp);
+	if (++vwk->nki == NKEV)
+		vwk_kq_flush(vwk);
+}
+
+static void
+vwk_kev(struct vwk *vwk, const struct kevent *kp)
+{
+	int i, j;
+	struct sess *sp;
+	struct sess *ss[NKEV];
+
+	AN(kp->udata);
+	if (kp->udata == vwk->pipes) {
+		j = 0;
+		i = read(vwk->pipes[0], ss, sizeof ss);
+		if (i == -1 && errno == EAGAIN)
+			return;
+		while (i >= sizeof ss[0]) {
+			CHECK_OBJ_NOTNULL(ss[j], SESS_MAGIC);
+			assert(ss[j]->fd >= 0);
+			AZ(ss[j]->obj);
+			VTAILQ_INSERT_TAIL(&vwk->sesshead, ss[j], list);
+			vwk_kq_sess(vwk, ss[j], EV_ADD | EV_ONESHOT);
+			j++;
+			i -= sizeof ss[0];
+		}
+		assert(i == 0);
+		return;
+	}
+	CAST_OBJ_NOTNULL(sp, kp->udata, SESS_MAGIC);
+	DSL(0x04, SLT_Debug, sp->vsl_id, "KQ: sp %p kev data %lu flags 0x%x%s",
+	    sp, (unsigned long)kp->data, kp->flags,
+	    (kp->flags & EV_EOF) ? " EOF" : "");
+
+	assert((sp->vsl_id & VSL_IDENTMASK) == kp->ident);
+	assert((sp->vsl_id & VSL_IDENTMASK) == sp->fd);
+	if (kp->data > 0) {
+		i = HTC_Rx(sp->htc);
+		if (i == 0) {
+			vwk_kq_sess(vwk, sp, EV_ADD | EV_ONESHOT);
+			return;	/* more needed */
+		}
+		VTAILQ_REMOVE(&vwk->sesshead, sp, list);
+		SES_Handle(sp, i);
+		return;
+	} else if (kp->flags & EV_EOF) {
+		VTAILQ_REMOVE(&vwk->sesshead, sp, list);
+		SES_Delete(sp, "EOF");
+		return;
+	} else {
+		VSL(SLT_Debug, sp->vsl_id, "KQ: sp %p kev data %lu flags 0x%x%s",
+		    sp, (unsigned long)kp->data, kp->flags,
+		    (kp->flags & EV_EOF) ? " EOF" : "");
+	}
+}
+
+/*--------------------------------------------------------------------*/
+
+static void *
+vwk_thread(void *priv)
+{
+	struct vwk *vwk;
+	struct kevent ke[NKEV], *kp;
+	int j, n, dotimer;
+	double deadline;
+	struct sess *sp;
+
+	CAST_OBJ_NOTNULL(vwk, priv, VWK_MAGIC);
+	THR_SetName("cache-kqueue");
+
+	vwk->kq = kqueue();
+	assert(vwk->kq >= 0);
+
+	j = 0;
+	EV_SET(&ke[j], 0, EVFILT_TIMER, EV_ADD, 0, 100, NULL);
+	j++;
+	EV_SET(&ke[j], vwk->pipes[0], EVFILT_READ, EV_ADD, 0, 0, vwk->pipes);
+	j++;
+	AZ(kevent(vwk->kq, ke, j, NULL, 0, NULL));
+
+	vwk->nki = 0;
+	while (1) {
+		dotimer = 0;
+		n = kevent(vwk->kq, vwk->ki, vwk->nki, ke, NKEV, NULL);
+		assert(n >= 1 && n <= NKEV);
+		vwk->nki = 0;
+		for (kp = ke, j = 0; j < n; j++, kp++) {
+			if (kp->filter == EVFILT_TIMER) {
+				dotimer = 1;
+				continue;
+			}
+			assert(kp->filter == EVFILT_READ);
+			vwk_kev(vwk, kp);
+		}
+		if (!dotimer)
+			continue;
+		/*
+		 * Make sure we have no pending changes for the fd's
+		 * we are about to close, in case the accept(2) in the
+		 * other thread creates new fd's betwen our close and
+		 * the kevent(2) at the top of this loop, the kernel
+		 * would not know we meant "the old fd of this number".
+		 */
+		vwk_kq_flush(vwk);
+		deadline = VTIM_real() - params->sess_timeout;
+		for (;;) {
+			sp = VTAILQ_FIRST(&vwk->sesshead);
+			if (sp == NULL)
+				break;
+			if (sp->t_open > deadline)
+				break;
+			VTAILQ_REMOVE(&vwk->sesshead, sp, list);
+			// XXX: not yet (void)VTCP_linger(sp->fd, 0);
+			SES_Delete(sp, "timeout");
+		}
+	}
+}
+
+/*--------------------------------------------------------------------*/
+
+static void
+vwk_pass(void *priv, const struct sess *sp)
+{
+	struct vwk *vwk;
+
+	CAST_OBJ_NOTNULL(vwk, priv, VWK_MAGIC);
+	assert(sizeof sp == write(vwk->pipes[1], &sp, sizeof sp));
+}
+
+/*--------------------------------------------------------------------*/
+
+static void *
+vwk_init(void)
+{
+	int i;
+	struct vwk *vwk;
+
+	ALLOC_OBJ(vwk, VWK_MAGIC);
+	AN(vwk);
+
+	VTAILQ_INIT(&vwk->sesshead);
+	AZ(pipe(vwk->pipes));
+
+	i = fcntl(vwk->pipes[0], F_GETFL);
+	assert(i != -1);
+	i |= O_NONBLOCK;
+	i = fcntl(vwk->pipes[0], F_SETFL, i);
+	assert(i != -1);
+
+	AZ(pthread_create(&vwk->thread, NULL, vwk_thread, vwk));
+	return (vwk);
+}
+
+/*--------------------------------------------------------------------*/
+
+const struct waiter waiter_kqueue = {
+	.name =		"kqueue",
+	.init =		vwk_init,
+	.pass =		vwk_pass,
+};
+
+#endif /* defined(HAVE_KQUEUE) */
diff --git a/bin/varnishd/waiter/cache_waiter_poll.c b/bin/varnishd/waiter/cache_waiter_poll.c
new file mode 100644
index 0000000..2617365
--- /dev/null
+++ b/bin/varnishd/waiter/cache_waiter_poll.c
@@ -0,0 +1,233 @@
+/*-
+ * Copyright (c) 2006 Verdens Gang AS
+ * Copyright (c) 2006-2010 Varnish Software AS
+ * All rights reserved.
+ *
+ * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ */
+
+#include "config.h"
+
+#include <poll.h>
+#include <stdlib.h>
+
+#include "cache.h"
+
+#include "waiter/cache_waiter.h"
+#include "vtim.h"
+
+#define NEEV	128
+
+struct vwp {
+	unsigned		magic;
+#define VWP_MAGIC		0x4b2cc735
+	int			pipes[2];
+	pthread_t		poll_thread;
+	struct pollfd		*pollfd;
+	unsigned		npoll;
+	unsigned		hpoll;
+
+	VTAILQ_HEAD(,sess)	sesshead;
+};
+
+/*--------------------------------------------------------------------*/
+
+static void
+vwp_pollspace(struct vwp *vwp, unsigned fd)
+{
+	struct pollfd *newpollfd = vwp->pollfd;
+	unsigned newnpoll;
+
+	if (fd < vwp->npoll)
+		return;
+	newnpoll = vwp->npoll;
+	if (newnpoll == 0)
+		newnpoll = 1;
+	while (fd >= newnpoll)
+		newnpoll = newnpoll * 2;
+	VSL(SLT_Debug, 0, "Acceptor poll space increased to %u", newnpoll);
+	newpollfd = realloc(newpollfd, newnpoll * sizeof *newpollfd);
+	XXXAN(newpollfd);
+	memset(newpollfd + vwp->npoll, 0,
+	    (newnpoll - vwp->npoll) * sizeof *newpollfd);
+	vwp->pollfd = newpollfd;
+	while (vwp->npoll < newnpoll)
+		vwp->pollfd[vwp->npoll++].fd = -1;
+	assert(fd < vwp->npoll);
+}
+
+/*--------------------------------------------------------------------*/
+
+static void
+vwp_poll(struct vwp *vwp, int fd)
+{
+
+	assert(fd >= 0);
+	vwp_pollspace(vwp, (unsigned)fd);
+	assert(fd < vwp->npoll);
+
+	if (vwp->hpoll < fd)
+		vwp->hpoll = fd;
+
+	assert(vwp->pollfd[fd].fd == -1);
+	assert(vwp->pollfd[fd].events == 0);
+	assert(vwp->pollfd[fd].revents == 0);
+
+	vwp->pollfd[fd].fd = fd;
+	vwp->pollfd[fd].events = POLLIN;
+}
+
+static void
+vwp_unpoll(struct vwp *vwp, int fd)
+{
+
+	assert(fd >= 0);
+	assert(fd < vwp->npoll);
+	vwp_pollspace(vwp, (unsigned)fd);
+
+	assert(vwp->pollfd[fd].fd == fd);
+	assert(vwp->pollfd[fd].events == POLLIN);
+	assert(vwp->pollfd[fd].revents == 0);
+
+	vwp->pollfd[fd].fd = -1;
+	vwp->pollfd[fd].events = 0;
+}
+
+/*--------------------------------------------------------------------*/
+
+static void *
+vwp_main(void *priv)
+{
+	int v, v2;
+	struct vwp *vwp;
+	struct sess *ss[NEEV], *sp, *sp2;
+	double deadline;
+	int i, j, fd;
+
+	CAST_OBJ_NOTNULL(vwp, priv, VWP_MAGIC);
+	THR_SetName("cache-poll");
+
+	vwp_poll(vwp, vwp->pipes[0]);
+
+	while (1) {
+		assert(vwp->hpoll < vwp->npoll);
+		while (vwp->hpoll > 0 && vwp->pollfd[vwp->hpoll].fd == -1)
+			vwp->hpoll--;
+		assert(vwp->pipes[0] <= vwp->hpoll);
+		assert(vwp->pollfd[vwp->pipes[0]].fd == vwp->pipes[0]);
+		assert(vwp->pollfd[vwp->pipes[1]].fd == -1);
+		v = poll(vwp->pollfd, vwp->hpoll + 1, 100);
+		assert(v >= 0);
+		deadline = VTIM_real() - params->sess_timeout;
+		v2 = v;
+		VTAILQ_FOREACH_SAFE(sp, &vwp->sesshead, list, sp2) {
+			if (v != 0 && v2 == 0)
+				break;
+			CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+			fd = sp->fd;
+			assert(fd >= 0);
+			assert(fd <= vwp->hpoll);
+			assert(fd < vwp->npoll);
+			assert(vwp->pollfd[fd].fd == fd);
+			if (vwp->pollfd[fd].revents) {
+				v2--;
+				i = HTC_Rx(sp->htc);
+				if (vwp->pollfd[fd].revents != POLLIN)
+					VSL(SLT_Debug, fd, "Poll: %x / %d",
+					    vwp->pollfd[fd].revents, i);
+				vwp->pollfd[fd].revents = 0;
+				VTAILQ_REMOVE(&vwp->sesshead, sp, list);
+				if (i == 0) {
+					/* Mov to front of list for speed */
+					VTAILQ_INSERT_HEAD(&vwp->sesshead, sp, list);
+				} else {
+					vwp_unpoll(vwp, fd);
+					SES_Handle(sp, i);
+				}
+			} else if (sp->t_open <= deadline) {
+				VTAILQ_REMOVE(&vwp->sesshead, sp, list);
+				vwp_unpoll(vwp, fd);
+				// XXX: not yet (void)VTCP_linger(sp->fd, 0);
+				SES_Delete(sp, "timeout");
+			}
+		}
+		if (v2 && vwp->pollfd[vwp->pipes[0]].revents) {
+
+			if (vwp->pollfd[vwp->pipes[0]].revents != POLLIN)
+				VSL(SLT_Debug, 0, "pipe.revents= 0x%x",
+				    vwp->pollfd[vwp->pipes[0]].revents);
+			assert(vwp->pollfd[vwp->pipes[0]].revents == POLLIN);
+			vwp->pollfd[vwp->pipes[0]].revents = 0;
+			v2--;
+			i = read(vwp->pipes[0], ss, sizeof ss);
+			assert(i >= 0);
+			assert(((unsigned)i % sizeof ss[0]) == 0);
+			for (j = 0; j * sizeof ss[0] < i; j++) {
+				CHECK_OBJ_NOTNULL(ss[j], SESS_MAGIC);
+				assert(ss[j]->fd >= 0);
+				VTAILQ_INSERT_TAIL(&vwp->sesshead, ss[j], list);
+				vwp_poll(vwp, ss[j]->fd);
+			}
+		}
+		assert(v2 == 0);
+	}
+	NEEDLESS_RETURN(NULL);
+}
+
+/*--------------------------------------------------------------------*/
+
+static void
+vwp_poll_pass(void *priv, const struct sess *sp)
+{
+	struct vwp *vwp;
+
+	CAST_OBJ_NOTNULL(vwp, priv, VWP_MAGIC);
+
+	assert(sizeof sp == write(vwp->pipes[1], &sp, sizeof sp));
+}
+
+/*--------------------------------------------------------------------*/
+
+static void *
+vwp_poll_init(void)
+{
+	struct vwp *vwp;
+
+	ALLOC_OBJ(vwp, VWP_MAGIC);
+	AN(vwp);
+	VTAILQ_INIT(&vwp->sesshead);
+	AZ(pipe(vwp->pipes));
+	vwp_pollspace(vwp, 256);
+	AZ(pthread_create(&vwp->poll_thread, NULL, vwp_main, vwp));
+	return (vwp);
+}
+
+/*--------------------------------------------------------------------*/
+
+const struct waiter waiter_poll = {
+	.name =		"poll",
+	.init =		vwp_poll_init,
+	.pass =		vwp_poll_pass,
+};
diff --git a/bin/varnishd/waiter/cache_waiter_ports.c b/bin/varnishd/waiter/cache_waiter_ports.c
new file mode 100644
index 0000000..022131b
--- /dev/null
+++ b/bin/varnishd/waiter/cache_waiter_ports.c
@@ -0,0 +1,283 @@
+/*-
+ * Copyright (c) 2006 Verdens Gang AS
+ * Copyright (c) 2006 Varnish Software AS
+ * Copyright (c) 2007 OmniTI Computer Consulting, Inc.
+ * Copyright (c) 2007 Theo Schlossnagle
+ * Copyright (c) 2010 UPLEX, Nils Goroll
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ */
+
+#include "config.h"
+
+#if defined(HAVE_PORT_CREATE)
+
+#include <sys/time.h>
+
+#include <math.h>
+#include <port.h>
+#include <stdio.h>
+#include <stdlib.h>
+
+#include "cache.h"
+
+#include "waiter/cache_waiter.h"
+#include "vtim.h"
+
+#define MAX_EVENTS 256
+
+struct vws {
+	unsigned		magic;
+#define VWS_MAGIC		0x0b771473
+	pthread_t		ports_thread;
+	int			dport;
+	VTAILQ_HEAD(,sess)	sesshead;
+};
+
+static inline void
+vws_add(struct vws *vws, int fd, void *data)
+{
+	/*
+	 * POLLIN should be all we need here
+	 *
+	 */
+	AZ(port_associate(vws->dport, PORT_SOURCE_FD, fd, POLLIN, data));
+}
+
+static inline void
+vws_del(struct vws *vws, int fd)
+{
+	port_dissociate(vws->dport, PORT_SOURCE_FD, fd);
+}
+
+static inline void
+vws_port_ev(struct vws *vws, port_event_t *ev) {
+	struct sess *sp;
+	if(ev->portev_source == PORT_SOURCE_USER) {
+		CAST_OBJ_NOTNULL(sp, ev->portev_user, SESS_MAGIC);
+		assert(sp->fd >= 0);
+		AZ(sp->obj);
+		VTAILQ_INSERT_TAIL(&vws->sesshead, sp, list);
+		vws_add(vws, sp->fd, sp);
+	} else {
+		int i;
+		assert(ev->portev_source == PORT_SOURCE_FD);
+		CAST_OBJ_NOTNULL(sp, ev->portev_user, SESS_MAGIC);
+		assert(sp->fd >= 0);
+		if(ev->portev_events & POLLERR) {
+			vws_del(vws, sp->fd);
+			VTAILQ_REMOVE(&vws->sesshead, sp, list);
+			SES_Delete(sp, "EOF");
+			return;
+		}
+		i = HTC_Rx(sp->htc);
+
+		if (i == 0) {
+			/* incomplete header, wait for more data */
+			vws_add(vws, sp->fd, sp);
+			return;
+		}
+
+		/*
+		 * note: the original man page for port_associate(3C) states:
+		 *
+		 *    When an event for a PORT_SOURCE_FD object is retrieved,
+		 *    the object no longer has an association with the port.
+		 *
+		 * This can be read along the lines of sparing the
+		 * port_dissociate after port_getn(), but in fact,
+		 * port_dissociate should be used
+		 *
+		 * Ref: http://opensolaris.org/jive/thread.jspa?threadID=129476&tstart=0
+		 */
+		vws_del(vws, sp->fd);
+		VTAILQ_REMOVE(&vws->sesshead, sp, list);
+
+		/* SES_Handle will also handle errors */
+		SES_Handle(sp, i);
+	}
+	return;
+}
+
+static void *
+vws_thread(void *priv)
+{
+	struct sess *sp;
+	struct vws *vws;
+
+	CAST_OBJ_NOTNULL(vws, priv, VWS_MAGIC);
+	/*
+	 * timeouts:
+	 *
+	 * min_ts : Minimum timeout for port_getn
+	 * min_t  : ^ equivalent in floating point representation
+	 *
+	 * max_ts : Maximum timeout for port_getn
+	 * max_t  : ^ equivalent in floating point representation
+	 *
+	 * with (nevents == 1), we should always choose the correct port_getn
+	 * timeout to check session timeouts, so max is just a safety measure
+	 * (if this implementation is correct, it could be set to an "infinte"
+	 *  value)
+	 *
+	 * with (nevents > 1), min and max define the acceptable range for
+	 * - additional latency of keep-alive connections and
+	 * - additional tolerance for handling session timeouts
+	 *
+	 */
+	static struct timespec min_ts = {0L,    100L /*ms*/  * 1000L /*us*/  * 1000L /*ns*/};
+	static double          min_t  = 0.1; /* 100    ms*/
+	static struct timespec max_ts = {1L, 0L};		/* 1 second */
+	static double	       max_t  = 1.0;			/* 1 second */
+
+	/* XXX: These should probably go in vws ? */
+	struct timespec ts;
+	struct timespec *timeout;
+
+	vws->dport = port_create();
+	assert(vws->dport >= 0);
+
+	timeout = &max_ts;
+
+	while (1) {
+		port_event_t ev[MAX_EVENTS];
+		int nevents, ei, ret;
+		double now, deadline;
+
+		/*
+		 * XXX Do we want to scale this up dynamically to increase
+		 *     efficiency in high throughput situations? - would need to
+		 *     start with one to keep latency low at any rate
+		 *
+		 *     Note: when increasing nevents, we must lower min_ts
+		 *	     and max_ts
+		 */
+		nevents = 1;
+
+		/*
+		 * see disucssion in
+		 * - https://issues.apache.org/bugzilla/show_bug.cgi?id=47645
+		 * - http://mail.opensolaris.org/pipermail/networking-discuss/2009-August/011979.html
+		 *
+		 * comment from apr/poll/unix/port.c :
+		 *
+		 * This confusing API can return an event at the same time
+		 * that it reports EINTR or ETIME.
+		 *
+		 */
+
+		ret = port_getn(vws->dport, ev, MAX_EVENTS, &nevents, timeout);
+
+		if (ret < 0)
+			assert((errno == EINTR) || (errno == ETIME));
+
+		for (ei = 0; ei < nevents; ei++)
+			vws_port_ev(vws, ev + ei);
+
+		/* check for timeouts */
+		now = VTIM_real();
+		deadline = now - params->sess_timeout;
+
+		/*
+		 * This loop assumes that the oldest sessions are always at the
+		 * beginning of the list (which is the case if we guarantee to
+		 * enqueue at the tail only
+		 *
+		 */
+
+		for (;;) {
+			sp = VTAILQ_FIRST(&vws->sesshead);
+			if (sp == NULL)
+				break;
+			if (sp->t_open > deadline) {
+				break;
+			}
+			VTAILQ_REMOVE(&vws->sesshead, sp, list);
+			if(sp->fd != -1) {
+				vws_del(vws, sp->fd);
+			}
+			SES_Delete(sp, "timeout");
+		}
+
+		/*
+		 * Calculate the timeout for the next get_portn
+		 */
+
+		if (sp) {
+			double tmo = (sp->t_open + params->sess_timeout) - now;
+
+			/* we should have removed all sps whose timeout has passed */
+			assert(tmo > 0.0);
+
+			if (tmo < min_t) {
+				timeout = &min_ts;
+			} else if (tmo > max_t) {
+				timeout = &max_ts;
+			} else {
+				ts = VTIM_timespec(tmo);
+				timeout = &ts;
+			}
+		} else {
+			timeout = &max_ts;
+		}
+	}
+}
+
+/*--------------------------------------------------------------------*/
+
+static void
+vws_pass(void *priv, const struct sess *sp)
+{
+	int r;
+	struct vws *vws;
+
+	CAST_OBJ_NOTNULL(vws, priv, VWS_MAGIC);
+	while((r = port_send(vws->dport, 0, TRUST_ME(sp))) == -1 &&
+		errno == EAGAIN);
+	AZ(r);
+}
+
+/*--------------------------------------------------------------------*/
+
+static void *
+vws_init(void)
+{
+	struct vws *vws;
+
+	ALLOC_OBJ(vws, VWS_MAGIC);
+	AN(vws);
+	VTAILQ_INIT(&vws->sesshead);
+	AZ(pthread_create(&vws->ports_thread, NULL, vws_thread, vws));
+	return (vws);
+}
+
+/*--------------------------------------------------------------------*/
+
+const struct waiter waiter_ports = {
+	.name =		"ports",
+	.init =		vws_init,
+	.pass =		vws_pass
+};
+
+#endif /* defined(HAVE_PORT_CREATE) */



From geoff at varnish-cache.org  Mon Jan  9 20:52:00 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:00 +0100
Subject: [experimental-ims] f98e2b8 remove unused variable
Message-ID: <E1RkMCG-00077t-0m@project.varnish-software.com>

commit f98e2b83dd9683d2081a0ad6d8f219e98e8023ee
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Sep 20 14:25:40 2011 +0000

    remove unused variable

diff --git a/lib/libvarnishcompat/srandomdev.c b/lib/libvarnishcompat/srandomdev.c
index d0332c9..83b22c4 100644
--- a/lib/libvarnishcompat/srandomdev.c
+++ b/lib/libvarnishcompat/srandomdev.c
@@ -61,7 +61,6 @@ srandomdev(void)
 {
 	struct timeval tv;
 	unsigned long seed;
-	int fd;
 
 	if (trydev("/dev/urandom", &seed)) {
 		if (trydev("/dev/random", &seed)) {



From geoff at varnish-cache.org  Mon Jan  9 20:52:44 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:44 +0100
Subject: [experimental-ims] 7813b33 Compile fix.
Message-ID: <E1RkMCy-0008VI-IQ@project.varnish-software.com>

commit 7813b33f99ee139e0ef93766e928af19da2a2cac
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Nov 29 21:05:23 2011 +0000

    Compile fix.

diff --git a/bin/varnishd/waiter/cache_waiter_epoll.c b/bin/varnishd/waiter/cache_waiter_epoll.c
index 4d45909..06cd8bb 100644
--- a/bin/varnishd/waiter/cache_waiter_epoll.c
+++ b/bin/varnishd/waiter/cache_waiter_epoll.c
@@ -119,7 +119,6 @@ vwe_eev(struct vwe *vwe, const struct epoll_event *ep)
 			while (i >= sizeof ss[0]) {
 				CHECK_OBJ_NOTNULL(ss[j], SESS_MAGIC);
 				assert(ss[j]->fd >= 0);
-				AZ(ss[j]->obj);
 				VTAILQ_INSERT_TAIL(&vwe->sesshead, ss[j], list);
 				vwe_cond_modadd(vwe, ss[j]->fd, ss[j]);
 				j++;



From geoff at varnish-cache.org  Mon Jan  9 20:52:28 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:28 +0100
Subject: [experimental-ims] aef8337 Use new-style backend naming in output
Message-ID: <E1RkMCi-00080v-K3@project.varnish-software.com>

commit aef83370f8ad506dbca5b9df0494ca9b04712945
Author: Tollef Fog Heen <tfheen at varnish-software.com>
Date:   Tue Nov 1 10:43:02 2011 +0100

    Use new-style backend naming in output
    
    Prefer vcl_name(ipv4,ipv6,port) to the old vcl_name in output.

diff --git a/bin/varnishd/cache_backend.c b/bin/varnishd/cache_backend.c
index 7bacd26..1a3ae5b 100644
--- a/bin/varnishd/cache_backend.c
+++ b/bin/varnishd/cache_backend.c
@@ -143,8 +143,8 @@ bes_conn_try(const struct sess *sp, struct vbc *vc, const struct vdi_simple *vs)
 {
 	int s;
 	struct backend *bp = vs->backend;
-	char abuf1[VTCP_ADDRBUFSIZE], abuf2[VTCP_ADDRBUFSIZE];
-	char pbuf1[VTCP_PORTBUFSIZE], pbuf2[VTCP_PORTBUFSIZE];
+	char abuf1[VTCP_ADDRBUFSIZE];
+	char pbuf1[VTCP_PORTBUFSIZE];
 
 	CHECK_OBJ_NOTNULL(vs, VDI_SIMPLE_MAGIC);
 
@@ -185,10 +185,8 @@ bes_conn_try(const struct sess *sp, struct vbc *vc, const struct vdi_simple *vs)
 	} else {
 		vc->vsl_id = s | VSL_BACKENDMARKER;
 		VTCP_myname(s, abuf1, sizeof abuf1, pbuf1, sizeof pbuf1);
-		VTCP_name(vc->addr, vc->addrlen,
-		    abuf2, sizeof abuf2, pbuf2, sizeof pbuf2);
-		WSL(sp->wrk, SLT_BackendOpen, vc->vsl_id, "%s %s %s %s %s",
-		    vs->backend->vcl_name, abuf1, pbuf1, abuf2, pbuf2);
+		WSL(sp->wrk, SLT_BackendOpen, vc->vsl_id, "%s %s %s ",
+		    vs->backend->display_name, abuf1, pbuf1);
 	}
 
 }
@@ -347,13 +345,13 @@ vbe_GetVbe(const struct sess *sp, struct vdi_simple *vs)
 			/* XXX locking of stats */
 			VSC_C_main->backend_reuse += 1;
 			WSP(sp, SLT_Backend, "%d %s %s",
-			    vc->fd, sp->director->vcl_name, bp->vcl_name);
+			    vc->fd, sp->director->vcl_name, bp->display_name);
 			vc->vdis = vs;
 			vc->recycled = 1;
 			return (vc);
 		}
 		VSC_C_main->backend_toolate++;
-		WSL(sp->wrk, SLT_BackendClose, vc->vsl_id, "%s", bp->vcl_name);
+		WSL(sp->wrk, SLT_BackendClose, vc->vsl_id, "%s", bp->display_name);
 
 		/* Checkpoint log to flush all info related to this connection
 		   before the OS reuses the FD */
@@ -388,7 +386,7 @@ vbe_GetVbe(const struct sess *sp, struct vdi_simple *vs)
 	vc->backend = bp;
 	VSC_C_main->backend_conn++;
 	WSP(sp, SLT_Backend, "%d %s %s",
-	    vc->fd, sp->director->vcl_name, bp->vcl_name);
+	    vc->fd, sp->director->vcl_name, bp->display_name);
 	vc->vdis = vs;
 	return (vc);
 }
diff --git a/bin/varnishd/cache_backend.h b/bin/varnishd/cache_backend.h
index 47e709d..8937673 100644
--- a/bin/varnishd/cache_backend.h
+++ b/bin/varnishd/cache_backend.h
@@ -115,6 +115,7 @@ struct backend {
 	struct lock		mtx;
 
 	char			*vcl_name;
+	char			*display_name;
 	char			*ipv4_addr;
 	char			*ipv6_addr;
 	char			*port;
diff --git a/bin/varnishd/cache_backend_cfg.c b/bin/varnishd/cache_backend_cfg.c
index b10512d..a4f3285 100644
--- a/bin/varnishd/cache_backend_cfg.c
+++ b/bin/varnishd/cache_backend_cfg.c
@@ -213,6 +213,7 @@ VBE_AddBackend(struct cli *cli, const struct vrt_backend *vb)
 	 * so we cannot simply reference the VCL's copy of things.
 	 */
 	REPLACE(b->vcl_name, vb->vcl_name);
+	REPLACE(b->display_name, buf);
 	REPLACE(b->ipv4_addr, vb->ipv4_addr);
 	REPLACE(b->ipv6_addr, vb->ipv6_addr);
 	REPLACE(b->port, vb->port);
@@ -283,9 +284,9 @@ cli_debug_backend(struct cli *cli, const char * const *av, void *priv)
 	ASSERT_CLI();
 	VTAILQ_FOREACH(b, &backends, list) {
 		CHECK_OBJ_NOTNULL(b, BACKEND_MAGIC);
-		VCLI_Out(cli, "%p %s(%s,%s,:%s) %d %d\n",
-		    b, b->vcl_name, b->ipv4_addr, b->ipv6_addr, b->port,
-		    b->refcount, b->n_conn);
+		VCLI_Out(cli, "%p %s %d %d\n",
+			 b, b->display_name,
+			 b->refcount, b->n_conn);
 	}
 }
 
diff --git a/bin/varnishd/cache_dir.c b/bin/varnishd/cache_dir.c
index e95b015..59ef5fc 100644
--- a/bin/varnishd/cache_dir.c
+++ b/bin/varnishd/cache_dir.c
@@ -50,7 +50,7 @@ VDI_CloseFd(struct worker *wrk)
 
 	bp = wrk->vbc->backend;
 
-	WSL(wrk, SLT_BackendClose, wrk->vbc->vsl_id, "%s", bp->vcl_name);
+	WSL(wrk, SLT_BackendClose, wrk->vbc->vsl_id, "%s", bp->display_name);
 
 	/* Checkpoint log to flush all info related to this connection
 	   before the OS reuses the FD */
@@ -78,7 +78,7 @@ VDI_RecycleFd(struct worker *wrk)
 
 	bp = wrk->vbc->backend;
 
-	WSL(wrk, SLT_BackendReuse, wrk->vbc->vsl_id, "%s", bp->vcl_name);
+	WSL(wrk, SLT_BackendReuse, wrk->vbc->vsl_id, "%s", bp->display_name);
 	/*
 	 * Flush the shmlog, so that another session reusing this backend
 	 * will log chronologically later than our use of it.
diff --git a/bin/varnishd/cache_panic.c b/bin/varnishd/cache_panic.c
index 231adf1..4076d26 100644
--- a/bin/varnishd/cache_panic.c
+++ b/bin/varnishd/cache_panic.c
@@ -96,7 +96,7 @@ pan_vbc(const struct vbc *vbc)
 	be = vbc->backend;
 
 	VSB_printf(vsp, "  backend = %p fd = %d {\n", be, vbc->fd);
-	VSB_printf(vsp, "    vcl_name = \"%s\",\n", be->vcl_name);
+	VSB_printf(vsp, "    display_name = \"%s\",\n", be->display_name);
 	VSB_printf(vsp, "  },\n");
 }
 



From geoff at varnish-cache.org  Mon Jan  9 20:51:57 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:51:57 +0100
Subject: [experimental-ims] 652b2f6 Take another stab at getting OS/X to
	compile again.
Message-ID: <E1RkMCD-00072j-AA@project.varnish-software.com>

commit 652b2f61a59a5698bb5f15ccf0a8d1b31d519f69
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Sep 19 13:36:53 2011 +0000

    Take another stab at getting OS/X to compile again.

diff --git a/bin/varnishd/cache_pool.c b/bin/varnishd/cache_pool.c
index 675fea6..811b233 100644
--- a/bin/varnishd/cache_pool.c
+++ b/bin/varnishd/cache_pool.c
@@ -49,6 +49,7 @@
 #include <math.h>
 #include <stdlib.h>
 #include <string.h>
+#include <time.h>
 #include <unistd.h>
 
 #include "vcl.h"



From geoff at varnish-cache.org  Mon Jan  9 20:52:27 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:27 +0100
Subject: [experimental-ims] e9e02a4 Support for \t\n in varnishncsa format
	strings
Message-ID: <E1RkMCh-0007yX-K1@project.varnish-software.com>

commit e9e02a49c2a5606264b3ee9cd8e1c795639f4d61
Author: Tollef Fog Heen <tfheen at varnish-software.com>
Date:   Mon Oct 31 14:35:28 2011 +0100

    Support for \t\n in varnishncsa format strings

diff --git a/bin/varnishncsa/varnishncsa.c b/bin/varnishncsa/varnishncsa.c
index f699d9b..8d969eb 100644
--- a/bin/varnishncsa/varnishncsa.c
+++ b/bin/varnishncsa/varnishncsa.c
@@ -629,6 +629,14 @@ h_ncsa(void *priv, enum VSL_tag_e tag, unsigned fd,
 
 	for (p = format; *p != '\0'; p++) {
 
+		/* allow the most essential escape sequences in format. */
+		if (*p == '\\') {
+			p++;
+			if (*p == 't') VSB_putc(os, '\t');
+			if (*p == 'n') VSB_putc(os, '\n');
+			continue;
+		}
+
 		if (*p != '%') {
 			VSB_putc(os, *p);
 			continue;
diff --git a/doc/sphinx/reference/varnishncsa.rst b/doc/sphinx/reference/varnishncsa.rst
index ddb6538..8499d29 100644
--- a/doc/sphinx/reference/varnishncsa.rst
+++ b/doc/sphinx/reference/varnishncsa.rst
@@ -54,6 +54,8 @@ The following options are available:
 
             %h %l %u %t "%r" %s %b "%{Referer}i" "%{User-agent}i"
 
+	    Escape sequences \\n and \\t are supported.
+
 	    Supported formatters are:
 
 	      %b 



From geoff at varnish-cache.org  Mon Jan  9 20:52:30 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:30 +0100
Subject: [experimental-ims] a2dbc59 Renovate ban related statistics,
	add long descriptions etc.
Message-ID: <E1RkMCk-00086P-RY@project.varnish-software.com>

commit a2dbc597f6bb7a9bc16345432cdd753caef16986
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Nov 9 10:59:26 2011 +0000

    Renovate ban related statistics, add long descriptions etc.

diff --git a/bin/varnishd/cache_ban.c b/bin/varnishd/cache_ban.c
index 648997c..ac45144 100644
--- a/bin/varnishd/cache_ban.c
+++ b/bin/varnishd/cache_ban.c
@@ -376,7 +376,6 @@ void
 BAN_Insert(struct ban *b)
 {
 	struct ban  *bi, *be;
-	unsigned pcount;
 	ssize_t ln;
 	double t0;
 
@@ -402,8 +401,10 @@ BAN_Insert(struct ban *b)
 	Lck_Lock(&ban_mtx);
 	VTAILQ_INSERT_HEAD(&ban_head, b, list);
 	ban_start = b;
-	VSC_C_main->n_ban++;
-	VSC_C_main->n_ban_add++;
+	VSC_C_main->bans++;
+	VSC_C_main->bans_added++;
+	if (b->flags & BAN_F_REQ)
+		VSC_C_main->bans_req++;
 
 	be = VTAILQ_LAST(&ban_head, banhead_s);
 	if (params->ban_dups && be != b)
@@ -419,7 +420,6 @@ BAN_Insert(struct ban *b)
 
 	/* Hunt down duplicates, and mark them as gone */
 	bi = b;
-	pcount = 0;
 	Lck_Lock(&ban_mtx);
 	while(bi != be) {
 		bi = VTAILQ_NEXT(bi, list);
@@ -429,11 +429,10 @@ BAN_Insert(struct ban *b)
 		if (memcmp(b->spec + 8, bi->spec + 8, ln - 8))
 			continue;
 		bi->flags |= BAN_F_GONE;
-		VSC_C_main->n_ban_gone++;
-		pcount++;
+		VSC_C_main->bans_gone++;
+		VSC_C_main->bans_dups++;
 	}
 	be->refcount--;
-	VSC_C_main->n_ban_dups += pcount;
 	Lck_Unlock(&ban_mtx);
 }
 
@@ -520,20 +519,28 @@ BAN_Reload(const uint8_t *ban, unsigned len)
 
 	t0 = ban_time(ban);
 	assert(len == ban_len(ban));
+
+	Lck_Lock(&ban_mtx);
+
 	VTAILQ_FOREACH(b, &ban_head, list) {
 		t1 = ban_time(b->spec);
 		assert (t1 < t2);
 		t2 = t1;
-		if (t1 == t0)
+		if (t1 == t0) {
+			Lck_Unlock(&ban_mtx);
 			return;
+		}
 		if (t1 < t0)
 			break;
-		if (!memcmp(b->spec + 8, ban + 8, len - 8))
+		if (!memcmp(b->spec + 8, ban + 8, len - 8)) {
 			gone |= BAN_F_GONE;
+			VSC_C_main->bans_dups++;
+			VSC_C_main->bans_gone++;
+		}
 	}
 
-	VSC_C_main->n_ban++;
-	VSC_C_main->n_ban_add++;
+	VSC_C_main->bans++;
+	VSC_C_main->bans_added++;
 
 	b2 = BAN_New();
 	AN(b2);
@@ -552,9 +559,13 @@ BAN_Reload(const uint8_t *ban, unsigned len)
 	for (b = VTAILQ_NEXT(b2, list); b != NULL; b = VTAILQ_NEXT(b, list)) {
 		if (b->flags & BAN_F_GONE)
 			continue;
-		if (!memcmp(b->spec + 8, ban + 8, len - 8))
+		if (!memcmp(b->spec + 8, ban + 8, len - 8)) {
 			b->flags |= BAN_F_GONE;
+			VSC_C_main->bans_dups++;
+			VSC_C_main->bans_gone++;
+		}
 	}
+	Lck_Unlock(&ban_mtx);
 }
 
 /*--------------------------------------------------------------------
@@ -705,8 +716,8 @@ ban_check_object(struct object *o, const struct sess *sp, int has_req)
 	}
 
 	Lck_Lock(&ban_mtx);
-	VSC_C_main->n_ban_obj_test++;
-	VSC_C_main->n_ban_re_test += tests;
+	VSC_C_main->bans_tested++;
+	VSC_C_main->bans_tests_tested += tests;
 
 	if (b == oc->ban && skipped > 0) {
 		AZ(has_req);
@@ -758,9 +769,11 @@ ban_CheckLast(void)
 	b = VTAILQ_LAST(&ban_head, banhead_s);
 	if (b != VTAILQ_FIRST(&ban_head) && b->refcount == 0) {
 		if (b->flags & BAN_F_GONE)
-			VSC_C_main->n_ban_gone--;
-		VSC_C_main->n_ban--;
-		VSC_C_main->n_ban_retire++;
+			VSC_C_main->bans_gone--;
+		if (b->flags & BAN_F_REQ)
+			VSC_C_main->bans_req--;
+		VSC_C_main->bans--;
+		VSC_C_main->bans_deleted++;
 		VTAILQ_REMOVE(&ban_head, b, list);
 	} else {
 		b = NULL;
@@ -888,7 +901,7 @@ ban_lurker_work(const struct sess *sp, unsigned pass)
 		if (!(b->flags & BAN_F_REQ)) {
 			if (!(b->flags & BAN_F_GONE)) {
 				b->flags |= BAN_F_GONE;
-				VSC_C_main->n_ban_gone++;
+				VSC_C_main->bans_gone++;
 			}
 			if (params->diag_bitmap & 0x80000)
 				VSL(SLT_Debug, 0, "lurker BAN %f now gone",
@@ -1090,6 +1103,6 @@ BAN_Init(void)
 	ban_magic = BAN_New();
 	AN(ban_magic);
 	ban_magic->flags |= BAN_F_GONE;
-	VSC_C_main->n_ban_gone++;
+	VSC_C_main->bans_gone++;
 	BAN_Insert(ban_magic);
 }
diff --git a/bin/varnishtest/tests/c00019.vtc b/bin/varnishtest/tests/c00019.vtc
index a27bb8e..5cc5973 100644
--- a/bin/varnishtest/tests/c00019.vtc
+++ b/bin/varnishtest/tests/c00019.vtc
@@ -16,7 +16,7 @@ varnish v1 -vcl+backend {} -start
 varnish v1 -cliok "ban.url FOO"
 
 # There is one "magic" ban from boot
-varnish v1 -expect n_ban_add == 2
+varnish v1 -expect bans_added == 2
 varnish v1 -cliok "ban.list"
 
 # Our fetch is not affected by the ban
@@ -31,12 +31,12 @@ client c1 {
 } -run
 
 varnish v1 -cliok "ban.list"
-varnish v1 -expect n_ban_obj_test == 0
-varnish v1 -expect n_ban_re_test == 0
+varnish v1 -expect bans_tested == 0
+varnish v1 -expect bans_tests_tested == 0
 
 # Add another ban
 varnish v1 -cliok "ban.url FOO"
-varnish v1 -expect n_ban_add == 3
+varnish v1 -expect bans_added == 3
 varnish v1 -cliok "ban.list"
 
 # The cached object will be band, and a new
@@ -47,8 +47,8 @@ client c1 {
 	expect resp.http.foo == 2
 } -run
 
-varnish v1 -expect n_ban_obj_test == 1
-varnish v1 -expect n_ban_re_test == 1
+varnish v1 -expect bans_tested == 1
+varnish v1 -expect bans_tests_tested == 1
 varnish v1 -cliok "ban.list"
 
 # Fetch the cached copy, just for grins
@@ -62,15 +62,15 @@ client c1 {
 # Now add another two bans, Kilroy should not be hit
 varnish v1 -cliok "ban.url KILROY"
 varnish v1 -cliok "ban.url FOO"
-varnish v1 -expect n_ban_add == 5
+varnish v1 -expect bans_added == 5
 
 # Enable dup removal of bans
 varnish v1 -cliok "param.set ban_dups on"
 
 # This should incapacitate the two previous FOO bans.
 varnish v1 -cliok "ban.url FOO"
-varnish v1 -expect n_ban_add == 6
-varnish v1 -expect n_ban_dups == 3
+varnish v1 -expect bans_added == 6
+varnish v1 -expect bans_dups == 3
 varnish v1 -cliok "ban.list"
 
 # And we should get a fresh object from backend
@@ -81,8 +81,8 @@ client c1 {
 } -run
 
 # With only two objects having ever been compared
-varnish v1 -expect n_ban_obj_test == 2
-varnish v1 -expect n_ban_re_test == 2
+varnish v1 -expect bans_tested == 2
+varnish v1 -expect bans_tests_tested == 2
 varnish v1 -cliok "ban.list"
 
 # Test a bogus regexp
diff --git a/bin/varnishtest/tests/c00049.vtc b/bin/varnishtest/tests/c00049.vtc
index facd979..59e6c14 100644
--- a/bin/varnishtest/tests/c00049.vtc
+++ b/bin/varnishtest/tests/c00049.vtc
@@ -49,8 +49,8 @@ client c1 {
 delay 0.1
 varnish v1 -cliok "ban req.url == /alpha"
 varnish v1 -cliok "ban.list"
-varnish v1 -expect n_ban == 2
-varnish v1 -expect n_ban_gone == 1
+varnish v1 -expect bans == 2
+varnish v1 -expect bans_gone == 1
 
 client c1 {
 	txreq -url "/beta"
@@ -61,7 +61,7 @@ client c1 {
 delay 0.1
 varnish v1 -cliok "ban obj.http.foo == /beta"
 varnish v1 -cliok "ban.list"
-varnish v1 -expect n_ban == 3
+varnish v1 -expect bans == 3
 
 client c1 {
 	txreq -url "/gamma"
@@ -72,7 +72,7 @@ client c1 {
 delay 0.1
 varnish v1 -cliok "ban obj.http.foo == /gamma"
 varnish v1 -cliok "ban.list"
-varnish v1 -expect n_ban == 4
+varnish v1 -expect bans == 4
 
 client c1 {
 	txreq -url "/delta"
@@ -83,22 +83,22 @@ client c1 {
 delay 0.1
 varnish v1 -cliok "ban req.url == /delta"
 
-varnish v1 -expect n_ban_gone == 1
+varnish v1 -expect bans_gone == 1
 varnish v1 -cliok "ban obj.http.foo == /gamma"
 # Dup-check should have added one
-varnish v1 -expect n_ban_gone == 2
+varnish v1 -expect bans_gone == 2
 
 varnish v1 -cliok "ban req.url == /epsilon"
 varnish v1 -cliok "ban.list"
-varnish v1 -expect n_ban == 7
-varnish v1 -expect n_ban_gone == 2
+varnish v1 -expect bans == 7
+varnish v1 -expect bans_gone == 2
 
 varnish v1 -cliok "param.set ban_lurker_sleep .01"
 delay 1
 varnish v1 -cliok "param.set ban_lurker_sleep .00"
 varnish v1 -cliok "ban.list"
-varnish v1 -expect n_ban == 7
-varnish v1 -expect n_ban_gone == 4
+varnish v1 -expect bans == 7
+varnish v1 -expect bans_gone == 4
 
 client c1 {
 	txreq -url "/alpha"
@@ -108,7 +108,7 @@ client c1 {
 
 delay 1
 varnish v1 -cliok "ban.list"
-varnish v1 -expect n_ban == 4
+varnish v1 -expect bans == 4
 
 client c1 {
 	txreq -url "/beta"
@@ -127,5 +127,7 @@ client c1 {
 
 delay 1
 varnish v1 -cliok "ban.list"
-varnish v1 -expect n_ban == 1
-varnish v1 -expect n_ban_gone == 0
+varnish v1 -expect bans == 1
+varnish v1 -expect bans_gone == 0
+varnish v1 -expect bans_added == 7
+varnish v1 -expect bans_deleted == 6
diff --git a/bin/varnishtest/tests/p00002.vtc b/bin/varnishtest/tests/p00002.vtc
index cef8f36..e3ea052 100644
--- a/bin/varnishtest/tests/p00002.vtc
+++ b/bin/varnishtest/tests/p00002.vtc
@@ -36,4 +36,4 @@ varnish v1 -start
 varnish v1 -cliok ban.list
 
 # Count of 3 here, because two "magic" bans are also there"
-varnish v1 -expect n_ban == 3
+varnish v1 -expect bans == 3
diff --git a/bin/varnishtest/tests/r00917.vtc b/bin/varnishtest/tests/r00917.vtc
index 6591079..9791034 100644
--- a/bin/varnishtest/tests/r00917.vtc
+++ b/bin/varnishtest/tests/r00917.vtc
@@ -21,4 +21,4 @@ foo
 
 varnish v1 -cliok ban.list
 
-varnish v1 -expect n_ban_add == 2
+varnish v1 -expect bans_added == 2
diff --git a/bin/varnishtest/tests/r01030.vtc b/bin/varnishtest/tests/r01030.vtc
index 97ef3d7..8351d81 100644
--- a/bin/varnishtest/tests/r01030.vtc
+++ b/bin/varnishtest/tests/r01030.vtc
@@ -26,7 +26,7 @@ varnish v1 -vcl+backend {
 } -start
 
 varnish v1 -cliok "param.set ban_lurker_sleep 0.01"
-varnish v1 -expect n_ban_obj_test == 0
+varnish v1 -expect bans_tests_tested == 0
 
 delay 0.01
 client c1 {
@@ -40,10 +40,10 @@ client c1 {
 } -run
 
 delay 0.1
-varnish v1 -expect n_ban_obj_test == 0
+varnish v1 -expect bans_tests_tested == 0
 
 delay 1.0
-varnish v1 -expect n_ban_obj_test == 1
+varnish v1 -expect bans_tests_tested == 1
 
 varnish v1 -cliok "param.set ban_lurker_sleep 5.01"
 
@@ -58,7 +58,7 @@ client c2 {
 } -run
 
 delay 0.1
-varnish v1 -expect n_ban_obj_test == 1
+varnish v1 -expect bans_tests_tested == 1
 
 delay 1.1
-varnish v1 -expect n_ban_obj_test == 2
+varnish v1 -expect bans_tests_tested == 2
diff --git a/include/tbl/vsc_fields.h b/include/tbl/vsc_fields.h
index abc5d18..738703c 100644
--- a/include/tbl/vsc_fields.h
+++ b/include/tbl/vsc_fields.h
@@ -41,6 +41,10 @@
  *    e - Explantion:	Short explanation of field (for screen use)
  *    d - Description:	Long explanation of field (for doc use)
  *
+ * Please describe Gauge variables as "Number of..." to indicate that
+ * this is a snapshot, and Counter variables as "Count of" to indicate
+ * accumulative count.
+ *
  * -----------------------
  * NB: Cleanup in progress
  * -----------------------
@@ -297,13 +301,49 @@ VSC_F(n_vcl,		uint64_t, 0, 'a', "N vcl total", "")
 VSC_F(n_vcl_avail,		uint64_t, 0, 'a', "N vcl available", "")
 VSC_F(n_vcl_discard,	uint64_t, 0, 'a', "N vcl discarded", "")
 
-VSC_F(n_ban,			uint64_t, 0, 'i', "N total active bans", "")
-VSC_F(n_ban_gone,		uint64_t, 0, 'i', "N total gone bans", "")
-VSC_F(n_ban_add,		uint64_t, 0, 'a', "N new bans added", "")
-VSC_F(n_ban_retire,		uint64_t, 0, 'a', "N old bans deleted", "")
-VSC_F(n_ban_obj_test,		uint64_t, 0, 'a', "N objects tested", "")
-VSC_F(n_ban_re_test,		uint64_t, 0, 'a', "N regexps tested against", "")
-VSC_F(n_ban_dups,		uint64_t, 0, 'a', "N duplicate bans removed", "")
+/**********************************************************************/
+
+VSC_F(bans,			uint64_t, 0, 'g',
+   "Count of bans",
+	"Number of all bans in system, including bans superseded"
+	" by newer bans and bans already checked by the ban-lurker."
+)
+VSC_F(bans_gone,		uint64_t, 0, 'g',
+    "Number of bans marked 'gone'",
+	"Number of bans which are no longer active, either because they"
+	" got checked by the ban-lurker or superseded by newer identical bans."
+)
+VSC_F(bans_req,			uint64_t, 0, 'g',
+    "Number of bans using req.*",
+	"Number of bans which use req.* variables.  These bans can not"
+	" be washed by the ban-lurker."
+)
+VSC_F(bans_added,		uint64_t, 0, 'c',
+    "Bans added",
+	"Counter of bans added to ban list."
+)
+VSC_F(bans_deleted,		uint64_t, 0, 'c',
+    "Bans deleted",
+	"Counter of bans deleted from ban list."
+)
+
+VSC_F(bans_tested,		uint64_t, 0, 'c',
+    "Bans tested against objects",
+	"Count of how many bans and objects have been tested against"
+	" each other."
+)
+VSC_F(bans_tests_tested,	uint64_t, 0, 'c',
+    "Ban tests tested against objects",
+	"Count of how many tests and objects have been tested against"
+	" each other.  'ban req.url == foo && req.http.host == bar'"
+	" counts as one in 'bans_tested' and as two in 'bans_tests_tested'"
+)
+VSC_F(bans_dups,		uint64_t, 0, 'c',
+    "Bans superseded by other bans",
+	"Count of bans replaced by later identical bans."
+)
+
+/**********************************************************************/
 
 VSC_F(hcb_nolock,		uint64_t, 0, 'a', "HCB Lookups without lock", "")
 VSC_F(hcb_lock,		uint64_t, 0, 'a', "HCB Lookups with lock", "")



From geoff at varnish-cache.org  Mon Jan  9 20:52:28 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:28 +0100
Subject: [experimental-ims] dd4d545 Merge branch 'master' of
	ssh://git.varnish-cache.org/git/varnish-cache
Message-ID: <E1RkMCi-00081U-RF@project.varnish-software.com>

commit dd4d54557c7b862f762bbf8eabac3f9fe54c64b4
Merge: d8b670d 374c3a2
Author: Per Buer <perbu at varnish-software.com>
Date:   Thu Nov 3 09:24:03 2011 +0100

    Merge branch 'master' of ssh://git.varnish-cache.org/git/varnish-cache



From geoff at varnish-cache.org  Mon Jan  9 20:52:31 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:31 +0100
Subject: [experimental-ims] 4805cac Clean up some formatting leading to error
	messages in rst2man that made it into the man pages
Message-ID: <E1RkMCl-00087U-Gq@project.varnish-software.com>

commit 4805cacd6166d8b7f57d61aa016ab61d0556b1ee
Author: Lasse Karstensen <lasse at varnish-software.com>
Date:   Thu Nov 10 11:00:45 2011 +0100

    Clean up some formatting leading to error messages in rst2man that made it into the man pages
    
    Conflicts:
    
    	bin/varnishd/mgt/mgt_param.c

diff --git a/bin/varnishd/mgt/mgt_param.c b/bin/varnishd/mgt/mgt_param.c
index e2eb16b..44f3b9b 100644
--- a/bin/varnishd/mgt/mgt_param.c
+++ b/bin/varnishd/mgt/mgt_param.c
@@ -711,6 +711,7 @@ static const struct parspec input_parspec[] = {
 		"  0x00000002 - Ignore non-esi elements\n"
 		"  0x00000004 - Emit parsing debug records\n"
 		"  0x00000008 - Force-split parser input (debugging)\n"
+		"\n"
 		"Use 0x notation and do the bitor in your head :-)\n",
 		0,
 		"0", "bitmap" },
@@ -843,6 +844,7 @@ static const struct parspec input_parspec[] = {
 		"  0x00040000 - release VCL early.\n"
 		"  0x00080000 - ban-lurker debugging.\n"
 		"  0x80000000 - do edge-detection on digest.\n"
+		"\n"
 		"Use 0x notation and do the bitor in your head :-)\n",
 		0,
 		"0", "bitmap" },
@@ -880,7 +882,7 @@ static const struct parspec input_parspec[] = {
 		"uncompress compressed objects on demand. Varnish will also "
 		"rewrite the Accept-Encoding header of clients indicating "
 		"support for gzip to:\n"
-		"Accept-Encoding: gzip\n\n"
+		"  Accept-Encoding: gzip\n\n"
 		"Clients that do not support gzip will have their "
 		"Accept-Encoding header removed. For more information on how "
 		"gzip is implemented please see the chapter on gzip in the "
@@ -891,6 +893,7 @@ static const struct parspec input_parspec[] = {
 		"Where temporary space for gzip/gunzip is allocated:\n"
 		"  0 - malloc\n"
 		"  2 - thread workspace\n"
+		"\n"
 		"If you have much gzip/gunzip activity, it may be an"
 		" advantage to use workspace for these allocations to reduce"
 		" malloc activity.  Be aware that gzip needs 256+KB and gunzip"
@@ -1186,6 +1189,7 @@ MCF_DumpRst(void)
 	const char *p, *q;
 	int i;
 
+	printf("\n.. The following is the autogenerated output from varnishd -x dumprst\n\n");
 	for (i = 0; i < nparspec; i++) {
 		pp = parspec[i];
 		printf("%s\n", pp->name);



From geoff at varnish-cache.org  Mon Jan  9 20:52:48 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:48 +0100
Subject: [experimental-ims] 02fd3c7 Implement a getxid method that doesn't
	require full resurrection of zombie objects.
Message-ID: <E1RkMD2-0000CU-UK@project.varnish-software.com>

commit 02fd3c797265b7edbd7e5a7759b431cfd5afa91d
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Dec 6 11:03:36 2011 +0000

    Implement a getxid method that doesn't require full resurrection
    of zombie objects.

diff --git a/bin/varnishd/storage/storage_persistent_silo.c b/bin/varnishd/storage/storage_persistent_silo.c
index b7ef952..7cafe4f 100644
--- a/bin/varnishd/storage/storage_persistent_silo.c
+++ b/bin/varnishd/storage/storage_persistent_silo.c
@@ -369,6 +369,35 @@ smp_loaded_st(const struct smp_sc *sc, const struct smp_seg *sg,
  * objcore methods for persistent objects
  */
 
+static unsigned __match_proto__(getxid_f)
+smp_oc_getxid(struct worker *wrk, struct objcore *oc)
+{
+	struct object *o;
+	struct smp_seg *sg;
+	struct smp_object *so;
+
+	(void)wrk;
+	CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
+
+	CAST_OBJ_NOTNULL(sg, oc->priv, SMP_SEG_MAGIC);
+	so = smp_find_so(sg, oc->priv2);
+
+	o = (void*)(sg->sc->base + so->ptr);
+	/*
+	 * The object may not be in this segment since we allocate it
+	 * In a separate operation than the smp_object.  We could check
+	 * that it is in a later segment, but that would be complicated.
+	 * XXX: For now, be happy if it is inside th silo
+	 */
+	ASSERT_PTR_IN_SILO(sg->sc, o);
+	CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC);
+	return (o->xid);
+}
+
+/*---------------------------------------------------------------------
+ * objcore methods for persistent objects
+ */
+
 static struct object *
 smp_oc_getobj(struct worker *wrk, struct objcore *oc)
 {
@@ -506,6 +535,7 @@ smp_oc_getlru(const struct objcore *oc)
 }
 
 static struct objcore_methods smp_oc_methods = {
+	.getxid =		smp_oc_getxid,
 	.getobj =		smp_oc_getobj,
 	.updatemeta =		smp_oc_updatemeta,
 	.freeobj =		smp_oc_freeobj,



From geoff at varnish-cache.org  Mon Jan  9 20:52:16 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:16 +0100
Subject: [experimental-ims] 58db1e6 Isolate VTCP in its own include file
Message-ID: <E1RkMCW-0007aq-15@project.varnish-software.com>

commit 58db1e659a6b6055e27246047ec28218f991d2b2
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sun Oct 9 18:02:15 2011 +0000

    Isolate VTCP in its own include file

diff --git a/bin/varnishd/cache_acceptor.c b/bin/varnishd/cache_acceptor.c
index af5edec..87a9180 100644
--- a/bin/varnishd/cache_acceptor.c
+++ b/bin/varnishd/cache_acceptor.c
@@ -33,6 +33,7 @@
 #include "cache.h"
 
 #include "vcli.h"
+#include "vtcp.h"
 #include "cli_priv.h"
 
 static pthread_t	VCA_thread;
diff --git a/bin/varnishd/cache_backend.c b/bin/varnishd/cache_backend.c
index a724923..3ed7fcd 100644
--- a/bin/varnishd/cache_backend.c
+++ b/bin/varnishd/cache_backend.c
@@ -39,6 +39,7 @@
 
 #include "cache_backend.h"
 #include "vrt.h"
+#include "vtcp.h"
 
 /*--------------------------------------------------------------------
  * The "simple" director really isn't, since thats where all the actual
diff --git a/bin/varnishd/cache_backend_poll.c b/bin/varnishd/cache_backend_poll.c
index be8ee6a..c90cda9 100644
--- a/bin/varnishd/cache_backend_poll.c
+++ b/bin/varnishd/cache_backend_poll.c
@@ -45,6 +45,7 @@
 #include "cli_priv.h"
 #include "cache.h"
 #include "vrt.h"
+#include "vtcp.h"
 #include "cache_backend.h"
 
 /* Default averaging rate, we want something pretty responsive */
diff --git a/bin/varnishd/cache_center.c b/bin/varnishd/cache_center.c
index 0b7f8bf..1415170 100644
--- a/bin/varnishd/cache_center.c
+++ b/bin/varnishd/cache_center.c
@@ -70,6 +70,7 @@ DOT acceptor -> start [style=bold,color=green]
 #include "cache.h"
 
 #include "vcl.h"
+#include "vtcp.h"
 #include "cli_priv.h"
 #include "hash_slinger.h"
 #include "stevedore.h"
diff --git a/bin/varnishd/cache_dir.c b/bin/varnishd/cache_dir.c
index 03fdf26..ef882bf 100644
--- a/bin/varnishd/cache_dir.c
+++ b/bin/varnishd/cache_dir.c
@@ -34,6 +34,7 @@
 
 #include "cache.h"
 #include "cache_backend.h"
+#include "vtcp.h"
 
 /* Close a connection ------------------------------------------------*/
 
diff --git a/bin/varnishd/cache_fetch.c b/bin/varnishd/cache_fetch.c
index df56c7a..d18e726 100644
--- a/bin/varnishd/cache_fetch.c
+++ b/bin/varnishd/cache_fetch.c
@@ -38,6 +38,7 @@
 #include "stevedore.h"
 #include "cli_priv.h"
 #include "vct.h"
+#include "vtcp.h"
 
 static unsigned fetchfrag;
 
diff --git a/bin/varnishd/cache_pipe.c b/bin/varnishd/cache_pipe.c
index e02da80..74343ba 100644
--- a/bin/varnishd/cache_pipe.c
+++ b/bin/varnishd/cache_pipe.c
@@ -35,6 +35,7 @@
 #include <poll.h>
 
 #include "cache.h"
+#include "vtcp.h"
 
 static int
 rdf(int fd0, int fd1)
diff --git a/bin/varnishd/cache_pool.c b/bin/varnishd/cache_pool.c
index 0380b55..e146043 100644
--- a/bin/varnishd/cache_pool.c
+++ b/bin/varnishd/cache_pool.c
@@ -50,6 +50,7 @@
 
 #include "cache_waiter.h"
 #include "hash_slinger.h"
+#include "vtcp.h"
 
 /*--------------------------------------------------------------------
  * MAC OS/X is incredibly moronic when it comes to time and such...
diff --git a/bin/varnishd/cache_vrt_var.c b/bin/varnishd/cache_vrt_var.c
index ae65058..1343ae4 100644
--- a/bin/varnishd/cache_vrt_var.c
+++ b/bin/varnishd/cache_vrt_var.c
@@ -36,6 +36,7 @@
 #include "cache.h"
 
 #include "vrt.h"
+#include "vtcp.h"
 #include "vrt_obj.h"
 #include "cache_backend.h"
 #include "hash_slinger.h"
diff --git a/bin/varnishd/mgt_child.c b/bin/varnishd/mgt_child.c
index ba2effe..6c1cf71 100644
--- a/bin/varnishd/mgt_child.c
+++ b/bin/varnishd/mgt_child.c
@@ -52,6 +52,7 @@
 #include "vlu.h"
 #include "vss.h"
 #include "vbm.h"
+#include "vtcp.h"
 
 pid_t		child_pid = -1;
 
diff --git a/bin/varnishd/mgt_cli.c b/bin/varnishd/mgt_cli.c
index eccf268..4488d51 100644
--- a/bin/varnishd/mgt_cli.c
+++ b/bin/varnishd/mgt_cli.c
@@ -51,6 +51,7 @@
 #include "vev.h"
 #include "vlu.h"
 #include "vss.h"
+#include "vtcp.h"
 
 
 #include "mgt.h"
diff --git a/bin/varnishtest/vtc_client.c b/bin/varnishtest/vtc_client.c
index 2c75622..55e25fa 100644
--- a/bin/varnishtest/vtc_client.c
+++ b/bin/varnishtest/vtc_client.c
@@ -38,6 +38,7 @@
 #include "vtc.h"
 
 #include "vss.h"
+#include "vtcp.h"
 #include "libvarnish.h"
 
 struct client {
diff --git a/bin/varnishtest/vtc_http.c b/bin/varnishtest/vtc_http.c
index 4582d4c..50816bc 100644
--- a/bin/varnishtest/vtc_http.c
+++ b/bin/varnishtest/vtc_http.c
@@ -41,6 +41,7 @@
 #include "libvarnish.h"
 
 #include "vtc.h"
+#include "vtcp.h"
 
 #include "vgz.h"
 
diff --git a/bin/varnishtest/vtc_server.c b/bin/varnishtest/vtc_server.c
index d541d67..afe62af 100644
--- a/bin/varnishtest/vtc_server.c
+++ b/bin/varnishtest/vtc_server.c
@@ -37,6 +37,7 @@
 #include <sys/socket.h>
 
 #include "vtc.h"
+#include "vtcp.h"
 
 #include "libvarnish.h"
 #include "vss.h"
diff --git a/bin/varnishtest/vtc_varnish.c b/bin/varnishtest/vtc_varnish.c
index 7362f1f..8b310f6 100644
--- a/bin/varnishtest/vtc_varnish.c
+++ b/bin/varnishtest/vtc_varnish.c
@@ -46,6 +46,7 @@
 #include <sys/socket.h>
 
 #include "vtc.h"
+#include "vtcp.h"
 
 #include "libvarnish.h"
 
diff --git a/include/Makefile.am b/include/Makefile.am
index 0f1782f..66c2ddd 100644
--- a/include/Makefile.am
+++ b/include/Makefile.am
@@ -55,7 +55,8 @@ nobase_noinst_HEADERS = \
 	vrt_obj.h \
 	vsb.h \
 	vsha256.h \
-	vss.h
+	vss.h \
+	vtcp.h
 
 tbl/vrt_stv_var.h tbl/vcl_returns.h vcl.h vrt_obj.h: $(top_srcdir)/lib/libvcl/generate.py $(top_srcdir)/include/vrt.h
 	mkdir -p tbl
diff --git a/include/libvarnish.h b/include/libvarnish.h
index b016848..6008781 100644
--- a/include/libvarnish.h
+++ b/include/libvarnish.h
@@ -48,40 +48,6 @@ typedef void sub_func_f(void*);
 int SUB_run(struct vsb *sb, sub_func_f *func, void *priv, const char *name,
     int maxlines);
 
-/* from libvarnish/tcp.c */
-/* NI_MAXHOST and NI_MAXSERV are ridiculously long for numeric format */
-#define VTCP_ADDRBUFSIZE		64
-#define VTCP_PORTBUFSIZE		16
-
-#if (defined (__SVR4) && defined (__sun)) || defined (__NetBSD__)
-/*
- * Solaris returns EINVAL if the other end unexepectedly reset the
- * connection.  This is a bug in Solaris and documented behaviour on NetBSD.
- */
-#define VTCP_Check(a) ((a) == 0 || errno == ECONNRESET || errno == ENOTCONN \
-    || errno == EINVAL)
-#else
-#define VTCP_Check(a) ((a) == 0 || errno == ECONNRESET || errno == ENOTCONN)
-#endif
-
-#define VTCP_Assert(a) assert(VTCP_Check(a))
-
-void VTCP_myname(int sock, char *abuf, unsigned alen, char *pbuf, unsigned plen);
-void VTCP_hisname(int sock, char *abuf, unsigned alen, char *pbuf, unsigned plen);
-int VTCP_filter_http(int sock);
-int VTCP_blocking(int sock);
-int VTCP_nonblocking(int sock);
-int VTCP_linger(int sock, int linger);
-#ifdef SOL_SOCKET
-int VTCP_port(const struct sockaddr_storage *addr);
-void VTCP_name(const struct sockaddr_storage *addr, unsigned l, char *abuf,
-    unsigned alen, char *pbuf, unsigned plen);
-int VTCP_connect(int s, const struct sockaddr_storage *name, socklen_t namelen,
-    int msec);
-void VTCP_close(int *s);
-void VTCP_set_read_timeout(int s, double seconds);
-#endif
-
 /* from libvarnish/time.c */
 #define TIM_FORMAT_SIZE 30
 void TIM_format(double t, char *p);
diff --git a/include/vtcp.h b/include/vtcp.h
new file mode 100644
index 0000000..022f101
--- /dev/null
+++ b/include/vtcp.h
@@ -0,0 +1,64 @@
+/*-
+ * Copyright (c) 2006 Verdens Gang AS
+ * Copyright (c) 2006-2011 Varnish Software AS
+ * All rights reserved.
+ *
+ * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ */
+
+/* from libvarnish/tcp.c */
+/* NI_MAXHOST and NI_MAXSERV are ridiculously long for numeric format */
+#define VTCP_ADDRBUFSIZE		64
+#define VTCP_PORTBUFSIZE		16
+
+#if (defined (__SVR4) && defined (__sun)) || defined (__NetBSD__)
+/*
+ * Solaris returns EINVAL if the other end unexepectedly reset the
+ * connection.  This is a bug in Solaris and documented behaviour on NetBSD.
+ */
+#define VTCP_Check(a) ((a) == 0 || errno == ECONNRESET || errno == ENOTCONN \
+    || errno == EINVAL)
+#else
+#define VTCP_Check(a) ((a) == 0 || errno == ECONNRESET || errno == ENOTCONN)
+#endif
+
+#define VTCP_Assert(a) assert(VTCP_Check(a))
+
+void VTCP_myname(int sock, char *abuf, unsigned alen, char *pbuf, unsigned plen);
+void VTCP_hisname(int sock, char *abuf, unsigned alen, char *pbuf, unsigned plen);
+int VTCP_filter_http(int sock);
+int VTCP_blocking(int sock);
+int VTCP_nonblocking(int sock);
+int VTCP_linger(int sock, int linger);
+
+#ifdef SOL_SOCKET
+int VTCP_port(const struct sockaddr_storage *addr);
+void VTCP_name(const struct sockaddr_storage *addr, unsigned l, char *abuf,
+    unsigned alen, char *pbuf, unsigned plen);
+int VTCP_connect(int s, const struct sockaddr_storage *name, socklen_t namelen,
+    int msec);
+void VTCP_close(int *s);
+void VTCP_set_read_timeout(int s, double seconds);
+#endif
diff --git a/lib/libvarnish/tcp.c b/lib/libvarnish/tcp.c
index c612e97..e8932e8 100644
--- a/lib/libvarnish/tcp.c
+++ b/lib/libvarnish/tcp.c
@@ -51,8 +51,7 @@
 #include <unistd.h>
 #include <math.h>
 
-#include "config.h"
-
+#include "vtcp.h"
 #include "libvarnish.h"
 
 /*--------------------------------------------------------------------*/
diff --git a/lib/libvarnish/vss.c b/lib/libvarnish/vss.c
index 42b4db7..ff19ac7 100644
--- a/lib/libvarnish/vss.c
+++ b/lib/libvarnish/vss.c
@@ -44,6 +44,7 @@
 #include <unistd.h>
 
 #include "libvarnish.h"
+#include "vtcp.h"
 #include "vss.h"
 
 /* lightweight addrinfo */
diff --git a/lib/libvmod_std/vmod_std.c b/lib/libvmod_std/vmod_std.c
index 544e2dc..847c705 100644
--- a/lib/libvmod_std/vmod_std.c
+++ b/lib/libvmod_std/vmod_std.c
@@ -32,6 +32,7 @@
 #include <syslog.h>
 #include <netinet/in.h>
 #include "vrt.h"
+#include "vtcp.h"
 #include "../../bin/varnishd/cache.h"
 
 #include "vcc_if.h"



From geoff at varnish-cache.org  Mon Jan  9 20:51:55 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:51:55 +0100
Subject: [experimental-ims] fc59c74 Give each pool a list of sockets to
	accept() from
Message-ID: <E1RkMCB-0006yv-Lg@project.varnish-software.com>

commit fc59c7455129e230fbead7627d522f6a7e8c07d1
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sat Sep 17 16:57:20 2011 +0000

    Give each pool a list of sockets to accept() from

diff --git a/bin/varnishd/cache_pool.c b/bin/varnishd/cache_pool.c
index 6fc7737..05ce529 100644
--- a/bin/varnishd/cache_pool.c
+++ b/bin/varnishd/cache_pool.c
@@ -62,6 +62,13 @@ static void *waiter_priv;
 
 VTAILQ_HEAD(workerhead, worker);
 
+struct poolsock {
+	unsigned			magic;
+#define POOLSOCK_MAGIC			0x1b0a2d38
+	VTAILQ_ENTRY(poolsock)		list;
+	int				sock;
+};
+
 /* Number of work requests queued in excess of worker threads available */
 
 struct pool {
@@ -70,6 +77,7 @@ struct pool {
 	struct lock		mtx;
 	struct workerhead	idle;
 	VTAILQ_HEAD(, sess)	queue;
+	VTAILQ_HEAD(, poolsock)	socks;
 	unsigned		nthr;
 	unsigned		lqueue;
 	unsigned		last_lqueue;
@@ -237,6 +245,29 @@ Pool_QueueSession(struct sess *sp)
  * Add (more) thread pools
  */
 
+static struct pool *
+pool_mkpool(void)
+{
+	struct pool *pp;
+	struct listen_sock *ls;
+	struct poolsock *ps;
+
+	ALLOC_OBJ(pp, POOL_MAGIC);
+	XXXAN(pp);
+	Lck_New(&pp->mtx, lck_wq);
+	VTAILQ_INIT(&pp->queue);
+	VTAILQ_INIT(&pp->idle);
+	VTAILQ_INIT(&pp->socks);
+
+	VTAILQ_FOREACH(ls, &heritage.socks, list) {
+		ALLOC_OBJ(ps, POOLSOCK_MAGIC);
+		XXXAN(ps);
+		ps->sock = ls->sock;
+		VTAILQ_INSERT_TAIL(&pp->socks, ps, list);
+	}
+	return (pp);
+}
+
 static void
 wrk_addpools(const unsigned pools)
 {
@@ -251,12 +282,8 @@ wrk_addpools(const unsigned pools)
 	owq = wq;
 	wq = pwq;
 	for (u = nwq; u < pools; u++) {
-		wq[u] = calloc(sizeof *wq[0], 1);
+		wq[u] = pool_mkpool();
 		XXXAN(wq[u]);
-		wq[u]->magic = POOL_MAGIC;
-		Lck_New(&wq[u]->mtx, lck_wq);
-		VTAILQ_INIT(&wq[u]->queue);
-		VTAILQ_INIT(&wq[u]->idle);
 	}
 	(void)owq;	/* XXX: avoid race, leak it. */
 	nwq = pools;



From geoff at varnish-cache.org  Mon Jan  9 20:52:25 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:25 +0100
Subject: [experimental-ims] 5e673e1 Start to push struct sess out of the
	"fetch non-esi body from backend" code path,
	to pave the road for Martins streaming code.
Message-ID: <E1RkMCf-0007td-EG@project.varnish-software.com>

commit 5e673e14c020a4cf3ce2f246268a0fbfd0de7bfb
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Oct 24 09:50:18 2011 +0000

    Start to push struct sess out of the "fetch non-esi body from backend"
    code path, to pave the road for Martins streaming code.

diff --git a/bin/varnishd/cache.h b/bin/varnishd/cache.h
index e94180a..018355e 100644
--- a/bin/varnishd/cache.h
+++ b/bin/varnishd/cache.h
@@ -710,8 +710,8 @@ void Fetch_Init(void);
 struct vgz;
 
 enum vgz_flag { VGZ_NORMAL, VGZ_ALIGN, VGZ_RESET, VGZ_FINISH };
-struct vgz *VGZ_NewUngzip(struct sess *sp, const char *id);
-struct vgz *VGZ_NewGzip(struct sess *sp, const char *id);
+struct vgz *VGZ_NewUngzip(struct worker *wrk, int vsl_id, const char *id);
+struct vgz *VGZ_NewGzip(struct worker *wrk, int vsl_id, const char *id);
 void VGZ_Ibuf(struct vgz *, const void *, ssize_t len);
 int VGZ_IbufEmpty(const struct vgz *vg);
 void VGZ_Obuf(struct vgz *, void *, ssize_t len);
diff --git a/bin/varnishd/cache_center.c b/bin/varnishd/cache_center.c
index fd3238d..d60f17e 100644
--- a/bin/varnishd/cache_center.c
+++ b/bin/varnishd/cache_center.c
@@ -901,7 +901,7 @@ cnt_streambody(struct sess *sp)
 	sp->wrk->sctx = &sctx;
 
 	if (sp->wrk->res_mode & RES_GUNZIP) {
-		sctx.vgz = VGZ_NewUngzip(sp, "U S -");
+		sctx.vgz = VGZ_NewUngzip(sp->wrk, sp->vsl_id, "U S -");
 		sctx.obuf = obuf;
 		sctx.obuf_len = sizeof (obuf);
 	}
diff --git a/bin/varnishd/cache_esi_deliver.c b/bin/varnishd/cache_esi_deliver.c
index f7daeeb..aaf5f7a 100644
--- a/bin/varnishd/cache_esi_deliver.c
+++ b/bin/varnishd/cache_esi_deliver.c
@@ -267,7 +267,7 @@ ESI_Deliver(struct sess *sp)
 	}
 
 	if (isgzip && !sp->wrk->gzip_resp) {
-		vgz = VGZ_NewUngzip(sp, "U D E");
+		vgz = VGZ_NewUngzip(sp->wrk, sp->vsl_id, "U D E");
 
 		/* Feed a gzip header to gunzip to make it happy */
 		VGZ_Ibuf(vgz, gzip_hdr, sizeof gzip_hdr);
diff --git a/bin/varnishd/cache_esi_fetch.c b/bin/varnishd/cache_esi_fetch.c
index 1db5259..f69e900 100644
--- a/bin/varnishd/cache_esi_fetch.c
+++ b/bin/varnishd/cache_esi_fetch.c
@@ -301,7 +301,7 @@ vfp_esi_begin(struct sess *sp, size_t estimate)
 
 	AZ(sp->wrk->vgz_rx);
 	if (sp->wrk->is_gzip && sp->wrk->do_gunzip) {
-		sp->wrk->vgz_rx = VGZ_NewUngzip(sp, "U F E");
+		sp->wrk->vgz_rx = VGZ_NewUngzip(sp->wrk, sp->vsl_id, "U F E");
 		VEP_Init(sp, NULL);
 	} else if (sp->wrk->is_gunzip && sp->wrk->do_gzip) {
 		ALLOC_OBJ(vef, VEF_MAGIC);
@@ -309,18 +309,18 @@ vfp_esi_begin(struct sess *sp, size_t estimate)
 		//vef = (void*)WS_Alloc(sp->ws, sizeof *vef);
 		//memset(vef, 0, sizeof *vef);
 		//vef->magic = VEF_MAGIC;
-		vef->vgz = VGZ_NewGzip(sp, "G F E");
+		vef->vgz = VGZ_NewGzip(sp->wrk, sp->vsl_id, "G F E");
 		AZ(sp->wrk->vef_priv);
 		sp->wrk->vef_priv = vef;
 		VEP_Init(sp, vfp_vep_callback);
 	} else if (sp->wrk->is_gzip) {
-		sp->wrk->vgz_rx = VGZ_NewUngzip(sp, "U F E");
+		sp->wrk->vgz_rx = VGZ_NewUngzip(sp->wrk, sp->vsl_id, "U F E");
 		ALLOC_OBJ(vef, VEF_MAGIC);
 		AN(vef);
 		//vef = (void*)WS_Alloc(sp->ws, sizeof *vef);
 		//memset(vef, 0, sizeof *vef);
 		//vef->magic = VEF_MAGIC;
-		vef->vgz = VGZ_NewGzip(sp, "G F E");
+		vef->vgz = VGZ_NewGzip(sp->wrk, sp->vsl_id, "G F E");
 		AZ(sp->wrk->vef_priv);
 		sp->wrk->vef_priv = vef;
 		VEP_Init(sp, vfp_vep_callback);
diff --git a/bin/varnishd/cache_gzip.c b/bin/varnishd/cache_gzip.c
index 156ac52..787ff16 100644
--- a/bin/varnishd/cache_gzip.c
+++ b/bin/varnishd/cache_gzip.c
@@ -77,7 +77,8 @@ struct vgz {
 	unsigned		magic;
 #define VGZ_MAGIC		0x162df0cb
 	enum {VGZ_GZ,VGZ_UN}	dir;
-	struct sess		*sess;
+	struct worker		*wrk;
+	int			vsl_id;
 	const char		*id;
 	struct ws		*tmp;
 	char			*tmp_snapshot;
@@ -113,11 +114,13 @@ vgz_free(voidpf opaque, voidpf address)
  */
 
 static struct vgz *
-vgz_alloc_vgz(struct sess *sp, const char *id)
+vgz_alloc_vgz(struct worker *wrk, int vsl_id, const char *id)
 {
 	struct vgz *vg;
-	struct ws *ws = sp->wrk->ws;
+	struct ws *ws;
 
+	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
+	ws = wrk->ws;
 	WS_Assert(ws);
 	// XXX: we restore workspace in esi:include
 	// vg = (void*)WS_Alloc(ws, sizeof *vg);
@@ -125,22 +128,17 @@ vgz_alloc_vgz(struct sess *sp, const char *id)
 	AN(vg);
 	memset(vg, 0, sizeof *vg);
 	vg->magic = VGZ_MAGIC;
-	vg->sess = sp;
+	vg->wrk = wrk;
+	vg->vsl_id = vsl_id;
 	vg->id = id;
 
 	switch (params->gzip_tmp_space) {
 	case 0:
-		/* malloc, the default */
-		break;
 	case 1:
-		vg->tmp = sp->ws;
-		vg->tmp_snapshot = WS_Snapshot(vg->tmp);
-		vg->vz.zalloc = vgz_alloc;
-		vg->vz.zfree = vgz_free;
-		vg->vz.opaque = vg;
+		/* malloc, the default */
 		break;
 	case 2:
-		vg->tmp = sp->wrk->ws;
+		vg->tmp = wrk->ws;
 		vg->tmp_snapshot = WS_Snapshot(vg->tmp);
 		vg->vz.zalloc = vgz_alloc;
 		vg->vz.zfree = vgz_free;
@@ -153,12 +151,12 @@ vgz_alloc_vgz(struct sess *sp, const char *id)
 }
 
 struct vgz *
-VGZ_NewUngzip(struct sess *sp, const char *id)
+VGZ_NewUngzip(struct worker *wrk, int vsl_id, const char *id)
 {
 	struct vgz *vg;
 
-	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
-	vg = vgz_alloc_vgz(sp, id);
+	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
+	vg = vgz_alloc_vgz(wrk, vsl_id, id);
 	vg->dir = VGZ_UN;
 	VSC_C_main->n_gunzip++;
 
@@ -173,13 +171,13 @@ VGZ_NewUngzip(struct sess *sp, const char *id)
 }
 
 struct vgz *
-VGZ_NewGzip(struct sess *sp, const char *id)
+VGZ_NewGzip(struct worker *wrk, int vsl_id, const char *id)
 {
 	struct vgz *vg;
 	int i;
 
-	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
-	vg = vgz_alloc_vgz(sp, id);
+	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
+	vg = vgz_alloc_vgz(wrk, vsl_id, id);
 	vg->dir = VGZ_GZ;
 	VSC_C_main->n_gzip++;
 
@@ -413,7 +411,7 @@ VGZ_Destroy(struct vgz **vgp)
 	CHECK_OBJ_NOTNULL(vg, VGZ_MAGIC);
 	*vgp = NULL;
 
-	WSP(vg->sess, SLT_Gzip, "%s %jd %jd %jd %jd %jd",
+	WSL(vg->wrk, SLT_Gzip, vg->vsl_id, "%s %jd %jd %jd %jd %jd",
 	    vg->id,
 	    (intmax_t)vg->vz.total_in,
 	    (intmax_t)vg->vz.total_out,
@@ -440,7 +438,7 @@ vfp_gunzip_begin(struct sess *sp, size_t estimate)
 {
 	(void)estimate;
 	AZ(sp->wrk->vgz_rx);
-	sp->wrk->vgz_rx = VGZ_NewUngzip(sp, "U F -");
+	sp->wrk->vgz_rx = VGZ_NewUngzip(sp->wrk, sp->vsl_id, "U F -");
 }
 
 static int __match_proto__()
@@ -516,7 +514,7 @@ vfp_gzip_begin(struct sess *sp, size_t estimate)
 	(void)estimate;
 
 	AZ(sp->wrk->vgz_rx);
-	sp->wrk->vgz_rx = VGZ_NewGzip(sp, "G F -");
+	sp->wrk->vgz_rx = VGZ_NewGzip(sp->wrk, sp->vsl_id, "G F -");
 }
 
 static int __match_proto__()
@@ -598,7 +596,7 @@ static void __match_proto__()
 vfp_testgzip_begin(struct sess *sp, size_t estimate)
 {
 	(void)estimate;
-	sp->wrk->vgz_rx = VGZ_NewUngzip(sp, "u F -");
+	sp->wrk->vgz_rx = VGZ_NewUngzip(sp->wrk, sp->vsl_id, "u F -");
 	CHECK_OBJ_NOTNULL(sp->wrk->vgz_rx, VGZ_MAGIC);
 }
 
diff --git a/bin/varnishd/cache_response.c b/bin/varnishd/cache_response.c
index 4f0334b..5521bc0 100644
--- a/bin/varnishd/cache_response.c
+++ b/bin/varnishd/cache_response.c
@@ -151,7 +151,7 @@ RES_BuildHttp(const struct sess *sp)
  */
 
 static void
-res_WriteGunzipObj(struct sess *sp)
+res_WriteGunzipObj(const struct sess *sp)
 {
 	struct storage *st;
 	unsigned u = 0;
@@ -162,7 +162,7 @@ res_WriteGunzipObj(struct sess *sp)
 
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 
-	vg = VGZ_NewUngzip(sp, "U D -");
+	vg = VGZ_NewUngzip(sp->wrk, sp->vsl_id, "U D -");
 
 	VGZ_Obuf(vg, obuf, sizeof obuf);
 	VTAILQ_FOREACH(st, &sp->obj->store, list) {
diff --git a/bin/varnishd/mgt/mgt_param.c b/bin/varnishd/mgt/mgt_param.c
index 1ca6a74..8665ff5 100644
--- a/bin/varnishd/mgt/mgt_param.c
+++ b/bin/varnishd/mgt/mgt_param.c
@@ -882,7 +882,6 @@ static const struct parspec input_parspec[] = {
 	{ "gzip_tmp_space", tweak_uint, &master.gzip_tmp_space, 0, 2,
 		"Where temporary space for gzip/gunzip is allocated:\n"
 		"  0 - malloc\n"
-		"  1 - session workspace\n"
 		"  2 - thread workspace\n"
 		"If you have much gzip/gunzip activity, it may be an"
 		" advantage to use workspace for these allocations to reduce"



From geoff at varnish-cache.org  Mon Jan  9 20:52:17 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:17 +0100
Subject: [experimental-ims] 7a0dc3f Add default values for some fields when
	logging imcomplete records. Allow %r format to log incomplete
	records too. Update docs to reflect new defaults
Message-ID: <E1RkMCX-0007ej-Pk@project.varnish-software.com>

commit 7a0dc3f78987b37edc927c231a42879915e9f2ac
Author: Andreas Plesner Jacobsen <apj at mutt.dk>
Date:   Mon Oct 10 14:55:55 2011 +0200

    Add default values for some fields when logging imcomplete records.
    Allow %r format to log incomplete records too.
    Update docs to reflect new defaults
    
    Fixes #1028

diff --git a/bin/varnishncsa/varnishncsa.c b/bin/varnishncsa/varnishncsa.c
index 26ede48..47e5e2a 100644
--- a/bin/varnishncsa/varnishncsa.c
+++ b/bin/varnishncsa/varnishncsa.c
@@ -603,7 +603,7 @@ h_ncsa(void *priv, enum VSL_tag_e tag, unsigned fd,
 			break;
 
 		case 'H':
-			VSB_cat(os, lp->df_H);
+			VSB_cat(os, lp->df_H ? lp->df_H : "HTTP/1.0");
 			break;
 
 		case 'h':
@@ -617,7 +617,7 @@ h_ncsa(void *priv, enum VSL_tag_e tag, unsigned fd,
 			break;
 
 		case 'm':
-			VSB_cat(os, lp->df_m);
+			VSB_cat(os, lp->df_m ? lp->df_m : "-");
 			break;
 
 		case 'q':
@@ -629,24 +629,19 @@ h_ncsa(void *priv, enum VSL_tag_e tag, unsigned fd,
 			 * Fake "%r".  This would be a lot easier if Varnish
 			 * normalized the request URL.
 			 */
-			if (!lp->df_m ||
-			    !req_header(lp, "Host") ||
-			    !lp->df_U ||
-			    !lp->df_H) {
-				clean_logline(lp);
-				return (reopen);
-			}
-			VSB_cat(os, lp->df_m);
+			VSB_cat(os, lp->df_m ? lp->df_m : "-");
 			VSB_putc(os, ' ');
 			if (req_header(lp, "Host")) {
 				if (strncmp(req_header(lp, "Host"), "http://", 7) != 0)
 					VSB_cat(os, "http://");
 				VSB_cat(os, req_header(lp, "Host"));
+			} else {
+				VSB_cat(os, "http://localhost");
 			}
-			VSB_cat(os, lp->df_U);
+			VSB_cat(os, lp->df_U ? lp->df_U : "-");
 			VSB_cat(os, lp->df_q ? lp->df_q : "");
 			VSB_putc(os, ' ');
-			VSB_cat(os, lp->df_H);
+			VSB_cat(os, lp->df_H ? lp->df_H : "HTTP/1.0");
 			break;
 
 		case 's':
@@ -661,7 +656,7 @@ h_ncsa(void *priv, enum VSL_tag_e tag, unsigned fd,
 			break;
 
 		case 'U':
-			VSB_cat(os, lp->df_U);
+			VSB_cat(os, lp->df_U ? lp->df_U : "-");
 			break;
 
 		case 'u':
diff --git a/doc/sphinx/reference/varnishncsa.rst b/doc/sphinx/reference/varnishncsa.rst
index a4bac0a..e654e37 100644
--- a/doc/sphinx/reference/varnishncsa.rst
+++ b/doc/sphinx/reference/varnishncsa.rst
@@ -62,10 +62,12 @@ The following options are available:
    	         bytes are sent.
 
 	      %H 
-	         The request protocol
+	         The request protocol. Defaults to HTTP/1.0 if not
+                 known.
 
               %h
-	         Remote host
+	         Remote host. Defaults to '-' if not known.
+                 Defaults to 127.0.0.1 for backend requests.
 
 	      %{X}i
 	         The contents of request header line X.
@@ -74,16 +76,18 @@ The following options are available:
 	         Remote logname (always '-')
 
 	      %m
-	         Request method
+	         Request method. Defaults to '-' if not known.
 
 	      %q
-	         The query string, if no query string exists, an empty string.
+	         The query string, if no query string exists, an
+                 empty string.
 
 	      %{X}o
 	         The contents of response header line X.
 
 	      %r
-	         The first line of the request
+	         The first line of the request. Synthesized from other
+                 fields, so it may not be the request verbatim.
 
 	      %s
 	         Status sent to the client
@@ -93,7 +97,8 @@ The following options are available:
 	         format.
 
 	      %U
-	         The request URL without any query string.
+	         The request URL without any query string. Defaults to
+                 '-' if not known.
 
 	      %u
 	         Remote user from auth



From geoff at varnish-cache.org  Mon Jan  9 20:52:41 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:41 +0100
Subject: [experimental-ims] a77b576 Don't let VCC compilers or other
	subprocesses kill our VSM.
Message-ID: <E1RkMCv-0008N3-0j@project.varnish-software.com>

commit a77b5767b42ddd8ee89c9b4ffd4c75522df4442d
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sun Nov 20 23:27:03 2011 +0000

    Don't let VCC compilers or other subprocesses kill our VSM.

diff --git a/bin/varnishd/mgt/mgt_shmem.c b/bin/varnishd/mgt/mgt_shmem.c
index 9f88f58..79f8c41 100644
--- a/bin/varnishd/mgt/mgt_shmem.c
+++ b/bin/varnishd/mgt/mgt_shmem.c
@@ -293,6 +293,9 @@ void
 mgt_shm_atexit(void)
 {
 
+	/* Do not let VCC kill our VSM */
+	if (getpid() != mgt_pid)
+		return;
 	if (heritage.vsm != NULL)
 		VSM_common_delete(&heritage.vsm);
 }



From geoff at varnish-cache.org  Mon Jan  9 20:52:44 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:44 +0100
Subject: [experimental-ims] 49c619b Move htc from worker to busyobj
Message-ID: <E1RkMCy-0008UU-4f@project.varnish-software.com>

commit 49c619b275799f619ea69a300e5c6b9c430da80d
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Nov 29 18:22:33 2011 +0000

    Move htc from worker to busyobj

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index 5196575..125dd53 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -315,7 +315,6 @@ struct worker {
 	/* Lookup stuff */
 	struct SHA256Context	*sha256ctx;
 
-	struct http_conn	htc[1];
 	struct ws		ws[1];
 	struct http		*bereq;
 	struct http		*beresp;
@@ -503,6 +502,7 @@ struct busyobj {
 	struct vgz		*vgz_rx;
 
 	struct exp		exp;
+	struct http_conn	htc;
 };
 
 /* Object structure --------------------------------------------------*/
diff --git a/bin/varnishd/cache/cache_esi_fetch.c b/bin/varnishd/cache/cache_esi_fetch.c
index 3dbc5ce..71dd815 100644
--- a/bin/varnishd/cache/cache_esi_fetch.c
+++ b/bin/varnishd/cache/cache_esi_fetch.c
@@ -336,7 +336,7 @@ vfp_esi_bytes(struct worker *w, struct http_conn *htc, ssize_t bytes)
 	CHECK_OBJ_NOTNULL(w, WORKER_MAGIC);
 	AZ(w->busyobj->fetch_failed);
 	AN(w->busyobj->vep);
-	assert(w->htc == htc);
+	assert(&w->busyobj->htc == htc);
 	if (w->busyobj->is_gzip && w->do_gunzip)
 		i = vfp_esi_bytes_gu(w, htc, bytes);
 	else if (w->busyobj->is_gunzip && w->do_gzip)
diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c
index 2a2de5b..0b86821 100644
--- a/bin/varnishd/cache/cache_fetch.c
+++ b/bin/varnishd/cache/cache_fetch.c
@@ -383,10 +383,13 @@ FetchHdr(struct sess *sp)
 	struct http *hp;
 	int retry = -1;
 	int i;
+	struct http_conn *htc;
 
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 	CHECK_OBJ_NOTNULL(sp->wrk, WORKER_MAGIC);
 	w = sp->wrk;
+	CHECK_OBJ_NOTNULL(w->busyobj, BUSYOBJ_MAGIC);
+	htc = &w->busyobj->htc;
 
 	AN(sp->director);
 	AZ(sp->wrk->obj);
@@ -437,12 +440,13 @@ FetchHdr(struct sess *sp)
 
 	/* Receive response */
 
-	HTC_Init(w->htc, w->ws, vc->fd, vc->vsl_id, cache_param->http_resp_size,
+	HTC_Init(htc, w->ws, vc->fd, vc->vsl_id,
+	    cache_param->http_resp_size,
 	    cache_param->http_resp_hdr_len);
 
 	VTCP_set_read_timeout(vc->fd, vc->first_byte_timeout);
 
-	i = HTC_Rx(w->htc);
+	i = HTC_Rx(htc);
 
 	if (i < 0) {
 		WSP(sp, SLT_FetchError, "http first read error: %d %d (%s)",
@@ -456,7 +460,7 @@ FetchHdr(struct sess *sp)
 	VTCP_set_read_timeout(vc->fd, vc->between_bytes_timeout);
 
 	while (i == 0) {
-		i = HTC_Rx(w->htc);
+		i = HTC_Rx(htc);
 		if (i < 0) {
 			WSP(sp, SLT_FetchError,
 			    "http first read error: %d %d (%s)",
@@ -469,7 +473,7 @@ FetchHdr(struct sess *sp)
 
 	hp = w->beresp;
 
-	if (http_DissectResponse(w, w->htc, hp)) {
+	if (http_DissectResponse(w, htc, hp)) {
 		WSP(sp, SLT_FetchError, "http format error");
 		VDI_CloseFd(sp->wrk);
 		/* XXX: other cleanup ? */
@@ -487,13 +491,17 @@ FetchBody(struct worker *w, struct object *obj)
 	struct storage *st;
 	int mklen;
 	ssize_t cl;
+	struct http_conn *htc;
 
 	CHECK_OBJ_NOTNULL(w, WORKER_MAGIC);
+	CHECK_OBJ_NOTNULL(w->busyobj, BUSYOBJ_MAGIC);
 	AZ(w->fetch_obj);
 	CHECK_OBJ_NOTNULL(w->vbc, VBC_MAGIC);
 	CHECK_OBJ_NOTNULL(obj, OBJECT_MAGIC);
 	CHECK_OBJ_NOTNULL(obj->http, HTTP_MAGIC);
 
+	htc = &w->busyobj->htc;
+
 	if (w->busyobj->vfp == NULL)
 		w->busyobj->vfp = &vfp_nop;
 
@@ -519,21 +527,21 @@ FetchBody(struct worker *w, struct object *obj)
 	case BS_LENGTH:
 		cl = fetch_number( w->h_content_length, 10);
 		w->busyobj->vfp->begin(w, cl > 0 ? cl : 0);
-		cls = fetch_straight(w, w->htc, cl);
+		cls = fetch_straight(w, htc, cl);
 		mklen = 1;
 		if (w->busyobj->vfp->end(w))
 			cls = -1;
 		break;
 	case BS_CHUNKED:
 		w->busyobj->vfp->begin(w, cl);
-		cls = fetch_chunked(w, w->htc);
+		cls = fetch_chunked(w, htc);
 		mklen = 1;
 		if (w->busyobj->vfp->end(w))
 			cls = -1;
 		break;
 	case BS_EOF:
 		w->busyobj->vfp->begin(w, cl);
-		cls = fetch_eof(w, w->htc);
+		cls = fetch_eof(w, htc);
 		mklen = 1;
 		if (w->busyobj->vfp->end(w))
 			cls = -1;
diff --git a/bin/varnishd/storage/stevedore.c b/bin/varnishd/storage/stevedore.c
index 7c364c0..d0b4226 100644
--- a/bin/varnishd/storage/stevedore.c
+++ b/bin/varnishd/storage/stevedore.c
@@ -131,7 +131,7 @@ stv_pick_stevedore(struct worker *wrk, const char **hint)
 			return (stv_transient);
 
 		/* Hint was not valid, nuke it */
-		WSL(wrk, SLT_Debug, wrk->htc->vsl_id,
+		WSL(wrk, SLT_Debug, 0, 			/* XXX VSL_id ?? */
 		    "Storage hint not usable");
 		*hint = NULL;
 	}



From geoff at varnish-cache.org  Mon Jan  9 20:52:50 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:50 +0100
Subject: [experimental-ims] dcd622d Split long line
Message-ID: <E1RkMD4-0000FG-9K@project.varnish-software.com>

commit dcd622df236dfa496b4565eeff9555ad7f37fae5
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu Dec 8 08:29:19 2011 +0000

    Split long line

diff --git a/bin/varnishd/waiter/cache_waiter_poll.c b/bin/varnishd/waiter/cache_waiter_poll.c
index b4863a8..f6be13d 100644
--- a/bin/varnishd/waiter/cache_waiter_poll.c
+++ b/bin/varnishd/waiter/cache_waiter_poll.c
@@ -161,7 +161,8 @@ vwp_main(void *priv)
 				VTAILQ_REMOVE(&vwp->sesshead, sp, list);
 				if (i == 0) {
 					/* Mov to front of list for speed */
-					VTAILQ_INSERT_HEAD(&vwp->sesshead, sp, list);
+					VTAILQ_INSERT_HEAD(&vwp->sesshead,
+					    sp, list);
 				} else {
 					vwp_unpoll(vwp, fd);
 					SES_Handle(sp, i);



From geoff at varnish-cache.org  Mon Jan  9 20:52:41 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:41 +0100
Subject: [experimental-ims] 5a7a850 New VSL api
Message-ID: <E1RkMCv-0008Oc-Ka@project.varnish-software.com>

commit 5a7a850f2d42aea69a782cff92d50aad88cfce43
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Nov 22 10:35:34 2011 +0000

    New VSL api

diff --git a/include/vapi/vsl.h b/include/vapi/vsl.h
index fb02051..35e09dc 100644
--- a/include/vapi/vsl.h
+++ b/include/vapi/vsl.h
@@ -26,6 +26,17 @@
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
  *
+ * This is the public API for the VSL access.
+ *
+ * VSL is a "subclass" of VSM.
+ *
+ * VSL can either read from VSM or from a file.
+ *
+ * When reading from a file, the filename is passed in with:
+ *	VSL_Arg(vd, "r", "/some/file");
+ * and once VSL_Dispatch()/VSL_NextSLT() will indicate EOF by returning -2.
+ * Another file can then be opened with VSL_Arg() and processed.
+ * 
  */
 
 #ifndef VAPI_VSL_H_INCLUDED
@@ -39,21 +50,6 @@ struct VSM_data;
  * VSL level access functions
  */
 
-void VSL_Setup(struct VSM_data *vd);
-	/*
-	 * Setup vd for use with VSL functions.
-	 * Must be called once before any other VSL function is called.
-	 */
-
-int VSL_Open(struct VSM_data *vd, int diag);
-	/*
-	 * Attempt to open the VSM (unless -r given)
-	 * If diag is non-zero, diagnostics are emitted.
-	 * Returns:
-	 *	0 on success
-	 *	!= 0 on failure
-	 */
-
 #define VSL_ARGS	"bCcdI:i:k:n:r:s:X:x:m:"
 #define VSL_b_USAGE	"[-b]"
 #define VSL_c_USAGE	"[-c]"
@@ -68,6 +64,7 @@ int VSL_Open(struct VSM_data *vd, int diag);
 #define VSL_s_USAGE	"[-s skip]"
 #define VSL_x_USAGE	"[-x tag]"
 #define VSL_X_USAGE	"[-X regexp]"
+
 #define VSL_USAGE	"[-bCcd] "		\
 			VSL_i_USAGE " "		\
 			VSL_I_USAGE " "		\
@@ -83,24 +80,79 @@ int VSL_Arg(struct VSM_data *vd, int arg, const char *opt);
 	/*
 	 * Handle standard log-presenter arguments
 	 * Return:
-	 *	-1 error
+	 *	-1 error, VSM_Error() returns diagnostic string
 	 *	 0 not handled
 	 *	 1 Handled.
 	 */
 
 typedef int VSL_handler_f(void *priv, enum VSL_tag_e tag, unsigned fd,
     unsigned len, unsigned spec, const char *ptr, uint64_t bitmap);
+	/*
+	 * This is the call-back function you must provide.
+	 *	priv is whatever you asked for it to be.
+	 *	tag is the SLT_mumble tag
+	 *	fd is the filedescriptor associated with this record
+	 *	len is the length of the data at ptr
+	 *	spec are the VSL_S_* flags
+	 *	ptr points to the data, beware of non-printables.
+	 *	bitmap is XXX ???
+	 */
 
 #define VSL_S_CLIENT	(1 << 0)
 #define VSL_S_BACKEND	(1 << 1)
+
 VSL_handler_f VSL_H_Print;
-struct VSM_data;
-void VSL_Select(const struct VSM_data *vd, unsigned tag);
-void VSL_NonBlocking(const struct VSM_data *vd, int nb);
+	/*
+	 * This call-back function will printf() the record to the FILE *
+	 * specified in priv.
+	 */
+
+void VSL_Select(struct VSM_data *vd, enum VSL_tag_e tag);
+	/*
+	 * This adds tags which shall always be selected, similar to using
+	 * the '-i' option.
+	 * VSL_Select()/-i takes precedence over all other filtering.
+	 */
+
 int VSL_Dispatch(struct VSM_data *vd, VSL_handler_f *func, void *priv);
-int VSL_NextLog(const struct VSM_data *lh, uint32_t **pp, uint64_t *bitmap);
-int VSL_Matched(const struct VSM_data *vd, uint64_t bitmap);
+	/*
+	 * Call func(priv, ...) for all filtered VSL records.
+	 *
+	 * Return values:
+	 *	!=0:	Non-zero return value from func()
+	 *	0:	no VSL records.
+	 *	-1:	VSL chunk was abandonned.
+	 *	-2:	End of file (-r) / -k arg exhausted / "done"
+	 */
+
+int VSL_NextSLT(struct VSM_data *lh, uint32_t **pp, uint64_t *bitmap);
+	/*
+	 * Return raw pointer to next filtered VSL record.
+	 *
+	 * Return values:
+	 *	1:	Valid VSL record at *pp
+	 *	0:	no VSL records
+	 *	-1:	VSL cunkwas abandonned
+	 *	-2:	End of file (-r) / -k arg exhausted / "done"
+	 */
+
+int VSL_Matched(struct VSM_data *vd, uint64_t bitmap);
+	/*
+	 */
+
 int VSL_Name2Tag(const char *name, int l);
+	/*
+	 * Convert string to tag number (= enum VSL_tag_e)
+	 *
+	 * Return values:
+	 *	>=0:	Tag number
+	 *	-1:	No tag matches
+	 *	-2:	Multiple tags match substring
+	 */
+
 extern const char *VSL_tags[256];
+	/*
+	 * Tag to string array.  Contains NULL for invalid tags.
+	 */
 
 #endif /* VAPI_VSL_H_INCLUDED */
diff --git a/include/vapi/vsm.h b/include/vapi/vsm.h
index 277a670..9635e8f 100644
--- a/include/vapi/vsm.h
+++ b/include/vapi/vsm.h
@@ -83,7 +83,7 @@ int VSM_n_Arg(struct VSM_data *vd, const char *n_arg);
 	 * Can also be, and normally is done through VSC_Arg()/VSL_Arg().
 	 * Returns:
 	 *	 1 on success
-	 *	 <0 on failure, use VSM_Error() to get diagnostics.
+	 *	 <0 on failure, VSM_Error() returns diagnostic string
 	 */
 
 const char *VSM_Name(const struct VSM_data *vd);
@@ -97,7 +97,7 @@ int VSM_Open(struct VSM_data *vd);
 	 * If diag is non-zero, diagnostics are emitted.
 	 * Returns:
 	 *	0 on success
-	 *	<0 on failure, use VSM_Error() to get diagnostics.
+	 *	<0 on failure, VSM_Error() returns diagnostic string
 	 */
 
 int VSM_Abandonned(const struct VSM_data *vd);
diff --git a/lib/libvarnishapi/vsl.c b/lib/libvarnishapi/vsl.c
index e1ace33..d45e1c3 100644
--- a/lib/libvarnishapi/vsl.c
+++ b/lib/libvarnishapi/vsl.c
@@ -51,37 +51,40 @@
 #include "vsl_api.h"
 #include "vsm_api.h"
 
-static void VSL_Close(struct VSM_data *vd);
-
 /*--------------------------------------------------------------------*/
 
 const char *VSL_tags[256] = {
-#define SLTM(foo)       [SLT_##foo] = #foo,
-#include "tbl/vsl_tags.h"
-#undef SLTM
+#  define SLTM(foo)       [SLT_##foo] = #foo,
+#  include "tbl/vsl_tags.h"
+#  undef SLTM
 };
 
 /*--------------------------------------------------------------------*/
 
-void
-VSL_Setup(struct VSM_data *vd)
+struct vsl *
+vsl_Setup(struct VSM_data *vd)
 {
 	struct vsl *vsl;
 
 	CHECK_OBJ_NOTNULL(vd, VSM_MAGIC);
-	AZ(vd->vsl);
-	ALLOC_OBJ(vd->vsl, VSL_MAGIC);
-	AN(vd->vsl);
-	vsl = vd->vsl;
-	vsl->regflags = 0;
-	vsl->vbm_supress = vbit_init(256);
-	vsl->vbm_select = vbit_init(256);
-	vsl->r_fd = -1;
-	vsl->num_matchers = 0;
-	VTAILQ_INIT(&vsl->matchers);
+	if (vd->vsl == NULL) {
+		ALLOC_OBJ(vd->vsl, VSL_MAGIC);
+		AN(vd->vsl);
+		vsl = vd->vsl;
+		vsl->regflags = 0;
+		vsl->vbm_supress = vbit_init(256);
+		vsl->vbm_select = vbit_init(256);
+		vsl->r_fd = -1;
+		vsl->num_matchers = 0;
+		VTAILQ_INIT(&vsl->matchers);
+	}
+	CHECK_OBJ_NOTNULL(vd->vsl, VSL_MAGIC);
+	return (vd->vsl);
 }
 
-/*--------------------------------------------------------------------*/
+/*--------------------------------------------------------------------
+ * Called from VSM_Delete()
+ */
 
 void
 VSL_Delete(struct VSM_data *vd)
@@ -93,41 +96,66 @@ VSL_Delete(struct VSM_data *vd)
 	vd->vsl = NULL;
 	CHECK_OBJ_NOTNULL(vsl, VSL_MAGIC);
 
+	if (vsl->r_fd > STDIN_FILENO)
+		(void)close(vsl->r_fd);
 	vbit_destroy(vsl->vbm_supress);
 	vbit_destroy(vsl->vbm_select);
 	free(vsl->rbuf);
-
 	FREE_OBJ(vsl);
 }
 
 /*--------------------------------------------------------------------*/
 
 void
-VSL_Select(const struct VSM_data *vd, unsigned tag)
+VSL_Select(struct VSM_data *vd, enum VSL_tag_e tag)
 {
-	struct vsl *vsl;
+	struct vsl *vsl = vsl_Setup(vd);
 
-	CHECK_OBJ_NOTNULL(vd, VSM_MAGIC);
-	vsl = vd->vsl;
-	CHECK_OBJ_NOTNULL(vsl, VSL_MAGIC);
-	vbit_set(vsl->vbm_select, tag);
+	vbit_set(vsl->vbm_select, (int)tag);
 }
 
+/*--------------------------------------------------------------------
+ */
 
-/*--------------------------------------------------------------------*/
+static int
+vsl_open(struct VSM_data *vd)
+{
+	struct vsl *vsl = vsl_Setup(vd);
+	int i;
 
-void
-VSL_NonBlocking(const struct VSM_data *vd, int nb)
+	assert(vsl->r_fd < 0);
+	i = VSM_Open(vd);
+	if (i)
+		return (i);
+	if (!VSM_Get(vd, &vsl->vf, VSL_CLASS, NULL, NULL)) {
+		VSM_Close(vd);
+		return (vsm_diag(vd, "No VSL chunk found "
+		    " (child not started ?)\n"));
+	}
+	vsl->log_start = vsl->vf.b;
+	vsl->log_end = vsl->vf.e;
+	vsl->log_ptr = vsl->log_start + 1;
+	if (!vsl->d_opt) {
+		while (*vsl->log_ptr != VSL_ENDMARKER)
+			vsl->log_ptr = VSL_NEXT(vsl->log_ptr);
+	}
+	return (0);
+}
+
+/*--------------------------------------------------------------------
+ */
+
+static void
+vsl_close(struct VSM_data *vd)
 {
-	struct vsl *vsl;
+	struct vsl *vsl = vsl_Setup(vd);
 
-	CHECK_OBJ_NOTNULL(vd, VSM_MAGIC);
-	vsl = vd->vsl;
-	CHECK_OBJ_NOTNULL(vsl, VSL_MAGIC);
-	if (nb)
-		vsl->flags |= F_NON_BLOCKING;
-	else
-		vsl->flags &= ~F_NON_BLOCKING;
+	assert(vsl->r_fd < 0);
+	VSM_Close(vd);
+	memset(&vsl->vf, 0, sizeof vsl->vf);
+	vsl->log_start = NULL;
+	vsl->log_end = NULL;
+	vsl->log_ptr = NULL;
 }
 
 /*--------------------------------------------------------------------
@@ -140,20 +168,19 @@ VSL_NonBlocking(const struct VSM_data *vd, int nb)
  */
 
 static int
-vsl_nextlog(struct vsl *vsl, uint32_t **pp)
+vsl_nextslt(struct VSM_data *vd, uint32_t **pp)
 {
+	struct vsl *vsl = vsl_Setup(vd);
 	unsigned l;
 	uint32_t t;
 	int i;
 
-	CHECK_OBJ_NOTNULL(vsl, VSL_MAGIC);
-
 	*pp = NULL;
 	if (vsl->r_fd != -1) {
 		assert(vsl->rbuflen >= 8);
 		i = read(vsl->r_fd, vsl->rbuf, 8);
 		if (i == 0)
-			return (-1);
+			return (-2);
 		if (i != 8)
 			return (-1);
 		l = 2 + VSL_WORDS(VSL_LEN(vsl->rbuf));
@@ -169,9 +196,11 @@ vsl_nextlog(struct vsl *vsl, uint32_t **pp)
 		*pp = vsl->rbuf;
 		return (1);
 	}
+
+	if (vsl->log_ptr == NULL && vsl_open(vd))
+		return (0);
+
 	while (1) {
-		if (vsl->log_ptr == NULL)
-			return (0);
 		assert(vsl->log_ptr >= vsl->log_start + 1);
 		assert(vsl->log_ptr < vsl->log_end);
 		t = *vsl->log_ptr;
@@ -209,60 +238,65 @@ vsl_nextlog(struct vsl *vsl, uint32_t **pp)
 }
 
 int
-VSL_NextLog(const struct VSM_data *vd, uint32_t **pp, uint64_t *bits)
+VSL_NextSLT(struct VSM_data *vd, uint32_t **pp, uint64_t *bits)
 {
-	struct vsl *vsl;
+	struct vsl *vsl = vsl_Setup(vd);
 	uint32_t *p;
 	unsigned char t;
 	int i;
+	struct vsl_re_match *vrm;
+	int j;
 
-	CHECK_OBJ_NOTNULL(vd, VSM_MAGIC);
-	vsl = vd->vsl;
-	CHECK_OBJ_NOTNULL(vsl, VSL_MAGIC);
+	if (bits != NULL)
+		*bits = 0;
 
 	while (1) {
-		i = vsl_nextlog(vsl, &p);
-		if (i <= 0)
+		i = vsl_nextslt(vd, &p);
+		if (i < 0)
+			return (i);
+		if (i == 0 && (vsl->d_opt || vsl->r_fd >= 0))
+			return (i);
+		if (i == 0 && !VSM_StillValid(vd, &vsl->vf)) {
+			vsl_close(vd);
 			return (i);
-		t = VSL_TAG(p);
-		if (vsl->skip) {
-			--vsl->skip;
-			continue;
-		} else if (vsl->keep) {
-			if (--vsl->keep == 0)
-				return (-1);
 		}
 
+		t = VSL_TAG(p);
 		if (vbit_test(vsl->vbm_select, t)) {
-			*pp = p;
-			return (1);
-		}
-		if (vbit_test(vsl->vbm_supress, t))
+			/* nothing */
+		} else if (vbit_test(vsl->vbm_supress, t)) {
 			continue;
-		if (vsl->b_opt && !VSL_BACKEND(p))
+		} else if (vsl->b_opt && !VSL_BACKEND(p)) {
 			continue;
-		if (vsl->c_opt && !VSL_CLIENT(p))
+		} else if (vsl->c_opt && !VSL_CLIENT(p)) {
 			continue;
-		if (vsl->regincl != NULL) {
+		} else if (vsl->regincl != NULL) {
 			i = VRE_exec(vsl->regincl, VSL_DATA(p), VSL_LEN(p),
 			    0, 0, NULL, 0, NULL);
 			if (i == VRE_ERROR_NOMATCH)
 				continue;
-		}
-		if (vsl->regexcl != NULL) {
+		} else if (vsl->regexcl != NULL) {
 			i = VRE_exec(vsl->regexcl, VSL_DATA(p), VSL_LEN(p),
 			    0, 0, NULL, 0, NULL);
 			if (i != VRE_ERROR_NOMATCH)
 				continue;
 		}
+
+		if (vsl->skip) {
+			--vsl->skip;
+			continue;
+		} else if (vsl->keep) {
+			if (--vsl->keep == 0)
+				return (-2);
+		}
+
 		if (bits != NULL) {
-			struct vsl_re_match *vrm;
-			int j = 0;
+			j = 0;
 			VTAILQ_FOREACH(vrm, &vsl->matchers, next) {
 				if (vrm->tag == t) {
 					i = VRE_exec(vrm->re, VSL_DATA(p),
 					    VSL_LEN(p), 0, 0, NULL, 0, NULL);
-					if (i >= 0)
+					if (i >= 0) 	/* XXX ?? */
 						*bits |= (uintmax_t)1 << j;
 				}
 				j++;
@@ -278,41 +312,13 @@ VSL_NextLog(const struct VSM_data *vd, uint32_t **pp, uint64_t *bits)
 int
 VSL_Dispatch(struct VSM_data *vd, VSL_handler_f *func, void *priv)
 {
-	struct vsl *vsl;
 	int i;
 	unsigned u, l, s;
 	uint32_t *p;
 	uint64_t bitmap;
-	int tmo;
-
-	CHECK_OBJ_NOTNULL(vd, VSM_MAGIC);
-	vsl = vd->vsl;
-	CHECK_OBJ_NOTNULL(vsl, VSL_MAGIC);
 
-	tmo = 0;
 	while (1) {
-		bitmap = 0;
-		i = VSL_NextLog(vd, &p, &bitmap);
-		if (i == 0) {
-			if (vsl->r_fd != -1)
-				return(0);
-			if (vsl->flags & F_NON_BLOCKING)
-				return (0);
-			if (VSM_StillValid(vd, &vsl->vf) != 1) {
-				VSL_Close(vd);
-				if (VSL_Open(vd, 0))
-					return (-1);
-				AN(vsl->log_ptr);
-				assert(vsl->log_ptr >= vsl->log_start + 1);
-				assert(vsl->log_ptr < vsl->log_end);
-				continue;
-			}
-			tmo += SLEEP_USEC;
-			if (tmo > TIMEOUT_USEC)
-				return (0);
-			(void)usleep(SLEEP_USEC);
-			continue;
-		}
+		i = VSL_NextSLT(vd, &p, &bitmap);
 		if (i <= 0)
 			return (i);
 		u = VSL_ID(p);
@@ -322,9 +328,10 @@ VSL_Dispatch(struct VSM_data *vd, VSL_handler_f *func, void *priv)
 			s |= VSL_S_CLIENT;
 		if (VSL_BACKEND(p))
 			s |= VSL_S_BACKEND;
-		if (func(priv, (enum VSL_tag_e)VSL_TAG(p),
-		    u, l, s, VSL_DATA(p), bitmap))
-			return (1);
+		i = func(priv, (enum VSL_tag_e)VSL_TAG(p),
+		    u, l, s, VSL_DATA(p), bitmap);
+		if (i)
+			return (i);
 	}
 }
 
@@ -362,64 +369,14 @@ VSL_H_Print(void *priv, enum VSL_tag_e tag, unsigned fd, unsigned len,
 
 /*--------------------------------------------------------------------*/
 
-static void
-VSL_Close(struct VSM_data *vd)
-{
-	struct vsl *vsl;
-
-	CHECK_OBJ_NOTNULL(vd, VSM_MAGIC);
-	vsl = vd->vsl;
-	CHECK_OBJ_NOTNULL(vsl, VSL_MAGIC);
-
-	VSM_Close(vd);
-	vsl->log_start = NULL;
-	vsl->log_end = NULL;
-	vsl->log_ptr = NULL;
-}
-
-/*--------------------------------------------------------------------*/
-
 int
-VSL_Open(struct VSM_data *vd, int diag)
+VSL_Matched(struct VSM_data *vd, uint64_t bitmap)
 {
-	struct vsl *vsl;
-	int i;
-
-	CHECK_OBJ_NOTNULL(vd, VSM_MAGIC);
-	vsl = vd->vsl;
-	CHECK_OBJ_NOTNULL(vsl, VSL_MAGIC);
-
-	if (vsl->r_fd == -1) {
-		i = VSM_Open(vd, diag);
-		if (i)
-			return (i);
-		if (!VSM_Get(vd, &vsl->vf, VSL_CLASS, NULL, NULL)) {
-			VSM_Close(vd);
-			if (diag)
-				vd->diag(vd->priv,
-				    "No VSL chunk found "
-				    " (child not started ?)\n");
-			return (1);
-		}
-		vsl->log_start = vsl->vf.b;
-		vsl->log_end = vsl->vf.e;
-		vsl->log_ptr = vsl->log_start + 1;
-		if (!vsl->d_opt) {
-			while (*vsl->log_ptr != VSL_ENDMARKER)
-				vsl->log_ptr = VSL_NEXT(vsl->log_ptr);
-		}
-	}
-	return (0);
-}
+	struct vsl *vsl = vsl_Setup(vd);
 
-/*--------------------------------------------------------------------*/
-
-int
-VSL_Matched(const struct VSM_data *vd, uint64_t bitmap)
-{
-	if (vd->vsl->num_matchers > 0) {
+	if (vsl->num_matchers > 0) {
 		uint64_t t;
-		t = vd->vsl->num_matchers | (vd->vsl->num_matchers - 1);
+		t = vsl->num_matchers | (vsl->num_matchers - 1);
 		return (bitmap == t);
 	}
 	return (1);
diff --git a/lib/libvarnishapi/vsl_api.h b/lib/libvarnishapi/vsl_api.h
index 013d1ae..0b38539 100644
--- a/lib/libvarnishapi/vsl_api.h
+++ b/lib/libvarnishapi/vsl_api.h
@@ -67,7 +67,6 @@ struct vsl {
 
 	unsigned		flags;
 #define F_SEEN_IX		(1 << 0)
-#define F_NON_BLOCKING		(1 << 1)
 
 	/*
 	 * Bit map of programatically selected tags, that cannot be suppressed.
@@ -89,3 +88,5 @@ struct vsl {
 	unsigned long		keep;
 };
 
+struct vsl *vsl_Setup(struct VSM_data *vd);
+
diff --git a/lib/libvarnishapi/vsl_arg.c b/lib/libvarnishapi/vsl_arg.c
index b855628..100d75a 100644
--- a/lib/libvarnishapi/vsl_arg.c
+++ b/lib/libvarnishapi/vsl_arg.c
@@ -82,21 +82,23 @@ VSL_Name2Tag(const char *name, int l)
 /*--------------------------------------------------------------------*/
 
 static int
-vsl_r_arg(const struct VSM_data *vd, const char *opt)
+vsl_r_arg(struct VSM_data *vd, const char *opt)
 {
+	struct vsl *vsl = vsl_Setup(vd);
 
-	CHECK_OBJ_NOTNULL(vd, VSM_MAGIC);
+	if (vsl->r_fd > STDIN_FILENO)
+		(void)close(vsl->r_fd);
 	if (!strcmp(opt, "-"))
-		vd->vsl->r_fd = STDIN_FILENO;
+		vsl->r_fd = STDIN_FILENO;
 	else
-		vd->vsl->r_fd = open(opt, O_RDONLY);
-	if (vd->vsl->r_fd < 0) {
-		perror(opt);
-		return (-1);
-	} else if (vd->vsl->rbuflen == 0) {
-		vd->vsl->rbuf = malloc(1024);
-		AN(vd->vsl->rbuf);
-		vd->vsl->rbuflen = 1024;
+		vsl->r_fd = open(opt, O_RDONLY);
+	if (vsl->r_fd < 0)
+		return (vsm_diag(vd,
+		    "Could not open %s: %s", opt, strerror(errno)));
+	if (vsl->rbuflen == 0) {
+		vsl->rbuflen = BUFSIZ;
+		vsl->rbuf = malloc(vsl->rbuflen);
+		AN(vsl->rbuf);
 	}
 	return (1);
 }
@@ -104,43 +106,41 @@ vsl_r_arg(const struct VSM_data *vd, const char *opt)
 /*--------------------------------------------------------------------*/
 
 static int
-vsl_IX_arg(const struct VSM_data *vd, const char *opt, int arg)
+vsl_IX_arg(struct VSM_data *vd, const char *opt, int arg)
 {
+	struct vsl *vsl = vsl_Setup(vd);
 	vre_t **rp;
 	const char *error;
 	int erroroffset;
 
 	CHECK_OBJ_NOTNULL(vd, VSM_MAGIC);
 	if (arg == 'I')
-		rp = &vd->vsl->regincl;
+		rp = &vsl->regincl;
 	else
-		rp = &vd->vsl->regexcl;
-	if (*rp != NULL) {
-		fprintf(stderr, "Option %c can only be given once", arg);
-		return (-1);
-	}
-	*rp = VRE_compile(opt, vd->vsl->regflags, &error, &erroroffset);
-	if (*rp == NULL) {
-		fprintf(stderr, "Illegal regex: %s\n", error);
-		return (-1);
-	}
+		rp = &vsl->regexcl;
+	if (*rp != NULL)
+		return (vsm_diag(vd, "Option %c can only be given once", arg));
+	*rp = VRE_compile(opt, vsl->regflags, &error, &erroroffset);
+	if (*rp == NULL)
+		return (vsm_diag(vd, "Illegal regex: %s\n", error));
 	return (1);
 }
 
 /*--------------------------------------------------------------------*/
 
 static int
-vsl_ix_arg(const struct VSM_data *vd, const char *opt, int arg)
+vsl_ix_arg(struct VSM_data *vd, const char *opt, int arg)
 {
+	struct vsl *vsl = vsl_Setup(vd);
 	int i, l;
 	const char *b, *e;
 
 	CHECK_OBJ_NOTNULL(vd, VSM_MAGIC);
 	/* If first option is 'i', set all bits for supression */
-	if (arg == 'i' && !(vd->vsl->flags & F_SEEN_IX))
+	if (arg == 'i' && !(vsl->flags & F_SEEN_IX))
 		for (i = 0; i < 256; i++)
-			vbit_set(vd->vsl->vbm_supress, i);
-	vd->vsl->flags |= F_SEEN_IX;
+			vbit_set(vsl->vbm_supress, i);
+	vsl->flags |= F_SEEN_IX;
 
 	for (b = opt; *b; b = e) {
 		while (isspace(*b))
@@ -156,17 +156,15 @@ vsl_ix_arg(const struct VSM_data *vd, const char *opt, int arg)
 		i = VSL_Name2Tag(b, l);
 		if (i >= 0) {
 			if (arg == 'x')
-				vbit_set(vd->vsl->vbm_supress, i);
+				vbit_set(vsl->vbm_supress, i);
 			else
-				vbit_clr(vd->vsl->vbm_supress, i);
+				vbit_clr(vsl->vbm_supress, i);
 		} else if (i == -2) {
-			fprintf(stderr,
-			    "\"%*.*s\" matches multiple tags\n", l, l, b);
-			return (-1);
+			return (vsm_diag(vd, 
+			    "\"%*.*s\" matches multiple tags\n", l, l, b));
 		} else {
-			fprintf(stderr,
-			    "Could not match \"%*.*s\" to any tag\n", l, l, b);
-			return (-1);
+			return (vsm_diag(vd, 
+			    "Could not match \"%*.*s\" to any tag\n", l, l, b));
 		}
 	}
 	return (1);
@@ -176,8 +174,9 @@ vsl_ix_arg(const struct VSM_data *vd, const char *opt, int arg)
 
 
 static int
-vsl_m_arg(const struct VSM_data *vd, const char *opt)
+vsl_m_arg(struct VSM_data *vd, const char *opt)
 {
+	struct vsl *vsl = vsl_Setup(vd);
 	struct vsl_re_match *m;
 	const char *error;
 	char *o, *regex;
@@ -185,10 +184,9 @@ vsl_m_arg(const struct VSM_data *vd, const char *opt)
 
 	CHECK_OBJ_NOTNULL(vd, VSM_MAGIC);
 
-	if (!strchr(opt, ':')) {
-		fprintf(stderr, "No : found in -m option %s\n", opt);
-		return (-1);
-	}
+	if (!strchr(opt, ':'))
+		return (vsm_diag(vd,
+		    "No : found in -m option %s\n", opt));
 
 	o = strdup(opt);
 	AN(o);
@@ -200,21 +198,21 @@ vsl_m_arg(const struct VSM_data *vd, const char *opt)
 	AN(m);
 	m->tag = VSL_Name2Tag(o, -1);
 	if (m->tag < 0) {
-		fprintf(stderr, "Illegal tag %s specified\n", o);
+		(void)vsm_diag(vd, "Illegal tag %s specified\n", o);
 		free(o);
 		FREE_OBJ(m);
 		return (-1);
 	}
 	/* Get tag, regex */
-	m->re = VRE_compile(regex, vd->vsl->regflags, &error, &erroroffset);
+	m->re = VRE_compile(regex, vsl->regflags, &error, &erroroffset);
 	if (m->re == NULL) {
-		fprintf(stderr, "Illegal regex: %s\n", error);
+		(void)vsm_diag(vd, "Illegal regex: %s\n", error);
 		free(o);
 		FREE_OBJ(m);
 		return (-1);
 	}
-	vd->vsl->num_matchers++;
-	VTAILQ_INSERT_TAIL(&vd->vsl->matchers, m, next);
+	vsl->num_matchers++;
+	VTAILQ_INSERT_TAIL(&vsl->matchers, m, next);
 	free(o);
 	return (1);
 }
@@ -222,40 +220,34 @@ vsl_m_arg(const struct VSM_data *vd, const char *opt)
 /*--------------------------------------------------------------------*/
 
 static int
-vsl_s_arg(const struct VSM_data *vd, const char *opt)
+vsl_s_arg(struct VSM_data *vd, const char *opt)
 {
+	struct vsl *vsl = vsl_Setup(vd);
 	char *end;
 
 	CHECK_OBJ_NOTNULL(vd, VSM_MAGIC);
-	if (*opt == '\0') {
-		fprintf(stderr, "number required for -s\n");
-		return (-1);
-	}
-	vd->vsl->skip = strtoul(opt, &end, 10);
-	if (*end != '\0') {
-		fprintf(stderr, "invalid number for -s\n");
-		return (-1);
-	}
+	if (*opt == '\0')
+		return (vsm_diag(vd, "number required for -s\n"));
+	vsl->skip = strtoul(opt, &end, 10);
+	if (*end != '\0')
+		return (vsm_diag(vd, "invalid number for -k\n"));
 	return (1);
 }
 
 /*--------------------------------------------------------------------*/
 
 static int
-vsl_k_arg(const struct VSM_data *vd, const char *opt)
+vsl_k_arg(struct VSM_data *vd, const char *opt)
 {
+	struct vsl *vsl = vsl_Setup(vd);
 	char *end;
 
 	CHECK_OBJ_NOTNULL(vd, VSM_MAGIC);
-	if (*opt == '\0') {
-		fprintf(stderr, "number required for -k\n");
-		return (-1);
-	}
-	vd->vsl->keep = strtoul(opt, &end, 10);
-	if (*end != '\0') {
-		fprintf(stderr, "invalid number for -k\n");
-		return (-1);
-	}
+	if (*opt == '\0') 
+		return (vsm_diag(vd, "number required for -k\n"));
+	vsl->keep = strtoul(opt, &end, 10);
+	if (*end != '\0') 
+		return (vsm_diag(vd, "invalid number for -k\n"));
 	return (1);
 }
 
@@ -264,14 +256,13 @@ vsl_k_arg(const struct VSM_data *vd, const char *opt)
 int
 VSL_Arg(struct VSM_data *vd, int arg, const char *opt)
 {
+	struct vsl *vsl = vsl_Setup(vd);
 
-	CHECK_OBJ_NOTNULL(vd, VSM_MAGIC);
 	switch (arg) {
-	case 'b': vd->vsl->b_opt = !vd->vsl->b_opt; return (1);
-	case 'c': vd->vsl->c_opt = !vd->vsl->c_opt; return (1);
+	case 'b': vsl->b_opt = !vsl->b_opt; return (1);
+	case 'c': vsl->c_opt = !vsl->c_opt; return (1);
 	case 'd':
-		vd->vsl->d_opt = !vd->vsl->d_opt;
-		vd->vsl->flags |= F_NON_BLOCKING;
+		vsl->d_opt = !vsl->d_opt;
 		return (1);
 	case 'i': case 'x': return (vsl_ix_arg(vd, opt, arg));
 	case 'k': return (vsl_k_arg(vd, opt));
@@ -280,7 +271,7 @@ VSL_Arg(struct VSM_data *vd, int arg, const char *opt)
 	case 's': return (vsl_s_arg(vd, opt));
 	case 'I': case 'X': return (vsl_IX_arg(vd, opt, arg));
 	case 'm': return (vsl_m_arg(vd, opt));
-	case 'C': vd->vsl->regflags = VRE_CASELESS; return (1);
+	case 'C': vsl->regflags = VRE_CASELESS; return (1);
 	default:
 		return (0);
 	}
diff --git a/lib/libvarnishapi/vsm.c b/lib/libvarnishapi/vsm.c
index d136c98..1d762e5 100644
--- a/lib/libvarnishapi/vsm.c
+++ b/lib/libvarnishapi/vsm.c
@@ -75,7 +75,7 @@ VSM_New(void)
 
 /*--------------------------------------------------------------------*/
 
-static int
+int
 vsm_diag(struct VSM_data *vd, const char *fmt, ...)
 {
 	va_list ap;
diff --git a/lib/libvarnishapi/vsm_api.h b/lib/libvarnishapi/vsm_api.h
index 6690ba6..79be982 100644
--- a/lib/libvarnishapi/vsm_api.h
+++ b/lib/libvarnishapi/vsm_api.h
@@ -47,14 +47,10 @@ struct VSM_data {
 	char			*b;
 	char			*e;
 
-	/* Stuff for backwards compat */
-	struct VSM_fantom	compat_vf;
-
-	/* Stuff relating the stats fields start here */
-
 	struct vsc		*vsc;
 	struct vsl		*vsl;
 };
 
+int vsm_diag(struct VSM_data *vd, const char *fmt, ...);
 void VSC_Delete(struct VSM_data *vd);
 void VSL_Delete(struct VSM_data *vd);



From geoff at varnish-cache.org  Mon Jan  9 20:52:14 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:14 +0100
Subject: [experimental-ims] 8b98c9e Move more table-generating #includes into
	tbl/ subdir
Message-ID: <E1RkMCU-0007Xt-QN@project.varnish-software.com>

commit 8b98c9ed9befa096e2fe0b862ed9720a71e33646
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sat Oct 8 14:43:13 2011 +0000

    Move more table-generating #includes into tbl/ subdir

diff --git a/bin/varnishd/cache.h b/bin/varnishd/cache.h
index 11e952e..f46634a 100644
--- a/bin/varnishd/cache.h
+++ b/bin/varnishd/cache.h
@@ -804,7 +804,7 @@ ssize_t HTC_Read(struct http_conn *htc, void *d, size_t len);
 int HTC_Complete(struct http_conn *htc);
 
 #define HTTPH(a, b, c, d, e, f, g) extern char b[];
-#include "http_headers.h"
+#include "tbl/http_headers.h"
 #undef HTTPH
 
 /* cache_main.c */
@@ -924,7 +924,7 @@ void VCL_Rel(struct VCL_conf **vcc);
 void VCL_Poll(void);
 
 #define VCL_MET_MAC(l,u,b) void VCL_##l##_method(struct sess *);
-#include "vcl_returns.h"
+#include "tbl/vcl_returns.h"
 #undef VCL_MET_MAC
 
 /* cache_vrt.c */
diff --git a/bin/varnishd/cache_ban.c b/bin/varnishd/cache_ban.c
index 97d961d..fe1efd0 100644
--- a/bin/varnishd/cache_ban.c
+++ b/bin/varnishd/cache_ban.c
@@ -127,7 +127,7 @@ static const struct pvar {
 	uint8_t			tag;
 } pvars[] = {
 #define PVAR(a, b, c)	{ (a), (b), (c) },
-#include "ban_vars.h"
+#include "tbl/ban_vars.h"
 #undef PVAR
 	{ 0, 0, 0}
 };
diff --git a/bin/varnishd/cache_http.c b/bin/varnishd/cache_http.c
index 25a6627..bd0380b 100644
--- a/bin/varnishd/cache_http.c
+++ b/bin/varnishd/cache_http.c
@@ -41,7 +41,7 @@
 #include "cache.h"
 
 #define HTTPH(a, b, c, d, e, f, g) char b[] = "*" a ":";
-#include "http_headers.h"
+#include "tbl/http_headers.h"
 #undef HTTPH
 
 /*lint -save -e773 not () */
@@ -91,7 +91,7 @@ static struct http_msg {
 	const char	*txt;
 } http_msg[] = {
 #define HTTP_RESP(n, t)	{ n, t},
-#include "http_response.h"
+#include "tbl/http_response.h"
 	{ 0, NULL }
 };
 
@@ -830,7 +830,7 @@ http_EstimateWS(const struct http *fm, unsigned how, uint16_t *nhd)
 #define HTTPH(a, b, c, d, e, f, g) \
 		if (((e) & how) && http_IsHdr(&fm->hd[u], (b))) \
 			continue;
-#include "http_headers.h"
+#include "tbl/http_headers.h"
 #undef HTTPH
 		l += PRNDUP(Tlen(fm->hd[u]) + 1);
 		(*nhd)++;
@@ -859,7 +859,7 @@ http_FilterFields(struct worker *w, unsigned vsl_id, struct http *to,
 #define HTTPH(a, b, c, d, e, f, g) \
 		if (((e) & how) && http_IsHdr(&fm->hd[u], (b))) \
 			continue;
-#include "http_headers.h"
+#include "tbl/http_headers.h"
 #undef HTTPH
 		http_copyheader(w, vsl_id, to, fm, u);
 	}
@@ -1111,6 +1111,6 @@ HTTP_Init(void)
 {
 
 #define HTTPH(a, b, c, d, e, f, g) b[0] = (char)strlen(b + 1);
-#include "http_headers.h"
+#include "tbl/http_headers.h"
 #undef HTTPH
 }
diff --git a/bin/varnishd/cache_vcl.c b/bin/varnishd/cache_vcl.c
index ac712fb..8848ddd 100644
--- a/bin/varnishd/cache_vcl.c
+++ b/bin/varnishd/cache_vcl.c
@@ -331,7 +331,7 @@ VCL_##func##_method(struct sess *sp)					\
 	assert(!((1U << sp->handling) & ~bitmap));			\
 }
 
-#include "vcl_returns.h"
+#include "tbl/vcl_returns.h"
 #undef VCL_MET_MAC
 
 /*--------------------------------------------------------------------*/
diff --git a/bin/varnishd/stevedore.c b/bin/varnishd/stevedore.c
index c1e52cb..143e534 100644
--- a/bin/varnishd/stevedore.c
+++ b/bin/varnishd/stevedore.c
@@ -597,5 +597,5 @@ VRT_Stv_##nm(const char *nm)			\
 	return (stv->var_##nm(stv));		\
 }
 
-#include "vrt_stv_var.h"
+#include "tbl/vrt_stv_var.h"
 #undef VRTSTVVAR
diff --git a/bin/varnishd/stevedore.h b/bin/varnishd/stevedore.h
index 0ccf834..9cd0b7b 100644
--- a/bin/varnishd/stevedore.h
+++ b/bin/varnishd/stevedore.h
@@ -47,7 +47,7 @@ typedef void storage_close_f(const struct stevedore *);
 
 /* Prototypes for VCL variable responders */
 #define VRTSTVTYPE(ct) typedef ct storage_var_##ct(const struct stevedore *);
-#include "vrt_stv_var.h"
+#include "tbl/vrt_stv_var.h"
 #undef VRTSTVTYPE
 
 /*--------------------------------------------------------------------*/
@@ -77,7 +77,7 @@ struct stevedore {
 	struct lru		*lru;
 
 #define VRTSTVVAR(nm, vtype, ctype, dval) storage_var_##ctype *var_##nm;
-#include "vrt_stv_var.h"
+#include "tbl/vrt_stv_var.h"
 #undef VRTSTVVAR
 
 	/* private fields */
diff --git a/include/Makefile.am b/include/Makefile.am
index 3f2994c..de1b19a 100644
--- a/include/Makefile.am
+++ b/include/Makefile.am
@@ -11,7 +11,7 @@ pkginclude_HEADERS = \
 	vsm.h
 
 nobase_noinst_HEADERS = \
-	ban_vars.h \
+	tbl/ban_vars.h \
 	binary_heap.h \
 	cli_common.h \
 	cli_priv.h \
@@ -20,8 +20,8 @@ nobase_noinst_HEADERS = \
 	compat/execinfo.h \
 	compat/srandomdev.h \
 	flopen.h \
-	http_headers.h \
-	http_response.h \
+	tbl/http_headers.h \
+	tbl/http_response.h \
 	libvarnish.h \
 	libvcl.h \
 	miniobj.h \
@@ -98,7 +98,9 @@ vmod_abi.h: vcs_version.h
 	fi ; \
 	echo "#define VMOD_ABI_Version \"@PACKAGE_STRING@ $$GITID\"" > vmod_abi.h
 
-CLEANFILES = vcl_returns.h \
+CLEANFILES = \
+	tbl/vcl_returns.h \
+	tbl/vrt_stv_var.h \
 	vcl.h \
 	vrt_obj.h \
 	vmod_abi.h
diff --git a/include/ban_vars.h b/include/ban_vars.h
deleted file mode 100644
index 669b9b1..0000000
--- a/include/ban_vars.h
+++ /dev/null
@@ -1,37 +0,0 @@
-/*-
- * Copyright (c) 2008-2011 Varnish Software AS
- * All rights reserved.
- *
- * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- *
- * Define which variables we can ban on, and which function does it.
- *
- */
-
-#define PVAR_HTTP	1
-#define PVAR_REQ	2
-
-PVAR("req.url",		PVAR_REQ,		BAN_ARG_URL)
-PVAR("req.http.",	PVAR_REQ|PVAR_HTTP,	BAN_ARG_REQHTTP)
-PVAR("obj.http.",	PVAR_HTTP,		BAN_ARG_OBJHTTP)
diff --git a/include/http_headers.h b/include/http_headers.h
deleted file mode 100644
index afb463c..0000000
--- a/include/http_headers.h
+++ /dev/null
@@ -1,103 +0,0 @@
-/*-
- * Copyright (c) 2006 Verdens Gang AS
- * Copyright (c) 2006-2010 Varnish Software AS
- * All rights reserved.
- *
- * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- *
- * Argument list:
- * ---------------------------------------
- * a	Http header name
- * b	session field name
- * c	Request(1)/Response(2) bitfield
- * d	(obsolete)
- * e	Supress header in filter ops
- * f	unused
- * g	unused
- *
- * see [RFC2616 13.5.1 End-to-end and Hop-by-hop Headers]
- *
- */
-
-/*lint -save -e525 -e539 */
-
-#ifndef HTTPH_R_PASS
-#define HTTPH_R_PASS	(1 << 0)	/* Request (c->b) in pass mode */
-#define HTTPH_A_PASS	(1 << 1)	/* Response (b->c)in pass mode */
-#define HTTPH_R_PIPE	(1 << 2)	/* Request (c->b) in pipe mode */
-#define HTTPH_R_FETCH	(1 << 3)	/* Request (c->b) for fetch */
-#define HTTPH_A_INS	(1 << 4)	/* Response (b->o) for insert */
-#define HTTPH_A_DELIVER	(1 << 5)	/* Response (o->c) for deliver */
-#endif
-
-HTTPH("Keep-Alive",		H_Keep_Alive,		3, 3, HTTPH_R_PASS | HTTPH_A_PASS | HTTPH_R_FETCH,					0, 0)	/* RFC2068 */
-HTTPH("Accept",			H_Accept,		1, 0, 0,										0, 0)	/* RFC2616 14.1 */
-HTTPH("Accept-Charset",		H_Accept_Charset,	1, 0, 0,										0, 0)	/* RFC2616 14.2 */
-HTTPH("Accept-Encoding",	H_Accept_Encoding,	1, 0, 0,										0, 0)	/* RFC2616 14.3 */
-HTTPH("Accept-Language",	H_Accept_Language,	1, 0, 0,										0, 0)	/* RFC2616 14.4 */
-HTTPH("Accept-Ranges",		H_Accept_Ranges,	2, 3,				    HTTPH_R_FETCH | HTTPH_A_INS,			0, 0)	/* RFC2616 14.5 */
-HTTPH("Age",			H_Age,			2, 0,						    HTTPH_A_INS,			0, 0)	/* RFC2616 14.6 */
-HTTPH("Allow",			H_Allow,		2, 0, 0,										0, 0)	/* RFC2616 14.7 */
-HTTPH("Authorization",		H_Authorization,	1, 0, 0,										0, 0)	/* RFC2616 14.8 */
-HTTPH("Cache-Control",		H_Cache_Control,	3, 3,				    HTTPH_R_FETCH,					0, 0)	/* RFC2616 14.9 */
-HTTPH("Connection",		H_Connection,		3, 3, HTTPH_R_PASS | HTTPH_A_PASS | HTTPH_R_FETCH | HTTPH_A_INS,			0, 0)	/* RFC2616 14.10 */
-HTTPH("Content-Encoding",	H_Content_Encoding,	2, 0, 0,										0, 0)	/* RFC2616 14.11 */
-HTTPH("Content-Langugae",	H_Content_Language,	2, 0, 0,										0, 0)	/* RFC2616 14.12 */
-HTTPH("Content-Length",		H_Content_Length,	2, 2,				    HTTPH_R_FETCH | HTTPH_A_INS,			0, 0)	/* RFC2616 14.13 */
-HTTPH("Content-Location",	H_Content_Location,	2, 0, 0,										0, 0)	/* RFC2616 14.14 */
-HTTPH("Content-MD5",		H_Content_MD5,		2, 0, 0,										0, 0)	/* RFC2616 14.15 */
-HTTPH("Content-Range",		H_Content_Range,	2, 3,				    HTTPH_R_FETCH | HTTPH_A_INS,			0, 0)	/* RFC2616 14.16 */
-HTTPH("Content-Type",		H_Content_Type,		2, 0, 0,										0, 0)	/* RFC2616 14.17 */
-HTTPH("Date",			H_Date,			2, 0,								HTTPH_A_DELIVER,	0, 0)	/* RFC2616 14.18 */
-HTTPH("ETag",			H_ETag,			2, 0, 0,										0, 0)	/* RFC2616 14.19 */
-HTTPH("Expect",			H_Expect,		1, 0, 0,										0, 0)	/* RFC2616 14.20 */
-HTTPH("Expires",		H_Expires,		2, 0, 0,										0, 0)	/* RFC2616 14.21 */
-HTTPH("From",			H_From,			1, 0, 0,										0, 0)	/* RFC2616 14.22 */
-HTTPH("Host",			H_Host,			1, 0, 0,										0, 0)	/* RFC2616 14.23 */
-HTTPH("If-Match",		H_If_Match,		1, 1,				    HTTPH_R_FETCH,					0, 0)	/* RFC2616 14.24 */
-HTTPH("If-Modified-Since",	H_If_Modified_Since,	1, 1,				    HTTPH_R_FETCH,					0, 0)	/* RFC2616 14.25 */
-HTTPH("If-None-Match",		H_If_None_Match,	1, 1,				    HTTPH_R_FETCH,					0, 0)	/* RFC2616 14.26 */
-HTTPH("If-Range",		H_If_Range,		1, 1,				    HTTPH_R_FETCH,					0, 0)	/* RFC2616 14.27 */
-HTTPH("If-Unmodified-Since",	H_If_Unmodifed_Since,	1, 1,				    HTTPH_R_FETCH,					0, 0)	/* RFC2616 14.28 */
-HTTPH("Last-Modified",		H_Last_Modified,	2, 0, 0,										0, 0)	/* RFC2616 14.29 */
-HTTPH("Location",		H_Location,		2, 0, 0,										0, 0)	/* RFC2616 14.30 */
-HTTPH("Max-Forwards",		H_Max_Forwards,		1, 0, 0,										0, 0)	/* RFC2616 14.31 */
-HTTPH("Pragma",			H_Pragma,		1, 0, 0,										0, 0)	/* RFC2616 14.32 */
-HTTPH("Proxy-Authenticate",	H_Proxy_Authenticate,	2, 3,				    HTTPH_R_FETCH | HTTPH_A_INS,			0, 0)	/* RFC2616 14.33 */
-HTTPH("Proxy-Authorization",	H_Proxy_Authorization,	1, 3,				    HTTPH_R_FETCH | HTTPH_A_INS,			0, 0)	/* RFC2616 14.34 */
-HTTPH("Range",			H_Range,		1, 0,				    HTTPH_R_FETCH | HTTPH_A_INS,			0, 0)	/* RFC2616 14.35 */
-HTTPH("Referer",		H_Referer,		1, 0, 0,										0, 0)	/* RFC2616 14.36 */
-HTTPH("Retry-After",		H_Retry_After,		2, 0, 0,										0, 0)	/* RFC2616 14.37 */
-HTTPH("Server",			H_Server,		2, 0, 0,										0, 0)	/* RFC2616 14.38 */
-HTTPH("TE",			H_TE,			1, 3, HTTPH_R_PASS | HTTPH_A_PASS | HTTPH_R_FETCH | HTTPH_A_INS,			0, 0)	/* RFC2616 14.39 */
-HTTPH("Trailer",		H_Trailer,		1, 3, HTTPH_R_PASS | HTTPH_A_PASS | HTTPH_R_FETCH | HTTPH_A_INS,			0, 0)	/* RFC2616 14.40 */
-HTTPH("Transfer-Encoding",	H_Transfer_Encoding,	2, 3, HTTPH_R_PASS | HTTPH_A_PASS | HTTPH_R_FETCH | HTTPH_A_INS,			0, 0)	/* RFC2616 14.41 */
-HTTPH("Upgrade",		H_Upgrade,		2, 3, HTTPH_R_PASS | HTTPH_A_PASS | HTTPH_R_FETCH | HTTPH_A_INS,			0, 0)	/* RFC2616 14.42 */
-HTTPH("User-Agent",		H_User_Agent,		1, 0, 0,										0, 0)	/* RFC2616 14.43 */
-HTTPH("Vary",			H_Vary,			2, 0, 0,										0, 0)	/* RFC2616 14.44 */
-HTTPH("Via",			H_Via,			2, 0, 0,										0, 0)	/* RFC2616 14.45 */
-HTTPH("Warning",		H_Warning,		2, 0, 0,										0, 0)	/* RFC2616 14.46 */
-HTTPH("WWW-Authenticate",	H_WWW_Authenticate,	2, 0, 0,										0, 0)	/* RFC2616 14.47 */
-
-/*lint -restore */
diff --git a/include/http_response.h b/include/http_response.h
deleted file mode 100644
index f60820c..0000000
--- a/include/http_response.h
+++ /dev/null
@@ -1,70 +0,0 @@
-/*-
- * Copyright (c) 2006 Verdens Gang AS
- * Copyright (c) 2006-2009 Varnish Software AS
- * All rights reserved.
- *
- * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- *
- */
-
-HTTP_RESP(101, "Switching Protocols")
-HTTP_RESP(200, "OK")
-HTTP_RESP(201, "Created")
-HTTP_RESP(202, "Accepted")
-HTTP_RESP(203, "Non-Authoritative Information")
-HTTP_RESP(204, "No Content")
-HTTP_RESP(205, "Reset Content")
-HTTP_RESP(206, "Partial Content")
-HTTP_RESP(300, "Multiple Choices")
-HTTP_RESP(301, "Moved Permanently")
-HTTP_RESP(302, "Found")
-HTTP_RESP(303, "See Other")
-HTTP_RESP(304, "Not Modified")
-HTTP_RESP(305, "Use Proxy")
-HTTP_RESP(306, "(Unused)")
-HTTP_RESP(307, "Temporary Redirect")
-HTTP_RESP(400, "Bad Request")
-HTTP_RESP(401, "Unauthorized")
-HTTP_RESP(402, "Payment Required")
-HTTP_RESP(403, "Forbidden")
-HTTP_RESP(404, "Not Found")
-HTTP_RESP(405, "Method Not Allowed")
-HTTP_RESP(406, "Not Acceptable")
-HTTP_RESP(407, "Proxy Authentication Required")
-HTTP_RESP(408, "Request Timeout")
-HTTP_RESP(409, "Conflict")
-HTTP_RESP(410, "Gone")
-HTTP_RESP(411, "Length Required")
-HTTP_RESP(412, "Precondition Failed")
-HTTP_RESP(413, "Request Entity Too Large")
-HTTP_RESP(414, "Request-URI Too Long")
-HTTP_RESP(415, "Unsupported Media Type")
-HTTP_RESP(416, "Requested Range Not Satisfiable")
-HTTP_RESP(417, "Expectation Failed")
-HTTP_RESP(500, "Internal Server Error")
-HTTP_RESP(501, "Not Implemented")
-HTTP_RESP(502, "Bad Gateway")
-HTTP_RESP(503, "Service Unavailable")
-HTTP_RESP(504, "Gateway Timeout")
-HTTP_RESP(505, "HTTP Version Not Supported")
diff --git a/include/tbl/ban_vars.h b/include/tbl/ban_vars.h
new file mode 100644
index 0000000..669b9b1
--- /dev/null
+++ b/include/tbl/ban_vars.h
@@ -0,0 +1,37 @@
+/*-
+ * Copyright (c) 2008-2011 Varnish Software AS
+ * All rights reserved.
+ *
+ * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * Define which variables we can ban on, and which function does it.
+ *
+ */
+
+#define PVAR_HTTP	1
+#define PVAR_REQ	2
+
+PVAR("req.url",		PVAR_REQ,		BAN_ARG_URL)
+PVAR("req.http.",	PVAR_REQ|PVAR_HTTP,	BAN_ARG_REQHTTP)
+PVAR("obj.http.",	PVAR_HTTP,		BAN_ARG_OBJHTTP)
diff --git a/include/tbl/http_headers.h b/include/tbl/http_headers.h
new file mode 100644
index 0000000..afb463c
--- /dev/null
+++ b/include/tbl/http_headers.h
@@ -0,0 +1,103 @@
+/*-
+ * Copyright (c) 2006 Verdens Gang AS
+ * Copyright (c) 2006-2010 Varnish Software AS
+ * All rights reserved.
+ *
+ * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * Argument list:
+ * ---------------------------------------
+ * a	Http header name
+ * b	session field name
+ * c	Request(1)/Response(2) bitfield
+ * d	(obsolete)
+ * e	Supress header in filter ops
+ * f	unused
+ * g	unused
+ *
+ * see [RFC2616 13.5.1 End-to-end and Hop-by-hop Headers]
+ *
+ */
+
+/*lint -save -e525 -e539 */
+
+#ifndef HTTPH_R_PASS
+#define HTTPH_R_PASS	(1 << 0)	/* Request (c->b) in pass mode */
+#define HTTPH_A_PASS	(1 << 1)	/* Response (b->c)in pass mode */
+#define HTTPH_R_PIPE	(1 << 2)	/* Request (c->b) in pipe mode */
+#define HTTPH_R_FETCH	(1 << 3)	/* Request (c->b) for fetch */
+#define HTTPH_A_INS	(1 << 4)	/* Response (b->o) for insert */
+#define HTTPH_A_DELIVER	(1 << 5)	/* Response (o->c) for deliver */
+#endif
+
+HTTPH("Keep-Alive",		H_Keep_Alive,		3, 3, HTTPH_R_PASS | HTTPH_A_PASS | HTTPH_R_FETCH,					0, 0)	/* RFC2068 */
+HTTPH("Accept",			H_Accept,		1, 0, 0,										0, 0)	/* RFC2616 14.1 */
+HTTPH("Accept-Charset",		H_Accept_Charset,	1, 0, 0,										0, 0)	/* RFC2616 14.2 */
+HTTPH("Accept-Encoding",	H_Accept_Encoding,	1, 0, 0,										0, 0)	/* RFC2616 14.3 */
+HTTPH("Accept-Language",	H_Accept_Language,	1, 0, 0,										0, 0)	/* RFC2616 14.4 */
+HTTPH("Accept-Ranges",		H_Accept_Ranges,	2, 3,				    HTTPH_R_FETCH | HTTPH_A_INS,			0, 0)	/* RFC2616 14.5 */
+HTTPH("Age",			H_Age,			2, 0,						    HTTPH_A_INS,			0, 0)	/* RFC2616 14.6 */
+HTTPH("Allow",			H_Allow,		2, 0, 0,										0, 0)	/* RFC2616 14.7 */
+HTTPH("Authorization",		H_Authorization,	1, 0, 0,										0, 0)	/* RFC2616 14.8 */
+HTTPH("Cache-Control",		H_Cache_Control,	3, 3,				    HTTPH_R_FETCH,					0, 0)	/* RFC2616 14.9 */
+HTTPH("Connection",		H_Connection,		3, 3, HTTPH_R_PASS | HTTPH_A_PASS | HTTPH_R_FETCH | HTTPH_A_INS,			0, 0)	/* RFC2616 14.10 */
+HTTPH("Content-Encoding",	H_Content_Encoding,	2, 0, 0,										0, 0)	/* RFC2616 14.11 */
+HTTPH("Content-Langugae",	H_Content_Language,	2, 0, 0,										0, 0)	/* RFC2616 14.12 */
+HTTPH("Content-Length",		H_Content_Length,	2, 2,				    HTTPH_R_FETCH | HTTPH_A_INS,			0, 0)	/* RFC2616 14.13 */
+HTTPH("Content-Location",	H_Content_Location,	2, 0, 0,										0, 0)	/* RFC2616 14.14 */
+HTTPH("Content-MD5",		H_Content_MD5,		2, 0, 0,										0, 0)	/* RFC2616 14.15 */
+HTTPH("Content-Range",		H_Content_Range,	2, 3,				    HTTPH_R_FETCH | HTTPH_A_INS,			0, 0)	/* RFC2616 14.16 */
+HTTPH("Content-Type",		H_Content_Type,		2, 0, 0,										0, 0)	/* RFC2616 14.17 */
+HTTPH("Date",			H_Date,			2, 0,								HTTPH_A_DELIVER,	0, 0)	/* RFC2616 14.18 */
+HTTPH("ETag",			H_ETag,			2, 0, 0,										0, 0)	/* RFC2616 14.19 */
+HTTPH("Expect",			H_Expect,		1, 0, 0,										0, 0)	/* RFC2616 14.20 */
+HTTPH("Expires",		H_Expires,		2, 0, 0,										0, 0)	/* RFC2616 14.21 */
+HTTPH("From",			H_From,			1, 0, 0,										0, 0)	/* RFC2616 14.22 */
+HTTPH("Host",			H_Host,			1, 0, 0,										0, 0)	/* RFC2616 14.23 */
+HTTPH("If-Match",		H_If_Match,		1, 1,				    HTTPH_R_FETCH,					0, 0)	/* RFC2616 14.24 */
+HTTPH("If-Modified-Since",	H_If_Modified_Since,	1, 1,				    HTTPH_R_FETCH,					0, 0)	/* RFC2616 14.25 */
+HTTPH("If-None-Match",		H_If_None_Match,	1, 1,				    HTTPH_R_FETCH,					0, 0)	/* RFC2616 14.26 */
+HTTPH("If-Range",		H_If_Range,		1, 1,				    HTTPH_R_FETCH,					0, 0)	/* RFC2616 14.27 */
+HTTPH("If-Unmodified-Since",	H_If_Unmodifed_Since,	1, 1,				    HTTPH_R_FETCH,					0, 0)	/* RFC2616 14.28 */
+HTTPH("Last-Modified",		H_Last_Modified,	2, 0, 0,										0, 0)	/* RFC2616 14.29 */
+HTTPH("Location",		H_Location,		2, 0, 0,										0, 0)	/* RFC2616 14.30 */
+HTTPH("Max-Forwards",		H_Max_Forwards,		1, 0, 0,										0, 0)	/* RFC2616 14.31 */
+HTTPH("Pragma",			H_Pragma,		1, 0, 0,										0, 0)	/* RFC2616 14.32 */
+HTTPH("Proxy-Authenticate",	H_Proxy_Authenticate,	2, 3,				    HTTPH_R_FETCH | HTTPH_A_INS,			0, 0)	/* RFC2616 14.33 */
+HTTPH("Proxy-Authorization",	H_Proxy_Authorization,	1, 3,				    HTTPH_R_FETCH | HTTPH_A_INS,			0, 0)	/* RFC2616 14.34 */
+HTTPH("Range",			H_Range,		1, 0,				    HTTPH_R_FETCH | HTTPH_A_INS,			0, 0)	/* RFC2616 14.35 */
+HTTPH("Referer",		H_Referer,		1, 0, 0,										0, 0)	/* RFC2616 14.36 */
+HTTPH("Retry-After",		H_Retry_After,		2, 0, 0,										0, 0)	/* RFC2616 14.37 */
+HTTPH("Server",			H_Server,		2, 0, 0,										0, 0)	/* RFC2616 14.38 */
+HTTPH("TE",			H_TE,			1, 3, HTTPH_R_PASS | HTTPH_A_PASS | HTTPH_R_FETCH | HTTPH_A_INS,			0, 0)	/* RFC2616 14.39 */
+HTTPH("Trailer",		H_Trailer,		1, 3, HTTPH_R_PASS | HTTPH_A_PASS | HTTPH_R_FETCH | HTTPH_A_INS,			0, 0)	/* RFC2616 14.40 */
+HTTPH("Transfer-Encoding",	H_Transfer_Encoding,	2, 3, HTTPH_R_PASS | HTTPH_A_PASS | HTTPH_R_FETCH | HTTPH_A_INS,			0, 0)	/* RFC2616 14.41 */
+HTTPH("Upgrade",		H_Upgrade,		2, 3, HTTPH_R_PASS | HTTPH_A_PASS | HTTPH_R_FETCH | HTTPH_A_INS,			0, 0)	/* RFC2616 14.42 */
+HTTPH("User-Agent",		H_User_Agent,		1, 0, 0,										0, 0)	/* RFC2616 14.43 */
+HTTPH("Vary",			H_Vary,			2, 0, 0,										0, 0)	/* RFC2616 14.44 */
+HTTPH("Via",			H_Via,			2, 0, 0,										0, 0)	/* RFC2616 14.45 */
+HTTPH("Warning",		H_Warning,		2, 0, 0,										0, 0)	/* RFC2616 14.46 */
+HTTPH("WWW-Authenticate",	H_WWW_Authenticate,	2, 0, 0,										0, 0)	/* RFC2616 14.47 */
+
+/*lint -restore */
diff --git a/include/tbl/http_response.h b/include/tbl/http_response.h
new file mode 100644
index 0000000..f60820c
--- /dev/null
+++ b/include/tbl/http_response.h
@@ -0,0 +1,70 @@
+/*-
+ * Copyright (c) 2006 Verdens Gang AS
+ * Copyright (c) 2006-2009 Varnish Software AS
+ * All rights reserved.
+ *
+ * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ */
+
+HTTP_RESP(101, "Switching Protocols")
+HTTP_RESP(200, "OK")
+HTTP_RESP(201, "Created")
+HTTP_RESP(202, "Accepted")
+HTTP_RESP(203, "Non-Authoritative Information")
+HTTP_RESP(204, "No Content")
+HTTP_RESP(205, "Reset Content")
+HTTP_RESP(206, "Partial Content")
+HTTP_RESP(300, "Multiple Choices")
+HTTP_RESP(301, "Moved Permanently")
+HTTP_RESP(302, "Found")
+HTTP_RESP(303, "See Other")
+HTTP_RESP(304, "Not Modified")
+HTTP_RESP(305, "Use Proxy")
+HTTP_RESP(306, "(Unused)")
+HTTP_RESP(307, "Temporary Redirect")
+HTTP_RESP(400, "Bad Request")
+HTTP_RESP(401, "Unauthorized")
+HTTP_RESP(402, "Payment Required")
+HTTP_RESP(403, "Forbidden")
+HTTP_RESP(404, "Not Found")
+HTTP_RESP(405, "Method Not Allowed")
+HTTP_RESP(406, "Not Acceptable")
+HTTP_RESP(407, "Proxy Authentication Required")
+HTTP_RESP(408, "Request Timeout")
+HTTP_RESP(409, "Conflict")
+HTTP_RESP(410, "Gone")
+HTTP_RESP(411, "Length Required")
+HTTP_RESP(412, "Precondition Failed")
+HTTP_RESP(413, "Request Entity Too Large")
+HTTP_RESP(414, "Request-URI Too Long")
+HTTP_RESP(415, "Unsupported Media Type")
+HTTP_RESP(416, "Requested Range Not Satisfiable")
+HTTP_RESP(417, "Expectation Failed")
+HTTP_RESP(500, "Internal Server Error")
+HTTP_RESP(501, "Not Implemented")
+HTTP_RESP(502, "Bad Gateway")
+HTTP_RESP(503, "Service Unavailable")
+HTTP_RESP(504, "Gateway Timeout")
+HTTP_RESP(505, "HTTP Version Not Supported")
diff --git a/lib/libvcl/generate.py b/lib/libvcl/generate.py
index 779091e..335cfd5 100755
--- a/lib/libvcl/generate.py
+++ b/lib/libvcl/generate.py
@@ -671,7 +671,7 @@ for i in returns:
 
 #######################################################################
 
-fo = open(buildroot + "/include/vcl_returns.h", "w")
+fo = open(buildroot + "/include/tbl/vcl_returns.h", "w")
 
 file_header(fo)
 
@@ -891,7 +891,7 @@ fo.close()
 
 #######################################################################
 
-fo = open(buildroot + "/include/vrt_stv_var.h", "w")
+fo = open(buildroot + "/include/tbl/vrt_stv_var.h", "w")
 
 file_header(fo)
 
diff --git a/lib/libvcl/vcc_action.c b/lib/libvcl/vcc_action.c
index 8c538ef..c11f5b2 100644
--- a/lib/libvcl/vcc_action.c
+++ b/lib/libvcl/vcc_action.c
@@ -269,7 +269,7 @@ parse_return(struct vcc *tl)
 			retval = 1;					\
 		}							\
 	} while (0);
-#include "vcl_returns.h"
+#include "tbl/vcl_returns.h"
 #undef VCL_RET_MAC
 	if (!retval) {
 		VSB_printf(tl->sb, "Expected return action name.\n");
@@ -330,7 +330,7 @@ static struct action_table {
 
 #define VCL_RET_MAC(l, U, B)						\
 	{ #l,			parse_new_syntax },
-#include "vcl_returns.h"
+#include "tbl/vcl_returns.h"
 #undef VCL_RET_MAC
 
 	/* Keep list sorted from here */
diff --git a/lib/libvcl/vcc_compile.c b/lib/libvcl/vcc_compile.c
index 36efa4b..a09f396 100644
--- a/lib/libvcl/vcc_compile.c
+++ b/lib/libvcl/vcc_compile.c
@@ -70,7 +70,7 @@
 
 struct method method_tab[] = {
 #define VCL_MET_MAC(l,U,m)	{ "vcl_"#l, m, VCL_MET_##U },
-#include "vcl_returns.h"
+#include "tbl/vcl_returns.h"
 #undef VCL_MET_MAC
 	{ NULL, 0U, 0}
 };
@@ -374,7 +374,7 @@ EmitStruct(const struct vcc *tl)
 	Fc(tl, 0, "\t.srcbody = srcbody,\n");
 #define VCL_MET_MAC(l,u,b) \
 	Fc(tl, 0, "\t." #l "_func = VGC_function_vcl_" #l ",\n");
-#include "vcl_returns.h"
+#include "tbl/vcl_returns.h"
 #undef VCL_MET_MAC
 	Fc(tl, 0, "};\n");
 }
@@ -722,7 +722,7 @@ VCC_Return_Name(unsigned method)
 
 	switch (method) {
 #define VCL_RET_MAC(l, U, B) case VCL_RET_##U: return(#l);
-#include "vcl_returns.h"
+#include "tbl/vcl_returns.h"
 #undef VCL_RET_MAC
 	default:
 		return (NULL);
diff --git a/lib/libvcl/vcc_storage.c b/lib/libvcl/vcc_storage.c
index 77c657c..5aff6e7 100644
--- a/lib/libvcl/vcc_storage.c
+++ b/lib/libvcl/vcc_storage.c
@@ -80,7 +80,7 @@ vcc_Stv_mkvar(struct vcc *tl, const struct token *t, enum var_type fmt)
 	v->name = TlDupTok(tl, t);
 	v->r_methods = 0
 #define VCL_MET_MAC(l,u,b)	| VCL_MET_##u
-#include "vcl_returns.h"
+#include "tbl/vcl_returns.h"
 #undef VCL_MET_MAC
 		;
 	v->fmt = fmt;
@@ -93,7 +93,7 @@ static struct stvars {
 	enum var_type	fmt;
 } stvars[] = {
 #define VRTSTVVAR(nm, vtype, ctype, dval)	{ #nm, vtype },
-#include "vrt_stv_var.h"
+#include "tbl/vrt_stv_var.h"
 #undef VRTSTVVAR
 	{ NULL,			BOOL }
 };
diff --git a/lib/libvcl/vcc_xref.c b/lib/libvcl/vcc_xref.c
index 560fe78..56a2747 100644
--- a/lib/libvcl/vcc_xref.c
+++ b/lib/libvcl/vcc_xref.c
@@ -226,7 +226,7 @@ vcc_CheckActionRecurse(struct vcc *tl, struct proc *p, unsigned bitmap)
 			VSB_printf(tl->sb, "Invalid return \"" #l "\"\n");\
 			vcc_ErrWhere(tl, p->return_tok[VCL_RET_##U]);	\
 		}
-#include "vcl_returns.h"
+#include "tbl/vcl_returns.h"
 #undef VCL_RET_MAC
 
 		VSB_printf(tl->sb, "\n...in subroutine \"%.*s\"\n",
@@ -271,7 +271,7 @@ vcc_checkaction1(struct vcc *tl, const struct symbol *sym)
 		if (m->ret_bitmap & ((1 << VCL_RET_##U)))	\
 			VSB_printf(tl->sb, " \"%s\"", #l);
 
-#include "vcl_returns.h"
+#include "tbl/vcl_returns.h"
 #undef VCL_RET_MAC
 		VSB_printf(tl->sb, "\n");
 		tl->err = 1;



From geoff at varnish-cache.org  Mon Jan  9 20:52:16 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:16 +0100
Subject: [experimental-ims] 6888981 Add necessary #includes
Message-ID: <E1RkMCW-0007cS-L1@project.varnish-software.com>

commit 6888981c59d95d9832f5fac29a5b2e6aae7752fa
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sun Oct 9 20:14:45 2011 +0000

    Add necessary #includes

diff --git a/bin/varnishd/varnishd.c b/bin/varnishd/varnishd.c
index bf61ea3..9ba9570 100644
--- a/bin/varnishd/varnishd.c
+++ b/bin/varnishd/varnishd.c
@@ -40,6 +40,7 @@
 #include <stdlib.h>
 #include <string.h>
 #include <syslog.h>
+#include <time.h>
 #include <unistd.h>
 
 #include <sys/utsname.h>
diff --git a/bin/varnishd/vsm.c b/bin/varnishd/vsm.c
index dc265ba..d1717fe 100644
--- a/bin/varnishd/vsm.c
+++ b/bin/varnishd/vsm.c
@@ -47,13 +47,15 @@
 
 #include "config.h"
 
-#include <unistd.h>
-#include <string.h>
+#include <stdint.h>
 #include <stdio.h>
+#include <string.h>
+#include <unistd.h>
 
 #include "common.h"
-#include "vsm.h"
+
 #include "vmb.h"
+#include "vsm.h"
 #include "vtim.h"
 
 /* These two come from beyond (mgt_shmem.c actually) */
diff --git a/bin/varnishtest/vtc.c b/bin/varnishtest/vtc.c
index 4404de9..5c6bbfd 100644
--- a/bin/varnishtest/vtc.c
+++ b/bin/varnishtest/vtc.c
@@ -28,14 +28,15 @@
 
 #include "config.h"
 
-#include <stdio.h>
-#include <string.h>
 #include <ctype.h>
 #include <stdarg.h>
+#include <stdint.h>
+#include <stdio.h>
 #include <stdlib.h>
-#include <unistd.h>
+#include <string.h>
 #include <sys/types.h>
 #include <sys/wait.h>
+#include <unistd.h>
 
 #include "vtc.h"
 



From geoff at varnish-cache.org  Mon Jan  9 20:52:19 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:19 +0100
Subject: [experimental-ims] ac66673 Quit early if setting blocking mode fails.
Message-ID: <E1RkMCZ-0007iV-8g@project.varnish-software.com>

commit ac666733b7985436b77f10f5276909e7e65933c9
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu Oct 13 10:11:42 2011 +0000

    Quit early if setting blocking mode fails.
    
    Submitted by:	Nils Goroll

diff --git a/bin/varnishd/cache_center.c b/bin/varnishd/cache_center.c
index 0a81925..47139c7 100644
--- a/bin/varnishd/cache_center.c
+++ b/bin/varnishd/cache_center.c
@@ -1573,14 +1573,21 @@ CNT_Session(struct sess *sp)
 	AZ(w->do_esi);
 
 	/*
-	 * Whenever we come in from the acceptor we need to set blocking
-	 * mode, but there is no point in setting it when we come from
+	 * Whenever we come in from the acceptor or waiter, we need to set
+	 * blocking mode, but there is no point in setting it when we come from
 	 * ESI or when a parked sessions returns.
-	 * It would be simpler to do this in the acceptor, but we'd rather
-	 * do the syscall in the worker thread.
+	 * It would be simpler to do this in the acceptor or waiter, but we'd
+	 * rather do the syscall in the worker thread.
+	 * On systems which return errors for ioctl, we close early
 	 */
-	if (sp->step == STP_FIRST || sp->step == STP_START)
-		(void)VTCP_blocking(sp->fd);
+	if ((sp->step == STP_FIRST || sp->step == STP_START) &&
+	    VTCP_blocking(sp->fd)) {
+		if (errno == ECONNRESET)
+			SES_Close(sp, "remote closed");
+		else
+			SES_Close(sp, "error");
+		sp->step = STP_DONE;
+	}
 
 	/*
 	 * NB: Once done is set, we can no longer touch sp!



From geoff at varnish-cache.org  Mon Jan  9 20:51:46 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:51:46 +0100
Subject: [experimental-ims] eea97db Prototype adjustment for last commit
Message-ID: <E1RkMC2-0006lY-Bw@project.varnish-software.com>

commit eea97dbea388be4d52a5919b0ec938bae4d0e6e3
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Sep 7 19:54:35 2011 +0000

    Prototype adjustment for last commit

diff --git a/lib/libvcl/vcc_compile.c b/lib/libvcl/vcc_compile.c
index c2a52d4..f4a85a0 100644
--- a/lib/libvcl/vcc_compile.c
+++ b/lib/libvcl/vcc_compile.c
@@ -322,7 +322,7 @@ EmitFiniFunc(const struct vcc *tl)
 {
 	unsigned u;
 
-	Fc(tl, 0, "\nstatic int\nVGC_Fini(struct cli *cli)\n{\n\n");
+	Fc(tl, 0, "\nstatic void\nVGC_Fini(struct cli *cli)\n{\n\n");
 
 	/*
 	 * We do this here, so we are sure they happen before any
@@ -333,7 +333,6 @@ EmitFiniFunc(const struct vcc *tl)
 
 	AZ(VSB_finish(tl->ff));
 	VSB_cat(tl->fc, VSB_data(tl->ff));
-	Fc(tl, 0, "\treturn(0);\n");
 	Fc(tl, 0, "}\n");
 }
 



From geoff at varnish-cache.org  Mon Jan  9 20:51:59 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:51:59 +0100
Subject: [experimental-ims] 96a4a90 Try to make it easier to see why this test
	fails.
Message-ID: <E1RkMCF-00076p-I9@project.varnish-software.com>

commit 96a4a9034bb0766228720bccba04cba3726f24b1
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Sep 20 12:15:54 2011 +0000

    Try to make it easier to see why this test fails.

diff --git a/bin/varnishtest/tests/r00962.vtc b/bin/varnishtest/tests/r00962.vtc
index 66f447b..faf4b4b 100644
--- a/bin/varnishtest/tests/r00962.vtc
+++ b/bin/varnishtest/tests/r00962.vtc
@@ -32,6 +32,12 @@ varnish v1 -cliok "debug.persistent s0 dump"
 varnish v1 -cliok "debug.persistent s0 sync"
 varnish v1 -stop
 
+server s1 {
+	rxreq 
+	txresp -status 400 -msg "Persistent Object Not Found"
+} -start
+
+
 varnish v2 \
 	-arg "-pdiag_bitmap=0x20000" \
 	-storage "-spersistent,${tmpdir}/_.per2,10m -spersistent,${tmpdir}/_.per1,10m" \
@@ -40,6 +46,7 @@ varnish v2 \
 client c1 -connect ${v2_sock} {
 	txreq -url "/"
 	rxresp
+	expect resp.msg != "Persistent Object Not Found"
 	expect resp.status == 200
 	expect resp.http.X-Varnish == "1001"
 } -run



From geoff at varnish-cache.org  Mon Jan  9 20:52:27 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:27 +0100
Subject: [experimental-ims] ea6f0a4 Build without rst2man and rst2html
Message-ID: <E1RkMCh-0007xH-13@project.varnish-software.com>

commit ea6f0a4af5e35db76151da65b6ab7f24e0404134
Author: Tollef Fog Heen <tfheen at varnish-software.com>
Date:   Wed Oct 26 14:11:39 2011 +0200

    Build without rst2man and rst2html

diff --git a/redhat/varnish.spec b/redhat/varnish.spec
index 0e3f1e6..5cf9c97 100644
--- a/redhat/varnish.spec
+++ b/redhat/varnish.spec
@@ -91,9 +91,9 @@ cp bin/varnishd/default.vcl etc/zope-plone.vcl examples
 # Remove "--disable static" if you want to build static libraries 
 # jemalloc is not compatible with Red Hat's ppc64 RHEL kernel :-(
 %ifarch ppc64 ppc
-	%configure --disable-static --localstatedir=/var/lib --without-jemalloc
+	%configure --disable-static --localstatedir=/var/lib --without-jemalloc  --without-rst2man --without-rst2html
 %else
-	%configure --disable-static --localstatedir=/var/lib
+	%configure --disable-static --localstatedir=/var/lib --without-rst2man --without-rst2html
 %endif
 
 # We have to remove rpath - not allowed in Fedora



From geoff at varnish-cache.org  Mon Jan  9 20:52:30 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:30 +0100
Subject: [experimental-ims] 2befbaf Add a missing WS_Release()
Message-ID: <E1RkMCk-00085n-H5@project.varnish-software.com>

commit 2befbaf97ce358dcb8d74c5e7e5ff8b3b38f87ff
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Nov 8 14:50:11 2011 +0000

    Add a missing WS_Release()

diff --git a/bin/varnishd/cache_vrt_re.c b/bin/varnishd/cache_vrt_re.c
index 955b7bf..d6dc5d5 100644
--- a/bin/varnishd/cache_vrt_re.c
+++ b/bin/varnishd/cache_vrt_re.c
@@ -143,6 +143,7 @@ VRT_regsub(const struct sess *sp, int all, const char *str, void *re,
 		i = VRE_exec(t, str, len, 0, options, ovector, 30,
 		    &params->vre_limits);
 		if (i < VRE_ERROR_NOMATCH ) {
+			WS_Release(sp->http->ws, 0);
 			WSP(sp, SLT_VCL_Error,
 			    "Regexp matching returned %d", i);
 			return(str);



From geoff at varnish-cache.org  Mon Jan  9 20:52:48 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:48 +0100
Subject: [experimental-ims] 6d66ab6 Minor polishing
Message-ID: <E1RkMD2-0000BV-DV@project.varnish-software.com>

commit 6d66ab671ceaa31e719e868f235bd4833666e0c8
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Dec 6 08:19:05 2011 +0000

    Minor polishing

diff --git a/bin/varnishd/cache/cache_esi_parse.c b/bin/varnishd/cache/cache_esi_parse.c
index 71ec16d..41992ab 100644
--- a/bin/varnishd/cache/cache_esi_parse.c
+++ b/bin/varnishd/cache/cache_esi_parse.c
@@ -86,7 +86,7 @@ struct vep_state {
 	uint32_t		crcp;
 	ssize_t			o_last;
 
-const char		*hack_p;
+	const char		*hack_p;
 	const char		*ver_p;
 
 	const char		*until;
@@ -548,16 +548,16 @@ vep_do_include(struct vep_state *vep, enum dowhat what)
  */
 
 void
-VEP_Parse(const struct worker *w, const char *p, size_t l)
+VEP_Parse(const struct worker *wrk, const char *p, size_t l)
 {
 	struct vep_state *vep;
 	const char *e;
 	struct vep_match *vm;
 	int i;
 
-	CHECK_OBJ_NOTNULL(w, WORKER_MAGIC);
-	CHECK_OBJ_NOTNULL(w->busyobj, BUSYOBJ_MAGIC);
-	vep = w->busyobj->vep;
+	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
+	CHECK_OBJ_NOTNULL(wrk->busyobj, BUSYOBJ_MAGIC);
+	vep = wrk->busyobj->vep;
 	CHECK_OBJ_NOTNULL(vep, VEP_MAGIC);
 	assert(l > 0);
 
@@ -980,13 +980,13 @@ VEP_Parse(const struct worker *w, const char *p, size_t l)
  */
 
 static ssize_t __match_proto__()
-vep_default_cb(struct worker *w, ssize_t l, enum vgz_flag flg)
+vep_default_cb(struct worker *wrk, ssize_t l, enum vgz_flag flg)
 {
 	struct vep_state *vep;
 
-	CHECK_OBJ_NOTNULL(w, WORKER_MAGIC);
-	CHECK_OBJ_NOTNULL(w->busyobj, BUSYOBJ_MAGIC);
-	vep = w->busyobj->vep;
+	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
+	CHECK_OBJ_NOTNULL(wrk->busyobj, BUSYOBJ_MAGIC);
+	vep = wrk->busyobj->vep;
 	CHECK_OBJ_NOTNULL(vep, VEP_MAGIC);
 	(void)flg;
 	vep->cb_x += l;
@@ -997,22 +997,20 @@ vep_default_cb(struct worker *w, ssize_t l, enum vgz_flag flg)
  */
 
 void
-VEP_Init(struct worker *w, vep_callback_t *cb)
+VEP_Init(struct worker *wrk, vep_callback_t *cb)
 {
 	struct vep_state *vep;
 
-	CHECK_OBJ_NOTNULL(w, WORKER_MAGIC);
-	CHECK_OBJ_NOTNULL(w->busyobj, BUSYOBJ_MAGIC);
-	AZ(w->busyobj->vep);
-	vep = (void*)WS_Alloc(w->ws, sizeof *vep);
+	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
+	CHECK_OBJ_NOTNULL(wrk->busyobj, BUSYOBJ_MAGIC);
+	AZ(wrk->busyobj->vep);
+	vep = (void*)WS_Alloc(wrk->ws, sizeof *vep);
 	AN(vep);
 
-	memset(vep, 0, sizeof *vep);
-	vep->magic = VEP_MAGIC;
-	vep->wrk = w;
+	vep->wrk = wrk;
 	vep->vsb = VSB_new_auto();
 	AN(vep->vsb);
-	w->busyobj->vep = vep;
+	wrk->busyobj->vep = vep;
 
 	if (cb != NULL) {
 		vep->dogzip = 1;
@@ -1043,14 +1041,14 @@ VEP_Init(struct worker *w, vep_callback_t *cb)
  */
 
 struct vsb *
-VEP_Finish(const struct worker *w)
+VEP_Finish(const struct worker *wrk)
 {
 	struct vep_state *vep;
 	ssize_t l, lcb;
 
-	CHECK_OBJ_NOTNULL(w, WORKER_MAGIC);
-	CHECK_OBJ_NOTNULL(w->busyobj, BUSYOBJ_MAGIC);
-	vep = w->busyobj->vep;
+	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
+	CHECK_OBJ_NOTNULL(wrk->busyobj, BUSYOBJ_MAGIC);
+	vep = wrk->busyobj->vep;
 	CHECK_OBJ_NOTNULL(vep, VEP_MAGIC);
 
 	if (vep->o_pending)
@@ -1061,8 +1059,7 @@ VEP_Finish(const struct worker *w)
 	}
 	(void)vep->cb(vep->wrk, 0, VGZ_FINISH);
 
-	w->busyobj->vep = NULL;
-
+	wrk->busyobj->vep = NULL;
 	AZ(VSB_finish(vep->vsb));
 	l = VSB_len(vep->vsb);
 	if (vep->esi_found && l > 0)



From geoff at varnish-cache.org  Mon Jan  9 20:52:33 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:33 +0100
Subject: [experimental-ims] 2144dc7 Use better criteria for determining if
	child CLI connection is hosed.
Message-ID: <E1RkMCn-0008Af-9y@project.varnish-software.com>

commit 2144dc7885a425926be4a74ecb89c5b38d062ac8
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sun Nov 13 22:02:44 2011 +0000

    Use better criteria for determining if child CLI connection is hosed.

diff --git a/bin/varnishd/mgt/mgt_cli.c b/bin/varnishd/mgt/mgt_cli.c
index 3a6f365..70e8557 100644
--- a/bin/varnishd/mgt/mgt_cli.c
+++ b/bin/varnishd/mgt/mgt_cli.c
@@ -171,7 +171,8 @@ mcf_askchild(struct cli *cli, const char * const *av, void *priv)
 		return;
 	}
 	VSB_delete(vsb);
-	(void)VCLI_ReadResult(cli_i, &u, &q, mgt_param.cli_timeout);
+	if (VCLI_ReadResult(cli_i, &u, &q, mgt_param.cli_timeout))
+		MGT_Child_Cli_Fail();
 	VCLI_SetResult(cli, u);
 	VCLI_Out(cli, "%s", q);
 	free(q);
@@ -220,11 +221,10 @@ mgt_cli_askchild(unsigned *status, char **resp, const char *fmt, ...) {
 		return (CLIS_COMMS);
 	}
 
-	(void)VCLI_ReadResult(cli_i, &u, resp, mgt_param.cli_timeout);
+	if (VCLI_ReadResult(cli_i, &u, resp, mgt_param.cli_timeout))
+		MGT_Child_Cli_Fail();
 	if (status != NULL)
 		*status = u;
-	if (u == CLIS_COMMS)
-		MGT_Child_Cli_Fail();
 	return (u == CLIS_OK ? 0 : u);
 }
 



From geoff at varnish-cache.org  Mon Jan  9 20:52:34 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:34 +0100
Subject: [experimental-ims] 9857918 Add VCLI_Overflow() so we can stop
	rendering bans when the CLI output buffer is full.
Message-ID: <E1RkMCo-0008Bk-48@project.varnish-software.com>

commit 9857918cf7916d860fe3f4358960467359c6fd5c
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Nov 14 11:26:44 2011 +0000

    Add VCLI_Overflow() so we can stop rendering bans when the CLI
    output buffer is full.

diff --git a/bin/varnishd/cache/cache_ban.c b/bin/varnishd/cache/cache_ban.c
index 768e631..ca2b616 100644
--- a/bin/varnishd/cache/cache_ban.c
+++ b/bin/varnishd/cache/cache_ban.c
@@ -1071,6 +1071,8 @@ ccf_ban_list(struct cli *cli, const char * const *av, void *priv)
 		    b->flags & BAN_F_GONE ? "G" : " ");
 		ban_render(cli, b->spec);
 		VCLI_Out(cli, "\n");
+		if (VCLI_Overflow(cli))
+			break;
 		if (cache_param->diag_bitmap & 0x80000) {
 			Lck_Lock(&ban_mtx);
 			struct objcore *oc;
diff --git a/include/vcli_priv.h b/include/vcli_priv.h
index 60d6d69..a265d2e 100644
--- a/include/vcli_priv.h
+++ b/include/vcli_priv.h
@@ -52,6 +52,7 @@ struct cli_proto {
 };
 
 /* The implementation must provide these functions */
+int VCLI_Overflow(struct cli *cli);
 void VCLI_Out(struct cli *cli, const char *fmt, ...);
 void VCLI_Quote(struct cli *cli, const char *str);
 void VCLI_SetResult(struct cli *cli, unsigned r);
diff --git a/lib/libvarnish/cli_common.c b/lib/libvarnish/cli_common.c
index 7b09bdb..aa5fd63 100644
--- a/lib/libvarnish/cli_common.c
+++ b/lib/libvarnish/cli_common.c
@@ -70,6 +70,17 @@ VCLI_Out(struct cli *cli, const char *fmt, ...)
 }
 
 /*lint -e{818} cli could be const */
+int
+VCLI_Overflow(struct cli *cli)
+{
+	CHECK_OBJ_NOTNULL(cli, CLI_MAGIC);
+	if (cli->result == CLIS_TRUNCATED ||
+	    VSB_len(cli->sb) >= *cli->limit)
+		return (1);
+	return (0);
+}
+
+/*lint -e{818} cli could be const */
 void
 VCLI_Quote(struct cli *cli, const char *s)
 {



From geoff at varnish-cache.org  Mon Jan  9 20:52:25 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 09 Jan 2012 21:52:25 +0100
Subject: [experimental-ims] 3999568 Shave some session from FetchBody
Message-ID: <E1RkMCf-0007tz-LU@project.varnish-software.com>

commit 3999568c2dc6be3f801484cea7cfa614535a9508
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Oct 24 12:19:26 2011 +0000

    Shave some session from FetchBody

diff --git a/bin/varnishd/cache.h b/bin/varnishd/cache.h
index 59d5fa8..71cc776 100644
--- a/bin/varnishd/cache.h
+++ b/bin/varnishd/cache.h
@@ -702,7 +702,7 @@ int EXP_NukeOne(struct worker *w, struct lru *lru);
 /* cache_fetch.c */
 struct storage *FetchStorage(const struct sess *sp, ssize_t sz);
 int FetchHdr(struct sess *sp);
-int FetchBody(struct sess *sp);
+int FetchBody(struct sess *sp, struct object *obj);
 int FetchReqBody(struct sess *sp);
 void Fetch_Init(void);
 
diff --git a/bin/varnishd/cache_center.c b/bin/varnishd/cache_center.c
index 243123d..13d8083 100644
--- a/bin/varnishd/cache_center.c
+++ b/bin/varnishd/cache_center.c
@@ -846,7 +846,7 @@ cnt_fetchbody(struct sess *sp)
 	}
 
 	/* Use unmodified headers*/
-	i = FetchBody(sp);
+	i = FetchBody(sp, sp->obj);
 
 	sp->wrk->h_content_length = NULL;
 
@@ -910,7 +910,7 @@ cnt_streambody(struct sess *sp)
 
 	AssertObjCorePassOrBusy(sp->obj->objcore);
 
-	i = FetchBody(sp);
+	i = FetchBody(sp, sp->obj);
 
 	sp->wrk->h_content_length = NULL;
 
diff --git a/bin/varnishd/cache_fetch.c b/bin/varnishd/cache_fetch.c
index ba91720..eae583d 100644
--- a/bin/varnishd/cache_fetch.c
+++ b/bin/varnishd/cache_fetch.c
@@ -477,7 +477,7 @@ FetchHdr(struct sess *sp)
 /*--------------------------------------------------------------------*/
 
 int
-FetchBody(struct sess *sp)
+FetchBody(struct sess *sp, struct object *obj)
 {
 	int cls;
 	struct storage *st;
@@ -487,17 +487,17 @@ FetchBody(struct sess *sp)
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 	CHECK_OBJ_NOTNULL(sp->wrk, WORKER_MAGIC);
 	w = sp->wrk;
-	CHECK_OBJ_NOTNULL(sp->obj, OBJECT_MAGIC);
-	CHECK_OBJ_NOTNULL(sp->obj->http, HTTP_MAGIC);
+	CHECK_OBJ_NOTNULL(obj, OBJECT_MAGIC);
+	CHECK_OBJ_NOTNULL(obj->http, HTTP_MAGIC);
 
 	if (w->vfp == NULL)
 		w->vfp = &vfp_nop;
 
 	AN(sp->director);
-	AssertObjCorePassOrBusy(sp->obj->objcore);
+	AssertObjCorePassOrBusy(obj->objcore);
 
 	AZ(w->vgz_rx);
-	AZ(VTAILQ_FIRST(&sp->obj->store));
+	AZ(VTAILQ_FIRST(&obj->store));
 	switch (w->body_status) {
 	case BS_NONE:
 		cls = 0;
@@ -541,57 +541,57 @@ FetchBody(struct sess *sp)
 	 */
 	AZ(vfp_nop_end(sp));
 
-	WSL(w, SLT_Fetch_Body, sp->wrk->vbc->vsl_id, "%u(%s) cls %d mklen %u",
+	WSL(w, SLT_Fetch_Body, w->vbc->vsl_id, "%u(%s) cls %d mklen %u",
 	    w->body_status, body_status(w->body_status),
 	    cls, mklen);
 
 	if (w->body_status == BS_ERROR) {
-		VDI_CloseFd(sp->wrk);
+		VDI_CloseFd(w);
 		return (__LINE__);
 	}
 
 	if (cls < 0) {
 		w->stats.fetch_failed++;
 		/* XXX: Wouldn't this store automatically be released ? */
-		while (!VTAILQ_EMPTY(&sp->obj->store)) {
-			st = VTAILQ_FIRST(&sp->obj->store);
-			VTAILQ_REMOVE(&sp->obj->store, st, list);
+		while (!VTAILQ_EMPTY(&obj->store)) {
+			st = VTAILQ_FIRST(&obj->store);
+			VTAILQ_REMOVE(&obj->store, st, list);
 			STV_free(st);
 		}
-		VDI_CloseFd(sp->wrk);
-		sp->obj->len = 0;
+		VDI_CloseFd(w);
+		obj->len = 0;
 		return (__LINE__);
 	}
 
 	if (cls == 0 && w->do_close)
 		cls = 1;
 
-	WSL(w, SLT_Length, sp->wrk->vbc->vsl_id, "%u", sp->obj->len);
+	WSL(w, SLT_Length, w->vbc->vsl_id, "%u", obj->len);
 
 	{
 	/* Sanity check fetch methods accounting */
 		ssize_t uu;
 
 		uu = 0;
-		VTAILQ_FOREACH(st, &sp->obj->store, list)
+		VTAILQ_FOREACH(st, &obj->store, list)
 			uu += st->len;
 		if (sp->objcore == NULL || (sp->objcore->flags & OC_F_PASS))
 			/* Streaming might have started freeing stuff */
-			assert (uu <= sp->obj->len);
+			assert (uu <= obj->len);
 		else
-			assert(uu == sp->obj->len);
+			assert(uu == obj->len);
 	}
 
 	if (mklen > 0) {
-		http_Unset(sp->obj->http, H_Content_Length);
-		http_PrintfHeader(w, sp->vsl_id, sp->obj->http,
-		    "Content-Length: %jd", (intmax_t)sp->obj->len);
+		http_Unset(obj->http, H_Content_Length);
+		http_PrintfHeader(w, sp->vsl_id, obj->http,
+		    "Content-Length: %jd", (intmax_t)obj->len);
 	}
 
 	if (cls)
-		VDI_CloseFd(sp->wrk);
+		VDI_CloseFd(w);
 	else
-		VDI_RecycleFd(sp->wrk);
+		VDI_RecycleFd(w);
 
 	return (0);
 }



From geoff at varnish-cache.org  Mon Jan  9 23:03:26 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Tue, 10 Jan 2012 00:03:26 +0100
Subject: [experimental-ims] 03e0bcf Write some generic code for a memory pool
	and apply it it VBC as the initial guineapig.
Message-ID: <E1RkOFS-0007m3-Qu@project.varnish-software.com>

commit 03e0bcf1beebc82633cbba0a861060a650288058
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Dec 12 11:54:59 2011 +0000

    Write some generic code for a memory pool and apply it it VBC as
    the initial guineapig.
    
    We have a number of pools of objects we maintain dynamically and we
    will get more in the future, so having a generic facility makes sense.
    
    One particular aspect of some of our pools, is that the desired size
    of object is variable over time, for instance workspaces which
    depend on parameters etc.
    
    Using generic code will allow us to use systematic parameters and
    VSC stats for all pools, hopefully with some memory savings involved.

diff --git a/bin/varnishd/Makefile.am b/bin/varnishd/Makefile.am
index a4bae7f..0c3427e 100644
--- a/bin/varnishd/Makefile.am
+++ b/bin/varnishd/Makefile.am
@@ -34,6 +34,7 @@ varnishd_SOURCES = \
 	cache/cache_httpconn.c \
 	cache/cache_lck.c \
 	cache/cache_main.c \
+	cache/cache_mempool.c \
 	cache/cache_panic.c \
 	cache/cache_pipe.c \
 	cache/cache_pool.c \
diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index ed6a3fd..edfe75f 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -99,10 +99,12 @@ struct cli;
 struct cli_proto;
 struct director;
 struct iovec;
+struct mempool;
 struct objcore;
 struct object;
 struct objhead;
 struct pool;
+struct poolparam;
 struct sess;
 struct sesspool;
 struct vbc;
@@ -667,6 +669,7 @@ void VDI_CloseFd(struct worker *wrk, struct vbc **vbp);
 void VDI_RecycleFd(struct worker *wrk, struct vbc **vbp);
 void VDI_AddHostHeader(struct worker *wrk, const struct vbc *vbc);
 void VBE_Poll(void);
+void VDI_Init(void);
 
 /* cache_backend_cfg.c */
 void VBE_InitCfg(void);
@@ -849,6 +852,15 @@ int Lck_CondWait(pthread_cond_t *cond, struct lock *lck, struct timespec *ts);
 #include "tbl/locks.h"
 #undef LOCK
 
+/* cache_mempool.c */
+struct mempool * MPL_New(const char *name, struct lock *mtx,
+    volatile struct poolparam *pp, volatile unsigned *cur_size);
+void *MPL_GetLocked(struct mempool *mpl, unsigned *size);
+void *MPL_Get(struct mempool *mpl, unsigned *size);
+void MPL_FreeLocked(struct mempool *mpl, void *item);
+void MPL_Free(struct mempool *mpl, void *item);
+
+
 /* cache_panic.c */
 void PAN_Init(void);
 
diff --git a/bin/varnishd/cache/cache_backend.c b/bin/varnishd/cache/cache_backend.c
index 4d2f18a..333b9bf 100644
--- a/bin/varnishd/cache/cache_backend.c
+++ b/bin/varnishd/cache/cache_backend.c
@@ -43,6 +43,16 @@
 #include "vrt.h"
 #include "vtcp.h"
 
+static struct mempool	*vbcpool;
+
+static struct poolparam vbcpp = {
+	.min_pool	= 10,
+	.max_pool	= 100,
+	.max_age	= 60,
+};
+
+static unsigned		vbcps = sizeof(struct vbc);
+
 /*--------------------------------------------------------------------
  * The "simple" director really isn't, since thats where all the actual
  * connections happen.  Nontheless, pretend it is simple by sequestering
@@ -82,14 +92,7 @@ VBE_ReleaseConn(struct vbc *vc)
 	CHECK_OBJ_NOTNULL(vc, VBC_MAGIC);
 	assert(vc->backend == NULL);
 	assert(vc->fd < 0);
-
-	vc->addr = NULL;
-	vc->addrlen = 0;
-	vc->recycled = 0;
-	Lck_Lock(&VBE_mtx);
-	VSC_C_main->n_vbc--;
-	Lck_Unlock(&VBE_mtx);
-	FREE_OBJ(vc);
+	MPL_Free(vbcpool, vc);
 }
 
 #define FIND_TMO(tmx, dst, sp, be)		\
@@ -221,12 +224,10 @@ vbe_NewConn(void)
 {
 	struct vbc *vc;
 
-	ALLOC_OBJ(vc, VBC_MAGIC);
+	vc = MPL_Get(vbcpool, NULL);
 	XXXAN(vc);
+	vc->magic = VBC_MAGIC;
 	vc->fd = -1;
-	Lck_Lock(&VBE_mtx);
-	VSC_C_main->n_vbc++;
-	Lck_Unlock(&VBE_mtx);
 	return (vc);
 }
 
@@ -517,3 +518,11 @@ VRT_init_dir_simple(struct cli *cli, struct director **bp, int idx,
 
 	bp[idx] = &vs->dir;
 }
+
+void
+VDI_Init(void)
+{
+
+	vbcpool = MPL_New("vbc", NULL, &vbcpp, &vbcps);
+	AN(vbcpool);
+}
diff --git a/bin/varnishd/cache/cache_backend.h b/bin/varnishd/cache/cache_backend.h
index 72a1283..a3ae5c2 100644
--- a/bin/varnishd/cache/cache_backend.h
+++ b/bin/varnishd/cache/cache_backend.h
@@ -170,7 +170,6 @@ void VBE_ReleaseConn(struct vbc *vc);
 struct backend *vdi_get_backend_if_simple(const struct director *d);
 
 /* cache_backend_cfg.c */
-extern struct lock VBE_mtx;
 void VBE_DropRefConn(struct backend *);
 void VBE_DropRefVcl(struct backend *);
 void VBE_DropRefLocked(struct backend *b);
diff --git a/bin/varnishd/cache/cache_backend_cfg.c b/bin/varnishd/cache/cache_backend_cfg.c
index a47aa3c..c2d022d 100644
--- a/bin/varnishd/cache/cache_backend_cfg.c
+++ b/bin/varnishd/cache/cache_backend_cfg.c
@@ -43,9 +43,6 @@
 #include "vcli_priv.h"
 #include "vrt.h"
 
-struct lock VBE_mtx;
-
-
 /*
  * The list of backends is not locked, it is only ever accessed from
  * the CLI thread, so there is no need.
@@ -502,6 +499,5 @@ void
 VBE_InitCfg(void)
 {
 
-	Lck_New(&VBE_mtx, lck_vbe);
 	CLI_AddFuncs(backend_cmds);
 }
diff --git a/bin/varnishd/cache/cache_main.c b/bin/varnishd/cache/cache_main.c
index a10b0fe..2189104 100644
--- a/bin/varnishd/cache/cache_main.c
+++ b/bin/varnishd/cache/cache_main.c
@@ -117,6 +117,7 @@ child_main(void)
 
 	HTTP_Init();
 
+	VDI_Init();
 	VBO_Init();
 	VBE_InitCfg();
 	VBP_Init();
diff --git a/bin/varnishd/cache/cache_mempool.c b/bin/varnishd/cache/cache_mempool.c
new file mode 100644
index 0000000..6677c84
--- /dev/null
+++ b/bin/varnishd/cache/cache_mempool.c
@@ -0,0 +1,164 @@
+/*-
+ * Copyright (c) 2011 Varnish Software AS
+ * All rights reserved.
+ *
+ * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * Generic memory pool
+ */
+
+#include <stddef.h>
+#include <stdlib.h>
+
+#include "config.h"
+
+#include "cache.h"
+
+struct memitem {
+	unsigned			magic;
+#define MEMITEM_MAGIC			0x42e55401
+	VTAILQ_ENTRY(memitem)		list;
+	unsigned			size;
+	double				payload;
+};
+
+struct mempool {
+	unsigned			magic;
+#define MEMPOOL_MAGIC			0x37a75a8d
+	VTAILQ_HEAD(,memitem)		list;
+	struct lock			*mtx;
+	struct lock			imtx;
+	const char			*name;
+	volatile struct poolparam	*param;
+	volatile unsigned		*cur_size;
+	struct VSC_C_mempool		*vsc;
+};
+
+struct mempool *
+MPL_New(const char *name,
+	struct lock *mtx,
+	volatile struct poolparam *pp,
+	volatile unsigned *cur_size)
+{
+	struct mempool *mpl;
+
+	ALLOC_OBJ(mpl, MEMPOOL_MAGIC);
+	AN(mpl);
+	mpl->name = name;
+	mpl->param = pp;
+	mpl->cur_size = cur_size;
+	mpl->mtx = mtx;
+	VTAILQ_INIT(&mpl->list);
+	Lck_New(&mpl->imtx, lck_mempool);
+	if (mpl->mtx == NULL)
+		mpl->mtx = &mpl->imtx;
+	/* XXX: prealloc min_pool */
+	mpl->vsc = VSM_Alloc(sizeof *mpl->vsc,
+	    VSC_CLASS, VSC_TYPE_MEMPOOL, name);
+	AN(mpl->vsc);
+	return (mpl);
+}
+
+void *
+MPL_GetLocked(struct mempool *mpl, unsigned *size)
+{
+	struct memitem *mi;
+	unsigned tsz;
+
+	CHECK_OBJ_NOTNULL(mpl, MEMPOOL_MAGIC);
+	Lck_AssertHeld(mpl->mtx);
+
+	mpl->vsc->allocs++;
+	mpl->vsc->live++;
+	do {
+		mi = VTAILQ_FIRST(&mpl->list);
+		if (mi == NULL)
+			break;
+		mpl->vsc->pool--;
+		CHECK_OBJ_NOTNULL(mi, MEMITEM_MAGIC);
+		VTAILQ_REMOVE(&mpl->list, mi, list);
+		if (mi->size < *mpl->cur_size) {
+			mpl->vsc->toosmall++;
+			FREE_OBJ(mi);
+			mi = NULL;
+		} else {
+			mpl->vsc->recycle++;
+		}
+	} while (mi == NULL);
+	if (mi == NULL) {
+		tsz = *mpl->cur_size;
+		mi = calloc(sizeof *mi + tsz, 1);
+		AN(mi);
+		mi->magic = MEMITEM_MAGIC;
+		mi->size = tsz;
+	}
+	if (size != NULL)
+		*size = mi->size;
+	return (&mi->payload);
+}
+
+void
+MPL_FreeLocked(struct mempool *mpl, void *item)
+{
+	struct memitem *mi;
+
+	CHECK_OBJ_NOTNULL(mpl, MEMPOOL_MAGIC);
+	Lck_AssertHeld(mpl->mtx);
+
+	mpl->vsc->frees++;
+	mpl->vsc->live--;
+
+	mi = (void*)((uintptr_t)item - offsetof(struct memitem, payload));
+	CHECK_OBJ_NOTNULL(mi, MEMITEM_MAGIC);
+
+	if (mi->size < *mpl->cur_size) {
+		mpl->vsc->toosmall++;
+		FREE_OBJ(mi);
+	} else {
+		mpl->vsc->pool++;
+		memset(item, 0, mi->size);
+		VTAILQ_INSERT_HEAD(&mpl->list, mi, list);
+	}
+}
+
+void *
+MPL_Get(struct mempool *mpl, unsigned *size)
+{
+	void *p;
+
+	CHECK_OBJ_NOTNULL(mpl, MEMPOOL_MAGIC);
+	Lck_Lock(mpl->mtx);
+	p = MPL_GetLocked(mpl, size);
+	Lck_Unlock(mpl->mtx);
+	return (p);
+}
+
+void
+MPL_Free(struct mempool *mpl, void *item)
+{
+	CHECK_OBJ_NOTNULL(mpl, MEMPOOL_MAGIC);
+	Lck_Lock(mpl->mtx);
+	MPL_FreeLocked(mpl, item);
+	Lck_Unlock(mpl->mtx);
+}
diff --git a/bin/varnishd/common/params.h b/bin/varnishd/common/params.h
index 8fd777f..3f1821f 100644
--- a/bin/varnishd/common/params.h
+++ b/bin/varnishd/common/params.h
@@ -33,6 +33,12 @@
 
 #define VSM_CLASS_PARAM		"Params"
 
+struct poolparam {
+	unsigned		min_pool;
+	unsigned		max_pool;
+	double			max_age;
+};
+
 struct params {
 
 	/* Unprivileged user / group */
diff --git a/include/tbl/locks.h b/include/tbl/locks.h
index d86da01..697752d 100644
--- a/include/tbl/locks.h
+++ b/include/tbl/locks.h
@@ -46,9 +46,9 @@ LOCK(lru)
 LOCK(cli)
 LOCK(ban)
 LOCK(vbp)
-LOCK(vbe)
 LOCK(backend)
 LOCK(vcapace)
 LOCK(nbusyobj)
 LOCK(busyobj)
+LOCK(mempool)
 /*lint -restore */
diff --git a/include/tbl/vsc_all.h b/include/tbl/vsc_all.h
index 4d7b927..ae42507 100644
--- a/include/tbl/vsc_all.h
+++ b/include/tbl/vsc_all.h
@@ -56,3 +56,9 @@ VSC_DO(VBE, vbe, VSC_TYPE_VBE)
 #include "tbl/vsc_fields.h"
 #undef VSC_DO_VBE
 VSC_DONE(VBE, vbe, VSC_TYPE_VBE)
+
+VSC_DO(MEMPOOL, mempool, VSC_TYPE_MEMPOOL)
+#define VSC_DO_MEMPOOL
+#include "tbl/vsc_fields.h"
+#undef VSC_DO_MEMPOOL
+VSC_DONE(MEMPOOL, mempool, VSC_TYPE_MEMPOOL)
diff --git a/include/tbl/vsc_f_main.h b/include/tbl/vsc_f_main.h
index ab8415c..c8c4ecb 100644
--- a/include/tbl/vsc_f_main.h
+++ b/include/tbl/vsc_f_main.h
@@ -259,8 +259,6 @@ VSC_F(n_objectcore,		uint64_t, 1, 'i', "N struct objectcore", "")
 VSC_F(n_objecthead,		uint64_t, 1, 'i', "N struct objecthead", "")
 VSC_F(n_waitinglist,		uint64_t, 1, 'i', "N struct waitinglist", "")
 
-VSC_F(n_vbc,		uint64_t, 0, 'i', "N struct vbc", "")
-
 VSC_F(n_backend,		uint64_t, 0, 'i', "N backends", "")
 
 VSC_F(n_expired,		uint64_t, 0, 'i', "N expired objects", "")
diff --git a/include/tbl/vsc_fields.h b/include/tbl/vsc_fields.h
index 60cde5d..cc72903 100644
--- a/include/tbl/vsc_fields.h
+++ b/include/tbl/vsc_fields.h
@@ -106,3 +106,15 @@ VSC_F(happy,		uint64_t, 0, 'b', "Happy health probes", "")
 
 #endif
 
+/**********************************************************************/
+#ifdef VSC_DO_MEMPOOL
+
+VSC_F(allocs,			uint64_t, 0, 'c', "Allocations", "")
+VSC_F(frees,			uint64_t, 0, 'c', "Frees", "")
+VSC_F(live,			uint64_t, 0, 'g', "In use", "")
+VSC_F(pool,			uint64_t, 0, 'g', "In Pool", "")
+VSC_F(recycle,			uint64_t, 0, 'g', "Recycled from pool", "")
+VSC_F(timeout,			uint64_t, 0, 'g', "Timed out from pool", "")
+VSC_F(toosmall,			uint64_t, 0, 'g', "Too small to recycle", "")
+
+#endif
diff --git a/include/vapi/vsc_int.h b/include/vapi/vsc_int.h
index 6ceb723..95346e3 100644
--- a/include/vapi/vsc_int.h
+++ b/include/vapi/vsc_int.h
@@ -28,13 +28,14 @@
  *
  */
 
-#define VSC_CLASS          "Stat"
+#define VSC_CLASS		"Stat"
 
 #define VSC_TYPE_MAIN		""
-#define VSC_TYPE_SMA	"SMA"
-#define VSC_TYPE_SMF	"SMF"
-#define VSC_TYPE_VBE	"VBE"
-#define VSC_TYPE_LCK	"LCK"
+#define VSC_TYPE_SMA		"SMA"
+#define VSC_TYPE_SMF		"SMF"
+#define VSC_TYPE_VBE		"VBE"
+#define VSC_TYPE_LCK		"LCK"
+#define VSC_TYPE_MEMPOOL	"MEMPOOL"
 
 #define VSC_F(n, t, l, f, e, d)	t n;
 



From geoff at varnish-cache.org  Mon Jan  9 23:03:26 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Tue, 10 Jan 2012 00:03:26 +0100
Subject: [experimental-ims] 7f3b2a7 Correct ACL syntax
Message-ID: <E1RkOFS-0007m9-TJ@project.varnish-software.com>

commit 7f3b2a79dcc033d249811ff246fdb22dd8ece61d
Author: Andreas Plesner Jacobsen <apj at mutt.dk>
Date:   Tue Dec 13 15:35:03 2011 +0100

    Correct ACL syntax

diff --git a/doc/sphinx/tutorial/purging.rst b/doc/sphinx/tutorial/purging.rst
index cea1ccd..ecf0e96 100644
--- a/doc/sphinx/tutorial/purging.rst
+++ b/doc/sphinx/tutorial/purging.rst
@@ -29,7 +29,7 @@ following VCL in place::
 
   acl purge {
 	  "localhost";
-	  "192.168.55.0/24";
+	  "192.168.55.0"/24;
   }
   
   sub vcl_recv {



From geoff at varnish-cache.org  Mon Jan  9 23:03:26 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Tue, 10 Jan 2012 00:03:26 +0100
Subject: [experimental-ims] cf024ae Introduce a generic handler for memory
	pool parameters.
Message-ID: <E1RkOFS-0007mF-Uf@project.varnish-software.com>

commit cf024ae0766179d693b6d5b0d74754134df54647
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Dec 14 10:51:23 2011 +0000

    Introduce a generic handler for memory pool parameters.
    
    Minor improvements to a couple param-tweakers as consequence.

diff --git a/bin/varnishd/cache/cache_backend.c b/bin/varnishd/cache/cache_backend.c
index 333b9bf..c1bc055 100644
--- a/bin/varnishd/cache/cache_backend.c
+++ b/bin/varnishd/cache/cache_backend.c
@@ -45,12 +45,6 @@
 
 static struct mempool	*vbcpool;
 
-static struct poolparam vbcpp = {
-	.min_pool	= 10,
-	.max_pool	= 100,
-	.max_age	= 60,
-};
-
 static unsigned		vbcps = sizeof(struct vbc);
 
 /*--------------------------------------------------------------------
@@ -523,6 +517,6 @@ void
 VDI_Init(void)
 {
 
-	vbcpool = MPL_New("vbc", NULL, &vbcpp, &vbcps);
+	vbcpool = MPL_New("vbc", NULL, &cache_param->vbc_pool, &vbcps);
 	AN(vbcpool);
 }
diff --git a/bin/varnishd/common/params.h b/bin/varnishd/common/params.h
index 3f1821f..8198a78 100644
--- a/bin/varnishd/common/params.h
+++ b/bin/varnishd/common/params.h
@@ -200,4 +200,6 @@ struct params {
 	/* VSM dimensions */
 	ssize_t			vsm_space;
 	ssize_t			vsl_space;
+
+	struct poolparam	vbc_pool;
 };
diff --git a/bin/varnishd/mgt/mgt_param.c b/bin/varnishd/mgt/mgt_param.c
index cc14e41..fedc0d1 100644
--- a/bin/varnishd/mgt/mgt_param.c
+++ b/bin/varnishd/mgt/mgt_param.c
@@ -102,33 +102,49 @@ tweak_timeout(struct cli *cli, const struct parspec *par, const char *arg)
 	tweak_generic_timeout(cli, dest, arg);
 }
 
-static void
-tweak_timeout_double(struct cli *cli, const struct parspec *par,
-    const char *arg)
+/*--------------------------------------------------------------------*/
+
+static int
+tweak_generic_timeout_double(struct cli *cli, volatile double *dest,
+    const char *arg, double min, double max)
 {
-	volatile double *dest;
 	double u;
+	char *p;
 
-	dest = par->priv;
 	if (arg != NULL) {
-		u = strtod(arg, NULL);
-		if (u < par->min) {
+		p = NULL;
+		u = strtod(arg, &p);
+		if (*arg == '\0' || *p != '\0') {
+			VCLI_Out(cli, "Not a number(%s)\n", arg);
+			VCLI_SetResult(cli, CLIS_PARAM);
+			return (-1);
+		}
+		if (u < min) {
 			VCLI_Out(cli,
-			    "Timeout must be greater or equal to %.g\n",
-				 par->min);
+			    "Timeout must be greater or equal to %.g\n", min);
 			VCLI_SetResult(cli, CLIS_PARAM);
-			return;
+			return (-1);
 		}
-		if (u > par->max) {
+		if (u > max) {
 			VCLI_Out(cli,
-			    "Timeout must be less than or equal to %.g\n",
-				 par->max);
+			    "Timeout must be less than or equal to %.g\n", max);
 			VCLI_SetResult(cli, CLIS_PARAM);
-			return;
+			return (-1);
 		}
 		*dest = u;
 	} else
 		VCLI_Out(cli, "%.6f", *dest);
+	return (0);
+}
+
+static void
+tweak_timeout_double(struct cli *cli, const struct parspec *par,
+    const char *arg)
+{
+	volatile double *dest;
+
+	dest = par->priv;
+	(void)tweak_generic_timeout_double(cli, dest, arg, par->min, par->max);
 }
 
 /*--------------------------------------------------------------------*/
@@ -138,11 +154,19 @@ tweak_generic_double(struct cli *cli, const struct parspec *par,
     const char *arg)
 {
 	volatile double *dest;
+	char *p;
 	double u;
 
 	dest = par->priv;
 	if (arg != NULL) {
-		u = strtod(arg, NULL);
+		p = NULL;
+		u = strtod(arg, &p);
+		if (arg == '\0' || *p != '\0') {
+			VCLI_Out(cli,
+			    "Not a number (%s)\n", arg);
+			VCLI_SetResult(cli, CLIS_PARAM);
+			return;
+		}
 		if (u < par->min) {
 			VCLI_Out(cli,
 			    "Must be greater or equal to %.g\n",
@@ -206,26 +230,33 @@ tweak_bool(struct cli *cli, const struct parspec *par, const char *arg)
 
 /*--------------------------------------------------------------------*/
 
-void
+int
 tweak_generic_uint(struct cli *cli, volatile unsigned *dest, const char *arg,
     unsigned min, unsigned max)
 {
 	unsigned u;
+	char *p;
 
 	if (arg != NULL) {
+		p = NULL;
 		if (!strcasecmp(arg, "unlimited"))
 			u = UINT_MAX;
 		else
-			u = strtoul(arg, NULL, 0);
+			u = strtoul(arg, &p, 0);
+		if (*arg == '\0' || *p != '\0') {
+			VCLI_Out(cli, "Not a number (%s)\n", arg);
+			VCLI_SetResult(cli, CLIS_PARAM);
+			return (-1);
+		}
 		if (u < min) {
 			VCLI_Out(cli, "Must be at least %u\n", min);
 			VCLI_SetResult(cli, CLIS_PARAM);
-			return;
+			return (-1);
 		}
 		if (u > max) {
 			VCLI_Out(cli, "Must be no more than %u\n", max);
 			VCLI_SetResult(cli, CLIS_PARAM);
-			return;
+			return (-1);
 		}
 		*dest = u;
 	} else if (*dest == UINT_MAX) {
@@ -233,6 +264,7 @@ tweak_generic_uint(struct cli *cli, volatile unsigned *dest, const char *arg,
 	} else {
 		VCLI_Out(cli, "%u", *dest);
 	}
+	return (0);
 }
 
 /*--------------------------------------------------------------------*/
@@ -243,7 +275,8 @@ tweak_uint(struct cli *cli, const struct parspec *par, const char *arg)
 	volatile unsigned *dest;
 
 	dest = par->priv;
-	tweak_generic_uint(cli, dest, arg, (uint)par->min, (uint)par->max);
+	(void)tweak_generic_uint(cli, dest, arg,
+	    (uint)par->min, (uint)par->max);
 }
 
 /*--------------------------------------------------------------------*/
@@ -554,6 +587,56 @@ tweak_diag_bitmap(struct cli *cli, const struct parspec *par, const char *arg)
 
 /*--------------------------------------------------------------------*/
 
+static void
+tweak_poolparam(struct cli *cli, const struct parspec *par, const char *arg)
+{
+	volatile struct poolparam *pp, px;
+	char **av;
+
+	pp = par->priv;
+	if (arg == NULL) {
+		VCLI_Out(cli, "%u,%u,%g",
+		    pp->min_pool, pp->max_pool, pp->max_age);
+	} else {
+		av = VAV_Parse(arg, NULL, ARGV_COMMA);
+		do {
+			if (av[0] != NULL) {
+				VCLI_Out(cli, "Parse error: %s", av[0]);
+				VCLI_SetResult(cli, CLIS_PARAM);
+				break;
+			}
+			if (av[1] == NULL || av[2] == NULL || av[3] == NULL) {
+				VCLI_Out(cli,
+				    "Three fields required:"
+				    " min_pool, max_pool and max_age\n");
+				VCLI_SetResult(cli, CLIS_PARAM);
+				break;
+			}
+			px = *pp;
+			if (tweak_generic_uint(cli, &px.min_pool, av[1],
+			    (uint)par->min, (uint)par->max))
+				break;
+			if (tweak_generic_uint(cli, &px.max_pool, av[2],
+			    (uint)par->min, (uint)par->max))
+				break;
+			if (tweak_generic_timeout_double(cli, &px.max_age,
+			    av[3], 0, 1e6))
+				break;
+			if (px.min_pool > px.max_pool) {
+				VCLI_Out(cli,
+				    "min_pool cannot be larger"
+				    " than max_pool\n");
+				VCLI_SetResult(cli, CLIS_PARAM);
+				break;
+			}
+			*pp = px;
+		} while(0);
+		
+	}
+}
+
+/*--------------------------------------------------------------------*/
+
 /*
  * Make sure to end all lines with either a space or newline of the
  * formatting will go haywire.
@@ -1113,6 +1196,15 @@ static const struct parspec input_parspec[] = {
 		0,
 		"true", ""},
 
+	{ "vbc_pool", tweak_poolparam, &mgt_param.vbc_pool, 0, 10000,
+		"Parameters for backend connection memory pool.\n"
+		"The three numbers are:\n"
+		"   min_pool -- minimum size of free pool.\n"
+		"   max_pool -- maximum size of free pool.\n"
+		"   max_age -- max age of free element.\n",
+		0,
+		"10,100,10", ""},
+
 	{ NULL, NULL, NULL }
 };
 
diff --git a/bin/varnishd/mgt/mgt_param.h b/bin/varnishd/mgt/mgt_param.h
index 2d4a97f..275dfe6 100644
--- a/bin/varnishd/mgt/mgt_param.h
+++ b/bin/varnishd/mgt/mgt_param.h
@@ -49,7 +49,7 @@ struct parspec {
 	const char	*units;
 };
 
-void tweak_generic_uint(struct cli *cli,
+int tweak_generic_uint(struct cli *cli,
     volatile unsigned *dest, const char *arg, unsigned min, unsigned max);
 void tweak_uint(struct cli *cli, const struct parspec *par, const char *arg);
 void tweak_timeout(struct cli *cli,
diff --git a/bin/varnishd/mgt/mgt_pool.c b/bin/varnishd/mgt/mgt_pool.c
index badb3f4..e6320fd 100644
--- a/bin/varnishd/mgt/mgt_pool.c
+++ b/bin/varnishd/mgt/mgt_pool.c
@@ -59,7 +59,7 @@ tweak_thread_pool_min(struct cli *cli, const struct parspec *par,
     const char *arg)
 {
 
-	tweak_generic_uint(cli, &mgt_param.wthread_min, arg,
+	(void)tweak_generic_uint(cli, &mgt_param.wthread_min, arg,
 	    (unsigned)par->min, mgt_param.wthread_max);
 }
 
@@ -86,7 +86,7 @@ tweak_stack_size(struct cli *cli, const struct parspec *par,
 		arg = buf;
 	}
 
-	tweak_generic_uint(cli, &mgt_param.wthread_stacksize, arg,
+	(void)tweak_generic_uint(cli, &mgt_param.wthread_stacksize, arg,
 	    low, (uint)par->max);
 }
 
@@ -98,7 +98,7 @@ tweak_thread_pool_max(struct cli *cli, const struct parspec *par,
 {
 
 	(void)par;
-	tweak_generic_uint(cli, &mgt_param.wthread_max, arg,
+	(void)tweak_generic_uint(cli, &mgt_param.wthread_max, arg,
 	    mgt_param.wthread_min, UINT_MAX);
 }
 
diff --git a/bin/varnishtest/tests/b00034.vtc b/bin/varnishtest/tests/b00034.vtc
new file mode 100644
index 0000000..3579fcb
--- /dev/null
+++ b/bin/varnishtest/tests/b00034.vtc
@@ -0,0 +1,13 @@
+varnishtest "mempool param handling"
+
+server s1 {
+} -start
+
+varnish v1 -vcl+backend {}
+
+varnish v1 -cliok "param.set vbc_pool 1,10,1"
+varnish v1 -clierr 106 "param.set vbc_pool 10"
+varnish v1 -clierr 106 "param.set vbc_pool 10,1,1"
+varnish v1 -clierr 106 "param.set vbc_pool a,10,10"
+varnish v1 -clierr 106 "param.set vbc_pool 10,a,10"
+varnish v1 -clierr 106 "param.set vbc_pool 10,10,a"



From geoff at varnish-cache.org  Mon Jan  9 23:03:26 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Tue, 10 Jan 2012 00:03:26 +0100
Subject: [experimental-ims] a0ac5ac Merge branch 'master' of
	ssh://git.varnish-cache.org/git/varnish-cache
Message-ID: <E1RkOFS-0007mO-Vy@project.varnish-software.com>

commit a0ac5ac55dafb1d70501ab0a05164e62c3eb716c
Merge: cf024ae 7f3b2a7
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Dec 14 10:52:14 2011 +0000

    Merge branch 'master' of ssh://git.varnish-cache.org/git/varnish-cache



From geoff at varnish-cache.org  Mon Jan  9 23:03:27 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Tue, 10 Jan 2012 00:03:27 +0100
Subject: [experimental-ims] 65d9628 Count Trylocks that succeed
Message-ID: <E1RkOFT-0007mn-3x@project.varnish-software.com>

commit 65d9628481ea72ae2fc939f34391bfc01f92c179
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu Dec 15 08:35:02 2011 +0000

    Count Trylocks that succeed

diff --git a/bin/varnishd/cache/cache_lck.c b/bin/varnishd/cache/cache_lck.c
index 78b5e42..cff9845 100644
--- a/bin/varnishd/cache/cache_lck.c
+++ b/bin/varnishd/cache/cache_lck.c
@@ -118,6 +118,7 @@ Lck__Trylock(struct lock *lck, const char *p, const char *f, int l)
 	if (r == 0) {
 		AZ(ilck->held);
 		ilck->held = 1;
+		ilck->stat->locks++;
 		ilck->owner = pthread_self();
 	}
 	return (r);



From geoff at varnish-cache.org  Mon Jan  9 23:03:27 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Tue, 10 Jan 2012 00:03:27 +0100
Subject: [experimental-ims] 89ad25e Some clarification on how 'now' works I'm
	not sure if we should go into more detail about how the whole
	thing works
Message-ID: <E1RkOFT-0007mX-1N@project.varnish-software.com>

commit 89ad25ed13e8bbc3aca06a10acb4b1822ecf8e2a
Author: Per Buer <perbu at varnish-software.com>
Date:   Wed Dec 14 16:04:39 2011 +0100

    Some clarification on how 'now' works
    I'm not sure if we should go into more detail about how the whole thing works

diff --git a/doc/sphinx/reference/vcl.rst b/doc/sphinx/reference/vcl.rst
index 5b2e6d0..2c185ed 100644
--- a/doc/sphinx/reference/vcl.rst
+++ b/doc/sphinx/reference/vcl.rst
@@ -679,7 +679,8 @@ made available to the handler subroutines through global variables.
 The following variables are always available:
 
 now   
-  The current time, in seconds since the epoch.
+  The current time, in seconds since the epoch. When used in string context
+  it returns a formatted string.
 
 The following variables are available in backend declarations:
 



From geoff at varnish-cache.org  Mon Jan  9 23:03:27 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Tue, 10 Jan 2012 00:03:27 +0100
Subject: [experimental-ims] 9f5becc FlexeLinting
Message-ID: <E1RkOFT-0007my-6J@project.varnish-software.com>

commit 9f5becc0c3d1a409d9e3ac753762cd6ce7149e25
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu Dec 15 09:00:49 2011 +0000

    FlexeLinting

diff --git a/bin/varnishd/mgt/mgt_param.c b/bin/varnishd/mgt/mgt_param.c
index fedc0d1..6faf0c5 100644
--- a/bin/varnishd/mgt/mgt_param.c
+++ b/bin/varnishd/mgt/mgt_param.c
@@ -161,7 +161,7 @@ tweak_generic_double(struct cli *cli, const struct parspec *par,
 	if (arg != NULL) {
 		p = NULL;
 		u = strtod(arg, &p);
-		if (arg == '\0' || *p != '\0') {
+		if (*p != '\0') {
 			VCLI_Out(cli,
 			    "Not a number (%s)\n", arg);
 			VCLI_SetResult(cli, CLIS_PARAM);
@@ -241,12 +241,13 @@ tweak_generic_uint(struct cli *cli, volatile unsigned *dest, const char *arg,
 		p = NULL;
 		if (!strcasecmp(arg, "unlimited"))
 			u = UINT_MAX;
-		else
+		else {
 			u = strtoul(arg, &p, 0);
-		if (*arg == '\0' || *p != '\0') {
-			VCLI_Out(cli, "Not a number (%s)\n", arg);
-			VCLI_SetResult(cli, CLIS_PARAM);
-			return (-1);
+			if (*arg == '\0' || *p != '\0') {
+				VCLI_Out(cli, "Not a number (%s)\n", arg);
+				VCLI_SetResult(cli, CLIS_PARAM);
+				return (-1);
+			}
 		}
 		if (u < min) {
 			VCLI_Out(cli, "Must be at least %u\n", min);



From geoff at varnish-cache.org  Mon Jan  9 23:03:27 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Tue, 10 Jan 2012 00:03:27 +0100
Subject: [experimental-ims] b83c68d Formailize some of the macro-crutches I
	use to explain stuff to FlexeLint.
Message-ID: <E1RkOFT-0007nB-9E@project.varnish-software.com>

commit b83c68dd51ed522aac12129ecf6e45adbc28ea4f
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu Dec 15 09:21:45 2011 +0000

    Formailize some of the macro-crutches I use to explain stuff to FlexeLint.

diff --git a/bin/varnishd/common/common.h b/bin/varnishd/common/common.h
index 2ff5623..3509f25 100644
--- a/bin/varnishd/common/common.h
+++ b/bin/varnishd/common/common.h
@@ -42,6 +42,26 @@
 
 struct cli;
 
+/**********************************************************************
+ * FlexeLint and compiler shutuppery
+ */
+
+/*
+ * In OO-light situations, functions have to match their prototype
+ * even if that means not const'ing a const'able argument.
+ * The typedef should be specified as argument to the macro.
+ */
+#define __match_proto__(xxx)		/*lint -e{818} */
+
+/*
+ * State variables may change value before we use the last value we
+ * set them to.
+ * Pass no argument.
+ */
+#define __state_variable__(xxx)		/*lint -esym(838,xxx) */
+
+/**********************************************************************/
+
 /* Name of transient storage */
 #define TRANSIENT_STORAGE	"Transient"
 
@@ -52,8 +72,6 @@ extern pid_t mgt_pid;
 extern struct vsb *vident;		// XXX: -> heritage ?
 int Symbol_Lookup(struct vsb *vsb, void *ptr);
 
-/* Help shut up FlexeLint */
-#define __match_proto__(xxx) /*lint -e{818} */
 
 /* Really belongs in mgt.h, but storage_file chokes on both */
 void mgt_child_inherit(int fd, const char *what);



From geoff at varnish-cache.org  Mon Jan  9 23:03:27 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Tue, 10 Jan 2012 00:03:27 +0100
Subject: [experimental-ims] e9b524b Add the mempool guard-thread
Message-ID: <E1RkOFT-0007nP-B0@project.varnish-software.com>

commit e9b524b71aca73e048dd38ea8238f323e8be9558
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu Dec 15 09:32:09 2011 +0000

    Add the mempool guard-thread

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index edfe75f..9601270 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -855,9 +855,7 @@ int Lck_CondWait(pthread_cond_t *cond, struct lock *lck, struct timespec *ts);
 /* cache_mempool.c */
 struct mempool * MPL_New(const char *name, struct lock *mtx,
     volatile struct poolparam *pp, volatile unsigned *cur_size);
-void *MPL_GetLocked(struct mempool *mpl, unsigned *size);
 void *MPL_Get(struct mempool *mpl, unsigned *size);
-void MPL_FreeLocked(struct mempool *mpl, void *item);
 void MPL_Free(struct mempool *mpl, void *item);
 
 
diff --git a/bin/varnishd/cache/cache_mempool.c b/bin/varnishd/cache/cache_mempool.c
index 6677c84..6f46a2a 100644
--- a/bin/varnishd/cache/cache_mempool.c
+++ b/bin/varnishd/cache/cache_mempool.c
@@ -35,6 +35,8 @@
 
 #include "cache.h"
 
+#include "vtim.h"
+
 struct memitem {
 	unsigned			magic;
 #define MEMITEM_MAGIC			0x42e55401
@@ -47,14 +49,114 @@ struct mempool {
 	unsigned			magic;
 #define MEMPOOL_MAGIC			0x37a75a8d
 	VTAILQ_HEAD(,memitem)		list;
+	VTAILQ_HEAD(,memitem)		surplus;
 	struct lock			*mtx;
 	struct lock			imtx;
 	const char			*name;
 	volatile struct poolparam	*param;
 	volatile unsigned		*cur_size;
 	struct VSC_C_mempool		*vsc;
+	unsigned			n_pool;
+	pthread_t			thread;
 };
 
+/*---------------------------------------------------------------------
+ */
+
+static struct memitem *
+mpl_alloc(const struct mempool *mpl)
+{
+	unsigned tsz;
+	struct memitem *mi;
+
+	CHECK_OBJ_NOTNULL(mpl, MEMPOOL_MAGIC);
+	tsz = *mpl->cur_size;
+	mi = calloc(sizeof *mi + tsz, 1);
+	AN(mi);
+	mi->magic = MEMITEM_MAGIC;
+	mi->size = tsz;
+	return (mi);
+}
+
+/*---------------------------------------------------------------------
+ * Pool-guard
+ *   Attempt to keep number of free items in pool inside bounds with
+ *   minimum locking activity.
+ */
+
+static void *
+mpl_guard(void *priv)
+{
+	struct mempool *mpl;
+	struct memitem *mi = NULL;
+	double mpl_slp __state_variable__(mpl_slp);
+
+	CAST_OBJ_NOTNULL(mpl, priv, MEMPOOL_MAGIC);
+	mpl_slp = 0.15;	// random
+	while (1) {
+		VTIM_sleep(mpl_slp);
+		mpl_slp = 0.814;	// random
+
+		if (mi != NULL && (mpl->n_pool > mpl->param->max_pool ||
+		    mi->size < *mpl->cur_size)) {
+			FREE_OBJ(mi);
+			mi = NULL;
+		}
+
+		if (mi == NULL && mpl->n_pool < mpl->param->min_pool)
+			mi = mpl_alloc(mpl);
+
+		if (mpl->n_pool < mpl->param->min_pool && mi != NULL) {
+			/* can do */
+		} else if (mpl->n_pool > mpl->param->max_pool && mi == NULL) {
+			/* can do */
+		} else if (!VTAILQ_EMPTY(&mpl->surplus)) {
+			/* can do */
+		} else {
+			continue;	/* cannot do */
+		}
+
+		mpl_slp = 0.314;
+
+		if (Lck_Trylock(mpl->mtx))
+			continue;
+
+		mpl_slp = .01;
+
+		if (mpl->n_pool < mpl->param->min_pool &&
+		    mi != NULL && mi->size >= *mpl->cur_size) {
+			CHECK_OBJ_NOTNULL(mi, MEMITEM_MAGIC);
+			mpl->vsc->pool++;
+			mpl->n_pool++;
+			VTAILQ_INSERT_HEAD(&mpl->list, mi, list);
+			mi = NULL;
+		}
+		if (mpl->n_pool > mpl->param->max_pool && mi == NULL) {
+			mi = VTAILQ_FIRST(&mpl->list);
+			CHECK_OBJ_NOTNULL(mi, MEMITEM_MAGIC);
+			mpl->vsc->pool--;
+			mpl->n_pool--;
+			VTAILQ_REMOVE(&mpl->list, mi, list);
+		}
+		if (mi == NULL) {
+			mi = VTAILQ_FIRST(&mpl->surplus);
+			if (mi != NULL) {
+				CHECK_OBJ_NOTNULL(mi, MEMITEM_MAGIC);
+				VTAILQ_REMOVE(&mpl->surplus, mi, list);
+			}
+		}
+		Lck_Unlock(mpl->mtx);
+
+		if (mi != NULL) {
+			FREE_OBJ(mi);
+			mi = NULL;
+		}
+	}
+}
+
+/*---------------------------------------------------------------------
+ */
+
 struct mempool *
 MPL_New(const char *name,
 	struct lock *mtx,
@@ -70,6 +172,7 @@ MPL_New(const char *name,
 	mpl->cur_size = cur_size;
 	mpl->mtx = mtx;
 	VTAILQ_INIT(&mpl->list);
+	VTAILQ_INIT(&mpl->surplus);
 	Lck_New(&mpl->imtx, lck_mempool);
 	if (mpl->mtx == NULL)
 		mpl->mtx = &mpl->imtx;
@@ -77,88 +180,77 @@ MPL_New(const char *name,
 	mpl->vsc = VSM_Alloc(sizeof *mpl->vsc,
 	    VSC_CLASS, VSC_TYPE_MEMPOOL, name);
 	AN(mpl->vsc);
+	AZ(pthread_create(&mpl->thread, NULL, mpl_guard, mpl));
 	return (mpl);
 }
 
 void *
-MPL_GetLocked(struct mempool *mpl, unsigned *size)
+MPL_Get(struct mempool *mpl, unsigned *size)
 {
 	struct memitem *mi;
-	unsigned tsz;
 
 	CHECK_OBJ_NOTNULL(mpl, MEMPOOL_MAGIC);
-	Lck_AssertHeld(mpl->mtx);
+
+	Lck_Lock(mpl->mtx);
 
 	mpl->vsc->allocs++;
 	mpl->vsc->live++;
+
 	do {
 		mi = VTAILQ_FIRST(&mpl->list);
 		if (mi == NULL)
 			break;
 		mpl->vsc->pool--;
+		mpl->n_pool--;
 		CHECK_OBJ_NOTNULL(mi, MEMITEM_MAGIC);
 		VTAILQ_REMOVE(&mpl->list, mi, list);
 		if (mi->size < *mpl->cur_size) {
 			mpl->vsc->toosmall++;
-			FREE_OBJ(mi);
+			VTAILQ_INSERT_HEAD(&mpl->surplus, mi, list);
 			mi = NULL;
 		} else {
 			mpl->vsc->recycle++;
 		}
 	} while (mi == NULL);
-	if (mi == NULL) {
-		tsz = *mpl->cur_size;
-		mi = calloc(sizeof *mi + tsz, 1);
-		AN(mi);
-		mi->magic = MEMITEM_MAGIC;
-		mi->size = tsz;
-	}
+
+	Lck_Unlock(mpl->mtx);
+
+	if (mi == NULL)
+		mi = mpl_alloc(mpl);
 	if (size != NULL)
 		*size = mi->size;
-	return (&mi->payload);
+
+	/* Throw away sizeof info for FlexeLint: */
+	return ((void*)(uintptr_t)&mi->payload);
 }
 
 void
-MPL_FreeLocked(struct mempool *mpl, void *item)
+MPL_Free(struct mempool *mpl, void *item)
 {
 	struct memitem *mi;
 
 	CHECK_OBJ_NOTNULL(mpl, MEMPOOL_MAGIC);
-	Lck_AssertHeld(mpl->mtx);
-
-	mpl->vsc->frees++;
-	mpl->vsc->live--;
 
 	mi = (void*)((uintptr_t)item - offsetof(struct memitem, payload));
 	CHECK_OBJ_NOTNULL(mi, MEMITEM_MAGIC);
+	memset(item, 0, mi->size);
+
+	Lck_Lock(mpl->mtx);
+
+	mpl->vsc->frees++;
+	mpl->vsc->live--;
 
 	if (mi->size < *mpl->cur_size) {
 		mpl->vsc->toosmall++;
-		FREE_OBJ(mi);
+		VTAILQ_INSERT_HEAD(&mpl->surplus, mi, list);
+	} else if (mpl->n_pool >= mpl->param->max_pool) {
+		mpl->vsc->surplus++;
+		VTAILQ_INSERT_HEAD(&mpl->surplus, mi, list);
 	} else {
 		mpl->vsc->pool++;
-		memset(item, 0, mi->size);
+		mpl->n_pool++;
 		VTAILQ_INSERT_HEAD(&mpl->list, mi, list);
 	}
-}
-
-void *
-MPL_Get(struct mempool *mpl, unsigned *size)
-{
-	void *p;
-
-	CHECK_OBJ_NOTNULL(mpl, MEMPOOL_MAGIC);
-	Lck_Lock(mpl->mtx);
-	p = MPL_GetLocked(mpl, size);
-	Lck_Unlock(mpl->mtx);
-	return (p);
-}
 
-void
-MPL_Free(struct mempool *mpl, void *item)
-{
-	CHECK_OBJ_NOTNULL(mpl, MEMPOOL_MAGIC);
-	Lck_Lock(mpl->mtx);
-	MPL_FreeLocked(mpl, item);
 	Lck_Unlock(mpl->mtx);
 }
diff --git a/bin/varnishd/mgt/mgt_param.c b/bin/varnishd/mgt/mgt_param.c
index 6faf0c5..1935344 100644
--- a/bin/varnishd/mgt/mgt_param.c
+++ b/bin/varnishd/mgt/mgt_param.c
@@ -632,7 +632,6 @@ tweak_poolparam(struct cli *cli, const struct parspec *par, const char *arg)
 			}
 			*pp = px;
 		} while(0);
-		
 	}
 }
 
diff --git a/include/tbl/vsc_fields.h b/include/tbl/vsc_fields.h
index cc72903..58f4d88 100644
--- a/include/tbl/vsc_fields.h
+++ b/include/tbl/vsc_fields.h
@@ -116,5 +116,6 @@ VSC_F(pool,			uint64_t, 0, 'g', "In Pool", "")
 VSC_F(recycle,			uint64_t, 0, 'g', "Recycled from pool", "")
 VSC_F(timeout,			uint64_t, 0, 'g', "Timed out from pool", "")
 VSC_F(toosmall,			uint64_t, 0, 'g', "Too small to recycle", "")
+VSC_F(surplus,			uint64_t, 0, 'g', "Too many for pool", "")
 
 #endif



From geoff at varnish-cache.org  Mon Jan  9 23:03:27 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Tue, 10 Jan 2012 00:03:27 +0100
Subject: [experimental-ims] 9f4bb5a Gcc is silly...
Message-ID: <E1RkOFT-0007nY-D0@project.varnish-software.com>

commit 9f4bb5a36442811cf823cb977511fc0964d62cef
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu Dec 15 10:23:56 2011 +0000

    Gcc is silly...

diff --git a/bin/varnishd/cache/cache_mempool.c b/bin/varnishd/cache/cache_mempool.c
index 6f46a2a..ac4abf2 100644
--- a/bin/varnishd/cache/cache_mempool.c
+++ b/bin/varnishd/cache/cache_mempool.c
@@ -152,6 +152,7 @@ mpl_guard(void *priv)
 			mi = NULL;
 		}
 	}
+	NEEDLESS_RETURN(NULL);
 }
 
 /*---------------------------------------------------------------------



From geoff at varnish-cache.org  Mon Jan  9 23:03:27 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Tue, 10 Jan 2012 00:03:27 +0100
Subject: [experimental-ims] 3e1787c Remove two old relics
Message-ID: <E1RkOFT-0007ni-H1@project.varnish-software.com>

commit 3e1787cdd2df94894dc695e1101002cf7d0db578
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu Dec 15 21:54:57 2011 +0000

    Remove two old relics

diff --git a/bin/varnishd/c.sh b/bin/varnishd/c.sh
deleted file mode 100644
index 8bab772..0000000
--- a/bin/varnishd/c.sh
+++ /dev/null
@@ -1,15 +0,0 @@
-#!/bin/sh
-
-# A client side script to test the ESI parsing, see s.sh for serverside
-
-set -e
-
-echo ""
-while true
-do
-	sleep 1
-	echo ""
-	fetch -o - -q http://localhost:8080/ | hexdump -C |
-		 sed 's/$//'
-	echo ""
-done
diff --git a/bin/varnishd/s.sh b/bin/varnishd/s.sh
deleted file mode 100644
index 469cfb5..0000000
--- a/bin/varnishd/s.sh
+++ /dev/null
@@ -1,82 +0,0 @@
-#!/bin/sh
-
-# A server side test-script for pushing the ESI XML parser over a
-# storage boundary.
-# The crucial trick here is that we send these objects HTTP/0.9 style
-# so that cache_fetch puts the first 128k in one storage object and
-# the rest in another, thus by putting around 128K space in our test
-# data we can put it right before, over and after the storage boundary.
-#
-# Use c.sh as the client side, and run varnish with this vcl:
-#
-# backend b1 {
-#         set backend.host = "Localhost";
-#         set backend.port = "8081";
-# }
-# 
-# sub vcl_recv {
-# 	pass;
-# }
-# 
-# sub vcl_fetch {
-# 	esi;
-# }
-
-serve () (
-	(
-	echo 'HTTP/1.0 200 OK'
-	echo ''
-	echo "$1"
-	dd if=/dev/zero bs=$2 count=1 2>/dev/null | tr '\0' ' ' 
-	cat
-	sleep .1
-	) | nc -l 8081
-)
-
-
-
-if false ; then
-    echo -n "<esi:remove> foo </esi:remove> bar" | serve Test01 1
-    echo -n "<esi:remove> foo </esi:remove> bar" | serve Test02 2
-    # Unterminated CDATA
-    echo -n "<esi:remove> foo </esi:remove> { <![CDATA[foo]] }" | serve Test03 10
-
-    for i in `jot 40 131020`
-    do
-	echo -n "<esi:remove> foo </esi:remove> bar" | serve Test04::$i $i
-    done
-
-    for i in `jot 40 131036`
-    do
-	echo -n "<!--esi foo --> bar" | serve Test05::$i $i
-    done
-
-    for i in `jot 22 131040`
-    do
-	echo -n "<![CDATA[foo]]>" | serve Test06::$i $i
-    done
-
-    echo -n "<esi:remove> " | serve Test07 10
-
-    echo -n "<!--esi " | serve Test08 10
-
-    for i in `jot 10 131042`
-    do
-	echo -n " > " | serve "Test09:$i <esi:remove" $i
-    done
-
-    (
-    echo -n "<esi:remove  "
-    dd if=/dev/zero bs=32768 count=1 2>/dev/null | tr '\0' ' ' 
-    echo -n ">"
-    ) | serve "Test10"  131030
-
-    echo -n " ]]> " | serve "Test11:131048 <![CDATA[  " 131048
-    echo -n " bar" | serve "Test12 foo <esi:comment comment=\"Humbug!\"/> " 1
-    echo -n " bar" | serve "Test13 foo <esi:foo> " 1
-fi
-
-while true
-do
-    echo -n " <esi:say "Hi Mom">" | serve "Test13 foo <esi:foo> " 1
-done



From geoff at varnish-cache.org  Mon Jan  9 23:03:27 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Tue, 10 Jan 2012 00:03:27 +0100
Subject: [experimental-ims] 23e4c60 updated syntax,
	thanks to Chris Handy for spotting it
Message-ID: <E1RkOFT-0007o8-O2@project.varnish-software.com>

commit 23e4c6034feb5839dc78eb954baa36701692a790
Author: Per Buer <perbu at varnish-software.com>
Date:   Sun Dec 18 10:16:16 2011 +0100

    updated syntax, thanks to Chris Handy for spotting it

diff --git a/doc/sphinx/tutorial/increasing_your_hitrate.rst b/doc/sphinx/tutorial/increasing_your_hitrate.rst
index 31e32d0..0f0cbe8 100644
--- a/doc/sphinx/tutorial/increasing_your_hitrate.rst
+++ b/doc/sphinx/tutorial/increasing_your_hitrate.rst
@@ -32,8 +32,8 @@ Tool: varnishlog
 
 When you have identified the an URL which is frequently sent to the
 backend you can use varnishlog to have a look at the whole request.
-``varnishlog -c -o /foo/bar`` will give the whole (-o) requests coming
-from the client (-c) matching /foo/bar. 
+``varnishlog -c -m 'RxURL:^/foo/bar`` will give the whole (-o)
+requests coming from the client (-c) matching /foo/bar.
 
 For extended diagnostics headers, see
 http://www.varnish-cache.org/trac/wiki/VCLExampleHitMissHeader



From geoff at varnish-cache.org  Mon Jan  9 23:03:27 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Tue, 10 Jan 2012 00:03:27 +0100
Subject: [experimental-ims] 28af712 Stylistic nitpickery while I wait for the
	construction noises to allow intelligent hacking again:
Message-ID: <E1RkOFT-0007nu-KQ@project.varnish-software.com>

commit 28af712ca43b363a8f6c0c813df67f8661dd2be9
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Fri Dec 16 09:42:19 2011 +0000

    Stylistic nitpickery while I wait for the construction noises to
    allow intelligent hacking again:
    
    Wrap lines longer than 80 char and fix inconsistent copy&paste in
    copyright/disclaimer text.

diff --git a/bin/varnishadm/varnishadm.c b/bin/varnishadm/varnishadm.c
index 4570e3c..bb5cc8e 100644
--- a/bin/varnishadm/varnishadm.c
+++ b/bin/varnishadm/varnishadm.c
@@ -14,7 +14,7 @@
  *    notice, this list of conditions and the following disclaimer in the
  *    documentation and/or other materials provided with the distribution.
  *
- * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
@@ -224,7 +224,8 @@ pass(int sock)
 		if (fds[0].revents & POLLIN) {
 			/* Get rid of the prompt, kinda hackish */
 			u = write(1, "\r           \r", 13);
-			u = VCLI_ReadResult(fds[0].fd, &status, &answer, timeout);
+			u = VCLI_ReadResult(fds[0].fd, &status, &answer,
+			    timeout);
 			if (u) {
 				if (status == CLIS_COMMS)
 					RL_EXIT(0);
diff --git a/bin/varnishd/cache/cache_acceptor.c b/bin/varnishd/cache/cache_acceptor.c
index 8c83121..661b053 100644
--- a/bin/varnishd/cache/cache_acceptor.c
+++ b/bin/varnishd/cache/cache_acceptor.c
@@ -123,7 +123,8 @@ VCA_Prep(struct sess *sp)
 	sp->addr = WS_Dup(sp->ws, addr);
 	sp->port = WS_Dup(sp->ws, port);
 	if (cache_param->log_local_addr) {
-		AZ(getsockname(sp->fd, (void*)&sp->mysockaddr, &sp->mysockaddrlen));
+		AZ(getsockname(sp->fd, (void*)&sp->mysockaddr,
+		    &sp->mysockaddrlen));
 		VTCP_name(&sp->mysockaddr, sp->mysockaddrlen,
 		    addr, sizeof addr, port, sizeof port);
 		WSP(sp, SLT_SessionOpen, "%s %s %s %s",
diff --git a/bin/varnishd/cache/cache_backend.c b/bin/varnishd/cache/cache_backend.c
index c1bc055..6cfcaf3 100644
--- a/bin/varnishd/cache/cache_backend.c
+++ b/bin/varnishd/cache/cache_backend.c
@@ -354,7 +354,8 @@ vbe_GetVbe(const struct sess *sp, struct vdi_simple *vs)
 			return (vc);
 		}
 		VSC_C_main->backend_toolate++;
-		WSL(sp->wrk, SLT_BackendClose, vc->vsl_id, "%s", bp->display_name);
+		WSL(sp->wrk, SLT_BackendClose, vc->vsl_id, "%s",
+		   bp->display_name);
 
 		/* Checkpoint log to flush all info related to this connection
 		   before the OS reuses the FD */
diff --git a/bin/varnishd/cache/cache_backend.h b/bin/varnishd/cache/cache_backend.h
index a3ae5c2..ad71169 100644
--- a/bin/varnishd/cache/cache_backend.h
+++ b/bin/varnishd/cache/cache_backend.h
@@ -175,7 +175,8 @@ void VBE_DropRefVcl(struct backend *);
 void VBE_DropRefLocked(struct backend *b);
 
 /* cache_backend_poll.c */
-void VBP_Insert(struct backend *b, struct vrt_backend_probe const *p, const char *hosthdr);
+void VBP_Insert(struct backend *b, struct vrt_backend_probe const *p,
+    const char *hosthdr);
 void VBP_Remove(struct backend *b, struct vrt_backend_probe const *p);
 void VBP_Use(const struct backend *b, const struct vrt_backend_probe const *p);
 void VBP_Summary(struct cli *cli, const struct vbp_target *vt);
diff --git a/bin/varnishd/cache/cache_backend_poll.c b/bin/varnishd/cache/cache_backend_poll.c
index efd64cb..85ad69e 100644
--- a/bin/varnishd/cache/cache_backend_poll.c
+++ b/bin/varnishd/cache/cache_backend_poll.c
@@ -474,7 +474,8 @@ vbp_new_vcl(const struct vrt_backend_probe *p, const char *hosthdr)
  */
 
 void
-VBP_Insert(struct backend *b, const struct vrt_backend_probe *p, const char *hosthdr)
+VBP_Insert(struct backend *b, const struct vrt_backend_probe *p,
+    const char *hosthdr)
 {
 	struct vbp_target *vt;
 	struct vbp_vcl *vcl;
diff --git a/bin/varnishd/cache/cache_center.c b/bin/varnishd/cache/cache_center.c
index 8826ae2..aed15ee 100644
--- a/bin/varnishd/cache/cache_center.c
+++ b/bin/varnishd/cache/cache_center.c
@@ -237,7 +237,8 @@ cnt_prepresp(struct sess *sp)
 
 	sp->t_resp = W_TIM_real(wrk);
 	if (wrk->obj->objcore != NULL) {
-		if ((sp->t_resp - wrk->obj->last_lru) > cache_param->lru_timeout &&
+		if ((sp->t_resp - wrk->obj->last_lru) >
+		    cache_param->lru_timeout &&
 		    EXP_Touch(wrk->obj->objcore))
 			wrk->obj->last_lru = sp->t_resp;
 		wrk->obj->last_use = sp->t_resp;	/* XXX: locking ? */
@@ -1025,7 +1026,8 @@ cnt_first(struct sess *sp)
 	sp->ws_ses = WS_Snapshot(sp->ws);
 
 	/* Receive a HTTP protocol request */
-	HTC_Init(sp->htc, sp->ws, sp->fd, sp->vsl_id, cache_param->http_req_size,
+	HTC_Init(sp->htc, sp->ws, sp->fd, sp->vsl_id,
+	    cache_param->http_req_size,
 	    cache_param->http_req_hdr_len);
 	wrk->acct_tmp.sess++;
 
@@ -1758,7 +1760,8 @@ static struct cli_proto debug_cmds[] = {
 	{ "debug.xid", "debug.xid",
 		"\tExamine or set XID\n", 0, 1, "d", cli_debug_xid },
 	{ "debug.srandom", "debug.srandom",
-		"\tSeed the random(3) function\n", 0, 1, "d", cli_debug_srandom },
+		"\tSeed the random(3) function\n", 0, 1, "d",
+		cli_debug_srandom },
 	{ NULL }
 };
 
diff --git a/bin/varnishd/cache/cache_esi_deliver.c b/bin/varnishd/cache/cache_esi_deliver.c
index 0d6b74c..e961819 100644
--- a/bin/varnishd/cache/cache_esi_deliver.c
+++ b/bin/varnishd/cache/cache_esi_deliver.c
@@ -313,7 +313,8 @@ ESI_Deliver(struct sess *sp)
 					 * We have a gzip'ed VEC and delivers
 					 * a gzip'ed ESI response.
 					 */
-					(void)WRW_Write(sp->wrk, st->ptr + off, l2);
+					(void)WRW_Write(sp->wrk,
+					    st->ptr + off, l2);
 				} else if (sp->wrk->gzip_resp) {
 					/*
 					 * A gzip'ed ESI response, but the VEC
@@ -339,7 +340,8 @@ ESI_Deliver(struct sess *sp)
 					/*
 					 * Ungzip'ed VEC, ungzip'ed ESI response
 					 */
-					(void)WRW_Write(sp->wrk, st->ptr + off, l2);
+					(void)WRW_Write(sp->wrk,
+					    st->ptr + off, l2);
 				}
 				off += l2;
 				if (off == st->len) {
diff --git a/bin/varnishd/cache/cache_pool.c b/bin/varnishd/cache/cache_pool.c
index a3cb805..afa069a 100644
--- a/bin/varnishd/cache/cache_pool.c
+++ b/bin/varnishd/cache/cache_pool.c
@@ -385,9 +385,9 @@ pool_breed(struct pool *qp, const pthread_attr_t *tp_attr)
 	 * If we need more threads, and have space, create
 	 * one more thread.
 	 */
-	if (qp->nthr < cache_param->wthread_min ||	/* Not enough threads yet */
-	    (qp->lqueue > cache_param->wthread_add_threshold && /* more needed */
-	    qp->lqueue > qp->last_lqueue)) {	/* not getting better since last */
+	if (qp->nthr < cache_param->wthread_min || /* Not enough threads yet */
+	    (qp->lqueue > cache_param->wthread_add_threshold && /* need more  */
+	    qp->lqueue > qp->last_lqueue)) { /* not getting better since last */
 		if (qp->nthr > cache_param->wthread_max) {
 			Lck_Lock(&pool_mtx);
 			VSC_C_main->threads_limited++;
@@ -481,7 +481,8 @@ pool_herder(void *priv)
 		pp->nqueued = pp->ndropped = 0;
 		w = VTAILQ_LAST(&pp->idle, workerhead);
 		if (w != NULL &&
-		    (w->lastused < t_idle || pp->nthr > cache_param->wthread_max)) {
+		    (w->lastused < t_idle ||
+		    pp->nthr > cache_param->wthread_max)) {
 			VTAILQ_REMOVE(&pp->idle, w, list);
 		} else
 			w = NULL;
diff --git a/bin/varnishd/cache/cache_response.c b/bin/varnishd/cache/cache_response.c
index 7c63d51..43feae9 100644
--- a/bin/varnishd/cache/cache_response.c
+++ b/bin/varnishd/cache/cache_response.c
@@ -113,8 +113,8 @@ RES_BuildHttp(const struct sess *sp)
 	http_ClrHeader(sp->wrk->resp);
 	sp->wrk->resp->logtag = HTTP_Tx;
 	http_CopyResp(sp->wrk->resp, sp->wrk->obj->http);
-	http_FilterFields(sp->wrk, sp->vsl_id, sp->wrk->resp, sp->wrk->obj->http,
-	    HTTPH_A_DELIVER);
+	http_FilterFields(sp->wrk, sp->vsl_id, sp->wrk->resp,
+	    sp->wrk->obj->http, HTTPH_A_DELIVER);
 
 	if (!(sp->wrk->res_mode & RES_LEN)) {
 		http_Unset(sp->wrk->resp, H_Content_Length);
@@ -129,7 +129,8 @@ RES_BuildHttp(const struct sess *sp)
 		    "Transfer-Encoding: chunked");
 
 	VTIM_format(VTIM_real(), time_str);
-	http_PrintfHeader(sp->wrk, sp->vsl_id, sp->wrk->resp, "Date: %s", time_str);
+	http_PrintfHeader(sp->wrk, sp->vsl_id, sp->wrk->resp,
+	    "Date: %s", time_str);
 
 	if (sp->xid != sp->wrk->obj->xid)
 		http_PrintfHeader(sp->wrk, sp->vsl_id, sp->wrk->resp,
diff --git a/bin/varnishd/cache/cache_vrt_var.c b/bin/varnishd/cache/cache_vrt_var.c
index 0662e26..54124b2 100644
--- a/bin/varnishd/cache/cache_vrt_var.c
+++ b/bin/varnishd/cache/cache_vrt_var.c
@@ -88,12 +88,12 @@ VRT_DO_HDR(req,   proto,	sp->http,		HTTP_HDR_PROTO)
 VRT_DO_HDR(bereq, request,	sp->wrk->busyobj->bereq,	HTTP_HDR_REQ)
 VRT_DO_HDR(bereq, url,		sp->wrk->busyobj->bereq,	HTTP_HDR_URL)
 VRT_DO_HDR(bereq, proto,	sp->wrk->busyobj->bereq,	HTTP_HDR_PROTO)
-VRT_DO_HDR(obj,   proto,	sp->wrk->obj->http,		HTTP_HDR_PROTO)
-VRT_DO_HDR(obj,   response,	sp->wrk->obj->http,		HTTP_HDR_RESPONSE)
+VRT_DO_HDR(obj,   proto,	sp->wrk->obj->http,	HTTP_HDR_PROTO)
+VRT_DO_HDR(obj,   response,	sp->wrk->obj->http,	HTTP_HDR_RESPONSE)
 VRT_DO_HDR(resp,  proto,	sp->wrk->resp,		HTTP_HDR_PROTO)
 VRT_DO_HDR(resp,  response,	sp->wrk->resp,		HTTP_HDR_RESPONSE)
 VRT_DO_HDR(beresp,  proto,	sp->wrk->busyobj->beresp,	HTTP_HDR_PROTO)
-VRT_DO_HDR(beresp,  response,	sp->wrk->busyobj->beresp,	HTTP_HDR_RESPONSE)
+VRT_DO_HDR(beresp,  response,	sp->wrk->busyobj->beresp, HTTP_HDR_RESPONSE)
 
 /*--------------------------------------------------------------------*/
 
diff --git a/bin/varnishd/cache/cache_ws.c b/bin/varnishd/cache/cache_ws.c
index 01937b3..b5f48d3 100644
--- a/bin/varnishd/cache/cache_ws.c
+++ b/bin/varnishd/cache/cache_ws.c
@@ -14,7 +14,7 @@
  *    notice, this list of conditions and the following disclaimer in the
  *    documentation and/or other materials provided with the distribution.
  *
- * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
diff --git a/bin/varnishd/default.vcl b/bin/varnishd/default.vcl
index 412b6e9..dc4f555 100644
--- a/bin/varnishd/default.vcl
+++ b/bin/varnishd/default.vcl
@@ -16,15 +16,17 @@
  *
  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
- * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE
- * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
- * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR 
- * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
- * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
- * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
- * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+
  *
  * The default VCL code.
  *
diff --git a/bin/varnishd/hash/hash_critbit.c b/bin/varnishd/hash/hash_critbit.c
index 0b3a7b1..9935fc2 100644
--- a/bin/varnishd/hash/hash_critbit.c
+++ b/bin/varnishd/hash/hash_critbit.c
@@ -191,7 +191,8 @@ hcb_crit_bit(const struct objhead *oh1, const struct objhead *oh2,
  */
 
 static struct objhead *
-hcb_insert(struct worker *wrk, struct hcb_root *root, struct objhead *oh, int has_lock)
+hcb_insert(struct worker *wrk, struct hcb_root *root, struct objhead *oh,
+    int has_lock)
 {
 	volatile uintptr_t *p;
 	uintptr_t pp;
diff --git a/bin/varnishd/mgt/mgt.h b/bin/varnishd/mgt/mgt.h
index 52463e6..0e3eb9e 100644
--- a/bin/varnishd/mgt/mgt.h
+++ b/bin/varnishd/mgt/mgt.h
@@ -48,7 +48,8 @@ void MGT_Child_Cli_Fail(void);
 /* mgt_cli.c */
 
 typedef void mgt_cli_close_f(void *priv);
-void mgt_cli_setup(int fdi, int fdo, int verbose, const char *ident, mgt_cli_close_f *close_func, void *priv);
+void mgt_cli_setup(int fdi, int fdo, int verbose, const char *ident,
+    mgt_cli_close_f *close_func, void *priv);
 int mgt_cli_askchild(unsigned *status, char **resp, const char *fmt, ...);
 void mgt_cli_start_child(int fdi, int fdo);
 void mgt_cli_stop_child(void);
diff --git a/bin/varnishd/mgt/mgt_cli.c b/bin/varnishd/mgt/mgt_cli.c
index b141f5b..cbea81f 100644
--- a/bin/varnishd/mgt/mgt_cli.c
+++ b/bin/varnishd/mgt/mgt_cli.c
@@ -374,7 +374,8 @@ mgt_cli_callback2(const struct vev *e, int what)
 /*--------------------------------------------------------------------*/
 
 void
-mgt_cli_setup(int fdi, int fdo, int verbose, const char *ident, mgt_cli_close_f *closefunc, void *priv)
+mgt_cli_setup(int fdi, int fdo, int verbose, const char *ident,
+    mgt_cli_close_f *closefunc, void *priv)
 {
 	struct cli *cli;
 	struct vev *ev;
diff --git a/bin/varnishd/mgt/mgt_param.c b/bin/varnishd/mgt/mgt_param.c
index 1935344..128364c 100644
--- a/bin/varnishd/mgt/mgt_param.c
+++ b/bin/varnishd/mgt/mgt_param.c
@@ -797,7 +797,8 @@ static const struct parspec input_parspec[] = {
 		"See setsockopt(2) under SO_SNDTIMEO for more information.",
 		DELAYED_EFFECT,
 		"600", "seconds" },
-	{ "idle_send_timeout", tweak_timeout, &mgt_param.idle_send_timeout, 0, 0,
+	{ "idle_send_timeout", tweak_timeout, &mgt_param.idle_send_timeout,
+		0, 0,
 		"Time to wait with no data sent. "
 		"If no data has been transmitted in this many\n"
                 "seconds the session is closed. \n"
@@ -1409,7 +1410,8 @@ MCF_DumpRst(void)
 	const char *p, *q;
 	int i;
 
-	printf("\n.. The following is the autogenerated output from varnishd -x dumprst\n\n");
+	printf("\n.. The following is the autogenerated "
+	    "output from varnishd -x dumprst\n\n");
 	for (i = 0; i < nparspec; i++) {
 		pp = parspec[i];
 		printf("%s\n", pp->name);
diff --git a/bin/varnishd/mgt/mgt_pool.c b/bin/varnishd/mgt/mgt_pool.c
index e6320fd..d34eef2 100644
--- a/bin/varnishd/mgt/mgt_pool.c
+++ b/bin/varnishd/mgt/mgt_pool.c
@@ -135,7 +135,8 @@ const struct parspec WRK_parspec[] = {
 		"Minimum is 2 threads.",
 		EXPERIMENTAL | DELAYED_EFFECT,
 		"5", "threads" },
-	{ "thread_pool_timeout", tweak_timeout, &mgt_param.wthread_timeout, 1, 0,
+	{ "thread_pool_timeout", tweak_timeout, &mgt_param.wthread_timeout,
+		1, 0,
 		"Thread idle threshold.\n"
 		"\n"
 		"Threads in excess of thread_pool_min, which have been idle "
diff --git a/bin/varnishd/mgt/mgt_sandbox_solaris.c b/bin/varnishd/mgt/mgt_sandbox_solaris.c
index 114d6a4..e043fef 100644
--- a/bin/varnishd/mgt/mgt_sandbox_solaris.c
+++ b/bin/varnishd/mgt/mgt_sandbox_solaris.c
@@ -137,7 +137,8 @@ mgt_sandbox_solaris_init(void)
 
 	if (! (priv_all = priv_allocset())) {
 		REPORT(LOG_ERR,
-		    "Child start warning: mgt_sandbox_init - priv_allocset failed: errno=%d (%s)",
+		    "Child start warning: "
+		    " mgt_sandbox_init - priv_allocset failed: errno=%d (%s)",
 		    errno, strerror(errno));
 		return;
 	}
@@ -164,7 +165,8 @@ mgt_sandbox_solaris_privsep(void)
                 if (getuid() != mgt_param.uid)
                         XXXAZ(setuid(mgt_param.uid));
         } else {
-                REPORT(LOG_INFO, "Privilege %s missing, will not change uid/gid",
+                REPORT(LOG_INFO,
+		    "Privilege %s missing, will not change uid/gid",
 		    PRIV_PROC_SETID);
         }
 }
@@ -194,7 +196,8 @@ mgt_sandbox_solaris_fini(void)
 	    !(inheritable = priv_allocset()) ||
 	    !(permitted = priv_allocset())) {
 		REPORT(LOG_ERR,
-		    "Child start warning: mgt_sandbox_waive - priv_allocset failed: errno=%d (%s)",
+		    "Child start warning: "
+		    " mgt_sandbox_waive - priv_allocset failed: errno=%d (%s)",
 		    errno, strerror(errno));
 		return;
 	}
@@ -218,7 +221,8 @@ mgt_sandbox_solaris_fini(void)
 #define SETPPRIV(which, set)						\
 	if (setppriv(PRIV_OFF, which, set))				\
 		REPORT(LOG_ERR,						\
-		    "Child start warning: Waiving privileges failed on %s: errno=%d (%s)", \
+		    "Child start warning: "				\
+		    " Waiving privileges failed on %s: errno=%d (%s)",	\
 		    #which, errno, strerror(errno));
 
 	SETPPRIV(PRIV_LIMIT, permitted);
diff --git a/bin/varnishd/waiter/cache_waiter_kqueue.c b/bin/varnishd/waiter/cache_waiter_kqueue.c
index 23df3de..7ceb80c 100644
--- a/bin/varnishd/waiter/cache_waiter_kqueue.c
+++ b/bin/varnishd/waiter/cache_waiter_kqueue.c
@@ -131,7 +131,8 @@ vwk_kev(struct vwk *vwk, const struct kevent *kp)
 		SES_Delete(sp, "EOF");
 		return;
 	} else {
-		VSL(SLT_Debug, sp->vsl_id, "KQ: sp %p kev data %lu flags 0x%x%s",
+		VSL(SLT_Debug, sp->vsl_id,
+		    "KQ: sp %p kev data %lu flags 0x%x%s",
 		    sp, (unsigned long)kp->data, kp->flags,
 		    (kp->flags & EV_EOF) ? " EOF" : "");
 	}
diff --git a/bin/varnishd/waiter/cache_waiter_ports.c b/bin/varnishd/waiter/cache_waiter_ports.c
index 17d7ce0..ce2afc3 100644
--- a/bin/varnishd/waiter/cache_waiter_ports.c
+++ b/bin/varnishd/waiter/cache_waiter_ports.c
@@ -15,7 +15,7 @@
  *    notice, this list of conditions and the following disclaimer in the
  *    documentation and/or other materials provided with the distribution.
  *
- * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
@@ -108,7 +108,8 @@ vws_port_ev(struct vws *vws, port_event_t *ev) {
 		 * port_dissociate after port_getn(), but in fact,
 		 * port_dissociate should be used
 		 *
-		 * Ref: http://opensolaris.org/jive/thread.jspa?threadID=129476&tstart=0
+		 * Ref: http://opensolaris.org/jive/thread.jspa?\
+		 *          threadID=129476&tstart=0
 		 */
 		vws_del(vws, sp->fd);
 		VTAILQ_REMOVE(&vws->sesshead, sp, list);
@@ -145,7 +146,7 @@ vws_thread(void *priv)
 	 * - additional tolerance for handling session timeouts
 	 *
 	 */
-	static struct timespec min_ts = {0L,    100L /*ms*/  * 1000L /*us*/  * 1000L /*ns*/};
+	static struct timespec min_ts = {0L, 100L * 1000L * 1000L /*ns*/};
 	static double          min_t  = 0.1; /* 100    ms*/
 	static struct timespec max_ts = {1L, 0L};		/* 1 second */
 	static double	       max_t  = 1.0;			/* 1 second */
@@ -177,7 +178,8 @@ vws_thread(void *priv)
 		/*
 		 * see disucssion in
 		 * - https://issues.apache.org/bugzilla/show_bug.cgi?id=47645
-		 * - http://mail.opensolaris.org/pipermail/networking-discuss/2009-August/011979.html
+		 * - http://mail.opensolaris.org/pipermail/\
+		 *       networking-discuss/2009-August/011979.html
 		 *
 		 * comment from apr/poll/unix/port.c :
 		 *
@@ -224,9 +226,13 @@ vws_thread(void *priv)
 		 */
 
 		if (sp) {
-			double tmo = (sp->t_open + cache_param->sess_timeout) - now;
+			double tmo =
+			    (sp->t_open + cache_param->sess_timeout) - now;
 
-			/* we should have removed all sps whose timeout has passed */
+			/*
+			 * we should have removed all sps whose timeout
+			 * has passed
+			 */
 			assert(tmo > 0.0);
 
 			if (tmo < min_t) {
diff --git a/bin/varnishhist/varnishhist.c b/bin/varnishhist/varnishhist.c
index 852ff41..03bfd88 100644
--- a/bin/varnishhist/varnishhist.c
+++ b/bin/varnishhist/varnishhist.c
@@ -106,9 +106,21 @@ struct profile {
 	int hist_low;
 	int hist_high;
 } profiles[] = {
-	{ .name = "responsetime", .tag = SLT_ReqEnd, .field = 5, .hist_low = -6, .hist_high = 3 },
-	{ .name = "size", .tag = SLT_Length, .field = 1, .hist_low = 1, .hist_high = 8 },
-	{ 0 }
+	{
+		.name = "responsetime",
+		.tag = SLT_ReqEnd,
+		.field = 5,
+		.hist_low = -6,
+		.hist_high = 3
+	}, {
+		.name = "size",
+		.tag = SLT_Length,
+		.field = 1,
+		.hist_low = 1,
+		.hist_high = 8
+	}, {
+		.name = 0,
+	}
 };
 
 static struct profile *active_profile;
@@ -335,7 +347,8 @@ static void
 usage(void)
 {
 	fprintf(stderr, "usage: varnishhist "
-	    "%s [-p profile] [-f field_num] [-R max] [-r min] [-V] [-w delay]\n", VSL_USAGE);
+	    "%s [-p profile] [-f field_num] "
+	    "[-R max] [-r min] [-V] [-w delay]\n", VSL_USAGE);
 	exit(1);
 }
 
@@ -386,7 +399,8 @@ main(int argc, char **argv)
 		}
 	}
 	if (profile) {
-		for (active_profile = profiles; active_profile->name; active_profile++) {
+		for (active_profile = profiles; active_profile->name;
+		    active_profile++) {
 			if (strcmp(active_profile->name, profile) == 0) {
 				break;
 			}
diff --git a/bin/varnishlog/varnishlog.c b/bin/varnishlog/varnishlog.c
index 45b4327..a606861 100644
--- a/bin/varnishlog/varnishlog.c
+++ b/bin/varnishlog/varnishlog.c
@@ -270,7 +270,8 @@ static void
 usage(void)
 {
 	fprintf(stderr, "usage: varnishlog "
-	    "%s [-aDV] [-o [tag regex]] [-n varnish_name] [-P file] [-w file]\n", VSL_USAGE);
+	    "%s [-aDV] [-o [tag regex]] [-n varnish_name]"
+	    " [-P file] [-w file]\n", VSL_USAGE);
 	exit(1);
 }
 
diff --git a/bin/varnishncsa/varnishncsa.c b/bin/varnishncsa/varnishncsa.c
index 5561adc..9da6172 100644
--- a/bin/varnishncsa/varnishncsa.c
+++ b/bin/varnishncsa/varnishncsa.c
@@ -102,7 +102,8 @@ static struct logline {
 	char *df_u;			/* %u, Remote user */
 	char *df_ttfb;			/* Time to first byte */
 	const char *df_hitmiss;		/* Whether this is a hit or miss */
-	const char *df_handling;	/* How the request was handled (hit/miss/pass/pipe) */
+	const char *df_handling;	/* How the request was handled
+					   (hit/miss/pass/pipe) */
 	int active;			/* Is log line in an active trans */
 	int complete;			/* Is log line complete */
 	uint64_t bitmap;		/* Bitmap for regex matches */
@@ -549,7 +550,9 @@ collect_client(struct logline *lp, enum VSL_tag_e tag, unsigned spec,
 		char ttfb[64];
 		if (!lp->active)
 			break;
-		if (lp->df_ttfb != NULL || sscanf(ptr, "%*u %*u.%*u %ld.%*u %*u.%*u %s", &l, ttfb) != 2) {
+		if (lp->df_ttfb != NULL ||
+		    sscanf(ptr, "%*u %*u.%*u %ld.%*u %*u.%*u %s", &l, ttfb)
+		    != 2) {
 			clean_logline(lp);
 			break;
 		}
@@ -679,7 +682,8 @@ h_ncsa(void *priv, enum VSL_tag_e tag, unsigned fd,
 			VSB_cat(os, lp->df_m ? lp->df_m : "-");
 			VSB_putc(os, ' ');
 			if (req_header(lp, "Host")) {
-				if (strncmp(req_header(lp, "Host"), "http://", 7) != 0)
+				if (strncmp(req_header(lp, "Host"),
+				    "http://", 7) != 0)
 					VSB_cat(os, "http://");
 				VSB_cat(os, req_header(lp, "Host"));
 			} else {
@@ -698,7 +702,8 @@ h_ncsa(void *priv, enum VSL_tag_e tag, unsigned fd,
 
 		case 't':
 			/* %t */
-			strftime(tbuf, sizeof tbuf, "[%d/%b/%Y:%T %z]", &lp->df_t);
+			strftime(tbuf, sizeof tbuf,
+			    "[%d/%b/%Y:%T %z]", &lp->df_t);
 			VSB_cat(os, tbuf);
 			break;
 
@@ -753,20 +758,23 @@ h_ncsa(void *priv, enum VSL_tag_e tag, unsigned fd,
 				p = tmp;
 				break;
 			case 'x':
-				if (strcmp(fname, "Varnish:time_firstbyte") == 0) {
+				if (!strcmp(fname, "Varnish:time_firstbyte")) {
 					VSB_cat(os, lp->df_ttfb);
 					p = tmp;
 					break;
-				} else if (strcmp(fname, "Varnish:hitmiss") == 0) {
-					VSB_cat(os, (lp->df_hitmiss ? lp->df_hitmiss : "-"));
+				} else if (!strcmp(fname, "Varnish:hitmiss")) {
+					VSB_cat(os, (lp->df_hitmiss ?
+					    lp->df_hitmiss : "-"));
 					p = tmp;
 					break;
-				} else if (strcmp(fname, "Varnish:handling") == 0) {
-					VSB_cat(os, (lp->df_handling ? lp->df_handling : "-"));
+				} else if (!strcmp(fname, "Varnish:handling")) {
+					VSB_cat(os, (lp->df_handling ?
+					    lp->df_handling : "-"));
 					p = tmp;
 					break;
-				} else if (strncmp(fname, "VCL_Log:", 8) == 0) {
-					// support pulling entries logged with std.log() into output.
+				} else if (!strncmp(fname, "VCL_Log:", 8)) {
+					// support pulling entries logged
+					// with std.log() into output.
 					// Format: %{VCL_Log:keyname}x
 					// Logging: std.log("keyname:value")
 					h = vcl_log(lp, fname+8);
@@ -775,7 +783,8 @@ h_ncsa(void *priv, enum VSL_tag_e tag, unsigned fd,
 					break;
 				}
 			default:
-				fprintf(stderr, "Unknown format starting at: %s\n", --p);
+				fprintf(stderr,
+				    "Unknown format starting at: %s\n", --p);
 				exit(1);
 			}
 			break;
@@ -783,7 +792,8 @@ h_ncsa(void *priv, enum VSL_tag_e tag, unsigned fd,
 			/* Fall through if we haven't handled something */
 			/* FALLTHROUGH*/
 		default:
-			fprintf(stderr, "Unknown format starting at: %s\n", --p);
+			fprintf(stderr,
+			    "Unknown format starting at: %s\n", --p);
 			exit(1);
 		}
 	}
@@ -854,15 +864,18 @@ main(int argc, char *argv[])
 			break;
 		case 'f':
 			if (format_flag) {
-				fprintf(stderr, "-f and -F can not be combined\n");
+				fprintf(stderr,
+				    "-f and -F can not be combined\n");
 				exit(1);
 			}
-			format = "%{X-Forwarded-For}i %l %u %t \"%r\" %s %b \"%{Referer}i\" \"%{User-agent}i\"";
+			format = "%{X-Forwarded-For}i %l %u %t \"%r\""
+			    " %s %b \"%{Referer}i\" \"%{User-agent}i\"";
 			format_flag = 1;
 			break;
 		case 'F':
 			if (format_flag) {
-				fprintf(stderr, "-f and -F can not be combined\n");
+				fprintf(stderr,
+				    "-f and -F can not be combined\n");
 				exit(1);
 			}
 			format_flag = 1;
diff --git a/bin/varnishstat/varnishstat.c b/bin/varnishstat/varnishstat.c
index 5d9226f..859c379 100644
--- a/bin/varnishstat/varnishstat.c
+++ b/bin/varnishstat/varnishstat.c
@@ -241,7 +241,9 @@ usage(void)
 	    "The varnishd instance to get logs from");
 	fprintf(stderr, FMT, "-V", "Display the version number and exit");
 	fprintf(stderr, FMT, "-w delay",
-	    "Wait delay seconds between updates.  Default is 1 second. Can also be be used with -1, -x or -j for repeated output.");
+	    "Wait delay seconds between updates."
+	    "  Default is 1 second."
+	    " Can also be be used with -1, -x or -j for repeated output.");
 	fprintf(stderr, FMT, "-x",
 	    "Print statistics to stdout as XML.");
 	fprintf(stderr, FMT, "-j",
diff --git a/bin/varnishstat/varnishstat.h b/bin/varnishstat/varnishstat.h
index 0a5d80f..c229ed2 100644
--- a/bin/varnishstat/varnishstat.h
+++ b/bin/varnishstat/varnishstat.h
@@ -36,4 +36,5 @@
 #include "vcs.h"
 
 
-void do_curses(struct VSM_data *vd, const struct VSC_C_main *VSC_C_main, int delay);
+void do_curses(struct VSM_data *vd, const struct VSC_C_main *VSC_C_main,
+    int delay);
diff --git a/bin/varnishstat/varnishstat.xsd b/bin/varnishstat/varnishstat.xsd
index 84b0a6a..e9c5f3c 100644
--- a/bin/varnishstat/varnishstat.xsd
+++ b/bin/varnishstat/varnishstat.xsd
@@ -1,19 +1,19 @@
 <?xml version="1.0"?>
 <xs:schema xmlns:xs="http://www.w3.org/2001/XMLSchema">
-	<xs:element name="varnishstat">
-		<xs:complexType>
+    <xs:element name="varnishstat">
+	<xs:complexType>
+	    <xs:sequence>
+		<xs:element name="stat" maxOccurs="unbounded">
+		    <xs:complexType>
 			<xs:sequence>
-				<xs:element name="stat" maxOccurs="unbounded">
-					<xs:complexType>
-						<xs:sequence>
-							<xs:element name="name" type="xs:string"/>
-							<xs:element name="value" type="xs:integer"/>
-							<xs:element name="description" type="xs:string"/>
-						</xs:sequence>
-					</xs:complexType>
-				</xs:element>
+			    <xs:element name="name" type="xs:string"/>
+			    <xs:element name="value" type="xs:integer"/>
+			    <xs:element name="description" type="xs:string"/>
 			</xs:sequence>
-			<xs:attribute name="timestamp" type="xs:dateTime"/>
-		</xs:complexType>
-	</xs:element>
+		    </xs:complexType>
+		</xs:element>
+	    </xs:sequence>
+	    <xs:attribute name="timestamp" type="xs:dateTime"/>
+	</xs:complexType>
+    </xs:element>
 </xs:schema>
diff --git a/bin/varnishstat/varnishstat_curses.c b/bin/varnishstat/varnishstat_curses.c
index d4e714c..3a672e1 100644
--- a/bin/varnishstat/varnishstat_curses.c
+++ b/bin/varnishstat/varnishstat_curses.c
@@ -174,7 +174,8 @@ do_curses(struct VSM_data *vd, const struct VSC_C_main *VSC_C_main,
 			 * Break to outher loop if we need to re-read file.
 			 * Only check if it looks like nothing is happening.
 			 */
-			act = VSC_C_main->cache_hit + VSC_C_main->cache_miss + 1;
+			act = VSC_C_main->cache_hit +
+			    VSC_C_main->cache_miss + 1;
 			lact = act;
 
 			AZ(gettimeofday(&tv, NULL));
diff --git a/bin/varnishtest/vtc_log.c b/bin/varnishtest/vtc_log.c
index 890a98b..402b49f 100644
--- a/bin/varnishtest/vtc_log.c
+++ b/bin/varnishtest/vtc_log.c
@@ -229,7 +229,8 @@ vtc_dump(struct vtclog *vl, int lvl, const char *pfx, const char *str, int len)
 
 //lint -e{818}
 void
-vtc_hexdump(struct vtclog *vl, int lvl, const char *pfx, const unsigned char *str, int len)
+vtc_hexdump(struct vtclog *vl, int lvl, const char *pfx,
+    const unsigned char *str, int len)
 {
 	int nl = 1;
 	unsigned l;
diff --git a/bin/varnishtop/varnishtop.c b/bin/varnishtop/varnishtop.c
index 62efb15..716b35b 100644
--- a/bin/varnishtop/varnishtop.c
+++ b/bin/varnishtop/varnishtop.c
@@ -327,7 +327,8 @@ main(int argc, char **argv)
 			errno = 0;
 			period = strtol(optarg, NULL, 0);
 			if (errno != 0)  {
-				fprintf(stderr, "Syntax error, %s is not a number", optarg);
+				fprintf(stderr,
+				    "Syntax error, %s is not a number", optarg);
 				exit(1);
 			}
 			break;
diff --git a/include/tbl/vsc_f_main.h b/include/tbl/vsc_f_main.h
index c8c4ecb..d63b187 100644
--- a/include/tbl/vsc_f_main.h
+++ b/include/tbl/vsc_f_main.h
@@ -20,7 +20,7 @@
  * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION, "")
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
@@ -136,7 +136,8 @@ VSC_F(fetch_chunked,		uint64_t, 1, 'a', "Fetch chunked", "")
 VSC_F(fetch_eof,		uint64_t, 1, 'a', "Fetch EOF", "")
 VSC_F(fetch_bad,		uint64_t, 1, 'a', "Fetch had bad headers", "")
 VSC_F(fetch_close,		uint64_t, 1, 'a', "Fetch wanted close", "")
-VSC_F(fetch_oldhttp,		uint64_t, 1, 'a', "Fetch pre HTTP/1.1 closed", "")
+VSC_F(fetch_oldhttp,		uint64_t, 1, 'a',
+    "Fetch pre HTTP/1.1 closed", "")
 VSC_F(fetch_zero,		uint64_t, 1, 'a', "Fetch zero len", "")
 VSC_F(fetch_failed,		uint64_t, 1, 'a', "Fetch failed", "")
 VSC_F(fetch_1xx,		uint64_t, 1, 'a', "Fetch no body (1xx)", "")
@@ -293,9 +294,11 @@ VSC_F(sess_herd,		uint64_t, 1, 'a', "Session herd", "")
 
 VSC_F(shm_records,		uint64_t, 0, 'a', "SHM records", "")
 VSC_F(shm_writes,		uint64_t, 0, 'a', "SHM writes", "")
-VSC_F(shm_flushes,		uint64_t, 0, 'a', "SHM flushes due to overflow", "")
+VSC_F(shm_flushes,		uint64_t, 0, 'a',
+    "SHM flushes due to overflow", "")
 VSC_F(shm_cont,		uint64_t, 0, 'a', "SHM MTX contention", "")
-VSC_F(shm_cycles,		uint64_t, 0, 'a', "SHM cycles through buffer", "")
+VSC_F(shm_cycles,		uint64_t, 0, 'a',
+    "SHM cycles through buffer", "")
 
 VSC_F(sms_nreq,		uint64_t, 0, 'a', "SMS allocator requests", "")
 VSC_F(sms_nobj,		uint64_t, 0, 'i', "SMS outstanding allocations", "")
@@ -353,19 +356,24 @@ VSC_F(bans_dups,		uint64_t, 0, 'c',
 
 /**********************************************************************/
 
-VSC_F(hcb_nolock,		uint64_t, 0, 'a', "HCB Lookups without lock", "")
+VSC_F(hcb_nolock,		uint64_t, 0, 'a',
+    "HCB Lookups without lock", "")
 VSC_F(hcb_lock,		uint64_t, 0, 'a', "HCB Lookups with lock", "")
 VSC_F(hcb_insert,		uint64_t, 0, 'a', "HCB Inserts", "")
 
-VSC_F(esi_errors,		uint64_t, 0, 'a', "ESI parse errors (unlock)", "")
-VSC_F(esi_warnings,		uint64_t, 0, 'a', "ESI parse warnings (unlock)", "")
+VSC_F(esi_errors,		uint64_t, 0, 'a',
+    "ESI parse errors (unlock)", "")
+VSC_F(esi_warnings,		uint64_t, 0, 'a',
+    "ESI parse warnings (unlock)", "")
 VSC_F(client_drop_late,	uint64_t, 0, 'a', "Connection dropped late", "")
 VSC_F(uptime,		uint64_t, 0, 'a', "Client uptime", "")
 
 VSC_F(dir_dns_lookups,	uint64_t, 0, 'a', "DNS director lookups", "")
 VSC_F(dir_dns_failed,	uint64_t, 0, 'a', "DNS director failed lookups", "")
-VSC_F(dir_dns_hit,		uint64_t, 0, 'a', "DNS director cached lookups hit", "")
-VSC_F(dir_dns_cache_full,	uint64_t, 0, 'a', "DNS director full dnscache", "")
+VSC_F(dir_dns_hit,		uint64_t, 0, 'a',
+    "DNS director cached lookups hit", "")
+VSC_F(dir_dns_cache_full,	uint64_t, 0, 'a',
+    "DNS director full dnscache", "")
 
 VSC_F(vmods,		uint64_t, 0, 'i', "Loaded VMODs", "")
 
diff --git a/include/tbl/vsc_fields.h b/include/tbl/vsc_fields.h
index 58f4d88..2671558 100644
--- a/include/tbl/vsc_fields.h
+++ b/include/tbl/vsc_fields.h
@@ -20,7 +20,7 @@
  * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION, "")
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
diff --git a/include/vlu.h b/include/vlu.h
index e69c6ae..a9fe074 100644
--- a/include/vlu.h
+++ b/include/vlu.h
@@ -11,7 +11,7 @@
  *    notice, this list of conditions and the following disclaimer in the
  *    documentation and/or other materials provided with the distribution.
  *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE
diff --git a/include/vtcp.h b/include/vtcp.h
index 6ad260e..77f86ed 100644
--- a/include/vtcp.h
+++ b/include/vtcp.h
@@ -54,8 +54,10 @@ VTCP_Check(int a)
 
 #define VTCP_Assert(a) assert(VTCP_Check(a))
 
-void VTCP_myname(int sock, char *abuf, unsigned alen, char *pbuf, unsigned plen);
-void VTCP_hisname(int sock, char *abuf, unsigned alen, char *pbuf, unsigned plen);
+void VTCP_myname(int sock, char *abuf, unsigned alen,
+    char *pbuf, unsigned plen);
+void VTCP_hisname(int sock, char *abuf, unsigned alen,
+    char *pbuf, unsigned plen);
 int VTCP_filter_http(int sock);
 int VTCP_blocking(int sock);
 int VTCP_nonblocking(int sock);
diff --git a/lib/libvarnish/vct.c b/lib/libvarnish/vct.c
index f0e1635..c82e972 100644
--- a/lib/libvarnish/vct.c
+++ b/lib/libvarnish/vct.c
@@ -13,10 +13,10 @@
  *    notice, this list of conditions and the following disclaimer in the
  *    documentation and/or other materials provided with the distribution.
  *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
diff --git a/lib/libvarnish/vfil.c b/lib/libvarnish/vfil.c
index ca1d965..6bcbde5 100644
--- a/lib/libvarnish/vfil.c
+++ b/lib/libvarnish/vfil.c
@@ -110,7 +110,8 @@ VFIL_readfile(const char *pfx, const char *fn, ssize_t *sz)
 	if (fn[0] == '/')
 		fd = open(fn, O_RDONLY);
 	else if (pfx != NULL) {
-		bprintf(fnb, "/%s/%s", pfx, fn); /* XXX: graceful length check */
+		bprintf(fnb, "/%s/%s", pfx, fn);
+		    /* XXX: graceful length check */
 		fd = open(fnb, O_RDONLY);
 	} else
 		fd = open(fn, O_RDONLY);
diff --git a/lib/libvarnish/vlu.c b/lib/libvarnish/vlu.c
index f149a57..610ca8b 100644
--- a/lib/libvarnish/vlu.c
+++ b/lib/libvarnish/vlu.c
@@ -11,10 +11,10 @@
  *    notice, this list of conditions and the following disclaimer in the
  *    documentation and/or other materials provided with the distribution.
  *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
diff --git a/lib/libvarnish/vtcp.c b/lib/libvarnish/vtcp.c
index d15f516..e361738 100644
--- a/lib/libvarnish/vtcp.c
+++ b/lib/libvarnish/vtcp.c
@@ -199,7 +199,8 @@ VTCP_nonblocking(int sock)
  */
 
 int
-VTCP_connect(int s, const struct sockaddr_storage *name, socklen_t namelen, int msec)
+VTCP_connect(int s, const struct sockaddr_storage *name, socklen_t namelen,
+    int msec)
 {
 	int i, k;
 	socklen_t l;
@@ -271,7 +272,8 @@ VTCP_set_read_timeout(int s, double seconds)
 	 * timeout does not get set. Needs to be fixed in Solaris, there is
 	 * nothing we can do about this.
 	 */
-	VTCP_Assert(setsockopt(s, SOL_SOCKET, SO_RCVTIMEO, &timeout, sizeof timeout));
+	VTCP_Assert(setsockopt(s, SOL_SOCKET, SO_RCVTIMEO,
+	    &timeout, sizeof timeout));
 #else
 	(void)s;
 #endif
diff --git a/lib/libvarnishapi/vsc.c b/lib/libvarnishapi/vsc.c
index b399c98..accef82 100644
--- a/lib/libvarnishapi/vsc.c
+++ b/lib/libvarnishapi/vsc.c
@@ -379,9 +379,8 @@ VSC_Iter(struct VSM_data *vd, VSC_iter_f *func, void *priv)
 			VSM_Close(vd);
 			if (!vd->head && VSM_Open(vd))
 				return (-1);
-			if (!VSM_Get(vd, &vsc->iter_fantom, VSC_CLASS, "", "")) {
+			if (!VSM_Get(vd, &vsc->iter_fantom, VSC_CLASS, "", ""))
 				return (-1);
-			}
 		}
 		AN(vd->head);
 		/* Tell app that list will be nuked */
diff --git a/lib/libvcl/Makefile.am b/lib/libvcl/Makefile.am
index 44e2957..f135e43 100644
--- a/lib/libvcl/Makefile.am
+++ b/lib/libvcl/Makefile.am
@@ -33,8 +33,9 @@ libvcl_la_SOURCES = \
 EXTRA_DIST = \
 	generate.py
 
-vcc_obj.c vcc_fixed_token.c vcc_token_defs.h: $(srcdir)/generate.py $(top_srcdir)/include/vrt.h
-	@PYTHON@ $(top_srcdir)/lib/libvcl/generate.py $(top_srcdir) $(top_builddir)
+vcc_obj.c vcc_fixed_token.c vcc_token_defs.h: \
+	$(srcdir)/generate.py $(top_srcdir)/include/vrt.h
+	@PYTHON@ $(srcdir)/generate.py $(srcdir) $(top_builddir)
 
 CLEANFILES = $(builddir)/vcc_token_defs.h \
 	$(builddir)/vcc_fixed_token.c \
diff --git a/lib/libvcl/vcc_compile.h b/lib/libvcl/vcc_compile.h
index b67e4ca..caacd73 100644
--- a/lib/libvcl/vcc_compile.h
+++ b/lib/libvcl/vcc_compile.h
@@ -103,7 +103,8 @@ enum symkind {
 #undef VCC_SYMB
 };
 
-typedef void sym_expr_t(struct vcc *tl, struct expr **, const struct symbol *sym);
+typedef void sym_expr_t(struct vcc *tl, struct expr **,
+    const struct symbol *sym);
 typedef struct symbol *sym_wildcard_t(struct vcc *tl, const struct token *t,
     const struct symbol *sym);
 
@@ -234,7 +235,8 @@ void vcc_ResetFldSpec(struct fld_spec *f);
 void vcc_IsField(struct vcc *tl, struct token **t, struct fld_spec *fs);
 void vcc_FieldsOk(struct vcc *tl, const struct fld_spec *fs);
 
-void Emit_Sockaddr(struct vcc *tl, const struct token *t_host, const char *port);
+void Emit_Sockaddr(struct vcc *tl, const struct token *t_host,
+    const char *port);
 
 /* vcc_compile.c */
 extern struct method method_tab[];
@@ -292,7 +294,8 @@ char *vcc_regexp(struct vcc *tl);
 
 /* vcc_symb.c */
 struct symbol *VCC_AddSymbolStr(struct vcc *tl, const char *name, enum symkind);
-struct symbol *VCC_AddSymbolTok(struct vcc *tl, const struct token *t, enum symkind kind);
+struct symbol *VCC_AddSymbolTok(struct vcc *tl, const struct token *t,
+    enum symkind kind);
 struct symbol *VCC_GetSymbolTok(struct vcc *tl, const struct token *tok,
     enum symkind);
 struct symbol *VCC_FindSymbol(struct vcc *tl,
diff --git a/lib/libvcl/vcc_vmod.c b/lib/libvcl/vcc_vmod.c
index 55b43d7..bd9f366 100644
--- a/lib/libvcl/vcc_vmod.c
+++ b/lib/libvcl/vcc_vmod.c
@@ -163,7 +163,8 @@ vcc_ParseImport(struct vcc *tl)
 		p = *spec;
 		if (!strcmp(p, "INIT")) {
 			p += strlen(p) + 1;
-			Fi(tl, 0, "\t%s(&vmod_priv_%.*s, &VCL_conf);\n", p, PF(mod));
+			Fi(tl, 0, "\t%s(&vmod_priv_%.*s, &VCL_conf);\n",
+			    p, PF(mod));
 		} else {
 			sym = VCC_AddSymbolStr(tl, p, SYM_FUNC);
 			ERRCHK(tl);
diff --git a/lib/libvmod_std/Makefile.am b/lib/libvmod_std/Makefile.am
index 065948a..c4128ea 100644
--- a/lib/libvmod_std/Makefile.am
+++ b/lib/libvmod_std/Makefile.am
@@ -8,6 +8,7 @@ INCLUDES = \
 dist_man_MANS = vmod_std.3
 
 vmoddir = $(pkglibdir)/vmods
+vmod_srcdir = $(top_srcdir)/lib/libvmod_std
 vmod_LTLIBRARIES = libvmod_std.la
 
 libvmod_std_la_LDFLAGS = -module -export-dynamic -avoid-version
@@ -19,8 +20,8 @@ libvmod_std_la_SOURCES = \
 	vmod_std_fileread.c \
 	vmod_std_conversions.c
 
-vcc_if.c vcc_if.h: $(top_srcdir)/lib/libvmod_std/vmod.py $(top_srcdir)/lib/libvmod_std/vmod.vcc
-	@PYTHON@ $(top_srcdir)/lib/libvmod_std/vmod.py $(top_srcdir)/lib/libvmod_std/vmod.vcc
+vcc_if.c vcc_if.h: $(vmod_srcdir)/vmod.py $(vmod_srcdir)/vmod.vcc 
+	@PYTHON@ $(vmod_srcdir)/vmod.py $(vmod_srcdir)/vmod.vcc
 
 EXTRA_DIST = vmod.py vmod.vcc
 
diff --git a/lib/libvmod_std/vmod.py b/lib/libvmod_std/vmod.py
index 6a9b4fc..e409628 100755
--- a/lib/libvmod_std/vmod.py
+++ b/lib/libvmod_std/vmod.py
@@ -223,12 +223,14 @@ while True:
 		if at == "ENUM":
 			if tq == None:
 				raise Exception(
-				    "Argument type '%s' needs qualifier {...}" % at)
+				    "Argument type '%s' needs qualifier {...}"
+				    % at)
 			at=parse_enum(tq)
 
 		elif tq != None:
 			raise Exception(
-			    "Argument type '%s' cannot be qualified with {...}" % at)
+			    "Argument type '%s' cannot be qualified with {...}"
+			    % at)
 
 		vargs.append(at)
 
@@ -246,7 +248,8 @@ def dumps(s):
 #######################################################################
 
 if initname != "":
-	plist += "int " + initname + "(struct vmod_priv *, const struct VCL_conf *);\n"
+	plist += "int " + initname 
+	plist += "(struct vmod_priv *, const struct VCL_conf *);\n"
 	pstruct += "\tvmod_init_f\t*_init;\n"
 	pinit += "\t" + initname + ",\n"
 	slist += '\t"INIT\\0Vmod_Func_' + modname + '._init",\n'
diff --git a/lib/libvmod_std/vmod_std_fileread.c b/lib/libvmod_std/vmod_std_fileread.c
index 89e4f50..154d8ca 100644
--- a/lib/libvmod_std/vmod_std_fileread.c
+++ b/lib/libvmod_std/vmod_std_fileread.c
@@ -59,8 +59,8 @@ struct frfile {
 	VTAILQ_ENTRY(frfile)		list;
 };
 
-static VTAILQ_HEAD(, frfile)		frlist = VTAILQ_HEAD_INITIALIZER(frlist);
-static pthread_mutex_t			frmtx = PTHREAD_MUTEX_INITIALIZER;
+static VTAILQ_HEAD(, frfile)	frlist = VTAILQ_HEAD_INITIALIZER(frlist);
+static pthread_mutex_t		frmtx = PTHREAD_MUTEX_INITIALIZER;
 
 static void
 free_frfile(void *ptr)



From geoff at varnish-cache.org  Mon Jan  9 23:03:27 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Tue, 10 Jan 2012 00:03:27 +0100
Subject: [experimental-ims] 55d8e88 Split waiter::cache and waiter:mgt out
	from waiter::common
Message-ID: <E1RkOFT-0007oJ-RT@project.varnish-software.com>

commit 55d8e88dfface89b6d4a38f3afc48863ccdc2b04
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Dec 19 10:04:28 2011 +0000

    Split waiter::cache and waiter:mgt out from waiter::common

diff --git a/bin/varnishd/Makefile.am b/bin/varnishd/Makefile.am
index 0c3427e..8a925b9 100644
--- a/bin/varnishd/Makefile.am
+++ b/bin/varnishd/Makefile.am
@@ -76,7 +76,8 @@ varnishd_SOURCES = \
 	storage/storage_persistent_subr.c \
 	storage/storage_synth.c \
 	storage/storage_umem.c \
-	waiter/waiter_common.c \
+	waiter/mgt_waiter.c \
+	waiter/cache_waiter.c \
 	waiter/cache_waiter_epoll.c \
 	waiter/cache_waiter_kqueue.c \
 	waiter/cache_waiter_poll.c \
@@ -96,7 +97,7 @@ noinst_HEADERS = \
 	mgt/mgt_param.h \
 	storage/storage.h \
 	storage/storage_persistent.h \
-	waiter/cache_waiter.h
+	waiter/waiter.h
 
 varnishd_CFLAGS = \
 	@PCRE_CFLAGS@ \
diff --git a/bin/varnishd/cache/cache_main.c b/bin/varnishd/cache/cache_main.c
index 2189104..48e934d 100644
--- a/bin/varnishd/cache/cache_main.c
+++ b/bin/varnishd/cache/cache_main.c
@@ -35,7 +35,7 @@
 #include "cache.h"
 #include "common/heritage.h"
 
-#include "waiter/cache_waiter.h"
+#include "waiter/waiter.h"
 #include "hash/hash_slinger.h"
 
 volatile struct params	*cache_param;
diff --git a/bin/varnishd/cache/cache_panic.c b/bin/varnishd/cache/cache_panic.c
index 94a4c09..6daf807 100644
--- a/bin/varnishd/cache/cache_panic.c
+++ b/bin/varnishd/cache/cache_panic.c
@@ -42,7 +42,7 @@
 #include "common/heritage.h"
 
 #include "cache_backend.h"
-#include "waiter/cache_waiter.h"
+#include "waiter/waiter.h"
 #include "vcl.h"
 
 /*
diff --git a/bin/varnishd/cache/cache_pool.c b/bin/varnishd/cache/cache_pool.c
index afa069a..a4d8a28 100644
--- a/bin/varnishd/cache/cache_pool.c
+++ b/bin/varnishd/cache/cache_pool.c
@@ -48,7 +48,7 @@
 #include "cache.h"
 #include "common/heritage.h"
 
-#include "waiter/cache_waiter.h"
+#include "waiter/waiter.h"
 #include "vtcp.h"
 #include "vtim.h"
 
diff --git a/bin/varnishd/cache/cache_session.c b/bin/varnishd/cache/cache_session.c
index 21d7f51..19292ea 100644
--- a/bin/varnishd/cache/cache_session.c
+++ b/bin/varnishd/cache/cache_session.c
@@ -40,7 +40,7 @@
 
 #include "cache.h"
 
-#include "waiter/cache_waiter.h"
+#include "waiter/waiter.h"
 
 /*--------------------------------------------------------------------*/
 
diff --git a/bin/varnishd/mgt/mgt_param.c b/bin/varnishd/mgt/mgt_param.c
index 128364c..f7f1dd0 100644
--- a/bin/varnishd/mgt/mgt_param.c
+++ b/bin/varnishd/mgt/mgt_param.c
@@ -43,7 +43,7 @@
 #include "common/params.h"
 
 #include "mgt/mgt_param.h"
-#include "waiter/cache_waiter.h"
+#include "waiter/waiter.h"
 #include "vav.h"
 #include "vcli.h"
 #include "vcli_common.h"
diff --git a/bin/varnishd/waiter/cache_waiter.c b/bin/varnishd/waiter/cache_waiter.c
new file mode 100644
index 0000000..bf5717b
--- /dev/null
+++ b/bin/varnishd/waiter/cache_waiter.c
@@ -0,0 +1,58 @@
+/*-
+ * Copyright (c) 2006 Verdens Gang AS
+ * Copyright (c) 2006-2011 Varnish Software AS
+ * All rights reserved.
+ *
+ * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ */
+
+#include "config.h"
+#include <stdio.h>
+#include <unistd.h>
+#include <string.h>
+
+#include "common/common.h"
+
+#include "waiter/waiter.h"
+
+const char *
+WAIT_GetName(void)
+{
+
+	if (waiter != NULL)
+		return (waiter->name);
+	else
+		return ("no_waiter");
+}
+
+void
+WAIT_Init(void)
+{
+
+	AN(waiter);
+	AN(waiter->name);
+	AN(waiter->init);
+	AN(waiter->pass);
+}
diff --git a/bin/varnishd/waiter/cache_waiter.h b/bin/varnishd/waiter/cache_waiter.h
deleted file mode 100644
index 1d6e679..0000000
--- a/bin/varnishd/waiter/cache_waiter.h
+++ /dev/null
@@ -1,64 +0,0 @@
-/*-
- * Copyright (c) 2006 Verdens Gang AS
- * Copyright (c) 2006-2011 Varnish Software AS
- * All rights reserved.
- *
- * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- *
- */
-
-struct sess;
-
-typedef void* waiter_init_f(void);
-typedef void waiter_pass_f(void *priv, const struct sess *);
-
-struct waiter {
-	const char		*name;
-	waiter_init_f		*init;
-	waiter_pass_f		*pass;
-};
-
-extern struct waiter const * waiter;
-
-#if defined(HAVE_EPOLL_CTL)
-extern const struct waiter waiter_epoll;
-#endif
-
-#if defined(HAVE_KQUEUE)
-extern const struct waiter waiter_kqueue;
-#endif
-
-#if defined(HAVE_PORT_CREATE)
-extern const struct waiter waiter_ports;
-#endif
-
-
-/* cache_session.c */
-void SES_Handle(struct sess *sp, int status);
-
-/* cache_waiter.c */
-extern const struct waiter waiter_poll;
-const char *WAIT_GetName(void);
-void WAIT_tweak_waiter(struct cli *cli, const char *arg);
-void WAIT_Init(void);
diff --git a/bin/varnishd/waiter/cache_waiter_epoll.c b/bin/varnishd/waiter/cache_waiter_epoll.c
index 06cd8bb..ca0d29f 100644
--- a/bin/varnishd/waiter/cache_waiter_epoll.c
+++ b/bin/varnishd/waiter/cache_waiter_epoll.c
@@ -42,7 +42,7 @@
 
 #include "cache/cache.h"
 
-#include "waiter/cache_waiter.h"
+#include "waiter/waiter.h"
 #include "vtim.h"
 
 #ifndef EPOLLRDHUP
diff --git a/bin/varnishd/waiter/cache_waiter_kqueue.c b/bin/varnishd/waiter/cache_waiter_kqueue.c
index 7ceb80c..e9ec1ec 100644
--- a/bin/varnishd/waiter/cache_waiter_kqueue.c
+++ b/bin/varnishd/waiter/cache_waiter_kqueue.c
@@ -44,7 +44,7 @@
 
 #include "cache/cache.h"
 
-#include "waiter/cache_waiter.h"
+#include "waiter/waiter.h"
 #include "vtim.h"
 
 #define NKEV	100
diff --git a/bin/varnishd/waiter/cache_waiter_poll.c b/bin/varnishd/waiter/cache_waiter_poll.c
index f6be13d..681b48d 100644
--- a/bin/varnishd/waiter/cache_waiter_poll.c
+++ b/bin/varnishd/waiter/cache_waiter_poll.c
@@ -35,7 +35,7 @@
 
 #include "cache/cache.h"
 
-#include "waiter/cache_waiter.h"
+#include "waiter/waiter.h"
 #include "vtim.h"
 
 #define NEEV	128
diff --git a/bin/varnishd/waiter/cache_waiter_ports.c b/bin/varnishd/waiter/cache_waiter_ports.c
index ce2afc3..7717196 100644
--- a/bin/varnishd/waiter/cache_waiter_ports.c
+++ b/bin/varnishd/waiter/cache_waiter_ports.c
@@ -42,7 +42,7 @@
 
 #include "cache/cache.h"
 
-#include "waiter/cache_waiter.h"
+#include "waiter/waiter.h"
 #include "vtim.h"
 
 #define MAX_EVENTS 256
diff --git a/bin/varnishd/waiter/mgt_waiter.c b/bin/varnishd/waiter/mgt_waiter.c
new file mode 100644
index 0000000..d2816a8
--- /dev/null
+++ b/bin/varnishd/waiter/mgt_waiter.c
@@ -0,0 +1,90 @@
+/*-
+ * Copyright (c) 2006 Verdens Gang AS
+ * Copyright (c) 2006-2011 Varnish Software AS
+ * All rights reserved.
+ *
+ * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ */
+
+#include "config.h"
+#include <stdio.h>
+#include <unistd.h>
+#include <string.h>
+
+#include "common/common.h"
+
+#include "waiter/waiter.h"
+#include "vcli.h"
+#include "vcli_priv.h"
+
+static const struct waiter *const vca_waiters[] = {
+    #if defined(HAVE_KQUEUE)
+	&waiter_kqueue,
+    #endif
+    #if defined(HAVE_EPOLL_CTL)
+	&waiter_epoll,
+    #endif
+    #if defined(HAVE_PORT_CREATE)
+	&waiter_ports,
+    #endif
+	&waiter_poll,
+	NULL,
+};
+
+struct waiter const *waiter;
+
+void
+WAIT_tweak_waiter(struct cli *cli, const char *arg)
+{
+	int i;
+
+	ASSERT_MGT();
+
+	if (arg == NULL) {
+		if (waiter == NULL)
+			VCLI_Out(cli, "default");
+		else
+			VCLI_Out(cli, "%s", waiter->name);
+
+		VCLI_Out(cli, " (");
+		for (i = 0; vca_waiters[i] != NULL; i++)
+			VCLI_Out(cli, "%s%s", i == 0 ? "" : ", ",
+			    vca_waiters[i]->name);
+		VCLI_Out(cli, ")");
+		return;
+	}
+	if (!strcmp(arg, "default")) {
+		waiter = vca_waiters[0];
+		return;
+	}
+	for (i = 0; vca_waiters[i]; i++) {
+		if (!strcmp(arg, vca_waiters[i]->name)) {
+			waiter = vca_waiters[i];
+			return;
+		}
+	}
+	VCLI_Out(cli, "Unknown waiter");
+	VCLI_SetResult(cli, CLIS_PARAM);
+}
diff --git a/bin/varnishd/waiter/waiter.h b/bin/varnishd/waiter/waiter.h
new file mode 100644
index 0000000..e9c3cdc
--- /dev/null
+++ b/bin/varnishd/waiter/waiter.h
@@ -0,0 +1,65 @@
+/*-
+ * Copyright (c) 2006 Verdens Gang AS
+ * Copyright (c) 2006-2011 Varnish Software AS
+ * All rights reserved.
+ *
+ * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ */
+
+struct sess;
+
+typedef void* waiter_init_f(void);
+typedef void waiter_pass_f(void *priv, const struct sess *);
+
+struct waiter {
+	const char		*name;
+	waiter_init_f		*init;
+	waiter_pass_f		*pass;
+};
+
+/* mgt_waiter.c */
+extern struct waiter const * waiter;
+void WAIT_tweak_waiter(struct cli *cli, const char *arg);
+
+#if defined(HAVE_EPOLL_CTL)
+extern const struct waiter waiter_epoll;
+#endif
+
+#if defined(HAVE_KQUEUE)
+extern const struct waiter waiter_kqueue;
+#endif
+
+#if defined(HAVE_PORT_CREATE)
+extern const struct waiter waiter_ports;
+#endif
+
+extern const struct waiter waiter_poll;
+
+/* cache_session.c */
+void SES_Handle(struct sess *sp, int status);
+
+/* cache_waiter.c */
+const char *WAIT_GetName(void);
+void WAIT_Init(void);
diff --git a/bin/varnishd/waiter/waiter_common.c b/bin/varnishd/waiter/waiter_common.c
deleted file mode 100644
index e09a296..0000000
--- a/bin/varnishd/waiter/waiter_common.c
+++ /dev/null
@@ -1,113 +0,0 @@
-/*-
- * Copyright (c) 2006 Verdens Gang AS
- * Copyright (c) 2006-2011 Varnish Software AS
- * All rights reserved.
- *
- * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- *
- */
-
-#include "config.h"
-#include <stdio.h>
-#include <unistd.h>
-#include <string.h>
-
-#include "common/common.h"
-
-#include "waiter/cache_waiter.h"
-#include "vcli.h"
-#include "vcli_priv.h"
-
-static const struct waiter * const vca_waiters[] = {
-    #if defined(HAVE_KQUEUE)
-	&waiter_kqueue,
-    #endif
-    #if defined(HAVE_EPOLL_CTL)
-	&waiter_epoll,
-    #endif
-    #if defined(HAVE_PORT_CREATE)
-	&waiter_ports,
-    #endif
-	&waiter_poll,
-	NULL,
-};
-
-struct waiter const * waiter;
-
-const char *
-WAIT_GetName(void)
-{
-
-	if (waiter != NULL)
-		return (waiter->name);
-	else
-		return ("no_waiter");
-}
-
-void
-WAIT_tweak_waiter(struct cli *cli, const char *arg)
-{
-	int i;
-
-	ASSERT_MGT();
-
-	if (arg == NULL) {
-		if (waiter == NULL)
-			VCLI_Out(cli, "default");
-		else
-			VCLI_Out(cli, "%s", waiter->name);
-
-		VCLI_Out(cli, " (");
-		for (i = 0; vca_waiters[i] != NULL; i++)
-			VCLI_Out(cli, "%s%s", i == 0 ? "" : ", ",
-			    vca_waiters[i]->name);
-		VCLI_Out(cli, ")");
-		return;
-	}
-	if (!strcmp(arg, "default")) {
-		waiter = NULL;
-		return;
-	}
-	for (i = 0; vca_waiters[i]; i++) {
-		if (!strcmp(arg, vca_waiters[i]->name)) {
-			waiter = vca_waiters[i];
-			return;
-		}
-	}
-	VCLI_Out(cli, "Unknown waiter");
-	VCLI_SetResult(cli, CLIS_PARAM);
-}
-
-void
-WAIT_Init(void)
-{
-
-	if (waiter == NULL)
-		waiter = vca_waiters[0];
-
-	AN(waiter);
-	AN(waiter->name);
-	AN(waiter->init);
-	AN(waiter->pass);
-}



From geoff at varnish-cache.org  Mon Jan  9 23:03:27 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Tue, 10 Jan 2012 00:03:27 +0100
Subject: [experimental-ims] ae3733f Move Pool_Wait() to WAIT_Enter() where it
	belongs
Message-ID: <E1RkOFT-0007oV-TE@project.varnish-software.com>

commit ae3733fb53a2c4fb21a720eb834dcc43b90d0577
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Dec 19 10:18:46 2011 +0000

    Move Pool_Wait() to WAIT_Enter() where it belongs

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index 9601270..0270f3b 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -868,7 +868,6 @@ void PipeSession(struct sess *sp);
 /* cache_pool.c */
 void Pool_Init(void);
 void Pool_Work_Thread(void *priv, struct worker *w);
-void Pool_Wait(struct sess *sp);
 int Pool_Schedule(struct pool *pp, struct sess *sp);
 
 #define WRW_IsReleased(w)	((w)->wrw.wfd == NULL)
@@ -893,6 +892,7 @@ void SES_Charge(struct sess *sp);
 struct sesspool *SES_NewPool(struct pool *pp);
 void SES_DeletePool(struct sesspool *sp, struct worker *wrk);
 int SES_Schedule(struct sess *sp);
+void SES_Handle(struct sess *sp, int status);
 
 
 /* cache_shmlog.c */
@@ -964,6 +964,11 @@ void ESI_DeliverChild(const struct sess *);
 /* cache_vrt_vmod.c */
 void VMOD_Init(void);
 
+/* cache_waiter.c */
+void WAIT_Enter(struct sess *sp);
+void WAIT_Init(void);
+const char *WAIT_GetName(void);
+
 /* cache_wrk.c */
 
 void WRK_Init(void);
diff --git a/bin/varnishd/cache/cache_center.c b/bin/varnishd/cache/cache_center.c
index aed15ee..520c620 100644
--- a/bin/varnishd/cache/cache_center.c
+++ b/bin/varnishd/cache/cache_center.c
@@ -121,7 +121,7 @@ cnt_wait(struct sess *sp)
 		WSP(sp, SLT_Debug, "herding");
 		wrk->stats.sess_herd++;
 		SES_Charge(sp);
-		Pool_Wait(sp);
+		WAIT_Enter(sp);
 		return (1);
 	}
 	if (i == 1) {
@@ -432,7 +432,7 @@ WSP(sp, SLT_Debug, "PHK req %.9f resp %.9f end %.9f open %.9f",
 		return (0);
 	}
 	wrk->stats.sess_herd++;
-	Pool_Wait(sp);
+	WAIT_Enter(sp);
 	return (1);
 }
 
diff --git a/bin/varnishd/cache/cache_pool.c b/bin/varnishd/cache/cache_pool.c
index a4d8a28..e59bbd0 100644
--- a/bin/varnishd/cache/cache_pool.c
+++ b/bin/varnishd/cache/cache_pool.c
@@ -48,8 +48,6 @@
 #include "cache.h"
 #include "common/heritage.h"
 
-#include "waiter/waiter.h"
-#include "vtcp.h"
 #include "vtim.h"
 
 /*--------------------------------------------------------------------
@@ -82,8 +80,6 @@ pthread_condattr_setclock(pthread_condattr_t *attr, int foo)
 }
 #endif /* !CLOCK_MONOTONIC */
 
-static void *waiter_priv;
-
 VTAILQ_HEAD(workerhead, worker);
 
 struct poolsock {
@@ -350,29 +346,6 @@ Pool_Schedule(struct pool *pp, struct sess *sp)
 }
 
 /*--------------------------------------------------------------------
- * Wait for another request
- */
-
-void
-Pool_Wait(struct sess *sp)
-{
-
-	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
-	CHECK_OBJ_NOTNULL(sp->wrk, WORKER_MAGIC);
-	AZ(sp->vcl);
-	assert(sp->fd >= 0);
-	sp->wrk = NULL;
-
-	/*
-	 * Set nonblocking in the worker-thread, before passing to the
-	 * acceptor thread, to reduce syscall density of the latter.
-	 */
-	if (VTCP_nonblocking(sp->fd))
-		SES_Close(sp, "remote closed");
-	waiter->pass(waiter_priv, sp);
-}
-
-/*--------------------------------------------------------------------
  * Create another thread, if necessary & possible
  */
 
@@ -588,7 +561,6 @@ void
 Pool_Init(void)
 {
 
-	waiter_priv = waiter->init();
 	Lck_New(&pool_mtx, lck_wq);
 	AZ(pthread_create(&thr_pool_herder, NULL, pool_poolherder, NULL));
 }
diff --git a/bin/varnishd/waiter/cache_waiter.c b/bin/varnishd/waiter/cache_waiter.c
index bf5717b..8aa8684 100644
--- a/bin/varnishd/waiter/cache_waiter.c
+++ b/bin/varnishd/waiter/cache_waiter.c
@@ -33,10 +33,14 @@
 #include <unistd.h>
 #include <string.h>
 
-#include "common/common.h"
+#include "cache/cache.h"
 
 #include "waiter/waiter.h"
 
+#include "vtcp.h"
+
+static void *waiter_priv;
+
 const char *
 WAIT_GetName(void)
 {
@@ -55,4 +59,24 @@ WAIT_Init(void)
 	AN(waiter->name);
 	AN(waiter->init);
 	AN(waiter->pass);
+	waiter_priv = waiter->init();
+}
+
+void
+WAIT_Enter(struct sess *sp)
+{
+
+	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+	CHECK_OBJ_NOTNULL(sp->wrk, WORKER_MAGIC);
+	AZ(sp->vcl);
+	assert(sp->fd >= 0);
+	sp->wrk = NULL;
+
+	/*
+	* Set nonblocking in the worker-thread, before passing to the
+	* acceptor thread, to reduce syscall density of the latter.
+	*/
+	if (VTCP_nonblocking(sp->fd))
+		SES_Close(sp, "remote closed");
+	waiter->pass(waiter_priv, sp);
 }
diff --git a/bin/varnishd/waiter/waiter.h b/bin/varnishd/waiter/waiter.h
index e9c3cdc..9f48884 100644
--- a/bin/varnishd/waiter/waiter.h
+++ b/bin/varnishd/waiter/waiter.h
@@ -56,10 +56,3 @@ extern const struct waiter waiter_ports;
 #endif
 
 extern const struct waiter waiter_poll;
-
-/* cache_session.c */
-void SES_Handle(struct sess *sp, int status);
-
-/* cache_waiter.c */
-const char *WAIT_GetName(void);
-void WAIT_Init(void);



From geoff at varnish-cache.org  Mon Jan  9 23:03:27 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Tue, 10 Jan 2012 00:03:27 +0100
Subject: [experimental-ims] 6f8d983 add a label so we can ref this in the user
	guide / tutorial
Message-ID: <E1RkOFT-0007oh-VF@project.varnish-software.com>

commit 6f8d983c178876d08631f9f15f66b95cd3b77dee
Author: Per Buer <perbu at varnish-software.com>
Date:   Mon Dec 19 11:24:42 2011 +0100

    add a label so we can ref this in the user guide / tutorial

diff --git a/doc/sphinx/reference/varnishlog.rst b/doc/sphinx/reference/varnishlog.rst
index 9e6ada9..a90231f 100644
--- a/doc/sphinx/reference/varnishlog.rst
+++ b/doc/sphinx/reference/varnishlog.rst
@@ -1,3 +1,5 @@
+_reference-varnishlog:
+
 ==========
 varnishlog
 ==========



From geoff at varnish-cache.org  Mon Jan  9 23:03:28 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Tue, 10 Jan 2012 00:03:28 +0100
Subject: [experimental-ims] e578219 Add a label,
	fix a typo and remove -o (now default)
Message-ID: <E1RkOFU-0007os-1O@project.varnish-software.com>

commit e5782194340ab76c06e8e4bc242f244fd18791f4
Author: Per Buer <perbu at varnish-software.com>
Date:   Mon Dec 19 11:25:39 2011 +0100

    Add a label, fix a typo and remove -o (now default)

diff --git a/doc/sphinx/tutorial/logging.rst b/doc/sphinx/tutorial/logging.rst
index 070dc3d..331742f 100644
--- a/doc/sphinx/tutorial/logging.rst
+++ b/doc/sphinx/tutorial/logging.rst
@@ -1,3 +1,4 @@
+.. _tutorial-logging:
 
 Logging in Varnish
 ------------------
@@ -8,7 +9,7 @@ memory segment. When the end of the segment is reached we start over,
 overwriting old data. This is much, much faster then logging to a file
 and it doesn't require disk space.
 
-The flip side is that is you forget to have program actually write the
+The flip side is that if you forget to have program actually write the
 logs to disk they will disappear.
 
 varnishlog is one of the programs you can use to look at what Varnish
@@ -69,9 +70,6 @@ want to know are:
  show all cookie headers coming from the clients:
  ``$ varnishlog -c -i RxHeader -I Cookie``
 
--o 
- Group log entries by request ID.
-
 
 Now that Varnish seem to work OK it's time to put Varnish on port 80
 while we tune it.



From geoff at varnish-cache.org  Mon Jan  9 23:03:28 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Tue, 10 Jan 2012 00:03:28 +0100
Subject: [experimental-ims] 7402560 remove -i and -I,
	add -m and describe it. Simplify the docs. The user only needs to
	care about -m, really
Message-ID: <E1RkOFU-0007p4-3Z@project.varnish-software.com>

commit 7402560ab9ab74043b00b360e24edf9bb58f8060
Author: Per Buer <perbu at varnish-software.com>
Date:   Mon Dec 19 11:30:26 2011 +0100

    remove -i and -I, add -m and describe it. Simplify the docs. The user only needs to care about -m, really

diff --git a/doc/sphinx/tutorial/logging.rst b/doc/sphinx/tutorial/logging.rst
index 331742f..68f1d87 100644
--- a/doc/sphinx/tutorial/logging.rst
+++ b/doc/sphinx/tutorial/logging.rst
@@ -60,16 +60,9 @@ want to know are:
 -c 
  Same as -b but for client side traffic.
 
--i tag
- Only show lines with a certain tag. "varnishlog -i SessionOpen" 
- will only give you new sessions. Note that the tags are case 
- insensitive. 
-
--I Regex
- Filter the data through a regex and only show the matching lines. To
- show all cookie headers coming from the clients:
- ``$ varnishlog -c -i RxHeader -I Cookie``
-
+-m tag:regex
+ Only list transactions where the tag matches a regular expression. If
+ it matches you will get the whole transaction.
 
 Now that Varnish seem to work OK it's time to put Varnish on port 80
 while we tune it.



From geoff at varnish-cache.org  Mon Jan  9 23:03:28 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Tue, 10 Jan 2012 00:03:28 +0100
Subject: [experimental-ims] f111f6f Remove the ref to -o for varnishlog Add a
	link to the varnishlog man page and the logging chapter in the
	tutorial
Message-ID: <E1RkOFU-0007pE-5R@project.varnish-software.com>

commit f111f6f2df427527a5e88cc54d30ba2c02a493e6
Author: Per Buer <perbu at varnish-software.com>
Date:   Mon Dec 19 11:31:44 2011 +0100

    Remove the ref to -o for varnishlog
    Add a link to the varnishlog man page and the logging chapter in the tutorial

diff --git a/doc/sphinx/tutorial/increasing_your_hitrate.rst b/doc/sphinx/tutorial/increasing_your_hitrate.rst
index 0f0cbe8..c5f0c6a 100644
--- a/doc/sphinx/tutorial/increasing_your_hitrate.rst
+++ b/doc/sphinx/tutorial/increasing_your_hitrate.rst
@@ -31,9 +31,11 @@ Tool: varnishlog
 ~~~~~~~~~~~~~~~~
 
 When you have identified the an URL which is frequently sent to the
-backend you can use varnishlog to have a look at the whole request.
-``varnishlog -c -m 'RxURL:^/foo/bar`` will give the whole (-o)
-requests coming from the client (-c) matching /foo/bar.
+backend you can use varnishlog to have a look at the request.
+``varnishlog -c -m 'RxURL:^/foo/bar`` will show you the requests
+coming from the client (-c) matching /foo/bar.
+
+For more information on how varnishlog works please see :ref:`tutorial-logging`.
 
 For extended diagnostics headers, see
 http://www.varnish-cache.org/trac/wiki/VCLExampleHitMissHeader



From geoff at varnish-cache.org  Mon Jan  9 23:03:28 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Tue, 10 Jan 2012 00:03:28 +0100
Subject: [experimental-ims] 59a3b91 Merge branch 'master' of
	ssh://git.varnish-cache.org/git/varnish-cache
Message-ID: <E1RkOFU-0007pR-8E@project.varnish-software.com>

commit 59a3b914752e25787807784f9ec4f464402340ec
Merge: f111f6f ae3733f
Author: Per Buer <perbu at varnish-software.com>
Date:   Mon Dec 19 11:31:56 2011 +0100

    Merge branch 'master' of ssh://git.varnish-cache.org/git/varnish-cache



From geoff at varnish-cache.org  Mon Jan  9 23:03:28 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Tue, 10 Jan 2012 00:03:28 +0100
Subject: [experimental-ims] 4c1e826 Add an explict enum to tell a worker what
	to do
Message-ID: <E1RkOFU-0007pg-BH@project.varnish-software.com>

commit 4c1e82666039e0ffcd5a9b56ba9c342a59375d56
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Dec 19 10:35:44 2011 +0000

    Add an explict enum to tell a worker what to do

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index 0270f3b..14b9fe6 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -289,6 +289,13 @@ struct wrk_accept {
 
 /*--------------------------------------------------------------------*/
 
+enum e_do_what {
+	pool_do_inval = 0,
+	pool_do_sess,
+	pool_do_accept,
+	pool_do_die,
+};
+
 struct worker {
 	unsigned		magic;
 #define WORKER_MAGIC		0x6391adcf
@@ -301,6 +308,8 @@ struct worker {
 	struct dstat		stats;
 
 	/* Pool stuff */
+	enum e_do_what		do_what;
+
 	double			lastused;
 
 	struct wrw		wrw;
diff --git a/bin/varnishd/cache/cache_pool.c b/bin/varnishd/cache/cache_pool.c
index e59bbd0..1195b45 100644
--- a/bin/varnishd/cache/cache_pool.c
+++ b/bin/varnishd/cache/cache_pool.c
@@ -168,6 +168,7 @@ pool_accept(struct pool *pp, struct worker *w, const struct poolsock *ps)
 		assert(sizeof *wa2 == WS_Reserve(w2->ws, sizeof *wa2));
 		wa2 = (void*)w2->ws->f;
 		memcpy(wa2, wa, sizeof *wa);
+		w2->do_what = pool_do_accept;
 		AZ(pthread_cond_signal(&w2->cond));
 	}
 }
@@ -190,6 +191,7 @@ Pool_Work_Thread(void *priv, struct worker *w)
 	while (1) {
 
 		Lck_AssertHeld(&pp->mtx);
+		w->do_what = pool_do_inval;
 
 		CHECK_OBJ_NOTNULL(w, WORKER_MAGIC);
 		CHECK_OBJ_NOTNULL(w->resp, HTTP_MAGIC);
@@ -201,6 +203,7 @@ Pool_Work_Thread(void *priv, struct worker *w)
 			/* Process queued requests, if any */
 			assert(pp->lqueue > 0);
 			VTAILQ_REMOVE(&pp->queue, w->sp, poollist);
+			w->do_what = pool_do_sess;
 			pp->lqueue--;
 		} else if (!VTAILQ_EMPTY(&pp->socks)) {
 			/* Accept on a socket */
@@ -215,6 +218,7 @@ Pool_Work_Thread(void *priv, struct worker *w)
 				continue;
 			}
 			VTAILQ_INSERT_TAIL(&pp->socks, ps, list);
+			w->do_what = pool_do_accept;
 		} else if (VTAILQ_EMPTY(&pp->socks)) {
 			/* Nothing to do: To sleep, perchance to dream ... */
 			if (isnan(w->lastused))
@@ -225,29 +229,27 @@ Pool_Work_Thread(void *priv, struct worker *w)
 			(void)Lck_CondWait(&w->cond, &pp->mtx, NULL);
 		}
 
-		/*
-		 * If we got neither session or accepted a socket, we were
-		 * woken up to die to cull the herd.
-		 */
-		if (w->sp == NULL && w->ws->r == NULL)
+		if (w->do_what == pool_do_die)
 			break;
 
 		Lck_Unlock(&pp->mtx);
 
-		if (w->sp == NULL) {
+		if (w->do_what == pool_do_accept) {
 			/* Turn accepted socket into a session */
-			assert(w->ws->r != NULL);
+			AZ(w->sp);
+			AN(w->ws->r);
 			w->sp = SES_New(w, pp->sesspool);
 			if (w->sp == NULL)
 				VCA_FailSess(w);
 			else
 				VCA_SetupSess(w);
 			WS_Release(w->ws, 0);
+			w->do_what = pool_do_sess;
 		}
-		assert(w->ws->r == NULL);
 
-		if (w->sp != NULL) {
+		if (w->do_what == pool_do_sess) {
 			CHECK_OBJ_NOTNULL(w->sp, SESS_MAGIC);
+			AZ(w->ws->r);
 
 			stats_clean = 0;
 			w->lastused = NAN;
@@ -270,6 +272,8 @@ Pool_Work_Thread(void *priv, struct worker *w)
 				if (w->vcl != NULL)
 					VCL_Rel(&w->vcl);
 			}
+		} else {
+			WRONG("Invalid w->do_what");
 		}
 		stats_clean = WRK_TrySumStat(w);
 		Lck_Lock(&pp->mtx);
@@ -297,6 +301,7 @@ pool_queue(struct pool *pp, struct sess *sp)
 		VTAILQ_REMOVE(&pp->idle, w, list);
 		Lck_Unlock(&pp->mtx);
 		w->sp = sp;
+		w->do_what = pool_do_sess;
 		AZ(pthread_cond_signal(&w->cond));
 		return (0);
 	}
@@ -469,6 +474,7 @@ pool_herder(void *priv)
 			VSC_C_main->threads_destroyed++;
 			Lck_Unlock(&pool_mtx);
 			AZ(w->sp);
+			w->do_what = pool_do_die;
 			AZ(pthread_cond_signal(&w->cond));
 		}
 	}



From geoff at varnish-cache.org  Mon Jan  9 23:03:28 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Tue, 10 Jan 2012 00:03:28 +0100
Subject: [experimental-ims] 198823d add a ref to man varnishlog
Message-ID: <E1RkOFU-0007q6-IF@project.varnish-software.com>

commit 198823dc4c715589bc03f7b6c45a05ac13db4e3e
Author: Per Buer <perbu at varnish-software.com>
Date:   Mon Dec 19 12:29:11 2011 +0100

    add a ref to man varnishlog

diff --git a/doc/sphinx/tutorial/increasing_your_hitrate.rst b/doc/sphinx/tutorial/increasing_your_hitrate.rst
index c5f0c6a..83236d6 100644
--- a/doc/sphinx/tutorial/increasing_your_hitrate.rst
+++ b/doc/sphinx/tutorial/increasing_your_hitrate.rst
@@ -35,7 +35,8 @@ backend you can use varnishlog to have a look at the request.
 ``varnishlog -c -m 'RxURL:^/foo/bar`` will show you the requests
 coming from the client (-c) matching /foo/bar.
 
-For more information on how varnishlog works please see :ref:`tutorial-logging`.
+For more information on how varnishlog works please see
+:ref:`tutorial-logging` or man :ref:`ref-varnishlog`.
 
 For extended diagnostics headers, see
 http://www.varnish-cache.org/trac/wiki/VCLExampleHitMissHeader



From geoff at varnish-cache.org  Mon Jan  9 23:03:28 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Tue, 10 Jan 2012 00:03:28 +0100
Subject: [experimental-ims] ee43963 Force file permissions 0755 on compiled
	vcl .so file to make sure it is readable by the unprivileged user.
Message-ID: <E1RkOFU-0007qY-PS@project.varnish-software.com>

commit ee439631b413cc5505e384c233ca36930cd33a70
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Mon Dec 19 14:59:12 2011 +0100

    Force file permissions 0755 on compiled vcl .so file to make sure it
    is readable by the unprivileged user.
    
    Fixes: #1072

diff --git a/bin/varnishd/mgt/mgt_vcc.c b/bin/varnishd/mgt/mgt_vcc.c
index ec117db..12433ad 100644
--- a/bin/varnishd/mgt/mgt_vcc.c
+++ b/bin/varnishd/mgt/mgt_vcc.c
@@ -37,6 +37,7 @@
 #include <stdlib.h>
 #include <string.h>
 #include <unistd.h>
+#include <sys/stat.h>
 
 #include "mgt/mgt.h"
 
@@ -272,6 +273,14 @@ mgt_run_cc(const char *vcl, struct vsb *sb, int C_flag)
 	if (!i)
 		i = VSUB_run(sb, run_dlopen, of, "dlopen", 10);
 
+	/* Ensure the file is readable to the unprivileged user */
+	if (!i) {
+		i = chmod(of, 0755);
+		if (i)
+			VSB_printf(sb, "Failed to set permissions on %s: %s",
+				   of, strerror(errno));
+	}
+
 	if (i) {
 		(void)unlink(of);
 		return (NULL);



From geoff at varnish-cache.org  Mon Jan  9 23:03:29 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Tue, 10 Jan 2012 00:03:29 +0100
Subject: [experimental-ims] 203ab6e Fix syntax for 3.0
Message-ID: <E1RkOFV-0007rZ-8T@project.varnish-software.com>

commit 203ab6e7109b5d382e802c747f9b273c5f605bf7
Author: Andreas Plesner Jacobsen <apj at mutt.dk>
Date:   Tue Dec 20 16:18:12 2011 +0100

    Fix syntax for 3.0

diff --git a/doc/sphinx/tutorial/cookies.rst b/doc/sphinx/tutorial/cookies.rst
index ab59e21..0278712 100644
--- a/doc/sphinx/tutorial/cookies.rst
+++ b/doc/sphinx/tutorial/cookies.rst
@@ -50,7 +50,7 @@ cookies named COOKIE1 and COOKIE2 and you can marvel at it::
 
   sub vcl_recv {
     if (req.http.Cookie) {
-      set req.http.Cookie = ";" req.http.Cookie;
+      set req.http.Cookie = ";" + req.http.Cookie;
       set req.http.Cookie = regsuball(req.http.Cookie, "; +", ";");
       set req.http.Cookie = regsuball(req.http.Cookie, ";(COOKIE1|COOKIE2)=", "; \1=");
       set req.http.Cookie = regsuball(req.http.Cookie, ";[^ ][^;]*", "");



From geoff at varnish-cache.org  Mon Jan  9 23:03:29 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Tue, 10 Jan 2012 00:03:29 +0100
Subject: [experimental-ims] 2b6717a typos
Message-ID: <E1RkOFV-0007s5-GJ@project.varnish-software.com>

commit 2b6717a07db47d14fc232de4d9311a821663addf
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Dec 20 20:28:50 2011 +0000

    typos

diff --git a/doc/sphinx/phk/thetoolsweworkwith.rst b/doc/sphinx/phk/thetoolsweworkwith.rst
index cff365b..6921926 100644
--- a/doc/sphinx/phk/thetoolsweworkwith.rst
+++ b/doc/sphinx/phk/thetoolsweworkwith.rst
@@ -65,7 +65,7 @@ text actually means:
 	an uppercase letter or another underscore are always reserved
 	for any use.
 
-Feel free to guess.
+Feel free to guess, there's more such on pdf page 200 of the draft.
 
 Next, they broke the upper/lower rule, by adding special keywords
 in mixed case, probably because they thought it looked nicer::
@@ -103,8 +103,8 @@ aspects.
 
 For instance, neither pthreads nor C1X-threads offer a "assert I'm
 holding this mutex locked" facility.  I will posit that you cannot
-successfully develop real-world threaded programs and APIs that,
-or without wasting a lot of time debugging silly mistakes.
+successfully develop real-world threaded programs and APIs without
+that, or without wasting a lot of time debugging silly mistakes.
 
 If you look in the Varnish source code, which uses pthreads, you
 will see that I have wrapped pthread mutexes in my own little



From geoff at varnish-cache.org  Mon Jan  9 23:03:29 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Tue, 10 Jan 2012 00:03:29 +0100
Subject: [experimental-ims] 676064d Add timeouts and comprehensive test-case
	for memorypool gymnastics.
Message-ID: <E1RkOFV-0007sJ-M1@project.varnish-software.com>

commit 676064df3932eb64b2aaa8e526c2cd7d5a45d226
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Dec 21 11:16:23 2011 +0000

    Add timeouts and comprehensive test-case for memorypool gymnastics.

diff --git a/bin/varnishd/cache/cache_mempool.c b/bin/varnishd/cache/cache_mempool.c
index ac4abf2..92e38d2 100644
--- a/bin/varnishd/cache/cache_mempool.c
+++ b/bin/varnishd/cache/cache_mempool.c
@@ -42,14 +42,17 @@ struct memitem {
 #define MEMITEM_MAGIC			0x42e55401
 	VTAILQ_ENTRY(memitem)		list;
 	unsigned			size;
-	double				payload;
+	double				touched;
+	char				payload;
 };
 
+VTAILQ_HEAD(memhead_s, memitem);
+
 struct mempool {
 	unsigned			magic;
 #define MEMPOOL_MAGIC			0x37a75a8d
-	VTAILQ_HEAD(,memitem)		list;
-	VTAILQ_HEAD(,memitem)		surplus;
+	struct memhead_s		list;
+	struct memhead_s		surplus;
 	struct lock			*mtx;
 	struct lock			imtx;
 	const char			*name;
@@ -58,6 +61,7 @@ struct mempool {
 	struct VSC_C_mempool		*vsc;
 	unsigned			n_pool;
 	pthread_t			thread;
+	double				t_now;
 };
 
 /*---------------------------------------------------------------------
@@ -81,21 +85,26 @@ mpl_alloc(const struct mempool *mpl)
 /*---------------------------------------------------------------------
  * Pool-guard
  *   Attempt to keep number of free items in pool inside bounds with
- *   minimum locking activity.
+ *   minimum locking activity, and keep an eye on items at the tail
+ *   of the list not getting too old.
  */
 
+#include <stdio.h>
+
 static void *
 mpl_guard(void *priv)
 {
 	struct mempool *mpl;
 	struct memitem *mi = NULL;
 	double mpl_slp __state_variable__(mpl_slp);
+	double last = 0;
 
 	CAST_OBJ_NOTNULL(mpl, priv, MEMPOOL_MAGIC);
 	mpl_slp = 0.15;	// random
 	while (1) {
 		VTIM_sleep(mpl_slp);
 		mpl_slp = 0.814;	// random
+		mpl->t_now = VTIM_real();
 
 		if (mi != NULL && (mpl->n_pool > mpl->param->max_pool ||
 		    mi->size < *mpl->cur_size)) {
@@ -106,45 +115,69 @@ mpl_guard(void *priv)
 		if (mi == NULL && mpl->n_pool < mpl->param->min_pool)
 			mi = mpl_alloc(mpl);
 
+
 		if (mpl->n_pool < mpl->param->min_pool && mi != NULL) {
 			/* can do */
 		} else if (mpl->n_pool > mpl->param->max_pool && mi == NULL) {
 			/* can do */
 		} else if (!VTAILQ_EMPTY(&mpl->surplus)) {
 			/* can do */
+		} else if (last + .1 * mpl->param->max_age < mpl->t_now) {
+			/* should do */
 		} else {
-			continue;	/* cannot do */
+			continue;	/* nothing to do */
 		}
 
-		mpl_slp = 0.314;
+		mpl_slp = 0.314;	// random
 
 		if (Lck_Trylock(mpl->mtx))
 			continue;
 
-		mpl_slp = .01;
-
 		if (mpl->n_pool < mpl->param->min_pool &&
 		    mi != NULL && mi->size >= *mpl->cur_size) {
 			CHECK_OBJ_NOTNULL(mi, MEMITEM_MAGIC);
 			mpl->vsc->pool++;
 			mpl->n_pool++;
+			mi->touched = mpl->t_now;
 			VTAILQ_INSERT_HEAD(&mpl->list, mi, list);
 			mi = NULL;
+			mpl_slp = .01;	// random
+
 		}
 		if (mpl->n_pool > mpl->param->max_pool && mi == NULL) {
 			mi = VTAILQ_FIRST(&mpl->list);
 			CHECK_OBJ_NOTNULL(mi, MEMITEM_MAGIC);
 			mpl->vsc->pool--;
+			mpl->vsc->surplus++;
 			mpl->n_pool--;
 			VTAILQ_REMOVE(&mpl->list, mi, list);
+			mpl_slp = .01;	// random
 		}
 		if (mi == NULL) {
 			mi = VTAILQ_FIRST(&mpl->surplus);
 			if (mi != NULL) {
 				CHECK_OBJ_NOTNULL(mi, MEMITEM_MAGIC);
 				VTAILQ_REMOVE(&mpl->surplus, mi, list);
+				mpl_slp = .01;	// random
 			}
 		}
+		if (mi == NULL && mpl->n_pool > mpl->param->min_pool) {
+			mi = VTAILQ_LAST(&mpl->list, memhead_s);
+			CHECK_OBJ_NOTNULL(mi, MEMITEM_MAGIC);
+			if (mi->touched + mpl->param->max_age < mpl->t_now) {
+				mpl->vsc->pool--;
+				mpl->vsc->timeout++;
+				mpl->n_pool--;
+				VTAILQ_REMOVE(&mpl->list, mi, list);
+				mpl_slp = .01;	// random
+			} else {
+				mi = NULL;
+				last = mpl->t_now;
+			}
+		} else if (mpl->n_pool <= mpl->param->min_pool) {
+			last = mpl->t_now;
+		}
+
 		Lck_Unlock(mpl->mtx);
 
 		if (mi != NULL) {
@@ -250,6 +283,7 @@ MPL_Free(struct mempool *mpl, void *item)
 	} else {
 		mpl->vsc->pool++;
 		mpl->n_pool++;
+		mi->touched = mpl->t_now;
 		VTAILQ_INSERT_HEAD(&mpl->list, mi, list);
 	}
 
diff --git a/bin/varnishtest/tests/c00050.vtc b/bin/varnishtest/tests/c00050.vtc
new file mode 100644
index 0000000..340563b
--- /dev/null
+++ b/bin/varnishtest/tests/c00050.vtc
@@ -0,0 +1,29 @@
+varnishtest "Memory pool gymnastics"
+
+server s1 { } -start
+
+varnish v1 -vcl+backend {} -start
+
+delay 1
+
+varnish v1 -expect MEMPOOL.vbc.pool == 10
+
+varnish v1 -cliok "param.set vbc_pool 90,100,100"
+
+delay 1
+
+varnish v1 -expect MEMPOOL.vbc.pool == 90
+
+varnish v1 -cliok "param.set vbc_pool 50,80,100"
+
+delay 1
+
+varnish v1 -expect MEMPOOL.vbc.pool == 80
+varnish v1 -expect MEMPOOL.vbc.surplus == 10
+
+varnish v1 -cliok "param.set vbc_pool 10,80,1"
+
+delay 1
+
+varnish v1 -expect MEMPOOL.vbc.pool == 10
+varnish v1 -expect MEMPOOL.vbc.timeout == 70



From geoff at varnish-cache.org  Mon Jan  9 23:03:29 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Tue, 10 Jan 2012 00:03:29 +0100
Subject: [experimental-ims] 159c511 Further mempool-polishing:
Message-ID: <E1RkOFV-0007sh-V5@project.varnish-software.com>

commit 159c511d774bed9cc1b6774941640a75b4418931
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu Dec 22 19:26:46 2011 +0000

    Further mempool-polishing:
    
    Drop the mtx arg to MPL_New(), it wouldn't help.
    
    Store the pool name locally.
    
    Fix wrong type on some stats counters.
    
    Copy gauge counters from reliable values.
    
    Add stats-counter for dry pool allocations.
    
    Add MPL_Destroy() which tickles the guard thread into self-destroying
    the pool.  Only allowed if no live objects.

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index 46a0d26..d3b34b7 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -871,8 +871,9 @@ int Lck_CondWait(pthread_cond_t *cond, struct lock *lck, struct timespec *ts);
 #undef LOCK
 
 /* cache_mempool.c */
-struct mempool * MPL_New(const char *name, struct lock *mtx,
-    volatile struct poolparam *pp, volatile unsigned *cur_size);
+struct mempool * MPL_New(const char *name, volatile struct poolparam *pp,
+    volatile unsigned *cur_size);
+void MPL_Destroy(struct mempool **mpp);
 void *MPL_Get(struct mempool *mpl, unsigned *size);
 void MPL_Free(struct mempool *mpl, void *item);
 
diff --git a/bin/varnishd/cache/cache_backend.c b/bin/varnishd/cache/cache_backend.c
index 6cfcaf3..48c01a3 100644
--- a/bin/varnishd/cache/cache_backend.c
+++ b/bin/varnishd/cache/cache_backend.c
@@ -518,6 +518,6 @@ void
 VDI_Init(void)
 {
 
-	vbcpool = MPL_New("vbc", NULL, &cache_param->vbc_pool, &vbcps);
+	vbcpool = MPL_New("vbc", &cache_param->vbc_pool, &vbcps);
 	AN(vbcpool);
 }
diff --git a/bin/varnishd/cache/cache_mempool.c b/bin/varnishd/cache/cache_mempool.c
index 92e38d2..a89f1e7 100644
--- a/bin/varnishd/cache/cache_mempool.c
+++ b/bin/varnishd/cache/cache_mempool.c
@@ -51,17 +51,18 @@ VTAILQ_HEAD(memhead_s, memitem);
 struct mempool {
 	unsigned			magic;
 #define MEMPOOL_MAGIC			0x37a75a8d
+	char				name[8];
 	struct memhead_s		list;
 	struct memhead_s		surplus;
-	struct lock			*mtx;
-	struct lock			imtx;
-	const char			*name;
+	struct lock			mtx;
 	volatile struct poolparam	*param;
 	volatile unsigned		*cur_size;
+	uint64_t			live;
 	struct VSC_C_mempool		*vsc;
 	unsigned			n_pool;
 	pthread_t			thread;
 	double				t_now;
+	int				self_destruct;
 };
 
 /*---------------------------------------------------------------------
@@ -124,20 +125,50 @@ mpl_guard(void *priv)
 			/* can do */
 		} else if (last + .1 * mpl->param->max_age < mpl->t_now) {
 			/* should do */
+		} else if (mpl->self_destruct) {
+			/* can do */
 		} else {
 			continue;	/* nothing to do */
 		}
 
 		mpl_slp = 0.314;	// random
 
-		if (Lck_Trylock(mpl->mtx))
+		if (Lck_Trylock(&mpl->mtx))
 			continue;
 
+		if (mpl->self_destruct) {
+			AZ(mpl->live);
+			while (1) {
+				if (mi == NULL) {
+					mi = VTAILQ_FIRST(&mpl->list);
+					if (mi != NULL) {
+						mpl->vsc->pool = --mpl->n_pool;
+						VTAILQ_REMOVE(&mpl->list,
+						    mi, list);
+					}
+				}
+				if (mi == NULL) {
+					mi = VTAILQ_FIRST(&mpl->surplus);
+					if (mi != NULL)
+						VTAILQ_REMOVE(&mpl->surplus,
+						    mi, list);
+				}
+				if (mi == NULL) 
+					break;
+				FREE_OBJ(mi);
+				mi = NULL;
+			}
+			VSM_Free(mpl->vsc);
+			Lck_Unlock(&mpl->mtx);
+			Lck_Delete(&mpl->mtx);
+			FREE_OBJ(mpl);
+			break;
+		}
+
 		if (mpl->n_pool < mpl->param->min_pool &&
 		    mi != NULL && mi->size >= *mpl->cur_size) {
 			CHECK_OBJ_NOTNULL(mi, MEMITEM_MAGIC);
-			mpl->vsc->pool++;
-			mpl->n_pool++;
+			mpl->vsc->pool = ++mpl->n_pool;
 			mi->touched = mpl->t_now;
 			VTAILQ_INSERT_HEAD(&mpl->list, mi, list);
 			mi = NULL;
@@ -147,9 +178,8 @@ mpl_guard(void *priv)
 		if (mpl->n_pool > mpl->param->max_pool && mi == NULL) {
 			mi = VTAILQ_FIRST(&mpl->list);
 			CHECK_OBJ_NOTNULL(mi, MEMITEM_MAGIC);
-			mpl->vsc->pool--;
+			mpl->vsc->pool = --mpl->n_pool;
 			mpl->vsc->surplus++;
-			mpl->n_pool--;
 			VTAILQ_REMOVE(&mpl->list, mi, list);
 			mpl_slp = .01;	// random
 		}
@@ -165,9 +195,8 @@ mpl_guard(void *priv)
 			mi = VTAILQ_LAST(&mpl->list, memhead_s);
 			CHECK_OBJ_NOTNULL(mi, MEMITEM_MAGIC);
 			if (mi->touched + mpl->param->max_age < mpl->t_now) {
-				mpl->vsc->pool--;
+				mpl->vsc->pool = --mpl->n_pool;
 				mpl->vsc->timeout++;
-				mpl->n_pool--;
 				VTAILQ_REMOVE(&mpl->list, mi, list);
 				mpl_slp = .01;	// random
 			} else {
@@ -178,22 +207,22 @@ mpl_guard(void *priv)
 			last = mpl->t_now;
 		}
 
-		Lck_Unlock(mpl->mtx);
+		Lck_Unlock(&mpl->mtx);
 
 		if (mi != NULL) {
 			FREE_OBJ(mi);
 			mi = NULL;
 		}
 	}
-	NEEDLESS_RETURN(NULL);
+	return (NULL);
 }
 
 /*---------------------------------------------------------------------
+ * Create a new memory pool, and start the guard thread for it.
  */
 
 struct mempool *
 MPL_New(const char *name,
-	struct lock *mtx,
 	volatile struct poolparam *pp,
 	volatile unsigned *cur_size)
 {
@@ -201,23 +230,44 @@ MPL_New(const char *name,
 
 	ALLOC_OBJ(mpl, MEMPOOL_MAGIC);
 	AN(mpl);
-	mpl->name = name;
+	bprintf(mpl->name, "%s", name);
 	mpl->param = pp;
 	mpl->cur_size = cur_size;
-	mpl->mtx = mtx;
 	VTAILQ_INIT(&mpl->list);
 	VTAILQ_INIT(&mpl->surplus);
-	Lck_New(&mpl->imtx, lck_mempool);
-	if (mpl->mtx == NULL)
-		mpl->mtx = &mpl->imtx;
+	Lck_New(&mpl->mtx, lck_mempool);
 	/* XXX: prealloc min_pool */
 	mpl->vsc = VSM_Alloc(sizeof *mpl->vsc,
-	    VSC_CLASS, VSC_TYPE_MEMPOOL, name);
+	    VSC_CLASS, VSC_TYPE_MEMPOOL, mpl->name);
 	AN(mpl->vsc);
 	AZ(pthread_create(&mpl->thread, NULL, mpl_guard, mpl));
+	AZ(pthread_detach(mpl->thread));
 	return (mpl);
 }
 
+/*---------------------------------------------------------------------
+ * Destroy a memory pool.  There must be no live items, and we cheat
+ * and leave all the hard work to the guard thread.
+ */
+
+void
+MPL_Destroy(struct mempool **mpp)
+{
+	struct mempool *mpl;
+
+	AN(mpp);
+	mpl = *mpp;
+	*mpp = NULL;
+	CHECK_OBJ_NOTNULL(mpl, MEMPOOL_MAGIC);
+	Lck_Lock(&mpl->mtx);
+	AZ(mpl->live);
+	mpl->self_destruct = 1;
+	Lck_Unlock(&mpl->mtx);
+}
+
+/*---------------------------------------------------------------------
+ */
+
 void *
 MPL_Get(struct mempool *mpl, unsigned *size)
 {
@@ -225,17 +275,18 @@ MPL_Get(struct mempool *mpl, unsigned *size)
 
 	CHECK_OBJ_NOTNULL(mpl, MEMPOOL_MAGIC);
 
-	Lck_Lock(mpl->mtx);
+	Lck_Lock(&mpl->mtx);
 
 	mpl->vsc->allocs++;
-	mpl->vsc->live++;
+	mpl->vsc->live = ++mpl->live;
 
 	do {
 		mi = VTAILQ_FIRST(&mpl->list);
-		if (mi == NULL)
+		if (mi == NULL) {
+			mpl->vsc->randry++;
 			break;
-		mpl->vsc->pool--;
-		mpl->n_pool--;
+		}
+		mpl->vsc->pool = --mpl->n_pool;
 		CHECK_OBJ_NOTNULL(mi, MEMITEM_MAGIC);
 		VTAILQ_REMOVE(&mpl->list, mi, list);
 		if (mi->size < *mpl->cur_size) {
@@ -247,7 +298,7 @@ MPL_Get(struct mempool *mpl, unsigned *size)
 		}
 	} while (mi == NULL);
 
-	Lck_Unlock(mpl->mtx);
+	Lck_Unlock(&mpl->mtx);
 
 	if (mi == NULL)
 		mi = mpl_alloc(mpl);
@@ -269,10 +320,10 @@ MPL_Free(struct mempool *mpl, void *item)
 	CHECK_OBJ_NOTNULL(mi, MEMITEM_MAGIC);
 	memset(item, 0, mi->size);
 
-	Lck_Lock(mpl->mtx);
+	Lck_Lock(&mpl->mtx);
 
 	mpl->vsc->frees++;
-	mpl->vsc->live--;
+	mpl->vsc->live = --mpl->live;
 
 	if (mi->size < *mpl->cur_size) {
 		mpl->vsc->toosmall++;
@@ -281,11 +332,10 @@ MPL_Free(struct mempool *mpl, void *item)
 		mpl->vsc->surplus++;
 		VTAILQ_INSERT_HEAD(&mpl->surplus, mi, list);
 	} else {
-		mpl->vsc->pool++;
-		mpl->n_pool++;
+		mpl->vsc->pool = ++mpl->n_pool;
 		mi->touched = mpl->t_now;
 		VTAILQ_INSERT_HEAD(&mpl->list, mi, list);
 	}
 
-	Lck_Unlock(mpl->mtx);
+	Lck_Unlock(&mpl->mtx);
 }
diff --git a/include/tbl/vsc_fields.h b/include/tbl/vsc_fields.h
index 2671558..3d1e30a 100644
--- a/include/tbl/vsc_fields.h
+++ b/include/tbl/vsc_fields.h
@@ -109,13 +109,14 @@ VSC_F(happy,		uint64_t, 0, 'b', "Happy health probes", "")
 /**********************************************************************/
 #ifdef VSC_DO_MEMPOOL
 
-VSC_F(allocs,			uint64_t, 0, 'c', "Allocations", "")
-VSC_F(frees,			uint64_t, 0, 'c', "Frees", "")
 VSC_F(live,			uint64_t, 0, 'g', "In use", "")
 VSC_F(pool,			uint64_t, 0, 'g', "In Pool", "")
-VSC_F(recycle,			uint64_t, 0, 'g', "Recycled from pool", "")
-VSC_F(timeout,			uint64_t, 0, 'g', "Timed out from pool", "")
-VSC_F(toosmall,			uint64_t, 0, 'g', "Too small to recycle", "")
-VSC_F(surplus,			uint64_t, 0, 'g', "Too many for pool", "")
+VSC_F(allocs,			uint64_t, 0, 'c', "Allocations", "")
+VSC_F(frees,			uint64_t, 0, 'c', "Frees", "")
+VSC_F(recycle,			uint64_t, 0, 'c', "Recycled from pool", "")
+VSC_F(timeout,			uint64_t, 0, 'c', "Timed out from pool", "")
+VSC_F(toosmall,			uint64_t, 0, 'c', "Too small to recycle", "")
+VSC_F(surplus,			uint64_t, 0, 'c', "Too many for pool", "")
+VSC_F(randry,			uint64_t, 0, 'c', "Pool ran dry", "")
 
 #endif



From geoff at varnish-cache.org  Mon Jan  9 23:03:30 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Tue, 10 Jan 2012 00:03:30 +0100
Subject: [experimental-ims] a5708d5 Minor reorg of kqueue waiter for improved
	clarity
Message-ID: <E1RkOFW-0007t2-54@project.varnish-software.com>

commit a5708d520d9e452880aa5a58f8c1065c8061e489
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu Dec 22 21:07:24 2011 +0000

    Minor reorg of kqueue waiter for improved clarity

diff --git a/bin/varnishd/waiter/cache_waiter_kqueue.c b/bin/varnishd/waiter/cache_waiter_kqueue.c
index e9ec1ec..7dcf7f5 100644
--- a/bin/varnishd/waiter/cache_waiter_kqueue.c
+++ b/bin/varnishd/waiter/cache_waiter_kqueue.c
@@ -86,30 +86,41 @@ vwk_kq_sess(struct vwk *vwk, struct sess *sp, short arm)
 		vwk_kq_flush(vwk);
 }
 
+/*--------------------------------------------------------------------*/
+
 static void
-vwk_kev(struct vwk *vwk, const struct kevent *kp)
+vwk_pipe_ev(struct vwk *vwk, const struct kevent *kp)
 {
 	int i, j;
-	struct sess *sp;
 	struct sess *ss[NKEV];
 
 	AN(kp->udata);
-	if (kp->udata == vwk->pipes) {
-		j = 0;
-		i = read(vwk->pipes[0], ss, sizeof ss);
-		if (i == -1 && errno == EAGAIN)
-			return;
-		while (i >= sizeof ss[0]) {
-			CHECK_OBJ_NOTNULL(ss[j], SESS_MAGIC);
-			assert(ss[j]->fd >= 0);
-			VTAILQ_INSERT_TAIL(&vwk->sesshead, ss[j], list);
-			vwk_kq_sess(vwk, ss[j], EV_ADD | EV_ONESHOT);
-			j++;
-			i -= sizeof ss[0];
-		}
-		assert(i == 0);
+	assert(kp->udata == vwk->pipes);
+	j = 0;
+	i = read(vwk->pipes[0], ss, sizeof ss);
+	if (i == -1 && errno == EAGAIN)
 		return;
+	while (i >= sizeof ss[0]) {
+		CHECK_OBJ_NOTNULL(ss[j], SESS_MAGIC);
+		assert(ss[j]->fd >= 0);
+		VTAILQ_INSERT_TAIL(&vwk->sesshead, ss[j], list);
+		vwk_kq_sess(vwk, ss[j], EV_ADD | EV_ONESHOT);
+		j++;
+		i -= sizeof ss[0];
 	}
+	assert(i == 0);
+}
+
+/*--------------------------------------------------------------------*/
+
+static void
+vwk_sess_ev(struct vwk *vwk, const struct kevent *kp)
+{
+	int i;
+	struct sess *sp;
+
+	AN(kp->udata);
+	assert(kp->udata != vwk->pipes);
 	CAST_OBJ_NOTNULL(sp, kp->udata, SESS_MAGIC);
 	DSL(0x04, SLT_Debug, sp->vsl_id, "KQ: sp %p kev data %lu flags 0x%x%s",
 	    sp, (unsigned long)kp->data, kp->flags,
@@ -171,10 +182,13 @@ vwk_thread(void *priv)
 		for (kp = ke, j = 0; j < n; j++, kp++) {
 			if (kp->filter == EVFILT_TIMER) {
 				dotimer = 1;
-				continue;
+			} else if (kp->filter == EVFILT_READ &&
+			    kp->udata == vwk->pipes) {
+				vwk_pipe_ev(vwk, kp);
+			} else {
+				assert(kp->filter == EVFILT_READ);
+				vwk_sess_ev(vwk, kp);
 			}
-			assert(kp->filter == EVFILT_READ);
-			vwk_kev(vwk, kp);
 		}
 		if (!dotimer)
 			continue;



From geoff at varnish-cache.org  Mon Jan  9 23:03:30 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Tue, 10 Jan 2012 00:03:30 +0100
Subject: [experimental-ims] a87ec7a Shift some code to more appropriate
	location
Message-ID: <E1RkOFW-0007tI-Bl@project.varnish-software.com>

commit a87ec7adb127158fc70635ab3fb61ddc9180e426
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu Dec 22 21:51:36 2011 +0000

    Shift some code to more appropriate location

diff --git a/bin/varnishd/cache/cache_pool.c b/bin/varnishd/cache/cache_pool.c
index ee71572..1a67548 100644
--- a/bin/varnishd/cache/cache_pool.c
+++ b/bin/varnishd/cache/cache_pool.c
@@ -331,28 +331,10 @@ pool_queue(struct pool *pp, struct sess *sp)
 int
 Pool_Schedule(struct pool *pp, struct sess *sp)
 {
+
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 	AZ(sp->wrk);
-	if (pool_queue(pp, sp) == 0)
-		return(0);
-
-	VSC_C_main->client_drop_late++;
-
-	/*
-	 * Couldn't queue it -- kill it.
-	 *
-	 * XXX: a notice might be polite, but would potentially
-	 * XXX: sleep whichever thread got us here
-	 */
-	sp->t_end = VTIM_real();
-	if (sp->vcl != NULL) {
-		/*
-		 * A session parked on a busy object can come here
-		 * after it wakes up.  Loose the VCL reference.
-		 */
-		VCL_Rel(&sp->vcl);
-	}
-	return (1);
+	return(pool_queue(pp, sp));
 }
 
 /*--------------------------------------------------------------------
diff --git a/bin/varnishd/cache/cache_session.c b/bin/varnishd/cache/cache_session.c
index 9c5e82b..1b1f87c 100644
--- a/bin/varnishd/cache/cache_session.c
+++ b/bin/varnishd/cache/cache_session.c
@@ -42,6 +42,7 @@
 #include "cache.h"
 
 #include "waiter/waiter.h"
+#include "vtim.h"
 
 /*--------------------------------------------------------------------*/
 
@@ -266,6 +267,15 @@ SES_Schedule(struct sess *sp)
 	AN(pp->pool);
 
 	if (Pool_Schedule(pp->pool, sp)) {
+		VSC_C_main->client_drop_late++;
+		sp->t_end = VTIM_real();
+		if (sp->vcl != NULL) {
+			/*
+			 * A session parked on a busy object can come here
+			 * after it wakes up.  Loose the VCL reference.
+			 */
+			VCL_Rel(&sp->vcl);
+		}
 		SES_Delete(sp, "dropped");
 		return (1);
 	}



From geoff at varnish-cache.org  Mon Jan  9 23:03:30 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Tue, 10 Jan 2012 00:03:30 +0100
Subject: [experimental-ims] a9852b4 Rework the code that receives the client
	request.
Message-ID: <E1RkOFW-0007tc-Ht@project.varnish-software.com>

commit a9852b4c420afea2bf7191d1b5cb47dc64b92399
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu Dec 22 23:51:16 2011 +0000

    Rework the code that receives the client request.
    
    Collect all the code in cnt_wait()
    
    Add a new -3 return code from HTC which says that all we've seen so
    far is white space.  Use the RFC2616 "LWS" definition of white-space.
    
    Rename "sess_timeout" parameter to "timeout_idle" and describe it more
    precisely.  Make it a double, so more precise values can be specified.
    The sessions is considered idle even if we receive white-space.
    
    Rename "session_linger" parameter to "timeout_linger", make it a double
    and give it units of seconds like other timeouts.
    
    Add new parameter "timeout_req" which controls how long time we give
    the client to send the request headers, from the first non-white char.
    Set it to two seconds.  Experimentation/Experience requested.
    
    "timeout_req" also gives us positive control of any kind of
    "slowlaris" attack scenarios.
    
    I'm trying to group parameters more sensibly, since we sort them
    by name, calling them all timeout_*, pool_* etc, makes some sense I hope.

diff --git a/bin/varnishd/cache/cache_acceptor.c b/bin/varnishd/cache/cache_acceptor.c
index 1c6d6fb..1f9616a 100644
--- a/bin/varnishd/cache/cache_acceptor.c
+++ b/bin/varnishd/cache/cache_acceptor.c
@@ -276,6 +276,7 @@ VCA_SetupSess(struct worker *w)
 	sp->vsl_id = wa->acceptsock | VSL_CLIENTMARKER ;
 	wa->acceptsock = -1;
 	sp->t_open = VTIM_real();
+	sp->t_req = sp->t_open;
 	sp->t_idle = sp->t_open;
 	sp->mylsock = wa->acceptlsock;
 	CHECK_OBJ_NOTNULL(sp->mylsock, LISTEN_SOCK_MAGIC);
@@ -293,7 +294,7 @@ static void *
 vca_acct(void *arg)
 {
 #ifdef SO_RCVTIMEO_WORKS
-	double sess_timeout = 0;
+	double timeout_idle = 0;
 #endif
 #ifdef SO_SNDTIMEO_WORKS
 	double send_timeout = 0;
@@ -333,10 +334,10 @@ vca_acct(void *arg)
 		}
 #endif
 #ifdef SO_RCVTIMEO_WORKS
-		if (cache_param->sess_timeout != sess_timeout) {
+		if (cache_param->timeout_idle != timeout_idle) {
 			need_test = 1;
-			sess_timeout = cache_param->sess_timeout;
-			tv_rcvtimeo = VTIM_timeval(sess_timeout);
+			timeout_idle = cache_param->timeout_idle;
+			tv_rcvtimeo = VTIM_timeval(timeout_idle);
 			VTAILQ_FOREACH(ls, &heritage.socks, list) {
 				if (ls->sock < 0)
 					continue;
diff --git a/bin/varnishd/cache/cache_center.c b/bin/varnishd/cache/cache_center.c
index f926932..b14ec24 100644
--- a/bin/varnishd/cache/cache_center.c
+++ b/bin/varnishd/cache/cache_center.c
@@ -98,9 +98,10 @@ DOT }
 static int
 cnt_wait(struct sess *sp)
 {
-	int i;
+	int i, j, tmo;
 	struct pollfd pfd[1];
 	struct worker *wrk;
+	double now, when;
 
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 	wrk = sp->wrk;
@@ -110,33 +111,59 @@ cnt_wait(struct sess *sp)
 	AZ(sp->esi_level);
 	assert(sp->xid == 0);
 
-	i = HTC_Complete(sp->htc);
-	if (i == 0 && cache_param->session_linger > 0) {
+	assert(!isnan(sp->t_req));
+	tmo = (int)(1e3 * cache_param->timeout_linger);
+	while (1) {
 		pfd[0].fd = sp->fd;
 		pfd[0].events = POLLIN;
 		pfd[0].revents = 0;
-		i = poll(pfd, 1, cache_param->session_linger);
-		if (i)
+		j = poll(pfd, 1, tmo);
+		assert(j >= 0);
+		if (j != 0)
 			i = HTC_Rx(sp->htc);
+		else
+			i = HTC_Complete(sp->htc);
+		if (i == -1) {
+			SES_Close(sp, "EOF");
+			break;
+		}
+		if (i == -2) {
+			SES_Close(sp, "overflow");
+			break;
+		}
+		now = VTIM_real();
+		if (i == 1) {
+			/* Got it, run with it */
+			sp->t_req = now;
+			sp->step = STP_START;
+			return (0);
+		}
+		if (i == -3) {
+			/* Nothing but whitespace */
+			when = sp->t_idle + cache_param->timeout_idle;
+			if (when < now) {
+				SES_Close(sp, "timeout");
+				break;
+			}
+			when = sp->t_idle + cache_param->timeout_linger;
+			tmo = (int)(1e3 * (when - now));
+			if (when < now || tmo == 0) {
+				sp->t_req = NAN;
+				wrk->stats.sess_herd++;
+				SES_Charge(sp);
+				WAIT_Enter(sp);
+				return (1);
+			}
+		} else {
+			/* Working on it */
+			when = sp->t_req + cache_param->timeout_req;
+			tmo = (int)(1e3 * (when - now));
+			if (when < now || tmo == 0) {
+				SES_Close(sp, "req timeout");
+				break;
+			}
+		}
 	}
-	if (i == 0) {
-		WSP(sp, SLT_Debug, "herding");
-		wrk->stats.sess_herd++;
-		SES_Charge(sp);
-		WAIT_Enter(sp);
-		return (1);
-	}
-	if (i == 1) {
-		sp->step = STP_START;
-		return (0);
-	}
-	if (i == -2) {
-		SES_Close(sp, "overflow");
-	} else if (i == -1 && Tlen(sp->htc->rxbuf) == 0 &&
-	    (errno == 0 || errno == ECONNRESET))
-		SES_Close(sp, "EOF");
-	else
-		SES_Close(sp, "error");
 	sp->step = STP_DONE;
 	return (0);
 }
@@ -366,8 +393,6 @@ cnt_done(struct sess *sp)
 
 
 	sp->t_idle = W_TIM_real(wrk);
-WSP(sp, SLT_Debug, "PHK req %.9f resp %.9f end %.9f open %.9f",
-    sp->t_req, sp->t_resp, sp->t_idle,  sp->t_open);
 	if (sp->xid == 0) {
 		sp->t_resp = sp->t_idle;
 	} else {
@@ -418,17 +443,20 @@ WSP(sp, SLT_Debug, "PHK req %.9f resp %.9f end %.9f open %.9f",
 	i = HTC_Reinit(sp->htc);
 	if (i == 1) {
 		wrk->stats.sess_pipeline++;
+		sp->t_req = sp->t_idle;
 		sp->step = STP_START;
 		return (0);
 	}
 	if (Tlen(sp->htc->rxbuf)) {
 		wrk->stats.sess_readahead++;
 		sp->step = STP_WAIT;
+		sp->t_req = sp->t_idle;
 		return (0);
 	}
-	if (cache_param->session_linger > 0) {
+	if (cache_param->timeout_linger > 0.) {
 		wrk->stats.sess_linger++;
 		sp->step = STP_WAIT;
+		sp->t_req = sp->t_idle;		// XXX: not quite correct
 		return (0);
 	}
 	wrk->stats.sess_herd++;
@@ -1574,7 +1602,7 @@ cnt_start(struct sess *sp)
 
 	/* Update stats of various sorts */
 	wrk->stats.client_req++;
-	sp->t_req = W_TIM_real(wrk);
+	assert(!isnan(sp->t_req));
 	wrk->acct_tmp.req++;
 
 	/* Assign XID and log */
diff --git a/bin/varnishd/cache/cache_httpconn.c b/bin/varnishd/cache/cache_httpconn.c
index 9e0a052..1abd2a4 100644
--- a/bin/varnishd/cache/cache_httpconn.c
+++ b/bin/varnishd/cache/cache_httpconn.c
@@ -50,6 +50,7 @@
  * Check if we have a complete HTTP request or response yet
  *
  * Return values:
+ *	-3  All whitespace so far
  *	 0  No, keep trying
  *	>0  Yes, it is this many bytes long.
  */
@@ -62,13 +63,13 @@ htc_header_complete(txt *t)
 	Tcheck(*t);
 	assert(*t->e == '\0');
 	/* Skip any leading white space */
-	for (p = t->b ; vct_issp(*p); p++)
+	for (p = t->b ; vct_islws(*p); p++)
 		continue;
 	if (p == t->e) {
 		/* All white space */
 		t->e = t->b;
 		*t->e = '\0';
-		return (0);
+		return (-3);
 	}
 	while (1) {
 		p = strchr(p, '\n');
@@ -133,6 +134,8 @@ HTC_Reinit(struct http_conn *htc)
 }
 
 /*--------------------------------------------------------------------
+ * Return -3 if it's all whitespace so far
+ * Return 0 if we need more text
  * Return 1 if we have a complete HTTP procol header
  */
 
@@ -143,9 +146,8 @@ HTC_Complete(struct http_conn *htc)
 
 	CHECK_OBJ_NOTNULL(htc, HTTP_CONN_MAGIC);
 	i = htc_header_complete(&htc->rxbuf);
-	assert(i >= 0);
-	if (i == 0)
-		return (0);
+	if (i <= 0)
+		return (i);
 	WS_ReleaseP(htc->ws, htc->rxbuf.e);
 	AZ(htc->pipeline.b);
 	AZ(htc->pipeline.e);
@@ -160,8 +162,9 @@ HTC_Complete(struct http_conn *htc)
 /*--------------------------------------------------------------------
  * Receive more HTTP protocol bytes
  * Returns:
+ *	-3 all whitespace so far
  *	-2 overflow
- *	-1 error
+ *	-1 error/EOF
  *	 0 more needed
  *	 1 got complete HTTP header
  */
diff --git a/bin/varnishd/common/params.h b/bin/varnishd/common/params.h
index a024452..0a476d4 100644
--- a/bin/varnishd/common/params.h
+++ b/bin/varnishd/common/params.h
@@ -85,8 +85,9 @@ struct params {
 
 	unsigned		shm_reclen;
 
-	/* Acceptor hints */
-	unsigned		sess_timeout;
+	double			timeout_linger;
+	double			timeout_idle;
+	double			timeout_req;
 	unsigned		pipe_timeout;
 	unsigned		send_timeout;
 	unsigned		idle_send_timeout;
@@ -140,9 +141,6 @@ struct params {
 	double			first_byte_timeout;
 	double			between_bytes_timeout;
 
-	/* How long to linger on sessions */
-	unsigned		session_linger;
-
 	/* CLI buffer size */
 	unsigned		cli_buffer;
 
diff --git a/bin/varnishd/mgt/mgt_param.c b/bin/varnishd/mgt/mgt_param.c
index c236323..2f7bd53 100644
--- a/bin/varnishd/mgt/mgt_param.c
+++ b/bin/varnishd/mgt/mgt_param.c
@@ -773,12 +773,19 @@ static const struct parspec input_parspec[] = {
 		"cache at the end of ttl+grace+keep.",
 		DELAYED_EFFECT,
 		"0", "seconds" },
-	{ "sess_timeout", tweak_timeout, &mgt_param.sess_timeout, 0, 0,
-		"Idle timeout for persistent sessions. "
-		"If a HTTP request has not been received in this many "
-		"seconds, the session is closed.",
+	{ "timeout_idle", tweak_timeout_double, &mgt_param.timeout_idle,
+		0, UINT_MAX,
+		"Idle timeout for client connections.\n"
+		"A connection is considered idle, until we receive"
+		" a non-white-space character on it.",
 		0,
 		"5", "seconds" },
+	{ "timeout_req", tweak_timeout_double, &mgt_param.timeout_req,
+		0, UINT_MAX,
+		"Max time to receive clients request header, measured"
+		" from first non-white-space character to double CRNL.",
+		0,
+		"2", "seconds" },
 	{ "expiry_sleep", tweak_timeout_double, &mgt_param.expiry_sleep, 0, 60,
 		"How long the expiry thread sleeps when there is nothing "
 		"for it to do.\n",
@@ -993,18 +1000,18 @@ static const struct parspec input_parspec[] = {
 		"it.\n",
 		0,
 		"100000", "sessions" },
-	{ "session_linger", tweak_uint,
-		&mgt_param.session_linger,0, UINT_MAX,
-		"How long time the workerthread lingers on the session "
-		"to see if a new request appears right away.\n"
-		"If sessions are reused, as much as half of all reuses "
+	{ "timeout_linger", tweak_timeout_double, &mgt_param.timeout_linger,
+		0, UINT_MAX,
+		"How long time the workerthread lingers on an idle session "
+		"before handing it over to the waiter.\n"
+		"When sessions are reused, as much as half of all reuses "
 		"happen within the first 100 msec of the previous request "
 		"completing.\n"
 		"Setting this too high results in worker threads not doing "
 		"anything for their keep, setting it too low just means that "
 		"more sessions take a detour around the waiter.",
 		EXPERIMENTAL,
-		"50", "ms" },
+		"0.050", "seconds" },
 	{ "log_hashstring", tweak_bool, &mgt_param.log_hash, 0, 0,
 		"Log the hash string components to shared memory log.\n",
 		0,
diff --git a/bin/varnishd/waiter/cache_waiter_epoll.c b/bin/varnishd/waiter/cache_waiter_epoll.c
index e8f5e2a..6ad5be5 100644
--- a/bin/varnishd/waiter/cache_waiter_epoll.c
+++ b/bin/varnishd/waiter/cache_waiter_epoll.c
@@ -183,7 +183,7 @@ vwe_thread(void *priv)
 			continue;
 
 		/* check for timeouts */
-		deadline = now - cache_param->sess_timeout;
+		deadline = now - cache_param->timeout_idle;
 		for (;;) {
 			sp = VTAILQ_FIRST(&vwe->sesshead);
 			if (sp == NULL)
@@ -201,13 +201,13 @@ vwe_thread(void *priv)
 /*--------------------------------------------------------------------*/
 
 static void *
-vwe_sess_timeout_ticker(void *priv)
+vwe_timeout_idle_ticker(void *priv)
 {
 	char ticker = 'R';
 	struct vwe *vwe;
 
 	CAST_OBJ_NOTNULL(vwe, priv, VWE_MAGIC);
-	THR_SetName("cache-epoll-sess_timeout_ticker");
+	THR_SetName("cache-epoll-timeout_idle_ticker");
 
 	while (1) {
 		/* ticking */
@@ -255,7 +255,7 @@ vwe_init(void)
 	assert(i != -1);
 
 	AZ(pthread_create(&vwe->timer_thread,
-	    NULL, vwe_sess_timeout_ticker, vwe));
+	    NULL, vwe_timeout_idle_ticker, vwe));
 	AZ(pthread_create(&vwe->epoll_thread, NULL, vwe_thread, vwe));
 	return(vwe);
 }
diff --git a/bin/varnishd/waiter/cache_waiter_kqueue.c b/bin/varnishd/waiter/cache_waiter_kqueue.c
index 707c902..423bf26 100644
--- a/bin/varnishd/waiter/cache_waiter_kqueue.c
+++ b/bin/varnishd/waiter/cache_waiter_kqueue.c
@@ -195,7 +195,7 @@ vwk_thread(void *priv)
 		 * would not know we meant "the old fd of this number".
 		 */
 		vwk_kq_flush(vwk);
-		deadline = now - cache_param->sess_timeout;
+		deadline = now - cache_param->timeout_idle;
 		for (;;) {
 			sp = VTAILQ_FIRST(&vwk->sesshead);
 			if (sp == NULL)
diff --git a/bin/varnishd/waiter/cache_waiter_poll.c b/bin/varnishd/waiter/cache_waiter_poll.c
index 3a39c6b..a31bec1 100644
--- a/bin/varnishd/waiter/cache_waiter_poll.c
+++ b/bin/varnishd/waiter/cache_waiter_poll.c
@@ -141,7 +141,7 @@ vwp_main(void *priv)
 		v = poll(vwp->pollfd, vwp->hpoll + 1, 100);
 		assert(v >= 0);
 		now = VTIM_real();
-		deadline = now - cache_param->sess_timeout;
+		deadline = now - cache_param->timeout_idle;
 		v2 = v;
 		VTAILQ_FOREACH_SAFE(sp, &vwp->sesshead, list, sp2) {
 			if (v != 0 && v2 == 0)
diff --git a/bin/varnishd/waiter/cache_waiter_ports.c b/bin/varnishd/waiter/cache_waiter_ports.c
index 0d05775..4f39ca4 100644
--- a/bin/varnishd/waiter/cache_waiter_ports.c
+++ b/bin/varnishd/waiter/cache_waiter_ports.c
@@ -191,7 +191,7 @@ vws_thread(void *priv)
 			vws_port_ev(vws, ev + ei, now);
 
 		/* check for timeouts */
-		deadline = now - cache_param->sess_timeout;
+		deadline = now - cache_param->timeout_idle;
 
 		/*
 		 * This loop assumes that the oldest sessions are always at the
@@ -220,7 +220,7 @@ vws_thread(void *priv)
 
 		if (sp) {
 			double tmo =
-			    (sp->t_open + cache_param->sess_timeout) - now;
+			    (sp->t_open + cache_param->timeout_idle) - now;
 
 			/*
 			 * we should have removed all sps whose timeout
diff --git a/bin/varnishtest/tests/r00262.vtc b/bin/varnishtest/tests/r00262.vtc
index 1ef7255..b3c1cf9 100644
--- a/bin/varnishtest/tests/r00262.vtc
+++ b/bin/varnishtest/tests/r00262.vtc
@@ -7,7 +7,7 @@ server s1 {
 		-body "012345\n"
 } -start
 
-varnish v1 -arg "-p session_linger=20" -vcl+backend { } -start 
+varnish v1 -arg "-p timeout_linger=20" -vcl+backend { } -start 
 
 client c1 {
 	send "GET / HTTP/1.1\r\n\r\n\r\n"
diff --git a/bin/varnishtest/tests/r00524.vtc b/bin/varnishtest/tests/r00524.vtc
index 157eebe..cd43e97 100644
--- a/bin/varnishtest/tests/r00524.vtc
+++ b/bin/varnishtest/tests/r00524.vtc
@@ -21,7 +21,7 @@ varnish v1 -vcl+backend {
 	sub vcl_fetch {
 		set beresp.do_esi = true;
 	}
-} -cliok "param.set sess_timeout 60" -start
+} -cliok "param.set timeout_idle 60" -start
 
 client c1 {
 	txreq -proto HTTP/1.0 -hdr "Connection: kEep-alive"
diff --git a/bin/varnishtest/tests/r00641.vtc b/bin/varnishtest/tests/r00641.vtc
index ef825d7..da54121 100644
--- a/bin/varnishtest/tests/r00641.vtc
+++ b/bin/varnishtest/tests/r00641.vtc
@@ -18,7 +18,7 @@ varnish v1 -vcl+backend {
 	sub vcl_fetch {
 		set beresp.do_esi = true;
 	}
-} -cliok "param.set sess_timeout 60" -start
+} -cliok "param.set timeout_idle 60" -start
 
 client c1 {
 	txreq -proto HTTP/1.1



From geoff at varnish-cache.org  Mon Jan  9 23:03:30 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Tue, 10 Jan 2012 00:03:30 +0100
Subject: [experimental-ims] 2c90d78 Remove redundant field
Message-ID: <E1RkOFW-0007tk-Jr@project.varnish-software.com>

commit 2c90d78747a527fc8d55893d4d8eed68fe1120be
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Fri Dec 23 07:21:21 2011 +0000

    Remove redundant field

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index 7e7088e..b674bd7 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -198,7 +198,6 @@ struct http_conn {
 /*--------------------------------------------------------------------*/
 
 struct acct {
-	double			first;
 #define ACCT(foo)	uint64_t	foo;
 #include "tbl/acct_fields.h"
 #undef ACCT
diff --git a/bin/varnishd/cache/cache_center.c b/bin/varnishd/cache/cache_center.c
index b14ec24..052a68b 100644
--- a/bin/varnishd/cache/cache_center.c
+++ b/bin/varnishd/cache/cache_center.c
@@ -1019,7 +1019,8 @@ cnt_streambody(struct sess *sp)
 }
 
 /*--------------------------------------------------------------------
- * The very first request
+ * A freshly accepted socket
+ *
 DOT subgraph xcluster_first {
 DOT	first [
 DOT		shape=box
@@ -1040,16 +1041,10 @@ cnt_first(struct sess *sp)
 	wrk = sp->wrk;
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 
-	/*
-	 * XXX: If we don't have acceptfilters we are somewhat subject
-	 * XXX: to DoS'ing here.  One remedy would be to set a shorter
-	 * XXX: SO_RCVTIMEO and once we have received something here
-	 * XXX: increase it to the normal value.
-	 */
-
 	assert(sp->xid == 0);
 	assert(sp->restarts == 0);
 	AZ(sp->esi_level);
+
 	VTCP_name(&sp->sockaddr, sp->sockaddrlen,
 	    sp->addr, sizeof sp->addr, sp->port, sizeof sp->port);
 	if (cache_param->log_local_addr) {
@@ -1063,7 +1058,6 @@ cnt_first(struct sess *sp)
 		WSP(sp, SLT_SessionOpen, "%s %s %s",
 		    sp->addr, sp->port, sp->mylsock->name);
 	}
-	sp->acct_ses.first = sp->t_open;
 
 	/* Receive a HTTP protocol request */
 	HTC_Init(sp->htc, sp->ws, sp->fd, sp->vsl_id,
diff --git a/bin/varnishd/cache/cache_session.c b/bin/varnishd/cache/cache_session.c
index 46bfb8d..1292e5e 100644
--- a/bin/varnishd/cache/cache_session.c
+++ b/bin/varnishd/cache/cache_session.c
@@ -345,6 +345,7 @@ SES_Delete(struct sess *sp, const char *reason, double now)
 		SES_Close(sp, reason);
 	if (isnan(now))
 		now = VTIM_real();
+	assert(!isnan(sp->t_open));
 	assert(sp->fd < 0);
 
 	AZ(sp->vcl);
@@ -354,11 +355,10 @@ SES_Delete(struct sess *sp, const char *reason, double now)
 		strcpy(sp->addr, "-");
 
 	b = &sp->acct_ses;
-	assert(!isnan(b->first));
 
 	VSL(SLT_StatSess, sp->vsl_id, "%s %s %.0f %ju %ju %ju %ju %ju %ju %ju",
 	    sp->addr, sp->port,
-	    now - b->first, 	// XXX ??? 
+	    now - sp->t_open,
 	    b->sess, b->req, b->pipe, b->pass,
 	    b->fetch, b->hdrbytes, b->bodybytes);
 



From geoff at varnish-cache.org  Mon Jan  9 23:03:30 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Tue, 10 Jan 2012 00:03:30 +0100
Subject: [experimental-ims] bd76542 Collapse two functions for clarity
Message-ID: <E1RkOFW-0007ts-MO@project.varnish-software.com>

commit bd765420099d605322eae4049f56a8998544e837
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Fri Dec 23 07:28:30 2011 +0000

    Collapse two functions for clarity

diff --git a/bin/varnishd/cache/cache_acceptor.c b/bin/varnishd/cache/cache_acceptor.c
index 1f9616a..da35db4 100644
--- a/bin/varnishd/cache/cache_acceptor.c
+++ b/bin/varnishd/cache/cache_acceptor.c
@@ -128,32 +128,6 @@ sock_test(int fd)
 }
 
 /*--------------------------------------------------------------------
- * Called once the workerthread gets hold of the session, to fix up
- * any socket options that need it.
- */
-
-static void
-vca_prep(const struct sess *sp)
-{
-
-	if (need_test)
-		sock_test(sp->fd);
-	if (need_linger)
-		VTCP_Assert(setsockopt(sp->fd, SOL_SOCKET, SO_LINGER,
-		    &linger, sizeof linger));
-#ifdef SO_SNDTIMEO_WORKS
-	if (need_sndtimeo)
-		VTCP_Assert(setsockopt(sp->fd, SOL_SOCKET, SO_SNDTIMEO,
-		    &tv_sndtimeo, sizeof tv_sndtimeo));
-#endif
-#ifdef SO_RCVTIMEO_WORKS
-	if (need_rcvtimeo)
-		VTCP_Assert(setsockopt(sp->fd, SOL_SOCKET, SO_RCVTIMEO,
-		    &tv_rcvtimeo, sizeof tv_rcvtimeo));
-#endif
-}
-
-/*--------------------------------------------------------------------
  * If accept(2)'ing fails, we pace ourselves to relive any resource
  * shortage if possible.
  */
@@ -285,7 +259,22 @@ VCA_SetupSess(struct worker *w)
 	sp->sockaddrlen = wa->acceptaddrlen;
 	vca_pace_good();
 	w->stats.sess_conn++;
-	vca_prep(sp);
+
+	if (need_test)
+		sock_test(sp->fd);
+	if (need_linger)
+		VTCP_Assert(setsockopt(sp->fd, SOL_SOCKET, SO_LINGER,
+		    &linger, sizeof linger));
+#ifdef SO_SNDTIMEO_WORKS
+	if (need_sndtimeo)
+		VTCP_Assert(setsockopt(sp->fd, SOL_SOCKET, SO_SNDTIMEO,
+		    &tv_sndtimeo, sizeof tv_sndtimeo));
+#endif
+#ifdef SO_RCVTIMEO_WORKS
+	if (need_rcvtimeo)
+		VTCP_Assert(setsockopt(sp->fd, SOL_SOCKET, SO_RCVTIMEO,
+		    &tv_rcvtimeo, sizeof tv_rcvtimeo));
+#endif
 }
 
 /*--------------------------------------------------------------------*/



From geoff at varnish-cache.org  Mon Jan  9 23:03:30 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Tue, 10 Jan 2012 00:03:30 +0100
Subject: [experimental-ims] c0442f9 Only allocate sp->req where we need it.
Message-ID: <E1RkOFW-0007uD-So@project.varnish-software.com>

commit c0442f9041083486532712589bc7584ef9f9c53b
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Fri Dec 23 08:55:52 2011 +0000

    Only allocate sp->req where we need it.
    
    It's lifetime is now controlled by the timeout_linger parameter, so that
    the waiters won't have to think about it at all.

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index b674bd7..3c37176 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -922,6 +922,8 @@ struct sesspool *SES_NewPool(struct pool *pp, unsigned pool_no);
 void SES_DeletePool(struct sesspool *sp, struct worker *wrk);
 int SES_Schedule(struct sess *sp);
 void SES_Handle(struct sess *sp, double now);
+void SES_GetReq(struct sess *sp);
+void SES_ReleaseReq(struct sess *sp);
 
 /* cache_shmlog.c */
 extern struct VSC_C_main *VSC_C_main;
diff --git a/bin/varnishd/cache/cache_center.c b/bin/varnishd/cache/cache_center.c
index 6587bf2..a119360 100644
--- a/bin/varnishd/cache/cache_center.c
+++ b/bin/varnishd/cache/cache_center.c
@@ -37,7 +37,7 @@
  * a dot(1) graph in the source code comments.  So to see the big picture,
  * extract the DOT lines and run though dot(1), for instance with the
  * command:
- *	sed -n '/^DOT/s///p' cache_center.c | dot -Tps > /tmp/_.ps
+ *	sed -n '/^DOT/s///p' cache/cache_center.c | dot -Tps > /tmp/_.ps
  */
 
 /*
@@ -91,7 +91,8 @@ DOT	]
 DOT	herding [shape=hexagon]
 DOT	wait -> start [label="got req"]
 DOT	wait -> "SES_Delete()" [label="errors"]
-DOT	wait -> herding [label="timeout"]
+DOT	wait -> herding [label="timeout_linger"]
+DOT	herding -> wait [label="fd read_ready"]
 DOT }
  */
 
@@ -106,6 +107,9 @@ cnt_wait(struct sess *sp)
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 	wrk = sp->wrk;
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
+	if (sp->req == NULL)
+		SES_GetReq(sp);
+
 	AZ(sp->vcl);
 	AZ(wrk->obj);
 	AZ(sp->esi_level);
@@ -150,6 +154,7 @@ cnt_wait(struct sess *sp)
 				sp->t_req = NAN;
 				wrk->stats.sess_herd++;
 				SES_Charge(sp);
+				SES_ReleaseReq(sp);
 				WAIT_Enter(sp);
 				return (1);
 			}
@@ -350,10 +355,10 @@ cnt_deliver(struct sess *sp)
  *
 DOT	DONE [
 DOT		shape=hexagon
-DOT		label="Request completed"
+DOT		label="cnt_done:\nRequest completed"
 DOT	]
 DOT	ESI_RESP [ shape=hexagon ]
-DOT	DONE -> start
+DOT	DONE -> start [label="full pipeline"]
 DOT	DONE -> wait
 DOT	DONE -> ESI_RESP
  */
@@ -446,21 +451,11 @@ cnt_done(struct sess *sp)
 		sp->step = STP_START;
 		return (0);
 	}
-	if (Tlen(sp->htc->rxbuf)) {
+	if (Tlen(sp->htc->rxbuf))
 		wrk->stats.sess_readahead++;
-		sp->step = STP_WAIT;
-		sp->t_req = sp->t_idle;
-		return (0);
-	}
-	if (cache_param->timeout_linger > 0.) {
-		wrk->stats.sess_linger++;
-		sp->step = STP_WAIT;
-		sp->t_req = sp->t_idle;		// XXX: not quite correct
-		return (0);
-	}
-	wrk->stats.sess_herd++;
-	WAIT_Enter(sp);
-	return (1);
+	sp->step = STP_WAIT;
+	sp->t_req = sp->t_idle;
+	return (0);
 }
 
 /*--------------------------------------------------------------------
@@ -1023,7 +1018,7 @@ cnt_streambody(struct sess *sp)
 DOT subgraph xcluster_first {
 DOT	first [
 DOT		shape=box
-DOT		label="first\nConfigure data structures"
+DOT		label="cnt_first:\nSockaddr's"
 DOT	]
 DOT }
 DOT first -> wait
@@ -1567,11 +1562,11 @@ cnt_recv(struct sess *sp)
 
 /*--------------------------------------------------------------------
  * START
- * Handle a request, wherever it came from recv/restart.
+ * Handle a request.
  *
 DOT start [
 DOT	shape=box
-DOT	label="Dissect request\nHandle expect"
+DOT	label="cnt_start:\nDissect request\nHandle expect"
 DOT ]
 DOT start -> recv [style=bold,color=green]
 DOT start -> DONE [label=errors]
@@ -1588,6 +1583,7 @@ cnt_start(struct sess *sp)
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 	wrk = sp->wrk;
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
+	CHECK_OBJ_NOTNULL(sp->req, REQ_MAGIC);
 	AZ(sp->restarts);
 	AZ(wrk->obj);
 	AZ(sp->vcl);
@@ -1684,8 +1680,10 @@ CNT_Session(struct sess *sp)
 	struct worker *wrk;
 
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+#if 0
 	CHECK_OBJ_NOTNULL(sp->req, REQ_MAGIC);
 	MPL_AssertSane(sp->req);
+#endif
 	wrk = sp->wrk;
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 
@@ -1723,8 +1721,10 @@ CNT_Session(struct sess *sp)
 	 */
 	for (done = 0; !done; ) {
 		assert(sp->wrk == wrk);
+#if 0
 		CHECK_OBJ_NOTNULL(sp->req, REQ_MAGIC);
 		MPL_AssertSane(sp->req);
+#endif
 		/*
 		 * This is a good place to be paranoid about the various
 		 * pointers still pointing to the things we expect.
diff --git a/bin/varnishd/cache/cache_session.c b/bin/varnishd/cache/cache_session.c
index 1292e5e..530c96e 100644
--- a/bin/varnishd/cache/cache_session.c
+++ b/bin/varnishd/cache/cache_session.c
@@ -218,9 +218,6 @@ SES_New(struct worker *wrk, struct sesspool *pp)
 	if (sm == NULL)
 		return (NULL);
 	sp = &sm->sess;
-	sp->req = MPL_Get(pp->mpl_req, NULL);
-	AN(sp->req);
-	sp->req->magic = REQ_MAGIC;
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 	return (sp);
 }
@@ -245,24 +242,33 @@ SES_Alloc(void)
 }
 
 /*--------------------------------------------------------------------
- * Schedule a session back on a work-thread from its pool
  */
 
-int
-SES_Schedule(struct sess *sp)
+static struct sesspool *
+ses_getpool(const struct sess *sp)
 {
 	struct sessmem *sm;
 	struct sesspool *pp;
 
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
-
-	AZ(sp->wrk);
-
 	sm = sp->mem;
 	CHECK_OBJ_NOTNULL(sm, SESSMEM_MAGIC);
-
 	pp = sm->pool;
 	CHECK_OBJ_NOTNULL(pp, SESSPOOL_MAGIC);
+	return (pp);
+}
+
+/*--------------------------------------------------------------------
+ * Schedule a session back on a work-thread from its pool
+ */
+
+int
+SES_Schedule(struct sess *sp)
+{
+	struct sesspool *pp;
+
+	pp = ses_getpool(sp);
+	AZ(sp->wrk);
 
 	AN(pp->pool);
 
@@ -330,14 +336,10 @@ SES_Delete(struct sess *sp, const char *reason, double now)
 	struct worker *wrk;
 	struct sesspool *pp;
 
+	pp = ses_getpool(sp);
 
-	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
-	CHECK_OBJ_NOTNULL(sp->req, REQ_MAGIC);
-	MPL_AssertSane(sp->req);
 	sm = sp->mem;
 	CHECK_OBJ_NOTNULL(sm, SESSMEM_MAGIC);
-	pp = sm->pool;
-	CHECK_OBJ_NOTNULL(pp, SESSPOOL_MAGIC);
 	wrk = sp->wrk;
 	CHECK_OBJ_ORNULL(wrk, WORKER_MAGIC);
 
@@ -348,6 +350,9 @@ SES_Delete(struct sess *sp, const char *reason, double now)
 	assert(!isnan(sp->t_open));
 	assert(sp->fd < 0);
 
+	if (sp->req != NULL)
+		SES_ReleaseReq(sp);
+
 	AZ(sp->vcl);
 	if (*sp->addr == '\0')
 		strcpy(sp->addr, "-");
@@ -362,11 +367,6 @@ SES_Delete(struct sess *sp, const char *reason, double now)
 	    b->sess, b->req, b->pipe, b->pass,
 	    b->fetch, b->hdrbytes, b->bodybytes);
 
-	CHECK_OBJ_NOTNULL(sp->req, REQ_MAGIC);
-	MPL_AssertSane(sp->req);
-	MPL_Free(pp->mpl_req, sp->req);
-	sp->req = NULL;
-
 	if (sm->workspace != cache_param->sess_workspace ||
 	    sm->nhttp != (uint16_t)cache_param->http_max_hdr ||
 	    pp->nsess > cache_param->max_sess) {
@@ -392,45 +392,73 @@ SES_Delete(struct sess *sp, const char *reason, double now)
 }
 
 /*--------------------------------------------------------------------
+ * Alloc/Free/Clean sp->req
+ */
+
+void
+SES_GetReq(struct sess *sp)
+{
+	struct sesspool *pp;
+
+	pp = ses_getpool(sp);
+	AZ(sp->req);
+	sp->req = MPL_Get(pp->mpl_req, NULL);
+	AN(sp->req);
+	sp->req->magic = REQ_MAGIC;
+}
+
+void
+SES_ReleaseReq(struct sess *sp)
+{
+	struct sesspool *pp;
+
+	pp = ses_getpool(sp);
+	CHECK_OBJ_NOTNULL(sp->req, REQ_MAGIC);
+	MPL_AssertSane(sp->req);
+	MPL_Free(pp->mpl_req, sp->req);
+	sp->req = NULL;
+}
+
+/*--------------------------------------------------------------------
  * Create and delete pools
  */
 
 struct sesspool *
-SES_NewPool(struct pool *pp, unsigned pool_no)
+SES_NewPool(struct pool *wp, unsigned pool_no)
 {
-	struct sesspool *sp;
+	struct sesspool *pp;
 	char nb[8];
 
-	ALLOC_OBJ(sp, SESSPOOL_MAGIC);
-	AN(sp);
-	sp->pool = pp;
-	VTAILQ_INIT(&sp->freelist);
-	Lck_New(&sp->mtx, lck_sessmem);
+	ALLOC_OBJ(pp, SESSPOOL_MAGIC);
+	AN(pp);
+	pp->pool = wp;
+	VTAILQ_INIT(&pp->freelist);
+	Lck_New(&pp->mtx, lck_sessmem);
 	bprintf(nb, "req%u", pool_no);
-	sp->req_size = sizeof (struct req);
-	sp->mpl_req = MPL_New(nb, &cache_param->req_pool, &sp->req_size);
-	return (sp);
+	pp->req_size = sizeof (struct req);
+	pp->mpl_req = MPL_New(nb, &cache_param->req_pool, &pp->req_size);
+	return (pp);
 }
 
 void
-SES_DeletePool(struct sesspool *sp, struct worker *wrk)
+SES_DeletePool(struct sesspool *pp, struct worker *wrk)
 {
 	struct sessmem *sm;
 
-	CHECK_OBJ_NOTNULL(sp, SESSPOOL_MAGIC);
+	CHECK_OBJ_NOTNULL(pp, SESSPOOL_MAGIC);
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
-	Lck_Lock(&sp->mtx);
-	while (!VTAILQ_EMPTY(&sp->freelist)) {
-		sm = VTAILQ_FIRST(&sp->freelist);
+	Lck_Lock(&pp->mtx);
+	while (!VTAILQ_EMPTY(&pp->freelist)) {
+		sm = VTAILQ_FIRST(&pp->freelist);
 		CHECK_OBJ_NOTNULL(sm, SESSMEM_MAGIC);
-		VTAILQ_REMOVE(&sp->freelist, sm, list);
+		VTAILQ_REMOVE(&pp->freelist, sm, list);
 		FREE_OBJ(sm);
 		wrk->stats.sessmem_free++;
-		sp->nsess--;
+		pp->nsess--;
 	}
-	AZ(sp->nsess);
-	Lck_Unlock(&sp->mtx);
-	Lck_Delete(&sp->mtx);
-	MPL_Destroy(&sp->mpl_req);
-	FREE_OBJ(sp);
+	AZ(pp->nsess);
+	Lck_Unlock(&pp->mtx);
+	Lck_Delete(&pp->mtx);
+	MPL_Destroy(&pp->mpl_req);
+	FREE_OBJ(pp);
 }
diff --git a/bin/varnishd/waiter/cache_waiter.c b/bin/varnishd/waiter/cache_waiter.c
index 8aa8684..44e9721 100644
--- a/bin/varnishd/waiter/cache_waiter.c
+++ b/bin/varnishd/waiter/cache_waiter.c
@@ -68,6 +68,7 @@ WAIT_Enter(struct sess *sp)
 
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 	CHECK_OBJ_NOTNULL(sp->wrk, WORKER_MAGIC);
+	AZ(sp->req);
 	AZ(sp->vcl);
 	assert(sp->fd >= 0);
 	sp->wrk = NULL;



From geoff at varnish-cache.org  Mon Jan  9 23:03:30 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Tue, 10 Jan 2012 00:03:30 +0100
Subject: [experimental-ims] 4354cd7 Delete the session directly if we never
	manage to collect a full request header: There is no point in
	going through cnt_done to do that.
Message-ID: <E1RkOFW-0007u5-Oi@project.varnish-software.com>

commit 4354cd713f2db1c55db169927e0d5e001a5a19ff
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Fri Dec 23 07:37:20 2011 +0000

    Delete the session directly if we never manage to collect a full
    request header:  There is no point in going through cnt_done to do that.

diff --git a/bin/varnishd/cache/cache_center.c b/bin/varnishd/cache/cache_center.c
index 052a68b..6587bf2 100644
--- a/bin/varnishd/cache/cache_center.c
+++ b/bin/varnishd/cache/cache_center.c
@@ -86,11 +86,11 @@ static unsigned xids;
 DOT subgraph xcluster_wait {
 DOT	wait [
 DOT		shape=box
-DOT		label="wait for\nrequest"
+DOT		label="cnt_wait:\nwait for\nrequest"
 DOT	]
 DOT	herding [shape=hexagon]
 DOT	wait -> start [label="got req"]
-DOT	wait -> DONE [label="errors"]
+DOT	wait -> "SES_Delete()" [label="errors"]
 DOT	wait -> herding [label="timeout"]
 DOT }
  */
@@ -119,31 +119,30 @@ cnt_wait(struct sess *sp)
 		pfd[0].revents = 0;
 		j = poll(pfd, 1, tmo);
 		assert(j >= 0);
+		now = VTIM_real();
 		if (j != 0)
 			i = HTC_Rx(sp->htc);
 		else
 			i = HTC_Complete(sp->htc);
-		if (i == -1) {
-			SES_Close(sp, "EOF");
-			break;
-		}
-		if (i == -2) {
-			SES_Close(sp, "overflow");
-			break;
-		}
-		now = VTIM_real();
 		if (i == 1) {
 			/* Got it, run with it */
 			sp->t_req = now;
-			sp->step = STP_START;
-			return (0);
+			break;
+		}
+		if (i == -1) {
+			SES_Delete(sp, "EOF", now);
+			return (1);
+		}
+		if (i == -2) {
+			SES_Delete(sp, "overflow", now);
+			return (1);
 		}
 		if (i == -3) {
 			/* Nothing but whitespace */
 			when = sp->t_idle + cache_param->timeout_idle;
 			if (when < now) {
-				SES_Close(sp, "timeout");
-				break;
+				SES_Delete(sp, "timeout", now);
+				return (1);
 			}
 			when = sp->t_idle + cache_param->timeout_linger;
 			tmo = (int)(1e3 * (when - now));
@@ -159,12 +158,12 @@ cnt_wait(struct sess *sp)
 			when = sp->t_req + cache_param->timeout_req;
 			tmo = (int)(1e3 * (when - now));
 			if (when < now || tmo == 0) {
-				SES_Close(sp, "req timeout");
-				break;
+				SES_Delete(sp, "req timeout", now);
+				return (1);
 			}
 		}
 	}
-	sp->step = STP_DONE;
+	sp->step = STP_START;
 	return (0);
 }
 



From geoff at varnish-cache.org  Mon Jan  9 23:03:31 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Tue, 10 Jan 2012 00:03:31 +0100
Subject: [experimental-ims] 9173a98 Move xid from sess to req
Message-ID: <E1RkOFX-0007uT-1r@project.varnish-software.com>

commit 9173a98a9f63de2fa47b04e8eb0094f4e5029ee6
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Fri Dec 23 11:48:11 2011 +0000

    Move xid from sess to req

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index 3c37176..f45235b 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -580,6 +580,8 @@ struct object {
 struct req {
 	unsigned		magic;
 #define REQ_MAGIC		0x2751aaa1
+
+	unsigned		xid;
 };
 
 /*--------------------------------------------------------------------*/
@@ -625,7 +627,6 @@ struct sess {
 
 	/* Request related fields ------------------------------------*/
 
-	unsigned		xid;
 
 	int			restarts;
 	int			esi_level;
diff --git a/bin/varnishd/cache/cache_center.c b/bin/varnishd/cache/cache_center.c
index a119360..1de6ed7 100644
--- a/bin/varnishd/cache/cache_center.c
+++ b/bin/varnishd/cache/cache_center.c
@@ -113,7 +113,7 @@ cnt_wait(struct sess *sp)
 	AZ(sp->vcl);
 	AZ(wrk->obj);
 	AZ(sp->esi_level);
-	assert(sp->xid == 0);
+	assert(sp->req->xid == 0);
 
 	assert(!isnan(sp->t_req));
 	tmo = (int)(1e3 * cache_param->timeout_linger);
@@ -397,7 +397,7 @@ cnt_done(struct sess *sp)
 
 
 	sp->t_idle = W_TIM_real(wrk);
-	if (sp->xid == 0) {
+	if (sp->req->xid == 0) {
 		sp->t_resp = sp->t_idle;
 	} else {
 		dp = sp->t_resp - sp->t_req;
@@ -410,9 +410,9 @@ cnt_done(struct sess *sp)
 			    (uintmax_t)sp->req_bodybytes);
 		}
 		WSP(sp, SLT_ReqEnd, "%u %.9f %.9f %.9f %.9f %.9f",
-		    sp->xid, sp->t_req, sp->t_idle, dh, dp, da);
+		    sp->req->xid, sp->t_req, sp->t_idle, dh, dp, da);
 	}
-	sp->xid = 0;
+	sp->req->xid = 0;
 	WSL_Flush(wrk, 0);
 
 	sp->t_req = NAN;
@@ -503,7 +503,7 @@ cnt_error(struct sess *sp)
 			return(0);
 		}
 		AN(wrk->obj);
-		wrk->obj->xid = sp->xid;
+		wrk->obj->xid = sp->req->xid;
 		wrk->obj->exp.entered = sp->t_req;
 	} else {
 		CHECK_OBJ_NOTNULL(wrk->busyobj, BUSYOBJ_MAGIC);
@@ -869,7 +869,7 @@ cnt_fetchbody(struct sess *sp)
 		VSB_delete(vary);
 	}
 
-	wrk->obj->xid = sp->xid;
+	wrk->obj->xid = sp->req->xid;
 	wrk->obj->response = sp->err_code;
 	WS_Assert(wrk->obj->ws_o);
 
@@ -1035,7 +1035,7 @@ cnt_first(struct sess *sp)
 	wrk = sp->wrk;
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 
-	assert(sp->xid == 0);
+	AZ(sp->req);
 	assert(sp->restarts == 0);
 	AZ(sp->esi_level);
 
@@ -1595,8 +1595,8 @@ cnt_start(struct sess *sp)
 	wrk->acct_tmp.req++;
 
 	/* Assign XID and log */
-	sp->xid = ++xids;				/* XXX not locked */
-	WSP(sp, SLT_ReqStart, "%s %s %u", sp->addr, sp->port,  sp->xid);
+	sp->req->xid = ++xids;				/* XXX not locked */
+	WSP(sp, SLT_ReqStart, "%s %s %u", sp->addr, sp->port,  sp->req->xid);
 
 	/* Borrow VCL reference from worker thread */
 	VCL_Refresh(&wrk->vcl);
diff --git a/bin/varnishd/cache/cache_cli.c b/bin/varnishd/cache/cache_cli.c
index 98d89c3..abb9e43 100644
--- a/bin/varnishd/cache/cache_cli.c
+++ b/bin/varnishd/cache/cache_cli.c
@@ -132,6 +132,7 @@ cli_debug_sizeof(struct cli *cli, const char * const *av, void *priv)
 	SZOF(struct objcore);
 	SZOF(struct objhead);
 	SZOF(struct sess);
+	SZOF(struct req);
 	SZOF(struct vbc);
 	SZOF(struct VSC_C_main);
 	SZOF(struct lock);
diff --git a/bin/varnishd/cache/cache_esi_deliver.c b/bin/varnishd/cache/cache_esi_deliver.c
index e961819..fb0771d 100644
--- a/bin/varnishd/cache/cache_esi_deliver.c
+++ b/bin/varnishd/cache/cache_esi_deliver.c
@@ -91,7 +91,7 @@ ved_include(struct sess *sp, const char *src, const char *host)
 	/* Client content already taken care of */
 	http_Unset(sp->http, H_Content_Length);
 
-	sxid = sp->xid;
+	sxid = sp->req->xid;
 	while (1) {
 		sp->wrk = w;
 		CNT_Session(sp);
@@ -102,7 +102,7 @@ ved_include(struct sess *sp, const char *src, const char *host)
 		DSL(0x20, SLT_Debug, sp->vsl_id, "loop waiting for ESI");
 		(void)usleep(10000);
 	}
-	sp->xid = sxid;
+	sp->req->xid = sxid;
 	AN(sp->wrk);
 	assert(sp->step == STP_DONE);
 	sp->esi_level--;
diff --git a/bin/varnishd/cache/cache_http.c b/bin/varnishd/cache/cache_http.c
index 4a6ba82..803fed2 100644
--- a/bin/varnishd/cache/cache_http.c
+++ b/bin/varnishd/cache/cache_http.c
@@ -880,7 +880,7 @@ http_FilterHeader(const struct sess *sp, unsigned how)
 	else
 		http_copyh(hp, sp->http, HTTP_HDR_PROTO);
 	http_FilterFields(sp->wrk, sp->vsl_id, hp, sp->http, how);
-	http_PrintfHeader(sp->wrk, sp->vsl_id, hp, "X-Varnish: %u", sp->xid);
+	http_PrintfHeader(sp->wrk, sp->vsl_id, hp, "X-Varnish: %u", sp->req->xid);
 }
 
 /*--------------------------------------------------------------------
diff --git a/bin/varnishd/cache/cache_panic.c b/bin/varnishd/cache/cache_panic.c
index 6daf807..238c597 100644
--- a/bin/varnishd/cache/cache_panic.c
+++ b/bin/varnishd/cache/cache_panic.c
@@ -235,7 +235,7 @@ pan_sess(const struct sess *sp)
 	VSB_printf(pan_vsp, "sp = %p {\n", sp);
 	VSB_printf(pan_vsp,
 	    "  fd = %d, id = %u, xid = %u,\n",
-	    sp->fd, sp->vsl_id & VSL_IDENTMASK, sp->xid);
+	    sp->fd, sp->vsl_id & VSL_IDENTMASK, sp->req->xid);
 	VSB_printf(pan_vsp, "  client = %s %s,\n",
 	    sp->addr ? sp->addr : "?.?.?.?",
 	    sp->port ? sp->port : "?");
diff --git a/bin/varnishd/cache/cache_response.c b/bin/varnishd/cache/cache_response.c
index 43feae9..cbddb7d 100644
--- a/bin/varnishd/cache/cache_response.c
+++ b/bin/varnishd/cache/cache_response.c
@@ -132,12 +132,12 @@ RES_BuildHttp(const struct sess *sp)
 	http_PrintfHeader(sp->wrk, sp->vsl_id, sp->wrk->resp,
 	    "Date: %s", time_str);
 
-	if (sp->xid != sp->wrk->obj->xid)
+	if (sp->req->xid != sp->wrk->obj->xid)
 		http_PrintfHeader(sp->wrk, sp->vsl_id, sp->wrk->resp,
-		    "X-Varnish: %u %u", sp->xid, sp->wrk->obj->xid);
+		    "X-Varnish: %u %u", sp->req->xid, sp->wrk->obj->xid);
 	else
 		http_PrintfHeader(sp->wrk, sp->vsl_id, sp->wrk->resp,
-		    "X-Varnish: %u", sp->xid);
+		    "X-Varnish: %u", sp->req->xid);
 	http_PrintfHeader(sp->wrk, sp->vsl_id, sp->wrk->resp, "Age: %.0f",
 	    sp->wrk->obj->exp.age + sp->t_resp - sp->wrk->obj->exp.entered);
 	http_SetHeader(sp->wrk, sp->vsl_id, sp->wrk->resp, "Via: 1.1 varnish");
diff --git a/bin/varnishd/cache/cache_rfc2616.c b/bin/varnishd/cache/cache_rfc2616.c
index 6e7efea..2878d9a 100644
--- a/bin/varnishd/cache/cache_rfc2616.c
+++ b/bin/varnishd/cache/cache_rfc2616.c
@@ -169,7 +169,7 @@ RFC2616_Ttl(const struct sess *sp)
 	/* calculated TTL, Our time, Date, Expires, max-age, age */
 	WSP(sp, SLT_TTL,
 	    "%u RFC %.0f %.0f %.0f %.0f %.0f %.0f %.0f %u",
-	    sp->xid, expp->ttl, -1., -1., expp->entered,
+	    sp->req->xid, expp->ttl, -1., -1., expp->entered,
 	    expp->age, h_date, h_expires, max_age);
 }
 
diff --git a/bin/varnishd/cache/cache_vrt_var.c b/bin/varnishd/cache/cache_vrt_var.c
index 7431c1b..6f5544b 100644
--- a/bin/varnishd/cache/cache_vrt_var.c
+++ b/bin/varnishd/cache/cache_vrt_var.c
@@ -416,11 +416,11 @@ VRT_DO_EXP(obj, sp->wrk->obj->exp, keep, 0,
    vrt_wsp_exp(sp, sp->wrk->obj->xid, &sp->wrk->obj->exp);)
 
 VRT_DO_EXP(beresp, sp->wrk->busyobj->exp, grace, 0,
-   vrt_wsp_exp(sp, sp->xid, &sp->wrk->busyobj->exp);)
+   vrt_wsp_exp(sp, sp->req->xid, &sp->wrk->busyobj->exp);)
 VRT_DO_EXP(beresp, sp->wrk->busyobj->exp, ttl, 0,
-   vrt_wsp_exp(sp, sp->xid, &sp->wrk->busyobj->exp);)
+   vrt_wsp_exp(sp, sp->req->xid, &sp->wrk->busyobj->exp);)
 VRT_DO_EXP(beresp, sp->wrk->busyobj->exp, keep, 0,
-   vrt_wsp_exp(sp, sp->xid, &sp->wrk->busyobj->exp);)
+   vrt_wsp_exp(sp, sp->req->xid, &sp->wrk->busyobj->exp);)
 
 /*--------------------------------------------------------------------
  * req.xid
@@ -433,9 +433,9 @@ VRT_r_req_xid(struct sess *sp)
 	int size;
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 
-	size = snprintf(NULL, 0, "%u", sp->xid) + 1;
+	size = snprintf(NULL, 0, "%u", sp->req->xid) + 1;
 	AN(p = WS_Alloc(sp->http->ws, size));
-	assert(snprintf(p, size, "%u", sp->xid) < size);
+	assert(snprintf(p, size, "%u", sp->req->xid) < size);
 	return (p);
 }
 



From geoff at varnish-cache.org  Mon Jan  9 23:03:31 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Tue, 10 Jan 2012 00:03:31 +0100
Subject: [experimental-ims] 3abbb12 Move another 56 bytes from sess to req
Message-ID: <E1RkOFX-0007um-G6@project.varnish-software.com>

commit 3abbb1270c6cdfd2511ddca4f316c98e7353efd7
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Fri Dec 23 12:30:26 2011 +0000

    Move another 56 bytes from sess to req

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index 1b2f1a7..578d548 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -605,6 +605,16 @@ struct req {
 	unsigned char		sendbody;
 	unsigned char		wantbody;
 
+	uint16_t		err_code;
+	const char		*err_reason;
+
+	struct director		*director;
+	struct VCL_conf		*vcl;
+
+	uint64_t		req_bodybytes;
+	char			*ws_req;	/* WS above request data */
+
+	double			t_resp;
 };
 
 /*--------------------------------------------------------------------*/
@@ -656,19 +666,9 @@ struct sess {
 	struct http		*http0;
 
 	struct ws		ws[1];
-	char			*ws_req;	/* WS above request data */
-
-	uint16_t		err_code;
-	const char		*err_reason;
-
-	struct director		*director;
-	struct VCL_conf		*vcl;
-
-	uint64_t		req_bodybytes;
 
 	/* Timestamps, all on TIM_real() timescale */
 	double			t_req;
-	double			t_resp;
 
 	/* TBD fields ------------------------------------------------*/
 
diff --git a/bin/varnishd/cache/cache_backend.c b/bin/varnishd/cache/cache_backend.c
index 48c01a3..0b3c261 100644
--- a/bin/varnishd/cache/cache_backend.c
+++ b/bin/varnishd/cache/cache_backend.c
@@ -348,7 +348,7 @@ vbe_GetVbe(const struct sess *sp, struct vdi_simple *vs)
 			/* XXX locking of stats */
 			VSC_C_main->backend_reuse += 1;
 			WSP(sp, SLT_Backend, "%d %s %s",
-			    vc->fd, sp->director->vcl_name, bp->display_name);
+			    vc->fd, sp->req->director->vcl_name, bp->display_name);
 			vc->vdis = vs;
 			vc->recycled = 1;
 			return (vc);
@@ -390,7 +390,7 @@ vbe_GetVbe(const struct sess *sp, struct vdi_simple *vs)
 	vc->backend = bp;
 	VSC_C_main->backend_conn++;
 	WSP(sp, SLT_Backend, "%d %s %s",
-	    vc->fd, sp->director->vcl_name, bp->display_name);
+	    vc->fd, sp->req->director->vcl_name, bp->display_name);
 	vc->vdis = vs;
 	return (vc);
 }
diff --git a/bin/varnishd/cache/cache_center.c b/bin/varnishd/cache/cache_center.c
index cb5d161..6eef7cb 100644
--- a/bin/varnishd/cache/cache_center.c
+++ b/bin/varnishd/cache/cache_center.c
@@ -110,10 +110,11 @@ cnt_wait(struct sess *sp)
 	if (sp->req == NULL)
 		SES_GetReq(sp);
 
-	AZ(sp->vcl);
+	AZ(sp->req->vcl);
 	AZ(wrk->obj);
 	AZ(sp->req->esi_level);
 	assert(sp->req->xid == 0);
+	sp->req->t_resp = NAN;
 
 	assert(!isnan(sp->t_req));
 	tmo = (int)(1e3 * cache_param->timeout_linger);
@@ -210,7 +211,7 @@ cnt_prepresp(struct sess *sp)
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 
 	CHECK_OBJ_NOTNULL(wrk->obj, OBJECT_MAGIC);
-	CHECK_OBJ_NOTNULL(sp->vcl, VCL_CONF_MAGIC);
+	CHECK_OBJ_NOTNULL(sp->req->vcl, VCL_CONF_MAGIC);
 
 	if (wrk->busyobj != NULL) {
 		CHECK_OBJ_NOTNULL(wrk->busyobj, BUSYOBJ_MAGIC);
@@ -268,13 +269,13 @@ cnt_prepresp(struct sess *sp)
 		}
 	}
 
-	sp->t_resp = W_TIM_real(wrk);
+	sp->req->t_resp = W_TIM_real(wrk);
 	if (wrk->obj->objcore != NULL) {
-		if ((sp->t_resp - wrk->obj->last_lru) >
+		if ((sp->req->t_resp - wrk->obj->last_lru) >
 		    cache_param->lru_timeout &&
 		    EXP_Touch(wrk->obj->objcore))
-			wrk->obj->last_lru = sp->t_resp;
-		wrk->obj->last_use = sp->t_resp;	/* XXX: locking ? */
+			wrk->obj->last_lru = sp->req->t_resp;
+		wrk->obj->last_use = sp->req->t_resp;	/* XXX: locking ? */
 	}
 	http_Setup(wrk->resp, wrk->ws);
 	RES_BuildHttp(sp);
@@ -295,7 +296,7 @@ cnt_prepresp(struct sess *sp)
 		}
 		AZ(wrk->obj);
 		sp->req->restarts++;
-		sp->director = NULL;
+		sp->req->director = NULL;
 		http_Setup(wrk->resp, NULL);
 		sp->step = STP_RECV;
 		return (0);
@@ -336,7 +337,7 @@ cnt_deliver(struct sess *sp)
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 
 	AZ(sp->wrk->busyobj);
-	sp->director = NULL;
+	sp->req->director = NULL;
 	sp->req->restarts = 0;
 
 	RES_WriteObj(sp);
@@ -373,11 +374,11 @@ cnt_done(struct sess *sp)
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 	wrk = sp->wrk;
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
-	CHECK_OBJ_ORNULL(sp->vcl, VCL_CONF_MAGIC);
+	CHECK_OBJ_ORNULL(sp->req->vcl, VCL_CONF_MAGIC);
 
 	AZ(wrk->obj);
 	AZ(wrk->busyobj);
-	sp->director = NULL;
+	sp->req->director = NULL;
 	sp->req->restarts = 0;
 
 	wrk->busyobj = NULL;
@@ -388,26 +389,26 @@ cnt_done(struct sess *sp)
 	if (sp->req->esi_level > 0)
 		return (1);
 
-	if (sp->vcl != NULL) {
+	if (sp->req->vcl != NULL) {
 		if (wrk->vcl != NULL)
 			VCL_Rel(&wrk->vcl);
-		wrk->vcl = sp->vcl;
-		sp->vcl = NULL;
+		wrk->vcl = sp->req->vcl;
+		sp->req->vcl = NULL;
 	}
 
 
 	sp->t_idle = W_TIM_real(wrk);
 	if (sp->req->xid == 0) {
-		sp->t_resp = sp->t_idle;
+		sp->req->t_resp = sp->t_idle;
 	} else {
-		dp = sp->t_resp - sp->t_req;
-		da = sp->t_idle - sp->t_resp;
+		dp = sp->req->t_resp - sp->t_req;
+		da = sp->t_idle - sp->req->t_resp;
 		dh = sp->t_req - sp->t_open;
 		/* XXX: Add StatReq == StatSess */
 		/* XXX: Workaround for pipe */
 		if (sp->fd >= 0) {
 			WSP(sp, SLT_Length, "%ju",
-			    (uintmax_t)sp->req_bodybytes);
+			    (uintmax_t)sp->req->req_bodybytes);
 		}
 		WSP(sp, SLT_ReqEnd, "%u %.9f %.9f %.9f %.9f %.9f",
 		    sp->req->xid, sp->t_req, sp->t_idle, dh, dp, da);
@@ -416,9 +417,9 @@ cnt_done(struct sess *sp)
 	WSL_Flush(wrk, 0);
 
 	sp->t_req = NAN;
-	sp->t_resp = NAN;
+	sp->req->t_resp = NAN;
 
-	sp->req_bodybytes = 0;
+	sp->req->req_bodybytes = 0;
 
 	sp->req->hash_always_miss = 0;
 	sp->req->hash_ignore_busy = 0;
@@ -496,7 +497,7 @@ cnt_error(struct sess *sp)
 			    (uint16_t)cache_param->http_max_hdr);
 		if (wrk->obj == NULL) {
 			sp->req->doclose = "Out of objects";
-			sp->director = NULL;
+			sp->req->director = NULL;
 			http_Setup(wrk->busyobj->beresp, NULL);
 			http_Setup(wrk->busyobj->bereq, NULL);
 			sp->step = STP_DONE;
@@ -512,27 +513,27 @@ cnt_error(struct sess *sp)
 	CHECK_OBJ_NOTNULL(wrk->obj, OBJECT_MAGIC);
 	h = wrk->obj->http;
 
-	if (sp->err_code < 100 || sp->err_code > 999)
-		sp->err_code = 501;
+	if (sp->req->err_code < 100 || sp->req->err_code > 999)
+		sp->req->err_code = 501;
 
 	http_PutProtocol(wrk, sp->vsl_id, h, "HTTP/1.1");
-	http_PutStatus(h, sp->err_code);
+	http_PutStatus(h, sp->req->err_code);
 	VTIM_format(W_TIM_real(wrk), date);
 	http_PrintfHeader(wrk, sp->vsl_id, h, "Date: %s", date);
 	http_SetHeader(wrk, sp->vsl_id, h, "Server: Varnish");
 
-	if (sp->err_reason != NULL)
-		http_PutResponse(wrk, sp->vsl_id, h, sp->err_reason);
+	if (sp->req->err_reason != NULL)
+		http_PutResponse(wrk, sp->vsl_id, h, sp->req->err_reason);
 	else
 		http_PutResponse(wrk, sp->vsl_id, h,
-		    http_StatusMessage(sp->err_code));
+		    http_StatusMessage(sp->req->err_code));
 	VCL_error_method(sp);
 
 	if (sp->req->handling == VCL_RET_RESTART &&
 	    sp->req->restarts <  cache_param->max_restarts) {
 		HSH_Drop(wrk);
 		VBO_DerefBusyObj(wrk, &wrk->busyobj);
-		sp->director = NULL;
+		sp->req->director = NULL;
 		sp->req->restarts++;
 		sp->step = STP_RECV;
 		return (0);
@@ -545,8 +546,8 @@ cnt_error(struct sess *sp)
 	sp->req->wantbody = 1;
 
 	assert(sp->req->handling == VCL_RET_DELIVER);
-	sp->err_code = 0;
-	sp->err_reason = NULL;
+	sp->req->err_code = 0;
+	sp->req->err_reason = NULL;
 	http_Setup(wrk->busyobj->bereq, NULL);
 	VBO_DerefBusyObj(wrk, &wrk->busyobj);
 	sp->step = STP_PREPRESP;
@@ -592,10 +593,10 @@ cnt_fetch(struct sess *sp)
 	wrk = sp->wrk;
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 
-	CHECK_OBJ_NOTNULL(sp->vcl, VCL_CONF_MAGIC);
+	CHECK_OBJ_NOTNULL(sp->req->vcl, VCL_CONF_MAGIC);
 	CHECK_OBJ_NOTNULL(wrk->busyobj, BUSYOBJ_MAGIC);
 
-	AN(sp->director);
+	AN(sp->req->director);
 	AZ(wrk->busyobj->vbc);
 	AZ(wrk->busyobj->should_close);
 	AZ(wrk->storage_hint);
@@ -617,7 +618,7 @@ cnt_fetch(struct sess *sp)
 
 	if (i) {
 		sp->req->handling = VCL_RET_ERROR;
-		sp->err_code = 503;
+		sp->req->err_code = 503;
 	} else {
 		/*
 		 * These two headers can be spread over multiple actual headers
@@ -634,7 +635,7 @@ cnt_fetch(struct sess *sp)
 		 */
 		wrk->busyobj->body_status = RFC2616_Body(sp);
 
-		sp->err_code = http_GetStatus(wrk->busyobj->beresp);
+		sp->req->err_code = http_GetStatus(wrk->busyobj->beresp);
 
 		/*
 		 * What does RFC2616 think about TTL ?
@@ -678,7 +679,7 @@ cnt_fetch(struct sess *sp)
 		wrk->objcore = NULL;
 	}
 	VBO_DerefBusyObj(wrk, &wrk->busyobj);
-	sp->director = NULL;
+	sp->req->director = NULL;
 	wrk->storage_hint = NULL;
 
 	switch (sp->req->handling) {
@@ -847,7 +848,7 @@ cnt_fetchbody(struct sess *sp)
 		wrk->busyobj->exp.keep = 0.0;
 	}
 	if (wrk->obj == NULL) {
-		sp->err_code = 503;
+		sp->req->err_code = 503;
 		sp->step = STP_ERROR;
 		VDI_CloseFd(wrk, &wrk->busyobj->vbc);
 		VBO_DerefBusyObj(wrk, &wrk->busyobj);
@@ -870,7 +871,7 @@ cnt_fetchbody(struct sess *sp)
 	}
 
 	wrk->obj->xid = sp->req->xid;
-	wrk->obj->response = sp->err_code;
+	wrk->obj->response = sp->req->err_code;
 	WS_Assert(wrk->obj->ws_o);
 
 	/* Filter into object */
@@ -915,13 +916,13 @@ cnt_fetchbody(struct sess *sp)
 	wrk->busyobj->vfp = NULL;
 	assert(WRW_IsReleased(wrk));
 	AZ(wrk->busyobj->vbc);
-	AN(sp->director);
+	AN(sp->req->director);
 
 	if (i) {
 		HSH_Drop(wrk);
 		VBO_DerefBusyObj(wrk, &wrk->busyobj);
 		AZ(wrk->obj);
-		sp->err_code = 503;
+		sp->req->err_code = 503;
 		sp->step = STP_ERROR;
 		return (0);
 	}
@@ -984,7 +985,7 @@ cnt_streambody(struct sess *sp)
 	http_Setup(wrk->busyobj->beresp, NULL);
 	wrk->busyobj->vfp = NULL;
 	AZ(wrk->busyobj->vbc);
-	AN(sp->director);
+	AN(sp->req->director);
 
 	if (!i && wrk->obj->objcore != NULL) {
 		EXP_Insert(wrk->obj);
@@ -995,7 +996,7 @@ cnt_streambody(struct sess *sp)
 		sp->req->doclose = "Stream error";
 	}
 	wrk->acct_tmp.fetch++;
-	sp->director = NULL;
+	sp->req->director = NULL;
 	sp->req->restarts = 0;
 
 	RES_StreamEnd(sp);
@@ -1089,7 +1090,7 @@ cnt_hit(struct sess *sp)
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 
 	CHECK_OBJ_NOTNULL(wrk->obj, OBJECT_MAGIC);
-	CHECK_OBJ_NOTNULL(sp->vcl, VCL_CONF_MAGIC);
+	CHECK_OBJ_NOTNULL(sp->req->vcl, VCL_CONF_MAGIC);
 	AZ(wrk->busyobj);
 
 	assert(!(wrk->obj->objcore->flags & OC_F_PASS));
@@ -1117,7 +1118,7 @@ cnt_hit(struct sess *sp)
 		sp->step = STP_ERROR;
 		return (0);
 	case VCL_RET_RESTART:
-		sp->director = NULL;
+		sp->req->director = NULL;
 		sp->req->restarts++;
 		sp->step = STP_RECV;
 		return (0);
@@ -1166,7 +1167,7 @@ cnt_lookup(struct sess *sp)
 	wrk = sp->wrk;
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 
-	CHECK_OBJ_NOTNULL(sp->vcl, VCL_CONF_MAGIC);
+	CHECK_OBJ_NOTNULL(sp->req->vcl, VCL_CONF_MAGIC);
 	AZ(wrk->busyobj);
 
 	if (sp->req->hash_objhead == NULL) {
@@ -1275,7 +1276,7 @@ cnt_miss(struct sess *sp)
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 	wrk = sp->wrk;
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
-	CHECK_OBJ_NOTNULL(sp->vcl, VCL_CONF_MAGIC);
+	CHECK_OBJ_NOTNULL(sp->req->vcl, VCL_CONF_MAGIC);
 
 	AZ(wrk->obj);
 	AN(wrk->objcore);
@@ -1369,7 +1370,7 @@ cnt_pass(struct sess *sp)
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 	wrk = sp->wrk;
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
-	CHECK_OBJ_NOTNULL(sp->vcl, VCL_CONF_MAGIC);
+	CHECK_OBJ_NOTNULL(sp->req->vcl, VCL_CONF_MAGIC);
 	AZ(wrk->obj);
 	AZ(wrk->busyobj);
 
@@ -1429,7 +1430,7 @@ cnt_pipe(struct sess *sp)
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 	wrk = sp->wrk;
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
-	CHECK_OBJ_NOTNULL(sp->vcl, VCL_CONF_MAGIC);
+	CHECK_OBJ_NOTNULL(sp->req->vcl, VCL_CONF_MAGIC);
 	AZ(wrk->busyobj);
 
 	wrk->acct_tmp.pipe++;
@@ -1482,15 +1483,15 @@ cnt_recv(struct sess *sp)
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 	wrk = sp->wrk;
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
-	CHECK_OBJ_NOTNULL(sp->vcl, VCL_CONF_MAGIC);
+	CHECK_OBJ_NOTNULL(sp->req->vcl, VCL_CONF_MAGIC);
 	AZ(wrk->obj);
 	AZ(wrk->busyobj);
 	assert(wrk->wrw.ciov == wrk->wrw.siov);
 
 	/* By default we use the first backend */
-	AZ(sp->director);
-	sp->director = sp->vcl->director[0];
-	AN(sp->director);
+	AZ(sp->req->director);
+	sp->req->director = sp->req->vcl->director[0];
+	AN(sp->req->director);
 
 	sp->req->disable_esi = 0;
 	sp->req->hash_always_miss = 0;
@@ -1503,8 +1504,8 @@ cnt_recv(struct sess *sp)
 	recv_handling = sp->req->handling;
 
 	if (sp->req->restarts >= cache_param->max_restarts) {
-		if (sp->err_code == 0)
-			sp->err_code = 503;
+		if (sp->req->err_code == 0)
+			sp->req->err_code = 503;
 		sp->step = STP_ERROR;
 		return (0);
 	}
@@ -1584,7 +1585,7 @@ cnt_start(struct sess *sp)
 	CHECK_OBJ_NOTNULL(sp->req, REQ_MAGIC);
 	AZ(sp->req->restarts);
 	AZ(wrk->obj);
-	AZ(sp->vcl);
+	AZ(sp->req->vcl);
 	EXP_Clr(&sp->req->exp);
 	AZ(sp->req->esi_level);
 
@@ -1599,7 +1600,7 @@ cnt_start(struct sess *sp)
 
 	/* Borrow VCL reference from worker thread */
 	VCL_Refresh(&wrk->vcl);
-	sp->vcl = wrk->vcl;
+	sp->req->vcl = wrk->vcl;
 	wrk->vcl = NULL;
 
 	http_Setup(sp->http, sp->ws);
@@ -1613,13 +1614,13 @@ cnt_start(struct sess *sp)
 	}
 
 	/* Catch request snapshot */
-	sp->ws_req = WS_Snapshot(sp->ws);
+	sp->req->ws_req = WS_Snapshot(sp->ws);
 
 	/* Catch original request, before modification */
 	HTTP_Copy(sp->http0, sp->http);
 
 	if (done != 0) {
-		sp->err_code = done;
+		sp->req->err_code = done;
 		sp->step = STP_ERROR;
 		return (0);
 	}
@@ -1633,7 +1634,7 @@ cnt_start(struct sess *sp)
 	 */
 	if (http_GetHdr(sp->http, H_Expect, &p)) {
 		if (strcasecmp(p, "100-continue")) {
-			sp->err_code = 417;
+			sp->req->err_code = 417;
 			sp->step = STP_ERROR;
 			return (0);
 		}
@@ -1661,14 +1662,21 @@ cnt_start(struct sess *sp)
 static void
 cnt_diag(struct sess *sp, const char *state)
 {
+	void *vcl;
+
+	if (sp->req == NULL)
+		vcl = NULL;
+	else
+		vcl = sp->req->vcl;
+
 	if (sp->wrk != NULL) {
 		WSP(sp, SLT_Debug, "thr %p STP_%s sp %p obj %p vcl %p",
-		    pthread_self(), state, sp, sp->wrk->obj, sp->vcl);
+		    pthread_self(), state, sp, sp->wrk->obj, vcl);
 		WSL_Flush(sp->wrk, 0);
 	} else {
 		VSL(SLT_Debug, sp->vsl_id,
 		    "thr %p STP_%s sp %p obj %p vcl %p",
-		    pthread_self(), state, sp, sp->wrk->obj, sp->vcl);
+		    pthread_self(), state, sp, sp->wrk->obj, vcl);
 	}
 }
 
diff --git a/bin/varnishd/cache/cache_dir.c b/bin/varnishd/cache/cache_dir.c
index 5902ce8..4ec81fb 100644
--- a/bin/varnishd/cache/cache_dir.c
+++ b/bin/varnishd/cache/cache_dir.c
@@ -105,7 +105,7 @@ VDI_GetFd(const struct director *d, struct sess *sp)
 
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 	if (d == NULL)
-		d = sp->director;
+		d = sp->req->director;
 	CHECK_OBJ_NOTNULL(d, DIRECTOR_MAGIC);
 	return (d->getfd(d, sp));
 }
diff --git a/bin/varnishd/cache/cache_dir_dns.c b/bin/varnishd/cache/cache_dir_dns.c
index 56bce48..517caaa 100644
--- a/bin/varnishd/cache/cache_dir_dns.c
+++ b/bin/varnishd/cache/cache_dir_dns.c
@@ -403,8 +403,8 @@ vdi_dns_healthy(const struct director *dir, const struct sess *sp)
 	int i;
 
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
-	CHECK_OBJ_NOTNULL(sp->director, DIRECTOR_MAGIC);
-	CAST_OBJ_NOTNULL(vs, sp->director->priv, VDI_DNS_MAGIC);
+	CHECK_OBJ_NOTNULL(sp->req->director, DIRECTOR_MAGIC);
+	CAST_OBJ_NOTNULL(vs, sp->req->director->priv, VDI_DNS_MAGIC);
 
 	dir = vdi_dns_find_backend(sp, vs);
 
diff --git a/bin/varnishd/cache/cache_esi_deliver.c b/bin/varnishd/cache/cache_esi_deliver.c
index 36ff704..e1d3945 100644
--- a/bin/varnishd/cache/cache_esi_deliver.c
+++ b/bin/varnishd/cache/cache_esi_deliver.c
@@ -80,7 +80,7 @@ ved_include(struct sess *sp, const char *src, const char *host)
 	 * XXX: or not (for session/backend coupling).  Until then
 	 * XXX: make sure we don't trip up the check in vcl_recv.
 	 */
-	sp->director = NULL;
+	sp->req->director = NULL;
 	sp->step = STP_RECV;
 	http_ForceGet(sp->http);
 
diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c
index 79d0665..9f1d822 100644
--- a/bin/varnishd/cache/cache_fetch.c
+++ b/bin/varnishd/cache/cache_fetch.c
@@ -390,7 +390,7 @@ FetchHdr(struct sess *sp, int need_host_hdr)
 	CHECK_OBJ_NOTNULL(wrk->busyobj, BUSYOBJ_MAGIC);
 	htc = &wrk->busyobj->htc;
 
-	AN(sp->director);
+	AN(sp->req->director);
 	AZ(sp->wrk->obj);
 
 	if (sp->wrk->objcore != NULL) {		/* pass has no objcore */
diff --git a/bin/varnishd/cache/cache_hash.c b/bin/varnishd/cache/cache_hash.c
index 627db48..dbcca41 100644
--- a/bin/varnishd/cache/cache_hash.c
+++ b/bin/varnishd/cache/cache_hash.c
@@ -302,7 +302,7 @@ HSH_Lookup(struct sess *sp, struct objhead **poh)
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 	CHECK_OBJ_NOTNULL(sp->wrk, WORKER_MAGIC);
 	CHECK_OBJ_NOTNULL(sp->http, HTTP_MAGIC);
-	AN(sp->director);
+	AN(sp->req->director);
 	AN(hash);
 	wrk = sp->wrk;
 
@@ -393,7 +393,7 @@ HSH_Lookup(struct sess *sp, struct objhead **poh)
 	if (oc == NULL			/* We found no live object */
 	    && grace_oc != NULL		/* There is a grace candidate */
 	    && (busy_oc != NULL		/* Somebody else is already busy */
-	    || !VDI_Healthy(sp->director, sp))) {
+	    || !VDI_Healthy(sp->req->director, sp))) {
 					/* Or it is impossible to fetch */
 		o = oc_getobj(sp->wrk, grace_oc);
 		CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC);
diff --git a/bin/varnishd/cache/cache_panic.c b/bin/varnishd/cache/cache_panic.c
index 44c8fc3..9710517 100644
--- a/bin/varnishd/cache/cache_panic.c
+++ b/bin/varnishd/cache/cache_panic.c
@@ -254,10 +254,10 @@ pan_sess(const struct sess *sp)
 		VSB_printf(pan_vsp, "  handling = %s,\n", hand);
 	else
 		VSB_printf(pan_vsp, "  handling = 0x%x,\n", sp->req->handling);
-	if (sp->err_code)
+	if (sp->req->err_code)
 		VSB_printf(pan_vsp,
-		    "  err_code = %d, err_reason = %s,\n", sp->err_code,
-		    sp->err_reason ? sp->err_reason : "(null)");
+		    "  err_code = %d, err_reason = %s,\n", sp->req->err_code,
+		    sp->req->err_reason ? sp->req->err_reason : "(null)");
 
 	VSB_printf(pan_vsp, "  restarts = %d, esi_level = %d\n",
 	    sp->req->restarts, sp->req->esi_level);
@@ -271,8 +271,8 @@ pan_sess(const struct sess *sp)
 	if (sp->wrk != NULL)
 		pan_wrk(sp->wrk);
 
-	if (VALID_OBJ(sp->vcl, VCL_CONF_MAGIC))
-		pan_vcl(sp->vcl);
+	if (VALID_OBJ(sp->req->vcl, VCL_CONF_MAGIC))
+		pan_vcl(sp->req->vcl);
 
 	if (VALID_OBJ(sp->wrk->obj, OBJECT_MAGIC))
 		pan_object(sp->wrk->obj);
diff --git a/bin/varnishd/cache/cache_pipe.c b/bin/varnishd/cache/cache_pipe.c
index cc98fba..8bf76f3 100644
--- a/bin/varnishd/cache/cache_pipe.c
+++ b/bin/varnishd/cache/cache_pipe.c
@@ -94,7 +94,7 @@ PipeSession(struct sess *sp)
 		return;
 	}
 
-	sp->t_resp = VTIM_real();
+	sp->req->t_resp = VTIM_real();
 
 	memset(fds, 0, sizeof fds);
 
diff --git a/bin/varnishd/cache/cache_response.c b/bin/varnishd/cache/cache_response.c
index e5dbf86..fe5f29f 100644
--- a/bin/varnishd/cache/cache_response.c
+++ b/bin/varnishd/cache/cache_response.c
@@ -139,7 +139,7 @@ RES_BuildHttp(const struct sess *sp)
 		http_PrintfHeader(sp->wrk, sp->vsl_id, sp->wrk->resp,
 		    "X-Varnish: %u", sp->req->xid);
 	http_PrintfHeader(sp->wrk, sp->vsl_id, sp->wrk->resp, "Age: %.0f",
-	    sp->wrk->obj->exp.age + sp->t_resp - sp->wrk->obj->exp.entered);
+	    sp->wrk->obj->exp.age + sp->req->t_resp - sp->wrk->obj->exp.entered);
 	http_SetHeader(sp->wrk, sp->vsl_id, sp->wrk->resp, "Via: 1.1 varnish");
 	http_PrintfHeader(sp->wrk, sp->vsl_id, sp->wrk->resp, "Connection: %s",
 	    sp->req->doclose ? "close" : "keep-alive");
diff --git a/bin/varnishd/cache/cache_rfc2616.c b/bin/varnishd/cache/cache_rfc2616.c
index 2878d9a..5c5e5e2 100644
--- a/bin/varnishd/cache/cache_rfc2616.c
+++ b/bin/varnishd/cache/cache_rfc2616.c
@@ -98,7 +98,7 @@ RFC2616_Ttl(const struct sess *sp)
 	if (http_GetHdr(hp, H_Date, &p))
 		h_date = VTIM_parse(p);
 
-	switch (sp->err_code) {
+	switch (sp->req->err_code) {
 	default:
 		expp->ttl = -1.;
 		break;
diff --git a/bin/varnishd/cache/cache_session.c b/bin/varnishd/cache/cache_session.c
index f1f502a..42ce7df 100644
--- a/bin/varnishd/cache/cache_session.c
+++ b/bin/varnishd/cache/cache_session.c
@@ -82,7 +82,7 @@ SES_Charge(struct sess *sp)
 {
 	struct acct *a = &sp->wrk->acct_tmp;
 
-	sp->req_bodybytes += a->bodybytes;
+	sp->req->req_bodybytes += a->bodybytes;
 
 #define ACCT(foo)				\
 	sp->wrk->stats.s_##foo += a->foo;	\
@@ -170,7 +170,6 @@ ses_setup(struct sessmem *sm)
 	sp->t_open = NAN;
 	sp->t_idle = NAN;
 	sp->t_req = NAN;
-	sp->t_resp = NAN;
 
 	WS_Init(sp->ws, "sess", sm->wsp, sm->workspace);
 	sp->http = sm->http[0];
@@ -274,12 +273,12 @@ SES_Schedule(struct sess *sp)
 	if (Pool_Schedule(pp->pool, sp)) {
 		VSC_C_main->client_drop_late++;
 		sp->t_idle = VTIM_real();
-		if (sp->vcl != NULL) {
+		if (sp->req->vcl != NULL) {
 			/*
 			 * A session parked on a busy object can come here
 			 * after it wakes up.  Loose the VCL reference.
 			 */
-			VCL_Rel(&sp->vcl);
+			VCL_Rel(&sp->req->vcl);
 		}
 		SES_Delete(sp, "dropped", sp->t_idle);
 		return (1);
@@ -349,10 +348,11 @@ SES_Delete(struct sess *sp, const char *reason, double now)
 	assert(!isnan(sp->t_open));
 	assert(sp->fd < 0);
 
-	if (sp->req != NULL)
+	if (sp->req != NULL) {
+		AZ(sp->req->vcl);
 		SES_ReleaseReq(sp);
+	}
 
-	AZ(sp->vcl);
 	if (*sp->addr == '\0')
 		strcpy(sp->addr, "-");
 	if (*sp->port == '\0')
diff --git a/bin/varnishd/cache/cache_vcl.c b/bin/varnishd/cache/cache_vcl.c
index 06d4bd7..4cf61c4 100644
--- a/bin/varnishd/cache/cache_vcl.c
+++ b/bin/varnishd/cache/cache_vcl.c
@@ -336,7 +336,7 @@ VCL_##func##_method(struct sess *sp)					\
 	sp->req->handling = 0;						\
 	sp->req->cur_method = VCL_MET_ ## upper;				\
 	WSP(sp, SLT_VCL_call, "%s", #func);				\
-	(void)sp->vcl->func##_func(sp);					\
+	(void)sp->req->vcl->func##_func(sp);					\
 	WSP(sp, SLT_VCL_return, "%s", VCL_Return_Name(sp->req->handling));	\
 	sp->req->cur_method = 0;						\
 	assert((1U << sp->req->handling) & bitmap);				\
diff --git a/bin/varnishd/cache/cache_vrt.c b/bin/varnishd/cache/cache_vrt.c
index 9ec88cc..e078f8f 100644
--- a/bin/varnishd/cache/cache_vrt.c
+++ b/bin/varnishd/cache/cache_vrt.c
@@ -60,8 +60,8 @@ VRT_error(struct sess *sp, unsigned code, const char *reason)
 	    reason : "(null)");
 	if (code < 100 || code > 999)
 		code = 503;
-	sp->err_code = (uint16_t)code;
-	sp->err_reason = reason ? reason : http_StatusMessage(sp->err_code);
+	sp->req->err_code = (uint16_t)code;
+	sp->req->err_reason = reason ? reason : http_StatusMessage(sp->req->err_code);
 }
 
 /*--------------------------------------------------------------------*/
@@ -75,7 +75,7 @@ VRT_count(const struct sess *sp, unsigned u)
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 	if (cache_param->vcl_trace)
 		WSP(sp, SLT_VCL_trace, "%u %u.%u", u,
-		    sp->vcl->ref[u].line, sp->vcl->ref[u].pos);
+		    sp->req->vcl->ref[u].line, sp->req->vcl->ref[u].pos);
 }
 
 /*--------------------------------------------------------------------*/
@@ -351,7 +351,7 @@ VRT_backend_string(const struct sess *sp, const struct director *d)
 {
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 	if (d == NULL)
-		d = sp->director;
+		d = sp->req->director;
 	if (d == NULL)
 		return (NULL);
 	return (d->vcl_name);
@@ -372,7 +372,7 @@ VRT_Rollback(struct sess *sp)
 {
 
 	HTTP_Copy(sp->http, sp->http0);
-	WS_Reset(sp->ws, sp->ws_req);
+	WS_Reset(sp->ws, sp->req->ws_req);
 }
 
 /*--------------------------------------------------------------------*/
diff --git a/bin/varnishd/cache/cache_vrt_var.c b/bin/varnishd/cache/cache_vrt_var.c
index 8c95d5b..aef935b 100644
--- a/bin/varnishd/cache/cache_vrt_var.c
+++ b/bin/varnishd/cache/cache_vrt_var.c
@@ -307,14 +307,14 @@ void
 VRT_l_req_backend(struct sess *sp, struct director *be)
 {
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
-	sp->director = be;
+	sp->req->director = be;
 }
 
 struct director * __match_proto__()
 VRT_r_req_backend(struct sess *sp)
 {
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
-	return (sp->director);
+	return (sp->req->director);
 }
 
 /*--------------------------------------------------------------------*/
@@ -548,7 +548,7 @@ unsigned
 VRT_r_req_backend_healthy(const struct sess *sp)
 {
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
-	CHECK_OBJ_NOTNULL(sp->director, DIRECTOR_MAGIC);
-	return (VDI_Healthy(sp->director, sp));
+	CHECK_OBJ_NOTNULL(sp->req->director, DIRECTOR_MAGIC);
+	return (VDI_Healthy(sp->req->director, sp));
 }
 
diff --git a/bin/varnishd/cache/cache_wrw.c b/bin/varnishd/cache/cache_wrw.c
index 96fa3e5..c344f9c 100644
--- a/bin/varnishd/cache/cache_wrw.c
+++ b/bin/varnishd/cache/cache_wrw.c
@@ -159,7 +159,7 @@ WRW_Flush(struct worker *wrk)
 			 * counter to prevent slowlaris attacks
 			*/
 
-			if (VTIM_real() - wrk->sp->t_resp >
+			if (VTIM_real() - wrk->sp->req->t_resp >
 			    cache_param->send_timeout) {
 				WSL(wrk, SLT_Debug, *wrw->wfd,
 				    "Hit total send timeout, "
diff --git a/bin/varnishd/waiter/cache_waiter.c b/bin/varnishd/waiter/cache_waiter.c
index 44e9721..b0f461a 100644
--- a/bin/varnishd/waiter/cache_waiter.c
+++ b/bin/varnishd/waiter/cache_waiter.c
@@ -69,7 +69,6 @@ WAIT_Enter(struct sess *sp)
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 	CHECK_OBJ_NOTNULL(sp->wrk, WORKER_MAGIC);
 	AZ(sp->req);
-	AZ(sp->vcl);
 	assert(sp->fd >= 0);
 	sp->wrk = NULL;
 



From geoff at varnish-cache.org  Mon Jan  9 23:03:31 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Tue, 10 Jan 2012 00:03:31 +0100
Subject: [experimental-ims] a1d9126 Move another 8 bytes from sess to req
Message-ID: <E1RkOFX-0007v3-Kr@project.varnish-software.com>

commit a1d91261e761afa4f534911cb98a93d1473d78b2
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Fri Dec 23 12:55:42 2011 +0000

    Move another 8 bytes from sess to req

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index bf4754b..7517d1c 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -617,6 +617,7 @@ struct req {
 	double			t_resp;
 
 	struct http_conn	htc[1];
+	char			*client_identity;
 
 };
 
@@ -646,7 +647,6 @@ struct sess {
 	/* formatted ascii client address */
 	char			addr[ADDR_BUFSIZE];
 	char			port[PORT_BUFSIZE];
-	char			*client_identity;
 
 	VTAILQ_ENTRY(sess)	poollist;
 	struct acct		acct_ses;
diff --git a/bin/varnishd/cache/cache_center.c b/bin/varnishd/cache/cache_center.c
index b6d7a67..5816119 100644
--- a/bin/varnishd/cache/cache_center.c
+++ b/bin/varnishd/cache/cache_center.c
@@ -1498,7 +1498,7 @@ cnt_recv(struct sess *sp)
 	sp->req->disable_esi = 0;
 	sp->req->hash_always_miss = 0;
 	sp->req->hash_ignore_busy = 0;
-	sp->client_identity = NULL;
+	sp->req->client_identity = NULL;
 
 	http_CollectHdr(sp->http, H_Cache_Control);
 
diff --git a/bin/varnishd/cache/cache_dir_random.c b/bin/varnishd/cache/cache_dir_random.c
index 28a079c..b323343 100644
--- a/bin/varnishd/cache/cache_dir_random.c
+++ b/bin/varnishd/cache/cache_dir_random.c
@@ -105,8 +105,8 @@ vdi_random_init_seed(const struct vdi_random *vs, const struct sess *sp)
 
 	switch (vs->criteria) {
 	case c_client:
-		if (sp->client_identity != NULL)
-			p = sp->client_identity;
+		if (sp->req->client_identity != NULL)
+			p = sp->req->client_identity;
 		else
 			p = sp->addr;
 		retval = vdi_random_sha(p, strlen(p));
diff --git a/bin/varnishd/cache/cache_vrt_var.c b/bin/varnishd/cache/cache_vrt_var.c
index aef935b..7ac9768 100644
--- a/bin/varnishd/cache/cache_vrt_var.c
+++ b/bin/varnishd/cache/cache_vrt_var.c
@@ -209,8 +209,8 @@ const char * __match_proto__()
 VRT_r_client_identity(struct sess *sp)
 {
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
-	if (sp->client_identity != NULL)
-		return (sp->client_identity);
+	if (sp->req->client_identity != NULL)
+		return (sp->req->client_identity);
 	else
 		return (sp->addr);
 }
@@ -224,7 +224,7 @@ VRT_l_client_identity(struct sess *sp, const char *str, ...)
 	va_start(ap, str);
 	b = VRT_String(sp->http->ws, NULL, str, ap);
 	va_end(ap);
-	sp->client_identity = b;
+	sp->req->client_identity = b;
 }
 
 /*--------------------------------------------------------------------*/



From geoff at varnish-cache.org  Mon Jan  9 23:03:31 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Tue, 10 Jan 2012 00:03:31 +0100
Subject: [experimental-ims] e2081b7 Consequence changes of http/ws moving to
	struct req
Message-ID: <E1RkOFX-0007vJ-Q0@project.varnish-software.com>

commit e2081b7df599b83e31d486a1896f1dd9f8d7ac01
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sun Dec 25 10:09:19 2011 +0000

    Consequence changes of http/ws moving to struct req

diff --git a/bin/varnishtest/tests/r00693.vtc b/bin/varnishtest/tests/r00693.vtc
index ce65be5..24b6827 100644
--- a/bin/varnishtest/tests/r00693.vtc
+++ b/bin/varnishtest/tests/r00693.vtc
@@ -17,7 +17,7 @@ server s1 {
 
 } -start
 
-varnish v1 -arg "-p sess_workspace=1024" -vcl+backend {
+varnish v1 -arg "-p sess_workspace=3072" -vcl+backend {
 
 	sub vcl_recv {
 		set req.http.foo = 
@@ -35,23 +35,6 @@ varnish v1 -arg "-p sess_workspace=1024" -vcl+backend {
 			"0123456789abcdef" + "0123456789abcdef" +
 			"0123456789abcdef" + "0123456789abcdef" +
 			"0123456789abcdef" + "0123456789abcdef" +
-			"0123456789abcdef" + "0123456789abcdef" +
-			"0123456789abcdef" + "0123456789abcdef" +
-			"0123456789abcdef" + "0123456789abcdef" +
-			"0123456789abcdef" + "0123456789abcdef" +
-			"0123456789abcdef" + "0123456789abcdef" +
-			"0123456789abcdef" + "0123456789abcdef" +
-			"0123456789abcdef" + "0123456789abcdef" +
-			"0123456789abcdef" + "0123456789abcdef" +
-			"0123456789abcdef" + "0123456789abcdef" +
-			"0123456789abcdef" + "0123456789abcdef" +
-			"0123456789abcdef" + "0123456789abcdef" +
-			"0123456789abcdef" + "0123456789abcdef" +
-			"0123456789abcdef" + "0123456789abcdef" +
-			"0123456789abcdef" + "0123456789abcdef" +
-			"0123456789abcdef" + "0123456789abcdef" +
-			"0123456789abcdef" + "0123456789abcdef" +
-			"0123456789abcdef" + "0123456789abcdef" +
 			"0123456789abcde";
 		set req.http.baz = "BAZ";
 		return (pass);



From geoff at varnish-cache.org  Mon Jan  9 23:03:31 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Tue, 10 Jan 2012 00:03:31 +0100
Subject: [experimental-ims] ac2e067 Remove the old sessmem allocator/freelist
	stuff, and replace with a run of the mill mempool.
Message-ID: <E1RkOFX-0007vb-Vw@project.varnish-software.com>

commit ac2e067b0daced3cc24b16b0b14b7be10c5ceb27
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sun Dec 25 10:28:52 2011 +0000

    Remove the old sessmem allocator/freelist stuff, and replace with
    a run of the mill mempool.

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index d1dc4fb..9743398 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -634,9 +634,9 @@ struct sess {
 
 	/* Cross references ------------------------------------------*/
 
+	struct sesspool		*sesspool;
 	struct worker		*wrk;
 	struct req		*req;
-	struct sessmem		*mem;
 
 	/* Session related fields ------------------------------------*/
 
diff --git a/bin/varnishd/cache/cache_session.c b/bin/varnishd/cache/cache_session.c
index 92a7357..552c078 100644
--- a/bin/varnishd/cache/cache_session.c
+++ b/bin/varnishd/cache/cache_session.c
@@ -44,31 +44,16 @@
 #include "waiter/waiter.h"
 #include "vtim.h"
 
-/*--------------------------------------------------------------------*/
-
-struct sessmem {
-	unsigned		magic;
-#define SESSMEM_MAGIC		0x555859c5
-
-	struct sesspool		*pool;
-
-	unsigned		workspace;
-	uint16_t		nhttp;
-	void			*wsp;
-	VTAILQ_ENTRY(sessmem)	list;
+static unsigned ses_size = sizeof (struct sess);
 
-	struct sess		sess;
-};
+/*--------------------------------------------------------------------*/
 
 struct sesspool {
 	unsigned		magic;
 #define SESSPOOL_MAGIC		0xd916e202
 	struct pool		*pool;
-	VTAILQ_HEAD(,sessmem)	freelist;
-	struct lock		mtx;
-	unsigned		nsess;
-	unsigned		dly_free_cnt;
 	struct mempool		*mpl_req;
+	struct mempool		*mpl_sess;
 };
 
 /*--------------------------------------------------------------------
@@ -92,77 +77,14 @@ SES_Charge(struct sess *sp)
 }
 
 /*--------------------------------------------------------------------
- * This function allocates a session + assorted peripheral data
- * structures in one single malloc operation.
- */
-
-static struct sessmem *
-ses_sm_alloc(void)
-{
-	struct sessmem *sm;
-	unsigned char *p, *q;
-	unsigned nws;
-	uint16_t nhttp;
-	unsigned l, hl;
-
-	/*
-	 * It is not necessary to lock these, but we need to
-	 * cache them locally, to make sure we get a consistent
-	 * view of the value.
-	 */
-	nws = cache_param->sess_workspace;
-	nhttp = (uint16_t)cache_param->http_max_hdr;
-
-	hl = HTTP_estimate(nhttp);
-	l = sizeof *sm + nws + 2 * hl;
-	VSC_C_main->sessmem_size = l;
-	p = malloc(l);
-	if (p == NULL)
-		return (NULL);
-	q = p + l;
-
-	/* Don't waste time zeroing the workspace */
-	memset(p, 0, l - nws);
-
-	sm = (void*)p;
-	p += sizeof *sm;
-
-	sm->magic = SESSMEM_MAGIC;
-	sm->workspace = nws;
-	sm->nhttp = nhttp;
-
-	sm->http[0] = HTTP_create(p, nhttp);
-	p += hl;
-
-	sm->http[1] = HTTP_create(p, nhttp);
-	p += hl;
-
-	sm->wsp = p;
-	p += nws;
-
-	assert(p == q);
-
-	return (sm);
-}
-
-/*--------------------------------------------------------------------
  * This prepares a session for use, based on its sessmem structure.
  */
 
 static void
-ses_setup(struct sessmem *sm)
+ses_setup(struct sess *sp)
 {
-	struct sess *sp;
-
-	CHECK_OBJ_NOTNULL(sm, SESSMEM_MAGIC);
-	sp = &sm->sess;
-	memset(sp, 0, sizeof *sp);
 
-	/* We assume that the sess has been zeroed by the time we get here */
-	AZ(sp->magic);
-
-	sp->magic = SESS_MAGIC;
-	sp->mem = sm;
+	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 	sp->sockaddrlen = sizeof(sp->sockaddr);
 	sp->mysockaddrlen = sizeof(sp->mysockaddr);
 	sp->sockaddr.ss_family = sp->mysockaddr.ss_family = PF_UNSPEC;
@@ -178,39 +100,14 @@ ses_setup(struct sessmem *sm)
 struct sess *
 SES_New(struct worker *wrk, struct sesspool *pp)
 {
-	struct sessmem *sm;
 	struct sess *sp;
-	int do_alloc;
 
+	(void)wrk;					// XXX
 	CHECK_OBJ_NOTNULL(pp, SESSPOOL_MAGIC);
-
-	do_alloc = 0;
-	Lck_Lock(&pp->mtx);
-	sm = VTAILQ_FIRST(&pp->freelist);
-	if (sm != NULL) {
-		VTAILQ_REMOVE(&pp->freelist, sm, list);
-	} else if (pp->nsess < cache_param->max_sess) {
-		pp->nsess++;
-		do_alloc = 1;
-	}
-	wrk->stats.sessmem_free += pp->dly_free_cnt;
-	pp->dly_free_cnt = 0;
-	Lck_Unlock(&pp->mtx);
-	if (do_alloc) {
-		sm = ses_sm_alloc();
-		if (sm != NULL) {
-			wrk->stats.sessmem_alloc++;
-			sm->pool = pp;
-			ses_setup(sm);
-		} else {
-			wrk->stats.sessmem_fail++;
-		}
-	} else if (sm == NULL) {
-		wrk->stats.sessmem_limit++;
-	}
-	if (sm == NULL)
-		return (NULL);
-	sp = &sm->sess;
+	sp = MPL_Get(pp->mpl_sess, NULL);
+	sp->magic = SESS_MAGIC;
+	sp->sesspool = pp;
+	ses_setup(sp);
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 	return (sp);
 }
@@ -223,35 +120,15 @@ struct sess *
 SES_Alloc(void)
 {
 	struct sess *sp;
-	struct sessmem *sm;
 
-	sm = ses_sm_alloc();
-	AN(sm);
-	ses_setup(sm);
-	sp = &sm->sess;
-	sp->sockaddrlen = 0;
+	ALLOC_OBJ(sp, SESS_MAGIC);
+	AN(sp);
+	ses_setup(sp);
 	/* XXX: sp->req ? */
 	return (sp);
 }
 
 /*--------------------------------------------------------------------
- */
-
-static struct sesspool *
-ses_getpool(const struct sess *sp)
-{
-	struct sessmem *sm;
-	struct sesspool *pp;
-
-	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
-	sm = sp->mem;
-	CHECK_OBJ_NOTNULL(sm, SESSMEM_MAGIC);
-	pp = sm->pool;
-	CHECK_OBJ_NOTNULL(pp, SESSPOOL_MAGIC);
-	return (pp);
-}
-
-/*--------------------------------------------------------------------
  * Schedule a session back on a work-thread from its pool
  */
 
@@ -260,9 +137,10 @@ SES_Schedule(struct sess *sp)
 {
 	struct sesspool *pp;
 
-	pp = ses_getpool(sp);
+	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 	AZ(sp->wrk);
-
+	pp = sp->sesspool;
+	CHECK_OBJ_NOTNULL(pp, SESSPOOL_MAGIC);
 	AN(pp->pool);
 
 	if (Pool_Schedule(pp->pool, sp)) {
@@ -325,14 +203,14 @@ void
 SES_Delete(struct sess *sp, const char *reason, double now)
 {
 	struct acct *b;
-	struct sessmem *sm;
 	struct worker *wrk;
 	struct sesspool *pp;
 
-	pp = ses_getpool(sp);
+	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+	pp = sp->sesspool;
+	CHECK_OBJ_NOTNULL(pp, SESSPOOL_MAGIC);
+	AN(pp->pool);
 
-	sm = sp->mem;
-	CHECK_OBJ_NOTNULL(sm, SESSMEM_MAGIC);
 	wrk = sp->wrk;
 	CHECK_OBJ_ORNULL(wrk, WORKER_MAGIC);
 
@@ -361,28 +239,8 @@ SES_Delete(struct sess *sp, const char *reason, double now)
 	    b->sess, b->req, b->pipe, b->pass,
 	    b->fetch, b->hdrbytes, b->bodybytes);
 
-	if (sm->workspace != cache_param->sess_workspace ||
-	    sm->nhttp != (uint16_t)cache_param->http_max_hdr ||
-	    pp->nsess > cache_param->max_sess) {
-		free(sm);
-		Lck_Lock(&pp->mtx);
-		if (wrk != NULL)
-			wrk->stats.sessmem_free++;
-		else
-			pp->dly_free_cnt++;
-		pp->nsess--;
-		Lck_Unlock(&pp->mtx);
-	} else {
-		/* Clean and prepare for reuse */
-		ses_setup(sm);
-		Lck_Lock(&pp->mtx);
-		if (wrk != NULL) {
-			wrk->stats.sessmem_free += pp->dly_free_cnt;
-			pp->dly_free_cnt = 0;
-		}
-		VTAILQ_INSERT_HEAD(&pp->freelist, sm, list);
-		Lck_Unlock(&pp->mtx);
-	}
+	MPL_Free(pp->mpl_sess, sp);
+
 }
 
 /*--------------------------------------------------------------------
@@ -397,7 +255,11 @@ SES_GetReq(struct sess *sp)
 	unsigned sz, hl;
 	char *p;
 
-	pp = ses_getpool(sp);
+	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+	pp = sp->sesspool;
+	CHECK_OBJ_NOTNULL(pp, SESSPOOL_MAGIC);
+	AN(pp->pool);
+
 	AZ(sp->req);
 	sp->req = MPL_Get(pp->mpl_req, &sz);
 	AN(sp->req);
@@ -427,7 +289,10 @@ SES_ReleaseReq(struct sess *sp)
 {
 	struct sesspool *pp;
 
-	pp = ses_getpool(sp);
+	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+	pp = sp->sesspool;
+	CHECK_OBJ_NOTNULL(pp, SESSPOOL_MAGIC);
+	AN(pp->pool);
 	CHECK_OBJ_NOTNULL(sp->req, REQ_MAGIC);
 	MPL_AssertSane(sp->req);
 	MPL_Free(pp->mpl_req, sp->req);
@@ -447,33 +312,20 @@ SES_NewPool(struct pool *wp, unsigned pool_no)
 	ALLOC_OBJ(pp, SESSPOOL_MAGIC);
 	AN(pp);
 	pp->pool = wp;
-	VTAILQ_INIT(&pp->freelist);
-	Lck_New(&pp->mtx, lck_sessmem);
 	bprintf(nb, "req%u", pool_no);
 	pp->mpl_req = MPL_New(nb, &cache_param->req_pool,
 	    &cache_param->sess_workspace);
+	pp->mpl_sess = MPL_New(nb, &cache_param->sess_pool, &ses_size);
 	return (pp);
 }
 
 void
 SES_DeletePool(struct sesspool *pp, struct worker *wrk)
 {
-	struct sessmem *sm;
 
 	CHECK_OBJ_NOTNULL(pp, SESSPOOL_MAGIC);
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
-	Lck_Lock(&pp->mtx);
-	while (!VTAILQ_EMPTY(&pp->freelist)) {
-		sm = VTAILQ_FIRST(&pp->freelist);
-		CHECK_OBJ_NOTNULL(sm, SESSMEM_MAGIC);
-		VTAILQ_REMOVE(&pp->freelist, sm, list);
-		FREE_OBJ(sm);
-		wrk->stats.sessmem_free++;
-		pp->nsess--;
-	}
-	AZ(pp->nsess);
-	Lck_Unlock(&pp->mtx);
-	Lck_Delete(&pp->mtx);
+	MPL_Destroy(&pp->mpl_sess);
 	MPL_Destroy(&pp->mpl_req);
 	FREE_OBJ(pp);
 }
diff --git a/bin/varnishd/common/params.h b/bin/varnishd/common/params.h
index 0a476d4..ca39ff7 100644
--- a/bin/varnishd/common/params.h
+++ b/bin/varnishd/common/params.h
@@ -201,4 +201,5 @@ struct params {
 
 	struct poolparam	vbc_pool;
 	struct poolparam	req_pool;
+	struct poolparam	sess_pool;
 };
diff --git a/bin/varnishd/mgt/mgt_param.c b/bin/varnishd/mgt/mgt_param.c
index 81aaf0d..36f96cd 100644
--- a/bin/varnishd/mgt/mgt_param.c
+++ b/bin/varnishd/mgt/mgt_param.c
@@ -1221,6 +1221,14 @@ static const struct parspec input_parspec[] = {
 		"   max_age -- max age of free element.\n",
 		0,
 		"10,100,10", ""},
+	{ "pool_sess", tweak_poolparam, &mgt_param.sess_pool, 0, 10000,
+		"Parameters for per worker pool session memory pool.\n"
+		"The three numbers are:\n"
+		"   min_pool -- minimum size of free pool.\n"
+		"   max_pool -- maximum size of free pool.\n"
+		"   max_age -- max age of free element.\n",
+		0,
+		"10,100,10", ""},
 
 	{ NULL, NULL, NULL }
 };



From geoff at varnish-cache.org  Mon Jan  9 23:03:32 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Tue, 10 Jan 2012 00:03:32 +0100
Subject: [experimental-ims] 718e877 Wrap lines
Message-ID: <E1RkOFY-0007vx-4x@project.varnish-software.com>

commit 718e87721891249567bac736ea5e03c11ee63f77
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sun Dec 25 11:05:14 2011 +0000

    Wrap lines

diff --git a/bin/varnishd/cache/cache_backend.c b/bin/varnishd/cache/cache_backend.c
index 0b3c261..24aedb4 100644
--- a/bin/varnishd/cache/cache_backend.c
+++ b/bin/varnishd/cache/cache_backend.c
@@ -348,7 +348,8 @@ vbe_GetVbe(const struct sess *sp, struct vdi_simple *vs)
 			/* XXX locking of stats */
 			VSC_C_main->backend_reuse += 1;
 			WSP(sp, SLT_Backend, "%d %s %s",
-			    vc->fd, sp->req->director->vcl_name, bp->display_name);
+			    vc->fd, sp->req->director->vcl_name,
+			    bp->display_name);
 			vc->vdis = vs;
 			vc->recycled = 1;
 			return (vc);
diff --git a/bin/varnishd/cache/cache_hash.c b/bin/varnishd/cache/cache_hash.c
index 5d8c880..9f2c053 100644
--- a/bin/varnishd/cache/cache_hash.c
+++ b/bin/varnishd/cache/cache_hash.c
@@ -307,7 +307,8 @@ HSH_Lookup(struct sess *sp, struct objhead **poh)
 	wrk = sp->wrk;
 
 	HSH_Prealloc(sp);
-	memcpy(sp->wrk->nobjhead->digest, sp->req->digest, sizeof sp->req->digest);
+	memcpy(sp->wrk->nobjhead->digest, sp->req->digest,
+	    sizeof sp->req->digest);
 	if (cache_param->diag_bitmap & 0x80000000)
 		hsh_testmagic(sp->wrk->nobjhead->digest);
 
diff --git a/bin/varnishd/cache/cache_http.c b/bin/varnishd/cache/cache_http.c
index cb17df4..f441ef2 100644
--- a/bin/varnishd/cache/cache_http.c
+++ b/bin/varnishd/cache/cache_http.c
@@ -880,7 +880,8 @@ http_FilterHeader(const struct sess *sp, unsigned how)
 	else
 		http_copyh(hp, sp->req->http, HTTP_HDR_PROTO);
 	http_FilterFields(sp->wrk, sp->vsl_id, hp, sp->req->http, how);
-	http_PrintfHeader(sp->wrk, sp->vsl_id, hp, "X-Varnish: %u", sp->req->xid);
+	http_PrintfHeader(sp->wrk, sp->vsl_id, hp,
+	    "X-Varnish: %u", sp->req->xid);
 }
 
 /*--------------------------------------------------------------------
diff --git a/bin/varnishd/cache/cache_pipe.c b/bin/varnishd/cache/cache_pipe.c
index a8282d7..21fefef 100644
--- a/bin/varnishd/cache/cache_pipe.c
+++ b/bin/varnishd/cache/cache_pipe.c
@@ -84,7 +84,8 @@ PipeSession(struct sess *sp)
 
 	if (sp->req->htc->pipeline.b != NULL)
 		sp->wrk->acct_tmp.bodybytes +=
-		    WRW_Write(w, sp->req->htc->pipeline.b, Tlen(sp->req->htc->pipeline));
+		    WRW_Write(w, sp->req->htc->pipeline.b,
+		    Tlen(sp->req->htc->pipeline));
 
 	i = WRW_FlushRelease(w);
 
diff --git a/bin/varnishd/cache/cache_response.c b/bin/varnishd/cache/cache_response.c
index 6b2ea78..2398e5b 100644
--- a/bin/varnishd/cache/cache_response.c
+++ b/bin/varnishd/cache/cache_response.c
@@ -139,7 +139,8 @@ RES_BuildHttp(const struct sess *sp)
 		http_PrintfHeader(sp->wrk, sp->vsl_id, sp->wrk->resp,
 		    "X-Varnish: %u", sp->req->xid);
 	http_PrintfHeader(sp->wrk, sp->vsl_id, sp->wrk->resp, "Age: %.0f",
-	    sp->wrk->obj->exp.age + sp->req->t_resp - sp->wrk->obj->exp.entered);
+	    sp->wrk->obj->exp.age + sp->req->t_resp -
+	    sp->wrk->obj->exp.entered);
 	http_SetHeader(sp->wrk, sp->vsl_id, sp->wrk->resp, "Via: 1.1 varnish");
 	http_PrintfHeader(sp->wrk, sp->vsl_id, sp->wrk->resp, "Connection: %s",
 	    sp->req->doclose ? "close" : "keep-alive");
diff --git a/bin/varnishd/cache/cache_vary.c b/bin/varnishd/cache/cache_vary.c
index d524fd3..453b52d 100644
--- a/bin/varnishd/cache/cache_vary.c
+++ b/bin/varnishd/cache/cache_vary.c
@@ -187,7 +187,8 @@ VRY_Match(const struct sess *sp, const uint8_t *vary)
 		if (i == 1) {
 			/* Build a new entry */
 
-			i = http_GetHdr(sp->req->http, (const char*)(vary+2), &h);
+			i = http_GetHdr(sp->req->http,
+			    (const char*)(vary+2), &h);
 			if (i) {
 				/* Trim trailing space */
 				e = strchr(h, '\0');
diff --git a/bin/varnishd/cache/cache_vcl.c b/bin/varnishd/cache/cache_vcl.c
index 4cf61c4..ad5acf0 100644
--- a/bin/varnishd/cache/cache_vcl.c
+++ b/bin/varnishd/cache/cache_vcl.c
@@ -334,12 +334,12 @@ VCL_##func##_method(struct sess *sp)					\
 {									\
 									\
 	sp->req->handling = 0;						\
-	sp->req->cur_method = VCL_MET_ ## upper;				\
+	sp->req->cur_method = VCL_MET_ ## upper;			\
 	WSP(sp, SLT_VCL_call, "%s", #func);				\
-	(void)sp->req->vcl->func##_func(sp);					\
-	WSP(sp, SLT_VCL_return, "%s", VCL_Return_Name(sp->req->handling));	\
-	sp->req->cur_method = 0;						\
-	assert((1U << sp->req->handling) & bitmap);				\
+	(void)sp->req->vcl->func##_func(sp);				\
+	WSP(sp, SLT_VCL_return, "%s", VCL_Return_Name(sp->req->handling)); \
+	sp->req->cur_method = 0;					\
+	assert((1U << sp->req->handling) & bitmap);			\
 	assert(!((1U << sp->req->handling) & ~bitmap));			\
 }
 



From geoff at varnish-cache.org  Mon Jan  9 23:03:32 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Tue, 10 Jan 2012 00:03:32 +0100
Subject: [experimental-ims] 2955f0b Wrap a line
Message-ID: <E1RkOFY-0007wI-B0@project.varnish-software.com>

commit 2955f0ba479c4e1b036c2a7b77c7381169f6b348
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sun Dec 25 14:36:42 2011 +0000

    Wrap a line

diff --git a/bin/varnishd/cache/cache_vrt.c b/bin/varnishd/cache/cache_vrt.c
index da50c44..67875bd 100644
--- a/bin/varnishd/cache/cache_vrt.c
+++ b/bin/varnishd/cache/cache_vrt.c
@@ -61,7 +61,8 @@ VRT_error(const struct sess *sp, unsigned code, const char *reason)
 	if (code < 100 || code > 999)
 		code = 503;
 	sp->req->err_code = (uint16_t)code;
-	sp->req->err_reason = reason ? reason : http_StatusMessage(sp->req->err_code);
+	sp->req->err_reason =
+	    reason ? reason : http_StatusMessage(sp->req->err_code);
 }
 
 /*--------------------------------------------------------------------*/



From geoff at varnish-cache.org  Mon Jan  9 23:03:32 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Tue, 10 Jan 2012 00:03:32 +0100
Subject: [experimental-ims] 41608b6 Always call struct worker variables "wrk"
	to ease grep'ing
Message-ID: <E1RkOFY-0007wU-FG@project.varnish-software.com>

commit 41608b69413078adb0d22db0c664665e3843c6df
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Dec 26 09:04:54 2011 +0000

    Always call struct worker variables "wrk" to ease grep'ing

diff --git a/bin/varnishd/cache/cache_acceptor.c b/bin/varnishd/cache/cache_acceptor.c
index da35db4..d55545b 100644
--- a/bin/varnishd/cache/cache_acceptor.c
+++ b/bin/varnishd/cache/cache_acceptor.c
@@ -220,15 +220,15 @@ VCA_Accept(struct listen_sock *ls, struct wrk_accept *wa)
  */
 
 void
-VCA_FailSess(struct worker *w)
+VCA_FailSess(struct worker *wrk)
 {
 	struct wrk_accept *wa;
 
-	CHECK_OBJ_NOTNULL(w, WORKER_MAGIC);
-	CAST_OBJ_NOTNULL(wa, (void*)w->ws->f, WRK_ACCEPT_MAGIC);
-	AZ(w->sp);
+	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
+	CAST_OBJ_NOTNULL(wa, (void*)wrk->ws->f, WRK_ACCEPT_MAGIC);
+	AZ(wrk->sp);
 	AZ(close(wa->acceptsock));
-	w->stats.sess_drop++;
+	wrk->stats.sess_drop++;
 	vca_pace_bad();
 }
 
@@ -237,14 +237,14 @@ VCA_FailSess(struct worker *w)
  */
 
 void
-VCA_SetupSess(struct worker *w)
+VCA_SetupSess(struct worker *wrk)
 {
 	struct sess *sp;
 	struct wrk_accept *wa;
 
-	CHECK_OBJ_NOTNULL(w, WORKER_MAGIC);
-	CAST_OBJ_NOTNULL(wa, (void*)w->ws->f, WRK_ACCEPT_MAGIC);
-	sp = w->sp;
+	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
+	CAST_OBJ_NOTNULL(wa, (void*)wrk->ws->f, WRK_ACCEPT_MAGIC);
+	sp = wrk->sp;
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 	sp->fd = wa->acceptsock;
 	sp->vsl_id = wa->acceptsock | VSL_CLIENTMARKER ;
@@ -258,7 +258,7 @@ VCA_SetupSess(struct worker *w)
 	memcpy(&sp->sockaddr, &wa->acceptaddr, wa->acceptaddrlen);
 	sp->sockaddrlen = wa->acceptaddrlen;
 	vca_pace_good();
-	w->stats.sess_conn++;
+	wrk->stats.sess_conn++;
 
 	if (need_test)
 		sock_test(sp->fd);
diff --git a/bin/varnishd/cache/cache_cli.c b/bin/varnishd/cache/cache_cli.c
index abb9e43..2432691 100644
--- a/bin/varnishd/cache/cache_cli.c
+++ b/bin/varnishd/cache/cache_cli.c
@@ -127,6 +127,7 @@ cli_debug_sizeof(struct cli *cli, const char * const *av, void *priv)
 	SZOF(struct http_conn);
 	SZOF(struct acct);
 	SZOF(struct worker);
+	SZOF(struct wrk_accept);
 	SZOF(struct storage);
 	SZOF(struct object);
 	SZOF(struct objcore);
diff --git a/bin/varnishd/cache/cache_pool.c b/bin/varnishd/cache/cache_pool.c
index 7b1471e..2236f0b 100644
--- a/bin/varnishd/cache/cache_pool.c
+++ b/bin/varnishd/cache/cache_pool.c
@@ -129,20 +129,20 @@ static pthread_t		thr_pool_herder;
  */
 
 static int
-pool_accept(struct pool *pp, struct worker *w, const struct poolsock *ps)
+pool_accept(struct pool *pp, struct worker *wrk, const struct poolsock *ps)
 {
-	struct worker *w2;
+	struct worker *wrk2;
 	struct wrk_accept *wa, *wa2;
 
 	CHECK_OBJ_NOTNULL(pp, POOL_MAGIC);
-	CHECK_OBJ_NOTNULL(w, WORKER_MAGIC);
+	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 	CHECK_OBJ_NOTNULL(ps, POOLSOCK_MAGIC);
 
 	CHECK_OBJ_NOTNULL(ps->lsock, LISTEN_SOCK_MAGIC);
 	Lck_AssertHeld(&pp->mtx);
 	Lck_Unlock(&pp->mtx);
-	assert(sizeof *wa == WS_Reserve(w->ws, sizeof *wa));
-	wa = (void*)w->ws->f;
+	assert(sizeof *wa == WS_Reserve(wrk->ws, sizeof *wa));
+	wa = (void*)wrk->ws->f;
 	while (1) {
 		memset(wa, 0, sizeof *wa);
 		wa->magic = WRK_ACCEPT_MAGIC;
@@ -153,23 +153,23 @@ pool_accept(struct pool *pp, struct worker *w, const struct poolsock *ps)
 			return (-1);
 		}
 		if (VCA_Accept(ps->lsock, wa) < 0) {
-			w->stats.sess_fail++;
+			wrk->stats.sess_fail++;
 			/* We're going to pace in vca anyway... */
-			(void)WRK_TrySumStat(w);
+			(void)WRK_TrySumStat(wrk);
 			continue;
 		}
 
 		Lck_Lock(&pp->mtx);
 		if (VTAILQ_EMPTY(&pp->idle))
 			return (0);
-		w2 = VTAILQ_FIRST(&pp->idle);
-		VTAILQ_REMOVE(&pp->idle, w2, list);
+		wrk2 = VTAILQ_FIRST(&pp->idle);
+		VTAILQ_REMOVE(&pp->idle, wrk2, list);
 		Lck_Unlock(&pp->mtx);
-		assert(sizeof *wa2 == WS_Reserve(w2->ws, sizeof *wa2));
-		wa2 = (void*)w2->ws->f;
+		assert(sizeof *wa2 == WS_Reserve(wrk2->ws, sizeof *wa2));
+		wa2 = (void*)wrk2->ws->f;
 		memcpy(wa2, wa, sizeof *wa);
-		w2->do_what = pool_do_accept;
-		AZ(pthread_cond_signal(&w2->cond));
+		wrk2->do_what = pool_do_accept;
+		AZ(pthread_cond_signal(&wrk2->cond));
 	}
 }
 
@@ -178,111 +178,111 @@ pool_accept(struct pool *pp, struct worker *w, const struct poolsock *ps)
  */
 
 void
-Pool_Work_Thread(void *priv, struct worker *w)
+Pool_Work_Thread(void *priv, struct worker *wrk)
 {
 	struct pool *pp;
 	int stats_clean, i;
 	struct poolsock *ps;
 
 	CAST_OBJ_NOTNULL(pp, priv, POOL_MAGIC);
-	w->pool = pp;
+	wrk->pool = pp;
 	Lck_Lock(&pp->mtx);
 	stats_clean = 1;
 	while (1) {
 
 		Lck_AssertHeld(&pp->mtx);
-		w->do_what = pool_do_inval;
+		wrk->do_what = pool_do_inval;
 
-		CHECK_OBJ_NOTNULL(w, WORKER_MAGIC);
+		CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 
-		WS_Reset(w->ws, NULL);
+		WS_Reset(wrk->ws, NULL);
 
-		w->sp = VTAILQ_FIRST(&pp->queue);
-		if (w->sp != NULL) {
+		wrk->sp = VTAILQ_FIRST(&pp->queue);
+		if (wrk->sp != NULL) {
 			/* Process queued requests, if any */
 			assert(pp->lqueue > 0);
-			VTAILQ_REMOVE(&pp->queue, w->sp, poollist);
-			w->do_what = pool_do_sess;
+			VTAILQ_REMOVE(&pp->queue, wrk->sp, poollist);
+			wrk->do_what = pool_do_sess;
 			pp->lqueue--;
 		} else if (!VTAILQ_EMPTY(&pp->socks)) {
 			/* Accept on a socket */
 			ps = VTAILQ_FIRST(&pp->socks);
 			VTAILQ_REMOVE(&pp->socks, ps, list);
-			i = pool_accept(pp, w, ps);
+			i = pool_accept(pp, wrk, ps);
 			Lck_AssertHeld(&pp->mtx);
 			if (i < 0) {
 				/* Socket Shutdown */
 				FREE_OBJ(ps);
-				WS_Release(w->ws, 0);
+				WS_Release(wrk->ws, 0);
 				continue;
 			}
 			VTAILQ_INSERT_TAIL(&pp->socks, ps, list);
-			w->do_what = pool_do_accept;
+			wrk->do_what = pool_do_accept;
 		} else if (VTAILQ_EMPTY(&pp->socks)) {
 			/* Nothing to do: To sleep, perchance to dream ... */
-			if (isnan(w->lastused))
-				w->lastused = VTIM_real();
-			VTAILQ_INSERT_HEAD(&pp->idle, w, list);
+			if (isnan(wrk->lastused))
+				wrk->lastused = VTIM_real();
+			VTAILQ_INSERT_HEAD(&pp->idle, wrk, list);
 			if (!stats_clean)
-				WRK_SumStat(w);
-			(void)Lck_CondWait(&w->cond, &pp->mtx, NULL);
+				WRK_SumStat(wrk);
+			(void)Lck_CondWait(&wrk->cond, &pp->mtx, NULL);
 		}
 
-		if (w->do_what == pool_do_die)
+		if (wrk->do_what == pool_do_die)
 			break;
 
 		Lck_Unlock(&pp->mtx);
 
-		if (w->do_what == pool_do_accept) {
+		if (wrk->do_what == pool_do_accept) {
 			/* Turn accepted socket into a session */
-			AZ(w->sp);
-			AN(w->ws->r);
-			w->sp = SES_New(pp->sesspool);
-			if (w->sp == NULL) {
-				VCA_FailSess(w);
-				w->do_what = pool_do_nothing;
+			AZ(wrk->sp);
+			AN(wrk->ws->r);
+			wrk->sp = SES_New(pp->sesspool);
+			if (wrk->sp == NULL) {
+				VCA_FailSess(wrk);
+				wrk->do_what = pool_do_nothing;
 			} else {
-				VCA_SetupSess(w);
-				w->sp->step = STP_FIRST;
-				w->do_what = pool_do_sess;
+				VCA_SetupSess(wrk);
+				wrk->sp->step = STP_FIRST;
+				wrk->do_what = pool_do_sess;
 			}
-			WS_Release(w->ws, 0);
+			WS_Release(wrk->ws, 0);
 		}
 
-		if (w->do_what == pool_do_sess) {
-			CHECK_OBJ_NOTNULL(w->sp, SESS_MAGIC);
-			AZ(w->ws->r);
+		if (wrk->do_what == pool_do_sess) {
+			CHECK_OBJ_NOTNULL(wrk->sp, SESS_MAGIC);
+			AZ(wrk->ws->r);
 
 			stats_clean = 0;
-			w->lastused = NAN;
-			w->storage_hint = NULL;
+			wrk->lastused = NAN;
+			wrk->storage_hint = NULL;
 
-			AZ(w->sp->wrk);
-			THR_SetSession(w->sp);
-			w->sp->wrk = w;
-			CNT_Session(w->sp);
+			AZ(wrk->sp->wrk);
+			THR_SetSession(wrk->sp);
+			wrk->sp->wrk = wrk;
+			CNT_Session(wrk->sp);
 			THR_SetSession(NULL);
-			w->sp = NULL;
+			wrk->sp = NULL;
 
-			WS_Assert(w->ws);
-			AZ(w->busyobj);
-			AZ(w->wrw.wfd);
-			AZ(w->storage_hint);
-			assert(w->wlp == w->wlb);
+			WS_Assert(wrk->ws);
+			AZ(wrk->busyobj);
+			AZ(wrk->wrw.wfd);
+			AZ(wrk->storage_hint);
+			assert(wrk->wlp == wrk->wlb);
 			if (cache_param->diag_bitmap & 0x00040000) {
-				if (w->vcl != NULL)
-					VCL_Rel(&w->vcl);
+				if (wrk->vcl != NULL)
+					VCL_Rel(&wrk->vcl);
 			}
-		} else if (w->do_what == pool_do_nothing) {
+		} else if (wrk->do_what == pool_do_nothing) {
 			/* we already did */
 		} else {
-			WRONG("Invalid w->do_what");
+			WRONG("Invalid wrk->do_what");
 		}
-		stats_clean = WRK_TrySumStat(w);
+		stats_clean = WRK_TrySumStat(wrk);
 		Lck_Lock(&pp->mtx);
 	}
 	Lck_Unlock(&pp->mtx);
-	w->pool = NULL;
+	wrk->pool = NULL;
 }
 
 /*--------------------------------------------------------------------
@@ -294,18 +294,18 @@ Pool_Work_Thread(void *priv, struct worker *w)
 static int
 pool_queue(struct pool *pp, struct sess *sp)
 {
-	struct worker *w;
+	struct worker *wrk;
 
 	Lck_Lock(&pp->mtx);
 
 	/* If there are idle threads, we tickle the first one into action */
-	w = VTAILQ_FIRST(&pp->idle);
-	if (w != NULL) {
-		VTAILQ_REMOVE(&pp->idle, w, list);
+	wrk = VTAILQ_FIRST(&pp->idle);
+	if (wrk != NULL) {
+		VTAILQ_REMOVE(&pp->idle, wrk, list);
 		Lck_Unlock(&pp->mtx);
-		w->sp = sp;
-		w->do_what = pool_do_sess;
-		AZ(pthread_cond_signal(&w->cond));
+		wrk->sp = sp;
+		wrk->do_what = pool_do_sess;
+		AZ(pthread_cond_signal(&wrk->cond));
 		return (0);
 	}
 
@@ -397,7 +397,7 @@ pool_herder(void *priv)
 	pthread_attr_t tp_attr;
 	struct timespec ts;
 	double t_idle;
-	struct worker *w;
+	struct worker *wrk;
 	int i;
 
 	CAST_OBJ_NOTNULL(pp, priv, POOL_MAGIC);
@@ -442,25 +442,25 @@ pool_herder(void *priv)
 		VSC_C_main->sess_queued += pp->nqueued;
 		VSC_C_main->sess_dropped += pp->ndropped;
 		pp->nqueued = pp->ndropped = 0;
-		w = VTAILQ_LAST(&pp->idle, workerhead);
-		if (w != NULL &&
-		    (w->lastused < t_idle ||
+		wrk = VTAILQ_LAST(&pp->idle, workerhead);
+		if (wrk != NULL &&
+		    (wrk->lastused < t_idle ||
 		    pp->nthr > cache_param->wthread_max)) {
-			VTAILQ_REMOVE(&pp->idle, w, list);
+			VTAILQ_REMOVE(&pp->idle, wrk, list);
 		} else
-			w = NULL;
+			wrk = NULL;
 		Lck_Unlock(&pp->mtx);
 
 		/* And give it a kiss on the cheek... */
-		if (w != NULL) {
+		if (wrk != NULL) {
 			pp->nthr--;
 			Lck_Lock(&pool_mtx);
 			VSC_C_main->threads--;
 			VSC_C_main->threads_destroyed++;
 			Lck_Unlock(&pool_mtx);
-			AZ(w->sp);
-			w->do_what = pool_do_die;
-			AZ(pthread_cond_signal(&w->cond));
+			AZ(wrk->sp);
+			wrk->do_what = pool_do_die;
+			AZ(pthread_cond_signal(&wrk->cond));
 		}
 	}
 	NEEDLESS_RETURN(NULL);
diff --git a/bin/varnishd/cache/cache_response.c b/bin/varnishd/cache/cache_response.c
index 5356b7b..5c10c35 100644
--- a/bin/varnishd/cache/cache_response.c
+++ b/bin/varnishd/cache/cache_response.c
@@ -359,52 +359,52 @@ RES_StreamStart(struct sess *sp)
 }
 
 void
-RES_StreamPoll(struct worker *w)
+RES_StreamPoll(struct worker *wrk)
 {
 	struct stream_ctx *sctx;
 	struct storage *st;
 	ssize_t l, l2;
 	void *ptr;
 
-	CHECK_OBJ_NOTNULL(w, WORKER_MAGIC);
-	CHECK_OBJ_NOTNULL(w->busyobj->fetch_obj, OBJECT_MAGIC);
-	sctx = w->sctx;
+	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
+	CHECK_OBJ_NOTNULL(wrk->busyobj->fetch_obj, OBJECT_MAGIC);
+	sctx = wrk->sctx;
 	CHECK_OBJ_NOTNULL(sctx, STREAM_CTX_MAGIC);
-	if (w->busyobj->fetch_obj->len == sctx->stream_next)
+	if (wrk->busyobj->fetch_obj->len == sctx->stream_next)
 		return;
-	assert(w->busyobj->fetch_obj->len > sctx->stream_next);
+	assert(wrk->busyobj->fetch_obj->len > sctx->stream_next);
 	l = sctx->stream_front;
-	VTAILQ_FOREACH(st, &w->busyobj->fetch_obj->store, list) {
+	VTAILQ_FOREACH(st, &wrk->busyobj->fetch_obj->store, list) {
 		if (st->len + l <= sctx->stream_next) {
 			l += st->len;
 			continue;
 		}
 		l2 = st->len + l - sctx->stream_next;
 		ptr = st->ptr + (sctx->stream_next - l);
-		if (w->res_mode & RES_GUNZIP) {
-			(void)VGZ_WrwGunzip(w, sctx->vgz, ptr, l2,
+		if (wrk->res_mode & RES_GUNZIP) {
+			(void)VGZ_WrwGunzip(wrk, sctx->vgz, ptr, l2,
 			    sctx->obuf, sctx->obuf_len, &sctx->obuf_ptr);
 		} else {
-			(void)WRW_Write(w, ptr, l2);
+			(void)WRW_Write(wrk, ptr, l2);
 		}
 		l += st->len;
 		sctx->stream_next += l2;
 	}
-	if (!(w->res_mode & RES_GUNZIP))
-		(void)WRW_Flush(w);
+	if (!(wrk->res_mode & RES_GUNZIP))
+		(void)WRW_Flush(wrk);
 
-	if (w->busyobj->fetch_obj->objcore == NULL ||
-	    (w->busyobj->fetch_obj->objcore->flags & OC_F_PASS)) {
+	if (wrk->busyobj->fetch_obj->objcore == NULL ||
+	    (wrk->busyobj->fetch_obj->objcore->flags & OC_F_PASS)) {
 		/*
 		 * This is a pass object, release storage as soon as we
 		 * have delivered it.
 		 */
 		while (1) {
-			st = VTAILQ_FIRST(&w->busyobj->fetch_obj->store);
+			st = VTAILQ_FIRST(&wrk->busyobj->fetch_obj->store);
 			if (st == NULL ||
 			    sctx->stream_front + st->len > sctx->stream_next)
 				break;
-			VTAILQ_REMOVE(&w->busyobj->fetch_obj->store, st, list);
+			VTAILQ_REMOVE(&wrk->busyobj->fetch_obj->store, st, list);
 			sctx->stream_front += st->len;
 			STV_free(st);
 		}
diff --git a/bin/varnishd/cache/cache_shmlog.c b/bin/varnishd/cache/cache_shmlog.c
index b7c43a1..c29f5a1 100644
--- a/bin/varnishd/cache/cache_shmlog.c
+++ b/bin/varnishd/cache/cache_shmlog.c
@@ -35,7 +35,7 @@
 #include "cache.h"
 #include "common/heritage.h"
 
-#include "cache_backend.h"	// For w->vbc
+#include "cache_backend.h"	// For wrk->vbc
 
 #include "vmb.h"
 #include "vtim.h"
@@ -180,30 +180,30 @@ VSL(enum VSL_tag_e tag, int id, const char *fmt, ...)
 /*--------------------------------------------------------------------*/
 
 void
-WSL_Flush(struct worker *w, int overflow)
+WSL_Flush(struct worker *wrk, int overflow)
 {
 	uint32_t *p;
 	unsigned l;
 
-	l = pdiff(w->wlb, w->wlp);
+	l = pdiff(wrk->wlb, wrk->wlp);
 	if (l == 0)
 		return;
 
 	assert(l >= 8);
 
-	p = vsl_get(l - 8, w->wlr, overflow);
+	p = vsl_get(l - 8, wrk->wlr, overflow);
 
-	memcpy(p + 1, w->wlb + 1, l - 4);
+	memcpy(p + 1, wrk->wlb + 1, l - 4);
 	VWMB();
-	p[0] = w->wlb[0];
-	w->wlp = w->wlb;
-	w->wlr = 0;
+	p[0] = wrk->wlb[0];
+	wrk->wlp = wrk->wlb;
+	wrk->wlr = 0;
 }
 
 /*--------------------------------------------------------------------*/
 
 void
-WSLR(struct worker *w, enum VSL_tag_e tag, int id, txt t)
+WSLR(struct worker *wrk, enum VSL_tag_e tag, int id, txt t)
 {
 	unsigned l, mlen;
 
@@ -217,25 +217,25 @@ WSLR(struct worker *w, enum VSL_tag_e tag, int id, txt t)
 		t.e = t.b + l;
 	}
 
-	assert(w->wlp < w->wle);
+	assert(wrk->wlp < wrk->wle);
 
 	/* Wrap if necessary */
-	if (VSL_END(w->wlp, l) >= w->wle)
-		WSL_Flush(w, 1);
-	assert (VSL_END(w->wlp, l) < w->wle);
-	memcpy(VSL_DATA(w->wlp), t.b, l);
-	vsl_hdr(tag, w->wlp, l, id);
-	w->wlp = VSL_END(w->wlp, l);
-	assert(w->wlp < w->wle);
-	w->wlr++;
+	if (VSL_END(wrk->wlp, l) >= wrk->wle)
+		WSL_Flush(wrk, 1);
+	assert (VSL_END(wrk->wlp, l) < wrk->wle);
+	memcpy(VSL_DATA(wrk->wlp), t.b, l);
+	vsl_hdr(tag, wrk->wlp, l, id);
+	wrk->wlp = VSL_END(wrk->wlp, l);
+	assert(wrk->wlp < wrk->wle);
+	wrk->wlr++;
 	if (cache_param->diag_bitmap & 0x10000)
-		WSL_Flush(w, 0);
+		WSL_Flush(wrk, 0);
 }
 
 /*--------------------------------------------------------------------*/
 
 static void
-wsl(struct worker *w, enum VSL_tag_e tag, int id, const char *fmt, va_list ap)
+wsl(struct worker *wrk, enum VSL_tag_e tag, int id, const char *fmt, va_list ap)
 {
 	char *p;
 	unsigned n, mlen;
@@ -247,38 +247,38 @@ wsl(struct worker *w, enum VSL_tag_e tag, int id, const char *fmt, va_list ap)
 	if (strchr(fmt, '%') == NULL) {
 		t.b = TRUST_ME(fmt);
 		t.e = strchr(t.b, '\0');
-		WSLR(w, tag, id, t);
+		WSLR(wrk, tag, id, t);
 	} else {
-		assert(w->wlp < w->wle);
+		assert(wrk->wlp < wrk->wle);
 
 		/* Wrap if we cannot fit a full size record */
-		if (VSL_END(w->wlp, mlen) >= w->wle)
-			WSL_Flush(w, 1);
+		if (VSL_END(wrk->wlp, mlen) >= wrk->wle)
+			WSL_Flush(wrk, 1);
 
-		p = VSL_DATA(w->wlp);
+		p = VSL_DATA(wrk->wlp);
 		n = vsnprintf(p, mlen, fmt, ap);
 		if (n > mlen)
 			n = mlen;	/* we truncate long fields */
-		vsl_hdr(tag, w->wlp, n, id);
-		w->wlp = VSL_END(w->wlp, n);
-		assert(w->wlp < w->wle);
-		w->wlr++;
+		vsl_hdr(tag, wrk->wlp, n, id);
+		wrk->wlp = VSL_END(wrk->wlp, n);
+		assert(wrk->wlp < wrk->wle);
+		wrk->wlr++;
 	}
 	if (cache_param->diag_bitmap & 0x10000)
-		WSL_Flush(w, 0);
+		WSL_Flush(wrk, 0);
 }
 
 /*--------------------------------------------------------------------*/
 
 void
-WSL(struct worker *w, enum VSL_tag_e tag, int id, const char *fmt, ...)
+WSL(struct worker *wrk, enum VSL_tag_e tag, int id, const char *fmt, ...)
 {
 	va_list ap;
 
-	CHECK_OBJ_NOTNULL(w, WORKER_MAGIC);
+	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 	AN(fmt);
 	va_start(ap, fmt);
-	wsl(w, tag, id, fmt, ap);
+	wsl(wrk, tag, id, fmt, ap);
 	va_end(ap);
 }
 
@@ -286,16 +286,16 @@ WSL(struct worker *w, enum VSL_tag_e tag, int id, const char *fmt, ...)
 /*--------------------------------------------------------------------*/
 
 void
-WSLB(struct worker *w, enum VSL_tag_e tag, const char *fmt, ...)
+WSLB(struct worker *wrk, enum VSL_tag_e tag, const char *fmt, ...)
 {
 	va_list ap;
 
-	CHECK_OBJ_NOTNULL(w, WORKER_MAGIC);
-	CHECK_OBJ_NOTNULL(w->busyobj, BUSYOBJ_MAGIC);
-	CHECK_OBJ_NOTNULL(w->busyobj->vbc, VBC_MAGIC);
+	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
+	CHECK_OBJ_NOTNULL(wrk->busyobj, BUSYOBJ_MAGIC);
+	CHECK_OBJ_NOTNULL(wrk->busyobj->vbc, VBC_MAGIC);
 	AN(fmt);
 	va_start(ap, fmt);
-	wsl(w, tag, w->busyobj->vbc->vsl_id, fmt, ap);
+	wsl(wrk, tag, wrk->busyobj->vbc->vsl_id, fmt, ap);
 	va_end(ap);
 }
 
diff --git a/bin/varnishd/cache/cache_wrk.c b/bin/varnishd/cache/cache_wrk.c
index ce665bc..2e933c5 100644
--- a/bin/varnishd/cache/cache_wrk.c
+++ b/bin/varnishd/cache/cache_wrk.c
@@ -94,7 +94,7 @@ static void *
 wrk_bgthread(void *arg)
 {
 	struct bgthread *bt;
-	struct worker ww;
+	struct worker wrk;
 	struct sess *sp;
 	uint32_t logbuf[1024];	/* XXX:  size ? */
 
@@ -102,11 +102,11 @@ wrk_bgthread(void *arg)
 	THR_SetName(bt->name);
 	sp = SES_Alloc();
 	XXXAN(sp);
-	memset(&ww, 0, sizeof ww);
-	sp->wrk = &ww;
-	ww.magic = WORKER_MAGIC;
-	ww.wlp = ww.wlb = logbuf;
-	ww.wle = logbuf + (sizeof logbuf) / 4;
+	memset(&wrk, 0, sizeof wrk);
+	sp->wrk = &wrk;
+	wrk.magic = WORKER_MAGIC;
+	wrk.wlp = wrk.wlb = logbuf;
+	wrk.wle = logbuf + (sizeof logbuf) / 4;
 
 	(void)bt->func(sp, bt->priv);
 
diff --git a/bin/varnishd/hash/hash_critbit.c b/bin/varnishd/hash/hash_critbit.c
index 9935fc2..3b55490 100644
--- a/bin/varnishd/hash/hash_critbit.c
+++ b/bin/varnishd/hash/hash_critbit.c
@@ -348,11 +348,11 @@ static void *
 hcb_cleaner(void *priv)
 {
 	struct hcb_y *y, *y2;
-	struct worker ww;
+	struct worker wrk;
 	struct objhead *oh, *oh2;
 
-	memset(&ww, 0, sizeof ww);
-	ww.magic = WORKER_MAGIC;
+	memset(&wrk, 0, sizeof wrk);
+	wrk.magic = WORKER_MAGIC;
 
 	THR_SetName("hcb_cleaner");
 	(void)priv;
@@ -363,13 +363,13 @@ hcb_cleaner(void *priv)
 		}
 		VTAILQ_FOREACH_SAFE(oh, &dead_h, hoh_list, oh2) {
 			VTAILQ_REMOVE(&dead_h, oh, hoh_list);
-			HSH_DeleteObjHead(&ww, oh);
+			HSH_DeleteObjHead(&wrk, oh);
 		}
 		Lck_Lock(&hcb_mtx);
 		VSTAILQ_CONCAT(&dead_y, &cool_y);
 		VTAILQ_CONCAT(&dead_h, &cool_h, hoh_list);
 		Lck_Unlock(&hcb_mtx);
-		WRK_SumStat(&ww);
+		WRK_SumStat(&wrk);
 		VTIM_sleep(cache_param->critbit_cooloff);
 	}
 	NEEDLESS_RETURN(NULL);
@@ -386,6 +386,7 @@ hcb_start(void)
 	(void)oh;
 	CLI_AddFuncs(hcb_cmds);
 	Lck_New(&hcb_mtx, lck_hcb);
+	/* XXX: use WRK_BgThread */
 	AZ(pthread_create(&tp, NULL, hcb_cleaner, NULL));
 	memset(&hcb_root, 0, sizeof hcb_root);
 	hcb_build_bittbl();



From geoff at varnish-cache.org  Mon Jan  9 23:03:33 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Tue, 10 Jan 2012 00:03:33 +0100
Subject: [experimental-ims] a2e72e8 Add vsc for requested and needed sizes for
	mempool.
Message-ID: <E1RkOFZ-0007y7-Cu@project.varnish-software.com>

commit a2e72e8fa66d871903f99a4973687ba7c3c6a7c8
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Dec 26 16:36:09 2011 +0000

    Add vsc for requested and needed sizes for mempool.
    
    Fix sess mempool names

diff --git a/bin/varnishd/cache/cache_mempool.c b/bin/varnishd/cache/cache_mempool.c
index 6f556f2..a9d4249 100644
--- a/bin/varnishd/cache/cache_mempool.c
+++ b/bin/varnishd/cache/cache_mempool.c
@@ -80,6 +80,8 @@ mpl_alloc(const struct mempool *mpl)
 	AN(mi);
 	mi->magic = MEMITEM_MAGIC;
 	mi->size = tsz;
+	mpl->vsc->sz_wanted = tsz;
+	mpl->vsc->sz_needed = tsz + sizeof *mi;
 	return (mi);
 }
 
diff --git a/bin/varnishd/cache/cache_session.c b/bin/varnishd/cache/cache_session.c
index ee5060f..d1dbaba 100644
--- a/bin/varnishd/cache/cache_session.c
+++ b/bin/varnishd/cache/cache_session.c
@@ -318,6 +318,7 @@ SES_NewPool(struct pool *wp, unsigned pool_no)
 	bprintf(nb, "req%u", pool_no);
 	pp->mpl_req = MPL_New(nb, &cache_param->req_pool,
 	    &cache_param->workspace_client);
+	bprintf(nb, "sess%u", pool_no);
 	pp->mpl_sess = MPL_New(nb, &cache_param->sess_pool, &ses_size);
 	return (pp);
 }



From geoff at varnish-cache.org  Mon Jan  9 23:03:33 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Tue, 10 Jan 2012 00:03:33 +0100
Subject: [experimental-ims] 9e0f29c Attempt to close a theoretical race,
	which POSIX_MISTAKE makes it impossible to deal with correctly.
Message-ID: <E1RkOFZ-0007yS-J0@project.varnish-software.com>

commit 9e0f29cdcdb3c96ac559692f21bdeaa7ecc9272a
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Dec 27 14:55:59 2011 +0000

    Attempt to close a theoretical race, which POSIX_MISTAKE makes it
    impossible to deal with correctly.
    
    Pointed out by:		Dong Feng

diff --git a/bin/varnishd/cache/cache_lck.c b/bin/varnishd/cache/cache_lck.c
index cff9845..2910582 100644
--- a/bin/varnishd/cache/cache_lck.c
+++ b/bin/varnishd/cache/cache_lck.c
@@ -98,6 +98,17 @@ Lck__Unlock(struct lock *lck, const char *p, const char *f, int l)
 	assert(pthread_equal(ilck->owner, pthread_self()));
 	AN(ilck->held);
 	ilck->held = 0;
+	/*
+	 * #ifdef POSIX_STUPIDITY:
+	 * The pthread_t type has no defined assignment or comparison
+	 * operators, this is why pthread_equal() is necessary.
+	 * Unfortunately POSIX forgot to define a NULL value for pthread_t
+	 * so you can never unset a pthread_t variable.
+	 * We hack it and fill it with zero bits, hoping for sane
+	 * implementations of pthread.
+	 * #endif
+	 */ 
+	memset(&ilck->owner, 0, sizeof ilck->owner);
 	AZ(pthread_mutex_unlock(&ilck->mtx));
 	if (cache_param->diag_bitmap & 0x8)
 		VSL(SLT_Debug, 0, "MTX_UNLOCK(%s,%s,%d,%s)", p, f, l, ilck->w);



From geoff at varnish-cache.org  Mon Jan  9 23:03:33 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Tue, 10 Jan 2012 00:03:33 +0100
Subject: [experimental-ims] 473e58c Collapse two VTAILQ_ENTRY() in struct sess,
	saving 16 bytes.
Message-ID: <E1RkOFZ-0007z1-S8@project.varnish-software.com>

commit 473e58c0b53edde3795bfc4637d2b71fb3166383
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Jan 2 10:01:47 2012 +0000

    Collapse two VTAILQ_ENTRY() in struct sess, saving 16 bytes.

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index 42562b5..a15f272 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -636,6 +636,8 @@ struct sess {
 	struct worker		*wrk;
 	struct req		*req;
 
+	VTAILQ_ENTRY(sess)	list;
+
 	/* Session related fields ------------------------------------*/
 
 	int			fd;
@@ -651,11 +653,8 @@ struct sess {
 	char			addr[ADDR_BUFSIZE];
 	char			port[PORT_BUFSIZE];
 
-	VTAILQ_ENTRY(sess)	poollist;
 	struct acct		acct_ses;
 
-	VTAILQ_ENTRY(sess)	list;
-
 	/* Timestamps, all on TIM_real() timescale */
 	double			t_open;		/* fd accepted */
 	double			t_idle;		/* fd accepted or resp sent */
diff --git a/bin/varnishd/cache/cache_pool.c b/bin/varnishd/cache/cache_pool.c
index 9b0760c..84a6cfd 100644
--- a/bin/varnishd/cache/cache_pool.c
+++ b/bin/varnishd/cache/cache_pool.c
@@ -201,7 +201,7 @@ Pool_Work_Thread(void *priv, struct worker *wrk)
 		if (wrk->sp != NULL) {
 			/* Process queued requests, if any */
 			assert(pp->lqueue > 0);
-			VTAILQ_REMOVE(&pp->queue, wrk->sp, poollist);
+			VTAILQ_REMOVE(&pp->queue, wrk->sp, list);
 			wrk->do_what = pool_do_sess;
 			pp->lqueue--;
 		} else if (!VTAILQ_EMPTY(&pp->socks)) {
@@ -314,7 +314,7 @@ pool_queue(struct pool *pp, struct sess *sp)
 		return (-1);
 	}
 
-	VTAILQ_INSERT_TAIL(&pp->queue, sp, poollist);
+	VTAILQ_INSERT_TAIL(&pp->queue, sp, list);
 	pp->nqueued++;
 	pp->lqueue++;
 	Lck_Unlock(&pp->mtx);



From geoff at varnish-cache.org  Mon Jan  9 23:03:33 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Tue, 10 Jan 2012 00:03:33 +0100
Subject: [experimental-ims] c632af8 Pack the memitem a bit smarter and drop
	the payload element which costs us 8 bytes per allocation.
Message-ID: <E1RkOFZ-0007zA-UL@project.varnish-software.com>

commit c632af8afa5976b05fe499c71d83f0bacde7de8d
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Jan 2 10:22:37 2012 +0000

    Pack the memitem a bit smarter and drop the payload element which costs
    us 8 bytes per allocation.

diff --git a/bin/varnishd/cache/cache_mempool.c b/bin/varnishd/cache/cache_mempool.c
index a9d4249..468d5a8 100644
--- a/bin/varnishd/cache/cache_mempool.c
+++ b/bin/varnishd/cache/cache_mempool.c
@@ -29,6 +29,7 @@
  */
 
 #include <stddef.h>
+#include <stdio.h>
 #include <stdlib.h>
 
 #include "config.h"
@@ -40,10 +41,9 @@
 struct memitem {
 	unsigned			magic;
 #define MEMITEM_MAGIC			0x42e55401
-	VTAILQ_ENTRY(memitem)		list;
 	unsigned			size;
+	VTAILQ_ENTRY(memitem)		list;
 	double				touched;
-	char				payload;
 };
 
 VTAILQ_HEAD(memhead_s, memitem);
@@ -92,8 +92,6 @@ mpl_alloc(const struct mempool *mpl)
  *   of the list not getting too old.
  */
 
-#include <stdio.h>
-
 static void *
 mpl_guard(void *priv)
 {
@@ -309,7 +307,7 @@ MPL_Get(struct mempool *mpl, unsigned *size)
 
 	CHECK_OBJ_NOTNULL(mi, MEMITEM_MAGIC);
 	/* Throw away sizeof info for FlexeLint: */
-	return ((void*)(uintptr_t)&mi->payload);
+	return ((void*)(uintptr_t)(mi+1));
 }
 
 void
@@ -320,7 +318,7 @@ MPL_Free(struct mempool *mpl, void *item)
 	CHECK_OBJ_NOTNULL(mpl, MEMPOOL_MAGIC);
 	AN(item);
 
-	mi = (void*)((uintptr_t)item - offsetof(struct memitem, payload));
+	mi = (void*)((uintptr_t)item - sizeof(*mi));
 	CHECK_OBJ_NOTNULL(mi, MEMITEM_MAGIC);
 	memset(item, 0, mi->size);
 
@@ -348,6 +346,6 @@ void
 MPL_AssertSane(void *item)
 {
 	struct memitem *mi;
-	mi = (void*)((uintptr_t)item - offsetof(struct memitem, payload));
+	mi = (void*)((uintptr_t)item - sizeof(*mi));
 	CHECK_OBJ_NOTNULL(mi, MEMITEM_MAGIC);
 }



From geoff at varnish-cache.org  Mon Jan  9 23:03:34 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Tue, 10 Jan 2012 00:03:34 +0100
Subject: [experimental-ims] de08ec0 Shave 16 bytes of the ADDR_BUFSIZE to push
	struct sess(+memitem) down to 512 bytes on epoll systems.
Message-ID: <E1RkOFa-0007zX-3n@project.varnish-software.com>

commit de08ec088e0a8ca1264d0df46d037b01bdaeb091
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Jan 2 10:48:57 2012 +0000

    Shave 16 bytes of the ADDR_BUFSIZE to push struct sess(+memitem) down
    to 512 bytes on epoll systems.

diff --git a/bin/varnishd/common/common.h b/bin/varnishd/common/common.h
index 7fcc7ec..f665e52 100644
--- a/bin/varnishd/common/common.h
+++ b/bin/varnishd/common/common.h
@@ -63,9 +63,10 @@ struct cli;
 /**********************************************************************
  * NI_MAXHOST and less so NI_MAXSERV, are ridiculously large for numeric
  * representations of TCP/IP socket addresses, so we use our own.
+ * <netinet/in6.h>::INET6_ADDRSTRLEN is 46
  */
 
-#define ADDR_BUFSIZE	64
+#define ADDR_BUFSIZE	48
 #define PORT_BUFSIZE	8
 
 



From geoff at varnish-cache.org  Mon Jan  9 23:03:34 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Tue, 10 Jan 2012 00:03:34 +0100
Subject: [experimental-ims] a4e2d47 All relevant BSD's have a testable kqueue
	now.
Message-ID: <E1RkOFa-0007zs-8d@project.varnish-software.com>

commit a4e2d478d5d8bab31a654248b9509c84adc9f1f1
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Jan 2 12:10:57 2012 +0000

    All relevant BSD's have a testable kqueue now.

diff --git a/configure.ac b/configure.ac
index 8404de4..01eac28 100644
--- a/configure.ac
+++ b/configure.ac
@@ -334,16 +334,7 @@ AC_ARG_ENABLE(kqueue,
     [enable_kqueue=yes])
 
 if test "$enable_kqueue" = yes; then
-	case $target in
-	*-*-freebsd* | *-*-darwin9* | *-*-darwin11* | *-*-netbsd* )
-		AC_CHECK_FUNCS([kqueue])
-		;;
-	*-*-bsd*)
-		# No other BSD has a sufficiently recent implementation
-		AC_MSG_WARN([won't look for kqueue() on $target])
-		ac_cv_func_kqueue=no
-		;;
-	esac
+	AC_CHECK_FUNCS([kqueue])
 else
 	ac_cv_func_kqueue=no
 fi



From geoff at varnish-cache.org  Mon Jan  9 23:03:34 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Tue, 10 Jan 2012 00:03:34 +0100
Subject: [experimental-ims] b70a93c Fix typo
Message-ID: <E1RkOFa-00080P-GX@project.varnish-software.com>

commit b70a93cff1df0763a99a6b0fbad952ee86c24be3
Author: Tollef Fog Heen <tfheen at varnish-software.com>
Date:   Mon Jan 2 12:59:42 2012 +0100

    Fix typo

diff --git a/doc/sphinx/reference/varnish-cli.rst b/doc/sphinx/reference/varnish-cli.rst
index 69b6546..a097248 100644
--- a/doc/sphinx/reference/varnish-cli.rst
+++ b/doc/sphinx/reference/varnish-cli.rst
@@ -196,7 +196,7 @@ consists of a field, an operator, and an argument.  Conditions can be
 ANDed together with "&&".
 
 A field can be any of the variables from VCL, for instance req.url,
-req.http.host or obj.set-cookie.
+req.http.host or obj.http.set-cookie.
 
 Operators are "==" for direct comparision, "~" for a regular
 expression match, and ">" or "<" for size comparisons.  Prepending



From geoff at varnish-cache.org  Mon Jan  9 23:03:34 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Tue, 10 Jan 2012 00:03:34 +0100
Subject: [experimental-ims] b6efb69 Remove outdated comment
Message-ID: <E1RkOFa-00080Y-KY@project.varnish-software.com>

commit b6efb69ef5ecfd82d35adf85134d553381297325
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Jan 3 08:15:12 2012 +0000

    Remove outdated comment

diff --git a/configure.ac b/configure.ac
index 01eac28..72d2b7c 100644
--- a/configure.ac
+++ b/configure.ac
@@ -320,12 +320,6 @@ AC_CHECK_FUNCS([clock_gettime])
 AC_CHECK_FUNCS([gethrtime]) 
 LIBS="${save_LIBS}"
 
-# Check which mechanism to use for the acceptor.  We look for kqueue
-# only on platforms on which we know that it works, because there are
-# platforms where a simple AC_CHECK_FUNCS([kqueue]) would succeed but
-# the build would fail.  We also allow the user to disable mechanisms
-# he doesn't want to use.
-
 # --enable-kqueue
 AC_ARG_ENABLE(kqueue,
     AS_HELP_STRING([--enable-kqueue],



From geoff at varnish-cache.org  Mon Jan  9 23:03:34 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Tue, 10 Jan 2012 00:03:34 +0100
Subject: [experimental-ims] 0a96b48 Add explanation for Varnish "hashing" in
	light of advisories.
Message-ID: <E1RkOFa-00080m-Nr@project.varnish-software.com>

commit 0a96b4875c06f2320b43cd0833d35e33ba047dc7
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Jan 3 18:10:29 2012 +0000

    Add explanation for Varnish "hashing" in light of advisories.

diff --git a/doc/sphinx/phk/index.rst b/doc/sphinx/phk/index.rst
index 822434c..58935c7 100644
--- a/doc/sphinx/phk/index.rst
+++ b/doc/sphinx/phk/index.rst
@@ -8,6 +8,7 @@ You may or may not want to know what Poul-Henning think.
 
 .. toctree::
 
+	varnish_does_not_hash.rst
 	thetoolsweworkwith.rst
 	three-zero.rst
 	ssl.rst
diff --git a/doc/sphinx/phk/varnish_does_not_hash.rst b/doc/sphinx/phk/varnish_does_not_hash.rst
new file mode 100644
index 0000000..8393c87
--- /dev/null
+++ b/doc/sphinx/phk/varnish_does_not_hash.rst
@@ -0,0 +1,145 @@
+.. _phk_varnish_does_not_hash:
+
+=====================
+Varnish Does Not Hash
+=====================
+
+A spate of security advisories related to hash-collisions have made
+a lot of people stare at Varnish and wonder if it is affected.
+
+The answer is no, but the explanation is probably not what most of
+you expected:
+
+Varnish does not hash, at least not by default, and
+even if it does, it's still as immune to the attacks as can be.
+
+To understand what is going on, I have to introduce a concept from
+Shannons information theory: "entropy."
+
+Entropy is hard to explain, and according to legend, that is exactly
+why Shannon recycled that term from thermodynamics.
+
+In this context, we can get away with thinking about entropy as how
+much our "keys" differ::
+
+	Low entropy (1 bit):
+		/foo/bar/barf/some/cms/content/article?article=2
+		/foo/bar/barf/some/cms/content/article?article=3
+
+	High entropy (65 bits):
+		/i?ee30d0770eb460634e9d5dcfb562a2c5.html
+		/i?bca3633d52607f38a107cb5297fd66e5.html
+
+Hashing consists of calculating a hash-index from the key and
+storing the objects in an array indexed by that key.
+
+Typically, but not always, the key is a string and the index is a
+(smallish) integer, and the job of the hash-function is to squeeze
+the key into the integer, without loosing any of the entropy.
+
+Needless to say, the more entropy you have to begin with, the more
+of it you can afford to loose, and loose some you almost invariably
+will.
+
+There are two families of hash-functions, the fast ones, and the good
+ones, and the security advisories are about the fast ones.
+
+The good ones are slower, but probably not so much slower that you
+care, and therefore, if you want to fix your web-app:
+
+Change::
+	foo=somedict[$somekey]
+To::
+	foo=somedict[md5($somekey)]
+
+and forget about the advisories.
+
+Yes, that's right: Cryptographic hash algorithms are the good ones,
+they are built to not throw any entropy away, and they are built to
+have very hard to predict collisions, which is exactly the problem
+with the fast hash-functions in the advisories.
+
+-----------------
+What Varnish Does
+-----------------
+
+The way to avoid having hash-collisions is to not use a hash:  Use a
+tree instead, there every object has its own place and there are no
+collisions.
+
+Varnish does that, but with a twist.
+
+The "keys" in varnish can be very long, by default they consist of::
+
+	sub vcl_hash {
+	    hash_data(req.url);
+	    if (req.http.host) {
+		hash_data(req.http.host);
+	    } else {
+		hash_data(server.ip);
+	    }
+	    return (hash);
+	}
+
+But some users will add cookies, user identification and many other
+bits and pieces of string in there, and in the end the keys can be
+kilobytes in length, and quite often, as in the first example above,
+the first difference may not come until pretty far into the keys.
+
+Trees generally need to have a copy of the key around to be able
+to tell if they have a match, and more importantly to compare
+tree-leaves in order to "re-balance" the tree and other such arcanae
+of data structures.
+
+This would add another per-object memory load to Varnish, and it
+would feel particularly silly to store 48 identical characters for
+each object in the far too common case seen above.
+
+But furthermore, we want the tree to be very fast to do lookups in,
+preferably it should be lockless for lookups, and that means that
+we cannot (realistically) use any of the "smart" trees which
+automatically balance themselves etc.
+
+You (generally) don't need a "smart" tree if your keys look
+like random data in the order they arrive, but we can pretty
+much expect the opposite as article number 4, 5, 6 etc are added
+to the CMS in the first example.
+
+But we can make the keys look random, and make them small and fixed
+size at the same time, and the perfect functions designed for just
+that task are the "good" hash-functions, the cryptographic ones.
+
+So what Varnish does is "key-compression":  All the strings hash_data()
+are fed, are pushed through a cryptographic hash algorithm called
+SHA256, which, as the name says, always spits out 256 bits (= 32
+bytes), no matter how many bits you feed it.
+
+This does not eliminate the key-storage requirement, but now all
+the keys are 32 bytes and can be put directly into the data structure::
+
+	struct objhead {
+		[...]
+		unsigned char           digest[DIGEST_LEN];
+	};
+
+In the example above, the output of SHA256 for the 1 bit difference
+in entropy becomes::
+
+	/foo/bar/barf/some/cms/content/article?article=2
+	-> 14f0553caa5c796650ec82256e3f111ae2f20020a4b9029f135a01610932054e
+	/foo/bar/barf/some/cms/content/article?article=3
+	-> 4d45b9544077921575c3c5a2a14c779bff6c4830d1fbafe4bd7e03e5dd93ca05
+
+That should be random enough.
+
+But the key-compression does introduce a risk of collisions, since
+not even SHA256 can guarantee different outputs for all possible
+inputs:  Try pushing all the possible 33 bytes long files through
+SHA256 and sooner or later you will get collisions.
+
+The risk of collision is very small however, and I can all but
+promise you, that you will be fully offset in fame and money for
+any inconvenience a collision might cause, because you will
+be the first person to find a SHA256 collision.
+
+Poul-Henning, 2012-01-03



From geoff at varnish-cache.org  Mon Jan  9 23:03:34 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Tue, 10 Jan 2012 00:03:34 +0100
Subject: [experimental-ims] 84f098e Get rid of unused variable,
	fixes compilation problem with newer gcc
Message-ID: <E1RkOFa-000814-Ul@project.varnish-software.com>

commit 84f098e0b70a7aca500fd8969929438c821bd392
Author: Tollef Fog Heen <tfheen at varnish-software.com>
Date:   Mon Jan 9 14:00:30 2012 +0100

    Get rid of unused variable, fixes compilation problem with newer gcc

diff --git a/bin/varnishstat/varnishstat_curses.c b/bin/varnishstat/varnishstat_curses.c
index 3a672e1..5997096 100644
--- a/bin/varnishstat/varnishstat_curses.c
+++ b/bin/varnishstat/varnishstat_curses.c
@@ -145,7 +145,6 @@ do_curses(struct VSM_data *vd, const struct VSC_C_main *VSC_C_main,
 	time_t rt;
 	int ch, line;
 	struct pt *pt;
-	double act, lact;
 
 	(void)initscr();
 	AC(raw());
@@ -167,17 +166,12 @@ do_curses(struct VSM_data *vd, const struct VSC_C_main *VSC_C_main,
 		lt = 0;
 		lhit = 0;
 		lmiss = 0;
-		lact = 0;
 
 		while (1) {
 			/*
 			 * Break to outher loop if we need to re-read file.
 			 * Only check if it looks like nothing is happening.
 			 */
-			act = VSC_C_main->cache_hit +
-			    VSC_C_main->cache_miss + 1;
-			lact = act;
-
 			AZ(gettimeofday(&tv, NULL));
 			tt = tv.tv_usec * 1e-6 + tv.tv_sec;
 			lt = tt - lt;



From geoff at varnish-cache.org  Mon Jan  9 23:03:34 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Tue, 10 Jan 2012 00:03:34 +0100
Subject: [experimental-ims] 1fbff54 Merge branch 'master' of
	ssh://git.varnish-cache.org/git/varnish-cache
Message-ID: <E1RkOFa-00080w-S5@project.varnish-software.com>

commit 1fbff541f44cda7b9c3bd07739372099bc8df9e5
Merge: 0a96b48 b70a93c
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Jan 3 18:11:00 2012 +0000

    Merge branch 'master' of ssh://git.varnish-cache.org/git/varnish-cache



From geoff at varnish-cache.org  Mon Jan  9 23:03:35 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Tue, 10 Jan 2012 00:03:35 +0100
Subject: [experimental-ims] 6bec2c8 Fix a buglet in handling non-ascii strings
	in VCL,
	now that we (I) have decided what the strategy is going to be for
	synthetic in the future.
Message-ID: <E1RkOFb-00081D-1u@project.varnish-software.com>

commit 6bec2c8361f4bb87e84e7760d0970f3d96e9d07f
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Jan 9 15:21:35 2012 +0000

    Fix a buglet in handling non-ascii strings in VCL, now that we (I)
    have decided what the strategy is going to be for synthetic in
    the future.
    
    Fixes	#545

diff --git a/bin/varnishtest/tests/r00545.vtc b/bin/varnishtest/tests/r00545.vtc
new file mode 100644
index 0000000..0888af1
--- /dev/null
+++ b/bin/varnishtest/tests/r00545.vtc
@@ -0,0 +1,18 @@
+varnishtest "High-bit chars"
+
+server s1 {
+	rxreq
+	txresp
+} -start
+
+varnish v1 -vcl+backend {
+	sub vcl_deliver {
+		set resp.http.foo = "???";
+	}
+} -start
+
+client c1 {
+	txreq
+	rxresp
+	expect resp.http.foo == "???"
+} -run
diff --git a/lib/libvcl/vcc_compile.c b/lib/libvcl/vcc_compile.c
index fddd019..27447fe 100644
--- a/lib/libvcl/vcc_compile.c
+++ b/lib/libvcl/vcc_compile.c
@@ -229,7 +229,7 @@ EncString(struct vsb *sb, const char *b, const char *e, int mode)
 			if (isgraph(*b))
 				VSB_printf(sb, "%c", *b);
 			else
-				VSB_printf(sb, "\\%03o", *b);
+				VSB_printf(sb, "\\%03o", (uint8_t)*b);
 			break;
 		}
 	}
@@ -241,7 +241,7 @@ EncToken(struct vsb *sb, const struct token *t)
 {
 
 	assert(t->tok == CSTR);
-	EncString(sb, t->dec, NULL, 0);
+	EncString(sb, t->dec, NULL, 1);
 }
 
 /*--------------------------------------------------------------------



From geoff at varnish-cache.org  Mon Jan  9 23:03:35 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Tue, 10 Jan 2012 00:03:35 +0100
Subject: [experimental-ims] 5ecc60f Merge master -> experimental-ims
Message-ID: <E1RkOFb-00081O-6T@project.varnish-software.com>

commit 5ecc60f79fb0eb64d8945832ad4c9c3b7579bc69
Merge: 9e5fe24 6bec2c8
Author: Geoff Simmons <geoff at uplex.de>
Date:   Mon Jan 9 22:47:17 2012 +0100

    Merge master -> experimental-ims

diff --cc bin/varnishd/cache/cache.h
index 521786b,6b0d0a9..898f3ba
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@@ -594,39 -604,49 +604,51 @@@ struct req 
  	char			*client_identity;
  
  	/* HTTP request */
- 	const char		*doclose;
  	struct http		*http;
  	struct http		*http0;
+ 	struct http		*resp;
  
  	struct ws		ws[1];
- 	char			*ws_ses;	/* WS above session data */
- 	char			*ws_req;	/* WS above request data */
+ 	struct object		*obj;
+ 	struct objcore		*objcore;
+ 	/* Lookup stuff */
+ 	struct SHA256Context	*sha256ctx;
+ 	/* This is only here so VRT can find it */
+ 	const char		*storage_hint;
  
- 	unsigned char		digest[DIGEST_LEN];
+ 	/* ESI delivery stuff */
+ 	int			gzip_resp;
+ 	ssize_t			l_crc;
+ 	uint32_t		crc;
  
- 	/* Built Vary string */
- 	uint8_t			*vary_b;
- 	uint8_t			*vary_l;
- 	uint8_t			*vary_e;
  
- 	struct http_conn	htc[1];
+ };
  
- 	/* Timestamps, all on TIM_real() timescale */
- 	double			t_open;
- 	double			t_req;
- 	double			t_resp;
- 	double			t_end;
+ /*--------------------------------------------------------------------
+  * Struct sess is a high memory-load structure because sessions typically
+  * hang around the waiter for relatively long time.
+  *
+  * The size goal for struct sess + struct memitem is <512 bytes
+  *
+  * Getting down to the next relevant size (<256 bytes because of how malloc
+  * works, is not realistic without a lot of code changes.
+  */
  
- 	/* Acceptable grace period */
- 	struct exp		exp;
+ struct sess {
+ 	unsigned		magic;
+ #define SESS_MAGIC		0x2c2f9c5a
  
  	enum step		step;
- 	unsigned		cur_method;
- 	unsigned		handling;
- 	unsigned char		sendbody;
- 	unsigned char		wantbody;
- 	uint16_t		err_code;
- 	const char		*err_reason;
+ 	int			fd;
+ 	unsigned		vsl_id;
+ 
++	struct object		*stale_obj;
++	
+ 	/* Cross references ------------------------------------------*/
+ 
+ 	struct sesspool		*sesspool;
+ 	struct worker		*wrk;
+ 	struct req		*req;
  
  	VTAILQ_ENTRY(sess)	list;
  
diff --cc bin/varnishd/cache/cache_center.c
index dd4e1bd,4967c82..22430ca
--- a/bin/varnishd/cache/cache_center.c
+++ b/bin/varnishd/cache/cache_center.c
@@@ -619,10 -641,9 +641,10 @@@ cnt_fetch(struct sess *sp, struct worke
  		EXP_Clr(&wrk->busyobj->exp);
  		wrk->busyobj->exp.entered = W_TIM_real(wrk);
  		RFC2616_Ttl(sp);
 +		wrk->busyobj->exp.keep = cache_param->default_keep;
  
  		/* pass from vclrecv{} has negative TTL */
- 		if (wrk->objcore == NULL)
+ 		if (req->objcore == NULL)
  			wrk->busyobj->exp.ttl = -1.;
  
  		AZ(wrk->busyobj->do_esi);
@@@ -790,14 -810,10 +811,14 @@@ cnt_fetchbody(struct sess *sp, struct w
  
  	l = http_EstimateWS(wrk->busyobj->beresp,
  	    pass ? HTTPH_R_PASS : HTTPH_A_INS, &nhttp);
 +        if (sp->stale_obj) {
 +            l += http_EstimateWS(sp->stale_obj->http, 0, &stale_nhttp);
 +            nhttp += stale_nhttp;
 +        }
  
  	/* Create Vary instructions */
- 	if (wrk->objcore != NULL) {
- 		CHECK_OBJ_NOTNULL(wrk->objcore, OBJCORE_MAGIC);
+ 	if (req->objcore != NULL) {
+ 		CHECK_OBJ_NOTNULL(req->objcore, OBJCORE_MAGIC);
  		vary = VRY_Create(sp, wrk->busyobj->beresp);
  		if (vary != NULL) {
  			varyl = VSB_len(vary);
@@@ -835,10 -851,9 +856,11 @@@
  		VBO_DerefBusyObj(wrk, &wrk->busyobj);
  		return (0);
  	}
- 	CHECK_OBJ_NOTNULL(wrk->obj, OBJECT_MAGIC);
+ 	CHECK_OBJ_NOTNULL(req->obj, OBJECT_MAGIC);
+ 
 +	wrk->busyobj->exp.keep = sp->exp.keep;
 +
- 	wrk->storage_hint = NULL;
+ 	req->storage_hint = NULL;
  
  	if (wrk->busyobj->do_gzip ||
  	    (wrk->busyobj->is_gzip && !wrk->busyobj->do_gunzip))
@@@ -862,32 -877,14 +884,32 @@@
  
  	hp2->logtag = HTTP_Obj;
  	http_CopyResp(hp2, hp);
 +        
  	http_FilterFields(wrk, sp->vsl_id, hp2, hp,
  	    pass ? HTTPH_R_PASS : HTTPH_A_INS);
 +
 +        /*
 +	 * If we found a candidate for conditional backend request, attempt it
 +         * now. If backend responds with 304, http_Check304() merges stale_obj
 +         * into wrk->obj, any other response is handled as usual. In either case,
 +         * the stale_obj is no longer needed in the cache, so discard it.
 +         */
 +        if (sp->stale_obj) {
 +            http_Check304(sp);
 +            if (wrk->busyobj->beresp->status == 304)
 +                assert(wrk->obj->http->status == 200);
 +	    EXP_Clr(&sp->stale_obj->exp);
 +	    EXP_Rearm(sp->stale_obj);
 +	    HSH_Deref(sp->wrk, NULL, &sp->stale_obj);
 +	    AZ(sp->stale_obj);
 +        }
  	http_CopyHome(wrk, sp->vsl_id, hp2);
  
 -	if (http_GetHdr(hp, H_Last_Modified, &b))
 -		req->obj->last_modified = VTIM_parse(b);
 +	if (http_GetHdr(hp, H_Last_Modified, &b)
 +            || http_GetHdr(wrk->obj->http, H_Last_Modified, &b))
 +		wrk->obj->last_modified = VTIM_parse(b);
  	else
- 		wrk->obj->last_modified = floor(wrk->busyobj->exp.entered);
+ 		req->obj->last_modified = floor(wrk->busyobj->exp.entered);
  
  	assert(WRW_IsReleased(wrk));
  
@@@ -1227,11 -1217,9 +1242,11 @@@ cnt_lookup(struct sess *sp, struct work
  
  	if (oc->flags & OC_F_PASS) {
  		wrk->stats.cache_hitpass++;
- 		WSP(sp, SLT_HitPass, "%u", wrk->obj->xid);
- 		(void)HSH_Deref(wrk, NULL, &wrk->obj);
+ 		WSP(sp, SLT_HitPass, "%u", req->obj->xid);
+ 		(void)HSH_Deref(wrk, NULL, &req->obj);
 +                if (sp->stale_obj != NULL)
 +                    (void)HSH_Deref(wrk, NULL, &sp->stale_obj);
- 		wrk->objcore = NULL;
+ 		req->objcore = NULL;
  		sp->step = STP_PASS;
  		return (0);
  	}
@@@ -1297,18 -1284,12 +1311,18 @@@ cnt_miss(struct sess *sp, struct worke
  	wrk->first_byte_timeout = 0;
  	wrk->between_bytes_timeout = 0;
  
 +        /* If a candidate for a conditional backend request was found,
 +         * add If-Modified-Since and/or If-None-Match to the bereq.
 +         */
 +        if (sp->stale_obj)
 +                http_CheckRefresh(sp);
 +
  	VCL_miss_method(sp);
  
- 	switch(sp->handling) {
+ 	switch(req->handling) {
  	case VCL_RET_ERROR:
- 		AZ(HSH_Deref(wrk, wrk->objcore, NULL));
- 		wrk->objcore = NULL;
+ 		AZ(HSH_Deref(wrk, req->objcore, NULL));
+ 		req->objcore = NULL;
  		http_Setup(wrk->busyobj->bereq, NULL);
  		VBO_DerefBusyObj(wrk, &wrk->busyobj);
  		sp->step = STP_ERROR;
diff --cc bin/varnishd/cache/cache_hash.c
index c56a2d5,d967291..75f568c
--- a/bin/varnishd/cache/cache_hash.c
+++ b/bin/varnishd/cache/cache_hash.c
@@@ -303,10 -302,9 +304,10 @@@ HSH_Lookup(struct sess *sp, struct objh
  
  	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
  	CHECK_OBJ_NOTNULL(sp->wrk, WORKER_MAGIC);
- 	CHECK_OBJ_NOTNULL(sp->http, HTTP_MAGIC);
- 	AN(sp->director);
+ 	CHECK_OBJ_NOTNULL(sp->req->http, HTTP_MAGIC);
+ 	AN(sp->req->director);
  	AN(hash);
 +        AZ(sp->stale_obj);
  	wrk = sp->wrk;
  
  	HSH_Prealloc(sp);
diff --cc bin/varnishd/cache/cache_http.c
index b314b55,f441ef2..ef7bbc5
--- a/bin/varnishd/cache/cache_http.c
+++ b/bin/varnishd/cache/cache_http.c
@@@ -912,93 -878,12 +912,94 @@@ http_FilterHeader(const struct sess *sp
  	if (how == HTTPH_R_FETCH)
  		http_SetH(hp, HTTP_HDR_PROTO, "HTTP/1.1");
  	else
- 		http_copyh(hp, sp->http, HTTP_HDR_PROTO);
- 	http_FilterFields(sp->wrk, sp->vsl_id, hp, sp->http, how);
- 	http_PrintfHeader(sp->wrk, sp->vsl_id, hp, "X-Varnish: %u", sp->xid);
+ 		http_copyh(hp, sp->req->http, HTTP_HDR_PROTO);
+ 	http_FilterFields(sp->wrk, sp->vsl_id, hp, sp->req->http, how);
+ 	http_PrintfHeader(sp->wrk, sp->vsl_id, hp,
+ 	    "X-Varnish: %u", sp->req->xid);
  }
  
 +/*-------------------------------------------------------------------
 + * This function checks for sp->freshen_obj.  If present, HSH_Lookup()
 + * found an expired object that qualifies for a refresh check,
 + * so add the appropriate headers.
 + */
 +
 +void
 +http_CheckRefresh(struct sess *sp)
 +{
 +	struct object *freshen_obj;
 +	struct http *obj_hp, *bereq_hp;
 +	char *p;
 +
 +	freshen_obj = sp->stale_obj;
 +	CHECK_OBJ_NOTNULL(freshen_obj, OBJECT_MAGIC);
 +	bereq_hp = sp->wrk->busyobj->bereq;
 +	CHECK_OBJ_NOTNULL(bereq_hp, HTTP_MAGIC);
 +	obj_hp = freshen_obj->http;
 +	CHECK_OBJ_NOTNULL(obj_hp, HTTP_MAGIC);
 +
 +	if(http_GetHdr(obj_hp, H_ETag, &p))
 +		http_PrintfHeader(sp->wrk, sp->fd, bereq_hp, "If-None-Match: %s", p);
 +
 +	if(http_GetHdr(obj_hp, H_Last_Modified, &p))
 +		http_PrintfHeader(sp->wrk, sp->fd, bereq_hp, "If-Modified-Since: %s",p);
 +}
 +
 +/*-------------------------------------------------------------------
 + * Called after fetch and sp->freshen_obj present.  Check
 + * response and handle as needed.
 + */
 +
 +void
 +http_Check304(struct sess *sp)
 +{
 +	struct object *o, *o_stale;
 +	char *p;
 +
 +	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 +	o_stale = sp->stale_obj;
 +	CHECK_OBJ_NOTNULL(o_stale, OBJECT_MAGIC);
 +	o = sp->wrk->obj;
 +	CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC);
 +
 +	if (sp->wrk->busyobj->beresp->status != 304) {
 +	    /*
 +	     * IMS/INM headers may have been removed in VCL, so only count a
 +	     * non-validating response if they were present in the request.
 +	     */
 +	    if (http_GetHdr(sp->wrk->busyobj->bereq, H_If_Modified_Since, &p)
 +		|| http_GetHdr(sp->wrk->busyobj->bereq, H_If_None_Match, &p))
 +		sp->wrk->stats.fetch_not_validated++;
 +	    return;
 +	}
 +
 +	/* 
 +	 * Copy headers we need from the stale object into the 304 response
 +	 */
 +	http_FilterMissingFields(sp->wrk, sp->fd, o->http, o_stale->http);
 +
 +	/*
 +	 * Dup the stale object's storage in to the new object
 +	 * and reset Content-Length from the size of the storage.
 +	 */
 +	STV_dup(sp, o_stale, o);
 +	http_Unset(o->http, H_Content_Length);
 +	http_PrintfHeader(sp->wrk, sp->fd, o->http, "Content-Length: %u", o->len);
 +
 +	http_SetResp(o->http, "HTTP/1.1", 200, "Ok Not Modified");
 +	http_SetH(o->http, HTTP_HDR_REQ, "GET");
 +	http_copyh(o->http, sp->wrk->busyobj->bereq, HTTP_HDR_URL);
 +
 +	/*
 +	 * XXX: Are we copying all the necessary fields from stale_obj?
 +	 *	Should we copy o_stale->hits into o->hits?
 +	 */
 +	o->response = 200;
 +	o->gziped = o_stale->gziped;
 +
 +        AZ(o_stale->objcore->flags & OC_F_BUSY);
 +}
 +
  /*--------------------------------------------------------------------
   * This function copies any header fields which reference foreign
   * storage into our own WS.
diff --cc bin/varnishd/cache/cache_vrt.c
index 290b6bf,2623bcc..50a7b7f
--- a/bin/varnishd/cache/cache_vrt.c
+++ b/bin/varnishd/cache/cache_vrt.c
@@@ -106,16 -107,12 +107,16 @@@ vrt_selecthttp(const struct sess *sp, e
  		hp = sp->wrk->busyobj->beresp;
  		break;
  	case HDR_RESP:
- 		hp = sp->wrk->resp;
+ 		hp = sp->req->resp;
  		break;
  	case HDR_OBJ:
- 		CHECK_OBJ_NOTNULL(sp->wrk->obj, OBJECT_MAGIC);
- 		hp = sp->wrk->obj->http;
+ 		CHECK_OBJ_NOTNULL(sp->req->obj, OBJECT_MAGIC);
+ 		hp = sp->req->obj->http;
  		break;
 +        case HDR_STALE_OBJ:
 +		CHECK_OBJ_NOTNULL(sp->stale_obj, OBJECT_MAGIC);
 +		hp = sp->stale_obj->http;
 +		break;
  	default:
  		INCOMPL();
  	}
diff --cc bin/varnishd/cache/cache_vrt_var.c
index b5a5bac,e051dbb..5f60a5c
--- a/bin/varnishd/cache/cache_vrt_var.c
+++ b/bin/varnishd/cache/cache_vrt_var.c
@@@ -82,32 -78,22 +82,32 @@@ const char *							
  VRT_r_##obj##_##hdr(const struct sess *sp)			\
  {								\
  	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);			\
 -	CHECK_OBJ_NOTNULL(http, HTTP_MAGIC);			\
 -	return (http->hd[fld].b);				\
 -}
 -
 -VRT_DO_HDR(req,   request,	sp->req->http,		HTTP_HDR_REQ)
 -VRT_DO_HDR(req,   url,		sp->req->http,		HTTP_HDR_URL)
 -VRT_DO_HDR(req,   proto,	sp->req->http,		HTTP_HDR_PROTO)
 -VRT_DO_HDR(bereq, request,	sp->wrk->busyobj->bereq,	HTTP_HDR_REQ)
 -VRT_DO_HDR(bereq, url,		sp->wrk->busyobj->bereq,	HTTP_HDR_URL)
 -VRT_DO_HDR(bereq, proto,	sp->wrk->busyobj->bereq,	HTTP_HDR_PROTO)
 -VRT_DO_HDR(obj,   proto,	sp->req->obj->http,	HTTP_HDR_PROTO)
 -VRT_DO_HDR(obj,   response,	sp->req->obj->http,	HTTP_HDR_RESPONSE)
 -VRT_DO_HDR(resp,  proto,	sp->req->resp,		HTTP_HDR_PROTO)
 -VRT_DO_HDR(resp,  response,	sp->req->resp,		HTTP_HDR_RESPONSE)
 -VRT_DO_HDR(beresp,  proto,	sp->wrk->busyobj->beresp,	HTTP_HDR_PROTO)
 -VRT_DO_HDR(beresp,  response,	sp->wrk->busyobj->beresp, HTTP_HDR_RESPONSE)
 +        if (!nullable || cont != NULL) {			\
 +            CHECK_OBJ_NOTNULL(cont->http, HTTP_MAGIC);		\
 +            return (cont->http->hd[fld].b);			\
 +        }							\
 +        ILLEGAL_R(sp, #obj, #hdr);				\
 +        return(NULL);                                           \
 +}                                                               \
 +
 +#define VRT_DO_HDR(obj, hdr, cont, http, fld, nullable)		\
 +VRT_DO_HDR_l(obj, hdr, cont, http, fld)				\
 +VRT_DO_HDR_r(obj, hdr, cont, http, fld, nullable)		\
 +
- VRT_DO_HDR(req,		request,	sp,			http,	HTTP_HDR_REQ,		0)
- VRT_DO_HDR(req,		url,		sp,			http,	HTTP_HDR_URL,		0)
- VRT_DO_HDR(req,		proto,		sp,			http,	HTTP_HDR_PROTO,		0)
++VRT_DO_HDR(req,		request,	sp->req,		http,	HTTP_HDR_REQ,		0)
++VRT_DO_HDR(req,		url,		sp->req,		http,	HTTP_HDR_URL,		0)
++VRT_DO_HDR(req,		proto,		sp->req,		http,	HTTP_HDR_PROTO,		0)
 +VRT_DO_HDR(bereq,	request,	sp->wrk->busyobj,	bereq,	HTTP_HDR_REQ,		0)
 +VRT_DO_HDR(bereq,	url,		sp->wrk->busyobj,	bereq,	HTTP_HDR_URL,		0)
 +VRT_DO_HDR(bereq,	proto,		sp->wrk->busyobj,	bereq,	HTTP_HDR_PROTO,		0)
- VRT_DO_HDR(obj,		proto,		sp->wrk->obj,		http,	HTTP_HDR_PROTO,		0)
- VRT_DO_HDR(obj,		response,	sp->wrk->obj,		http,	HTTP_HDR_RESPONSE,	0)
- VRT_DO_HDR(resp,	proto,		sp->wrk,		resp,	HTTP_HDR_PROTO,		0)
- VRT_DO_HDR(resp,	response,	sp->wrk,		resp,	HTTP_HDR_RESPONSE,	0)
++VRT_DO_HDR(obj,		proto,		sp->req->obj,		http,	HTTP_HDR_PROTO,		0)
++VRT_DO_HDR(obj,		response,	sp->req->obj,		http,	HTTP_HDR_RESPONSE,	0)
++VRT_DO_HDR(resp,	proto,		sp->req,		resp,	HTTP_HDR_PROTO,		0)
++VRT_DO_HDR(resp,	response,	sp->req,		resp,	HTTP_HDR_RESPONSE,	0)
 +VRT_DO_HDR(beresp,	proto,		sp->wrk->busyobj,	beresp,	HTTP_HDR_PROTO,		0)
 +VRT_DO_HDR(beresp,	response,	sp->wrk->busyobj,	beresp,	HTTP_HDR_RESPONSE,	0)
 +VRT_DO_HDR_r(stale_obj,	proto,		sp->stale_obj,		http,	HTTP_HDR_PROTO,		1)
 +VRT_DO_HDR_r(stale_obj,	response,	sp->stale_obj,		http,	HTTP_HDR_RESPONSE,	1)
  
  /*--------------------------------------------------------------------*/
  
@@@ -126,21 -111,12 +126,21 @@@ VRT_r_##obj##_status(const struct sess 
  {								\
  								\
  	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);			\
 -	return(http->status);					\
 +        if (nullable && cont == NULL) {				\
 +            ILLEGAL_R(sp, #obj, "status");			\
 +            return (503);                                       \
 +        }                                                       \
 +	return(cont->http->status);				\
  }
  
 -VRT_DO_STATUS(obj, sp->req->obj->http)
 -VRT_DO_STATUS(beresp, sp->wrk->busyobj->beresp)
 -VRT_DO_STATUS(resp, sp->req->resp)
 +#define VRT_DO_STATUS(obj, cont, http, nullable)		\
 +VRT_DO_STATUS_l(obj, cont, http)				\
 +VRT_DO_STATUS_r(obj, cont, http, nullable)			\
 +
- VRT_DO_STATUS(obj,		sp->wrk->obj,		http,	0)
++VRT_DO_STATUS(obj,		sp->req->obj,		http,	0)
 +VRT_DO_STATUS(beresp,		sp->wrk->busyobj,	beresp,	0)
- VRT_DO_STATUS(resp,		sp->wrk,		resp,	0)
++VRT_DO_STATUS(resp,		sp->req,		resp,	0)
 +VRT_DO_STATUS_r(stale_obj,	sp->stale_obj,		http,	1)
  
  /*--------------------------------------------------------------------*/
  
@@@ -433,30 -401,26 +433,30 @@@ vrt_wsp_exp(const struct sess *sp, unsi
  	    sp->t_req, e->age + (sp->t_req - e->entered));
  }
  
- VRT_DO_EXP(req, sp, ttl, 0, 0, )
- VRT_DO_EXP(req, sp, grace, 0, 0, )
- VRT_DO_EXP(req, sp, keep, 0, 0, )
 -VRT_DO_EXP(req, sp->req->exp, ttl, 0, )
 -VRT_DO_EXP(req, sp->req->exp, grace, 0, )
 -VRT_DO_EXP(req, sp->req->exp, keep, 0, )
++VRT_DO_EXP(req, sp->req, ttl, 0, 0, )
++VRT_DO_EXP(req, sp->req, grace, 0, 0, )
++VRT_DO_EXP(req, sp->req, keep, 0, 0, )
  
- VRT_DO_EXP(obj, sp->wrk->obj, grace, 0, 0,
-    EXP_Rearm(sp->wrk->obj);
-    vrt_wsp_exp(sp, sp->wrk->obj->xid, &sp->wrk->obj->exp);)
- VRT_DO_EXP(obj, sp->wrk->obj, ttl, (sp->t_req - sp->wrk->obj->exp.entered), 0,
-    EXP_Rearm(sp->wrk->obj);
-    vrt_wsp_exp(sp, sp->wrk->obj->xid, &sp->wrk->obj->exp);)
- VRT_DO_EXP(obj, sp->wrk->obj, keep, 0, 0,
-    EXP_Rearm(sp->wrk->obj);
-    vrt_wsp_exp(sp, sp->wrk->obj->xid, &sp->wrk->obj->exp);)
 -VRT_DO_EXP(obj, sp->req->obj->exp, grace, 0,
++VRT_DO_EXP(obj, sp->req->obj, grace, 0, 0,
+    EXP_Rearm(sp->req->obj);
+    vrt_wsp_exp(sp, sp->req->obj->xid, &sp->req->obj->exp);)
 -VRT_DO_EXP(obj, sp->req->obj->exp, ttl, (sp->t_req - sp->req->obj->exp.entered),
++VRT_DO_EXP(obj, sp->req->obj, ttl, (sp->t_req - sp->req->obj->exp.entered), 0,
+    EXP_Rearm(sp->req->obj);
+    vrt_wsp_exp(sp, sp->req->obj->xid, &sp->req->obj->exp);)
 -VRT_DO_EXP(obj, sp->req->obj->exp, keep, 0,
++VRT_DO_EXP(obj, sp->req->obj, keep, 0, 0,
+    EXP_Rearm(sp->req->obj);
+    vrt_wsp_exp(sp, sp->req->obj->xid, &sp->req->obj->exp);)
  
 -VRT_DO_EXP(beresp, sp->wrk->busyobj->exp, grace, 0,
 +VRT_DO_EXP(beresp, sp->wrk->busyobj, grace, 0, 0,
-    vrt_wsp_exp(sp, sp->xid, &sp->wrk->busyobj->exp);)
+    vrt_wsp_exp(sp, sp->req->xid, &sp->wrk->busyobj->exp);)
 -VRT_DO_EXP(beresp, sp->wrk->busyobj->exp, ttl, 0,
 +VRT_DO_EXP(beresp, sp->wrk->busyobj, ttl, 0, 0,
-    vrt_wsp_exp(sp, sp->xid, &sp->wrk->busyobj->exp);)
+    vrt_wsp_exp(sp, sp->req->xid, &sp->wrk->busyobj->exp);)
 -VRT_DO_EXP(beresp, sp->wrk->busyobj->exp, keep, 0,
 +VRT_DO_EXP(beresp, sp->wrk->busyobj, keep, 0, 0,
-    vrt_wsp_exp(sp, sp->xid, &sp->wrk->busyobj->exp);)
+    vrt_wsp_exp(sp, sp->req->xid, &sp->wrk->busyobj->exp);)
 +    
 +VRT_DO_EXP_r(stale_obj, sp->stale_obj, grace, 0, 1)
 +VRT_DO_EXP_r(stale_obj, sp->stale_obj, ttl, 0, 1)
 +VRT_DO_EXP_r(stale_obj, sp->stale_obj, keep, 0, 1)
  
  /*--------------------------------------------------------------------
   * req.xid
@@@ -569,56 -531,24 +569,56 @@@ VRT_r_obj_hits(const struct sess *sp
  {
  
  	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
- 	CHECK_OBJ_NOTNULL(sp->wrk->obj, OBJECT_MAGIC);	/* XXX */
- 	return (sp->wrk->obj->hits);
+ 	CHECK_OBJ_NOTNULL(sp->req->obj, OBJECT_MAGIC);	/* XXX */
+ 	return (sp->req->obj->hits);
  }
  
 +int
 +VRT_r_stale_obj_hits(const struct sess *sp)
 +{
 +
 +	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 +        if (sp->stale_obj == NULL) {
 +            ILLEGAL_R(sp, "stale_obj", "hits");
 +            return (0);
 +        }
 +	CHECK_OBJ(sp->stale_obj, OBJECT_MAGIC);	/* XXX */
 +	return (sp->stale_obj->hits);
 +}
 +
  double
  VRT_r_obj_lastuse(const struct sess *sp)
  {
  
  	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
- 	CHECK_OBJ_NOTNULL(sp->wrk->obj, OBJECT_MAGIC);	/* XXX */
- 	return (VTIM_real() - sp->wrk->obj->last_use);
+ 	CHECK_OBJ_NOTNULL(sp->req->obj, OBJECT_MAGIC);	/* XXX */
+ 	return (VTIM_real() - sp->req->obj->last_use);
  }
  
 +double
 +VRT_r_stale_obj_lastuse(const struct sess *sp)
 +{
 +
 +	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 +        if (sp->stale_obj == NULL) {
 +            ILLEGAL_R(sp, "stale_obj", "lastuse");
 +            return (0);
 +        }
 +	CHECK_OBJ(sp->stale_obj, OBJECT_MAGIC);	/* XXX */
 +	return (VTIM_real() - sp->stale_obj->last_use);
 +}
 +
  unsigned
  VRT_r_req_backend_healthy(const struct sess *sp)
  {
  	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
- 	CHECK_OBJ_NOTNULL(sp->director, DIRECTOR_MAGIC);
- 	return (VDI_Healthy(sp->director, sp));
+ 	CHECK_OBJ_NOTNULL(sp->req->director, DIRECTOR_MAGIC);
+ 	return (VDI_Healthy(sp->req->director, sp));
  }
  
 +unsigned
 +VRT_r_stale_obj(const struct sess *sp)
 +{
 +        CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 +        return (sp->stale_obj != NULL);
 +}
diff --cc bin/varnishd/mgt/mgt_param.c
index 71f08eb,c488f23..b1d562d
--- a/bin/varnishd/mgt/mgt_param.c
+++ b/bin/varnishd/mgt/mgt_param.c
@@@ -687,15 -767,22 +767,22 @@@ static const struct parspec input_parsp
  		"around this long, making it available for conditional "
  		"backend fetches.  "
  		"That means that the object will be removed from the "
 -		"cache at the end of ttl+grace+keep.",
 +		"cache at the end of ttl+max(grace,keep).",
  		DELAYED_EFFECT,
 -		"0", "seconds" },
 +		"10", "seconds" },
- 	{ "sess_timeout", tweak_timeout, &mgt_param.sess_timeout, 0, 0,
- 		"Idle timeout for persistent sessions. "
- 		"If a HTTP request has not been received in this many "
- 		"seconds, the session is closed.",
+ 	{ "timeout_idle", tweak_timeout_double, &mgt_param.timeout_idle,
+ 		0, UINT_MAX,
+ 		"Idle timeout for client connections.\n"
+ 		"A connection is considered idle, until we receive"
+ 		" a non-white-space character on it.",
  		0,
  		"5", "seconds" },
+ 	{ "timeout_req", tweak_timeout_double, &mgt_param.timeout_req,
+ 		0, UINT_MAX,
+ 		"Max time to receive clients request header, measured"
+ 		" from first non-white-space character to double CRNL.",
+ 		0,
+ 		"2", "seconds" },
  	{ "expiry_sleep", tweak_timeout_double, &mgt_param.expiry_sleep, 0, 60,
  		"How long the expiry thread sleeps when there is nothing "
  		"for it to do.\n",
diff --cc include/tbl/vsc_f_main.h
index 5b9abf6,df7984a..910d2d9
--- a/include/tbl/vsc_f_main.h
+++ b/include/tbl/vsc_f_main.h
@@@ -142,40 -143,8 +143,10 @@@ VSC_F(fetch_failed,		uint64_t, 1, 'a', 
  VSC_F(fetch_1xx,		uint64_t, 1, 'a', "Fetch no body (1xx)", "")
  VSC_F(fetch_204,		uint64_t, 1, 'a', "Fetch no body (204)", "")
  VSC_F(fetch_304,		uint64_t, 1, 'a', "Fetch no body (304)", "")
 +VSC_F(fetch_not_validated,	uint64_t, 1, 'c', "Non-validating responses",
 +      "Count of backend responses to conditional requests with status != 304")
  
  /*---------------------------------------------------------------------
-  * Session Memory
-  *    see: cache_session.c
-  */
- 
- VSC_F(sessmem_size,		uint64_t, 1, 'g',
-     "Session mem size",
- 	"Bytes of memory allocated for last allocated session."
- )
- 
- VSC_F(sessmem_alloc,		uint64_t, 1, 'c',
-     "Session mem allocated",
- 	"Count of all allocations of session memory."
- )
- 
- VSC_F(sessmem_free,		uint64_t, 1, 'c',
-     "Session mem freed",
- 	"Count of all frees of session memory."
- )
- 
- VSC_F(sessmem_fail,		uint64_t, 1, 'c',
-     "Session mem alloc failed",
- 	"Count of session memory allocation failures."
- )
- 
- VSC_F(sessmem_limit,		uint64_t, 1, 'c',
-     "Session mem alloc limited",
- 	"Count of session memory allocations blocked by limit (max_sess)."
- )
- 
- /*---------------------------------------------------------------------
   * Pools, threads, and sessions
   *    see: cache_pool.c
   *
diff --cc include/vrt.h
index bbcc412,bbf16ac..d121d2a
--- a/include/vrt.h
+++ b/include/vrt.h
@@@ -156,10 -156,10 +156,10 @@@ void VRT_purge(const struct sess *sp, d
  
  void VRT_count(const struct sess *, unsigned);
  int VRT_rewrite(const char *, const char *);
- void VRT_error(struct sess *, unsigned, const char *);
+ void VRT_error(const struct sess *, unsigned, const char *);
  int VRT_switch_config(const char *);
  
 -enum gethdr_e { HDR_REQ, HDR_RESP, HDR_OBJ, HDR_BEREQ, HDR_BERESP };
 +enum gethdr_e { HDR_REQ, HDR_RESP, HDR_OBJ, HDR_STALE_OBJ, HDR_BEREQ, HDR_BERESP };
  char *VRT_GetHdr(const struct sess *, enum gethdr_e where, const char *);
  void VRT_SetHdr(const struct sess *, enum gethdr_e where, const char *,
      const char *, ...);



From geoff at varnish-cache.org  Mon Jan  9 23:03:35 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Tue, 10 Jan 2012 00:03:35 +0100
Subject: [experimental-ims] 51ee983 Bugfixes for merge master ->
	experimental-ims to pass varnishtest
Message-ID: <E1RkOFb-00081Y-9K@project.varnish-software.com>

commit 51ee983341dbac603510cdf657fa31d8476fdc19
Author: Geoff Simmons <geoff at uplex.de>
Date:   Tue Jan 10 00:01:50 2012 +0100

    Bugfixes for merge master -> experimental-ims to pass varnishtest

diff --git a/bin/varnishd/cache/cache_center.c b/bin/varnishd/cache/cache_center.c
index 22430ca..1883943 100644
--- a/bin/varnishd/cache/cache_center.c
+++ b/bin/varnishd/cache/cache_center.c
@@ -858,7 +858,7 @@ cnt_fetchbody(struct sess *sp, struct worker *wrk, struct req *req)
 	}
 	CHECK_OBJ_NOTNULL(req->obj, OBJECT_MAGIC);
 
-	wrk->busyobj->exp.keep = sp->exp.keep;
+	wrk->busyobj->exp.keep = req->exp.keep;
 
 	req->storage_hint = NULL;
 
@@ -897,7 +897,7 @@ cnt_fetchbody(struct sess *sp, struct worker *wrk, struct req *req)
         if (sp->stale_obj) {
             http_Check304(sp);
             if (wrk->busyobj->beresp->status == 304)
-                assert(wrk->obj->http->status == 200);
+                assert(req->obj->http->status == 200);
 	    EXP_Clr(&sp->stale_obj->exp);
 	    EXP_Rearm(sp->stale_obj);
 	    HSH_Deref(sp->wrk, NULL, &sp->stale_obj);
@@ -906,8 +906,8 @@ cnt_fetchbody(struct sess *sp, struct worker *wrk, struct req *req)
 	http_CopyHome(wrk, sp->vsl_id, hp2);
 
 	if (http_GetHdr(hp, H_Last_Modified, &b)
-            || http_GetHdr(wrk->obj->http, H_Last_Modified, &b))
-		wrk->obj->last_modified = VTIM_parse(b);
+            || http_GetHdr(req->obj->http, H_Last_Modified, &b))
+		req->obj->last_modified = VTIM_parse(b);
 	else
 		req->obj->last_modified = floor(wrk->busyobj->exp.entered);
 
diff --git a/bin/varnishd/cache/cache_hash.c b/bin/varnishd/cache/cache_hash.c
index 75f568c..6ae193d 100644
--- a/bin/varnishd/cache/cache_hash.c
+++ b/bin/varnishd/cache/cache_hash.c
@@ -472,7 +472,7 @@ HSH_Lookup(struct sess *sp, struct objhead **poh)
 
         /* If we're not serving a valid or graced object and we saved stale_o,
 	 * it is a candidate for the conditional backend request. */
-        AZ(oc && !sp->hash_always_miss);
+        AZ(oc && !sp->req->hash_always_miss);
         AZ(busy_oc);
         if (stale_o != NULL) {
                 AZ(stale_o->objcore->flags & OC_F_BUSY);
diff --git a/bin/varnishd/cache/cache_http.c b/bin/varnishd/cache/cache_http.c
index ef7bbc5..4f48d52 100644
--- a/bin/varnishd/cache/cache_http.c
+++ b/bin/varnishd/cache/cache_http.c
@@ -959,7 +959,7 @@ http_Check304(struct sess *sp)
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 	o_stale = sp->stale_obj;
 	CHECK_OBJ_NOTNULL(o_stale, OBJECT_MAGIC);
-	o = sp->wrk->obj;
+	o = sp->req->obj;
 	CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC);
 
 	if (sp->wrk->busyobj->beresp->status != 304) {
@@ -984,7 +984,8 @@ http_Check304(struct sess *sp)
 	 */
 	STV_dup(sp, o_stale, o);
 	http_Unset(o->http, H_Content_Length);
-	http_PrintfHeader(sp->wrk, sp->fd, o->http, "Content-Length: %u", o->len);
+	http_PrintfHeader(sp->wrk, sp->fd, o->http, "Content-Length: %u",
+	    o->len);
 
 	http_SetResp(o->http, "HTTP/1.1", 200, "Ok Not Modified");
 	http_SetH(o->http, HTTP_HDR_REQ, "GET");
diff --git a/bin/varnishd/storage/stevedore.c b/bin/varnishd/storage/stevedore.c
index 432a5b2..150a142 100644
--- a/bin/varnishd/storage/stevedore.c
+++ b/bin/varnishd/storage/stevedore.c
@@ -378,7 +378,7 @@ STV_alloc(struct worker *w, size_t size)
 {
 	struct object *obj = w->busyobj->fetch_obj;
 	if (obj == NULL)
-		obj = w->obj;
+		obj = w->sp->req->obj;
 
 	return (stv_alloc(w, obj, size));
 }
diff --git a/bin/varnishtest/tests/i00000.vtc b/bin/varnishtest/tests/i00000.vtc
index 4c4aeb5..d274f95 100644
--- a/bin/varnishtest/tests/i00000.vtc
+++ b/bin/varnishtest/tests/i00000.vtc
@@ -491,8 +491,7 @@ varnish v1 -badvcl {
 
 varnish v1 -cliok "param.show default_keep"
 
-varnish v1 -cliok "param.set default_keep 30s"
+varnish v1 -cliok "param.set default_keep 30"
 
-## XXX: This is not caught as an error
-#varnish v1 -clierr 106 "param.set default_keep foo"
+varnish v1 -clierr 106 "param.set default_keep foo"
 



From geoff at varnish-cache.org  Mon Jan  9 23:03:31 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Tue, 10 Jan 2012 00:03:31 +0100
Subject: [experimental-ims] 8129edf Forgot to commit the vmod_std changes.
Message-ID: <E1RkOFX-0007vU-St@project.varnish-software.com>

commit 8129edff14a76b76caf19ca5dc3cfc9ba92891ae
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sun Dec 25 10:09:48 2011 +0000

    Forgot to commit the vmod_std changes.

diff --git a/lib/libvmod_std/vmod_std.c b/lib/libvmod_std/vmod_std.c
index fc371e7..b2ab419 100644
--- a/lib/libvmod_std/vmod_std.c
+++ b/lib/libvmod_std/vmod_std.c
@@ -191,7 +191,7 @@ vmod_collect(struct sess *sp, enum gethdr_e e, const char *h)
 	(void)sp;
 	(void)h;
 	if (e == HDR_REQ)
-		http_CollectHdr(sp->http, h);
+		http_CollectHdr(sp->req->http, h);
 	else if (e == HDR_BERESP && sp->wrk->busyobj != NULL)
 		http_CollectHdr(sp->wrk->busyobj->beresp, h);
 }



From geoff at varnish-cache.org  Mon Jan  9 23:03:32 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Tue, 10 Jan 2012 00:03:32 +0100
Subject: [experimental-ims] 624b373 Move sha256 hash context from worker to req
Message-ID: <E1RkOFY-0007xL-U1@project.varnish-software.com>

commit 624b3731b13b23fbd60499170dfb2458ef111b6e
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Dec 26 10:12:24 2011 +0000

    Move sha256 hash context from worker to req

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index 71b2c83..372a32d 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -324,9 +324,6 @@ struct worker {
 	uint32_t		*wlb, *wlp, *wle;
 	unsigned		wlr;
 
-	/* Lookup stuff */
-	struct SHA256Context	*sha256ctx;
-
 	struct ws		ws[1];
 
 	struct busyobj		*busyobj;
@@ -622,6 +619,9 @@ struct req {
 	struct ws		ws[1];
 	struct object		*obj;
 	struct objcore		*objcore;
+	/* Lookup stuff */
+	struct SHA256Context	*sha256ctx;
+
 };
 
 /*--------------------------------------------------------------------*/
diff --git a/bin/varnishd/cache/cache_center.c b/bin/varnishd/cache/cache_center.c
index 8ee4470..7e9c77f 100644
--- a/bin/varnishd/cache/cache_center.c
+++ b/bin/varnishd/cache/cache_center.c
@@ -1502,10 +1502,10 @@ cnt_recv(struct sess *sp, struct worker *wrk, struct req *req)
 		}
 	}
 
-	SHA256_Init(wrk->sha256ctx);
+	SHA256_Init(req->sha256ctx);
 	VCL_hash_method(sp);
 	assert(req->handling == VCL_RET_HASH);
-	SHA256_Final(req->digest, wrk->sha256ctx);
+	SHA256_Final(req->digest, req->sha256ctx);
 
 	if (!strcmp(req->http->hd[HTTP_HDR_REQ].b, "HEAD"))
 		req->wantbody = 0;
diff --git a/bin/varnishd/cache/cache_hash.c b/bin/varnishd/cache/cache_hash.c
index 19395d5..643eb94 100644
--- a/bin/varnishd/cache/cache_hash.c
+++ b/bin/varnishd/cache/cache_hash.c
@@ -158,8 +158,8 @@ HSH_AddString(const struct sess *sp, const char *str)
 		str = "";
 	l = strlen(str);
 
-	SHA256_Update(sp->wrk->sha256ctx, str, l);
-	SHA256_Update(sp->wrk->sha256ctx, "#", 1);
+	SHA256_Update(sp->req->sha256ctx, str, l);
+	SHA256_Update(sp->req->sha256ctx, "#", 1);
 
 	if (cache_param->log_hash)
 		WSP(sp, SLT_Hash, "%s", str);
diff --git a/bin/varnishd/cache/cache_session.c b/bin/varnishd/cache/cache_session.c
index ee5060f..d26bd8f 100644
--- a/bin/varnishd/cache/cache_session.c
+++ b/bin/varnishd/cache/cache_session.c
@@ -42,6 +42,7 @@
 #include "cache.h"
 
 #include "waiter/waiter.h"
+#include "vsha256.h"
 #include "vtim.h"
 
 static unsigned ses_size = sizeof (struct sess);
@@ -270,7 +271,10 @@ SES_GetReq(struct sess *sp)
 	nhttp = (uint16_t)cache_param->http_max_hdr;
 	hl = HTTP_estimate(nhttp);
 
-	xxxassert(sz > 3 * hl + 128);
+	xxxassert(sz > 3 * hl + sizeof(struct SHA256Context) + 128);
+
+	sp->req->sha256ctx = (void*)p;
+	p += sizeof(struct SHA256Context);
 
 	sp->req->http = HTTP_create(p, nhttp);
 	p += hl;		// XXX: align ?
diff --git a/bin/varnishd/cache/cache_wrk.c b/bin/varnishd/cache/cache_wrk.c
index 2e933c5..6087991 100644
--- a/bin/varnishd/cache/cache_wrk.c
+++ b/bin/varnishd/cache/cache_wrk.c
@@ -37,7 +37,6 @@
 #include "cache.h"
 
 #include "hash/hash_slinger.h"
-#include "vsha256.h"
 
 static struct lock		wstat_mtx;
 
@@ -140,7 +139,6 @@ wrk_thread_real(void *priv, unsigned shm_workspace, unsigned sess_workspace,
 	/* XXX: can we trust these to be properly aligned ? */
 	unsigned char ws[sess_workspace];
 	struct iovec iov[siov];
-	struct SHA256Context sha256;
 
 	THR_SetName("cache-worker");
 	w = &ww;
@@ -149,7 +147,6 @@ wrk_thread_real(void *priv, unsigned shm_workspace, unsigned sess_workspace,
 	w->lastused = NAN;
 	w->wlb = w->wlp = wlog;
 	w->wle = wlog + (sizeof wlog) / 4;
-	w->sha256ctx = &sha256;
 	w->wrw.iov = iov;
 	w->wrw.siov = siov;
 	w->wrw.ciov = siov;



From geoff at varnish-cache.org  Mon Jan  9 23:03:29 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Tue, 10 Jan 2012 00:03:29 +0100
Subject: [experimental-ims] 9a7413d Sort the sess fields in per session and
	per request fields, plus a few I have not made up my mind about yet.
Message-ID: <E1RkOFV-0007sY-RY@project.varnish-software.com>

commit 9a7413d2a7d28333112cb6326e785582373b6684
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu Dec 22 18:41:03 2011 +0000

    Sort the sess fields in per session and per request fields, plus a few
    I have not made up my mind about yet.

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index 08f8adc..46a0d26 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -581,18 +581,13 @@ struct object {
 struct sess {
 	unsigned		magic;
 #define SESS_MAGIC		0x2c2f9c5a
-	int			fd;
-	unsigned		vsl_id;
-	unsigned		xid;
 
-	int			restarts;
-	int			esi_level;
-	int			disable_esi;
+	struct worker		*wrk;
 
-	uint8_t			hash_ignore_busy;
-	uint8_t			hash_always_miss;
+	/* Session related fields ------------------------------------*/
 
-	struct worker		*wrk;
+	int			fd;
+	unsigned		vsl_id;
 
 	socklen_t		sockaddrlen;
 	socklen_t		mysockaddrlen;
@@ -605,6 +600,29 @@ struct sess {
 	char			port[PORT_BUFSIZE];
 	char			*client_identity;
 
+	/* Various internal stuff */
+	struct sessmem		*mem;
+
+	VTAILQ_ENTRY(sess)	poollist;
+	struct acct		acct_ses;
+
+	VTAILQ_ENTRY(sess)	list;
+
+#if defined(HAVE_EPOLL_CTL)
+	struct epoll_event ev;
+#endif
+
+	/* Request related fields ------------------------------------*/
+
+	unsigned		xid;
+
+	int			restarts;
+	int			esi_level;
+	int			disable_esi;
+
+	uint8_t			hash_ignore_busy;
+	uint8_t			hash_always_miss;
+
 	/* HTTP request */
 	const char		*doclose;
 	struct http		*http;
@@ -620,14 +638,6 @@ struct sess {
 	uint8_t			*vary_l;
 	uint8_t			*vary_e;
 
-	struct http_conn	htc[1];
-
-	/* Timestamps, all on TIM_real() timescale */
-	double			t_open;
-	double			t_req;
-	double			t_resp;
-	double			t_end;
-
 	/* Acceptable grace period */
 	struct exp		exp;
 
@@ -639,24 +649,24 @@ struct sess {
 	uint16_t		err_code;
 	const char		*err_reason;
 
-	VTAILQ_ENTRY(sess)	list;
+	/* The busy objhead we sleep on */
+	struct objhead		*hash_objhead;
 
 	struct director		*director;
 	struct VCL_conf		*vcl;
 
-	/* The busy objhead we sleep on */
-	struct objhead		*hash_objhead;
+	uint64_t		req_bodybytes;
 
-	/* Various internal stuff */
-	struct sessmem		*mem;
+	/* TBD fields ------------------------------------------------*/
 
-	VTAILQ_ENTRY(sess)	poollist;
-	uint64_t		req_bodybytes;
-	struct acct		acct_ses;
+	struct http_conn	htc[1];
+
+	/* Timestamps, all on TIM_real() timescale */
+	double			t_open;
+	double			t_req;
+	double			t_resp;
+	double			t_end;
 
-#if defined(HAVE_EPOLL_CTL)
-	struct epoll_event ev;
-#endif
 };
 
 /* Prototypes etc ----------------------------------------------------*/



From geoff at varnish-cache.org  Mon Jan  9 23:03:32 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Tue, 10 Jan 2012 00:03:32 +0100
Subject: [experimental-ims] 6c14864 move obj and objcore from worker to req
Message-ID: <E1RkOFY-0007x0-QJ@project.varnish-software.com>

commit 6c148642883f466b15281ad8799ecbce23c3954b
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Dec 26 10:02:54 2011 +0000

    move obj and objcore from worker to req

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index c919b32..71b2c83 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -329,8 +329,6 @@ struct worker {
 
 	struct ws		ws[1];
 
-	struct object		*obj;
-	struct objcore		*objcore;
 	struct busyobj		*busyobj;
 
 	/* This is only here so VRT can find it */
@@ -622,6 +620,8 @@ struct req {
 	struct http		*resp;
 
 	struct ws		ws[1];
+	struct object		*obj;
+	struct objcore		*objcore;
 };
 
 /*--------------------------------------------------------------------*/
diff --git a/bin/varnishd/cache/cache_backend.c b/bin/varnishd/cache/cache_backend.c
index 24aedb4..b683300 100644
--- a/bin/varnishd/cache/cache_backend.c
+++ b/bin/varnishd/cache/cache_backend.c
@@ -227,7 +227,7 @@ vbe_NewConn(void)
 
 /*--------------------------------------------------------------------
  * It evaluates if a backend is healthy _for_a_specific_object_.
- * That means that it relies on sp->wrk->objcore->objhead. This is mainly for
+ * That means that it relies on sp->req->objcore->objhead. This is mainly for
  * saint-mode, but also takes backend->healthy into account. If
  * cache_param->saintmode_threshold is 0, this is basically just a test of
  * backend->healthy.
@@ -274,11 +274,11 @@ vbe_Healthy(const struct vdi_simple *vs, const struct sess *sp)
 	if (threshold == 0)
 		return (1);
 
-	if (sp->wrk->objcore == NULL)
+	if (sp->req->objcore == NULL)
 		return (1);
 
 	now = sp->t_req;
-	target = (uintptr_t)(sp->wrk->objcore->objhead);
+	target = (uintptr_t)(sp->req->objcore->objhead);
 
 	old = NULL;
 	retval = 1;
diff --git a/bin/varnishd/cache/cache_center.c b/bin/varnishd/cache/cache_center.c
index 298b7b4..8ee4470 100644
--- a/bin/varnishd/cache/cache_center.c
+++ b/bin/varnishd/cache/cache_center.c
@@ -115,7 +115,7 @@ cnt_wait(struct sess *sp, struct worker *wrk, struct req *req)
 	}
 
 	AZ(req->vcl);
-	AZ(wrk->obj);
+	AZ(req->obj);
 	AZ(req->esi_level);
 	assert(req->xid == 0);
 	req->t_resp = NAN;
@@ -213,13 +213,13 @@ cnt_prepresp(struct sess *sp, struct worker *wrk, struct req *req)
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
 
-	CHECK_OBJ_NOTNULL(wrk->obj, OBJECT_MAGIC);
+	CHECK_OBJ_NOTNULL(req->obj, OBJECT_MAGIC);
 	CHECK_OBJ_NOTNULL(req->vcl, VCL_CONF_MAGIC);
 
 	if (wrk->busyobj != NULL) {
 		CHECK_OBJ_NOTNULL(wrk->busyobj, BUSYOBJ_MAGIC);
 		AN(wrk->busyobj->do_stream);
-		AssertObjCorePassOrBusy(wrk->obj->objcore);
+		AssertObjCorePassOrBusy(req->obj->objcore);
 	}
 
 	wrk->res_mode = 0;
@@ -233,7 +233,7 @@ cnt_prepresp(struct sess *sp, struct worker *wrk, struct req *req)
 	    !wrk->busyobj->do_gzip && !wrk->busyobj->do_gunzip)
 		wrk->res_mode |= RES_LEN;
 
-	if (!req->disable_esi && wrk->obj->esidata != NULL) {
+	if (!req->disable_esi && req->obj->esidata != NULL) {
 		/* In ESI mode, we don't know the aggregate length */
 		wrk->res_mode &= ~RES_LEN;
 		wrk->res_mode |= RES_ESI;
@@ -244,7 +244,7 @@ cnt_prepresp(struct sess *sp, struct worker *wrk, struct req *req)
 		wrk->res_mode |= RES_ESI_CHILD;
 	}
 
-	if (cache_param->http_gzip_support && wrk->obj->gziped &&
+	if (cache_param->http_gzip_support && req->obj->gziped &&
 	    !RFC2616_Req_Gzip(sp)) {
 		/*
 		 * We don't know what it uncompresses to
@@ -255,7 +255,7 @@ cnt_prepresp(struct sess *sp, struct worker *wrk, struct req *req)
 	}
 
 	if (!(wrk->res_mode & (RES_LEN|RES_CHUNKED|RES_EOF))) {
-		if (wrk->obj->len == 0 &&
+		if (req->obj->len == 0 &&
 		    (wrk->busyobj == NULL || !wrk->busyobj->do_stream))
 			/*
 			 * If the object is empty, neither ESI nor GUNZIP
@@ -273,12 +273,12 @@ cnt_prepresp(struct sess *sp, struct worker *wrk, struct req *req)
 	}
 
 	req->t_resp = W_TIM_real(wrk);
-	if (wrk->obj->objcore != NULL) {
-		if ((req->t_resp - wrk->obj->last_lru) >
+	if (req->obj->objcore != NULL) {
+		if ((req->t_resp - req->obj->last_lru) >
 		    cache_param->lru_timeout &&
-		    EXP_Touch(wrk->obj->objcore))
-			wrk->obj->last_lru = req->t_resp;
-		wrk->obj->last_use = req->t_resp;	/* XXX: locking ? */
+		    EXP_Touch(req->obj->objcore))
+			req->obj->last_lru = req->t_resp;
+		req->obj->last_use = req->t_resp;	/* XXX: locking ? */
 	}
 	http_Setup(req->resp, req->ws);
 	RES_BuildHttp(sp);
@@ -295,9 +295,9 @@ cnt_prepresp(struct sess *sp, struct worker *wrk, struct req *req)
 			HSH_Drop(wrk);
 			VBO_DerefBusyObj(wrk, &wrk->busyobj);
 		} else {
-			(void)HSH_Deref(wrk, NULL, &wrk->obj);
+			(void)HSH_Deref(wrk, NULL, &req->obj);
 		}
-		AZ(wrk->obj);
+		AZ(req->obj);
 		req->restarts++;
 		req->director = NULL;
 		http_Setup(req->resp, NULL);
@@ -307,7 +307,7 @@ cnt_prepresp(struct sess *sp, struct worker *wrk, struct req *req)
 		WRONG("Illegal action in vcl_deliver{}");
 	}
 	if (wrk->busyobj != NULL && wrk->busyobj->do_stream) {
-		AssertObjCorePassOrBusy(wrk->obj->objcore);
+		AssertObjCorePassOrBusy(req->obj->objcore);
 		sp->step = STP_STREAMBODY;
 	} else {
 		sp->step = STP_DELIVER;
@@ -345,7 +345,7 @@ cnt_deliver(struct sess *sp, struct worker *wrk, struct req *req)
 
 	assert(WRW_IsReleased(wrk));
 	assert(wrk->wrw.ciov == wrk->wrw.siov);
-	(void)HSH_Deref(wrk, NULL, &wrk->obj);
+	(void)HSH_Deref(wrk, NULL, &req->obj);
 	http_Setup(req->resp, NULL);
 	sp->step = STP_DONE;
 	return (0);
@@ -376,7 +376,7 @@ cnt_done(struct sess *sp, struct worker *wrk, struct req *req)
 	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
 	CHECK_OBJ_ORNULL(req->vcl, VCL_CONF_MAGIC);
 
-	AZ(wrk->obj);
+	AZ(req->obj);
 	AZ(wrk->busyobj);
 	req->director = NULL;
 	req->restarts = 0;
@@ -484,17 +484,17 @@ cnt_error(struct sess *sp, struct worker *wrk, struct req *req)
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
 
-	if (wrk->obj == NULL) {
+	if (req->obj == NULL) {
 		HSH_Prealloc(sp);
 		AZ(wrk->busyobj);
 		wrk->busyobj = VBO_GetBusyObj(wrk);
-		wrk->obj = STV_NewObject(wrk, NULL, cache_param->http_resp_size,
+		req->obj = STV_NewObject(wrk, NULL, cache_param->http_resp_size,
 		     (uint16_t)cache_param->http_max_hdr);
-		if (wrk->obj == NULL)
-			wrk->obj = STV_NewObject(wrk, TRANSIENT_STORAGE,
+		if (req->obj == NULL)
+			req->obj = STV_NewObject(wrk, TRANSIENT_STORAGE,
 			    cache_param->http_resp_size,
 			    (uint16_t)cache_param->http_max_hdr);
-		if (wrk->obj == NULL) {
+		if (req->obj == NULL) {
 			req->doclose = "Out of objects";
 			req->director = NULL;
 			http_Setup(wrk->busyobj->beresp, NULL);
@@ -502,15 +502,15 @@ cnt_error(struct sess *sp, struct worker *wrk, struct req *req)
 			sp->step = STP_DONE;
 			return(0);
 		}
-		AN(wrk->obj);
-		wrk->obj->xid = req->xid;
-		wrk->obj->exp.entered = sp->t_req;
+		AN(req->obj);
+		req->obj->xid = req->xid;
+		req->obj->exp.entered = sp->t_req;
 	} else {
 		CHECK_OBJ_NOTNULL(wrk->busyobj, BUSYOBJ_MAGIC);
 		/* XXX: Null the headers ? */
 	}
-	CHECK_OBJ_NOTNULL(wrk->obj, OBJECT_MAGIC);
-	h = wrk->obj->http;
+	CHECK_OBJ_NOTNULL(req->obj, OBJECT_MAGIC);
+	h = req->obj->http;
 
 	if (req->err_code < 100 || req->err_code > 999)
 		req->err_code = 501;
@@ -643,7 +643,7 @@ cnt_fetch(struct sess *sp, struct worker *wrk, struct req *req)
 		RFC2616_Ttl(sp);
 
 		/* pass from vclrecv{} has negative TTL */
-		if (wrk->objcore == NULL)
+		if (req->objcore == NULL)
 			wrk->busyobj->exp.ttl = -1.;
 
 		AZ(wrk->busyobj->do_esi);
@@ -652,12 +652,12 @@ cnt_fetch(struct sess *sp, struct worker *wrk, struct req *req)
 
 		switch (req->handling) {
 		case VCL_RET_HIT_FOR_PASS:
-			if (wrk->objcore != NULL)
-				wrk->objcore->flags |= OC_F_PASS;
+			if (req->objcore != NULL)
+				req->objcore->flags |= OC_F_PASS;
 			sp->step = STP_FETCHBODY;
 			return (0);
 		case VCL_RET_DELIVER:
-			AssertObjCorePassOrBusy(wrk->objcore);
+			AssertObjCorePassOrBusy(req->objcore);
 			sp->step = STP_FETCHBODY;
 			return (0);
 		default:
@@ -671,10 +671,10 @@ cnt_fetch(struct sess *sp, struct worker *wrk, struct req *req)
 	/* Clean up partial fetch */
 	AZ(wrk->busyobj->vbc);
 
-	if (wrk->objcore != NULL) {
-		CHECK_OBJ_NOTNULL(wrk->objcore, OBJCORE_MAGIC);
-		AZ(HSH_Deref(wrk, wrk->objcore, NULL));
-		wrk->objcore = NULL;
+	if (req->objcore != NULL) {
+		CHECK_OBJ_NOTNULL(req->objcore, OBJCORE_MAGIC);
+		AZ(HSH_Deref(wrk, req->objcore, NULL));
+		req->objcore = NULL;
 	}
 	VBO_DerefBusyObj(wrk, &wrk->busyobj);
 	req->director = NULL;
@@ -733,7 +733,7 @@ cnt_fetchbody(struct sess *sp, struct worker *wrk, struct req *req)
 	assert(req->handling == VCL_RET_HIT_FOR_PASS ||
 	    req->handling == VCL_RET_DELIVER);
 
-	if (wrk->objcore == NULL) {
+	if (req->objcore == NULL) {
 		/* This is a pass from vcl_recv */
 		pass = 1;
 		/* VCL may have fiddled this, but that doesn't help */
@@ -812,8 +812,8 @@ cnt_fetchbody(struct sess *sp, struct worker *wrk, struct req *req)
 	    pass ? HTTPH_R_PASS : HTTPH_A_INS, &nhttp);
 
 	/* Create Vary instructions */
-	if (wrk->objcore != NULL) {
-		CHECK_OBJ_NOTNULL(wrk->objcore, OBJCORE_MAGIC);
+	if (req->objcore != NULL) {
+		CHECK_OBJ_NOTNULL(req->objcore, OBJCORE_MAGIC);
 		vary = VRY_Create(sp, wrk->busyobj->beresp);
 		if (vary != NULL) {
 			varyl = VSB_len(vary);
@@ -829,51 +829,51 @@ cnt_fetchbody(struct sess *sp, struct worker *wrk, struct req *req)
 	l += strlen("Content-Length: XxxXxxXxxXxxXxxXxx") + sizeof(void *);
 
 	if (wrk->busyobj->exp.ttl < cache_param->shortlived ||
-	    wrk->objcore == NULL)
+	    req->objcore == NULL)
 		wrk->storage_hint = TRANSIENT_STORAGE;
 
-	wrk->obj = STV_NewObject(wrk, wrk->storage_hint, l, nhttp);
-	if (wrk->obj == NULL) {
+	req->obj = STV_NewObject(wrk, wrk->storage_hint, l, nhttp);
+	if (req->obj == NULL) {
 		/*
 		 * Try to salvage the transaction by allocating a
 		 * shortlived object on Transient storage.
 		 */
-		wrk->obj = STV_NewObject(wrk, TRANSIENT_STORAGE, l, nhttp);
+		req->obj = STV_NewObject(wrk, TRANSIENT_STORAGE, l, nhttp);
 		if (wrk->busyobj->exp.ttl > cache_param->shortlived)
 			wrk->busyobj->exp.ttl = cache_param->shortlived;
 		wrk->busyobj->exp.grace = 0.0;
 		wrk->busyobj->exp.keep = 0.0;
 	}
-	if (wrk->obj == NULL) {
+	if (req->obj == NULL) {
 		req->err_code = 503;
 		sp->step = STP_ERROR;
 		VDI_CloseFd(wrk, &wrk->busyobj->vbc);
 		VBO_DerefBusyObj(wrk, &wrk->busyobj);
 		return (0);
 	}
-	CHECK_OBJ_NOTNULL(wrk->obj, OBJECT_MAGIC);
+	CHECK_OBJ_NOTNULL(req->obj, OBJECT_MAGIC);
 
 	wrk->storage_hint = NULL;
 
 	if (wrk->busyobj->do_gzip ||
 	    (wrk->busyobj->is_gzip && !wrk->busyobj->do_gunzip))
-		wrk->obj->gziped = 1;
+		req->obj->gziped = 1;
 
 	if (vary != NULL) {
-		wrk->obj->vary = (void *)WS_Alloc(wrk->obj->http->ws, varyl);
-		AN(wrk->obj->vary);
-		memcpy(wrk->obj->vary, VSB_data(vary), varyl);
-		VRY_Validate(wrk->obj->vary);
+		req->obj->vary = (void *)WS_Alloc(req->obj->http->ws, varyl);
+		AN(req->obj->vary);
+		memcpy(req->obj->vary, VSB_data(vary), varyl);
+		VRY_Validate(req->obj->vary);
 		VSB_delete(vary);
 	}
 
-	wrk->obj->xid = req->xid;
-	wrk->obj->response = req->err_code;
-	WS_Assert(wrk->obj->ws_o);
+	req->obj->xid = req->xid;
+	req->obj->response = req->err_code;
+	WS_Assert(req->obj->ws_o);
 
 	/* Filter into object */
 	hp = wrk->busyobj->beresp;
-	hp2 = wrk->obj->http;
+	hp2 = req->obj->http;
 
 	hp2->logtag = HTTP_Obj;
 	http_CopyResp(hp2, hp);
@@ -882,9 +882,9 @@ cnt_fetchbody(struct sess *sp, struct worker *wrk, struct req *req)
 	http_CopyHome(wrk, sp->vsl_id, hp2);
 
 	if (http_GetHdr(hp, H_Last_Modified, &b))
-		wrk->obj->last_modified = VTIM_parse(b);
+		req->obj->last_modified = VTIM_parse(b);
 	else
-		wrk->obj->last_modified = floor(wrk->busyobj->exp.entered);
+		req->obj->last_modified = floor(wrk->busyobj->exp.entered);
 
 	assert(WRW_IsReleased(wrk));
 
@@ -893,12 +893,12 @@ cnt_fetchbody(struct sess *sp, struct worker *wrk, struct req *req)
 	 * Notice that vcl_deliver{} could still nuke the headers
 	 * that allow the 304, in which case we return 200 non-stream.
 	 */
-	if (wrk->obj->response == 200 &&
+	if (req->obj->response == 200 &&
 	    req->http->conds &&
 	    RFC2616_Do_Cond(sp))
 		wrk->busyobj->do_stream = 0;
 
-	AssertObjCorePassOrBusy(wrk->obj->objcore);
+	AssertObjCorePassOrBusy(req->obj->objcore);
 
 	if (wrk->busyobj->do_stream) {
 		sp->step = STP_PREPRESP;
@@ -906,7 +906,7 @@ cnt_fetchbody(struct sess *sp, struct worker *wrk, struct req *req)
 	}
 
 	/* Use unmodified headers*/
-	i = FetchBody(wrk, wrk->obj);
+	i = FetchBody(wrk, req->obj);
 
 	http_Setup(wrk->busyobj->bereq, NULL);
 	http_Setup(wrk->busyobj->beresp, NULL);
@@ -918,16 +918,16 @@ cnt_fetchbody(struct sess *sp, struct worker *wrk, struct req *req)
 	if (i) {
 		HSH_Drop(wrk);
 		VBO_DerefBusyObj(wrk, &wrk->busyobj);
-		AZ(wrk->obj);
+		AZ(req->obj);
 		req->err_code = 503;
 		sp->step = STP_ERROR;
 		return (0);
 	}
 
-	if (wrk->obj->objcore != NULL) {
-		EXP_Insert(wrk->obj);
-		AN(wrk->obj->objcore);
-		AN(wrk->obj->objcore->ban);
+	if (req->obj->objcore != NULL) {
+		EXP_Insert(req->obj);
+		AN(req->obj->objcore);
+		AN(req->obj->objcore->ban);
 		HSH_Unbusy(wrk);
 	}
 	VBO_DerefBusyObj(wrk, &wrk->busyobj);
@@ -973,9 +973,9 @@ cnt_streambody(struct sess *sp, struct worker *wrk, struct req *req)
 
 	RES_StreamStart(sp);
 
-	AssertObjCorePassOrBusy(wrk->obj->objcore);
+	AssertObjCorePassOrBusy(req->obj->objcore);
 
-	i = FetchBody(wrk, wrk->obj);
+	i = FetchBody(wrk, req->obj);
 
 	http_Setup(wrk->busyobj->bereq, NULL);
 	http_Setup(wrk->busyobj->beresp, NULL);
@@ -983,10 +983,10 @@ cnt_streambody(struct sess *sp, struct worker *wrk, struct req *req)
 	AZ(wrk->busyobj->vbc);
 	AN(req->director);
 
-	if (!i && wrk->obj->objcore != NULL) {
-		EXP_Insert(wrk->obj);
-		AN(wrk->obj->objcore);
-		AN(wrk->obj->objcore->ban);
+	if (!i && req->obj->objcore != NULL) {
+		EXP_Insert(req->obj);
+		AN(req->obj->objcore);
+		AN(req->obj->objcore->ban);
 		HSH_Unbusy(wrk);
 	} else {
 		req->doclose = "Stream error";
@@ -1002,7 +1002,7 @@ cnt_streambody(struct sess *sp, struct worker *wrk, struct req *req)
 	wrk->sctx = NULL;
 	assert(WRW_IsReleased(wrk));
 	assert(wrk->wrw.ciov == wrk->wrw.siov);
-	(void)HSH_Deref(wrk, NULL, &wrk->obj);
+	(void)HSH_Deref(wrk, NULL, &req->obj);
 	VBO_DerefBusyObj(wrk, &wrk->busyobj);
 	http_Setup(req->resp, NULL);
 	sp->step = STP_DONE;
@@ -1075,11 +1075,11 @@ cnt_hit(struct sess *sp, struct worker *wrk, struct req *req)
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
 
-	CHECK_OBJ_NOTNULL(wrk->obj, OBJECT_MAGIC);
+	CHECK_OBJ_NOTNULL(req->obj, OBJECT_MAGIC);
 	CHECK_OBJ_NOTNULL(req->vcl, VCL_CONF_MAGIC);
 	AZ(wrk->busyobj);
 
-	assert(!(wrk->obj->objcore->flags & OC_F_PASS));
+	assert(!(req->obj->objcore->flags & OC_F_PASS));
 
 	VCL_hit_method(sp);
 
@@ -1093,8 +1093,8 @@ cnt_hit(struct sess *sp, struct worker *wrk, struct req *req)
 	}
 
 	/* Drop our object, we won't need it */
-	(void)HSH_Deref(wrk, NULL, &wrk->obj);
-	wrk->objcore = NULL;
+	(void)HSH_Deref(wrk, NULL, &req->obj);
+	req->objcore = NULL;
 
 	switch(req->handling) {
 	case VCL_RET_PASS:
@@ -1200,7 +1200,7 @@ cnt_lookup(struct sess *sp, struct worker *wrk, struct req *req)
 		req->vary_l = NULL;
 		req->vary_e = NULL;
 
-		wrk->objcore = oc;
+		req->objcore = oc;
 		CHECK_OBJ_NOTNULL(wrk->busyobj, BUSYOBJ_MAGIC);
 		sp->step = STP_MISS;
 		return (0);
@@ -1208,7 +1208,7 @@ cnt_lookup(struct sess *sp, struct worker *wrk, struct req *req)
 
 	o = oc_getobj(wrk, oc);
 	CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC);
-	wrk->obj = o;
+	req->obj = o;
 
 	WS_Release(req->ws, 0);
 	req->vary_b = NULL;
@@ -1217,15 +1217,15 @@ cnt_lookup(struct sess *sp, struct worker *wrk, struct req *req)
 
 	if (oc->flags & OC_F_PASS) {
 		wrk->stats.cache_hitpass++;
-		WSP(sp, SLT_HitPass, "%u", wrk->obj->xid);
-		(void)HSH_Deref(wrk, NULL, &wrk->obj);
-		wrk->objcore = NULL;
+		WSP(sp, SLT_HitPass, "%u", req->obj->xid);
+		(void)HSH_Deref(wrk, NULL, &req->obj);
+		req->objcore = NULL;
 		sp->step = STP_PASS;
 		return (0);
 	}
 
 	wrk->stats.cache_hit++;
-	WSP(sp, SLT_Hit, "%u", wrk->obj->xid);
+	WSP(sp, SLT_Hit, "%u", req->obj->xid);
 	sp->step = STP_HIT;
 	return (0);
 }
@@ -1262,8 +1262,8 @@ cnt_miss(struct sess *sp, struct worker *wrk, struct req *req)
 	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
 	CHECK_OBJ_NOTNULL(req->vcl, VCL_CONF_MAGIC);
 
-	AZ(wrk->obj);
-	AN(wrk->objcore);
+	AZ(req->obj);
+	AN(req->objcore);
 	CHECK_OBJ_NOTNULL(wrk->busyobj, BUSYOBJ_MAGIC);
 	WS_Reset(wrk->ws, NULL);
 	wrk->busyobj = VBO_GetBusyObj(wrk);
@@ -1288,15 +1288,15 @@ cnt_miss(struct sess *sp, struct worker *wrk, struct req *req)
 
 	switch(req->handling) {
 	case VCL_RET_ERROR:
-		AZ(HSH_Deref(wrk, wrk->objcore, NULL));
-		wrk->objcore = NULL;
+		AZ(HSH_Deref(wrk, req->objcore, NULL));
+		req->objcore = NULL;
 		http_Setup(wrk->busyobj->bereq, NULL);
 		VBO_DerefBusyObj(wrk, &wrk->busyobj);
 		sp->step = STP_ERROR;
 		return (0);
 	case VCL_RET_PASS:
-		AZ(HSH_Deref(wrk, wrk->objcore, NULL));
-		wrk->objcore = NULL;
+		AZ(HSH_Deref(wrk, req->objcore, NULL));
+		req->objcore = NULL;
 		VBO_DerefBusyObj(wrk, &wrk->busyobj);
 		sp->step = STP_PASS;
 		return (0);
@@ -1305,8 +1305,8 @@ cnt_miss(struct sess *sp, struct worker *wrk, struct req *req)
 		sp->step = STP_FETCH;
 		return (0);
 	case VCL_RET_RESTART:
-		AZ(HSH_Deref(wrk, wrk->objcore, NULL));
-		wrk->objcore = NULL;
+		AZ(HSH_Deref(wrk, req->objcore, NULL));
+		req->objcore = NULL;
 		VBO_DerefBusyObj(wrk, &wrk->busyobj);
 		INCOMPL();
 	default:
@@ -1353,7 +1353,7 @@ cnt_pass(struct sess *sp, struct worker *wrk, struct req *req)
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
 	CHECK_OBJ_NOTNULL(req->vcl, VCL_CONF_MAGIC);
-	AZ(wrk->obj);
+	AZ(req->obj);
 	AZ(wrk->busyobj);
 
 	wrk->busyobj = VBO_GetBusyObj(wrk);
@@ -1464,7 +1464,7 @@ cnt_recv(struct sess *sp, struct worker *wrk, struct req *req)
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
 	CHECK_OBJ_NOTNULL(req->vcl, VCL_CONF_MAGIC);
-	AZ(wrk->obj);
+	AZ(req->obj);
 	AZ(wrk->busyobj);
 	assert(wrk->wrw.ciov == wrk->wrw.siov);
 
@@ -1562,7 +1562,7 @@ cnt_start(struct sess *sp, struct worker *wrk, struct req *req)
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
 	AZ(req->restarts);
-	AZ(wrk->obj);
+	AZ(req->obj);
 	AZ(req->vcl);
 	EXP_Clr(&req->exp);
 	AZ(req->esi_level);
@@ -1641,20 +1641,24 @@ static void
 cnt_diag(struct sess *sp, const char *state)
 {
 	void *vcl;
+	void *obj;
 
-	if (sp->req == NULL)
+	if (sp->req == NULL) {
 		vcl = NULL;
-	else
+		obj = NULL;
+	} else {
 		vcl = sp->req->vcl;
+		obj = sp->req->obj;
+	}
 
 	if (sp->wrk != NULL) {
 		WSP(sp, SLT_Debug, "thr %p STP_%s sp %p obj %p vcl %p",
-		    pthread_self(), state, sp, sp->wrk->obj, vcl);
+		    pthread_self(), state, sp, obj, vcl);
 		WSL_Flush(sp->wrk, 0);
 	} else {
 		VSL(SLT_Debug, sp->vsl_id,
 		    "thr %p STP_%s sp %p obj %p vcl %p",
-		    pthread_self(), state, sp, sp->wrk->obj, vcl);
+		    pthread_self(), state, sp, obj, vcl);
 	}
 }
 
@@ -1681,9 +1685,6 @@ CNT_Session(struct sess *sp)
 	    sp->step == STP_LOOKUP ||
 	    sp->step == STP_RECV);
 
-	AZ(wrk->obj);
-	AZ(wrk->objcore);
-
 	/*
 	 * Whenever we come in from the acceptor or waiter, we need to set
 	 * blocking mode, but there is no point in setting it when we come from
@@ -1716,7 +1717,6 @@ CNT_Session(struct sess *sp)
 		 */
 		CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 		CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
-		CHECK_OBJ_ORNULL(wrk->obj, OBJECT_MAGIC);
 		CHECK_OBJ_ORNULL(wrk->nobjhead, OBJHEAD_MAGIC);
 		WS_Assert(wrk->ws);
 
@@ -1736,8 +1736,6 @@ CNT_Session(struct sess *sp)
 		CHECK_OBJ_ORNULL(wrk->nobjhead, OBJHEAD_MAGIC);
 	}
 	WSL_Flush(wrk, 0);
-	AZ(wrk->obj);
-	AZ(wrk->objcore);
 #define ACCT(foo)	AZ(wrk->acct_tmp.foo);
 #include "tbl/acct_fields.h"
 #undef ACCT
diff --git a/bin/varnishd/cache/cache_esi_deliver.c b/bin/varnishd/cache/cache_esi_deliver.c
index be8d185..0c58395 100644
--- a/bin/varnishd/cache/cache_esi_deliver.c
+++ b/bin/varnishd/cache/cache_esi_deliver.c
@@ -58,8 +58,8 @@ ved_include(struct sess *sp, const char *src, const char *host)
 
 	(void)WRW_FlushRelease(w);
 
-	obj = sp->wrk->obj;
-	sp->wrk->obj = NULL;
+	obj = sp->req->obj;
+	sp->req->obj = NULL;
 	res_mode = sp->wrk->res_mode;
 
 	/* Reset request to status before we started messing with it */
@@ -106,7 +106,7 @@ ved_include(struct sess *sp, const char *src, const char *host)
 	AN(sp->wrk);
 	assert(sp->step == STP_DONE);
 	sp->req->esi_level--;
-	sp->wrk->obj = obj;
+	sp->req->obj = obj;
 	sp->wrk->res_mode = res_mode;
 
 	/* Reset the workspace */
@@ -231,7 +231,7 @@ ESI_Deliver(struct sess *sp)
 	int i;
 
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
-	st = sp->wrk->obj->esidata;
+	st = sp->req->obj->esidata;
 	AN(st);
 	assert(sizeof obuf >= 1024);
 
@@ -276,7 +276,7 @@ ESI_Deliver(struct sess *sp)
 		obufl = 0;
 	}
 
-	st = VTAILQ_FIRST(&sp->wrk->obj->store);
+	st = VTAILQ_FIRST(&sp->req->obj->store);
 	off = 0;
 
 	while (p < e) {
@@ -435,7 +435,7 @@ ved_deliver_byterange(const struct sess *sp, ssize_t low, ssize_t high)
 
 //printf("BR %jd %jd\n", low, high);
 	lx = 0;
-	VTAILQ_FOREACH(st, &sp->wrk->obj->store, list) {
+	VTAILQ_FOREACH(st, &sp->req->obj->store, list) {
 		p = st->ptr;
 		l = st->len;
 //printf("[0-] %jd %jd\n", lx, lx + l);
@@ -476,8 +476,8 @@ ESI_DeliverChild(const struct sess *sp)
 	uint32_t ilen;
 	uint8_t *dbits;
 
-	if (!sp->wrk->obj->gziped) {
-		VTAILQ_FOREACH(st, &sp->wrk->obj->store, list)
+	if (!sp->req->obj->gziped) {
+		VTAILQ_FOREACH(st, &sp->req->obj->store, list)
 			ved_pretend_gzip(sp, st->ptr, st->len);
 		return;
 	}
@@ -489,7 +489,7 @@ ESI_DeliverChild(const struct sess *sp)
 
 	dbits = (void*)WS_Alloc(sp->wrk->ws, 8);
 	AN(dbits);
-	obj = sp->wrk->obj;
+	obj = sp->req->obj;
 	CHECK_OBJ_NOTNULL(obj, OBJECT_MAGIC);
 	start = obj->gzip_start;
 	last = obj->gzip_last;
@@ -554,7 +554,7 @@ ESI_DeliverChild(const struct sess *sp)
 	}
 	if (lpad > 0)
 		(void)WRW_Write(sp->wrk, dbits + 1, lpad);
-	st = VTAILQ_LAST(&sp->wrk->obj->store, storagehead);
+	st = VTAILQ_LAST(&sp->req->obj->store, storagehead);
 	assert(st->len > 8);
 
 	p = st->ptr + st->len - 8;
diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c
index 2e65591..7070612 100644
--- a/bin/varnishd/cache/cache_fetch.c
+++ b/bin/varnishd/cache/cache_fetch.c
@@ -391,11 +391,11 @@ FetchHdr(struct sess *sp, int need_host_hdr)
 	htc = &wrk->busyobj->htc;
 
 	AN(sp->req->director);
-	AZ(sp->wrk->obj);
+	AZ(sp->req->obj);
 
-	if (sp->wrk->objcore != NULL) {		/* pass has no objcore */
-		CHECK_OBJ_NOTNULL(sp->wrk->objcore, OBJCORE_MAGIC);
-		AN(sp->wrk->objcore->flags & OC_F_BUSY);
+	if (sp->req->objcore != NULL) {		/* pass has no objcore */
+		CHECK_OBJ_NOTNULL(sp->req->objcore, OBJCORE_MAGIC);
+		AN(sp->req->objcore->flags & OC_F_BUSY);
 	}
 
 	hp = wrk->busyobj->bereq;
diff --git a/bin/varnishd/cache/cache_gzip.c b/bin/varnishd/cache/cache_gzip.c
index 061348a..1b732ca 100644
--- a/bin/varnishd/cache/cache_gzip.c
+++ b/bin/varnishd/cache/cache_gzip.c
@@ -248,7 +248,7 @@ VGZ_ObufFull(const struct vgz *vg)
 
 /*--------------------------------------------------------------------
  * Keep the outbuffer supplied with storage and file it under the
- * sp->wrk->obj as it fills.
+ * sp->req->obj as it fills.
  */
 
 int
diff --git a/bin/varnishd/cache/cache_hash.c b/bin/varnishd/cache/cache_hash.c
index 9f2c053..19395d5 100644
--- a/bin/varnishd/cache/cache_hash.c
+++ b/bin/varnishd/cache/cache_hash.c
@@ -389,8 +389,8 @@ HSH_Lookup(struct sess *sp, struct objhead **poh)
 	 * XXX: serialize fetch of all Vary's if grace is possible.
 	 */
 
-	AZ(sp->wrk->objcore);
-	sp->wrk->objcore = grace_oc;		/* XXX: Hack-ish */
+	AZ(sp->req->objcore);
+	sp->req->objcore = grace_oc;		/* XXX: Hack-ish */
 	if (oc == NULL			/* We found no live object */
 	    && grace_oc != NULL		/* There is a grace candidate */
 	    && (busy_oc != NULL		/* Somebody else is already busy */
@@ -400,7 +400,7 @@ HSH_Lookup(struct sess *sp, struct objhead **poh)
 		CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC);
 		oc = grace_oc;
 	}
-	sp->wrk->objcore = NULL;
+	sp->req->objcore = NULL;
 
 	if (oc != NULL && !sp->req->hash_always_miss) {
 		o = oc_getobj(sp->wrk, oc);
@@ -580,13 +580,13 @@ HSH_Drop(struct worker *wrk)
 	struct object *o;
 
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
-	o = wrk->obj;
+	o = wrk->sp->req->obj;
 	CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC);
 	AssertObjCorePassOrBusy(o->objcore);
 	o->exp.ttl = -1.;
 	if (o->objcore != NULL)		/* Pass has no objcore */
 		HSH_Unbusy(wrk);
-	(void)HSH_Deref(wrk, NULL, &wrk->obj);
+	(void)HSH_Deref(wrk, NULL, &wrk->sp->req->obj);
 }
 
 void
@@ -597,7 +597,7 @@ HSH_Unbusy(struct worker *wrk)
 	struct objcore *oc;
 
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
-	o = wrk->obj;
+	o = wrk->sp->req->obj;
 	CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC);
 	oc = o->objcore;
 	CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
diff --git a/bin/varnishd/cache/cache_panic.c b/bin/varnishd/cache/cache_panic.c
index 80f61a4..b09bcb8 100644
--- a/bin/varnishd/cache/cache_panic.c
+++ b/bin/varnishd/cache/cache_panic.c
@@ -274,8 +274,8 @@ pan_sess(const struct sess *sp)
 	if (VALID_OBJ(sp->req->vcl, VCL_CONF_MAGIC))
 		pan_vcl(sp->req->vcl);
 
-	if (VALID_OBJ(sp->wrk->obj, OBJECT_MAGIC))
-		pan_object(sp->wrk->obj);
+	if (VALID_OBJ(sp->req->obj, OBJECT_MAGIC))
+		pan_object(sp->req->obj);
 
 	VSB_printf(pan_vsp, "},\n");
 }
diff --git a/bin/varnishd/cache/cache_response.c b/bin/varnishd/cache/cache_response.c
index 5c10c35..1abb12b 100644
--- a/bin/varnishd/cache/cache_response.c
+++ b/bin/varnishd/cache/cache_response.c
@@ -41,7 +41,7 @@ res_dorange(const struct sess *sp, const char *r, ssize_t *plow, ssize_t *phigh)
 {
 	ssize_t low, high, has_low;
 
-	assert(sp->wrk->obj->response == 200);
+	assert(sp->req->obj->response == 200);
 	if (strncmp(r, "bytes=", 6))
 		return;
 	r += 6;
@@ -57,7 +57,7 @@ res_dorange(const struct sess *sp, const char *r, ssize_t *plow, ssize_t *phigh)
 		r++;
 	}
 
-	if (low >= sp->wrk->obj->len)
+	if (low >= sp->req->obj->len)
 		return;
 
 	if (*r != '-')
@@ -73,23 +73,23 @@ res_dorange(const struct sess *sp, const char *r, ssize_t *plow, ssize_t *phigh)
 			r++;
 		}
 		if (!has_low) {
-			low = sp->wrk->obj->len - high;
-			high = sp->wrk->obj->len - 1;
+			low = sp->req->obj->len - high;
+			high = sp->req->obj->len - 1;
 		}
 	} else
-		high = sp->wrk->obj->len - 1;
+		high = sp->req->obj->len - 1;
 	if (*r != '\0')
 		return;
 
-	if (high >= sp->wrk->obj->len)
-		high = sp->wrk->obj->len - 1;
+	if (high >= sp->req->obj->len)
+		high = sp->req->obj->len - 1;
 
 	if (low > high)
 		return;
 
 	http_PrintfHeader(sp->wrk, sp->vsl_id, sp->req->resp,
 	    "Content-Range: bytes %jd-%jd/%jd",
-	    (intmax_t)low, (intmax_t)high, (intmax_t)sp->wrk->obj->len);
+	    (intmax_t)low, (intmax_t)high, (intmax_t)sp->req->obj->len);
 	http_Unset(sp->req->resp, H_Content_Length);
 	assert(sp->wrk->res_mode & RES_LEN);
 	http_PrintfHeader(sp->wrk, sp->vsl_id, sp->req->resp,
@@ -112,9 +112,9 @@ RES_BuildHttp(const struct sess *sp)
 
 	http_ClrHeader(sp->req->resp);
 	sp->req->resp->logtag = HTTP_Tx;
-	http_CopyResp(sp->req->resp, sp->wrk->obj->http);
+	http_CopyResp(sp->req->resp, sp->req->obj->http);
 	http_FilterFields(sp->wrk, sp->vsl_id, sp->req->resp,
-	    sp->wrk->obj->http, HTTPH_A_DELIVER);
+	    sp->req->obj->http, HTTPH_A_DELIVER);
 
 	if (!(sp->wrk->res_mode & RES_LEN)) {
 		http_Unset(sp->req->resp, H_Content_Length);
@@ -132,15 +132,15 @@ RES_BuildHttp(const struct sess *sp)
 	http_PrintfHeader(sp->wrk, sp->vsl_id, sp->req->resp,
 	    "Date: %s", time_str);
 
-	if (sp->req->xid != sp->wrk->obj->xid)
+	if (sp->req->xid != sp->req->obj->xid)
 		http_PrintfHeader(sp->wrk, sp->vsl_id, sp->req->resp,
-		    "X-Varnish: %u %u", sp->req->xid, sp->wrk->obj->xid);
+		    "X-Varnish: %u %u", sp->req->xid, sp->req->obj->xid);
 	else
 		http_PrintfHeader(sp->wrk, sp->vsl_id, sp->req->resp,
 		    "X-Varnish: %u", sp->req->xid);
 	http_PrintfHeader(sp->wrk, sp->vsl_id, sp->req->resp, "Age: %.0f",
-	    sp->wrk->obj->exp.age + sp->req->t_resp -
-	    sp->wrk->obj->exp.entered);
+	    sp->req->obj->exp.age + sp->req->t_resp -
+	    sp->req->obj->exp.entered);
 	http_SetHeader(sp->wrk, sp->vsl_id, sp->req->resp, "Via: 1.1 varnish");
 	http_PrintfHeader(sp->wrk, sp->vsl_id, sp->req->resp, "Connection: %s",
 	    sp->req->doclose ? "close" : "keep-alive");
@@ -167,7 +167,7 @@ res_WriteGunzipObj(const struct sess *sp)
 	vg = VGZ_NewUngzip(sp->wrk, "U D -");
 
 	VGZ_Obuf(vg, obuf, sizeof obuf);
-	VTAILQ_FOREACH(st, &sp->wrk->obj->store, list) {
+	VTAILQ_FOREACH(st, &sp->req->obj->store, list) {
 		CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 		CHECK_OBJ_NOTNULL(st, STORAGE_MAGIC);
 		u += st->len;
@@ -185,7 +185,7 @@ res_WriteGunzipObj(const struct sess *sp)
 		(void)WRW_Flush(sp->wrk);
 	}
 	(void)VGZ_Destroy(&vg, sp->vsl_id);
-	assert(u == sp->wrk->obj->len);
+	assert(u == sp->req->obj->len);
 }
 
 /*--------------------------------------------------------------------*/
@@ -200,7 +200,7 @@ res_WriteDirObj(const struct sess *sp, ssize_t low, ssize_t high)
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 
 	ptr = 0;
-	VTAILQ_FOREACH(st, &sp->wrk->obj->store, list) {
+	VTAILQ_FOREACH(st, &sp->req->obj->store, list) {
 		CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 		CHECK_OBJ_NOTNULL(st, STORAGE_MAGIC);
 		u += st->len;
@@ -241,7 +241,7 @@ res_WriteDirObj(const struct sess *sp, ssize_t low, ssize_t high)
 		VSC_C_main->n_objwrite++;
 		(void)WRW_Write(sp->wrk, st->ptr + off, len);
 	}
-	assert(u == sp->wrk->obj->len);
+	assert(u == sp->req->obj->len);
 }
 
 /*--------------------------------------------------------------------
@@ -259,7 +259,7 @@ RES_WriteObj(struct sess *sp)
 
 	WRW_Reserve(sp->wrk, &sp->fd);
 
-	if (sp->wrk->obj->response == 200 &&
+	if (sp->req->obj->response == 200 &&
 	    sp->req->http->conds &&
 	    RFC2616_Do_Cond(sp)) {
 		sp->req->wantbody = 0;
@@ -272,13 +272,13 @@ RES_WriteObj(struct sess *sp)
 	 * If nothing special planned, we can attempt Range support
 	 */
 	low = 0;
-	high = sp->wrk->obj->len - 1;
+	high = sp->req->obj->len - 1;
 	if (
 	    sp->req->wantbody &&
 	    (sp->wrk->res_mode & RES_LEN) &&
 	    !(sp->wrk->res_mode & (RES_ESI|RES_ESI_CHILD|RES_GUNZIP)) &&
 	    cache_param->http_range_support &&
-	    sp->wrk->obj->response == 200 &&
+	    sp->req->obj->response == 200 &&
 	    http_GetHdr(sp->req->http, H_Range, &r))
 		res_dorange(sp, r, &low, &high);
 
@@ -303,14 +303,14 @@ RES_WriteObj(struct sess *sp)
 
 	if (!sp->req->wantbody) {
 		/* This was a HEAD or conditional request */
-	} else if (sp->wrk->obj->len == 0) {
+	} else if (sp->req->obj->len == 0) {
 		/* Nothing to do here */
 	} else if (sp->wrk->res_mode & RES_ESI) {
 		ESI_Deliver(sp);
 	} else if (sp->wrk->res_mode & RES_ESI_CHILD && sp->wrk->gzip_resp) {
 		ESI_DeliverChild(sp);
 	} else if (sp->wrk->res_mode & RES_ESI_CHILD &&
-	    !sp->wrk->gzip_resp && sp->wrk->obj->gziped) {
+	    !sp->wrk->gzip_resp && sp->req->obj->gziped) {
 		res_WriteGunzipObj(sp);
 	} else if (sp->wrk->res_mode & RES_GUNZIP) {
 		res_WriteGunzipObj(sp);
diff --git a/bin/varnishd/cache/cache_rfc2616.c b/bin/varnishd/cache/cache_rfc2616.c
index 04641ca..1b67d81 100644
--- a/bin/varnishd/cache/cache_rfc2616.c
+++ b/bin/varnishd/cache/cache_rfc2616.c
@@ -319,18 +319,18 @@ RFC2616_Do_Cond(const struct sess *sp)
 	   and If-Modified-Since if present*/
 
 	if (http_GetHdr(sp->req->http, H_If_Modified_Since, &p) ) {
-		if (!sp->wrk->obj->last_modified)
+		if (!sp->req->obj->last_modified)
 			return (0);
 		ims = VTIM_parse(p);
 		if (ims > sp->t_req)	/* [RFC2616 14.25] */
 			return (0);
-		if (sp->wrk->obj->last_modified > ims)
+		if (sp->req->obj->last_modified > ims)
 			return (0);
 		do_cond = 1;
 	}
 
 	if (http_GetHdr(sp->req->http, H_If_None_Match, &p) &&
-	    http_GetHdr(sp->wrk->obj->http, H_ETag, &e)) {
+	    http_GetHdr(sp->req->obj->http, H_ETag, &e)) {
 		if (strcmp(p,e) != 0)
 			return (0);
 		do_cond = 1;
diff --git a/bin/varnishd/cache/cache_vrt.c b/bin/varnishd/cache/cache_vrt.c
index 67875bd..2623bcc 100644
--- a/bin/varnishd/cache/cache_vrt.c
+++ b/bin/varnishd/cache/cache_vrt.c
@@ -110,8 +110,8 @@ vrt_selecthttp(const struct sess *sp, enum gethdr_e where)
 		hp = sp->req->resp;
 		break;
 	case HDR_OBJ:
-		CHECK_OBJ_NOTNULL(sp->wrk->obj, OBJECT_MAGIC);
-		hp = sp->wrk->obj->http;
+		CHECK_OBJ_NOTNULL(sp->req->obj, OBJECT_MAGIC);
+		hp = sp->req->obj->http;
 		break;
 	default:
 		INCOMPL();
@@ -401,8 +401,8 @@ VRT_synth_page(const struct sess *sp, unsigned flags, const char *str, ...)
 
 	(void)flags;
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
-	CHECK_OBJ_NOTNULL(sp->wrk->obj, OBJECT_MAGIC);
-	vsb = SMS_Makesynth(sp->wrk->obj);
+	CHECK_OBJ_NOTNULL(sp->req->obj, OBJECT_MAGIC);
+	vsb = SMS_Makesynth(sp->req->obj);
 	AN(vsb);
 
 	VSB_cat(vsb, str);
@@ -415,10 +415,10 @@ VRT_synth_page(const struct sess *sp, unsigned flags, const char *str, ...)
 		p = va_arg(ap, const char *);
 	}
 	va_end(ap);
-	SMS_Finish(sp->wrk->obj);
-	http_Unset(sp->wrk->obj->http, H_Content_Length);
-	http_PrintfHeader(sp->wrk, sp->vsl_id, sp->wrk->obj->http,
-	    "Content-Length: %zd", sp->wrk->obj->len);
+	SMS_Finish(sp->req->obj);
+	http_Unset(sp->req->obj->http, H_Content_Length);
+	http_PrintfHeader(sp->wrk, sp->vsl_id, sp->req->obj->http,
+	    "Content-Length: %zd", sp->req->obj->len);
 }
 
 /*--------------------------------------------------------------------*/
@@ -512,9 +512,9 @@ void
 VRT_purge(const struct sess *sp, double ttl, double grace)
 {
 	if (sp->req->cur_method == VCL_MET_HIT)
-		HSH_Purge(sp, sp->wrk->obj->objcore->objhead, ttl, grace);
+		HSH_Purge(sp, sp->req->obj->objcore->objhead, ttl, grace);
 	else if (sp->req->cur_method == VCL_MET_MISS)
-		HSH_Purge(sp, sp->wrk->objcore->objhead, ttl, grace);
+		HSH_Purge(sp, sp->req->objcore->objhead, ttl, grace);
 }
 
 /*--------------------------------------------------------------------
diff --git a/bin/varnishd/cache/cache_vrt_var.c b/bin/varnishd/cache/cache_vrt_var.c
index 6e5c751..cb66ff7 100644
--- a/bin/varnishd/cache/cache_vrt_var.c
+++ b/bin/varnishd/cache/cache_vrt_var.c
@@ -88,8 +88,8 @@ VRT_DO_HDR(req,   proto,	sp->req->http,		HTTP_HDR_PROTO)
 VRT_DO_HDR(bereq, request,	sp->wrk->busyobj->bereq,	HTTP_HDR_REQ)
 VRT_DO_HDR(bereq, url,		sp->wrk->busyobj->bereq,	HTTP_HDR_URL)
 VRT_DO_HDR(bereq, proto,	sp->wrk->busyobj->bereq,	HTTP_HDR_PROTO)
-VRT_DO_HDR(obj,   proto,	sp->wrk->obj->http,	HTTP_HDR_PROTO)
-VRT_DO_HDR(obj,   response,	sp->wrk->obj->http,	HTTP_HDR_RESPONSE)
+VRT_DO_HDR(obj,   proto,	sp->req->obj->http,	HTTP_HDR_PROTO)
+VRT_DO_HDR(obj,   response,	sp->req->obj->http,	HTTP_HDR_RESPONSE)
 VRT_DO_HDR(resp,  proto,	sp->req->resp,		HTTP_HDR_PROTO)
 VRT_DO_HDR(resp,  response,	sp->req->resp,		HTTP_HDR_RESPONSE)
 VRT_DO_HDR(beresp,  proto,	sp->wrk->busyobj->beresp,	HTTP_HDR_PROTO)
@@ -114,7 +114,7 @@ VRT_r_##obj##_status(const struct sess *sp)			\
 	return(http->status);					\
 }
 
-VRT_DO_STATUS(obj, sp->wrk->obj->http)
+VRT_DO_STATUS(obj, sp->req->obj->http)
 VRT_DO_STATUS(beresp, sp->wrk->busyobj->beresp)
 VRT_DO_STATUS(resp, sp->req->resp)
 
@@ -145,9 +145,9 @@ VRT_l_beresp_saintmode(const struct sess *sp, double a)
 	if (!vbc->backend)
 		return;
 	CHECK_OBJ_NOTNULL(vbc->backend, BACKEND_MAGIC);
-	if (!sp->wrk->objcore)
+	if (!sp->req->objcore)
 		return;
-	CHECK_OBJ_NOTNULL(sp->wrk->objcore, OBJCORE_MAGIC);
+	CHECK_OBJ_NOTNULL(sp->req->objcore, OBJCORE_MAGIC);
 
 	/* Setting a negative holdoff period is a mistake. Detecting this
 	 * when compiling the VCL would be better.
@@ -156,7 +156,7 @@ VRT_l_beresp_saintmode(const struct sess *sp, double a)
 
 	ALLOC_OBJ(new, TROUBLE_MAGIC);
 	AN(new);
-	new->target = (uintptr_t)(sp->wrk->objcore->objhead);
+	new->target = (uintptr_t)(sp->req->objcore->objhead);
 	new->timeout = sp->t_req + a;
 
 	/* Insert the new item on the list before the first item with a
@@ -405,15 +405,15 @@ VRT_DO_EXP(req, sp->req->exp, ttl, 0, )
 VRT_DO_EXP(req, sp->req->exp, grace, 0, )
 VRT_DO_EXP(req, sp->req->exp, keep, 0, )
 
-VRT_DO_EXP(obj, sp->wrk->obj->exp, grace, 0,
-   EXP_Rearm(sp->wrk->obj);
-   vrt_wsp_exp(sp, sp->wrk->obj->xid, &sp->wrk->obj->exp);)
-VRT_DO_EXP(obj, sp->wrk->obj->exp, ttl, (sp->t_req - sp->wrk->obj->exp.entered),
-   EXP_Rearm(sp->wrk->obj);
-   vrt_wsp_exp(sp, sp->wrk->obj->xid, &sp->wrk->obj->exp);)
-VRT_DO_EXP(obj, sp->wrk->obj->exp, keep, 0,
-   EXP_Rearm(sp->wrk->obj);
-   vrt_wsp_exp(sp, sp->wrk->obj->xid, &sp->wrk->obj->exp);)
+VRT_DO_EXP(obj, sp->req->obj->exp, grace, 0,
+   EXP_Rearm(sp->req->obj);
+   vrt_wsp_exp(sp, sp->req->obj->xid, &sp->req->obj->exp);)
+VRT_DO_EXP(obj, sp->req->obj->exp, ttl, (sp->t_req - sp->req->obj->exp.entered),
+   EXP_Rearm(sp->req->obj);
+   vrt_wsp_exp(sp, sp->req->obj->xid, &sp->req->obj->exp);)
+VRT_DO_EXP(obj, sp->req->obj->exp, keep, 0,
+   EXP_Rearm(sp->req->obj);
+   vrt_wsp_exp(sp, sp->req->obj->xid, &sp->req->obj->exp);)
 
 VRT_DO_EXP(beresp, sp->wrk->busyobj->exp, grace, 0,
    vrt_wsp_exp(sp, sp->req->xid, &sp->wrk->busyobj->exp);)
@@ -531,8 +531,8 @@ VRT_r_obj_hits(const struct sess *sp)
 {
 
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
-	CHECK_OBJ_NOTNULL(sp->wrk->obj, OBJECT_MAGIC);	/* XXX */
-	return (sp->wrk->obj->hits);
+	CHECK_OBJ_NOTNULL(sp->req->obj, OBJECT_MAGIC);	/* XXX */
+	return (sp->req->obj->hits);
 }
 
 double
@@ -540,8 +540,8 @@ VRT_r_obj_lastuse(const struct sess *sp)
 {
 
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
-	CHECK_OBJ_NOTNULL(sp->wrk->obj, OBJECT_MAGIC);	/* XXX */
-	return (VTIM_real() - sp->wrk->obj->last_use);
+	CHECK_OBJ_NOTNULL(sp->req->obj, OBJECT_MAGIC);	/* XXX */
+	return (VTIM_real() - sp->req->obj->last_use);
 }
 
 unsigned
diff --git a/bin/varnishd/storage/stevedore.c b/bin/varnishd/storage/stevedore.c
index a6156d4..ccd45b1 100644
--- a/bin/varnishd/storage/stevedore.c
+++ b/bin/varnishd/storage/stevedore.c
@@ -256,11 +256,11 @@ STV_MkObject(struct worker *wrk, void *ptr, unsigned ltot,
 	VTAILQ_INIT(&o->store);
 	wrk->stats.n_object++;
 
-	if (wrk->objcore != NULL) {
-		CHECK_OBJ_NOTNULL(wrk->objcore, OBJCORE_MAGIC);
+	if (wrk->sp->req->objcore != NULL) {
+		CHECK_OBJ_NOTNULL(wrk->sp->req->objcore, OBJCORE_MAGIC);
 
-		o->objcore = wrk->objcore;
-		wrk->objcore = NULL;     /* refcnt follows pointer. */
+		o->objcore = wrk->sp->req->objcore;
+		wrk->sp->req->objcore = NULL;     /* refcnt follows pointer. */
 		BAN_NewObjCore(o->objcore);
 
 		o->objcore->methods = &default_oc_methods;
diff --git a/bin/varnishd/storage/storage_persistent.c b/bin/varnishd/storage/storage_persistent.c
index 0791941..f90594e 100644
--- a/bin/varnishd/storage/storage_persistent.c
+++ b/bin/varnishd/storage/storage_persistent.c
@@ -474,10 +474,10 @@ smp_allocobj(struct stevedore *stv, struct worker *wrk, unsigned ltot,
 	struct objcore *oc;
 	unsigned objidx;
 
-	if (wrk->objcore == NULL)
+	if (wrk->sp->req->objcore == NULL)
 		return (NULL);		/* from cnt_error */
 	CAST_OBJ_NOTNULL(sc, stv->priv, SMP_SC_MAGIC);
-	AN(wrk->objcore);
+	AN(wrk->sp->req->objcore);
 	AN(wrk->busyobj->exp.ttl > 0.);
 
 	ltot = IRNUP(sc, ltot);



From geoff at varnish-cache.org  Mon Jan  9 23:03:32 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Tue, 10 Jan 2012 00:03:32 +0100
Subject: [experimental-ims] df85ad8 Introduce a local "req" parameter in the
	cnt_* states,
	and pass wrk and req in as paramters to shave a few source lines
Message-ID: <E1RkOFY-0007wo-Kh@project.varnish-software.com>

commit df85ad845ef28488781130a3115a117e09227660
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Dec 26 09:48:23 2011 +0000

    Introduce a local "req" parameter in the cnt_* states, and pass wrk and req
    in as paramters to shave a few source lines

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index c048b5a..c919b32 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -127,7 +127,7 @@ typedef struct {
 /*--------------------------------------------------------------------*/
 
 enum step {
-#define STEP(l, u)	STP_##u,
+#define STEP(l, u, arg)	STP_##u,
 #include "tbl/steps.h"
 #undef STEP
 };
diff --git a/bin/varnishd/cache/cache_center.c b/bin/varnishd/cache/cache_center.c
index 0154577..298b7b4 100644
--- a/bin/varnishd/cache/cache_center.c
+++ b/bin/varnishd/cache/cache_center.c
@@ -97,28 +97,28 @@ DOT }
  */
 
 static int
-cnt_wait(struct sess *sp)
+cnt_wait(struct sess *sp, struct worker *wrk, struct req *req)
 {
 	int i, j, tmo;
 	struct pollfd pfd[1];
-	struct worker *wrk;
 	double now, when;
 
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
-	wrk = sp->wrk;
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
-	if (sp->req == NULL) {
+	if (req == NULL) {
 		SES_GetReq(sp);
-		HTC_Init(sp->req->htc, sp->req->ws, sp->fd, sp->vsl_id,
+		req = sp->req;
+		CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
+		HTC_Init(req->htc, req->ws, sp->fd, sp->vsl_id,
 		    cache_param->http_req_size,
 		    cache_param->http_req_hdr_len);
 	}
 
-	AZ(sp->req->vcl);
+	AZ(req->vcl);
 	AZ(wrk->obj);
-	AZ(sp->req->esi_level);
-	assert(sp->req->xid == 0);
-	sp->req->t_resp = NAN;
+	AZ(req->esi_level);
+	assert(req->xid == 0);
+	req->t_resp = NAN;
 
 	assert(!isnan(sp->t_req));
 	tmo = (int)(1e3 * cache_param->timeout_linger);
@@ -130,9 +130,9 @@ cnt_wait(struct sess *sp)
 		assert(j >= 0);
 		now = VTIM_real();
 		if (j != 0)
-			i = HTC_Rx(sp->req->htc);
+			i = HTC_Rx(req->htc);
 		else
-			i = HTC_Complete(sp->req->htc);
+			i = HTC_Complete(req->htc);
 		if (i == 1) {
 			/* Got it, run with it */
 			sp->t_req = now;
@@ -206,16 +206,15 @@ DOT }
  */
 
 static int
-cnt_prepresp(struct sess *sp)
+cnt_prepresp(struct sess *sp, struct worker *wrk, struct req *req)
 {
-	struct worker *wrk;
 
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
-	wrk = sp->wrk;
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
+	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
 
 	CHECK_OBJ_NOTNULL(wrk->obj, OBJECT_MAGIC);
-	CHECK_OBJ_NOTNULL(sp->req->vcl, VCL_CONF_MAGIC);
+	CHECK_OBJ_NOTNULL(req->vcl, VCL_CONF_MAGIC);
 
 	if (wrk->busyobj != NULL) {
 		CHECK_OBJ_NOTNULL(wrk->busyobj, BUSYOBJ_MAGIC);
@@ -234,13 +233,13 @@ cnt_prepresp(struct sess *sp)
 	    !wrk->busyobj->do_gzip && !wrk->busyobj->do_gunzip)
 		wrk->res_mode |= RES_LEN;
 
-	if (!sp->req->disable_esi && wrk->obj->esidata != NULL) {
+	if (!req->disable_esi && wrk->obj->esidata != NULL) {
 		/* In ESI mode, we don't know the aggregate length */
 		wrk->res_mode &= ~RES_LEN;
 		wrk->res_mode |= RES_ESI;
 	}
 
-	if (sp->req->esi_level > 0) {
+	if (req->esi_level > 0) {
 		wrk->res_mode &= ~RES_LEN;
 		wrk->res_mode |= RES_ESI_CHILD;
 	}
@@ -263,32 +262,32 @@ cnt_prepresp(struct sess *sp)
 			 * can make it any different size
 			 */
 			wrk->res_mode |= RES_LEN;
-		else if (!sp->req->wantbody) {
+		else if (!req->wantbody) {
 			/* Nothing */
-		} else if (sp->req->http->protover >= 11) {
+		} else if (req->http->protover >= 11) {
 			wrk->res_mode |= RES_CHUNKED;
 		} else {
 			wrk->res_mode |= RES_EOF;
-			sp->req->doclose = "EOF mode";
+			req->doclose = "EOF mode";
 		}
 	}
 
-	sp->req->t_resp = W_TIM_real(wrk);
+	req->t_resp = W_TIM_real(wrk);
 	if (wrk->obj->objcore != NULL) {
-		if ((sp->req->t_resp - wrk->obj->last_lru) >
+		if ((req->t_resp - wrk->obj->last_lru) >
 		    cache_param->lru_timeout &&
 		    EXP_Touch(wrk->obj->objcore))
-			wrk->obj->last_lru = sp->req->t_resp;
-		wrk->obj->last_use = sp->req->t_resp;	/* XXX: locking ? */
+			wrk->obj->last_lru = req->t_resp;
+		wrk->obj->last_use = req->t_resp;	/* XXX: locking ? */
 	}
-	http_Setup(sp->req->resp, sp->req->ws);
+	http_Setup(req->resp, req->ws);
 	RES_BuildHttp(sp);
 	VCL_deliver_method(sp);
-	switch (sp->req->handling) {
+	switch (req->handling) {
 	case VCL_RET_DELIVER:
 		break;
 	case VCL_RET_RESTART:
-		if (sp->req->restarts >= cache_param->max_restarts)
+		if (req->restarts >= cache_param->max_restarts)
 			break;
 		if (wrk->busyobj != NULL) {
 			AN(wrk->busyobj->do_stream);
@@ -299,9 +298,9 @@ cnt_prepresp(struct sess *sp)
 			(void)HSH_Deref(wrk, NULL, &wrk->obj);
 		}
 		AZ(wrk->obj);
-		sp->req->restarts++;
-		sp->req->director = NULL;
-		http_Setup(sp->req->resp, NULL);
+		req->restarts++;
+		req->director = NULL;
+		http_Setup(req->resp, NULL);
 		sp->step = STP_RECV;
 		return (0);
 	default:
@@ -332,24 +331,22 @@ DOT deliver -> DONE [style=bold,color=blue]
  */
 
 static int
-cnt_deliver(struct sess *sp)
+cnt_deliver(struct sess *sp, struct worker *wrk, struct req *req)
 {
-	struct worker *wrk;
-
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
-	wrk = sp->wrk;
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
+	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
 
 	AZ(sp->wrk->busyobj);
-	sp->req->director = NULL;
-	sp->req->restarts = 0;
+	req->director = NULL;
+	req->restarts = 0;
 
 	RES_WriteObj(sp);
 
 	assert(WRW_IsReleased(wrk));
 	assert(wrk->wrw.ciov == wrk->wrw.siov);
 	(void)HSH_Deref(wrk, NULL, &wrk->obj);
-	http_Setup(sp->req->resp, NULL);
+	http_Setup(req->resp, NULL);
 	sp->step = STP_DONE;
 	return (0);
 }
@@ -369,72 +366,71 @@ DOT	DONE -> ESI_RESP
  */
 
 static int
-cnt_done(struct sess *sp)
+cnt_done(struct sess *sp, struct worker *wrk, struct req *req)
 {
 	double dh, dp, da;
 	int i;
-	struct worker *wrk;
 
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
-	wrk = sp->wrk;
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
-	CHECK_OBJ_ORNULL(sp->req->vcl, VCL_CONF_MAGIC);
+	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
+	CHECK_OBJ_ORNULL(req->vcl, VCL_CONF_MAGIC);
 
 	AZ(wrk->obj);
 	AZ(wrk->busyobj);
-	sp->req->director = NULL;
-	sp->req->restarts = 0;
+	req->director = NULL;
+	req->restarts = 0;
 
 	wrk->busyobj = NULL;
 
 	SES_Charge(sp);
 
 	/* If we did an ESI include, don't mess up our state */
-	if (sp->req->esi_level > 0)
+	if (req->esi_level > 0)
 		return (1);
 
-	if (sp->req->vcl != NULL) {
+	if (req->vcl != NULL) {
 		if (wrk->vcl != NULL)
 			VCL_Rel(&wrk->vcl);
-		wrk->vcl = sp->req->vcl;
-		sp->req->vcl = NULL;
+		wrk->vcl = req->vcl;
+		req->vcl = NULL;
 	}
 
 
 	sp->t_idle = W_TIM_real(wrk);
-	if (sp->req->xid == 0) {
-		sp->req->t_resp = sp->t_idle;
+	if (req->xid == 0) {
+		req->t_resp = sp->t_idle;
 	} else {
-		dp = sp->req->t_resp - sp->t_req;
-		da = sp->t_idle - sp->req->t_resp;
+		dp = req->t_resp - sp->t_req;
+		da = sp->t_idle - req->t_resp;
 		dh = sp->t_req - sp->t_open;
 		/* XXX: Add StatReq == StatSess */
 		/* XXX: Workaround for pipe */
 		if (sp->fd >= 0) {
 			WSP(sp, SLT_Length, "%ju",
-			    (uintmax_t)sp->req->req_bodybytes);
+			    (uintmax_t)req->req_bodybytes);
 		}
 		WSP(sp, SLT_ReqEnd, "%u %.9f %.9f %.9f %.9f %.9f",
-		    sp->req->xid, sp->t_req, sp->t_idle, dh, dp, da);
+		    req->xid, sp->t_req, sp->t_idle, dh, dp, da);
 	}
-	sp->req->xid = 0;
+	req->xid = 0;
 	WSL_Flush(wrk, 0);
 
 	sp->t_req = NAN;
-	sp->req->t_resp = NAN;
+	req->t_resp = NAN;
 
-	sp->req->req_bodybytes = 0;
+	req->req_bodybytes = 0;
 
-	sp->req->hash_always_miss = 0;
-	sp->req->hash_ignore_busy = 0;
+	req->hash_always_miss = 0;
+	req->hash_ignore_busy = 0;
 
-	if (sp->fd >= 0 && sp->req->doclose != NULL) {
+	if (sp->fd >= 0 && req->doclose != NULL) {
 		/*
 		 * This is an orderly close of the connection; ditch nolinger
 		 * before we close, to get queued data transmitted.
 		 */
 		// XXX: not yet (void)VTCP_linger(sp->fd, 0);
-		SES_Close(sp, sp->req->doclose);
+		SES_Close(sp, req->doclose);
 	}
 
 	if (sp->fd < 0) {
@@ -446,17 +442,17 @@ cnt_done(struct sess *sp)
 	if (wrk->stats.client_req >= cache_param->wthread_stats_rate)
 		WRK_SumStat(wrk);
 	/* Reset the workspace to the session-watermark */
-	WS_Reset(sp->req->ws, NULL);
+	WS_Reset(req->ws, NULL);
 	WS_Reset(wrk->ws, NULL);
 
-	i = HTC_Reinit(sp->req->htc);
+	i = HTC_Reinit(req->htc);
 	if (i == 1) {
 		wrk->stats.sess_pipeline++;
 		sp->t_req = sp->t_idle;
 		sp->step = STP_START;
 		return (0);
 	}
-	if (Tlen(sp->req->htc->rxbuf))
+	if (Tlen(req->htc->rxbuf))
 		wrk->stats.sess_readahead++;
 	sp->step = STP_WAIT;
 	sp->t_req = sp->t_idle;
@@ -479,15 +475,14 @@ DOT rsterr [label="RESTART",shape=plaintext]
  */
 
 static int
-cnt_error(struct sess *sp)
+cnt_error(struct sess *sp, struct worker *wrk, struct req *req)
 {
 	struct http *h;
 	char date[40];
-	struct worker *wrk;
 
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
-	wrk = sp->wrk;
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
+	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
 
 	if (wrk->obj == NULL) {
 		HSH_Prealloc(sp);
@@ -500,15 +495,15 @@ cnt_error(struct sess *sp)
 			    cache_param->http_resp_size,
 			    (uint16_t)cache_param->http_max_hdr);
 		if (wrk->obj == NULL) {
-			sp->req->doclose = "Out of objects";
-			sp->req->director = NULL;
+			req->doclose = "Out of objects";
+			req->director = NULL;
 			http_Setup(wrk->busyobj->beresp, NULL);
 			http_Setup(wrk->busyobj->bereq, NULL);
 			sp->step = STP_DONE;
 			return(0);
 		}
 		AN(wrk->obj);
-		wrk->obj->xid = sp->req->xid;
+		wrk->obj->xid = req->xid;
 		wrk->obj->exp.entered = sp->t_req;
 	} else {
 		CHECK_OBJ_NOTNULL(wrk->busyobj, BUSYOBJ_MAGIC);
@@ -517,41 +512,41 @@ cnt_error(struct sess *sp)
 	CHECK_OBJ_NOTNULL(wrk->obj, OBJECT_MAGIC);
 	h = wrk->obj->http;
 
-	if (sp->req->err_code < 100 || sp->req->err_code > 999)
-		sp->req->err_code = 501;
+	if (req->err_code < 100 || req->err_code > 999)
+		req->err_code = 501;
 
 	http_PutProtocol(wrk, sp->vsl_id, h, "HTTP/1.1");
-	http_PutStatus(h, sp->req->err_code);
+	http_PutStatus(h, req->err_code);
 	VTIM_format(W_TIM_real(wrk), date);
 	http_PrintfHeader(wrk, sp->vsl_id, h, "Date: %s", date);
 	http_SetHeader(wrk, sp->vsl_id, h, "Server: Varnish");
 
-	if (sp->req->err_reason != NULL)
-		http_PutResponse(wrk, sp->vsl_id, h, sp->req->err_reason);
+	if (req->err_reason != NULL)
+		http_PutResponse(wrk, sp->vsl_id, h, req->err_reason);
 	else
 		http_PutResponse(wrk, sp->vsl_id, h,
-		    http_StatusMessage(sp->req->err_code));
+		    http_StatusMessage(req->err_code));
 	VCL_error_method(sp);
 
-	if (sp->req->handling == VCL_RET_RESTART &&
-	    sp->req->restarts <  cache_param->max_restarts) {
+	if (req->handling == VCL_RET_RESTART &&
+	    req->restarts <  cache_param->max_restarts) {
 		HSH_Drop(wrk);
 		VBO_DerefBusyObj(wrk, &wrk->busyobj);
-		sp->req->director = NULL;
-		sp->req->restarts++;
+		req->director = NULL;
+		req->restarts++;
 		sp->step = STP_RECV;
 		return (0);
-	} else if (sp->req->handling == VCL_RET_RESTART)
-		sp->req->handling = VCL_RET_DELIVER;
+	} else if (req->handling == VCL_RET_RESTART)
+		req->handling = VCL_RET_DELIVER;
 
 
 	/* We always close when we take this path */
-	sp->req->doclose = "error";
-	sp->req->wantbody = 1;
+	req->doclose = "error";
+	req->wantbody = 1;
 
-	assert(sp->req->handling == VCL_RET_DELIVER);
-	sp->req->err_code = 0;
-	sp->req->err_reason = NULL;
+	assert(req->handling == VCL_RET_DELIVER);
+	req->err_code = 0;
+	req->err_reason = NULL;
 	http_Setup(wrk->busyobj->bereq, NULL);
 	VBO_DerefBusyObj(wrk, &wrk->busyobj);
 	sp->step = STP_PREPRESP;
@@ -588,19 +583,18 @@ DOT errfetch [label="ERROR",shape=plaintext]
  */
 
 static int
-cnt_fetch(struct sess *sp)
+cnt_fetch(struct sess *sp, struct worker *wrk, struct req *req)
 {
 	int i, need_host_hdr;
-	struct worker *wrk;
 
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
-	wrk = sp->wrk;
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
+	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
 
-	CHECK_OBJ_NOTNULL(sp->req->vcl, VCL_CONF_MAGIC);
+	CHECK_OBJ_NOTNULL(req->vcl, VCL_CONF_MAGIC);
 	CHECK_OBJ_NOTNULL(wrk->busyobj, BUSYOBJ_MAGIC);
 
-	AN(sp->req->director);
+	AN(req->director);
 	AZ(wrk->busyobj->vbc);
 	AZ(wrk->busyobj->should_close);
 	AZ(wrk->storage_hint);
@@ -621,8 +615,8 @@ cnt_fetch(struct sess *sp)
 	}
 
 	if (i) {
-		sp->req->handling = VCL_RET_ERROR;
-		sp->req->err_code = 503;
+		req->handling = VCL_RET_ERROR;
+		req->err_code = 503;
 	} else {
 		/*
 		 * These two headers can be spread over multiple actual headers
@@ -639,7 +633,7 @@ cnt_fetch(struct sess *sp)
 		 */
 		wrk->busyobj->body_status = RFC2616_Body(sp);
 
-		sp->req->err_code = http_GetStatus(wrk->busyobj->beresp);
+		req->err_code = http_GetStatus(wrk->busyobj->beresp);
 
 		/*
 		 * What does RFC2616 think about TTL ?
@@ -656,7 +650,7 @@ cnt_fetch(struct sess *sp)
 
 		VCL_fetch_method(sp);
 
-		switch (sp->req->handling) {
+		switch (req->handling) {
 		case VCL_RET_HIT_FOR_PASS:
 			if (wrk->objcore != NULL)
 				wrk->objcore->flags |= OC_F_PASS;
@@ -683,12 +677,12 @@ cnt_fetch(struct sess *sp)
 		wrk->objcore = NULL;
 	}
 	VBO_DerefBusyObj(wrk, &wrk->busyobj);
-	sp->req->director = NULL;
+	req->director = NULL;
 	wrk->storage_hint = NULL;
 
-	switch (sp->req->handling) {
+	switch (req->handling) {
 	case VCL_RET_RESTART:
-		sp->req->restarts++;
+		req->restarts++;
 		sp->step = STP_RECV;
 		return (0);
 	case VCL_RET_ERROR:
@@ -721,7 +715,7 @@ DOT fetchbody2 -> prepresp [style=bold,color=blue]
 
 
 static int
-cnt_fetchbody(struct sess *sp)
+cnt_fetchbody(struct sess *sp, struct worker *wrk, struct req *req)
 {
 	int i;
 	struct http *hp, *hp2;
@@ -730,22 +724,21 @@ cnt_fetchbody(struct sess *sp)
 	unsigned l;
 	struct vsb *vary = NULL;
 	int varyl = 0, pass;
-	struct worker *wrk;
 
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
-	wrk = sp->wrk;
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
+	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
 	CHECK_OBJ_NOTNULL(wrk->busyobj, BUSYOBJ_MAGIC);
 
-	assert(sp->req->handling == VCL_RET_HIT_FOR_PASS ||
-	    sp->req->handling == VCL_RET_DELIVER);
+	assert(req->handling == VCL_RET_HIT_FOR_PASS ||
+	    req->handling == VCL_RET_DELIVER);
 
 	if (wrk->objcore == NULL) {
 		/* This is a pass from vcl_recv */
 		pass = 1;
 		/* VCL may have fiddled this, but that doesn't help */
 		wrk->busyobj->exp.ttl = -1.;
-	} else if (sp->req->handling == VCL_RET_HIT_FOR_PASS) {
+	} else if (req->handling == VCL_RET_HIT_FOR_PASS) {
 		/* pass from vcl_fetch{} -> hit-for-pass */
 		/* XXX: the bereq was not filtered pass... */
 		pass = 1;
@@ -810,9 +803,9 @@ cnt_fetchbody(struct sess *sp)
 	else if (wrk->busyobj->is_gzip)
 		wrk->busyobj->vfp = &vfp_testgzip;
 
-	if (wrk->busyobj->do_esi || sp->req->esi_level > 0)
+	if (wrk->busyobj->do_esi || req->esi_level > 0)
 		wrk->busyobj->do_stream = 0;
-	if (!sp->req->wantbody)
+	if (!req->wantbody)
 		wrk->busyobj->do_stream = 0;
 
 	l = http_EstimateWS(wrk->busyobj->beresp,
@@ -852,7 +845,7 @@ cnt_fetchbody(struct sess *sp)
 		wrk->busyobj->exp.keep = 0.0;
 	}
 	if (wrk->obj == NULL) {
-		sp->req->err_code = 503;
+		req->err_code = 503;
 		sp->step = STP_ERROR;
 		VDI_CloseFd(wrk, &wrk->busyobj->vbc);
 		VBO_DerefBusyObj(wrk, &wrk->busyobj);
@@ -874,8 +867,8 @@ cnt_fetchbody(struct sess *sp)
 		VSB_delete(vary);
 	}
 
-	wrk->obj->xid = sp->req->xid;
-	wrk->obj->response = sp->req->err_code;
+	wrk->obj->xid = req->xid;
+	wrk->obj->response = req->err_code;
 	WS_Assert(wrk->obj->ws_o);
 
 	/* Filter into object */
@@ -901,7 +894,7 @@ cnt_fetchbody(struct sess *sp)
 	 * that allow the 304, in which case we return 200 non-stream.
 	 */
 	if (wrk->obj->response == 200 &&
-	    sp->req->http->conds &&
+	    req->http->conds &&
 	    RFC2616_Do_Cond(sp))
 		wrk->busyobj->do_stream = 0;
 
@@ -920,13 +913,13 @@ cnt_fetchbody(struct sess *sp)
 	wrk->busyobj->vfp = NULL;
 	assert(WRW_IsReleased(wrk));
 	AZ(wrk->busyobj->vbc);
-	AN(sp->req->director);
+	AN(req->director);
 
 	if (i) {
 		HSH_Drop(wrk);
 		VBO_DerefBusyObj(wrk, &wrk->busyobj);
 		AZ(wrk->obj);
-		sp->req->err_code = 503;
+		req->err_code = 503;
 		sp->step = STP_ERROR;
 		return (0);
 	}
@@ -955,17 +948,16 @@ DOT streambody -> DONE [style=bold,color=cyan]
  */
 
 static int
-cnt_streambody(struct sess *sp)
+cnt_streambody(struct sess *sp, struct worker *wrk, struct req *req)
 {
 	int i;
 	struct stream_ctx sctx;
 	uint8_t obuf[sp->wrk->res_mode & RES_GUNZIP ?
 	    cache_param->gzip_stack_buffer : 1];
-	struct worker *wrk;
 
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
-	wrk = sp->wrk;
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
+	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
 
 	CHECK_OBJ_NOTNULL(wrk->busyobj, BUSYOBJ_MAGIC);
 	memset(&sctx, 0, sizeof sctx);
@@ -989,7 +981,7 @@ cnt_streambody(struct sess *sp)
 	http_Setup(wrk->busyobj->beresp, NULL);
 	wrk->busyobj->vfp = NULL;
 	AZ(wrk->busyobj->vbc);
-	AN(sp->req->director);
+	AN(req->director);
 
 	if (!i && wrk->obj->objcore != NULL) {
 		EXP_Insert(wrk->obj);
@@ -997,11 +989,11 @@ cnt_streambody(struct sess *sp)
 		AN(wrk->obj->objcore->ban);
 		HSH_Unbusy(wrk);
 	} else {
-		sp->req->doclose = "Stream error";
+		req->doclose = "Stream error";
 	}
 	wrk->acct_tmp.fetch++;
-	sp->req->director = NULL;
-	sp->req->restarts = 0;
+	req->director = NULL;
+	req->restarts = 0;
 
 	RES_StreamEnd(sp);
 	if (wrk->res_mode & RES_GUNZIP)
@@ -1012,7 +1004,7 @@ cnt_streambody(struct sess *sp)
 	assert(wrk->wrw.ciov == wrk->wrw.siov);
 	(void)HSH_Deref(wrk, NULL, &wrk->obj);
 	VBO_DerefBusyObj(wrk, &wrk->busyobj);
-	http_Setup(sp->req->resp, NULL);
+	http_Setup(req->resp, NULL);
 	sp->step = STP_DONE;
 	return (0);
 }
@@ -1030,18 +1022,14 @@ DOT first -> wait
  */
 
 static int
-cnt_first(struct sess *sp)
+cnt_first(struct sess *sp, struct worker *wrk)
 {
-	struct worker *wrk;
 	char laddr[ADDR_BUFSIZE];
 	char lport[PORT_BUFSIZE];
 
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
-	wrk = sp->wrk;
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 
-	AZ(sp->req);
-
 	VTCP_name(&sp->sockaddr, sp->sockaddrlen,
 	    sp->addr, sizeof sp->addr, sp->port, sizeof sp->port);
 	if (cache_param->log_local_addr) {
@@ -1081,23 +1069,21 @@ DOT hit -> prepresp [label="deliver",style=bold,color=green]
  */
 
 static int
-cnt_hit(struct sess *sp)
+cnt_hit(struct sess *sp, struct worker *wrk, struct req *req)
 {
-	struct worker *wrk;
-
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
-	wrk = sp->wrk;
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
+	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
 
 	CHECK_OBJ_NOTNULL(wrk->obj, OBJECT_MAGIC);
-	CHECK_OBJ_NOTNULL(sp->req->vcl, VCL_CONF_MAGIC);
+	CHECK_OBJ_NOTNULL(req->vcl, VCL_CONF_MAGIC);
 	AZ(wrk->busyobj);
 
 	assert(!(wrk->obj->objcore->flags & OC_F_PASS));
 
 	VCL_hit_method(sp);
 
-	if (sp->req->handling == VCL_RET_DELIVER) {
+	if (req->handling == VCL_RET_DELIVER) {
 		/* Dispose of any body part of the request */
 		(void)FetchReqBody(sp);
 		//AZ(wrk->busyobj->bereq->ws);
@@ -1110,7 +1096,7 @@ cnt_hit(struct sess *sp)
 	(void)HSH_Deref(wrk, NULL, &wrk->obj);
 	wrk->objcore = NULL;
 
-	switch(sp->req->handling) {
+	switch(req->handling) {
 	case VCL_RET_PASS:
 		sp->step = STP_PASS;
 		return (0);
@@ -1118,8 +1104,8 @@ cnt_hit(struct sess *sp)
 		sp->step = STP_ERROR;
 		return (0);
 	case VCL_RET_RESTART:
-		sp->req->director = NULL;
-		sp->req->restarts++;
+		req->director = NULL;
+		req->restarts++;
 		sp->step = STP_RECV;
 		return (0);
 	default:
@@ -1156,32 +1142,31 @@ DOT lookup -> miss [label="no",style=bold,color=blue]
  */
 
 static int
-cnt_lookup(struct sess *sp)
+cnt_lookup(struct sess *sp, struct worker *wrk, struct req *req)
 {
 	struct objcore *oc;
 	struct object *o;
 	struct objhead *oh;
-	struct worker *wrk;
 
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
-	wrk = sp->wrk;
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
+	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
 
-	CHECK_OBJ_NOTNULL(sp->req->vcl, VCL_CONF_MAGIC);
+	CHECK_OBJ_NOTNULL(req->vcl, VCL_CONF_MAGIC);
 	AZ(wrk->busyobj);
 
-	if (sp->req->hash_objhead == NULL) {
+	if (req->hash_objhead == NULL) {
 		/* Not a waiting list return */
-		AZ(sp->req->vary_b);
-		AZ(sp->req->vary_l);
-		AZ(sp->req->vary_e);
-		(void)WS_Reserve(sp->req->ws, 0);
+		AZ(req->vary_b);
+		AZ(req->vary_l);
+		AZ(req->vary_e);
+		(void)WS_Reserve(req->ws, 0);
 	} else {
-		AN(sp->req->ws->r);
+		AN(req->ws->r);
 	}
-	sp->req->vary_b = (void*)sp->req->ws->f;
-	sp->req->vary_e = (void*)sp->req->ws->r;
-	sp->req->vary_b[2] = '\0';
+	req->vary_b = (void*)req->ws->f;
+	req->vary_e = (void*)req->ws->r;
+	req->vary_b[2] = '\0';
 
 	oc = HSH_Lookup(sp, &oh);
 
@@ -1203,17 +1188,17 @@ cnt_lookup(struct sess *sp)
 	if (oc->flags & OC_F_BUSY) {
 		wrk->stats.cache_miss++;
 
-		if (sp->req->vary_l != NULL) {
-			assert(oc->busyobj->vary == sp->req->vary_b);
+		if (req->vary_l != NULL) {
+			assert(oc->busyobj->vary == req->vary_b);
 			VRY_Validate(oc->busyobj->vary);
-			WS_ReleaseP(sp->req->ws, (void*)sp->req->vary_l);
+			WS_ReleaseP(req->ws, (void*)req->vary_l);
 		} else {
 			AZ(oc->busyobj->vary);
-			WS_Release(sp->req->ws, 0);
+			WS_Release(req->ws, 0);
 		}
-		sp->req->vary_b = NULL;
-		sp->req->vary_l = NULL;
-		sp->req->vary_e = NULL;
+		req->vary_b = NULL;
+		req->vary_l = NULL;
+		req->vary_e = NULL;
 
 		wrk->objcore = oc;
 		CHECK_OBJ_NOTNULL(wrk->busyobj, BUSYOBJ_MAGIC);
@@ -1225,10 +1210,10 @@ cnt_lookup(struct sess *sp)
 	CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC);
 	wrk->obj = o;
 
-	WS_Release(sp->req->ws, 0);
-	sp->req->vary_b = NULL;
-	sp->req->vary_l = NULL;
-	sp->req->vary_e = NULL;
+	WS_Release(req->ws, 0);
+	req->vary_b = NULL;
+	req->vary_l = NULL;
+	req->vary_e = NULL;
 
 	if (oc->flags & OC_F_PASS) {
 		wrk->stats.cache_hitpass++;
@@ -1269,14 +1254,13 @@ DOT
  */
 
 static int
-cnt_miss(struct sess *sp)
+cnt_miss(struct sess *sp, struct worker *wrk, struct req *req)
 {
-	struct worker *wrk;
 
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
-	wrk = sp->wrk;
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
-	CHECK_OBJ_NOTNULL(sp->req->vcl, VCL_CONF_MAGIC);
+	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
+	CHECK_OBJ_NOTNULL(req->vcl, VCL_CONF_MAGIC);
 
 	AZ(wrk->obj);
 	AN(wrk->objcore);
@@ -1302,7 +1286,7 @@ cnt_miss(struct sess *sp)
 
 	VCL_miss_method(sp);
 
-	switch(sp->req->handling) {
+	switch(req->handling) {
 	case VCL_RET_ERROR:
 		AZ(HSH_Deref(wrk, wrk->objcore, NULL));
 		wrk->objcore = NULL;
@@ -1363,14 +1347,12 @@ DOT err_pass [label="ERROR",shape=plaintext]
  */
 
 static int
-cnt_pass(struct sess *sp)
+cnt_pass(struct sess *sp, struct worker *wrk, struct req *req)
 {
-	struct worker *wrk;
-
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
-	wrk = sp->wrk;
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
-	CHECK_OBJ_NOTNULL(sp->req->vcl, VCL_CONF_MAGIC);
+	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
+	CHECK_OBJ_NOTNULL(req->vcl, VCL_CONF_MAGIC);
 	AZ(wrk->obj);
 	AZ(wrk->busyobj);
 
@@ -1384,15 +1366,15 @@ cnt_pass(struct sess *sp)
 	wrk->first_byte_timeout = 0;
 	wrk->between_bytes_timeout = 0;
 	VCL_pass_method(sp);
-	if (sp->req->handling == VCL_RET_ERROR) {
+	if (req->handling == VCL_RET_ERROR) {
 		http_Setup(wrk->busyobj->bereq, NULL);
 		VBO_DerefBusyObj(wrk, &wrk->busyobj);
 		sp->step = STP_ERROR;
 		return (0);
 	}
-	assert(sp->req->handling == VCL_RET_PASS);
+	assert(req->handling == VCL_RET_PASS);
 	wrk->acct_tmp.pass++;
-	sp->req->sendbody = 1;
+	req->sendbody = 1;
 	sp->step = STP_FETCH;
 	return (0);
 }
@@ -1423,14 +1405,13 @@ DOT err_pipe [label="ERROR",shape=plaintext]
  */
 
 static int
-cnt_pipe(struct sess *sp)
+cnt_pipe(struct sess *sp, struct worker *wrk, struct req *req)
 {
-	struct worker *wrk;
 
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
-	wrk = sp->wrk;
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
-	CHECK_OBJ_NOTNULL(sp->req->vcl, VCL_CONF_MAGIC);
+	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
+	CHECK_OBJ_NOTNULL(req->vcl, VCL_CONF_MAGIC);
 	AZ(wrk->busyobj);
 
 	wrk->acct_tmp.pipe++;
@@ -1442,9 +1423,9 @@ cnt_pipe(struct sess *sp)
 
 	VCL_pipe_method(sp);
 
-	if (sp->req->handling == VCL_RET_ERROR)
+	if (req->handling == VCL_RET_ERROR)
 		INCOMPL();
-	assert(sp->req->handling == VCL_RET_PIPE);
+	assert(req->handling == VCL_RET_PIPE);
 
 	PipeSession(sp);
 	assert(WRW_IsReleased(wrk));
@@ -1475,37 +1456,36 @@ DOT recv -> hash [label="lookup",style=bold,color=green]
  */
 
 static int
-cnt_recv(struct sess *sp)
+cnt_recv(struct sess *sp, struct worker *wrk, struct req *req)
 {
-	struct worker *wrk;
 	unsigned recv_handling;
 
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
-	wrk = sp->wrk;
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
-	CHECK_OBJ_NOTNULL(sp->req->vcl, VCL_CONF_MAGIC);
+	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
+	CHECK_OBJ_NOTNULL(req->vcl, VCL_CONF_MAGIC);
 	AZ(wrk->obj);
 	AZ(wrk->busyobj);
 	assert(wrk->wrw.ciov == wrk->wrw.siov);
 
 	/* By default we use the first backend */
-	AZ(sp->req->director);
-	sp->req->director = sp->req->vcl->director[0];
-	AN(sp->req->director);
+	AZ(req->director);
+	req->director = req->vcl->director[0];
+	AN(req->director);
 
-	sp->req->disable_esi = 0;
-	sp->req->hash_always_miss = 0;
-	sp->req->hash_ignore_busy = 0;
-	sp->req->client_identity = NULL;
+	req->disable_esi = 0;
+	req->hash_always_miss = 0;
+	req->hash_ignore_busy = 0;
+	req->client_identity = NULL;
 
-	http_CollectHdr(sp->req->http, H_Cache_Control);
+	http_CollectHdr(req->http, H_Cache_Control);
 
 	VCL_recv_method(sp);
-	recv_handling = sp->req->handling;
+	recv_handling = req->handling;
 
-	if (sp->req->restarts >= cache_param->max_restarts) {
-		if (sp->req->err_code == 0)
-			sp->req->err_code = 503;
+	if (req->restarts >= cache_param->max_restarts) {
+		if (req->err_code == 0)
+			req->err_code = 503;
 		sp->step = STP_ERROR;
 		return (0);
 	}
@@ -1514,32 +1494,32 @@ cnt_recv(struct sess *sp)
 	     (recv_handling != VCL_RET_PIPE) &&
 	     (recv_handling != VCL_RET_PASS)) {
 		if (RFC2616_Req_Gzip(sp)) {
-			http_Unset(sp->req->http, H_Accept_Encoding);
-			http_SetHeader(wrk, sp->vsl_id, sp->req->http,
+			http_Unset(req->http, H_Accept_Encoding);
+			http_SetHeader(wrk, sp->vsl_id, req->http,
 			    "Accept-Encoding: gzip");
 		} else {
-			http_Unset(sp->req->http, H_Accept_Encoding);
+			http_Unset(req->http, H_Accept_Encoding);
 		}
 	}
 
 	SHA256_Init(wrk->sha256ctx);
 	VCL_hash_method(sp);
-	assert(sp->req->handling == VCL_RET_HASH);
-	SHA256_Final(sp->req->digest, wrk->sha256ctx);
+	assert(req->handling == VCL_RET_HASH);
+	SHA256_Final(req->digest, wrk->sha256ctx);
 
-	if (!strcmp(sp->req->http->hd[HTTP_HDR_REQ].b, "HEAD"))
-		sp->req->wantbody = 0;
+	if (!strcmp(req->http->hd[HTTP_HDR_REQ].b, "HEAD"))
+		req->wantbody = 0;
 	else
-		sp->req->wantbody = 1;
+		req->wantbody = 1;
 
-	sp->req->sendbody = 0;
+	req->sendbody = 0;
 	switch(recv_handling) {
 	case VCL_RET_LOOKUP:
 		/* XXX: discard req body, if any */
 		sp->step = STP_LOOKUP;
 		return (0);
 	case VCL_RET_PIPE:
-		if (sp->req->esi_level > 0) {
+		if (req->esi_level > 0) {
 			/* XXX: VSL something */
 			INCOMPL();
 			/* sp->step = STP_DONE; */
@@ -1572,22 +1552,20 @@ DOT start -> DONE [label=errors]
  */
 
 static int
-cnt_start(struct sess *sp)
+cnt_start(struct sess *sp, struct worker *wrk, struct req *req)
 {
 	uint16_t done;
 	char *p;
 	const char *r = "HTTP/1.1 100 Continue\r\n\r\n";
-	struct worker *wrk;
 
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
-	wrk = sp->wrk;
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
-	CHECK_OBJ_NOTNULL(sp->req, REQ_MAGIC);
-	AZ(sp->req->restarts);
+	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
+	AZ(req->restarts);
 	AZ(wrk->obj);
-	AZ(sp->req->vcl);
-	EXP_Clr(&sp->req->exp);
-	AZ(sp->req->esi_level);
+	AZ(req->vcl);
+	EXP_Clr(&req->exp);
+	AZ(req->esi_level);
 
 	/* Update stats of various sorts */
 	wrk->stats.client_req++;
@@ -1595,15 +1573,15 @@ cnt_start(struct sess *sp)
 	wrk->acct_tmp.req++;
 
 	/* Assign XID and log */
-	sp->req->xid = ++xids;				/* XXX not locked */
-	WSP(sp, SLT_ReqStart, "%s %s %u", sp->addr, sp->port,  sp->req->xid);
+	req->xid = ++xids;				/* XXX not locked */
+	WSP(sp, SLT_ReqStart, "%s %s %u", sp->addr, sp->port,  req->xid);
 
 	/* Borrow VCL reference from worker thread */
 	VCL_Refresh(&wrk->vcl);
-	sp->req->vcl = wrk->vcl;
+	req->vcl = wrk->vcl;
 	wrk->vcl = NULL;
 
-	http_Setup(sp->req->http, sp->req->ws);
+	http_Setup(req->http, req->ws);
 	done = http_DissectRequest(sp);
 
 	/* If we could not even parse the request, just close */
@@ -1614,27 +1592,27 @@ cnt_start(struct sess *sp)
 	}
 
 	/* Catch request snapshot */
-	sp->req->ws_req = WS_Snapshot(sp->req->ws);
+	req->ws_req = WS_Snapshot(req->ws);
 
 	/* Catch original request, before modification */
-	HTTP_Copy(sp->req->http0, sp->req->http);
+	HTTP_Copy(req->http0, req->http);
 
 	if (done != 0) {
-		sp->req->err_code = done;
+		req->err_code = done;
 		sp->step = STP_ERROR;
 		return (0);
 	}
 
-	sp->req->doclose = http_DoConnection(sp->req->http);
+	req->doclose = http_DoConnection(req->http);
 
 	/* XXX: Handle TRACE & OPTIONS of Max-Forwards = 0 */
 
 	/*
 	 * Handle Expect headers
 	 */
-	if (http_GetHdr(sp->req->http, H_Expect, &p)) {
+	if (http_GetHdr(req->http, H_Expect, &p)) {
 		if (strcasecmp(p, "100-continue")) {
-			sp->req->err_code = 417;
+			req->err_code = 417;
 			sp->step = STP_ERROR;
 			return (0);
 		}
@@ -1645,7 +1623,7 @@ cnt_start(struct sess *sp)
 		 * XXX: because we use http0 as our basis.  Believed
 		 * XXX: safe, but potentially confusing.
 		 */
-		http_Unset(sp->req->http, H_Expect);
+		http_Unset(req->http, H_Expect);
 	}
 
 	sp->step = STP_RECV;
@@ -1688,8 +1666,8 @@ CNT_Session(struct sess *sp)
 
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 #if 0
-	CHECK_OBJ_NOTNULL(sp->req, REQ_MAGIC);
-	MPL_AssertSane(sp->req);
+	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
+	MPL_AssertSane(req);
 #endif
 	wrk = sp->wrk;
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
@@ -1729,8 +1707,8 @@ CNT_Session(struct sess *sp)
 	for (done = 0; !done; ) {
 		assert(sp->wrk == wrk);
 #if 0
-		CHECK_OBJ_NOTNULL(sp->req, REQ_MAGIC);
-		MPL_AssertSane(sp->req);
+		CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
+		MPL_AssertSane(req);
 #endif
 		/*
 		 * This is a good place to be paranoid about the various
@@ -1743,11 +1721,11 @@ CNT_Session(struct sess *sp)
 		WS_Assert(wrk->ws);
 
 		switch (sp->step) {
-#define STEP(l,u) \
+#define STEP(l,u,arg) \
 		    case STP_##u: \
 			if (cache_param->diag_bitmap & 0x01) \
 				cnt_diag(sp, #u); \
-			done = cnt_##l(sp); \
+			done = cnt_##l arg; \
 		        break;
 #include "tbl/steps.h"
 #undef STEP
diff --git a/bin/varnishd/cache/cache_panic.c b/bin/varnishd/cache/cache_panic.c
index 9c26eef..80f61a4 100644
--- a/bin/varnishd/cache/cache_panic.c
+++ b/bin/varnishd/cache/cache_panic.c
@@ -238,7 +238,7 @@ pan_sess(const struct sess *sp)
 	    sp->addr ? sp->addr : "?.?.?.?",
 	    sp->port ? sp->port : "?");
 	switch (sp->step) {
-#define STEP(l, u) case STP_##u: stp = "STP_" #u; break;
+#define STEP(l, u, arg) case STP_##u: stp = "STP_" #u; break;
 #include "tbl/steps.h"
 #undef STEP
 		default: stp = NULL;
diff --git a/include/tbl/steps.h b/include/tbl/steps.h
index c88e434..6fcd74e 100644
--- a/include/tbl/steps.h
+++ b/include/tbl/steps.h
@@ -29,20 +29,20 @@
  */
 
 /*lint -save -e525 -e539 */
-STEP(wait,	WAIT)
-STEP(first,	FIRST)
-STEP(recv,	RECV)
-STEP(start,	START)
-STEP(pipe,	PIPE)
-STEP(pass,	PASS)
-STEP(lookup,	LOOKUP)
-STEP(miss,	MISS)
-STEP(hit,	HIT)
-STEP(fetch,	FETCH)
-STEP(fetchbody,	FETCHBODY)
-STEP(streambody,STREAMBODY)
-STEP(prepresp,	PREPRESP)
-STEP(deliver,	DELIVER)
-STEP(error,	ERROR)
-STEP(done,	DONE)
+STEP(wait,		WAIT,		(sp, sp->wrk, sp->req))
+STEP(first,		FIRST,		(sp, sp->wrk))
+STEP(recv,		RECV,		(sp, sp->wrk, sp->req))
+STEP(start,		START,		(sp, sp->wrk, sp->req))
+STEP(pipe,		PIPE,		(sp, sp->wrk, sp->req))
+STEP(pass,		PASS,		(sp, sp->wrk, sp->req))
+STEP(lookup,		LOOKUP,		(sp, sp->wrk, sp->req))
+STEP(miss,		MISS,		(sp, sp->wrk, sp->req))
+STEP(hit,		HIT,		(sp, sp->wrk, sp->req))
+STEP(fetch,		FETCH,		(sp, sp->wrk, sp->req))
+STEP(fetchbody,		FETCHBODY,	(sp, sp->wrk, sp->req))
+STEP(streambody,	STREAMBODY,	(sp, sp->wrk, sp->req))
+STEP(prepresp,		PREPRESP,	(sp, sp->wrk, sp->req))
+STEP(deliver,		DELIVER,	(sp, sp->wrk, sp->req))
+STEP(error,		ERROR,		(sp, sp->wrk, sp->req))
+STEP(done,		DONE,		(sp, sp->wrk, sp->req))
 /*lint -restore */



From geoff at varnish-cache.org  Mon Jan  9 23:03:29 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Tue, 10 Jan 2012 00:03:29 +0100
Subject: [experimental-ims] ad22891 Loose the session watermark
Message-ID: <E1RkOFV-0007rL-5Q@project.varnish-software.com>

commit ad228918a305d4971143d14c93b05ecfe3a6e703
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Dec 20 11:43:48 2011 +0000

    Loose the session watermark

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index 2c8d730..08f8adc 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -611,7 +611,6 @@ struct sess {
 	struct http		*http0;
 
 	struct ws		ws[1];
-	char			*ws_ses;	/* WS above session data */
 	char			*ws_req;	/* WS above request data */
 
 	unsigned char		digest[DIGEST_LEN];
diff --git a/bin/varnishd/cache/cache_acceptor.c b/bin/varnishd/cache/cache_acceptor.c
index 2814f87..a9b8c0b 100644
--- a/bin/varnishd/cache/cache_acceptor.c
+++ b/bin/varnishd/cache/cache_acceptor.c
@@ -32,7 +32,7 @@
  * The actual acceptance is done from cache_pool.c, by calling
  * into VCA_Accept() in this file.
  *
- * Once the session is allocated we move into it with a call to 
+ * Once the session is allocated we move into it with a call to
  * VCA_SetupSess().
  *
  * If we fail to allocate a session we call VCA_FailSess() to clean up
diff --git a/bin/varnishd/cache/cache_center.c b/bin/varnishd/cache/cache_center.c
index 6b9b353..e7c7fe2 100644
--- a/bin/varnishd/cache/cache_center.c
+++ b/bin/varnishd/cache/cache_center.c
@@ -414,7 +414,7 @@ WSP(sp, SLT_Debug, "PHK req %.9f resp %.9f end %.9f open %.9f",
 	if (wrk->stats.client_req >= cache_param->wthread_stats_rate)
 		WRK_SumStat(wrk);
 	/* Reset the workspace to the session-watermark */
-	WS_Reset(sp->ws, sp->ws_ses);
+	WS_Reset(sp->ws, NULL);
 	WS_Reset(wrk->ws, NULL);
 
 	i = HTC_Reinit(sp->htc);
@@ -1039,9 +1039,6 @@ cnt_first(struct sess *sp)
 	}
 	sp->acct_ses.first = sp->t_open;
 
-	/* Record the session watermark */
-	sp->ws_ses = WS_Snapshot(sp->ws);
-
 	/* Receive a HTTP protocol request */
 	HTC_Init(sp->htc, sp->ws, sp->fd, sp->vsl_id,
 	    cache_param->http_req_size,



From geoff at varnish-cache.org  Mon Jan  9 23:03:29 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Tue, 10 Jan 2012 00:03:29 +0100
Subject: [experimental-ims] ffb8801 Simplify the dance around sock acceptance.
Message-ID: <E1RkOFV-0007r4-1V@project.varnish-software.com>

commit ffb8801b909d842072d4c0282d2820e7ee7aa029
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Dec 20 09:54:05 2011 +0000

    Simplify the dance around sock acceptance.

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index d1d67b6..2c8d730 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -293,6 +293,7 @@ enum e_do_what {
 	pool_do_inval = 0,
 	pool_do_sess,
 	pool_do_accept,
+	pool_do_nothing,
 	pool_do_die,
 };
 
@@ -662,7 +663,6 @@ struct sess {
 /* Prototypes etc ----------------------------------------------------*/
 
 /* cache_acceptor.c */
-void VCA_Prep(const struct sess *sp);
 void VCA_Init(void);
 void VCA_Shutdown(void);
 int VCA_Accept(struct listen_sock *ls, struct wrk_accept *wa);
diff --git a/bin/varnishd/cache/cache_acceptor.c b/bin/varnishd/cache/cache_acceptor.c
index 6ef9aec..2814f87 100644
--- a/bin/varnishd/cache/cache_acceptor.c
+++ b/bin/varnishd/cache/cache_acceptor.c
@@ -26,6 +26,17 @@
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
  *
+ * This source file has the various trickery surrounding the accept/listen
+ * sockets.
+ *
+ * The actual acceptance is done from cache_pool.c, by calling
+ * into VCA_Accept() in this file.
+ *
+ * Once the session is allocated we move into it with a call to 
+ * VCA_SetupSess().
+ *
+ * If we fail to allocate a session we call VCA_FailSess() to clean up
+ * and initiate pacing.
  */
 
 #include "config.h"
@@ -41,6 +52,9 @@
 static pthread_t	VCA_thread;
 static struct timeval	tv_sndtimeo;
 static struct timeval	tv_rcvtimeo;
+static int hack_ready;
+static double vca_pace = 0.0;
+static struct lock pace_mtx;
 
 /*--------------------------------------------------------------------
  * We want to get out of any kind of trouble-hit TCP connections as fast
@@ -54,6 +68,12 @@ static const struct linger linger = {
 
 static unsigned char	need_sndtimeo, need_rcvtimeo, need_linger, need_test;
 
+/*--------------------------------------------------------------------
+ * Some kernels have bugs/limitations with respect to which options are
+ * inherited from the accept/listen socket, so we have to keep track of
+ * which, if any, sockopts we have to set on the accepted socket.
+ */
+
 static void
 sock_test(int fd)
 {
@@ -108,12 +128,12 @@ sock_test(int fd)
 }
 
 /*--------------------------------------------------------------------
- * Called once the workerthread gets hold of the session, to do setup
- * setup overhead, we don't want to bother the acceptor thread with.
+ * Called once the workerthread gets hold of the session, to fix up
+ * any socket options that need it.
  */
 
-void
-VCA_Prep(const struct sess *sp)
+static void
+vca_prep(const struct sess *sp)
 {
 
 	if (need_test)
@@ -138,9 +158,6 @@ VCA_Prep(const struct sess *sp)
  * shortage if possible.
  */
 
-static double vca_pace = 0.0;
-static struct lock pace_mtx;
-
 static void
 vca_pace_check(void)
 {
@@ -183,8 +200,6 @@ vca_pace_good(void)
  * Accept on a listen socket, and handle error returns.
  */
 
-static int hack_ready;
-
 int
 VCA_Accept(struct listen_sock *ls, struct wrk_accept *wa)
 {
@@ -243,7 +258,9 @@ VCA_FailSess(struct worker *w)
 	vca_pace_bad();
 }
 
-/*--------------------------------------------------------------------*/
+/*--------------------------------------------------------------------
+ * We have allocated a session, move our info into it.
+ */
 
 void
 VCA_SetupSess(struct worker *w)
@@ -268,6 +285,7 @@ VCA_SetupSess(struct worker *w)
 	sp->step = STP_FIRST;
 	vca_pace_good();
 	w->stats.sess_conn++;
+	vca_prep(sp);
 }
 
 /*--------------------------------------------------------------------*/
@@ -363,7 +381,7 @@ ccf_listen_address(struct cli *cli, const char * const *av, void *priv)
 
 	/*
 	 * This CLI command is primarily used by varnishtest.  Don't
-	 * respond until liste(2) has been called, in order to avoid
+	 * respond until listen(2) has been called, in order to avoid
 	 * a race where varnishtest::client would attempt to connect(2)
 	 * before listen(2) has been called.
 	 */
diff --git a/bin/varnishd/cache/cache_center.c b/bin/varnishd/cache/cache_center.c
index a6ea594..6b9b353 100644
--- a/bin/varnishd/cache/cache_center.c
+++ b/bin/varnishd/cache/cache_center.c
@@ -1039,8 +1039,6 @@ cnt_first(struct sess *sp)
 	}
 	sp->acct_ses.first = sp->t_open;
 
-	VCA_Prep(sp);
-
 	/* Record the session watermark */
 	sp->ws_ses = WS_Snapshot(sp->ws);
 
diff --git a/bin/varnishd/cache/cache_pool.c b/bin/varnishd/cache/cache_pool.c
index 1195b45..c557edd 100644
--- a/bin/varnishd/cache/cache_pool.c
+++ b/bin/varnishd/cache/cache_pool.c
@@ -239,12 +239,14 @@ Pool_Work_Thread(void *priv, struct worker *w)
 			AZ(w->sp);
 			AN(w->ws->r);
 			w->sp = SES_New(w, pp->sesspool);
-			if (w->sp == NULL)
+			if (w->sp == NULL) {
 				VCA_FailSess(w);
-			else
+				w->do_what = pool_do_nothing;
+			} else {
 				VCA_SetupSess(w);
+				w->do_what = pool_do_sess;
+			}
 			WS_Release(w->ws, 0);
-			w->do_what = pool_do_sess;
 		}
 
 		if (w->do_what == pool_do_sess) {
@@ -272,6 +274,8 @@ Pool_Work_Thread(void *priv, struct worker *w)
 				if (w->vcl != NULL)
 					VCL_Rel(&w->vcl);
 			}
+		} else if (w->do_what == pool_do_nothing) {
+			/* we already did */
 		} else {
 			WRONG("Invalid w->do_what");
 		}



From geoff at varnish-cache.org  Mon Jan  9 23:03:33 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Tue, 10 Jan 2012 00:03:33 +0100
Subject: [experimental-ims] f5577b7 Move more stuff from worker to req
Message-ID: <E1RkOFZ-0007xT-1p@project.varnish-software.com>

commit f5577b72696ea7035c732293bd60fc0a5794932c
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Dec 26 10:22:01 2011 +0000

    Move more stuff from worker to req

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index 372a32d..42562b5 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -328,17 +328,9 @@ struct worker {
 
 	struct busyobj		*busyobj;
 
-	/* This is only here so VRT can find it */
-	const char		*storage_hint;
-
 	/* Stream state */
 	struct stream_ctx	*sctx;
 
-	/* ESI delivery stuff */
-	int			gzip_resp;
-	ssize_t			l_crc;
-	uint32_t		crc;
-
 	/* Timeouts */
 	double			connect_timeout;
 	double			first_byte_timeout;
@@ -621,6 +613,14 @@ struct req {
 	struct objcore		*objcore;
 	/* Lookup stuff */
 	struct SHA256Context	*sha256ctx;
+	/* This is only here so VRT can find it */
+	const char		*storage_hint;
+
+	/* ESI delivery stuff */
+	int			gzip_resp;
+	ssize_t			l_crc;
+	uint32_t		crc;
+
 
 };
 
diff --git a/bin/varnishd/cache/cache_center.c b/bin/varnishd/cache/cache_center.c
index 7e9c77f..81a0c67 100644
--- a/bin/varnishd/cache/cache_center.c
+++ b/bin/varnishd/cache/cache_center.c
@@ -597,7 +597,7 @@ cnt_fetch(struct sess *sp, struct worker *wrk, struct req *req)
 	AN(req->director);
 	AZ(wrk->busyobj->vbc);
 	AZ(wrk->busyobj->should_close);
-	AZ(wrk->storage_hint);
+	AZ(req->storage_hint);
 
 	http_Setup(wrk->busyobj->beresp, wrk->ws);
 
@@ -678,7 +678,7 @@ cnt_fetch(struct sess *sp, struct worker *wrk, struct req *req)
 	}
 	VBO_DerefBusyObj(wrk, &wrk->busyobj);
 	req->director = NULL;
-	wrk->storage_hint = NULL;
+	req->storage_hint = NULL;
 
 	switch (req->handling) {
 	case VCL_RET_RESTART:
@@ -830,9 +830,9 @@ cnt_fetchbody(struct sess *sp, struct worker *wrk, struct req *req)
 
 	if (wrk->busyobj->exp.ttl < cache_param->shortlived ||
 	    req->objcore == NULL)
-		wrk->storage_hint = TRANSIENT_STORAGE;
+		req->storage_hint = TRANSIENT_STORAGE;
 
-	req->obj = STV_NewObject(wrk, wrk->storage_hint, l, nhttp);
+	req->obj = STV_NewObject(wrk, req->storage_hint, l, nhttp);
 	if (req->obj == NULL) {
 		/*
 		 * Try to salvage the transaction by allocating a
@@ -853,7 +853,7 @@ cnt_fetchbody(struct sess *sp, struct worker *wrk, struct req *req)
 	}
 	CHECK_OBJ_NOTNULL(req->obj, OBJECT_MAGIC);
 
-	wrk->storage_hint = NULL;
+	req->storage_hint = NULL;
 
 	if (wrk->busyobj->do_gzip ||
 	    (wrk->busyobj->is_gzip && !wrk->busyobj->do_gunzip))
diff --git a/bin/varnishd/cache/cache_esi_deliver.c b/bin/varnishd/cache/cache_esi_deliver.c
index 0c58395..a7e80cd 100644
--- a/bin/varnishd/cache/cache_esi_deliver.c
+++ b/bin/varnishd/cache/cache_esi_deliver.c
@@ -194,8 +194,8 @@ ved_pretend_gzip(const struct sess *sp, const uint8_t *p, ssize_t l)
 			(void)WRW_Write(sp->wrk, buf2, sizeof buf2);
 		}
 		(void)WRW_Write(sp->wrk, p, lx);
-		sp->wrk->crc = crc32(sp->wrk->crc, p, lx);
-		sp->wrk->l_crc += lx;
+		sp->req->crc = crc32(sp->req->crc, p, lx);
+		sp->req->l_crc += lx;
 		l -= lx;
 		p += lx;
 	}
@@ -251,18 +251,18 @@ ESI_Deliver(struct sess *sp)
 		/*
 		 * Only the top level document gets to decide this.
 		 */
-		sp->wrk->gzip_resp = 0;
+		sp->req->gzip_resp = 0;
 		if (isgzip && !(sp->wrk->res_mode & RES_GUNZIP)) {
 			assert(sizeof gzip_hdr == 10);
 			/* Send out the gzip header */
 			(void)WRW_Write(sp->wrk, gzip_hdr, 10);
-			sp->wrk->l_crc = 0;
-			sp->wrk->gzip_resp = 1;
-			sp->wrk->crc = crc32(0L, Z_NULL, 0);
+			sp->req->l_crc = 0;
+			sp->req->gzip_resp = 1;
+			sp->req->crc = crc32(0L, Z_NULL, 0);
 		}
 	}
 
-	if (isgzip && !sp->wrk->gzip_resp) {
+	if (isgzip && !sp->req->gzip_resp) {
 		vgz = VGZ_NewUngzip(sp->wrk, "U D E");
 
 		/* Feed a gzip header to gunzip to make it happy */
@@ -291,10 +291,10 @@ ESI_Deliver(struct sess *sp)
 				l_icrc = ved_decode_len(&p);
 				icrc = vbe32dec(p);
 				p += 4;
-				if (sp->wrk->gzip_resp) {
-					sp->wrk->crc = crc32_combine(
-					    sp->wrk->crc, icrc, l_icrc);
-					sp->wrk->l_crc += l_icrc;
+				if (sp->req->gzip_resp) {
+					sp->req->crc = crc32_combine(
+					    sp->req->crc, icrc, l_icrc);
+					sp->req->l_crc += l_icrc;
 				}
 			}
 			/*
@@ -308,14 +308,14 @@ ESI_Deliver(struct sess *sp)
 					l2 = st->len - off;
 				l -= l2;
 
-				if (sp->wrk->gzip_resp && isgzip) {
+				if (sp->req->gzip_resp && isgzip) {
 					/*
 					 * We have a gzip'ed VEC and delivers
 					 * a gzip'ed ESI response.
 					 */
 					(void)WRW_Write(sp->wrk,
 					    st->ptr + off, l2);
-				} else if (sp->wrk->gzip_resp) {
+				} else if (sp->req->gzip_resp) {
 					/*
 					 * A gzip'ed ESI response, but the VEC
 					 * was not gzip'ed.
@@ -403,7 +403,7 @@ ESI_Deliver(struct sess *sp)
 			(void)WRW_Write(sp->wrk, obuf, obufl);
 		(void)VGZ_Destroy(&vgz, sp->vsl_id);
 	}
-	if (sp->wrk->gzip_resp && sp->req->esi_level == 0) {
+	if (sp->req->gzip_resp && sp->req->esi_level == 0) {
 		/* Emit a gzip literal block with finish bit set */
 		tailbuf[0] = 0x01;
 		tailbuf[1] = 0x00;
@@ -412,10 +412,10 @@ ESI_Deliver(struct sess *sp)
 		tailbuf[4] = 0xff;
 
 		/* Emit CRC32 */
-		vle32enc(tailbuf + 5, sp->wrk->crc);
+		vle32enc(tailbuf + 5, sp->req->crc);
 
 		/* MOD(2^32) length */
-		vle32enc(tailbuf + 9, sp->wrk->l_crc);
+		vle32enc(tailbuf + 9, sp->req->l_crc);
 
 		(void)WRW_Write(sp->wrk, tailbuf, 13);
 	}
@@ -560,6 +560,6 @@ ESI_DeliverChild(const struct sess *sp)
 	p = st->ptr + st->len - 8;
 	icrc = vle32dec(p);
 	ilen = vle32dec(p + 4);
-	sp->wrk->crc = crc32_combine(sp->wrk->crc, icrc, ilen);
-	sp->wrk->l_crc += ilen;
+	sp->req->crc = crc32_combine(sp->req->crc, icrc, ilen);
+	sp->req->l_crc += ilen;
 }
diff --git a/bin/varnishd/cache/cache_pool.c b/bin/varnishd/cache/cache_pool.c
index 2236f0b..9b0760c 100644
--- a/bin/varnishd/cache/cache_pool.c
+++ b/bin/varnishd/cache/cache_pool.c
@@ -255,7 +255,6 @@ Pool_Work_Thread(void *priv, struct worker *wrk)
 
 			stats_clean = 0;
 			wrk->lastused = NAN;
-			wrk->storage_hint = NULL;
 
 			AZ(wrk->sp->wrk);
 			THR_SetSession(wrk->sp);
@@ -267,7 +266,6 @@ Pool_Work_Thread(void *priv, struct worker *wrk)
 			WS_Assert(wrk->ws);
 			AZ(wrk->busyobj);
 			AZ(wrk->wrw.wfd);
-			AZ(wrk->storage_hint);
 			assert(wrk->wlp == wrk->wlb);
 			if (cache_param->diag_bitmap & 0x00040000) {
 				if (wrk->vcl != NULL)
diff --git a/bin/varnishd/cache/cache_response.c b/bin/varnishd/cache/cache_response.c
index 1abb12b..fc94c22 100644
--- a/bin/varnishd/cache/cache_response.c
+++ b/bin/varnishd/cache/cache_response.c
@@ -307,10 +307,10 @@ RES_WriteObj(struct sess *sp)
 		/* Nothing to do here */
 	} else if (sp->wrk->res_mode & RES_ESI) {
 		ESI_Deliver(sp);
-	} else if (sp->wrk->res_mode & RES_ESI_CHILD && sp->wrk->gzip_resp) {
+	} else if (sp->wrk->res_mode & RES_ESI_CHILD && sp->req->gzip_resp) {
 		ESI_DeliverChild(sp);
 	} else if (sp->wrk->res_mode & RES_ESI_CHILD &&
-	    !sp->wrk->gzip_resp && sp->req->obj->gziped) {
+	    !sp->req->gzip_resp && sp->req->obj->gziped) {
 		res_WriteGunzipObj(sp);
 	} else if (sp->wrk->res_mode & RES_GUNZIP) {
 		res_WriteGunzipObj(sp);
diff --git a/bin/varnishd/cache/cache_vrt_var.c b/bin/varnishd/cache/cache_vrt_var.c
index cb66ff7..e051dbb 100644
--- a/bin/varnishd/cache/cache_vrt_var.c
+++ b/bin/varnishd/cache/cache_vrt_var.c
@@ -283,8 +283,8 @@ const char * __match_proto__()
 VRT_r_beresp_storage(struct sess *sp)
 {
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
-	if (sp->wrk->storage_hint != NULL)
-		return (sp->wrk->storage_hint);
+	if (sp->req->storage_hint != NULL)
+		return (sp->req->storage_hint);
 	else
 		return (NULL);
 }
@@ -298,7 +298,7 @@ VRT_l_beresp_storage(struct sess *sp, const char *str, ...)
 	va_start(ap, str);
 	b = VRT_String(sp->wrk->ws, NULL, str, ap);
 	va_end(ap);
-	sp->wrk->storage_hint = b;
+	sp->req->storage_hint = b;
 }
 
 /*--------------------------------------------------------------------*/



From geoff at varnish-cache.org  Mon Jan  9 23:03:29 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Tue, 10 Jan 2012 00:03:29 +0100
Subject: [experimental-ims] c57dab7 Curse WG14
Message-ID: <E1RkOFV-0007ro-Dw@project.varnish-software.com>

commit c57dab774f86e66334c304990ca2feacbb8ce840
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Dec 20 20:09:54 2011 +0000

    Curse WG14

diff --git a/doc/sphinx/phk/index.rst b/doc/sphinx/phk/index.rst
index 6264f55..822434c 100644
--- a/doc/sphinx/phk/index.rst
+++ b/doc/sphinx/phk/index.rst
@@ -8,6 +8,7 @@ You may or may not want to know what Poul-Henning think.
 
 .. toctree::
 
+	thetoolsweworkwith.rst
 	three-zero.rst
 	ssl.rst
 	gzip.rst
diff --git a/doc/sphinx/phk/thetoolsweworkwith.rst b/doc/sphinx/phk/thetoolsweworkwith.rst
new file mode 100644
index 0000000..cff365b
--- /dev/null
+++ b/doc/sphinx/phk/thetoolsweworkwith.rst
@@ -0,0 +1,211 @@
+.. _phk_thetoolsweworkwith:
+
+======================
+The Tools We Work With
+======================
+
+"Only amateurs were limited by their tools" is an old wisdom, and
+the world is littered with art and architecture that very much
+proves this point.
+
+But as amazing as the Aquaeduct of Segovia is, tools are the reason
+why it looks nowhere near as fantastic as the Sydney Opera House.
+
+Concrete has been known since antiquity, but steel-reinforced
+concrete and massive numerical calculations of stress-distribution,
+is the tools that makes the difference between using concrete as a
+filler material between stones, and as gravity-defying curved but
+perfectly safe load-bearing wall.
+
+My tool for writing Varnish is the C-language which in many ways
+is unique amongst all of the computer programming languages for
+having no ambitions.
+
+The C language was invented as a portable assembler language, it
+doesn't do objects and garbage-collection, it does numbers and
+pointers, just like your CPU.
+
+Compared to the high ambitions, then as now, of new programming
+languages, that was almost ridiculous unambitious.  Other people
+were trying to make their programming languages provably correct,
+or safe for multiprogramming and quite an effort went into using
+natural languages as programming languages.
+
+But C was written to write programs, not to research computer science
+and that's exactly what made it useful and popular.
+
+Unfortunately C fell in bad company over the years, and the reason
+for this outburst is that I just browsed the latest draft from the
+ISO-C standardisation working-group 14.
+
+I won't claim that it is enough to make grown men cry, but it
+certainly was enough to make me angry.
+
+Let me give you an example of their utter sillyness:
+
+The book which defined the C langauge had a list af reserved
+identifiers, all of them lower-case words.  The UNIX libraries
+defined a lot of functions, all of them lower-case words.
+
+When compiled, the assembler saw all of these words prefixed
+with an underscore, which made it easy to mix assembler and
+C code.
+
+All the macros for the C-preprocessor on the other hand, were
+UPPERCASE, making them easy to spot.
+
+Which meant that if you mixed upper and lower case, in your
+identifiers, you were safe: That wouldn't collide with anything.
+
+First the ISO-C standards people got confused about the leading
+underscore, and I'll leave you guessing as to what the current
+text actually means:
+
+	All identifiers that begin with an underscore and either
+	an uppercase letter or another underscore are always reserved
+	for any use.
+
+Feel free to guess.
+
+Next, they broke the upper/lower rule, by adding special keywords
+in mixed case, probably because they thought it looked nicer::
+
+	_Atomic, _Bool, _Noreturn &c
+
+Then, presumably, somebody pointed out that this looked ugly::
+
+	void _Noreturn foo(int bar);
+
+So they have come up with a #include file called <stdnoreturn.h>
+so that instead you can write::
+
+	#include <nostdreturn.h>
+	void noreturn foo(int bar);
+
+The <nostdreturn.h> file according to the standard shall have
+exactly this content::
+
+	#define noreturn _Noreturn
+
+Are you crying or laughing yet ?   You should be.
+
+Another thing brought by the new draft is an entirely new thread
+API, which is incompatible with the POSIX 'pthread' API which have
+been used for about 20 years now.
+
+If they had improved on the shortcomings of the pthreads, I would
+have cheered them on, because there are some very annoying mistakes
+in pthreads.
+
+But they didn't, in fact, as far as I can tell, the C1X draft's
+threads are worse than the 20 years older pthreads in all relevant
+aspects.
+
+For instance, neither pthreads nor C1X-threads offer a "assert I'm
+holding this mutex locked" facility.  I will posit that you cannot
+successfully develop real-world threaded programs and APIs that,
+or without wasting a lot of time debugging silly mistakes.
+
+If you look in the Varnish source code, which uses pthreads, you
+will see that I have wrapped pthread mutexes in my own little
+datastructure, to be able to do those asserts, and to get some
+usable statistics on lock-contention.
+
+Another example where C1X did not improve on pthreads at all, was
+in timed sleeps, where you say "get me this lock, but give up if
+it takes longer than X time".
+
+The way both pthreads and C1X threads do this, is you specify a UTC
+wall clock time you want to sleep until.
+
+The only problem with that is that UTC wall clock time is not
+continuous when implemented on a computer, and it may not even be
+monotonously increasing, since NTPD or other timesync facilites may
+step the clock backwards, particularly in the first minutes after
+boot.
+
+If the practice of saying "get me this lock before 16:00Z" was
+widespread, I could see the point, but I have actually never seen
+that in any source code.  What I have seen are wrappers that take
+the general shape of::
+
+	int
+	get_lock_timed(lock, timeout)
+	{
+		while (timeout > 0) {
+			t0 = time();
+			i = get_lock_before(lock, t + timeout));
+			if (i == WASLOCKED)
+				return (i);
+			t1 = time();
+			timeout -= (t1 - t0);
+		}
+		return (TIMEDOUT);
+	}
+
+Because it's not like the call is actually guaranteed to return at
+16:00Z if you ask it to, you are only promised it will not return
+later than that, so you have to wrap the call in a loop.
+
+Whoever defined the select(2) and poll(2) systemcalls knew better
+than the POSIX and ISO-C group-think:  They specifed a maximum
+duration for the call, because then it doesn't matter what time
+it is, only how long time has transpired.
+
+Ohh, and setting the stack-size for a new thread ?
+That is appearantly "too dangerous" so there is no argument in the
+C1X API for doing so, a clear step backwards from pthreads.
+
+But guess what:  Thread stacks are like T-shirts:  There is no "one
+size fits all."
+
+I have no idea what the "danger" they perceived were, my best
+guess is that feared it might make the API useful ?
+
+This single idiocy will single-handedly doom the C1X thread API
+to uselessness.
+
+Now, don't get me wrong:  There are lot of ways to improve the C
+language that would make sense:  Bitmaps, defined structure packing
+(think: communication protocol packets), big/little endian variables
+(data sharing), sensible handling of linked lists etc.
+
+As ugly as it is, even the printf()/scanf() format strings could
+be improved, by offering a sensible plugin mechanism, which the
+compiler can understand and use to issue warnings.
+
+Heck, even a simple basic object facility would be good addition,
+now that C++ have become this huge bloated monster language.
+
+But none of that is appearantly as important as <stdnoreturn.h>
+and a new, crippled and therefore useless thread API.
+
+The neat thing about the C language, and the one feature that made
+it so popular, is that not even an ISO-C working group can prevent
+you from implementing all these things using macros and other tricks.
+
+But it would be better to have them in the language, so the compiler
+could issue sensible warnings and programmers won't have to write
+monsters like::
+
+    #define VTAILQ_INSERT_BEFORE(listelm, elm, field) do {              \
+        (elm)->field.vtqe_prev = (listelm)->field.vtqe_prev;            \
+        VTAILQ_NEXT((elm), field) = (listelm);                          \
+        *(listelm)->field.vtqe_prev = (elm);                            \
+        (listelm)->field.vtqe_prev = &VTAILQ_NEXT((elm), field);        \
+    } while (0)
+
+To put an element on a linked list.
+
+I could go on like this, but it would rapidly become boring for
+both you and me, because the current C1X draft is 701 pages, and
+it contains not a single explanatory example if how to use any of
+the verbiage in pratice.
+
+Compare this with The C Programming Language, a book of 274 pages
+which in addition to define the C language, taught people how to
+program through well-thought-out examples.
+
+From where I sit, ISO WG14 are destroying the C language I use and love.
+
+Poul-Henning, 2011-12-20



From geoff at varnish-cache.org  Mon Jan  9 23:03:33 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Tue, 10 Jan 2012 00:03:33 +0100
Subject: [experimental-ims] c522799 Misleading use of key= instead of key:
	which is correct.
Message-ID: <E1RkOFZ-0007ye-Lc@project.varnish-software.com>

commit c522799663d9fba682a74dde6736c7dbe4286f74
Author: Lasse Karstensen <lasse at varnish-software.com>
Date:   Wed Dec 28 15:45:36 2011 +0100

    Misleading use of key= instead of key: which is correct.

diff --git a/doc/sphinx/reference/varnishncsa.rst b/doc/sphinx/reference/varnishncsa.rst
index 8499d29..91622d8 100644
--- a/doc/sphinx/reference/varnishncsa.rst
+++ b/doc/sphinx/reference/varnishncsa.rst
@@ -120,7 +120,7 @@ The following options are available:
 		     cache hit, miss, pass, pipe or error.
 	
 		   VCL_Log:key
-		     Output value set by std.log("key=value") in VCL.
+		     Output value set by std.log("key:value") in VCL.
 		     
 
 -m tag:regex only list records where tag matches regex. Multiple



From geoff at varnish-cache.org  Mon Jan  9 23:03:32 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Tue, 10 Jan 2012 00:03:32 +0100
Subject: [experimental-ims] 110b8e7 Use WRK_BgThread() instead of hand-rolling
Message-ID: <E1RkOFY-0007wg-Hi@project.varnish-software.com>

commit 110b8e719bdf062b449faf4b79dd360d4fba82c2
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Dec 26 09:12:08 2011 +0000

    Use WRK_BgThread() instead of hand-rolling

diff --git a/bin/varnishd/hash/hash_critbit.c b/bin/varnishd/hash/hash_critbit.c
index 3b55490..5fafac5 100644
--- a/bin/varnishd/hash/hash_critbit.c
+++ b/bin/varnishd/hash/hash_critbit.c
@@ -344,17 +344,13 @@ static struct cli_proto hcb_cmds[] = {
 
 /*--------------------------------------------------------------------*/
 
-static void *
-hcb_cleaner(void *priv)
+static void * __match_proto__(bgthread_t)
+hcb_cleaner(struct sess *sp, void *priv)
 {
 	struct hcb_y *y, *y2;
-	struct worker wrk;
+	struct worker *wrk = sp->wrk;
 	struct objhead *oh, *oh2;
 
-	memset(&wrk, 0, sizeof wrk);
-	wrk.magic = WORKER_MAGIC;
-
-	THR_SetName("hcb_cleaner");
 	(void)priv;
 	while (1) {
 		VSTAILQ_FOREACH_SAFE(y, &dead_y, list, y2) {
@@ -363,13 +359,13 @@ hcb_cleaner(void *priv)
 		}
 		VTAILQ_FOREACH_SAFE(oh, &dead_h, hoh_list, oh2) {
 			VTAILQ_REMOVE(&dead_h, oh, hoh_list);
-			HSH_DeleteObjHead(&wrk, oh);
+			HSH_DeleteObjHead(wrk, oh);
 		}
 		Lck_Lock(&hcb_mtx);
 		VSTAILQ_CONCAT(&dead_y, &cool_y);
 		VTAILQ_CONCAT(&dead_h, &cool_h, hoh_list);
 		Lck_Unlock(&hcb_mtx);
-		WRK_SumStat(&wrk);
+		WRK_SumStat(wrk);
 		VTIM_sleep(cache_param->critbit_cooloff);
 	}
 	NEEDLESS_RETURN(NULL);
@@ -386,8 +382,7 @@ hcb_start(void)
 	(void)oh;
 	CLI_AddFuncs(hcb_cmds);
 	Lck_New(&hcb_mtx, lck_hcb);
-	/* XXX: use WRK_BgThread */
-	AZ(pthread_create(&tp, NULL, hcb_cleaner, NULL));
+	WRK_BgThread(&tp, "hcb-cleaner", hcb_cleaner, NULL);
 	memset(&hcb_root, 0, sizeof hcb_root);
 	hcb_build_bittbl();
 }



From geoff at varnish-cache.org  Mon Jan  9 23:03:30 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Tue, 10 Jan 2012 00:03:30 +0100
Subject: [experimental-ims] 23b1f97 Add an empty struct req,
	and start managing it with a mempool per worker pool.
Message-ID: <E1RkOFW-0007sp-1z@project.varnish-software.com>

commit 23b1f97765ba6dc44c688187747c77b772f966c7
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu Dec 22 20:46:12 2011 +0000

    Add an empty struct req, and start managing it with a mempool per
    worker pool.

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index d3b34b7..bcf4f54 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -576,13 +576,24 @@ struct object {
 
 };
 
-/* -------------------------------------------------------------------*/
+/*--------------------------------------------------------------------*/
+
+struct req {
+	unsigned		magic;
+#define REQ_MAGIC		0x2751aaa1
+};
+
+/*--------------------------------------------------------------------*/
 
 struct sess {
 	unsigned		magic;
 #define SESS_MAGIC		0x2c2f9c5a
 
+	/* Cross references ------------------------------------------*/
+
 	struct worker		*wrk;
+	struct req		*req;
+	struct sessmem		*mem;
 
 	/* Session related fields ------------------------------------*/
 
@@ -600,9 +611,6 @@ struct sess {
 	char			port[PORT_BUFSIZE];
 	char			*client_identity;
 
-	/* Various internal stuff */
-	struct sessmem		*mem;
-
 	VTAILQ_ENTRY(sess)	poollist;
 	struct acct		acct_ses;
 
@@ -871,6 +879,7 @@ int Lck_CondWait(pthread_cond_t *cond, struct lock *lck, struct timespec *ts);
 #undef LOCK
 
 /* cache_mempool.c */
+void MPL_AssertSane(void *item);
 struct mempool * MPL_New(const char *name, volatile struct poolparam *pp,
     volatile unsigned *cur_size);
 void MPL_Destroy(struct mempool **mpp);
@@ -908,7 +917,7 @@ struct sess *SES_Alloc(void);
 void SES_Close(struct sess *sp, const char *reason);
 void SES_Delete(struct sess *sp, const char *reason);
 void SES_Charge(struct sess *sp);
-struct sesspool *SES_NewPool(struct pool *pp);
+struct sesspool *SES_NewPool(struct pool *pp, unsigned pool_no);
 void SES_DeletePool(struct sesspool *sp, struct worker *wrk);
 int SES_Schedule(struct sess *sp);
 void SES_Handle(struct sess *sp, int status);
diff --git a/bin/varnishd/cache/cache_acceptor.c b/bin/varnishd/cache/cache_acceptor.c
index a9b8c0b..87e51a9 100644
--- a/bin/varnishd/cache/cache_acceptor.c
+++ b/bin/varnishd/cache/cache_acceptor.c
@@ -282,7 +282,6 @@ VCA_SetupSess(struct worker *w)
 	assert(wa->acceptaddrlen <= sp->sockaddrlen);
 	memcpy(&sp->sockaddr, &wa->acceptaddr, wa->acceptaddrlen);
 	sp->sockaddrlen = wa->acceptaddrlen;
-	sp->step = STP_FIRST;
 	vca_pace_good();
 	w->stats.sess_conn++;
 	vca_prep(sp);
diff --git a/bin/varnishd/cache/cache_center.c b/bin/varnishd/cache/cache_center.c
index e7c7fe2..333db31 100644
--- a/bin/varnishd/cache/cache_center.c
+++ b/bin/varnishd/cache/cache_center.c
@@ -1665,6 +1665,8 @@ CNT_Session(struct sess *sp)
 	struct worker *wrk;
 
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+	CHECK_OBJ_NOTNULL(sp->req, REQ_MAGIC);
+	MPL_AssertSane(sp->req);
 	wrk = sp->wrk;
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 
@@ -1702,6 +1704,8 @@ CNT_Session(struct sess *sp)
 	 */
 	for (done = 0; !done; ) {
 		assert(sp->wrk == wrk);
+		CHECK_OBJ_NOTNULL(sp->req, REQ_MAGIC);
+		MPL_AssertSane(sp->req);
 		/*
 		 * This is a good place to be paranoid about the various
 		 * pointers still pointing to the things we expect.
diff --git a/bin/varnishd/cache/cache_mempool.c b/bin/varnishd/cache/cache_mempool.c
index a89f1e7..6f556f2 100644
--- a/bin/varnishd/cache/cache_mempool.c
+++ b/bin/varnishd/cache/cache_mempool.c
@@ -153,7 +153,7 @@ mpl_guard(void *priv)
 						VTAILQ_REMOVE(&mpl->surplus,
 						    mi, list);
 				}
-				if (mi == NULL) 
+				if (mi == NULL)
 					break;
 				FREE_OBJ(mi);
 				mi = NULL;
@@ -305,6 +305,7 @@ MPL_Get(struct mempool *mpl, unsigned *size)
 	if (size != NULL)
 		*size = mi->size;
 
+	CHECK_OBJ_NOTNULL(mi, MEMITEM_MAGIC);
 	/* Throw away sizeof info for FlexeLint: */
 	return ((void*)(uintptr_t)&mi->payload);
 }
@@ -315,6 +316,7 @@ MPL_Free(struct mempool *mpl, void *item)
 	struct memitem *mi;
 
 	CHECK_OBJ_NOTNULL(mpl, MEMPOOL_MAGIC);
+	AN(item);
 
 	mi = (void*)((uintptr_t)item - offsetof(struct memitem, payload));
 	CHECK_OBJ_NOTNULL(mi, MEMITEM_MAGIC);
@@ -339,3 +341,11 @@ MPL_Free(struct mempool *mpl, void *item)
 
 	Lck_Unlock(&mpl->mtx);
 }
+
+void
+MPL_AssertSane(void *item)
+{
+	struct memitem *mi;
+	mi = (void*)((uintptr_t)item - offsetof(struct memitem, payload));
+	CHECK_OBJ_NOTNULL(mi, MEMITEM_MAGIC);
+}
diff --git a/bin/varnishd/cache/cache_pool.c b/bin/varnishd/cache/cache_pool.c
index c557edd..ee71572 100644
--- a/bin/varnishd/cache/cache_pool.c
+++ b/bin/varnishd/cache/cache_pool.c
@@ -244,6 +244,7 @@ Pool_Work_Thread(void *priv, struct worker *w)
 				w->do_what = pool_do_nothing;
 			} else {
 				VCA_SetupSess(w);
+				w->sp->step = STP_FIRST;
 				w->do_what = pool_do_sess;
 			}
 			WS_Release(w->ws, 0);
@@ -490,7 +491,7 @@ pool_herder(void *priv)
  */
 
 static struct pool *
-pool_mkpool(void)
+pool_mkpool(unsigned pool_no)
 {
 	struct pool *pp;
 	struct listen_sock *ls;
@@ -504,7 +505,7 @@ pool_mkpool(void)
 	VTAILQ_INIT(&pp->queue);
 	VTAILQ_INIT(&pp->idle);
 	VTAILQ_INIT(&pp->socks);
-	pp->sesspool = SES_NewPool(pp);
+	pp->sesspool = SES_NewPool(pp, pool_no);
 	AN(pp->sesspool);
 
 	VTAILQ_FOREACH(ls, &heritage.socks, list) {
@@ -545,7 +546,7 @@ pool_poolherder(void *priv)
 	nwq = 0;
 	while (1) {
 		if (nwq < cache_param->wthread_pools) {
-			pp = pool_mkpool();
+			pp = pool_mkpool(nwq);
 			if (pp != NULL) {
 				VTAILQ_INSERT_TAIL(&pools, pp, list);
 				VSC_C_main->pools++;
diff --git a/bin/varnishd/cache/cache_session.c b/bin/varnishd/cache/cache_session.c
index 65dd5c7..790ea2c 100644
--- a/bin/varnishd/cache/cache_session.c
+++ b/bin/varnishd/cache/cache_session.c
@@ -36,6 +36,7 @@
 #include "config.h"
 
 #include <math.h>
+#include <stdio.h>
 #include <stdlib.h>
 
 #include "cache.h"
@@ -67,6 +68,8 @@ struct sesspool {
 	struct lock		mtx;
 	unsigned		nsess;
 	unsigned		dly_free_cnt;
+	unsigned		req_size;
+	struct mempool		*mpl_req;
 };
 
 /*--------------------------------------------------------------------
@@ -214,6 +217,9 @@ SES_New(struct worker *wrk, struct sesspool *pp)
 	if (sm == NULL)
 		return (NULL);
 	sp = &sm->sess;
+	sp->req = MPL_Get(pp->mpl_req, NULL);
+	AN(sp->req);
+	sp->req->magic = REQ_MAGIC;
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 	return (sp);
 }
@@ -233,6 +239,7 @@ SES_Alloc(void)
 	ses_setup(sm);
 	sp = &sm->sess;
 	sp->sockaddrlen = 0;
+	/* XXX: sp->req ? */
 	return (sp);
 }
 
@@ -326,6 +333,8 @@ SES_Delete(struct sess *sp, const char *reason)
 
 
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+	CHECK_OBJ_NOTNULL(sp->req, REQ_MAGIC);
+	MPL_AssertSane(sp->req);
 	sm = sp->mem;
 	CHECK_OBJ_NOTNULL(sm, SESSMEM_MAGIC);
 	pp = sm->pool;
@@ -353,6 +362,11 @@ SES_Delete(struct sess *sp, const char *reason)
 	    b->sess, b->req, b->pipe, b->pass,
 	    b->fetch, b->hdrbytes, b->bodybytes);
 
+	CHECK_OBJ_NOTNULL(sp->req, REQ_MAGIC);
+	MPL_AssertSane(sp->req);
+	MPL_Free(pp->mpl_req, sp->req);
+	sp->req = NULL;
+
 	if (sm->workspace != cache_param->sess_workspace ||
 	    sm->nhttp != (uint16_t)cache_param->http_max_hdr ||
 	    pp->nsess > cache_param->max_sess) {
@@ -382,15 +396,19 @@ SES_Delete(struct sess *sp, const char *reason)
  */
 
 struct sesspool *
-SES_NewPool(struct pool *pp)
+SES_NewPool(struct pool *pp, unsigned pool_no)
 {
 	struct sesspool *sp;
+	char nb[8];
 
 	ALLOC_OBJ(sp, SESSPOOL_MAGIC);
 	AN(sp);
 	sp->pool = pp;
 	VTAILQ_INIT(&sp->freelist);
 	Lck_New(&sp->mtx, lck_sessmem);
+	bprintf(nb, "req%u", pool_no);
+	sp->req_size = sizeof (struct req);
+	sp->mpl_req = MPL_New(nb, &cache_param->req_pool, &sp->req_size);
 	return (sp);
 }
 
@@ -413,5 +431,6 @@ SES_DeletePool(struct sesspool *sp, struct worker *wrk)
 	AZ(sp->nsess);
 	Lck_Unlock(&sp->mtx);
 	Lck_Delete(&sp->mtx);
+	MPL_Destroy(&sp->mpl_req);
 	FREE_OBJ(sp);
 }
diff --git a/bin/varnishd/common/params.h b/bin/varnishd/common/params.h
index 8198a78..a024452 100644
--- a/bin/varnishd/common/params.h
+++ b/bin/varnishd/common/params.h
@@ -202,4 +202,5 @@ struct params {
 	ssize_t			vsl_space;
 
 	struct poolparam	vbc_pool;
+	struct poolparam	req_pool;
 };
diff --git a/bin/varnishd/mgt/mgt_param.c b/bin/varnishd/mgt/mgt_param.c
index f7f1dd0..c236323 100644
--- a/bin/varnishd/mgt/mgt_param.c
+++ b/bin/varnishd/mgt/mgt_param.c
@@ -1197,7 +1197,7 @@ static const struct parspec input_parspec[] = {
 		0,
 		"true", ""},
 
-	{ "vbc_pool", tweak_poolparam, &mgt_param.vbc_pool, 0, 10000,
+	{ "pool_vbc", tweak_poolparam, &mgt_param.vbc_pool, 0, 10000,
 		"Parameters for backend connection memory pool.\n"
 		"The three numbers are:\n"
 		"   min_pool -- minimum size of free pool.\n"
@@ -1206,6 +1206,15 @@ static const struct parspec input_parspec[] = {
 		0,
 		"10,100,10", ""},
 
+	{ "pool_req", tweak_poolparam, &mgt_param.req_pool, 0, 10000,
+		"Parameters for per worker pool request memory pool.\n"
+		"The three numbers are:\n"
+		"   min_pool -- minimum size of free pool.\n"
+		"   max_pool -- maximum size of free pool.\n"
+		"   max_age -- max age of free element.\n",
+		0,
+		"10,100,10", ""},
+
 	{ NULL, NULL, NULL }
 };
 
diff --git a/bin/varnishtest/tests/b00034.vtc b/bin/varnishtest/tests/b00034.vtc
index 3579fcb..8e23d60 100644
--- a/bin/varnishtest/tests/b00034.vtc
+++ b/bin/varnishtest/tests/b00034.vtc
@@ -5,9 +5,9 @@ server s1 {
 
 varnish v1 -vcl+backend {}
 
-varnish v1 -cliok "param.set vbc_pool 1,10,1"
-varnish v1 -clierr 106 "param.set vbc_pool 10"
-varnish v1 -clierr 106 "param.set vbc_pool 10,1,1"
-varnish v1 -clierr 106 "param.set vbc_pool a,10,10"
-varnish v1 -clierr 106 "param.set vbc_pool 10,a,10"
-varnish v1 -clierr 106 "param.set vbc_pool 10,10,a"
+varnish v1 -cliok "param.set pool_vbc 1,10,1"
+varnish v1 -clierr 106 "param.set pool_vbc 10"
+varnish v1 -clierr 106 "param.set pool_vbc 10,1,1"
+varnish v1 -clierr 106 "param.set pool_vbc a,10,10"
+varnish v1 -clierr 106 "param.set pool_vbc 10,a,10"
+varnish v1 -clierr 106 "param.set pool_vbc 10,10,a"
diff --git a/bin/varnishtest/tests/c00050.vtc b/bin/varnishtest/tests/c00050.vtc
index 340563b..04d8f2c 100644
--- a/bin/varnishtest/tests/c00050.vtc
+++ b/bin/varnishtest/tests/c00050.vtc
@@ -8,20 +8,20 @@ delay 1
 
 varnish v1 -expect MEMPOOL.vbc.pool == 10
 
-varnish v1 -cliok "param.set vbc_pool 90,100,100"
+varnish v1 -cliok "param.set pool_vbc 90,100,100"
 
 delay 1
 
 varnish v1 -expect MEMPOOL.vbc.pool == 90
 
-varnish v1 -cliok "param.set vbc_pool 50,80,100"
+varnish v1 -cliok "param.set pool_vbc 50,80,100"
 
 delay 1
 
 varnish v1 -expect MEMPOOL.vbc.pool == 80
 varnish v1 -expect MEMPOOL.vbc.surplus == 10
 
-varnish v1 -cliok "param.set vbc_pool 10,80,1"
+varnish v1 -cliok "param.set pool_vbc 10,80,1"
 
 delay 1
 



From geoff at varnish-cache.org  Mon Jan  9 23:03:29 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Tue, 10 Jan 2012 00:03:29 +0100
Subject: [experimental-ims] 8313da8 Another typo
Message-ID: <E1RkOFV-0007sB-J6@project.varnish-software.com>

commit 8313da8d5db143038e72157493cd33c95da2e77a
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Dec 20 20:30:28 2011 +0000

    Another typo

diff --git a/doc/sphinx/phk/thetoolsweworkwith.rst b/doc/sphinx/phk/thetoolsweworkwith.rst
index 6921926..2631c73 100644
--- a/doc/sphinx/phk/thetoolsweworkwith.rst
+++ b/doc/sphinx/phk/thetoolsweworkwith.rst
@@ -200,7 +200,7 @@ To put an element on a linked list.
 I could go on like this, but it would rapidly become boring for
 both you and me, because the current C1X draft is 701 pages, and
 it contains not a single explanatory example if how to use any of
-the verbiage in pratice.
+the verbiage in practice.
 
 Compare this with The C Programming Language, a book of 274 pages
 which in addition to define the C language, taught people how to



From geoff at varnish-cache.org  Mon Jan  9 23:03:31 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Tue, 10 Jan 2012 00:03:31 +0100
Subject: [experimental-ims] 097e646 Move another 144 bytes from sess to req
Message-ID: <E1RkOFX-0007ue-DI@project.varnish-software.com>

commit 097e646a65fc157751f93bbd24005eaa1e34d160
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Fri Dec 23 12:15:35 2011 +0000

    Move another 144 bytes from sess to req

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index f45235b..1b2f1a7 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -582,6 +582,29 @@ struct req {
 #define REQ_MAGIC		0x2751aaa1
 
 	unsigned		xid;
+	int			restarts;
+	int			esi_level;
+	int			disable_esi;
+	uint8_t			hash_ignore_busy;
+	uint8_t			hash_always_miss;
+
+	/* The busy objhead we sleep on */
+	struct objhead		*hash_objhead;
+
+	/* Built Vary string */
+	uint8_t			*vary_b;
+	uint8_t			*vary_l;
+	uint8_t			*vary_e;
+
+	unsigned char		digest[DIGEST_LEN];
+
+	const char		*doclose;
+	struct exp		exp;
+	unsigned		cur_method;
+	unsigned		handling;
+	unsigned char		sendbody;
+	unsigned char		wantbody;
+
 };
 
 /*--------------------------------------------------------------------*/
@@ -624,46 +647,20 @@ struct sess {
 #if defined(HAVE_EPOLL_CTL)
 	struct epoll_event ev;
 #endif
+	enum step		step;
 
 	/* Request related fields ------------------------------------*/
 
-
-	int			restarts;
-	int			esi_level;
-	int			disable_esi;
-
-	uint8_t			hash_ignore_busy;
-	uint8_t			hash_always_miss;
-
 	/* HTTP request */
-	const char		*doclose;
 	struct http		*http;
 	struct http		*http0;
 
 	struct ws		ws[1];
 	char			*ws_req;	/* WS above request data */
 
-	unsigned char		digest[DIGEST_LEN];
-
-	/* Built Vary string */
-	uint8_t			*vary_b;
-	uint8_t			*vary_l;
-	uint8_t			*vary_e;
-
-	/* Acceptable grace period */
-	struct exp		exp;
-
-	enum step		step;
-	unsigned		cur_method;
-	unsigned		handling;
-	unsigned char		sendbody;
-	unsigned char		wantbody;
 	uint16_t		err_code;
 	const char		*err_reason;
 
-	/* The busy objhead we sleep on */
-	struct objhead		*hash_objhead;
-
 	struct director		*director;
 	struct VCL_conf		*vcl;
 
diff --git a/bin/varnishd/cache/cache_center.c b/bin/varnishd/cache/cache_center.c
index 1de6ed7..cb5d161 100644
--- a/bin/varnishd/cache/cache_center.c
+++ b/bin/varnishd/cache/cache_center.c
@@ -112,7 +112,7 @@ cnt_wait(struct sess *sp)
 
 	AZ(sp->vcl);
 	AZ(wrk->obj);
-	AZ(sp->esi_level);
+	AZ(sp->req->esi_level);
 	assert(sp->req->xid == 0);
 
 	assert(!isnan(sp->t_req));
@@ -229,13 +229,13 @@ cnt_prepresp(struct sess *sp)
 	    !wrk->busyobj->do_gzip && !wrk->busyobj->do_gunzip)
 		wrk->res_mode |= RES_LEN;
 
-	if (!sp->disable_esi && wrk->obj->esidata != NULL) {
+	if (!sp->req->disable_esi && wrk->obj->esidata != NULL) {
 		/* In ESI mode, we don't know the aggregate length */
 		wrk->res_mode &= ~RES_LEN;
 		wrk->res_mode |= RES_ESI;
 	}
 
-	if (sp->esi_level > 0) {
+	if (sp->req->esi_level > 0) {
 		wrk->res_mode &= ~RES_LEN;
 		wrk->res_mode |= RES_ESI_CHILD;
 	}
@@ -258,13 +258,13 @@ cnt_prepresp(struct sess *sp)
 			 * can make it any different size
 			 */
 			wrk->res_mode |= RES_LEN;
-		else if (!sp->wantbody) {
+		else if (!sp->req->wantbody) {
 			/* Nothing */
 		} else if (sp->http->protover >= 11) {
 			wrk->res_mode |= RES_CHUNKED;
 		} else {
 			wrk->res_mode |= RES_EOF;
-			sp->doclose = "EOF mode";
+			sp->req->doclose = "EOF mode";
 		}
 	}
 
@@ -279,11 +279,11 @@ cnt_prepresp(struct sess *sp)
 	http_Setup(wrk->resp, wrk->ws);
 	RES_BuildHttp(sp);
 	VCL_deliver_method(sp);
-	switch (sp->handling) {
+	switch (sp->req->handling) {
 	case VCL_RET_DELIVER:
 		break;
 	case VCL_RET_RESTART:
-		if (sp->restarts >= cache_param->max_restarts)
+		if (sp->req->restarts >= cache_param->max_restarts)
 			break;
 		if (wrk->busyobj != NULL) {
 			AN(wrk->busyobj->do_stream);
@@ -294,7 +294,7 @@ cnt_prepresp(struct sess *sp)
 			(void)HSH_Deref(wrk, NULL, &wrk->obj);
 		}
 		AZ(wrk->obj);
-		sp->restarts++;
+		sp->req->restarts++;
 		sp->director = NULL;
 		http_Setup(wrk->resp, NULL);
 		sp->step = STP_RECV;
@@ -337,7 +337,7 @@ cnt_deliver(struct sess *sp)
 
 	AZ(sp->wrk->busyobj);
 	sp->director = NULL;
-	sp->restarts = 0;
+	sp->req->restarts = 0;
 
 	RES_WriteObj(sp);
 
@@ -378,14 +378,14 @@ cnt_done(struct sess *sp)
 	AZ(wrk->obj);
 	AZ(wrk->busyobj);
 	sp->director = NULL;
-	sp->restarts = 0;
+	sp->req->restarts = 0;
 
 	wrk->busyobj = NULL;
 
 	SES_Charge(sp);
 
 	/* If we did an ESI include, don't mess up our state */
-	if (sp->esi_level > 0)
+	if (sp->req->esi_level > 0)
 		return (1);
 
 	if (sp->vcl != NULL) {
@@ -420,16 +420,16 @@ cnt_done(struct sess *sp)
 
 	sp->req_bodybytes = 0;
 
-	sp->hash_always_miss = 0;
-	sp->hash_ignore_busy = 0;
+	sp->req->hash_always_miss = 0;
+	sp->req->hash_ignore_busy = 0;
 
-	if (sp->fd >= 0 && sp->doclose != NULL) {
+	if (sp->fd >= 0 && sp->req->doclose != NULL) {
 		/*
 		 * This is an orderly close of the connection; ditch nolinger
 		 * before we close, to get queued data transmitted.
 		 */
 		// XXX: not yet (void)VTCP_linger(sp->fd, 0);
-		SES_Close(sp, sp->doclose);
+		SES_Close(sp, sp->req->doclose);
 	}
 
 	if (sp->fd < 0) {
@@ -495,7 +495,7 @@ cnt_error(struct sess *sp)
 			    cache_param->http_resp_size,
 			    (uint16_t)cache_param->http_max_hdr);
 		if (wrk->obj == NULL) {
-			sp->doclose = "Out of objects";
+			sp->req->doclose = "Out of objects";
 			sp->director = NULL;
 			http_Setup(wrk->busyobj->beresp, NULL);
 			http_Setup(wrk->busyobj->bereq, NULL);
@@ -528,23 +528,23 @@ cnt_error(struct sess *sp)
 		    http_StatusMessage(sp->err_code));
 	VCL_error_method(sp);
 
-	if (sp->handling == VCL_RET_RESTART &&
-	    sp->restarts <  cache_param->max_restarts) {
+	if (sp->req->handling == VCL_RET_RESTART &&
+	    sp->req->restarts <  cache_param->max_restarts) {
 		HSH_Drop(wrk);
 		VBO_DerefBusyObj(wrk, &wrk->busyobj);
 		sp->director = NULL;
-		sp->restarts++;
+		sp->req->restarts++;
 		sp->step = STP_RECV;
 		return (0);
-	} else if (sp->handling == VCL_RET_RESTART)
-		sp->handling = VCL_RET_DELIVER;
+	} else if (sp->req->handling == VCL_RET_RESTART)
+		sp->req->handling = VCL_RET_DELIVER;
 
 
 	/* We always close when we take this path */
-	sp->doclose = "error";
-	sp->wantbody = 1;
+	sp->req->doclose = "error";
+	sp->req->wantbody = 1;
 
-	assert(sp->handling == VCL_RET_DELIVER);
+	assert(sp->req->handling == VCL_RET_DELIVER);
 	sp->err_code = 0;
 	sp->err_reason = NULL;
 	http_Setup(wrk->busyobj->bereq, NULL);
@@ -616,7 +616,7 @@ cnt_fetch(struct sess *sp)
 	}
 
 	if (i) {
-		sp->handling = VCL_RET_ERROR;
+		sp->req->handling = VCL_RET_ERROR;
 		sp->err_code = 503;
 	} else {
 		/*
@@ -651,7 +651,7 @@ cnt_fetch(struct sess *sp)
 
 		VCL_fetch_method(sp);
 
-		switch (sp->handling) {
+		switch (sp->req->handling) {
 		case VCL_RET_HIT_FOR_PASS:
 			if (wrk->objcore != NULL)
 				wrk->objcore->flags |= OC_F_PASS;
@@ -681,9 +681,9 @@ cnt_fetch(struct sess *sp)
 	sp->director = NULL;
 	wrk->storage_hint = NULL;
 
-	switch (sp->handling) {
+	switch (sp->req->handling) {
 	case VCL_RET_RESTART:
-		sp->restarts++;
+		sp->req->restarts++;
 		sp->step = STP_RECV;
 		return (0);
 	case VCL_RET_ERROR:
@@ -732,15 +732,15 @@ cnt_fetchbody(struct sess *sp)
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 	CHECK_OBJ_NOTNULL(wrk->busyobj, BUSYOBJ_MAGIC);
 
-	assert(sp->handling == VCL_RET_HIT_FOR_PASS ||
-	    sp->handling == VCL_RET_DELIVER);
+	assert(sp->req->handling == VCL_RET_HIT_FOR_PASS ||
+	    sp->req->handling == VCL_RET_DELIVER);
 
 	if (wrk->objcore == NULL) {
 		/* This is a pass from vcl_recv */
 		pass = 1;
 		/* VCL may have fiddled this, but that doesn't help */
 		wrk->busyobj->exp.ttl = -1.;
-	} else if (sp->handling == VCL_RET_HIT_FOR_PASS) {
+	} else if (sp->req->handling == VCL_RET_HIT_FOR_PASS) {
 		/* pass from vcl_fetch{} -> hit-for-pass */
 		/* XXX: the bereq was not filtered pass... */
 		pass = 1;
@@ -805,9 +805,9 @@ cnt_fetchbody(struct sess *sp)
 	else if (wrk->busyobj->is_gzip)
 		wrk->busyobj->vfp = &vfp_testgzip;
 
-	if (wrk->busyobj->do_esi || sp->esi_level > 0)
+	if (wrk->busyobj->do_esi || sp->req->esi_level > 0)
 		wrk->busyobj->do_stream = 0;
-	if (!sp->wantbody)
+	if (!sp->req->wantbody)
 		wrk->busyobj->do_stream = 0;
 
 	l = http_EstimateWS(wrk->busyobj->beresp,
@@ -992,11 +992,11 @@ cnt_streambody(struct sess *sp)
 		AN(wrk->obj->objcore->ban);
 		HSH_Unbusy(wrk);
 	} else {
-		sp->doclose = "Stream error";
+		sp->req->doclose = "Stream error";
 	}
 	wrk->acct_tmp.fetch++;
 	sp->director = NULL;
-	sp->restarts = 0;
+	sp->req->restarts = 0;
 
 	RES_StreamEnd(sp);
 	if (wrk->res_mode & RES_GUNZIP)
@@ -1036,8 +1036,6 @@ cnt_first(struct sess *sp)
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 
 	AZ(sp->req);
-	assert(sp->restarts == 0);
-	AZ(sp->esi_level);
 
 	VTCP_name(&sp->sockaddr, sp->sockaddrlen,
 	    sp->addr, sizeof sp->addr, sp->port, sizeof sp->port);
@@ -1098,7 +1096,7 @@ cnt_hit(struct sess *sp)
 
 	VCL_hit_method(sp);
 
-	if (sp->handling == VCL_RET_DELIVER) {
+	if (sp->req->handling == VCL_RET_DELIVER) {
 		/* Dispose of any body part of the request */
 		(void)FetchReqBody(sp);
 		//AZ(wrk->busyobj->bereq->ws);
@@ -1111,7 +1109,7 @@ cnt_hit(struct sess *sp)
 	(void)HSH_Deref(wrk, NULL, &wrk->obj);
 	wrk->objcore = NULL;
 
-	switch(sp->handling) {
+	switch(sp->req->handling) {
 	case VCL_RET_PASS:
 		sp->step = STP_PASS;
 		return (0);
@@ -1120,7 +1118,7 @@ cnt_hit(struct sess *sp)
 		return (0);
 	case VCL_RET_RESTART:
 		sp->director = NULL;
-		sp->restarts++;
+		sp->req->restarts++;
 		sp->step = STP_RECV;
 		return (0);
 	default:
@@ -1171,18 +1169,18 @@ cnt_lookup(struct sess *sp)
 	CHECK_OBJ_NOTNULL(sp->vcl, VCL_CONF_MAGIC);
 	AZ(wrk->busyobj);
 
-	if (sp->hash_objhead == NULL) {
+	if (sp->req->hash_objhead == NULL) {
 		/* Not a waiting list return */
-		AZ(sp->vary_b);
-		AZ(sp->vary_l);
-		AZ(sp->vary_e);
+		AZ(sp->req->vary_b);
+		AZ(sp->req->vary_l);
+		AZ(sp->req->vary_e);
 		(void)WS_Reserve(sp->ws, 0);
 	} else {
 		AN(sp->ws->r);
 	}
-	sp->vary_b = (void*)sp->ws->f;
-	sp->vary_e = (void*)sp->ws->r;
-	sp->vary_b[2] = '\0';
+	sp->req->vary_b = (void*)sp->ws->f;
+	sp->req->vary_e = (void*)sp->ws->r;
+	sp->req->vary_b[2] = '\0';
 
 	oc = HSH_Lookup(sp, &oh);
 
@@ -1204,17 +1202,17 @@ cnt_lookup(struct sess *sp)
 	if (oc->flags & OC_F_BUSY) {
 		wrk->stats.cache_miss++;
 
-		if (sp->vary_l != NULL) {
-			assert(oc->busyobj->vary == sp->vary_b);
+		if (sp->req->vary_l != NULL) {
+			assert(oc->busyobj->vary == sp->req->vary_b);
 			VRY_Validate(oc->busyobj->vary);
-			WS_ReleaseP(sp->ws, (void*)sp->vary_l);
+			WS_ReleaseP(sp->ws, (void*)sp->req->vary_l);
 		} else {
 			AZ(oc->busyobj->vary);
 			WS_Release(sp->ws, 0);
 		}
-		sp->vary_b = NULL;
-		sp->vary_l = NULL;
-		sp->vary_e = NULL;
+		sp->req->vary_b = NULL;
+		sp->req->vary_l = NULL;
+		sp->req->vary_e = NULL;
 
 		wrk->objcore = oc;
 		CHECK_OBJ_NOTNULL(wrk->busyobj, BUSYOBJ_MAGIC);
@@ -1227,9 +1225,9 @@ cnt_lookup(struct sess *sp)
 	wrk->obj = o;
 
 	WS_Release(sp->ws, 0);
-	sp->vary_b = NULL;
-	sp->vary_l = NULL;
-	sp->vary_e = NULL;
+	sp->req->vary_b = NULL;
+	sp->req->vary_l = NULL;
+	sp->req->vary_e = NULL;
 
 	if (oc->flags & OC_F_PASS) {
 		wrk->stats.cache_hitpass++;
@@ -1303,7 +1301,7 @@ cnt_miss(struct sess *sp)
 
 	VCL_miss_method(sp);
 
-	switch(sp->handling) {
+	switch(sp->req->handling) {
 	case VCL_RET_ERROR:
 		AZ(HSH_Deref(wrk, wrk->objcore, NULL));
 		wrk->objcore = NULL;
@@ -1385,15 +1383,15 @@ cnt_pass(struct sess *sp)
 	wrk->first_byte_timeout = 0;
 	wrk->between_bytes_timeout = 0;
 	VCL_pass_method(sp);
-	if (sp->handling == VCL_RET_ERROR) {
+	if (sp->req->handling == VCL_RET_ERROR) {
 		http_Setup(wrk->busyobj->bereq, NULL);
 		VBO_DerefBusyObj(wrk, &wrk->busyobj);
 		sp->step = STP_ERROR;
 		return (0);
 	}
-	assert(sp->handling == VCL_RET_PASS);
+	assert(sp->req->handling == VCL_RET_PASS);
 	wrk->acct_tmp.pass++;
-	sp->sendbody = 1;
+	sp->req->sendbody = 1;
 	sp->step = STP_FETCH;
 	return (0);
 }
@@ -1443,9 +1441,9 @@ cnt_pipe(struct sess *sp)
 
 	VCL_pipe_method(sp);
 
-	if (sp->handling == VCL_RET_ERROR)
+	if (sp->req->handling == VCL_RET_ERROR)
 		INCOMPL();
-	assert(sp->handling == VCL_RET_PIPE);
+	assert(sp->req->handling == VCL_RET_PIPE);
 
 	PipeSession(sp);
 	assert(WRW_IsReleased(wrk));
@@ -1494,17 +1492,17 @@ cnt_recv(struct sess *sp)
 	sp->director = sp->vcl->director[0];
 	AN(sp->director);
 
-	sp->disable_esi = 0;
-	sp->hash_always_miss = 0;
-	sp->hash_ignore_busy = 0;
+	sp->req->disable_esi = 0;
+	sp->req->hash_always_miss = 0;
+	sp->req->hash_ignore_busy = 0;
 	sp->client_identity = NULL;
 
 	http_CollectHdr(sp->http, H_Cache_Control);
 
 	VCL_recv_method(sp);
-	recv_handling = sp->handling;
+	recv_handling = sp->req->handling;
 
-	if (sp->restarts >= cache_param->max_restarts) {
+	if (sp->req->restarts >= cache_param->max_restarts) {
 		if (sp->err_code == 0)
 			sp->err_code = 503;
 		sp->step = STP_ERROR;
@@ -1525,22 +1523,22 @@ cnt_recv(struct sess *sp)
 
 	SHA256_Init(wrk->sha256ctx);
 	VCL_hash_method(sp);
-	assert(sp->handling == VCL_RET_HASH);
-	SHA256_Final(sp->digest, wrk->sha256ctx);
+	assert(sp->req->handling == VCL_RET_HASH);
+	SHA256_Final(sp->req->digest, wrk->sha256ctx);
 
 	if (!strcmp(sp->http->hd[HTTP_HDR_REQ].b, "HEAD"))
-		sp->wantbody = 0;
+		sp->req->wantbody = 0;
 	else
-		sp->wantbody = 1;
+		sp->req->wantbody = 1;
 
-	sp->sendbody = 0;
+	sp->req->sendbody = 0;
 	switch(recv_handling) {
 	case VCL_RET_LOOKUP:
 		/* XXX: discard req body, if any */
 		sp->step = STP_LOOKUP;
 		return (0);
 	case VCL_RET_PIPE:
-		if (sp->esi_level > 0) {
+		if (sp->req->esi_level > 0) {
 			/* XXX: VSL something */
 			INCOMPL();
 			/* sp->step = STP_DONE; */
@@ -1584,10 +1582,11 @@ cnt_start(struct sess *sp)
 	wrk = sp->wrk;
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 	CHECK_OBJ_NOTNULL(sp->req, REQ_MAGIC);
-	AZ(sp->restarts);
+	AZ(sp->req->restarts);
 	AZ(wrk->obj);
 	AZ(sp->vcl);
-	AZ(sp->esi_level);
+	EXP_Clr(&sp->req->exp);
+	AZ(sp->req->esi_level);
 
 	/* Update stats of various sorts */
 	wrk->stats.client_req++;
@@ -1625,7 +1624,7 @@ cnt_start(struct sess *sp)
 		return (0);
 	}
 
-	sp->doclose = http_DoConnection(sp->http);
+	sp->req->doclose = http_DoConnection(sp->http);
 
 	/* XXX: Handle TRACE & OPTIONS of Max-Forwards = 0 */
 
diff --git a/bin/varnishd/cache/cache_dir_random.c b/bin/varnishd/cache/cache_dir_random.c
index d6570ed..28a079c 100644
--- a/bin/varnishd/cache/cache_dir_random.c
+++ b/bin/varnishd/cache/cache_dir_random.c
@@ -112,8 +112,8 @@ vdi_random_init_seed(const struct vdi_random *vs, const struct sess *sp)
 		retval = vdi_random_sha(p, strlen(p));
 		break;
 	case c_hash:
-		AN(sp->digest);
-		retval = scalbn(vle32dec(sp->digest), -32);
+		AN(sp->req->digest);
+		retval = scalbn(vle32dec(sp->req->digest), -32);
 		break;
 	case c_random:
 	default:
diff --git a/bin/varnishd/cache/cache_esi_deliver.c b/bin/varnishd/cache/cache_esi_deliver.c
index fb0771d..36ff704 100644
--- a/bin/varnishd/cache/cache_esi_deliver.c
+++ b/bin/varnishd/cache/cache_esi_deliver.c
@@ -52,9 +52,9 @@ ved_include(struct sess *sp, const char *src, const char *host)
 
 	w = sp->wrk;
 
-	if (sp->esi_level >= cache_param->max_esi_depth)
+	if (sp->req->esi_level >= cache_param->max_esi_depth)
 		return;
-	sp->esi_level++;
+	sp->req->esi_level++;
 
 	(void)WRW_FlushRelease(w);
 
@@ -105,7 +105,7 @@ ved_include(struct sess *sp, const char *src, const char *host)
 	sp->req->xid = sxid;
 	AN(sp->wrk);
 	assert(sp->step == STP_DONE);
-	sp->esi_level--;
+	sp->req->esi_level--;
 	sp->wrk->obj = obj;
 	sp->wrk->res_mode = res_mode;
 
@@ -247,7 +247,7 @@ ESI_Deliver(struct sess *sp)
 		isgzip = 0;
 	}
 
-	if (sp->esi_level == 0) {
+	if (sp->req->esi_level == 0) {
 		/*
 		 * Only the top level document gets to decide this.
 		 */
@@ -403,7 +403,7 @@ ESI_Deliver(struct sess *sp)
 			(void)WRW_Write(sp->wrk, obuf, obufl);
 		(void)VGZ_Destroy(&vgz, sp->vsl_id);
 	}
-	if (sp->wrk->gzip_resp && sp->esi_level == 0) {
+	if (sp->wrk->gzip_resp && sp->req->esi_level == 0) {
 		/* Emit a gzip literal block with finish bit set */
 		tailbuf[0] = 0x01;
 		tailbuf[1] = 0x00;
diff --git a/bin/varnishd/cache/cache_expire.c b/bin/varnishd/cache/cache_expire.c
index 761a62e..e93a1aa 100644
--- a/bin/varnishd/cache/cache_expire.c
+++ b/bin/varnishd/cache/cache_expire.c
@@ -116,8 +116,8 @@ EXP_Keep(const struct sess *sp, const struct object *o)
 	r = (double)cache_param->default_keep;
 	if (o->exp.keep > 0.)
 		r = o->exp.keep;
-	if (sp != NULL && sp->exp.keep > 0. && sp->exp.keep < r)
-		r = sp->exp.keep;
+	if (sp != NULL && sp->req->exp.keep > 0. && sp->req->exp.keep < r)
+		r = sp->req->exp.keep;
 	return (EXP_Ttl(sp, o) + r);
 }
 
@@ -129,8 +129,8 @@ EXP_Grace(const struct sess *sp, const struct object *o)
 	r = (double)cache_param->default_grace;
 	if (o->exp.grace >= 0.)
 		r = o->exp.grace;
-	if (sp != NULL && sp->exp.grace > 0. && sp->exp.grace < r)
-		r = sp->exp.grace;
+	if (sp != NULL && sp->req->exp.grace > 0. && sp->req->exp.grace < r)
+		r = sp->req->exp.grace;
 	return (EXP_Ttl(sp, o) + r);
 }
 
@@ -140,8 +140,8 @@ EXP_Ttl(const struct sess *sp, const struct object *o)
 	double r;
 
 	r = o->exp.ttl;
-	if (sp != NULL && sp->exp.ttl > 0. && sp->exp.ttl < r)
-		r = sp->exp.ttl;
+	if (sp != NULL && sp->req->exp.ttl > 0. && sp->req->exp.ttl < r)
+		r = sp->req->exp.ttl;
 	return (o->exp.entered + r);
 }
 
diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c
index 936dc30..79d0665 100644
--- a/bin/varnishd/cache/cache_fetch.c
+++ b/bin/varnishd/cache/cache_fetch.c
@@ -349,7 +349,7 @@ FetchReqBody(struct sess *sp)
 			if (rdcnt <= 0)
 				return (1);
 			content_length -= rdcnt;
-			if (!sp->sendbody)
+			if (!sp->req->sendbody)
 				continue;
 			(void)WRW_Write(sp->wrk, buf, rdcnt); /* XXX: stats ? */
 			if (WRW_Flush(sp->wrk))
diff --git a/bin/varnishd/cache/cache_hash.c b/bin/varnishd/cache/cache_hash.c
index 6f6a85f..627db48 100644
--- a/bin/varnishd/cache/cache_hash.c
+++ b/bin/varnishd/cache/cache_hash.c
@@ -262,7 +262,7 @@ HSH_Insert(const struct sess *sp)
 	if (cache_param->diag_bitmap & 0x80000000)
 		hsh_testmagic(sp->wrk->nobjhead->digest);
 
-	AZ(sp->hash_objhead);
+	AZ(sp->req);
 	AN(wrk->nobjhead);
 	oh = hash->lookup(sp, wrk->nobjhead);
 	CHECK_OBJ_NOTNULL(oh, OBJHEAD_MAGIC);
@@ -307,18 +307,18 @@ HSH_Lookup(struct sess *sp, struct objhead **poh)
 	wrk = sp->wrk;
 
 	HSH_Prealloc(sp);
-	memcpy(sp->wrk->nobjhead->digest, sp->digest, sizeof sp->digest);
+	memcpy(sp->wrk->nobjhead->digest, sp->req->digest, sizeof sp->req->digest);
 	if (cache_param->diag_bitmap & 0x80000000)
 		hsh_testmagic(sp->wrk->nobjhead->digest);
 
-	if (sp->hash_objhead != NULL) {
+	if (sp->req->hash_objhead != NULL) {
 		/*
 		 * This sess came off the waiting list, and brings a
 		 * oh refcnt with it.
 		 */
-		CHECK_OBJ_NOTNULL(sp->hash_objhead, OBJHEAD_MAGIC);
-		oh = sp->hash_objhead;
-		sp->hash_objhead = NULL;
+		CHECK_OBJ_NOTNULL(sp->req->hash_objhead, OBJHEAD_MAGIC);
+		oh = sp->req->hash_objhead;
+		sp->req->hash_objhead = NULL;
 	} else {
 		AN(wrk->nobjhead);
 		oh = hash->lookup(sp, wrk->nobjhead);
@@ -340,7 +340,7 @@ HSH_Lookup(struct sess *sp, struct objhead **poh)
 
 		if (oc->flags & OC_F_BUSY) {
 			CHECK_OBJ_NOTNULL(oc->busyobj, BUSYOBJ_MAGIC);
-			if (sp->hash_ignore_busy)
+			if (sp->req->hash_ignore_busy)
 				continue;
 
 			if (oc->busyobj->vary != NULL &&
@@ -401,7 +401,7 @@ HSH_Lookup(struct sess *sp, struct objhead **poh)
 	}
 	sp->wrk->objcore = NULL;
 
-	if (oc != NULL && !sp->hash_always_miss) {
+	if (oc != NULL && !sp->req->hash_always_miss) {
 		o = oc_getobj(sp->wrk, oc);
 		CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC);
 		assert(oc->objhead == oh);
@@ -419,7 +419,7 @@ HSH_Lookup(struct sess *sp, struct objhead **poh)
 
 	if (busy_oc != NULL) {
 		/* There are one or more busy objects, wait for them */
-		if (sp->esi_level == 0) {
+		if (sp->req->esi_level == 0) {
 			CHECK_OBJ_NOTNULL(sp->wrk->nwaitinglist,
 			    WAITINGLIST_MAGIC);
 			if (oh->waitinglist == NULL) {
@@ -437,7 +437,7 @@ HSH_Lookup(struct sess *sp, struct objhead **poh)
 		 * back when the sess comes off the waiting list and
 		 * calls us again
 		 */
-		sp->hash_objhead = oh;
+		sp->req->hash_objhead = oh;
 		sp->wrk = NULL;
 		Lck_Unlock(&oh->mtx);
 		return (NULL);
@@ -452,9 +452,9 @@ HSH_Lookup(struct sess *sp, struct objhead **poh)
 	AZ(wrk->busyobj);
 	wrk->busyobj = VBO_GetBusyObj(wrk);
 
-	VRY_Validate(sp->vary_b);
-	if (sp->vary_l != NULL)
-		wrk->busyobj->vary = sp->vary_b;
+	VRY_Validate(sp->req->vary_b);
+	if (sp->req->vary_l != NULL)
+		wrk->busyobj->vary = sp->req->vary_b;
 	else
 		wrk->busyobj->vary = NULL;
 	oc->busyobj = wrk->busyobj;
diff --git a/bin/varnishd/cache/cache_panic.c b/bin/varnishd/cache/cache_panic.c
index 238c597..44c8fc3 100644
--- a/bin/varnishd/cache/cache_panic.c
+++ b/bin/varnishd/cache/cache_panic.c
@@ -245,7 +245,7 @@ pan_sess(const struct sess *sp)
 #undef STEP
 		default: stp = NULL;
 	}
-	hand = VCL_Return_Name(sp->handling);
+	hand = VCL_Return_Name(sp->req->handling);
 	if (stp != NULL)
 		VSB_printf(pan_vsp, "  step = %s,\n", stp);
 	else
@@ -253,14 +253,14 @@ pan_sess(const struct sess *sp)
 	if (hand != NULL)
 		VSB_printf(pan_vsp, "  handling = %s,\n", hand);
 	else
-		VSB_printf(pan_vsp, "  handling = 0x%x,\n", sp->handling);
+		VSB_printf(pan_vsp, "  handling = 0x%x,\n", sp->req->handling);
 	if (sp->err_code)
 		VSB_printf(pan_vsp,
 		    "  err_code = %d, err_reason = %s,\n", sp->err_code,
 		    sp->err_reason ? sp->err_reason : "(null)");
 
 	VSB_printf(pan_vsp, "  restarts = %d, esi_level = %d\n",
-	    sp->restarts, sp->esi_level);
+	    sp->req->restarts, sp->req->esi_level);
 
 	if (sp->wrk->busyobj != NULL)
 		pan_busyobj(sp->wrk->busyobj);
diff --git a/bin/varnishd/cache/cache_response.c b/bin/varnishd/cache/cache_response.c
index cbddb7d..e5dbf86 100644
--- a/bin/varnishd/cache/cache_response.c
+++ b/bin/varnishd/cache/cache_response.c
@@ -142,7 +142,7 @@ RES_BuildHttp(const struct sess *sp)
 	    sp->wrk->obj->exp.age + sp->t_resp - sp->wrk->obj->exp.entered);
 	http_SetHeader(sp->wrk, sp->vsl_id, sp->wrk->resp, "Via: 1.1 varnish");
 	http_PrintfHeader(sp->wrk, sp->vsl_id, sp->wrk->resp, "Connection: %s",
-	    sp->doclose ? "close" : "keep-alive");
+	    sp->req->doclose ? "close" : "keep-alive");
 }
 
 /*--------------------------------------------------------------------
@@ -261,7 +261,7 @@ RES_WriteObj(struct sess *sp)
 	if (sp->wrk->obj->response == 200 &&
 	    sp->http->conds &&
 	    RFC2616_Do_Cond(sp)) {
-		sp->wantbody = 0;
+		sp->req->wantbody = 0;
 		http_SetResp(sp->wrk->resp, "HTTP/1.1", 304, "Not Modified");
 		http_Unset(sp->wrk->resp, H_Content_Length);
 		http_Unset(sp->wrk->resp, H_Transfer_Encoding);
@@ -273,7 +273,7 @@ RES_WriteObj(struct sess *sp)
 	low = 0;
 	high = sp->wrk->obj->len - 1;
 	if (
-	    sp->wantbody &&
+	    sp->req->wantbody &&
 	    (sp->wrk->res_mode & RES_LEN) &&
 	    !(sp->wrk->res_mode & (RES_ESI|RES_ESI_CHILD|RES_GUNZIP)) &&
 	    cache_param->http_range_support &&
@@ -294,13 +294,13 @@ RES_WriteObj(struct sess *sp)
 		sp->wrk->acct_tmp.hdrbytes +=
 		    http_Write(sp->wrk, sp->vsl_id, sp->wrk->resp, 1);
 
-	if (!sp->wantbody)
+	if (!sp->req->wantbody)
 		sp->wrk->res_mode &= ~RES_CHUNKED;
 
 	if (sp->wrk->res_mode & RES_CHUNKED)
 		WRW_Chunked(sp->wrk);
 
-	if (!sp->wantbody) {
+	if (!sp->req->wantbody) {
 		/* This was a HEAD or conditional request */
 	} else if (sp->wrk->obj->len == 0) {
 		/* Nothing to do here */
@@ -336,7 +336,7 @@ RES_StreamStart(struct sess *sp)
 	CHECK_OBJ_NOTNULL(sctx, STREAM_CTX_MAGIC);
 
 	AZ(sp->wrk->res_mode & RES_ESI_CHILD);
-	AN(sp->wantbody);
+	AN(sp->req->wantbody);
 
 	WRW_Reserve(sp->wrk, &sp->fd);
 	/*
diff --git a/bin/varnishd/cache/cache_session.c b/bin/varnishd/cache/cache_session.c
index 530c96e..f1f502a 100644
--- a/bin/varnishd/cache/cache_session.c
+++ b/bin/varnishd/cache/cache_session.c
@@ -171,7 +171,6 @@ ses_setup(struct sessmem *sm)
 	sp->t_idle = NAN;
 	sp->t_req = NAN;
 	sp->t_resp = NAN;
-	EXP_Clr(&sp->exp);
 
 	WS_Init(sp->ws, "sess", sm->wsp, sm->workspace);
 	sp->http = sm->http[0];
diff --git a/bin/varnishd/cache/cache_vary.c b/bin/varnishd/cache/cache_vary.c
index f55607b..562f6df 100644
--- a/bin/varnishd/cache/cache_vary.c
+++ b/bin/varnishd/cache/cache_vary.c
@@ -176,7 +176,7 @@ vry_cmp(const uint8_t * const *v1, uint8_t * const *v2)
 int
 VRY_Match(struct sess *sp, const uint8_t *vary)
 {
-	uint8_t *vsp = sp->vary_b;
+	uint8_t *vsp = sp->req->vary_b;
 	char *h, *e;
 	unsigned lh, ln;
 	int i, retval = 1, oflo = 0;
@@ -202,8 +202,8 @@ VRY_Match(struct sess *sp, const uint8_t *vary)
 
 			/* Length of the entire new vary entry */
 			ln = 2 + vary[2] + 2 + (lh == 0xffff ? 0 : lh);
-			if (vsp + ln >= sp->vary_e) {
-				vsp = sp->vary_b;
+			if (vsp + ln >= sp->req->vary_e) {
+				vsp = sp->req->vary_b;
 				oflo = 1;
 			}
 
@@ -211,7 +211,7 @@ VRY_Match(struct sess *sp, const uint8_t *vary)
 			 * We MUST have space for one entry and the end marker
 			 * after it, which prevents old junk from confusing us
 			 */
-			assert(vsp + ln + 2 < sp->vary_e);
+			assert(vsp + ln + 2 < sp->req->vary_e);
 
 			vbe16enc(vsp, (uint16_t)lh);
 			memcpy(vsp + 2, vary + 2, vary[2] + 2);
@@ -229,20 +229,20 @@ VRY_Match(struct sess *sp, const uint8_t *vary)
 		vsp += vry_len(vsp);
 		vary += vry_len(vary);
 	}
-	if (vsp + 3 > sp->vary_e)
+	if (vsp + 3 > sp->req->vary_e)
 		oflo = 1;
 
 	if (oflo) {
 		/* XXX: Should log this */
-		vsp = sp->vary_b;
+		vsp = sp->req->vary_b;
 	}
 	vsp[0] = 0xff;
 	vsp[1] = 0xff;
 	vsp[2] = 0;
 	if (oflo)
-		sp->vary_l = NULL;
+		sp->req->vary_l = NULL;
 	else
-		sp->vary_l = vsp + 3;
+		sp->req->vary_l = vsp + 3;
 	return (retval);
 }
 
diff --git a/bin/varnishd/cache/cache_vcl.c b/bin/varnishd/cache/cache_vcl.c
index 068b482..06d4bd7 100644
--- a/bin/varnishd/cache/cache_vcl.c
+++ b/bin/varnishd/cache/cache_vcl.c
@@ -333,14 +333,14 @@ void									\
 VCL_##func##_method(struct sess *sp)					\
 {									\
 									\
-	sp->handling = 0;						\
-	sp->cur_method = VCL_MET_ ## upper;				\
+	sp->req->handling = 0;						\
+	sp->req->cur_method = VCL_MET_ ## upper;				\
 	WSP(sp, SLT_VCL_call, "%s", #func);				\
 	(void)sp->vcl->func##_func(sp);					\
-	WSP(sp, SLT_VCL_return, "%s", VCL_Return_Name(sp->handling));	\
-	sp->cur_method = 0;						\
-	assert((1U << sp->handling) & bitmap);				\
-	assert(!((1U << sp->handling) & ~bitmap));			\
+	WSP(sp, SLT_VCL_return, "%s", VCL_Return_Name(sp->req->handling));	\
+	sp->req->cur_method = 0;						\
+	assert((1U << sp->req->handling) & bitmap);				\
+	assert(!((1U << sp->req->handling) & ~bitmap));			\
 }
 
 #include "tbl/vcl_returns.h"
diff --git a/bin/varnishd/cache/cache_vrt.c b/bin/varnishd/cache/cache_vrt.c
index a696f64..9ec88cc 100644
--- a/bin/varnishd/cache/cache_vrt.c
+++ b/bin/varnishd/cache/cache_vrt.c
@@ -248,7 +248,7 @@ VRT_handling(struct sess *sp, unsigned hand)
 	}
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 	assert(hand < VCL_RET_MAX);
-	sp->handling = hand;
+	sp->req->handling = hand;
 }
 
 /*--------------------------------------------------------------------
@@ -510,9 +510,9 @@ VRT_ban_string(struct sess *sp, const char *str)
 void
 VRT_purge(const struct sess *sp, double ttl, double grace)
 {
-	if (sp->cur_method == VCL_MET_HIT)
+	if (sp->req->cur_method == VCL_MET_HIT)
 		HSH_Purge(sp, sp->wrk->obj->objcore->objhead, ttl, grace);
-	else if (sp->cur_method == VCL_MET_MISS)
+	else if (sp->req->cur_method == VCL_MET_MISS)
 		HSH_Purge(sp, sp->wrk->objcore->objhead, ttl, grace);
 }
 
diff --git a/bin/varnishd/cache/cache_vrt_var.c b/bin/varnishd/cache/cache_vrt_var.c
index 6f5544b..8c95d5b 100644
--- a/bin/varnishd/cache/cache_vrt_var.c
+++ b/bin/varnishd/cache/cache_vrt_var.c
@@ -327,15 +327,15 @@ VRT_l_req_esi(struct sess *sp, unsigned process_esi)
 	 * Only allow you to turn of esi in the main request
 	 * else everything gets confused
 	 */
-	if(sp->esi_level == 0)
-		sp->disable_esi = !process_esi;
+	if(sp->req->esi_level == 0)
+		sp->req->disable_esi = !process_esi;
 }
 
 unsigned __match_proto__()
 VRT_r_req_esi(struct sess *sp)
 {
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
-	return (!sp->disable_esi);
+	return (!sp->req->disable_esi);
 }
 
 int
@@ -343,7 +343,7 @@ VRT_r_req_esi_level(const struct sess *sp)
 {
 
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
-	return(sp->esi_level);
+	return(sp->req->esi_level);
 }
 
 /*--------------------------------------------------------------------*/
@@ -364,7 +364,7 @@ VRT_r_req_restarts(const struct sess *sp)
 {
 
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
-	return (sp->restarts);
+	return (sp->req->restarts);
 }
 
 /*--------------------------------------------------------------------
@@ -401,9 +401,9 @@ vrt_wsp_exp(const struct sess *sp, unsigned xid, const struct exp *e)
 	    sp->t_req, e->age + (sp->t_req - e->entered));
 }
 
-VRT_DO_EXP(req, sp->exp, ttl, 0, )
-VRT_DO_EXP(req, sp->exp, grace, 0, )
-VRT_DO_EXP(req, sp->exp, keep, 0, )
+VRT_DO_EXP(req, sp->req->exp, ttl, 0, )
+VRT_DO_EXP(req, sp->req->exp, grace, 0, )
+VRT_DO_EXP(req, sp->req->exp, keep, 0, )
 
 VRT_DO_EXP(obj, sp->wrk->obj->exp, grace, 0,
    EXP_Rearm(sp->wrk->obj);
@@ -447,7 +447,7 @@ VRT_l_req_##hash_var(struct sess *sp, unsigned val)		\
 {								\
 								\
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);			\
-	sp->hash_var = val ? 1 : 0;				\
+	sp->req->hash_var = val ? 1 : 0;				\
 }								\
 								\
 unsigned __match_proto__()					\
@@ -455,7 +455,7 @@ VRT_r_req_##hash_var(struct sess *sp)				\
 {								\
 								\
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);			\
-	return(sp->hash_var);					\
+	return(sp->req->hash_var);					\
 }
 
 REQ_BOOL(hash_ignore_busy)



From geoff at varnish-cache.org  Mon Jan  9 23:03:34 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Tue, 10 Jan 2012 00:03:34 +0100
Subject: [experimental-ims] 3d43f62 Add support for banning on http.status
Message-ID: <E1RkOFa-00080A-C6@project.varnish-software.com>

commit 3d43f62b2e886ff115caa803eb45bccf72c806ce
Author: Tollef Fog Heen <tfheen at varnish-software.com>
Date:   Mon Jan 2 12:58:03 2012 +0100

    Add support for banning on http.status
    
    Fixes: #1076

diff --git a/bin/varnishd/cache/cache_ban.c b/bin/varnishd/cache/cache_ban.c
index a884983..451e504 100644
--- a/bin/varnishd/cache/cache_ban.c
+++ b/bin/varnishd/cache/cache_ban.c
@@ -113,9 +113,10 @@ static bgthread_t ban_lurker;
 #define	BAN_OPER_MATCH	0x12
 #define	BAN_OPER_NMATCH	0x13
 
-#define BAN_ARG_URL	0x18
-#define BAN_ARG_REQHTTP	0x19
-#define BAN_ARG_OBJHTTP	0x1a
+#define BAN_ARG_URL		0x18
+#define BAN_ARG_REQHTTP		0x19
+#define BAN_ARG_OBJHTTP		0x1a
+#define BAN_ARG_OBJSTATUS	0x1b
 
 /*--------------------------------------------------------------------
  * Variables we can purge on
@@ -609,6 +610,7 @@ ban_evaluate(const uint8_t *bs, const struct http *objhttp,
 	struct ban_test bt;
 	const uint8_t *be;
 	char *arg1;
+	char buf[10];
 
 	be = bs + ban_len(bs);
 	bs += 13;
@@ -628,6 +630,10 @@ ban_evaluate(const uint8_t *bs, const struct http *objhttp,
 		case BAN_ARG_OBJHTTP:
 			(void)http_GetHdr(objhttp, bt.arg1_spec, &arg1);
 			break;
+		case BAN_ARG_OBJSTATUS:
+			arg1 = buf;
+			sprintf(buf, "%d", objhttp->status);
+			break;
 		default:
 			INCOMPL();
 		}
diff --git a/include/tbl/ban_vars.h b/include/tbl/ban_vars.h
index 669b9b1..65b17e0 100644
--- a/include/tbl/ban_vars.h
+++ b/include/tbl/ban_vars.h
@@ -35,3 +35,4 @@
 PVAR("req.url",		PVAR_REQ,		BAN_ARG_URL)
 PVAR("req.http.",	PVAR_REQ|PVAR_HTTP,	BAN_ARG_REQHTTP)
 PVAR("obj.http.",	PVAR_HTTP,		BAN_ARG_OBJHTTP)
+PVAR("obj.status",	0,			BAN_ARG_OBJSTATUS)



From geoff at varnish-cache.org  Mon Jan  9 23:03:28 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Tue, 10 Jan 2012 00:03:28 +0100
Subject: [experimental-ims] 3673967 Merge branch 'master' of
	ssh://git.varnish-cache.org/git/varnish-cache
Message-ID: <E1RkOFU-0007pq-E3@project.varnish-software.com>

commit 3673967c6bf261a2c1e1e8220239c98d71ad7393
Merge: 4c1e826 59a3b91
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Dec 19 10:36:01 2011 +0000

    Merge branch 'master' of ssh://git.varnish-cache.org/git/varnish-cache



From geoff at varnish-cache.org  Mon Jan  9 23:03:28 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Tue, 10 Jan 2012 00:03:28 +0100
Subject: [experimental-ims] 8624d49 Move the client IP+port onto the sess
	structure instead of the sess workspace.
Message-ID: <E1RkOFU-0007qm-U4@project.varnish-software.com>

commit 8624d492a504d40f0794b00a6b036bfd929822d9
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Dec 20 09:01:14 2011 +0000

    Move the client IP+port onto the sess structure instead of the
    sess workspace.

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index 14b9fe6..d1d67b6 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -600,8 +600,8 @@ struct sess {
 	struct listen_sock	*mylsock;
 
 	/* formatted ascii client address */
-	char			*addr;
-	char			*port;
+	char			addr[ADDR_BUFSIZE];
+	char			port[PORT_BUFSIZE];
 	char			*client_identity;
 
 	/* HTTP request */
@@ -662,7 +662,7 @@ struct sess {
 /* Prototypes etc ----------------------------------------------------*/
 
 /* cache_acceptor.c */
-void VCA_Prep(struct sess *sp);
+void VCA_Prep(const struct sess *sp);
 void VCA_Init(void);
 void VCA_Shutdown(void);
 int VCA_Accept(struct listen_sock *ls, struct wrk_accept *wa);
@@ -997,7 +997,6 @@ void WS_ReleaseP(struct ws *ws, char *ptr);
 void WS_Assert(const struct ws *ws);
 void WS_Reset(struct ws *ws, char *p);
 char *WS_Alloc(struct ws *ws, unsigned bytes);
-char *WS_Dup(struct ws *ws, const char *);
 char *WS_Snapshot(struct ws *ws);
 unsigned WS_Free(const struct ws *ws);
 
diff --git a/bin/varnishd/cache/cache_acceptor.c b/bin/varnishd/cache/cache_acceptor.c
index 661b053..6ef9aec 100644
--- a/bin/varnishd/cache/cache_acceptor.c
+++ b/bin/varnishd/cache/cache_acceptor.c
@@ -113,27 +113,9 @@ sock_test(int fd)
  */
 
 void
-VCA_Prep(struct sess *sp)
+VCA_Prep(const struct sess *sp)
 {
-	char addr[VTCP_ADDRBUFSIZE];
-	char port[VTCP_PORTBUFSIZE];
-
-	VTCP_name(&sp->sockaddr, sp->sockaddrlen,
-	    addr, sizeof addr, port, sizeof port);
-	sp->addr = WS_Dup(sp->ws, addr);
-	sp->port = WS_Dup(sp->ws, port);
-	if (cache_param->log_local_addr) {
-		AZ(getsockname(sp->fd, (void*)&sp->mysockaddr,
-		    &sp->mysockaddrlen));
-		VTCP_name(&sp->mysockaddr, sp->mysockaddrlen,
-		    addr, sizeof addr, port, sizeof port);
-		WSP(sp, SLT_SessionOpen, "%s %s %s %s",
-		    sp->addr, sp->port, addr, port);
-	} else {
-		WSP(sp, SLT_SessionOpen, "%s %s %s",
-		    sp->addr, sp->port, sp->mylsock->name);
-	}
-	sp->acct_ses.first = sp->t_open;
+
 	if (need_test)
 		sock_test(sp->fd);
 	if (need_linger)
diff --git a/bin/varnishd/cache/cache_center.c b/bin/varnishd/cache/cache_center.c
index 520c620..a6ea594 100644
--- a/bin/varnishd/cache/cache_center.c
+++ b/bin/varnishd/cache/cache_center.c
@@ -64,6 +64,8 @@ DOT acceptor -> first [style=bold,color=green]
 
 #include "cache.h"
 
+#include "common/heritage.h"
+
 #include "hash/hash_slinger.h"
 #include "vcl.h"
 #include "vcli_priv.h"
@@ -1005,6 +1007,8 @@ static int
 cnt_first(struct sess *sp)
 {
 	struct worker *wrk;
+	char laddr[ADDR_BUFSIZE];
+	char lport[PORT_BUFSIZE];
 
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 	wrk = sp->wrk;
@@ -1020,6 +1024,21 @@ cnt_first(struct sess *sp)
 	assert(sp->xid == 0);
 	assert(sp->restarts == 0);
 	AZ(sp->esi_level);
+	VTCP_name(&sp->sockaddr, sp->sockaddrlen,
+	    sp->addr, sizeof sp->addr, sp->port, sizeof sp->port);
+	if (cache_param->log_local_addr) {
+		AZ(getsockname(sp->fd, (void*)&sp->mysockaddr,
+		    &sp->mysockaddrlen));
+		VTCP_name(&sp->mysockaddr, sp->mysockaddrlen,
+		    laddr, sizeof laddr, lport, sizeof lport);
+		WSP(sp, SLT_SessionOpen, "%s %s %s %s",
+		    sp->addr, sp->port, laddr, lport);
+	} else {
+		WSP(sp, SLT_SessionOpen, "%s %s %s",
+		    sp->addr, sp->port, sp->mylsock->name);
+	}
+	sp->acct_ses.first = sp->t_open;
+
 	VCA_Prep(sp);
 
 	/* Record the session watermark */
diff --git a/bin/varnishd/cache/cache_session.c b/bin/varnishd/cache/cache_session.c
index 19292ea..65dd5c7 100644
--- a/bin/varnishd/cache/cache_session.c
+++ b/bin/varnishd/cache/cache_session.c
@@ -321,7 +321,6 @@ SES_Delete(struct sess *sp, const char *reason)
 {
 	struct acct *b;
 	struct sessmem *sm;
-	static char noaddr[] = "-";
 	struct worker *wrk;
 	struct sesspool *pp;
 
@@ -340,10 +339,10 @@ SES_Delete(struct sess *sp, const char *reason)
 	assert(sp->fd < 0);
 
 	AZ(sp->vcl);
-	if (sp->addr == NULL)
-		sp->addr = noaddr;
-	if (sp->port == NULL)
-		sp->port = noaddr;
+	if (*sp->addr == '\0')
+		strcpy(sp->addr, "-");
+	if (*sp->port == '\0')
+		strcpy(sp->addr, "-");
 
 	b = &sp->acct_ses;
 	assert(!isnan(b->first));
diff --git a/bin/varnishd/cache/cache_ws.c b/bin/varnishd/cache/cache_ws.c
index b5f48d3..be38496 100644
--- a/bin/varnishd/cache/cache_ws.c
+++ b/bin/varnishd/cache/cache_ws.c
@@ -116,22 +116,6 @@ WS_Alloc(struct ws *ws, unsigned bytes)
 	return (r);
 }
 
-char *
-WS_Dup(struct ws *ws, const char *s)
-{
-	unsigned l;
-	char *p;
-
-	WS_Assert(ws);
-	l = strlen(s) + 1;
-	p = WS_Alloc(ws, l);
-	if (p != NULL)
-		memcpy(p, s, l);
-	DSL(0x02, SLT_Debug, 0, "WS_Dup(%p, \"%s\") = %p", ws, s, p);
-	WS_Assert(ws);
-	return (p);
-}
-
 unsigned
 WS_Free(const struct ws *ws)
 {
diff --git a/bin/varnishd/common/common.h b/bin/varnishd/common/common.h
index 3509f25..7fcc7ec 100644
--- a/bin/varnishd/common/common.h
+++ b/bin/varnishd/common/common.h
@@ -60,6 +60,15 @@ struct cli;
  */
 #define __state_variable__(xxx)		/*lint -esym(838,xxx) */
 
+/**********************************************************************
+ * NI_MAXHOST and less so NI_MAXSERV, are ridiculously large for numeric
+ * representations of TCP/IP socket addresses, so we use our own.
+ */
+
+#define ADDR_BUFSIZE	64
+#define PORT_BUFSIZE	8
+
+
 /**********************************************************************/
 
 /* Name of transient storage */
diff --git a/bin/varnishtest/tests/r00693.vtc b/bin/varnishtest/tests/r00693.vtc
index 37b0a98..ce65be5 100644
--- a/bin/varnishtest/tests/r00693.vtc
+++ b/bin/varnishtest/tests/r00693.vtc
@@ -51,8 +51,8 @@ varnish v1 -arg "-p sess_workspace=1024" -vcl+backend {
 			"0123456789abcdef" + "0123456789abcdef" +
 			"0123456789abcdef" + "0123456789abcdef" +
 			"0123456789abcdef" + "0123456789abcdef" +
-			"0123456789abcdef" +
-			"0123456";
+			"0123456789abcdef" + "0123456789abcdef" +
+			"0123456789abcde";
 		set req.http.baz = "BAZ";
 		return (pass);
 	}



From geoff at varnish-cache.org  Mon Jan  9 23:03:33 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Tue, 10 Jan 2012 00:03:33 +0100
Subject: [experimental-ims] 1b75dc8 These escaped last commit.
Message-ID: <E1RkOFZ-0007yJ-Fp@project.varnish-software.com>

commit 1b75dc8d2b1a98297eb4adc5df080e80c2edc333
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Dec 26 16:45:11 2011 +0000

    These escaped last commit.

diff --git a/include/tbl/vsc_fields.h b/include/tbl/vsc_fields.h
index 3d1e30a..2d83f69 100644
--- a/include/tbl/vsc_fields.h
+++ b/include/tbl/vsc_fields.h
@@ -111,6 +111,8 @@ VSC_F(happy,		uint64_t, 0, 'b', "Happy health probes", "")
 
 VSC_F(live,			uint64_t, 0, 'g', "In use", "")
 VSC_F(pool,			uint64_t, 0, 'g', "In Pool", "")
+VSC_F(sz_wanted,		uint64_t, 0, 'g', "Size requested", "")
+VSC_F(sz_needed,		uint64_t, 0, 'g', "Size allocated", "")
 VSC_F(allocs,			uint64_t, 0, 'c', "Allocations", "")
 VSC_F(frees,			uint64_t, 0, 'c', "Frees", "")
 VSC_F(recycle,			uint64_t, 0, 'c', "Recycled from pool", "")



From geoff at varnish-cache.org  Mon Jan  9 23:03:33 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Tue, 10 Jan 2012 00:03:33 +0100
Subject: [experimental-ims] 0b1e63e Fix inconsistant man page,
	as reported by Chris Adams
Message-ID: <E1RkOFZ-0007yp-O6@project.varnish-software.com>

commit 0b1e63e3c5b70ba7915a633278127c82d5111556
Author: Lasse Karstensen <lasse at varnish-software.com>
Date:   Wed Dec 28 16:17:57 2011 +0100

    Fix inconsistant man page, as reported by Chris Adams

diff --git a/doc/sphinx/reference/varnishncsa.rst b/doc/sphinx/reference/varnishncsa.rst
index 91622d8..0b3f97f 100644
--- a/doc/sphinx/reference/varnishncsa.rst
+++ b/doc/sphinx/reference/varnishncsa.rst
@@ -15,7 +15,7 @@ Display Varnish logs in Apache / NCSA combined log format
 SYNOPSIS
 ========
 
-varnishncsa [-a] [-b] [-C] [-c] [-D] [-d] [-f] [-F format] [-I regex]
+varnishncsa [-a] [-C] [-D] [-d] [-f] [-F format] [-I regex]
 [-i tag] [-n varnish_name] [-m tag:regex ...] [-P file] [-r file] [-V] [-w file] 
 [-X regex] [-x tag]
 
@@ -30,16 +30,8 @@ The following options are available:
 
 -a          When writing to a file, append to it rather than overwrite it.
 
--b          Include log entries which result from communication with a 
-	    backend server.  If neither -b nor -c is
-	    specified, varnishncsa acts as if they both were.
-
 -C          Ignore case when matching regular expressions.
 
--c          Include log entries which result from communication 
-	    with a client.  If neither -b nor -c is specified, 
-	    varnishncsa acts as if they both were.
-
 -D          Daemonize.
 
 -d          Process old log entries on startup.  Normally, varnishncsa 



From geoff at varnish-cache.org  Mon Jan  9 23:03:30 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Tue, 10 Jan 2012 00:03:30 +0100
Subject: [experimental-ims] f7ef385 Longer delays
Message-ID: <E1RkOFW-0007uL-UQ@project.varnish-software.com>

commit f7ef385a7a708b35164f4eaa64ab4aa1e10f2a46
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Fri Dec 23 09:21:52 2011 +0000

    Longer delays

diff --git a/bin/varnishtest/tests/c00050.vtc b/bin/varnishtest/tests/c00050.vtc
index 04d8f2c..2e787cb 100644
--- a/bin/varnishtest/tests/c00050.vtc
+++ b/bin/varnishtest/tests/c00050.vtc
@@ -4,26 +4,26 @@ server s1 { } -start
 
 varnish v1 -vcl+backend {} -start
 
-delay 1
+delay 2
 
 varnish v1 -expect MEMPOOL.vbc.pool == 10
 
 varnish v1 -cliok "param.set pool_vbc 90,100,100"
 
-delay 1
+delay 2
 
 varnish v1 -expect MEMPOOL.vbc.pool == 90
 
 varnish v1 -cliok "param.set pool_vbc 50,80,100"
 
-delay 1
+delay 2
 
 varnish v1 -expect MEMPOOL.vbc.pool == 80
 varnish v1 -expect MEMPOOL.vbc.surplus == 10
 
 varnish v1 -cliok "param.set pool_vbc 10,80,1"
 
-delay 1
+delay 2
 
 varnish v1 -expect MEMPOOL.vbc.pool == 10
 varnish v1 -expect MEMPOOL.vbc.timeout == 70



From geoff at varnish-cache.org  Mon Jan  9 23:03:34 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Tue, 10 Jan 2012 00:03:34 +0100
Subject: [experimental-ims] 58a062a Pack struct sess slightly smarter,
	and get its allocation size (incl memitem) down to 512 bytes,
	despite ridiculously large sockaddr_storage structs.
Message-ID: <E1RkOFa-0007zJ-1J@project.varnish-software.com>

commit 58a062a6e39a11a3dd436be5c03792ef1f868f05
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Jan 2 10:23:18 2012 +0000

    Pack struct sess slightly smarter, and get its allocation size
    (incl memitem) down to 512 bytes, despite ridiculously large
    sockaddr_storage structs.

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index a15f272..75bc803 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -630,6 +630,10 @@ struct sess {
 	unsigned		magic;
 #define SESS_MAGIC		0x2c2f9c5a
 
+	enum step		step;
+	int			fd;
+	unsigned		vsl_id;
+
 	/* Cross references ------------------------------------------*/
 
 	struct sesspool		*sesspool;
@@ -640,9 +644,6 @@ struct sess {
 
 	/* Session related fields ------------------------------------*/
 
-	int			fd;
-	unsigned		vsl_id;
-
 	socklen_t		sockaddrlen;
 	socklen_t		mysockaddrlen;
 	struct sockaddr_storage	sockaddr;
@@ -658,18 +659,11 @@ struct sess {
 	/* Timestamps, all on TIM_real() timescale */
 	double			t_open;		/* fd accepted */
 	double			t_idle;		/* fd accepted or resp sent */
+	double			t_req;
 
 #if defined(HAVE_EPOLL_CTL)
 	struct epoll_event ev;
 #endif
-	enum step		step;
-
-	/* Request related fields ------------------------------------*/
-
-
-	/* Timestamps, all on TIM_real() timescale */
-	double			t_req;
-
 };
 
 /* Prototypes etc ----------------------------------------------------*/



From geoff at varnish-cache.org  Mon Jan  9 23:03:31 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Tue, 10 Jan 2012 00:03:31 +0100
Subject: [experimental-ims] 45202f7 This is an interim snapshot committed
	mostly for debugging purposes, do not run in production!
Message-ID: <E1RkOFX-0007vA-Nv@project.varnish-software.com>

commit 45202f7f39093f95a21e7efbb90c95c1eda96d33
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sun Dec 25 10:07:31 2011 +0000

    This is an interim snapshot committed mostly for debugging purposes,
    do not run in production!
    
    Allocate the workspace as part of struct req, and move sp->ws,
    sp->http and sp->http0 to struct req.
    
    Allocate the req->http[0] on the workspace, so that changes to
    nhttp does not force us to dump the mempool.

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index 7517d1c..d1dc4fb 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -619,6 +619,11 @@ struct req {
 	struct http_conn	htc[1];
 	char			*client_identity;
 
+	/* HTTP request */
+	struct http		*http;
+	struct http		*http0;
+
+	struct ws		ws[1];
 };
 
 /*--------------------------------------------------------------------*/
@@ -664,11 +669,6 @@ struct sess {
 
 	/* Request related fields ------------------------------------*/
 
-	/* HTTP request */
-	struct http		*http;
-	struct http		*http0;
-
-	struct ws		ws[1];
 
 	/* Timestamps, all on TIM_real() timescale */
 	double			t_req;
diff --git a/bin/varnishd/cache/cache_ban.c b/bin/varnishd/cache/cache_ban.c
index 2badb76..9162f50 100644
--- a/bin/varnishd/cache/cache_ban.c
+++ b/bin/varnishd/cache/cache_ban.c
@@ -675,6 +675,7 @@ ban_check_object(struct object *o, const struct sess *sp, int has_req)
 	struct objcore *oc;
 	struct ban * volatile b0;
 	unsigned tests, skipped;
+	struct http *http;
 
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 	CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC);
@@ -688,6 +689,11 @@ ban_check_object(struct object *o, const struct sess *sp, int has_req)
 	if (b0 == oc->ban)
 		return (0);
 
+	if (has_req)
+		http = sp->req->http;
+	else
+		http = NULL;
+
 	/*
 	 * This loop is safe without locks, because we know we hold
 	 * a refcount on a ban somewhere in the list and we do not
@@ -711,7 +717,7 @@ ban_check_object(struct object *o, const struct sess *sp, int has_req)
 			 * be other bans that match, so we soldier on
 			 */
 			skipped++;
-		} else if (ban_evaluate(b->spec, o->http, sp->http, &tests))
+		} else if (ban_evaluate(b->spec, o->http, http, &tests))
 			break;
 	}
 
diff --git a/bin/varnishd/cache/cache_center.c b/bin/varnishd/cache/cache_center.c
index 5816119..6bf3f6a 100644
--- a/bin/varnishd/cache/cache_center.c
+++ b/bin/varnishd/cache/cache_center.c
@@ -109,7 +109,7 @@ cnt_wait(struct sess *sp)
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 	if (sp->req == NULL) {
 		SES_GetReq(sp);
-		HTC_Init(sp->req->htc, sp->ws, sp->fd, sp->vsl_id,
+		HTC_Init(sp->req->htc, sp->req->ws, sp->fd, sp->vsl_id,
 		    cache_param->http_req_size,
 		    cache_param->http_req_hdr_len);
 	}
@@ -160,8 +160,6 @@ cnt_wait(struct sess *sp)
 				wrk->stats.sess_herd++;
 				SES_Charge(sp);
 				SES_ReleaseReq(sp);
-				WS_Release(sp->ws, 0);
-				WS_Reset(sp->ws, NULL);
 				WAIT_Enter(sp);
 				return (1);
 			}
@@ -267,7 +265,7 @@ cnt_prepresp(struct sess *sp)
 			wrk->res_mode |= RES_LEN;
 		else if (!sp->req->wantbody) {
 			/* Nothing */
-		} else if (sp->http->protover >= 11) {
+		} else if (sp->req->http->protover >= 11) {
 			wrk->res_mode |= RES_CHUNKED;
 		} else {
 			wrk->res_mode |= RES_EOF;
@@ -448,7 +446,7 @@ cnt_done(struct sess *sp)
 	if (wrk->stats.client_req >= cache_param->wthread_stats_rate)
 		WRK_SumStat(wrk);
 	/* Reset the workspace to the session-watermark */
-	WS_Reset(sp->ws, NULL);
+	WS_Reset(sp->req->ws, NULL);
 	WS_Reset(wrk->ws, NULL);
 
 	i = HTC_Reinit(sp->req->htc);
@@ -903,7 +901,7 @@ cnt_fetchbody(struct sess *sp)
 	 * that allow the 304, in which case we return 200 non-stream.
 	 */
 	if (wrk->obj->response == 200 &&
-	    sp->http->conds &&
+	    sp->req->http->conds &&
 	    RFC2616_Do_Cond(sp))
 		wrk->busyobj->do_stream = 0;
 
@@ -1177,12 +1175,12 @@ cnt_lookup(struct sess *sp)
 		AZ(sp->req->vary_b);
 		AZ(sp->req->vary_l);
 		AZ(sp->req->vary_e);
-		(void)WS_Reserve(sp->ws, 0);
+		(void)WS_Reserve(sp->req->ws, 0);
 	} else {
-		AN(sp->ws->r);
+		AN(sp->req->ws->r);
 	}
-	sp->req->vary_b = (void*)sp->ws->f;
-	sp->req->vary_e = (void*)sp->ws->r;
+	sp->req->vary_b = (void*)sp->req->ws->f;
+	sp->req->vary_e = (void*)sp->req->ws->r;
 	sp->req->vary_b[2] = '\0';
 
 	oc = HSH_Lookup(sp, &oh);
@@ -1208,10 +1206,10 @@ cnt_lookup(struct sess *sp)
 		if (sp->req->vary_l != NULL) {
 			assert(oc->busyobj->vary == sp->req->vary_b);
 			VRY_Validate(oc->busyobj->vary);
-			WS_ReleaseP(sp->ws, (void*)sp->req->vary_l);
+			WS_ReleaseP(sp->req->ws, (void*)sp->req->vary_l);
 		} else {
 			AZ(oc->busyobj->vary);
-			WS_Release(sp->ws, 0);
+			WS_Release(sp->req->ws, 0);
 		}
 		sp->req->vary_b = NULL;
 		sp->req->vary_l = NULL;
@@ -1227,7 +1225,7 @@ cnt_lookup(struct sess *sp)
 	CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC);
 	wrk->obj = o;
 
-	WS_Release(sp->ws, 0);
+	WS_Release(sp->req->ws, 0);
 	sp->req->vary_b = NULL;
 	sp->req->vary_l = NULL;
 	sp->req->vary_e = NULL;
@@ -1500,7 +1498,7 @@ cnt_recv(struct sess *sp)
 	sp->req->hash_ignore_busy = 0;
 	sp->req->client_identity = NULL;
 
-	http_CollectHdr(sp->http, H_Cache_Control);
+	http_CollectHdr(sp->req->http, H_Cache_Control);
 
 	VCL_recv_method(sp);
 	recv_handling = sp->req->handling;
@@ -1516,11 +1514,11 @@ cnt_recv(struct sess *sp)
 	     (recv_handling != VCL_RET_PIPE) &&
 	     (recv_handling != VCL_RET_PASS)) {
 		if (RFC2616_Req_Gzip(sp)) {
-			http_Unset(sp->http, H_Accept_Encoding);
-			http_SetHeader(wrk, sp->vsl_id, sp->http,
+			http_Unset(sp->req->http, H_Accept_Encoding);
+			http_SetHeader(wrk, sp->vsl_id, sp->req->http,
 			    "Accept-Encoding: gzip");
 		} else {
-			http_Unset(sp->http, H_Accept_Encoding);
+			http_Unset(sp->req->http, H_Accept_Encoding);
 		}
 	}
 
@@ -1529,7 +1527,7 @@ cnt_recv(struct sess *sp)
 	assert(sp->req->handling == VCL_RET_HASH);
 	SHA256_Final(sp->req->digest, wrk->sha256ctx);
 
-	if (!strcmp(sp->http->hd[HTTP_HDR_REQ].b, "HEAD"))
+	if (!strcmp(sp->req->http->hd[HTTP_HDR_REQ].b, "HEAD"))
 		sp->req->wantbody = 0;
 	else
 		sp->req->wantbody = 1;
@@ -1605,7 +1603,7 @@ cnt_start(struct sess *sp)
 	sp->req->vcl = wrk->vcl;
 	wrk->vcl = NULL;
 
-	http_Setup(sp->http, sp->ws);
+	http_Setup(sp->req->http, sp->req->ws);
 	done = http_DissectRequest(sp);
 
 	/* If we could not even parse the request, just close */
@@ -1616,10 +1614,10 @@ cnt_start(struct sess *sp)
 	}
 
 	/* Catch request snapshot */
-	sp->req->ws_req = WS_Snapshot(sp->ws);
+	sp->req->ws_req = WS_Snapshot(sp->req->ws);
 
 	/* Catch original request, before modification */
-	HTTP_Copy(sp->http0, sp->http);
+	HTTP_Copy(sp->req->http0, sp->req->http);
 
 	if (done != 0) {
 		sp->req->err_code = done;
@@ -1627,14 +1625,14 @@ cnt_start(struct sess *sp)
 		return (0);
 	}
 
-	sp->req->doclose = http_DoConnection(sp->http);
+	sp->req->doclose = http_DoConnection(sp->req->http);
 
 	/* XXX: Handle TRACE & OPTIONS of Max-Forwards = 0 */
 
 	/*
 	 * Handle Expect headers
 	 */
-	if (http_GetHdr(sp->http, H_Expect, &p)) {
+	if (http_GetHdr(sp->req->http, H_Expect, &p)) {
 		if (strcasecmp(p, "100-continue")) {
 			sp->req->err_code = 417;
 			sp->step = STP_ERROR;
@@ -1647,7 +1645,7 @@ cnt_start(struct sess *sp)
 		 * XXX: because we use http0 as our basis.  Believed
 		 * XXX: safe, but potentially confusing.
 		 */
-		http_Unset(sp->http, H_Expect);
+		http_Unset(sp->req->http, H_Expect);
 	}
 
 	sp->step = STP_RECV;
diff --git a/bin/varnishd/cache/cache_dir_dns.c b/bin/varnishd/cache/cache_dir_dns.c
index 517caaa..80310b9 100644
--- a/bin/varnishd/cache/cache_dir_dns.c
+++ b/bin/varnishd/cache/cache_dir_dns.c
@@ -350,7 +350,7 @@ vdi_dns_find_backend(const struct sess *sp, struct vdi_dns *vs)
 	if (sp->wrk->busyobj != NULL && sp->wrk->busyobj->bereq)
 		hp = sp->wrk->busyobj->bereq;
 	else
-		hp = sp->http;
+		hp = sp->req->http;
 
 
 	CHECK_OBJ_NOTNULL(hp, HTTP_MAGIC);
diff --git a/bin/varnishd/cache/cache_esi_deliver.c b/bin/varnishd/cache/cache_esi_deliver.c
index e1d3945..be8d185 100644
--- a/bin/varnishd/cache/cache_esi_deliver.c
+++ b/bin/varnishd/cache/cache_esi_deliver.c
@@ -63,17 +63,17 @@ ved_include(struct sess *sp, const char *src, const char *host)
 	res_mode = sp->wrk->res_mode;
 
 	/* Reset request to status before we started messing with it */
-	HTTP_Copy(sp->http, sp->http0);
+	HTTP_Copy(sp->req->http, sp->req->http0);
 
 	/* Take a workspace snapshot */
-	sp_ws_wm = WS_Snapshot(sp->ws);
+	sp_ws_wm = WS_Snapshot(sp->req->ws);
 	wrk_ws_wm = WS_Snapshot(w->ws);
 
-	http_SetH(sp->http, HTTP_HDR_URL, src);
+	http_SetH(sp->req->http, HTTP_HDR_URL, src);
 	if (host != NULL && *host != '\0')  {
-		http_Unset(sp->http, H_Host);
-		http_Unset(sp->http, H_If_Modified_Since);
-		http_SetHeader(w, sp->vsl_id, sp->http, host);
+		http_Unset(sp->req->http, H_Host);
+		http_Unset(sp->req->http, H_If_Modified_Since);
+		http_SetHeader(w, sp->vsl_id, sp->req->http, host);
 	}
 	/*
 	 * XXX: We should decide if we should cache the director
@@ -82,14 +82,14 @@ ved_include(struct sess *sp, const char *src, const char *host)
 	 */
 	sp->req->director = NULL;
 	sp->step = STP_RECV;
-	http_ForceGet(sp->http);
+	http_ForceGet(sp->req->http);
 
 	/* Don't do conditionals */
-	sp->http->conds = 0;
-	http_Unset(sp->http, H_If_Modified_Since);
+	sp->req->http->conds = 0;
+	http_Unset(sp->req->http, H_If_Modified_Since);
 
 	/* Client content already taken care of */
-	http_Unset(sp->http, H_Content_Length);
+	http_Unset(sp->req->http, H_Content_Length);
 
 	sxid = sp->req->xid;
 	while (1) {
@@ -110,7 +110,7 @@ ved_include(struct sess *sp, const char *src, const char *host)
 	sp->wrk->res_mode = res_mode;
 
 	/* Reset the workspace */
-	WS_Reset(sp->ws, sp_ws_wm);
+	WS_Reset(sp->req->ws, sp_ws_wm);
 	WS_Reset(w->ws, wrk_ws_wm);
 
 	WRW_Reserve(sp->wrk, &sp->fd);
diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c
index dbf30e6..1e75f64 100644
--- a/bin/varnishd/cache/cache_fetch.c
+++ b/bin/varnishd/cache/cache_fetch.c
@@ -336,7 +336,7 @@ FetchReqBody(struct sess *sp)
 	char *ptr, *endp;
 	int rdcnt;
 
-	if (http_GetHdr(sp->http, H_Content_Length, &ptr)) {
+	if (http_GetHdr(sp->req->http, H_Content_Length, &ptr)) {
 
 		content_length = strtoul(ptr, &endp, 10);
 		/* XXX should check result of conversion */
@@ -356,7 +356,7 @@ FetchReqBody(struct sess *sp)
 				return (2);
 		}
 	}
-	if (http_GetHdr(sp->http, H_Transfer_Encoding, NULL)) {
+	if (http_GetHdr(sp->req->http, H_Transfer_Encoding, NULL)) {
 		/* XXX: Handle chunked encoding. */
 		WSP(sp, SLT_Debug, "Transfer-Encoding in request");
 		return (1);
diff --git a/bin/varnishd/cache/cache_hash.c b/bin/varnishd/cache/cache_hash.c
index dbcca41..5d8c880 100644
--- a/bin/varnishd/cache/cache_hash.c
+++ b/bin/varnishd/cache/cache_hash.c
@@ -301,7 +301,7 @@ HSH_Lookup(struct sess *sp, struct objhead **poh)
 
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 	CHECK_OBJ_NOTNULL(sp->wrk, WORKER_MAGIC);
-	CHECK_OBJ_NOTNULL(sp->http, HTTP_MAGIC);
+	CHECK_OBJ_NOTNULL(sp->req->http, HTTP_MAGIC);
 	AN(sp->req->director);
 	AN(hash);
 	wrk = sp->wrk;
diff --git a/bin/varnishd/cache/cache_http.c b/bin/varnishd/cache/cache_http.c
index 93bd896..f7dad38 100644
--- a/bin/varnishd/cache/cache_http.c
+++ b/bin/varnishd/cache/cache_http.c
@@ -660,7 +660,7 @@ http_DissectRequest(struct sess *sp)
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 	htc = sp->req->htc;
 	CHECK_OBJ_NOTNULL(htc, HTTP_CONN_MAGIC);
-	hp = sp->http;
+	hp = sp->req->http;
 	CHECK_OBJ_NOTNULL(hp, HTTP_MAGIC);
 
 	hp->logtag = HTTP_Rx;
@@ -873,13 +873,13 @@ http_FilterHeader(const struct sess *sp, unsigned how)
 	CHECK_OBJ_NOTNULL(hp, HTTP_MAGIC);
 	hp->logtag = HTTP_Tx;
 
-	http_copyh(hp, sp->http, HTTP_HDR_REQ);
-	http_copyh(hp, sp->http, HTTP_HDR_URL);
+	http_copyh(hp, sp->req->http, HTTP_HDR_REQ);
+	http_copyh(hp, sp->req->http, HTTP_HDR_URL);
 	if (how == HTTPH_R_FETCH)
 		http_SetH(hp, HTTP_HDR_PROTO, "HTTP/1.1");
 	else
-		http_copyh(hp, sp->http, HTTP_HDR_PROTO);
-	http_FilterFields(sp->wrk, sp->vsl_id, hp, sp->http, how);
+		http_copyh(hp, sp->req->http, HTTP_HDR_PROTO);
+	http_FilterFields(sp->wrk, sp->vsl_id, hp, sp->req->http, how);
 	http_PrintfHeader(sp->wrk, sp->vsl_id, hp, "X-Varnish: %u", sp->req->xid);
 }
 
diff --git a/bin/varnishd/cache/cache_panic.c b/bin/varnishd/cache/cache_panic.c
index 9710517..a35c258 100644
--- a/bin/varnishd/cache/cache_panic.c
+++ b/bin/varnishd/cache/cache_panic.c
@@ -265,8 +265,8 @@ pan_sess(const struct sess *sp)
 	if (sp->wrk->busyobj != NULL)
 		pan_busyobj(sp->wrk->busyobj);
 
-	pan_ws(sp->ws, 2);
-	pan_http("req", sp->http, 2);
+	pan_ws(sp->req->ws, 2);
+	pan_http("req", sp->req->http, 2);
 
 	if (sp->wrk != NULL)
 		pan_wrk(sp->wrk);
diff --git a/bin/varnishd/cache/cache_response.c b/bin/varnishd/cache/cache_response.c
index fe5f29f..6b2ea78 100644
--- a/bin/varnishd/cache/cache_response.c
+++ b/bin/varnishd/cache/cache_response.c
@@ -259,7 +259,7 @@ RES_WriteObj(struct sess *sp)
 	WRW_Reserve(sp->wrk, &sp->fd);
 
 	if (sp->wrk->obj->response == 200 &&
-	    sp->http->conds &&
+	    sp->req->http->conds &&
 	    RFC2616_Do_Cond(sp)) {
 		sp->req->wantbody = 0;
 		http_SetResp(sp->wrk->resp, "HTTP/1.1", 304, "Not Modified");
@@ -278,7 +278,7 @@ RES_WriteObj(struct sess *sp)
 	    !(sp->wrk->res_mode & (RES_ESI|RES_ESI_CHILD|RES_GUNZIP)) &&
 	    cache_param->http_range_support &&
 	    sp->wrk->obj->response == 200 &&
-	    http_GetHdr(sp->http, H_Range, &r))
+	    http_GetHdr(sp->req->http, H_Range, &r))
 		res_dorange(sp, r, &low, &high);
 
 	/*
diff --git a/bin/varnishd/cache/cache_rfc2616.c b/bin/varnishd/cache/cache_rfc2616.c
index 5c5e5e2..04641ca 100644
--- a/bin/varnishd/cache/cache_rfc2616.c
+++ b/bin/varnishd/cache/cache_rfc2616.c
@@ -291,7 +291,7 @@ RFC2616_Req_Gzip(const struct sess *sp)
 	 * p104 says to not do q values for x-gzip, so we just test
 	 * for its existence.
 	 */
-	if (http_GetHdrData(sp->http, H_Accept_Encoding, "x-gzip", NULL))
+	if (http_GetHdrData(sp->req->http, H_Accept_Encoding, "x-gzip", NULL))
 		return (1);
 
 	/*
@@ -299,7 +299,7 @@ RFC2616_Req_Gzip(const struct sess *sp)
 	 * We do not care a hoot if the client prefers some other
 	 * compression more than gzip: Varnish only does gzip.
 	 */
-	if (http_GetHdrQ(sp->http, H_Accept_Encoding, "gzip") > 0.)
+	if (http_GetHdrQ(sp->req->http, H_Accept_Encoding, "gzip") > 0.)
 		return (1);
 
 	/* Bad client, no gzip. */
@@ -318,7 +318,7 @@ RFC2616_Do_Cond(const struct sess *sp)
 	/* RFC 2616 13.3.4 states we need to match both ETag
 	   and If-Modified-Since if present*/
 
-	if (http_GetHdr(sp->http, H_If_Modified_Since, &p) ) {
+	if (http_GetHdr(sp->req->http, H_If_Modified_Since, &p) ) {
 		if (!sp->wrk->obj->last_modified)
 			return (0);
 		ims = VTIM_parse(p);
@@ -329,7 +329,7 @@ RFC2616_Do_Cond(const struct sess *sp)
 		do_cond = 1;
 	}
 
-	if (http_GetHdr(sp->http, H_If_None_Match, &p) &&
+	if (http_GetHdr(sp->req->http, H_If_None_Match, &p) &&
 	    http_GetHdr(sp->wrk->obj->http, H_ETag, &e)) {
 		if (strcmp(p,e) != 0)
 			return (0);
diff --git a/bin/varnishd/cache/cache_session.c b/bin/varnishd/cache/cache_session.c
index 42ce7df..92a7357 100644
--- a/bin/varnishd/cache/cache_session.c
+++ b/bin/varnishd/cache/cache_session.c
@@ -55,7 +55,6 @@ struct sessmem {
 	unsigned		workspace;
 	uint16_t		nhttp;
 	void			*wsp;
-	struct http		*http[2];
 	VTAILQ_ENTRY(sessmem)	list;
 
 	struct sess		sess;
@@ -69,7 +68,6 @@ struct sesspool {
 	struct lock		mtx;
 	unsigned		nsess;
 	unsigned		dly_free_cnt;
-	unsigned		req_size;
 	struct mempool		*mpl_req;
 };
 
@@ -82,6 +80,7 @@ SES_Charge(struct sess *sp)
 {
 	struct acct *a = &sp->wrk->acct_tmp;
 
+	CHECK_OBJ_NOTNULL(sp->req, REQ_MAGIC);
 	sp->req->req_bodybytes += a->bodybytes;
 
 #define ACCT(foo)				\
@@ -170,10 +169,6 @@ ses_setup(struct sessmem *sm)
 	sp->t_open = NAN;
 	sp->t_idle = NAN;
 	sp->t_req = NAN;
-
-	WS_Init(sp->ws, "sess", sm->wsp, sm->workspace);
-	sp->http = sm->http[0];
-	sp->http0 = sm->http[1];
 }
 
 /*--------------------------------------------------------------------
@@ -273,7 +268,7 @@ SES_Schedule(struct sess *sp)
 	if (Pool_Schedule(pp->pool, sp)) {
 		VSC_C_main->client_drop_late++;
 		sp->t_idle = VTIM_real();
-		if (sp->req->vcl != NULL) {
+		if (sp->req != NULL && sp->req->vcl != NULL) {
 			/*
 			 * A session parked on a busy object can come here
 			 * after it wakes up.  Loose the VCL reference.
@@ -391,19 +386,40 @@ SES_Delete(struct sess *sp, const char *reason, double now)
 }
 
 /*--------------------------------------------------------------------
- * Alloc/Free/Clean sp->req
+ * Alloc/Free sp->req
  */
 
 void
 SES_GetReq(struct sess *sp)
 {
 	struct sesspool *pp;
+	uint16_t nhttp;
+	unsigned sz, hl;
+	char *p;
 
 	pp = ses_getpool(sp);
 	AZ(sp->req);
-	sp->req = MPL_Get(pp->mpl_req, NULL);
+	sp->req = MPL_Get(pp->mpl_req, &sz);
 	AN(sp->req);
 	sp->req->magic = REQ_MAGIC;
+
+	p = (char*)(sp->req + 1);
+	sz -= sizeof *sp->req;
+
+	nhttp = (uint16_t)cache_param->http_max_hdr;
+	hl = HTTP_estimate(nhttp);
+
+	xxxassert(sz > 2 * hl + 128);
+
+	sp->req->http = HTTP_create(p, nhttp);
+	p += hl;
+	sz -= hl;
+
+	sp->req->http0 = HTTP_create(p, nhttp);
+	p += hl;
+	sz -= hl;
+	
+	WS_Init(sp->req->ws, "req", p, sz);
 }
 
 void
@@ -434,8 +450,8 @@ SES_NewPool(struct pool *wp, unsigned pool_no)
 	VTAILQ_INIT(&pp->freelist);
 	Lck_New(&pp->mtx, lck_sessmem);
 	bprintf(nb, "req%u", pool_no);
-	pp->req_size = sizeof (struct req);
-	pp->mpl_req = MPL_New(nb, &cache_param->req_pool, &pp->req_size);
+	pp->mpl_req = MPL_New(nb, &cache_param->req_pool,
+	    &cache_param->sess_workspace);
 	return (pp);
 }
 
diff --git a/bin/varnishd/cache/cache_vary.c b/bin/varnishd/cache/cache_vary.c
index 562f6df..a5070cd 100644
--- a/bin/varnishd/cache/cache_vary.c
+++ b/bin/varnishd/cache/cache_vary.c
@@ -96,7 +96,7 @@ VRY_Create(const struct sess *sp, const struct http *hp)
 		    (char)(1 + (q - p)), (int)(q - p), p, 0);
 		AZ(VSB_finish(sbh));
 
-		if (http_GetHdr(sp->http, VSB_data(sbh), &h)) {
+		if (http_GetHdr(sp->req->http, VSB_data(sbh), &h)) {
 			AZ(vct_issp(*h));
 			/* Trim trailing space */
 			e = strchr(h, '\0');
@@ -187,7 +187,7 @@ VRY_Match(struct sess *sp, const uint8_t *vary)
 		if (i == 1) {
 			/* Build a new entry */
 
-			i = http_GetHdr(sp->http, (const char*)(vary+2), &h);
+			i = http_GetHdr(sp->req->http, (const char*)(vary+2), &h);
 			if (i) {
 				/* Trim trailing space */
 				e = strchr(h, '\0');
diff --git a/bin/varnishd/cache/cache_vrt.c b/bin/varnishd/cache/cache_vrt.c
index e078f8f..7f27e8b 100644
--- a/bin/varnishd/cache/cache_vrt.c
+++ b/bin/varnishd/cache/cache_vrt.c
@@ -97,7 +97,7 @@ vrt_selecthttp(const struct sess *sp, enum gethdr_e where)
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 	switch (where) {
 	case HDR_REQ:
-		hp = sp->http;
+		hp = sp->req->http;
 		break;
 	case HDR_BEREQ:
 		hp = sp->wrk->busyobj->bereq;
@@ -307,7 +307,7 @@ VRT_IP_string(const struct sess *sp, const struct sockaddr_storage *sa)
 		INCOMPL();
 	}
 	XXXAN(len);
-	AN(p = WS_Alloc(sp->http->ws, len));
+	AN(p = WS_Alloc(sp->req->http->ws, len));
 	AN(inet_ntop(sa->ss_family, addr, p, len));
 	return (p);
 }
@@ -319,7 +319,7 @@ VRT_int_string(const struct sess *sp, int num)
 	int size;
 
 	size = snprintf(NULL, 0, "%d", num) + 1;
-	AN(p = WS_Alloc(sp->http->ws, size));
+	AN(p = WS_Alloc(sp->req->http->ws, size));
 	assert(snprintf(p, size, "%d", num) < size);
 	return (p);
 }
@@ -331,7 +331,7 @@ VRT_double_string(const struct sess *sp, double num)
 	int size;
 
 	size = snprintf(NULL, 0, "%.3f", num) + 1;
-	AN(p = WS_Alloc(sp->http->ws, size));
+	AN(p = WS_Alloc(sp->req->http->ws, size));
 	assert(snprintf(p, size, "%.3f", num) < size);
 	return (p);
 }
@@ -341,7 +341,7 @@ VRT_time_string(const struct sess *sp, double t)
 {
 	char *p;
 
-	AN(p = WS_Alloc(sp->http->ws, VTIM_FORMAT_SIZE));
+	AN(p = WS_Alloc(sp->req->http->ws, VTIM_FORMAT_SIZE));
 	VTIM_format(t, p);
 	return (p);
 }
@@ -371,8 +371,8 @@ void
 VRT_Rollback(struct sess *sp)
 {
 
-	HTTP_Copy(sp->http, sp->http0);
-	WS_Reset(sp->ws, sp->req->ws_req);
+	HTTP_Copy(sp->req->http, sp->req->http0);
+	WS_Reset(sp->req->ws, sp->req->ws_req);
 }
 
 /*--------------------------------------------------------------------*/
@@ -384,7 +384,7 @@ VRT_panic(const struct sess *sp, const char *str, ...)
 	char *b;
 
 	va_start(ap, str);
-	b = VRT_String(sp->http->ws, "PANIC: ", str, ap);
+	b = VRT_String(sp->req->http->ws, "PANIC: ", str, ap);
 	va_end(ap);
 	VAS_Fail("VCL", "", 0, b, 0, 2);
 }
diff --git a/bin/varnishd/cache/cache_vrt_re.c b/bin/varnishd/cache/cache_vrt_re.c
index e9ac9de..9864611 100644
--- a/bin/varnishd/cache/cache_vrt_re.c
+++ b/bin/varnishd/cache/cache_vrt_re.c
@@ -110,8 +110,8 @@ VRT_regsub(const struct sess *sp, int all, const char *str, void *re,
 		return(str);
 	}
 
-	u = WS_Reserve(sp->http->ws, 0);
-	res.e = res.b = b0 = sp->http->ws->f;
+	u = WS_Reserve(sp->req->http->ws, 0);
+	res.e = res.b = b0 = sp->req->http->ws->f;
 	res.e += u;
 
 	do {
@@ -143,7 +143,7 @@ VRT_regsub(const struct sess *sp, int all, const char *str, void *re,
 		i = VRE_exec(t, str, len, 0, options, ovector, 30,
 		    &cache_param->vre_limits);
 		if (i < VRE_ERROR_NOMATCH ) {
-			WS_Release(sp->http->ws, 0);
+			WS_Release(sp->req->http->ws, 0);
 			WSP(sp, SLT_VCL_Error,
 			    "Regexp matching returned %d", i);
 			return(str);
@@ -153,10 +153,10 @@ VRT_regsub(const struct sess *sp, int all, const char *str, void *re,
 	/* Copy suffix to match */
 	Tadd(&res, str, len+1);
 	if (res.b >= res.e) {
-		WS_Release(sp->http->ws, 0);
+		WS_Release(sp->req->http->ws, 0);
 		return (str);
 	}
 	Tcheck(res);
-	WS_ReleaseP(sp->http->ws, res.b);
+	WS_ReleaseP(sp->req->http->ws, res.b);
 	return (b0);
 }
diff --git a/bin/varnishd/cache/cache_vrt_var.c b/bin/varnishd/cache/cache_vrt_var.c
index 7ac9768..d128818 100644
--- a/bin/varnishd/cache/cache_vrt_var.c
+++ b/bin/varnishd/cache/cache_vrt_var.c
@@ -82,9 +82,9 @@ VRT_r_##obj##_##hdr(const struct sess *sp)			\
 	return (http->hd[fld].b);				\
 }
 
-VRT_DO_HDR(req,   request,	sp->http,		HTTP_HDR_REQ)
-VRT_DO_HDR(req,   url,		sp->http,		HTTP_HDR_URL)
-VRT_DO_HDR(req,   proto,	sp->http,		HTTP_HDR_PROTO)
+VRT_DO_HDR(req,   request,	sp->req->http,		HTTP_HDR_REQ)
+VRT_DO_HDR(req,   url,		sp->req->http,		HTTP_HDR_URL)
+VRT_DO_HDR(req,   proto,	sp->req->http,		HTTP_HDR_PROTO)
 VRT_DO_HDR(bereq, request,	sp->wrk->busyobj->bereq,	HTTP_HDR_REQ)
 VRT_DO_HDR(bereq, url,		sp->wrk->busyobj->bereq,	HTTP_HDR_URL)
 VRT_DO_HDR(bereq, proto,	sp->wrk->busyobj->bereq,	HTTP_HDR_PROTO)
@@ -222,7 +222,7 @@ VRT_l_client_identity(struct sess *sp, const char *str, ...)
 	char *b;
 
 	va_start(ap, str);
-	b = VRT_String(sp->http->ws, NULL, str, ap);
+	b = VRT_String(sp->req->http->ws, NULL, str, ap);
 	va_end(ap);
 	sp->req->client_identity = b;
 }
@@ -434,7 +434,7 @@ VRT_r_req_xid(struct sess *sp)
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 
 	size = snprintf(NULL, 0, "%u", sp->req->xid) + 1;
-	AN(p = WS_Alloc(sp->http->ws, size));
+	AN(p = WS_Alloc(sp->req->http->ws, size));
 	assert(snprintf(p, size, "%u", sp->req->xid) < size);
 	return (p);
 }
diff --git a/bin/varnishd/mgt/mgt_param.c b/bin/varnishd/mgt/mgt_param.c
index 2f7bd53..81aaf0d 100644
--- a/bin/varnishd/mgt/mgt_param.c
+++ b/bin/varnishd/mgt/mgt_param.c
@@ -688,7 +688,7 @@ static const struct parspec input_parspec[] = {
 		0,
 		"120", "seconds" },
 	{ "sess_workspace",
-		tweak_bytes_u, &mgt_param.sess_workspace, 1024, UINT_MAX,
+		tweak_bytes_u, &mgt_param.sess_workspace, 3072, UINT_MAX,
 		"Bytes of HTTP protocol workspace allocated for sessions. "
 		"This space must be big enough for the entire HTTP protocol "
 		"header and any edits done to it in the VCL code.\n"



From geoff at varnish-cache.org  Mon Jan  9 23:03:32 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Tue, 10 Jan 2012 00:03:32 +0100
Subject: [experimental-ims] 372312e Move resp.* to workspace_client and struct
	req
Message-ID: <E1RkOFY-0007w6-7V@project.varnish-software.com>

commit 372312e0fc5047be926ff7a5d6dc501fe578628d
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sun Dec 25 14:22:32 2011 +0000

    Move resp.* to workspace_client and struct req
    
    Retire a tiresome testcase to .disabled.
    
    Constify some VRT prototypes.

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index 50c1ec0..c048b5a 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -329,9 +329,6 @@ struct worker {
 
 	struct ws		ws[1];
 
-
-	struct http		*resp;
-
 	struct object		*obj;
 	struct objcore		*objcore;
 	struct busyobj		*busyobj;
@@ -622,6 +619,7 @@ struct req {
 	/* HTTP request */
 	struct http		*http;
 	struct http		*http0;
+	struct http		*resp;
 
 	struct ws		ws[1];
 };
diff --git a/bin/varnishd/cache/cache_center.c b/bin/varnishd/cache/cache_center.c
index 6bf3f6a..0154577 100644
--- a/bin/varnishd/cache/cache_center.c
+++ b/bin/varnishd/cache/cache_center.c
@@ -281,7 +281,7 @@ cnt_prepresp(struct sess *sp)
 			wrk->obj->last_lru = sp->req->t_resp;
 		wrk->obj->last_use = sp->req->t_resp;	/* XXX: locking ? */
 	}
-	http_Setup(wrk->resp, wrk->ws);
+	http_Setup(sp->req->resp, sp->req->ws);
 	RES_BuildHttp(sp);
 	VCL_deliver_method(sp);
 	switch (sp->req->handling) {
@@ -301,7 +301,7 @@ cnt_prepresp(struct sess *sp)
 		AZ(wrk->obj);
 		sp->req->restarts++;
 		sp->req->director = NULL;
-		http_Setup(wrk->resp, NULL);
+		http_Setup(sp->req->resp, NULL);
 		sp->step = STP_RECV;
 		return (0);
 	default:
@@ -349,7 +349,7 @@ cnt_deliver(struct sess *sp)
 	assert(WRW_IsReleased(wrk));
 	assert(wrk->wrw.ciov == wrk->wrw.siov);
 	(void)HSH_Deref(wrk, NULL, &wrk->obj);
-	http_Setup(wrk->resp, NULL);
+	http_Setup(sp->req->resp, NULL);
 	sp->step = STP_DONE;
 	return (0);
 }
@@ -1012,7 +1012,7 @@ cnt_streambody(struct sess *sp)
 	assert(wrk->wrw.ciov == wrk->wrw.siov);
 	(void)HSH_Deref(wrk, NULL, &wrk->obj);
 	VBO_DerefBusyObj(wrk, &wrk->busyobj);
-	http_Setup(wrk->resp, NULL);
+	http_Setup(sp->req->resp, NULL);
 	sp->step = STP_DONE;
 	return (0);
 }
diff --git a/bin/varnishd/cache/cache_panic.c b/bin/varnishd/cache/cache_panic.c
index a35c258..9c26eef 100644
--- a/bin/varnishd/cache/cache_panic.c
+++ b/bin/varnishd/cache/cache_panic.c
@@ -201,8 +201,6 @@ pan_wrk(const struct worker *wrk)
 		pan_http("bereq", wrk->busyobj->bereq, 4);
 	if (wrk->busyobj != NULL && wrk->busyobj->beresp->ws != NULL)
 		pan_http("beresp", wrk->busyobj->beresp, 4);
-	if (wrk->resp->ws != NULL)
-		pan_http("resp", wrk->resp, 4);
 	VSB_printf(pan_vsp, "  },\n");
 }
 
@@ -267,6 +265,8 @@ pan_sess(const struct sess *sp)
 
 	pan_ws(sp->req->ws, 2);
 	pan_http("req", sp->req->http, 2);
+	if (sp->req->resp->ws != NULL)
+		pan_http("resp", sp->req->resp, 4);
 
 	if (sp->wrk != NULL)
 		pan_wrk(sp->wrk);
diff --git a/bin/varnishd/cache/cache_pool.c b/bin/varnishd/cache/cache_pool.c
index 8ab73fe..7b1471e 100644
--- a/bin/varnishd/cache/cache_pool.c
+++ b/bin/varnishd/cache/cache_pool.c
@@ -194,7 +194,6 @@ Pool_Work_Thread(void *priv, struct worker *w)
 		w->do_what = pool_do_inval;
 
 		CHECK_OBJ_NOTNULL(w, WORKER_MAGIC);
-		CHECK_OBJ_NOTNULL(w->resp, HTTP_MAGIC);
 
 		WS_Reset(w->ws, NULL);
 
@@ -267,7 +266,6 @@ Pool_Work_Thread(void *priv, struct worker *w)
 
 			WS_Assert(w->ws);
 			AZ(w->busyobj);
-			AZ(w->resp->ws);
 			AZ(w->wrw.wfd);
 			AZ(w->storage_hint);
 			assert(w->wlp == w->wlb);
diff --git a/bin/varnishd/cache/cache_response.c b/bin/varnishd/cache/cache_response.c
index 2398e5b..5356b7b 100644
--- a/bin/varnishd/cache/cache_response.c
+++ b/bin/varnishd/cache/cache_response.c
@@ -87,14 +87,14 @@ res_dorange(const struct sess *sp, const char *r, ssize_t *plow, ssize_t *phigh)
 	if (low > high)
 		return;
 
-	http_PrintfHeader(sp->wrk, sp->vsl_id, sp->wrk->resp,
+	http_PrintfHeader(sp->wrk, sp->vsl_id, sp->req->resp,
 	    "Content-Range: bytes %jd-%jd/%jd",
 	    (intmax_t)low, (intmax_t)high, (intmax_t)sp->wrk->obj->len);
-	http_Unset(sp->wrk->resp, H_Content_Length);
+	http_Unset(sp->req->resp, H_Content_Length);
 	assert(sp->wrk->res_mode & RES_LEN);
-	http_PrintfHeader(sp->wrk, sp->vsl_id, sp->wrk->resp,
+	http_PrintfHeader(sp->wrk, sp->vsl_id, sp->req->resp,
 	    "Content-Length: %jd", (intmax_t)(1 + high - low));
-	http_SetResp(sp->wrk->resp, "HTTP/1.1", 206, "Partial Content");
+	http_SetResp(sp->req->resp, "HTTP/1.1", 206, "Partial Content");
 
 	*plow = low;
 	*phigh = high;
@@ -110,39 +110,39 @@ RES_BuildHttp(const struct sess *sp)
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 
 
-	http_ClrHeader(sp->wrk->resp);
-	sp->wrk->resp->logtag = HTTP_Tx;
-	http_CopyResp(sp->wrk->resp, sp->wrk->obj->http);
-	http_FilterFields(sp->wrk, sp->vsl_id, sp->wrk->resp,
+	http_ClrHeader(sp->req->resp);
+	sp->req->resp->logtag = HTTP_Tx;
+	http_CopyResp(sp->req->resp, sp->wrk->obj->http);
+	http_FilterFields(sp->wrk, sp->vsl_id, sp->req->resp,
 	    sp->wrk->obj->http, HTTPH_A_DELIVER);
 
 	if (!(sp->wrk->res_mode & RES_LEN)) {
-		http_Unset(sp->wrk->resp, H_Content_Length);
+		http_Unset(sp->req->resp, H_Content_Length);
 	} else if (cache_param->http_range_support) {
 		/* We only accept ranges if we know the length */
-		http_SetHeader(sp->wrk, sp->vsl_id, sp->wrk->resp,
+		http_SetHeader(sp->wrk, sp->vsl_id, sp->req->resp,
 		    "Accept-Ranges: bytes");
 	}
 
 	if (sp->wrk->res_mode & RES_CHUNKED)
-		http_SetHeader(sp->wrk, sp->vsl_id, sp->wrk->resp,
+		http_SetHeader(sp->wrk, sp->vsl_id, sp->req->resp,
 		    "Transfer-Encoding: chunked");
 
 	VTIM_format(VTIM_real(), time_str);
-	http_PrintfHeader(sp->wrk, sp->vsl_id, sp->wrk->resp,
+	http_PrintfHeader(sp->wrk, sp->vsl_id, sp->req->resp,
 	    "Date: %s", time_str);
 
 	if (sp->req->xid != sp->wrk->obj->xid)
-		http_PrintfHeader(sp->wrk, sp->vsl_id, sp->wrk->resp,
+		http_PrintfHeader(sp->wrk, sp->vsl_id, sp->req->resp,
 		    "X-Varnish: %u %u", sp->req->xid, sp->wrk->obj->xid);
 	else
-		http_PrintfHeader(sp->wrk, sp->vsl_id, sp->wrk->resp,
+		http_PrintfHeader(sp->wrk, sp->vsl_id, sp->req->resp,
 		    "X-Varnish: %u", sp->req->xid);
-	http_PrintfHeader(sp->wrk, sp->vsl_id, sp->wrk->resp, "Age: %.0f",
+	http_PrintfHeader(sp->wrk, sp->vsl_id, sp->req->resp, "Age: %.0f",
 	    sp->wrk->obj->exp.age + sp->req->t_resp -
 	    sp->wrk->obj->exp.entered);
-	http_SetHeader(sp->wrk, sp->vsl_id, sp->wrk->resp, "Via: 1.1 varnish");
-	http_PrintfHeader(sp->wrk, sp->vsl_id, sp->wrk->resp, "Connection: %s",
+	http_SetHeader(sp->wrk, sp->vsl_id, sp->req->resp, "Via: 1.1 varnish");
+	http_PrintfHeader(sp->wrk, sp->vsl_id, sp->req->resp, "Connection: %s",
 	    sp->req->doclose ? "close" : "keep-alive");
 }
 
@@ -263,9 +263,9 @@ RES_WriteObj(struct sess *sp)
 	    sp->req->http->conds &&
 	    RFC2616_Do_Cond(sp)) {
 		sp->req->wantbody = 0;
-		http_SetResp(sp->wrk->resp, "HTTP/1.1", 304, "Not Modified");
-		http_Unset(sp->wrk->resp, H_Content_Length);
-		http_Unset(sp->wrk->resp, H_Transfer_Encoding);
+		http_SetResp(sp->req->resp, "HTTP/1.1", 304, "Not Modified");
+		http_Unset(sp->req->resp, H_Content_Length);
+		http_Unset(sp->req->resp, H_Transfer_Encoding);
 	}
 
 	/*
@@ -286,14 +286,14 @@ RES_WriteObj(struct sess *sp)
 	 * Always remove C-E if client don't grok it
 	 */
 	if (sp->wrk->res_mode & RES_GUNZIP)
-		http_Unset(sp->wrk->resp, H_Content_Encoding);
+		http_Unset(sp->req->resp, H_Content_Encoding);
 
 	/*
 	 * Send HTTP protocol header, unless interior ESI object
 	 */
 	if (!(sp->wrk->res_mode & RES_ESI_CHILD))
 		sp->wrk->acct_tmp.hdrbytes +=
-		    http_Write(sp->wrk, sp->vsl_id, sp->wrk->resp, 1);
+		    http_Write(sp->wrk, sp->vsl_id, sp->req->resp, 1);
 
 	if (!sp->req->wantbody)
 		sp->wrk->res_mode &= ~RES_CHUNKED;
@@ -344,15 +344,15 @@ RES_StreamStart(struct sess *sp)
 	 * Always remove C-E if client don't grok it
 	 */
 	if (sp->wrk->res_mode & RES_GUNZIP)
-		http_Unset(sp->wrk->resp, H_Content_Encoding);
+		http_Unset(sp->req->resp, H_Content_Encoding);
 
 	if (!(sp->wrk->res_mode & RES_CHUNKED) &&
 	    sp->wrk->busyobj->h_content_length != NULL)
-		http_PrintfHeader(sp->wrk, sp->vsl_id, sp->wrk->resp,
+		http_PrintfHeader(sp->wrk, sp->vsl_id, sp->req->resp,
 		    "Content-Length: %s", sp->wrk->busyobj->h_content_length);
 
 	sp->wrk->acct_tmp.hdrbytes +=
-	    http_Write(sp->wrk, sp->vsl_id, sp->wrk->resp, 1);
+	    http_Write(sp->wrk, sp->vsl_id, sp->req->resp, 1);
 
 	if (sp->wrk->res_mode & RES_CHUNKED)
 		WRW_Chunked(sp->wrk);
diff --git a/bin/varnishd/cache/cache_session.c b/bin/varnishd/cache/cache_session.c
index 0091407..ee5060f 100644
--- a/bin/varnishd/cache/cache_session.c
+++ b/bin/varnishd/cache/cache_session.c
@@ -270,14 +270,18 @@ SES_GetReq(struct sess *sp)
 	nhttp = (uint16_t)cache_param->http_max_hdr;
 	hl = HTTP_estimate(nhttp);
 
-	xxxassert(sz > 2 * hl + 128);
+	xxxassert(sz > 3 * hl + 128);
 
 	sp->req->http = HTTP_create(p, nhttp);
-	p += hl;
+	p += hl;		// XXX: align ?
 	sz -= hl;
 
 	sp->req->http0 = HTTP_create(p, nhttp);
-	p += hl;
+	p += hl;		// XXX: align ?
+	sz -= hl;
+
+	sp->req->resp = HTTP_create(p, nhttp);
+	p += hl;		// XXX: align ?
 	sz -= hl;
 
 	WS_Init(sp->req->ws, "req", p, sz);
diff --git a/bin/varnishd/cache/cache_vrt.c b/bin/varnishd/cache/cache_vrt.c
index 7f27e8b..da50c44 100644
--- a/bin/varnishd/cache/cache_vrt.c
+++ b/bin/varnishd/cache/cache_vrt.c
@@ -52,7 +52,7 @@ const void * const vrt_magic_string_end = &vrt_magic_string_end;
 /*--------------------------------------------------------------------*/
 
 void
-VRT_error(struct sess *sp, unsigned code, const char *reason)
+VRT_error(const struct sess *sp, unsigned code, const char *reason)
 {
 
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
@@ -106,7 +106,7 @@ vrt_selecthttp(const struct sess *sp, enum gethdr_e where)
 		hp = sp->wrk->busyobj->beresp;
 		break;
 	case HDR_RESP:
-		hp = sp->wrk->resp;
+		hp = sp->req->resp;
 		break;
 	case HDR_OBJ:
 		CHECK_OBJ_NOTNULL(sp->wrk->obj, OBJECT_MAGIC);
@@ -239,7 +239,7 @@ VRT_SetHdr(const struct sess *sp , enum gethdr_e where, const char *hdr,
 /*--------------------------------------------------------------------*/
 
 void
-VRT_handling(struct sess *sp, unsigned hand)
+VRT_handling(const struct sess *sp, unsigned hand)
 {
 
 	if (sp == NULL) {
@@ -368,7 +368,7 @@ VRT_bool_string(const struct sess *sp, unsigned val)
 /*--------------------------------------------------------------------*/
 
 void
-VRT_Rollback(struct sess *sp)
+VRT_Rollback(const struct sess *sp)
 {
 
 	HTTP_Copy(sp->req->http, sp->req->http0);
diff --git a/bin/varnishd/cache/cache_vrt_var.c b/bin/varnishd/cache/cache_vrt_var.c
index d128818..6e5c751 100644
--- a/bin/varnishd/cache/cache_vrt_var.c
+++ b/bin/varnishd/cache/cache_vrt_var.c
@@ -90,8 +90,8 @@ VRT_DO_HDR(bereq, url,		sp->wrk->busyobj->bereq,	HTTP_HDR_URL)
 VRT_DO_HDR(bereq, proto,	sp->wrk->busyobj->bereq,	HTTP_HDR_PROTO)
 VRT_DO_HDR(obj,   proto,	sp->wrk->obj->http,	HTTP_HDR_PROTO)
 VRT_DO_HDR(obj,   response,	sp->wrk->obj->http,	HTTP_HDR_RESPONSE)
-VRT_DO_HDR(resp,  proto,	sp->wrk->resp,		HTTP_HDR_PROTO)
-VRT_DO_HDR(resp,  response,	sp->wrk->resp,		HTTP_HDR_RESPONSE)
+VRT_DO_HDR(resp,  proto,	sp->req->resp,		HTTP_HDR_PROTO)
+VRT_DO_HDR(resp,  response,	sp->req->resp,		HTTP_HDR_RESPONSE)
 VRT_DO_HDR(beresp,  proto,	sp->wrk->busyobj->beresp,	HTTP_HDR_PROTO)
 VRT_DO_HDR(beresp,  response,	sp->wrk->busyobj->beresp, HTTP_HDR_RESPONSE)
 
@@ -116,7 +116,7 @@ VRT_r_##obj##_status(const struct sess *sp)			\
 
 VRT_DO_STATUS(obj, sp->wrk->obj->http)
 VRT_DO_STATUS(beresp, sp->wrk->busyobj->beresp)
-VRT_DO_STATUS(resp, sp->wrk->resp)
+VRT_DO_STATUS(resp, sp->req->resp)
 
 /*--------------------------------------------------------------------*/
 
@@ -205,8 +205,8 @@ VBERESP(beresp, unsigned, do_stream, busyobj->do_stream)
 
 /*--------------------------------------------------------------------*/
 
-const char * __match_proto__()
-VRT_r_client_identity(struct sess *sp)
+const char *
+VRT_r_client_identity(const struct sess *sp)
 {
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 	if (sp->req->client_identity != NULL)
@@ -216,7 +216,7 @@ VRT_r_client_identity(struct sess *sp)
 }
 
 void
-VRT_l_client_identity(struct sess *sp, const char *str, ...)
+VRT_l_client_identity(const struct sess *sp, const char *str, ...)
 {
 	va_list ap;
 	char *b;
@@ -304,14 +304,14 @@ VRT_l_beresp_storage(struct sess *sp, const char *str, ...)
 /*--------------------------------------------------------------------*/
 
 void
-VRT_l_req_backend(struct sess *sp, struct director *be)
+VRT_l_req_backend(const struct sess *sp, struct director *be)
 {
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 	sp->req->director = be;
 }
 
-struct director * __match_proto__()
-VRT_r_req_backend(struct sess *sp)
+struct director *
+VRT_r_req_backend(const struct sess *sp)
 {
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 	return (sp->req->director);
@@ -320,7 +320,7 @@ VRT_r_req_backend(struct sess *sp)
 /*--------------------------------------------------------------------*/
 
 void
-VRT_l_req_esi(struct sess *sp, unsigned process_esi)
+VRT_l_req_esi(const struct sess *sp, unsigned process_esi)
 {
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 	/*
@@ -331,8 +331,8 @@ VRT_l_req_esi(struct sess *sp, unsigned process_esi)
 		sp->req->disable_esi = !process_esi;
 }
 
-unsigned __match_proto__()
-VRT_r_req_esi(struct sess *sp)
+unsigned
+VRT_r_req_esi(const struct sess *sp)
 {
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 	return (!sp->req->disable_esi);
diff --git a/bin/varnishd/cache/cache_wrk.c b/bin/varnishd/cache/cache_wrk.c
index 6b1d9dd..ce665bc 100644
--- a/bin/varnishd/cache/cache_wrk.c
+++ b/bin/varnishd/cache/cache_wrk.c
@@ -133,13 +133,12 @@ WRK_BgThread(pthread_t *thr, const char *name, bgthread_t *func, void *priv)
 
 static void *
 wrk_thread_real(void *priv, unsigned shm_workspace, unsigned sess_workspace,
-    uint16_t nhttp, unsigned http_space, unsigned siov)
+    unsigned siov)
 {
 	struct worker *w, ww;
 	uint32_t wlog[shm_workspace / 4];
 	/* XXX: can we trust these to be properly aligned ? */
 	unsigned char ws[sess_workspace];
-	unsigned char http2[http_space];
 	struct iovec iov[siov];
 	struct SHA256Context sha256;
 
@@ -151,7 +150,6 @@ wrk_thread_real(void *priv, unsigned shm_workspace, unsigned sess_workspace,
 	w->wlb = w->wlp = wlog;
 	w->wle = wlog + (sizeof wlog) / 4;
 	w->sha256ctx = &sha256;
-	w->resp = HTTP_create(http2, nhttp);
 	w->wrw.iov = iov;
 	w->wrw.siov = siov;
 	w->wrw.ciov = siov;
@@ -187,8 +185,7 @@ WRK_thread(void *priv)
 		siov = IOV_MAX;
 	return (wrk_thread_real(priv,
 	    cache_param->shm_workspace,
-	    cache_param->wthread_workspace,
-	    nhttp, HTTP_estimate(nhttp), siov));
+	    cache_param->wthread_workspace, siov));
 }
 
 void
diff --git a/bin/varnishtest/tests.disabled/r00693.vtc b/bin/varnishtest/tests.disabled/r00693.vtc
new file mode 100644
index 0000000..ff99ca7
--- /dev/null
+++ b/bin/varnishtest/tests.disabled/r00693.vtc
@@ -0,0 +1,65 @@
+varnishtest "check boundary condition on vrt_assemble_string()"
+
+feature 64bit
+
+server s1 {
+	rxreq
+	expect req.http.baz == <undef>
+	txresp -status 201
+
+	rxreq
+	expect req.http.baz == <undef>
+	txresp -status 202
+
+	rxreq
+	expect req.http.baz == "BAZ"
+	txresp -status 203
+
+} -start
+
+varnish v1 -arg "-p workspace_client=3072" -vcl+backend {
+
+	sub vcl_recv {
+		set req.http.foo = 
+			req.http.bar +
+			"0123456789abcdef" + "0123456789abcdef" +
+			"0123456789abcdef" + "0123456789abcdef" +
+			"0123456789abcdef" + "0123456789abcdef" +
+			"0123456789abcdef" + "0123456789abcdef" +
+			"0123456789abcdef" + "0123456789abcdef" +
+			"0123456789abcdef" + "0123456789abcdef" +
+			"0123456789abcdef" + "0123456789abcdef" +
+			"0123456789abcdef" + "0123456789abcdef" +
+			"0123456789abcdef" + "0123456789abcdef" +
+			"0123456789abcdef" + "0123456789abcdef" +
+			"0123456789abcdef" + "0123456789abcdef" +
+			"0123456789abcdef" + "0123456789abcdef" +
+			"0123456789abcdef" + "0123456789abcdef" +
+			"0123456789abcde";
+		set req.http.baz = "BAZ";
+		return (pass);
+	}
+	sub vcl_hash {
+		hash_data(req.url);
+		return (hash);
+	}
+} -start
+
+client c1 {
+	# This should soak up all bytes but the last in the workspace
+	txreq -hdr "foo: x" -hdr "bar: A" 
+	rxresp
+	expect resp.status == 201
+
+	# This should soak up all bytes in the workspace
+	txreq -hdr "foo: x" -hdr "bar: AB" 
+	rxresp
+	expect resp.status == 202
+
+	# This overcommits the workspace, failing the "bar" set,
+	# Thus allowing the "baz" set to work.
+	txreq -hdr "foo: x" -hdr "bar: ABC" 
+	rxresp
+	expect resp.status == 203
+
+} -run
diff --git a/bin/varnishtest/tests/r00693.vtc b/bin/varnishtest/tests/r00693.vtc
deleted file mode 100644
index ff99ca7..0000000
--- a/bin/varnishtest/tests/r00693.vtc
+++ /dev/null
@@ -1,65 +0,0 @@
-varnishtest "check boundary condition on vrt_assemble_string()"
-
-feature 64bit
-
-server s1 {
-	rxreq
-	expect req.http.baz == <undef>
-	txresp -status 201
-
-	rxreq
-	expect req.http.baz == <undef>
-	txresp -status 202
-
-	rxreq
-	expect req.http.baz == "BAZ"
-	txresp -status 203
-
-} -start
-
-varnish v1 -arg "-p workspace_client=3072" -vcl+backend {
-
-	sub vcl_recv {
-		set req.http.foo = 
-			req.http.bar +
-			"0123456789abcdef" + "0123456789abcdef" +
-			"0123456789abcdef" + "0123456789abcdef" +
-			"0123456789abcdef" + "0123456789abcdef" +
-			"0123456789abcdef" + "0123456789abcdef" +
-			"0123456789abcdef" + "0123456789abcdef" +
-			"0123456789abcdef" + "0123456789abcdef" +
-			"0123456789abcdef" + "0123456789abcdef" +
-			"0123456789abcdef" + "0123456789abcdef" +
-			"0123456789abcdef" + "0123456789abcdef" +
-			"0123456789abcdef" + "0123456789abcdef" +
-			"0123456789abcdef" + "0123456789abcdef" +
-			"0123456789abcdef" + "0123456789abcdef" +
-			"0123456789abcdef" + "0123456789abcdef" +
-			"0123456789abcde";
-		set req.http.baz = "BAZ";
-		return (pass);
-	}
-	sub vcl_hash {
-		hash_data(req.url);
-		return (hash);
-	}
-} -start
-
-client c1 {
-	# This should soak up all bytes but the last in the workspace
-	txreq -hdr "foo: x" -hdr "bar: A" 
-	rxresp
-	expect resp.status == 201
-
-	# This should soak up all bytes in the workspace
-	txreq -hdr "foo: x" -hdr "bar: AB" 
-	rxresp
-	expect resp.status == 202
-
-	# This overcommits the workspace, failing the "bar" set,
-	# Thus allowing the "baz" set to work.
-	txreq -hdr "foo: x" -hdr "bar: ABC" 
-	rxresp
-	expect resp.status == 203
-
-} -run
diff --git a/include/vrt.h b/include/vrt.h
index 72c4cfe..bbf16ac 100644
--- a/include/vrt.h
+++ b/include/vrt.h
@@ -156,14 +156,14 @@ void VRT_purge(const struct sess *sp, double ttl, double grace);
 
 void VRT_count(const struct sess *, unsigned);
 int VRT_rewrite(const char *, const char *);
-void VRT_error(struct sess *, unsigned, const char *);
+void VRT_error(const struct sess *, unsigned, const char *);
 int VRT_switch_config(const char *);
 
 enum gethdr_e { HDR_REQ, HDR_RESP, HDR_OBJ, HDR_BEREQ, HDR_BERESP };
 char *VRT_GetHdr(const struct sess *, enum gethdr_e where, const char *);
 void VRT_SetHdr(const struct sess *, enum gethdr_e where, const char *,
     const char *, ...);
-void VRT_handling(struct sess *sp, unsigned hand);
+void VRT_handling(const struct sess *sp, unsigned hand);
 
 void VRT_hashdata(const struct sess *sp, const char *str, ...);
 
@@ -172,7 +172,7 @@ int VRT_strcmp(const char *s1, const char *s2);
 void VRT_memmove(void *dst, const void *src, unsigned len);
 
 void VRT_ESI(const struct sess *sp);
-void VRT_Rollback(struct sess *sp);
+void VRT_Rollback(const struct sess *sp);
 
 /* Synthetic pages */
 void VRT_synth_page(const struct sess *sp, unsigned flags, const char *, ...);
diff --git a/lib/libvcl/generate.py b/lib/libvcl/generate.py
index dde0e89..b91f675 100755
--- a/lib/libvcl/generate.py
+++ b/lib/libvcl/generate.py
@@ -113,7 +113,7 @@ sp_variables = (
 		'STRING',
 		( 'proc',),
 		( 'proc',),
-		'struct sess *'
+		'const struct sess *'
 	),
 	('server.ip',
 		'IP',
@@ -167,7 +167,7 @@ sp_variables = (
 		'BACKEND',
 		( 'proc',),
 		( 'proc',),
-		'struct sess *'
+		'const struct sess *'
 	),
 	('req.restarts',
 		'INT',
@@ -209,7 +209,7 @@ sp_variables = (
 		'BOOL',
 		( 'recv', 'fetch', 'deliver', 'error',),
 		( 'recv', 'fetch', 'deliver', 'error',),
-		'struct sess *'
+		'const struct sess *'
 	),
 	('req.can_gzip',
 		'BOOL',



From geoff at varnish-cache.org  Mon Jan  9 23:03:30 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Tue, 10 Jan 2012 00:03:30 +0100
Subject: [experimental-ims] 559283b Rename sp->t_end to sp->t_idle,
	which marks when we start listening for a request.
Message-ID: <E1RkOFW-0007tQ-Ec@project.varnish-software.com>

commit 559283b13b680af6f2f54de986fed2ac6d43d56a
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu Dec 22 22:22:23 2011 +0000

    Rename sp->t_end to sp->t_idle, which marks when we start listening
    for a request.
    
    t_open and t_idle are session timestamps.
    
    Have the waiters pass in a T(now) timestamp

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index 58419bc..7e7088e 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -616,6 +616,10 @@ struct sess {
 
 	VTAILQ_ENTRY(sess)	list;
 
+	/* Timestamps, all on TIM_real() timescale */
+	double			t_open;		/* fd accepted */
+	double			t_idle;		/* fd accepted or resp sent */
+
 #if defined(HAVE_EPOLL_CTL)
 	struct epoll_event ev;
 #endif
@@ -665,15 +669,13 @@ struct sess {
 
 	uint64_t		req_bodybytes;
 
-	/* TBD fields ------------------------------------------------*/
-
-	struct http_conn	htc[1];
-
 	/* Timestamps, all on TIM_real() timescale */
-	double			t_open;
 	double			t_req;
 	double			t_resp;
-	double			t_end;
+
+	/* TBD fields ------------------------------------------------*/
+
+	struct http_conn	htc[1];
 
 };
 
@@ -915,12 +917,12 @@ void WRW_Sendfile(struct worker *w, int fd, off_t off, unsigned len);
 struct sess *SES_New(struct worker *wrk, struct sesspool *pp);
 struct sess *SES_Alloc(void);
 void SES_Close(struct sess *sp, const char *reason);
-void SES_Delete(struct sess *sp, const char *reason);
+void SES_Delete(struct sess *sp, const char *reason, double now);
 void SES_Charge(struct sess *sp);
 struct sesspool *SES_NewPool(struct pool *pp, unsigned pool_no);
 void SES_DeletePool(struct sesspool *sp, struct worker *wrk);
 int SES_Schedule(struct sess *sp);
-void SES_Handle(struct sess *sp);
+void SES_Handle(struct sess *sp, double now);
 
 /* cache_shmlog.c */
 extern struct VSC_C_main *VSC_C_main;
diff --git a/bin/varnishd/cache/cache_acceptor.c b/bin/varnishd/cache/cache_acceptor.c
index 87e51a9..1c6d6fb 100644
--- a/bin/varnishd/cache/cache_acceptor.c
+++ b/bin/varnishd/cache/cache_acceptor.c
@@ -276,7 +276,7 @@ VCA_SetupSess(struct worker *w)
 	sp->vsl_id = wa->acceptsock | VSL_CLIENTMARKER ;
 	wa->acceptsock = -1;
 	sp->t_open = VTIM_real();
-	sp->t_end = sp->t_open;
+	sp->t_idle = sp->t_open;
 	sp->mylsock = wa->acceptlsock;
 	CHECK_OBJ_NOTNULL(sp->mylsock, LISTEN_SOCK_MAGIC);
 	assert(wa->acceptaddrlen <= sp->sockaddrlen);
diff --git a/bin/varnishd/cache/cache_center.c b/bin/varnishd/cache/cache_center.c
index 1d92ded..f926932 100644
--- a/bin/varnishd/cache/cache_center.c
+++ b/bin/varnishd/cache/cache_center.c
@@ -365,15 +365,14 @@ cnt_done(struct sess *sp)
 	}
 
 
-	sp->t_end = W_TIM_real(wrk);
+	sp->t_idle = W_TIM_real(wrk);
 WSP(sp, SLT_Debug, "PHK req %.9f resp %.9f end %.9f open %.9f",
-    sp->t_req, sp->t_resp, sp->t_end,  sp->t_open);
+    sp->t_req, sp->t_resp, sp->t_idle,  sp->t_open);
 	if (sp->xid == 0) {
-		// sp->t_req = sp->t_end;
-		sp->t_resp = sp->t_end;
+		sp->t_resp = sp->t_idle;
 	} else {
 		dp = sp->t_resp - sp->t_req;
-		da = sp->t_end - sp->t_resp;
+		da = sp->t_idle - sp->t_resp;
 		dh = sp->t_req - sp->t_open;
 		/* XXX: Add StatReq == StatSess */
 		/* XXX: Workaround for pipe */
@@ -382,17 +381,16 @@ WSP(sp, SLT_Debug, "PHK req %.9f resp %.9f end %.9f open %.9f",
 			    (uintmax_t)sp->req_bodybytes);
 		}
 		WSP(sp, SLT_ReqEnd, "%u %.9f %.9f %.9f %.9f %.9f",
-		    sp->xid, sp->t_req, sp->t_end, dh, dp, da);
+		    sp->xid, sp->t_req, sp->t_idle, dh, dp, da);
 	}
 	sp->xid = 0;
 	WSL_Flush(wrk, 0);
 
-	sp->t_open = sp->t_end;
+	sp->t_req = NAN;
 	sp->t_resp = NAN;
 
 	sp->req_bodybytes = 0;
 
-	sp->t_req = NAN;
 	sp->hash_always_miss = 0;
 	sp->hash_ignore_busy = 0;
 
@@ -407,7 +405,7 @@ WSP(sp, SLT_Debug, "PHK req %.9f resp %.9f end %.9f open %.9f",
 
 	if (sp->fd < 0) {
 		wrk->stats.sess_closed++;
-		SES_Delete(sp, NULL);
+		SES_Delete(sp, NULL, NAN);
 		return (1);
 	}
 
diff --git a/bin/varnishd/cache/cache_session.c b/bin/varnishd/cache/cache_session.c
index 1b1f87c..46bfb8d 100644
--- a/bin/varnishd/cache/cache_session.c
+++ b/bin/varnishd/cache/cache_session.c
@@ -168,9 +168,9 @@ ses_setup(struct sessmem *sm)
 	sp->mysockaddrlen = sizeof(sp->mysockaddr);
 	sp->sockaddr.ss_family = sp->mysockaddr.ss_family = PF_UNSPEC;
 	sp->t_open = NAN;
+	sp->t_idle = NAN;
 	sp->t_req = NAN;
 	sp->t_resp = NAN;
-	sp->t_end = NAN;
 	EXP_Clr(&sp->exp);
 
 	WS_Init(sp->ws, "sess", sm->wsp, sm->workspace);
@@ -268,7 +268,7 @@ SES_Schedule(struct sess *sp)
 
 	if (Pool_Schedule(pp->pool, sp)) {
 		VSC_C_main->client_drop_late++;
-		sp->t_end = VTIM_real();
+		sp->t_idle = VTIM_real();
 		if (sp->vcl != NULL) {
 			/*
 			 * A session parked on a busy object can come here
@@ -276,7 +276,7 @@ SES_Schedule(struct sess *sp)
 			 */
 			VCL_Rel(&sp->vcl);
 		}
-		SES_Delete(sp, "dropped");
+		SES_Delete(sp, "dropped", sp->t_idle);
 		return (1);
 	}
 	return (0);
@@ -284,20 +284,21 @@ SES_Schedule(struct sess *sp)
 
 /*--------------------------------------------------------------------
  * Handle a session (from waiter)
- *
- * Status: see HTC_Rx()
  */
 
 void
-SES_Handle(struct sess *sp)
+SES_Handle(struct sess *sp, double now)
 {
 
 	sp->step = STP_WAIT;
+	sp->t_req = now;
 	(void)SES_Schedule(sp);
 }
 
 /*--------------------------------------------------------------------
  * Close a sessions connection.
+ * XXX: Technically speaking we should catch a t_end timestamp here
+ * XXX: for SES_Delete() to use.
  */
 
 void
@@ -322,7 +323,7 @@ SES_Close(struct sess *sp, const char *reason)
  */
 
 void
-SES_Delete(struct sess *sp, const char *reason)
+SES_Delete(struct sess *sp, const char *reason, double now)
 {
 	struct acct *b;
 	struct sessmem *sm;
@@ -340,9 +341,10 @@ SES_Delete(struct sess *sp, const char *reason)
 	wrk = sp->wrk;
 	CHECK_OBJ_ORNULL(wrk, WORKER_MAGIC);
 
-
 	if (reason != NULL)
 		SES_Close(sp, reason);
+	if (isnan(now))
+		now = VTIM_real();
 	assert(sp->fd < 0);
 
 	AZ(sp->vcl);
@@ -353,10 +355,10 @@ SES_Delete(struct sess *sp, const char *reason)
 
 	b = &sp->acct_ses;
 	assert(!isnan(b->first));
-	assert(!isnan(sp->t_end));
 
 	VSL(SLT_StatSess, sp->vsl_id, "%s %s %.0f %ju %ju %ju %ju %ju %ju %ju",
-	    sp->addr, sp->port, sp->t_end - b->first,
+	    sp->addr, sp->port,
+	    now - b->first, 	// XXX ??? 
 	    b->sess, b->req, b->pipe, b->pass,
 	    b->fetch, b->hdrbytes, b->bodybytes);
 
diff --git a/bin/varnishd/waiter/cache_waiter_epoll.c b/bin/varnishd/waiter/cache_waiter_epoll.c
index e71633a..e8f5e2a 100644
--- a/bin/varnishd/waiter/cache_waiter_epoll.c
+++ b/bin/varnishd/waiter/cache_waiter_epoll.c
@@ -104,7 +104,7 @@ vwe_cond_modadd(struct vwe *vwe, int fd, void *data)
 }
 
 static void
-vwe_eev(struct vwe *vwe, const struct epoll_event *ep)
+vwe_eev(struct vwe *vwe, const struct epoll_event *ep, double now)
 {
 	struct sess *ss[NEEV], *sp;
 	int i, j;
@@ -130,16 +130,16 @@ vwe_eev(struct vwe *vwe, const struct epoll_event *ep)
 		CAST_OBJ_NOTNULL(sp, ep->data.ptr, SESS_MAGIC);
 		if (ep->events & EPOLLIN || ep->events & EPOLLPRI) {
 			VTAILQ_REMOVE(&vwe->sesshead, sp, list);
-			SES_Handle(sp);
+			SES_Handle(sp, now);
 		} else if (ep->events & EPOLLERR) {
 			VTAILQ_REMOVE(&vwe->sesshead, sp, list);
-			SES_Delete(sp, "ERR");
+			SES_Delete(sp, "ERR", now);
 		} else if (ep->events & EPOLLHUP) {
 			VTAILQ_REMOVE(&vwe->sesshead, sp, list);
-			SES_Delete(sp, "HUP");
+			SES_Delete(sp, "HUP", now);
 		} else if (ep->events & EPOLLRDHUP) {
 			VTAILQ_REMOVE(&vwe->sesshead, sp, list);
-			SES_Delete(sp, "RHUP");
+			SES_Delete(sp, "RHUP", now);
 		}
 	}
 }
@@ -152,7 +152,7 @@ vwe_thread(void *priv)
 	struct epoll_event ev[NEEV], *ep;
 	struct sess *sp;
 	char junk;
-	double deadline;
+	double now, deadline;
 	int dotimer, i, n;
 	struct vwe *vwe;
 
@@ -169,6 +169,7 @@ vwe_thread(void *priv)
 	while (1) {
 		dotimer = 0;
 		n = epoll_wait(vwe->epfd, ev, NEEV, -1);
+		now = VTIM_real();
 		for (ep = ev, i = 0; i < n; i++, ep++) {
 			if (ep->data.ptr == vwe->timer_pipes &&
 			    (ep->events == EPOLLIN || ep->events == EPOLLPRI))
@@ -176,22 +177,22 @@ vwe_thread(void *priv)
 				assert(read(vwe->timer_pipes[0], &junk, 1));
 				dotimer = 1;
 			} else
-				vwe_eev(vwe, ep);
+				vwe_eev(vwe, ep, now);
 		}
 		if (!dotimer)
 			continue;
 
 		/* check for timeouts */
-		deadline = VTIM_real() - cache_param->sess_timeout;
+		deadline = now - cache_param->sess_timeout;
 		for (;;) {
 			sp = VTAILQ_FIRST(&vwe->sesshead);
 			if (sp == NULL)
 				break;
-			if (sp->t_open > deadline)
+			if (sp->t_idle > deadline)
 				break;
 			VTAILQ_REMOVE(&vwe->sesshead, sp, list);
 			// XXX: not yet VTCP_linger(sp->fd, 0);
-			SES_Delete(sp, "timeout");
+			SES_Delete(sp, "timeout", now);
 		}
 	}
 	return (NULL);
diff --git a/bin/varnishd/waiter/cache_waiter_kqueue.c b/bin/varnishd/waiter/cache_waiter_kqueue.c
index e11f945..707c902 100644
--- a/bin/varnishd/waiter/cache_waiter_kqueue.c
+++ b/bin/varnishd/waiter/cache_waiter_kqueue.c
@@ -114,7 +114,7 @@ vwk_pipe_ev(struct vwk *vwk, const struct kevent *kp)
 /*--------------------------------------------------------------------*/
 
 static void
-vwk_sess_ev(struct vwk *vwk, const struct kevent *kp)
+vwk_sess_ev(struct vwk *vwk, const struct kevent *kp, double now)
 {
 	struct sess *sp;
 
@@ -129,11 +129,11 @@ vwk_sess_ev(struct vwk *vwk, const struct kevent *kp)
 	assert((sp->vsl_id & VSL_IDENTMASK) == sp->fd);
 	if (kp->data > 0) {
 		VTAILQ_REMOVE(&vwk->sesshead, sp, list);
-		SES_Handle(sp);
+		SES_Handle(sp, now);
 		return;
 	} else if (kp->flags & EV_EOF) {
 		VTAILQ_REMOVE(&vwk->sesshead, sp, list);
-		SES_Delete(sp, "EOF");
+		SES_Delete(sp, "EOF", now);
 		return;
 	} else {
 		VSL(SLT_Debug, sp->vsl_id,
@@ -151,7 +151,7 @@ vwk_thread(void *priv)
 	struct vwk *vwk;
 	struct kevent ke[NKEV], *kp;
 	int j, n, dotimer;
-	double deadline;
+	double now, deadline;
 	struct sess *sp;
 
 	CAST_OBJ_NOTNULL(vwk, priv, VWK_MAGIC);
@@ -171,6 +171,7 @@ vwk_thread(void *priv)
 	while (1) {
 		dotimer = 0;
 		n = kevent(vwk->kq, vwk->ki, vwk->nki, ke, NKEV, NULL);
+		now = VTIM_real();
 		assert(n >= 1 && n <= NKEV);
 		vwk->nki = 0;
 		for (kp = ke, j = 0; j < n; j++, kp++) {
@@ -181,7 +182,7 @@ vwk_thread(void *priv)
 				vwk_pipe_ev(vwk, kp);
 			} else {
 				assert(kp->filter == EVFILT_READ);
-				vwk_sess_ev(vwk, kp);
+				vwk_sess_ev(vwk, kp, now);
 			}
 		}
 		if (!dotimer)
@@ -194,16 +195,16 @@ vwk_thread(void *priv)
 		 * would not know we meant "the old fd of this number".
 		 */
 		vwk_kq_flush(vwk);
-		deadline = VTIM_real() - cache_param->sess_timeout;
+		deadline = now - cache_param->sess_timeout;
 		for (;;) {
 			sp = VTAILQ_FIRST(&vwk->sesshead);
 			if (sp == NULL)
 				break;
-			if (sp->t_open > deadline)
+			if (sp->t_idle > deadline)
 				break;
 			VTAILQ_REMOVE(&vwk->sesshead, sp, list);
 			// XXX: not yet (void)VTCP_linger(sp->fd, 0);
-			SES_Delete(sp, "timeout");
+			SES_Delete(sp, "timeout", now);
 		}
 	}
 }
diff --git a/bin/varnishd/waiter/cache_waiter_poll.c b/bin/varnishd/waiter/cache_waiter_poll.c
index cf750ad..3a39c6b 100644
--- a/bin/varnishd/waiter/cache_waiter_poll.c
+++ b/bin/varnishd/waiter/cache_waiter_poll.c
@@ -123,7 +123,7 @@ vwp_main(void *priv)
 	int v, v2;
 	struct vwp *vwp;
 	struct sess *ss[NEEV], *sp, *sp2;
-	double deadline;
+	double now, deadline;
 	int i, j, fd;
 
 	CAST_OBJ_NOTNULL(vwp, priv, VWP_MAGIC);
@@ -140,7 +140,8 @@ vwp_main(void *priv)
 		assert(vwp->pollfd[vwp->pipes[1]].fd == -1);
 		v = poll(vwp->pollfd, vwp->hpoll + 1, 100);
 		assert(v >= 0);
-		deadline = VTIM_real() - cache_param->sess_timeout;
+		now = VTIM_real();
+		deadline = now - cache_param->sess_timeout;
 		v2 = v;
 		VTAILQ_FOREACH_SAFE(sp, &vwp->sesshead, list, sp2) {
 			if (v != 0 && v2 == 0)
@@ -156,12 +157,12 @@ vwp_main(void *priv)
 				vwp->pollfd[fd].revents = 0;
 				VTAILQ_REMOVE(&vwp->sesshead, sp, list);
 				vwp_unpoll(vwp, fd);
-				SES_Handle(sp);
-			} else if (sp->t_open <= deadline) {
+				SES_Handle(sp, now);
+			} else if (sp->t_idle <= deadline) {
 				VTAILQ_REMOVE(&vwp->sesshead, sp, list);
 				vwp_unpoll(vwp, fd);
 				// XXX: not yet (void)VTCP_linger(sp->fd, 0);
-				SES_Delete(sp, "timeout");
+				SES_Delete(sp, "timeout", now);
 			}
 		}
 		if (v2 && vwp->pollfd[vwp->pipes[0]].revents) {
diff --git a/bin/varnishd/waiter/cache_waiter_ports.c b/bin/varnishd/waiter/cache_waiter_ports.c
index 09db406..0d05775 100644
--- a/bin/varnishd/waiter/cache_waiter_ports.c
+++ b/bin/varnishd/waiter/cache_waiter_ports.c
@@ -72,7 +72,7 @@ vws_del(struct vws *vws, int fd)
 }
 
 static inline void
-vws_port_ev(struct vws *vws, port_event_t *ev) {
+vws_port_ev(struct vws *vws, port_event_t *ev, double now) {
 	struct sess *sp;
 	if(ev->portev_source == PORT_SOURCE_USER) {
 		CAST_OBJ_NOTNULL(sp, ev->portev_user, SESS_MAGIC);
@@ -87,7 +87,7 @@ vws_port_ev(struct vws *vws, port_event_t *ev) {
 		if(ev->portev_events & POLLERR) {
 			vws_del(vws, sp->fd);
 			VTAILQ_REMOVE(&vws->sesshead, sp, list);
-			SES_Delete(sp, "EOF");
+			SES_Delete(sp, "EOF", now);
 			return;
 		}
 
@@ -108,7 +108,7 @@ vws_port_ev(struct vws *vws, port_event_t *ev) {
 		VTAILQ_REMOVE(&vws->sesshead, sp, list);
 
 		/* SES_Handle will also handle errors */
-		SES_Handle(sp);
+		SES_Handle(sp, now);
 	}
 	return;
 }
@@ -182,15 +182,15 @@ vws_thread(void *priv)
 		 */
 
 		ret = port_getn(vws->dport, ev, MAX_EVENTS, &nevents, timeout);
+		now = VTIM_real();
 
 		if (ret < 0)
 			assert((errno == EINTR) || (errno == ETIME));
 
 		for (ei = 0; ei < nevents; ei++)
-			vws_port_ev(vws, ev + ei);
+			vws_port_ev(vws, ev + ei, now);
 
 		/* check for timeouts */
-		now = VTIM_real();
 		deadline = now - cache_param->sess_timeout;
 
 		/*
@@ -204,14 +204,14 @@ vws_thread(void *priv)
 			sp = VTAILQ_FIRST(&vws->sesshead);
 			if (sp == NULL)
 				break;
-			if (sp->t_open > deadline) {
+			if (sp->t_idle > deadline) {
 				break;
 			}
 			VTAILQ_REMOVE(&vws->sesshead, sp, list);
 			if(sp->fd != -1) {
 				vws_del(vws, sp->fd);
 			}
-			SES_Delete(sp, "timeout");
+			SES_Delete(sp, "timeout", now);
 		}
 
 		/*



From geoff at varnish-cache.org  Mon Jan  9 23:03:32 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Tue, 10 Jan 2012 00:03:32 +0100
Subject: [experimental-ims] ff61fc3 Rename the 'sess_workspace' param to
	'workspace_client'.
Message-ID: <E1RkOFY-0007vk-1t@project.varnish-software.com>

commit ff61fc3496a5f299617c7a026f10d129ef0e9297
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sun Dec 25 10:55:47 2011 +0000

    Rename the 'sess_workspace' param to 'workspace_client'.
    
    I'm not quite there yet, but in the end we will have two workspaces:
    req->ws for req.* and resp.* and busyobj->ws for bereq.* and beresp.*
    
    Various argument changes as consequence of the sess->req moves.

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index 9743398..50c1ec0 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -761,7 +761,7 @@ int FetchError(struct worker *w, const char *error);
 int FetchError2(struct worker *w, const char *error, const char *more);
 int FetchHdr(struct sess *sp, int need_host_hdr);
 int FetchBody(struct worker *w, struct object *obj);
-int FetchReqBody(struct sess *sp);
+int FetchReqBody(const struct sess *sp);
 void Fetch_Init(void);
 
 /* cache_gzip.c */
@@ -825,7 +825,7 @@ double http_GetHdrQ(const struct http *hp, const char *hdr, const char *field);
 uint16_t http_GetStatus(const struct http *hp);
 const char *http_GetReq(const struct http *hp);
 int http_HdrIs(const struct http *hp, const char *hdr, const char *val);
-uint16_t http_DissectRequest(struct sess *sp);
+uint16_t http_DissectRequest(const struct sess *sp);
 uint16_t http_DissectResponse(struct worker *w, const struct http_conn *htc,
     struct http *sp);
 const char *http_DoConnection(const struct http *hp);
@@ -910,13 +910,13 @@ void WRW_Sendfile(struct worker *w, int fd, off_t off, unsigned len);
 #endif  /* SENDFILE_WORKS */
 
 /* cache_session.c [SES] */
-struct sess *SES_New(struct worker *wrk, struct sesspool *pp);
+struct sess *SES_New(struct sesspool *pp);
 struct sess *SES_Alloc(void);
 void SES_Close(struct sess *sp, const char *reason);
 void SES_Delete(struct sess *sp, const char *reason, double now);
 void SES_Charge(struct sess *sp);
 struct sesspool *SES_NewPool(struct pool *pp, unsigned pool_no);
-void SES_DeletePool(struct sesspool *sp, struct worker *wrk);
+void SES_DeletePool(struct sesspool *sp);
 int SES_Schedule(struct sess *sp);
 void SES_Handle(struct sess *sp, double now);
 void SES_GetReq(struct sess *sp);
@@ -966,7 +966,7 @@ void RES_StreamPoll(struct worker *);
 
 /* cache_vary.c */
 struct vsb *VRY_Create(const struct sess *sp, const struct http *hp);
-int VRY_Match(struct sess *sp, const uint8_t *vary);
+int VRY_Match(const struct sess *sp, const uint8_t *vary);
 void VRY_Validate(const uint8_t *vary);
 
 /* cache_vcl.c */
diff --git a/bin/varnishd/cache/cache_ban.c b/bin/varnishd/cache/cache_ban.c
index 9162f50..a884983 100644
--- a/bin/varnishd/cache/cache_ban.c
+++ b/bin/varnishd/cache/cache_ban.c
@@ -618,9 +618,11 @@ ban_evaluate(const uint8_t *bs, const struct http *objhttp,
 		arg1 = NULL;
 		switch (bt.arg1) {
 		case BAN_ARG_URL:
+			AN(reqhttp);
 			arg1 = reqhttp->hd[HTTP_HDR_URL].b;
 			break;
 		case BAN_ARG_REQHTTP:
+			AN(reqhttp);
 			(void)http_GetHdr(reqhttp, bt.arg1_spec, &arg1);
 			break;
 		case BAN_ARG_OBJHTTP:
diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c
index 1e75f64..2e65591 100644
--- a/bin/varnishd/cache/cache_fetch.c
+++ b/bin/varnishd/cache/cache_fetch.c
@@ -329,7 +329,7 @@ fetch_eof(struct worker *wrk, struct http_conn *htc)
  */
 
 int
-FetchReqBody(struct sess *sp)
+FetchReqBody(const struct sess *sp)
 {
 	unsigned long content_length;
 	char buf[8192];
diff --git a/bin/varnishd/cache/cache_http.c b/bin/varnishd/cache/cache_http.c
index f7dad38..cb17df4 100644
--- a/bin/varnishd/cache/cache_http.c
+++ b/bin/varnishd/cache/cache_http.c
@@ -651,7 +651,7 @@ http_ProtoVer(struct http *hp)
 /*--------------------------------------------------------------------*/
 
 uint16_t
-http_DissectRequest(struct sess *sp)
+http_DissectRequest(const struct sess *sp)
 {
 	struct http_conn *htc;
 	struct http *hp;
diff --git a/bin/varnishd/cache/cache_pool.c b/bin/varnishd/cache/cache_pool.c
index 1a67548..8ab73fe 100644
--- a/bin/varnishd/cache/cache_pool.c
+++ b/bin/varnishd/cache/cache_pool.c
@@ -238,7 +238,7 @@ Pool_Work_Thread(void *priv, struct worker *w)
 			/* Turn accepted socket into a session */
 			AZ(w->sp);
 			AN(w->ws->r);
-			w->sp = SES_New(w, pp->sesspool);
+			w->sp = SES_New(pp->sesspool);
 			if (w->sp == NULL) {
 				VCA_FailSess(w);
 				w->do_what = pool_do_nothing;
@@ -538,7 +538,7 @@ pool_poolherder(void *priv)
 		}
 		/* XXX: remove pools */
 		if (0)
-			SES_DeletePool(NULL, NULL);
+			SES_DeletePool(NULL);
 		(void)sleep(1);
 		u = 0;
 		VTAILQ_FOREACH(pp, &pools, list)
diff --git a/bin/varnishd/cache/cache_session.c b/bin/varnishd/cache/cache_session.c
index 552c078..0091407 100644
--- a/bin/varnishd/cache/cache_session.c
+++ b/bin/varnishd/cache/cache_session.c
@@ -98,11 +98,10 @@ ses_setup(struct sess *sp)
  */
 
 struct sess *
-SES_New(struct worker *wrk, struct sesspool *pp)
+SES_New(struct sesspool *pp)
 {
 	struct sess *sp;
 
-	(void)wrk;					// XXX
 	CHECK_OBJ_NOTNULL(pp, SESSPOOL_MAGIC);
 	sp = MPL_Get(pp->mpl_sess, NULL);
 	sp->magic = SESS_MAGIC;
@@ -280,7 +279,7 @@ SES_GetReq(struct sess *sp)
 	sp->req->http0 = HTTP_create(p, nhttp);
 	p += hl;
 	sz -= hl;
-	
+
 	WS_Init(sp->req->ws, "req", p, sz);
 }
 
@@ -314,17 +313,16 @@ SES_NewPool(struct pool *wp, unsigned pool_no)
 	pp->pool = wp;
 	bprintf(nb, "req%u", pool_no);
 	pp->mpl_req = MPL_New(nb, &cache_param->req_pool,
-	    &cache_param->sess_workspace);
+	    &cache_param->workspace_client);
 	pp->mpl_sess = MPL_New(nb, &cache_param->sess_pool, &ses_size);
 	return (pp);
 }
 
 void
-SES_DeletePool(struct sesspool *pp, struct worker *wrk)
+SES_DeletePool(struct sesspool *pp)
 {
 
 	CHECK_OBJ_NOTNULL(pp, SESSPOOL_MAGIC);
-	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 	MPL_Destroy(&pp->mpl_sess);
 	MPL_Destroy(&pp->mpl_req);
 	FREE_OBJ(pp);
diff --git a/bin/varnishd/cache/cache_vary.c b/bin/varnishd/cache/cache_vary.c
index a5070cd..d524fd3 100644
--- a/bin/varnishd/cache/cache_vary.c
+++ b/bin/varnishd/cache/cache_vary.c
@@ -174,7 +174,7 @@ vry_cmp(const uint8_t * const *v1, uint8_t * const *v2)
 }
 
 int
-VRY_Match(struct sess *sp, const uint8_t *vary)
+VRY_Match(const struct sess *sp, const uint8_t *vary)
 {
 	uint8_t *vsp = sp->req->vary_b;
 	char *h, *e;
diff --git a/bin/varnishd/common/params.h b/bin/varnishd/common/params.h
index ca39ff7..6c1899a 100644
--- a/bin/varnishd/common/params.h
+++ b/bin/varnishd/common/params.h
@@ -75,7 +75,7 @@ struct params {
 	unsigned		queue_max;
 
 	/* Memory allocation hints */
-	unsigned		sess_workspace;
+	unsigned		workspace_client;
 	unsigned		shm_workspace;
 	unsigned		http_req_size;
 	unsigned		http_req_hdr_len;
diff --git a/bin/varnishd/mgt/mgt_param.c b/bin/varnishd/mgt/mgt_param.c
index 36f96cd..c488f23 100644
--- a/bin/varnishd/mgt/mgt_param.c
+++ b/bin/varnishd/mgt/mgt_param.c
@@ -687,12 +687,9 @@ static const struct parspec input_parspec[] = {
 		"flush of the cache use \"ban.url .\"",
 		0,
 		"120", "seconds" },
-	{ "sess_workspace",
-		tweak_bytes_u, &mgt_param.sess_workspace, 3072, UINT_MAX,
-		"Bytes of HTTP protocol workspace allocated for sessions. "
-		"This space must be big enough for the entire HTTP protocol "
-		"header and any edits done to it in the VCL code.\n"
-		"Minimum is 1024 bytes.",
+	{ "workspace_client",
+		tweak_bytes_u, &mgt_param.workspace_client, 3072, UINT_MAX,
+		"Bytes of HTTP protocol workspace for clients HTTP req/resp.",
 		DELAYED_EFFECT,
 		"64k", "bytes" },
 	{ "http_req_hdr_len",
diff --git a/bin/varnishtest/tests/e00022.vtc b/bin/varnishtest/tests/e00022.vtc
index 39302b4..897f3ee 100644
--- a/bin/varnishtest/tests/e00022.vtc
+++ b/bin/varnishtest/tests/e00022.vtc
@@ -14,7 +14,7 @@ server s1 {
 	}
 } -start
 
-varnish v1 -arg "-p sess_workspace=131072 -p thread_pool_stack=262144" -vcl+backend {
+varnish v1 -arg "-p thread_pool_stack=262144" -vcl+backend {
 	sub vcl_recv {
 		set req.esi = true;
 	}
diff --git a/bin/varnishtest/tests/r00693.vtc b/bin/varnishtest/tests/r00693.vtc
index 24b6827..ff99ca7 100644
--- a/bin/varnishtest/tests/r00693.vtc
+++ b/bin/varnishtest/tests/r00693.vtc
@@ -17,7 +17,7 @@ server s1 {
 
 } -start
 
-varnish v1 -arg "-p sess_workspace=3072" -vcl+backend {
+varnish v1 -arg "-p workspace_client=3072" -vcl+backend {
 
 	sub vcl_recv {
 		set req.http.foo = 
diff --git a/include/tbl/vsc_f_main.h b/include/tbl/vsc_f_main.h
index d63b187..4963b7d 100644
--- a/include/tbl/vsc_f_main.h
+++ b/include/tbl/vsc_f_main.h
@@ -145,36 +145,6 @@ VSC_F(fetch_204,		uint64_t, 1, 'a', "Fetch no body (204)", "")
 VSC_F(fetch_304,		uint64_t, 1, 'a', "Fetch no body (304)", "")
 
 /*---------------------------------------------------------------------
- * Session Memory
- *    see: cache_session.c
- */
-
-VSC_F(sessmem_size,		uint64_t, 1, 'g',
-    "Session mem size",
-	"Bytes of memory allocated for last allocated session."
-)
-
-VSC_F(sessmem_alloc,		uint64_t, 1, 'c',
-    "Session mem allocated",
-	"Count of all allocations of session memory."
-)
-
-VSC_F(sessmem_free,		uint64_t, 1, 'c',
-    "Session mem freed",
-	"Count of all frees of session memory."
-)
-
-VSC_F(sessmem_fail,		uint64_t, 1, 'c',
-    "Session mem alloc failed",
-	"Count of session memory allocation failures."
-)
-
-VSC_F(sessmem_limit,		uint64_t, 1, 'c',
-    "Session mem alloc limited",
-	"Count of session memory allocations blocked by limit (max_sess)."
-)
-
-/*---------------------------------------------------------------------
  * Pools, threads, and sessions
  *    see: cache_pool.c
  *
@@ -252,8 +222,6 @@ VSC_F(busyobj_free,		uint64_t, 1, 'c',
 
 /*---------------------------------------------------------------------*/
 
-VSC_F(n_sess_mem,		uint64_t, 0, 'i', "N struct sess_mem", "")
-VSC_F(n_sess,			uint64_t, 0, 'i', "N struct sess", "")
 VSC_F(n_object,			uint64_t, 1, 'i', "N struct object", "")
 VSC_F(n_vampireobject,		uint64_t, 1, 'i', "N unresurrected objects", "")
 VSC_F(n_objectcore,		uint64_t, 1, 'i', "N struct objectcore", "")



From geoff at varnish-cache.org  Mon Jan  9 23:03:34 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Tue, 10 Jan 2012 00:03:34 +0100
Subject: [experimental-ims] 8a37b31 Comment on struct sess size issues.
Message-ID: <E1RkOFa-0007zj-6B@project.varnish-software.com>

commit 8a37b3122217fde89d98cc3b5ce0c9452ed843a1
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Jan 2 11:04:29 2012 +0000

    Comment on struct sess size issues.

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index 75bc803..6b0d0a9 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -624,7 +624,15 @@ struct req {
 
 };
 
-/*--------------------------------------------------------------------*/
+/*--------------------------------------------------------------------
+ * Struct sess is a high memory-load structure because sessions typically
+ * hang around the waiter for relatively long time.
+ *
+ * The size goal for struct sess + struct memitem is <512 bytes
+ *
+ * Getting down to the next relevant size (<256 bytes because of how malloc
+ * works, is not realistic without a lot of code changes.
+ */
 
 struct sess {
 	unsigned		magic;



From geoff at varnish-cache.org  Mon Jan  9 23:03:30 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Tue, 10 Jan 2012 00:03:30 +0100
Subject: [experimental-ims] 3d186d9 Relieve waiters of the task of receiving
	the request header,
	instead throw the session at a worker thread the moment it becomes
	readable.
Message-ID: <E1RkOFW-0007t9-8b@project.varnish-software.com>

commit 3d186d949c28d7a5cef6a0b82eb9a22df21bad62
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu Dec 22 21:38:58 2011 +0000

    Relieve waiters of the task of receiving the request header, instead
    throw the session at a worker thread the moment it becomes readable.

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index bcf4f54..58419bc 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -920,8 +920,7 @@ void SES_Charge(struct sess *sp);
 struct sesspool *SES_NewPool(struct pool *pp, unsigned pool_no);
 void SES_DeletePool(struct sesspool *sp, struct worker *wrk);
 int SES_Schedule(struct sess *sp);
-void SES_Handle(struct sess *sp, int status);
-
+void SES_Handle(struct sess *sp);
 
 /* cache_shmlog.c */
 extern struct VSC_C_main *VSC_C_main;
diff --git a/bin/varnishd/cache/cache_center.c b/bin/varnishd/cache/cache_center.c
index 333db31..1d92ded 100644
--- a/bin/varnishd/cache/cache_center.c
+++ b/bin/varnishd/cache/cache_center.c
@@ -1675,7 +1675,7 @@ CNT_Session(struct sess *sp)
 	 */
 	assert(
 	    sp->step == STP_FIRST ||
-	    sp->step == STP_START ||
+	    sp->step == STP_WAIT ||
 	    sp->step == STP_LOOKUP ||
 	    sp->step == STP_RECV);
 
diff --git a/bin/varnishd/cache/cache_session.c b/bin/varnishd/cache/cache_session.c
index 790ea2c..9c5e82b 100644
--- a/bin/varnishd/cache/cache_session.c
+++ b/bin/varnishd/cache/cache_session.c
@@ -279,23 +279,11 @@ SES_Schedule(struct sess *sp)
  */
 
 void
-SES_Handle(struct sess *sp, int status)
+SES_Handle(struct sess *sp)
 {
 
-	switch (status) {
-	case -2:
-		SES_Delete(sp, "blast");
-		break;
-	case -1:
-		SES_Delete(sp, "no request");
-		break;
-	case 1:
-		sp->step = STP_START;
-		(void)SES_Schedule(sp);
-		break;
-	default:
-		WRONG("Unexpected return from HTC_Rx()");
-	}
+	sp->step = STP_WAIT;
+	(void)SES_Schedule(sp);
 }
 
 /*--------------------------------------------------------------------
diff --git a/bin/varnishd/waiter/cache_waiter_epoll.c b/bin/varnishd/waiter/cache_waiter_epoll.c
index ca0d29f..e71633a 100644
--- a/bin/varnishd/waiter/cache_waiter_epoll.c
+++ b/bin/varnishd/waiter/cache_waiter_epoll.c
@@ -129,13 +129,8 @@ vwe_eev(struct vwe *vwe, const struct epoll_event *ep)
 	} else {
 		CAST_OBJ_NOTNULL(sp, ep->data.ptr, SESS_MAGIC);
 		if (ep->events & EPOLLIN || ep->events & EPOLLPRI) {
-			i = HTC_Rx(sp->htc);
-			if (i == 0) {
-				vwe_modadd(vwe, sp->fd, sp, EPOLL_CTL_MOD);
-				return;	/* more needed */
-			}
 			VTAILQ_REMOVE(&vwe->sesshead, sp, list);
-			SES_Handle(sp, i);
+			SES_Handle(sp);
 		} else if (ep->events & EPOLLERR) {
 			VTAILQ_REMOVE(&vwe->sesshead, sp, list);
 			SES_Delete(sp, "ERR");
diff --git a/bin/varnishd/waiter/cache_waiter_kqueue.c b/bin/varnishd/waiter/cache_waiter_kqueue.c
index 7dcf7f5..e11f945 100644
--- a/bin/varnishd/waiter/cache_waiter_kqueue.c
+++ b/bin/varnishd/waiter/cache_waiter_kqueue.c
@@ -116,7 +116,6 @@ vwk_pipe_ev(struct vwk *vwk, const struct kevent *kp)
 static void
 vwk_sess_ev(struct vwk *vwk, const struct kevent *kp)
 {
-	int i;
 	struct sess *sp;
 
 	AN(kp->udata);
@@ -129,13 +128,8 @@ vwk_sess_ev(struct vwk *vwk, const struct kevent *kp)
 	assert((sp->vsl_id & VSL_IDENTMASK) == kp->ident);
 	assert((sp->vsl_id & VSL_IDENTMASK) == sp->fd);
 	if (kp->data > 0) {
-		i = HTC_Rx(sp->htc);
-		if (i == 0) {
-			vwk_kq_sess(vwk, sp, EV_ADD | EV_ONESHOT);
-			return;	/* more needed */
-		}
 		VTAILQ_REMOVE(&vwk->sesshead, sp, list);
-		SES_Handle(sp, i);
+		SES_Handle(sp);
 		return;
 	} else if (kp->flags & EV_EOF) {
 		VTAILQ_REMOVE(&vwk->sesshead, sp, list);
diff --git a/bin/varnishd/waiter/cache_waiter_poll.c b/bin/varnishd/waiter/cache_waiter_poll.c
index 681b48d..cf750ad 100644
--- a/bin/varnishd/waiter/cache_waiter_poll.c
+++ b/bin/varnishd/waiter/cache_waiter_poll.c
@@ -153,20 +153,10 @@ vwp_main(void *priv)
 			assert(vwp->pollfd[fd].fd == fd);
 			if (vwp->pollfd[fd].revents) {
 				v2--;
-				i = HTC_Rx(sp->htc);
-				if (vwp->pollfd[fd].revents != POLLIN)
-					VSL(SLT_Debug, fd, "Poll: %x / %d",
-					    vwp->pollfd[fd].revents, i);
 				vwp->pollfd[fd].revents = 0;
 				VTAILQ_REMOVE(&vwp->sesshead, sp, list);
-				if (i == 0) {
-					/* Mov to front of list for speed */
-					VTAILQ_INSERT_HEAD(&vwp->sesshead,
-					    sp, list);
-				} else {
-					vwp_unpoll(vwp, fd);
-					SES_Handle(sp, i);
-				}
+				vwp_unpoll(vwp, fd);
+				SES_Handle(sp);
 			} else if (sp->t_open <= deadline) {
 				VTAILQ_REMOVE(&vwp->sesshead, sp, list);
 				vwp_unpoll(vwp, fd);
diff --git a/bin/varnishd/waiter/cache_waiter_ports.c b/bin/varnishd/waiter/cache_waiter_ports.c
index 7717196..09db406 100644
--- a/bin/varnishd/waiter/cache_waiter_ports.c
+++ b/bin/varnishd/waiter/cache_waiter_ports.c
@@ -90,13 +90,6 @@ vws_port_ev(struct vws *vws, port_event_t *ev) {
 			SES_Delete(sp, "EOF");
 			return;
 		}
-		i = HTC_Rx(sp->htc);
-
-		if (i == 0) {
-			/* incomplete header, wait for more data */
-			vws_add(vws, sp->fd, sp);
-			return;
-		}
 
 		/*
 		 * note: the original man page for port_associate(3C) states:
@@ -115,7 +108,7 @@ vws_port_ev(struct vws *vws, port_event_t *ev) {
 		VTAILQ_REMOVE(&vws->sesshead, sp, list);
 
 		/* SES_Handle will also handle errors */
-		SES_Handle(sp, i);
+		SES_Handle(sp);
 	}
 	return;
 }



From geoff at varnish-cache.org  Mon Jan  9 23:03:33 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Tue, 10 Jan 2012 00:03:33 +0100
Subject: [experimental-ims] 903f1a0 Reduce the scope of the SHA256 contexts
	memory demand, we only need it during vcl_hash{}
Message-ID: <E1RkOFZ-0007xf-5t@project.varnish-software.com>

commit 903f1a06e7e4ebf40c99a158fa0e62b9bf01ee6c
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Dec 26 10:32:58 2011 +0000

    Reduce the scope of the SHA256 contexts memory demand, we only
    need it during vcl_hash{}

diff --git a/bin/varnishd/cache/cache_center.c b/bin/varnishd/cache/cache_center.c
index 81a0c67..4967c82 100644
--- a/bin/varnishd/cache/cache_center.c
+++ b/bin/varnishd/cache/cache_center.c
@@ -1459,6 +1459,7 @@ static int
 cnt_recv(struct sess *sp, struct worker *wrk, struct req *req)
 {
 	unsigned recv_handling;
+	struct SHA256Context sha256ctx;
 
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
@@ -1502,10 +1503,12 @@ cnt_recv(struct sess *sp, struct worker *wrk, struct req *req)
 		}
 	}
 
+	req->sha256ctx = &sha256ctx;
 	SHA256_Init(req->sha256ctx);
 	VCL_hash_method(sp);
 	assert(req->handling == VCL_RET_HASH);
 	SHA256_Final(req->digest, req->sha256ctx);
+	req->sha256ctx = NULL;
 
 	if (!strcmp(req->http->hd[HTTP_HDR_REQ].b, "HEAD"))
 		req->wantbody = 0;
diff --git a/bin/varnishd/cache/cache_hash.c b/bin/varnishd/cache/cache_hash.c
index 643eb94..d967291 100644
--- a/bin/varnishd/cache/cache_hash.c
+++ b/bin/varnishd/cache/cache_hash.c
@@ -158,6 +158,7 @@ HSH_AddString(const struct sess *sp, const char *str)
 		str = "";
 	l = strlen(str);
 
+	AN(sp->req->sha256ctx);
 	SHA256_Update(sp->req->sha256ctx, str, l);
 	SHA256_Update(sp->req->sha256ctx, "#", 1);
 
diff --git a/bin/varnishd/cache/cache_session.c b/bin/varnishd/cache/cache_session.c
index d26bd8f..ee5060f 100644
--- a/bin/varnishd/cache/cache_session.c
+++ b/bin/varnishd/cache/cache_session.c
@@ -42,7 +42,6 @@
 #include "cache.h"
 
 #include "waiter/waiter.h"
-#include "vsha256.h"
 #include "vtim.h"
 
 static unsigned ses_size = sizeof (struct sess);
@@ -271,10 +270,7 @@ SES_GetReq(struct sess *sp)
 	nhttp = (uint16_t)cache_param->http_max_hdr;
 	hl = HTTP_estimate(nhttp);
 
-	xxxassert(sz > 3 * hl + sizeof(struct SHA256Context) + 128);
-
-	sp->req->sha256ctx = (void*)p;
-	p += sizeof(struct SHA256Context);
+	xxxassert(sz > 3 * hl + 128);
 
 	sp->req->http = HTTP_create(p, nhttp);
 	p += hl;		// XXX: align ?



From geoff at varnish-cache.org  Mon Jan  9 23:03:31 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Tue, 10 Jan 2012 00:03:31 +0100
Subject: [experimental-ims] f918ddb Move the HTC from sess to req,
	that's another 64 bytes.
Message-ID: <E1RkOFX-0007uv-IS@project.varnish-software.com>

commit f918ddbfaaf2b8c449f6595331fcdc6664bada2f
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Fri Dec 23 12:47:32 2011 +0000

    Move the HTC from sess to req, that's another 64 bytes.

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index 578d548..bf4754b 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -615,6 +615,9 @@ struct req {
 	char			*ws_req;	/* WS above request data */
 
 	double			t_resp;
+
+	struct http_conn	htc[1];
+
 };
 
 /*--------------------------------------------------------------------*/
@@ -670,10 +673,6 @@ struct sess {
 	/* Timestamps, all on TIM_real() timescale */
 	double			t_req;
 
-	/* TBD fields ------------------------------------------------*/
-
-	struct http_conn	htc[1];
-
 };
 
 /* Prototypes etc ----------------------------------------------------*/
diff --git a/bin/varnishd/cache/cache_center.c b/bin/varnishd/cache/cache_center.c
index 6eef7cb..b6d7a67 100644
--- a/bin/varnishd/cache/cache_center.c
+++ b/bin/varnishd/cache/cache_center.c
@@ -107,8 +107,12 @@ cnt_wait(struct sess *sp)
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 	wrk = sp->wrk;
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
-	if (sp->req == NULL)
+	if (sp->req == NULL) {
 		SES_GetReq(sp);
+		HTC_Init(sp->req->htc, sp->ws, sp->fd, sp->vsl_id,
+		    cache_param->http_req_size,
+		    cache_param->http_req_hdr_len);
+	}
 
 	AZ(sp->req->vcl);
 	AZ(wrk->obj);
@@ -126,9 +130,9 @@ cnt_wait(struct sess *sp)
 		assert(j >= 0);
 		now = VTIM_real();
 		if (j != 0)
-			i = HTC_Rx(sp->htc);
+			i = HTC_Rx(sp->req->htc);
 		else
-			i = HTC_Complete(sp->htc);
+			i = HTC_Complete(sp->req->htc);
 		if (i == 1) {
 			/* Got it, run with it */
 			sp->t_req = now;
@@ -156,6 +160,8 @@ cnt_wait(struct sess *sp)
 				wrk->stats.sess_herd++;
 				SES_Charge(sp);
 				SES_ReleaseReq(sp);
+				WS_Release(sp->ws, 0);
+				WS_Reset(sp->ws, NULL);
 				WAIT_Enter(sp);
 				return (1);
 			}
@@ -445,14 +451,14 @@ cnt_done(struct sess *sp)
 	WS_Reset(sp->ws, NULL);
 	WS_Reset(wrk->ws, NULL);
 
-	i = HTC_Reinit(sp->htc);
+	i = HTC_Reinit(sp->req->htc);
 	if (i == 1) {
 		wrk->stats.sess_pipeline++;
 		sp->t_req = sp->t_idle;
 		sp->step = STP_START;
 		return (0);
 	}
-	if (Tlen(sp->htc->rxbuf))
+	if (Tlen(sp->req->htc->rxbuf))
 		wrk->stats.sess_readahead++;
 	sp->step = STP_WAIT;
 	sp->t_req = sp->t_idle;
@@ -1052,10 +1058,6 @@ cnt_first(struct sess *sp)
 		    sp->addr, sp->port, sp->mylsock->name);
 	}
 
-	/* Receive a HTTP protocol request */
-	HTC_Init(sp->htc, sp->ws, sp->fd, sp->vsl_id,
-	    cache_param->http_req_size,
-	    cache_param->http_req_hdr_len);
 	wrk->acct_tmp.sess++;
 
 	sp->step = STP_WAIT;
diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c
index 9f1d822..dbf30e6 100644
--- a/bin/varnishd/cache/cache_fetch.c
+++ b/bin/varnishd/cache/cache_fetch.c
@@ -345,7 +345,7 @@ FetchReqBody(struct sess *sp)
 				rdcnt = sizeof buf;
 			else
 				rdcnt = content_length;
-			rdcnt = HTC_Read(sp->wrk, sp->htc, buf, rdcnt);
+			rdcnt = HTC_Read(sp->wrk, sp->req->htc, buf, rdcnt);
 			if (rdcnt <= 0)
 				return (1);
 			content_length -= rdcnt;
diff --git a/bin/varnishd/cache/cache_http.c b/bin/varnishd/cache/cache_http.c
index 803fed2..93bd896 100644
--- a/bin/varnishd/cache/cache_http.c
+++ b/bin/varnishd/cache/cache_http.c
@@ -658,7 +658,7 @@ http_DissectRequest(struct sess *sp)
 	uint16_t retval;
 
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
-	htc = sp->htc;
+	htc = sp->req->htc;
 	CHECK_OBJ_NOTNULL(htc, HTTP_CONN_MAGIC);
 	hp = sp->http;
 	CHECK_OBJ_NOTNULL(hp, HTTP_MAGIC);
diff --git a/bin/varnishd/cache/cache_pipe.c b/bin/varnishd/cache/cache_pipe.c
index 8bf76f3..a8282d7 100644
--- a/bin/varnishd/cache/cache_pipe.c
+++ b/bin/varnishd/cache/cache_pipe.c
@@ -82,9 +82,9 @@ PipeSession(struct sess *sp)
 	sp->wrk->acct_tmp.hdrbytes +=
 	    http_Write(w, sp->vsl_id, sp->wrk->busyobj->bereq, 0);
 
-	if (sp->htc->pipeline.b != NULL)
+	if (sp->req->htc->pipeline.b != NULL)
 		sp->wrk->acct_tmp.bodybytes +=
-		    WRW_Write(w, sp->htc->pipeline.b, Tlen(sp->htc->pipeline));
+		    WRW_Write(w, sp->req->htc->pipeline.b, Tlen(sp->req->htc->pipeline));
 
 	i = WRW_FlushRelease(w);
 



From geoff at varnish-cache.org  Mon Jan  9 23:03:28 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Tue, 10 Jan 2012 00:03:28 +0100
Subject: [experimental-ims] 0d03649 clean up refs
Message-ID: <E1RkOFU-0007qL-Li@project.varnish-software.com>

commit 0d0364905b418d43e55197efee41b04eea3e5e50
Author: Per Buer <perbu at varnish-software.com>
Date:   Mon Dec 19 12:29:33 2011 +0100

    clean up refs

diff --git a/doc/sphinx/reference/varnishd.rst b/doc/sphinx/reference/varnishd.rst
index 5d1fc77..4e2b3f2 100644
--- a/doc/sphinx/reference/varnishd.rst
+++ b/doc/sphinx/reference/varnishd.rst
@@ -1,3 +1,5 @@
+.. _ref-varnishd:
+
 =========
  varnishd
 =========
diff --git a/doc/sphinx/reference/varnishlog.rst b/doc/sphinx/reference/varnishlog.rst
index a90231f..f626332 100644
--- a/doc/sphinx/reference/varnishlog.rst
+++ b/doc/sphinx/reference/varnishlog.rst
@@ -1,4 +1,4 @@
-_reference-varnishlog:
+.. _ref-varnishlog:
 
 ==========
 varnishlog



From geoff at varnish-cache.org  Mon Jan  9 23:03:33 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Tue, 10 Jan 2012 00:03:33 +0100
Subject: [experimental-ims] b099f90 Shave 88 bytes of struct dstat and struct
	worker by not marking stats counters as worker-local unless
	they actually are.
Message-ID: <E1RkOFZ-0007xv-9q@project.varnish-software.com>

commit b099f90867d1c99e45164ec5aa577d606fdd5596
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Dec 26 16:27:42 2011 +0000

    Shave 88 bytes of struct dstat and struct worker by not marking
    stats counters as worker-local unless they actually are.

diff --git a/bin/varnishd/cache/cache_cli.c b/bin/varnishd/cache/cache_cli.c
index 2432691..de8265d 100644
--- a/bin/varnishd/cache/cache_cli.c
+++ b/bin/varnishd/cache/cache_cli.c
@@ -137,6 +137,8 @@ cli_debug_sizeof(struct cli *cli, const char * const *av, void *priv)
 	SZOF(struct vbc);
 	SZOF(struct VSC_C_main);
 	SZOF(struct lock);
+	SZOF(struct wrw);
+	SZOF(struct dstat);
 #if 0
 #define OFOF(foo, bar)	{ foo __foo; VCLI_Out(cli, \
     "%-30s = 0x%4zx @ 0x%4zx\n", \
diff --git a/include/tbl/vsc_f_main.h b/include/tbl/vsc_f_main.h
index 4963b7d..df7984a 100644
--- a/include/tbl/vsc_f_main.h
+++ b/include/tbl/vsc_f_main.h
@@ -150,56 +150,56 @@ VSC_F(fetch_304,		uint64_t, 1, 'a', "Fetch no body (304)", "")
  *
  */
 
-VSC_F(pools,			uint64_t, 1, 'g',
+VSC_F(pools,			uint64_t, 0, 'g',
     "Number of thread pools",
 	"Number of thread pools.  See also param wthread_pools."
 	"  NB: Presently pools cannot be removed once created."
 )
 
-VSC_F(threads,			uint64_t, 1, 'g',
+VSC_F(threads,			uint64_t, 0, 'g',
     "Total number of threads",
 	"Number of threads in all pools."
 	"  See also params thread_pools, thread_pool_min & thread_pool_max."
 )
 
-VSC_F(threads_limited,		uint64_t, 1, 'c',
+VSC_F(threads_limited,		uint64_t, 0, 'c',
     "Threads hit max",
 	"Number of times more threads were needed, but limit was reached"
 	" in a thread pool."
 	"  See also param thread_pool_max."
 )
 
-VSC_F(threads_created,		uint64_t, 1, 'c',
+VSC_F(threads_created,		uint64_t, 0, 'c',
     "Threads created",
 	"Total number of threads created in all pools."
 )
 
-VSC_F(threads_destroyed,	uint64_t, 1, 'c',
+VSC_F(threads_destroyed,	uint64_t, 0, 'c',
     "Threads destoryed",
 	"Total number of threads destroyed in all pools."
 )
 
-VSC_F(threads_failed,		uint64_t, 1, 'c',
+VSC_F(threads_failed,		uint64_t, 0, 'c',
     "Thread creation failed",
 	"Number of times creating a thread failed."
 	"  See VSL::Debug for diagnostics."
 	"  See also param thread_fail_delay."
 )
 
-VSC_F(thread_queue_len,		uint64_t, 1, 'g',
+VSC_F(thread_queue_len,		uint64_t, 0, 'g',
     "Length of session queue",
 	"Length of session queue waiting for threads."
 	"  NB: Only updates once per second."
 	"  See also param queue_max."
 )
 
-VSC_F(sess_queued,		uint64_t, 1, 'c',
+VSC_F(sess_queued,		uint64_t, 0, 'c',
     "Sessions queued for thread",
 	"Number of times session was queued waiting for a thread."
 	"  See also param queue_max."
 )
 
-VSC_F(sess_dropped,		uint64_t, 1, 'c',
+VSC_F(sess_dropped,		uint64_t, 0, 'c',
     "Sessions dropped for thread",
 	"Number of times session was dropped because the queue were too"
 	" long already."
@@ -210,12 +210,12 @@ VSC_F(sess_dropped,		uint64_t, 1, 'c',
  * BusyObj
  */
 
-VSC_F(busyobj_alloc,		uint64_t, 1, 'c',
+VSC_F(busyobj_alloc,		uint64_t, 0, 'c',
     "Busyobj allocations",
 	"Number of busyobj structures allocated."
 )
 
-VSC_F(busyobj_free,		uint64_t, 1, 'c',
+VSC_F(busyobj_free,		uint64_t, 0, 'c',
     "Busyobj freed",
 	"Number of busyobj structures freed."
 )
@@ -246,17 +246,17 @@ VSC_F(n_objwrite,		uint64_t, 0, 'a', "Objects sent with write",
 VSC_F(n_objoverflow,	uint64_t, 1, 'a',
 					"Objects overflowing workspace", "")
 
-VSC_F(s_sess,		uint64_t, 1, 'a', "Total Sessions", "")
-VSC_F(s_req,		uint64_t, 1, 'a', "Total Requests", "")
-VSC_F(s_pipe,		uint64_t, 1, 'a', "Total pipe", "")
-VSC_F(s_pass,		uint64_t, 1, 'a', "Total pass", "")
-VSC_F(s_fetch,		uint64_t, 1, 'a', "Total fetch", "")
+VSC_F(s_sess,			uint64_t, 1, 'a', "Total Sessions", "")
+VSC_F(s_req,			uint64_t, 1, 'a', "Total Requests", "")
+VSC_F(s_pipe,			uint64_t, 1, 'a', "Total pipe", "")
+VSC_F(s_pass,			uint64_t, 1, 'a', "Total pass", "")
+VSC_F(s_fetch,			uint64_t, 1, 'a', "Total fetch", "")
 VSC_F(s_hdrbytes,		uint64_t, 1, 'a', "Total header bytes", "")
 VSC_F(s_bodybytes,		uint64_t, 1, 'a', "Total body bytes", "")
 
 VSC_F(sess_closed,		uint64_t, 1, 'a', "Session Closed", "")
-VSC_F(sess_pipeline,	uint64_t, 1, 'a', "Session Pipeline", "")
-VSC_F(sess_readahead,	uint64_t, 1, 'a', "Session Read Ahead", "")
+VSC_F(sess_pipeline,		uint64_t, 1, 'a', "Session Pipeline", "")
+VSC_F(sess_readahead,		uint64_t, 1, 'a', "Session Read Ahead", "")
 VSC_F(sess_linger,		uint64_t, 1, 'a', "Session Linger", "")
 VSC_F(sess_herd,		uint64_t, 1, 'a', "Session herd", "")
 
@@ -264,11 +264,11 @@ VSC_F(shm_records,		uint64_t, 0, 'a', "SHM records", "")
 VSC_F(shm_writes,		uint64_t, 0, 'a', "SHM writes", "")
 VSC_F(shm_flushes,		uint64_t, 0, 'a',
     "SHM flushes due to overflow", "")
-VSC_F(shm_cont,		uint64_t, 0, 'a', "SHM MTX contention", "")
+VSC_F(shm_cont,			uint64_t, 0, 'a', "SHM MTX contention", "")
 VSC_F(shm_cycles,		uint64_t, 0, 'a',
     "SHM cycles through buffer", "")
 
-VSC_F(sms_nreq,		uint64_t, 0, 'a', "SMS allocator requests", "")
+VSC_F(sms_nreq,			uint64_t, 0, 'a', "SMS allocator requests", "")
 VSC_F(sms_nobj,		uint64_t, 0, 'i', "SMS outstanding allocations", "")
 VSC_F(sms_nbytes,		uint64_t, 0, 'i', "SMS outstanding bytes", "")
 VSC_F(sms_balloc,		uint64_t, 0, 'i', "SMS bytes allocated", "")
@@ -276,9 +276,9 @@ VSC_F(sms_bfree,		uint64_t, 0, 'i', "SMS bytes freed", "")
 
 VSC_F(backend_req,		uint64_t, 0, 'a', "Backend requests made", "")
 
-VSC_F(n_vcl,		uint64_t, 0, 'a', "N vcl total", "")
+VSC_F(n_vcl,			uint64_t, 0, 'a', "N vcl total", "")
 VSC_F(n_vcl_avail,		uint64_t, 0, 'a', "N vcl available", "")
-VSC_F(n_vcl_discard,	uint64_t, 0, 'a', "N vcl discarded", "")
+VSC_F(n_vcl_discard,		uint64_t, 0, 'a', "N vcl discarded", "")
 
 /**********************************************************************/
 



From geoff at varnish-cache.org  Mon Jan  9 23:03:29 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Tue, 10 Jan 2012 00:03:29 +0100
Subject: [experimental-ims] bc72cff Pave a bit of road for major surgery
Message-ID: <E1RkOFV-0007sS-P0@project.varnish-software.com>

commit bc72cff0f52ed41010a3fe0b0c93eedf84c85ab4
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu Dec 22 15:08:13 2011 +0000

    Pave a bit of road for major surgery

diff --git a/bin/varnishd/cache/cache_vrt_var.c b/bin/varnishd/cache/cache_vrt_var.c
index 54124b2..7431c1b 100644
--- a/bin/varnishd/cache/cache_vrt_var.c
+++ b/bin/varnishd/cache/cache_vrt_var.c
@@ -441,21 +441,21 @@ VRT_r_req_xid(struct sess *sp)
 
 /*--------------------------------------------------------------------*/
 
-#define REQ_BOOL(which)						\
+#define REQ_BOOL(hash_var)					\
 void __match_proto__()						\
-VRT_l_req_##which(struct sess *sp, unsigned val)		\
+VRT_l_req_##hash_var(struct sess *sp, unsigned val)		\
 {								\
 								\
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);			\
-	sp->which = val ? 1 : 0;				\
+	sp->hash_var = val ? 1 : 0;				\
 }								\
 								\
 unsigned __match_proto__()					\
-VRT_r_req_##which(struct sess *sp)				\
+VRT_r_req_##hash_var(struct sess *sp)				\
 {								\
 								\
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);			\
-	return(sp->which);					\
+	return(sp->hash_var);					\
 }
 
 REQ_BOOL(hash_ignore_busy)



From perbu at varnish-cache.org  Wed Jan 11 09:26:59 2012
From: perbu at varnish-cache.org (Per Buer)
Date: Wed, 11 Jan 2012 10:26:59 +0100
Subject: [master] 2bcd2f0 Added an introduction. Brushed up the chapter on
	varnish in a virtualized environment
Message-ID: <E1RkuSR-0001Yb-7w@project.varnish-software.com>

commit 2bcd2f0fbea8c1ab2e4b623aa7b71cb139b54659
Author: Per Buer <perbu at varnish-software.com>
Date:   Wed Jan 11 10:26:56 2012 +0100

    Added an introduction. Brushed up the chapter on varnish in a virtualized environment

diff --git a/doc/sphinx/tutorial/index.rst b/doc/sphinx/tutorial/index.rst
index 32c2264..0bf3137 100644
--- a/doc/sphinx/tutorial/index.rst
+++ b/doc/sphinx/tutorial/index.rst
@@ -14,6 +14,7 @@ separate topic. Good luck.
 
 .. toctree:: :maxdepth: 1
 
+        indroduction
 	backend_servers
         starting_varnish
 	logging
diff --git a/doc/sphinx/tutorial/introduction.rst b/doc/sphinx/tutorial/introduction.rst
new file mode 100644
index 0000000..17d7b6a
--- /dev/null
+++ b/doc/sphinx/tutorial/introduction.rst
@@ -0,0 +1,37 @@
+.. _tutorial-intro:
+
+What is Varnish?
+----------------
+
+Varnish Cache is a Varnish Cache is a web application accelerator also
+known as a caching HTTP reverse proxy. You install it in front of any
+server that speaks HTTP and configure it to cache the
+contents. Varnish Cache is really, really fast. It typically speeds up
+delivery with a factor of 300 - 1000x, depending on your architecture.
+
+
+Performance
+~~~~~~~~~~~
+
+Varnish performs really, really well. It is usually bound by the speed
+of the network, effectivly turning performance into a non-issue. We've
+seen Varnish delivering 20 Gbps on regular off-the-shelf hardware.
+
+Flexibility
+~~~~~~~~~~~
+
+One of the key features of Varnish Cache, in addition to it's
+performance, is the flexibility of it's configuration language,
+VCL. VCL enables you to write policies on how incoming requests should
+be handled. In such a policy you can decide what content you want to
+serve, from where you want to get the content and how the request or
+response should be altered. You can read more about this in our
+tutorial.
+
+
+Supported plattforms
+~~~~~~~~~~~~~~~~~~~~
+
+Varnish is written to run on modern versions of Linux and FreeBSD and
+the best experience is had on those plattforms. Thanks to our
+contributors it also runs on NetBSD, OpenBSD and OS X.
diff --git a/doc/sphinx/tutorial/virtualised.rst b/doc/sphinx/tutorial/virtualised.rst
index ce4b9ad..ea8ade5 100644
--- a/doc/sphinx/tutorial/virtualised.rst
+++ b/doc/sphinx/tutorial/virtualised.rst
@@ -1,16 +1,19 @@
 
-Running inside a virtual machine (VM)
--------------------------------------
+Running Varnish in a virtualized environment
+--------------------------------------------
 
-It is possible, but not recommended for high performance, to run Varnish on virtualised 
-hardware.
+It is possible, but not recommended for high performance, to run
+Varnish on virtualised hardware. Reduced disk- and network performance
+will reduce the performance a bit so make sure your system has good IO
+performance.
 
 OpenVZ
-''''''
+~~~~~~
 
-If you are running on 64bit OpenVZ (or Parallels VPS), you must reduce the 
-maximum stack size before starting Varnish. The default allocates to much memory per thread, 
-which will make varnish fail as soon as the number of threads (==traffic) increases.
+If you are running on 64bit OpenVZ (or Parallels VPS), you must reduce
+the maximum stack size before starting Varnish. The default allocates
+to much memory per thread, which will make varnish fail as soon as the
+number of threads (==traffic) increases.
 
 Reduce the maximum stack size by running::
 



From perbu at varnish-cache.org  Wed Jan 11 11:48:27 2012
From: perbu at varnish-cache.org (Per Buer)
Date: Wed, 11 Jan 2012 12:48:27 +0100
Subject: [master] bf78ea9 typo. Thanks scoof
Message-ID: <E1RkwfL-00061N-1O@project.varnish-software.com>

commit bf78ea99790454329cb2d72b7f4903a9cad123bc
Author: Per Buer <perbu at varnish-software.com>
Date:   Wed Jan 11 12:48:23 2012 +0100

    typo. Thanks scoof

diff --git a/doc/sphinx/tutorial/introduction.rst b/doc/sphinx/tutorial/introduction.rst
index 17d7b6a..0d43623 100644
--- a/doc/sphinx/tutorial/introduction.rst
+++ b/doc/sphinx/tutorial/introduction.rst
@@ -3,11 +3,11 @@
 What is Varnish?
 ----------------
 
-Varnish Cache is a Varnish Cache is a web application accelerator also
-known as a caching HTTP reverse proxy. You install it in front of any
-server that speaks HTTP and configure it to cache the
-contents. Varnish Cache is really, really fast. It typically speeds up
-delivery with a factor of 300 - 1000x, depending on your architecture.
+Varnish Cache is a web application accelerator also known as a caching
+HTTP reverse proxy. You install it in front of any server that speaks
+HTTP and configure it to cache the contents. Varnish Cache is really,
+really fast. It typically speeds up delivery with a factor of 300 -
+1000x, depending on your architecture.
 
 
 Performance



From tfheen at varnish-cache.org  Wed Jan 11 13:07:42 2012
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Wed, 11 Jan 2012 14:07:42 +0100
Subject: [master] 0802b1f Handle the case of sub being NULL
Message-ID: <E1Rkxu2-0007Zm-0Z@project.varnish-software.com>

commit 0802b1fe539325b44f316f3c598a6607638a83c0
Author: Tollef Fog Heen <tfheen at varnish-software.com>
Date:   Wed Jan 11 14:07:02 2012 +0100

    Handle the case of sub being NULL
    
    if sub is null (because it's the contents of a non-existing header),
    pretend it's "" instead.

diff --git a/bin/varnishd/cache/cache_vrt_re.c b/bin/varnishd/cache/cache_vrt_re.c
index 9864611..867ec86 100644
--- a/bin/varnishd/cache/cache_vrt_re.c
+++ b/bin/varnishd/cache/cache_vrt_re.c
@@ -96,6 +96,8 @@ VRT_regsub(const struct sess *sp, int all, const char *str, void *re,
 	AN(re);
 	if (str == NULL)
 		str = "";
+	if (sub == NULL)
+		sub = "";
 	t = re;
 	memset(ovector, 0, sizeof(ovector));
 	len = strlen(str);
diff --git a/bin/varnishtest/tests/c00001.vtc b/bin/varnishtest/tests/c00001.vtc
index d9b6c41..f1a6824 100644
--- a/bin/varnishtest/tests/c00001.vtc
+++ b/bin/varnishtest/tests/c00001.vtc
@@ -21,6 +21,8 @@ varnish v1 -vcl+backend {
 		    regsub(beresp.http.Foobar, "(b)(a)(r)(f)", "\0\4\3\2\\p");
 		set beresp.http.Snafu6 =
 		    regsub(beresp.http.Foobar, "(b)(a)(r)(f)", "\4\&\3\2p\");
+		set beresp.http.Snafu7 =
+		    regsub(beresp.http.Foobar, "ar", req.http.nosuchheader);
 	}
 } -start 
 
@@ -37,4 +39,5 @@ client c1 {
 	# NB: have to escape the \\ in the next two lines
 	expect resp.http.snafu5 == "_barffra\\p_"
 	expect resp.http.snafu6 == "_f&rap\\_"
+	expect resp.http.snafu7 == "_bf_"
 } -run



From drwilco at varnish-cache.org  Sat Jan 14 12:36:59 2012
From: drwilco at varnish-cache.org (Rogier Mulhuijzen)
Date: Sat, 14 Jan 2012 13:36:59 +0100
Subject: [master] b73b1a0 Update .gitignore
Message-ID: <E1Rm2qx-0004zu-7Q@project.varnish-software.com>

commit b73b1a02dad3c7691d9047c4d76f89271c9c50e5
Author: Rogier 'DocWilco' Mulhuijzen <github at bsdchicks.com>
Date:   Sat Jan 14 13:36:10 2012 +0100

    Update .gitignore
    
    I work with NetBeans, and it insists on this being present.

diff --git a/.gitignore b/.gitignore
index 14c5393..6680eb8 100644
--- a/.gitignore
+++ b/.gitignore
@@ -82,3 +82,6 @@ TAGS
 /doc/*.html
 /doc/sphinx/=build/
 /doc/sphinx/conf.py
+
+# NetBeans insists on this
+/nbproject/private/



From drwilco at varnish-cache.org  Sat Jan 14 13:08:13 2012
From: drwilco at varnish-cache.org (Rogier Mulhuijzen)
Date: Sat, 14 Jan 2012 14:08:13 +0100
Subject: [master] 872c478 Actually use a default (80) if no port is specified
Message-ID: <E1Rm3LB-0004rK-L9@project.varnish-software.com>

commit 872c478170e0640444fe5c173845f8667f9adae7
Author: Rogier 'DocWilco' Mulhuijzen <github at bsdchicks.com>
Date:   Sat Jan 14 13:37:57 2012 +0100

    Actually use a default (80) if no port is specified

diff --git a/lib/libvcl/vcc_dir_dns.c b/lib/libvcl/vcc_dir_dns.c
index bdc0a21..174d1ab 100644
--- a/lib/libvcl/vcc_dir_dns.c
+++ b/lib/libvcl/vcc_dir_dns.c
@@ -88,7 +88,7 @@ print_backend(struct vcc *tl,
 	if (serial >= 0)
 		Fb(tl, 0, "[%d]", serial);
 	Fb(tl, 0, "\",\n");
-	Emit_Sockaddr(tl, &tmptok, b_defaults.port);
+	Emit_Sockaddr(tl, &tmptok, b_defaults.port ? b_defaults.port : "80");
 
 	Fb(tl, 0, "\t.hosthdr = \"");
 	if (b_defaults.hostheader != NULL)



From apj at varnish-cache.org  Sun Jan 15 16:44:31 2012
From: apj at varnish-cache.org (Andreas Plesner Jacobsen)
Date: Sun, 15 Jan 2012 17:44:31 +0100
Subject: [master] 77a7e83 Update syntax for 3.0
Message-ID: <E1RmTC3-0005l5-Rj@project.varnish-software.com>

commit 77a7e8309c5ab30b81e98fdb3a1ac2b0140c9dd7
Author: Andreas Plesner Jacobsen <apj at mutt.dk>
Date:   Sun Jan 15 17:44:17 2012 +0100

    Update syntax for 3.0

diff --git a/doc/sphinx/tutorial/handling_misbehaving_servers.rst b/doc/sphinx/tutorial/handling_misbehaving_servers.rst
index c258f46..6cd7111 100644
--- a/doc/sphinx/tutorial/handling_misbehaving_servers.rst
+++ b/doc/sphinx/tutorial/handling_misbehaving_servers.rst
@@ -69,7 +69,7 @@ can be enabled in VCL:::
   sub vcl_fetch {
     if (beresp.status == 500) { 
       set beresp.saintmode = 10s;
-      restart;
+      return(restart);
     }
     set beresp.grace = 5m;
   } 



From drwilco at varnish-cache.org  Sun Jan 15 17:16:47 2012
From: drwilco at varnish-cache.org (Rogier Mulhuijzen)
Date: Sun, 15 Jan 2012 18:16:47 +0100
Subject: [master] 3eea8da Use sp->vsl_id as thread identifier
Message-ID: <E1RmThH-0001H6-Jm@project.varnish-software.com>

commit 3eea8dacc142df1b2c460b80113c8171c214f5f5
Author: Rogier 'DocWilco' Mulhuijzen <github at bsdchicks.com>
Date:   Sun Jan 15 16:39:42 2012 +0100

    Use sp->vsl_id as thread identifier
    
    POSIX threads don't have a thread identifier that can be printed
    in a portable manner.

diff --git a/bin/varnishd/cache/cache_center.c b/bin/varnishd/cache/cache_center.c
index 4967c82..69e854e 100644
--- a/bin/varnishd/cache/cache_center.c
+++ b/bin/varnishd/cache/cache_center.c
@@ -1655,13 +1655,13 @@ cnt_diag(struct sess *sp, const char *state)
 	}
 
 	if (sp->wrk != NULL) {
-		WSP(sp, SLT_Debug, "thr %p STP_%s sp %p obj %p vcl %p",
-		    pthread_self(), state, sp, obj, vcl);
+		WSP(sp, SLT_Debug, "vsl_id %u STP_%s sp %p obj %p vcl %p",
+		    sp->vsl_id, state, sp, obj, vcl);
 		WSL_Flush(sp->wrk, 0);
 	} else {
 		VSL(SLT_Debug, sp->vsl_id,
-		    "thr %p STP_%s sp %p obj %p vcl %p",
-		    pthread_self(), state, sp, obj, vcl);
+		    "vsl_id %u STP_%s sp %p obj %p vcl %p",
+		    sp->vsl_id, state, sp, obj, vcl);
 	}
 }
 



From drwilco at varnish-cache.org  Sun Jan 15 17:16:47 2012
From: drwilco at varnish-cache.org (Rogier Mulhuijzen)
Date: Sun, 15 Jan 2012 18:16:47 +0100
Subject: [master] d0be5ad Add __printflike and fix everything that brings to
	light
Message-ID: <E1RmThH-0001HB-MP@project.varnish-software.com>

commit d0be5ad9818095aa17f5dda12adf42d5e63c3ddf
Author: Rogier 'DocWilco' Mulhuijzen <github at bsdchicks.com>
Date:   Sun Jan 15 17:37:03 2012 +0100

    Add __printflike and fix everything that brings to light

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index 6b0d0a9..356d138 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -809,7 +809,8 @@ void http_PutStatus(struct http *to, uint16_t status);
 void http_PutResponse(struct worker *w, unsigned vsl_id, const struct http *to,
     const char *response);
 void http_PrintfHeader(struct worker *w, unsigned vsl_id, struct http *to,
-    const char *fmt, ...);
+    const char *fmt, ...)
+    __printflike(4, 5);
 void http_SetHeader(struct worker *w, unsigned vsl_id, struct http *to,
     const char *hdr);
 void http_SetH(const struct http *to, unsigned n, const char *fm);
@@ -928,10 +929,13 @@ void *VSM_Alloc(unsigned size, const char *class, const char *type,
     const char *ident);
 void VSM_Free(void *ptr);
 #ifdef VSL_ENDMARKER
-void VSL(enum VSL_tag_e tag, int id, const char *fmt, ...);
+void VSL(enum VSL_tag_e tag, int id, const char *fmt, ...)
+    __printflike(3, 4);
 void WSLR(struct worker *w, enum VSL_tag_e tag, int id, txt t);
-void WSL(struct worker *w, enum VSL_tag_e tag, int id, const char *fmt, ...);
-void WSLB(struct worker *w, enum VSL_tag_e tag, const char *fmt, ...);
+void WSL(struct worker *w, enum VSL_tag_e tag, int id, const char *fmt, ...)
+    __printflike(4, 5);
+void WSLB(struct worker *w, enum VSL_tag_e tag, const char *fmt, ...)
+    __printflike(3, 4);
 
 void WSL_Flush(struct worker *w, int overflow);
 
diff --git a/bin/varnishd/cache/cache_center.c b/bin/varnishd/cache/cache_center.c
index 69e854e..d80842a 100644
--- a/bin/varnishd/cache/cache_center.c
+++ b/bin/varnishd/cache/cache_center.c
@@ -1776,7 +1776,7 @@ cli_debug_srandom(struct cli *cli, const char * const *av, void *priv)
 		seed = strtoul(av[2], NULL, 0);
 	srandom(seed);
 	srand48(random());
-	VCLI_Out(cli, "Random(3) seeded with %lu", seed);
+	VCLI_Out(cli, "Random(3) seeded with %u", seed);
 }
 
 static struct cli_proto debug_cmds[] = {
diff --git a/bin/varnishd/cache/cache_shmlog.c b/bin/varnishd/cache/cache_shmlog.c
index c29f5a1..a839f5a 100644
--- a/bin/varnishd/cache/cache_shmlog.c
+++ b/bin/varnishd/cache/cache_shmlog.c
@@ -236,6 +236,10 @@ WSLR(struct worker *wrk, enum VSL_tag_e tag, int id, txt t)
 
 static void
 wsl(struct worker *wrk, enum VSL_tag_e tag, int id, const char *fmt, va_list ap)
+    __printflike(4, 0);
+
+static void
+wsl(struct worker *wrk, enum VSL_tag_e tag, int id, const char *fmt, va_list ap)
 {
 	char *p;
 	unsigned n, mlen;
diff --git a/bin/varnishd/cache/cache_vrt_vmod.c b/bin/varnishd/cache/cache_vrt_vmod.c
index 6b3b846..93d82f9 100644
--- a/bin/varnishd/cache/cache_vrt_vmod.c
+++ b/bin/varnishd/cache/cache_vrt_vmod.c
@@ -104,7 +104,8 @@ VRT_Vmod_Init(void **hdl, void *ptr, int len, const char *nm,
 		AN(w);
 		if (strcmp(x, nm)) {
 			VCLI_Out(cli, "Loading VMOD %s from %s:\n", nm, path);
-			VCLI_Out(cli, "File contain wrong VMOD (\"%s\")\n", x);
+			VCLI_Out(cli, "File contain wrong VMOD (\"%s\")\n",
+			    (char *) x);
 			VCLI_Out(cli, "Check relative pathnames ?.\n");
 			(void)dlclose(v->hdl);
 			FREE_OBJ(v);
diff --git a/bin/varnishd/mgt/mgt.h b/bin/varnishd/mgt/mgt.h
index 0e3eb9e..ffaef56 100644
--- a/bin/varnishd/mgt/mgt.h
+++ b/bin/varnishd/mgt/mgt.h
@@ -50,7 +50,8 @@ void MGT_Child_Cli_Fail(void);
 typedef void mgt_cli_close_f(void *priv);
 void mgt_cli_setup(int fdi, int fdo, int verbose, const char *ident,
     mgt_cli_close_f *close_func, void *priv);
-int mgt_cli_askchild(unsigned *status, char **resp, const char *fmt, ...);
+int mgt_cli_askchild(unsigned *status, char **resp, const char *fmt, ...)
+    __printflike(3, 4);
 void mgt_cli_start_child(int fdi, int fdo);
 void mgt_cli_stop_child(void);
 void mgt_cli_telnet(const char *T_arg);
diff --git a/bin/varnishd/mgt/mgt_param.c b/bin/varnishd/mgt/mgt_param.c
index c488f23..8123bfc 100644
--- a/bin/varnishd/mgt/mgt_param.c
+++ b/bin/varnishd/mgt/mgt_param.c
@@ -261,7 +261,7 @@ tweak_generic_uint(struct cli *cli, volatile unsigned *dest, const char *arg,
 		}
 		*dest = u;
 	} else if (*dest == UINT_MAX) {
-		VCLI_Out(cli, "unlimited", *dest);
+		VCLI_Out(cli, "unlimited");
 	} else {
 		VCLI_Out(cli, "%u", *dest);
 	}
diff --git a/bin/varnishreplay/varnishreplay.c b/bin/varnishreplay/varnishreplay.c
index 8e694da..02ad83e 100644
--- a/bin/varnishreplay/varnishreplay.c
+++ b/bin/varnishreplay/varnishreplay.c
@@ -44,6 +44,7 @@
 #include <string.h>
 #include <unistd.h>
 
+#include "vdef.h"
 #include "vapi/vsl.h"
 #include "vapi/vsm.h"
 #include "vas.h"
@@ -171,6 +172,10 @@ static pthread_mutex_t log_mutex = PTHREAD_MUTEX_INITIALIZER;
 
 static void
 thread_log(int lvl, int errcode, const char *fmt, ...)
+    __printflike(3, 4);
+
+static void
+thread_log(int lvl, int errcode, const char *fmt, ...)
 {
 	va_list ap;
 
diff --git a/bin/varnishtest/vtc.c b/bin/varnishtest/vtc.c
index 9581b3c..b8556ce 100644
--- a/bin/varnishtest/vtc.c
+++ b/bin/varnishtest/vtc.c
@@ -449,7 +449,7 @@ cmd_random(CMD_ARGS)
 		l = random();
 		if (l == random_expect[i])
 			continue;
-		vtc_log(vl, 4, "random[%d] = 0x%x (expect 0x%x)",
+		vtc_log(vl, 4, "random[%d] = 0x%x (expect 0x%lx)",
 		    i, l, random_expect[i]);
 		vtc_log(vl, 1, "SKIPPING test: unknown srandom(1) sequence.");
 		vtc_stop = 1;
@@ -538,17 +538,17 @@ exec_file(const char *fn, const char *script, const char *tmpdir,
 
 	/* Apply extmacro definitions */
 	VTAILQ_FOREACH(m, &extmacro_list, list)
-		macro_def(vltop, NULL, m->name, m->val);
+		macro_def(vltop, NULL, m->name, "%s", m->val);
 
 	/* Other macro definitions */
 	cwd = getcwd(NULL, PATH_MAX);
-	macro_def(vltop, NULL, "pwd", cwd);
+	macro_def(vltop, NULL, "pwd", "%s", cwd);
 	macro_def(vltop, NULL, "topbuild", "%s/%s", cwd, TOP_BUILDDIR);
 	macro_def(vltop, NULL, "bad_ip", "10.255.255.255");
 
 	/* Move into our tmpdir */
 	AZ(chdir(tmpdir));
-	macro_def(vltop, NULL, "tmpdir", tmpdir);
+	macro_def(vltop, NULL, "tmpdir", "%s", tmpdir);
 
 	/* Drop file to tell what was going on here */
 	f = fopen("INFO", "w");
diff --git a/bin/varnishtest/vtc.h b/bin/varnishtest/vtc.h
index ee5308a..be20cb9 100644
--- a/bin/varnishtest/vtc.h
+++ b/bin/varnishtest/vtc.h
@@ -77,7 +77,8 @@ void cmd_server_genvcl(struct vsb *vsb);
 void vtc_loginit(char *buf, unsigned buflen);
 struct vtclog *vtc_logopen(const char *id);
 void vtc_logclose(struct vtclog *vl);
-void vtc_log(struct vtclog *vl, int lvl, const char *fmt, ...);
+void vtc_log(struct vtclog *vl, int lvl, const char *fmt, ...)
+    __printflike(3, 4);
 void vtc_dump(struct vtclog *vl, int lvl, const char *pfx,
     const char *str, int len);
 void vtc_hexdump(struct vtclog *vl, int lvl, const char *pfx,
@@ -87,7 +88,9 @@ int exec_file(const char *fn, const char *script, const char *tmpdir,
     char *logbuf, unsigned loglen);
 
 void macro_def(struct vtclog *vl, const char *instance, const char *name,
-    const char *fmt, ...);
+    const char *fmt, ...)
+    __printflike(4, 5);
 struct vsb *macro_expand(struct vtclog *vl, const char *text);
 
-void extmacro_def(const char *name, const char *fmt, ...);
+void extmacro_def(const char *name, const char *fmt, ...)
+    __printflike(2, 3);
diff --git a/bin/varnishtest/vtc_http.c b/bin/varnishtest/vtc_http.c
index d3753a5..00be0be 100644
--- a/bin/varnishtest/vtc_http.c
+++ b/bin/varnishtest/vtc_http.c
@@ -133,13 +133,13 @@ synth_body(const char *len, int rnd)
 static void
 http_write(const struct http *hp, int lvl, const char *pfx)
 {
-	int l;
+	ssize_t l;
 
 	AZ(VSB_finish(hp->vsb));
 	vtc_dump(hp->vl, lvl, pfx, VSB_data(hp->vsb), VSB_len(hp->vsb));
 	l = write(hp->fd, VSB_data(hp->vsb), VSB_len(hp->vsb));
 	if (l != VSB_len(hp->vsb))
-		vtc_log(hp->vl, hp->fatal, "Write failed: (%d vs %d) %s",
+		vtc_log(hp->vl, hp->fatal, "Write failed: (%zd vs %zd) %s",
 		    l, VSB_len(hp->vsb), strerror(errno));
 }
 
@@ -387,7 +387,7 @@ http_rxchunk(struct http *hp)
 	bprintf(hp->chunklen, "%d", i);
 	if ((q == hp->rxbuf + l) ||
 		(*q != '\0' && !vct_islws(*q))) {
-		vtc_log(hp->vl, hp->fatal, "chunked fail %02x @ %d",
+		vtc_log(hp->vl, hp->fatal, "chunked fail %02x @ %td",
 		    *q, q - (hp->rxbuf + l));
 	}
 	assert(q != hp->rxbuf + l);
diff --git a/bin/varnishtest/vtc_sema.c b/bin/varnishtest/vtc_sema.c
index 48d4c68..fd4c561 100644
--- a/bin/varnishtest/vtc_sema.c
+++ b/bin/varnishtest/vtc_sema.c
@@ -63,7 +63,7 @@ sema_new(char *name, struct vtclog *vl)
 	AN(r);
 	r->name = name;
 	if (*name != 'r')
-		vtc_log(vl, 0, "Sema name must start with 'r' (%s)", *name);
+		vtc_log(vl, 0, "Sema name must start with 'r' (%s)", name);
 
 	AZ(pthread_mutex_init(&r->mtx, NULL));
 	AZ(pthread_cond_init(&r->cond, NULL));
diff --git a/include/vcli_common.h b/include/vcli_common.h
index 8d0a623..fbb4e9f 100644
--- a/include/vcli_common.h
+++ b/include/vcli_common.h
@@ -28,6 +28,8 @@
  *
  */
 
+#include "vdef.h"
+
 struct vlu;
 struct VCLS;
 
diff --git a/include/vcli_priv.h b/include/vcli_priv.h
index a265d2e..0129b83 100644
--- a/include/vcli_priv.h
+++ b/include/vcli_priv.h
@@ -53,6 +53,7 @@ struct cli_proto {
 
 /* The implementation must provide these functions */
 int VCLI_Overflow(struct cli *cli);
-void VCLI_Out(struct cli *cli, const char *fmt, ...);
+void VCLI_Out(struct cli *cli, const char *fmt, ...)
+    __printflike(2, 3);
 void VCLI_Quote(struct cli *cli, const char *str);
 void VCLI_SetResult(struct cli *cli, unsigned r);
diff --git a/include/vdef.h b/include/vdef.h
index c9b24c5..63a2a6b 100644
--- a/include/vdef.h
+++ b/include/vdef.h
@@ -1,10 +1,14 @@
 /*-
  * Copyright (c) 2006 Verdens Gang AS
- * Copyright (c) 2006-2011 Varnish Software AS
+ * Copyright (c) 2006-2012 Varnish Software AS
+ * Copyright (c) 2012 Fastly Inc
  * All rights reserved.
  *
  * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
+ * Author: Rogier 'DocWilco' Mulhuijzen <rogier at fastly.com>
  *
+ * Inspired by FreeBSD's <sys/cdefs.h>
+ * 
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
@@ -28,6 +32,9 @@
  *
  */
 
+#ifndef VDEF_H_INCLUDED
+#define VDEF_H_INCLUDED
+
 /* Safe printf into a fixed-size buffer */
 #define bprintf(buf, fmt, ...)						\
 	do {								\
@@ -42,3 +49,22 @@
 		    < sizeof buf);					\
 	} while (0)
 
+
+#ifndef __GNUC_PREREQ
+# if defined __GNUC__ && defined __GNUC_MINOR__
+#  define __GNUC_PREREQ(maj, min) \
+	(__GNUC__ > (maj) || (__GNUC__ == (maj) && __GNUC_MINOR__ >= (min)))
+# else
+#  define __GNUC_PREREQ(maj, min) 0
+# endif
+#endif
+
+#ifndef __printflike
+# if __GNUC_PREREQ(2, 95) || defined(__INTEL_COMPILER)
+#  define __printflike(f,a) __attribute__((format(printf, f, a)))
+# else
+#  define __printflike(f,a)
+# endif
+#endif
+
+#endif /* VDEF_H_INCLUDED */
diff --git a/include/vsb.h b/include/vsb.h
index 0a8e2bf..e17fe5b 100644
--- a/include/vsb.h
+++ b/include/vsb.h
@@ -31,6 +31,8 @@
 #ifndef VSB_H_INCLUDED
 #define VSB_H_INCLUDED
 
+#include "vdef.h"
+
 /*
  * Structure definition
  */
@@ -49,10 +51,6 @@ struct vsb {
 	int		 s_flags;	/* flags */
 };
 
-#ifndef __printflike
-#define __printflike(a,b)
-#endif
-
 #ifdef __cplusplus
 extern "C" {
 #endif
diff --git a/lib/libvarnishapi/vsc.c b/lib/libvarnishapi/vsc.c
index accef82..5f992cf 100644
--- a/lib/libvarnishapi/vsc.c
+++ b/lib/libvarnishapi/vsc.c
@@ -39,6 +39,7 @@
 
 #include "miniobj.h"
 #include "vas.h"
+#include "vdef.h"
 
 #include "vapi/vsc.h"
 #include "vapi/vsm.h"
diff --git a/lib/libvarnishapi/vsl.c b/lib/libvarnishapi/vsl.c
index 6950c1b..8e7001a 100644
--- a/lib/libvarnishapi/vsl.c
+++ b/lib/libvarnishapi/vsl.c
@@ -41,6 +41,7 @@
 
 #include "miniobj.h"
 #include "vas.h"
+#include "vdef.h"
 
 #include "vapi/vsl.h"
 #include "vapi/vsm.h"
diff --git a/lib/libvarnishapi/vsl_arg.c b/lib/libvarnishapi/vsl_arg.c
index 068d539..8587fb1 100644
--- a/lib/libvarnishapi/vsl_arg.c
+++ b/lib/libvarnishapi/vsl_arg.c
@@ -43,6 +43,7 @@
 
 #include "miniobj.h"
 #include "vas.h"
+#include "vdef.h"
 
 #include "vapi/vsl.h"
 #include "vapi/vsm.h"
diff --git a/lib/libvarnishapi/vsm.c b/lib/libvarnishapi/vsm.c
index e11fe26..c175f10 100644
--- a/lib/libvarnishapi/vsm.c
+++ b/lib/libvarnishapi/vsm.c
@@ -44,6 +44,7 @@
 
 #include "miniobj.h"
 #include "vas.h"
+#include "vdef.h"
 
 #include "vapi/vsm.h"
 #include "vapi/vsm_int.h"
diff --git a/lib/libvarnishapi/vsm_api.h b/lib/libvarnishapi/vsm_api.h
index 79be982..af99798 100644
--- a/lib/libvarnishapi/vsm_api.h
+++ b/lib/libvarnishapi/vsm_api.h
@@ -51,6 +51,7 @@ struct VSM_data {
 	struct vsl		*vsl;
 };
 
-int vsm_diag(struct VSM_data *vd, const char *fmt, ...);
+int vsm_diag(struct VSM_data *vd, const char *fmt, ...)
+    __printflike(2, 3);
 void VSC_Delete(struct VSM_data *vd);
 void VSL_Delete(struct VSM_data *vd);
diff --git a/lib/libvcl/vcc_acl.c b/lib/libvcl/vcc_acl.c
index c962044..e6a1065 100644
--- a/lib/libvcl/vcc_acl.c
+++ b/lib/libvcl/vcc_acl.c
@@ -362,12 +362,12 @@ vcc_acl_emit(const struct vcc *tl, const char *acln, int anon)
 
 	Fh(tl, 0, "\n");
 	Fh(tl, 0, "\ta = p;\n");
-	Fh(tl, 0, "\tVRT_memmove(&fam, a + %d, sizeof fam);\n",
+	Fh(tl, 0, "\tVRT_memmove(&fam, a + %zd, sizeof fam);\n",
 	    offsetof(struct sockaddr, sa_family));
 	Fh(tl, 0, "\tif (fam == %d)\n", PF_INET);
-	Fh(tl, 0, "\t\ta += %d;\n", offsetof(struct sockaddr_in, sin_addr));
+	Fh(tl, 0, "\t\ta += %zd;\n", offsetof(struct sockaddr_in, sin_addr));
 	Fh(tl, 0, "\telse if (fam == %d)\n", PF_INET6);
-	Fh(tl, 0, "\t\ta += %d;\n", offsetof(struct sockaddr_in6, sin6_addr));
+	Fh(tl, 0, "\t\ta += %zd;\n", offsetof(struct sockaddr_in6, sin6_addr));
 	Fh(tl, 0, "\telse {\n");
 	Fh(tl, 0, "\t\tVRT_acl_log(sp, \"NO_FAM %s\");\n", acln);
 	Fh(tl, 0, "\t\treturn(0);\n");
@@ -422,8 +422,7 @@ vcc_acl_emit(const struct vcc *tl, const char *acln, int anon)
 
 		if (!anon) {
 			Fh(tl, 0, "\t%*sVRT_acl_log(sp, \"%sMATCH %s \" ",
-			    -i, "", ae->not ? "NEG_" : "", acln,
-			    PF(ae->t_addr));
+			    -i, "", ae->not ? "NEG_" : "", acln);
 			EncToken(tl->fh, ae->t_addr);
 			if (ae->t_mask != NULL)
 				Fh(tl, 0, " \"/%.*s\" ", PF(ae->t_mask));
diff --git a/lib/libvcl/vcc_backend.c b/lib/libvcl/vcc_backend.c
index 03482c4..fb160e7 100644
--- a/lib/libvcl/vcc_backend.c
+++ b/lib/libvcl/vcc_backend.c
@@ -353,7 +353,7 @@ vcc_ParseProbeSpec(struct vcc *tl)
 	if (t_initial != NULL)
 		Fh(tl, 0, "\t.initial = %u,\n", initial);
 	else
-		Fh(tl, 0, "\t.initial = ~0U,\n", initial);
+		Fh(tl, 0, "\t.initial = ~0U,\n");
 	if (status > 0)
 		Fh(tl, 0, "\t.exp_status = %u,\n", status);
 	Fh(tl, 0, "};\n");
diff --git a/lib/libvcl/vcc_compile.c b/lib/libvcl/vcc_compile.c
index 27447fe..66d89f5 100644
--- a/lib/libvcl/vcc_compile.c
+++ b/lib/libvcl/vcc_compile.c
@@ -286,7 +286,7 @@ LocTable(const struct vcc *tl)
 				pos++;
 
 		}
-		Fc(tl, 0, "  [%3u] = { %d, %8u, %4u, %3u, 0, ",
+		Fc(tl, 0, "  [%3u] = { %d, %8tu, %4u, %3u, 0, ",
 		    t->cnt, sp->idx, t->b - sp->b, lin, pos + 1);
 		if (t->tok == CSRC)
 			Fc(tl, 0, " \"C{\"},\n");
diff --git a/lib/libvcl/vcc_compile.h b/lib/libvcl/vcc_compile.h
index caacd73..b64564e 100644
--- a/lib/libvcl/vcc_compile.h
+++ b/lib/libvcl/vcc_compile.h
@@ -247,11 +247,16 @@ extern struct method method_tab[];
  * I -> Initializer function
  * F -> Finish function
  */
-void Fh(const struct vcc *tl, int indent, const char *fmt, ...);
-void Fc(const struct vcc *tl, int indent, const char *fmt, ...);
-void Fb(const struct vcc *tl, int indent, const char *fmt, ...);
-void Fi(const struct vcc *tl, int indent, const char *fmt, ...);
-void Ff(const struct vcc *tl, int indent, const char *fmt, ...);
+void Fh(const struct vcc *tl, int indent, const char *fmt, ...)
+    __printflike(3, 4);
+void Fc(const struct vcc *tl, int indent, const char *fmt, ...)
+    __printflike(3, 4);
+void Fb(const struct vcc *tl, int indent, const char *fmt, ...)
+    __printflike(3, 4);
+void Fi(const struct vcc *tl, int indent, const char *fmt, ...)
+    __printflike(3, 4);
+void Ff(const struct vcc *tl, int indent, const char *fmt, ...)
+    __printflike(3, 4);
 void EncToken(struct vsb *sb, const struct token *t);
 int IsMethod(const struct token *t);
 void *TlAlloc(struct vcc *tl, unsigned len);
diff --git a/lib/libvcl/vcc_dir_dns.c b/lib/libvcl/vcc_dir_dns.c
index 174d1ab..55ed921 100644
--- a/lib/libvcl/vcc_dir_dns.c
+++ b/lib/libvcl/vcc_dir_dns.c
@@ -92,9 +92,9 @@ print_backend(struct vcc *tl,
 
 	Fb(tl, 0, "\t.hosthdr = \"");
 	if (b_defaults.hostheader != NULL)
-		Fb(tl,0, b_defaults.hostheader);
+		Fb(tl, 0, "%s", b_defaults.hostheader);
 	else
-		Fb(tl,0, strip);
+		Fb(tl, 0, "%s", strip);
 	Fb(tl, 0, "\",\n");
 
 	Fb(tl, 0, "\t.saintmode_threshold = %d,\n",b_defaults.saint);
diff --git a/lib/libvcl/vcc_expr.c b/lib/libvcl/vcc_expr.c
index 3dfeeaa..82e53ef 100644
--- a/lib/libvcl/vcc_expr.c
+++ b/lib/libvcl/vcc_expr.c
@@ -253,6 +253,10 @@ vcc_new_expr(void)
 
 static struct expr *
 vcc_mk_expr(enum var_type fmt, const char *str, ...)
+    __printflike(2, 3);
+
+static struct expr *
+vcc_mk_expr(enum var_type fmt, const char *str, ...)
 {
 	va_list ap;
 	struct expr *e;
@@ -537,7 +541,7 @@ vcc_Eval_Func(struct vcc *tl, struct expr **e, const struct symbol *sym)
 			r = strchr(sym->name, '.');
 			AN(r);
 			e1 = vcc_mk_expr(VOID, "&vmod_priv_%.*s",
-			    r - sym->name, sym->name);
+			    (int) (r - sym->name), sym->name);
 			p += strlen(p) + 1;
 		} else if (fmt == VOID && !strcmp(p, "PRIV_CALL")) {
 			bprintf(buf, "vmod_priv_%u", tl->nvmodpriv++);
diff --git a/lib/libvcl/vcc_parse.c b/lib/libvcl/vcc_parse.c
index 4023287..d8e74a4 100644
--- a/lib/libvcl/vcc_parse.c
+++ b/lib/libvcl/vcc_parse.c
@@ -154,7 +154,7 @@ vcc_Compound(struct vcc *tl)
 			return;
 		case CSRC:
 			Fb(tl, 1, "%.*s\n",
-			    tl->t->e - (tl->t->b + 2),
+			    (int) (tl->t->e - (tl->t->b + 2)),
 			    tl->t->b + 1);
 			vcc_NextToken(tl);
 			break;
@@ -274,7 +274,7 @@ vcc_Parse(struct vcc *tl)
 		switch (tl->t->tok) {
 		case CSRC:
 			Fc(tl, 0, "%.*s\n",
-			    tl->t->e - (tl->t->b + 4), tl->t->b + 2);
+			    (int) (tl->t->e - (tl->t->b + 4)), tl->t->b + 2);
 			vcc_NextToken(tl);
 			break;
 		case EOI:



From phk at varnish-cache.org  Sun Jan 15 19:31:42 2012
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Sun, 15 Jan 2012 20:31:42 +0100
Subject: [master] 5b8d896 By Slink, via Geoff:
Message-ID: <E1RmVnq-0000rG-5g@project.varnish-software.com>

commit 5b8d8964491a85d06c5c496d8242bd6dfa14ec7e
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sun Jan 15 19:30:53 2012 +0000

    By Slink, via Geoff:
    
    The attached patch is a one-liner for waiter/cache_waiter_ports.c,
    the code was referencing sp->t_open where it should have been
    sp->t_idle, which was causing assertion failures on idle session
    timeouts.

diff --git a/bin/varnishd/waiter/cache_waiter_ports.c b/bin/varnishd/waiter/cache_waiter_ports.c
index 4f39ca4..87eaeac 100644
--- a/bin/varnishd/waiter/cache_waiter_ports.c
+++ b/bin/varnishd/waiter/cache_waiter_ports.c
@@ -3,7 +3,7 @@
  * Copyright (c) 2006 Varnish Software AS
  * Copyright (c) 2007 OmniTI Computer Consulting, Inc.
  * Copyright (c) 2007 Theo Schlossnagle
- * Copyright (c) 2010 UPLEX, Nils Goroll
+ * Copyright (c) 2010-2012 UPLEX, Nils Goroll
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -220,7 +220,7 @@ vws_thread(void *priv)
 
 		if (sp) {
 			double tmo =
-			    (sp->t_open + cache_param->timeout_idle) - now;
+			    (sp->t_idle + cache_param->timeout_idle) - now;
 
 			/*
 			 * we should have removed all sps whose timeout



From drwilco at varnish-cache.org  Sun Jan 15 22:54:01 2012
From: drwilco at varnish-cache.org (Rogier Mulhuijzen)
Date: Sun, 15 Jan 2012 23:54:01 +0100
Subject: [master] ca587d1 Simplify vry_cmp's signature
Message-ID: <E1RmYxd-0000Yf-NY@project.varnish-software.com>

commit ca587d1331272972ccd2b8c3e6fc589e3768007a
Author: Rogier 'DocWilco' Mulhuijzen <github at bsdchicks.com>
Date:   Sun Jan 15 23:19:55 2012 +0100

    Simplify vry_cmp's signature

diff --git a/bin/varnishd/cache/cache_vary.c b/bin/varnishd/cache/cache_vary.c
index 453b52d..9ad41d0 100644
--- a/bin/varnishd/cache/cache_vary.c
+++ b/bin/varnishd/cache/cache_vary.c
@@ -145,18 +145,18 @@ vry_len(const uint8_t *p)
  * Compare two vary entries
  */
 static int
-vry_cmp(const uint8_t * const *v1, uint8_t * const *v2)
+vry_cmp(const uint8_t *v1, const uint8_t *v2)
 {
 	unsigned retval = 0;
 
-	if (!memcmp(*v1, *v2, vry_len(*v1))) {
+	if (!memcmp(v1, v2, vry_len(v1))) {
 		/* Same same */
 		retval = 0;
-	} else if (memcmp((*v1) + 2, (*v2) + 2, (*v1)[2] + 2)) {
+	} else if (memcmp(v1 + 2, v2 + 2, v1[2] + 2)) {
 		/* Different header */
 		retval = 1;
 	} else if (cache_param->http_gzip_support &&
-	    !strcasecmp(H_Accept_Encoding, (const char*)((*v1)+2))) {
+	    !strcasecmp(H_Accept_Encoding, (const char*) v1 + 2)) {
 		/*
 		 * If we do gzip processing, we do not vary on Accept-Encoding,
 		 * because we want everybody to get the gzip'ed object, and
@@ -183,7 +183,7 @@ VRY_Match(const struct sess *sp, const uint8_t *vary)
 
 	AN(vsp);
 	while (vary[2]) {
-		i = vry_cmp(&vary, &vsp);
+		i = vry_cmp(vary, vsp);
 		if (i == 1) {
 			/* Build a new entry */
 
@@ -222,7 +222,7 @@ VRY_Match(const struct sess *sp, const uint8_t *vary)
 			} else
 				vsp[2 + vary[2] + 2 + 2] = '\0';
 
-			i = vry_cmp(&vary, &vsp);
+			i = vry_cmp(vary, vsp);
 			assert(i != 1);	/* hdr must be the same now */
 		}
 		if (i != 0)



From tfheen at varnish-cache.org  Mon Jan 16 09:13:08 2012
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Mon, 16 Jan 2012 10:13:08 +0100
Subject: [master] 0471b9b Add support for the %{format}t construct to
	varnishncsa
Message-ID: <E1Rmicm-0006xs-6R@project.varnish-software.com>

commit 0471b9bf34c477db127aa11132e4efd99a68ef37
Author: Tollef Fog Heen <tfheen at varnish-software.com>
Date:   Mon Jan 16 10:12:14 2012 +0100

    Add support for the %{format}t construct to varnishncsa

diff --git a/bin/varnishncsa/varnishncsa.c b/bin/varnishncsa/varnishncsa.c
index 9da6172..9abb9b6 100644
--- a/bin/varnishncsa/varnishncsa.c
+++ b/bin/varnishncsa/varnishncsa.c
@@ -757,6 +757,11 @@ h_ncsa(void *priv, enum VSL_tag_e tag, unsigned fd,
 				VSB_cat(os, h ? h : "-");
 				p = tmp;
 				break;
+			case 't':
+				strftime(tbuf, sizeof tbuf, fname, &lp->df_t);
+				VSB_cat(os, tbuf);
+				p = tmp;
+				break;
 			case 'x':
 				if (!strcmp(fname, "Varnish:time_firstbyte")) {
 					VSB_cat(os, lp->df_ttfb);
diff --git a/doc/sphinx/reference/varnishncsa.rst b/doc/sphinx/reference/varnishncsa.rst
index 0b3f97f..9373f5c 100644
--- a/doc/sphinx/reference/varnishncsa.rst
+++ b/doc/sphinx/reference/varnishncsa.rst
@@ -90,6 +90,11 @@ The following options are available:
 	         Time when the request was received, in HTTP date/time
 	         format.
 
+	      %{X}t
+	         Time when the request was received, in the format
+		 specified by X.  The time specification format is the
+		 same as for strftime(3).
+
 	      %U
 	         The request URL without any query string. Defaults to
                  '-' if not known.



From geoff at varnish-cache.org  Mon Jan 16 16:40:52 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 16 Jan 2012 17:40:52 +0100
Subject: [experimental-ims] c656cdc Fix conditional backend request in
	streaming mode
Message-ID: <E1Rmpc4-0007DC-FJ@project.varnish-software.com>

commit c656cdcc45534f2722810ced60d0025193cc93db
Author: Jocelyn De La Rosa <jocelyn.delarosa at smartjog.com>
Date:   Tue Jan 10 15:56:48 2012 +0100

    Fix conditional backend request in streaming mode
    
    Signed-off-by: Geoff Simmons <geoff at uplex.de>

diff --git a/bin/varnishd/cache/cache_center.c b/bin/varnishd/cache/cache_center.c
index 1883943..04a2444 100644
--- a/bin/varnishd/cache/cache_center.c
+++ b/bin/varnishd/cache/cache_center.c
@@ -896,8 +896,10 @@ cnt_fetchbody(struct sess *sp, struct worker *wrk, struct req *req)
          */
         if (sp->stale_obj) {
             http_Check304(sp);
-            if (wrk->busyobj->beresp->status == 304)
+            if (wrk->busyobj->beresp->status == 304) {
                 assert(req->obj->http->status == 200);
+		wrk->busyobj->do_stream = 0;
+	    }
 	    EXP_Clr(&sp->stale_obj->exp);
 	    EXP_Rearm(sp->stale_obj);
 	    HSH_Deref(sp->wrk, NULL, &sp->stale_obj);



From geoff at varnish-cache.org  Mon Jan 16 16:40:52 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 16 Jan 2012 17:40:52 +0100
Subject: [experimental-ims] 42654fd Bugfixes in experimental-ims: - TTL was
	set to -1 after a 304 response - Operation with do_stream =
	true experimental-ims now passes (my version of) the rapid
	expire stress test
Message-ID: <E1Rmpc4-0007DH-H9@project.varnish-software.com>

commit 42654fd13f6f5427bbc57553dc50486f4ed38fa4
Author: Geoff Simmons <geoff at uplex.de>
Date:   Mon Jan 16 17:36:21 2012 +0100

    Bugfixes in experimental-ims:
    - TTL was set to -1 after a 304 response
    - Operation with do_stream = true
    experimental-ims now passes (my version of) the rapid expire stress test

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index 898f3ba..c7924a9 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -810,7 +810,7 @@ void http_FilterHeader(const struct sess *sp, unsigned how);
 /* Check if a refresh should be done */
 void http_CheckRefresh(struct sess *sp);
 /* Check if we got 304 response */
-void http_Check304(struct sess *sp);
+void http_Check304(struct sess *sp, struct busyobj *busyobj);
 
 void http_PutProtocol(struct worker *w, unsigned vsl_id, const struct http *to,
     const char *protocol);
diff --git a/bin/varnishd/cache/cache_center.c b/bin/varnishd/cache/cache_center.c
index 04a2444..3a91016 100644
--- a/bin/varnishd/cache/cache_center.c
+++ b/bin/varnishd/cache/cache_center.c
@@ -73,6 +73,8 @@ DOT acceptor -> first [style=bold,color=green]
 #include "vtcp.h"
 #include "vtim.h"
 
+#include "storage/storage.h"
+
 #ifndef HAVE_SRANDOMDEV
 #include "compat/srandomdev.h"
 #endif
@@ -633,6 +635,15 @@ cnt_fetch(struct sess *sp, struct worker *wrk, struct req *req)
 		 */
 		wrk->busyobj->body_status = RFC2616_Body(sp);
 
+		/*
+		 * If we found a candidate for conditional backend
+		 * request, check now if the backend responded with 304,
+		 * and merge headers from stale_obj into the busyobj if so.
+		 * Any other response is handled as usual.
+		 */
+		if (sp->stale_obj)
+			http_Check304(sp, wrk->busyobj);
+
 		req->err_code = http_GetStatus(wrk->busyobj->beresp);
 
 		/*
@@ -651,6 +662,12 @@ cnt_fetch(struct sess *sp, struct worker *wrk, struct req *req)
 
 		VCL_fetch_method(sp);
 
+		/* Cancel streaming if a stale object was validated    */
+		/* XXX: But not if original beresp.status != 304       */
+		/*      See also AZ(sp->stale_obj) in cnt_streambody() */
+		if (sp->stale_obj)
+			wrk->busyobj->do_stream = 0;
+
 		switch (req->handling) {
 		case VCL_RET_HIT_FOR_PASS:
 			if (req->objcore != NULL)
@@ -888,23 +905,6 @@ cnt_fetchbody(struct sess *sp, struct worker *wrk, struct req *req)
 	http_FilterFields(wrk, sp->vsl_id, hp2, hp,
 	    pass ? HTTPH_R_PASS : HTTPH_A_INS);
 
-        /*
-	 * If we found a candidate for conditional backend request, attempt it
-         * now. If backend responds with 304, http_Check304() merges stale_obj
-         * into wrk->obj, any other response is handled as usual. In either case,
-         * the stale_obj is no longer needed in the cache, so discard it.
-         */
-        if (sp->stale_obj) {
-            http_Check304(sp);
-            if (wrk->busyobj->beresp->status == 304) {
-                assert(req->obj->http->status == 200);
-		wrk->busyobj->do_stream = 0;
-	    }
-	    EXP_Clr(&sp->stale_obj->exp);
-	    EXP_Rearm(sp->stale_obj);
-	    HSH_Deref(sp->wrk, NULL, &sp->stale_obj);
-	    AZ(sp->stale_obj);
-        }
 	http_CopyHome(wrk, sp->vsl_id, hp2);
 
 	if (http_GetHdr(hp, H_Last_Modified, &b)
@@ -935,6 +935,25 @@ cnt_fetchbody(struct sess *sp, struct worker *wrk, struct req *req)
 	/* Use unmodified headers*/
 	i = FetchBody(wrk, req->obj);
 
+	/*
+	 * If a stale_obj was found, dup its storage into the new obj,
+	 * reset Content-Length from the size of the storage, and discard
+	 * the stale_obj.
+	 */
+	if (sp->stale_obj) {
+		STV_dup(sp, sp->stale_obj, req->obj);
+		assert(sp->stale_obj->len == req->obj->len);
+		
+		http_Unset(req->obj->http, H_Content_Length);
+		http_PrintfHeader(sp->wrk, sp->fd, req->obj->http,
+		    "Content-Length: %u", req->obj->len);
+		
+		EXP_Clr(&sp->stale_obj->exp);
+		EXP_Rearm(sp->stale_obj);
+		HSH_Deref(sp->wrk, NULL, &sp->stale_obj);
+		AZ(sp->stale_obj);
+	}
+
 	http_Setup(wrk->busyobj->bereq, NULL);
 	http_Setup(wrk->busyobj->beresp, NULL);
 	wrk->busyobj->vfp = NULL;
@@ -985,6 +1004,7 @@ cnt_streambody(struct sess *sp, struct worker *wrk, struct req *req)
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
+	AZ(sp->stale_obj);
 
 	CHECK_OBJ_NOTNULL(wrk->busyobj, BUSYOBJ_MAGIC);
 	memset(&sctx, 0, sizeof sctx);
diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c
index 98b88c0..7070612 100644
--- a/bin/varnishd/cache/cache_fetch.c
+++ b/bin/varnishd/cache/cache_fetch.c
@@ -509,12 +509,7 @@ FetchBody(struct worker *wrk, struct object *obj)
 	AssertObjCorePassOrBusy(obj->objcore);
 
 	AZ(bo->vgz_rx);
-
-        /* If we've freshened from another object and got a "Not Modified"
-         * response, then we have already duped the other object's body.
-         */
-        if (bo->beresp->status != 304)
-        	AZ(VTAILQ_FIRST(&obj->store));
+	AZ(VTAILQ_FIRST(&obj->store));
 
 	bo->fetch_obj = obj;
 	bo->fetch_failed = 0;
diff --git a/bin/varnishd/cache/cache_hash.c b/bin/varnishd/cache/cache_hash.c
index 6ae193d..827d6af 100644
--- a/bin/varnishd/cache/cache_hash.c
+++ b/bin/varnishd/cache/cache_hash.c
@@ -307,7 +307,7 @@ HSH_Lookup(struct sess *sp, struct objhead **poh)
 	CHECK_OBJ_NOTNULL(sp->req->http, HTTP_MAGIC);
 	AN(sp->req->director);
 	AN(hash);
-        AZ(sp->stale_obj);
+	AZ(sp->stale_obj);
 	wrk = sp->wrk;
 
 	HSH_Prealloc(sp);
diff --git a/bin/varnishd/cache/cache_http.c b/bin/varnishd/cache/cache_http.c
index 4f48d52..7ea85f2 100644
--- a/bin/varnishd/cache/cache_http.c
+++ b/bin/varnishd/cache/cache_http.c
@@ -34,7 +34,6 @@
 #include <stdio.h>
 
 #include "cache.h"
-#include "storage/storage.h"
 
 #include "vct.h"
 
@@ -951,52 +950,46 @@ http_CheckRefresh(struct sess *sp)
  */
 
 void
-http_Check304(struct sess *sp)
+http_Check304(struct sess *sp, struct busyobj *busyobj)
 {
-	struct object *o, *o_stale;
+	struct object *o_stale;
 	char *p;
 
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+	CHECK_OBJ_NOTNULL(busyobj, BUSYOBJ_MAGIC);
 	o_stale = sp->stale_obj;
 	CHECK_OBJ_NOTNULL(o_stale, OBJECT_MAGIC);
-	o = sp->req->obj;
-	CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC);
 
-	if (sp->wrk->busyobj->beresp->status != 304) {
+	if (busyobj->beresp->status != 304) {
 	    /*
 	     * IMS/INM headers may have been removed in VCL, so only count a
 	     * non-validating response if they were present in the request.
 	     */
-	    if (http_GetHdr(sp->wrk->busyobj->bereq, H_If_Modified_Since, &p)
-		|| http_GetHdr(sp->wrk->busyobj->bereq, H_If_None_Match, &p))
+	    if (http_GetHdr(busyobj->bereq, H_If_Modified_Since, &p)
+		|| http_GetHdr(busyobj->bereq, H_If_None_Match, &p))
 		sp->wrk->stats.fetch_not_validated++;
+
+	    HSH_Deref(sp->wrk, NULL, &sp->stale_obj);
+	    AZ(sp->stale_obj);
 	    return;
 	}
 
 	/* 
 	 * Copy headers we need from the stale object into the 304 response
 	 */
-	http_FilterMissingFields(sp->wrk, sp->fd, o->http, o_stale->http);
-
-	/*
-	 * Dup the stale object's storage in to the new object
-	 * and reset Content-Length from the size of the storage.
-	 */
-	STV_dup(sp, o_stale, o);
-	http_Unset(o->http, H_Content_Length);
-	http_PrintfHeader(sp->wrk, sp->fd, o->http, "Content-Length: %u",
-	    o->len);
+	http_FilterMissingFields(sp->wrk, sp->fd, busyobj->beresp,
+	    o_stale->http);
 
-	http_SetResp(o->http, "HTTP/1.1", 200, "Ok Not Modified");
-	http_SetH(o->http, HTTP_HDR_REQ, "GET");
-	http_copyh(o->http, sp->wrk->busyobj->bereq, HTTP_HDR_URL);
+	http_SetResp(busyobj->beresp, "HTTP/1.1", 200, "Ok Not Modified");
+	http_SetH(busyobj->beresp, HTTP_HDR_REQ, "GET");
+	http_copyh(busyobj->beresp, busyobj->bereq, HTTP_HDR_URL);
 
 	/*
 	 * XXX: Are we copying all the necessary fields from stale_obj?
 	 *	Should we copy o_stale->hits into o->hits?
+	 *      What about do_esi and do_g(un)zip?
 	 */
-	o->response = 200;
-	o->gziped = o_stale->gziped;
+	busyobj->is_gzip = o_stale->gziped;
 
         AZ(o_stale->objcore->flags & OC_F_BUSY);
 }
diff --git a/bin/varnishtest/tests/i00001.vtc b/bin/varnishtest/tests/i00001.vtc
index 6383a1f..1d25a8f 100644
--- a/bin/varnishtest/tests/i00001.vtc
+++ b/bin/varnishtest/tests/i00001.vtc
@@ -19,10 +19,10 @@ server s1 {
         txresp -status 304
 } -start
 
-varnish v1 -vcl+backend {
-        sub vcl_fetch {
-            set beresp.ttl = 0.5s;
-        }
+varnish v1 -arg "-t 0.5" -vcl+backend {
+	sub vcl_fetch {
+	    set beresp.http.X-TTL = beresp.ttl;
+	}
 } -start
 
 client c1 {
@@ -32,6 +32,7 @@ client c1 {
         expect resp.http.content-length == 6
         expect resp.bodylen == 6
         expect resp.http.Age == 0
+	expect resp.http.X-TTL == 0.500
 } -run
 
 delay 0.6
@@ -40,6 +41,7 @@ client c1 -run
 
 varnish v1 -expect fetch_304 == 1
 varnish v1 -expect fetch_not_validated == 0
+varnish v1 -expect cache_hitpass == 0
 
 server s1 -wait
 varnish v1 -stop
diff --git a/bin/varnishtest/tests/i00003.vtc b/bin/varnishtest/tests/i00003.vtc
index fb66266..b4220e1 100644
--- a/bin/varnishtest/tests/i00003.vtc
+++ b/bin/varnishtest/tests/i00003.vtc
@@ -52,6 +52,47 @@ varnish v1 -expect fetch_not_validated == 0
 server s1 -wait
 varnish v1 -stop
 
+## Verify that backend conditionals work correctly with streaming
+## i.e. streaming is cancelled if a stale object is delivered
+
+server s1 {
+        rxreq
+        expect req.url == "/foo"
+        txresp -status 200 \
+    	    -hdr "Last-Modified: Thu, 26 Jun 2008 12:00:01 GMT" \
+            -hdr "ETag: foo" \
+            -body "abcdefghijklmonpqrstuvwxyz"
+
+        rxreq
+        expect req.url == "/foo"
+        expect req.http.If-Modified-Since == "Thu, 26 Jun 2008 12:00:01 GMT"
+        expect req.http.If-None-Match == "foo"
+        txresp -status 304
+} -start
+
+varnish v1 -vcl {
+        backend s1 {
+                .host = "${s1_addr}"; .port = "${s1_port}";
+        }
+
+        sub vcl_fetch {
+            set beresp.ttl = 0.5s;
+	    set beresp.do_stream = true;
+        }
+} -start
+
+client c1 -run
+
+delay 0.6
+
+client c1 -run
+
+varnish v1 -expect fetch_304 == 1
+varnish v1 -expect fetch_not_validated == 0
+
+server s1 -wait
+varnish v1 -stop
+
 ## Verify that if a client sends a conditional request to Varnish, then Varnish
 ## can return a 304 response to the client after it got 304 from the backend
 
@@ -99,9 +140,8 @@ varnish v1 -expect fetch_not_validated == 0
 server s2 -wait
 varnish v1 -stop
 
-##
-## If stale_obj has a gzipped body, make sure it interacts correctly with clients
-##
+## If stale_obj has a gzipped body, make sure it interacts correctly
+## with clients
 
 server s2 {
         rxreq
@@ -144,3 +184,6 @@ client c2 {
 
 varnish v1 -expect fetch_304 == 1
 varnish v1 -expect fetch_not_validated == 0
+
+server s2 -wait
+varnish v1 -stop
diff --git a/bin/varnishtest/tests/i00004.vtc b/bin/varnishtest/tests/i00004.vtc
index 77fb9a6..a173e29 100644
--- a/bin/varnishtest/tests/i00004.vtc
+++ b/bin/varnishtest/tests/i00004.vtc
@@ -49,6 +49,7 @@ client c1 -run
 
 varnish v1 -expect fetch_304 == 1
 varnish v1 -expect fetch_not_validated == 0
+varnish v1 -expect cache_hitpass == 0
 
 server s1 -wait
 varnish v1 -stop



From geoff at varnish-cache.org  Mon Jan 16 16:58:30 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 16 Jan 2012 17:58:30 +0100
Subject: [experimental-ims] bf78ea9 typo. Thanks scoof
Message-ID: <E1Rmpt8-0000n6-UG@project.varnish-software.com>

commit bf78ea99790454329cb2d72b7f4903a9cad123bc
Author: Per Buer <perbu at varnish-software.com>
Date:   Wed Jan 11 12:48:23 2012 +0100

    typo. Thanks scoof

diff --git a/doc/sphinx/tutorial/introduction.rst b/doc/sphinx/tutorial/introduction.rst
index 17d7b6a..0d43623 100644
--- a/doc/sphinx/tutorial/introduction.rst
+++ b/doc/sphinx/tutorial/introduction.rst
@@ -3,11 +3,11 @@
 What is Varnish?
 ----------------
 
-Varnish Cache is a Varnish Cache is a web application accelerator also
-known as a caching HTTP reverse proxy. You install it in front of any
-server that speaks HTTP and configure it to cache the
-contents. Varnish Cache is really, really fast. It typically speeds up
-delivery with a factor of 300 - 1000x, depending on your architecture.
+Varnish Cache is a web application accelerator also known as a caching
+HTTP reverse proxy. You install it in front of any server that speaks
+HTTP and configure it to cache the contents. Varnish Cache is really,
+really fast. It typically speeds up delivery with a factor of 300 -
+1000x, depending on your architecture.
 
 
 Performance



From geoff at varnish-cache.org  Mon Jan 16 16:58:30 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 16 Jan 2012 17:58:30 +0100
Subject: [experimental-ims] 2bcd2f0 Added an introduction. Brushed up the
	chapter on varnish in a virtualized environment
Message-ID: <E1Rmpt8-0000n1-T0@project.varnish-software.com>

commit 2bcd2f0fbea8c1ab2e4b623aa7b71cb139b54659
Author: Per Buer <perbu at varnish-software.com>
Date:   Wed Jan 11 10:26:56 2012 +0100

    Added an introduction. Brushed up the chapter on varnish in a virtualized environment

diff --git a/doc/sphinx/tutorial/index.rst b/doc/sphinx/tutorial/index.rst
index 32c2264..0bf3137 100644
--- a/doc/sphinx/tutorial/index.rst
+++ b/doc/sphinx/tutorial/index.rst
@@ -14,6 +14,7 @@ separate topic. Good luck.
 
 .. toctree:: :maxdepth: 1
 
+        indroduction
 	backend_servers
         starting_varnish
 	logging
diff --git a/doc/sphinx/tutorial/introduction.rst b/doc/sphinx/tutorial/introduction.rst
new file mode 100644
index 0000000..17d7b6a
--- /dev/null
+++ b/doc/sphinx/tutorial/introduction.rst
@@ -0,0 +1,37 @@
+.. _tutorial-intro:
+
+What is Varnish?
+----------------
+
+Varnish Cache is a Varnish Cache is a web application accelerator also
+known as a caching HTTP reverse proxy. You install it in front of any
+server that speaks HTTP and configure it to cache the
+contents. Varnish Cache is really, really fast. It typically speeds up
+delivery with a factor of 300 - 1000x, depending on your architecture.
+
+
+Performance
+~~~~~~~~~~~
+
+Varnish performs really, really well. It is usually bound by the speed
+of the network, effectivly turning performance into a non-issue. We've
+seen Varnish delivering 20 Gbps on regular off-the-shelf hardware.
+
+Flexibility
+~~~~~~~~~~~
+
+One of the key features of Varnish Cache, in addition to it's
+performance, is the flexibility of it's configuration language,
+VCL. VCL enables you to write policies on how incoming requests should
+be handled. In such a policy you can decide what content you want to
+serve, from where you want to get the content and how the request or
+response should be altered. You can read more about this in our
+tutorial.
+
+
+Supported plattforms
+~~~~~~~~~~~~~~~~~~~~
+
+Varnish is written to run on modern versions of Linux and FreeBSD and
+the best experience is had on those plattforms. Thanks to our
+contributors it also runs on NetBSD, OpenBSD and OS X.
diff --git a/doc/sphinx/tutorial/virtualised.rst b/doc/sphinx/tutorial/virtualised.rst
index ce4b9ad..ea8ade5 100644
--- a/doc/sphinx/tutorial/virtualised.rst
+++ b/doc/sphinx/tutorial/virtualised.rst
@@ -1,16 +1,19 @@
 
-Running inside a virtual machine (VM)
--------------------------------------
+Running Varnish in a virtualized environment
+--------------------------------------------
 
-It is possible, but not recommended for high performance, to run Varnish on virtualised 
-hardware.
+It is possible, but not recommended for high performance, to run
+Varnish on virtualised hardware. Reduced disk- and network performance
+will reduce the performance a bit so make sure your system has good IO
+performance.
 
 OpenVZ
-''''''
+~~~~~~
 
-If you are running on 64bit OpenVZ (or Parallels VPS), you must reduce the 
-maximum stack size before starting Varnish. The default allocates to much memory per thread, 
-which will make varnish fail as soon as the number of threads (==traffic) increases.
+If you are running on 64bit OpenVZ (or Parallels VPS), you must reduce
+the maximum stack size before starting Varnish. The default allocates
+to much memory per thread, which will make varnish fail as soon as the
+number of threads (==traffic) increases.
 
 Reduce the maximum stack size by running::
 



From geoff at varnish-cache.org  Mon Jan 16 16:58:31 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 16 Jan 2012 17:58:31 +0100
Subject: [experimental-ims] b73b1a0 Update .gitignore
Message-ID: <E1Rmpt9-0000nL-0p@project.varnish-software.com>

commit b73b1a02dad3c7691d9047c4d76f89271c9c50e5
Author: Rogier 'DocWilco' Mulhuijzen <github at bsdchicks.com>
Date:   Sat Jan 14 13:36:10 2012 +0100

    Update .gitignore
    
    I work with NetBeans, and it insists on this being present.

diff --git a/.gitignore b/.gitignore
index 14c5393..6680eb8 100644
--- a/.gitignore
+++ b/.gitignore
@@ -82,3 +82,6 @@ TAGS
 /doc/*.html
 /doc/sphinx/=build/
 /doc/sphinx/conf.py
+
+# NetBeans insists on this
+/nbproject/private/



From geoff at varnish-cache.org  Mon Jan 16 16:58:30 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 16 Jan 2012 17:58:30 +0100
Subject: [experimental-ims] 0802b1f Handle the case of sub being NULL
Message-ID: <E1Rmpt8-0000nC-VL@project.varnish-software.com>

commit 0802b1fe539325b44f316f3c598a6607638a83c0
Author: Tollef Fog Heen <tfheen at varnish-software.com>
Date:   Wed Jan 11 14:07:02 2012 +0100

    Handle the case of sub being NULL
    
    if sub is null (because it's the contents of a non-existing header),
    pretend it's "" instead.

diff --git a/bin/varnishd/cache/cache_vrt_re.c b/bin/varnishd/cache/cache_vrt_re.c
index 9864611..867ec86 100644
--- a/bin/varnishd/cache/cache_vrt_re.c
+++ b/bin/varnishd/cache/cache_vrt_re.c
@@ -96,6 +96,8 @@ VRT_regsub(const struct sess *sp, int all, const char *str, void *re,
 	AN(re);
 	if (str == NULL)
 		str = "";
+	if (sub == NULL)
+		sub = "";
 	t = re;
 	memset(ovector, 0, sizeof(ovector));
 	len = strlen(str);
diff --git a/bin/varnishtest/tests/c00001.vtc b/bin/varnishtest/tests/c00001.vtc
index d9b6c41..f1a6824 100644
--- a/bin/varnishtest/tests/c00001.vtc
+++ b/bin/varnishtest/tests/c00001.vtc
@@ -21,6 +21,8 @@ varnish v1 -vcl+backend {
 		    regsub(beresp.http.Foobar, "(b)(a)(r)(f)", "\0\4\3\2\\p");
 		set beresp.http.Snafu6 =
 		    regsub(beresp.http.Foobar, "(b)(a)(r)(f)", "\4\&\3\2p\");
+		set beresp.http.Snafu7 =
+		    regsub(beresp.http.Foobar, "ar", req.http.nosuchheader);
 	}
 } -start 
 
@@ -37,4 +39,5 @@ client c1 {
 	# NB: have to escape the \\ in the next two lines
 	expect resp.http.snafu5 == "_barffra\\p_"
 	expect resp.http.snafu6 == "_f&rap\\_"
+	expect resp.http.snafu7 == "_bf_"
 } -run



From geoff at varnish-cache.org  Mon Jan 16 16:58:31 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 16 Jan 2012 17:58:31 +0100
Subject: [experimental-ims] 77a7e83 Update syntax for 3.0
Message-ID: <E1Rmpt9-0000nh-4R@project.varnish-software.com>

commit 77a7e8309c5ab30b81e98fdb3a1ac2b0140c9dd7
Author: Andreas Plesner Jacobsen <apj at mutt.dk>
Date:   Sun Jan 15 17:44:17 2012 +0100

    Update syntax for 3.0

diff --git a/doc/sphinx/tutorial/handling_misbehaving_servers.rst b/doc/sphinx/tutorial/handling_misbehaving_servers.rst
index c258f46..6cd7111 100644
--- a/doc/sphinx/tutorial/handling_misbehaving_servers.rst
+++ b/doc/sphinx/tutorial/handling_misbehaving_servers.rst
@@ -69,7 +69,7 @@ can be enabled in VCL:::
   sub vcl_fetch {
     if (beresp.status == 500) { 
       set beresp.saintmode = 10s;
-      restart;
+      return(restart);
     }
     set beresp.grace = 5m;
   } 



From geoff at varnish-cache.org  Mon Jan 16 16:58:31 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 16 Jan 2012 17:58:31 +0100
Subject: [experimental-ims] 872c478 Actually use a default (80) if no port is
	specified
Message-ID: <E1Rmpt9-0000nU-2I@project.varnish-software.com>

commit 872c478170e0640444fe5c173845f8667f9adae7
Author: Rogier 'DocWilco' Mulhuijzen <github at bsdchicks.com>
Date:   Sat Jan 14 13:37:57 2012 +0100

    Actually use a default (80) if no port is specified

diff --git a/lib/libvcl/vcc_dir_dns.c b/lib/libvcl/vcc_dir_dns.c
index bdc0a21..174d1ab 100644
--- a/lib/libvcl/vcc_dir_dns.c
+++ b/lib/libvcl/vcc_dir_dns.c
@@ -88,7 +88,7 @@ print_backend(struct vcc *tl,
 	if (serial >= 0)
 		Fb(tl, 0, "[%d]", serial);
 	Fb(tl, 0, "\",\n");
-	Emit_Sockaddr(tl, &tmptok, b_defaults.port);
+	Emit_Sockaddr(tl, &tmptok, b_defaults.port ? b_defaults.port : "80");
 
 	Fb(tl, 0, "\t.hosthdr = \"");
 	if (b_defaults.hostheader != NULL)



From geoff at varnish-cache.org  Mon Jan 16 16:58:31 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 16 Jan 2012 17:58:31 +0100
Subject: [experimental-ims] 3eea8da Use sp->vsl_id as thread identifier
Message-ID: <E1Rmpt9-0000ns-6Z@project.varnish-software.com>

commit 3eea8dacc142df1b2c460b80113c8171c214f5f5
Author: Rogier 'DocWilco' Mulhuijzen <github at bsdchicks.com>
Date:   Sun Jan 15 16:39:42 2012 +0100

    Use sp->vsl_id as thread identifier
    
    POSIX threads don't have a thread identifier that can be printed
    in a portable manner.

diff --git a/bin/varnishd/cache/cache_center.c b/bin/varnishd/cache/cache_center.c
index 4967c82..69e854e 100644
--- a/bin/varnishd/cache/cache_center.c
+++ b/bin/varnishd/cache/cache_center.c
@@ -1655,13 +1655,13 @@ cnt_diag(struct sess *sp, const char *state)
 	}
 
 	if (sp->wrk != NULL) {
-		WSP(sp, SLT_Debug, "thr %p STP_%s sp %p obj %p vcl %p",
-		    pthread_self(), state, sp, obj, vcl);
+		WSP(sp, SLT_Debug, "vsl_id %u STP_%s sp %p obj %p vcl %p",
+		    sp->vsl_id, state, sp, obj, vcl);
 		WSL_Flush(sp->wrk, 0);
 	} else {
 		VSL(SLT_Debug, sp->vsl_id,
-		    "thr %p STP_%s sp %p obj %p vcl %p",
-		    pthread_self(), state, sp, obj, vcl);
+		    "vsl_id %u STP_%s sp %p obj %p vcl %p",
+		    sp->vsl_id, state, sp, obj, vcl);
 	}
 }
 



From geoff at varnish-cache.org  Mon Jan 16 16:58:31 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 16 Jan 2012 17:58:31 +0100
Subject: [experimental-ims] d0be5ad Add __printflike and fix everything that
	brings to light
Message-ID: <E1Rmpt9-0000o7-A4@project.varnish-software.com>

commit d0be5ad9818095aa17f5dda12adf42d5e63c3ddf
Author: Rogier 'DocWilco' Mulhuijzen <github at bsdchicks.com>
Date:   Sun Jan 15 17:37:03 2012 +0100

    Add __printflike and fix everything that brings to light

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index 6b0d0a9..356d138 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -809,7 +809,8 @@ void http_PutStatus(struct http *to, uint16_t status);
 void http_PutResponse(struct worker *w, unsigned vsl_id, const struct http *to,
     const char *response);
 void http_PrintfHeader(struct worker *w, unsigned vsl_id, struct http *to,
-    const char *fmt, ...);
+    const char *fmt, ...)
+    __printflike(4, 5);
 void http_SetHeader(struct worker *w, unsigned vsl_id, struct http *to,
     const char *hdr);
 void http_SetH(const struct http *to, unsigned n, const char *fm);
@@ -928,10 +929,13 @@ void *VSM_Alloc(unsigned size, const char *class, const char *type,
     const char *ident);
 void VSM_Free(void *ptr);
 #ifdef VSL_ENDMARKER
-void VSL(enum VSL_tag_e tag, int id, const char *fmt, ...);
+void VSL(enum VSL_tag_e tag, int id, const char *fmt, ...)
+    __printflike(3, 4);
 void WSLR(struct worker *w, enum VSL_tag_e tag, int id, txt t);
-void WSL(struct worker *w, enum VSL_tag_e tag, int id, const char *fmt, ...);
-void WSLB(struct worker *w, enum VSL_tag_e tag, const char *fmt, ...);
+void WSL(struct worker *w, enum VSL_tag_e tag, int id, const char *fmt, ...)
+    __printflike(4, 5);
+void WSLB(struct worker *w, enum VSL_tag_e tag, const char *fmt, ...)
+    __printflike(3, 4);
 
 void WSL_Flush(struct worker *w, int overflow);
 
diff --git a/bin/varnishd/cache/cache_center.c b/bin/varnishd/cache/cache_center.c
index 69e854e..d80842a 100644
--- a/bin/varnishd/cache/cache_center.c
+++ b/bin/varnishd/cache/cache_center.c
@@ -1776,7 +1776,7 @@ cli_debug_srandom(struct cli *cli, const char * const *av, void *priv)
 		seed = strtoul(av[2], NULL, 0);
 	srandom(seed);
 	srand48(random());
-	VCLI_Out(cli, "Random(3) seeded with %lu", seed);
+	VCLI_Out(cli, "Random(3) seeded with %u", seed);
 }
 
 static struct cli_proto debug_cmds[] = {
diff --git a/bin/varnishd/cache/cache_shmlog.c b/bin/varnishd/cache/cache_shmlog.c
index c29f5a1..a839f5a 100644
--- a/bin/varnishd/cache/cache_shmlog.c
+++ b/bin/varnishd/cache/cache_shmlog.c
@@ -236,6 +236,10 @@ WSLR(struct worker *wrk, enum VSL_tag_e tag, int id, txt t)
 
 static void
 wsl(struct worker *wrk, enum VSL_tag_e tag, int id, const char *fmt, va_list ap)
+    __printflike(4, 0);
+
+static void
+wsl(struct worker *wrk, enum VSL_tag_e tag, int id, const char *fmt, va_list ap)
 {
 	char *p;
 	unsigned n, mlen;
diff --git a/bin/varnishd/cache/cache_vrt_vmod.c b/bin/varnishd/cache/cache_vrt_vmod.c
index 6b3b846..93d82f9 100644
--- a/bin/varnishd/cache/cache_vrt_vmod.c
+++ b/bin/varnishd/cache/cache_vrt_vmod.c
@@ -104,7 +104,8 @@ VRT_Vmod_Init(void **hdl, void *ptr, int len, const char *nm,
 		AN(w);
 		if (strcmp(x, nm)) {
 			VCLI_Out(cli, "Loading VMOD %s from %s:\n", nm, path);
-			VCLI_Out(cli, "File contain wrong VMOD (\"%s\")\n", x);
+			VCLI_Out(cli, "File contain wrong VMOD (\"%s\")\n",
+			    (char *) x);
 			VCLI_Out(cli, "Check relative pathnames ?.\n");
 			(void)dlclose(v->hdl);
 			FREE_OBJ(v);
diff --git a/bin/varnishd/mgt/mgt.h b/bin/varnishd/mgt/mgt.h
index 0e3eb9e..ffaef56 100644
--- a/bin/varnishd/mgt/mgt.h
+++ b/bin/varnishd/mgt/mgt.h
@@ -50,7 +50,8 @@ void MGT_Child_Cli_Fail(void);
 typedef void mgt_cli_close_f(void *priv);
 void mgt_cli_setup(int fdi, int fdo, int verbose, const char *ident,
     mgt_cli_close_f *close_func, void *priv);
-int mgt_cli_askchild(unsigned *status, char **resp, const char *fmt, ...);
+int mgt_cli_askchild(unsigned *status, char **resp, const char *fmt, ...)
+    __printflike(3, 4);
 void mgt_cli_start_child(int fdi, int fdo);
 void mgt_cli_stop_child(void);
 void mgt_cli_telnet(const char *T_arg);
diff --git a/bin/varnishd/mgt/mgt_param.c b/bin/varnishd/mgt/mgt_param.c
index c488f23..8123bfc 100644
--- a/bin/varnishd/mgt/mgt_param.c
+++ b/bin/varnishd/mgt/mgt_param.c
@@ -261,7 +261,7 @@ tweak_generic_uint(struct cli *cli, volatile unsigned *dest, const char *arg,
 		}
 		*dest = u;
 	} else if (*dest == UINT_MAX) {
-		VCLI_Out(cli, "unlimited", *dest);
+		VCLI_Out(cli, "unlimited");
 	} else {
 		VCLI_Out(cli, "%u", *dest);
 	}
diff --git a/bin/varnishreplay/varnishreplay.c b/bin/varnishreplay/varnishreplay.c
index 8e694da..02ad83e 100644
--- a/bin/varnishreplay/varnishreplay.c
+++ b/bin/varnishreplay/varnishreplay.c
@@ -44,6 +44,7 @@
 #include <string.h>
 #include <unistd.h>
 
+#include "vdef.h"
 #include "vapi/vsl.h"
 #include "vapi/vsm.h"
 #include "vas.h"
@@ -171,6 +172,10 @@ static pthread_mutex_t log_mutex = PTHREAD_MUTEX_INITIALIZER;
 
 static void
 thread_log(int lvl, int errcode, const char *fmt, ...)
+    __printflike(3, 4);
+
+static void
+thread_log(int lvl, int errcode, const char *fmt, ...)
 {
 	va_list ap;
 
diff --git a/bin/varnishtest/vtc.c b/bin/varnishtest/vtc.c
index 9581b3c..b8556ce 100644
--- a/bin/varnishtest/vtc.c
+++ b/bin/varnishtest/vtc.c
@@ -449,7 +449,7 @@ cmd_random(CMD_ARGS)
 		l = random();
 		if (l == random_expect[i])
 			continue;
-		vtc_log(vl, 4, "random[%d] = 0x%x (expect 0x%x)",
+		vtc_log(vl, 4, "random[%d] = 0x%x (expect 0x%lx)",
 		    i, l, random_expect[i]);
 		vtc_log(vl, 1, "SKIPPING test: unknown srandom(1) sequence.");
 		vtc_stop = 1;
@@ -538,17 +538,17 @@ exec_file(const char *fn, const char *script, const char *tmpdir,
 
 	/* Apply extmacro definitions */
 	VTAILQ_FOREACH(m, &extmacro_list, list)
-		macro_def(vltop, NULL, m->name, m->val);
+		macro_def(vltop, NULL, m->name, "%s", m->val);
 
 	/* Other macro definitions */
 	cwd = getcwd(NULL, PATH_MAX);
-	macro_def(vltop, NULL, "pwd", cwd);
+	macro_def(vltop, NULL, "pwd", "%s", cwd);
 	macro_def(vltop, NULL, "topbuild", "%s/%s", cwd, TOP_BUILDDIR);
 	macro_def(vltop, NULL, "bad_ip", "10.255.255.255");
 
 	/* Move into our tmpdir */
 	AZ(chdir(tmpdir));
-	macro_def(vltop, NULL, "tmpdir", tmpdir);
+	macro_def(vltop, NULL, "tmpdir", "%s", tmpdir);
 
 	/* Drop file to tell what was going on here */
 	f = fopen("INFO", "w");
diff --git a/bin/varnishtest/vtc.h b/bin/varnishtest/vtc.h
index ee5308a..be20cb9 100644
--- a/bin/varnishtest/vtc.h
+++ b/bin/varnishtest/vtc.h
@@ -77,7 +77,8 @@ void cmd_server_genvcl(struct vsb *vsb);
 void vtc_loginit(char *buf, unsigned buflen);
 struct vtclog *vtc_logopen(const char *id);
 void vtc_logclose(struct vtclog *vl);
-void vtc_log(struct vtclog *vl, int lvl, const char *fmt, ...);
+void vtc_log(struct vtclog *vl, int lvl, const char *fmt, ...)
+    __printflike(3, 4);
 void vtc_dump(struct vtclog *vl, int lvl, const char *pfx,
     const char *str, int len);
 void vtc_hexdump(struct vtclog *vl, int lvl, const char *pfx,
@@ -87,7 +88,9 @@ int exec_file(const char *fn, const char *script, const char *tmpdir,
     char *logbuf, unsigned loglen);
 
 void macro_def(struct vtclog *vl, const char *instance, const char *name,
-    const char *fmt, ...);
+    const char *fmt, ...)
+    __printflike(4, 5);
 struct vsb *macro_expand(struct vtclog *vl, const char *text);
 
-void extmacro_def(const char *name, const char *fmt, ...);
+void extmacro_def(const char *name, const char *fmt, ...)
+    __printflike(2, 3);
diff --git a/bin/varnishtest/vtc_http.c b/bin/varnishtest/vtc_http.c
index d3753a5..00be0be 100644
--- a/bin/varnishtest/vtc_http.c
+++ b/bin/varnishtest/vtc_http.c
@@ -133,13 +133,13 @@ synth_body(const char *len, int rnd)
 static void
 http_write(const struct http *hp, int lvl, const char *pfx)
 {
-	int l;
+	ssize_t l;
 
 	AZ(VSB_finish(hp->vsb));
 	vtc_dump(hp->vl, lvl, pfx, VSB_data(hp->vsb), VSB_len(hp->vsb));
 	l = write(hp->fd, VSB_data(hp->vsb), VSB_len(hp->vsb));
 	if (l != VSB_len(hp->vsb))
-		vtc_log(hp->vl, hp->fatal, "Write failed: (%d vs %d) %s",
+		vtc_log(hp->vl, hp->fatal, "Write failed: (%zd vs %zd) %s",
 		    l, VSB_len(hp->vsb), strerror(errno));
 }
 
@@ -387,7 +387,7 @@ http_rxchunk(struct http *hp)
 	bprintf(hp->chunklen, "%d", i);
 	if ((q == hp->rxbuf + l) ||
 		(*q != '\0' && !vct_islws(*q))) {
-		vtc_log(hp->vl, hp->fatal, "chunked fail %02x @ %d",
+		vtc_log(hp->vl, hp->fatal, "chunked fail %02x @ %td",
 		    *q, q - (hp->rxbuf + l));
 	}
 	assert(q != hp->rxbuf + l);
diff --git a/bin/varnishtest/vtc_sema.c b/bin/varnishtest/vtc_sema.c
index 48d4c68..fd4c561 100644
--- a/bin/varnishtest/vtc_sema.c
+++ b/bin/varnishtest/vtc_sema.c
@@ -63,7 +63,7 @@ sema_new(char *name, struct vtclog *vl)
 	AN(r);
 	r->name = name;
 	if (*name != 'r')
-		vtc_log(vl, 0, "Sema name must start with 'r' (%s)", *name);
+		vtc_log(vl, 0, "Sema name must start with 'r' (%s)", name);
 
 	AZ(pthread_mutex_init(&r->mtx, NULL));
 	AZ(pthread_cond_init(&r->cond, NULL));
diff --git a/include/vcli_common.h b/include/vcli_common.h
index 8d0a623..fbb4e9f 100644
--- a/include/vcli_common.h
+++ b/include/vcli_common.h
@@ -28,6 +28,8 @@
  *
  */
 
+#include "vdef.h"
+
 struct vlu;
 struct VCLS;
 
diff --git a/include/vcli_priv.h b/include/vcli_priv.h
index a265d2e..0129b83 100644
--- a/include/vcli_priv.h
+++ b/include/vcli_priv.h
@@ -53,6 +53,7 @@ struct cli_proto {
 
 /* The implementation must provide these functions */
 int VCLI_Overflow(struct cli *cli);
-void VCLI_Out(struct cli *cli, const char *fmt, ...);
+void VCLI_Out(struct cli *cli, const char *fmt, ...)
+    __printflike(2, 3);
 void VCLI_Quote(struct cli *cli, const char *str);
 void VCLI_SetResult(struct cli *cli, unsigned r);
diff --git a/include/vdef.h b/include/vdef.h
index c9b24c5..63a2a6b 100644
--- a/include/vdef.h
+++ b/include/vdef.h
@@ -1,10 +1,14 @@
 /*-
  * Copyright (c) 2006 Verdens Gang AS
- * Copyright (c) 2006-2011 Varnish Software AS
+ * Copyright (c) 2006-2012 Varnish Software AS
+ * Copyright (c) 2012 Fastly Inc
  * All rights reserved.
  *
  * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
+ * Author: Rogier 'DocWilco' Mulhuijzen <rogier at fastly.com>
  *
+ * Inspired by FreeBSD's <sys/cdefs.h>
+ * 
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
@@ -28,6 +32,9 @@
  *
  */
 
+#ifndef VDEF_H_INCLUDED
+#define VDEF_H_INCLUDED
+
 /* Safe printf into a fixed-size buffer */
 #define bprintf(buf, fmt, ...)						\
 	do {								\
@@ -42,3 +49,22 @@
 		    < sizeof buf);					\
 	} while (0)
 
+
+#ifndef __GNUC_PREREQ
+# if defined __GNUC__ && defined __GNUC_MINOR__
+#  define __GNUC_PREREQ(maj, min) \
+	(__GNUC__ > (maj) || (__GNUC__ == (maj) && __GNUC_MINOR__ >= (min)))
+# else
+#  define __GNUC_PREREQ(maj, min) 0
+# endif
+#endif
+
+#ifndef __printflike
+# if __GNUC_PREREQ(2, 95) || defined(__INTEL_COMPILER)
+#  define __printflike(f,a) __attribute__((format(printf, f, a)))
+# else
+#  define __printflike(f,a)
+# endif
+#endif
+
+#endif /* VDEF_H_INCLUDED */
diff --git a/include/vsb.h b/include/vsb.h
index 0a8e2bf..e17fe5b 100644
--- a/include/vsb.h
+++ b/include/vsb.h
@@ -31,6 +31,8 @@
 #ifndef VSB_H_INCLUDED
 #define VSB_H_INCLUDED
 
+#include "vdef.h"
+
 /*
  * Structure definition
  */
@@ -49,10 +51,6 @@ struct vsb {
 	int		 s_flags;	/* flags */
 };
 
-#ifndef __printflike
-#define __printflike(a,b)
-#endif
-
 #ifdef __cplusplus
 extern "C" {
 #endif
diff --git a/lib/libvarnishapi/vsc.c b/lib/libvarnishapi/vsc.c
index accef82..5f992cf 100644
--- a/lib/libvarnishapi/vsc.c
+++ b/lib/libvarnishapi/vsc.c
@@ -39,6 +39,7 @@
 
 #include "miniobj.h"
 #include "vas.h"
+#include "vdef.h"
 
 #include "vapi/vsc.h"
 #include "vapi/vsm.h"
diff --git a/lib/libvarnishapi/vsl.c b/lib/libvarnishapi/vsl.c
index 6950c1b..8e7001a 100644
--- a/lib/libvarnishapi/vsl.c
+++ b/lib/libvarnishapi/vsl.c
@@ -41,6 +41,7 @@
 
 #include "miniobj.h"
 #include "vas.h"
+#include "vdef.h"
 
 #include "vapi/vsl.h"
 #include "vapi/vsm.h"
diff --git a/lib/libvarnishapi/vsl_arg.c b/lib/libvarnishapi/vsl_arg.c
index 068d539..8587fb1 100644
--- a/lib/libvarnishapi/vsl_arg.c
+++ b/lib/libvarnishapi/vsl_arg.c
@@ -43,6 +43,7 @@
 
 #include "miniobj.h"
 #include "vas.h"
+#include "vdef.h"
 
 #include "vapi/vsl.h"
 #include "vapi/vsm.h"
diff --git a/lib/libvarnishapi/vsm.c b/lib/libvarnishapi/vsm.c
index e11fe26..c175f10 100644
--- a/lib/libvarnishapi/vsm.c
+++ b/lib/libvarnishapi/vsm.c
@@ -44,6 +44,7 @@
 
 #include "miniobj.h"
 #include "vas.h"
+#include "vdef.h"
 
 #include "vapi/vsm.h"
 #include "vapi/vsm_int.h"
diff --git a/lib/libvarnishapi/vsm_api.h b/lib/libvarnishapi/vsm_api.h
index 79be982..af99798 100644
--- a/lib/libvarnishapi/vsm_api.h
+++ b/lib/libvarnishapi/vsm_api.h
@@ -51,6 +51,7 @@ struct VSM_data {
 	struct vsl		*vsl;
 };
 
-int vsm_diag(struct VSM_data *vd, const char *fmt, ...);
+int vsm_diag(struct VSM_data *vd, const char *fmt, ...)
+    __printflike(2, 3);
 void VSC_Delete(struct VSM_data *vd);
 void VSL_Delete(struct VSM_data *vd);
diff --git a/lib/libvcl/vcc_acl.c b/lib/libvcl/vcc_acl.c
index c962044..e6a1065 100644
--- a/lib/libvcl/vcc_acl.c
+++ b/lib/libvcl/vcc_acl.c
@@ -362,12 +362,12 @@ vcc_acl_emit(const struct vcc *tl, const char *acln, int anon)
 
 	Fh(tl, 0, "\n");
 	Fh(tl, 0, "\ta = p;\n");
-	Fh(tl, 0, "\tVRT_memmove(&fam, a + %d, sizeof fam);\n",
+	Fh(tl, 0, "\tVRT_memmove(&fam, a + %zd, sizeof fam);\n",
 	    offsetof(struct sockaddr, sa_family));
 	Fh(tl, 0, "\tif (fam == %d)\n", PF_INET);
-	Fh(tl, 0, "\t\ta += %d;\n", offsetof(struct sockaddr_in, sin_addr));
+	Fh(tl, 0, "\t\ta += %zd;\n", offsetof(struct sockaddr_in, sin_addr));
 	Fh(tl, 0, "\telse if (fam == %d)\n", PF_INET6);
-	Fh(tl, 0, "\t\ta += %d;\n", offsetof(struct sockaddr_in6, sin6_addr));
+	Fh(tl, 0, "\t\ta += %zd;\n", offsetof(struct sockaddr_in6, sin6_addr));
 	Fh(tl, 0, "\telse {\n");
 	Fh(tl, 0, "\t\tVRT_acl_log(sp, \"NO_FAM %s\");\n", acln);
 	Fh(tl, 0, "\t\treturn(0);\n");
@@ -422,8 +422,7 @@ vcc_acl_emit(const struct vcc *tl, const char *acln, int anon)
 
 		if (!anon) {
 			Fh(tl, 0, "\t%*sVRT_acl_log(sp, \"%sMATCH %s \" ",
-			    -i, "", ae->not ? "NEG_" : "", acln,
-			    PF(ae->t_addr));
+			    -i, "", ae->not ? "NEG_" : "", acln);
 			EncToken(tl->fh, ae->t_addr);
 			if (ae->t_mask != NULL)
 				Fh(tl, 0, " \"/%.*s\" ", PF(ae->t_mask));
diff --git a/lib/libvcl/vcc_backend.c b/lib/libvcl/vcc_backend.c
index 03482c4..fb160e7 100644
--- a/lib/libvcl/vcc_backend.c
+++ b/lib/libvcl/vcc_backend.c
@@ -353,7 +353,7 @@ vcc_ParseProbeSpec(struct vcc *tl)
 	if (t_initial != NULL)
 		Fh(tl, 0, "\t.initial = %u,\n", initial);
 	else
-		Fh(tl, 0, "\t.initial = ~0U,\n", initial);
+		Fh(tl, 0, "\t.initial = ~0U,\n");
 	if (status > 0)
 		Fh(tl, 0, "\t.exp_status = %u,\n", status);
 	Fh(tl, 0, "};\n");
diff --git a/lib/libvcl/vcc_compile.c b/lib/libvcl/vcc_compile.c
index 27447fe..66d89f5 100644
--- a/lib/libvcl/vcc_compile.c
+++ b/lib/libvcl/vcc_compile.c
@@ -286,7 +286,7 @@ LocTable(const struct vcc *tl)
 				pos++;
 
 		}
-		Fc(tl, 0, "  [%3u] = { %d, %8u, %4u, %3u, 0, ",
+		Fc(tl, 0, "  [%3u] = { %d, %8tu, %4u, %3u, 0, ",
 		    t->cnt, sp->idx, t->b - sp->b, lin, pos + 1);
 		if (t->tok == CSRC)
 			Fc(tl, 0, " \"C{\"},\n");
diff --git a/lib/libvcl/vcc_compile.h b/lib/libvcl/vcc_compile.h
index caacd73..b64564e 100644
--- a/lib/libvcl/vcc_compile.h
+++ b/lib/libvcl/vcc_compile.h
@@ -247,11 +247,16 @@ extern struct method method_tab[];
  * I -> Initializer function
  * F -> Finish function
  */
-void Fh(const struct vcc *tl, int indent, const char *fmt, ...);
-void Fc(const struct vcc *tl, int indent, const char *fmt, ...);
-void Fb(const struct vcc *tl, int indent, const char *fmt, ...);
-void Fi(const struct vcc *tl, int indent, const char *fmt, ...);
-void Ff(const struct vcc *tl, int indent, const char *fmt, ...);
+void Fh(const struct vcc *tl, int indent, const char *fmt, ...)
+    __printflike(3, 4);
+void Fc(const struct vcc *tl, int indent, const char *fmt, ...)
+    __printflike(3, 4);
+void Fb(const struct vcc *tl, int indent, const char *fmt, ...)
+    __printflike(3, 4);
+void Fi(const struct vcc *tl, int indent, const char *fmt, ...)
+    __printflike(3, 4);
+void Ff(const struct vcc *tl, int indent, const char *fmt, ...)
+    __printflike(3, 4);
 void EncToken(struct vsb *sb, const struct token *t);
 int IsMethod(const struct token *t);
 void *TlAlloc(struct vcc *tl, unsigned len);
diff --git a/lib/libvcl/vcc_dir_dns.c b/lib/libvcl/vcc_dir_dns.c
index 174d1ab..55ed921 100644
--- a/lib/libvcl/vcc_dir_dns.c
+++ b/lib/libvcl/vcc_dir_dns.c
@@ -92,9 +92,9 @@ print_backend(struct vcc *tl,
 
 	Fb(tl, 0, "\t.hosthdr = \"");
 	if (b_defaults.hostheader != NULL)
-		Fb(tl,0, b_defaults.hostheader);
+		Fb(tl, 0, "%s", b_defaults.hostheader);
 	else
-		Fb(tl,0, strip);
+		Fb(tl, 0, "%s", strip);
 	Fb(tl, 0, "\",\n");
 
 	Fb(tl, 0, "\t.saintmode_threshold = %d,\n",b_defaults.saint);
diff --git a/lib/libvcl/vcc_expr.c b/lib/libvcl/vcc_expr.c
index 3dfeeaa..82e53ef 100644
--- a/lib/libvcl/vcc_expr.c
+++ b/lib/libvcl/vcc_expr.c
@@ -253,6 +253,10 @@ vcc_new_expr(void)
 
 static struct expr *
 vcc_mk_expr(enum var_type fmt, const char *str, ...)
+    __printflike(2, 3);
+
+static struct expr *
+vcc_mk_expr(enum var_type fmt, const char *str, ...)
 {
 	va_list ap;
 	struct expr *e;
@@ -537,7 +541,7 @@ vcc_Eval_Func(struct vcc *tl, struct expr **e, const struct symbol *sym)
 			r = strchr(sym->name, '.');
 			AN(r);
 			e1 = vcc_mk_expr(VOID, "&vmod_priv_%.*s",
-			    r - sym->name, sym->name);
+			    (int) (r - sym->name), sym->name);
 			p += strlen(p) + 1;
 		} else if (fmt == VOID && !strcmp(p, "PRIV_CALL")) {
 			bprintf(buf, "vmod_priv_%u", tl->nvmodpriv++);
diff --git a/lib/libvcl/vcc_parse.c b/lib/libvcl/vcc_parse.c
index 4023287..d8e74a4 100644
--- a/lib/libvcl/vcc_parse.c
+++ b/lib/libvcl/vcc_parse.c
@@ -154,7 +154,7 @@ vcc_Compound(struct vcc *tl)
 			return;
 		case CSRC:
 			Fb(tl, 1, "%.*s\n",
-			    tl->t->e - (tl->t->b + 2),
+			    (int) (tl->t->e - (tl->t->b + 2)),
 			    tl->t->b + 1);
 			vcc_NextToken(tl);
 			break;
@@ -274,7 +274,7 @@ vcc_Parse(struct vcc *tl)
 		switch (tl->t->tok) {
 		case CSRC:
 			Fc(tl, 0, "%.*s\n",
-			    tl->t->e - (tl->t->b + 4), tl->t->b + 2);
+			    (int) (tl->t->e - (tl->t->b + 4)), tl->t->b + 2);
 			vcc_NextToken(tl);
 			break;
 		case EOI:



From geoff at varnish-cache.org  Mon Jan 16 16:58:31 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 16 Jan 2012 17:58:31 +0100
Subject: [experimental-ims] 5b8d896 By Slink, via Geoff:
Message-ID: <E1Rmpt9-0000oN-Ce@project.varnish-software.com>

commit 5b8d8964491a85d06c5c496d8242bd6dfa14ec7e
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sun Jan 15 19:30:53 2012 +0000

    By Slink, via Geoff:
    
    The attached patch is a one-liner for waiter/cache_waiter_ports.c,
    the code was referencing sp->t_open where it should have been
    sp->t_idle, which was causing assertion failures on idle session
    timeouts.

diff --git a/bin/varnishd/waiter/cache_waiter_ports.c b/bin/varnishd/waiter/cache_waiter_ports.c
index 4f39ca4..87eaeac 100644
--- a/bin/varnishd/waiter/cache_waiter_ports.c
+++ b/bin/varnishd/waiter/cache_waiter_ports.c
@@ -3,7 +3,7 @@
  * Copyright (c) 2006 Varnish Software AS
  * Copyright (c) 2007 OmniTI Computer Consulting, Inc.
  * Copyright (c) 2007 Theo Schlossnagle
- * Copyright (c) 2010 UPLEX, Nils Goroll
+ * Copyright (c) 2010-2012 UPLEX, Nils Goroll
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -220,7 +220,7 @@ vws_thread(void *priv)
 
 		if (sp) {
 			double tmo =
-			    (sp->t_open + cache_param->timeout_idle) - now;
+			    (sp->t_idle + cache_param->timeout_idle) - now;
 
 			/*
 			 * we should have removed all sps whose timeout



From geoff at varnish-cache.org  Mon Jan 16 16:58:31 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 16 Jan 2012 17:58:31 +0100
Subject: [experimental-ims] ca587d1 Simplify vry_cmp's signature
Message-ID: <E1Rmpt9-0000oY-Er@project.varnish-software.com>

commit ca587d1331272972ccd2b8c3e6fc589e3768007a
Author: Rogier 'DocWilco' Mulhuijzen <github at bsdchicks.com>
Date:   Sun Jan 15 23:19:55 2012 +0100

    Simplify vry_cmp's signature

diff --git a/bin/varnishd/cache/cache_vary.c b/bin/varnishd/cache/cache_vary.c
index 453b52d..9ad41d0 100644
--- a/bin/varnishd/cache/cache_vary.c
+++ b/bin/varnishd/cache/cache_vary.c
@@ -145,18 +145,18 @@ vry_len(const uint8_t *p)
  * Compare two vary entries
  */
 static int
-vry_cmp(const uint8_t * const *v1, uint8_t * const *v2)
+vry_cmp(const uint8_t *v1, const uint8_t *v2)
 {
 	unsigned retval = 0;
 
-	if (!memcmp(*v1, *v2, vry_len(*v1))) {
+	if (!memcmp(v1, v2, vry_len(v1))) {
 		/* Same same */
 		retval = 0;
-	} else if (memcmp((*v1) + 2, (*v2) + 2, (*v1)[2] + 2)) {
+	} else if (memcmp(v1 + 2, v2 + 2, v1[2] + 2)) {
 		/* Different header */
 		retval = 1;
 	} else if (cache_param->http_gzip_support &&
-	    !strcasecmp(H_Accept_Encoding, (const char*)((*v1)+2))) {
+	    !strcasecmp(H_Accept_Encoding, (const char*) v1 + 2)) {
 		/*
 		 * If we do gzip processing, we do not vary on Accept-Encoding,
 		 * because we want everybody to get the gzip'ed object, and
@@ -183,7 +183,7 @@ VRY_Match(const struct sess *sp, const uint8_t *vary)
 
 	AN(vsp);
 	while (vary[2]) {
-		i = vry_cmp(&vary, &vsp);
+		i = vry_cmp(vary, vsp);
 		if (i == 1) {
 			/* Build a new entry */
 
@@ -222,7 +222,7 @@ VRY_Match(const struct sess *sp, const uint8_t *vary)
 			} else
 				vsp[2 + vary[2] + 2 + 2] = '\0';
 
-			i = vry_cmp(&vary, &vsp);
+			i = vry_cmp(vary, vsp);
 			assert(i != 1);	/* hdr must be the same now */
 		}
 		if (i != 0)



From geoff at varnish-cache.org  Mon Jan 16 16:58:31 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 16 Jan 2012 17:58:31 +0100
Subject: [experimental-ims] 0471b9b Add support for the %{format}t construct
	to varnishncsa
Message-ID: <E1Rmpt9-0000of-Gt@project.varnish-software.com>

commit 0471b9bf34c477db127aa11132e4efd99a68ef37
Author: Tollef Fog Heen <tfheen at varnish-software.com>
Date:   Mon Jan 16 10:12:14 2012 +0100

    Add support for the %{format}t construct to varnishncsa

diff --git a/bin/varnishncsa/varnishncsa.c b/bin/varnishncsa/varnishncsa.c
index 9da6172..9abb9b6 100644
--- a/bin/varnishncsa/varnishncsa.c
+++ b/bin/varnishncsa/varnishncsa.c
@@ -757,6 +757,11 @@ h_ncsa(void *priv, enum VSL_tag_e tag, unsigned fd,
 				VSB_cat(os, h ? h : "-");
 				p = tmp;
 				break;
+			case 't':
+				strftime(tbuf, sizeof tbuf, fname, &lp->df_t);
+				VSB_cat(os, tbuf);
+				p = tmp;
+				break;
 			case 'x':
 				if (!strcmp(fname, "Varnish:time_firstbyte")) {
 					VSB_cat(os, lp->df_ttfb);
diff --git a/doc/sphinx/reference/varnishncsa.rst b/doc/sphinx/reference/varnishncsa.rst
index 0b3f97f..9373f5c 100644
--- a/doc/sphinx/reference/varnishncsa.rst
+++ b/doc/sphinx/reference/varnishncsa.rst
@@ -90,6 +90,11 @@ The following options are available:
 	         Time when the request was received, in HTTP date/time
 	         format.
 
+	      %{X}t
+	         Time when the request was received, in the format
+		 specified by X.  The time specification format is the
+		 same as for strftime(3).
+
 	      %U
 	         The request URL without any query string. Defaults to
                  '-' if not known.



From geoff at varnish-cache.org  Mon Jan 16 16:58:31 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 16 Jan 2012 17:58:31 +0100
Subject: [experimental-ims] b05baa6 Merge branch 'master' of
	ssh://git.varnish-cache.org/git/varnish-cache into experimental-ims
Message-ID: <E1Rmpt9-0000ou-Lg@project.varnish-software.com>

commit b05baa63fd93afb94c315a185cbc38b4dee2e4ba
Merge: 42654fd 0471b9b
Author: Geoff Simmons <geoff at uplex.de>
Date:   Mon Jan 16 17:45:14 2012 +0100

    Merge branch 'master' of ssh://git.varnish-cache.org/git/varnish-cache into experimental-ims



From phk at varnish-cache.org  Tue Jan 17 08:38:07 2012
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Tue, 17 Jan 2012 09:38:07 +0100
Subject: [master] 05184e9 Minor cleanups while waiting for tests
Message-ID: <E1Rn4YR-0000fC-PQ@project.varnish-software.com>

commit 05184e90cc52ab073da0a08ed9c83804a4611f19
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Jan 17 08:37:46 2012 +0000

    Minor cleanups while waiting for tests

diff --git a/bin/varnishd/cache/cache_center.c b/bin/varnishd/cache/cache_center.c
index d80842a..62108ec 100644
--- a/bin/varnishd/cache/cache_center.c
+++ b/bin/varnishd/cache/cache_center.c
@@ -1405,7 +1405,7 @@ DOT err_pipe [label="ERROR",shape=plaintext]
  */
 
 static int
-cnt_pipe(struct sess *sp, struct worker *wrk, struct req *req)
+cnt_pipe(struct sess *sp, struct worker *wrk, const struct req *req)
 {
 
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
diff --git a/bin/varnishd/cache/cache_lck.c b/bin/varnishd/cache/cache_lck.c
index 2910582..9fb34ae 100644
--- a/bin/varnishd/cache/cache_lck.c
+++ b/bin/varnishd/cache/cache_lck.c
@@ -107,7 +107,7 @@ Lck__Unlock(struct lock *lck, const char *p, const char *f, int l)
 	 * We hack it and fill it with zero bits, hoping for sane
 	 * implementations of pthread.
 	 * #endif
-	 */ 
+	 */
 	memset(&ilck->owner, 0, sizeof ilck->owner);
 	AZ(pthread_mutex_unlock(&ilck->mtx));
 	if (cache_param->diag_bitmap & 0x8)
diff --git a/bin/varnishd/cache/cache_response.c b/bin/varnishd/cache/cache_response.c
index fc94c22..dc3bdc2 100644
--- a/bin/varnishd/cache/cache_response.c
+++ b/bin/varnishd/cache/cache_response.c
@@ -40,8 +40,11 @@ static void
 res_dorange(const struct sess *sp, const char *r, ssize_t *plow, ssize_t *phigh)
 {
 	ssize_t low, high, has_low;
+	struct req *req;
 
-	assert(sp->req->obj->response == 200);
+	req = sp->req;
+	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
+	assert(req->obj->response == 200);
 	if (strncmp(r, "bytes=", 6))
 		return;
 	r += 6;
@@ -57,7 +60,7 @@ res_dorange(const struct sess *sp, const char *r, ssize_t *plow, ssize_t *phigh)
 		r++;
 	}
 
-	if (low >= sp->req->obj->len)
+	if (low >= req->obj->len)
 		return;
 
 	if (*r != '-')
@@ -73,28 +76,28 @@ res_dorange(const struct sess *sp, const char *r, ssize_t *plow, ssize_t *phigh)
 			r++;
 		}
 		if (!has_low) {
-			low = sp->req->obj->len - high;
-			high = sp->req->obj->len - 1;
+			low = req->obj->len - high;
+			high = req->obj->len - 1;
 		}
 	} else
-		high = sp->req->obj->len - 1;
+		high = req->obj->len - 1;
 	if (*r != '\0')
 		return;
 
-	if (high >= sp->req->obj->len)
-		high = sp->req->obj->len - 1;
+	if (high >= req->obj->len)
+		high = req->obj->len - 1;
 
 	if (low > high)
 		return;
 
-	http_PrintfHeader(sp->wrk, sp->vsl_id, sp->req->resp,
+	http_PrintfHeader(sp->wrk, sp->vsl_id, req->resp,
 	    "Content-Range: bytes %jd-%jd/%jd",
-	    (intmax_t)low, (intmax_t)high, (intmax_t)sp->req->obj->len);
-	http_Unset(sp->req->resp, H_Content_Length);
+	    (intmax_t)low, (intmax_t)high, (intmax_t)req->obj->len);
+	http_Unset(req->resp, H_Content_Length);
 	assert(sp->wrk->res_mode & RES_LEN);
-	http_PrintfHeader(sp->wrk, sp->vsl_id, sp->req->resp,
+	http_PrintfHeader(sp->wrk, sp->vsl_id, req->resp,
 	    "Content-Length: %jd", (intmax_t)(1 + high - low));
-	http_SetResp(sp->req->resp, "HTTP/1.1", 206, "Partial Content");
+	http_SetResp(req->resp, "HTTP/1.1", 206, "Partial Content");
 
 	*plow = low;
 	*phigh = high;
@@ -106,44 +109,46 @@ void
 RES_BuildHttp(const struct sess *sp)
 {
 	char time_str[30];
+	struct req *req;
 
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+	req = sp->req;
+	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
 
-
-	http_ClrHeader(sp->req->resp);
-	sp->req->resp->logtag = HTTP_Tx;
-	http_CopyResp(sp->req->resp, sp->req->obj->http);
-	http_FilterFields(sp->wrk, sp->vsl_id, sp->req->resp,
-	    sp->req->obj->http, HTTPH_A_DELIVER);
+	http_ClrHeader(req->resp);
+	req->resp->logtag = HTTP_Tx;
+	http_CopyResp(req->resp, req->obj->http);
+	http_FilterFields(sp->wrk, sp->vsl_id, req->resp,
+	    req->obj->http, HTTPH_A_DELIVER);
 
 	if (!(sp->wrk->res_mode & RES_LEN)) {
-		http_Unset(sp->req->resp, H_Content_Length);
+		http_Unset(req->resp, H_Content_Length);
 	} else if (cache_param->http_range_support) {
 		/* We only accept ranges if we know the length */
-		http_SetHeader(sp->wrk, sp->vsl_id, sp->req->resp,
+		http_SetHeader(sp->wrk, sp->vsl_id, req->resp,
 		    "Accept-Ranges: bytes");
 	}
 
 	if (sp->wrk->res_mode & RES_CHUNKED)
-		http_SetHeader(sp->wrk, sp->vsl_id, sp->req->resp,
+		http_SetHeader(sp->wrk, sp->vsl_id, req->resp,
 		    "Transfer-Encoding: chunked");
 
 	VTIM_format(VTIM_real(), time_str);
-	http_PrintfHeader(sp->wrk, sp->vsl_id, sp->req->resp,
+	http_PrintfHeader(sp->wrk, sp->vsl_id, req->resp,
 	    "Date: %s", time_str);
 
-	if (sp->req->xid != sp->req->obj->xid)
-		http_PrintfHeader(sp->wrk, sp->vsl_id, sp->req->resp,
-		    "X-Varnish: %u %u", sp->req->xid, sp->req->obj->xid);
+	if (req->xid != req->obj->xid)
+		http_PrintfHeader(sp->wrk, sp->vsl_id, req->resp,
+		    "X-Varnish: %u %u", req->xid, req->obj->xid);
 	else
-		http_PrintfHeader(sp->wrk, sp->vsl_id, sp->req->resp,
-		    "X-Varnish: %u", sp->req->xid);
-	http_PrintfHeader(sp->wrk, sp->vsl_id, sp->req->resp, "Age: %.0f",
-	    sp->req->obj->exp.age + sp->req->t_resp -
-	    sp->req->obj->exp.entered);
-	http_SetHeader(sp->wrk, sp->vsl_id, sp->req->resp, "Via: 1.1 varnish");
-	http_PrintfHeader(sp->wrk, sp->vsl_id, sp->req->resp, "Connection: %s",
-	    sp->req->doclose ? "close" : "keep-alive");
+		http_PrintfHeader(sp->wrk, sp->vsl_id, req->resp,
+		    "X-Varnish: %u", req->xid);
+	http_PrintfHeader(sp->wrk, sp->vsl_id, req->resp, "Age: %.0f",
+	    req->obj->exp.age + req->t_resp -
+	    req->obj->exp.entered);
+	http_SetHeader(sp->wrk, sp->vsl_id, req->resp, "Via: 1.1 varnish");
+	http_PrintfHeader(sp->wrk, sp->vsl_id, req->resp, "Connection: %s",
+	    req->doclose ? "close" : "keep-alive");
 }
 
 /*--------------------------------------------------------------------
@@ -254,63 +259,66 @@ RES_WriteObj(struct sess *sp)
 {
 	char *r;
 	ssize_t low, high;
+	struct req *req;
 
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+	req = sp->req;
+	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
 
 	WRW_Reserve(sp->wrk, &sp->fd);
 
-	if (sp->req->obj->response == 200 &&
-	    sp->req->http->conds &&
+	if (req->obj->response == 200 &&
+	    req->http->conds &&
 	    RFC2616_Do_Cond(sp)) {
-		sp->req->wantbody = 0;
-		http_SetResp(sp->req->resp, "HTTP/1.1", 304, "Not Modified");
-		http_Unset(sp->req->resp, H_Content_Length);
-		http_Unset(sp->req->resp, H_Transfer_Encoding);
+		req->wantbody = 0;
+		http_SetResp(req->resp, "HTTP/1.1", 304, "Not Modified");
+		http_Unset(req->resp, H_Content_Length);
+		http_Unset(req->resp, H_Transfer_Encoding);
 	}
 
 	/*
 	 * If nothing special planned, we can attempt Range support
 	 */
 	low = 0;
-	high = sp->req->obj->len - 1;
+	high = req->obj->len - 1;
 	if (
-	    sp->req->wantbody &&
+	    req->wantbody &&
 	    (sp->wrk->res_mode & RES_LEN) &&
 	    !(sp->wrk->res_mode & (RES_ESI|RES_ESI_CHILD|RES_GUNZIP)) &&
 	    cache_param->http_range_support &&
-	    sp->req->obj->response == 200 &&
-	    http_GetHdr(sp->req->http, H_Range, &r))
+	    req->obj->response == 200 &&
+	    http_GetHdr(req->http, H_Range, &r))
 		res_dorange(sp, r, &low, &high);
 
 	/*
 	 * Always remove C-E if client don't grok it
 	 */
 	if (sp->wrk->res_mode & RES_GUNZIP)
-		http_Unset(sp->req->resp, H_Content_Encoding);
+		http_Unset(req->resp, H_Content_Encoding);
 
 	/*
 	 * Send HTTP protocol header, unless interior ESI object
 	 */
 	if (!(sp->wrk->res_mode & RES_ESI_CHILD))
 		sp->wrk->acct_tmp.hdrbytes +=
-		    http_Write(sp->wrk, sp->vsl_id, sp->req->resp, 1);
+		    http_Write(sp->wrk, sp->vsl_id, req->resp, 1);
 
-	if (!sp->req->wantbody)
+	if (!req->wantbody)
 		sp->wrk->res_mode &= ~RES_CHUNKED;
 
 	if (sp->wrk->res_mode & RES_CHUNKED)
 		WRW_Chunked(sp->wrk);
 
-	if (!sp->req->wantbody) {
+	if (!req->wantbody) {
 		/* This was a HEAD or conditional request */
-	} else if (sp->req->obj->len == 0) {
+	} else if (req->obj->len == 0) {
 		/* Nothing to do here */
 	} else if (sp->wrk->res_mode & RES_ESI) {
 		ESI_Deliver(sp);
-	} else if (sp->wrk->res_mode & RES_ESI_CHILD && sp->req->gzip_resp) {
+	} else if (sp->wrk->res_mode & RES_ESI_CHILD && req->gzip_resp) {
 		ESI_DeliverChild(sp);
 	} else if (sp->wrk->res_mode & RES_ESI_CHILD &&
-	    !sp->req->gzip_resp && sp->req->obj->gziped) {
+	    !req->gzip_resp && req->obj->gziped) {
 		res_WriteGunzipObj(sp);
 	} else if (sp->wrk->res_mode & RES_GUNZIP) {
 		res_WriteGunzipObj(sp);



From phk at varnish-cache.org  Tue Jan 17 09:29:23 2012
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Tue, 17 Jan 2012 10:29:23 +0100
Subject: [master] 41a9730 Remove unused cruft from the HTTP header table.
Message-ID: <E1Rn5M3-0004v6-Va@project.varnish-software.com>

commit 41a9730fd0b96f2e92dafabf7bc6ce1fc3625716
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Jan 17 09:28:23 2012 +0000

    Remove unused cruft from the HTTP header table.
    
    Eliminate the HTTPH_A_DELIVER filter which only took out "Date:"
    because we always overwrite Date: anyway.

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index 356d138..8891f12 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -841,7 +841,7 @@ int HTC_Rx(struct http_conn *htc);
 ssize_t HTC_Read(struct worker *w, struct http_conn *htc, void *d, size_t len);
 int HTC_Complete(struct http_conn *htc);
 
-#define HTTPH(a, b, c, d, e, f, g) extern char b[];
+#define HTTPH(a, b, c) extern char b[];
 #include "tbl/http_headers.h"
 #undef HTTPH
 
diff --git a/bin/varnishd/cache/cache_center.c b/bin/varnishd/cache/cache_center.c
index 62108ec..df00fbc 100644
--- a/bin/varnishd/cache/cache_center.c
+++ b/bin/varnishd/cache/cache_center.c
@@ -1419,7 +1419,7 @@ cnt_pipe(struct sess *sp, struct worker *wrk, const struct req *req)
 	WS_Reset(wrk->ws, NULL);
 	wrk->busyobj = VBO_GetBusyObj(wrk);
 	http_Setup(wrk->busyobj->bereq, wrk->ws);
-	http_FilterHeader(sp, HTTPH_R_PIPE);
+	http_FilterHeader(sp, 0);
 
 	VCL_pipe_method(sp);
 
diff --git a/bin/varnishd/cache/cache_http.c b/bin/varnishd/cache/cache_http.c
index f441ef2..27a648f 100644
--- a/bin/varnishd/cache/cache_http.c
+++ b/bin/varnishd/cache/cache_http.c
@@ -37,7 +37,7 @@
 
 #include "vct.h"
 
-#define HTTPH(a, b, c, d, e, f, g) char b[] = "*" a ":";
+#define HTTPH(a, b, c) char b[] = "*" a ":";
 #include "tbl/http_headers.h"
 #undef HTTPH
 
@@ -824,8 +824,8 @@ http_EstimateWS(const struct http *fm, unsigned how, uint16_t *nhd)
 			continue;
 		if (fm->hdf[u] & HDF_FILTER)
 			continue;
-#define HTTPH(a, b, c, d, e, f, g) \
-		if (((e) & how) && http_IsHdr(&fm->hd[u], (b))) \
+#define HTTPH(a, b, c) \
+		if (((c) & how) && http_IsHdr(&fm->hd[u], (b))) \
 			continue;
 #include "tbl/http_headers.h"
 #undef HTTPH
@@ -853,8 +853,8 @@ http_FilterFields(struct worker *w, unsigned vsl_id, struct http *to,
 			continue;
 		if (fm->hdf[u] & HDF_FILTER)
 			continue;
-#define HTTPH(a, b, c, d, e, f, g) \
-		if (((e) & how) && http_IsHdr(&fm->hd[u], (b))) \
+#define HTTPH(a, b, c) \
+		if (((c) & how) && http_IsHdr(&fm->hd[u], (b))) \
 			continue;
 #include "tbl/http_headers.h"
 #undef HTTPH
@@ -1114,7 +1114,7 @@ void
 HTTP_Init(void)
 {
 
-#define HTTPH(a, b, c, d, e, f, g) b[0] = (char)strlen(b + 1);
+#define HTTPH(a, b, c) b[0] = (char)strlen(b + 1);
 #include "tbl/http_headers.h"
 #undef HTTPH
 }
diff --git a/bin/varnishd/cache/cache_response.c b/bin/varnishd/cache/cache_response.c
index dc3bdc2..3a19d4a 100644
--- a/bin/varnishd/cache/cache_response.c
+++ b/bin/varnishd/cache/cache_response.c
@@ -119,7 +119,7 @@ RES_BuildHttp(const struct sess *sp)
 	req->resp->logtag = HTTP_Tx;
 	http_CopyResp(req->resp, req->obj->http);
 	http_FilterFields(sp->wrk, sp->vsl_id, req->resp,
-	    req->obj->http, HTTPH_A_DELIVER);
+	    req->obj->http, 0);
 
 	if (!(sp->wrk->res_mode & RES_LEN)) {
 		http_Unset(req->resp, H_Content_Length);
diff --git a/include/tbl/http_headers.h b/include/tbl/http_headers.h
index afb463c..503c5c3 100644
--- a/include/tbl/http_headers.h
+++ b/include/tbl/http_headers.h
@@ -30,11 +30,7 @@
  * ---------------------------------------
  * a	Http header name
  * b	session field name
- * c	Request(1)/Response(2) bitfield
- * d	(obsolete)
- * e	Supress header in filter ops
- * f	unused
- * g	unused
+ * c	Supress header in filter ops
  *
  * see [RFC2616 13.5.1 End-to-end and Hop-by-hop Headers]
  *
@@ -44,60 +40,57 @@
 
 #ifndef HTTPH_R_PASS
 #define HTTPH_R_PASS	(1 << 0)	/* Request (c->b) in pass mode */
-#define HTTPH_A_PASS	(1 << 1)	/* Response (b->c)in pass mode */
-#define HTTPH_R_PIPE	(1 << 2)	/* Request (c->b) in pipe mode */
-#define HTTPH_R_FETCH	(1 << 3)	/* Request (c->b) for fetch */
-#define HTTPH_A_INS	(1 << 4)	/* Response (b->o) for insert */
-#define HTTPH_A_DELIVER	(1 << 5)	/* Response (o->c) for deliver */
+#define HTTPH_R_FETCH	(1 << 1)	/* Request (c->b) for fetch */
+#define HTTPH_A_INS	(1 << 2)	/* Response (b->o) for insert */
 #endif
 
-HTTPH("Keep-Alive",		H_Keep_Alive,		3, 3, HTTPH_R_PASS | HTTPH_A_PASS | HTTPH_R_FETCH,					0, 0)	/* RFC2068 */
-HTTPH("Accept",			H_Accept,		1, 0, 0,										0, 0)	/* RFC2616 14.1 */
-HTTPH("Accept-Charset",		H_Accept_Charset,	1, 0, 0,										0, 0)	/* RFC2616 14.2 */
-HTTPH("Accept-Encoding",	H_Accept_Encoding,	1, 0, 0,										0, 0)	/* RFC2616 14.3 */
-HTTPH("Accept-Language",	H_Accept_Language,	1, 0, 0,										0, 0)	/* RFC2616 14.4 */
-HTTPH("Accept-Ranges",		H_Accept_Ranges,	2, 3,				    HTTPH_R_FETCH | HTTPH_A_INS,			0, 0)	/* RFC2616 14.5 */
-HTTPH("Age",			H_Age,			2, 0,						    HTTPH_A_INS,			0, 0)	/* RFC2616 14.6 */
-HTTPH("Allow",			H_Allow,		2, 0, 0,										0, 0)	/* RFC2616 14.7 */
-HTTPH("Authorization",		H_Authorization,	1, 0, 0,										0, 0)	/* RFC2616 14.8 */
-HTTPH("Cache-Control",		H_Cache_Control,	3, 3,				    HTTPH_R_FETCH,					0, 0)	/* RFC2616 14.9 */
-HTTPH("Connection",		H_Connection,		3, 3, HTTPH_R_PASS | HTTPH_A_PASS | HTTPH_R_FETCH | HTTPH_A_INS,			0, 0)	/* RFC2616 14.10 */
-HTTPH("Content-Encoding",	H_Content_Encoding,	2, 0, 0,										0, 0)	/* RFC2616 14.11 */
-HTTPH("Content-Langugae",	H_Content_Language,	2, 0, 0,										0, 0)	/* RFC2616 14.12 */
-HTTPH("Content-Length",		H_Content_Length,	2, 2,				    HTTPH_R_FETCH | HTTPH_A_INS,			0, 0)	/* RFC2616 14.13 */
-HTTPH("Content-Location",	H_Content_Location,	2, 0, 0,										0, 0)	/* RFC2616 14.14 */
-HTTPH("Content-MD5",		H_Content_MD5,		2, 0, 0,										0, 0)	/* RFC2616 14.15 */
-HTTPH("Content-Range",		H_Content_Range,	2, 3,				    HTTPH_R_FETCH | HTTPH_A_INS,			0, 0)	/* RFC2616 14.16 */
-HTTPH("Content-Type",		H_Content_Type,		2, 0, 0,										0, 0)	/* RFC2616 14.17 */
-HTTPH("Date",			H_Date,			2, 0,								HTTPH_A_DELIVER,	0, 0)	/* RFC2616 14.18 */
-HTTPH("ETag",			H_ETag,			2, 0, 0,										0, 0)	/* RFC2616 14.19 */
-HTTPH("Expect",			H_Expect,		1, 0, 0,										0, 0)	/* RFC2616 14.20 */
-HTTPH("Expires",		H_Expires,		2, 0, 0,										0, 0)	/* RFC2616 14.21 */
-HTTPH("From",			H_From,			1, 0, 0,										0, 0)	/* RFC2616 14.22 */
-HTTPH("Host",			H_Host,			1, 0, 0,										0, 0)	/* RFC2616 14.23 */
-HTTPH("If-Match",		H_If_Match,		1, 1,				    HTTPH_R_FETCH,					0, 0)	/* RFC2616 14.24 */
-HTTPH("If-Modified-Since",	H_If_Modified_Since,	1, 1,				    HTTPH_R_FETCH,					0, 0)	/* RFC2616 14.25 */
-HTTPH("If-None-Match",		H_If_None_Match,	1, 1,				    HTTPH_R_FETCH,					0, 0)	/* RFC2616 14.26 */
-HTTPH("If-Range",		H_If_Range,		1, 1,				    HTTPH_R_FETCH,					0, 0)	/* RFC2616 14.27 */
-HTTPH("If-Unmodified-Since",	H_If_Unmodifed_Since,	1, 1,				    HTTPH_R_FETCH,					0, 0)	/* RFC2616 14.28 */
-HTTPH("Last-Modified",		H_Last_Modified,	2, 0, 0,										0, 0)	/* RFC2616 14.29 */
-HTTPH("Location",		H_Location,		2, 0, 0,										0, 0)	/* RFC2616 14.30 */
-HTTPH("Max-Forwards",		H_Max_Forwards,		1, 0, 0,										0, 0)	/* RFC2616 14.31 */
-HTTPH("Pragma",			H_Pragma,		1, 0, 0,										0, 0)	/* RFC2616 14.32 */
-HTTPH("Proxy-Authenticate",	H_Proxy_Authenticate,	2, 3,				    HTTPH_R_FETCH | HTTPH_A_INS,			0, 0)	/* RFC2616 14.33 */
-HTTPH("Proxy-Authorization",	H_Proxy_Authorization,	1, 3,				    HTTPH_R_FETCH | HTTPH_A_INS,			0, 0)	/* RFC2616 14.34 */
-HTTPH("Range",			H_Range,		1, 0,				    HTTPH_R_FETCH | HTTPH_A_INS,			0, 0)	/* RFC2616 14.35 */
-HTTPH("Referer",		H_Referer,		1, 0, 0,										0, 0)	/* RFC2616 14.36 */
-HTTPH("Retry-After",		H_Retry_After,		2, 0, 0,										0, 0)	/* RFC2616 14.37 */
-HTTPH("Server",			H_Server,		2, 0, 0,										0, 0)	/* RFC2616 14.38 */
-HTTPH("TE",			H_TE,			1, 3, HTTPH_R_PASS | HTTPH_A_PASS | HTTPH_R_FETCH | HTTPH_A_INS,			0, 0)	/* RFC2616 14.39 */
-HTTPH("Trailer",		H_Trailer,		1, 3, HTTPH_R_PASS | HTTPH_A_PASS | HTTPH_R_FETCH | HTTPH_A_INS,			0, 0)	/* RFC2616 14.40 */
-HTTPH("Transfer-Encoding",	H_Transfer_Encoding,	2, 3, HTTPH_R_PASS | HTTPH_A_PASS | HTTPH_R_FETCH | HTTPH_A_INS,			0, 0)	/* RFC2616 14.41 */
-HTTPH("Upgrade",		H_Upgrade,		2, 3, HTTPH_R_PASS | HTTPH_A_PASS | HTTPH_R_FETCH | HTTPH_A_INS,			0, 0)	/* RFC2616 14.42 */
-HTTPH("User-Agent",		H_User_Agent,		1, 0, 0,										0, 0)	/* RFC2616 14.43 */
-HTTPH("Vary",			H_Vary,			2, 0, 0,										0, 0)	/* RFC2616 14.44 */
-HTTPH("Via",			H_Via,			2, 0, 0,										0, 0)	/* RFC2616 14.45 */
-HTTPH("Warning",		H_Warning,		2, 0, 0,										0, 0)	/* RFC2616 14.46 */
-HTTPH("WWW-Authenticate",	H_WWW_Authenticate,	2, 0, 0,										0, 0)	/* RFC2616 14.47 */
+HTTPH("Keep-Alive",		H_Keep_Alive,		HTTPH_R_PASS | HTTPH_R_FETCH		  )	/* RFC2068 */
+HTTPH("Accept",			H_Accept,		0					  )	/* RFC2616 14.1 */
+HTTPH("Accept-Charset",		H_Accept_Charset,	0					  )	/* RFC2616 14.2 */
+HTTPH("Accept-Encoding",	H_Accept_Encoding,	0					  )	/* RFC2616 14.3 */
+HTTPH("Accept-Language",	H_Accept_Language,	0					  )	/* RFC2616 14.4 */
+HTTPH("Accept-Ranges",		H_Accept_Ranges,		       HTTPH_R_FETCH | HTTPH_A_INS)	/* RFC2616 14.5 */
+HTTPH("Age",			H_Age,						       HTTPH_A_INS)	/* RFC2616 14.6 */
+HTTPH("Allow",			H_Allow,		0					  )	/* RFC2616 14.7 */
+HTTPH("Authorization",		H_Authorization,	0					  )	/* RFC2616 14.8 */
+HTTPH("Cache-Control",		H_Cache_Control,		       HTTPH_R_FETCH		  )	/* RFC2616 14.9 */
+HTTPH("Connection",		H_Connection,		HTTPH_R_PASS | HTTPH_R_FETCH | HTTPH_A_INS)	/* RFC2616 14.10 */
+HTTPH("Content-Encoding",	H_Content_Encoding,	0					  )	/* RFC2616 14.11 */
+HTTPH("Content-Langugae",	H_Content_Language,	0					  )	/* RFC2616 14.12 */
+HTTPH("Content-Length",		H_Content_Length,		       HTTPH_R_FETCH | HTTPH_A_INS)	/* RFC2616 14.13 */
+HTTPH("Content-Location",	H_Content_Location,	0					  )	/* RFC2616 14.14 */
+HTTPH("Content-MD5",		H_Content_MD5,		0					  )	/* RFC2616 14.15 */
+HTTPH("Content-Range",		H_Content_Range,		       HTTPH_R_FETCH | HTTPH_A_INS)	/* RFC2616 14.16 */
+HTTPH("Content-Type",		H_Content_Type,		0					  )	/* RFC2616 14.17 */
+HTTPH("Date",			H_Date,			0					  )	/* RFC2616 14.18 */
+HTTPH("ETag",			H_ETag,			0					  )	/* RFC2616 14.19 */
+HTTPH("Expect",			H_Expect,		0					  )	/* RFC2616 14.20 */
+HTTPH("Expires",		H_Expires,		0					  )	/* RFC2616 14.21 */
+HTTPH("From",			H_From,			0					  )	/* RFC2616 14.22 */
+HTTPH("Host",			H_Host,			0					  )	/* RFC2616 14.23 */
+HTTPH("If-Match",		H_If_Match,			       HTTPH_R_FETCH		  )	/* RFC2616 14.24 */
+HTTPH("If-Modified-Since",	H_If_Modified_Since,		       HTTPH_R_FETCH		  )	/* RFC2616 14.25 */
+HTTPH("If-None-Match",		H_If_None_Match,		       HTTPH_R_FETCH		  )	/* RFC2616 14.26 */
+HTTPH("If-Range",		H_If_Range,			       HTTPH_R_FETCH		  )	/* RFC2616 14.27 */
+HTTPH("If-Unmodified-Since",	H_If_Unmodifed_Since,		       HTTPH_R_FETCH		  )	/* RFC2616 14.28 */
+HTTPH("Last-Modified",		H_Last_Modified,	0					  )	/* RFC2616 14.29 */
+HTTPH("Location",		H_Location,		0					  )	/* RFC2616 14.30 */
+HTTPH("Max-Forwards",		H_Max_Forwards,		0					  )	/* RFC2616 14.31 */
+HTTPH("Pragma",			H_Pragma,		0					  )	/* RFC2616 14.32 */
+HTTPH("Proxy-Authenticate",	H_Proxy_Authenticate,		       HTTPH_R_FETCH | HTTPH_A_INS)	/* RFC2616 14.33 */
+HTTPH("Proxy-Authorization",	H_Proxy_Authorization,		       HTTPH_R_FETCH | HTTPH_A_INS)	/* RFC2616 14.34 */
+HTTPH("Range",			H_Range,			       HTTPH_R_FETCH | HTTPH_A_INS)	/* RFC2616 14.35 */
+HTTPH("Referer",		H_Referer,		0					  )	/* RFC2616 14.36 */
+HTTPH("Retry-After",		H_Retry_After,		0					  )	/* RFC2616 14.37 */
+HTTPH("Server",			H_Server,		0					  )	/* RFC2616 14.38 */
+HTTPH("TE",			H_TE,			HTTPH_R_PASS | HTTPH_R_FETCH | HTTPH_A_INS)	/* RFC2616 14.39 */
+HTTPH("Trailer",		H_Trailer,		HTTPH_R_PASS | HTTPH_R_FETCH | HTTPH_A_INS)	/* RFC2616 14.40 */
+HTTPH("Transfer-Encoding",	H_Transfer_Encoding,	HTTPH_R_PASS | HTTPH_R_FETCH | HTTPH_A_INS)	/* RFC2616 14.41 */
+HTTPH("Upgrade",		H_Upgrade,		HTTPH_R_PASS | HTTPH_R_FETCH | HTTPH_A_INS)	/* RFC2616 14.42 */
+HTTPH("User-Agent",		H_User_Agent,		0					  )	/* RFC2616 14.43 */
+HTTPH("Vary",			H_Vary,			0					  )	/* RFC2616 14.44 */
+HTTPH("Via",			H_Via,			0					  )	/* RFC2616 14.45 */
+HTTPH("Warning",		H_Warning,		0					  )	/* RFC2616 14.46 */
+HTTPH("WWW-Authenticate",	H_WWW_Authenticate,	0					  )	/* RFC2616 14.47 */
 
 /*lint -restore */



From phk at varnish-cache.org  Tue Jan 17 10:16:32 2012
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Tue, 17 Jan 2012 11:16:32 +0100
Subject: [master] dd103d3 Cleanup http filtering operations:
Message-ID: <E1Rn65g-0007kP-5o@project.varnish-software.com>

commit dd103d37091b9951f7705b240913ce638ded640c
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Jan 17 10:15:14 2012 +0000

    Cleanup http filtering operations:
    
    Rename http_FilterHeader() to http_FilterReq()
    Add http_FilterResp()
    make http_FilterFields() static
    remove http_CopyResp()

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index 8891f12..817b7c7 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -797,12 +797,11 @@ void HTTP_Init(void);
 void http_ClrHeader(struct http *to);
 unsigned http_Write(struct worker *w, unsigned vsl_id, const struct http *hp,
     int resp);
-void http_CopyResp(struct http *to, const struct http *fm);
 void http_SetResp(struct http *to, const char *proto, uint16_t status,
     const char *response);
-void http_FilterFields(struct worker *w, unsigned vsl_id, struct http *to,
-    const struct http *fm, unsigned how);
-void http_FilterHeader(const struct sess *sp, unsigned how);
+void http_FilterReq(const struct sess *sp, unsigned how);
+void http_FilterResp(const struct sess *sp, const struct http *fm, struct http *to,
+    unsigned how);
 void http_PutProtocol(struct worker *w, unsigned vsl_id, const struct http *to,
     const char *protocol);
 void http_PutStatus(struct http *to, uint16_t status);
diff --git a/bin/varnishd/cache/cache_center.c b/bin/varnishd/cache/cache_center.c
index df00fbc..a9c233b 100644
--- a/bin/varnishd/cache/cache_center.c
+++ b/bin/varnishd/cache/cache_center.c
@@ -876,9 +876,7 @@ cnt_fetchbody(struct sess *sp, struct worker *wrk, struct req *req)
 	hp2 = req->obj->http;
 
 	hp2->logtag = HTTP_Obj;
-	http_CopyResp(hp2, hp);
-	http_FilterFields(wrk, sp->vsl_id, hp2, hp,
-	    pass ? HTTPH_R_PASS : HTTPH_A_INS);
+	http_FilterResp(sp, hp, hp2, pass ? HTTPH_R_PASS : HTTPH_A_INS);
 	http_CopyHome(wrk, sp->vsl_id, hp2);
 
 	if (http_GetHdr(hp, H_Last_Modified, &b))
@@ -1268,7 +1266,7 @@ cnt_miss(struct sess *sp, struct worker *wrk, struct req *req)
 	WS_Reset(wrk->ws, NULL);
 	wrk->busyobj = VBO_GetBusyObj(wrk);
 	http_Setup(wrk->busyobj->bereq, wrk->ws);
-	http_FilterHeader(sp, HTTPH_R_FETCH);
+	http_FilterReq(sp, HTTPH_R_FETCH);
 	http_ForceGet(wrk->busyobj->bereq);
 	if (cache_param->http_gzip_support) {
 		/*
@@ -1360,7 +1358,7 @@ cnt_pass(struct sess *sp, struct worker *wrk, struct req *req)
 	WS_Reset(wrk->ws, NULL);
 	wrk->busyobj = VBO_GetBusyObj(wrk);
 	http_Setup(wrk->busyobj->bereq, wrk->ws);
-	http_FilterHeader(sp, HTTPH_R_PASS);
+	http_FilterReq(sp, HTTPH_R_PASS);
 
 	wrk->connect_timeout = 0;
 	wrk->first_byte_timeout = 0;
@@ -1419,7 +1417,7 @@ cnt_pipe(struct sess *sp, struct worker *wrk, const struct req *req)
 	WS_Reset(wrk->ws, NULL);
 	wrk->busyobj = VBO_GetBusyObj(wrk);
 	http_Setup(wrk->busyobj->bereq, wrk->ws);
-	http_FilterHeader(sp, 0);
+	http_FilterReq(sp, 0);
 
 	VCL_pipe_method(sp);
 
diff --git a/bin/varnishd/cache/cache_http.c b/bin/varnishd/cache/cache_http.c
index 27a648f..9fc4e09 100644
--- a/bin/varnishd/cache/cache_http.c
+++ b/bin/varnishd/cache/cache_http.c
@@ -765,17 +765,6 @@ http_ForceGet(const struct http *to)
 }
 
 void
-http_CopyResp(struct http *to, const struct http *fm)
-{
-
-	CHECK_OBJ_NOTNULL(fm, HTTP_MAGIC);
-	CHECK_OBJ_NOTNULL(to, HTTP_MAGIC);
-	http_SetH(to, HTTP_HDR_PROTO, "HTTP/1.1");
-	to->status = fm->status;
-	http_copyh(to, fm, HTTP_HDR_RESPONSE);
-}
-
-void
 http_SetResp(struct http *to, const char *proto, uint16_t status,
     const char *response)
 {
@@ -838,8 +827,8 @@ http_EstimateWS(const struct http *fm, unsigned how, uint16_t *nhd)
 
 /*--------------------------------------------------------------------*/
 
-void
-http_FilterFields(struct worker *w, unsigned vsl_id, struct http *to,
+static void
+http_filterfields(struct worker *w, unsigned vsl_id, struct http *to,
     const struct http *fm, unsigned how)
 {
 	unsigned u;
@@ -865,7 +854,7 @@ http_FilterFields(struct worker *w, unsigned vsl_id, struct http *to,
 /*--------------------------------------------------------------------*/
 
 void
-http_FilterHeader(const struct sess *sp, unsigned how)
+http_FilterReq(const struct sess *sp, unsigned how)
 {
 	struct http *hp;
 
@@ -879,11 +868,26 @@ http_FilterHeader(const struct sess *sp, unsigned how)
 		http_SetH(hp, HTTP_HDR_PROTO, "HTTP/1.1");
 	else
 		http_copyh(hp, sp->req->http, HTTP_HDR_PROTO);
-	http_FilterFields(sp->wrk, sp->vsl_id, hp, sp->req->http, how);
+	http_filterfields(sp->wrk, sp->vsl_id, hp, sp->req->http, how);
 	http_PrintfHeader(sp->wrk, sp->vsl_id, hp,
 	    "X-Varnish: %u", sp->req->xid);
 }
 
+/*--------------------------------------------------------------------*/
+
+void
+http_FilterResp(const struct sess *sp, const struct http *fm, struct http *to,
+    unsigned how)
+{
+
+	CHECK_OBJ_NOTNULL(fm, HTTP_MAGIC);
+	CHECK_OBJ_NOTNULL(to, HTTP_MAGIC);
+	http_SetH(to, HTTP_HDR_PROTO, "HTTP/1.1");
+	to->status = fm->status;
+	http_copyh(to, fm, HTTP_HDR_RESPONSE);
+	http_filterfields(sp->wrk, sp->vsl_id, to, fm, how);
+}
+
 /*--------------------------------------------------------------------
  * This function copies any header fields which reference foreign
  * storage into our own WS.
diff --git a/bin/varnishd/cache/cache_response.c b/bin/varnishd/cache/cache_response.c
index 3a19d4a..5724f07 100644
--- a/bin/varnishd/cache/cache_response.c
+++ b/bin/varnishd/cache/cache_response.c
@@ -117,9 +117,7 @@ RES_BuildHttp(const struct sess *sp)
 
 	http_ClrHeader(req->resp);
 	req->resp->logtag = HTTP_Tx;
-	http_CopyResp(req->resp, req->obj->http);
-	http_FilterFields(sp->wrk, sp->vsl_id, req->resp,
-	    req->obj->http, 0);
+	http_FilterResp(sp, req->obj->http, req->resp, 0);
 
 	if (!(sp->wrk->res_mode & RES_LEN)) {
 		http_Unset(req->resp, H_Content_Length);



From phk at varnish-cache.org  Tue Jan 17 11:57:21 2012
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Tue, 17 Jan 2012 12:57:21 +0100
Subject: [master] 4c8f484 A bit of cleanup of the filtering stuff
Message-ID: <E1Rn7fF-0006JH-UF@project.varnish-software.com>

commit 4c8f484e24c5871dce6f6c4a19c40a6999fb5725
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Jan 17 11:57:11 2012 +0000

    A bit of cleanup of the filtering stuff

diff --git a/bin/varnishd/cache/cache_http.c b/bin/varnishd/cache/cache_http.c
index 9fc4e09..0b0b926 100644
--- a/bin/varnishd/cache/cache_http.c
+++ b/bin/varnishd/cache/cache_http.c
@@ -748,7 +748,7 @@ http_SetH(const struct http *to, unsigned n, const char *fm)
 }
 
 static void
-http_copyh(const struct http *to, const struct http *fm, unsigned n)
+http_linkh(const struct http *to, const struct http *fm, unsigned n)
 {
 
 	assert(n < HTTP_HDR_FIRST);
@@ -776,25 +776,6 @@ http_SetResp(struct http *to, const char *proto, uint16_t status,
 	http_SetH(to, HTTP_HDR_RESPONSE, response);
 }
 
-static void
-http_copyheader(struct worker *w, unsigned vsl_id, struct http *to,
-    const struct http *fm, unsigned n)
-{
-
-	CHECK_OBJ_NOTNULL(fm, HTTP_MAGIC);
-	CHECK_OBJ_NOTNULL(to, HTTP_MAGIC);
-	assert(n < fm->shd);
-	Tcheck(fm->hd[n]);
-	if (to->nhd < to->shd) {
-		to->hd[to->nhd] = fm->hd[n];
-		to->hdf[to->nhd] = 0;
-		to->nhd++;
-	} else  {
-		VSC_C_main->losthdr++;
-		WSLR(w, SLT_LostHeader, vsl_id, fm->hd[n]);
-	}
-}
-
 /*--------------------------------------------------------------------
  * Estimate how much workspace we need to Filter this header according
  * to 'how'.
@@ -847,7 +828,15 @@ http_filterfields(struct worker *w, unsigned vsl_id, struct http *to,
 			continue;
 #include "tbl/http_headers.h"
 #undef HTTPH
-		http_copyheader(w, vsl_id, to, fm, u);
+		Tcheck(fm->hd[u]);
+		if (to->nhd < to->shd) {
+			to->hd[to->nhd] = fm->hd[u];
+			to->hdf[to->nhd] = 0;
+			to->nhd++;
+		} else  {
+			VSC_C_main->losthdr++;
+			WSLR(w, SLT_LostHeader, vsl_id, fm->hd[u]);
+		}
 	}
 }
 
@@ -862,12 +851,12 @@ http_FilterReq(const struct sess *sp, unsigned how)
 	CHECK_OBJ_NOTNULL(hp, HTTP_MAGIC);
 	hp->logtag = HTTP_Tx;
 
-	http_copyh(hp, sp->req->http, HTTP_HDR_REQ);
-	http_copyh(hp, sp->req->http, HTTP_HDR_URL);
+	http_linkh(hp, sp->req->http, HTTP_HDR_REQ);
+	http_linkh(hp, sp->req->http, HTTP_HDR_URL);
 	if (how == HTTPH_R_FETCH)
 		http_SetH(hp, HTTP_HDR_PROTO, "HTTP/1.1");
 	else
-		http_copyh(hp, sp->req->http, HTTP_HDR_PROTO);
+		http_linkh(hp, sp->req->http, HTTP_HDR_PROTO);
 	http_filterfields(sp->wrk, sp->vsl_id, hp, sp->req->http, how);
 	http_PrintfHeader(sp->wrk, sp->vsl_id, hp,
 	    "X-Varnish: %u", sp->req->xid);
@@ -884,7 +873,7 @@ http_FilterResp(const struct sess *sp, const struct http *fm, struct http *to,
 	CHECK_OBJ_NOTNULL(to, HTTP_MAGIC);
 	http_SetH(to, HTTP_HDR_PROTO, "HTTP/1.1");
 	to->status = fm->status;
-	http_copyh(to, fm, HTTP_HDR_RESPONSE);
+	http_linkh(to, fm, HTTP_HDR_RESPONSE);
 	http_filterfields(sp->wrk, sp->vsl_id, to, fm, how);
 }
 



From geoff at varnish-cache.org  Tue Jan 17 13:21:27 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Tue, 17 Jan 2012 14:21:27 +0100
Subject: [experimental-ims] 05184e9 Minor cleanups while waiting for tests
Message-ID: <E1Rn8yd-0003Ae-Uv@project.varnish-software.com>

commit 05184e90cc52ab073da0a08ed9c83804a4611f19
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Jan 17 08:37:46 2012 +0000

    Minor cleanups while waiting for tests

diff --git a/bin/varnishd/cache/cache_center.c b/bin/varnishd/cache/cache_center.c
index d80842a..62108ec 100644
--- a/bin/varnishd/cache/cache_center.c
+++ b/bin/varnishd/cache/cache_center.c
@@ -1405,7 +1405,7 @@ DOT err_pipe [label="ERROR",shape=plaintext]
  */
 
 static int
-cnt_pipe(struct sess *sp, struct worker *wrk, struct req *req)
+cnt_pipe(struct sess *sp, struct worker *wrk, const struct req *req)
 {
 
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
diff --git a/bin/varnishd/cache/cache_lck.c b/bin/varnishd/cache/cache_lck.c
index 2910582..9fb34ae 100644
--- a/bin/varnishd/cache/cache_lck.c
+++ b/bin/varnishd/cache/cache_lck.c
@@ -107,7 +107,7 @@ Lck__Unlock(struct lock *lck, const char *p, const char *f, int l)
 	 * We hack it and fill it with zero bits, hoping for sane
 	 * implementations of pthread.
 	 * #endif
-	 */ 
+	 */
 	memset(&ilck->owner, 0, sizeof ilck->owner);
 	AZ(pthread_mutex_unlock(&ilck->mtx));
 	if (cache_param->diag_bitmap & 0x8)
diff --git a/bin/varnishd/cache/cache_response.c b/bin/varnishd/cache/cache_response.c
index fc94c22..dc3bdc2 100644
--- a/bin/varnishd/cache/cache_response.c
+++ b/bin/varnishd/cache/cache_response.c
@@ -40,8 +40,11 @@ static void
 res_dorange(const struct sess *sp, const char *r, ssize_t *plow, ssize_t *phigh)
 {
 	ssize_t low, high, has_low;
+	struct req *req;
 
-	assert(sp->req->obj->response == 200);
+	req = sp->req;
+	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
+	assert(req->obj->response == 200);
 	if (strncmp(r, "bytes=", 6))
 		return;
 	r += 6;
@@ -57,7 +60,7 @@ res_dorange(const struct sess *sp, const char *r, ssize_t *plow, ssize_t *phigh)
 		r++;
 	}
 
-	if (low >= sp->req->obj->len)
+	if (low >= req->obj->len)
 		return;
 
 	if (*r != '-')
@@ -73,28 +76,28 @@ res_dorange(const struct sess *sp, const char *r, ssize_t *plow, ssize_t *phigh)
 			r++;
 		}
 		if (!has_low) {
-			low = sp->req->obj->len - high;
-			high = sp->req->obj->len - 1;
+			low = req->obj->len - high;
+			high = req->obj->len - 1;
 		}
 	} else
-		high = sp->req->obj->len - 1;
+		high = req->obj->len - 1;
 	if (*r != '\0')
 		return;
 
-	if (high >= sp->req->obj->len)
-		high = sp->req->obj->len - 1;
+	if (high >= req->obj->len)
+		high = req->obj->len - 1;
 
 	if (low > high)
 		return;
 
-	http_PrintfHeader(sp->wrk, sp->vsl_id, sp->req->resp,
+	http_PrintfHeader(sp->wrk, sp->vsl_id, req->resp,
 	    "Content-Range: bytes %jd-%jd/%jd",
-	    (intmax_t)low, (intmax_t)high, (intmax_t)sp->req->obj->len);
-	http_Unset(sp->req->resp, H_Content_Length);
+	    (intmax_t)low, (intmax_t)high, (intmax_t)req->obj->len);
+	http_Unset(req->resp, H_Content_Length);
 	assert(sp->wrk->res_mode & RES_LEN);
-	http_PrintfHeader(sp->wrk, sp->vsl_id, sp->req->resp,
+	http_PrintfHeader(sp->wrk, sp->vsl_id, req->resp,
 	    "Content-Length: %jd", (intmax_t)(1 + high - low));
-	http_SetResp(sp->req->resp, "HTTP/1.1", 206, "Partial Content");
+	http_SetResp(req->resp, "HTTP/1.1", 206, "Partial Content");
 
 	*plow = low;
 	*phigh = high;
@@ -106,44 +109,46 @@ void
 RES_BuildHttp(const struct sess *sp)
 {
 	char time_str[30];
+	struct req *req;
 
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+	req = sp->req;
+	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
 
-
-	http_ClrHeader(sp->req->resp);
-	sp->req->resp->logtag = HTTP_Tx;
-	http_CopyResp(sp->req->resp, sp->req->obj->http);
-	http_FilterFields(sp->wrk, sp->vsl_id, sp->req->resp,
-	    sp->req->obj->http, HTTPH_A_DELIVER);
+	http_ClrHeader(req->resp);
+	req->resp->logtag = HTTP_Tx;
+	http_CopyResp(req->resp, req->obj->http);
+	http_FilterFields(sp->wrk, sp->vsl_id, req->resp,
+	    req->obj->http, HTTPH_A_DELIVER);
 
 	if (!(sp->wrk->res_mode & RES_LEN)) {
-		http_Unset(sp->req->resp, H_Content_Length);
+		http_Unset(req->resp, H_Content_Length);
 	} else if (cache_param->http_range_support) {
 		/* We only accept ranges if we know the length */
-		http_SetHeader(sp->wrk, sp->vsl_id, sp->req->resp,
+		http_SetHeader(sp->wrk, sp->vsl_id, req->resp,
 		    "Accept-Ranges: bytes");
 	}
 
 	if (sp->wrk->res_mode & RES_CHUNKED)
-		http_SetHeader(sp->wrk, sp->vsl_id, sp->req->resp,
+		http_SetHeader(sp->wrk, sp->vsl_id, req->resp,
 		    "Transfer-Encoding: chunked");
 
 	VTIM_format(VTIM_real(), time_str);
-	http_PrintfHeader(sp->wrk, sp->vsl_id, sp->req->resp,
+	http_PrintfHeader(sp->wrk, sp->vsl_id, req->resp,
 	    "Date: %s", time_str);
 
-	if (sp->req->xid != sp->req->obj->xid)
-		http_PrintfHeader(sp->wrk, sp->vsl_id, sp->req->resp,
-		    "X-Varnish: %u %u", sp->req->xid, sp->req->obj->xid);
+	if (req->xid != req->obj->xid)
+		http_PrintfHeader(sp->wrk, sp->vsl_id, req->resp,
+		    "X-Varnish: %u %u", req->xid, req->obj->xid);
 	else
-		http_PrintfHeader(sp->wrk, sp->vsl_id, sp->req->resp,
-		    "X-Varnish: %u", sp->req->xid);
-	http_PrintfHeader(sp->wrk, sp->vsl_id, sp->req->resp, "Age: %.0f",
-	    sp->req->obj->exp.age + sp->req->t_resp -
-	    sp->req->obj->exp.entered);
-	http_SetHeader(sp->wrk, sp->vsl_id, sp->req->resp, "Via: 1.1 varnish");
-	http_PrintfHeader(sp->wrk, sp->vsl_id, sp->req->resp, "Connection: %s",
-	    sp->req->doclose ? "close" : "keep-alive");
+		http_PrintfHeader(sp->wrk, sp->vsl_id, req->resp,
+		    "X-Varnish: %u", req->xid);
+	http_PrintfHeader(sp->wrk, sp->vsl_id, req->resp, "Age: %.0f",
+	    req->obj->exp.age + req->t_resp -
+	    req->obj->exp.entered);
+	http_SetHeader(sp->wrk, sp->vsl_id, req->resp, "Via: 1.1 varnish");
+	http_PrintfHeader(sp->wrk, sp->vsl_id, req->resp, "Connection: %s",
+	    req->doclose ? "close" : "keep-alive");
 }
 
 /*--------------------------------------------------------------------
@@ -254,63 +259,66 @@ RES_WriteObj(struct sess *sp)
 {
 	char *r;
 	ssize_t low, high;
+	struct req *req;
 
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+	req = sp->req;
+	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
 
 	WRW_Reserve(sp->wrk, &sp->fd);
 
-	if (sp->req->obj->response == 200 &&
-	    sp->req->http->conds &&
+	if (req->obj->response == 200 &&
+	    req->http->conds &&
 	    RFC2616_Do_Cond(sp)) {
-		sp->req->wantbody = 0;
-		http_SetResp(sp->req->resp, "HTTP/1.1", 304, "Not Modified");
-		http_Unset(sp->req->resp, H_Content_Length);
-		http_Unset(sp->req->resp, H_Transfer_Encoding);
+		req->wantbody = 0;
+		http_SetResp(req->resp, "HTTP/1.1", 304, "Not Modified");
+		http_Unset(req->resp, H_Content_Length);
+		http_Unset(req->resp, H_Transfer_Encoding);
 	}
 
 	/*
 	 * If nothing special planned, we can attempt Range support
 	 */
 	low = 0;
-	high = sp->req->obj->len - 1;
+	high = req->obj->len - 1;
 	if (
-	    sp->req->wantbody &&
+	    req->wantbody &&
 	    (sp->wrk->res_mode & RES_LEN) &&
 	    !(sp->wrk->res_mode & (RES_ESI|RES_ESI_CHILD|RES_GUNZIP)) &&
 	    cache_param->http_range_support &&
-	    sp->req->obj->response == 200 &&
-	    http_GetHdr(sp->req->http, H_Range, &r))
+	    req->obj->response == 200 &&
+	    http_GetHdr(req->http, H_Range, &r))
 		res_dorange(sp, r, &low, &high);
 
 	/*
 	 * Always remove C-E if client don't grok it
 	 */
 	if (sp->wrk->res_mode & RES_GUNZIP)
-		http_Unset(sp->req->resp, H_Content_Encoding);
+		http_Unset(req->resp, H_Content_Encoding);
 
 	/*
 	 * Send HTTP protocol header, unless interior ESI object
 	 */
 	if (!(sp->wrk->res_mode & RES_ESI_CHILD))
 		sp->wrk->acct_tmp.hdrbytes +=
-		    http_Write(sp->wrk, sp->vsl_id, sp->req->resp, 1);
+		    http_Write(sp->wrk, sp->vsl_id, req->resp, 1);
 
-	if (!sp->req->wantbody)
+	if (!req->wantbody)
 		sp->wrk->res_mode &= ~RES_CHUNKED;
 
 	if (sp->wrk->res_mode & RES_CHUNKED)
 		WRW_Chunked(sp->wrk);
 
-	if (!sp->req->wantbody) {
+	if (!req->wantbody) {
 		/* This was a HEAD or conditional request */
-	} else if (sp->req->obj->len == 0) {
+	} else if (req->obj->len == 0) {
 		/* Nothing to do here */
 	} else if (sp->wrk->res_mode & RES_ESI) {
 		ESI_Deliver(sp);
-	} else if (sp->wrk->res_mode & RES_ESI_CHILD && sp->req->gzip_resp) {
+	} else if (sp->wrk->res_mode & RES_ESI_CHILD && req->gzip_resp) {
 		ESI_DeliverChild(sp);
 	} else if (sp->wrk->res_mode & RES_ESI_CHILD &&
-	    !sp->req->gzip_resp && sp->req->obj->gziped) {
+	    !req->gzip_resp && req->obj->gziped) {
 		res_WriteGunzipObj(sp);
 	} else if (sp->wrk->res_mode & RES_GUNZIP) {
 		res_WriteGunzipObj(sp);



From geoff at varnish-cache.org  Tue Jan 17 13:21:28 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Tue, 17 Jan 2012 14:21:28 +0100
Subject: [experimental-ims] 41a9730 Remove unused cruft from the HTTP header
	table.
Message-ID: <E1Rn8ye-0003Ak-0U@project.varnish-software.com>

commit 41a9730fd0b96f2e92dafabf7bc6ce1fc3625716
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Jan 17 09:28:23 2012 +0000

    Remove unused cruft from the HTTP header table.
    
    Eliminate the HTTPH_A_DELIVER filter which only took out "Date:"
    because we always overwrite Date: anyway.

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index 356d138..8891f12 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -841,7 +841,7 @@ int HTC_Rx(struct http_conn *htc);
 ssize_t HTC_Read(struct worker *w, struct http_conn *htc, void *d, size_t len);
 int HTC_Complete(struct http_conn *htc);
 
-#define HTTPH(a, b, c, d, e, f, g) extern char b[];
+#define HTTPH(a, b, c) extern char b[];
 #include "tbl/http_headers.h"
 #undef HTTPH
 
diff --git a/bin/varnishd/cache/cache_center.c b/bin/varnishd/cache/cache_center.c
index 62108ec..df00fbc 100644
--- a/bin/varnishd/cache/cache_center.c
+++ b/bin/varnishd/cache/cache_center.c
@@ -1419,7 +1419,7 @@ cnt_pipe(struct sess *sp, struct worker *wrk, const struct req *req)
 	WS_Reset(wrk->ws, NULL);
 	wrk->busyobj = VBO_GetBusyObj(wrk);
 	http_Setup(wrk->busyobj->bereq, wrk->ws);
-	http_FilterHeader(sp, HTTPH_R_PIPE);
+	http_FilterHeader(sp, 0);
 
 	VCL_pipe_method(sp);
 
diff --git a/bin/varnishd/cache/cache_http.c b/bin/varnishd/cache/cache_http.c
index f441ef2..27a648f 100644
--- a/bin/varnishd/cache/cache_http.c
+++ b/bin/varnishd/cache/cache_http.c
@@ -37,7 +37,7 @@
 
 #include "vct.h"
 
-#define HTTPH(a, b, c, d, e, f, g) char b[] = "*" a ":";
+#define HTTPH(a, b, c) char b[] = "*" a ":";
 #include "tbl/http_headers.h"
 #undef HTTPH
 
@@ -824,8 +824,8 @@ http_EstimateWS(const struct http *fm, unsigned how, uint16_t *nhd)
 			continue;
 		if (fm->hdf[u] & HDF_FILTER)
 			continue;
-#define HTTPH(a, b, c, d, e, f, g) \
-		if (((e) & how) && http_IsHdr(&fm->hd[u], (b))) \
+#define HTTPH(a, b, c) \
+		if (((c) & how) && http_IsHdr(&fm->hd[u], (b))) \
 			continue;
 #include "tbl/http_headers.h"
 #undef HTTPH
@@ -853,8 +853,8 @@ http_FilterFields(struct worker *w, unsigned vsl_id, struct http *to,
 			continue;
 		if (fm->hdf[u] & HDF_FILTER)
 			continue;
-#define HTTPH(a, b, c, d, e, f, g) \
-		if (((e) & how) && http_IsHdr(&fm->hd[u], (b))) \
+#define HTTPH(a, b, c) \
+		if (((c) & how) && http_IsHdr(&fm->hd[u], (b))) \
 			continue;
 #include "tbl/http_headers.h"
 #undef HTTPH
@@ -1114,7 +1114,7 @@ void
 HTTP_Init(void)
 {
 
-#define HTTPH(a, b, c, d, e, f, g) b[0] = (char)strlen(b + 1);
+#define HTTPH(a, b, c) b[0] = (char)strlen(b + 1);
 #include "tbl/http_headers.h"
 #undef HTTPH
 }
diff --git a/bin/varnishd/cache/cache_response.c b/bin/varnishd/cache/cache_response.c
index dc3bdc2..3a19d4a 100644
--- a/bin/varnishd/cache/cache_response.c
+++ b/bin/varnishd/cache/cache_response.c
@@ -119,7 +119,7 @@ RES_BuildHttp(const struct sess *sp)
 	req->resp->logtag = HTTP_Tx;
 	http_CopyResp(req->resp, req->obj->http);
 	http_FilterFields(sp->wrk, sp->vsl_id, req->resp,
-	    req->obj->http, HTTPH_A_DELIVER);
+	    req->obj->http, 0);
 
 	if (!(sp->wrk->res_mode & RES_LEN)) {
 		http_Unset(req->resp, H_Content_Length);
diff --git a/include/tbl/http_headers.h b/include/tbl/http_headers.h
index afb463c..503c5c3 100644
--- a/include/tbl/http_headers.h
+++ b/include/tbl/http_headers.h
@@ -30,11 +30,7 @@
  * ---------------------------------------
  * a	Http header name
  * b	session field name
- * c	Request(1)/Response(2) bitfield
- * d	(obsolete)
- * e	Supress header in filter ops
- * f	unused
- * g	unused
+ * c	Supress header in filter ops
  *
  * see [RFC2616 13.5.1 End-to-end and Hop-by-hop Headers]
  *
@@ -44,60 +40,57 @@
 
 #ifndef HTTPH_R_PASS
 #define HTTPH_R_PASS	(1 << 0)	/* Request (c->b) in pass mode */
-#define HTTPH_A_PASS	(1 << 1)	/* Response (b->c)in pass mode */
-#define HTTPH_R_PIPE	(1 << 2)	/* Request (c->b) in pipe mode */
-#define HTTPH_R_FETCH	(1 << 3)	/* Request (c->b) for fetch */
-#define HTTPH_A_INS	(1 << 4)	/* Response (b->o) for insert */
-#define HTTPH_A_DELIVER	(1 << 5)	/* Response (o->c) for deliver */
+#define HTTPH_R_FETCH	(1 << 1)	/* Request (c->b) for fetch */
+#define HTTPH_A_INS	(1 << 2)	/* Response (b->o) for insert */
 #endif
 
-HTTPH("Keep-Alive",		H_Keep_Alive,		3, 3, HTTPH_R_PASS | HTTPH_A_PASS | HTTPH_R_FETCH,					0, 0)	/* RFC2068 */
-HTTPH("Accept",			H_Accept,		1, 0, 0,										0, 0)	/* RFC2616 14.1 */
-HTTPH("Accept-Charset",		H_Accept_Charset,	1, 0, 0,										0, 0)	/* RFC2616 14.2 */
-HTTPH("Accept-Encoding",	H_Accept_Encoding,	1, 0, 0,										0, 0)	/* RFC2616 14.3 */
-HTTPH("Accept-Language",	H_Accept_Language,	1, 0, 0,										0, 0)	/* RFC2616 14.4 */
-HTTPH("Accept-Ranges",		H_Accept_Ranges,	2, 3,				    HTTPH_R_FETCH | HTTPH_A_INS,			0, 0)	/* RFC2616 14.5 */
-HTTPH("Age",			H_Age,			2, 0,						    HTTPH_A_INS,			0, 0)	/* RFC2616 14.6 */
-HTTPH("Allow",			H_Allow,		2, 0, 0,										0, 0)	/* RFC2616 14.7 */
-HTTPH("Authorization",		H_Authorization,	1, 0, 0,										0, 0)	/* RFC2616 14.8 */
-HTTPH("Cache-Control",		H_Cache_Control,	3, 3,				    HTTPH_R_FETCH,					0, 0)	/* RFC2616 14.9 */
-HTTPH("Connection",		H_Connection,		3, 3, HTTPH_R_PASS | HTTPH_A_PASS | HTTPH_R_FETCH | HTTPH_A_INS,			0, 0)	/* RFC2616 14.10 */
-HTTPH("Content-Encoding",	H_Content_Encoding,	2, 0, 0,										0, 0)	/* RFC2616 14.11 */
-HTTPH("Content-Langugae",	H_Content_Language,	2, 0, 0,										0, 0)	/* RFC2616 14.12 */
-HTTPH("Content-Length",		H_Content_Length,	2, 2,				    HTTPH_R_FETCH | HTTPH_A_INS,			0, 0)	/* RFC2616 14.13 */
-HTTPH("Content-Location",	H_Content_Location,	2, 0, 0,										0, 0)	/* RFC2616 14.14 */
-HTTPH("Content-MD5",		H_Content_MD5,		2, 0, 0,										0, 0)	/* RFC2616 14.15 */
-HTTPH("Content-Range",		H_Content_Range,	2, 3,				    HTTPH_R_FETCH | HTTPH_A_INS,			0, 0)	/* RFC2616 14.16 */
-HTTPH("Content-Type",		H_Content_Type,		2, 0, 0,										0, 0)	/* RFC2616 14.17 */
-HTTPH("Date",			H_Date,			2, 0,								HTTPH_A_DELIVER,	0, 0)	/* RFC2616 14.18 */
-HTTPH("ETag",			H_ETag,			2, 0, 0,										0, 0)	/* RFC2616 14.19 */
-HTTPH("Expect",			H_Expect,		1, 0, 0,										0, 0)	/* RFC2616 14.20 */
-HTTPH("Expires",		H_Expires,		2, 0, 0,										0, 0)	/* RFC2616 14.21 */
-HTTPH("From",			H_From,			1, 0, 0,										0, 0)	/* RFC2616 14.22 */
-HTTPH("Host",			H_Host,			1, 0, 0,										0, 0)	/* RFC2616 14.23 */
-HTTPH("If-Match",		H_If_Match,		1, 1,				    HTTPH_R_FETCH,					0, 0)	/* RFC2616 14.24 */
-HTTPH("If-Modified-Since",	H_If_Modified_Since,	1, 1,				    HTTPH_R_FETCH,					0, 0)	/* RFC2616 14.25 */
-HTTPH("If-None-Match",		H_If_None_Match,	1, 1,				    HTTPH_R_FETCH,					0, 0)	/* RFC2616 14.26 */
-HTTPH("If-Range",		H_If_Range,		1, 1,				    HTTPH_R_FETCH,					0, 0)	/* RFC2616 14.27 */
-HTTPH("If-Unmodified-Since",	H_If_Unmodifed_Since,	1, 1,				    HTTPH_R_FETCH,					0, 0)	/* RFC2616 14.28 */
-HTTPH("Last-Modified",		H_Last_Modified,	2, 0, 0,										0, 0)	/* RFC2616 14.29 */
-HTTPH("Location",		H_Location,		2, 0, 0,										0, 0)	/* RFC2616 14.30 */
-HTTPH("Max-Forwards",		H_Max_Forwards,		1, 0, 0,										0, 0)	/* RFC2616 14.31 */
-HTTPH("Pragma",			H_Pragma,		1, 0, 0,										0, 0)	/* RFC2616 14.32 */
-HTTPH("Proxy-Authenticate",	H_Proxy_Authenticate,	2, 3,				    HTTPH_R_FETCH | HTTPH_A_INS,			0, 0)	/* RFC2616 14.33 */
-HTTPH("Proxy-Authorization",	H_Proxy_Authorization,	1, 3,				    HTTPH_R_FETCH | HTTPH_A_INS,			0, 0)	/* RFC2616 14.34 */
-HTTPH("Range",			H_Range,		1, 0,				    HTTPH_R_FETCH | HTTPH_A_INS,			0, 0)	/* RFC2616 14.35 */
-HTTPH("Referer",		H_Referer,		1, 0, 0,										0, 0)	/* RFC2616 14.36 */
-HTTPH("Retry-After",		H_Retry_After,		2, 0, 0,										0, 0)	/* RFC2616 14.37 */
-HTTPH("Server",			H_Server,		2, 0, 0,										0, 0)	/* RFC2616 14.38 */
-HTTPH("TE",			H_TE,			1, 3, HTTPH_R_PASS | HTTPH_A_PASS | HTTPH_R_FETCH | HTTPH_A_INS,			0, 0)	/* RFC2616 14.39 */
-HTTPH("Trailer",		H_Trailer,		1, 3, HTTPH_R_PASS | HTTPH_A_PASS | HTTPH_R_FETCH | HTTPH_A_INS,			0, 0)	/* RFC2616 14.40 */
-HTTPH("Transfer-Encoding",	H_Transfer_Encoding,	2, 3, HTTPH_R_PASS | HTTPH_A_PASS | HTTPH_R_FETCH | HTTPH_A_INS,			0, 0)	/* RFC2616 14.41 */
-HTTPH("Upgrade",		H_Upgrade,		2, 3, HTTPH_R_PASS | HTTPH_A_PASS | HTTPH_R_FETCH | HTTPH_A_INS,			0, 0)	/* RFC2616 14.42 */
-HTTPH("User-Agent",		H_User_Agent,		1, 0, 0,										0, 0)	/* RFC2616 14.43 */
-HTTPH("Vary",			H_Vary,			2, 0, 0,										0, 0)	/* RFC2616 14.44 */
-HTTPH("Via",			H_Via,			2, 0, 0,										0, 0)	/* RFC2616 14.45 */
-HTTPH("Warning",		H_Warning,		2, 0, 0,										0, 0)	/* RFC2616 14.46 */
-HTTPH("WWW-Authenticate",	H_WWW_Authenticate,	2, 0, 0,										0, 0)	/* RFC2616 14.47 */
+HTTPH("Keep-Alive",		H_Keep_Alive,		HTTPH_R_PASS | HTTPH_R_FETCH		  )	/* RFC2068 */
+HTTPH("Accept",			H_Accept,		0					  )	/* RFC2616 14.1 */
+HTTPH("Accept-Charset",		H_Accept_Charset,	0					  )	/* RFC2616 14.2 */
+HTTPH("Accept-Encoding",	H_Accept_Encoding,	0					  )	/* RFC2616 14.3 */
+HTTPH("Accept-Language",	H_Accept_Language,	0					  )	/* RFC2616 14.4 */
+HTTPH("Accept-Ranges",		H_Accept_Ranges,		       HTTPH_R_FETCH | HTTPH_A_INS)	/* RFC2616 14.5 */
+HTTPH("Age",			H_Age,						       HTTPH_A_INS)	/* RFC2616 14.6 */
+HTTPH("Allow",			H_Allow,		0					  )	/* RFC2616 14.7 */
+HTTPH("Authorization",		H_Authorization,	0					  )	/* RFC2616 14.8 */
+HTTPH("Cache-Control",		H_Cache_Control,		       HTTPH_R_FETCH		  )	/* RFC2616 14.9 */
+HTTPH("Connection",		H_Connection,		HTTPH_R_PASS | HTTPH_R_FETCH | HTTPH_A_INS)	/* RFC2616 14.10 */
+HTTPH("Content-Encoding",	H_Content_Encoding,	0					  )	/* RFC2616 14.11 */
+HTTPH("Content-Langugae",	H_Content_Language,	0					  )	/* RFC2616 14.12 */
+HTTPH("Content-Length",		H_Content_Length,		       HTTPH_R_FETCH | HTTPH_A_INS)	/* RFC2616 14.13 */
+HTTPH("Content-Location",	H_Content_Location,	0					  )	/* RFC2616 14.14 */
+HTTPH("Content-MD5",		H_Content_MD5,		0					  )	/* RFC2616 14.15 */
+HTTPH("Content-Range",		H_Content_Range,		       HTTPH_R_FETCH | HTTPH_A_INS)	/* RFC2616 14.16 */
+HTTPH("Content-Type",		H_Content_Type,		0					  )	/* RFC2616 14.17 */
+HTTPH("Date",			H_Date,			0					  )	/* RFC2616 14.18 */
+HTTPH("ETag",			H_ETag,			0					  )	/* RFC2616 14.19 */
+HTTPH("Expect",			H_Expect,		0					  )	/* RFC2616 14.20 */
+HTTPH("Expires",		H_Expires,		0					  )	/* RFC2616 14.21 */
+HTTPH("From",			H_From,			0					  )	/* RFC2616 14.22 */
+HTTPH("Host",			H_Host,			0					  )	/* RFC2616 14.23 */
+HTTPH("If-Match",		H_If_Match,			       HTTPH_R_FETCH		  )	/* RFC2616 14.24 */
+HTTPH("If-Modified-Since",	H_If_Modified_Since,		       HTTPH_R_FETCH		  )	/* RFC2616 14.25 */
+HTTPH("If-None-Match",		H_If_None_Match,		       HTTPH_R_FETCH		  )	/* RFC2616 14.26 */
+HTTPH("If-Range",		H_If_Range,			       HTTPH_R_FETCH		  )	/* RFC2616 14.27 */
+HTTPH("If-Unmodified-Since",	H_If_Unmodifed_Since,		       HTTPH_R_FETCH		  )	/* RFC2616 14.28 */
+HTTPH("Last-Modified",		H_Last_Modified,	0					  )	/* RFC2616 14.29 */
+HTTPH("Location",		H_Location,		0					  )	/* RFC2616 14.30 */
+HTTPH("Max-Forwards",		H_Max_Forwards,		0					  )	/* RFC2616 14.31 */
+HTTPH("Pragma",			H_Pragma,		0					  )	/* RFC2616 14.32 */
+HTTPH("Proxy-Authenticate",	H_Proxy_Authenticate,		       HTTPH_R_FETCH | HTTPH_A_INS)	/* RFC2616 14.33 */
+HTTPH("Proxy-Authorization",	H_Proxy_Authorization,		       HTTPH_R_FETCH | HTTPH_A_INS)	/* RFC2616 14.34 */
+HTTPH("Range",			H_Range,			       HTTPH_R_FETCH | HTTPH_A_INS)	/* RFC2616 14.35 */
+HTTPH("Referer",		H_Referer,		0					  )	/* RFC2616 14.36 */
+HTTPH("Retry-After",		H_Retry_After,		0					  )	/* RFC2616 14.37 */
+HTTPH("Server",			H_Server,		0					  )	/* RFC2616 14.38 */
+HTTPH("TE",			H_TE,			HTTPH_R_PASS | HTTPH_R_FETCH | HTTPH_A_INS)	/* RFC2616 14.39 */
+HTTPH("Trailer",		H_Trailer,		HTTPH_R_PASS | HTTPH_R_FETCH | HTTPH_A_INS)	/* RFC2616 14.40 */
+HTTPH("Transfer-Encoding",	H_Transfer_Encoding,	HTTPH_R_PASS | HTTPH_R_FETCH | HTTPH_A_INS)	/* RFC2616 14.41 */
+HTTPH("Upgrade",		H_Upgrade,		HTTPH_R_PASS | HTTPH_R_FETCH | HTTPH_A_INS)	/* RFC2616 14.42 */
+HTTPH("User-Agent",		H_User_Agent,		0					  )	/* RFC2616 14.43 */
+HTTPH("Vary",			H_Vary,			0					  )	/* RFC2616 14.44 */
+HTTPH("Via",			H_Via,			0					  )	/* RFC2616 14.45 */
+HTTPH("Warning",		H_Warning,		0					  )	/* RFC2616 14.46 */
+HTTPH("WWW-Authenticate",	H_WWW_Authenticate,	0					  )	/* RFC2616 14.47 */
 
 /*lint -restore */



From geoff at varnish-cache.org  Tue Jan 17 13:21:28 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Tue, 17 Jan 2012 14:21:28 +0100
Subject: [experimental-ims] dd103d3 Cleanup http filtering operations:
Message-ID: <E1Rn8ye-0003As-2D@project.varnish-software.com>

commit dd103d37091b9951f7705b240913ce638ded640c
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Jan 17 10:15:14 2012 +0000

    Cleanup http filtering operations:
    
    Rename http_FilterHeader() to http_FilterReq()
    Add http_FilterResp()
    make http_FilterFields() static
    remove http_CopyResp()

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index 8891f12..817b7c7 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -797,12 +797,11 @@ void HTTP_Init(void);
 void http_ClrHeader(struct http *to);
 unsigned http_Write(struct worker *w, unsigned vsl_id, const struct http *hp,
     int resp);
-void http_CopyResp(struct http *to, const struct http *fm);
 void http_SetResp(struct http *to, const char *proto, uint16_t status,
     const char *response);
-void http_FilterFields(struct worker *w, unsigned vsl_id, struct http *to,
-    const struct http *fm, unsigned how);
-void http_FilterHeader(const struct sess *sp, unsigned how);
+void http_FilterReq(const struct sess *sp, unsigned how);
+void http_FilterResp(const struct sess *sp, const struct http *fm, struct http *to,
+    unsigned how);
 void http_PutProtocol(struct worker *w, unsigned vsl_id, const struct http *to,
     const char *protocol);
 void http_PutStatus(struct http *to, uint16_t status);
diff --git a/bin/varnishd/cache/cache_center.c b/bin/varnishd/cache/cache_center.c
index df00fbc..a9c233b 100644
--- a/bin/varnishd/cache/cache_center.c
+++ b/bin/varnishd/cache/cache_center.c
@@ -876,9 +876,7 @@ cnt_fetchbody(struct sess *sp, struct worker *wrk, struct req *req)
 	hp2 = req->obj->http;
 
 	hp2->logtag = HTTP_Obj;
-	http_CopyResp(hp2, hp);
-	http_FilterFields(wrk, sp->vsl_id, hp2, hp,
-	    pass ? HTTPH_R_PASS : HTTPH_A_INS);
+	http_FilterResp(sp, hp, hp2, pass ? HTTPH_R_PASS : HTTPH_A_INS);
 	http_CopyHome(wrk, sp->vsl_id, hp2);
 
 	if (http_GetHdr(hp, H_Last_Modified, &b))
@@ -1268,7 +1266,7 @@ cnt_miss(struct sess *sp, struct worker *wrk, struct req *req)
 	WS_Reset(wrk->ws, NULL);
 	wrk->busyobj = VBO_GetBusyObj(wrk);
 	http_Setup(wrk->busyobj->bereq, wrk->ws);
-	http_FilterHeader(sp, HTTPH_R_FETCH);
+	http_FilterReq(sp, HTTPH_R_FETCH);
 	http_ForceGet(wrk->busyobj->bereq);
 	if (cache_param->http_gzip_support) {
 		/*
@@ -1360,7 +1358,7 @@ cnt_pass(struct sess *sp, struct worker *wrk, struct req *req)
 	WS_Reset(wrk->ws, NULL);
 	wrk->busyobj = VBO_GetBusyObj(wrk);
 	http_Setup(wrk->busyobj->bereq, wrk->ws);
-	http_FilterHeader(sp, HTTPH_R_PASS);
+	http_FilterReq(sp, HTTPH_R_PASS);
 
 	wrk->connect_timeout = 0;
 	wrk->first_byte_timeout = 0;
@@ -1419,7 +1417,7 @@ cnt_pipe(struct sess *sp, struct worker *wrk, const struct req *req)
 	WS_Reset(wrk->ws, NULL);
 	wrk->busyobj = VBO_GetBusyObj(wrk);
 	http_Setup(wrk->busyobj->bereq, wrk->ws);
-	http_FilterHeader(sp, 0);
+	http_FilterReq(sp, 0);
 
 	VCL_pipe_method(sp);
 
diff --git a/bin/varnishd/cache/cache_http.c b/bin/varnishd/cache/cache_http.c
index 27a648f..9fc4e09 100644
--- a/bin/varnishd/cache/cache_http.c
+++ b/bin/varnishd/cache/cache_http.c
@@ -765,17 +765,6 @@ http_ForceGet(const struct http *to)
 }
 
 void
-http_CopyResp(struct http *to, const struct http *fm)
-{
-
-	CHECK_OBJ_NOTNULL(fm, HTTP_MAGIC);
-	CHECK_OBJ_NOTNULL(to, HTTP_MAGIC);
-	http_SetH(to, HTTP_HDR_PROTO, "HTTP/1.1");
-	to->status = fm->status;
-	http_copyh(to, fm, HTTP_HDR_RESPONSE);
-}
-
-void
 http_SetResp(struct http *to, const char *proto, uint16_t status,
     const char *response)
 {
@@ -838,8 +827,8 @@ http_EstimateWS(const struct http *fm, unsigned how, uint16_t *nhd)
 
 /*--------------------------------------------------------------------*/
 
-void
-http_FilterFields(struct worker *w, unsigned vsl_id, struct http *to,
+static void
+http_filterfields(struct worker *w, unsigned vsl_id, struct http *to,
     const struct http *fm, unsigned how)
 {
 	unsigned u;
@@ -865,7 +854,7 @@ http_FilterFields(struct worker *w, unsigned vsl_id, struct http *to,
 /*--------------------------------------------------------------------*/
 
 void
-http_FilterHeader(const struct sess *sp, unsigned how)
+http_FilterReq(const struct sess *sp, unsigned how)
 {
 	struct http *hp;
 
@@ -879,11 +868,26 @@ http_FilterHeader(const struct sess *sp, unsigned how)
 		http_SetH(hp, HTTP_HDR_PROTO, "HTTP/1.1");
 	else
 		http_copyh(hp, sp->req->http, HTTP_HDR_PROTO);
-	http_FilterFields(sp->wrk, sp->vsl_id, hp, sp->req->http, how);
+	http_filterfields(sp->wrk, sp->vsl_id, hp, sp->req->http, how);
 	http_PrintfHeader(sp->wrk, sp->vsl_id, hp,
 	    "X-Varnish: %u", sp->req->xid);
 }
 
+/*--------------------------------------------------------------------*/
+
+void
+http_FilterResp(const struct sess *sp, const struct http *fm, struct http *to,
+    unsigned how)
+{
+
+	CHECK_OBJ_NOTNULL(fm, HTTP_MAGIC);
+	CHECK_OBJ_NOTNULL(to, HTTP_MAGIC);
+	http_SetH(to, HTTP_HDR_PROTO, "HTTP/1.1");
+	to->status = fm->status;
+	http_copyh(to, fm, HTTP_HDR_RESPONSE);
+	http_filterfields(sp->wrk, sp->vsl_id, to, fm, how);
+}
+
 /*--------------------------------------------------------------------
  * This function copies any header fields which reference foreign
  * storage into our own WS.
diff --git a/bin/varnishd/cache/cache_response.c b/bin/varnishd/cache/cache_response.c
index 3a19d4a..5724f07 100644
--- a/bin/varnishd/cache/cache_response.c
+++ b/bin/varnishd/cache/cache_response.c
@@ -117,9 +117,7 @@ RES_BuildHttp(const struct sess *sp)
 
 	http_ClrHeader(req->resp);
 	req->resp->logtag = HTTP_Tx;
-	http_CopyResp(req->resp, req->obj->http);
-	http_FilterFields(sp->wrk, sp->vsl_id, req->resp,
-	    req->obj->http, 0);
+	http_FilterResp(sp, req->obj->http, req->resp, 0);
 
 	if (!(sp->wrk->res_mode & RES_LEN)) {
 		http_Unset(req->resp, H_Content_Length);



From geoff at varnish-cache.org  Tue Jan 17 13:21:28 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Tue, 17 Jan 2012 14:21:28 +0100
Subject: [experimental-ims] 4c8f484 A bit of cleanup of the filtering stuff
Message-ID: <E1Rn8ye-0003B2-4J@project.varnish-software.com>

commit 4c8f484e24c5871dce6f6c4a19c40a6999fb5725
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Jan 17 11:57:11 2012 +0000

    A bit of cleanup of the filtering stuff

diff --git a/bin/varnishd/cache/cache_http.c b/bin/varnishd/cache/cache_http.c
index 9fc4e09..0b0b926 100644
--- a/bin/varnishd/cache/cache_http.c
+++ b/bin/varnishd/cache/cache_http.c
@@ -748,7 +748,7 @@ http_SetH(const struct http *to, unsigned n, const char *fm)
 }
 
 static void
-http_copyh(const struct http *to, const struct http *fm, unsigned n)
+http_linkh(const struct http *to, const struct http *fm, unsigned n)
 {
 
 	assert(n < HTTP_HDR_FIRST);
@@ -776,25 +776,6 @@ http_SetResp(struct http *to, const char *proto, uint16_t status,
 	http_SetH(to, HTTP_HDR_RESPONSE, response);
 }
 
-static void
-http_copyheader(struct worker *w, unsigned vsl_id, struct http *to,
-    const struct http *fm, unsigned n)
-{
-
-	CHECK_OBJ_NOTNULL(fm, HTTP_MAGIC);
-	CHECK_OBJ_NOTNULL(to, HTTP_MAGIC);
-	assert(n < fm->shd);
-	Tcheck(fm->hd[n]);
-	if (to->nhd < to->shd) {
-		to->hd[to->nhd] = fm->hd[n];
-		to->hdf[to->nhd] = 0;
-		to->nhd++;
-	} else  {
-		VSC_C_main->losthdr++;
-		WSLR(w, SLT_LostHeader, vsl_id, fm->hd[n]);
-	}
-}
-
 /*--------------------------------------------------------------------
  * Estimate how much workspace we need to Filter this header according
  * to 'how'.
@@ -847,7 +828,15 @@ http_filterfields(struct worker *w, unsigned vsl_id, struct http *to,
 			continue;
 #include "tbl/http_headers.h"
 #undef HTTPH
-		http_copyheader(w, vsl_id, to, fm, u);
+		Tcheck(fm->hd[u]);
+		if (to->nhd < to->shd) {
+			to->hd[to->nhd] = fm->hd[u];
+			to->hdf[to->nhd] = 0;
+			to->nhd++;
+		} else  {
+			VSC_C_main->losthdr++;
+			WSLR(w, SLT_LostHeader, vsl_id, fm->hd[u]);
+		}
 	}
 }
 
@@ -862,12 +851,12 @@ http_FilterReq(const struct sess *sp, unsigned how)
 	CHECK_OBJ_NOTNULL(hp, HTTP_MAGIC);
 	hp->logtag = HTTP_Tx;
 
-	http_copyh(hp, sp->req->http, HTTP_HDR_REQ);
-	http_copyh(hp, sp->req->http, HTTP_HDR_URL);
+	http_linkh(hp, sp->req->http, HTTP_HDR_REQ);
+	http_linkh(hp, sp->req->http, HTTP_HDR_URL);
 	if (how == HTTPH_R_FETCH)
 		http_SetH(hp, HTTP_HDR_PROTO, "HTTP/1.1");
 	else
-		http_copyh(hp, sp->req->http, HTTP_HDR_PROTO);
+		http_linkh(hp, sp->req->http, HTTP_HDR_PROTO);
 	http_filterfields(sp->wrk, sp->vsl_id, hp, sp->req->http, how);
 	http_PrintfHeader(sp->wrk, sp->vsl_id, hp,
 	    "X-Varnish: %u", sp->req->xid);
@@ -884,7 +873,7 @@ http_FilterResp(const struct sess *sp, const struct http *fm, struct http *to,
 	CHECK_OBJ_NOTNULL(to, HTTP_MAGIC);
 	http_SetH(to, HTTP_HDR_PROTO, "HTTP/1.1");
 	to->status = fm->status;
-	http_copyh(to, fm, HTTP_HDR_RESPONSE);
+	http_linkh(to, fm, HTTP_HDR_RESPONSE);
 	http_filterfields(sp->wrk, sp->vsl_id, to, fm, how);
 }
 



From geoff at varnish-cache.org  Tue Jan 17 13:21:28 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Tue, 17 Jan 2012 14:21:28 +0100
Subject: [experimental-ims] 5c39139 Merge master -> experimental-ims Retaining
	http_copyheader() in cache_http.c (factored out of master)
	because IMS uses it to copy from stale_obj to req->obj
Message-ID: <E1Rn8ye-0003BE-7E@project.varnish-software.com>

commit 5c39139c44eb4243aad50fc72701ca612523bf5e
Merge: b05baa6 4c8f484
Author: Geoff Simmons <geoff at uplex.de>
Date:   Tue Jan 17 14:18:37 2012 +0100

    Merge master -> experimental-ims
    Retaining http_copyheader() in cache_http.c (factored out of master)
    because IMS uses it to copy from stale_obj to req->obj

diff --cc bin/varnishd/cache/cache.h
index 833828d,817b7c7..4917a38
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@@ -800,18 -797,11 +800,17 @@@ void HTTP_Init(void)
  void http_ClrHeader(struct http *to);
  unsigned http_Write(struct worker *w, unsigned vsl_id, const struct http *hp,
      int resp);
- void http_CopyResp(struct http *to, const struct http *fm);
  void http_SetResp(struct http *to, const char *proto, uint16_t status,
      const char *response);
- void http_FilterFields(struct worker *w, unsigned vsl_id, struct http *to,
-     const struct http *fm, unsigned how);
- void http_FilterHeader(const struct sess *sp, unsigned how);
+ void http_FilterReq(const struct sess *sp, unsigned how);
+ void http_FilterResp(const struct sess *sp, const struct http *fm, struct http *to,
+     unsigned how);
 +
 +/* Check if a refresh should be done */
 +void http_CheckRefresh(struct sess *sp);
 +/* Check if we got 304 response */
 +void http_Check304(struct sess *sp, struct busyobj *busyobj);
 +
  void http_PutProtocol(struct worker *w, unsigned vsl_id, const struct http *to,
      const char *protocol);
  void http_PutStatus(struct http *to, uint16_t status);
diff --cc bin/varnishd/cache/cache_center.c
index 3b2e613,a9c233b..c16c42e
--- a/bin/varnishd/cache/cache_center.c
+++ b/bin/varnishd/cache/cache_center.c
@@@ -900,15 -876,10 +900,11 @@@ cnt_fetchbody(struct sess *sp, struct w
  	hp2 = req->obj->http;
  
  	hp2->logtag = HTTP_Obj;
- 	http_CopyResp(hp2, hp);
-         
- 	http_FilterFields(wrk, sp->vsl_id, hp2, hp,
- 	    pass ? HTTPH_R_PASS : HTTPH_A_INS);
- 
+ 	http_FilterResp(sp, hp, hp2, pass ? HTTPH_R_PASS : HTTPH_A_INS);
  	http_CopyHome(wrk, sp->vsl_id, hp2);
  
 -	if (http_GetHdr(hp, H_Last_Modified, &b))
 +	if (http_GetHdr(hp, H_Last_Modified, &b)
 +            || http_GetHdr(req->obj->http, H_Last_Modified, &b))
  		req->obj->last_modified = VTIM_parse(b);
  	else
  		req->obj->last_modified = floor(wrk->busyobj->exp.entered);
diff --cc bin/varnishd/cache/cache_http.c
index 7ea85f2,0b0b926..ceef821
--- a/bin/varnishd/cache/cache_http.c
+++ b/bin/varnishd/cache/cache_http.c
@@@ -917,83 -862,21 +906,100 @@@ http_FilterReq(const struct sess *sp, u
  	    "X-Varnish: %u", sp->req->xid);
  }
  
 +/*-------------------------------------------------------------------
 + * This function checks for sp->freshen_obj.  If present, HSH_Lookup()
 + * found an expired object that qualifies for a refresh check,
 + * so add the appropriate headers.
 + */
 +
 +void
 +http_CheckRefresh(struct sess *sp)
 +{
 +	struct object *freshen_obj;
 +	struct http *obj_hp, *bereq_hp;
 +	char *p;
 +
 +	freshen_obj = sp->stale_obj;
 +	CHECK_OBJ_NOTNULL(freshen_obj, OBJECT_MAGIC);
 +	bereq_hp = sp->wrk->busyobj->bereq;
 +	CHECK_OBJ_NOTNULL(bereq_hp, HTTP_MAGIC);
 +	obj_hp = freshen_obj->http;
 +	CHECK_OBJ_NOTNULL(obj_hp, HTTP_MAGIC);
 +
 +	if(http_GetHdr(obj_hp, H_ETag, &p))
- 		http_PrintfHeader(sp->wrk, sp->fd, bereq_hp, "If-None-Match: %s", p);
++		http_PrintfHeader(sp->wrk, sp->fd, bereq_hp,
++		    "If-None-Match: %s", p);
 +
 +	if(http_GetHdr(obj_hp, H_Last_Modified, &p))
- 		http_PrintfHeader(sp->wrk, sp->fd, bereq_hp, "If-Modified-Since: %s",p);
++		http_PrintfHeader(sp->wrk, sp->fd, bereq_hp,
++		    "If-Modified-Since: %s", p);
 +}
 +
 +/*-------------------------------------------------------------------
 + * Called after fetch and sp->freshen_obj present.  Check
 + * response and handle as needed.
 + */
 +
 +void
 +http_Check304(struct sess *sp, struct busyobj *busyobj)
 +{
 +	struct object *o_stale;
 +	char *p;
 +
 +	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 +	CHECK_OBJ_NOTNULL(busyobj, BUSYOBJ_MAGIC);
 +	o_stale = sp->stale_obj;
 +	CHECK_OBJ_NOTNULL(o_stale, OBJECT_MAGIC);
 +
 +	if (busyobj->beresp->status != 304) {
 +	    /*
 +	     * IMS/INM headers may have been removed in VCL, so only count a
 +	     * non-validating response if they were present in the request.
 +	     */
 +	    if (http_GetHdr(busyobj->bereq, H_If_Modified_Since, &p)
 +		|| http_GetHdr(busyobj->bereq, H_If_None_Match, &p))
 +		sp->wrk->stats.fetch_not_validated++;
 +
 +	    HSH_Deref(sp->wrk, NULL, &sp->stale_obj);
 +	    AZ(sp->stale_obj);
 +	    return;
 +	}
 +
 +	/* 
 +	 * Copy headers we need from the stale object into the 304 response
 +	 */
 +	http_FilterMissingFields(sp->wrk, sp->fd, busyobj->beresp,
 +	    o_stale->http);
 +
 +	http_SetResp(busyobj->beresp, "HTTP/1.1", 200, "Ok Not Modified");
 +	http_SetH(busyobj->beresp, HTTP_HDR_REQ, "GET");
- 	http_copyh(busyobj->beresp, busyobj->bereq, HTTP_HDR_URL);
++	http_linkh(busyobj->beresp, busyobj->bereq, HTTP_HDR_URL);
 +
 +	/*
 +	 * XXX: Are we copying all the necessary fields from stale_obj?
 +	 *	Should we copy o_stale->hits into o->hits?
 +	 *      What about do_esi and do_g(un)zip?
 +	 */
 +	busyobj->is_gzip = o_stale->gziped;
 +
 +        AZ(o_stale->objcore->flags & OC_F_BUSY);
 +}
 +
+ /*--------------------------------------------------------------------*/
+ 
+ void
+ http_FilterResp(const struct sess *sp, const struct http *fm, struct http *to,
+     unsigned how)
+ {
+ 
+ 	CHECK_OBJ_NOTNULL(fm, HTTP_MAGIC);
+ 	CHECK_OBJ_NOTNULL(to, HTTP_MAGIC);
+ 	http_SetH(to, HTTP_HDR_PROTO, "HTTP/1.1");
+ 	to->status = fm->status;
+ 	http_linkh(to, fm, HTTP_HDR_RESPONSE);
+ 	http_filterfields(sp->wrk, sp->vsl_id, to, fm, how);
+ }
+ 
  /*--------------------------------------------------------------------
   * This function copies any header fields which reference foreign
   * storage into our own WS.



From perbu at varnish-cache.org  Thu Jan 19 09:56:12 2012
From: perbu at varnish-cache.org (Per Buer)
Date: Thu, 19 Jan 2012 10:56:12 +0100
Subject: [master] c1635fa remove the experimental flag from http_range_support
Message-ID: <E1Rnoj6-0003Kc-1o@project.varnish-software.com>

commit c1635fa22d084267122508e99b887f4da7213a41
Author: Per Buer <perbu at varnish-software.com>
Date:   Thu Jan 19 10:54:34 2012 +0100

    remove the experimental flag from http_range_support

diff --git a/bin/varnishd/mgt/mgt_param.c b/bin/varnishd/mgt/mgt_param.c
index 8123bfc..c984aae 100644
--- a/bin/varnishd/mgt/mgt_param.c
+++ b/bin/varnishd/mgt/mgt_param.c
@@ -1068,7 +1068,7 @@ static const struct parspec input_parspec[] = {
 		"10", "objects" },
 	{ "http_range_support", tweak_bool, &mgt_param.http_range_support, 0, 0,
 		"Enable support for HTTP Range headers.\n",
-		EXPERIMENTAL,
+		0,
 		"on", "bool" },
 	{ "http_gzip_support", tweak_bool, &mgt_param.http_gzip_support, 0, 0,
 		"Enable gzip support. When enabled Varnish will compress "



From perbu at varnish-cache.org  Thu Jan 19 09:56:12 2012
From: perbu at varnish-cache.org (Per Buer)
Date: Thu, 19 Jan 2012 10:56:12 +0100
Subject: [master] bbecdd2 speling
Message-ID: <E1Rnoj6-0003Kh-47@project.varnish-software.com>

commit bbecdd22ea3c4f866b951a716acaa8e1dad575b8
Author: Per Buer <perbu at varnish-software.com>
Date:   Thu Jan 19 10:55:49 2012 +0100

    speling

diff --git a/doc/sphinx/tutorial/index.rst b/doc/sphinx/tutorial/index.rst
index 0bf3137..c3a05f5 100644
--- a/doc/sphinx/tutorial/index.rst
+++ b/doc/sphinx/tutorial/index.rst
@@ -14,7 +14,7 @@ separate topic. Good luck.
 
 .. toctree:: :maxdepth: 1
 
-        indroduction
+        introduction
 	backend_servers
         starting_varnish
 	logging



From geoff at varnish-cache.org  Thu Jan 19 11:26:22 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Thu, 19 Jan 2012 12:26:22 +0100
Subject: [experimental-ims] c1635fa remove the experimental flag from
	http_range_support
Message-ID: <E1Rnq8M-0002lT-2w@project.varnish-software.com>

commit c1635fa22d084267122508e99b887f4da7213a41
Author: Per Buer <perbu at varnish-software.com>
Date:   Thu Jan 19 10:54:34 2012 +0100

    remove the experimental flag from http_range_support

diff --git a/bin/varnishd/mgt/mgt_param.c b/bin/varnishd/mgt/mgt_param.c
index 8123bfc..c984aae 100644
--- a/bin/varnishd/mgt/mgt_param.c
+++ b/bin/varnishd/mgt/mgt_param.c
@@ -1068,7 +1068,7 @@ static const struct parspec input_parspec[] = {
 		"10", "objects" },
 	{ "http_range_support", tweak_bool, &mgt_param.http_range_support, 0, 0,
 		"Enable support for HTTP Range headers.\n",
-		EXPERIMENTAL,
+		0,
 		"on", "bool" },
 	{ "http_gzip_support", tweak_bool, &mgt_param.http_gzip_support, 0, 0,
 		"Enable gzip support. When enabled Varnish will compress "



From geoff at varnish-cache.org  Thu Jan 19 11:26:22 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Thu, 19 Jan 2012 12:26:22 +0100
Subject: [experimental-ims] bbecdd2 speling
Message-ID: <E1Rnq8M-0002lY-4H@project.varnish-software.com>

commit bbecdd22ea3c4f866b951a716acaa8e1dad575b8
Author: Per Buer <perbu at varnish-software.com>
Date:   Thu Jan 19 10:55:49 2012 +0100

    speling

diff --git a/doc/sphinx/tutorial/index.rst b/doc/sphinx/tutorial/index.rst
index 0bf3137..c3a05f5 100644
--- a/doc/sphinx/tutorial/index.rst
+++ b/doc/sphinx/tutorial/index.rst
@@ -14,7 +14,7 @@ separate topic. Good luck.
 
 .. toctree:: :maxdepth: 1
 
-        indroduction
+        introduction
 	backend_servers
         starting_varnish
 	logging



From geoff at varnish-cache.org  Thu Jan 19 11:26:22 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Thu, 19 Jan 2012 12:26:22 +0100
Subject: [experimental-ims] eb7cbda Merge branch 'master' of
	ssh://git.varnish-cache.org/git/varnish-cache into experimental-ims
Message-ID: <E1Rnq8M-0002le-5d@project.varnish-software.com>

commit eb7cbdac2056a4d8d901fa209b008b601a40764d
Merge: 5c39139 bbecdd2
Author: Geoff Simmons <geoff at uplex.de>
Date:   Thu Jan 19 12:20:14 2012 +0100

    Merge branch 'master' of ssh://git.varnish-cache.org/git/varnish-cache into experimental-ims



From tfheen at varnish-cache.org  Thu Jan 19 11:32:45 2012
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Thu, 19 Jan 2012 12:32:45 +0100
Subject: [master] 299d2da Force lookup of kss resources and fix cached
	creation of objects in Plone
Message-ID: <E1RnqEX-0004Oy-Eu@project.varnish-software.com>

commit 299d2da0d94cae74efce9f46dd5f5d0d654fe348
Author: Cleber J Santos <cleber at cleberjsantos.com.br>
Date:   Wed Jan 18 11:46:03 2012 -0200

    Force lookup of kss resources and fix cached creation of objects in Plone

diff --git a/etc/zope-plone.vcl b/etc/zope-plone.vcl
index b69f583..263e343 100644
--- a/etc/zope-plone.vcl
+++ b/etc/zope-plone.vcl
@@ -52,13 +52,18 @@ sub vcl_recv {
                         }
                         return(lookup);
                 }
+
+                # Do not cache the creation of objects in Plone
+                if (req.url ~ "createObject"){
+                        return(pass);
+                }
         }
 
         # Don't cache authenticated requests
         if (req.http.Cookie && req.http.Cookie ~ "__ac(|_(name|password|persistent))=") {
 
 		# Force lookup of specific urls unlikely to need protection
-		if (req.url ~ "\.(js|css)") {
+		if (req.url ~ "\.(js|css|kss)") {
                         remove req.http.cookie;
                         return(lookup);
                 }



From drwilco at varnish-cache.org  Thu Jan 19 12:58:35 2012
From: drwilco at varnish-cache.org (Rogier Mulhuijzen)
Date: Thu, 19 Jan 2012 13:58:35 +0100
Subject: [master] bf913ad Don't forget to do the stats when a request is
	aborted
Message-ID: <E1RnrZb-00087Z-02@project.varnish-software.com>

commit bf913ad2f50f12ec99696816ac7048b69c21d0cb
Author: Rogier 'DocWilco' Mulhuijzen <github at bsdchicks.com>
Date:   Wed Jan 18 18:16:35 2012 +0100

    Don't forget to do the stats when a request is aborted
    
    Without doing SES_Charge, the worker will panic at the end of
    CNT_Session() when a request is aborted before it is complete.
    
    Found by: Federico G. Schwindt <fgsch at lodoss.net>

diff --git a/bin/varnishd/cache/cache_center.c b/bin/varnishd/cache/cache_center.c
index a9c233b..b03ba4b 100644
--- a/bin/varnishd/cache/cache_center.c
+++ b/bin/varnishd/cache/cache_center.c
@@ -139,10 +139,12 @@ cnt_wait(struct sess *sp, struct worker *wrk, struct req *req)
 			break;
 		}
 		if (i == -1) {
+			SES_Charge(sp);
 			SES_Delete(sp, "EOF", now);
 			return (1);
 		}
 		if (i == -2) {
+			SES_Charge(sp);
 			SES_Delete(sp, "overflow", now);
 			return (1);
 		}
@@ -150,6 +152,7 @@ cnt_wait(struct sess *sp, struct worker *wrk, struct req *req)
 			/* Nothing but whitespace */
 			when = sp->t_idle + cache_param->timeout_idle;
 			if (when < now) {
+				SES_Charge(sp);
 				SES_Delete(sp, "timeout", now);
 				return (1);
 			}
@@ -168,6 +171,7 @@ cnt_wait(struct sess *sp, struct worker *wrk, struct req *req)
 			when = sp->t_req + cache_param->timeout_req;
 			tmo = (int)(1e3 * (when - now));
 			if (when < now || tmo == 0) {
+				SES_Charge(sp);
 				SES_Delete(sp, "req timeout", now);
 				return (1);
 			}



From martin at varnish-cache.org  Thu Jan 19 17:40:08 2012
From: martin at varnish-cache.org (Martin Blix Grydeland)
Date: Thu, 19 Jan 2012 18:40:08 +0100
Subject: [master] b499af9 Stop accept() timeouts from being counted as
	sess_fail in varnishstat
Message-ID: <E1Rnvy4-0007vd-JJ@project.varnish-software.com>

commit b499af9caad1a1bb1db5ed29c547573cc81fcd66
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Thu Jan 19 16:04:12 2012 +0100

    Stop accept() timeouts from being counted as sess_fail in varnishstat
    
    Varnish sets SO_RCVTIMEO to cache_param->timeout_idle on the listen
    sockets, so that the value is inherited to the accept()ed
    connections. This has the side effect of making accept() returning
    with EAGAIN after cache_param->timeout_idle seconds when there are no
    connections coming in, and each of these timeouts is counted as a
    sess_failed. Also the pacing sleep time is increased for each of these
    timeouts, causing Varnish to react slowly at first when connections
    start coming in.
    
    Patch changes to loop around the call to accept() while EAGAIN

diff --git a/bin/varnishd/cache/cache_acceptor.c b/bin/varnishd/cache/cache_acceptor.c
index d55545b..7918a6a 100644
--- a/bin/varnishd/cache/cache_acceptor.c
+++ b/bin/varnishd/cache/cache_acceptor.c
@@ -186,7 +186,10 @@ VCA_Accept(struct listen_sock *ls, struct wrk_accept *wa)
 		(void)usleep(100*1000);
 
 	wa->acceptaddrlen = sizeof wa->acceptaddr;
-	i = accept(ls->sock, (void*)&wa->acceptaddr, &wa->acceptaddrlen);
+	do {
+		i = accept(ls->sock, (void*)&wa->acceptaddr,
+			   &wa->acceptaddrlen);
+	} while (i < 0 && errno == EAGAIN);
 
 	if (i < 0) {
 		switch (errno) {



From apj at varnish-cache.org  Thu Jan 19 18:14:31 2012
From: apj at varnish-cache.org (Andreas Plesner Jacobsen)
Date: Thu, 19 Jan 2012 19:14:31 +0100
Subject: [master] cf0a45a Remove last traces of sess_workspace from logs
Message-ID: <E1RnwVL-00078L-Nn@project.varnish-software.com>

commit cf0a45afedf1a5ca696119ed2b1a1e99b1fb1a08
Author: Andreas Plesner Jacobsen <apj at mutt.dk>
Date:   Thu Jan 19 19:12:43 2012 +0100

    Remove last traces of sess_workspace from logs
    
    Found by: Federico G. Schwindt <fgsch at lodoss.net>

diff --git a/doc/sphinx/reference/params.rst b/doc/sphinx/reference/params.rst
index f161596..1f9a486 100644
--- a/doc/sphinx/reference/params.rst
+++ b/doc/sphinx/reference/params.rst
@@ -250,7 +250,7 @@ http_req_size
 	- Default: 32768
 
 	Maximum number of bytes of HTTP client request we will deal with.  This is a limit on all bytes up to the double blank line which ends the HTTP request.
-	The memory for the request is allocated from the session workspace (param: sess_workspace) and this parameter limits how much of that the request is allowed to take up.
+	The memory for the request is allocated from the client workspace (param: workspace_client) and this parameter limits how much of that the request is allowed to take up.
 
 http_resp_hdr_len
 	- Units: bytes
@@ -263,7 +263,7 @@ http_resp_size
 	- Default: 32768
 
 	Maximum number of bytes of HTTP backend resonse we will deal with.  This is a limit on all bytes up to the double blank line which ends the HTTP request.
-	The memory for the request is allocated from the worker workspace (param: sess_workspace) and this parameter limits how much of that the request is allowed to take up.
+	The memory for the request is allocated from the thread pool workspace (param: thread_pool_workspace) and this parameter limits how much of that the request is allowed to take up.
 
 listen_address
 	- Default: :80
@@ -378,14 +378,6 @@ sess_timeout
 
 	Idle timeout for persistent sessions. If a HTTP request has not been received in this many seconds, the session is closed.
 
-sess_workspace
-	- Units: bytes
-	- Default: 65536
-	- Flags: delayed
-
-	Bytes of HTTP protocol workspace allocated for sessions. This space must be big enough for the entire HTTP protocol header and any edits done to it in the VCL code.
-	Minimum is 1024 bytes.
-
 session_linger
 	- Units: ms
 	- Default: 50
diff --git a/doc/sphinx/reference/varnishd.rst b/doc/sphinx/reference/varnishd.rst
index 4e2b3f2..351c7b3 100644
--- a/doc/sphinx/reference/varnishd.rst
+++ b/doc/sphinx/reference/varnishd.rst
@@ -274,8 +274,10 @@ Here is a list of all parameters, current as of last time we remembered to updat
 produced from the same text you will find in the CLI if you use the param.show command, so should there be a new
 parameter which is not listed here, you can find the description using the CLI commands.
 
-Be aware that on 32 bit systems, certain default values, such as sess_workspace (=16k) and thread_pool_stack
-(=64k) are reduced relative to the values listed here, in order to conserve VM space.
+Be aware that on 32 bit systems, certain default values, such as
+workspace_client (=16k), thread_pool_workspace (=16k), http_resp_size (=8k),
+http_req_size (=12k), gzip_stack_buffer (=4k) and thread_pool_stack (=64k) are
+reduced relative to the values listed here, in order to conserve VM space.
 
 .. include:: params.rst
 
diff --git a/doc/sphinx/reference/vmod.rst b/doc/sphinx/reference/vmod.rst
index 2b515c1..57163c5 100644
--- a/doc/sphinx/reference/vmod.rst
+++ b/doc/sphinx/reference/vmod.rst
@@ -171,7 +171,7 @@ STRING_LIST
 	a function, we may relax that at a latter time.
 
 	If you don't want to bother with STRING_LIST, just use STRING
-	and make sure your sess_workspace param is big enough.
+	and make sure your thread_pool_workspace param is big enough.
 
 PRIV_VCL
 	See below



From phk at varnish-cache.org  Mon Jan 23 09:34:53 2012
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Mon, 23 Jan 2012 10:34:53 +0100
Subject: [master] 29fcad4 stylistic nit
Message-ID: <E1RpGIf-0008V4-Do@project.varnish-software.com>

commit 29fcad40204b0ac09d81139692c8f26ebf3f06cf
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Jan 23 09:34:41 2012 +0000

    stylistic nit

diff --git a/bin/varnishd/cache/cache_pool.c b/bin/varnishd/cache/cache_pool.c
index 84a6cfd..0f7c9e4 100644
--- a/bin/varnishd/cache/cache_pool.c
+++ b/bin/varnishd/cache/cache_pool.c
@@ -218,7 +218,7 @@ Pool_Work_Thread(void *priv, struct worker *wrk)
 			}
 			VTAILQ_INSERT_TAIL(&pp->socks, ps, list);
 			wrk->do_what = pool_do_accept;
-		} else if (VTAILQ_EMPTY(&pp->socks)) {
+		} else {
 			/* Nothing to do: To sleep, perchance to dream ... */
 			if (isnan(wrk->lastused))
 				wrk->lastused = VTIM_real();



From phk at varnish-cache.org  Mon Jan 23 10:07:16 2012
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Mon, 23 Jan 2012 11:07:16 +0100
Subject: [master] edf5144 Add the new pool-task API so Martin can see it.
Message-ID: <E1RpGo0-0003Aq-HJ@project.varnish-software.com>

commit edf5144fb9c87869638f3b3dc56ea6572e0d83cb
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Jan 23 10:06:46 2012 +0000

    Add the new pool-task API so Martin can see it.
    
    Presently unused and untested.

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index 817b7c7..463517c 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -286,6 +286,22 @@ struct wrk_accept {
 	struct listen_sock	*acceptlsock;
 };
 
+/* Worker pool stuff -------------------------------------------------*/
+
+typedef void pool_func_t(struct pool *pp, void *priv);
+
+struct pool_task {
+	VTAILQ_ENTRY(pool_task)		list;
+	pool_func_t			*func;
+	void				*priv;
+};
+
+enum pool_how {
+	POOL_NO_QUEUE,
+	POOL_QUEUE_FRONT,
+	POOL_QUEUE_BACK
+};
+
 /*--------------------------------------------------------------------*/
 
 enum e_do_what {
@@ -307,6 +323,10 @@ struct worker {
 	void			*nhashpriv;
 	struct dstat		stats;
 
+	/* New Pool stuff */
+	pool_func_t		*pool_func;
+	void			*pool_priv;
+
 	/* Pool stuff */
 	enum e_do_what		do_what;
 
@@ -648,6 +668,7 @@ struct sess {
 	struct worker		*wrk;
 	struct req		*req;
 
+	struct pool_task	task;
 	VTAILQ_ENTRY(sess)	list;
 
 	/* Session related fields ------------------------------------*/
@@ -894,6 +915,7 @@ void PipeSession(struct sess *sp);
 void Pool_Init(void);
 void Pool_Work_Thread(void *priv, struct worker *w);
 int Pool_Schedule(struct pool *pp, struct sess *sp);
+int Pool_Task(struct pool *pp, struct pool_task *task, enum pool_how how);
 
 #define WRW_IsReleased(w)	((w)->wrw.wfd == NULL)
 int WRW_Error(const struct worker *w);
diff --git a/bin/varnishd/cache/cache_pool.c b/bin/varnishd/cache/cache_pool.c
index 0f7c9e4..cc81b59 100644
--- a/bin/varnishd/cache/cache_pool.c
+++ b/bin/varnishd/cache/cache_pool.c
@@ -92,24 +92,26 @@ struct poolsock {
 /* Number of work requests queued in excess of worker threads available */
 
 struct pool {
-	unsigned		magic;
-#define POOL_MAGIC		0x606658fa
-	VTAILQ_ENTRY(pool)	list;
-
-	pthread_cond_t		herder_cond;
-	struct lock		herder_mtx;
-	pthread_t		herder_thr;
-
-	struct lock		mtx;
-	struct workerhead	idle;
-	VTAILQ_HEAD(, sess)	queue;
-	VTAILQ_HEAD(, poolsock)	socks;
-	unsigned		nthr;
-	unsigned		lqueue;
-	unsigned		last_lqueue;
-	uintmax_t		ndropped;
-	uintmax_t		nqueued;
-	struct sesspool		*sesspool;
+	unsigned			magic;
+#define POOL_MAGIC			0x606658fa
+	VTAILQ_ENTRY(pool)		list;
+
+	pthread_cond_t			herder_cond;
+	struct lock			herder_mtx;
+	pthread_t			herder_thr;
+
+	struct lock			mtx;
+	struct workerhead		idle;
+	VTAILQ_HEAD(, pool_task)	front_queue;
+	VTAILQ_HEAD(, pool_task)	back_queue;
+	VTAILQ_HEAD(, sess)		queue;
+	VTAILQ_HEAD(, poolsock)		socks;
+	unsigned			nthr;
+	unsigned			lqueue;
+	unsigned			last_lqueue;
+	uintmax_t			ndropped;
+	uintmax_t			nqueued;
+	struct sesspool			*sesspool;
 };
 
 static struct lock		pool_mtx;
@@ -174,6 +176,51 @@ pool_accept(struct pool *pp, struct worker *wrk, const struct poolsock *ps)
 }
 
 /*--------------------------------------------------------------------
+ * Enter a new task to be done
+ */
+
+int
+Pool_Task(struct pool *pp, struct pool_task *task, enum pool_how how)
+{
+	struct worker *wrk;
+	int retval = 0;
+
+	CHECK_OBJ_NOTNULL(pp, POOL_MAGIC);
+
+	Lck_Lock(&pp->mtx);
+
+	/*
+	 * The common case first:  Take an idle thread, do it.
+	 */
+
+	wrk = VTAILQ_FIRST(&pp->idle);
+	if (wrk != NULL) {
+		VTAILQ_REMOVE(&pp->idle, wrk, list);
+		Lck_Unlock(&pp->mtx);
+		wrk->pool_func = task->func;
+		wrk->pool_priv = task->priv;
+		AZ(pthread_cond_signal(&wrk->cond));
+		return (0);
+	}
+
+	switch (how) {
+	case POOL_NO_QUEUE:
+		retval = -1;
+		break;
+	case POOL_QUEUE_FRONT:
+		VTAILQ_INSERT_TAIL(&pp->front_queue, task, list);
+		break;
+	case POOL_QUEUE_BACK:
+		VTAILQ_INSERT_TAIL(&pp->back_queue, task, list);
+		break;
+	default:
+		WRONG("Unknown enum pool_how");
+	}
+	Lck_Unlock(&pp->mtx);
+	return (retval);
+}
+
+/*--------------------------------------------------------------------
  * This is the work function for worker threads in the pool.
  */
 
@@ -183,6 +230,7 @@ Pool_Work_Thread(void *priv, struct worker *wrk)
 	struct pool *pp;
 	int stats_clean, i;
 	struct poolsock *ps;
+	struct pool_task *tp;
 
 	CAST_OBJ_NOTNULL(pp, priv, POOL_MAGIC);
 	wrk->pool = pp;
@@ -197,6 +245,24 @@ Pool_Work_Thread(void *priv, struct worker *wrk)
 
 		WS_Reset(wrk->ws, NULL);
 
+		tp = VTAILQ_FIRST(&pp->front_queue);
+		if (tp != NULL)
+			VTAILQ_REMOVE(&pp->front_queue, tp, list);
+
+		if (tp == NULL) {
+			tp = VTAILQ_FIRST(&pp->back_queue);
+			if (tp != NULL)
+				VTAILQ_REMOVE(&pp->back_queue, tp, list);
+		}
+
+		if (tp != NULL) {
+			Lck_Unlock(&pp->mtx);
+			tp->func(pp, tp->priv);
+			stats_clean = WRK_TrySumStat(wrk);
+			Lck_Lock(&pp->mtx);
+			continue;
+		}
+
 		wrk->sp = VTAILQ_FIRST(&pp->queue);
 		if (wrk->sp != NULL) {
 			/* Process queued requests, if any */
@@ -483,6 +549,8 @@ pool_mkpool(unsigned pool_no)
 	VTAILQ_INIT(&pp->queue);
 	VTAILQ_INIT(&pp->idle);
 	VTAILQ_INIT(&pp->socks);
+	VTAILQ_INIT(&pp->front_queue);
+	VTAILQ_INIT(&pp->back_queue);
 	pp->sesspool = SES_NewPool(pp, pool_no);
 	AN(pp->sesspool);
 



From phk at varnish-cache.org  Mon Jan 23 10:19:03 2012
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Mon, 23 Jan 2012 11:19:03 +0100
Subject: [master] f965f4d Forgot to pass the struct wrk* argument to the task
	function.
Message-ID: <E1RpGzP-0004B0-Le@project.varnish-software.com>

commit f965f4dfe47ebfbd410e784c0c79e9e84455289b
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Jan 23 10:18:50 2012 +0000

    Forgot to pass the struct wrk* argument to the task function.

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index 463517c..5843fa4 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -288,7 +288,7 @@ struct wrk_accept {
 
 /* Worker pool stuff -------------------------------------------------*/
 
-typedef void pool_func_t(struct pool *pp, void *priv);
+typedef void pool_func_t(struct pool *pp, struct worker *wrk, void *priv);
 
 struct pool_task {
 	VTAILQ_ENTRY(pool_task)		list;
diff --git a/bin/varnishd/cache/cache_pool.c b/bin/varnishd/cache/cache_pool.c
index cc81b59..8b247fa 100644
--- a/bin/varnishd/cache/cache_pool.c
+++ b/bin/varnishd/cache/cache_pool.c
@@ -257,7 +257,7 @@ Pool_Work_Thread(void *priv, struct worker *wrk)
 
 		if (tp != NULL) {
 			Lck_Unlock(&pp->mtx);
-			tp->func(pp, tp->priv);
+			tp->func(pp, wrk, tp->priv);
 			stats_clean = WRK_TrySumStat(wrk);
 			Lck_Lock(&pp->mtx);
 			continue;



From phk at varnish-cache.org  Mon Jan 23 10:37:51 2012
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Mon, 23 Jan 2012 11:37:51 +0100
Subject: [master] 7fc7fa7 Move the "schedule a session" code from cache_pool
	to cache_session where it rightfully belongs.
Message-ID: <E1RpHHb-0006Di-QK@project.varnish-software.com>

commit 7fc7fa7d430bb715713db16ca98b8105958bccad
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Jan 23 10:37:18 2012 +0000

    Move the "schedule a session" code from cache_pool to cache_session
    where it rightfully belongs.

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index 5843fa4..2f48427 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -942,6 +942,8 @@ int SES_Schedule(struct sess *sp);
 void SES_Handle(struct sess *sp, double now);
 void SES_GetReq(struct sess *sp);
 void SES_ReleaseReq(struct sess *sp);
+pool_func_t SES_pool_task;
+
 
 /* cache_shmlog.c */
 extern struct VSC_C_main *VSC_C_main;
diff --git a/bin/varnishd/cache/cache_pool.c b/bin/varnishd/cache/cache_pool.c
index 8b247fa..a8a2b2a 100644
--- a/bin/varnishd/cache/cache_pool.c
+++ b/bin/varnishd/cache/cache_pool.c
@@ -186,6 +186,8 @@ Pool_Task(struct pool *pp, struct pool_task *task, enum pool_how how)
 	int retval = 0;
 
 	CHECK_OBJ_NOTNULL(pp, POOL_MAGIC);
+	AN(task);
+	AN(task->func);
 
 	Lck_Lock(&pp->mtx);
 
@@ -257,6 +259,7 @@ Pool_Work_Thread(void *priv, struct worker *wrk)
 
 		if (tp != NULL) {
 			Lck_Unlock(&pp->mtx);
+			AN(tp->func);
 			tp->func(pp, wrk, tp->priv);
 			stats_clean = WRK_TrySumStat(wrk);
 			Lck_Lock(&pp->mtx);
@@ -316,27 +319,8 @@ Pool_Work_Thread(void *priv, struct worker *wrk)
 		}
 
 		if (wrk->do_what == pool_do_sess) {
-			CHECK_OBJ_NOTNULL(wrk->sp, SESS_MAGIC);
-			AZ(wrk->ws->r);
-
 			stats_clean = 0;
-			wrk->lastused = NAN;
-
-			AZ(wrk->sp->wrk);
-			THR_SetSession(wrk->sp);
-			wrk->sp->wrk = wrk;
-			CNT_Session(wrk->sp);
-			THR_SetSession(NULL);
-			wrk->sp = NULL;
-
-			WS_Assert(wrk->ws);
-			AZ(wrk->busyobj);
-			AZ(wrk->wrw.wfd);
-			assert(wrk->wlp == wrk->wlb);
-			if (cache_param->diag_bitmap & 0x00040000) {
-				if (wrk->vcl != NULL)
-					VCL_Rel(&wrk->vcl);
-			}
+			SES_pool_task(pp, wrk, wrk->sp);
 		} else if (wrk->do_what == pool_do_nothing) {
 			/* we already did */
 		} else {
diff --git a/bin/varnishd/cache/cache_session.c b/bin/varnishd/cache/cache_session.c
index d1dbaba..ba5f005 100644
--- a/bin/varnishd/cache/cache_session.c
+++ b/bin/varnishd/cache/cache_session.c
@@ -127,6 +127,42 @@ SES_Alloc(void)
 	return (sp);
 }
 
+
+/*--------------------------------------------------------------------
+ * The pool-task function for sessions
+ */
+
+void
+SES_pool_task(struct pool *pp, struct worker *wrk, void *arg)
+{
+	struct sess *sp;
+
+	AN(pp);
+	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
+	CAST_OBJ_NOTNULL(sp, arg, SESS_MAGIC);
+
+	AZ(wrk->ws->r);
+	wrk->lastused = NAN;
+	THR_SetSession(sp);
+	// AZ(wrk->sp);
+	// wrk->sp = sp;
+	AZ(sp->wrk);
+	sp->wrk = wrk;
+	CNT_Session(sp);
+	sp = NULL;
+	/* Cannot access sp now */
+	THR_SetSession(NULL);
+	wrk->sp = NULL;
+	WS_Assert(wrk->ws);
+	AZ(wrk->busyobj);
+	AZ(wrk->wrw.wfd);
+	assert(wrk->wlp == wrk->wlb);
+	if (cache_param->diag_bitmap & 0x00040000) {
+		if (wrk->vcl != NULL)
+			VCL_Rel(&wrk->vcl);
+	}
+}
+
 /*--------------------------------------------------------------------
  * Schedule a session back on a work-thread from its pool
  */



From phk at varnish-cache.org  Mon Jan 23 11:10:15 2012
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Mon, 23 Jan 2012 12:10:15 +0100
Subject: [master] 273788a Move session scheduling to new task api
Message-ID: <E1RpHmx-00017n-0w@project.varnish-software.com>

commit 273788aee2c1892ed4a1334442dca28533f0bc94
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Jan 23 11:10:02 2012 +0000

    Move session scheduling to new task api

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index 2f48427..43b4eae 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -288,7 +288,7 @@ struct wrk_accept {
 
 /* Worker pool stuff -------------------------------------------------*/
 
-typedef void pool_func_t(struct pool *pp, struct worker *wrk, void *priv);
+typedef void pool_func_t(struct worker *wrk, void *priv);
 
 struct pool_task {
 	VTAILQ_ENTRY(pool_task)		list;
@@ -324,8 +324,7 @@ struct worker {
 	struct dstat		stats;
 
 	/* New Pool stuff */
-	pool_func_t		*pool_func;
-	void			*pool_priv;
+	struct pool_task	task;
 
 	/* Pool stuff */
 	enum e_do_what		do_what;
@@ -914,7 +913,6 @@ void PipeSession(struct sess *sp);
 /* cache_pool.c */
 void Pool_Init(void);
 void Pool_Work_Thread(void *priv, struct worker *w);
-int Pool_Schedule(struct pool *pp, struct sess *sp);
 int Pool_Task(struct pool *pp, struct pool_task *task, enum pool_how how);
 
 #define WRW_IsReleased(w)	((w)->wrw.wfd == NULL)
diff --git a/bin/varnishd/cache/cache_pool.c b/bin/varnishd/cache/cache_pool.c
index a8a2b2a..20dc38a 100644
--- a/bin/varnishd/cache/cache_pool.c
+++ b/bin/varnishd/cache/cache_pool.c
@@ -199,8 +199,8 @@ Pool_Task(struct pool *pp, struct pool_task *task, enum pool_how how)
 	if (wrk != NULL) {
 		VTAILQ_REMOVE(&pp->idle, wrk, list);
 		Lck_Unlock(&pp->mtx);
-		wrk->pool_func = task->func;
-		wrk->pool_priv = task->priv;
+		wrk->task.func = task->func;
+		wrk->task.priv = task->priv;
 		AZ(pthread_cond_signal(&wrk->cond));
 		return (0);
 	}
@@ -210,7 +210,15 @@ Pool_Task(struct pool *pp, struct pool_task *task, enum pool_how how)
 		retval = -1;
 		break;
 	case POOL_QUEUE_FRONT:
-		VTAILQ_INSERT_TAIL(&pp->front_queue, task, list);
+		/* If we have too much in the queue already, refuse. */
+		if (pp->lqueue > (cache_param->queue_max * pp->nthr) / 100) {
+			pp->ndropped++;
+			retval = -1;
+		} else {
+			VTAILQ_INSERT_TAIL(&pp->front_queue, task, list);
+			pp->nqueued++;
+			pp->lqueue++;
+		}
 		break;
 	case POOL_QUEUE_BACK:
 		VTAILQ_INSERT_TAIL(&pp->back_queue, task, list);
@@ -219,6 +227,8 @@ Pool_Task(struct pool *pp, struct pool_task *task, enum pool_how how)
 		WRONG("Unknown enum pool_how");
 	}
 	Lck_Unlock(&pp->mtx);
+	if (how == POOL_QUEUE_FRONT && !retval)
+		AZ(pthread_cond_signal(&pp->herder_cond));
 	return (retval);
 }
 
@@ -248,8 +258,10 @@ Pool_Work_Thread(void *priv, struct worker *wrk)
 		WS_Reset(wrk->ws, NULL);
 
 		tp = VTAILQ_FIRST(&pp->front_queue);
-		if (tp != NULL)
+		if (tp != NULL) {
+			pp->lqueue--;
 			VTAILQ_REMOVE(&pp->front_queue, tp, list);
+		}
 
 		if (tp == NULL) {
 			tp = VTAILQ_FIRST(&pp->back_queue);
@@ -260,7 +272,8 @@ Pool_Work_Thread(void *priv, struct worker *wrk)
 		if (tp != NULL) {
 			Lck_Unlock(&pp->mtx);
 			AN(tp->func);
-			tp->func(pp, wrk, tp->priv);
+			assert(wrk->pool == pp);
+			tp->func(wrk, tp->priv);
 			stats_clean = WRK_TrySumStat(wrk);
 			Lck_Lock(&pp->mtx);
 			continue;
@@ -297,6 +310,17 @@ Pool_Work_Thread(void *priv, struct worker *wrk)
 			(void)Lck_CondWait(&wrk->cond, &pp->mtx, NULL);
 		}
 
+		if (wrk->task.func != NULL) {
+			Lck_Unlock(&pp->mtx);
+			assert(wrk->pool == pp);
+			wrk->task.func(wrk, wrk->task.priv);
+			wrk->task.func = NULL;
+			wrk->task.priv = NULL;
+			stats_clean = WRK_TrySumStat(wrk);
+			Lck_Lock(&pp->mtx);
+			continue;
+		}
+
 		if (wrk->do_what == pool_do_die)
 			break;
 
@@ -320,7 +344,8 @@ Pool_Work_Thread(void *priv, struct worker *wrk)
 
 		if (wrk->do_what == pool_do_sess) {
 			stats_clean = 0;
-			SES_pool_task(pp, wrk, wrk->sp);
+			assert(wrk->pool == pp);
+			SES_pool_task(wrk, wrk->sp);
 		} else if (wrk->do_what == pool_do_nothing) {
 			/* we already did */
 		} else {
@@ -334,56 +359,6 @@ Pool_Work_Thread(void *priv, struct worker *wrk)
 }
 
 /*--------------------------------------------------------------------
- * Queue a workrequest if possible.
- *
- * Return zero if the request was queued, negative if it wasn't.
- */
-
-static int
-pool_queue(struct pool *pp, struct sess *sp)
-{
-	struct worker *wrk;
-
-	Lck_Lock(&pp->mtx);
-
-	/* If there are idle threads, we tickle the first one into action */
-	wrk = VTAILQ_FIRST(&pp->idle);
-	if (wrk != NULL) {
-		VTAILQ_REMOVE(&pp->idle, wrk, list);
-		Lck_Unlock(&pp->mtx);
-		wrk->sp = sp;
-		wrk->do_what = pool_do_sess;
-		AZ(pthread_cond_signal(&wrk->cond));
-		return (0);
-	}
-
-	/* If we have too much in the queue already, refuse. */
-	if (pp->lqueue > (cache_param->queue_max * pp->nthr) / 100) {
-		pp->ndropped++;
-		Lck_Unlock(&pp->mtx);
-		return (-1);
-	}
-
-	VTAILQ_INSERT_TAIL(&pp->queue, sp, list);
-	pp->nqueued++;
-	pp->lqueue++;
-	Lck_Unlock(&pp->mtx);
-	AZ(pthread_cond_signal(&pp->herder_cond));
-	return (0);
-}
-
-/*--------------------------------------------------------------------*/
-
-int
-Pool_Schedule(struct pool *pp, struct sess *sp)
-{
-
-	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
-	AZ(sp->wrk);
-	return(pool_queue(pp, sp));
-}
-
-/*--------------------------------------------------------------------
  * Create another thread, if necessary & possible
  */
 
diff --git a/bin/varnishd/cache/cache_session.c b/bin/varnishd/cache/cache_session.c
index ba5f005..871ed8f 100644
--- a/bin/varnishd/cache/cache_session.c
+++ b/bin/varnishd/cache/cache_session.c
@@ -133,19 +133,20 @@ SES_Alloc(void)
  */
 
 void
-SES_pool_task(struct pool *pp, struct worker *wrk, void *arg)
+SES_pool_task(struct worker *wrk, void *arg)
 {
 	struct sess *sp;
 
-	AN(pp);
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 	CAST_OBJ_NOTNULL(sp, arg, SESS_MAGIC);
 
 	AZ(wrk->ws->r);
 	wrk->lastused = NAN;
 	THR_SetSession(sp);
-	// AZ(wrk->sp);
-	// wrk->sp = sp;
+	if (wrk->sp == NULL)
+		wrk->sp = sp;
+	else
+		assert(wrk->sp == sp);
 	AZ(sp->wrk);
 	sp->wrk = wrk;
 	CNT_Session(sp);
@@ -178,7 +179,11 @@ SES_Schedule(struct sess *sp)
 	CHECK_OBJ_NOTNULL(pp, SESSPOOL_MAGIC);
 	AN(pp->pool);
 
-	if (Pool_Schedule(pp->pool, sp)) {
+	AZ(sp->wrk);
+	sp->task.func = SES_pool_task;
+	sp->task.priv = sp;
+
+	if (Pool_Task(pp->pool, &sp->task, POOL_QUEUE_FRONT)) {
 		VSC_C_main->client_drop_late++;
 		sp->t_idle = VTIM_real();
 		if (sp->req != NULL && sp->req->vcl != NULL) {



From phk at varnish-cache.org  Mon Jan 23 11:25:50 2012
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Mon, 23 Jan 2012 12:25:50 +0100
Subject: [master] 5b969ad Remove the now unused queue of sessions.
Message-ID: <E1RpI22-0002e7-2i@project.varnish-software.com>

commit 5b969ad5d9d7adc7d7cf09f5cc767965653238f7
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Jan 23 11:25:40 2012 +0000

    Remove the now unused queue of sessions.

diff --git a/bin/varnishd/cache/cache_pool.c b/bin/varnishd/cache/cache_pool.c
index 20dc38a..91c8e24 100644
--- a/bin/varnishd/cache/cache_pool.c
+++ b/bin/varnishd/cache/cache_pool.c
@@ -104,7 +104,6 @@ struct pool {
 	struct workerhead		idle;
 	VTAILQ_HEAD(, pool_task)	front_queue;
 	VTAILQ_HEAD(, pool_task)	back_queue;
-	VTAILQ_HEAD(, sess)		queue;
 	VTAILQ_HEAD(, poolsock)		socks;
 	unsigned			nthr;
 	unsigned			lqueue;
@@ -279,14 +278,7 @@ Pool_Work_Thread(void *priv, struct worker *wrk)
 			continue;
 		}
 
-		wrk->sp = VTAILQ_FIRST(&pp->queue);
-		if (wrk->sp != NULL) {
-			/* Process queued requests, if any */
-			assert(pp->lqueue > 0);
-			VTAILQ_REMOVE(&pp->queue, wrk->sp, list);
-			wrk->do_what = pool_do_sess;
-			pp->lqueue--;
-		} else if (!VTAILQ_EMPTY(&pp->socks)) {
+		if (!VTAILQ_EMPTY(&pp->socks)) {
 			/* Accept on a socket */
 			ps = VTAILQ_FIRST(&pp->socks);
 			VTAILQ_REMOVE(&pp->socks, ps, list);
@@ -505,7 +497,6 @@ pool_mkpool(unsigned pool_no)
 	XXXAN(pp);
 	Lck_New(&pp->mtx, lck_wq);
 
-	VTAILQ_INIT(&pp->queue);
 	VTAILQ_INIT(&pp->idle);
 	VTAILQ_INIT(&pp->socks);
 	VTAILQ_INIT(&pp->front_queue);



From phk at varnish-cache.org  Mon Jan 23 11:49:24 2012
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Mon, 23 Jan 2012 12:49:24 +0100
Subject: [master] 69256c8 Use the list in the embedded task object to hold
	idle threads that saves two pointers in workers.
Message-ID: <E1RpIOq-0005FC-Eh@project.varnish-software.com>

commit 69256c8e651979dce1001a4b5d172521786e18c3
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Jan 23 11:48:51 2012 +0000

    Use the list in the embedded task object to hold idle threads that saves
    two pointers in workers.

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index 43b4eae..8bf15ef 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -335,7 +335,6 @@ struct worker {
 
 	pthread_cond_t		cond;
 
-	VTAILQ_ENTRY(worker)	list;
 	struct sess		*sp;
 
 	struct VCL_conf		*vcl;
diff --git a/bin/varnishd/cache/cache_pool.c b/bin/varnishd/cache/cache_pool.c
index 91c8e24..656fbc8 100644
--- a/bin/varnishd/cache/cache_pool.c
+++ b/bin/varnishd/cache/cache_pool.c
@@ -80,7 +80,7 @@ pthread_condattr_setclock(pthread_condattr_t *attr, int foo)
 }
 #endif /* !CLOCK_MONOTONIC */
 
-VTAILQ_HEAD(workerhead, worker);
+VTAILQ_HEAD(taskhead, pool_task);
 
 struct poolsock {
 	unsigned			magic;
@@ -101,9 +101,9 @@ struct pool {
 	pthread_t			herder_thr;
 
 	struct lock			mtx;
-	struct workerhead		idle;
-	VTAILQ_HEAD(, pool_task)	front_queue;
-	VTAILQ_HEAD(, pool_task)	back_queue;
+	struct taskhead			idle_queue;
+	struct taskhead			front_queue;
+	struct taskhead			back_queue;
 	VTAILQ_HEAD(, poolsock)		socks;
 	unsigned			nthr;
 	unsigned			lqueue;
@@ -117,6 +117,27 @@ static struct lock		pool_mtx;
 static pthread_t		thr_pool_herder;
 
 /*--------------------------------------------------------------------
+ */
+
+static struct worker *
+pool_getidleworker(const struct pool *pp, int back)
+{
+	struct pool_task *pt;
+	struct worker *wrk;
+
+	CHECK_OBJ_NOTNULL(pp, POOL_MAGIC);
+	if (back)
+		pt = VTAILQ_LAST(&pp->idle_queue, taskhead);
+	else
+		pt = VTAILQ_FIRST(&pp->idle_queue);
+	if (pt == NULL)
+		return (NULL);
+	AZ(pt->func);
+	CAST_OBJ_NOTNULL(wrk, pt->priv, WORKER_MAGIC);
+	return (wrk);
+}
+
+/*--------------------------------------------------------------------
  * Nobody is accepting on this socket, so we do.
  *
  * As long as we can stick the accepted connection to another thread
@@ -161,10 +182,10 @@ pool_accept(struct pool *pp, struct worker *wrk, const struct poolsock *ps)
 		}
 
 		Lck_Lock(&pp->mtx);
-		if (VTAILQ_EMPTY(&pp->idle))
+		wrk2 = pool_getidleworker(pp, 0);
+		if (wrk2 == NULL)
 			return (0);
-		wrk2 = VTAILQ_FIRST(&pp->idle);
-		VTAILQ_REMOVE(&pp->idle, wrk2, list);
+		VTAILQ_REMOVE(&pp->idle_queue, &wrk2->task, list);
 		Lck_Unlock(&pp->mtx);
 		assert(sizeof *wa2 == WS_Reserve(wrk2->ws, sizeof *wa2));
 		wa2 = (void*)wrk2->ws->f;
@@ -194,9 +215,9 @@ Pool_Task(struct pool *pp, struct pool_task *task, enum pool_how how)
 	 * The common case first:  Take an idle thread, do it.
 	 */
 
-	wrk = VTAILQ_FIRST(&pp->idle);
+	wrk = pool_getidleworker(pp, 0);
 	if (wrk != NULL) {
-		VTAILQ_REMOVE(&pp->idle, wrk, list);
+		VTAILQ_REMOVE(&pp->idle_queue, &wrk->task, list);
 		Lck_Unlock(&pp->mtx);
 		wrk->task.func = task->func;
 		wrk->task.priv = task->priv;
@@ -296,7 +317,9 @@ Pool_Work_Thread(void *priv, struct worker *wrk)
 			/* Nothing to do: To sleep, perchance to dream ... */
 			if (isnan(wrk->lastused))
 				wrk->lastused = VTIM_real();
-			VTAILQ_INSERT_HEAD(&pp->idle, wrk, list);
+			wrk->task.func = NULL;
+			wrk->task.priv = wrk;
+			VTAILQ_INSERT_HEAD(&pp->idle_queue, &wrk->task, list);
 			if (!stats_clean)
 				WRK_SumStat(wrk);
 			(void)Lck_CondWait(&wrk->cond, &pp->mtx, NULL);
@@ -457,12 +480,12 @@ pool_herder(void *priv)
 		VSC_C_main->sess_queued += pp->nqueued;
 		VSC_C_main->sess_dropped += pp->ndropped;
 		pp->nqueued = pp->ndropped = 0;
-		wrk = VTAILQ_LAST(&pp->idle, workerhead);
+		wrk = pool_getidleworker(pp, 1);
 		if (wrk != NULL &&
 		    (wrk->lastused < t_idle ||
 		    pp->nthr > cache_param->wthread_max)) {
-			VTAILQ_REMOVE(&pp->idle, wrk, list);
-		} else
+			VTAILQ_REMOVE(&pp->idle_queue, &wrk->task, list);
+		} else 
 			wrk = NULL;
 		Lck_Unlock(&pp->mtx);
 
@@ -497,7 +520,7 @@ pool_mkpool(unsigned pool_no)
 	XXXAN(pp);
 	Lck_New(&pp->mtx, lck_wq);
 
-	VTAILQ_INIT(&pp->idle);
+	VTAILQ_INIT(&pp->idle_queue);
 	VTAILQ_INIT(&pp->socks);
 	VTAILQ_INIT(&pp->front_queue);
 	VTAILQ_INIT(&pp->back_queue);



From phk at varnish-cache.org  Mon Jan 23 12:33:51 2012
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Mon, 23 Jan 2012 13:33:51 +0100
Subject: [master] ac78724 Move the freshly acceptted session code from
	cache_pool.c to cache_session.c
Message-ID: <E1RpJ5r-0001qb-5E@project.varnish-software.com>

commit ac7872447397c2d9a2ff2f74c781a1ad6853aefa
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Jan 23 12:33:27 2012 +0000

    Move the freshly acceptted session code from cache_pool.c to
    cache_session.c

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index 8bf15ef..fffb777 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -928,7 +928,6 @@ void WRW_Sendfile(struct worker *w, int fd, off_t off, unsigned len);
 #endif  /* SENDFILE_WORKS */
 
 /* cache_session.c [SES] */
-struct sess *SES_New(struct sesspool *pp);
 struct sess *SES_Alloc(void);
 void SES_Close(struct sess *sp, const char *reason);
 void SES_Delete(struct sess *sp, const char *reason, double now);
@@ -940,7 +939,7 @@ void SES_Handle(struct sess *sp, double now);
 void SES_GetReq(struct sess *sp);
 void SES_ReleaseReq(struct sess *sp);
 pool_func_t SES_pool_task;
-
+pool_func_t SES_pool_accept_task;
 
 /* cache_shmlog.c */
 extern struct VSC_C_main *VSC_C_main;
diff --git a/bin/varnishd/cache/cache_pool.c b/bin/varnishd/cache/cache_pool.c
index 656fbc8..28ae2b1 100644
--- a/bin/varnishd/cache/cache_pool.c
+++ b/bin/varnishd/cache/cache_pool.c
@@ -190,7 +190,8 @@ pool_accept(struct pool *pp, struct worker *wrk, const struct poolsock *ps)
 		assert(sizeof *wa2 == WS_Reserve(wrk2->ws, sizeof *wa2));
 		wa2 = (void*)wrk2->ws->f;
 		memcpy(wa2, wa, sizeof *wa);
-		wrk2->do_what = pool_do_accept;
+		wrk2->task.func = SES_pool_accept_task;
+		wrk2->task.priv = pp->sesspool;
 		AZ(pthread_cond_signal(&wrk2->cond));
 	}
 }
@@ -342,19 +343,11 @@ Pool_Work_Thread(void *priv, struct worker *wrk)
 		Lck_Unlock(&pp->mtx);
 
 		if (wrk->do_what == pool_do_accept) {
-			/* Turn accepted socket into a session */
-			AZ(wrk->sp);
-			AN(wrk->ws->r);
-			wrk->sp = SES_New(pp->sesspool);
-			if (wrk->sp == NULL) {
-				VCA_FailSess(wrk);
+			SES_pool_accept_task(wrk, pp->sesspool);
+			if (wrk->sp == NULL)
 				wrk->do_what = pool_do_nothing;
-			} else {
-				VCA_SetupSess(wrk);
-				wrk->sp->step = STP_FIRST;
+			else
 				wrk->do_what = pool_do_sess;
-			}
-			WS_Release(wrk->ws, 0);
 		}
 
 		if (wrk->do_what == pool_do_sess) {
diff --git a/bin/varnishd/cache/cache_session.c b/bin/varnishd/cache/cache_session.c
index 871ed8f..1079bb9 100644
--- a/bin/varnishd/cache/cache_session.c
+++ b/bin/varnishd/cache/cache_session.c
@@ -97,8 +97,8 @@ ses_setup(struct sess *sp)
  * Get a new session, preferably by recycling an already ready one
  */
 
-struct sess *
-SES_New(struct sesspool *pp)
+static struct sess *
+ses_new(struct sesspool *pp)
 {
 	struct sess *sp;
 
@@ -127,6 +127,33 @@ SES_Alloc(void)
 	return (sp);
 }
 
+ /*--------------------------------------------------------------------
+ * The pool-task for a newly accepted session
+ */
+
+void
+SES_pool_accept_task(struct worker *wrk, void *arg)
+{
+	struct sesspool *pp;
+
+	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
+	CAST_OBJ_NOTNULL(pp, arg, SESSPOOL_MAGIC);
+
+	/* Turn accepted socket into a session */
+	AZ(wrk->sp);
+	AN(wrk->ws->r);
+	wrk->sp = ses_new(pp);
+	if (wrk->sp == NULL) {
+		VCA_FailSess(wrk);
+		return;
+	}
+	VCA_SetupSess(wrk);
+	wrk->sp->step = STP_FIRST;
+	WS_Release(wrk->ws, 0);
+	SES_pool_task(wrk, wrk->sp);
+}
+
+
 
 /*--------------------------------------------------------------------
  * The pool-task function for sessions



From phk at varnish-cache.org  Mon Jan 23 13:09:58 2012
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Mon, 23 Jan 2012 14:09:58 +0100
Subject: [master] b04b83b Move the accept sockets into the new pool-task API
	and GC all the cruft now not needed.
Message-ID: <E1RpJeo-0004v7-Vp@project.varnish-software.com>

commit b04b83baca73fff2fec268a16946f12665099215
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Jan 23 13:09:21 2012 +0000

    Move the accept sockets into the new pool-task API and GC all the
    cruft now not needed.

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index fffb777..3c20777 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -304,14 +304,6 @@ enum pool_how {
 
 /*--------------------------------------------------------------------*/
 
-enum e_do_what {
-	pool_do_inval = 0,
-	pool_do_sess,
-	pool_do_accept,
-	pool_do_nothing,
-	pool_do_die,
-};
-
 struct worker {
 	unsigned		magic;
 #define WORKER_MAGIC		0x6391adcf
@@ -323,12 +315,8 @@ struct worker {
 	void			*nhashpriv;
 	struct dstat		stats;
 
-	/* New Pool stuff */
 	struct pool_task	task;
 
-	/* Pool stuff */
-	enum e_do_what		do_what;
-
 	double			lastused;
 
 	struct wrw		wrw;
@@ -938,7 +926,6 @@ int SES_Schedule(struct sess *sp);
 void SES_Handle(struct sess *sp, double now);
 void SES_GetReq(struct sess *sp);
 void SES_ReleaseReq(struct sess *sp);
-pool_func_t SES_pool_task;
 pool_func_t SES_pool_accept_task;
 
 /* cache_shmlog.c */
diff --git a/bin/varnishd/cache/cache_pool.c b/bin/varnishd/cache/cache_pool.c
index 28ae2b1..dab5b11 100644
--- a/bin/varnishd/cache/cache_pool.c
+++ b/bin/varnishd/cache/cache_pool.c
@@ -85,8 +85,8 @@ VTAILQ_HEAD(taskhead, pool_task);
 struct poolsock {
 	unsigned			magic;
 #define POOLSOCK_MAGIC			0x1b0a2d38
-	VTAILQ_ENTRY(poolsock)		list;
 	struct listen_sock		*lsock;
+	struct pool_task		task;
 };
 
 /* Number of work requests queued in excess of worker threads available */
@@ -104,7 +104,6 @@ struct pool {
 	struct taskhead			idle_queue;
 	struct taskhead			front_queue;
 	struct taskhead			back_queue;
-	VTAILQ_HEAD(, poolsock)		socks;
 	unsigned			nthr;
 	unsigned			lqueue;
 	unsigned			last_lqueue;
@@ -126,6 +125,7 @@ pool_getidleworker(const struct pool *pp, int back)
 	struct worker *wrk;
 
 	CHECK_OBJ_NOTNULL(pp, POOL_MAGIC);
+	Lck_AssertHeld(&pp->mtx);
 	if (back)
 		pt = VTAILQ_LAST(&pp->idle_queue, taskhead);
 	else
@@ -141,28 +141,27 @@ pool_getidleworker(const struct pool *pp, int back)
  * Nobody is accepting on this socket, so we do.
  *
  * As long as we can stick the accepted connection to another thread
- * we do so, otherwise we return and handle it ourselves.
- *
- * Notice calling convention:  Called locked and returns locked, but
- * works lock in the meantime.
+ * we do so, otherwise we put the socket back on the "BACK" queue
+ * and handle the new connection ourselves.
  *
- * We store data about the accept in reserved workspace, it is only used
- * for a brief moment and it takes up around 144 bytes.
+ * We store data about the accept in reserved workspace on the reserved
+ * worker workspace.  SES_pool_accept_task() knows about this.
  */
 
-static int
-pool_accept(struct pool *pp, struct worker *wrk, const struct poolsock *ps)
+static void
+pool_accept(struct worker *wrk, void *arg)
 {
 	struct worker *wrk2;
 	struct wrk_accept *wa, *wa2;
+	struct pool *pp;
+	struct poolsock *ps;
 
-	CHECK_OBJ_NOTNULL(pp, POOL_MAGIC);
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
-	CHECK_OBJ_NOTNULL(ps, POOLSOCK_MAGIC);
+	pp = wrk->pool;
+	CHECK_OBJ_NOTNULL(pp, POOL_MAGIC);
+	CAST_OBJ_NOTNULL(ps, arg, POOLSOCK_MAGIC);
 
 	CHECK_OBJ_NOTNULL(ps->lsock, LISTEN_SOCK_MAGIC);
-	Lck_AssertHeld(&pp->mtx);
-	Lck_Unlock(&pp->mtx);
 	assert(sizeof *wa == WS_Reserve(wrk->ws, sizeof *wa));
 	wa = (void*)wrk->ws->f;
 	while (1) {
@@ -171,8 +170,9 @@ pool_accept(struct pool *pp, struct worker *wrk, const struct poolsock *ps)
 
 		if (ps->lsock->sock < 0) {
 			/* Socket Shutdown */
-			Lck_Lock(&pp->mtx);
-			return (-1);
+			FREE_OBJ(ps);
+			WS_Release(wrk->ws, 0);
+			return;
 		}
 		if (VCA_Accept(ps->lsock, wa) < 0) {
 			wrk->stats.sess_fail++;
@@ -183,8 +183,13 @@ pool_accept(struct pool *pp, struct worker *wrk, const struct poolsock *ps)
 
 		Lck_Lock(&pp->mtx);
 		wrk2 = pool_getidleworker(pp, 0);
-		if (wrk2 == NULL)
-			return (0);
+		if (wrk2 == NULL) {
+			/* No idle threads, do it ourselves */
+			Lck_Unlock(&pp->mtx);
+			AZ(Pool_Task(pp, &ps->task, POOL_QUEUE_BACK));
+			SES_pool_accept_task(wrk, pp->sesspool);
+			return;
+		}
 		VTAILQ_REMOVE(&pp->idle_queue, &wrk2->task, list);
 		Lck_Unlock(&pp->mtx);
 		assert(sizeof *wa2 == WS_Reserve(wrk2->ws, sizeof *wa2));
@@ -248,7 +253,7 @@ Pool_Task(struct pool *pp, struct pool_task *task, enum pool_how how)
 		WRONG("Unknown enum pool_how");
 	}
 	Lck_Unlock(&pp->mtx);
-	if (how == POOL_QUEUE_FRONT && !retval)
+	if (retval)
 		AZ(pthread_cond_signal(&pp->herder_cond));
 	return (retval);
 }
@@ -261,18 +266,14 @@ void
 Pool_Work_Thread(void *priv, struct worker *wrk)
 {
 	struct pool *pp;
-	int stats_clean, i;
-	struct poolsock *ps;
+	int stats_clean;
 	struct pool_task *tp;
 
 	CAST_OBJ_NOTNULL(pp, priv, POOL_MAGIC);
 	wrk->pool = pp;
-	Lck_Lock(&pp->mtx);
 	stats_clean = 1;
 	while (1) {
-
-		Lck_AssertHeld(&pp->mtx);
-		wrk->do_what = pool_do_inval;
+		Lck_Lock(&pp->mtx);
 
 		CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 
@@ -282,39 +283,13 @@ Pool_Work_Thread(void *priv, struct worker *wrk)
 		if (tp != NULL) {
 			pp->lqueue--;
 			VTAILQ_REMOVE(&pp->front_queue, tp, list);
-		}
-
-		if (tp == NULL) {
+		} else {
 			tp = VTAILQ_FIRST(&pp->back_queue);
 			if (tp != NULL)
 				VTAILQ_REMOVE(&pp->back_queue, tp, list);
 		}
 
-		if (tp != NULL) {
-			Lck_Unlock(&pp->mtx);
-			AN(tp->func);
-			assert(wrk->pool == pp);
-			tp->func(wrk, tp->priv);
-			stats_clean = WRK_TrySumStat(wrk);
-			Lck_Lock(&pp->mtx);
-			continue;
-		}
-
-		if (!VTAILQ_EMPTY(&pp->socks)) {
-			/* Accept on a socket */
-			ps = VTAILQ_FIRST(&pp->socks);
-			VTAILQ_REMOVE(&pp->socks, ps, list);
-			i = pool_accept(pp, wrk, ps);
-			Lck_AssertHeld(&pp->mtx);
-			if (i < 0) {
-				/* Socket Shutdown */
-				FREE_OBJ(ps);
-				WS_Release(wrk->ws, 0);
-				continue;
-			}
-			VTAILQ_INSERT_TAIL(&pp->socks, ps, list);
-			wrk->do_what = pool_do_accept;
-		} else {
+		if (tp == NULL) {
 			/* Nothing to do: To sleep, perchance to dream ... */
 			if (isnan(wrk->lastused))
 				wrk->lastused = VTIM_real();
@@ -324,45 +299,17 @@ Pool_Work_Thread(void *priv, struct worker *wrk)
 			if (!stats_clean)
 				WRK_SumStat(wrk);
 			(void)Lck_CondWait(&wrk->cond, &pp->mtx, NULL);
+			tp = &wrk->task;
 		}
-
-		if (wrk->task.func != NULL) {
-			Lck_Unlock(&pp->mtx);
-			assert(wrk->pool == pp);
-			wrk->task.func(wrk, wrk->task.priv);
-			wrk->task.func = NULL;
-			wrk->task.priv = NULL;
-			stats_clean = WRK_TrySumStat(wrk);
-			Lck_Lock(&pp->mtx);
-			continue;
-		}
-
-		if (wrk->do_what == pool_do_die)
-			break;
-
 		Lck_Unlock(&pp->mtx);
 
-		if (wrk->do_what == pool_do_accept) {
-			SES_pool_accept_task(wrk, pp->sesspool);
-			if (wrk->sp == NULL)
-				wrk->do_what = pool_do_nothing;
-			else
-				wrk->do_what = pool_do_sess;
-		}
+		if (tp->func == NULL)
+			break;
 
-		if (wrk->do_what == pool_do_sess) {
-			stats_clean = 0;
-			assert(wrk->pool == pp);
-			SES_pool_task(wrk, wrk->sp);
-		} else if (wrk->do_what == pool_do_nothing) {
-			/* we already did */
-		} else {
-			WRONG("Invalid wrk->do_what");
-		}
+		assert(wrk->pool == pp);
+		tp->func(wrk, tp->priv);
 		stats_clean = WRK_TrySumStat(wrk);
-		Lck_Lock(&pp->mtx);
 	}
-	Lck_Unlock(&pp->mtx);
 	wrk->pool = NULL;
 }
 
@@ -474,11 +421,10 @@ pool_herder(void *priv)
 		VSC_C_main->sess_dropped += pp->ndropped;
 		pp->nqueued = pp->ndropped = 0;
 		wrk = pool_getidleworker(pp, 1);
-		if (wrk != NULL &&
-		    (wrk->lastused < t_idle ||
+		if (wrk != NULL && (wrk->lastused < t_idle ||
 		    pp->nthr > cache_param->wthread_max)) {
 			VTAILQ_REMOVE(&pp->idle_queue, &wrk->task, list);
-		} else 
+		} else
 			wrk = NULL;
 		Lck_Unlock(&pp->mtx);
 
@@ -490,7 +436,8 @@ pool_herder(void *priv)
 			VSC_C_main->threads_destroyed++;
 			Lck_Unlock(&pool_mtx);
 			AZ(wrk->sp);
-			wrk->do_what = pool_do_die;
+			wrk->task.func = NULL;
+			wrk->task.priv = NULL;
 			AZ(pthread_cond_signal(&wrk->cond));
 		}
 	}
@@ -514,7 +461,6 @@ pool_mkpool(unsigned pool_no)
 	Lck_New(&pp->mtx, lck_wq);
 
 	VTAILQ_INIT(&pp->idle_queue);
-	VTAILQ_INIT(&pp->socks);
 	VTAILQ_INIT(&pp->front_queue);
 	VTAILQ_INIT(&pp->back_queue);
 	pp->sesspool = SES_NewPool(pp, pool_no);
@@ -526,7 +472,9 @@ pool_mkpool(unsigned pool_no)
 		ALLOC_OBJ(ps, POOLSOCK_MAGIC);
 		XXXAN(ps);
 		ps->lsock = ls;
-		VTAILQ_INSERT_TAIL(&pp->socks, ps, list);
+		ps->task.func = pool_accept;
+		ps->task.priv = ps;
+		AZ(Pool_Task(pp, &ps->task, POOL_QUEUE_BACK));
 	}
 
 	AZ(pthread_condattr_init(&cv_attr));
diff --git a/bin/varnishd/cache/cache_session.c b/bin/varnishd/cache/cache_session.c
index 1079bb9..3e144c1 100644
--- a/bin/varnishd/cache/cache_session.c
+++ b/bin/varnishd/cache/cache_session.c
@@ -127,40 +127,12 @@ SES_Alloc(void)
 	return (sp);
 }
 
- /*--------------------------------------------------------------------
- * The pool-task for a newly accepted session
- */
-
-void
-SES_pool_accept_task(struct worker *wrk, void *arg)
-{
-	struct sesspool *pp;
-
-	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
-	CAST_OBJ_NOTNULL(pp, arg, SESSPOOL_MAGIC);
-
-	/* Turn accepted socket into a session */
-	AZ(wrk->sp);
-	AN(wrk->ws->r);
-	wrk->sp = ses_new(pp);
-	if (wrk->sp == NULL) {
-		VCA_FailSess(wrk);
-		return;
-	}
-	VCA_SetupSess(wrk);
-	wrk->sp->step = STP_FIRST;
-	WS_Release(wrk->ws, 0);
-	SES_pool_task(wrk, wrk->sp);
-}
-
-
-
 /*--------------------------------------------------------------------
  * The pool-task function for sessions
  */
 
-void
-SES_pool_task(struct worker *wrk, void *arg)
+static void
+ses_pool_task(struct worker *wrk, void *arg)
 {
 	struct sess *sp;
 
@@ -192,6 +164,32 @@ SES_pool_task(struct worker *wrk, void *arg)
 }
 
 /*--------------------------------------------------------------------
+ * The pool-task for a newly accepted session
+ */
+
+void
+SES_pool_accept_task(struct worker *wrk, void *arg)
+{
+	struct sesspool *pp;
+
+	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
+	CAST_OBJ_NOTNULL(pp, arg, SESSPOOL_MAGIC);
+
+	/* Turn accepted socket into a session */
+	AZ(wrk->sp);
+	AN(wrk->ws->r);
+	wrk->sp = ses_new(pp);
+	if (wrk->sp == NULL) {
+		VCA_FailSess(wrk);
+		return;
+	}
+	VCA_SetupSess(wrk);
+	wrk->sp->step = STP_FIRST;
+	WS_Release(wrk->ws, 0);
+	ses_pool_task(wrk, wrk->sp);
+}
+
+/*--------------------------------------------------------------------
  * Schedule a session back on a work-thread from its pool
  */
 
@@ -207,7 +205,7 @@ SES_Schedule(struct sess *sp)
 	AN(pp->pool);
 
 	AZ(sp->wrk);
-	sp->task.func = SES_pool_task;
+	sp->task.func = ses_pool_task;
 	sp->task.priv = sp;
 
 	if (Pool_Task(pp->pool, &sp->task, POOL_QUEUE_FRONT)) {



From martin at varnish-cache.org  Mon Jan 23 14:11:34 2012
From: martin at varnish-cache.org (Martin Blix Grydeland)
Date: Mon, 23 Jan 2012 15:11:34 +0100
Subject: [master] e5a0c62 Don't try to stream when there is no body.
Message-ID: <E1RpKcQ-0004SS-0Z@project.varnish-software.com>

commit e5a0c62c450dea5efa86919f56fff5a97149b821
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Sun Jan 22 16:24:08 2012 +0100

    Don't try to stream when there is no body.

diff --git a/bin/varnishd/cache/cache_center.c b/bin/varnishd/cache/cache_center.c
index b03ba4b..28ef0bf 100644
--- a/bin/varnishd/cache/cache_center.c
+++ b/bin/varnishd/cache/cache_center.c
@@ -812,6 +812,10 @@ cnt_fetchbody(struct sess *sp, struct worker *wrk, struct req *req)
 	if (!req->wantbody)
 		wrk->busyobj->do_stream = 0;
 
+	/* No reason to try streaming a non-existing body */
+	if (wrk->busyobj->body_status == BS_NONE)
+		wrk->busyobj->do_stream = 0;
+
 	l = http_EstimateWS(wrk->busyobj->beresp,
 	    pass ? HTTPH_R_PASS : HTTPH_A_INS, &nhttp);
 



From phk at varnish-cache.org  Mon Jan 23 19:55:49 2012
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Mon, 23 Jan 2012 20:55:49 +0100
Subject: [master] 0d03ebb Remove an old wart of dubious origin. I belive that
	this originally was related to the entry parameters of
	cnt_error(), but I belive it has long been OBE,
	and now gets in the way of streaming.
Message-ID: <E1RpPzZ-000871-Ix@project.varnish-software.com>

commit 0d03ebb6b68fa50d88463566a40bb341698ec559
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Jan 23 19:54:31 2012 +0000

    Remove an old wart of dubious origin.  I belive that this originally
    was related to the entry parameters of cnt_error(), but I belive it
    has long been OBE, and now gets in the way of streaming.

diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c
index 7070612..28de3b6 100644
--- a/bin/varnishd/cache/cache_fetch.c
+++ b/bin/varnishd/cache/cache_fetch.c
@@ -578,12 +578,6 @@ FetchBody(struct worker *wrk, struct object *obj)
 
 	if (cls < 0) {
 		wrk->stats.fetch_failed++;
-		/* XXX: Wouldn't this store automatically be released ? */
-		while (!VTAILQ_EMPTY(&obj->store)) {
-			st = VTAILQ_FIRST(&obj->store);
-			VTAILQ_REMOVE(&obj->store, st, list);
-			STV_free(st);
-		}
 		VDI_CloseFd(wrk, &bo->vbc);
 		obj->len = 0;
 		return (__LINE__);



From phk at varnish-cache.org  Mon Jan 23 20:03:54 2012
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Mon, 23 Jan 2012 21:03:54 +0100
Subject: [master] 7bbc60f Untangle cnt_error() entrance criteria:
Message-ID: <E1RpQ7O-0003nn-C0@project.varnish-software.com>

commit 7bbc60f7bf32b0b6e3274445b900caaa20766cdf
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Jan 23 20:03:08 2012 +0000

    Untangle cnt_error() entrance criteria:
    
    We should never come in with a busyobj or obj.
    
    Always allocate obj from Transient storage.

diff --git a/bin/varnishd/cache/cache_center.c b/bin/varnishd/cache/cache_center.c
index 28ef0bf..0c08dd2 100644
--- a/bin/varnishd/cache/cache_center.c
+++ b/bin/varnishd/cache/cache_center.c
@@ -488,32 +488,25 @@ cnt_error(struct sess *sp, struct worker *wrk, struct req *req)
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
 
+	AZ(req->objcore);
+	AZ(req->obj);
+	AZ(wrk->busyobj);
+	wrk->busyobj = VBO_GetBusyObj(wrk);
+	req->obj = STV_NewObject(wrk, TRANSIENT_STORAGE,
+	    cache_param->http_resp_size,
+	    (uint16_t)cache_param->http_max_hdr);
 	if (req->obj == NULL) {
-		HSH_Prealloc(sp);
-		AZ(wrk->busyobj);
-		wrk->busyobj = VBO_GetBusyObj(wrk);
-		req->obj = STV_NewObject(wrk, NULL, cache_param->http_resp_size,
-		     (uint16_t)cache_param->http_max_hdr);
-		if (req->obj == NULL)
-			req->obj = STV_NewObject(wrk, TRANSIENT_STORAGE,
-			    cache_param->http_resp_size,
-			    (uint16_t)cache_param->http_max_hdr);
-		if (req->obj == NULL) {
-			req->doclose = "Out of objects";
-			req->director = NULL;
-			http_Setup(wrk->busyobj->beresp, NULL);
-			http_Setup(wrk->busyobj->bereq, NULL);
-			sp->step = STP_DONE;
-			return(0);
-		}
-		AN(req->obj);
-		req->obj->xid = req->xid;
-		req->obj->exp.entered = sp->t_req;
-	} else {
-		CHECK_OBJ_NOTNULL(wrk->busyobj, BUSYOBJ_MAGIC);
-		/* XXX: Null the headers ? */
+		req->doclose = "Out of objects";
+		req->director = NULL;
+		http_Setup(wrk->busyobj->beresp, NULL);
+		http_Setup(wrk->busyobj->bereq, NULL);
+		sp->step = STP_DONE;
+		return(0);
 	}
 	CHECK_OBJ_NOTNULL(req->obj, OBJECT_MAGIC);
+	req->obj->xid = req->xid;
+	req->obj->exp.entered = sp->t_req;
+
 	h = req->obj->http;
 
 	if (req->err_code < 100 || req->err_code > 999)



From phk at varnish-cache.org  Mon Jan 23 20:55:41 2012
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Mon, 23 Jan 2012 21:55:41 +0100
Subject: [master] 4c1abcb Go over the first couple of cache_center states with
	a bucket of polish.
Message-ID: <E1RpQvV-0003bm-3d@project.varnish-software.com>

commit 4c1abcb322f00f83c3655ee5f772202c4b1f2f18
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Jan 23 20:53:06 2012 +0000

    Go over the first couple of cache_center states with a bucket of polish.
    
    Note that we get into vcl_error{} on 413 and 417 also.
    
    Rearrange the order in which we do things in cnt_start, to be more
    consistent and correct: Process Connection: and Expect: headers
    before we snapshot and possibly head into vcl_error{}.

diff --git a/bin/varnishd/cache/cache_center.c b/bin/varnishd/cache/cache_center.c
index 0c08dd2..964bd3b 100644
--- a/bin/varnishd/cache/cache_center.c
+++ b/bin/varnishd/cache/cache_center.c
@@ -81,15 +81,15 @@ static unsigned xids;
 
 /*--------------------------------------------------------------------
  * WAIT
- * Wait (briefly) until we have a full request in our htc.
+ * Collect the request from the client.
  *
 DOT subgraph xcluster_wait {
 DOT	wait [
 DOT		shape=box
-DOT		label="cnt_wait:\nwait for\nrequest"
+DOT		label="cnt_wait:\nwait for\ncomplete\nrequest"
 DOT	]
 DOT	herding [shape=hexagon]
-DOT	wait -> start [label="got req"]
+DOT	wait -> start [label="got req",style=bold,color=green]
 DOT	wait -> "SES_Delete()" [label="errors"]
 DOT	wait -> herding [label="timeout_linger"]
 DOT	herding -> wait [label="fd read_ready"]
@@ -102,9 +102,11 @@ cnt_wait(struct sess *sp, struct worker *wrk, struct req *req)
 	int i, j, tmo;
 	struct pollfd pfd[1];
 	double now, when;
+	const char *why = NULL;
 
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
+
 	if (req == NULL) {
 		SES_GetReq(sp);
 		req = sp->req;
@@ -136,25 +138,20 @@ cnt_wait(struct sess *sp, struct worker *wrk, struct req *req)
 		if (i == 1) {
 			/* Got it, run with it */
 			sp->t_req = now;
+			sp->step = STP_START;
+			return (0);
+		} else if (i == -1) {
+			why = "EOF";
 			break;
-		}
-		if (i == -1) {
-			SES_Charge(sp);
-			SES_Delete(sp, "EOF", now);
-			return (1);
-		}
-		if (i == -2) {
-			SES_Charge(sp);
-			SES_Delete(sp, "overflow", now);
-			return (1);
-		}
-		if (i == -3) {
+		} else if (i == -2) {
+			why = "overflow";
+			break;
+		} else if (i == -3) {
 			/* Nothing but whitespace */
 			when = sp->t_idle + cache_param->timeout_idle;
 			if (when < now) {
-				SES_Charge(sp);
-				SES_Delete(sp, "timeout", now);
-				return (1);
+				why = "timeout";
+				break;
 			}
 			when = sp->t_idle + cache_param->timeout_linger;
 			tmo = (int)(1e3 * (when - now));
@@ -171,14 +168,14 @@ cnt_wait(struct sess *sp, struct worker *wrk, struct req *req)
 			when = sp->t_req + cache_param->timeout_req;
 			tmo = (int)(1e3 * (when - now));
 			if (when < now || tmo == 0) {
-				SES_Charge(sp);
-				SES_Delete(sp, "req timeout", now);
-				return (1);
+				why = "req timeout";
+				break;
 			}
 		}
 	}
-	sp->step = STP_START;
-	return (0);
+	SES_Charge(sp);
+	SES_Delete(sp, why, now);
+	return (1);
 }
 
 /*--------------------------------------------------------------------
@@ -1014,10 +1011,10 @@ cnt_streambody(struct sess *sp, struct worker *wrk, struct req *req)
 DOT subgraph xcluster_first {
 DOT	first [
 DOT		shape=box
-DOT		label="cnt_first:\nSockaddr's"
+DOT		label="cnt_first:\nrender\naddresses"
 DOT	]
 DOT }
-DOT first -> wait
+DOT first -> wait [style=bold,color=green]
  */
 
 static int
@@ -1543,7 +1540,7 @@ cnt_recv(struct sess *sp, struct worker *wrk, struct req *req)
 
 /*--------------------------------------------------------------------
  * START
- * Handle a request.
+ * First time we see a request
  *
 DOT start [
 DOT	shape=box
@@ -1556,7 +1553,6 @@ DOT start -> DONE [label=errors]
 static int
 cnt_start(struct sess *sp, struct worker *wrk, struct req *req)
 {
-	uint16_t done;
 	char *p;
 	const char *r = "HTTP/1.1 100 Continue\r\n\r\n";
 
@@ -1566,7 +1562,6 @@ cnt_start(struct sess *sp, struct worker *wrk, struct req *req)
 	AZ(req->restarts);
 	AZ(req->obj);
 	AZ(req->vcl);
-	EXP_Clr(&req->exp);
 	AZ(req->esi_level);
 
 	/* Update stats of various sorts */
@@ -1583,52 +1578,43 @@ cnt_start(struct sess *sp, struct worker *wrk, struct req *req)
 	req->vcl = wrk->vcl;
 	wrk->vcl = NULL;
 
+	EXP_Clr(&req->exp);
+
 	http_Setup(req->http, req->ws);
-	done = http_DissectRequest(sp);
+	req->err_code = http_DissectRequest(sp);
 
 	/* If we could not even parse the request, just close */
-	if (done == 400) {
+	if (req->err_code == 400) {
 		sp->step = STP_DONE;
 		SES_Close(sp, "junk");
 		return (0);
 	}
 
-	/* Catch request snapshot */
 	req->ws_req = WS_Snapshot(req->ws);
 
-	/* Catch original request, before modification */
-	HTTP_Copy(req->http0, req->http);
-
-	if (done != 0) {
-		req->err_code = done;
-		sp->step = STP_ERROR;
-		return (0);
-	}
-
 	req->doclose = http_DoConnection(req->http);
 
-	/* XXX: Handle TRACE & OPTIONS of Max-Forwards = 0 */
-
 	/*
-	 * Handle Expect headers
+	 * We want to deal with Expect: headers the first time we
+	 * attempt the request, and remove them before we move on.
 	 */
-	if (http_GetHdr(req->http, H_Expect, &p)) {
+	if (req->err_code == 0 && http_GetHdr(req->http, H_Expect, &p)) {
 		if (strcasecmp(p, "100-continue")) {
 			req->err_code = 417;
-			sp->step = STP_ERROR;
+		} else if (strlen(r) != write(sp->fd, r, strlen(r))) {
+			sp->step = STP_DONE;
+			SES_Close(sp, "remote closed");
 			return (0);
 		}
-
-		/* XXX: Don't bother with write failures for now */
-		(void)write(sp->fd, r, strlen(r));
-		/* XXX: When we do ESI includes, this is not removed
-		 * XXX: because we use http0 as our basis.  Believed
-		 * XXX: safe, but potentially confusing.
-		 */
-		http_Unset(req->http, H_Expect);
 	}
+	http_Unset(req->http, H_Expect);
 
-	sp->step = STP_RECV;
+	HTTP_Copy(req->http0, req->http);	/* Copy for restart/ESI use */
+
+	if (req->err_code)
+		sp->step = STP_ERROR;
+	else 
+		sp->step = STP_RECV;
 	return (0);
 }
 
diff --git a/bin/varnishd/default.vcl b/bin/varnishd/default.vcl
index dc4f555..18121b8 100644
--- a/bin/varnishd/default.vcl
+++ b/bin/varnishd/default.vcl
@@ -118,6 +118,9 @@ sub vcl_deliver {
     return (deliver);
 }
 
+/*
+ * We can come here "invisibly" with the following errors:  413, 417 & 503
+ */
 sub vcl_error {
     set obj.http.Content-Type = "text/html; charset=utf-8";
     set obj.http.Retry-After = "5";



From phk at varnish-cache.org  Mon Jan 23 22:13:55 2012
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Mon, 23 Jan 2012 23:13:55 +0100
Subject: [master] 3e0e0ed Fix a test description
Message-ID: <E1RpS9D-0005Lz-RA@project.varnish-software.com>

commit 3e0e0edc2b63c64b0db7cec436e9ae24795659a5
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Jan 23 22:09:04 2012 +0000

    Fix a test description

diff --git a/bin/varnishtest/tests/r00102.vtc b/bin/varnishtest/tests/r00102.vtc
index 6cf81cf..6d2d8aa 100644
--- a/bin/varnishtest/tests/r00102.vtc
+++ b/bin/varnishtest/tests/r00102.vtc
@@ -1,4 +1,4 @@
-varnishtest "Test VCL regsub()"
+varnishtest "Test POST->GET conversion"
 
 server s1 {
 	rxreq 



From phk at varnish-cache.org  Mon Jan 23 22:13:55 2012
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Mon, 23 Jan 2012 23:13:55 +0100
Subject: [master] a2c53c8 Be more careful about handling the request-body.
Message-ID: <E1RpS9D-0005M4-Su@project.varnish-software.com>

commit a2c53c83db535b4568722558d3c8bb5ff9b57dc7
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Jan 23 22:13:14 2012 +0000

    Be more careful about handling the request-body.

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index 3c20777..aefb077 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -592,7 +592,7 @@ struct req {
 	struct exp		exp;
 	unsigned		cur_method;
 	unsigned		handling;
-	unsigned char		sendbody;
+	unsigned char		reqbodydone;
 	unsigned char		wantbody;
 
 	uint16_t		err_code;
@@ -765,9 +765,9 @@ int EXP_NukeOne(struct worker *w, struct lru *lru);
 struct storage *FetchStorage(struct worker *w, ssize_t sz);
 int FetchError(struct worker *w, const char *error);
 int FetchError2(struct worker *w, const char *error, const char *more);
-int FetchHdr(struct sess *sp, int need_host_hdr);
+int FetchHdr(struct sess *sp, int need_host_hdr, int sendbody);
 int FetchBody(struct worker *w, struct object *obj);
-int FetchReqBody(const struct sess *sp);
+int FetchReqBody(const struct sess *sp, int sendbody);
 void Fetch_Init(void);
 
 /* cache_gzip.c */
diff --git a/bin/varnishd/cache/cache_acceptor.c b/bin/varnishd/cache/cache_acceptor.c
index 7918a6a..434b50c 100644
--- a/bin/varnishd/cache/cache_acceptor.c
+++ b/bin/varnishd/cache/cache_acceptor.c
@@ -253,8 +253,6 @@ VCA_SetupSess(struct worker *wrk)
 	sp->vsl_id = wa->acceptsock | VSL_CLIENTMARKER ;
 	wa->acceptsock = -1;
 	sp->t_open = VTIM_real();
-	sp->t_req = sp->t_open;
-	sp->t_idle = sp->t_open;
 	sp->mylsock = wa->acceptlsock;
 	CHECK_OBJ_NOTNULL(sp->mylsock, LISTEN_SOCK_MAGIC);
 	assert(wa->acceptaddrlen <= sp->sockaddrlen);
diff --git a/bin/varnishd/cache/cache_center.c b/bin/varnishd/cache/cache_center.c
index 964bd3b..b83cb93 100644
--- a/bin/varnishd/cache/cache_center.c
+++ b/bin/varnishd/cache/cache_center.c
@@ -83,6 +83,11 @@ static unsigned xids;
  * WAIT
  * Collect the request from the client.
  *
+ * We "abuse" sp->t_req a bit here:  On input it means "request reception
+ * started at xxx" and is used to trigger timeouts.  On return it means
+ * "we had full request headers by xxx" and is used for reporting by
+ * later steps.
+ *
 DOT subgraph xcluster_wait {
 DOT	wait [
 DOT		shape=box
@@ -446,17 +451,16 @@ cnt_done(struct sess *sp, struct worker *wrk, struct req *req)
 	WS_Reset(req->ws, NULL);
 	WS_Reset(wrk->ws, NULL);
 
+	sp->t_req = sp->t_idle;
 	i = HTC_Reinit(req->htc);
 	if (i == 1) {
 		wrk->stats.sess_pipeline++;
-		sp->t_req = sp->t_idle;
 		sp->step = STP_START;
-		return (0);
+	} else {
+		if (Tlen(req->htc->rxbuf))
+			wrk->stats.sess_readahead++;
+		sp->step = STP_WAIT;
 	}
-	if (Tlen(req->htc->rxbuf))
-		wrk->stats.sess_readahead++;
-	sp->step = STP_WAIT;
-	sp->t_req = sp->t_idle;
 	return (0);
 }
 
@@ -597,7 +601,7 @@ cnt_fetch(struct sess *sp, struct worker *wrk, struct req *req)
 
 	need_host_hdr = !http_GetHdr(wrk->busyobj->bereq, H_Host, NULL);
 
-	i = FetchHdr(sp, need_host_hdr);
+	i = FetchHdr(sp, need_host_hdr, req->objcore == NULL);
 	/*
 	 * If we recycle a backend connection, there is a finite chance
 	 * that the backend closed it before we get a request to it.
@@ -605,7 +609,7 @@ cnt_fetch(struct sess *sp, struct worker *wrk, struct req *req)
 	 */
 	if (i == 1) {
 		VSC_C_main->backend_retry++;
-		i = FetchHdr(sp, need_host_hdr);
+		i = FetchHdr(sp, need_host_hdr, req->objcore == NULL);
 	}
 
 	if (i) {
@@ -1042,6 +1046,8 @@ cnt_first(struct sess *sp, struct worker *wrk)
 
 	wrk->acct_tmp.sess++;
 
+	sp->t_req = sp->t_open;
+	sp->t_idle = sp->t_open;
 	sp->step = STP_WAIT;
 	return (0);
 }
@@ -1080,10 +1086,9 @@ cnt_hit(struct sess *sp, struct worker *wrk, struct req *req)
 	VCL_hit_method(sp);
 
 	if (req->handling == VCL_RET_DELIVER) {
-		/* Dispose of any body part of the request */
-		(void)FetchReqBody(sp);
 		//AZ(wrk->busyobj->bereq->ws);
 		//AZ(wrk->busyobj->beresp->ws);
+		(void)FetchReqBody(sp, 0);
 		sp->step = STP_PREPRESP;
 		return (0);
 	}
@@ -1117,10 +1122,6 @@ cnt_hit(struct sess *sp, struct worker *wrk, struct req *req)
  * encounter a busy object.
  *
 DOT subgraph xcluster_lookup {
-DOT	hash [
-DOT		shape=record
-DOT		label="vcl_hash()|req."
-DOT	]
 DOT	lookup [
 DOT		shape=diamond
 DOT		label="obj in cache ?\ncreate if not"
@@ -1129,7 +1130,6 @@ DOT	lookup2 [
 DOT		shape=diamond
 DOT		label="obj.f.pass ?"
 DOT	]
-DOT	hash -> lookup [label="hash",style=bold,color=green]
 DOT	lookup -> lookup2 [label="yes",style=bold,color=green]
 DOT }
 DOT lookup2 -> hit [label="no", style=bold,color=green]
@@ -1343,7 +1343,7 @@ DOT err_pass [label="ERROR",shape=plaintext]
  */
 
 static int
-cnt_pass(struct sess *sp, struct worker *wrk, struct req *req)
+cnt_pass(struct sess *sp, struct worker *wrk, const struct req *req)
 {
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
@@ -1370,7 +1370,6 @@ cnt_pass(struct sess *sp, struct worker *wrk, struct req *req)
 	}
 	assert(req->handling == VCL_RET_PASS);
 	wrk->acct_tmp.pass++;
-	req->sendbody = 1;
 	sp->step = STP_FETCH;
 	return (0);
 }
@@ -1434,11 +1433,19 @@ cnt_pipe(struct sess *sp, struct worker *wrk, const struct req *req)
 /*--------------------------------------------------------------------
  * RECV
  * We have a complete request, set everything up and start it.
+ * We can come here both with a request from the client and with
+ * a interior request during ESI delivery.
  *
 DOT subgraph xcluster_recv {
 DOT	recv [
 DOT		shape=record
-DOT		label="vcl_recv()|req."
+DOT		label="{cnt_recv:|{vcl_recv\{\}|req.*}}"
+DOT	]
+DOT }
+DOT subgraph xcluster_hash {
+DOT	hash [
+DOT		shape=record
+DOT		label="{cnt_recv:|{vcl_hash\{\}|req.*}}"
 DOT	]
 DOT }
 DOT ESI_REQ [ shape=hexagon ]
@@ -1449,6 +1456,7 @@ DOT recv -> pass2 [label="pass",style=bold,color=red]
 DOT recv -> err_recv [label="error"]
 DOT err_recv [label="ERROR",shape=plaintext]
 DOT recv -> hash [label="lookup",style=bold,color=green]
+DOT hash -> lookup [label="hash",style=bold,color=green]
  */
 
 static int
@@ -1481,6 +1489,11 @@ cnt_recv(struct sess *sp, struct worker *wrk, struct req *req)
 	recv_handling = req->handling;
 
 	if (req->restarts >= cache_param->max_restarts) {
+		/*
+		 * XXX: Why not before vcl_recv{} ?  We go to vcl_error{}
+		 * XXX: without vcl_recv{} on 413 and 417 already.
+		 * XXX tell vcl_error why we come
+		 */
 		if (req->err_code == 0)
 			req->err_code = 503;
 		sp->step = STP_ERROR;
@@ -1499,7 +1512,7 @@ cnt_recv(struct sess *sp, struct worker *wrk, struct req *req)
 		}
 	}
 
-	req->sha256ctx = &sha256ctx;
+	req->sha256ctx = &sha256ctx;	/* so HSH_AddString() can find it */
 	SHA256_Init(req->sha256ctx);
 	VCL_hash_method(sp);
 	assert(req->handling == VCL_RET_HASH);
@@ -1511,10 +1524,8 @@ cnt_recv(struct sess *sp, struct worker *wrk, struct req *req)
 	else
 		req->wantbody = 1;
 
-	req->sendbody = 0;
 	switch(recv_handling) {
 	case VCL_RET_LOOKUP:
-		/* XXX: discard req body, if any */
 		sp->step = STP_LOOKUP;
 		return (0);
 	case VCL_RET_PIPE:
@@ -1530,7 +1541,6 @@ cnt_recv(struct sess *sp, struct worker *wrk, struct req *req)
 		sp->step = STP_PASS;
 		return (0);
 	case VCL_RET_ERROR:
-		/* XXX: discard req body, if any */
 		sp->step = STP_ERROR;
 		return (0);
 	default:
@@ -1563,10 +1573,10 @@ cnt_start(struct sess *sp, struct worker *wrk, struct req *req)
 	AZ(req->obj);
 	AZ(req->vcl);
 	AZ(req->esi_level);
+	assert(!isnan(sp->t_req));
 
 	/* Update stats of various sorts */
 	wrk->stats.client_req++;
-	assert(!isnan(sp->t_req));
 	wrk->acct_tmp.req++;
 
 	/* Assign XID and log */
@@ -1609,11 +1619,14 @@ cnt_start(struct sess *sp, struct worker *wrk, struct req *req)
 	}
 	http_Unset(req->http, H_Expect);
 
+	/* XXX: pull in req-body and make it available instead. */
+	req->reqbodydone = 0;
+
 	HTTP_Copy(req->http0, req->http);	/* Copy for restart/ESI use */
 
 	if (req->err_code)
 		sp->step = STP_ERROR;
-	else 
+	else
 		sp->step = STP_RECV;
 	return (0);
 }
diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c
index 28de3b6..c3563a6 100644
--- a/bin/varnishd/cache/cache_fetch.c
+++ b/bin/varnishd/cache/cache_fetch.c
@@ -329,14 +329,20 @@ fetch_eof(struct worker *wrk, struct http_conn *htc)
  */
 
 int
-FetchReqBody(const struct sess *sp)
+FetchReqBody(const struct sess *sp, int sendbody)
 {
 	unsigned long content_length;
 	char buf[8192];
 	char *ptr, *endp;
 	int rdcnt;
 
+	if (sp->req->reqbodydone) {
+		AZ(sendbody);
+		return (0);
+	}
+		
 	if (http_GetHdr(sp->req->http, H_Content_Length, &ptr)) {
+		sp->req->reqbodydone = 1;
 
 		content_length = strtoul(ptr, &endp, 10);
 		/* XXX should check result of conversion */
@@ -349,11 +355,12 @@ FetchReqBody(const struct sess *sp)
 			if (rdcnt <= 0)
 				return (1);
 			content_length -= rdcnt;
-			if (!sp->req->sendbody)
-				continue;
-			(void)WRW_Write(sp->wrk, buf, rdcnt); /* XXX: stats ? */
-			if (WRW_Flush(sp->wrk))
-				return (2);
+			if (sendbody) {
+				/* XXX: stats ? */
+				(void)WRW_Write(sp->wrk, buf, rdcnt);
+				if (WRW_Flush(sp->wrk))
+					return (2);
+			}
 		}
 	}
 	if (http_GetHdr(sp->req->http, H_Transfer_Encoding, NULL)) {
@@ -375,7 +382,7 @@ FetchReqBody(const struct sess *sp)
  */
 
 int
-FetchHdr(struct sess *sp, int need_host_hdr)
+FetchHdr(struct sess *sp, int need_host_hdr, int sendbody)
 {
 	struct vbc *vc;
 	struct worker *wrk;
@@ -422,7 +429,7 @@ FetchHdr(struct sess *sp, int need_host_hdr)
 	(void)http_Write(wrk, vc->vsl_id, hp, 0);	/* XXX: stats ? */
 
 	/* Deal with any message-body the request might have */
-	i = FetchReqBody(sp);
+	i = FetchReqBody(sp, sendbody);
 	if (WRW_FlushRelease(wrk) || i > 0) {
 		WSP(sp, SLT_FetchError, "backend write error: %d (%s)",
 		    errno, strerror(errno));



From phk at varnish-cache.org  Tue Jan 24 10:50:01 2012
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Tue, 24 Jan 2012 11:50:01 +0100
Subject: [master] 4a75131 Add testcase for restart in vcl_miss{}
Message-ID: <E1Rpdwv-0003aE-7e@project.varnish-software.com>

commit 4a75131dbe4d95f6de7bc54848975c3da6e25abd
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Jan 24 10:49:45 2012 +0000

    Add testcase for restart in vcl_miss{}

diff --git a/bin/varnishtest/tests/v00037.vtc b/bin/varnishtest/tests/v00037.vtc
new file mode 100644
index 0000000..e7759a6
--- /dev/null
+++ b/bin/varnishtest/tests/v00037.vtc
@@ -0,0 +1,30 @@
+varnishtest "test restart in miss"
+
+server s1 {
+	rxreq
+	txresp -body "FOOBAR"
+} -start
+
+varnish v1 -vcl+backend {
+
+	sub vcl_recv {
+		if (req.restarts > 0) {
+			unset req.http.foobar;
+		}
+	}
+
+	sub vcl_miss {
+		if (req.http.foobar) {
+			return (restart);
+		}
+	}
+	sub vcl_deliver {
+		set resp.http.restarts = req.restarts;
+	}
+} -start
+
+client c1 {
+	txreq -hdr "foobar: snafu"
+	rxresp
+	expect resp.http.restarts == 1
+} -run



From phk at varnish-cache.org  Tue Jan 24 10:50:01 2012
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Tue, 24 Jan 2012 11:50:01 +0100
Subject: [master] 3d52af7 Continue polishing cache_center to make states and
	assumptions clearer for everybody.
Message-ID: <E1Rpdwv-0003a7-61@project.varnish-software.com>

commit 3d52af787968fa31c94f0f0429eb6cbdfcf4f1b5
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Jan 24 10:48:48 2012 +0000

    Continue polishing cache_center to make states and assumptions
    clearer for everybody.
    
    Add restart in vcl_miss{} implementation.

diff --git a/bin/varnishd/cache/cache_center.c b/bin/varnishd/cache/cache_center.c
index b83cb93..ea176cb 100644
--- a/bin/varnishd/cache/cache_center.c
+++ b/bin/varnishd/cache/cache_center.c
@@ -488,10 +488,10 @@ cnt_error(struct sess *sp, struct worker *wrk, struct req *req)
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
-
 	AZ(req->objcore);
 	AZ(req->obj);
 	AZ(wrk->busyobj);
+
 	wrk->busyobj = VBO_GetBusyObj(wrk);
 	req->obj = STV_NewObject(wrk, TRANSIENT_STORAGE,
 	    cache_param->http_resp_size,
@@ -1079,6 +1079,7 @@ cnt_hit(struct sess *sp, struct worker *wrk, struct req *req)
 
 	CHECK_OBJ_NOTNULL(req->obj, OBJECT_MAGIC);
 	CHECK_OBJ_NOTNULL(req->vcl, VCL_CONF_MAGIC);
+	AZ(req->objcore);
 	AZ(wrk->busyobj);
 
 	assert(!(req->obj->objcore->flags & OC_F_PASS));
@@ -1123,18 +1124,19 @@ cnt_hit(struct sess *sp, struct worker *wrk, struct req *req)
  *
 DOT subgraph xcluster_lookup {
 DOT	lookup [
-DOT		shape=diamond
-DOT		label="obj in cache ?\ncreate if not"
+DOT		shape=record
+DOT		label="{cnt_lookup:|hash lookup}"
 DOT	]
 DOT	lookup2 [
 DOT		shape=diamond
 DOT		label="obj.f.pass ?"
 DOT	]
-DOT	lookup -> lookup2 [label="yes",style=bold,color=green]
 DOT }
+DOT lookup -> lookup [label="Busy object found"]
+DOT lookup -> miss [label="not found\ncreate new",style=bold,color=blue]
+DOT lookup -> lookup2 [label="found",style=bold,color=green]
 DOT lookup2 -> hit [label="no", style=bold,color=green]
 DOT lookup2 -> pass [label="yes",style=bold,color=red]
-DOT lookup -> miss [label="no",style=bold,color=blue]
  */
 
 static int
@@ -1147,6 +1149,7 @@ cnt_lookup(struct sess *sp, struct worker *wrk, struct req *req)
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
+	AZ(req->objcore);
 
 	CHECK_OBJ_NOTNULL(req->vcl, VCL_CONF_MAGIC);
 	AZ(wrk->busyobj);
@@ -1165,18 +1168,19 @@ cnt_lookup(struct sess *sp, struct worker *wrk, struct req *req)
 	req->vary_b[2] = '\0';
 
 	oc = HSH_Lookup(sp, &oh);
+	AZ(req->objcore);
 
 	if (oc == NULL) {
 		/*
 		 * We lost the session to a busy object, disembark the
-		 * worker thread.   The hash code to restart the session,
-		 * still in STP_LOOKUP, later when the busy object isn't.
+		 * worker thread.   We return to STP_LOOKUP when the busy
+		 * object has been unbusied, and still have the hash digest
+		 * around to do the lookup with.
 		 * NB:  Do not access sp any more !
 		 */
 		return (1);
 	}
 
-
 	CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
 	CHECK_OBJ_NOTNULL(oh, OBJHEAD_MAGIC);
 
@@ -1215,7 +1219,7 @@ cnt_lookup(struct sess *sp, struct worker *wrk, struct req *req)
 		wrk->stats.cache_hitpass++;
 		WSP(sp, SLT_HitPass, "%u", req->obj->xid);
 		(void)HSH_Deref(wrk, NULL, &req->obj);
-		req->objcore = NULL;
+		AZ(req->objcore);
 		sp->step = STP_PASS;
 		return (0);
 	}
@@ -1257,10 +1261,10 @@ cnt_miss(struct sess *sp, struct worker *wrk, struct req *req)
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
 	CHECK_OBJ_NOTNULL(req->vcl, VCL_CONF_MAGIC);
-
-	AZ(req->obj);
-	AN(req->objcore);
+	CHECK_OBJ_NOTNULL(req->objcore, OBJCORE_MAGIC);
 	CHECK_OBJ_NOTNULL(wrk->busyobj, BUSYOBJ_MAGIC);
+	AZ(req->obj);
+
 	WS_Reset(wrk->ws, NULL);
 	wrk->busyobj = VBO_GetBusyObj(wrk);
 	http_Setup(wrk->busyobj->bereq, wrk->ws);
@@ -1276,38 +1280,36 @@ cnt_miss(struct sess *sp, struct worker *wrk, struct req *req)
 		http_SetHeader(wrk, sp->vsl_id, wrk->busyobj->bereq,
 		    "Accept-Encoding: gzip");
 	}
-	wrk->connect_timeout = 0;
-	wrk->first_byte_timeout = 0;
-	wrk->between_bytes_timeout = 0;
 
 	VCL_miss_method(sp);
 
+	if (req->handling == VCL_RET_FETCH) {
+		CHECK_OBJ_NOTNULL(wrk->busyobj, BUSYOBJ_MAGIC);
+		sp->step = STP_FETCH;
+		return (0);
+	}
+
+	AZ(HSH_Deref(wrk, req->objcore, NULL));
+	req->objcore = NULL;
+	http_Setup(wrk->busyobj->bereq, NULL);
+	VBO_DerefBusyObj(wrk, &wrk->busyobj);
+
 	switch(req->handling) {
 	case VCL_RET_ERROR:
-		AZ(HSH_Deref(wrk, req->objcore, NULL));
-		req->objcore = NULL;
-		http_Setup(wrk->busyobj->bereq, NULL);
-		VBO_DerefBusyObj(wrk, &wrk->busyobj);
 		sp->step = STP_ERROR;
-		return (0);
+		break;
 	case VCL_RET_PASS:
-		AZ(HSH_Deref(wrk, req->objcore, NULL));
-		req->objcore = NULL;
-		VBO_DerefBusyObj(wrk, &wrk->busyobj);
 		sp->step = STP_PASS;
-		return (0);
-	case VCL_RET_FETCH:
-		CHECK_OBJ_NOTNULL(wrk->busyobj, BUSYOBJ_MAGIC);
-		sp->step = STP_FETCH;
-		return (0);
+		break;
 	case VCL_RET_RESTART:
-		AZ(HSH_Deref(wrk, req->objcore, NULL));
-		req->objcore = NULL;
-		VBO_DerefBusyObj(wrk, &wrk->busyobj);
-		INCOMPL();
+		req->restarts++;
+		req->director = NULL;
+		sp->step = STP_RECV;
+		break;
 	default:
 		WRONG("Illegal action in vcl_miss{}");
 	}
+	return (0);
 }
 
 /*--------------------------------------------------------------------
@@ -1349,6 +1351,7 @@ cnt_pass(struct sess *sp, struct worker *wrk, const struct req *req)
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
 	CHECK_OBJ_NOTNULL(req->vcl, VCL_CONF_MAGIC);
+	AZ(req->objcore);
 	AZ(req->obj);
 	AZ(wrk->busyobj);
 
@@ -1358,10 +1361,8 @@ cnt_pass(struct sess *sp, struct worker *wrk, const struct req *req)
 	http_Setup(wrk->busyobj->bereq, wrk->ws);
 	http_FilterReq(sp, HTTPH_R_PASS);
 
-	wrk->connect_timeout = 0;
-	wrk->first_byte_timeout = 0;
-	wrk->between_bytes_timeout = 0;
 	VCL_pass_method(sp);
+
 	if (req->handling == VCL_RET_ERROR) {
 		http_Setup(wrk->busyobj->bereq, NULL);
 		VBO_DerefBusyObj(wrk, &wrk->busyobj);
@@ -1478,6 +1479,10 @@ cnt_recv(struct sess *sp, struct worker *wrk, struct req *req)
 	req->director = req->vcl->director[0];
 	AN(req->director);
 
+	wrk->connect_timeout = 0;
+	wrk->first_byte_timeout = 0;
+	wrk->between_bytes_timeout = 0;
+
 	req->disable_esi = 0;
 	req->hash_always_miss = 0;
 	req->hash_ignore_busy = 0;
diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c
index c3563a6..abef6b1 100644
--- a/bin/varnishd/cache/cache_fetch.c
+++ b/bin/varnishd/cache/cache_fetch.c
@@ -340,7 +340,7 @@ FetchReqBody(const struct sess *sp, int sendbody)
 		AZ(sendbody);
 		return (0);
 	}
-		
+
 	if (http_GetHdr(sp->req->http, H_Content_Length, &ptr)) {
 		sp->req->reqbodydone = 1;
 



From geoff at varnish-cache.org  Tue Jan 24 17:30:27 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Tue, 24 Jan 2012 18:30:27 +0100
Subject: [experimental-ims] 299d2da Force lookup of kss resources and fix
	cached creation of objects in Plone
Message-ID: <E1RpkCR-0005Nr-Hn@project.varnish-software.com>

commit 299d2da0d94cae74efce9f46dd5f5d0d654fe348
Author: Cleber J Santos <cleber at cleberjsantos.com.br>
Date:   Wed Jan 18 11:46:03 2012 -0200

    Force lookup of kss resources and fix cached creation of objects in Plone

diff --git a/etc/zope-plone.vcl b/etc/zope-plone.vcl
index b69f583..263e343 100644
--- a/etc/zope-plone.vcl
+++ b/etc/zope-plone.vcl
@@ -52,13 +52,18 @@ sub vcl_recv {
                         }
                         return(lookup);
                 }
+
+                # Do not cache the creation of objects in Plone
+                if (req.url ~ "createObject"){
+                        return(pass);
+                }
         }
 
         # Don't cache authenticated requests
         if (req.http.Cookie && req.http.Cookie ~ "__ac(|_(name|password|persistent))=") {
 
 		# Force lookup of specific urls unlikely to need protection
-		if (req.url ~ "\.(js|css)") {
+		if (req.url ~ "\.(js|css|kss)") {
                         remove req.http.cookie;
                         return(lookup);
                 }



From geoff at varnish-cache.org  Tue Jan 24 17:30:27 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Tue, 24 Jan 2012 18:30:27 +0100
Subject: [experimental-ims] 3cf4dfd Merge branch 'master' of
	ssh://git.varnish-cache.org/git/varnish-cache into experimental-ims
Message-ID: <E1RpkCR-0005Nx-Jd@project.varnish-software.com>

commit 3cf4dfdb08fbf29a57d99d334f69dc27fbe63afe
Merge: eb7cbda 299d2da
Author: Geoff Simmons <geoff at uplex.de>
Date:   Thu Jan 19 12:33:08 2012 +0100

    Merge branch 'master' of ssh://git.varnish-cache.org/git/varnish-cache into experimental-ims



From geoff at varnish-cache.org  Tue Jan 24 17:30:27 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Tue, 24 Jan 2012 18:30:27 +0100
Subject: [experimental-ims] bf913ad Don't forget to do the stats when a
	request is aborted
Message-ID: <E1RpkCR-0005O2-MA@project.varnish-software.com>

commit bf913ad2f50f12ec99696816ac7048b69c21d0cb
Author: Rogier 'DocWilco' Mulhuijzen <github at bsdchicks.com>
Date:   Wed Jan 18 18:16:35 2012 +0100

    Don't forget to do the stats when a request is aborted
    
    Without doing SES_Charge, the worker will panic at the end of
    CNT_Session() when a request is aborted before it is complete.
    
    Found by: Federico G. Schwindt <fgsch at lodoss.net>

diff --git a/bin/varnishd/cache/cache_center.c b/bin/varnishd/cache/cache_center.c
index a9c233b..b03ba4b 100644
--- a/bin/varnishd/cache/cache_center.c
+++ b/bin/varnishd/cache/cache_center.c
@@ -139,10 +139,12 @@ cnt_wait(struct sess *sp, struct worker *wrk, struct req *req)
 			break;
 		}
 		if (i == -1) {
+			SES_Charge(sp);
 			SES_Delete(sp, "EOF", now);
 			return (1);
 		}
 		if (i == -2) {
+			SES_Charge(sp);
 			SES_Delete(sp, "overflow", now);
 			return (1);
 		}
@@ -150,6 +152,7 @@ cnt_wait(struct sess *sp, struct worker *wrk, struct req *req)
 			/* Nothing but whitespace */
 			when = sp->t_idle + cache_param->timeout_idle;
 			if (when < now) {
+				SES_Charge(sp);
 				SES_Delete(sp, "timeout", now);
 				return (1);
 			}
@@ -168,6 +171,7 @@ cnt_wait(struct sess *sp, struct worker *wrk, struct req *req)
 			when = sp->t_req + cache_param->timeout_req;
 			tmo = (int)(1e3 * (when - now));
 			if (when < now || tmo == 0) {
+				SES_Charge(sp);
 				SES_Delete(sp, "req timeout", now);
 				return (1);
 			}



From geoff at varnish-cache.org  Tue Jan 24 17:30:27 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Tue, 24 Jan 2012 18:30:27 +0100
Subject: [experimental-ims] b499af9 Stop accept() timeouts from being counted
	as sess_fail in varnishstat
Message-ID: <E1RpkCR-0005OD-On@project.varnish-software.com>

commit b499af9caad1a1bb1db5ed29c547573cc81fcd66
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Thu Jan 19 16:04:12 2012 +0100

    Stop accept() timeouts from being counted as sess_fail in varnishstat
    
    Varnish sets SO_RCVTIMEO to cache_param->timeout_idle on the listen
    sockets, so that the value is inherited to the accept()ed
    connections. This has the side effect of making accept() returning
    with EAGAIN after cache_param->timeout_idle seconds when there are no
    connections coming in, and each of these timeouts is counted as a
    sess_failed. Also the pacing sleep time is increased for each of these
    timeouts, causing Varnish to react slowly at first when connections
    start coming in.
    
    Patch changes to loop around the call to accept() while EAGAIN

diff --git a/bin/varnishd/cache/cache_acceptor.c b/bin/varnishd/cache/cache_acceptor.c
index d55545b..7918a6a 100644
--- a/bin/varnishd/cache/cache_acceptor.c
+++ b/bin/varnishd/cache/cache_acceptor.c
@@ -186,7 +186,10 @@ VCA_Accept(struct listen_sock *ls, struct wrk_accept *wa)
 		(void)usleep(100*1000);
 
 	wa->acceptaddrlen = sizeof wa->acceptaddr;
-	i = accept(ls->sock, (void*)&wa->acceptaddr, &wa->acceptaddrlen);
+	do {
+		i = accept(ls->sock, (void*)&wa->acceptaddr,
+			   &wa->acceptaddrlen);
+	} while (i < 0 && errno == EAGAIN);
 
 	if (i < 0) {
 		switch (errno) {



From geoff at varnish-cache.org  Tue Jan 24 17:30:27 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Tue, 24 Jan 2012 18:30:27 +0100
Subject: [experimental-ims] cf0a45a Remove last traces of sess_workspace from
	logs
Message-ID: <E1RpkCR-0005OS-Qp@project.varnish-software.com>

commit cf0a45afedf1a5ca696119ed2b1a1e99b1fb1a08
Author: Andreas Plesner Jacobsen <apj at mutt.dk>
Date:   Thu Jan 19 19:12:43 2012 +0100

    Remove last traces of sess_workspace from logs
    
    Found by: Federico G. Schwindt <fgsch at lodoss.net>

diff --git a/doc/sphinx/reference/params.rst b/doc/sphinx/reference/params.rst
index f161596..1f9a486 100644
--- a/doc/sphinx/reference/params.rst
+++ b/doc/sphinx/reference/params.rst
@@ -250,7 +250,7 @@ http_req_size
 	- Default: 32768
 
 	Maximum number of bytes of HTTP client request we will deal with.  This is a limit on all bytes up to the double blank line which ends the HTTP request.
-	The memory for the request is allocated from the session workspace (param: sess_workspace) and this parameter limits how much of that the request is allowed to take up.
+	The memory for the request is allocated from the client workspace (param: workspace_client) and this parameter limits how much of that the request is allowed to take up.
 
 http_resp_hdr_len
 	- Units: bytes
@@ -263,7 +263,7 @@ http_resp_size
 	- Default: 32768
 
 	Maximum number of bytes of HTTP backend resonse we will deal with.  This is a limit on all bytes up to the double blank line which ends the HTTP request.
-	The memory for the request is allocated from the worker workspace (param: sess_workspace) and this parameter limits how much of that the request is allowed to take up.
+	The memory for the request is allocated from the thread pool workspace (param: thread_pool_workspace) and this parameter limits how much of that the request is allowed to take up.
 
 listen_address
 	- Default: :80
@@ -378,14 +378,6 @@ sess_timeout
 
 	Idle timeout for persistent sessions. If a HTTP request has not been received in this many seconds, the session is closed.
 
-sess_workspace
-	- Units: bytes
-	- Default: 65536
-	- Flags: delayed
-
-	Bytes of HTTP protocol workspace allocated for sessions. This space must be big enough for the entire HTTP protocol header and any edits done to it in the VCL code.
-	Minimum is 1024 bytes.
-
 session_linger
 	- Units: ms
 	- Default: 50
diff --git a/doc/sphinx/reference/varnishd.rst b/doc/sphinx/reference/varnishd.rst
index 4e2b3f2..351c7b3 100644
--- a/doc/sphinx/reference/varnishd.rst
+++ b/doc/sphinx/reference/varnishd.rst
@@ -274,8 +274,10 @@ Here is a list of all parameters, current as of last time we remembered to updat
 produced from the same text you will find in the CLI if you use the param.show command, so should there be a new
 parameter which is not listed here, you can find the description using the CLI commands.
 
-Be aware that on 32 bit systems, certain default values, such as sess_workspace (=16k) and thread_pool_stack
-(=64k) are reduced relative to the values listed here, in order to conserve VM space.
+Be aware that on 32 bit systems, certain default values, such as
+workspace_client (=16k), thread_pool_workspace (=16k), http_resp_size (=8k),
+http_req_size (=12k), gzip_stack_buffer (=4k) and thread_pool_stack (=64k) are
+reduced relative to the values listed here, in order to conserve VM space.
 
 .. include:: params.rst
 
diff --git a/doc/sphinx/reference/vmod.rst b/doc/sphinx/reference/vmod.rst
index 2b515c1..57163c5 100644
--- a/doc/sphinx/reference/vmod.rst
+++ b/doc/sphinx/reference/vmod.rst
@@ -171,7 +171,7 @@ STRING_LIST
 	a function, we may relax that at a latter time.
 
 	If you don't want to bother with STRING_LIST, just use STRING
-	and make sure your sess_workspace param is big enough.
+	and make sure your thread_pool_workspace param is big enough.
 
 PRIV_VCL
 	See below



From geoff at varnish-cache.org  Tue Jan 24 17:30:27 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Tue, 24 Jan 2012 18:30:27 +0100
Subject: [experimental-ims] 29fcad4 stylistic nit
Message-ID: <E1RpkCR-0005Oc-Sv@project.varnish-software.com>

commit 29fcad40204b0ac09d81139692c8f26ebf3f06cf
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Jan 23 09:34:41 2012 +0000

    stylistic nit

diff --git a/bin/varnishd/cache/cache_pool.c b/bin/varnishd/cache/cache_pool.c
index 84a6cfd..0f7c9e4 100644
--- a/bin/varnishd/cache/cache_pool.c
+++ b/bin/varnishd/cache/cache_pool.c
@@ -218,7 +218,7 @@ Pool_Work_Thread(void *priv, struct worker *wrk)
 			}
 			VTAILQ_INSERT_TAIL(&pp->socks, ps, list);
 			wrk->do_what = pool_do_accept;
-		} else if (VTAILQ_EMPTY(&pp->socks)) {
+		} else {
 			/* Nothing to do: To sleep, perchance to dream ... */
 			if (isnan(wrk->lastused))
 				wrk->lastused = VTIM_real();



From geoff at varnish-cache.org  Tue Jan 24 17:30:27 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Tue, 24 Jan 2012 18:30:27 +0100
Subject: [experimental-ims] edf5144 Add the new pool-task API so Martin can
	see it.
Message-ID: <E1RpkCR-0005Ok-UM@project.varnish-software.com>

commit edf5144fb9c87869638f3b3dc56ea6572e0d83cb
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Jan 23 10:06:46 2012 +0000

    Add the new pool-task API so Martin can see it.
    
    Presently unused and untested.

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index 817b7c7..463517c 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -286,6 +286,22 @@ struct wrk_accept {
 	struct listen_sock	*acceptlsock;
 };
 
+/* Worker pool stuff -------------------------------------------------*/
+
+typedef void pool_func_t(struct pool *pp, void *priv);
+
+struct pool_task {
+	VTAILQ_ENTRY(pool_task)		list;
+	pool_func_t			*func;
+	void				*priv;
+};
+
+enum pool_how {
+	POOL_NO_QUEUE,
+	POOL_QUEUE_FRONT,
+	POOL_QUEUE_BACK
+};
+
 /*--------------------------------------------------------------------*/
 
 enum e_do_what {
@@ -307,6 +323,10 @@ struct worker {
 	void			*nhashpriv;
 	struct dstat		stats;
 
+	/* New Pool stuff */
+	pool_func_t		*pool_func;
+	void			*pool_priv;
+
 	/* Pool stuff */
 	enum e_do_what		do_what;
 
@@ -648,6 +668,7 @@ struct sess {
 	struct worker		*wrk;
 	struct req		*req;
 
+	struct pool_task	task;
 	VTAILQ_ENTRY(sess)	list;
 
 	/* Session related fields ------------------------------------*/
@@ -894,6 +915,7 @@ void PipeSession(struct sess *sp);
 void Pool_Init(void);
 void Pool_Work_Thread(void *priv, struct worker *w);
 int Pool_Schedule(struct pool *pp, struct sess *sp);
+int Pool_Task(struct pool *pp, struct pool_task *task, enum pool_how how);
 
 #define WRW_IsReleased(w)	((w)->wrw.wfd == NULL)
 int WRW_Error(const struct worker *w);
diff --git a/bin/varnishd/cache/cache_pool.c b/bin/varnishd/cache/cache_pool.c
index 0f7c9e4..cc81b59 100644
--- a/bin/varnishd/cache/cache_pool.c
+++ b/bin/varnishd/cache/cache_pool.c
@@ -92,24 +92,26 @@ struct poolsock {
 /* Number of work requests queued in excess of worker threads available */
 
 struct pool {
-	unsigned		magic;
-#define POOL_MAGIC		0x606658fa
-	VTAILQ_ENTRY(pool)	list;
-
-	pthread_cond_t		herder_cond;
-	struct lock		herder_mtx;
-	pthread_t		herder_thr;
-
-	struct lock		mtx;
-	struct workerhead	idle;
-	VTAILQ_HEAD(, sess)	queue;
-	VTAILQ_HEAD(, poolsock)	socks;
-	unsigned		nthr;
-	unsigned		lqueue;
-	unsigned		last_lqueue;
-	uintmax_t		ndropped;
-	uintmax_t		nqueued;
-	struct sesspool		*sesspool;
+	unsigned			magic;
+#define POOL_MAGIC			0x606658fa
+	VTAILQ_ENTRY(pool)		list;
+
+	pthread_cond_t			herder_cond;
+	struct lock			herder_mtx;
+	pthread_t			herder_thr;
+
+	struct lock			mtx;
+	struct workerhead		idle;
+	VTAILQ_HEAD(, pool_task)	front_queue;
+	VTAILQ_HEAD(, pool_task)	back_queue;
+	VTAILQ_HEAD(, sess)		queue;
+	VTAILQ_HEAD(, poolsock)		socks;
+	unsigned			nthr;
+	unsigned			lqueue;
+	unsigned			last_lqueue;
+	uintmax_t			ndropped;
+	uintmax_t			nqueued;
+	struct sesspool			*sesspool;
 };
 
 static struct lock		pool_mtx;
@@ -174,6 +176,51 @@ pool_accept(struct pool *pp, struct worker *wrk, const struct poolsock *ps)
 }
 
 /*--------------------------------------------------------------------
+ * Enter a new task to be done
+ */
+
+int
+Pool_Task(struct pool *pp, struct pool_task *task, enum pool_how how)
+{
+	struct worker *wrk;
+	int retval = 0;
+
+	CHECK_OBJ_NOTNULL(pp, POOL_MAGIC);
+
+	Lck_Lock(&pp->mtx);
+
+	/*
+	 * The common case first:  Take an idle thread, do it.
+	 */
+
+	wrk = VTAILQ_FIRST(&pp->idle);
+	if (wrk != NULL) {
+		VTAILQ_REMOVE(&pp->idle, wrk, list);
+		Lck_Unlock(&pp->mtx);
+		wrk->pool_func = task->func;
+		wrk->pool_priv = task->priv;
+		AZ(pthread_cond_signal(&wrk->cond));
+		return (0);
+	}
+
+	switch (how) {
+	case POOL_NO_QUEUE:
+		retval = -1;
+		break;
+	case POOL_QUEUE_FRONT:
+		VTAILQ_INSERT_TAIL(&pp->front_queue, task, list);
+		break;
+	case POOL_QUEUE_BACK:
+		VTAILQ_INSERT_TAIL(&pp->back_queue, task, list);
+		break;
+	default:
+		WRONG("Unknown enum pool_how");
+	}
+	Lck_Unlock(&pp->mtx);
+	return (retval);
+}
+
+/*--------------------------------------------------------------------
  * This is the work function for worker threads in the pool.
  */
 
@@ -183,6 +230,7 @@ Pool_Work_Thread(void *priv, struct worker *wrk)
 	struct pool *pp;
 	int stats_clean, i;
 	struct poolsock *ps;
+	struct pool_task *tp;
 
 	CAST_OBJ_NOTNULL(pp, priv, POOL_MAGIC);
 	wrk->pool = pp;
@@ -197,6 +245,24 @@ Pool_Work_Thread(void *priv, struct worker *wrk)
 
 		WS_Reset(wrk->ws, NULL);
 
+		tp = VTAILQ_FIRST(&pp->front_queue);
+		if (tp != NULL)
+			VTAILQ_REMOVE(&pp->front_queue, tp, list);
+
+		if (tp == NULL) {
+			tp = VTAILQ_FIRST(&pp->back_queue);
+			if (tp != NULL)
+				VTAILQ_REMOVE(&pp->back_queue, tp, list);
+		}
+
+		if (tp != NULL) {
+			Lck_Unlock(&pp->mtx);
+			tp->func(pp, tp->priv);
+			stats_clean = WRK_TrySumStat(wrk);
+			Lck_Lock(&pp->mtx);
+			continue;
+		}
+
 		wrk->sp = VTAILQ_FIRST(&pp->queue);
 		if (wrk->sp != NULL) {
 			/* Process queued requests, if any */
@@ -483,6 +549,8 @@ pool_mkpool(unsigned pool_no)
 	VTAILQ_INIT(&pp->queue);
 	VTAILQ_INIT(&pp->idle);
 	VTAILQ_INIT(&pp->socks);
+	VTAILQ_INIT(&pp->front_queue);
+	VTAILQ_INIT(&pp->back_queue);
 	pp->sesspool = SES_NewPool(pp, pool_no);
 	AN(pp->sesspool);
 



From geoff at varnish-cache.org  Tue Jan 24 17:30:28 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Tue, 24 Jan 2012 18:30:28 +0100
Subject: [experimental-ims] f965f4d Forgot to pass the struct wrk* argument to
	the task function.
Message-ID: <E1RpkCS-0005Ow-0m@project.varnish-software.com>

commit f965f4dfe47ebfbd410e784c0c79e9e84455289b
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Jan 23 10:18:50 2012 +0000

    Forgot to pass the struct wrk* argument to the task function.

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index 463517c..5843fa4 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -288,7 +288,7 @@ struct wrk_accept {
 
 /* Worker pool stuff -------------------------------------------------*/
 
-typedef void pool_func_t(struct pool *pp, void *priv);
+typedef void pool_func_t(struct pool *pp, struct worker *wrk, void *priv);
 
 struct pool_task {
 	VTAILQ_ENTRY(pool_task)		list;
diff --git a/bin/varnishd/cache/cache_pool.c b/bin/varnishd/cache/cache_pool.c
index cc81b59..8b247fa 100644
--- a/bin/varnishd/cache/cache_pool.c
+++ b/bin/varnishd/cache/cache_pool.c
@@ -257,7 +257,7 @@ Pool_Work_Thread(void *priv, struct worker *wrk)
 
 		if (tp != NULL) {
 			Lck_Unlock(&pp->mtx);
-			tp->func(pp, tp->priv);
+			tp->func(pp, wrk, tp->priv);
 			stats_clean = WRK_TrySumStat(wrk);
 			Lck_Lock(&pp->mtx);
 			continue;



From geoff at varnish-cache.org  Tue Jan 24 17:30:28 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Tue, 24 Jan 2012 18:30:28 +0100
Subject: [experimental-ims] 7fc7fa7 Move the "schedule a session" code from
	cache_pool to cache_session where it rightfully belongs.
Message-ID: <E1RpkCS-0005P7-23@project.varnish-software.com>

commit 7fc7fa7d430bb715713db16ca98b8105958bccad
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Jan 23 10:37:18 2012 +0000

    Move the "schedule a session" code from cache_pool to cache_session
    where it rightfully belongs.

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index 5843fa4..2f48427 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -942,6 +942,8 @@ int SES_Schedule(struct sess *sp);
 void SES_Handle(struct sess *sp, double now);
 void SES_GetReq(struct sess *sp);
 void SES_ReleaseReq(struct sess *sp);
+pool_func_t SES_pool_task;
+
 
 /* cache_shmlog.c */
 extern struct VSC_C_main *VSC_C_main;
diff --git a/bin/varnishd/cache/cache_pool.c b/bin/varnishd/cache/cache_pool.c
index 8b247fa..a8a2b2a 100644
--- a/bin/varnishd/cache/cache_pool.c
+++ b/bin/varnishd/cache/cache_pool.c
@@ -186,6 +186,8 @@ Pool_Task(struct pool *pp, struct pool_task *task, enum pool_how how)
 	int retval = 0;
 
 	CHECK_OBJ_NOTNULL(pp, POOL_MAGIC);
+	AN(task);
+	AN(task->func);
 
 	Lck_Lock(&pp->mtx);
 
@@ -257,6 +259,7 @@ Pool_Work_Thread(void *priv, struct worker *wrk)
 
 		if (tp != NULL) {
 			Lck_Unlock(&pp->mtx);
+			AN(tp->func);
 			tp->func(pp, wrk, tp->priv);
 			stats_clean = WRK_TrySumStat(wrk);
 			Lck_Lock(&pp->mtx);
@@ -316,27 +319,8 @@ Pool_Work_Thread(void *priv, struct worker *wrk)
 		}
 
 		if (wrk->do_what == pool_do_sess) {
-			CHECK_OBJ_NOTNULL(wrk->sp, SESS_MAGIC);
-			AZ(wrk->ws->r);
-
 			stats_clean = 0;
-			wrk->lastused = NAN;
-
-			AZ(wrk->sp->wrk);
-			THR_SetSession(wrk->sp);
-			wrk->sp->wrk = wrk;
-			CNT_Session(wrk->sp);
-			THR_SetSession(NULL);
-			wrk->sp = NULL;
-
-			WS_Assert(wrk->ws);
-			AZ(wrk->busyobj);
-			AZ(wrk->wrw.wfd);
-			assert(wrk->wlp == wrk->wlb);
-			if (cache_param->diag_bitmap & 0x00040000) {
-				if (wrk->vcl != NULL)
-					VCL_Rel(&wrk->vcl);
-			}
+			SES_pool_task(pp, wrk, wrk->sp);
 		} else if (wrk->do_what == pool_do_nothing) {
 			/* we already did */
 		} else {
diff --git a/bin/varnishd/cache/cache_session.c b/bin/varnishd/cache/cache_session.c
index d1dbaba..ba5f005 100644
--- a/bin/varnishd/cache/cache_session.c
+++ b/bin/varnishd/cache/cache_session.c
@@ -127,6 +127,42 @@ SES_Alloc(void)
 	return (sp);
 }
 
+
+/*--------------------------------------------------------------------
+ * The pool-task function for sessions
+ */
+
+void
+SES_pool_task(struct pool *pp, struct worker *wrk, void *arg)
+{
+	struct sess *sp;
+
+	AN(pp);
+	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
+	CAST_OBJ_NOTNULL(sp, arg, SESS_MAGIC);
+
+	AZ(wrk->ws->r);
+	wrk->lastused = NAN;
+	THR_SetSession(sp);
+	// AZ(wrk->sp);
+	// wrk->sp = sp;
+	AZ(sp->wrk);
+	sp->wrk = wrk;
+	CNT_Session(sp);
+	sp = NULL;
+	/* Cannot access sp now */
+	THR_SetSession(NULL);
+	wrk->sp = NULL;
+	WS_Assert(wrk->ws);
+	AZ(wrk->busyobj);
+	AZ(wrk->wrw.wfd);
+	assert(wrk->wlp == wrk->wlb);
+	if (cache_param->diag_bitmap & 0x00040000) {
+		if (wrk->vcl != NULL)
+			VCL_Rel(&wrk->vcl);
+	}
+}
+
 /*--------------------------------------------------------------------
  * Schedule a session back on a work-thread from its pool
  */



From geoff at varnish-cache.org  Tue Jan 24 17:30:28 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Tue, 24 Jan 2012 18:30:28 +0100
Subject: [experimental-ims] 273788a Move session scheduling to new task api
Message-ID: <E1RpkCS-0005PI-4R@project.varnish-software.com>

commit 273788aee2c1892ed4a1334442dca28533f0bc94
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Jan 23 11:10:02 2012 +0000

    Move session scheduling to new task api

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index 2f48427..43b4eae 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -288,7 +288,7 @@ struct wrk_accept {
 
 /* Worker pool stuff -------------------------------------------------*/
 
-typedef void pool_func_t(struct pool *pp, struct worker *wrk, void *priv);
+typedef void pool_func_t(struct worker *wrk, void *priv);
 
 struct pool_task {
 	VTAILQ_ENTRY(pool_task)		list;
@@ -324,8 +324,7 @@ struct worker {
 	struct dstat		stats;
 
 	/* New Pool stuff */
-	pool_func_t		*pool_func;
-	void			*pool_priv;
+	struct pool_task	task;
 
 	/* Pool stuff */
 	enum e_do_what		do_what;
@@ -914,7 +913,6 @@ void PipeSession(struct sess *sp);
 /* cache_pool.c */
 void Pool_Init(void);
 void Pool_Work_Thread(void *priv, struct worker *w);
-int Pool_Schedule(struct pool *pp, struct sess *sp);
 int Pool_Task(struct pool *pp, struct pool_task *task, enum pool_how how);
 
 #define WRW_IsReleased(w)	((w)->wrw.wfd == NULL)
diff --git a/bin/varnishd/cache/cache_pool.c b/bin/varnishd/cache/cache_pool.c
index a8a2b2a..20dc38a 100644
--- a/bin/varnishd/cache/cache_pool.c
+++ b/bin/varnishd/cache/cache_pool.c
@@ -199,8 +199,8 @@ Pool_Task(struct pool *pp, struct pool_task *task, enum pool_how how)
 	if (wrk != NULL) {
 		VTAILQ_REMOVE(&pp->idle, wrk, list);
 		Lck_Unlock(&pp->mtx);
-		wrk->pool_func = task->func;
-		wrk->pool_priv = task->priv;
+		wrk->task.func = task->func;
+		wrk->task.priv = task->priv;
 		AZ(pthread_cond_signal(&wrk->cond));
 		return (0);
 	}
@@ -210,7 +210,15 @@ Pool_Task(struct pool *pp, struct pool_task *task, enum pool_how how)
 		retval = -1;
 		break;
 	case POOL_QUEUE_FRONT:
-		VTAILQ_INSERT_TAIL(&pp->front_queue, task, list);
+		/* If we have too much in the queue already, refuse. */
+		if (pp->lqueue > (cache_param->queue_max * pp->nthr) / 100) {
+			pp->ndropped++;
+			retval = -1;
+		} else {
+			VTAILQ_INSERT_TAIL(&pp->front_queue, task, list);
+			pp->nqueued++;
+			pp->lqueue++;
+		}
 		break;
 	case POOL_QUEUE_BACK:
 		VTAILQ_INSERT_TAIL(&pp->back_queue, task, list);
@@ -219,6 +227,8 @@ Pool_Task(struct pool *pp, struct pool_task *task, enum pool_how how)
 		WRONG("Unknown enum pool_how");
 	}
 	Lck_Unlock(&pp->mtx);
+	if (how == POOL_QUEUE_FRONT && !retval)
+		AZ(pthread_cond_signal(&pp->herder_cond));
 	return (retval);
 }
 
@@ -248,8 +258,10 @@ Pool_Work_Thread(void *priv, struct worker *wrk)
 		WS_Reset(wrk->ws, NULL);
 
 		tp = VTAILQ_FIRST(&pp->front_queue);
-		if (tp != NULL)
+		if (tp != NULL) {
+			pp->lqueue--;
 			VTAILQ_REMOVE(&pp->front_queue, tp, list);
+		}
 
 		if (tp == NULL) {
 			tp = VTAILQ_FIRST(&pp->back_queue);
@@ -260,7 +272,8 @@ Pool_Work_Thread(void *priv, struct worker *wrk)
 		if (tp != NULL) {
 			Lck_Unlock(&pp->mtx);
 			AN(tp->func);
-			tp->func(pp, wrk, tp->priv);
+			assert(wrk->pool == pp);
+			tp->func(wrk, tp->priv);
 			stats_clean = WRK_TrySumStat(wrk);
 			Lck_Lock(&pp->mtx);
 			continue;
@@ -297,6 +310,17 @@ Pool_Work_Thread(void *priv, struct worker *wrk)
 			(void)Lck_CondWait(&wrk->cond, &pp->mtx, NULL);
 		}
 
+		if (wrk->task.func != NULL) {
+			Lck_Unlock(&pp->mtx);
+			assert(wrk->pool == pp);
+			wrk->task.func(wrk, wrk->task.priv);
+			wrk->task.func = NULL;
+			wrk->task.priv = NULL;
+			stats_clean = WRK_TrySumStat(wrk);
+			Lck_Lock(&pp->mtx);
+			continue;
+		}
+
 		if (wrk->do_what == pool_do_die)
 			break;
 
@@ -320,7 +344,8 @@ Pool_Work_Thread(void *priv, struct worker *wrk)
 
 		if (wrk->do_what == pool_do_sess) {
 			stats_clean = 0;
-			SES_pool_task(pp, wrk, wrk->sp);
+			assert(wrk->pool == pp);
+			SES_pool_task(wrk, wrk->sp);
 		} else if (wrk->do_what == pool_do_nothing) {
 			/* we already did */
 		} else {
@@ -334,56 +359,6 @@ Pool_Work_Thread(void *priv, struct worker *wrk)
 }
 
 /*--------------------------------------------------------------------
- * Queue a workrequest if possible.
- *
- * Return zero if the request was queued, negative if it wasn't.
- */
-
-static int
-pool_queue(struct pool *pp, struct sess *sp)
-{
-	struct worker *wrk;
-
-	Lck_Lock(&pp->mtx);
-
-	/* If there are idle threads, we tickle the first one into action */
-	wrk = VTAILQ_FIRST(&pp->idle);
-	if (wrk != NULL) {
-		VTAILQ_REMOVE(&pp->idle, wrk, list);
-		Lck_Unlock(&pp->mtx);
-		wrk->sp = sp;
-		wrk->do_what = pool_do_sess;
-		AZ(pthread_cond_signal(&wrk->cond));
-		return (0);
-	}
-
-	/* If we have too much in the queue already, refuse. */
-	if (pp->lqueue > (cache_param->queue_max * pp->nthr) / 100) {
-		pp->ndropped++;
-		Lck_Unlock(&pp->mtx);
-		return (-1);
-	}
-
-	VTAILQ_INSERT_TAIL(&pp->queue, sp, list);
-	pp->nqueued++;
-	pp->lqueue++;
-	Lck_Unlock(&pp->mtx);
-	AZ(pthread_cond_signal(&pp->herder_cond));
-	return (0);
-}
-
-/*--------------------------------------------------------------------*/
-
-int
-Pool_Schedule(struct pool *pp, struct sess *sp)
-{
-
-	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
-	AZ(sp->wrk);
-	return(pool_queue(pp, sp));
-}
-
-/*--------------------------------------------------------------------
  * Create another thread, if necessary & possible
  */
 
diff --git a/bin/varnishd/cache/cache_session.c b/bin/varnishd/cache/cache_session.c
index ba5f005..871ed8f 100644
--- a/bin/varnishd/cache/cache_session.c
+++ b/bin/varnishd/cache/cache_session.c
@@ -133,19 +133,20 @@ SES_Alloc(void)
  */
 
 void
-SES_pool_task(struct pool *pp, struct worker *wrk, void *arg)
+SES_pool_task(struct worker *wrk, void *arg)
 {
 	struct sess *sp;
 
-	AN(pp);
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 	CAST_OBJ_NOTNULL(sp, arg, SESS_MAGIC);
 
 	AZ(wrk->ws->r);
 	wrk->lastused = NAN;
 	THR_SetSession(sp);
-	// AZ(wrk->sp);
-	// wrk->sp = sp;
+	if (wrk->sp == NULL)
+		wrk->sp = sp;
+	else
+		assert(wrk->sp == sp);
 	AZ(sp->wrk);
 	sp->wrk = wrk;
 	CNT_Session(sp);
@@ -178,7 +179,11 @@ SES_Schedule(struct sess *sp)
 	CHECK_OBJ_NOTNULL(pp, SESSPOOL_MAGIC);
 	AN(pp->pool);
 
-	if (Pool_Schedule(pp->pool, sp)) {
+	AZ(sp->wrk);
+	sp->task.func = SES_pool_task;
+	sp->task.priv = sp;
+
+	if (Pool_Task(pp->pool, &sp->task, POOL_QUEUE_FRONT)) {
 		VSC_C_main->client_drop_late++;
 		sp->t_idle = VTIM_real();
 		if (sp->req != NULL && sp->req->vcl != NULL) {



From geoff at varnish-cache.org  Tue Jan 24 17:30:28 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Tue, 24 Jan 2012 18:30:28 +0100
Subject: [experimental-ims] 5b969ad Remove the now unused queue of sessions.
Message-ID: <E1RpkCS-0005PU-6L@project.varnish-software.com>

commit 5b969ad5d9d7adc7d7cf09f5cc767965653238f7
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Jan 23 11:25:40 2012 +0000

    Remove the now unused queue of sessions.

diff --git a/bin/varnishd/cache/cache_pool.c b/bin/varnishd/cache/cache_pool.c
index 20dc38a..91c8e24 100644
--- a/bin/varnishd/cache/cache_pool.c
+++ b/bin/varnishd/cache/cache_pool.c
@@ -104,7 +104,6 @@ struct pool {
 	struct workerhead		idle;
 	VTAILQ_HEAD(, pool_task)	front_queue;
 	VTAILQ_HEAD(, pool_task)	back_queue;
-	VTAILQ_HEAD(, sess)		queue;
 	VTAILQ_HEAD(, poolsock)		socks;
 	unsigned			nthr;
 	unsigned			lqueue;
@@ -279,14 +278,7 @@ Pool_Work_Thread(void *priv, struct worker *wrk)
 			continue;
 		}
 
-		wrk->sp = VTAILQ_FIRST(&pp->queue);
-		if (wrk->sp != NULL) {
-			/* Process queued requests, if any */
-			assert(pp->lqueue > 0);
-			VTAILQ_REMOVE(&pp->queue, wrk->sp, list);
-			wrk->do_what = pool_do_sess;
-			pp->lqueue--;
-		} else if (!VTAILQ_EMPTY(&pp->socks)) {
+		if (!VTAILQ_EMPTY(&pp->socks)) {
 			/* Accept on a socket */
 			ps = VTAILQ_FIRST(&pp->socks);
 			VTAILQ_REMOVE(&pp->socks, ps, list);
@@ -505,7 +497,6 @@ pool_mkpool(unsigned pool_no)
 	XXXAN(pp);
 	Lck_New(&pp->mtx, lck_wq);
 
-	VTAILQ_INIT(&pp->queue);
 	VTAILQ_INIT(&pp->idle);
 	VTAILQ_INIT(&pp->socks);
 	VTAILQ_INIT(&pp->front_queue);



From geoff at varnish-cache.org  Tue Jan 24 17:30:28 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Tue, 24 Jan 2012 18:30:28 +0100
Subject: [experimental-ims] 69256c8 Use the list in the embedded task object
	to hold idle threads that saves two pointers in workers.
Message-ID: <E1RpkCS-0005Pf-89@project.varnish-software.com>

commit 69256c8e651979dce1001a4b5d172521786e18c3
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Jan 23 11:48:51 2012 +0000

    Use the list in the embedded task object to hold idle threads that saves
    two pointers in workers.

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index 43b4eae..8bf15ef 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -335,7 +335,6 @@ struct worker {
 
 	pthread_cond_t		cond;
 
-	VTAILQ_ENTRY(worker)	list;
 	struct sess		*sp;
 
 	struct VCL_conf		*vcl;
diff --git a/bin/varnishd/cache/cache_pool.c b/bin/varnishd/cache/cache_pool.c
index 91c8e24..656fbc8 100644
--- a/bin/varnishd/cache/cache_pool.c
+++ b/bin/varnishd/cache/cache_pool.c
@@ -80,7 +80,7 @@ pthread_condattr_setclock(pthread_condattr_t *attr, int foo)
 }
 #endif /* !CLOCK_MONOTONIC */
 
-VTAILQ_HEAD(workerhead, worker);
+VTAILQ_HEAD(taskhead, pool_task);
 
 struct poolsock {
 	unsigned			magic;
@@ -101,9 +101,9 @@ struct pool {
 	pthread_t			herder_thr;
 
 	struct lock			mtx;
-	struct workerhead		idle;
-	VTAILQ_HEAD(, pool_task)	front_queue;
-	VTAILQ_HEAD(, pool_task)	back_queue;
+	struct taskhead			idle_queue;
+	struct taskhead			front_queue;
+	struct taskhead			back_queue;
 	VTAILQ_HEAD(, poolsock)		socks;
 	unsigned			nthr;
 	unsigned			lqueue;
@@ -117,6 +117,27 @@ static struct lock		pool_mtx;
 static pthread_t		thr_pool_herder;
 
 /*--------------------------------------------------------------------
+ */
+
+static struct worker *
+pool_getidleworker(const struct pool *pp, int back)
+{
+	struct pool_task *pt;
+	struct worker *wrk;
+
+	CHECK_OBJ_NOTNULL(pp, POOL_MAGIC);
+	if (back)
+		pt = VTAILQ_LAST(&pp->idle_queue, taskhead);
+	else
+		pt = VTAILQ_FIRST(&pp->idle_queue);
+	if (pt == NULL)
+		return (NULL);
+	AZ(pt->func);
+	CAST_OBJ_NOTNULL(wrk, pt->priv, WORKER_MAGIC);
+	return (wrk);
+}
+
+/*--------------------------------------------------------------------
  * Nobody is accepting on this socket, so we do.
  *
  * As long as we can stick the accepted connection to another thread
@@ -161,10 +182,10 @@ pool_accept(struct pool *pp, struct worker *wrk, const struct poolsock *ps)
 		}
 
 		Lck_Lock(&pp->mtx);
-		if (VTAILQ_EMPTY(&pp->idle))
+		wrk2 = pool_getidleworker(pp, 0);
+		if (wrk2 == NULL)
 			return (0);
-		wrk2 = VTAILQ_FIRST(&pp->idle);
-		VTAILQ_REMOVE(&pp->idle, wrk2, list);
+		VTAILQ_REMOVE(&pp->idle_queue, &wrk2->task, list);
 		Lck_Unlock(&pp->mtx);
 		assert(sizeof *wa2 == WS_Reserve(wrk2->ws, sizeof *wa2));
 		wa2 = (void*)wrk2->ws->f;
@@ -194,9 +215,9 @@ Pool_Task(struct pool *pp, struct pool_task *task, enum pool_how how)
 	 * The common case first:  Take an idle thread, do it.
 	 */
 
-	wrk = VTAILQ_FIRST(&pp->idle);
+	wrk = pool_getidleworker(pp, 0);
 	if (wrk != NULL) {
-		VTAILQ_REMOVE(&pp->idle, wrk, list);
+		VTAILQ_REMOVE(&pp->idle_queue, &wrk->task, list);
 		Lck_Unlock(&pp->mtx);
 		wrk->task.func = task->func;
 		wrk->task.priv = task->priv;
@@ -296,7 +317,9 @@ Pool_Work_Thread(void *priv, struct worker *wrk)
 			/* Nothing to do: To sleep, perchance to dream ... */
 			if (isnan(wrk->lastused))
 				wrk->lastused = VTIM_real();
-			VTAILQ_INSERT_HEAD(&pp->idle, wrk, list);
+			wrk->task.func = NULL;
+			wrk->task.priv = wrk;
+			VTAILQ_INSERT_HEAD(&pp->idle_queue, &wrk->task, list);
 			if (!stats_clean)
 				WRK_SumStat(wrk);
 			(void)Lck_CondWait(&wrk->cond, &pp->mtx, NULL);
@@ -457,12 +480,12 @@ pool_herder(void *priv)
 		VSC_C_main->sess_queued += pp->nqueued;
 		VSC_C_main->sess_dropped += pp->ndropped;
 		pp->nqueued = pp->ndropped = 0;
-		wrk = VTAILQ_LAST(&pp->idle, workerhead);
+		wrk = pool_getidleworker(pp, 1);
 		if (wrk != NULL &&
 		    (wrk->lastused < t_idle ||
 		    pp->nthr > cache_param->wthread_max)) {
-			VTAILQ_REMOVE(&pp->idle, wrk, list);
-		} else
+			VTAILQ_REMOVE(&pp->idle_queue, &wrk->task, list);
+		} else 
 			wrk = NULL;
 		Lck_Unlock(&pp->mtx);
 
@@ -497,7 +520,7 @@ pool_mkpool(unsigned pool_no)
 	XXXAN(pp);
 	Lck_New(&pp->mtx, lck_wq);
 
-	VTAILQ_INIT(&pp->idle);
+	VTAILQ_INIT(&pp->idle_queue);
 	VTAILQ_INIT(&pp->socks);
 	VTAILQ_INIT(&pp->front_queue);
 	VTAILQ_INIT(&pp->back_queue);



From geoff at varnish-cache.org  Tue Jan 24 17:30:28 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Tue, 24 Jan 2012 18:30:28 +0100
Subject: [experimental-ims] ac78724 Move the freshly acceptted session code
	from cache_pool.c to cache_session.c
Message-ID: <E1RpkCS-0005Pr-AH@project.varnish-software.com>

commit ac7872447397c2d9a2ff2f74c781a1ad6853aefa
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Jan 23 12:33:27 2012 +0000

    Move the freshly acceptted session code from cache_pool.c to
    cache_session.c

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index 8bf15ef..fffb777 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -928,7 +928,6 @@ void WRW_Sendfile(struct worker *w, int fd, off_t off, unsigned len);
 #endif  /* SENDFILE_WORKS */
 
 /* cache_session.c [SES] */
-struct sess *SES_New(struct sesspool *pp);
 struct sess *SES_Alloc(void);
 void SES_Close(struct sess *sp, const char *reason);
 void SES_Delete(struct sess *sp, const char *reason, double now);
@@ -940,7 +939,7 @@ void SES_Handle(struct sess *sp, double now);
 void SES_GetReq(struct sess *sp);
 void SES_ReleaseReq(struct sess *sp);
 pool_func_t SES_pool_task;
-
+pool_func_t SES_pool_accept_task;
 
 /* cache_shmlog.c */
 extern struct VSC_C_main *VSC_C_main;
diff --git a/bin/varnishd/cache/cache_pool.c b/bin/varnishd/cache/cache_pool.c
index 656fbc8..28ae2b1 100644
--- a/bin/varnishd/cache/cache_pool.c
+++ b/bin/varnishd/cache/cache_pool.c
@@ -190,7 +190,8 @@ pool_accept(struct pool *pp, struct worker *wrk, const struct poolsock *ps)
 		assert(sizeof *wa2 == WS_Reserve(wrk2->ws, sizeof *wa2));
 		wa2 = (void*)wrk2->ws->f;
 		memcpy(wa2, wa, sizeof *wa);
-		wrk2->do_what = pool_do_accept;
+		wrk2->task.func = SES_pool_accept_task;
+		wrk2->task.priv = pp->sesspool;
 		AZ(pthread_cond_signal(&wrk2->cond));
 	}
 }
@@ -342,19 +343,11 @@ Pool_Work_Thread(void *priv, struct worker *wrk)
 		Lck_Unlock(&pp->mtx);
 
 		if (wrk->do_what == pool_do_accept) {
-			/* Turn accepted socket into a session */
-			AZ(wrk->sp);
-			AN(wrk->ws->r);
-			wrk->sp = SES_New(pp->sesspool);
-			if (wrk->sp == NULL) {
-				VCA_FailSess(wrk);
+			SES_pool_accept_task(wrk, pp->sesspool);
+			if (wrk->sp == NULL)
 				wrk->do_what = pool_do_nothing;
-			} else {
-				VCA_SetupSess(wrk);
-				wrk->sp->step = STP_FIRST;
+			else
 				wrk->do_what = pool_do_sess;
-			}
-			WS_Release(wrk->ws, 0);
 		}
 
 		if (wrk->do_what == pool_do_sess) {
diff --git a/bin/varnishd/cache/cache_session.c b/bin/varnishd/cache/cache_session.c
index 871ed8f..1079bb9 100644
--- a/bin/varnishd/cache/cache_session.c
+++ b/bin/varnishd/cache/cache_session.c
@@ -97,8 +97,8 @@ ses_setup(struct sess *sp)
  * Get a new session, preferably by recycling an already ready one
  */
 
-struct sess *
-SES_New(struct sesspool *pp)
+static struct sess *
+ses_new(struct sesspool *pp)
 {
 	struct sess *sp;
 
@@ -127,6 +127,33 @@ SES_Alloc(void)
 	return (sp);
 }
 
+ /*--------------------------------------------------------------------
+ * The pool-task for a newly accepted session
+ */
+
+void
+SES_pool_accept_task(struct worker *wrk, void *arg)
+{
+	struct sesspool *pp;
+
+	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
+	CAST_OBJ_NOTNULL(pp, arg, SESSPOOL_MAGIC);
+
+	/* Turn accepted socket into a session */
+	AZ(wrk->sp);
+	AN(wrk->ws->r);
+	wrk->sp = ses_new(pp);
+	if (wrk->sp == NULL) {
+		VCA_FailSess(wrk);
+		return;
+	}
+	VCA_SetupSess(wrk);
+	wrk->sp->step = STP_FIRST;
+	WS_Release(wrk->ws, 0);
+	SES_pool_task(wrk, wrk->sp);
+}
+
+
 
 /*--------------------------------------------------------------------
  * The pool-task function for sessions



From geoff at varnish-cache.org  Tue Jan 24 17:30:28 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Tue, 24 Jan 2012 18:30:28 +0100
Subject: [experimental-ims] b04b83b Move the accept sockets into the new
	pool-task API and GC all the cruft now not needed.
Message-ID: <E1RpkCS-0005Q2-C5@project.varnish-software.com>

commit b04b83baca73fff2fec268a16946f12665099215
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Jan 23 13:09:21 2012 +0000

    Move the accept sockets into the new pool-task API and GC all the
    cruft now not needed.

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index fffb777..3c20777 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -304,14 +304,6 @@ enum pool_how {
 
 /*--------------------------------------------------------------------*/
 
-enum e_do_what {
-	pool_do_inval = 0,
-	pool_do_sess,
-	pool_do_accept,
-	pool_do_nothing,
-	pool_do_die,
-};
-
 struct worker {
 	unsigned		magic;
 #define WORKER_MAGIC		0x6391adcf
@@ -323,12 +315,8 @@ struct worker {
 	void			*nhashpriv;
 	struct dstat		stats;
 
-	/* New Pool stuff */
 	struct pool_task	task;
 
-	/* Pool stuff */
-	enum e_do_what		do_what;
-
 	double			lastused;
 
 	struct wrw		wrw;
@@ -938,7 +926,6 @@ int SES_Schedule(struct sess *sp);
 void SES_Handle(struct sess *sp, double now);
 void SES_GetReq(struct sess *sp);
 void SES_ReleaseReq(struct sess *sp);
-pool_func_t SES_pool_task;
 pool_func_t SES_pool_accept_task;
 
 /* cache_shmlog.c */
diff --git a/bin/varnishd/cache/cache_pool.c b/bin/varnishd/cache/cache_pool.c
index 28ae2b1..dab5b11 100644
--- a/bin/varnishd/cache/cache_pool.c
+++ b/bin/varnishd/cache/cache_pool.c
@@ -85,8 +85,8 @@ VTAILQ_HEAD(taskhead, pool_task);
 struct poolsock {
 	unsigned			magic;
 #define POOLSOCK_MAGIC			0x1b0a2d38
-	VTAILQ_ENTRY(poolsock)		list;
 	struct listen_sock		*lsock;
+	struct pool_task		task;
 };
 
 /* Number of work requests queued in excess of worker threads available */
@@ -104,7 +104,6 @@ struct pool {
 	struct taskhead			idle_queue;
 	struct taskhead			front_queue;
 	struct taskhead			back_queue;
-	VTAILQ_HEAD(, poolsock)		socks;
 	unsigned			nthr;
 	unsigned			lqueue;
 	unsigned			last_lqueue;
@@ -126,6 +125,7 @@ pool_getidleworker(const struct pool *pp, int back)
 	struct worker *wrk;
 
 	CHECK_OBJ_NOTNULL(pp, POOL_MAGIC);
+	Lck_AssertHeld(&pp->mtx);
 	if (back)
 		pt = VTAILQ_LAST(&pp->idle_queue, taskhead);
 	else
@@ -141,28 +141,27 @@ pool_getidleworker(const struct pool *pp, int back)
  * Nobody is accepting on this socket, so we do.
  *
  * As long as we can stick the accepted connection to another thread
- * we do so, otherwise we return and handle it ourselves.
- *
- * Notice calling convention:  Called locked and returns locked, but
- * works lock in the meantime.
+ * we do so, otherwise we put the socket back on the "BACK" queue
+ * and handle the new connection ourselves.
  *
- * We store data about the accept in reserved workspace, it is only used
- * for a brief moment and it takes up around 144 bytes.
+ * We store data about the accept in reserved workspace on the reserved
+ * worker workspace.  SES_pool_accept_task() knows about this.
  */
 
-static int
-pool_accept(struct pool *pp, struct worker *wrk, const struct poolsock *ps)
+static void
+pool_accept(struct worker *wrk, void *arg)
 {
 	struct worker *wrk2;
 	struct wrk_accept *wa, *wa2;
+	struct pool *pp;
+	struct poolsock *ps;
 
-	CHECK_OBJ_NOTNULL(pp, POOL_MAGIC);
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
-	CHECK_OBJ_NOTNULL(ps, POOLSOCK_MAGIC);
+	pp = wrk->pool;
+	CHECK_OBJ_NOTNULL(pp, POOL_MAGIC);
+	CAST_OBJ_NOTNULL(ps, arg, POOLSOCK_MAGIC);
 
 	CHECK_OBJ_NOTNULL(ps->lsock, LISTEN_SOCK_MAGIC);
-	Lck_AssertHeld(&pp->mtx);
-	Lck_Unlock(&pp->mtx);
 	assert(sizeof *wa == WS_Reserve(wrk->ws, sizeof *wa));
 	wa = (void*)wrk->ws->f;
 	while (1) {
@@ -171,8 +170,9 @@ pool_accept(struct pool *pp, struct worker *wrk, const struct poolsock *ps)
 
 		if (ps->lsock->sock < 0) {
 			/* Socket Shutdown */
-			Lck_Lock(&pp->mtx);
-			return (-1);
+			FREE_OBJ(ps);
+			WS_Release(wrk->ws, 0);
+			return;
 		}
 		if (VCA_Accept(ps->lsock, wa) < 0) {
 			wrk->stats.sess_fail++;
@@ -183,8 +183,13 @@ pool_accept(struct pool *pp, struct worker *wrk, const struct poolsock *ps)
 
 		Lck_Lock(&pp->mtx);
 		wrk2 = pool_getidleworker(pp, 0);
-		if (wrk2 == NULL)
-			return (0);
+		if (wrk2 == NULL) {
+			/* No idle threads, do it ourselves */
+			Lck_Unlock(&pp->mtx);
+			AZ(Pool_Task(pp, &ps->task, POOL_QUEUE_BACK));
+			SES_pool_accept_task(wrk, pp->sesspool);
+			return;
+		}
 		VTAILQ_REMOVE(&pp->idle_queue, &wrk2->task, list);
 		Lck_Unlock(&pp->mtx);
 		assert(sizeof *wa2 == WS_Reserve(wrk2->ws, sizeof *wa2));
@@ -248,7 +253,7 @@ Pool_Task(struct pool *pp, struct pool_task *task, enum pool_how how)
 		WRONG("Unknown enum pool_how");
 	}
 	Lck_Unlock(&pp->mtx);
-	if (how == POOL_QUEUE_FRONT && !retval)
+	if (retval)
 		AZ(pthread_cond_signal(&pp->herder_cond));
 	return (retval);
 }
@@ -261,18 +266,14 @@ void
 Pool_Work_Thread(void *priv, struct worker *wrk)
 {
 	struct pool *pp;
-	int stats_clean, i;
-	struct poolsock *ps;
+	int stats_clean;
 	struct pool_task *tp;
 
 	CAST_OBJ_NOTNULL(pp, priv, POOL_MAGIC);
 	wrk->pool = pp;
-	Lck_Lock(&pp->mtx);
 	stats_clean = 1;
 	while (1) {
-
-		Lck_AssertHeld(&pp->mtx);
-		wrk->do_what = pool_do_inval;
+		Lck_Lock(&pp->mtx);
 
 		CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 
@@ -282,39 +283,13 @@ Pool_Work_Thread(void *priv, struct worker *wrk)
 		if (tp != NULL) {
 			pp->lqueue--;
 			VTAILQ_REMOVE(&pp->front_queue, tp, list);
-		}
-
-		if (tp == NULL) {
+		} else {
 			tp = VTAILQ_FIRST(&pp->back_queue);
 			if (tp != NULL)
 				VTAILQ_REMOVE(&pp->back_queue, tp, list);
 		}
 
-		if (tp != NULL) {
-			Lck_Unlock(&pp->mtx);
-			AN(tp->func);
-			assert(wrk->pool == pp);
-			tp->func(wrk, tp->priv);
-			stats_clean = WRK_TrySumStat(wrk);
-			Lck_Lock(&pp->mtx);
-			continue;
-		}
-
-		if (!VTAILQ_EMPTY(&pp->socks)) {
-			/* Accept on a socket */
-			ps = VTAILQ_FIRST(&pp->socks);
-			VTAILQ_REMOVE(&pp->socks, ps, list);
-			i = pool_accept(pp, wrk, ps);
-			Lck_AssertHeld(&pp->mtx);
-			if (i < 0) {
-				/* Socket Shutdown */
-				FREE_OBJ(ps);
-				WS_Release(wrk->ws, 0);
-				continue;
-			}
-			VTAILQ_INSERT_TAIL(&pp->socks, ps, list);
-			wrk->do_what = pool_do_accept;
-		} else {
+		if (tp == NULL) {
 			/* Nothing to do: To sleep, perchance to dream ... */
 			if (isnan(wrk->lastused))
 				wrk->lastused = VTIM_real();
@@ -324,45 +299,17 @@ Pool_Work_Thread(void *priv, struct worker *wrk)
 			if (!stats_clean)
 				WRK_SumStat(wrk);
 			(void)Lck_CondWait(&wrk->cond, &pp->mtx, NULL);
+			tp = &wrk->task;
 		}
-
-		if (wrk->task.func != NULL) {
-			Lck_Unlock(&pp->mtx);
-			assert(wrk->pool == pp);
-			wrk->task.func(wrk, wrk->task.priv);
-			wrk->task.func = NULL;
-			wrk->task.priv = NULL;
-			stats_clean = WRK_TrySumStat(wrk);
-			Lck_Lock(&pp->mtx);
-			continue;
-		}
-
-		if (wrk->do_what == pool_do_die)
-			break;
-
 		Lck_Unlock(&pp->mtx);
 
-		if (wrk->do_what == pool_do_accept) {
-			SES_pool_accept_task(wrk, pp->sesspool);
-			if (wrk->sp == NULL)
-				wrk->do_what = pool_do_nothing;
-			else
-				wrk->do_what = pool_do_sess;
-		}
+		if (tp->func == NULL)
+			break;
 
-		if (wrk->do_what == pool_do_sess) {
-			stats_clean = 0;
-			assert(wrk->pool == pp);
-			SES_pool_task(wrk, wrk->sp);
-		} else if (wrk->do_what == pool_do_nothing) {
-			/* we already did */
-		} else {
-			WRONG("Invalid wrk->do_what");
-		}
+		assert(wrk->pool == pp);
+		tp->func(wrk, tp->priv);
 		stats_clean = WRK_TrySumStat(wrk);
-		Lck_Lock(&pp->mtx);
 	}
-	Lck_Unlock(&pp->mtx);
 	wrk->pool = NULL;
 }
 
@@ -474,11 +421,10 @@ pool_herder(void *priv)
 		VSC_C_main->sess_dropped += pp->ndropped;
 		pp->nqueued = pp->ndropped = 0;
 		wrk = pool_getidleworker(pp, 1);
-		if (wrk != NULL &&
-		    (wrk->lastused < t_idle ||
+		if (wrk != NULL && (wrk->lastused < t_idle ||
 		    pp->nthr > cache_param->wthread_max)) {
 			VTAILQ_REMOVE(&pp->idle_queue, &wrk->task, list);
-		} else 
+		} else
 			wrk = NULL;
 		Lck_Unlock(&pp->mtx);
 
@@ -490,7 +436,8 @@ pool_herder(void *priv)
 			VSC_C_main->threads_destroyed++;
 			Lck_Unlock(&pool_mtx);
 			AZ(wrk->sp);
-			wrk->do_what = pool_do_die;
+			wrk->task.func = NULL;
+			wrk->task.priv = NULL;
 			AZ(pthread_cond_signal(&wrk->cond));
 		}
 	}
@@ -514,7 +461,6 @@ pool_mkpool(unsigned pool_no)
 	Lck_New(&pp->mtx, lck_wq);
 
 	VTAILQ_INIT(&pp->idle_queue);
-	VTAILQ_INIT(&pp->socks);
 	VTAILQ_INIT(&pp->front_queue);
 	VTAILQ_INIT(&pp->back_queue);
 	pp->sesspool = SES_NewPool(pp, pool_no);
@@ -526,7 +472,9 @@ pool_mkpool(unsigned pool_no)
 		ALLOC_OBJ(ps, POOLSOCK_MAGIC);
 		XXXAN(ps);
 		ps->lsock = ls;
-		VTAILQ_INSERT_TAIL(&pp->socks, ps, list);
+		ps->task.func = pool_accept;
+		ps->task.priv = ps;
+		AZ(Pool_Task(pp, &ps->task, POOL_QUEUE_BACK));
 	}
 
 	AZ(pthread_condattr_init(&cv_attr));
diff --git a/bin/varnishd/cache/cache_session.c b/bin/varnishd/cache/cache_session.c
index 1079bb9..3e144c1 100644
--- a/bin/varnishd/cache/cache_session.c
+++ b/bin/varnishd/cache/cache_session.c
@@ -127,40 +127,12 @@ SES_Alloc(void)
 	return (sp);
 }
 
- /*--------------------------------------------------------------------
- * The pool-task for a newly accepted session
- */
-
-void
-SES_pool_accept_task(struct worker *wrk, void *arg)
-{
-	struct sesspool *pp;
-
-	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
-	CAST_OBJ_NOTNULL(pp, arg, SESSPOOL_MAGIC);
-
-	/* Turn accepted socket into a session */
-	AZ(wrk->sp);
-	AN(wrk->ws->r);
-	wrk->sp = ses_new(pp);
-	if (wrk->sp == NULL) {
-		VCA_FailSess(wrk);
-		return;
-	}
-	VCA_SetupSess(wrk);
-	wrk->sp->step = STP_FIRST;
-	WS_Release(wrk->ws, 0);
-	SES_pool_task(wrk, wrk->sp);
-}
-
-
-
 /*--------------------------------------------------------------------
  * The pool-task function for sessions
  */
 
-void
-SES_pool_task(struct worker *wrk, void *arg)
+static void
+ses_pool_task(struct worker *wrk, void *arg)
 {
 	struct sess *sp;
 
@@ -192,6 +164,32 @@ SES_pool_task(struct worker *wrk, void *arg)
 }
 
 /*--------------------------------------------------------------------
+ * The pool-task for a newly accepted session
+ */
+
+void
+SES_pool_accept_task(struct worker *wrk, void *arg)
+{
+	struct sesspool *pp;
+
+	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
+	CAST_OBJ_NOTNULL(pp, arg, SESSPOOL_MAGIC);
+
+	/* Turn accepted socket into a session */
+	AZ(wrk->sp);
+	AN(wrk->ws->r);
+	wrk->sp = ses_new(pp);
+	if (wrk->sp == NULL) {
+		VCA_FailSess(wrk);
+		return;
+	}
+	VCA_SetupSess(wrk);
+	wrk->sp->step = STP_FIRST;
+	WS_Release(wrk->ws, 0);
+	ses_pool_task(wrk, wrk->sp);
+}
+
+/*--------------------------------------------------------------------
  * Schedule a session back on a work-thread from its pool
  */
 
@@ -207,7 +205,7 @@ SES_Schedule(struct sess *sp)
 	AN(pp->pool);
 
 	AZ(sp->wrk);
-	sp->task.func = SES_pool_task;
+	sp->task.func = ses_pool_task;
 	sp->task.priv = sp;
 
 	if (Pool_Task(pp->pool, &sp->task, POOL_QUEUE_FRONT)) {



From geoff at varnish-cache.org  Tue Jan 24 17:30:28 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Tue, 24 Jan 2012 18:30:28 +0100
Subject: [experimental-ims] e5a0c62 Don't try to stream when there is no body.
Message-ID: <E1RpkCS-0005QE-E5@project.varnish-software.com>

commit e5a0c62c450dea5efa86919f56fff5a97149b821
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Sun Jan 22 16:24:08 2012 +0100

    Don't try to stream when there is no body.

diff --git a/bin/varnishd/cache/cache_center.c b/bin/varnishd/cache/cache_center.c
index b03ba4b..28ef0bf 100644
--- a/bin/varnishd/cache/cache_center.c
+++ b/bin/varnishd/cache/cache_center.c
@@ -812,6 +812,10 @@ cnt_fetchbody(struct sess *sp, struct worker *wrk, struct req *req)
 	if (!req->wantbody)
 		wrk->busyobj->do_stream = 0;
 
+	/* No reason to try streaming a non-existing body */
+	if (wrk->busyobj->body_status == BS_NONE)
+		wrk->busyobj->do_stream = 0;
+
 	l = http_EstimateWS(wrk->busyobj->beresp,
 	    pass ? HTTPH_R_PASS : HTTPH_A_INS, &nhttp);
 



From geoff at varnish-cache.org  Tue Jan 24 17:30:28 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Tue, 24 Jan 2012 18:30:28 +0100
Subject: [experimental-ims] 0d03ebb Remove an old wart of dubious origin. I
	belive that this originally was related to the entry parameters
	of cnt_error(), but I belive it has long been OBE,
	and now gets in the way of streaming.
Message-ID: <E1RpkCS-0005QR-GR@project.varnish-software.com>

commit 0d03ebb6b68fa50d88463566a40bb341698ec559
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Jan 23 19:54:31 2012 +0000

    Remove an old wart of dubious origin.  I belive that this originally
    was related to the entry parameters of cnt_error(), but I belive it
    has long been OBE, and now gets in the way of streaming.

diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c
index 7070612..28de3b6 100644
--- a/bin/varnishd/cache/cache_fetch.c
+++ b/bin/varnishd/cache/cache_fetch.c
@@ -578,12 +578,6 @@ FetchBody(struct worker *wrk, struct object *obj)
 
 	if (cls < 0) {
 		wrk->stats.fetch_failed++;
-		/* XXX: Wouldn't this store automatically be released ? */
-		while (!VTAILQ_EMPTY(&obj->store)) {
-			st = VTAILQ_FIRST(&obj->store);
-			VTAILQ_REMOVE(&obj->store, st, list);
-			STV_free(st);
-		}
 		VDI_CloseFd(wrk, &bo->vbc);
 		obj->len = 0;
 		return (__LINE__);



From geoff at varnish-cache.org  Tue Jan 24 17:30:28 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Tue, 24 Jan 2012 18:30:28 +0100
Subject: [experimental-ims] 7bbc60f Untangle cnt_error() entrance criteria:
Message-ID: <E1RpkCS-0005Qc-IA@project.varnish-software.com>

commit 7bbc60f7bf32b0b6e3274445b900caaa20766cdf
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Jan 23 20:03:08 2012 +0000

    Untangle cnt_error() entrance criteria:
    
    We should never come in with a busyobj or obj.
    
    Always allocate obj from Transient storage.

diff --git a/bin/varnishd/cache/cache_center.c b/bin/varnishd/cache/cache_center.c
index 28ef0bf..0c08dd2 100644
--- a/bin/varnishd/cache/cache_center.c
+++ b/bin/varnishd/cache/cache_center.c
@@ -488,32 +488,25 @@ cnt_error(struct sess *sp, struct worker *wrk, struct req *req)
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
 
+	AZ(req->objcore);
+	AZ(req->obj);
+	AZ(wrk->busyobj);
+	wrk->busyobj = VBO_GetBusyObj(wrk);
+	req->obj = STV_NewObject(wrk, TRANSIENT_STORAGE,
+	    cache_param->http_resp_size,
+	    (uint16_t)cache_param->http_max_hdr);
 	if (req->obj == NULL) {
-		HSH_Prealloc(sp);
-		AZ(wrk->busyobj);
-		wrk->busyobj = VBO_GetBusyObj(wrk);
-		req->obj = STV_NewObject(wrk, NULL, cache_param->http_resp_size,
-		     (uint16_t)cache_param->http_max_hdr);
-		if (req->obj == NULL)
-			req->obj = STV_NewObject(wrk, TRANSIENT_STORAGE,
-			    cache_param->http_resp_size,
-			    (uint16_t)cache_param->http_max_hdr);
-		if (req->obj == NULL) {
-			req->doclose = "Out of objects";
-			req->director = NULL;
-			http_Setup(wrk->busyobj->beresp, NULL);
-			http_Setup(wrk->busyobj->bereq, NULL);
-			sp->step = STP_DONE;
-			return(0);
-		}
-		AN(req->obj);
-		req->obj->xid = req->xid;
-		req->obj->exp.entered = sp->t_req;
-	} else {
-		CHECK_OBJ_NOTNULL(wrk->busyobj, BUSYOBJ_MAGIC);
-		/* XXX: Null the headers ? */
+		req->doclose = "Out of objects";
+		req->director = NULL;
+		http_Setup(wrk->busyobj->beresp, NULL);
+		http_Setup(wrk->busyobj->bereq, NULL);
+		sp->step = STP_DONE;
+		return(0);
 	}
 	CHECK_OBJ_NOTNULL(req->obj, OBJECT_MAGIC);
+	req->obj->xid = req->xid;
+	req->obj->exp.entered = sp->t_req;
+
 	h = req->obj->http;
 
 	if (req->err_code < 100 || req->err_code > 999)



From geoff at varnish-cache.org  Tue Jan 24 17:30:28 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Tue, 24 Jan 2012 18:30:28 +0100
Subject: [experimental-ims] 4c1abcb Go over the first couple of cache_center
	states with a bucket of polish.
Message-ID: <E1RpkCS-0005Qn-Kr@project.varnish-software.com>

commit 4c1abcb322f00f83c3655ee5f772202c4b1f2f18
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Jan 23 20:53:06 2012 +0000

    Go over the first couple of cache_center states with a bucket of polish.
    
    Note that we get into vcl_error{} on 413 and 417 also.
    
    Rearrange the order in which we do things in cnt_start, to be more
    consistent and correct: Process Connection: and Expect: headers
    before we snapshot and possibly head into vcl_error{}.

diff --git a/bin/varnishd/cache/cache_center.c b/bin/varnishd/cache/cache_center.c
index 0c08dd2..964bd3b 100644
--- a/bin/varnishd/cache/cache_center.c
+++ b/bin/varnishd/cache/cache_center.c
@@ -81,15 +81,15 @@ static unsigned xids;
 
 /*--------------------------------------------------------------------
  * WAIT
- * Wait (briefly) until we have a full request in our htc.
+ * Collect the request from the client.
  *
 DOT subgraph xcluster_wait {
 DOT	wait [
 DOT		shape=box
-DOT		label="cnt_wait:\nwait for\nrequest"
+DOT		label="cnt_wait:\nwait for\ncomplete\nrequest"
 DOT	]
 DOT	herding [shape=hexagon]
-DOT	wait -> start [label="got req"]
+DOT	wait -> start [label="got req",style=bold,color=green]
 DOT	wait -> "SES_Delete()" [label="errors"]
 DOT	wait -> herding [label="timeout_linger"]
 DOT	herding -> wait [label="fd read_ready"]
@@ -102,9 +102,11 @@ cnt_wait(struct sess *sp, struct worker *wrk, struct req *req)
 	int i, j, tmo;
 	struct pollfd pfd[1];
 	double now, when;
+	const char *why = NULL;
 
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
+
 	if (req == NULL) {
 		SES_GetReq(sp);
 		req = sp->req;
@@ -136,25 +138,20 @@ cnt_wait(struct sess *sp, struct worker *wrk, struct req *req)
 		if (i == 1) {
 			/* Got it, run with it */
 			sp->t_req = now;
+			sp->step = STP_START;
+			return (0);
+		} else if (i == -1) {
+			why = "EOF";
 			break;
-		}
-		if (i == -1) {
-			SES_Charge(sp);
-			SES_Delete(sp, "EOF", now);
-			return (1);
-		}
-		if (i == -2) {
-			SES_Charge(sp);
-			SES_Delete(sp, "overflow", now);
-			return (1);
-		}
-		if (i == -3) {
+		} else if (i == -2) {
+			why = "overflow";
+			break;
+		} else if (i == -3) {
 			/* Nothing but whitespace */
 			when = sp->t_idle + cache_param->timeout_idle;
 			if (when < now) {
-				SES_Charge(sp);
-				SES_Delete(sp, "timeout", now);
-				return (1);
+				why = "timeout";
+				break;
 			}
 			when = sp->t_idle + cache_param->timeout_linger;
 			tmo = (int)(1e3 * (when - now));
@@ -171,14 +168,14 @@ cnt_wait(struct sess *sp, struct worker *wrk, struct req *req)
 			when = sp->t_req + cache_param->timeout_req;
 			tmo = (int)(1e3 * (when - now));
 			if (when < now || tmo == 0) {
-				SES_Charge(sp);
-				SES_Delete(sp, "req timeout", now);
-				return (1);
+				why = "req timeout";
+				break;
 			}
 		}
 	}
-	sp->step = STP_START;
-	return (0);
+	SES_Charge(sp);
+	SES_Delete(sp, why, now);
+	return (1);
 }
 
 /*--------------------------------------------------------------------
@@ -1014,10 +1011,10 @@ cnt_streambody(struct sess *sp, struct worker *wrk, struct req *req)
 DOT subgraph xcluster_first {
 DOT	first [
 DOT		shape=box
-DOT		label="cnt_first:\nSockaddr's"
+DOT		label="cnt_first:\nrender\naddresses"
 DOT	]
 DOT }
-DOT first -> wait
+DOT first -> wait [style=bold,color=green]
  */
 
 static int
@@ -1543,7 +1540,7 @@ cnt_recv(struct sess *sp, struct worker *wrk, struct req *req)
 
 /*--------------------------------------------------------------------
  * START
- * Handle a request.
+ * First time we see a request
  *
 DOT start [
 DOT	shape=box
@@ -1556,7 +1553,6 @@ DOT start -> DONE [label=errors]
 static int
 cnt_start(struct sess *sp, struct worker *wrk, struct req *req)
 {
-	uint16_t done;
 	char *p;
 	const char *r = "HTTP/1.1 100 Continue\r\n\r\n";
 
@@ -1566,7 +1562,6 @@ cnt_start(struct sess *sp, struct worker *wrk, struct req *req)
 	AZ(req->restarts);
 	AZ(req->obj);
 	AZ(req->vcl);
-	EXP_Clr(&req->exp);
 	AZ(req->esi_level);
 
 	/* Update stats of various sorts */
@@ -1583,52 +1578,43 @@ cnt_start(struct sess *sp, struct worker *wrk, struct req *req)
 	req->vcl = wrk->vcl;
 	wrk->vcl = NULL;
 
+	EXP_Clr(&req->exp);
+
 	http_Setup(req->http, req->ws);
-	done = http_DissectRequest(sp);
+	req->err_code = http_DissectRequest(sp);
 
 	/* If we could not even parse the request, just close */
-	if (done == 400) {
+	if (req->err_code == 400) {
 		sp->step = STP_DONE;
 		SES_Close(sp, "junk");
 		return (0);
 	}
 
-	/* Catch request snapshot */
 	req->ws_req = WS_Snapshot(req->ws);
 
-	/* Catch original request, before modification */
-	HTTP_Copy(req->http0, req->http);
-
-	if (done != 0) {
-		req->err_code = done;
-		sp->step = STP_ERROR;
-		return (0);
-	}
-
 	req->doclose = http_DoConnection(req->http);
 
-	/* XXX: Handle TRACE & OPTIONS of Max-Forwards = 0 */
-
 	/*
-	 * Handle Expect headers
+	 * We want to deal with Expect: headers the first time we
+	 * attempt the request, and remove them before we move on.
 	 */
-	if (http_GetHdr(req->http, H_Expect, &p)) {
+	if (req->err_code == 0 && http_GetHdr(req->http, H_Expect, &p)) {
 		if (strcasecmp(p, "100-continue")) {
 			req->err_code = 417;
-			sp->step = STP_ERROR;
+		} else if (strlen(r) != write(sp->fd, r, strlen(r))) {
+			sp->step = STP_DONE;
+			SES_Close(sp, "remote closed");
 			return (0);
 		}
-
-		/* XXX: Don't bother with write failures for now */
-		(void)write(sp->fd, r, strlen(r));
-		/* XXX: When we do ESI includes, this is not removed
-		 * XXX: because we use http0 as our basis.  Believed
-		 * XXX: safe, but potentially confusing.
-		 */
-		http_Unset(req->http, H_Expect);
 	}
+	http_Unset(req->http, H_Expect);
 
-	sp->step = STP_RECV;
+	HTTP_Copy(req->http0, req->http);	/* Copy for restart/ESI use */
+
+	if (req->err_code)
+		sp->step = STP_ERROR;
+	else 
+		sp->step = STP_RECV;
 	return (0);
 }
 
diff --git a/bin/varnishd/default.vcl b/bin/varnishd/default.vcl
index dc4f555..18121b8 100644
--- a/bin/varnishd/default.vcl
+++ b/bin/varnishd/default.vcl
@@ -118,6 +118,9 @@ sub vcl_deliver {
     return (deliver);
 }
 
+/*
+ * We can come here "invisibly" with the following errors:  413, 417 & 503
+ */
 sub vcl_error {
     set obj.http.Content-Type = "text/html; charset=utf-8";
     set obj.http.Retry-After = "5";



From geoff at varnish-cache.org  Tue Jan 24 17:30:28 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Tue, 24 Jan 2012 18:30:28 +0100
Subject: [experimental-ims] 3e0e0ed Fix a test description
Message-ID: <E1RpkCS-0005R3-Nk@project.varnish-software.com>

commit 3e0e0edc2b63c64b0db7cec436e9ae24795659a5
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Jan 23 22:09:04 2012 +0000

    Fix a test description

diff --git a/bin/varnishtest/tests/r00102.vtc b/bin/varnishtest/tests/r00102.vtc
index 6cf81cf..6d2d8aa 100644
--- a/bin/varnishtest/tests/r00102.vtc
+++ b/bin/varnishtest/tests/r00102.vtc
@@ -1,4 +1,4 @@
-varnishtest "Test VCL regsub()"
+varnishtest "Test POST->GET conversion"
 
 server s1 {
 	rxreq 



From geoff at varnish-cache.org  Tue Jan 24 17:30:28 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Tue, 24 Jan 2012 18:30:28 +0100
Subject: [experimental-ims] a2c53c8 Be more careful about handling the
	request-body.
Message-ID: <E1RpkCS-0005RG-R6@project.varnish-software.com>

commit a2c53c83db535b4568722558d3c8bb5ff9b57dc7
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Jan 23 22:13:14 2012 +0000

    Be more careful about handling the request-body.

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index 3c20777..aefb077 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -592,7 +592,7 @@ struct req {
 	struct exp		exp;
 	unsigned		cur_method;
 	unsigned		handling;
-	unsigned char		sendbody;
+	unsigned char		reqbodydone;
 	unsigned char		wantbody;
 
 	uint16_t		err_code;
@@ -765,9 +765,9 @@ int EXP_NukeOne(struct worker *w, struct lru *lru);
 struct storage *FetchStorage(struct worker *w, ssize_t sz);
 int FetchError(struct worker *w, const char *error);
 int FetchError2(struct worker *w, const char *error, const char *more);
-int FetchHdr(struct sess *sp, int need_host_hdr);
+int FetchHdr(struct sess *sp, int need_host_hdr, int sendbody);
 int FetchBody(struct worker *w, struct object *obj);
-int FetchReqBody(const struct sess *sp);
+int FetchReqBody(const struct sess *sp, int sendbody);
 void Fetch_Init(void);
 
 /* cache_gzip.c */
diff --git a/bin/varnishd/cache/cache_acceptor.c b/bin/varnishd/cache/cache_acceptor.c
index 7918a6a..434b50c 100644
--- a/bin/varnishd/cache/cache_acceptor.c
+++ b/bin/varnishd/cache/cache_acceptor.c
@@ -253,8 +253,6 @@ VCA_SetupSess(struct worker *wrk)
 	sp->vsl_id = wa->acceptsock | VSL_CLIENTMARKER ;
 	wa->acceptsock = -1;
 	sp->t_open = VTIM_real();
-	sp->t_req = sp->t_open;
-	sp->t_idle = sp->t_open;
 	sp->mylsock = wa->acceptlsock;
 	CHECK_OBJ_NOTNULL(sp->mylsock, LISTEN_SOCK_MAGIC);
 	assert(wa->acceptaddrlen <= sp->sockaddrlen);
diff --git a/bin/varnishd/cache/cache_center.c b/bin/varnishd/cache/cache_center.c
index 964bd3b..b83cb93 100644
--- a/bin/varnishd/cache/cache_center.c
+++ b/bin/varnishd/cache/cache_center.c
@@ -83,6 +83,11 @@ static unsigned xids;
  * WAIT
  * Collect the request from the client.
  *
+ * We "abuse" sp->t_req a bit here:  On input it means "request reception
+ * started at xxx" and is used to trigger timeouts.  On return it means
+ * "we had full request headers by xxx" and is used for reporting by
+ * later steps.
+ *
 DOT subgraph xcluster_wait {
 DOT	wait [
 DOT		shape=box
@@ -446,17 +451,16 @@ cnt_done(struct sess *sp, struct worker *wrk, struct req *req)
 	WS_Reset(req->ws, NULL);
 	WS_Reset(wrk->ws, NULL);
 
+	sp->t_req = sp->t_idle;
 	i = HTC_Reinit(req->htc);
 	if (i == 1) {
 		wrk->stats.sess_pipeline++;
-		sp->t_req = sp->t_idle;
 		sp->step = STP_START;
-		return (0);
+	} else {
+		if (Tlen(req->htc->rxbuf))
+			wrk->stats.sess_readahead++;
+		sp->step = STP_WAIT;
 	}
-	if (Tlen(req->htc->rxbuf))
-		wrk->stats.sess_readahead++;
-	sp->step = STP_WAIT;
-	sp->t_req = sp->t_idle;
 	return (0);
 }
 
@@ -597,7 +601,7 @@ cnt_fetch(struct sess *sp, struct worker *wrk, struct req *req)
 
 	need_host_hdr = !http_GetHdr(wrk->busyobj->bereq, H_Host, NULL);
 
-	i = FetchHdr(sp, need_host_hdr);
+	i = FetchHdr(sp, need_host_hdr, req->objcore == NULL);
 	/*
 	 * If we recycle a backend connection, there is a finite chance
 	 * that the backend closed it before we get a request to it.
@@ -605,7 +609,7 @@ cnt_fetch(struct sess *sp, struct worker *wrk, struct req *req)
 	 */
 	if (i == 1) {
 		VSC_C_main->backend_retry++;
-		i = FetchHdr(sp, need_host_hdr);
+		i = FetchHdr(sp, need_host_hdr, req->objcore == NULL);
 	}
 
 	if (i) {
@@ -1042,6 +1046,8 @@ cnt_first(struct sess *sp, struct worker *wrk)
 
 	wrk->acct_tmp.sess++;
 
+	sp->t_req = sp->t_open;
+	sp->t_idle = sp->t_open;
 	sp->step = STP_WAIT;
 	return (0);
 }
@@ -1080,10 +1086,9 @@ cnt_hit(struct sess *sp, struct worker *wrk, struct req *req)
 	VCL_hit_method(sp);
 
 	if (req->handling == VCL_RET_DELIVER) {
-		/* Dispose of any body part of the request */
-		(void)FetchReqBody(sp);
 		//AZ(wrk->busyobj->bereq->ws);
 		//AZ(wrk->busyobj->beresp->ws);
+		(void)FetchReqBody(sp, 0);
 		sp->step = STP_PREPRESP;
 		return (0);
 	}
@@ -1117,10 +1122,6 @@ cnt_hit(struct sess *sp, struct worker *wrk, struct req *req)
  * encounter a busy object.
  *
 DOT subgraph xcluster_lookup {
-DOT	hash [
-DOT		shape=record
-DOT		label="vcl_hash()|req."
-DOT	]
 DOT	lookup [
 DOT		shape=diamond
 DOT		label="obj in cache ?\ncreate if not"
@@ -1129,7 +1130,6 @@ DOT	lookup2 [
 DOT		shape=diamond
 DOT		label="obj.f.pass ?"
 DOT	]
-DOT	hash -> lookup [label="hash",style=bold,color=green]
 DOT	lookup -> lookup2 [label="yes",style=bold,color=green]
 DOT }
 DOT lookup2 -> hit [label="no", style=bold,color=green]
@@ -1343,7 +1343,7 @@ DOT err_pass [label="ERROR",shape=plaintext]
  */
 
 static int
-cnt_pass(struct sess *sp, struct worker *wrk, struct req *req)
+cnt_pass(struct sess *sp, struct worker *wrk, const struct req *req)
 {
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
@@ -1370,7 +1370,6 @@ cnt_pass(struct sess *sp, struct worker *wrk, struct req *req)
 	}
 	assert(req->handling == VCL_RET_PASS);
 	wrk->acct_tmp.pass++;
-	req->sendbody = 1;
 	sp->step = STP_FETCH;
 	return (0);
 }
@@ -1434,11 +1433,19 @@ cnt_pipe(struct sess *sp, struct worker *wrk, const struct req *req)
 /*--------------------------------------------------------------------
  * RECV
  * We have a complete request, set everything up and start it.
+ * We can come here both with a request from the client and with
+ * a interior request during ESI delivery.
  *
 DOT subgraph xcluster_recv {
 DOT	recv [
 DOT		shape=record
-DOT		label="vcl_recv()|req."
+DOT		label="{cnt_recv:|{vcl_recv\{\}|req.*}}"
+DOT	]
+DOT }
+DOT subgraph xcluster_hash {
+DOT	hash [
+DOT		shape=record
+DOT		label="{cnt_recv:|{vcl_hash\{\}|req.*}}"
 DOT	]
 DOT }
 DOT ESI_REQ [ shape=hexagon ]
@@ -1449,6 +1456,7 @@ DOT recv -> pass2 [label="pass",style=bold,color=red]
 DOT recv -> err_recv [label="error"]
 DOT err_recv [label="ERROR",shape=plaintext]
 DOT recv -> hash [label="lookup",style=bold,color=green]
+DOT hash -> lookup [label="hash",style=bold,color=green]
  */
 
 static int
@@ -1481,6 +1489,11 @@ cnt_recv(struct sess *sp, struct worker *wrk, struct req *req)
 	recv_handling = req->handling;
 
 	if (req->restarts >= cache_param->max_restarts) {
+		/*
+		 * XXX: Why not before vcl_recv{} ?  We go to vcl_error{}
+		 * XXX: without vcl_recv{} on 413 and 417 already.
+		 * XXX tell vcl_error why we come
+		 */
 		if (req->err_code == 0)
 			req->err_code = 503;
 		sp->step = STP_ERROR;
@@ -1499,7 +1512,7 @@ cnt_recv(struct sess *sp, struct worker *wrk, struct req *req)
 		}
 	}
 
-	req->sha256ctx = &sha256ctx;
+	req->sha256ctx = &sha256ctx;	/* so HSH_AddString() can find it */
 	SHA256_Init(req->sha256ctx);
 	VCL_hash_method(sp);
 	assert(req->handling == VCL_RET_HASH);
@@ -1511,10 +1524,8 @@ cnt_recv(struct sess *sp, struct worker *wrk, struct req *req)
 	else
 		req->wantbody = 1;
 
-	req->sendbody = 0;
 	switch(recv_handling) {
 	case VCL_RET_LOOKUP:
-		/* XXX: discard req body, if any */
 		sp->step = STP_LOOKUP;
 		return (0);
 	case VCL_RET_PIPE:
@@ -1530,7 +1541,6 @@ cnt_recv(struct sess *sp, struct worker *wrk, struct req *req)
 		sp->step = STP_PASS;
 		return (0);
 	case VCL_RET_ERROR:
-		/* XXX: discard req body, if any */
 		sp->step = STP_ERROR;
 		return (0);
 	default:
@@ -1563,10 +1573,10 @@ cnt_start(struct sess *sp, struct worker *wrk, struct req *req)
 	AZ(req->obj);
 	AZ(req->vcl);
 	AZ(req->esi_level);
+	assert(!isnan(sp->t_req));
 
 	/* Update stats of various sorts */
 	wrk->stats.client_req++;
-	assert(!isnan(sp->t_req));
 	wrk->acct_tmp.req++;
 
 	/* Assign XID and log */
@@ -1609,11 +1619,14 @@ cnt_start(struct sess *sp, struct worker *wrk, struct req *req)
 	}
 	http_Unset(req->http, H_Expect);
 
+	/* XXX: pull in req-body and make it available instead. */
+	req->reqbodydone = 0;
+
 	HTTP_Copy(req->http0, req->http);	/* Copy for restart/ESI use */
 
 	if (req->err_code)
 		sp->step = STP_ERROR;
-	else 
+	else
 		sp->step = STP_RECV;
 	return (0);
 }
diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c
index 28de3b6..c3563a6 100644
--- a/bin/varnishd/cache/cache_fetch.c
+++ b/bin/varnishd/cache/cache_fetch.c
@@ -329,14 +329,20 @@ fetch_eof(struct worker *wrk, struct http_conn *htc)
  */
 
 int
-FetchReqBody(const struct sess *sp)
+FetchReqBody(const struct sess *sp, int sendbody)
 {
 	unsigned long content_length;
 	char buf[8192];
 	char *ptr, *endp;
 	int rdcnt;
 
+	if (sp->req->reqbodydone) {
+		AZ(sendbody);
+		return (0);
+	}
+		
 	if (http_GetHdr(sp->req->http, H_Content_Length, &ptr)) {
+		sp->req->reqbodydone = 1;
 
 		content_length = strtoul(ptr, &endp, 10);
 		/* XXX should check result of conversion */
@@ -349,11 +355,12 @@ FetchReqBody(const struct sess *sp)
 			if (rdcnt <= 0)
 				return (1);
 			content_length -= rdcnt;
-			if (!sp->req->sendbody)
-				continue;
-			(void)WRW_Write(sp->wrk, buf, rdcnt); /* XXX: stats ? */
-			if (WRW_Flush(sp->wrk))
-				return (2);
+			if (sendbody) {
+				/* XXX: stats ? */
+				(void)WRW_Write(sp->wrk, buf, rdcnt);
+				if (WRW_Flush(sp->wrk))
+					return (2);
+			}
 		}
 	}
 	if (http_GetHdr(sp->req->http, H_Transfer_Encoding, NULL)) {
@@ -375,7 +382,7 @@ FetchReqBody(const struct sess *sp)
  */
 
 int
-FetchHdr(struct sess *sp, int need_host_hdr)
+FetchHdr(struct sess *sp, int need_host_hdr, int sendbody)
 {
 	struct vbc *vc;
 	struct worker *wrk;
@@ -422,7 +429,7 @@ FetchHdr(struct sess *sp, int need_host_hdr)
 	(void)http_Write(wrk, vc->vsl_id, hp, 0);	/* XXX: stats ? */
 
 	/* Deal with any message-body the request might have */
-	i = FetchReqBody(sp);
+	i = FetchReqBody(sp, sendbody);
 	if (WRW_FlushRelease(wrk) || i > 0) {
 		WSP(sp, SLT_FetchError, "backend write error: %d (%s)",
 		    errno, strerror(errno));



From geoff at varnish-cache.org  Tue Jan 24 17:30:28 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Tue, 24 Jan 2012 18:30:28 +0100
Subject: [experimental-ims] 3d52af7 Continue polishing cache_center to make
	states and assumptions clearer for everybody.
Message-ID: <E1RpkCS-0005RS-Ui@project.varnish-software.com>

commit 3d52af787968fa31c94f0f0429eb6cbdfcf4f1b5
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Jan 24 10:48:48 2012 +0000

    Continue polishing cache_center to make states and assumptions
    clearer for everybody.
    
    Add restart in vcl_miss{} implementation.

diff --git a/bin/varnishd/cache/cache_center.c b/bin/varnishd/cache/cache_center.c
index b83cb93..ea176cb 100644
--- a/bin/varnishd/cache/cache_center.c
+++ b/bin/varnishd/cache/cache_center.c
@@ -488,10 +488,10 @@ cnt_error(struct sess *sp, struct worker *wrk, struct req *req)
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
-
 	AZ(req->objcore);
 	AZ(req->obj);
 	AZ(wrk->busyobj);
+
 	wrk->busyobj = VBO_GetBusyObj(wrk);
 	req->obj = STV_NewObject(wrk, TRANSIENT_STORAGE,
 	    cache_param->http_resp_size,
@@ -1079,6 +1079,7 @@ cnt_hit(struct sess *sp, struct worker *wrk, struct req *req)
 
 	CHECK_OBJ_NOTNULL(req->obj, OBJECT_MAGIC);
 	CHECK_OBJ_NOTNULL(req->vcl, VCL_CONF_MAGIC);
+	AZ(req->objcore);
 	AZ(wrk->busyobj);
 
 	assert(!(req->obj->objcore->flags & OC_F_PASS));
@@ -1123,18 +1124,19 @@ cnt_hit(struct sess *sp, struct worker *wrk, struct req *req)
  *
 DOT subgraph xcluster_lookup {
 DOT	lookup [
-DOT		shape=diamond
-DOT		label="obj in cache ?\ncreate if not"
+DOT		shape=record
+DOT		label="{cnt_lookup:|hash lookup}"
 DOT	]
 DOT	lookup2 [
 DOT		shape=diamond
 DOT		label="obj.f.pass ?"
 DOT	]
-DOT	lookup -> lookup2 [label="yes",style=bold,color=green]
 DOT }
+DOT lookup -> lookup [label="Busy object found"]
+DOT lookup -> miss [label="not found\ncreate new",style=bold,color=blue]
+DOT lookup -> lookup2 [label="found",style=bold,color=green]
 DOT lookup2 -> hit [label="no", style=bold,color=green]
 DOT lookup2 -> pass [label="yes",style=bold,color=red]
-DOT lookup -> miss [label="no",style=bold,color=blue]
  */
 
 static int
@@ -1147,6 +1149,7 @@ cnt_lookup(struct sess *sp, struct worker *wrk, struct req *req)
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
+	AZ(req->objcore);
 
 	CHECK_OBJ_NOTNULL(req->vcl, VCL_CONF_MAGIC);
 	AZ(wrk->busyobj);
@@ -1165,18 +1168,19 @@ cnt_lookup(struct sess *sp, struct worker *wrk, struct req *req)
 	req->vary_b[2] = '\0';
 
 	oc = HSH_Lookup(sp, &oh);
+	AZ(req->objcore);
 
 	if (oc == NULL) {
 		/*
 		 * We lost the session to a busy object, disembark the
-		 * worker thread.   The hash code to restart the session,
-		 * still in STP_LOOKUP, later when the busy object isn't.
+		 * worker thread.   We return to STP_LOOKUP when the busy
+		 * object has been unbusied, and still have the hash digest
+		 * around to do the lookup with.
 		 * NB:  Do not access sp any more !
 		 */
 		return (1);
 	}
 
-
 	CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
 	CHECK_OBJ_NOTNULL(oh, OBJHEAD_MAGIC);
 
@@ -1215,7 +1219,7 @@ cnt_lookup(struct sess *sp, struct worker *wrk, struct req *req)
 		wrk->stats.cache_hitpass++;
 		WSP(sp, SLT_HitPass, "%u", req->obj->xid);
 		(void)HSH_Deref(wrk, NULL, &req->obj);
-		req->objcore = NULL;
+		AZ(req->objcore);
 		sp->step = STP_PASS;
 		return (0);
 	}
@@ -1257,10 +1261,10 @@ cnt_miss(struct sess *sp, struct worker *wrk, struct req *req)
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
 	CHECK_OBJ_NOTNULL(req->vcl, VCL_CONF_MAGIC);
-
-	AZ(req->obj);
-	AN(req->objcore);
+	CHECK_OBJ_NOTNULL(req->objcore, OBJCORE_MAGIC);
 	CHECK_OBJ_NOTNULL(wrk->busyobj, BUSYOBJ_MAGIC);
+	AZ(req->obj);
+
 	WS_Reset(wrk->ws, NULL);
 	wrk->busyobj = VBO_GetBusyObj(wrk);
 	http_Setup(wrk->busyobj->bereq, wrk->ws);
@@ -1276,38 +1280,36 @@ cnt_miss(struct sess *sp, struct worker *wrk, struct req *req)
 		http_SetHeader(wrk, sp->vsl_id, wrk->busyobj->bereq,
 		    "Accept-Encoding: gzip");
 	}
-	wrk->connect_timeout = 0;
-	wrk->first_byte_timeout = 0;
-	wrk->between_bytes_timeout = 0;
 
 	VCL_miss_method(sp);
 
+	if (req->handling == VCL_RET_FETCH) {
+		CHECK_OBJ_NOTNULL(wrk->busyobj, BUSYOBJ_MAGIC);
+		sp->step = STP_FETCH;
+		return (0);
+	}
+
+	AZ(HSH_Deref(wrk, req->objcore, NULL));
+	req->objcore = NULL;
+	http_Setup(wrk->busyobj->bereq, NULL);
+	VBO_DerefBusyObj(wrk, &wrk->busyobj);
+
 	switch(req->handling) {
 	case VCL_RET_ERROR:
-		AZ(HSH_Deref(wrk, req->objcore, NULL));
-		req->objcore = NULL;
-		http_Setup(wrk->busyobj->bereq, NULL);
-		VBO_DerefBusyObj(wrk, &wrk->busyobj);
 		sp->step = STP_ERROR;
-		return (0);
+		break;
 	case VCL_RET_PASS:
-		AZ(HSH_Deref(wrk, req->objcore, NULL));
-		req->objcore = NULL;
-		VBO_DerefBusyObj(wrk, &wrk->busyobj);
 		sp->step = STP_PASS;
-		return (0);
-	case VCL_RET_FETCH:
-		CHECK_OBJ_NOTNULL(wrk->busyobj, BUSYOBJ_MAGIC);
-		sp->step = STP_FETCH;
-		return (0);
+		break;
 	case VCL_RET_RESTART:
-		AZ(HSH_Deref(wrk, req->objcore, NULL));
-		req->objcore = NULL;
-		VBO_DerefBusyObj(wrk, &wrk->busyobj);
-		INCOMPL();
+		req->restarts++;
+		req->director = NULL;
+		sp->step = STP_RECV;
+		break;
 	default:
 		WRONG("Illegal action in vcl_miss{}");
 	}
+	return (0);
 }
 
 /*--------------------------------------------------------------------
@@ -1349,6 +1351,7 @@ cnt_pass(struct sess *sp, struct worker *wrk, const struct req *req)
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
 	CHECK_OBJ_NOTNULL(req->vcl, VCL_CONF_MAGIC);
+	AZ(req->objcore);
 	AZ(req->obj);
 	AZ(wrk->busyobj);
 
@@ -1358,10 +1361,8 @@ cnt_pass(struct sess *sp, struct worker *wrk, const struct req *req)
 	http_Setup(wrk->busyobj->bereq, wrk->ws);
 	http_FilterReq(sp, HTTPH_R_PASS);
 
-	wrk->connect_timeout = 0;
-	wrk->first_byte_timeout = 0;
-	wrk->between_bytes_timeout = 0;
 	VCL_pass_method(sp);
+
 	if (req->handling == VCL_RET_ERROR) {
 		http_Setup(wrk->busyobj->bereq, NULL);
 		VBO_DerefBusyObj(wrk, &wrk->busyobj);
@@ -1478,6 +1479,10 @@ cnt_recv(struct sess *sp, struct worker *wrk, struct req *req)
 	req->director = req->vcl->director[0];
 	AN(req->director);
 
+	wrk->connect_timeout = 0;
+	wrk->first_byte_timeout = 0;
+	wrk->between_bytes_timeout = 0;
+
 	req->disable_esi = 0;
 	req->hash_always_miss = 0;
 	req->hash_ignore_busy = 0;
diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c
index c3563a6..abef6b1 100644
--- a/bin/varnishd/cache/cache_fetch.c
+++ b/bin/varnishd/cache/cache_fetch.c
@@ -340,7 +340,7 @@ FetchReqBody(const struct sess *sp, int sendbody)
 		AZ(sendbody);
 		return (0);
 	}
-		
+
 	if (http_GetHdr(sp->req->http, H_Content_Length, &ptr)) {
 		sp->req->reqbodydone = 1;
 



From geoff at varnish-cache.org  Tue Jan 24 17:30:29 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Tue, 24 Jan 2012 18:30:29 +0100
Subject: [experimental-ims] 4a75131 Add testcase for restart in vcl_miss{}
Message-ID: <E1RpkCT-0005Rf-1T@project.varnish-software.com>

commit 4a75131dbe4d95f6de7bc54848975c3da6e25abd
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Jan 24 10:49:45 2012 +0000

    Add testcase for restart in vcl_miss{}

diff --git a/bin/varnishtest/tests/v00037.vtc b/bin/varnishtest/tests/v00037.vtc
new file mode 100644
index 0000000..e7759a6
--- /dev/null
+++ b/bin/varnishtest/tests/v00037.vtc
@@ -0,0 +1,30 @@
+varnishtest "test restart in miss"
+
+server s1 {
+	rxreq
+	txresp -body "FOOBAR"
+} -start
+
+varnish v1 -vcl+backend {
+
+	sub vcl_recv {
+		if (req.restarts > 0) {
+			unset req.http.foobar;
+		}
+	}
+
+	sub vcl_miss {
+		if (req.http.foobar) {
+			return (restart);
+		}
+	}
+	sub vcl_deliver {
+		set resp.http.restarts = req.restarts;
+	}
+} -start
+
+client c1 {
+	txreq -hdr "foobar: snafu"
+	rxresp
+	expect resp.http.restarts == 1
+} -run



From geoff at varnish-cache.org  Tue Jan 24 17:30:29 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Tue, 24 Jan 2012 18:30:29 +0100
Subject: [experimental-ims] c766bc8 Merge master -> experimental-ims
Message-ID: <E1RpkCT-0005Rp-5N@project.varnish-software.com>

commit c766bc8d3b4ed95aeee184d6a220776c4626cb3b
Merge: 3cf4dfd 4a75131
Author: Geoff Simmons <geoff at uplex.de>
Date:   Tue Jan 24 18:28:39 2012 +0100

    Merge master -> experimental-ims

diff --cc bin/varnishd/cache/cache_center.c
index c16c42e,ea176cb..398996d
--- a/bin/varnishd/cache/cache_center.c
+++ b/bin/varnishd/cache/cache_center.c
@@@ -826,12 -806,12 +824,16 @@@ cnt_fetchbody(struct sess *sp, struct w
  	if (!req->wantbody)
  		wrk->busyobj->do_stream = 0;
  
+ 	/* No reason to try streaming a non-existing body */
+ 	if (wrk->busyobj->body_status == BS_NONE)
+ 		wrk->busyobj->do_stream = 0;
+ 
  	l = http_EstimateWS(wrk->busyobj->beresp,
  	    pass ? HTTPH_R_PASS : HTTPH_A_INS, &nhttp);
 +        if (sp->stale_obj) {
 +            l += http_EstimateWS(sp->stale_obj->http, 0, &stale_nhttp);
 +            nhttp += stale_nhttp;
 +        }
  
  	/* Create Vary instructions */
  	if (req->objcore != NULL) {
@@@ -1262,9 -1219,7 +1264,9 @@@ cnt_lookup(struct sess *sp, struct work
  		wrk->stats.cache_hitpass++;
  		WSP(sp, SLT_HitPass, "%u", req->obj->xid);
  		(void)HSH_Deref(wrk, NULL, &req->obj);
 +                if (sp->stale_obj != NULL)
 +                    (void)HSH_Deref(wrk, NULL, &sp->stale_obj);
- 		req->objcore = NULL;
+ 		AZ(req->objcore);
  		sp->step = STP_PASS;
  		return (0);
  	}
@@@ -1325,41 -1280,32 +1327,38 @@@ cnt_miss(struct sess *sp, struct worke
  		http_SetHeader(wrk, sp->vsl_id, wrk->busyobj->bereq,
  		    "Accept-Encoding: gzip");
  	}
- 	wrk->connect_timeout = 0;
- 	wrk->first_byte_timeout = 0;
- 	wrk->between_bytes_timeout = 0;
  
 +        /* If a candidate for a conditional backend request was found,
 +         * add If-Modified-Since and/or If-None-Match to the bereq.
 +         */
 +        if (sp->stale_obj)
 +                http_CheckRefresh(sp);
 +
  	VCL_miss_method(sp);
  
+ 	if (req->handling == VCL_RET_FETCH) {
+ 		CHECK_OBJ_NOTNULL(wrk->busyobj, BUSYOBJ_MAGIC);
+ 		sp->step = STP_FETCH;
+ 		return (0);
+ 	}
+ 
+ 	AZ(HSH_Deref(wrk, req->objcore, NULL));
+ 	req->objcore = NULL;
+ 	http_Setup(wrk->busyobj->bereq, NULL);
+ 	VBO_DerefBusyObj(wrk, &wrk->busyobj);
+ 
  	switch(req->handling) {
  	case VCL_RET_ERROR:
- 		AZ(HSH_Deref(wrk, req->objcore, NULL));
- 		req->objcore = NULL;
- 		http_Setup(wrk->busyobj->bereq, NULL);
- 		VBO_DerefBusyObj(wrk, &wrk->busyobj);
  		sp->step = STP_ERROR;
- 		return (0);
+ 		break;
  	case VCL_RET_PASS:
- 		AZ(HSH_Deref(wrk, req->objcore, NULL));
- 		req->objcore = NULL;
- 		VBO_DerefBusyObj(wrk, &wrk->busyobj);
  		sp->step = STP_PASS;
- 		return (0);
- 	case VCL_RET_FETCH:
- 		CHECK_OBJ_NOTNULL(wrk->busyobj, BUSYOBJ_MAGIC);
- 		sp->step = STP_FETCH;
- 		return (0);
+ 		break;
  	case VCL_RET_RESTART:
- 		AZ(HSH_Deref(wrk, req->objcore, NULL));
- 		req->objcore = NULL;
- 		VBO_DerefBusyObj(wrk, &wrk->busyobj);
- 		INCOMPL();
+ 		req->restarts++;
+ 		req->director = NULL;
+ 		sp->step = STP_RECV;
+ 		break;
  	default:
  		WRONG("Illegal action in vcl_miss{}");
  	}



From phk at varnish-cache.org  Tue Jan 24 19:36:01 2012
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Tue, 24 Jan 2012 20:36:01 +0100
Subject: [master] 3a1b9ca Make this test case more robust (and faster) by
	using the CLI to control backend health, rather than rely on probes.
Message-ID: <E1Rpm9x-0003zP-1u@project.varnish-software.com>

commit 3a1b9ca8fcf1e9c901f6e57b8863ec678f6538a7
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Jan 24 19:35:28 2012 +0000

    Make this test case more robust (and faster) by using the
    CLI to control backend health, rather than rely on probes.
    
    Submitted by:	Federico G. Schwindt

diff --git a/bin/varnishtest/tests/v00036.vtc b/bin/varnishtest/tests/v00036.vtc
index e83acef..323f80c 100644
--- a/bin/varnishtest/tests/v00036.vtc
+++ b/bin/varnishtest/tests/v00036.vtc
@@ -1,127 +1,66 @@
 varnishtest "Test fallback director"
 
 server s1 {
-	rxreq
-	expect req.url == "/"
-	txresp -body "slash"
-	close
-
-	sema r1 sync 3
-
-	accept
-	rxreq
-	expect req.url == "/"
-	txresp -body "slash"
-	close
-
-	accept
-	rxreq
-	expect req.url == "/"
-	txresp -body "slash"
-	close
-
-	sema r3 sync 2
-
-	accept
-	rxreq
-	expect req.url == "/foo"
-	txresp -hdr "Foo: 1" -body "foobar"
-
+        rxreq
+        txresp -hdr "Foo: 1"
 } -start
 
 server s2 {
-	rxreq
-	expect req.url == "/"
-	txresp -body "slash"
-	close
-
-	sema r1 sync 3
-
-	accept
-	rxreq
-	expect req.url == "/"
-	txresp -body "slash"
-	close
-
-	sema r2 sync 2
-
-	accept
-	rxreq
-	expect req.url == "/foo"
-	txresp -hdr "Foo: 2" -body "foobar"
+        rxreq
+        txresp -hdr "Foo: 2"
 } -start
 
 server s3 {
-	rxreq
-	expect req.url == "/foo"
-	txresp -hdr "Foo: 3" -body "foobar"
+        rxreq
+        txresp -hdr "Foo: 3"
 } -start
 
 varnish v1 -vcl {
-
-	probe p1 {
-		.url = "/";
-		.timeout = 1s;
-		.interval = 1s;
-		.window = 4;
-		.threshold = 3;
-		.initial = 0;
-	}
-	probe p2 {
-		.url = "/";
-		.timeout = 1s;
-		.interval = 1s;
-		.window = 3;
-		.threshold = 2;
-		.initial = 0;
-	}
-
-	backend b1 {
-		.host = "${s1_addr}";
-		.port = "${s1_port}";
-		.max_connections = 1;
-		.probe = p1;
-	}
-	backend b2 {
-		.host = "${s2_addr}";
-		.port = "${s2_port}";
-		.max_connections = 1;
-		.probe = p2;
-	}
-	backend b3 {
-		.host = "${s3_addr}";
-		.port = "${s3_port}";
-	}
-	director f1 fallback {
-		{ .backend = b1; }
-		{ .backend = b2; }
-		{ .backend = b3; }
-	}
-
-	sub vcl_recv {
-		set req.backend = f1;
-		return(pass);
-	}
+        backend b1 {
+                .host = "${s1_addr}";
+                .port = "${s1_port}";
+        }
+        backend b2 {
+                .host = "${s2_addr}";
+                .port = "${s2_port}";
+        }
+        backend b3 {
+                .host = "${s3_addr}";
+                .port = "${s3_port}";
+        }
+        director f1 fallback {
+                { .backend = b1; }
+                { .backend = b2; }
+                { .backend = b3; }
+        }
+        sub vcl_recv {
+                set req.backend = f1;
+                return (pass);
+        }
 } -start
 
+varnish v1 -cliok "backend.set_health b1 sick"
+varnish v1 -cliok "backend.set_health b2 sick"
 
 client c1 {
-	# s1 & s2 are both sick, expect response from s3
-	txreq -url "/foo"
-	rxresp
-	expect resp.http.foo == "3"
+        # s1 & s2 are both sick, expect response from s3
+        txreq
+        rxresp
+        expect resp.http.foo == "3"
+} -run
 
-	sema r1 sync 3
+varnish v1 -cliok "backend.set_health b2 healthy"
 
-	# wait for s2 to become healthy
-	sema r2 sync 2
-	txreq -url "/foo"
-	rxresp
-	expect resp.http.foo == "2"
+client c1 {
+        txreq
+        rxresp
+        expect resp.http.foo == "2"
+} -run
 
-	# wait for s1 to become healthy
-	sema r3 sync 2
-	txreq -url "/foo"
-	rxresp
-	expect resp.http.foo == "1"
+varnish v1 -cliok "backend.set_health b1 healthy"
+
+client c1 {
+        txreq
+        rxresp
+        expect resp.http.foo == "1"
 } -run



From perbu at varnish-cache.org  Wed Jan 25 08:57:02 2012
From: perbu at varnish-cache.org (Per Buer)
Date: Wed, 25 Jan 2012 09:57:02 +0100
Subject: [master] b23cb23 cleaned out some cruft. Added links to the actual
	docs
Message-ID: <E1Rpyf8-0008WD-OM@project.varnish-software.com>

commit b23cb230ad691426096bac155d5230e00ea928cc
Author: Per Buer <perbu at varnish-software.com>
Date:   Wed Jan 25 09:56:41 2012 +0100

    cleaned out some cruft. Added links to the actual docs

diff --git a/doc/sphinx/faq/general.rst b/doc/sphinx/faq/general.rst
index b30f093..2444394 100644
--- a/doc/sphinx/faq/general.rst
+++ b/doc/sphinx/faq/general.rst
@@ -5,25 +5,7 @@ General questions
 What is Varnish?
 ================
 
-Varnish is a state-of-the-art, high-performance `web accelerator <http://en.wikipedia.org/wiki/Web_accelerator>`_. It uses the advanced features in Linux 2.6, FreeBSD 6/7 and Solaris 10 to achieve its high performance.
-
-Some of the features include
-
-* A modern design
-* VCL - a very flexible configuration language
-* Load balancing with health checking of backends
-* Partial support for ESI
-* URL rewriting
-* Graceful handling of "dead" backends
-
-Features to come (Experimental):
-
-* Support for Ranged headers
-* Support for persistent cache
-	
-
-Varnish is free software and is licenced under a modified BSD licence. Please read the introduction to get started with Varnish.
-
+Please see ref:`tutorial-intro`.
 
 How...
 ======
@@ -59,7 +41,9 @@ This can be achieved by removing the query parameters using a regexp::
 
 **How can I force a refresh on a object cached by varnish?**
 
-Refreshing is often called `purging <http://dictionary.reference.com/browse/PURGE>`_ a document.  You can purge at least 2 different ways in Varnish:
+Refreshing is often called `purging
+<http://dictionary.reference.com/browse/PURGE>`_ a document.  You can
+purge at least 2 different ways in Varnish:
 
 1. Command line
 
@@ -118,23 +102,6 @@ You can use the ``bereq`` object for altering requests going to the backend, but
                 return(fetch);
         }
 
-**How do I force the backend to send Vary headers?**
-
-We have anecdotal evidence of non-RFC2616 compliant backends, which support content negotiation, but which do not emit a Vary header, unless the request contains Accept headers.
-
-It may be appropriate to send no-op Accept headers to trick the backend into sending us the Vary header.
-
-The following should be sufficient for most cases::
-
-        Accept: */*
-        Accept-Language: *
-        Accept-Charset: *
-        Accept-Encoding: identity
-
-Note that Accept-Encoding can not be set to ``*``, as the backend might then send back a compressed response which the client would be unable to process.
-
-This can of course be implemented in VCL.
-
 **How can I customize the error messages that Varnish returns?**
 
 A custom error page can be generated by adding a ``vcl_error`` to your configuration file. The default error page looks like this::
@@ -171,32 +138,9 @@ This can be achieved by removing the query parameters using a regexp::
         }
 
 
-Where...
-========
-
-**Can I find varnish for my operating system?**
-
-We know that Varnish has been packaged for Debian, Ubuntu, RHEL,
-CentOS, (Open)SUSE, Gentoo and FreeBSD, possibly more.  Check whatever
-package manager you use. Or read :ref:`Installing Varnish on your computer <install-doc>`.
-
 Can I...
 ========
 
-**Can I use Varnish as a client-side proxy?**
-
-No.  Varnish needs all backends configured in the VCL.  Look at squid
-instead.
-
-**Can I run Varnish on a 32bit system?**
-
-Yes, recently somebody even claimed to run Varnish on his N900 mobile
-phone recently, but if you have the choice, go 64 bit from the start.
-
-Varnish is written to use Virtual Memory and on a 32bit system that
-really cramps your style, and you will have trouble configuring more
-than 2 GB of storage.
-
 **Can I run Varnish on the same system as Apache?**
 
 Yes, and many people do that with good success.
@@ -243,83 +187,28 @@ Yes, you need VCL code like this::
 		set req.backend = foobar;
 	}
 
-    (XXX: reference to docs, once written)
+    Please see :ref:`tutorial-advanced_backend_servers-directors_`.
+
 
 Why ...
 =======
 
 **Why does it look like Varnish sends all requests to the backend? I thought it was a cache?**
 
-There are 2 common reasons for this:
-	1. The object's ``ttl expired``. A common situation is that the backend does not set an expiry time on the requested image/file/webpage, so Varnish uses the default TTL (normally 120s).
-	2. Your site uses ``cookies``:
-            * By default, varnish will not cache ``responses`` from the backend that come with a ``Set-Cookie``: header.
-            * By default, varnish will not serve ``requests`` with a ``Cookie:`` header, but pass them to the backend instead. Check out [wiki:VCLExamples these VCL examples] on how to make varnish cache cookied/logged in users sanely.
-
-
-**Why are regular expressions case-sensitive?**
-
-Some HTTP headers, such as ``Host:`` and ``Location:`` contain fully
-qualified domain names, which by definition is not case-sensitive.
-Other HTTP headers are case-sensitive, most notably the URLs.
-Therefore a "one size fits all" solution is not possible.
-
-In previous releases, we used the POSIX regular expressions
-supplied with the operating system, and decided, because the
-most common use of regexps were on ```Host:``` headers, that
-they should not be case-sensitive.
-
-From version 2.1.0 and forward, we use PCRE regular expressions,
-where it *is* possible to control case-sensitivity in the
-individual regular expressions, so we decided that it would
-probably confuse people if we made the default case-insensitive.
-(We promise not to change our minds about this again.)
-
-To make a PCRE regex case insensitive, put ``(?i)`` at the start::
-
-	if (req.http.host ~ "(?i)example.com$") {
-		...
-	}
-
-See the `PCRE man pages <http://www.pcre.org/pcre.txt>`_ for more information.
-
-**Are regular expressions case sensitive or not? Can I change it?**
-
-In 2.1 and newer, regular expressions are case sensitive by default.  In earlier versions, they were case insensitive.
-
-To change this for a single regex in 2.1, use ``(?i)`` at the start.
-
-See the `PCRE man pages <http://www.pcre.org/pcre.txt>`_ for more information.
-
-
-**Why does the ``Via:`` header say 1.1 in Varnish 2.1.x?**
-
-The number in the ``Via:`` header is the HTTP protocol version
-supported/applied, not the software's version number.
-
-**Why did you call it *Varnish*?**
-
-Long story, but basically the instigator of Varnish spent a long
-time staring at an art-poster with the word "Vernisage" and ended
-up checking it in a dictionary, which gives the following three
-meanings of the word:
-
-r.v. var?nished, var?nish?ing, var?nish?es
-
-	  1. To cover with varnish.
-	  2. To give a smooth and glossy finish to.
-	  3. To give a deceptively attractive appearance to; gloss over.
 
-The three point describes happens to your backend system when you
-put Varnish in front of it.
+Please see ref:`tutorial-increasing_your_hitrate`.
 
 **Why does Varnish require the system to have a C compiler?**
 
-The :ref:`VCL <faq-vcl>` compiler generates C source as output (your config file), and uses the systems C-compiler to compile that into a shared library.  If there is no C compiler, Varnish will not work.
+The :ref:`VCL <faq-vcl>` compiler generates C source as output (your
+config file), and uses the systems C-compiler to compile that into a
+shared library.  If there is no C compiler, Varnish will not work.
 
 **Isn't that security problem?**
 
-The days when you could prevent people from running non-approved programs by removing the C compiler from your system ended roughly with the VAX 11/780 computer.
+The days when you could prevent people from running non-approved
+programs by removing the C compiler from your system ended roughly
+with the VAX 11/780 computer.
 
 Troubleshooting
 ===============



From perbu at varnish-cache.org  Wed Jan 25 08:57:02 2012
From: perbu at varnish-cache.org (Per Buer)
Date: Wed, 25 Jan 2012 09:57:02 +0100
Subject: [master] db55f5f add link
Message-ID: <E1Rpyf8-0008WJ-Py@project.varnish-software.com>

commit db55f5f340eea9ed96499a06dece00fde997b9c6
Author: Per Buer <perbu at varnish-software.com>
Date:   Wed Jan 25 09:56:57 2012 +0100

    add link

diff --git a/doc/sphinx/tutorial/advanced_backend_servers.rst b/doc/sphinx/tutorial/advanced_backend_servers.rst
index 307f715..b4206d9 100644
--- a/doc/sphinx/tutorial/advanced_backend_servers.rst
+++ b/doc/sphinx/tutorial/advanced_backend_servers.rst
@@ -40,6 +40,8 @@ really arbitrary data. You want to send mobile devices to a different
 backend? No problem. if (req.User-agent ~ /mobile/) .... should do the
 trick. 
 
+.. _tutorial-advanced_backend_servers-directors:
+
 Directors
 ---------
 



From apj at varnish-cache.org  Wed Jan 25 09:05:40 2012
From: apj at varnish-cache.org (Andreas Plesner Jacobsen)
Date: Wed, 25 Jan 2012 10:05:40 +0100
Subject: [master] a999224 Literal block colons in fully minimized form
Message-ID: <E1RpynU-0001uU-PY@project.varnish-software.com>

commit a9992243b02d0ff02b10fc226c7da9de7553ddbc
Author: Andreas Plesner Jacobsen <apj at mutt.dk>
Date:   Wed Jan 25 10:03:51 2012 +0100

    Literal block colons in fully minimized form

diff --git a/doc/sphinx/reference/varnish-cli.rst b/doc/sphinx/reference/varnish-cli.rst
index a097248..668ceab 100644
--- a/doc/sphinx/reference/varnish-cli.rst
+++ b/doc/sphinx/reference/varnish-cli.rst
@@ -54,7 +54,7 @@ Syntax
 
 Commands are usually terminated with a newline. Long command can be
 entered using sh style *here documents*. The format of here-documents
-is:::
+is::
 
    << word
 	here document
@@ -233,7 +233,7 @@ file on the fly.
 
 Use the unix file permissions to control access to the file.
 
-An authenticated session looks like this:::
+An authenticated session looks like this::
 
    critter phk> telnet localhost 1234
    Trying ::1...
@@ -273,7 +273,7 @@ following byte sequence:
 
 and dumping the resulting digest in lower-case hex.
 
-In the above example, the secret file contained foo\n and thus:::
+In the above example, the secret file contained foo\n and thus::
 
    critter phk> cat > _
    ixslvvxrgkjptxmcgnnsdxsvdmvfympg
@@ -300,18 +300,18 @@ EXAMPLES
 ========
 
 Simple example: All requests where req.url exactly matches the string
-/news are banned from the cache:::
+/news are banned from the cache::
 
     req.url == "/news"
 
 Example: Ban all documents where the name does not end with ".ogg",
-and where the size of the object is greater than 10 megabytes:::
+and where the size of the object is greater than 10 megabytes::
 
     req.url !~ "\.ogg$" && obj.size > 10MB
 
 Example: Ban all documents where the serving host is "example.com"
 or "www.example.com", and where the Set-Cookie header received from
-the backend contains "USERID=1663":::
+the backend contains "USERID=1663"::
 
     req.http.host ~ "^(?i)(www\.)example.com$" && obj.set-cookie ~ "USERID=1663"
 
diff --git a/doc/sphinx/reference/varnishadm.rst b/doc/sphinx/reference/varnishadm.rst
index bfad196..3521474 100644
--- a/doc/sphinx/reference/varnishadm.rst
+++ b/doc/sphinx/reference/varnishadm.rst
@@ -66,7 +66,7 @@ zero if the command succeeded, and non-zero otherwise.
 EXAMPLES
 ========
 
-Some ways you can use varnishadm:::
+Some ways you can use varnishadm::
 
            varnishadm -T localhost:999 -S /var/db/secret vcl.use foo
            echo vcl.use foo | varnishadm -T localhost:999 -S /var/db/secret
diff --git a/doc/sphinx/reference/varnishlog.rst b/doc/sphinx/reference/varnishlog.rst
index f626332..9a2868f 100644
--- a/doc/sphinx/reference/varnishlog.rst
+++ b/doc/sphinx/reference/varnishlog.rst
@@ -142,11 +142,11 @@ The following log entry tags are currently defined:
 EXAMPLES
 ========
 
-The following command line simply copies all log entries to a log file:::
+The following command line simply copies all log entries to a log file::
 
     $ varnishlog -w /var/log/varnish.log
 
-The following command line reads that same log file and displays requests for the front page:::
+The following command line reads that same log file and displays requests for the front page::
 
     $ varnishlog -r /var/log/varnish.log -c -m 'RxURL:^/$'
 
diff --git a/doc/sphinx/reference/varnishtop.rst b/doc/sphinx/reference/varnishtop.rst
index 9b3ca9f..86e6cf6 100644
--- a/doc/sphinx/reference/varnishtop.rst
+++ b/doc/sphinx/reference/varnishtop.rst
@@ -79,12 +79,12 @@ EXAMPLES
 ========
 
 The following example displays a continuously updated list of the most
-frequently requested URLs:::
+frequently requested URLs::
 
   varnishtop -i RxURL
 
 The following example displays a continuously updated list of the most
-commonly used user agents:::
+commonly used user agents::
 
   varnishtop -i RxHeader -C -I ^User-Agent
 
diff --git a/doc/sphinx/reference/vcl.rst b/doc/sphinx/reference/vcl.rst
index 2c185ed..1ef14d6 100644
--- a/doc/sphinx/reference/vcl.rst
+++ b/doc/sphinx/reference/vcl.rst
@@ -96,14 +96,14 @@ file as a quoted string.
 Backend declarations
 --------------------
 
-A backend declaration creates and initializes a named backend object:::
+A backend declaration creates and initializes a named backend object::
 
   backend www {
     .host = "www.example.com";
     .port = "http";
   }
 
-The backend object can later be used to select a backend at request time:::
+The backend object can later be used to select a backend at request time::
 
   if (req.http.host ~ "(?i)^(www.)?example.com$") {
     set req.backend = www;
@@ -118,7 +118,7 @@ backend connection, .first_byte_timeout for the time to wait for the
 first byte from the backend and .between_bytes_timeout for time to
 wait between each received byte.
 
-These can be set in the declaration like this:::
+These can be set in the declaration like this::
 
   backend www {
     .host = "www.example.com";
@@ -145,7 +145,7 @@ be used.
 There are several types of directors. The different director types
 use different algorithms to choose which backend to use.
 
-Configuring a director may look like this:::
+Configuring a director may look like this::
 
   director b2 random {
     .retries = 5;
@@ -312,7 +312,7 @@ Probes take the following parameters:
   Default is 2 seconds.
 
 A backend with a probe can be defined like this, together with the
-backend or director:::
+backend or director::
 
   backend www {
     .host = "www.example.com";
@@ -326,7 +326,7 @@ backend or director:::
     }
   }
 
-Or it can be defined separately and then referenced:::
+Or it can be defined separately and then referenced::
 
   probe healthcheck {
      .url = "/status.cgi";
@@ -361,7 +361,7 @@ ACLs
 ----
 
 An ACL declaration creates and initializes a named access control list
-which can later be used to match client addresses:::
+which can later be used to match client addresses::
 
   acl local {
     "localhost";         // myself
@@ -375,7 +375,7 @@ if it is preceded by a negation mark, it will reject any address it is
 compared to, which may not be what you intended.  If the entry is
 enclosed in parentheses, however, it will simply be ignored.
 
-To match an IP address against an ACL, simply use the match operator:::
+To match an IP address against an ACL, simply use the match operator::
 
   if (client.ip ~ local) {
     return (pipe);
@@ -390,7 +390,7 @@ PCRE(3) man page.
 
 To send flags to the PCRE engine, such as to turn on *case
 insensitivity* add the flag within parens following a question mark,
-like this:::
+like this::
 
   if (req.http.host ~ "(?i)example.com$") {
           ...
@@ -424,7 +424,7 @@ ban_url(regex)
 Subroutines
 ~~~~~~~~~~~
 
-A subroutine is used to group code for legibility or reusability:::
+A subroutine is used to group code for legibility or reusability::
   
   sub pipe_if_local {
     if (client.ip ~ local) {
@@ -646,7 +646,7 @@ appear in the source.
 The default versions distributed with Varnish will be implicitly
 concatenated as a last resort at the end.
 
-Example:::
+Example::
 
 	# in file "main.vcl"
 	include "backends.vcl";
@@ -891,7 +891,7 @@ resp.response
 resp.http.header
   The corresponding HTTP header.
 
-Values may be assigned to variables using the set keyword:::
+Values may be assigned to variables using the set keyword::
 
   sub vcl_recv {
     # Normalize the Host: header
@@ -900,7 +900,7 @@ Values may be assigned to variables using the set keyword:::
     }
   }
 
-HTTP headers can be removed entirely using the remove keyword:::
+HTTP headers can be removed entirely using the remove keyword::
 
   sub vcl_fetch {
     # Don't cache cookies
@@ -942,7 +942,7 @@ EXAMPLES
 
 The following code is the equivalent of the default configuration with
 the backend address set to "backend.example.com" and no backend port
-specified:::
+specified::
 
   backend default {
    .host = "backend.example.com";
@@ -954,7 +954,7 @@ specified:::
 
 The following example shows how to support multiple sites running on
 separate backends in the same Varnish instance, by selecting backends
-based on the request URL:::
+based on the request URL::
 
   backend www {
     .host = "www.example.com";
@@ -980,7 +980,7 @@ based on the request URL:::
 The following snippet demonstrates how to force a minimum TTL for
 all documents.  Note that this is not the same as setting the
 default_ttl run-time parameter, as that only affects document for
-which the backend did not specify a TTL:::
+which the backend did not specify a TTL::
   
   import std; # needed for std.log
 
@@ -992,7 +992,7 @@ which the backend did not specify a TTL:::
   }
 
 The following snippet demonstrates how to force Varnish to cache
-documents even when cookies are present:::
+documents even when cookies are present::
 
   sub vcl_recv {
     if (req.request == "GET" && req.http.cookie) {
@@ -1007,7 +1007,7 @@ documents even when cookies are present:::
   }
 
 The following code implements the HTTP PURGE method as used by Squid
-for object invalidation:::
+for object invalidation::
 
   acl purge {
     "localhost";



From apj at varnish-cache.org  Wed Jan 25 11:13:11 2012
From: apj at varnish-cache.org (Andreas Plesner Jacobsen)
Date: Wed, 25 Jan 2012 12:13:11 +0100
Subject: [master] 4f3e2e9 Rest of the literal block colons in fully minimized
	form
Message-ID: <E1Rq0mt-000634-0M@project.varnish-software.com>

commit 4f3e2e9268772ec0bc58a79e513105ce7d769403
Author: Andreas Plesner Jacobsen <apj at mutt.dk>
Date:   Wed Jan 25 12:12:42 2012 +0100

    Rest of the literal block colons in fully minimized form

diff --git a/doc/sphinx/installation/install.rst b/doc/sphinx/installation/install.rst
index fe8496d..4a836ff 100644
--- a/doc/sphinx/installation/install.rst
+++ b/doc/sphinx/installation/install.rst
@@ -114,7 +114,7 @@ Configuring and compiling
 ~~~~~~~~~~~~~~~~~~~~~~~~~
 
 Next, configuration: The configuration will need the dependencies
-above satisfied. Once that is taken care of:::
+above satisfied. Once that is taken care of::
 
 	cd varnish-cache
 	sh autogen.sh
diff --git a/doc/sphinx/tutorial/esi.rst b/doc/sphinx/tutorial/esi.rst
index 1d01cb8..1badd5d 100644
--- a/doc/sphinx/tutorial/esi.rst
+++ b/doc/sphinx/tutorial/esi.rst
@@ -26,7 +26,7 @@ Example: esi include
 ~~~~~~~~~~~~~~~~~~~~
 
 Lets see an example how this could be used. This simple cgi script
-outputs the date:::
+outputs the date::
 
      #!/bin/sh
      
@@ -34,7 +34,7 @@ outputs the date:::
      echo ''
      date "+%Y-%m-%d %H:%M"
 
-Now, lets have an HTML file that has an ESI include statement:::
+Now, lets have an HTML file that has an ESI include statement::
 
      <HTML>
      <BODY>
@@ -43,7 +43,7 @@ Now, lets have an HTML file that has an ESI include statement:::
      </BODY>
      </HTML>
 
-For ESI to work you need to activate ESI processing in VCL, like this:::
+For ESI to work you need to activate ESI processing in VCL, like this::
 
     sub vcl_fetch {
     	if (req.url == "/test.html") {
@@ -59,7 +59,7 @@ Example: esi remove
 ~~~~~~~~~~~~~~~~~~~
 
 The *remove* keyword allows you to remove output. You can use this to make
-a fall back of sorts, when ESI is not available, like this:::
+a fall back of sorts, when ESI is not available, like this::
 
   <esi:include src="http://www.example.com/ad.html"/> 
   <esi:remove> 
diff --git a/doc/sphinx/tutorial/handling_misbehaving_servers.rst b/doc/sphinx/tutorial/handling_misbehaving_servers.rst
index 6cd7111..406b4b3 100644
--- a/doc/sphinx/tutorial/handling_misbehaving_servers.rst
+++ b/doc/sphinx/tutorial/handling_misbehaving_servers.rst
@@ -26,7 +26,7 @@ requests somewhat stale content.
 
 So, in order to serve stale content we must first have some content to
 serve. So to make Varnish keep all objects for 30 minutes beyond their
-TTL use the following VCL:::
+TTL use the following VCL::
 
   sub vcl_fetch {
     set beresp.grace = 30m;
@@ -64,7 +64,7 @@ errors. You can instruct Varnish to try to handle this in a
 more-than-graceful way - enter *Saint mode*. Saint mode enables you to
 discard a certain page from one backend server and either try another
 server or serve stale content from cache. Lets have a look at how this
-can be enabled in VCL:::
+can be enabled in VCL::
 
   sub vcl_fetch {
     if (beresp.status == 500) { 
diff --git a/doc/sphinx/tutorial/troubleshooting.rst b/doc/sphinx/tutorial/troubleshooting.rst
index 7ab9435..5bbcf6c 100644
--- a/doc/sphinx/tutorial/troubleshooting.rst
+++ b/doc/sphinx/tutorial/troubleshooting.rst
@@ -55,7 +55,7 @@ Varnish is crashing
 
 When varnish goes bust the child processes crashes. Usually the mother
 process will manage this by restarting the child process again. Any
-errors will be logged in syslog. It might look like this:::
+errors will be logged in syslog. It might look like this::
 
        Mar  8 13:23:38 smoke varnishd[15670]: Child (15671) not responding to CLI, killing it.
        Mar  8 13:23:43 smoke varnishd[15670]: last message repeated 2 times
@@ -78,13 +78,13 @@ Varnish gives me Guru meditation
 First find the relevant log entries in varnishlog. That will probably
 give you a clue. Since varnishlog logs so much data it might be hard
 to track the entries down. You can set varnishlog to log all your 503
-errors by issuing the following command:::
+errors by issuing the following command::
 
    $ varnishlog -c -m TxStatus:503
 
 If the error happened just a short time ago the transaction might still
 be in the shared memory log segment. To get varnishlog to process the
-whole shared memory log just add the -d option:::
+whole shared memory log just add the -d option::
 
    $ varnishlog -d -c -m TxStatus:503
 
diff --git a/doc/sphinx/tutorial/vcl.rst b/doc/sphinx/tutorial/vcl.rst
index 3ed78c5..7171ab6 100644
--- a/doc/sphinx/tutorial/vcl.rst
+++ b/doc/sphinx/tutorial/vcl.rst
@@ -136,7 +136,7 @@ Example 1 - manipulating headers
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 Lets say we want to remove the cookie for all objects in the /static
-directory of our web server:::
+directory of our web server::
 
   sub vcl_recv {
     if (req.url ~ "^/images") {
@@ -154,7 +154,7 @@ Example 2 - manipulating beresp
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 Here we override the TTL of a object comming from the backend if it
-matches certain criteria:::
+matches certain criteria::
 
   sub vcl_fetch {
      if (req.url ~ "\.(png|gif|jpg)$") {



From phk at varnish-cache.org  Wed Jan 25 16:11:12 2012
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Wed, 25 Jan 2012 17:11:12 +0100
Subject: [master] cff8e37 Make the invalid domains FQDN to save time if you
	have a long search-list.
Message-ID: <E1Rq5RI-0001ZF-VD@project.varnish-software.com>

commit cff8e37bbac7c690a4566ca1a63cb509ff9852c2
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Jan 25 16:10:51 2012 +0000

    Make the invalid domains FQDN to save time if you have a long search-list.

diff --git a/bin/varnishtest/tests/v00002.vtc b/bin/varnishtest/tests/v00002.vtc
index 586cfac..d9d33d6 100644
--- a/bin/varnishtest/tests/v00002.vtc
+++ b/bin/varnishtest/tests/v00002.vtc
@@ -116,11 +116,11 @@ varnish v1 -badvcl {
 }
 
 varnish v1 -badvcl {
-	backend b1 { .host = "////"; }
+	backend b1 { .host = "////."; }
 }
 
 varnish v1 -badvcl {
-	backend b1 { .host = "127.0.0.1"; .port = "////"; }
+	backend b1 { .host = "127.0.0.1"; .port = "////."; }
 }
 
 varnish v1 -badvcl {



From phk at varnish-cache.org  Mon Jan 30 07:20:01 2012
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Mon, 30 Jan 2012 08:20:01 +0100
Subject: [master] 263383b Whitespace CDO.
Message-ID: <E1RrlWz-0004Ea-RK@project.varnish-software.com>

commit 263383ba0a32ab40a30ea03b8895f6efa5099b3f
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Jan 30 07:19:49 2012 +0000

    Whitespace CDO.

diff --git a/include/vdef.h b/include/vdef.h
index 63a2a6b..9beedb0 100644
--- a/include/vdef.h
+++ b/include/vdef.h
@@ -8,7 +8,7 @@
  * Author: Rogier 'DocWilco' Mulhuijzen <rogier at fastly.com>
  *
  * Inspired by FreeBSD's <sys/cdefs.h>
- * 
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
diff --git a/lib/libvmod_std/vmod.py b/lib/libvmod_std/vmod.py
index e409628..041edcb 100755
--- a/lib/libvmod_std/vmod.py
+++ b/lib/libvmod_std/vmod.py
@@ -248,7 +248,7 @@ def dumps(s):
 #######################################################################
 
 if initname != "":
-	plist += "int " + initname 
+	plist += "int " + initname
 	plist += "(struct vmod_priv *, const struct VCL_conf *);\n"
 	pstruct += "\tvmod_init_f\t*_init;\n"
 	pinit += "\t" + initname + ",\n"



From phk at varnish-cache.org  Mon Jan 30 08:09:36 2012
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Mon, 30 Jan 2012 09:09:36 +0100
Subject: [master] ef4231f Work on the dot-graph while I ponder how streaming
	fits in.
Message-ID: <E1RrmIy-0008NW-VO@project.varnish-software.com>

commit ef4231f97f2e5e3a93c51eeb45cfa576cc5313b2
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Jan 30 08:09:18 2012 +0000

    Work on the dot-graph while I ponder how streaming fits in.

diff --git a/bin/varnishd/cache/cache_center.c b/bin/varnishd/cache/cache_center.c
index ea176cb..487e664 100644
--- a/bin/varnishd/cache/cache_center.c
+++ b/bin/varnishd/cache/cache_center.c
@@ -556,28 +556,17 @@ cnt_error(struct sess *sp, struct worker *wrk, struct req *req)
  *
 DOT subgraph xcluster_fetch {
 DOT	fetch [
-DOT		shape=ellipse
-DOT		label="fetch hdr\nfrom backend\n(find obj.ttl)"
-DOT	]
-DOT	vcl_fetch [
 DOT		shape=record
-DOT		label="vcl_fetch()|req.\nbereq.\nberesp."
+DOT		label="{cnt_fetch:|fetch hdr\nfrom backend|(find obj.ttl)|{vcl_fetch\{\}|{req.|bereq.|beresp.}}|{<err>error?|<rst>restart?}|{<hfp>hit_for_pass?|<del>deliver?}}"
 DOT	]
-DOT	fetch -> vcl_fetch [style=bold,color=blue]
-DOT	fetch -> vcl_fetch [style=bold,color=red]
 DOT	fetch_pass [
 DOT		shape=ellipse
 DOT		label="obj.f.pass=true"
 DOT	]
 DOT	vcl_fetch -> fetch_pass [label="hit_for_pass",style=bold,color=red]
 DOT }
-DOT fetch_pass -> fetchbody [style=bold,color=red]
-DOT vcl_fetch -> fetchbody [label="deliver",style=bold,color=blue]
-DOT vcl_fetch -> rstfetch [label="restart",color=purple]
-DOT rstfetch [label="RESTART",shape=plaintext]
-DOT fetch -> errfetch
-DOT vcl_fetch -> errfetch [label="error"]
-DOT errfetch [label="ERROR",shape=plaintext]
+DOT fetch:hfp -> fetchbody [style=bold,color=red]
+DOT fetch:del -> fetchbody [label="deliver",style=bold,color=blue]
  */
 
 static int
@@ -1059,15 +1048,15 @@ cnt_first(struct sess *sp, struct worker *wrk)
 DOT subgraph xcluster_hit {
 DOT	hit [
 DOT		shape=record
-DOT		label="vcl_hit()|req.\nobj."
+DOT		label="{cnt_hit:|{vcl_hit()|{req.|obj.}}|{<err>error?|<del>deliver?|<rst>restart?|<pass>pass?}}"
 DOT	]
 DOT }
-DOT hit -> err_hit [label="error"]
-DOT err_hit [label="ERROR",shape=plaintext]
-DOT hit -> rst_hit [label="restart",color=purple]
-DOT rst_hit [label="RESTART",shape=plaintext]
-DOT hit -> pass [label=pass,style=bold,color=red]
-DOT hit -> prepresp [label="deliver",style=bold,color=green]
+XDOT hit:err -> err_hit [label="error"]
+XDOT err_hit [label="ERROR",shape=plaintext]
+XDOT hit:rst -> rst_hit [label="restart",color=purple]
+XDOT rst_hit [label="RESTART",shape=plaintext]
+DOT hit:pass -> pass [label=pass,style=bold,color=red]
+DOT hit:del -> prepresp [label="deliver",style=bold,color=green]
  */
 
 static int
@@ -1125,18 +1114,13 @@ cnt_hit(struct sess *sp, struct worker *wrk, struct req *req)
 DOT subgraph xcluster_lookup {
 DOT	lookup [
 DOT		shape=record
-DOT		label="{cnt_lookup:|hash lookup}"
-DOT	]
-DOT	lookup2 [
-DOT		shape=diamond
-DOT		label="obj.f.pass ?"
+DOT		label="{<top>cnt_lookup:|hash lookup|{<busy>busy ?|<miss>miss ?}|{<no>no|obj.f.pass?|<yes>yes}}"
 DOT	]
 DOT }
-DOT lookup -> lookup [label="Busy object found"]
-DOT lookup -> miss [label="not found\ncreate new",style=bold,color=blue]
-DOT lookup -> lookup2 [label="found",style=bold,color=green]
-DOT lookup2 -> hit [label="no", style=bold,color=green]
-DOT lookup2 -> pass [label="yes",style=bold,color=red]
+DOT lookup:busy -> lookup:top [label="(waitinglist)"]
+DOT lookup:miss -> miss [style=bold,color=blue]
+DOT lookup:no -> hit [style=bold,color=green]
+DOT lookup:yes -> pass [style=bold,color=red]
  */
 
 static int
@@ -1235,21 +1219,16 @@ cnt_lookup(struct sess *sp, struct worker *wrk, struct req *req)
  *
 DOT subgraph xcluster_miss {
 DOT	miss [
-DOT		shape=ellipse
-DOT		label="filter req.->bereq."
-DOT	]
-DOT	vcl_miss [
 DOT		shape=record
-DOT		label="vcl_miss()|req.\nbereq."
+DOT		label="{cnt_miss:|filter req.-\>bereq.|{vcl_miss\{\}|{req.*|bereq.*}}|{<pass>pass?|<err>error?|<restart>restart?|<fetch>fetch?}}"
 DOT	]
-DOT	miss -> vcl_miss [style=bold,color=blue]
 DOT }
-DOT vcl_miss -> rst_miss [label="restart",color=purple]
-DOT rst_miss [label="RESTART",shape=plaintext]
-DOT vcl_miss -> err_miss [label="error"]
-DOT err_miss [label="ERROR",shape=plaintext]
-DOT vcl_miss -> fetch [label="fetch",style=bold,color=blue]
-DOT vcl_miss -> pass [label="pass",style=bold,color=red]
+XDOT miss:restart -> rst_miss [label="restart",color=purple]
+XDOT rst_miss [label="RESTART",shape=plaintext]
+XDOT miss:err -> err_miss [label="error"]
+XDOT err_miss [label="ERROR",shape=plaintext]
+DOT miss:fetch -> fetch [label="fetch",style=bold,color=blue]
+DOT miss:pass -> pass [label="pass",style=bold,color=red]
 DOT
  */
 
@@ -1318,30 +1297,15 @@ cnt_miss(struct sess *sp, struct worker *wrk, struct req *req)
  *
 DOT subgraph xcluster_pass {
 DOT	pass [
-DOT		shape=ellipse
-DOT		label="deref obj."
-DOT	]
-DOT	pass2 [
-DOT		shape=ellipse
-DOT		label="filter req.->bereq."
-DOT	]
-DOT	vcl_pass [
 DOT		shape=record
-DOT		label="vcl_pass()|req.\nbereq."
-DOT	]
-DOT	pass_do [
-DOT		shape=ellipse
-DOT		label="create anon object\n"
+DOT		label="{cnt_pass:|(XXX: deref obj.)|filter req.*-\>bereq.|{vcl_pass\{\}|{req.*|bereq.*}}|{<err>error?|<rst>restart?}|<pass>create anon obj}"
 DOT	]
-DOT	pass -> pass2 [style=bold, color=red]
-DOT	pass2 -> vcl_pass [style=bold, color=red]
-DOT	vcl_pass -> pass_do [label="pass"] [style=bold, color=red]
 DOT }
-DOT pass_do -> fetch [style=bold, color=red]
-DOT vcl_pass -> rst_pass [label="restart",color=purple]
-DOT rst_pass [label="RESTART",shape=plaintext]
-DOT vcl_pass -> err_pass [label="error"]
-DOT err_pass [label="ERROR",shape=plaintext]
+DOT pass:pass -> fetch [style=bold, color=red]
+XDOT pass:rst -> rst_pass [label="restart",color=purple]
+XDOT rst_pass [label="RESTART",shape=plaintext]
+XDOT pass:err -> err_pass [label="error"]
+XDOT err_pass [label="ERROR",shape=plaintext]
  */
 
 static int
@@ -1440,7 +1404,7 @@ cnt_pipe(struct sess *sp, struct worker *wrk, const struct req *req)
 DOT subgraph xcluster_recv {
 DOT	recv [
 DOT		shape=record
-DOT		label="{cnt_recv:|{vcl_recv\{\}|req.*}}"
+DOT		label="{cnt_recv:|{vcl_recv\{\}|req.*}|{<pipe>pipe?|<pass>pass?|<error>error?|<lookup>lookup?}}"
 DOT	]
 DOT }
 DOT subgraph xcluster_hash {
@@ -1450,13 +1414,13 @@ DOT		label="{cnt_recv:|{vcl_hash\{\}|req.*}}"
 DOT	]
 DOT }
 DOT ESI_REQ [ shape=hexagon ]
-DOT RESTART -> recv
+DOT RESTART -> recv [color=purple]
 DOT ESI_REQ -> recv
-DOT recv -> pipe [label="pipe",style=bold,color=orange]
-DOT recv -> pass2 [label="pass",style=bold,color=red]
-DOT recv -> err_recv [label="error"]
-DOT err_recv [label="ERROR",shape=plaintext]
-DOT recv -> hash [label="lookup",style=bold,color=green]
+DOT recv:pipe -> pipe [style=bold,color=orange]
+DOT recv:pass -> pass [style=bold,color=red]
+#DOT recv:error -> err_recv
+#DOT err_recv [label="ERROR",shape=plaintext]
+DOT recv:lookup -> hash [style=bold,color=green]
 DOT hash -> lookup [label="hash",style=bold,color=green]
  */
 



From phk at varnish-cache.org  Mon Jan 30 08:52:07 2012
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Mon, 30 Jan 2012 09:52:07 +0100
Subject: [master] e815d5a NB:  VCL syntax change!
Message-ID: <E1Rrmy7-00039S-13@project.varnish-software.com>

commit e815d5a5a420e507749e8a30f695fbcd7b28f369
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Jan 30 08:48:35 2012 +0000

    NB:  VCL syntax change!
    
    Change
    	return (hit_for_pass)
    to:
    	set beresp.do_pass = true
    
    It was wrong to make hit_for_pass a return action, because it is not.
    It is a flag on the cached object, and that is all it is.
    
    With streaming becoming the default, having it as return-action would
    make writing vcl_fetch{} more cumbersome.

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index aefb077..ad4f426 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -522,6 +522,7 @@ struct busyobj {
 	unsigned		do_gzip;
 	unsigned		do_gunzip;
 	unsigned		do_stream;
+	unsigned		do_pass;
 };
 
 /* Object structure --------------------------------------------------*/
diff --git a/bin/varnishd/cache/cache_center.c b/bin/varnishd/cache/cache_center.c
index 487e664..7bd0cfb 100644
--- a/bin/varnishd/cache/cache_center.c
+++ b/bin/varnishd/cache/cache_center.c
@@ -634,15 +634,14 @@ cnt_fetch(struct sess *sp, struct worker *wrk, struct req *req)
 			wrk->busyobj->exp.ttl = -1.;
 
 		AZ(wrk->busyobj->do_esi);
-
+		AZ(wrk->busyobj->do_pass);
+		
 		VCL_fetch_method(sp);
 
+		if (req->objcore != NULL && wrk->busyobj->do_pass)
+			req->objcore->flags |= OC_F_PASS;
+
 		switch (req->handling) {
-		case VCL_RET_HIT_FOR_PASS:
-			if (req->objcore != NULL)
-				req->objcore->flags |= OC_F_PASS;
-			sp->step = STP_FETCHBODY;
-			return (0);
 		case VCL_RET_DELIVER:
 			AssertObjCorePassOrBusy(req->objcore);
 			sp->step = STP_FETCHBODY;
@@ -717,18 +716,15 @@ cnt_fetchbody(struct sess *sp, struct worker *wrk, struct req *req)
 	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
 	CHECK_OBJ_NOTNULL(wrk->busyobj, BUSYOBJ_MAGIC);
 
-	assert(req->handling == VCL_RET_HIT_FOR_PASS ||
-	    req->handling == VCL_RET_DELIVER);
+	assert(req->handling == VCL_RET_DELIVER);
 
 	if (req->objcore == NULL) {
 		/* This is a pass from vcl_recv */
 		pass = 1;
 		/* VCL may have fiddled this, but that doesn't help */
 		wrk->busyobj->exp.ttl = -1.;
-	} else if (req->handling == VCL_RET_HIT_FOR_PASS) {
-		/* pass from vcl_fetch{} -> hit-for-pass */
-		/* XXX: the bereq was not filtered pass... */
-		pass = 1;
+	} else if (wrk->busyobj->do_pass) {
+			pass = 1;
 	} else {
 		/* regular object */
 		pass = 0;
diff --git a/bin/varnishd/cache/cache_vrt_var.c b/bin/varnishd/cache/cache_vrt_var.c
index e051dbb..c4c0c56 100644
--- a/bin/varnishd/cache/cache_vrt_var.c
+++ b/bin/varnishd/cache/cache_vrt_var.c
@@ -202,6 +202,7 @@ VBERESP(beresp, unsigned, do_esi, busyobj->do_esi)
 VBERESP(beresp, unsigned, do_gzip, busyobj->do_gzip)
 VBERESP(beresp, unsigned, do_gunzip, busyobj->do_gunzip)
 VBERESP(beresp, unsigned, do_stream, busyobj->do_stream)
+VBERESP(beresp, unsigned, do_pass, busyobj->do_pass)
 
 /*--------------------------------------------------------------------*/
 
diff --git a/bin/varnishd/default.vcl b/bin/varnishd/default.vcl
index 18121b8..d0d4c2e 100644
--- a/bin/varnishd/default.vcl
+++ b/bin/varnishd/default.vcl
@@ -109,7 +109,7 @@ sub vcl_fetch {
 		 * Mark as "Hit-For-Pass" for the next 2 minutes
 		 */
 		set beresp.ttl = 120 s;
-		return (hit_for_pass);
+		set beresp.do_pass = true;
     }
     return (deliver);
 }
diff --git a/bin/varnishtest/tests/c00011.vtc b/bin/varnishtest/tests/c00011.vtc
index 1dd34a8..18ee757 100644
--- a/bin/varnishtest/tests/c00011.vtc
+++ b/bin/varnishtest/tests/c00011.vtc
@@ -11,7 +11,7 @@ server s1 {
 
 varnish v1 -vcl+backend { 
 	sub vcl_fetch {
-		return(hit_for_pass);
+		set beresp.do_pass = true;
 	}
 } -start
 
diff --git a/bin/varnishtest/tests/c00014.vtc b/bin/varnishtest/tests/c00014.vtc
index ced8117..03089d8 100644
--- a/bin/varnishtest/tests/c00014.vtc
+++ b/bin/varnishtest/tests/c00014.vtc
@@ -15,7 +15,7 @@ server s1 {
 
 varnish v1 -vcl+backend { 
 	sub vcl_fetch {
-		return(hit_for_pass);
+		set beresp.do_pass = true;
 	}
 } -start
 
diff --git a/bin/varnishtest/tests/e00011.vtc b/bin/varnishtest/tests/e00011.vtc
index e725498..8f03bd4 100644
--- a/bin/varnishtest/tests/e00011.vtc
+++ b/bin/varnishtest/tests/e00011.vtc
@@ -17,8 +17,8 @@ server s1 {
 
 varnish v1 -vcl+backend {
 	sub vcl_fetch {
+		set beresp.do_pass = true;
 		set beresp.do_esi = true;
-		return(hit_for_pass);
 	}
 } -start 
 
diff --git a/bin/varnishtest/tests/r00318.vtc b/bin/varnishtest/tests/r00318.vtc
index 0170c3d..4e66fad 100644
--- a/bin/varnishtest/tests/r00318.vtc
+++ b/bin/varnishtest/tests/r00318.vtc
@@ -8,7 +8,7 @@ server s1 {
 varnish v1 -vcl+backend {
 	sub vcl_fetch {
 		set beresp.do_esi = true;
-		return(hit_for_pass);
+		set beresp.do_pass = true;
 	}
 } -start
 
diff --git a/lib/libvcl/generate.py b/lib/libvcl/generate.py
index b91f675..1d397c8 100755
--- a/lib/libvcl/generate.py
+++ b/lib/libvcl/generate.py
@@ -89,7 +89,7 @@ returns =(
 	('hash',	('hash',)),
 	('miss',	('error', 'restart', 'pass', 'fetch',)),
 	('hit',		('error', 'restart', 'pass', 'deliver',)),
-	('fetch',	('error', 'restart', 'hit_for_pass', 'deliver',)),
+	('fetch',	('error', 'restart', 'deliver',)),
 	('deliver',	('restart', 'deliver',)),
 	('error',	('restart', 'deliver',)),
 	('init',	('ok',)),
@@ -331,6 +331,12 @@ sp_variables = (
 		( 'fetch',),
 		'const struct sess *'
 	),
+	('beresp.do_pass',
+		'BOOL',
+		( 'fetch',),
+		( 'fetch',),
+		'const struct sess *'
+	),
 	('beresp.ttl',
 		'DURATION',
 		( 'fetch',),



From phk at varnish-cache.org  Mon Jan 30 13:03:43 2012
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Mon, 30 Jan 2012 14:03:43 +0100
Subject: [master] 58e4a20 s/wrk->busyobj->/bo->/ in cnt_fetchbody() (with a
	few notable exceptions.)
Message-ID: <E1Rrqtb-0004Of-5B@project.varnish-software.com>

commit 58e4a209ee09783bb7ad70303fc2fb757070b312
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Jan 30 13:03:06 2012 +0000

    s/wrk->busyobj->/bo->/ in cnt_fetchbody() (with a few notable exceptions.)

diff --git a/bin/varnishd/cache/cache_center.c b/bin/varnishd/cache/cache_center.c
index 7bd0cfb..3286abf 100644
--- a/bin/varnishd/cache/cache_center.c
+++ b/bin/varnishd/cache/cache_center.c
@@ -710,11 +710,13 @@ cnt_fetchbody(struct sess *sp, struct worker *wrk, struct req *req)
 	unsigned l;
 	struct vsb *vary = NULL;
 	int varyl = 0, pass;
+	struct busyobj *bo;
 
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
-	CHECK_OBJ_NOTNULL(wrk->busyobj, BUSYOBJ_MAGIC);
+	bo = wrk->busyobj;
+	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
 
 	assert(req->handling == VCL_RET_DELIVER);
 
@@ -722,9 +724,9 @@ cnt_fetchbody(struct sess *sp, struct worker *wrk, struct req *req)
 		/* This is a pass from vcl_recv */
 		pass = 1;
 		/* VCL may have fiddled this, but that doesn't help */
-		wrk->busyobj->exp.ttl = -1.;
-	} else if (wrk->busyobj->do_pass) {
-			pass = 1;
+		bo->exp.ttl = -1.;
+	} else if (bo->do_pass) {
+		pass = 1;
 	} else {
 		/* regular object */
 		pass = 0;
@@ -745,63 +747,61 @@ cnt_fetchbody(struct sess *sp, struct worker *wrk, struct req *req)
 
 	/* We do nothing unless the param is set */
 	if (!cache_param->http_gzip_support)
-		wrk->busyobj->do_gzip = wrk->busyobj->do_gunzip = 0;
+		bo->do_gzip = bo->do_gunzip = 0;
 
-	wrk->busyobj->is_gzip =
-	    http_HdrIs(wrk->busyobj->beresp, H_Content_Encoding, "gzip");
+	bo->is_gzip = http_HdrIs(bo->beresp, H_Content_Encoding, "gzip");
 
-	wrk->busyobj->is_gunzip =
-	    !http_GetHdr(wrk->busyobj->beresp, H_Content_Encoding, NULL);
+	bo->is_gunzip = !http_GetHdr(bo->beresp, H_Content_Encoding, NULL);
 
 	/* It can't be both */
-	assert(wrk->busyobj->is_gzip == 0 || wrk->busyobj->is_gunzip == 0);
+	assert(bo->is_gzip == 0 || bo->is_gunzip == 0);
 
 	/* We won't gunzip unless it is gzip'ed */
-	if (wrk->busyobj->do_gunzip && !wrk->busyobj->is_gzip)
-		wrk->busyobj->do_gunzip = 0;
+	if (bo->do_gunzip && !bo->is_gzip)
+		bo->do_gunzip = 0;
 
 	/* If we do gunzip, remove the C-E header */
-	if (wrk->busyobj->do_gunzip)
-		http_Unset(wrk->busyobj->beresp, H_Content_Encoding);
+	if (bo->do_gunzip)
+		http_Unset(bo->beresp, H_Content_Encoding);
 
 	/* We wont gzip unless it is ungziped */
-	if (wrk->busyobj->do_gzip && !wrk->busyobj->is_gunzip)
-		wrk->busyobj->do_gzip = 0;
+	if (bo->do_gzip && !bo->is_gunzip)
+		bo->do_gzip = 0;
 
 	/* If we do gzip, add the C-E header */
-	if (wrk->busyobj->do_gzip)
-		http_SetHeader(wrk, sp->vsl_id, wrk->busyobj->beresp,
+	if (bo->do_gzip)
+		http_SetHeader(wrk, sp->vsl_id, bo->beresp,
 		    "Content-Encoding: gzip");
 
 	/* But we can't do both at the same time */
-	assert(wrk->busyobj->do_gzip == 0 || wrk->busyobj->do_gunzip == 0);
+	assert(bo->do_gzip == 0 || bo->do_gunzip == 0);
 
 	/* ESI takes precedence and handles gzip/gunzip itself */
-	if (wrk->busyobj->do_esi)
-		wrk->busyobj->vfp = &vfp_esi;
-	else if (wrk->busyobj->do_gunzip)
-		wrk->busyobj->vfp = &vfp_gunzip;
-	else if (wrk->busyobj->do_gzip)
-		wrk->busyobj->vfp = &vfp_gzip;
-	else if (wrk->busyobj->is_gzip)
-		wrk->busyobj->vfp = &vfp_testgzip;
-
-	if (wrk->busyobj->do_esi || req->esi_level > 0)
-		wrk->busyobj->do_stream = 0;
+	if (bo->do_esi)
+		bo->vfp = &vfp_esi;
+	else if (bo->do_gunzip)
+		bo->vfp = &vfp_gunzip;
+	else if (bo->do_gzip)
+		bo->vfp = &vfp_gzip;
+	else if (bo->is_gzip)
+		bo->vfp = &vfp_testgzip;
+
+	if (bo->do_esi || req->esi_level > 0)
+		bo->do_stream = 0;
 	if (!req->wantbody)
-		wrk->busyobj->do_stream = 0;
+		bo->do_stream = 0;
 
 	/* No reason to try streaming a non-existing body */
-	if (wrk->busyobj->body_status == BS_NONE)
-		wrk->busyobj->do_stream = 0;
+	if (bo->body_status == BS_NONE)
+		bo->do_stream = 0;
 
-	l = http_EstimateWS(wrk->busyobj->beresp,
+	l = http_EstimateWS(bo->beresp,
 	    pass ? HTTPH_R_PASS : HTTPH_A_INS, &nhttp);
 
 	/* Create Vary instructions */
 	if (req->objcore != NULL) {
 		CHECK_OBJ_NOTNULL(req->objcore, OBJCORE_MAGIC);
-		vary = VRY_Create(sp, wrk->busyobj->beresp);
+		vary = VRY_Create(sp, bo->beresp);
 		if (vary != NULL) {
 			varyl = VSB_len(vary);
 			assert(varyl > 0);
@@ -815,7 +815,7 @@ cnt_fetchbody(struct sess *sp, struct worker *wrk, struct req *req)
 	 */
 	l += strlen("Content-Length: XxxXxxXxxXxxXxxXxx") + sizeof(void *);
 
-	if (wrk->busyobj->exp.ttl < cache_param->shortlived ||
+	if (bo->exp.ttl < cache_param->shortlived ||
 	    req->objcore == NULL)
 		req->storage_hint = TRANSIENT_STORAGE;
 
@@ -826,15 +826,15 @@ cnt_fetchbody(struct sess *sp, struct worker *wrk, struct req *req)
 		 * shortlived object on Transient storage.
 		 */
 		req->obj = STV_NewObject(wrk, TRANSIENT_STORAGE, l, nhttp);
-		if (wrk->busyobj->exp.ttl > cache_param->shortlived)
-			wrk->busyobj->exp.ttl = cache_param->shortlived;
-		wrk->busyobj->exp.grace = 0.0;
-		wrk->busyobj->exp.keep = 0.0;
+		if (bo->exp.ttl > cache_param->shortlived)
+			bo->exp.ttl = cache_param->shortlived;
+		bo->exp.grace = 0.0;
+		bo->exp.keep = 0.0;
 	}
 	if (req->obj == NULL) {
 		req->err_code = 503;
 		sp->step = STP_ERROR;
-		VDI_CloseFd(wrk, &wrk->busyobj->vbc);
+		VDI_CloseFd(wrk, &bo->vbc);
 		VBO_DerefBusyObj(wrk, &wrk->busyobj);
 		return (0);
 	}
@@ -842,8 +842,8 @@ cnt_fetchbody(struct sess *sp, struct worker *wrk, struct req *req)
 
 	req->storage_hint = NULL;
 
-	if (wrk->busyobj->do_gzip ||
-	    (wrk->busyobj->is_gzip && !wrk->busyobj->do_gunzip))
+	if (bo->do_gzip ||
+	    (bo->is_gzip && !bo->do_gunzip))
 		req->obj->gziped = 1;
 
 	if (vary != NULL) {
@@ -859,7 +859,7 @@ cnt_fetchbody(struct sess *sp, struct worker *wrk, struct req *req)
 	WS_Assert(req->obj->ws_o);
 
 	/* Filter into object */
-	hp = wrk->busyobj->beresp;
+	hp = bo->beresp;
 	hp2 = req->obj->http;
 
 	hp2->logtag = HTTP_Obj;
@@ -869,7 +869,7 @@ cnt_fetchbody(struct sess *sp, struct worker *wrk, struct req *req)
 	if (http_GetHdr(hp, H_Last_Modified, &b))
 		req->obj->last_modified = VTIM_parse(b);
 	else
-		req->obj->last_modified = floor(wrk->busyobj->exp.entered);
+		req->obj->last_modified = floor(bo->exp.entered);
 
 	assert(WRW_IsReleased(wrk));
 
@@ -881,11 +881,11 @@ cnt_fetchbody(struct sess *sp, struct worker *wrk, struct req *req)
 	if (req->obj->response == 200 &&
 	    req->http->conds &&
 	    RFC2616_Do_Cond(sp))
-		wrk->busyobj->do_stream = 0;
+		bo->do_stream = 0;
 
 	AssertObjCorePassOrBusy(req->obj->objcore);
 
-	if (wrk->busyobj->do_stream) {
+	if (bo->do_stream) {
 		sp->step = STP_PREPRESP;
 		return (0);
 	}
@@ -893,11 +893,11 @@ cnt_fetchbody(struct sess *sp, struct worker *wrk, struct req *req)
 	/* Use unmodified headers*/
 	i = FetchBody(wrk, req->obj);
 
-	http_Setup(wrk->busyobj->bereq, NULL);
-	http_Setup(wrk->busyobj->beresp, NULL);
-	wrk->busyobj->vfp = NULL;
+	http_Setup(bo->bereq, NULL);
+	http_Setup(bo->beresp, NULL);
+	bo->vfp = NULL;
 	assert(WRW_IsReleased(wrk));
-	AZ(wrk->busyobj->vbc);
+	AZ(bo->vbc);
 	AN(req->director);
 
 	if (i) {



From geoff at varnish-cache.org  Mon Jan 30 15:09:52 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 30 Jan 2012 16:09:52 +0100
Subject: [experimental-ims] 3a1b9ca Make this test case more robust (and
	faster) by using the CLI to control backend health,
	rather than rely on probes.
Message-ID: <E1Rrsrg-000841-F2@project.varnish-software.com>

commit 3a1b9ca8fcf1e9c901f6e57b8863ec678f6538a7
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Jan 24 19:35:28 2012 +0000

    Make this test case more robust (and faster) by using the
    CLI to control backend health, rather than rely on probes.
    
    Submitted by:	Federico G. Schwindt

diff --git a/bin/varnishtest/tests/v00036.vtc b/bin/varnishtest/tests/v00036.vtc
index e83acef..323f80c 100644
--- a/bin/varnishtest/tests/v00036.vtc
+++ b/bin/varnishtest/tests/v00036.vtc
@@ -1,127 +1,66 @@
 varnishtest "Test fallback director"
 
 server s1 {
-	rxreq
-	expect req.url == "/"
-	txresp -body "slash"
-	close
-
-	sema r1 sync 3
-
-	accept
-	rxreq
-	expect req.url == "/"
-	txresp -body "slash"
-	close
-
-	accept
-	rxreq
-	expect req.url == "/"
-	txresp -body "slash"
-	close
-
-	sema r3 sync 2
-
-	accept
-	rxreq
-	expect req.url == "/foo"
-	txresp -hdr "Foo: 1" -body "foobar"
-
+        rxreq
+        txresp -hdr "Foo: 1"
 } -start
 
 server s2 {
-	rxreq
-	expect req.url == "/"
-	txresp -body "slash"
-	close
-
-	sema r1 sync 3
-
-	accept
-	rxreq
-	expect req.url == "/"
-	txresp -body "slash"
-	close
-
-	sema r2 sync 2
-
-	accept
-	rxreq
-	expect req.url == "/foo"
-	txresp -hdr "Foo: 2" -body "foobar"
+        rxreq
+        txresp -hdr "Foo: 2"
 } -start
 
 server s3 {
-	rxreq
-	expect req.url == "/foo"
-	txresp -hdr "Foo: 3" -body "foobar"
+        rxreq
+        txresp -hdr "Foo: 3"
 } -start
 
 varnish v1 -vcl {
-
-	probe p1 {
-		.url = "/";
-		.timeout = 1s;
-		.interval = 1s;
-		.window = 4;
-		.threshold = 3;
-		.initial = 0;
-	}
-	probe p2 {
-		.url = "/";
-		.timeout = 1s;
-		.interval = 1s;
-		.window = 3;
-		.threshold = 2;
-		.initial = 0;
-	}
-
-	backend b1 {
-		.host = "${s1_addr}";
-		.port = "${s1_port}";
-		.max_connections = 1;
-		.probe = p1;
-	}
-	backend b2 {
-		.host = "${s2_addr}";
-		.port = "${s2_port}";
-		.max_connections = 1;
-		.probe = p2;
-	}
-	backend b3 {
-		.host = "${s3_addr}";
-		.port = "${s3_port}";
-	}
-	director f1 fallback {
-		{ .backend = b1; }
-		{ .backend = b2; }
-		{ .backend = b3; }
-	}
-
-	sub vcl_recv {
-		set req.backend = f1;
-		return(pass);
-	}
+        backend b1 {
+                .host = "${s1_addr}";
+                .port = "${s1_port}";
+        }
+        backend b2 {
+                .host = "${s2_addr}";
+                .port = "${s2_port}";
+        }
+        backend b3 {
+                .host = "${s3_addr}";
+                .port = "${s3_port}";
+        }
+        director f1 fallback {
+                { .backend = b1; }
+                { .backend = b2; }
+                { .backend = b3; }
+        }
+        sub vcl_recv {
+                set req.backend = f1;
+                return (pass);
+        }
 } -start
 
+varnish v1 -cliok "backend.set_health b1 sick"
+varnish v1 -cliok "backend.set_health b2 sick"
 
 client c1 {
-	# s1 & s2 are both sick, expect response from s3
-	txreq -url "/foo"
-	rxresp
-	expect resp.http.foo == "3"
+        # s1 & s2 are both sick, expect response from s3
+        txreq
+        rxresp
+        expect resp.http.foo == "3"
+} -run
 
-	sema r1 sync 3
+varnish v1 -cliok "backend.set_health b2 healthy"
 
-	# wait for s2 to become healthy
-	sema r2 sync 2
-	txreq -url "/foo"
-	rxresp
-	expect resp.http.foo == "2"
+client c1 {
+        txreq
+        rxresp
+        expect resp.http.foo == "2"
+} -run
 
-	# wait for s1 to become healthy
-	sema r3 sync 2
-	txreq -url "/foo"
-	rxresp
-	expect resp.http.foo == "1"
+varnish v1 -cliok "backend.set_health b1 healthy"
+
+client c1 {
+        txreq
+        rxresp
+        expect resp.http.foo == "1"
 } -run



From geoff at varnish-cache.org  Mon Jan 30 15:09:52 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 30 Jan 2012 16:09:52 +0100
Subject: [experimental-ims] b23cb23 cleaned out some cruft. Added links to the
	actual docs
Message-ID: <E1Rrsrg-000846-Ga@project.varnish-software.com>

commit b23cb230ad691426096bac155d5230e00ea928cc
Author: Per Buer <perbu at varnish-software.com>
Date:   Wed Jan 25 09:56:41 2012 +0100

    cleaned out some cruft. Added links to the actual docs

diff --git a/doc/sphinx/faq/general.rst b/doc/sphinx/faq/general.rst
index b30f093..2444394 100644
--- a/doc/sphinx/faq/general.rst
+++ b/doc/sphinx/faq/general.rst
@@ -5,25 +5,7 @@ General questions
 What is Varnish?
 ================
 
-Varnish is a state-of-the-art, high-performance `web accelerator <http://en.wikipedia.org/wiki/Web_accelerator>`_. It uses the advanced features in Linux 2.6, FreeBSD 6/7 and Solaris 10 to achieve its high performance.
-
-Some of the features include
-
-* A modern design
-* VCL - a very flexible configuration language
-* Load balancing with health checking of backends
-* Partial support for ESI
-* URL rewriting
-* Graceful handling of "dead" backends
-
-Features to come (Experimental):
-
-* Support for Ranged headers
-* Support for persistent cache
-	
-
-Varnish is free software and is licenced under a modified BSD licence. Please read the introduction to get started with Varnish.
-
+Please see ref:`tutorial-intro`.
 
 How...
 ======
@@ -59,7 +41,9 @@ This can be achieved by removing the query parameters using a regexp::
 
 **How can I force a refresh on a object cached by varnish?**
 
-Refreshing is often called `purging <http://dictionary.reference.com/browse/PURGE>`_ a document.  You can purge at least 2 different ways in Varnish:
+Refreshing is often called `purging
+<http://dictionary.reference.com/browse/PURGE>`_ a document.  You can
+purge at least 2 different ways in Varnish:
 
 1. Command line
 
@@ -118,23 +102,6 @@ You can use the ``bereq`` object for altering requests going to the backend, but
                 return(fetch);
         }
 
-**How do I force the backend to send Vary headers?**
-
-We have anecdotal evidence of non-RFC2616 compliant backends, which support content negotiation, but which do not emit a Vary header, unless the request contains Accept headers.
-
-It may be appropriate to send no-op Accept headers to trick the backend into sending us the Vary header.
-
-The following should be sufficient for most cases::
-
-        Accept: */*
-        Accept-Language: *
-        Accept-Charset: *
-        Accept-Encoding: identity
-
-Note that Accept-Encoding can not be set to ``*``, as the backend might then send back a compressed response which the client would be unable to process.
-
-This can of course be implemented in VCL.
-
 **How can I customize the error messages that Varnish returns?**
 
 A custom error page can be generated by adding a ``vcl_error`` to your configuration file. The default error page looks like this::
@@ -171,32 +138,9 @@ This can be achieved by removing the query parameters using a regexp::
         }
 
 
-Where...
-========
-
-**Can I find varnish for my operating system?**
-
-We know that Varnish has been packaged for Debian, Ubuntu, RHEL,
-CentOS, (Open)SUSE, Gentoo and FreeBSD, possibly more.  Check whatever
-package manager you use. Or read :ref:`Installing Varnish on your computer <install-doc>`.
-
 Can I...
 ========
 
-**Can I use Varnish as a client-side proxy?**
-
-No.  Varnish needs all backends configured in the VCL.  Look at squid
-instead.
-
-**Can I run Varnish on a 32bit system?**
-
-Yes, recently somebody even claimed to run Varnish on his N900 mobile
-phone recently, but if you have the choice, go 64 bit from the start.
-
-Varnish is written to use Virtual Memory and on a 32bit system that
-really cramps your style, and you will have trouble configuring more
-than 2 GB of storage.
-
 **Can I run Varnish on the same system as Apache?**
 
 Yes, and many people do that with good success.
@@ -243,83 +187,28 @@ Yes, you need VCL code like this::
 		set req.backend = foobar;
 	}
 
-    (XXX: reference to docs, once written)
+    Please see :ref:`tutorial-advanced_backend_servers-directors_`.
+
 
 Why ...
 =======
 
 **Why does it look like Varnish sends all requests to the backend? I thought it was a cache?**
 
-There are 2 common reasons for this:
-	1. The object's ``ttl expired``. A common situation is that the backend does not set an expiry time on the requested image/file/webpage, so Varnish uses the default TTL (normally 120s).
-	2. Your site uses ``cookies``:
-            * By default, varnish will not cache ``responses`` from the backend that come with a ``Set-Cookie``: header.
-            * By default, varnish will not serve ``requests`` with a ``Cookie:`` header, but pass them to the backend instead. Check out [wiki:VCLExamples these VCL examples] on how to make varnish cache cookied/logged in users sanely.
-
-
-**Why are regular expressions case-sensitive?**
-
-Some HTTP headers, such as ``Host:`` and ``Location:`` contain fully
-qualified domain names, which by definition is not case-sensitive.
-Other HTTP headers are case-sensitive, most notably the URLs.
-Therefore a "one size fits all" solution is not possible.
-
-In previous releases, we used the POSIX regular expressions
-supplied with the operating system, and decided, because the
-most common use of regexps were on ```Host:``` headers, that
-they should not be case-sensitive.
-
-From version 2.1.0 and forward, we use PCRE regular expressions,
-where it *is* possible to control case-sensitivity in the
-individual regular expressions, so we decided that it would
-probably confuse people if we made the default case-insensitive.
-(We promise not to change our minds about this again.)
-
-To make a PCRE regex case insensitive, put ``(?i)`` at the start::
-
-	if (req.http.host ~ "(?i)example.com$") {
-		...
-	}
-
-See the `PCRE man pages <http://www.pcre.org/pcre.txt>`_ for more information.
-
-**Are regular expressions case sensitive or not? Can I change it?**
-
-In 2.1 and newer, regular expressions are case sensitive by default.  In earlier versions, they were case insensitive.
-
-To change this for a single regex in 2.1, use ``(?i)`` at the start.
-
-See the `PCRE man pages <http://www.pcre.org/pcre.txt>`_ for more information.
-
-
-**Why does the ``Via:`` header say 1.1 in Varnish 2.1.x?**
-
-The number in the ``Via:`` header is the HTTP protocol version
-supported/applied, not the software's version number.
-
-**Why did you call it *Varnish*?**
-
-Long story, but basically the instigator of Varnish spent a long
-time staring at an art-poster with the word "Vernisage" and ended
-up checking it in a dictionary, which gives the following three
-meanings of the word:
-
-r.v. var?nished, var?nish?ing, var?nish?es
-
-	  1. To cover with varnish.
-	  2. To give a smooth and glossy finish to.
-	  3. To give a deceptively attractive appearance to; gloss over.
 
-The three point describes happens to your backend system when you
-put Varnish in front of it.
+Please see ref:`tutorial-increasing_your_hitrate`.
 
 **Why does Varnish require the system to have a C compiler?**
 
-The :ref:`VCL <faq-vcl>` compiler generates C source as output (your config file), and uses the systems C-compiler to compile that into a shared library.  If there is no C compiler, Varnish will not work.
+The :ref:`VCL <faq-vcl>` compiler generates C source as output (your
+config file), and uses the systems C-compiler to compile that into a
+shared library.  If there is no C compiler, Varnish will not work.
 
 **Isn't that security problem?**
 
-The days when you could prevent people from running non-approved programs by removing the C compiler from your system ended roughly with the VAX 11/780 computer.
+The days when you could prevent people from running non-approved
+programs by removing the C compiler from your system ended roughly
+with the VAX 11/780 computer.
 
 Troubleshooting
 ===============



From geoff at varnish-cache.org  Mon Jan 30 15:09:52 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 30 Jan 2012 16:09:52 +0100
Subject: [experimental-ims] db55f5f add link
Message-ID: <E1Rrsrg-00084D-I3@project.varnish-software.com>

commit db55f5f340eea9ed96499a06dece00fde997b9c6
Author: Per Buer <perbu at varnish-software.com>
Date:   Wed Jan 25 09:56:57 2012 +0100

    add link

diff --git a/doc/sphinx/tutorial/advanced_backend_servers.rst b/doc/sphinx/tutorial/advanced_backend_servers.rst
index 307f715..b4206d9 100644
--- a/doc/sphinx/tutorial/advanced_backend_servers.rst
+++ b/doc/sphinx/tutorial/advanced_backend_servers.rst
@@ -40,6 +40,8 @@ really arbitrary data. You want to send mobile devices to a different
 backend? No problem. if (req.User-agent ~ /mobile/) .... should do the
 trick. 
 
+.. _tutorial-advanced_backend_servers-directors:
+
 Directors
 ---------
 



From geoff at varnish-cache.org  Mon Jan 30 15:09:52 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 30 Jan 2012 16:09:52 +0100
Subject: [experimental-ims] a999224 Literal block colons in fully minimized
	form
Message-ID: <E1Rrsrg-00084M-KC@project.varnish-software.com>

commit a9992243b02d0ff02b10fc226c7da9de7553ddbc
Author: Andreas Plesner Jacobsen <apj at mutt.dk>
Date:   Wed Jan 25 10:03:51 2012 +0100

    Literal block colons in fully minimized form

diff --git a/doc/sphinx/reference/varnish-cli.rst b/doc/sphinx/reference/varnish-cli.rst
index a097248..668ceab 100644
--- a/doc/sphinx/reference/varnish-cli.rst
+++ b/doc/sphinx/reference/varnish-cli.rst
@@ -54,7 +54,7 @@ Syntax
 
 Commands are usually terminated with a newline. Long command can be
 entered using sh style *here documents*. The format of here-documents
-is:::
+is::
 
    << word
 	here document
@@ -233,7 +233,7 @@ file on the fly.
 
 Use the unix file permissions to control access to the file.
 
-An authenticated session looks like this:::
+An authenticated session looks like this::
 
    critter phk> telnet localhost 1234
    Trying ::1...
@@ -273,7 +273,7 @@ following byte sequence:
 
 and dumping the resulting digest in lower-case hex.
 
-In the above example, the secret file contained foo\n and thus:::
+In the above example, the secret file contained foo\n and thus::
 
    critter phk> cat > _
    ixslvvxrgkjptxmcgnnsdxsvdmvfympg
@@ -300,18 +300,18 @@ EXAMPLES
 ========
 
 Simple example: All requests where req.url exactly matches the string
-/news are banned from the cache:::
+/news are banned from the cache::
 
     req.url == "/news"
 
 Example: Ban all documents where the name does not end with ".ogg",
-and where the size of the object is greater than 10 megabytes:::
+and where the size of the object is greater than 10 megabytes::
 
     req.url !~ "\.ogg$" && obj.size > 10MB
 
 Example: Ban all documents where the serving host is "example.com"
 or "www.example.com", and where the Set-Cookie header received from
-the backend contains "USERID=1663":::
+the backend contains "USERID=1663"::
 
     req.http.host ~ "^(?i)(www\.)example.com$" && obj.set-cookie ~ "USERID=1663"
 
diff --git a/doc/sphinx/reference/varnishadm.rst b/doc/sphinx/reference/varnishadm.rst
index bfad196..3521474 100644
--- a/doc/sphinx/reference/varnishadm.rst
+++ b/doc/sphinx/reference/varnishadm.rst
@@ -66,7 +66,7 @@ zero if the command succeeded, and non-zero otherwise.
 EXAMPLES
 ========
 
-Some ways you can use varnishadm:::
+Some ways you can use varnishadm::
 
            varnishadm -T localhost:999 -S /var/db/secret vcl.use foo
            echo vcl.use foo | varnishadm -T localhost:999 -S /var/db/secret
diff --git a/doc/sphinx/reference/varnishlog.rst b/doc/sphinx/reference/varnishlog.rst
index f626332..9a2868f 100644
--- a/doc/sphinx/reference/varnishlog.rst
+++ b/doc/sphinx/reference/varnishlog.rst
@@ -142,11 +142,11 @@ The following log entry tags are currently defined:
 EXAMPLES
 ========
 
-The following command line simply copies all log entries to a log file:::
+The following command line simply copies all log entries to a log file::
 
     $ varnishlog -w /var/log/varnish.log
 
-The following command line reads that same log file and displays requests for the front page:::
+The following command line reads that same log file and displays requests for the front page::
 
     $ varnishlog -r /var/log/varnish.log -c -m 'RxURL:^/$'
 
diff --git a/doc/sphinx/reference/varnishtop.rst b/doc/sphinx/reference/varnishtop.rst
index 9b3ca9f..86e6cf6 100644
--- a/doc/sphinx/reference/varnishtop.rst
+++ b/doc/sphinx/reference/varnishtop.rst
@@ -79,12 +79,12 @@ EXAMPLES
 ========
 
 The following example displays a continuously updated list of the most
-frequently requested URLs:::
+frequently requested URLs::
 
   varnishtop -i RxURL
 
 The following example displays a continuously updated list of the most
-commonly used user agents:::
+commonly used user agents::
 
   varnishtop -i RxHeader -C -I ^User-Agent
 
diff --git a/doc/sphinx/reference/vcl.rst b/doc/sphinx/reference/vcl.rst
index 2c185ed..1ef14d6 100644
--- a/doc/sphinx/reference/vcl.rst
+++ b/doc/sphinx/reference/vcl.rst
@@ -96,14 +96,14 @@ file as a quoted string.
 Backend declarations
 --------------------
 
-A backend declaration creates and initializes a named backend object:::
+A backend declaration creates and initializes a named backend object::
 
   backend www {
     .host = "www.example.com";
     .port = "http";
   }
 
-The backend object can later be used to select a backend at request time:::
+The backend object can later be used to select a backend at request time::
 
   if (req.http.host ~ "(?i)^(www.)?example.com$") {
     set req.backend = www;
@@ -118,7 +118,7 @@ backend connection, .first_byte_timeout for the time to wait for the
 first byte from the backend and .between_bytes_timeout for time to
 wait between each received byte.
 
-These can be set in the declaration like this:::
+These can be set in the declaration like this::
 
   backend www {
     .host = "www.example.com";
@@ -145,7 +145,7 @@ be used.
 There are several types of directors. The different director types
 use different algorithms to choose which backend to use.
 
-Configuring a director may look like this:::
+Configuring a director may look like this::
 
   director b2 random {
     .retries = 5;
@@ -312,7 +312,7 @@ Probes take the following parameters:
   Default is 2 seconds.
 
 A backend with a probe can be defined like this, together with the
-backend or director:::
+backend or director::
 
   backend www {
     .host = "www.example.com";
@@ -326,7 +326,7 @@ backend or director:::
     }
   }
 
-Or it can be defined separately and then referenced:::
+Or it can be defined separately and then referenced::
 
   probe healthcheck {
      .url = "/status.cgi";
@@ -361,7 +361,7 @@ ACLs
 ----
 
 An ACL declaration creates and initializes a named access control list
-which can later be used to match client addresses:::
+which can later be used to match client addresses::
 
   acl local {
     "localhost";         // myself
@@ -375,7 +375,7 @@ if it is preceded by a negation mark, it will reject any address it is
 compared to, which may not be what you intended.  If the entry is
 enclosed in parentheses, however, it will simply be ignored.
 
-To match an IP address against an ACL, simply use the match operator:::
+To match an IP address against an ACL, simply use the match operator::
 
   if (client.ip ~ local) {
     return (pipe);
@@ -390,7 +390,7 @@ PCRE(3) man page.
 
 To send flags to the PCRE engine, such as to turn on *case
 insensitivity* add the flag within parens following a question mark,
-like this:::
+like this::
 
   if (req.http.host ~ "(?i)example.com$") {
           ...
@@ -424,7 +424,7 @@ ban_url(regex)
 Subroutines
 ~~~~~~~~~~~
 
-A subroutine is used to group code for legibility or reusability:::
+A subroutine is used to group code for legibility or reusability::
   
   sub pipe_if_local {
     if (client.ip ~ local) {
@@ -646,7 +646,7 @@ appear in the source.
 The default versions distributed with Varnish will be implicitly
 concatenated as a last resort at the end.
 
-Example:::
+Example::
 
 	# in file "main.vcl"
 	include "backends.vcl";
@@ -891,7 +891,7 @@ resp.response
 resp.http.header
   The corresponding HTTP header.
 
-Values may be assigned to variables using the set keyword:::
+Values may be assigned to variables using the set keyword::
 
   sub vcl_recv {
     # Normalize the Host: header
@@ -900,7 +900,7 @@ Values may be assigned to variables using the set keyword:::
     }
   }
 
-HTTP headers can be removed entirely using the remove keyword:::
+HTTP headers can be removed entirely using the remove keyword::
 
   sub vcl_fetch {
     # Don't cache cookies
@@ -942,7 +942,7 @@ EXAMPLES
 
 The following code is the equivalent of the default configuration with
 the backend address set to "backend.example.com" and no backend port
-specified:::
+specified::
 
   backend default {
    .host = "backend.example.com";
@@ -954,7 +954,7 @@ specified:::
 
 The following example shows how to support multiple sites running on
 separate backends in the same Varnish instance, by selecting backends
-based on the request URL:::
+based on the request URL::
 
   backend www {
     .host = "www.example.com";
@@ -980,7 +980,7 @@ based on the request URL:::
 The following snippet demonstrates how to force a minimum TTL for
 all documents.  Note that this is not the same as setting the
 default_ttl run-time parameter, as that only affects document for
-which the backend did not specify a TTL:::
+which the backend did not specify a TTL::
   
   import std; # needed for std.log
 
@@ -992,7 +992,7 @@ which the backend did not specify a TTL:::
   }
 
 The following snippet demonstrates how to force Varnish to cache
-documents even when cookies are present:::
+documents even when cookies are present::
 
   sub vcl_recv {
     if (req.request == "GET" && req.http.cookie) {
@@ -1007,7 +1007,7 @@ documents even when cookies are present:::
   }
 
 The following code implements the HTTP PURGE method as used by Squid
-for object invalidation:::
+for object invalidation::
 
   acl purge {
     "localhost";



From geoff at varnish-cache.org  Mon Jan 30 15:09:52 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 30 Jan 2012 16:09:52 +0100
Subject: [experimental-ims] 4f3e2e9 Rest of the literal block colons in fully
	minimized form
Message-ID: <E1Rrsrg-00084Z-LW@project.varnish-software.com>

commit 4f3e2e9268772ec0bc58a79e513105ce7d769403
Author: Andreas Plesner Jacobsen <apj at mutt.dk>
Date:   Wed Jan 25 12:12:42 2012 +0100

    Rest of the literal block colons in fully minimized form

diff --git a/doc/sphinx/installation/install.rst b/doc/sphinx/installation/install.rst
index fe8496d..4a836ff 100644
--- a/doc/sphinx/installation/install.rst
+++ b/doc/sphinx/installation/install.rst
@@ -114,7 +114,7 @@ Configuring and compiling
 ~~~~~~~~~~~~~~~~~~~~~~~~~
 
 Next, configuration: The configuration will need the dependencies
-above satisfied. Once that is taken care of:::
+above satisfied. Once that is taken care of::
 
 	cd varnish-cache
 	sh autogen.sh
diff --git a/doc/sphinx/tutorial/esi.rst b/doc/sphinx/tutorial/esi.rst
index 1d01cb8..1badd5d 100644
--- a/doc/sphinx/tutorial/esi.rst
+++ b/doc/sphinx/tutorial/esi.rst
@@ -26,7 +26,7 @@ Example: esi include
 ~~~~~~~~~~~~~~~~~~~~
 
 Lets see an example how this could be used. This simple cgi script
-outputs the date:::
+outputs the date::
 
      #!/bin/sh
      
@@ -34,7 +34,7 @@ outputs the date:::
      echo ''
      date "+%Y-%m-%d %H:%M"
 
-Now, lets have an HTML file that has an ESI include statement:::
+Now, lets have an HTML file that has an ESI include statement::
 
      <HTML>
      <BODY>
@@ -43,7 +43,7 @@ Now, lets have an HTML file that has an ESI include statement:::
      </BODY>
      </HTML>
 
-For ESI to work you need to activate ESI processing in VCL, like this:::
+For ESI to work you need to activate ESI processing in VCL, like this::
 
     sub vcl_fetch {
     	if (req.url == "/test.html") {
@@ -59,7 +59,7 @@ Example: esi remove
 ~~~~~~~~~~~~~~~~~~~
 
 The *remove* keyword allows you to remove output. You can use this to make
-a fall back of sorts, when ESI is not available, like this:::
+a fall back of sorts, when ESI is not available, like this::
 
   <esi:include src="http://www.example.com/ad.html"/> 
   <esi:remove> 
diff --git a/doc/sphinx/tutorial/handling_misbehaving_servers.rst b/doc/sphinx/tutorial/handling_misbehaving_servers.rst
index 6cd7111..406b4b3 100644
--- a/doc/sphinx/tutorial/handling_misbehaving_servers.rst
+++ b/doc/sphinx/tutorial/handling_misbehaving_servers.rst
@@ -26,7 +26,7 @@ requests somewhat stale content.
 
 So, in order to serve stale content we must first have some content to
 serve. So to make Varnish keep all objects for 30 minutes beyond their
-TTL use the following VCL:::
+TTL use the following VCL::
 
   sub vcl_fetch {
     set beresp.grace = 30m;
@@ -64,7 +64,7 @@ errors. You can instruct Varnish to try to handle this in a
 more-than-graceful way - enter *Saint mode*. Saint mode enables you to
 discard a certain page from one backend server and either try another
 server or serve stale content from cache. Lets have a look at how this
-can be enabled in VCL:::
+can be enabled in VCL::
 
   sub vcl_fetch {
     if (beresp.status == 500) { 
diff --git a/doc/sphinx/tutorial/troubleshooting.rst b/doc/sphinx/tutorial/troubleshooting.rst
index 7ab9435..5bbcf6c 100644
--- a/doc/sphinx/tutorial/troubleshooting.rst
+++ b/doc/sphinx/tutorial/troubleshooting.rst
@@ -55,7 +55,7 @@ Varnish is crashing
 
 When varnish goes bust the child processes crashes. Usually the mother
 process will manage this by restarting the child process again. Any
-errors will be logged in syslog. It might look like this:::
+errors will be logged in syslog. It might look like this::
 
        Mar  8 13:23:38 smoke varnishd[15670]: Child (15671) not responding to CLI, killing it.
        Mar  8 13:23:43 smoke varnishd[15670]: last message repeated 2 times
@@ -78,13 +78,13 @@ Varnish gives me Guru meditation
 First find the relevant log entries in varnishlog. That will probably
 give you a clue. Since varnishlog logs so much data it might be hard
 to track the entries down. You can set varnishlog to log all your 503
-errors by issuing the following command:::
+errors by issuing the following command::
 
    $ varnishlog -c -m TxStatus:503
 
 If the error happened just a short time ago the transaction might still
 be in the shared memory log segment. To get varnishlog to process the
-whole shared memory log just add the -d option:::
+whole shared memory log just add the -d option::
 
    $ varnishlog -d -c -m TxStatus:503
 
diff --git a/doc/sphinx/tutorial/vcl.rst b/doc/sphinx/tutorial/vcl.rst
index 3ed78c5..7171ab6 100644
--- a/doc/sphinx/tutorial/vcl.rst
+++ b/doc/sphinx/tutorial/vcl.rst
@@ -136,7 +136,7 @@ Example 1 - manipulating headers
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 Lets say we want to remove the cookie for all objects in the /static
-directory of our web server:::
+directory of our web server::
 
   sub vcl_recv {
     if (req.url ~ "^/images") {
@@ -154,7 +154,7 @@ Example 2 - manipulating beresp
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 Here we override the TTL of a object comming from the backend if it
-matches certain criteria:::
+matches certain criteria::
 
   sub vcl_fetch {
      if (req.url ~ "\.(png|gif|jpg)$") {



From geoff at varnish-cache.org  Mon Jan 30 15:09:52 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 30 Jan 2012 16:09:52 +0100
Subject: [experimental-ims] cff8e37 Make the invalid domains FQDN to save time
	if you have a long search-list.
Message-ID: <E1Rrsrg-00084m-Ni@project.varnish-software.com>

commit cff8e37bbac7c690a4566ca1a63cb509ff9852c2
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Jan 25 16:10:51 2012 +0000

    Make the invalid domains FQDN to save time if you have a long search-list.

diff --git a/bin/varnishtest/tests/v00002.vtc b/bin/varnishtest/tests/v00002.vtc
index 586cfac..d9d33d6 100644
--- a/bin/varnishtest/tests/v00002.vtc
+++ b/bin/varnishtest/tests/v00002.vtc
@@ -116,11 +116,11 @@ varnish v1 -badvcl {
 }
 
 varnish v1 -badvcl {
-	backend b1 { .host = "////"; }
+	backend b1 { .host = "////."; }
 }
 
 varnish v1 -badvcl {
-	backend b1 { .host = "127.0.0.1"; .port = "////"; }
+	backend b1 { .host = "127.0.0.1"; .port = "////."; }
 }
 
 varnish v1 -badvcl {



From geoff at varnish-cache.org  Mon Jan 30 15:09:52 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 30 Jan 2012 16:09:52 +0100
Subject: [experimental-ims] 263383b Whitespace CDO.
Message-ID: <E1Rrsrg-00084y-PU@project.varnish-software.com>

commit 263383ba0a32ab40a30ea03b8895f6efa5099b3f
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Jan 30 07:19:49 2012 +0000

    Whitespace CDO.

diff --git a/include/vdef.h b/include/vdef.h
index 63a2a6b..9beedb0 100644
--- a/include/vdef.h
+++ b/include/vdef.h
@@ -8,7 +8,7 @@
  * Author: Rogier 'DocWilco' Mulhuijzen <rogier at fastly.com>
  *
  * Inspired by FreeBSD's <sys/cdefs.h>
- * 
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
diff --git a/lib/libvmod_std/vmod.py b/lib/libvmod_std/vmod.py
index e409628..041edcb 100755
--- a/lib/libvmod_std/vmod.py
+++ b/lib/libvmod_std/vmod.py
@@ -248,7 +248,7 @@ def dumps(s):
 #######################################################################
 
 if initname != "":
-	plist += "int " + initname 
+	plist += "int " + initname
 	plist += "(struct vmod_priv *, const struct VCL_conf *);\n"
 	pstruct += "\tvmod_init_f\t*_init;\n"
 	pinit += "\t" + initname + ",\n"



From geoff at varnish-cache.org  Mon Jan 30 15:09:52 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 30 Jan 2012 16:09:52 +0100
Subject: [experimental-ims] ef4231f Work on the dot-graph while I ponder how
	streaming fits in.
Message-ID: <E1Rrsrg-000859-SC@project.varnish-software.com>

commit ef4231f97f2e5e3a93c51eeb45cfa576cc5313b2
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Jan 30 08:09:18 2012 +0000

    Work on the dot-graph while I ponder how streaming fits in.

diff --git a/bin/varnishd/cache/cache_center.c b/bin/varnishd/cache/cache_center.c
index ea176cb..487e664 100644
--- a/bin/varnishd/cache/cache_center.c
+++ b/bin/varnishd/cache/cache_center.c
@@ -556,28 +556,17 @@ cnt_error(struct sess *sp, struct worker *wrk, struct req *req)
  *
 DOT subgraph xcluster_fetch {
 DOT	fetch [
-DOT		shape=ellipse
-DOT		label="fetch hdr\nfrom backend\n(find obj.ttl)"
-DOT	]
-DOT	vcl_fetch [
 DOT		shape=record
-DOT		label="vcl_fetch()|req.\nbereq.\nberesp."
+DOT		label="{cnt_fetch:|fetch hdr\nfrom backend|(find obj.ttl)|{vcl_fetch\{\}|{req.|bereq.|beresp.}}|{<err>error?|<rst>restart?}|{<hfp>hit_for_pass?|<del>deliver?}}"
 DOT	]
-DOT	fetch -> vcl_fetch [style=bold,color=blue]
-DOT	fetch -> vcl_fetch [style=bold,color=red]
 DOT	fetch_pass [
 DOT		shape=ellipse
 DOT		label="obj.f.pass=true"
 DOT	]
 DOT	vcl_fetch -> fetch_pass [label="hit_for_pass",style=bold,color=red]
 DOT }
-DOT fetch_pass -> fetchbody [style=bold,color=red]
-DOT vcl_fetch -> fetchbody [label="deliver",style=bold,color=blue]
-DOT vcl_fetch -> rstfetch [label="restart",color=purple]
-DOT rstfetch [label="RESTART",shape=plaintext]
-DOT fetch -> errfetch
-DOT vcl_fetch -> errfetch [label="error"]
-DOT errfetch [label="ERROR",shape=plaintext]
+DOT fetch:hfp -> fetchbody [style=bold,color=red]
+DOT fetch:del -> fetchbody [label="deliver",style=bold,color=blue]
  */
 
 static int
@@ -1059,15 +1048,15 @@ cnt_first(struct sess *sp, struct worker *wrk)
 DOT subgraph xcluster_hit {
 DOT	hit [
 DOT		shape=record
-DOT		label="vcl_hit()|req.\nobj."
+DOT		label="{cnt_hit:|{vcl_hit()|{req.|obj.}}|{<err>error?|<del>deliver?|<rst>restart?|<pass>pass?}}"
 DOT	]
 DOT }
-DOT hit -> err_hit [label="error"]
-DOT err_hit [label="ERROR",shape=plaintext]
-DOT hit -> rst_hit [label="restart",color=purple]
-DOT rst_hit [label="RESTART",shape=plaintext]
-DOT hit -> pass [label=pass,style=bold,color=red]
-DOT hit -> prepresp [label="deliver",style=bold,color=green]
+XDOT hit:err -> err_hit [label="error"]
+XDOT err_hit [label="ERROR",shape=plaintext]
+XDOT hit:rst -> rst_hit [label="restart",color=purple]
+XDOT rst_hit [label="RESTART",shape=plaintext]
+DOT hit:pass -> pass [label=pass,style=bold,color=red]
+DOT hit:del -> prepresp [label="deliver",style=bold,color=green]
  */
 
 static int
@@ -1125,18 +1114,13 @@ cnt_hit(struct sess *sp, struct worker *wrk, struct req *req)
 DOT subgraph xcluster_lookup {
 DOT	lookup [
 DOT		shape=record
-DOT		label="{cnt_lookup:|hash lookup}"
-DOT	]
-DOT	lookup2 [
-DOT		shape=diamond
-DOT		label="obj.f.pass ?"
+DOT		label="{<top>cnt_lookup:|hash lookup|{<busy>busy ?|<miss>miss ?}|{<no>no|obj.f.pass?|<yes>yes}}"
 DOT	]
 DOT }
-DOT lookup -> lookup [label="Busy object found"]
-DOT lookup -> miss [label="not found\ncreate new",style=bold,color=blue]
-DOT lookup -> lookup2 [label="found",style=bold,color=green]
-DOT lookup2 -> hit [label="no", style=bold,color=green]
-DOT lookup2 -> pass [label="yes",style=bold,color=red]
+DOT lookup:busy -> lookup:top [label="(waitinglist)"]
+DOT lookup:miss -> miss [style=bold,color=blue]
+DOT lookup:no -> hit [style=bold,color=green]
+DOT lookup:yes -> pass [style=bold,color=red]
  */
 
 static int
@@ -1235,21 +1219,16 @@ cnt_lookup(struct sess *sp, struct worker *wrk, struct req *req)
  *
 DOT subgraph xcluster_miss {
 DOT	miss [
-DOT		shape=ellipse
-DOT		label="filter req.->bereq."
-DOT	]
-DOT	vcl_miss [
 DOT		shape=record
-DOT		label="vcl_miss()|req.\nbereq."
+DOT		label="{cnt_miss:|filter req.-\>bereq.|{vcl_miss\{\}|{req.*|bereq.*}}|{<pass>pass?|<err>error?|<restart>restart?|<fetch>fetch?}}"
 DOT	]
-DOT	miss -> vcl_miss [style=bold,color=blue]
 DOT }
-DOT vcl_miss -> rst_miss [label="restart",color=purple]
-DOT rst_miss [label="RESTART",shape=plaintext]
-DOT vcl_miss -> err_miss [label="error"]
-DOT err_miss [label="ERROR",shape=plaintext]
-DOT vcl_miss -> fetch [label="fetch",style=bold,color=blue]
-DOT vcl_miss -> pass [label="pass",style=bold,color=red]
+XDOT miss:restart -> rst_miss [label="restart",color=purple]
+XDOT rst_miss [label="RESTART",shape=plaintext]
+XDOT miss:err -> err_miss [label="error"]
+XDOT err_miss [label="ERROR",shape=plaintext]
+DOT miss:fetch -> fetch [label="fetch",style=bold,color=blue]
+DOT miss:pass -> pass [label="pass",style=bold,color=red]
 DOT
  */
 
@@ -1318,30 +1297,15 @@ cnt_miss(struct sess *sp, struct worker *wrk, struct req *req)
  *
 DOT subgraph xcluster_pass {
 DOT	pass [
-DOT		shape=ellipse
-DOT		label="deref obj."
-DOT	]
-DOT	pass2 [
-DOT		shape=ellipse
-DOT		label="filter req.->bereq."
-DOT	]
-DOT	vcl_pass [
 DOT		shape=record
-DOT		label="vcl_pass()|req.\nbereq."
-DOT	]
-DOT	pass_do [
-DOT		shape=ellipse
-DOT		label="create anon object\n"
+DOT		label="{cnt_pass:|(XXX: deref obj.)|filter req.*-\>bereq.|{vcl_pass\{\}|{req.*|bereq.*}}|{<err>error?|<rst>restart?}|<pass>create anon obj}"
 DOT	]
-DOT	pass -> pass2 [style=bold, color=red]
-DOT	pass2 -> vcl_pass [style=bold, color=red]
-DOT	vcl_pass -> pass_do [label="pass"] [style=bold, color=red]
 DOT }
-DOT pass_do -> fetch [style=bold, color=red]
-DOT vcl_pass -> rst_pass [label="restart",color=purple]
-DOT rst_pass [label="RESTART",shape=plaintext]
-DOT vcl_pass -> err_pass [label="error"]
-DOT err_pass [label="ERROR",shape=plaintext]
+DOT pass:pass -> fetch [style=bold, color=red]
+XDOT pass:rst -> rst_pass [label="restart",color=purple]
+XDOT rst_pass [label="RESTART",shape=plaintext]
+XDOT pass:err -> err_pass [label="error"]
+XDOT err_pass [label="ERROR",shape=plaintext]
  */
 
 static int
@@ -1440,7 +1404,7 @@ cnt_pipe(struct sess *sp, struct worker *wrk, const struct req *req)
 DOT subgraph xcluster_recv {
 DOT	recv [
 DOT		shape=record
-DOT		label="{cnt_recv:|{vcl_recv\{\}|req.*}}"
+DOT		label="{cnt_recv:|{vcl_recv\{\}|req.*}|{<pipe>pipe?|<pass>pass?|<error>error?|<lookup>lookup?}}"
 DOT	]
 DOT }
 DOT subgraph xcluster_hash {
@@ -1450,13 +1414,13 @@ DOT		label="{cnt_recv:|{vcl_hash\{\}|req.*}}"
 DOT	]
 DOT }
 DOT ESI_REQ [ shape=hexagon ]
-DOT RESTART -> recv
+DOT RESTART -> recv [color=purple]
 DOT ESI_REQ -> recv
-DOT recv -> pipe [label="pipe",style=bold,color=orange]
-DOT recv -> pass2 [label="pass",style=bold,color=red]
-DOT recv -> err_recv [label="error"]
-DOT err_recv [label="ERROR",shape=plaintext]
-DOT recv -> hash [label="lookup",style=bold,color=green]
+DOT recv:pipe -> pipe [style=bold,color=orange]
+DOT recv:pass -> pass [style=bold,color=red]
+#DOT recv:error -> err_recv
+#DOT err_recv [label="ERROR",shape=plaintext]
+DOT recv:lookup -> hash [style=bold,color=green]
 DOT hash -> lookup [label="hash",style=bold,color=green]
  */
 



From geoff at varnish-cache.org  Mon Jan 30 15:09:52 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 30 Jan 2012 16:09:52 +0100
Subject: [experimental-ims] e815d5a NB:  VCL syntax change!
Message-ID: <E1Rrsrg-00085M-Ul@project.varnish-software.com>

commit e815d5a5a420e507749e8a30f695fbcd7b28f369
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Jan 30 08:48:35 2012 +0000

    NB:  VCL syntax change!
    
    Change
    	return (hit_for_pass)
    to:
    	set beresp.do_pass = true
    
    It was wrong to make hit_for_pass a return action, because it is not.
    It is a flag on the cached object, and that is all it is.
    
    With streaming becoming the default, having it as return-action would
    make writing vcl_fetch{} more cumbersome.

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index aefb077..ad4f426 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -522,6 +522,7 @@ struct busyobj {
 	unsigned		do_gzip;
 	unsigned		do_gunzip;
 	unsigned		do_stream;
+	unsigned		do_pass;
 };
 
 /* Object structure --------------------------------------------------*/
diff --git a/bin/varnishd/cache/cache_center.c b/bin/varnishd/cache/cache_center.c
index 487e664..7bd0cfb 100644
--- a/bin/varnishd/cache/cache_center.c
+++ b/bin/varnishd/cache/cache_center.c
@@ -634,15 +634,14 @@ cnt_fetch(struct sess *sp, struct worker *wrk, struct req *req)
 			wrk->busyobj->exp.ttl = -1.;
 
 		AZ(wrk->busyobj->do_esi);
-
+		AZ(wrk->busyobj->do_pass);
+		
 		VCL_fetch_method(sp);
 
+		if (req->objcore != NULL && wrk->busyobj->do_pass)
+			req->objcore->flags |= OC_F_PASS;
+
 		switch (req->handling) {
-		case VCL_RET_HIT_FOR_PASS:
-			if (req->objcore != NULL)
-				req->objcore->flags |= OC_F_PASS;
-			sp->step = STP_FETCHBODY;
-			return (0);
 		case VCL_RET_DELIVER:
 			AssertObjCorePassOrBusy(req->objcore);
 			sp->step = STP_FETCHBODY;
@@ -717,18 +716,15 @@ cnt_fetchbody(struct sess *sp, struct worker *wrk, struct req *req)
 	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
 	CHECK_OBJ_NOTNULL(wrk->busyobj, BUSYOBJ_MAGIC);
 
-	assert(req->handling == VCL_RET_HIT_FOR_PASS ||
-	    req->handling == VCL_RET_DELIVER);
+	assert(req->handling == VCL_RET_DELIVER);
 
 	if (req->objcore == NULL) {
 		/* This is a pass from vcl_recv */
 		pass = 1;
 		/* VCL may have fiddled this, but that doesn't help */
 		wrk->busyobj->exp.ttl = -1.;
-	} else if (req->handling == VCL_RET_HIT_FOR_PASS) {
-		/* pass from vcl_fetch{} -> hit-for-pass */
-		/* XXX: the bereq was not filtered pass... */
-		pass = 1;
+	} else if (wrk->busyobj->do_pass) {
+			pass = 1;
 	} else {
 		/* regular object */
 		pass = 0;
diff --git a/bin/varnishd/cache/cache_vrt_var.c b/bin/varnishd/cache/cache_vrt_var.c
index e051dbb..c4c0c56 100644
--- a/bin/varnishd/cache/cache_vrt_var.c
+++ b/bin/varnishd/cache/cache_vrt_var.c
@@ -202,6 +202,7 @@ VBERESP(beresp, unsigned, do_esi, busyobj->do_esi)
 VBERESP(beresp, unsigned, do_gzip, busyobj->do_gzip)
 VBERESP(beresp, unsigned, do_gunzip, busyobj->do_gunzip)
 VBERESP(beresp, unsigned, do_stream, busyobj->do_stream)
+VBERESP(beresp, unsigned, do_pass, busyobj->do_pass)
 
 /*--------------------------------------------------------------------*/
 
diff --git a/bin/varnishd/default.vcl b/bin/varnishd/default.vcl
index 18121b8..d0d4c2e 100644
--- a/bin/varnishd/default.vcl
+++ b/bin/varnishd/default.vcl
@@ -109,7 +109,7 @@ sub vcl_fetch {
 		 * Mark as "Hit-For-Pass" for the next 2 minutes
 		 */
 		set beresp.ttl = 120 s;
-		return (hit_for_pass);
+		set beresp.do_pass = true;
     }
     return (deliver);
 }
diff --git a/bin/varnishtest/tests/c00011.vtc b/bin/varnishtest/tests/c00011.vtc
index 1dd34a8..18ee757 100644
--- a/bin/varnishtest/tests/c00011.vtc
+++ b/bin/varnishtest/tests/c00011.vtc
@@ -11,7 +11,7 @@ server s1 {
 
 varnish v1 -vcl+backend { 
 	sub vcl_fetch {
-		return(hit_for_pass);
+		set beresp.do_pass = true;
 	}
 } -start
 
diff --git a/bin/varnishtest/tests/c00014.vtc b/bin/varnishtest/tests/c00014.vtc
index ced8117..03089d8 100644
--- a/bin/varnishtest/tests/c00014.vtc
+++ b/bin/varnishtest/tests/c00014.vtc
@@ -15,7 +15,7 @@ server s1 {
 
 varnish v1 -vcl+backend { 
 	sub vcl_fetch {
-		return(hit_for_pass);
+		set beresp.do_pass = true;
 	}
 } -start
 
diff --git a/bin/varnishtest/tests/e00011.vtc b/bin/varnishtest/tests/e00011.vtc
index e725498..8f03bd4 100644
--- a/bin/varnishtest/tests/e00011.vtc
+++ b/bin/varnishtest/tests/e00011.vtc
@@ -17,8 +17,8 @@ server s1 {
 
 varnish v1 -vcl+backend {
 	sub vcl_fetch {
+		set beresp.do_pass = true;
 		set beresp.do_esi = true;
-		return(hit_for_pass);
 	}
 } -start 
 
diff --git a/bin/varnishtest/tests/r00318.vtc b/bin/varnishtest/tests/r00318.vtc
index 0170c3d..4e66fad 100644
--- a/bin/varnishtest/tests/r00318.vtc
+++ b/bin/varnishtest/tests/r00318.vtc
@@ -8,7 +8,7 @@ server s1 {
 varnish v1 -vcl+backend {
 	sub vcl_fetch {
 		set beresp.do_esi = true;
-		return(hit_for_pass);
+		set beresp.do_pass = true;
 	}
 } -start
 
diff --git a/lib/libvcl/generate.py b/lib/libvcl/generate.py
index b91f675..1d397c8 100755
--- a/lib/libvcl/generate.py
+++ b/lib/libvcl/generate.py
@@ -89,7 +89,7 @@ returns =(
 	('hash',	('hash',)),
 	('miss',	('error', 'restart', 'pass', 'fetch',)),
 	('hit',		('error', 'restart', 'pass', 'deliver',)),
-	('fetch',	('error', 'restart', 'hit_for_pass', 'deliver',)),
+	('fetch',	('error', 'restart', 'deliver',)),
 	('deliver',	('restart', 'deliver',)),
 	('error',	('restart', 'deliver',)),
 	('init',	('ok',)),
@@ -331,6 +331,12 @@ sp_variables = (
 		( 'fetch',),
 		'const struct sess *'
 	),
+	('beresp.do_pass',
+		'BOOL',
+		( 'fetch',),
+		( 'fetch',),
+		'const struct sess *'
+	),
 	('beresp.ttl',
 		'DURATION',
 		( 'fetch',),



From geoff at varnish-cache.org  Mon Jan 30 15:09:53 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 30 Jan 2012 16:09:53 +0100
Subject: [experimental-ims] 58e4a20 s/wrk->busyobj->/bo->/ in cnt_fetchbody()
	(with a few notable exceptions.)
Message-ID: <E1Rrsrh-00085c-0X@project.varnish-software.com>

commit 58e4a209ee09783bb7ad70303fc2fb757070b312
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Jan 30 13:03:06 2012 +0000

    s/wrk->busyobj->/bo->/ in cnt_fetchbody() (with a few notable exceptions.)

diff --git a/bin/varnishd/cache/cache_center.c b/bin/varnishd/cache/cache_center.c
index 7bd0cfb..3286abf 100644
--- a/bin/varnishd/cache/cache_center.c
+++ b/bin/varnishd/cache/cache_center.c
@@ -710,11 +710,13 @@ cnt_fetchbody(struct sess *sp, struct worker *wrk, struct req *req)
 	unsigned l;
 	struct vsb *vary = NULL;
 	int varyl = 0, pass;
+	struct busyobj *bo;
 
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
-	CHECK_OBJ_NOTNULL(wrk->busyobj, BUSYOBJ_MAGIC);
+	bo = wrk->busyobj;
+	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
 
 	assert(req->handling == VCL_RET_DELIVER);
 
@@ -722,9 +724,9 @@ cnt_fetchbody(struct sess *sp, struct worker *wrk, struct req *req)
 		/* This is a pass from vcl_recv */
 		pass = 1;
 		/* VCL may have fiddled this, but that doesn't help */
-		wrk->busyobj->exp.ttl = -1.;
-	} else if (wrk->busyobj->do_pass) {
-			pass = 1;
+		bo->exp.ttl = -1.;
+	} else if (bo->do_pass) {
+		pass = 1;
 	} else {
 		/* regular object */
 		pass = 0;
@@ -745,63 +747,61 @@ cnt_fetchbody(struct sess *sp, struct worker *wrk, struct req *req)
 
 	/* We do nothing unless the param is set */
 	if (!cache_param->http_gzip_support)
-		wrk->busyobj->do_gzip = wrk->busyobj->do_gunzip = 0;
+		bo->do_gzip = bo->do_gunzip = 0;
 
-	wrk->busyobj->is_gzip =
-	    http_HdrIs(wrk->busyobj->beresp, H_Content_Encoding, "gzip");
+	bo->is_gzip = http_HdrIs(bo->beresp, H_Content_Encoding, "gzip");
 
-	wrk->busyobj->is_gunzip =
-	    !http_GetHdr(wrk->busyobj->beresp, H_Content_Encoding, NULL);
+	bo->is_gunzip = !http_GetHdr(bo->beresp, H_Content_Encoding, NULL);
 
 	/* It can't be both */
-	assert(wrk->busyobj->is_gzip == 0 || wrk->busyobj->is_gunzip == 0);
+	assert(bo->is_gzip == 0 || bo->is_gunzip == 0);
 
 	/* We won't gunzip unless it is gzip'ed */
-	if (wrk->busyobj->do_gunzip && !wrk->busyobj->is_gzip)
-		wrk->busyobj->do_gunzip = 0;
+	if (bo->do_gunzip && !bo->is_gzip)
+		bo->do_gunzip = 0;
 
 	/* If we do gunzip, remove the C-E header */
-	if (wrk->busyobj->do_gunzip)
-		http_Unset(wrk->busyobj->beresp, H_Content_Encoding);
+	if (bo->do_gunzip)
+		http_Unset(bo->beresp, H_Content_Encoding);
 
 	/* We wont gzip unless it is ungziped */
-	if (wrk->busyobj->do_gzip && !wrk->busyobj->is_gunzip)
-		wrk->busyobj->do_gzip = 0;
+	if (bo->do_gzip && !bo->is_gunzip)
+		bo->do_gzip = 0;
 
 	/* If we do gzip, add the C-E header */
-	if (wrk->busyobj->do_gzip)
-		http_SetHeader(wrk, sp->vsl_id, wrk->busyobj->beresp,
+	if (bo->do_gzip)
+		http_SetHeader(wrk, sp->vsl_id, bo->beresp,
 		    "Content-Encoding: gzip");
 
 	/* But we can't do both at the same time */
-	assert(wrk->busyobj->do_gzip == 0 || wrk->busyobj->do_gunzip == 0);
+	assert(bo->do_gzip == 0 || bo->do_gunzip == 0);
 
 	/* ESI takes precedence and handles gzip/gunzip itself */
-	if (wrk->busyobj->do_esi)
-		wrk->busyobj->vfp = &vfp_esi;
-	else if (wrk->busyobj->do_gunzip)
-		wrk->busyobj->vfp = &vfp_gunzip;
-	else if (wrk->busyobj->do_gzip)
-		wrk->busyobj->vfp = &vfp_gzip;
-	else if (wrk->busyobj->is_gzip)
-		wrk->busyobj->vfp = &vfp_testgzip;
-
-	if (wrk->busyobj->do_esi || req->esi_level > 0)
-		wrk->busyobj->do_stream = 0;
+	if (bo->do_esi)
+		bo->vfp = &vfp_esi;
+	else if (bo->do_gunzip)
+		bo->vfp = &vfp_gunzip;
+	else if (bo->do_gzip)
+		bo->vfp = &vfp_gzip;
+	else if (bo->is_gzip)
+		bo->vfp = &vfp_testgzip;
+
+	if (bo->do_esi || req->esi_level > 0)
+		bo->do_stream = 0;
 	if (!req->wantbody)
-		wrk->busyobj->do_stream = 0;
+		bo->do_stream = 0;
 
 	/* No reason to try streaming a non-existing body */
-	if (wrk->busyobj->body_status == BS_NONE)
-		wrk->busyobj->do_stream = 0;
+	if (bo->body_status == BS_NONE)
+		bo->do_stream = 0;
 
-	l = http_EstimateWS(wrk->busyobj->beresp,
+	l = http_EstimateWS(bo->beresp,
 	    pass ? HTTPH_R_PASS : HTTPH_A_INS, &nhttp);
 
 	/* Create Vary instructions */
 	if (req->objcore != NULL) {
 		CHECK_OBJ_NOTNULL(req->objcore, OBJCORE_MAGIC);
-		vary = VRY_Create(sp, wrk->busyobj->beresp);
+		vary = VRY_Create(sp, bo->beresp);
 		if (vary != NULL) {
 			varyl = VSB_len(vary);
 			assert(varyl > 0);
@@ -815,7 +815,7 @@ cnt_fetchbody(struct sess *sp, struct worker *wrk, struct req *req)
 	 */
 	l += strlen("Content-Length: XxxXxxXxxXxxXxxXxx") + sizeof(void *);
 
-	if (wrk->busyobj->exp.ttl < cache_param->shortlived ||
+	if (bo->exp.ttl < cache_param->shortlived ||
 	    req->objcore == NULL)
 		req->storage_hint = TRANSIENT_STORAGE;
 
@@ -826,15 +826,15 @@ cnt_fetchbody(struct sess *sp, struct worker *wrk, struct req *req)
 		 * shortlived object on Transient storage.
 		 */
 		req->obj = STV_NewObject(wrk, TRANSIENT_STORAGE, l, nhttp);
-		if (wrk->busyobj->exp.ttl > cache_param->shortlived)
-			wrk->busyobj->exp.ttl = cache_param->shortlived;
-		wrk->busyobj->exp.grace = 0.0;
-		wrk->busyobj->exp.keep = 0.0;
+		if (bo->exp.ttl > cache_param->shortlived)
+			bo->exp.ttl = cache_param->shortlived;
+		bo->exp.grace = 0.0;
+		bo->exp.keep = 0.0;
 	}
 	if (req->obj == NULL) {
 		req->err_code = 503;
 		sp->step = STP_ERROR;
-		VDI_CloseFd(wrk, &wrk->busyobj->vbc);
+		VDI_CloseFd(wrk, &bo->vbc);
 		VBO_DerefBusyObj(wrk, &wrk->busyobj);
 		return (0);
 	}
@@ -842,8 +842,8 @@ cnt_fetchbody(struct sess *sp, struct worker *wrk, struct req *req)
 
 	req->storage_hint = NULL;
 
-	if (wrk->busyobj->do_gzip ||
-	    (wrk->busyobj->is_gzip && !wrk->busyobj->do_gunzip))
+	if (bo->do_gzip ||
+	    (bo->is_gzip && !bo->do_gunzip))
 		req->obj->gziped = 1;
 
 	if (vary != NULL) {
@@ -859,7 +859,7 @@ cnt_fetchbody(struct sess *sp, struct worker *wrk, struct req *req)
 	WS_Assert(req->obj->ws_o);
 
 	/* Filter into object */
-	hp = wrk->busyobj->beresp;
+	hp = bo->beresp;
 	hp2 = req->obj->http;
 
 	hp2->logtag = HTTP_Obj;
@@ -869,7 +869,7 @@ cnt_fetchbody(struct sess *sp, struct worker *wrk, struct req *req)
 	if (http_GetHdr(hp, H_Last_Modified, &b))
 		req->obj->last_modified = VTIM_parse(b);
 	else
-		req->obj->last_modified = floor(wrk->busyobj->exp.entered);
+		req->obj->last_modified = floor(bo->exp.entered);
 
 	assert(WRW_IsReleased(wrk));
 
@@ -881,11 +881,11 @@ cnt_fetchbody(struct sess *sp, struct worker *wrk, struct req *req)
 	if (req->obj->response == 200 &&
 	    req->http->conds &&
 	    RFC2616_Do_Cond(sp))
-		wrk->busyobj->do_stream = 0;
+		bo->do_stream = 0;
 
 	AssertObjCorePassOrBusy(req->obj->objcore);
 
-	if (wrk->busyobj->do_stream) {
+	if (bo->do_stream) {
 		sp->step = STP_PREPRESP;
 		return (0);
 	}
@@ -893,11 +893,11 @@ cnt_fetchbody(struct sess *sp, struct worker *wrk, struct req *req)
 	/* Use unmodified headers*/
 	i = FetchBody(wrk, req->obj);
 
-	http_Setup(wrk->busyobj->bereq, NULL);
-	http_Setup(wrk->busyobj->beresp, NULL);
-	wrk->busyobj->vfp = NULL;
+	http_Setup(bo->bereq, NULL);
+	http_Setup(bo->beresp, NULL);
+	bo->vfp = NULL;
 	assert(WRW_IsReleased(wrk));
-	AZ(wrk->busyobj->vbc);
+	AZ(bo->vbc);
 	AN(req->director);
 
 	if (i) {



From geoff at varnish-cache.org  Mon Jan 30 15:09:53 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Mon, 30 Jan 2012 16:09:53 +0100
Subject: [experimental-ims] 66bc91d Merge master -> experimental-ims
Message-ID: <E1Rrsrh-00085n-2l@project.varnish-software.com>

commit 66bc91da9513da5350964f051d2eca585ab8b113
Merge: c766bc8 58e4a20
Author: Geoff Simmons <geoff at uplex.de>
Date:   Mon Jan 30 16:09:23 2012 +0100

    Merge master -> experimental-ims

diff --cc bin/varnishd/cache/cache_center.c
index 398996d,3286abf..48d53a7
--- a/bin/varnishd/cache/cache_center.c
+++ b/bin/varnishd/cache/cache_center.c
@@@ -657,21 -634,14 +646,20 @@@ cnt_fetch(struct sess *sp, struct worke
  			wrk->busyobj->exp.ttl = -1.;
  
  		AZ(wrk->busyobj->do_esi);
- 
+ 		AZ(wrk->busyobj->do_pass);
+ 		
  		VCL_fetch_method(sp);
  
 +		/* Cancel streaming if a stale object was validated    */
 +		/* XXX: But not if original beresp.status != 304       */
 +		/*      See also AZ(sp->stale_obj) in cnt_streambody() */
 +		if (sp->stale_obj)
 +			wrk->busyobj->do_stream = 0;
 +
+ 		if (req->objcore != NULL && wrk->busyobj->do_pass)
+ 			req->objcore->flags |= OC_F_PASS;
+ 
  		switch (req->handling) {
- 		case VCL_RET_HIT_FOR_PASS:
- 			if (req->objcore != NULL)
- 				req->objcore->flags |= OC_F_PASS;
- 			sp->step = STP_FETCHBODY;
- 			return (0);
  		case VCL_RET_DELIVER:
  			AssertObjCorePassOrBusy(req->objcore);
  			sp->step = STP_FETCHBODY;
@@@ -807,33 -774,29 +792,33 @@@ cnt_fetchbody(struct sess *sp, struct w
  		    "Content-Encoding: gzip");
  
  	/* But we can't do both at the same time */
- 	assert(wrk->busyobj->do_gzip == 0 || wrk->busyobj->do_gunzip == 0);
+ 	assert(bo->do_gzip == 0 || bo->do_gunzip == 0);
  
  	/* ESI takes precedence and handles gzip/gunzip itself */
- 	if (wrk->busyobj->do_esi)
- 		wrk->busyobj->vfp = &vfp_esi;
- 	else if (wrk->busyobj->do_gunzip)
- 		wrk->busyobj->vfp = &vfp_gunzip;
- 	else if (wrk->busyobj->do_gzip)
- 		wrk->busyobj->vfp = &vfp_gzip;
- 	else if (wrk->busyobj->is_gzip)
- 		wrk->busyobj->vfp = &vfp_testgzip;
- 
- 	if (wrk->busyobj->do_esi || req->esi_level > 0)
- 		wrk->busyobj->do_stream = 0;
+ 	if (bo->do_esi)
+ 		bo->vfp = &vfp_esi;
+ 	else if (bo->do_gunzip)
+ 		bo->vfp = &vfp_gunzip;
+ 	else if (bo->do_gzip)
+ 		bo->vfp = &vfp_gzip;
+ 	else if (bo->is_gzip)
+ 		bo->vfp = &vfp_testgzip;
+ 
+ 	if (bo->do_esi || req->esi_level > 0)
+ 		bo->do_stream = 0;
  	if (!req->wantbody)
- 		wrk->busyobj->do_stream = 0;
+ 		bo->do_stream = 0;
  
  	/* No reason to try streaming a non-existing body */
- 	if (wrk->busyobj->body_status == BS_NONE)
- 		wrk->busyobj->do_stream = 0;
+ 	if (bo->body_status == BS_NONE)
+ 		bo->do_stream = 0;
  
- 	l = http_EstimateWS(wrk->busyobj->beresp,
+ 	l = http_EstimateWS(bo->beresp,
  	    pass ? HTTPH_R_PASS : HTTPH_A_INS, &nhttp);
 +        if (sp->stale_obj) {
 +            l += http_EstimateWS(sp->stale_obj->http, 0, &stale_nhttp);
 +            nhttp += stale_nhttp;
 +        }
  
  	/* Create Vary instructions */
  	if (req->objcore != NULL) {
@@@ -877,12 -840,10 +862,12 @@@
  	}
  	CHECK_OBJ_NOTNULL(req->obj, OBJECT_MAGIC);
  
 +	wrk->busyobj->exp.keep = req->exp.keep;
 +
  	req->storage_hint = NULL;
  
- 	if (wrk->busyobj->do_gzip ||
- 	    (wrk->busyobj->is_gzip && !wrk->busyobj->do_gunzip))
+ 	if (bo->do_gzip ||
+ 	    (bo->is_gzip && !bo->do_gunzip))
  		req->obj->gziped = 1;
  
  	if (vary != NULL) {
@@@ -905,11 -866,10 +890,11 @@@
  	http_FilterResp(sp, hp, hp2, pass ? HTTPH_R_PASS : HTTPH_A_INS);
  	http_CopyHome(wrk, sp->vsl_id, hp2);
  
 -	if (http_GetHdr(hp, H_Last_Modified, &b))
 +	if (http_GetHdr(hp, H_Last_Modified, &b)
 +            || http_GetHdr(req->obj->http, H_Last_Modified, &b))
  		req->obj->last_modified = VTIM_parse(b);
  	else
- 		req->obj->last_modified = floor(wrk->busyobj->exp.entered);
+ 		req->obj->last_modified = floor(bo->exp.entered);
  
  	assert(WRW_IsReleased(wrk));
  
@@@ -933,30 -893,11 +918,30 @@@
  	/* Use unmodified headers*/
  	i = FetchBody(wrk, req->obj);
  
 +	/*
 +	 * If a stale_obj was found, dup its storage into the new obj,
 +	 * reset Content-Length from the size of the storage, and discard
 +	 * the stale_obj.
 +	 */
 +	if (sp->stale_obj) {
 +		STV_dup(sp, sp->stale_obj, req->obj);
 +		assert(sp->stale_obj->len == req->obj->len);
 +		
 +		http_Unset(req->obj->http, H_Content_Length);
 +		http_PrintfHeader(sp->wrk, sp->fd, req->obj->http,
 +		    "Content-Length: %u", req->obj->len);
 +		
 +		EXP_Clr(&sp->stale_obj->exp);
 +		EXP_Rearm(sp->stale_obj);
 +		HSH_Deref(sp->wrk, NULL, &sp->stale_obj);
 +		AZ(sp->stale_obj);
 +	}
 +
- 	http_Setup(wrk->busyobj->bereq, NULL);
- 	http_Setup(wrk->busyobj->beresp, NULL);
- 	wrk->busyobj->vfp = NULL;
+ 	http_Setup(bo->bereq, NULL);
+ 	http_Setup(bo->beresp, NULL);
+ 	bo->vfp = NULL;
  	assert(WRW_IsReleased(wrk));
- 	AZ(wrk->busyobj->vbc);
+ 	AZ(bo->vbc);
  	AN(req->director);
  
  	if (i) {



From phk at varnish-cache.org  Tue Jan 31 08:21:40 2012
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Tue, 31 Jan 2012 09:21:40 +0100
Subject: [master] ac35ccf Split "prepfetch" from "fetchbody"
Message-ID: <E1Rs8yC-0002TZ-71@project.varnish-software.com>

commit ac35ccf8beb8adcd888f71200dba2ea79a1da481
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Jan 31 08:21:29 2012 +0000

    Split "prepfetch" from "fetchbody"

diff --git a/bin/varnishd/cache/cache_center.c b/bin/varnishd/cache/cache_center.c
index 3286abf..d12101d 100644
--- a/bin/varnishd/cache/cache_center.c
+++ b/bin/varnishd/cache/cache_center.c
@@ -565,8 +565,8 @@ DOT		label="obj.f.pass=true"
 DOT	]
 DOT	vcl_fetch -> fetch_pass [label="hit_for_pass",style=bold,color=red]
 DOT }
-DOT fetch:hfp -> fetchbody [style=bold,color=red]
-DOT fetch:del -> fetchbody [label="deliver",style=bold,color=blue]
+DOT fetch:hfp -> prepfetch [style=bold,color=red]
+DOT fetch:del -> prepfetch [label="deliver",style=bold,color=blue]
  */
 
 static int
@@ -635,7 +635,7 @@ cnt_fetch(struct sess *sp, struct worker *wrk, struct req *req)
 
 		AZ(wrk->busyobj->do_esi);
 		AZ(wrk->busyobj->do_pass);
-		
+
 		VCL_fetch_method(sp);
 
 		if (req->objcore != NULL && wrk->busyobj->do_pass)
@@ -644,7 +644,7 @@ cnt_fetch(struct sess *sp, struct worker *wrk, struct req *req)
 		switch (req->handling) {
 		case VCL_RET_DELIVER:
 			AssertObjCorePassOrBusy(req->objcore);
-			sp->step = STP_FETCHBODY;
+			sp->step = STP_PREPFETCH;
 			return (0);
 		default:
 			break;
@@ -680,30 +680,22 @@ cnt_fetch(struct sess *sp, struct worker *wrk, struct req *req)
 }
 
 /*--------------------------------------------------------------------
- * Fetch response body from the backend
+ * Prepare to fetch body from backend
  *
 DOT subgraph xcluster_body {
-DOT	fetchbody [
-DOT		shape=diamond
-DOT		label="stream ?"
-DOT	]
-DOT	fetchbody2 [
-DOT		shape=ellipse
-DOT		label="fetch body\nfrom backend\n"
+DOT	prepfetch [
+DOT		shape=record
+DOT		label="{cnt_prepfetch:|error?|<out>stream ?}"
 DOT	]
 DOT }
-DOT fetchbody -> fetchbody2 [label=no,style=bold,color=red]
-DOT fetchbody -> fetchbody2 [style=bold,color=blue]
-DOT fetchbody -> prepresp [label=yes,style=bold,color=cyan]
-DOT fetchbody2 -> prepresp [style=bold,color=red]
-DOT fetchbody2 -> prepresp [style=bold,color=blue]
+DOT prepfetch:out -> fetchbody [style=bold,color=red]
+DOT prepfetch:out -> fetchbody [style=bold,color=blue]
+DOT prepfetch:out -> prepresp [label=yes,style=bold,color=cyan]
  */
 
-
 static int
-cnt_fetchbody(struct sess *sp, struct worker *wrk, struct req *req)
+cnt_prepfetch(struct sess *sp, struct worker *wrk, struct req *req)
 {
-	int i;
 	struct http *hp, *hp2;
 	char *b;
 	uint16_t nhttp;
@@ -888,7 +880,36 @@ cnt_fetchbody(struct sess *sp, struct worker *wrk, struct req *req)
 	if (bo->do_stream) {
 		sp->step = STP_PREPRESP;
 		return (0);
+	} else {
+		sp->step = STP_FETCHBODY;
+		return (0);
 	}
+}
+
+/*--------------------------------------------------------------------
+ * Actually fetch body from backend
+ *
+DOT subgraph xcluster_fetchbody {
+DOT	fetchbody [
+DOT		shape=record
+DOT		label="{cnt_fetchbody:|error ?|<out>success ?}"
+DOT	]
+DOT }
+DOT fetchbody:out -> prepresp [style=bold,color=red]
+DOT fetchbody:out -> prepresp [style=bold,color=blue]
+ */
+
+static int
+cnt_fetchbody(struct sess *sp, struct worker *wrk, struct req *req)
+{
+	int i;
+	struct busyobj *bo;
+
+	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
+	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
+	bo = wrk->busyobj;
+	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
 
 	/* Use unmodified headers*/
 	i = FetchBody(wrk, req->obj);
diff --git a/include/tbl/steps.h b/include/tbl/steps.h
index 6fcd74e..1e95529 100644
--- a/include/tbl/steps.h
+++ b/include/tbl/steps.h
@@ -39,6 +39,7 @@ STEP(lookup,		LOOKUP,		(sp, sp->wrk, sp->req))
 STEP(miss,		MISS,		(sp, sp->wrk, sp->req))
 STEP(hit,		HIT,		(sp, sp->wrk, sp->req))
 STEP(fetch,		FETCH,		(sp, sp->wrk, sp->req))
+STEP(prepfetch,		PREPFETCH,	(sp, sp->wrk, sp->req))
 STEP(fetchbody,		FETCHBODY,	(sp, sp->wrk, sp->req))
 STEP(streambody,	STREAMBODY,	(sp, sp->wrk, sp->req))
 STEP(prepresp,		PREPRESP,	(sp, sp->wrk, sp->req))



From phk at varnish-cache.org  Tue Jan 31 08:37:00 2012
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Tue, 31 Jan 2012 09:37:00 +0100
Subject: [master] 8a9ad30 More work on the dot graph
Message-ID: <E1Rs9D2-0003Bw-RP@project.varnish-software.com>

commit 8a9ad303b25a8cfeba5ef1add82be05d0264b153
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Jan 31 08:36:51 2012 +0000

    More work on the dot graph

diff --git a/bin/varnishd/cache/cache_center.c b/bin/varnishd/cache/cache_center.c
index d12101d..c933175 100644
--- a/bin/varnishd/cache/cache_center.c
+++ b/bin/varnishd/cache/cache_center.c
@@ -188,25 +188,13 @@ cnt_wait(struct sess *sp, struct worker *wrk, struct req *req)
  *
 DOT subgraph xcluster_prepresp {
 DOT	prepresp [
-DOT		shape=ellipse
-DOT		label="Filter obj.->resp."
-DOT	]
-DOT	vcl_deliver [
 DOT		shape=record
-DOT		label="vcl_deliver()|resp."
+DOT		label="{cnt_prepresp:|Filter obj.-\>resp.|{vcl_deliver\{\}|{req.|resp.}}|{error?|restart?}|stream ?}"
 DOT	]
-DOT	prepresp -> vcl_deliver [style=bold,color=green]
-DOT	prepresp -> vcl_deliver [style=bold,color=cyan]
-DOT	prepresp -> vcl_deliver [style=bold,color=red]
-DOT	prepresp -> vcl_deliver [style=bold,color=blue,]
-DOT	vcl_deliver -> deliver [style=bold,color=green,label=deliver]
-DOT	vcl_deliver -> deliver [style=bold,color=red]
-DOT	vcl_deliver -> deliver [style=bold,color=blue]
-DOT     vcl_deliver -> errdeliver [label="error"]
-DOT     errdeliver [label="ERROR",shape=plaintext]
-DOT     vcl_deliver -> rstdeliver [label="restart",color=purple]
-DOT     rstdeliver [label="RESTART",shape=plaintext]
-DOT     vcl_deliver -> streambody [style=bold,color=cyan,label="deliver"]
+DOT	prepresp -> deliver [style=bold,color=green,label=deliver]
+DOT	prepresp -> deliver [style=bold,color=red]
+DOT	prepresp -> deliver [style=bold,color=blue]
+DOT     prepresp -> streambody [style=bold,color=cyan,label="deliver"]
 DOT }
  *
  */
@@ -326,8 +314,8 @@ cnt_prepresp(struct sess *sp, struct worker *wrk, struct req *req)
  *
 DOT subgraph xcluster_deliver {
 DOT	deliver [
-DOT		shape=ellipse
-DOT		label="Send body"
+DOT		shape=record
+DOT		label="{cnt_deliver:|Send body}"
 DOT	]
 DOT }
 DOT deliver -> DONE [style=bold,color=green]
@@ -362,8 +350,8 @@ cnt_deliver(struct sess *sp, struct worker *wrk, struct req *req)
  * the client connection
  *
 DOT	DONE [
-DOT		shape=hexagon
-DOT		label="cnt_done:\nRequest completed"
+DOT		shape=record
+DOT		label="{cnt_done:|Request completed}"
 DOT	]
 DOT	ESI_RESP [ shape=hexagon ]
 DOT	DONE -> start [label="full pipeline"]
@@ -557,16 +545,11 @@ cnt_error(struct sess *sp, struct worker *wrk, struct req *req)
 DOT subgraph xcluster_fetch {
 DOT	fetch [
 DOT		shape=record
-DOT		label="{cnt_fetch:|fetch hdr\nfrom backend|(find obj.ttl)|{vcl_fetch\{\}|{req.|bereq.|beresp.}}|{<err>error?|<rst>restart?}|{<hfp>hit_for_pass?|<del>deliver?}}"
+DOT		label="{cnt_fetch:|fetch hdr\nfrom backend|(find obj.ttl)|{vcl_fetch\{\}|{req.|bereq.|beresp.}}|{<err>error?|<rst>restart?}}"
 DOT	]
-DOT	fetch_pass [
-DOT		shape=ellipse
-DOT		label="obj.f.pass=true"
-DOT	]
-DOT	vcl_fetch -> fetch_pass [label="hit_for_pass",style=bold,color=red]
 DOT }
-DOT fetch:hfp -> prepfetch [style=bold,color=red]
-DOT fetch:del -> prepfetch [label="deliver",style=bold,color=blue]
+DOT fetch -> prepfetch [style=bold,color=red]
+DOT fetch -> prepfetch [style=bold,color=blue]
  */
 
 static int
@@ -946,8 +929,8 @@ cnt_fetchbody(struct sess *sp, struct worker *wrk, struct req *req)
  * Stream the body as we fetch it
 DOT subgraph xstreambody {
 DOT	streambody [
-DOT		shape=ellipse
-DOT		label="streaming\nfetch/deliver"
+DOT		shape=record
+DOT		label="{cnt_streambody:|ping_pong\nfetch/deliver}"
 DOT	]
 DOT }
 DOT streambody -> DONE [style=bold,color=cyan]
@@ -1065,7 +1048,7 @@ cnt_first(struct sess *sp, struct worker *wrk)
 DOT subgraph xcluster_hit {
 DOT	hit [
 DOT		shape=record
-DOT		label="{cnt_hit:|{vcl_hit()|{req.|obj.}}|{<err>error?|<del>deliver?|<rst>restart?|<pass>pass?}}"
+DOT		label="{cnt_hit:|{vcl_hit()|{req.|obj.}}|{<err>error?|<rst>restart?}|{<del>deliver?|<pass>pass?}}"
 DOT	]
 DOT }
 XDOT hit:err -> err_hit [label="error"]
@@ -1237,13 +1220,9 @@ cnt_lookup(struct sess *sp, struct worker *wrk, struct req *req)
 DOT subgraph xcluster_miss {
 DOT	miss [
 DOT		shape=record
-DOT		label="{cnt_miss:|filter req.-\>bereq.|{vcl_miss\{\}|{req.*|bereq.*}}|{<pass>pass?|<err>error?|<restart>restart?|<fetch>fetch?}}"
+DOT		label="{cnt_miss:|filter req.-\>bereq.|{vcl_miss\{\}|{req.*|bereq.*}}|{<err>error?|<rst>restart?}|{<pass>pass?|<fetch>fetch?}}"
 DOT	]
 DOT }
-XDOT miss:restart -> rst_miss [label="restart",color=purple]
-XDOT rst_miss [label="RESTART",shape=plaintext]
-XDOT miss:err -> err_miss [label="error"]
-XDOT err_miss [label="ERROR",shape=plaintext]
 DOT miss:fetch -> fetch [label="fetch",style=bold,color=blue]
 DOT miss:pass -> pass [label="pass",style=bold,color=red]
 DOT



From geoff at varnish-cache.org  Tue Jan 31 20:30:15 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Tue, 31 Jan 2012 21:30:15 +0100
Subject: [experimental-ims] 3df9ebf Moved sp->stale_obj to busyobj->stale_obj
Message-ID: <E1RsKLH-00047Q-8X@project.varnish-software.com>

commit 3df9ebf5105a5da63c2a3085af772e5b3ab32f05
Author: Geoff Simmons <geoff at uplex.de>
Date:   Tue Jan 31 21:29:08 2012 +0100

    Moved sp->stale_obj to busyobj->stale_obj

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index b257efc..6c3fb57 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -509,6 +509,7 @@ struct busyobj {
 	struct http		*bereq;
 	struct http		*beresp;
 	struct object		*fetch_obj;
+	struct object		*stale_obj;
 	struct exp		exp;
 	struct http_conn	htc;
 
@@ -649,8 +650,6 @@ struct sess {
 	int			fd;
 	unsigned		vsl_id;
 
-	struct object		*stale_obj;
-	
 	/* Cross references ------------------------------------------*/
 
 	struct sesspool		*sesspool;
diff --git a/bin/varnishd/cache/cache_center.c b/bin/varnishd/cache/cache_center.c
index 48d53a7..8897e83 100644
--- a/bin/varnishd/cache/cache_center.c
+++ b/bin/varnishd/cache/cache_center.c
@@ -628,7 +628,7 @@ cnt_fetch(struct sess *sp, struct worker *wrk, struct req *req)
 		 * and merge headers from stale_obj into the busyobj if so.
 		 * Any other response is handled as usual.
 		 */
-		if (sp->stale_obj)
+		if (wrk->busyobj->stale_obj)
 			http_Check304(sp, wrk->busyobj);
 
 		req->err_code = http_GetStatus(wrk->busyobj->beresp);
@@ -650,10 +650,10 @@ cnt_fetch(struct sess *sp, struct worker *wrk, struct req *req)
 		
 		VCL_fetch_method(sp);
 
-		/* Cancel streaming if a stale object was validated    */
-		/* XXX: But not if original beresp.status != 304       */
-		/*      See also AZ(sp->stale_obj) in cnt_streambody() */
-		if (sp->stale_obj)
+		/* Cancel streaming if a stale object was validated         */
+		/* XXX: But not if original beresp.status != 304            */
+		/*      See also AZ(busyobj->stale_obj) in cnt_streambody() */
+		if (wrk->busyobj->stale_obj)
 			wrk->busyobj->do_stream = 0;
 
 		if (req->objcore != NULL && wrk->busyobj->do_pass)
@@ -815,8 +815,8 @@ cnt_fetchbody(struct sess *sp, struct worker *wrk, struct req *req)
 
 	l = http_EstimateWS(bo->beresp,
 	    pass ? HTTPH_R_PASS : HTTPH_A_INS, &nhttp);
-        if (sp->stale_obj) {
-            l += http_EstimateWS(sp->stale_obj->http, 0, &stale_nhttp);
+        if (bo->stale_obj) {
+            l += http_EstimateWS(bo->stale_obj->http, 0, &stale_nhttp);
             nhttp += stale_nhttp;
         }
 
@@ -923,18 +923,18 @@ cnt_fetchbody(struct sess *sp, struct worker *wrk, struct req *req)
 	 * reset Content-Length from the size of the storage, and discard
 	 * the stale_obj.
 	 */
-	if (sp->stale_obj) {
-		STV_dup(sp, sp->stale_obj, req->obj);
-		assert(sp->stale_obj->len == req->obj->len);
+	if (bo->stale_obj) {
+		STV_dup(sp, bo->stale_obj, req->obj);
+		assert(bo->stale_obj->len == req->obj->len);
 		
 		http_Unset(req->obj->http, H_Content_Length);
 		http_PrintfHeader(sp->wrk, sp->fd, req->obj->http,
 		    "Content-Length: %u", req->obj->len);
 		
-		EXP_Clr(&sp->stale_obj->exp);
-		EXP_Rearm(sp->stale_obj);
-		HSH_Deref(sp->wrk, NULL, &sp->stale_obj);
-		AZ(sp->stale_obj);
+		EXP_Clr(&bo->stale_obj->exp);
+		EXP_Rearm(bo->stale_obj);
+		HSH_Deref(sp->wrk, NULL, &bo->stale_obj);
+		AZ(bo->stale_obj);
 	}
 
 	http_Setup(bo->bereq, NULL);
@@ -987,9 +987,9 @@ cnt_streambody(struct sess *sp, struct worker *wrk, struct req *req)
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
-	AZ(sp->stale_obj);
 
 	CHECK_OBJ_NOTNULL(wrk->busyobj, BUSYOBJ_MAGIC);
+	AZ(wrk->busyobj->stale_obj);
 	memset(&sctx, 0, sizeof sctx);
 	sctx.magic = STREAM_CTX_MAGIC;
 	AZ(wrk->sctx);
@@ -1244,8 +1244,8 @@ cnt_lookup(struct sess *sp, struct worker *wrk, struct req *req)
 		wrk->stats.cache_hitpass++;
 		WSP(sp, SLT_HitPass, "%u", req->obj->xid);
 		(void)HSH_Deref(wrk, NULL, &req->obj);
-                if (sp->stale_obj != NULL)
-                    (void)HSH_Deref(wrk, NULL, &sp->stale_obj);
+                if (wrk->busyobj != NULL && wrk->busyobj->stale_obj != NULL)
+                    (void)HSH_Deref(wrk, NULL, &wrk->busyobj->stale_obj);
 		AZ(req->objcore);
 		sp->step = STP_PASS;
 		return (0);
@@ -1287,8 +1287,10 @@ cnt_miss(struct sess *sp, struct worker *wrk, struct req *req)
 	CHECK_OBJ_NOTNULL(wrk->busyobj, BUSYOBJ_MAGIC);
 	AZ(req->obj);
 
-	WS_Reset(wrk->ws, NULL);
-	wrk->busyobj = VBO_GetBusyObj(wrk);
+	if (!wrk->busyobj->stale_obj) {
+		WS_Reset(wrk->ws, NULL);
+		wrk->busyobj = VBO_GetBusyObj(wrk);
+	}
 	http_Setup(wrk->busyobj->bereq, wrk->ws);
 	http_FilterReq(sp, HTTPH_R_FETCH);
 	http_ForceGet(wrk->busyobj->bereq);
@@ -1306,7 +1308,7 @@ cnt_miss(struct sess *sp, struct worker *wrk, struct req *req)
         /* If a candidate for a conditional backend request was found,
          * add If-Modified-Since and/or If-None-Match to the bereq.
          */
-        if (sp->stale_obj)
+        if (wrk->busyobj->stale_obj)
                 http_CheckRefresh(sp);
 
 	VCL_miss_method(sp);
diff --git a/bin/varnishd/cache/cache_hash.c b/bin/varnishd/cache/cache_hash.c
index 827d6af..9585e7a 100644
--- a/bin/varnishd/cache/cache_hash.c
+++ b/bin/varnishd/cache/cache_hash.c
@@ -307,7 +307,6 @@ HSH_Lookup(struct sess *sp, struct objhead **poh)
 	CHECK_OBJ_NOTNULL(sp->req->http, HTTP_MAGIC);
 	AN(sp->req->director);
 	AN(hash);
-	AZ(sp->stale_obj);
 	wrk = sp->wrk;
 
 	HSH_Prealloc(sp);
@@ -470,27 +469,26 @@ HSH_Lookup(struct sess *sp, struct objhead **poh)
 		return (NULL);
 	}
 
+	/* Insert (precreated) objcore in objecthead */
+	oc = wrk->nobjcore;
+	wrk->nobjcore = NULL;
+	AN(oc->flags & OC_F_BUSY);
+	oc->refcnt = 1;
+
+	AZ(wrk->busyobj);
+	wrk->busyobj = VBO_GetBusyObj(wrk);
+
         /* If we're not serving a valid or graced object and we saved stale_o,
 	 * it is a candidate for the conditional backend request. */
-        AZ(oc && !sp->req->hash_always_miss);
         AZ(busy_oc);
         if (stale_o != NULL) {
                 AZ(stale_o->objcore->flags & OC_F_BUSY);
 		CHECK_OBJ_NOTNULL(stale_o->objcore, OBJCORE_MAGIC);
                 Lck_AssertHeld(&oh->mtx);
                 stale_o->objcore->refcnt++;
-                sp->stale_obj = stale_o;
+                wrk->busyobj->stale_obj = stale_o;
         }
 
-	/* Insert (precreated) objcore in objecthead */
-	oc = wrk->nobjcore;
-	wrk->nobjcore = NULL;
-	AN(oc->flags & OC_F_BUSY);
-	oc->refcnt = 1;
-
-	AZ(wrk->busyobj);
-	wrk->busyobj = VBO_GetBusyObj(wrk);
-
 	VRY_Validate(sp->req->vary_b);
 	if (sp->req->vary_l != NULL)
 		wrk->busyobj->vary = sp->req->vary_b;
diff --git a/bin/varnishd/cache/cache_http.c b/bin/varnishd/cache/cache_http.c
index ceef821..ac922b8 100644
--- a/bin/varnishd/cache/cache_http.c
+++ b/bin/varnishd/cache/cache_http.c
@@ -907,9 +907,8 @@ http_FilterReq(const struct sess *sp, unsigned how)
 }
 
 /*-------------------------------------------------------------------
- * This function checks for sp->freshen_obj.  If present, HSH_Lookup()
- * found an expired object that qualifies for a refresh check,
- * so add the appropriate headers.
+ * This function checks if a stale_obj was found in HSH_Lookup().
+ * If so, add the appropriate headers for backend validation.
  */
 
 void
@@ -919,7 +918,10 @@ http_CheckRefresh(struct sess *sp)
 	struct http *obj_hp, *bereq_hp;
 	char *p;
 
-	freshen_obj = sp->stale_obj;
+	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+	CHECK_OBJ_NOTNULL(sp->wrk, WORKER_MAGIC);
+	CHECK_OBJ_NOTNULL(sp->wrk->busyobj, BUSYOBJ_MAGIC);
+	freshen_obj = sp->wrk->busyobj->stale_obj;
 	CHECK_OBJ_NOTNULL(freshen_obj, OBJECT_MAGIC);
 	bereq_hp = sp->wrk->busyobj->bereq;
 	CHECK_OBJ_NOTNULL(bereq_hp, HTTP_MAGIC);
@@ -936,7 +938,7 @@ http_CheckRefresh(struct sess *sp)
 }
 
 /*-------------------------------------------------------------------
- * Called after fetch and sp->freshen_obj present.  Check
+ * Called after fetch for a backend conditional request.  Check
  * response and handle as needed.
  */
 
@@ -948,7 +950,7 @@ http_Check304(struct sess *sp, struct busyobj *busyobj)
 
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 	CHECK_OBJ_NOTNULL(busyobj, BUSYOBJ_MAGIC);
-	o_stale = sp->stale_obj;
+	o_stale = busyobj->stale_obj;
 	CHECK_OBJ_NOTNULL(o_stale, OBJECT_MAGIC);
 
 	if (busyobj->beresp->status != 304) {
@@ -960,8 +962,10 @@ http_Check304(struct sess *sp, struct busyobj *busyobj)
 		|| http_GetHdr(busyobj->bereq, H_If_None_Match, &p))
 		sp->wrk->stats.fetch_not_validated++;
 
-	    HSH_Deref(sp->wrk, NULL, &sp->stale_obj);
-	    AZ(sp->stale_obj);
+	    /* Discard the stale object */
+	    /* XXX: just deref, or force expire? */
+	    HSH_Deref(sp->wrk, NULL, &busyobj->stale_obj);
+	    AZ(busyobj->stale_obj);
 	    return;
 	}
 
diff --git a/bin/varnishd/cache/cache_vrt.c b/bin/varnishd/cache/cache_vrt.c
index 50a7b7f..9c7664c 100644
--- a/bin/varnishd/cache/cache_vrt.c
+++ b/bin/varnishd/cache/cache_vrt.c
@@ -114,8 +114,8 @@ vrt_selecthttp(const struct sess *sp, enum gethdr_e where)
 		hp = sp->req->obj->http;
 		break;
         case HDR_STALE_OBJ:
-		CHECK_OBJ_NOTNULL(sp->stale_obj, OBJECT_MAGIC);
-		hp = sp->stale_obj->http;
+		CHECK_OBJ_NOTNULL(sp->wrk->busyobj->stale_obj, OBJECT_MAGIC);
+		hp = sp->wrk->busyobj->stale_obj->http;
 		break;
 	default:
 		INCOMPL();
@@ -131,7 +131,7 @@ VRT_GetHdr(const struct sess *sp, enum gethdr_e where, const char *n)
 	struct http *hp;
 
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
-        if (where == HDR_STALE_OBJ && sp->stale_obj == NULL) {
+        if (where == HDR_STALE_OBJ && sp->wrk->busyobj->stale_obj == NULL) {
 	    WSP(sp, SLT_VCL_error,
 		"stale_obj does not exist (reading header %s)", n);
             return NULL;
diff --git a/bin/varnishd/cache/cache_vrt_var.c b/bin/varnishd/cache/cache_vrt_var.c
index d376df1..f94dc83 100644
--- a/bin/varnishd/cache/cache_vrt_var.c
+++ b/bin/varnishd/cache/cache_vrt_var.c
@@ -94,20 +94,34 @@ VRT_r_##obj##_##hdr(const struct sess *sp)			\
 VRT_DO_HDR_l(obj, hdr, cont, http, fld)				\
 VRT_DO_HDR_r(obj, hdr, cont, http, fld, nullable)		\
 
-VRT_DO_HDR(req,		request,	sp->req,		http,	HTTP_HDR_REQ,		0)
-VRT_DO_HDR(req,		url,		sp->req,		http,	HTTP_HDR_URL,		0)
-VRT_DO_HDR(req,		proto,		sp->req,		http,	HTTP_HDR_PROTO,		0)
-VRT_DO_HDR(bereq,	request,	sp->wrk->busyobj,	bereq,	HTTP_HDR_REQ,		0)
-VRT_DO_HDR(bereq,	url,		sp->wrk->busyobj,	bereq,	HTTP_HDR_URL,		0)
-VRT_DO_HDR(bereq,	proto,		sp->wrk->busyobj,	bereq,	HTTP_HDR_PROTO,		0)
-VRT_DO_HDR(obj,		proto,		sp->req->obj,		http,	HTTP_HDR_PROTO,		0)
-VRT_DO_HDR(obj,		response,	sp->req->obj,		http,	HTTP_HDR_RESPONSE,	0)
-VRT_DO_HDR(resp,	proto,		sp->req,		resp,	HTTP_HDR_PROTO,		0)
-VRT_DO_HDR(resp,	response,	sp->req,		resp,	HTTP_HDR_RESPONSE,	0)
-VRT_DO_HDR(beresp,	proto,		sp->wrk->busyobj,	beresp,	HTTP_HDR_PROTO,		0)
-VRT_DO_HDR(beresp,	response,	sp->wrk->busyobj,	beresp,	HTTP_HDR_RESPONSE,	0)
-VRT_DO_HDR_r(stale_obj,	proto,		sp->stale_obj,		http,	HTTP_HDR_PROTO,		1)
-VRT_DO_HDR_r(stale_obj,	response,	sp->stale_obj,		http,	HTTP_HDR_RESPONSE,	1)
+VRT_DO_HDR(req,		request,	sp->req,
+	   http,	HTTP_HDR_REQ,		0)
+VRT_DO_HDR(req,		url,		sp->req,
+    	   http,	HTTP_HDR_URL,		0)
+VRT_DO_HDR(req,		proto,		sp->req,
+	   http,	HTTP_HDR_PROTO,		0)
+VRT_DO_HDR(bereq,	request,	sp->wrk->busyobj,
+	   bereq,	HTTP_HDR_REQ,		0)
+VRT_DO_HDR(bereq,	url,		sp->wrk->busyobj,
+	   bereq,	HTTP_HDR_URL,		0)
+VRT_DO_HDR(bereq,	proto,		sp->wrk->busyobj,
+	   bereq,	HTTP_HDR_PROTO,		0)
+VRT_DO_HDR(obj,		proto,		sp->req->obj,
+	   http,	HTTP_HDR_PROTO,		0)
+VRT_DO_HDR(obj,		response,	sp->req->obj,
+	   http,	HTTP_HDR_RESPONSE,	0)
+VRT_DO_HDR(resp,	proto,		sp->req,
+	   resp,	HTTP_HDR_PROTO,		0)
+VRT_DO_HDR(resp,	response,	sp->req,
+	   resp,	HTTP_HDR_RESPONSE,	0)
+VRT_DO_HDR(beresp,	proto,		sp->wrk->busyobj,
+	   beresp,	HTTP_HDR_PROTO,		0)
+VRT_DO_HDR(beresp,	response,	sp->wrk->busyobj,
+	   beresp,	HTTP_HDR_RESPONSE,	0)
+VRT_DO_HDR_r(stale_obj,	proto,		sp->wrk->busyobj->stale_obj,
+	     http,	HTTP_HDR_PROTO,		1)
+VRT_DO_HDR_r(stale_obj,	response,	sp->wrk->busyobj->stale_obj,
+	     http,	HTTP_HDR_RESPONSE,	1)
 
 /*--------------------------------------------------------------------*/
 
@@ -137,10 +151,14 @@ VRT_r_##obj##_status(const struct sess *sp)			\
 VRT_DO_STATUS_l(obj, cont, http)				\
 VRT_DO_STATUS_r(obj, cont, http, nullable)			\
 
-VRT_DO_STATUS(obj,		sp->req->obj,		http,	0)
-VRT_DO_STATUS(beresp,		sp->wrk->busyobj,	beresp,	0)
-VRT_DO_STATUS(resp,		sp->req,		resp,	0)
-VRT_DO_STATUS_r(stale_obj,	sp->stale_obj,		http,	1)
+VRT_DO_STATUS(obj,		sp->req->obj,
+	      http,	0)
+VRT_DO_STATUS(beresp,		sp->wrk->busyobj,
+	      beresp,	0)
+VRT_DO_STATUS(resp,		sp->req,
+	      resp,	0)
+VRT_DO_STATUS_r(stale_obj,	sp->wrk->busyobj->stale_obj,
+		http,	1)
 
 /*--------------------------------------------------------------------*/
 
@@ -455,9 +473,9 @@ VRT_DO_EXP(beresp, sp->wrk->busyobj, ttl, 0, 0,
 VRT_DO_EXP(beresp, sp->wrk->busyobj, keep, 0, 0,
    vrt_wsp_exp(sp, sp->req->xid, &sp->wrk->busyobj->exp);)
     
-VRT_DO_EXP_r(stale_obj, sp->stale_obj, grace, 0, 1)
-VRT_DO_EXP_r(stale_obj, sp->stale_obj, ttl, 0, 1)
-VRT_DO_EXP_r(stale_obj, sp->stale_obj, keep, 0, 1)
+VRT_DO_EXP_r(stale_obj, sp->wrk->busyobj->stale_obj, grace, 0, 1)
+VRT_DO_EXP_r(stale_obj, sp->wrk->busyobj->stale_obj, ttl, 0, 1)
+VRT_DO_EXP_r(stale_obj, sp->wrk->busyobj->stale_obj, keep, 0, 1)
 
 /*--------------------------------------------------------------------
  * req.xid
@@ -579,12 +597,12 @@ VRT_r_stale_obj_hits(const struct sess *sp)
 {
 
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
-        if (sp->stale_obj == NULL) {
+        if (sp->wrk->busyobj->stale_obj == NULL) {
             ILLEGAL_R(sp, "stale_obj", "hits");
             return (0);
         }
-	CHECK_OBJ(sp->stale_obj, OBJECT_MAGIC);	/* XXX */
-	return (sp->stale_obj->hits);
+	CHECK_OBJ(sp->wrk->busyobj->stale_obj, OBJECT_MAGIC);	/* XXX */
+	return (sp->wrk->busyobj->stale_obj->hits);
 }
 
 double
@@ -601,12 +619,12 @@ VRT_r_stale_obj_lastuse(const struct sess *sp)
 {
 
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
-        if (sp->stale_obj == NULL) {
+        if (sp->wrk->busyobj->stale_obj == NULL) {
             ILLEGAL_R(sp, "stale_obj", "lastuse");
             return (0);
         }
-	CHECK_OBJ(sp->stale_obj, OBJECT_MAGIC);	/* XXX */
-	return (VTIM_real() - sp->stale_obj->last_use);
+	CHECK_OBJ(sp->wrk->busyobj->stale_obj, OBJECT_MAGIC);	/* XXX */
+	return (VTIM_real() - sp->wrk->busyobj->stale_obj->last_use);
 }
 
 unsigned
@@ -621,5 +639,5 @@ unsigned
 VRT_r_stale_obj(const struct sess *sp)
 {
         CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
-        return (sp->stale_obj != NULL);
+        return (sp->wrk->busyobj->stale_obj != NULL);
 }



From geoff at varnish-cache.org  Tue Jan 31 20:47:23 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Tue, 31 Jan 2012 21:47:23 +0100
Subject: [experimental-ims] ac35ccf Split "prepfetch" from "fetchbody"
Message-ID: <E1RsKbr-0005lI-9l@project.varnish-software.com>

commit ac35ccf8beb8adcd888f71200dba2ea79a1da481
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Jan 31 08:21:29 2012 +0000

    Split "prepfetch" from "fetchbody"

diff --git a/bin/varnishd/cache/cache_center.c b/bin/varnishd/cache/cache_center.c
index 3286abf..d12101d 100644
--- a/bin/varnishd/cache/cache_center.c
+++ b/bin/varnishd/cache/cache_center.c
@@ -565,8 +565,8 @@ DOT		label="obj.f.pass=true"
 DOT	]
 DOT	vcl_fetch -> fetch_pass [label="hit_for_pass",style=bold,color=red]
 DOT }
-DOT fetch:hfp -> fetchbody [style=bold,color=red]
-DOT fetch:del -> fetchbody [label="deliver",style=bold,color=blue]
+DOT fetch:hfp -> prepfetch [style=bold,color=red]
+DOT fetch:del -> prepfetch [label="deliver",style=bold,color=blue]
  */
 
 static int
@@ -635,7 +635,7 @@ cnt_fetch(struct sess *sp, struct worker *wrk, struct req *req)
 
 		AZ(wrk->busyobj->do_esi);
 		AZ(wrk->busyobj->do_pass);
-		
+
 		VCL_fetch_method(sp);
 
 		if (req->objcore != NULL && wrk->busyobj->do_pass)
@@ -644,7 +644,7 @@ cnt_fetch(struct sess *sp, struct worker *wrk, struct req *req)
 		switch (req->handling) {
 		case VCL_RET_DELIVER:
 			AssertObjCorePassOrBusy(req->objcore);
-			sp->step = STP_FETCHBODY;
+			sp->step = STP_PREPFETCH;
 			return (0);
 		default:
 			break;
@@ -680,30 +680,22 @@ cnt_fetch(struct sess *sp, struct worker *wrk, struct req *req)
 }
 
 /*--------------------------------------------------------------------
- * Fetch response body from the backend
+ * Prepare to fetch body from backend
  *
 DOT subgraph xcluster_body {
-DOT	fetchbody [
-DOT		shape=diamond
-DOT		label="stream ?"
-DOT	]
-DOT	fetchbody2 [
-DOT		shape=ellipse
-DOT		label="fetch body\nfrom backend\n"
+DOT	prepfetch [
+DOT		shape=record
+DOT		label="{cnt_prepfetch:|error?|<out>stream ?}"
 DOT	]
 DOT }
-DOT fetchbody -> fetchbody2 [label=no,style=bold,color=red]
-DOT fetchbody -> fetchbody2 [style=bold,color=blue]
-DOT fetchbody -> prepresp [label=yes,style=bold,color=cyan]
-DOT fetchbody2 -> prepresp [style=bold,color=red]
-DOT fetchbody2 -> prepresp [style=bold,color=blue]
+DOT prepfetch:out -> fetchbody [style=bold,color=red]
+DOT prepfetch:out -> fetchbody [style=bold,color=blue]
+DOT prepfetch:out -> prepresp [label=yes,style=bold,color=cyan]
  */
 
-
 static int
-cnt_fetchbody(struct sess *sp, struct worker *wrk, struct req *req)
+cnt_prepfetch(struct sess *sp, struct worker *wrk, struct req *req)
 {
-	int i;
 	struct http *hp, *hp2;
 	char *b;
 	uint16_t nhttp;
@@ -888,7 +880,36 @@ cnt_fetchbody(struct sess *sp, struct worker *wrk, struct req *req)
 	if (bo->do_stream) {
 		sp->step = STP_PREPRESP;
 		return (0);
+	} else {
+		sp->step = STP_FETCHBODY;
+		return (0);
 	}
+}
+
+/*--------------------------------------------------------------------
+ * Actually fetch body from backend
+ *
+DOT subgraph xcluster_fetchbody {
+DOT	fetchbody [
+DOT		shape=record
+DOT		label="{cnt_fetchbody:|error ?|<out>success ?}"
+DOT	]
+DOT }
+DOT fetchbody:out -> prepresp [style=bold,color=red]
+DOT fetchbody:out -> prepresp [style=bold,color=blue]
+ */
+
+static int
+cnt_fetchbody(struct sess *sp, struct worker *wrk, struct req *req)
+{
+	int i;
+	struct busyobj *bo;
+
+	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
+	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
+	bo = wrk->busyobj;
+	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
 
 	/* Use unmodified headers*/
 	i = FetchBody(wrk, req->obj);
diff --git a/include/tbl/steps.h b/include/tbl/steps.h
index 6fcd74e..1e95529 100644
--- a/include/tbl/steps.h
+++ b/include/tbl/steps.h
@@ -39,6 +39,7 @@ STEP(lookup,		LOOKUP,		(sp, sp->wrk, sp->req))
 STEP(miss,		MISS,		(sp, sp->wrk, sp->req))
 STEP(hit,		HIT,		(sp, sp->wrk, sp->req))
 STEP(fetch,		FETCH,		(sp, sp->wrk, sp->req))
+STEP(prepfetch,		PREPFETCH,	(sp, sp->wrk, sp->req))
 STEP(fetchbody,		FETCHBODY,	(sp, sp->wrk, sp->req))
 STEP(streambody,	STREAMBODY,	(sp, sp->wrk, sp->req))
 STEP(prepresp,		PREPRESP,	(sp, sp->wrk, sp->req))



From geoff at varnish-cache.org  Tue Jan 31 20:47:23 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Tue, 31 Jan 2012 21:47:23 +0100
Subject: [experimental-ims] 8a9ad30 More work on the dot graph
Message-ID: <E1RsKbr-0005lO-BZ@project.varnish-software.com>

commit 8a9ad303b25a8cfeba5ef1add82be05d0264b153
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Jan 31 08:36:51 2012 +0000

    More work on the dot graph

diff --git a/bin/varnishd/cache/cache_center.c b/bin/varnishd/cache/cache_center.c
index d12101d..c933175 100644
--- a/bin/varnishd/cache/cache_center.c
+++ b/bin/varnishd/cache/cache_center.c
@@ -188,25 +188,13 @@ cnt_wait(struct sess *sp, struct worker *wrk, struct req *req)
  *
 DOT subgraph xcluster_prepresp {
 DOT	prepresp [
-DOT		shape=ellipse
-DOT		label="Filter obj.->resp."
-DOT	]
-DOT	vcl_deliver [
 DOT		shape=record
-DOT		label="vcl_deliver()|resp."
+DOT		label="{cnt_prepresp:|Filter obj.-\>resp.|{vcl_deliver\{\}|{req.|resp.}}|{error?|restart?}|stream ?}"
 DOT	]
-DOT	prepresp -> vcl_deliver [style=bold,color=green]
-DOT	prepresp -> vcl_deliver [style=bold,color=cyan]
-DOT	prepresp -> vcl_deliver [style=bold,color=red]
-DOT	prepresp -> vcl_deliver [style=bold,color=blue,]
-DOT	vcl_deliver -> deliver [style=bold,color=green,label=deliver]
-DOT	vcl_deliver -> deliver [style=bold,color=red]
-DOT	vcl_deliver -> deliver [style=bold,color=blue]
-DOT     vcl_deliver -> errdeliver [label="error"]
-DOT     errdeliver [label="ERROR",shape=plaintext]
-DOT     vcl_deliver -> rstdeliver [label="restart",color=purple]
-DOT     rstdeliver [label="RESTART",shape=plaintext]
-DOT     vcl_deliver -> streambody [style=bold,color=cyan,label="deliver"]
+DOT	prepresp -> deliver [style=bold,color=green,label=deliver]
+DOT	prepresp -> deliver [style=bold,color=red]
+DOT	prepresp -> deliver [style=bold,color=blue]
+DOT     prepresp -> streambody [style=bold,color=cyan,label="deliver"]
 DOT }
  *
  */
@@ -326,8 +314,8 @@ cnt_prepresp(struct sess *sp, struct worker *wrk, struct req *req)
  *
 DOT subgraph xcluster_deliver {
 DOT	deliver [
-DOT		shape=ellipse
-DOT		label="Send body"
+DOT		shape=record
+DOT		label="{cnt_deliver:|Send body}"
 DOT	]
 DOT }
 DOT deliver -> DONE [style=bold,color=green]
@@ -362,8 +350,8 @@ cnt_deliver(struct sess *sp, struct worker *wrk, struct req *req)
  * the client connection
  *
 DOT	DONE [
-DOT		shape=hexagon
-DOT		label="cnt_done:\nRequest completed"
+DOT		shape=record
+DOT		label="{cnt_done:|Request completed}"
 DOT	]
 DOT	ESI_RESP [ shape=hexagon ]
 DOT	DONE -> start [label="full pipeline"]
@@ -557,16 +545,11 @@ cnt_error(struct sess *sp, struct worker *wrk, struct req *req)
 DOT subgraph xcluster_fetch {
 DOT	fetch [
 DOT		shape=record
-DOT		label="{cnt_fetch:|fetch hdr\nfrom backend|(find obj.ttl)|{vcl_fetch\{\}|{req.|bereq.|beresp.}}|{<err>error?|<rst>restart?}|{<hfp>hit_for_pass?|<del>deliver?}}"
+DOT		label="{cnt_fetch:|fetch hdr\nfrom backend|(find obj.ttl)|{vcl_fetch\{\}|{req.|bereq.|beresp.}}|{<err>error?|<rst>restart?}}"
 DOT	]
-DOT	fetch_pass [
-DOT		shape=ellipse
-DOT		label="obj.f.pass=true"
-DOT	]
-DOT	vcl_fetch -> fetch_pass [label="hit_for_pass",style=bold,color=red]
 DOT }
-DOT fetch:hfp -> prepfetch [style=bold,color=red]
-DOT fetch:del -> prepfetch [label="deliver",style=bold,color=blue]
+DOT fetch -> prepfetch [style=bold,color=red]
+DOT fetch -> prepfetch [style=bold,color=blue]
  */
 
 static int
@@ -946,8 +929,8 @@ cnt_fetchbody(struct sess *sp, struct worker *wrk, struct req *req)
  * Stream the body as we fetch it
 DOT subgraph xstreambody {
 DOT	streambody [
-DOT		shape=ellipse
-DOT		label="streaming\nfetch/deliver"
+DOT		shape=record
+DOT		label="{cnt_streambody:|ping_pong\nfetch/deliver}"
 DOT	]
 DOT }
 DOT streambody -> DONE [style=bold,color=cyan]
@@ -1065,7 +1048,7 @@ cnt_first(struct sess *sp, struct worker *wrk)
 DOT subgraph xcluster_hit {
 DOT	hit [
 DOT		shape=record
-DOT		label="{cnt_hit:|{vcl_hit()|{req.|obj.}}|{<err>error?|<del>deliver?|<rst>restart?|<pass>pass?}}"
+DOT		label="{cnt_hit:|{vcl_hit()|{req.|obj.}}|{<err>error?|<rst>restart?}|{<del>deliver?|<pass>pass?}}"
 DOT	]
 DOT }
 XDOT hit:err -> err_hit [label="error"]
@@ -1237,13 +1220,9 @@ cnt_lookup(struct sess *sp, struct worker *wrk, struct req *req)
 DOT subgraph xcluster_miss {
 DOT	miss [
 DOT		shape=record
-DOT		label="{cnt_miss:|filter req.-\>bereq.|{vcl_miss\{\}|{req.*|bereq.*}}|{<pass>pass?|<err>error?|<restart>restart?|<fetch>fetch?}}"
+DOT		label="{cnt_miss:|filter req.-\>bereq.|{vcl_miss\{\}|{req.*|bereq.*}}|{<err>error?|<rst>restart?}|{<pass>pass?|<fetch>fetch?}}"
 DOT	]
 DOT }
-XDOT miss:restart -> rst_miss [label="restart",color=purple]
-XDOT rst_miss [label="RESTART",shape=plaintext]
-XDOT miss:err -> err_miss [label="error"]
-XDOT err_miss [label="ERROR",shape=plaintext]
 DOT miss:fetch -> fetch [label="fetch",style=bold,color=blue]
 DOT miss:pass -> pass [label="pass",style=bold,color=red]
 DOT



From geoff at varnish-cache.org  Tue Jan 31 20:47:23 2012
From: geoff at varnish-cache.org (Geoff Simmons)
Date: Tue, 31 Jan 2012 21:47:23 +0100
Subject: [experimental-ims] 1f8cdd7 Merge branch 'master' of
	ssh://git.varnish-cache.org/git/varnish-cache into experimental-ims
Message-ID: <E1RsKbr-0005lW-Df@project.varnish-software.com>

commit 1f8cdd71888885c0509d86cb1af906d22c6fa053
Merge: 3df9ebf 8a9ad30
Author: Geoff Simmons <geoff at uplex.de>
Date:   Tue Jan 31 21:40:15 2012 +0100

    Merge branch 'master' of ssh://git.varnish-cache.org/git/varnish-cache into experimental-ims

diff --cc bin/varnishd/cache/cache_center.c
index 8897e83,c933175..623eefd
--- a/bin/varnishd/cache/cache_center.c
+++ b/bin/varnishd/cache/cache_center.c
@@@ -647,15 -618,9 +630,15 @@@ cnt_fetch(struct sess *sp, struct worke
  
  		AZ(wrk->busyobj->do_esi);
  		AZ(wrk->busyobj->do_pass);
- 		
+ 
  		VCL_fetch_method(sp);
  
 +		/* Cancel streaming if a stale object was validated         */
 +		/* XXX: But not if original beresp.status != 304            */
 +		/*      See also AZ(busyobj->stale_obj) in cnt_streambody() */
 +		if (wrk->busyobj->stale_obj)
 +			wrk->busyobj->do_stream = 0;
 +
  		if (req->objcore != NULL && wrk->busyobj->do_pass)
  			req->objcore->flags |= OC_F_PASS;
  
@@@ -698,33 -663,25 +681,25 @@@
  }
  
  /*--------------------------------------------------------------------
-  * Fetch response body from the backend
+  * Prepare to fetch body from backend
   *
  DOT subgraph xcluster_body {
- DOT	fetchbody [
- DOT		shape=diamond
- DOT		label="stream ?"
- DOT	]
- DOT	fetchbody2 [
- DOT		shape=ellipse
- DOT		label="fetch body\nfrom backend\n"
+ DOT	prepfetch [
+ DOT		shape=record
+ DOT		label="{cnt_prepfetch:|error?|<out>stream ?}"
  DOT	]
  DOT }
- DOT fetchbody -> fetchbody2 [label=no,style=bold,color=red]
- DOT fetchbody -> fetchbody2 [style=bold,color=blue]
- DOT fetchbody -> prepresp [label=yes,style=bold,color=cyan]
- DOT fetchbody2 -> prepresp [style=bold,color=red]
- DOT fetchbody2 -> prepresp [style=bold,color=blue]
+ DOT prepfetch:out -> fetchbody [style=bold,color=red]
+ DOT prepfetch:out -> fetchbody [style=bold,color=blue]
+ DOT prepfetch:out -> prepresp [label=yes,style=bold,color=cyan]
   */
  
- 
  static int
- cnt_fetchbody(struct sess *sp, struct worker *wrk, struct req *req)
+ cnt_prepfetch(struct sess *sp, struct worker *wrk, struct req *req)
  {
- 	int i;
  	struct http *hp, *hp2;
  	char *b;
 -	uint16_t nhttp;
 +	uint16_t nhttp, stale_nhttp;
  	unsigned l;
  	struct vsb *vary = NULL;
  	int varyl = 0, pass;