From phk at varnish-cache.org  Mon Jan  3 09:17:47 2011
From: phk at varnish-cache.org (phk at varnish-cache.org)
Date: Mon,  3 Jan 2011 10:17:47 +0100
Subject: r5672 - in trunk/varnish-cache: bin/varnishd bin/varnishtest/tests
	include lib/libvcl
Message-ID: <E1PZgXz-0007Uw-Jr@project.varnish-software.com>

Author: phk
Date: 2011-01-03 10:17:47 +0100 (Mon, 03 Jan 2011)
New Revision: 5672

Added:
   trunk/varnish-cache/include/ban_vars.h
Removed:
   trunk/varnish-cache/include/purge_vars.h
Modified:
   trunk/varnish-cache/bin/varnishd/cache_ban.c
   trunk/varnish-cache/bin/varnishd/heritage.h
   trunk/varnish-cache/bin/varnishd/mgt_param.c
   trunk/varnish-cache/bin/varnishtest/tests/b00008.vtc
   trunk/varnish-cache/bin/varnishtest/tests/b00033.vtc
   trunk/varnish-cache/bin/varnishtest/tests/c00006.vtc
   trunk/varnish-cache/bin/varnishtest/tests/c00007.vtc
   trunk/varnish-cache/bin/varnishtest/tests/c00019.vtc
   trunk/varnish-cache/bin/varnishtest/tests/c00021.vtc
   trunk/varnish-cache/bin/varnishtest/tests/c00022.vtc
   trunk/varnish-cache/bin/varnishtest/tests/c00033.vtc
   trunk/varnish-cache/bin/varnishtest/tests/p00002.vtc
   trunk/varnish-cache/bin/varnishtest/tests/p00003.vtc
   trunk/varnish-cache/bin/varnishtest/tests/r00502.vtc
   trunk/varnish-cache/bin/varnishtest/tests/v00011.vtc
   trunk/varnish-cache/bin/varnishtest/tests/v00018.vtc
   trunk/varnish-cache/include/Makefile.am
   trunk/varnish-cache/include/cli.h
   trunk/varnish-cache/include/vsc_fields.h
   trunk/varnish-cache/lib/libvcl/vcc_action.c
Log:
Finally pull together the courage to fix an old mistake of mine:

Rename 'purges' to 'bans' so that we can implement _real_ purges
also.

They really have been called 'bans' internally all the way through,
but for reasons now forgotten, we thought it would be too hard to
explain to people that purges were called bans in Varnish, so
the user-fronting side were named purges.

As it transpired, people were surprised that purges in varnish
did not free the storage right away, and various other kinds of
confusion also materialized.

The sky convinced me to implement real purges for efficiency
reasons when Vary is used, and of course now the name "purge"
was taken.

I'm sure people will curse about this change in the 2.x to 3.x
migration, but sooner or later it had to happen, and now it did.



Modified: trunk/varnish-cache/bin/varnishd/cache_ban.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_ban.c	2010-12-31 12:46:11 UTC (rev 5671)
+++ trunk/varnish-cache/bin/varnishd/cache_ban.c	2011-01-03 09:17:47 UTC (rev 5672)
@@ -26,7 +26,7 @@
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
  *
- * Ban ("purge") processing
+ * Ban processing
  *
  * A ban consists of a number of conditions (or tests), all of which must be
  * satisfied.  Here are some potential bans we could support:
@@ -224,7 +224,7 @@
 	ban_cond_f		*func;
 } pvars[] = {
 #define PVAR(a, b, c)	{ (a), (b), (c) },
-#include "purge_vars.h"
+#include "ban_vars.h"
 #undef PVAR
 	{ 0, 0, 0}
 };
@@ -323,11 +323,11 @@
 	Lck_Lock(&ban_mtx);
 	VTAILQ_INSERT_HEAD(&ban_head, b, list);
 	ban_start = b;
-	VSC_main->n_purge++;
-	VSC_main->n_purge_add++;
+	VSC_main->n_ban++;
+	VSC_main->n_ban_add++;
 
 	be = VTAILQ_LAST(&ban_head, banhead);
-	if (params->purge_dups && be != b)
+	if (params->ban_dups && be != b)
 		be->refcount++;
 	else
 		be = NULL;
@@ -352,7 +352,7 @@
 	}
 	Lck_Lock(&ban_mtx);
 	be->refcount--;
-	VSC_main->n_purge_dups += pcount;
+	VSC_main->n_ban_dups += pcount;
 	Lck_Unlock(&ban_mtx);
 }
 
@@ -386,8 +386,8 @@
 	Lck_AssertHeld(&ban_mtx);
 	b = VTAILQ_LAST(&ban_head, banhead);
 	if (b != VTAILQ_FIRST(&ban_head) && b->refcount == 0) {
-		VSC_main->n_purge--;
-		VSC_main->n_purge_retire++;
+		VSC_main->n_ban--;
+		VSC_main->n_ban_retire++;
 		VTAILQ_REMOVE(&ban_head, b, list);
 	} else {
 		b = NULL;
@@ -465,8 +465,8 @@
 		VTAILQ_INSERT_TAIL(&b0->objcore, oc, ban_list);
 		b0->refcount++;
 	}
-	VSC_main->n_purge_obj_test++;
-	VSC_main->n_purge_re_test += tests;
+	VSC_main->n_ban_obj_test++;
+	VSC_main->n_ban_re_test += tests;
 	Lck_Unlock(&ban_mtx);
 
 	if (b == oc->ban) {	/* not banned */
@@ -637,8 +637,8 @@
 			break;
 	}
 
-	VSC_main->n_purge++;
-	VSC_main->n_purge_add++;
+	VSC_main->n_ban++;
+	VSC_main->n_ban_add++;
 
 	b2 = BAN_New();
 	AN(b2);
@@ -699,7 +699,7 @@
  */
 
 static void
-ccf_purge(struct cli *cli, const char * const *av, void *priv)
+ccf_ban(struct cli *cli, const char * const *av, void *priv)
 {
 	int narg, i;
 	struct ban *b;
@@ -737,22 +737,22 @@
 }
 
 static void
-ccf_purge_url(struct cli *cli, const char * const *av, void *priv)
+ccf_ban_url(struct cli *cli, const char * const *av, void *priv)
 {
 	const char *aav[6];
 
 	(void)priv;
 	aav[0] = NULL;
-	aav[1] = "purge";
+	aav[1] = "ban";
 	aav[2] = "req.url";
 	aav[3] = "~";
 	aav[4] = av[2];
 	aav[5] = NULL;
-	ccf_purge(cli, aav, priv);
+	ccf_ban(cli, aav, priv);
 }
 
 static void
-ccf_purge_list(struct cli *cli, const char * const *av, void *priv)
+ccf_ban_list(struct cli *cli, const char * const *av, void *priv)
 {
 	struct ban *b, *bl = NULL;
 
@@ -786,15 +786,9 @@
 }
 
 static struct cli_proto ban_cmds[] = {
-	/*
-	 * XXX: COMPAT: Retain these two entries for entire 2.x series
-	 * XXX: COMPAT: to stay compatible with 1.x series syntax.
-	 */
-	{ CLI_HIDDEN("url.purge", 1, 1)		"h", ccf_purge_url },
-
-	{ CLI_PURGE_URL,			"", ccf_purge_url },
-	{ CLI_PURGE,				"", ccf_purge },
-	{ CLI_PURGE_LIST,			"", ccf_purge_list },
+	{ CLI_BAN_URL,				"", ccf_ban_url },
+	{ CLI_BAN,				"", ccf_ban },
+	{ CLI_BAN_LIST,				"", ccf_ban_list },
 	{ NULL }
 };
 

Modified: trunk/varnish-cache/bin/varnishd/heritage.h
===================================================================
--- trunk/varnish-cache/bin/varnishd/heritage.h	2010-12-31 12:46:11 UTC (rev 5671)
+++ trunk/varnish-cache/bin/varnishd/heritage.h	2011-01-03 09:17:47 UTC (rev 5672)
@@ -186,8 +186,8 @@
 	double			acceptor_sleep_incr;
 	double			acceptor_sleep_decay;
 
-	/* Get rid of duplicate purges */
-	unsigned		purge_dups;
+	/* Get rid of duplicate bans */
+	unsigned		ban_dups;
 
 	/* How long time does the ban lurker sleep */
 	double			ban_lurker_sleep;

Modified: trunk/varnish-cache/bin/varnishd/mgt_param.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/mgt_param.c	2010-12-31 12:46:11 UTC (rev 5671)
+++ trunk/varnish-cache/bin/varnishd/mgt_param.c	2011-01-03 09:17:47 UTC (rev 5672)
@@ -504,7 +504,7 @@
 		"Objects already cached will not be affected by changes "
 		"made until they are fetched from the backend again.\n"
 		"To force an immediate effect at the expense of a total "
-		"flush of the cache use \"purge.url .\"",
+		"flush of the cache use \"ban.url .\"",
 		0,
 		"120", "seconds" },
 	{ "sess_workspace", tweak_uint, &master.sess_workspace, 1024, UINT_MAX,
@@ -785,8 +785,8 @@
 		"The TTL assigned to the synthesized error pages\n",
 		0,
 		"0", "seconds" },
-	{ "purge_dups", tweak_bool, &master.purge_dups, 0, 0,
-		"Detect and eliminate duplicate purges.\n",
+	{ "ban_dups", tweak_bool, &master.ban_dups, 0, 0,
+		"Detect and eliminate duplicate bans.\n",
 		0,
 		"on", "bool" },
 	{ "syslog_cli_traffic", tweak_bool, &master.syslog_cli_traffic, 0, 0,
@@ -796,7 +796,7 @@
 	{ "ban_lurker_sleep", tweak_timeout_double,
 		&master.ban_lurker_sleep, 0, UINT_MAX,
 		"How long time does the ban lurker thread sleeps between "
-		"successfull attempts to push the last item up the purge "
+		"successfull attempts to push the last item up the ban "
 		" list.  It always sleeps a second when nothing can be done.\n"
 		"A value of zero disables the ban lurker.",
 		0,

Modified: trunk/varnish-cache/bin/varnishtest/tests/b00008.vtc
===================================================================
--- trunk/varnish-cache/bin/varnishtest/tests/b00008.vtc	2010-12-31 12:46:11 UTC (rev 5671)
+++ trunk/varnish-cache/bin/varnishtest/tests/b00008.vtc	2011-01-03 09:17:47 UTC (rev 5672)
@@ -12,7 +12,7 @@
 
 varnish v1 -cliok "help vcl.load"
 
-varnish v1 -cliok "help purge"
+varnish v1 -cliok "help ban"
 
 varnish v1 -clierr 101 "FOO?"
 

Modified: trunk/varnish-cache/bin/varnishtest/tests/b00033.vtc
===================================================================
--- trunk/varnish-cache/bin/varnishtest/tests/b00033.vtc	2010-12-31 12:46:11 UTC (rev 5671)
+++ trunk/varnish-cache/bin/varnishtest/tests/b00033.vtc	2011-01-03 09:17:47 UTC (rev 5672)
@@ -32,7 +32,7 @@
 	expect resp.bodylen == 6
 } -run
 
-varnish v1 -cliok "purge req.url ~ ."
+varnish v1 -cliok "ban req.url ~ ."
 	
 client c1 {
 	txreq -url /1

Modified: trunk/varnish-cache/bin/varnishtest/tests/c00006.vtc
===================================================================
--- trunk/varnish-cache/bin/varnishtest/tests/c00006.vtc	2010-12-31 12:46:11 UTC (rev 5671)
+++ trunk/varnish-cache/bin/varnishtest/tests/c00006.vtc	2011-01-03 09:17:47 UTC (rev 5672)
@@ -22,7 +22,7 @@
 
 client c1 -run
 
-varnish v1 -cli "purge.url foo"
+varnish v1 -cli "ban.url foo"
 
 client c1 {
 	txreq -url "/foo"

Modified: trunk/varnish-cache/bin/varnishtest/tests/c00007.vtc
===================================================================
--- trunk/varnish-cache/bin/varnishtest/tests/c00007.vtc	2010-12-31 12:46:11 UTC (rev 5671)
+++ trunk/varnish-cache/bin/varnishtest/tests/c00007.vtc	2011-01-03 09:17:47 UTC (rev 5672)
@@ -2,4 +2,4 @@
 
 test "Test banning a hash"
 
-varnish v1 -arg "-b 127.0.0.1:80 -a 127.0.0.1:0" -start -clierr 101 "purge.hash foo"
+varnish v1 -arg "-b 127.0.0.1:80 -a 127.0.0.1:0" -start -clierr 101 "ban.hash foo"

Modified: trunk/varnish-cache/bin/varnishtest/tests/c00019.vtc
===================================================================
--- trunk/varnish-cache/bin/varnishtest/tests/c00019.vtc	2010-12-31 12:46:11 UTC (rev 5671)
+++ trunk/varnish-cache/bin/varnishtest/tests/c00019.vtc	2011-01-03 09:17:47 UTC (rev 5672)
@@ -1,6 +1,6 @@
 # $Id$
 
-test "Check purge counters and duplicate purge elimination"
+test "Check ban counters and duplicate ban elimination"
 
 server s1 {
 	rxreq
@@ -15,14 +15,14 @@
 
 varnish v1 -vcl+backend {} -start
 
-varnish v1 -cliok "purge.url FOO"
+varnish v1 -cliok "ban.url FOO"
 
-# There is one "magic" purge from boot
-varnish v1 -expect n_purge_add == 2
-varnish v1 -cliok "purge.list"
+# There is one "magic" ban from boot
+varnish v1 -expect n_ban_add == 2
+varnish v1 -cliok "ban.list"
 
-# Our fetch is not affected by the purge
-# as the FOO-purge was preexisting
+# Our fetch is not affected by the ban
+# as the FOO-ban was preexisting
 client c1 {
 	txreq -url /BAR
 	rxresp
@@ -32,16 +32,16 @@
 	expect resp.http.foo == 1
 } -run
 
-varnish v1 -cliok "purge.list"
-varnish v1 -expect n_purge_obj_test == 0
-varnish v1 -expect n_purge_re_test == 0
+varnish v1 -cliok "ban.list"
+varnish v1 -expect n_ban_obj_test == 0
+varnish v1 -expect n_ban_re_test == 0
 
-# Add another purge
-varnish v1 -cliok "purge.url FOO"
-varnish v1 -expect n_purge_add == 3
-varnish v1 -cliok "purge.list"
+# Add another ban
+varnish v1 -cliok "ban.url FOO"
+varnish v1 -expect n_ban_add == 3
+varnish v1 -cliok "ban.list"
 
-# The cached object will be purged, and a new
+# The cached object will be band, and a new
 # fetched from the backend
 client c1 {
 	txreq -url /FOO
@@ -49,9 +49,9 @@
 	expect resp.http.foo == 2
 } -run
 
-varnish v1 -expect n_purge_obj_test == 1
-varnish v1 -expect n_purge_re_test == 1
-varnish v1 -cliok "purge.list"
+varnish v1 -expect n_ban_obj_test == 1
+varnish v1 -expect n_ban_re_test == 1
+varnish v1 -cliok "ban.list"
 
 # Fetch the cached copy, just for grins
 client c1 {
@@ -61,19 +61,19 @@
 } -run
 
 
-# Now add another two purges, Kilroy should not be hit
-varnish v1 -cliok "purge.url KILROY"
-varnish v1 -cliok "purge.url FOO"
-varnish v1 -expect n_purge_add == 5
+# Now add another two bans, Kilroy should not be hit
+varnish v1 -cliok "ban.url KILROY"
+varnish v1 -cliok "ban.url FOO"
+varnish v1 -expect n_ban_add == 5
 
-# Enable dup removal of purges
-varnish v1 -cliok "param.set purge_dups on"
+# Enable dup removal of bans
+varnish v1 -cliok "param.set ban_dups on"
 
-# This should incapacitate the two previous FOO purges.
-varnish v1 -cliok "purge.url FOO"
-varnish v1 -expect n_purge_add == 6
-varnish v1 -expect n_purge_dups == 3
-varnish v1 -cliok "purge.list"
+# This should incapacitate the two previous FOO bans.
+varnish v1 -cliok "ban.url FOO"
+varnish v1 -expect n_ban_add == 6
+varnish v1 -expect n_ban_dups == 3
+varnish v1 -cliok "ban.list"
 
 # And we should get a fresh object from backend
 client c1 {
@@ -83,10 +83,10 @@
 } -run
 
 # With only two objects having ever been compared
-varnish v1 -expect n_purge_obj_test == 2
-varnish v1 -expect n_purge_re_test == 2
-varnish v1 -cliok "purge.list"
+varnish v1 -expect n_ban_obj_test == 2
+varnish v1 -expect n_ban_re_test == 2
+varnish v1 -cliok "ban.list"
 
 # Test a bogus regexp
 
-varnish v1 -clierr 106 "purge.url [[["
+varnish v1 -clierr 106 "ban.url [[["

Modified: trunk/varnish-cache/bin/varnishtest/tests/c00021.vtc
===================================================================
--- trunk/varnish-cache/bin/varnishtest/tests/c00021.vtc	2010-12-31 12:46:11 UTC (rev 5671)
+++ trunk/varnish-cache/bin/varnishtest/tests/c00021.vtc	2011-01-03 09:17:47 UTC (rev 5672)
@@ -1,6 +1,6 @@
 # $Id$
 
-test "Test banning a url with cli:purge"
+test "Test banning a url with cli:ban"
 
 server s1 {
 	rxreq
@@ -31,17 +31,17 @@
 } -run
 
 # syntax checks
-varnish v1 -clierr 104 "purge"
-varnish v1 -clierr 104 "purge foo"
-varnish v1 -clierr 104 "purge foo bar"
-varnish v1 -clierr 106 "purge a b c && a"
-varnish v1 -clierr 106 "purge a b c && a b"
-varnish v1 -clierr 106 "purge a b c || a b c"
-varnish v1 -cliok "purge.list"
+varnish v1 -clierr 104 "ban"
+varnish v1 -clierr 104 "ban foo"
+varnish v1 -clierr 104 "ban foo bar"
+varnish v1 -clierr 106 "ban a b c && a"
+varnish v1 -clierr 106 "ban a b c && a b"
+varnish v1 -clierr 106 "ban a b c || a b c"
+varnish v1 -cliok "ban.list"
 
 # exact match, not matching
-varnish v1 -cliok "purge req.url == foo"
-varnish v1 -cliok "purge.list"
+varnish v1 -cliok "ban req.url == foo"
+varnish v1 -cliok "ban.list"
 
 client c1 {
 	txreq -url "/foo"
@@ -52,8 +52,8 @@
 } -run
 
 # exact match, matching
-varnish v1 -cliok "purge req.url == /foo"
-varnish v1 -cliok "purge.list"
+varnish v1 -cliok "ban req.url == /foo"
+varnish v1 -cliok "ban.list"
 client c1 {
 	txreq -url "/foo"
 	rxresp
@@ -63,8 +63,8 @@
 } -run
 
 # regexp nonmatch
-varnish v1 -cliok "purge req.url ~ bar"
-varnish v1 -cliok "purge.list"
+varnish v1 -cliok "ban req.url ~ bar"
+varnish v1 -cliok "ban.list"
 
 client c1 {
 	txreq -url "/foo"
@@ -76,8 +76,8 @@
 
 
 # regexp match
-varnish v1 -cliok "purge req.url ~ foo"
-varnish v1 -cliok "purge.list"
+varnish v1 -cliok "ban req.url ~ foo"
+varnish v1 -cliok "ban.list"
 
 client c1 {
 	txreq -url "/foo"
@@ -88,8 +88,8 @@
 } -run
 
 # header check, nonmatch
-varnish v1 -cliok "purge obj.http.foo != bar7"
-varnish v1 -cliok "purge.list"
+varnish v1 -cliok "ban obj.http.foo != bar7"
+varnish v1 -cliok "ban.list"
 
 client c1 {
 	txreq -url "/foo"
@@ -100,8 +100,8 @@
 } -run
 
 # header check, match
-varnish v1 -cliok "purge req.http.foo == barcheck"
-varnish v1 -cliok "purge.list"
+varnish v1 -cliok "ban req.http.foo == barcheck"
+varnish v1 -cliok "ban.list"
 
 client c1 {
 	txreq -url "/foo" -hdr "foo: barcheck" 
@@ -112,8 +112,8 @@
 } -run
 
 # header check, no header
-varnish v1 -cliok "purge req.url ~ foo && obj.http.bar == barcheck"
-varnish v1 -cliok "purge.list"
+varnish v1 -cliok "ban req.url ~ foo && obj.http.bar == barcheck"
+varnish v1 -cliok "ban.list"
 
 client c1 {
 	txreq -url "/foo"

Modified: trunk/varnish-cache/bin/varnishtest/tests/c00022.vtc
===================================================================
--- trunk/varnish-cache/bin/varnishtest/tests/c00022.vtc	2010-12-31 12:46:11 UTC (rev 5671)
+++ trunk/varnish-cache/bin/varnishtest/tests/c00022.vtc	2011-01-03 09:17:47 UTC (rev 5672)
@@ -1,6 +1,6 @@
 # $Id$
 
-test "Test banning a url with VCL purge"
+test "Test banning a url with VCL ban"
 
 server s1 {
 	rxreq
@@ -23,11 +23,11 @@
 varnish v1 -vcl+backend { 
 	sub vcl_recv {
 		if (req.request == "PURGE") {
-			purge (req.url == req.url);
+			ban (req.url == req.url);
 			error 410;
 		}
 		if (req.request == "PURGESTR") {
-			purge ("" + req.http.purge);
+			ban ("" + req.http.ban);
 			error 410;
 		}
 	}
@@ -39,7 +39,7 @@
 		.host = "127.0.0.1";
 	}
 	sub vcl_recv {
-		purge (req.foo == req.url);
+		ban (req.foo == req.url);
 	}
 }
 
@@ -49,7 +49,7 @@
 		.host = "127.0.0.1";
 	}
 	sub vcl_recv {
-		purge (req.http. == req.url);
+		ban (req.http. == req.url);
 	}
 }
 
@@ -62,13 +62,13 @@
 	expect resp.bodylen == 5
 } -run
 
-# Purge something else
+# Ban: something else
 client c1 {
 	txreq -req PURGE -url /foox
 	rxresp
 	expect resp.status == 410
 } -run
-varnish v1 -cliok "purge.list"
+varnish v1 -cliok "ban.list"
 
 # Still in cache
 client c1 {
@@ -79,13 +79,13 @@
 	expect resp.bodylen == 5
 } -run
 
-# Purge it
+# Ban: it
 client c1 {
 	txreq -req PURGE -url /foo
 	rxresp
 	expect resp.status == 410
 } -run
-varnish v1 -cliok "purge.list"
+varnish v1 -cliok "ban.list"
 
 # New obj 
 client c1 {
@@ -96,13 +96,13 @@
 	expect resp.bodylen == 6
 } -run
 
-# Purge everything else
+# Ban: everything else
 client c1 {
-	txreq -req PURGESTR -hdr "purge=req.url != /foo"
+	txreq -req PURGESTR -hdr "ban=req.url != /foo"
 	rxresp
 	expect resp.status == 410
 } -run
-varnish v1 -cliok "purge.list"
+varnish v1 -cliok "ban.list"
 
 # still there
 client c1 {
@@ -113,13 +113,13 @@
 	expect resp.bodylen == 6
 } -run
 
-# Purge it
+# Ban: it
 client c1 {
-	txreq -req PURGESTR -hdr "Purge: obj.http.foo == \"bar6\""
+	txreq -req PURGESTR -hdr "Ban: obj.http.foo == \"bar6\""
 	rxresp
 	expect resp.status == 410
 } -run
-varnish v1 -cliok "purge.list"
+varnish v1 -cliok "ban.list"
 
 # New one
 client c1 {
@@ -130,13 +130,13 @@
 	expect resp.bodylen == 7
 } -run
 
-# Purge something else
+# Ban: something else
 client c1 {
-	txreq -req PURGESTR -hdr "Purge: obj.http.foo == \"bar6\""
+	txreq -req PURGESTR -hdr "Ban: obj.http.foo == \"bar6\""
 	rxresp
 	expect resp.status == 410
 } -run
-varnish v1 -cliok "purge.list"
+varnish v1 -cliok "ban.list"
 
 # Still there
 client c1 {
@@ -149,11 +149,11 @@
 
 # Header match
 client c1 {
-	txreq -req PURGESTR -hdr "Purge: req.http.foo == \"barcheck\""
+	txreq -req PURGESTR -hdr "Ban: req.http.foo == \"barcheck\""
 	rxresp
 	expect resp.status == 410
 } -run
-varnish v1 -cliok "purge.list"
+varnish v1 -cliok "ban.list"
 
 client c1 {
 	txreq -url "/foo" -hdr "foo: barcheck" 
@@ -165,11 +165,11 @@
 
 # Header match
 client c1 {
-	txreq -req PURGESTR -hdr "Purge: obj.http.foo == \"barcheck\""
+	txreq -req PURGESTR -hdr "Ban: obj.http.foo == \"barcheck\""
 	rxresp
 	expect resp.status == 410
 } -run
-varnish v1 -cliok "purge.list"
+varnish v1 -cliok "ban.list"
 
 client c1 {
 	txreq -url "/foo"

Modified: trunk/varnish-cache/bin/varnishtest/tests/c00033.vtc
===================================================================
--- trunk/varnish-cache/bin/varnishtest/tests/c00033.vtc	2010-12-31 12:46:11 UTC (rev 5671)
+++ trunk/varnish-cache/bin/varnishtest/tests/c00033.vtc	2011-01-03 09:17:47 UTC (rev 5672)
@@ -1,6 +1,6 @@
 # $Id$
 
-test "real purges"
+test "real bans"
 
 server s1 {
 	rxreq

Modified: trunk/varnish-cache/bin/varnishtest/tests/p00002.vtc
===================================================================
--- trunk/varnish-cache/bin/varnishtest/tests/p00002.vtc	2010-12-31 12:46:11 UTC (rev 5671)
+++ trunk/varnish-cache/bin/varnishtest/tests/p00002.vtc	2011-01-03 09:17:47 UTC (rev 5672)
@@ -24,7 +24,7 @@
 	expect resp.http.foo == "foo"
 } -run
 
-varnish v1 -cliok "purge req.url == / && req.http.jam != session"
+varnish v1 -cliok "ban req.url == / && req.http.jam != session"
 varnish v1 -stop
 server s1 -wait
 
@@ -35,7 +35,7 @@
 
 varnish v1 -start
 
-varnish v1 -cliok purge.list
+varnish v1 -cliok ban.list
 
 # Count of 3 here, because two "magic" bans are also there"
-varnish v1 -expect n_purge == 3
+varnish v1 -expect n_ban == 3

Modified: trunk/varnish-cache/bin/varnishtest/tests/p00003.vtc
===================================================================
--- trunk/varnish-cache/bin/varnishtest/tests/p00003.vtc	2010-12-31 12:46:11 UTC (rev 5671)
+++ trunk/varnish-cache/bin/varnishtest/tests/p00003.vtc	2011-01-03 09:17:47 UTC (rev 5672)
@@ -15,7 +15,7 @@
 	-arg "-pban_lurker_sleep=0" \
 	-vcl+backend { } -start 
 
-varnish v1 -cliok purge.list
+varnish v1 -cliok ban.list
 
 client c1 {
 	txreq -url "/"
@@ -25,8 +25,8 @@
 	expect resp.http.foo == "foo"
 } -run
 
-varnish v1 -cliok "purge req.url == /"
-varnish v1 -cliok purge.list
+varnish v1 -cliok "ban req.url == /"
+varnish v1 -cliok ban.list
 varnish v1 -stop
 server s1 -wait
 
@@ -37,10 +37,10 @@
 
 varnish v1 -vcl+backend {} -start
 
-varnish v1 -cliok purge.list
+varnish v1 -cliok ban.list
 
 # Count of 2 here, because the "magic" ban is also there"
-# varnish v1 -expect n_purge == 2
+# varnish v1 -expect n_ban == 2
 
 
 client c1 {
@@ -52,7 +52,7 @@
 } -run
 
 
-varnish v1 -cliok purge.list
+varnish v1 -cliok ban.list
 
 varnish v1 -stop
 

Modified: trunk/varnish-cache/bin/varnishtest/tests/r00502.vtc
===================================================================
--- trunk/varnish-cache/bin/varnishtest/tests/r00502.vtc	2010-12-31 12:46:11 UTC (rev 5671)
+++ trunk/varnish-cache/bin/varnishtest/tests/r00502.vtc	2011-01-03 09:17:47 UTC (rev 5672)
@@ -1,6 +1,6 @@
 # $Id$
 
-test "multi element purge"
+test "multi element ban"
 
 server s1 {
 	rxreq
@@ -11,7 +11,7 @@
 
 varnish v1 -vcl+backend {
 	sub vcl_recv {
-		purge("req.url == / && obj.http.foo ~ bar1");
+		ban("req.url == / && obj.http.foo ~ bar1");
 	}
 } -start
 
@@ -27,4 +27,4 @@
 	expect resp.http.foo == "bar2"
 } -run
 
-varnish v1 -cliok purge.list
+varnish v1 -cliok ban.list

Modified: trunk/varnish-cache/bin/varnishtest/tests/v00011.vtc
===================================================================
--- trunk/varnish-cache/bin/varnishtest/tests/v00011.vtc	2010-12-31 12:46:11 UTC (rev 5671)
+++ trunk/varnish-cache/bin/varnishtest/tests/v00011.vtc	2011-01-03 09:17:47 UTC (rev 5672)
@@ -13,7 +13,7 @@
 varnish v1 -vcl+backend { 
 	sub vcl_recv {
 		if (req.request == "PURGE") {
-			purge_url("^/$");
+			ban_url("^/$");
 			error 209 "foo";
 		}
 	}

Modified: trunk/varnish-cache/bin/varnishtest/tests/v00018.vtc
===================================================================
--- trunk/varnish-cache/bin/varnishtest/tests/v00018.vtc	2010-12-31 12:46:11 UTC (rev 5671)
+++ trunk/varnish-cache/bin/varnishtest/tests/v00018.vtc	2011-01-03 09:17:47 UTC (rev 5672)
@@ -88,17 +88,17 @@
 
 varnish v1 -badvcl {
 	backend b { .host = "127.0.0.1"; }
-	sub vcl_recv { purge_url (if); }
+	sub vcl_recv { ban_url (if); }
 }
 
 varnish v1 -badvcl {
 	backend b { .host = "127.0.0.1"; }
-	sub vcl_recv { purge_hash (if); }
+	sub vcl_recv { ban_hash (if); }
 }
 
 varnish v1 -vcl {
 	backend b { .host = "127.0.0.1"; }
-	sub vcl_recv { purge_url ("foo"); }
+	sub vcl_recv { ban_url ("foo"); }
 }
 
 varnish v1 -badvcl {

Modified: trunk/varnish-cache/include/Makefile.am
===================================================================
--- trunk/varnish-cache/include/Makefile.am	2010-12-31 12:46:11 UTC (rev 5671)
+++ trunk/varnish-cache/include/Makefile.am	2011-01-03 09:17:47 UTC (rev 5672)
@@ -11,6 +11,7 @@
 
 nobase_noinst_HEADERS = \
 	argv.h \
+	ban_vars.h \
 	binary_heap.h \
 	cli.h \
 	cli_common.h \
@@ -30,7 +31,6 @@
 	libvcl.h \
 	miniobj.h \
 	persistent.h \
-	purge_vars.h \
 	svnid.h \
 	vas.h \
 	vsha256.h \

Copied: trunk/varnish-cache/include/ban_vars.h (from rev 5671, trunk/varnish-cache/include/purge_vars.h)
===================================================================
--- trunk/varnish-cache/include/ban_vars.h	                        (rev 0)
+++ trunk/varnish-cache/include/ban_vars.h	2011-01-03 09:17:47 UTC (rev 5672)
@@ -0,0 +1,39 @@
+/*-
+ * Copyright (c) 2008-2009 Linpro AS
+ * All rights reserved.
+ *
+ * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $Id$
+ *
+ * Define which variables we can ban on, and which function does it.
+ *
+ */
+
+#define PVAR_HTTP	1
+#define PVAR_REQ	2
+
+PVAR("req.url",		PVAR_REQ,		ban_cond_url)
+PVAR("req.http.",	PVAR_REQ|PVAR_HTTP,	ban_cond_req_http)
+PVAR("obj.http.",	PVAR_HTTP,		ban_cond_obj_http)

Modified: trunk/varnish-cache/include/cli.h
===================================================================
--- trunk/varnish-cache/include/cli.h	2010-12-31 12:46:11 UTC (rev 5671)
+++ trunk/varnish-cache/include/cli.h	2011-01-03 09:17:47 UTC (rev 5672)
@@ -60,24 +60,24 @@
 	    "\tReturns the TTL, size and checksum of the object.",	\
 	1, 1
 
-#define CLI_PURGE_URL							\
-	"purge.url",							\
-	"purge.url <regexp>",						\
+#define CLI_BAN_URL							\
+	"ban.url",							\
+	"ban.url <regexp>",						\
 	"\tAll objects where the urls matches regexp will be "		\
 	    "marked obsolete.",						\
 	1, 1
 
-#define CLI_PURGE							\
-	"purge",							\
-	"purge <field> <operator> <arg> [&& <field> <oper> <arg>]...",	\
+#define CLI_BAN								\
+	"ban",								\
+	"ban <field> <operator> <arg> [&& <field> <oper> <arg>]...",	\
 	"\tAll objects where the all the conditions match will be "	\
 	    "marked obsolete.",						\
 	3, UINT_MAX
 
-#define CLI_PURGE_LIST							\
-	"purge.list",							\
-	"purge.list",							\
-	"\tList the active purges.",					\
+#define CLI_BAN_LIST							\
+	"ban.list",							\
+	"ban.list",							\
+	"\tList the active bans.",					\
 	0, 0
 
 #define CLI_URL_STATUS							\

Deleted: trunk/varnish-cache/include/purge_vars.h
===================================================================
--- trunk/varnish-cache/include/purge_vars.h	2010-12-31 12:46:11 UTC (rev 5671)
+++ trunk/varnish-cache/include/purge_vars.h	2011-01-03 09:17:47 UTC (rev 5672)
@@ -1,39 +0,0 @@
-/*-
- * Copyright (c) 2008-2009 Linpro AS
- * All rights reserved.
- *
- * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- *
- * $Id$
- *
- * Define which variables we can purge on, and which function does it.
- *
- */
-
-#define PVAR_HTTP	1
-#define PVAR_REQ	2
-
-PVAR("req.url",		PVAR_REQ,		ban_cond_url)
-PVAR("req.http.",	PVAR_REQ|PVAR_HTTP,	ban_cond_req_http)
-PVAR("obj.http.",	PVAR_HTTP,		ban_cond_obj_http)

Modified: trunk/varnish-cache/include/vsc_fields.h
===================================================================
--- trunk/varnish-cache/include/vsc_fields.h	2010-12-31 12:46:11 UTC (rev 5671)
+++ trunk/varnish-cache/include/vsc_fields.h	2011-01-03 09:17:47 UTC (rev 5672)
@@ -131,12 +131,12 @@
 VSC_F(n_vcl_avail,		uint64_t, 0, 'a', "N vcl available")
 VSC_F(n_vcl_discard,	uint64_t, 0, 'a', "N vcl discarded")
 
-VSC_F(n_purge,		uint64_t, 0, 'i', "N total active purges")
-VSC_F(n_purge_add,		uint64_t, 0, 'a', "N new purges added")
-VSC_F(n_purge_retire,	uint64_t, 0, 'a', "N old purges deleted")
-VSC_F(n_purge_obj_test,	uint64_t, 0, 'a', "N objects tested")
-VSC_F(n_purge_re_test,	uint64_t, 0, 'a', "N regexps tested against")
-VSC_F(n_purge_dups,	uint64_t, 0, 'a', "N duplicate purges removed")
+VSC_F(n_ban,		uint64_t, 0, 'i', "N total active bans")
+VSC_F(n_ban_add,		uint64_t, 0, 'a', "N new bans added")
+VSC_F(n_ban_retire,	uint64_t, 0, 'a', "N old bans deleted")
+VSC_F(n_ban_obj_test,	uint64_t, 0, 'a', "N objects tested")
+VSC_F(n_ban_re_test,	uint64_t, 0, 'a', "N regexps tested against")
+VSC_F(n_ban_dups,	uint64_t, 0, 'a', "N duplicate bans removed")
 
 VSC_F(hcb_nolock,		uint64_t, 0, 'a', "HCB Lookups without lock")
 VSC_F(hcb_lock,		uint64_t, 0, 'a', "HCB Lookups with lock")

Modified: trunk/varnish-cache/lib/libvcl/vcc_action.c
===================================================================
--- trunk/varnish-cache/lib/libvcl/vcc_action.c	2010-12-31 12:46:11 UTC (rev 5671)
+++ trunk/varnish-cache/lib/libvcl/vcc_action.c	2011-01-03 09:17:47 UTC (rev 5672)
@@ -179,20 +179,20 @@
 
 /*--------------------------------------------------------------------*/
 
-static const struct purge_var {
+static const struct ban_var {
 	const char	*name;
 	unsigned	flag;
-} purge_var[] = {
+} ban_var[] = {
 #define PVAR(a, b, c)   { (a), (b) },
-#include "purge_vars.h"
+#include "ban_vars.h"
 #undef PVAR
         { 0, 0 }
 };
 
 static void
-parse_purge(struct vcc *tl)
+parse_ban(struct vcc *tl)
 {
-	const struct purge_var *pv;
+	const struct ban_var *pv;
 
 	vcc_NextToken(tl);
 
@@ -205,14 +205,14 @@
 		while (1) {
 			ExpectErr(tl, ID);
 
-			/* Check valididity of purge variable */
-			for (pv = purge_var; pv->name != NULL; pv++) {
+			/* Check valididity of ban variable */
+			for (pv = ban_var; pv->name != NULL; pv++) {
 				if (!strncmp(pv->name, tl->t->b,
 				    strlen(pv->name)))
 					break;
 			}
 			if (pv->name == NULL) {
-				vsb_printf(tl->sb, "Unknown purge variable.");
+				vsb_printf(tl->sb, "Unknown ban variable.");
 				vcc_ErrWhere(tl, tl->t);
 				return;
 			}
@@ -265,7 +265,7 @@
 /*--------------------------------------------------------------------*/
 
 static void
-parse_purge_url(struct vcc *tl)
+parse_ban_url(struct vcc *tl)
 {
 
 	vcc_NextToken(tl);
@@ -405,8 +405,8 @@
 	{ "esi",		parse_esi, VCL_MET_FETCH },
 	{ "hash_data",		parse_hash_data, VCL_MET_HASH },
 	{ "panic",		parse_panic },
-	{ "purge",		parse_purge },
-	{ "purge_url",		parse_purge_url },
+	{ "ban",		parse_ban },
+	{ "ban_url",		parse_ban_url },
 	{ "remove",		parse_unset }, /* backward compatibility */
 	{ "return",		parse_return },
 	{ "rollback",		parse_rollback },




From phk at varnish-cache.org  Mon Jan  3 09:40:10 2011
From: phk at varnish-cache.org (phk at varnish-cache.org)
Date: Mon,  3 Jan 2011 10:40:10 +0100
Subject: r5673 - in trunk/varnish-cache: bin/varnishd include
Message-ID: <E1PZgte-0008U1-A8@project.varnish-software.com>

Author: phk
Date: 2011-01-03 10:40:09 +0100 (Mon, 03 Jan 2011)
New Revision: 5673

Modified:
   trunk/varnish-cache/bin/varnishd/cache_pool.c
   trunk/varnish-cache/bin/varnishd/heritage.h
   trunk/varnish-cache/bin/varnishd/mgt_pool.c
   trunk/varnish-cache/include/vsc_fields.h
Log:
Another rename to reduce confusion:

When there is no idle workerthread available, an incoming requests will
get queued until a thread becomes available.

We called this "overflow" because it overflowed the worker thread pool.

However, "overflow" gives the impression that the request is somehow
lost, which it is not.

"queued" gives a much more precise idea what goes on.



Modified: trunk/varnish-cache/bin/varnishd/cache_pool.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_pool.c	2011-01-03 09:17:47 UTC (rev 5672)
+++ trunk/varnish-cache/bin/varnishd/cache_pool.c	2011-01-03 09:40:09 UTC (rev 5673)
@@ -70,17 +70,17 @@
 #define WQ_MAGIC		0x606658fa
 	struct lock		mtx;
 	struct workerhead	idle;
-	VTAILQ_HEAD(, workreq)	overflow;
+	VTAILQ_HEAD(, workreq)	queue;
 	unsigned		nthr;
-	unsigned		nqueue;
 	unsigned		lqueue;
+	unsigned		last_lqueue;
 	uintmax_t		ndrop;
-	uintmax_t		noverflow;
+	uintmax_t		nqueue;
 };
 
 static struct wq		**wq;
 static unsigned			nwq;
-static unsigned			ovfl_max;
+static unsigned			queue_max;
 static unsigned			nthr_max;
 
 static pthread_cond_t		herder_cond;
@@ -156,11 +156,11 @@
 	while (1) {
 		CHECK_OBJ_NOTNULL(w, WORKER_MAGIC);
 
-		/* Process overflow requests, if any */
-		w->wrq = VTAILQ_FIRST(&qp->overflow);
+		/* Process queued requests, if any */
+		w->wrq = VTAILQ_FIRST(&qp->queue);
 		if (w->wrq != NULL) {
-			VTAILQ_REMOVE(&qp->overflow, w->wrq, list);
-			qp->nqueue--;
+			VTAILQ_REMOVE(&qp->queue, w->wrq, list);
+			qp->lqueue--;
 		} else {
 			if (isnan(w->lastused))
 				w->lastused = TIM_real();
@@ -271,16 +271,16 @@
 		return (0);
 	}
 
-	/* If we have too much in the overflow already, refuse. */
-	if (qp->nqueue > ovfl_max) {
+	/* If we have too much in the queue already, refuse. */
+	if (qp->lqueue > queue_max) {
 		qp->ndrop++;
 		Lck_Unlock(&qp->mtx);
 		return (-1);
 	}
 
-	VTAILQ_INSERT_TAIL(&qp->overflow, wrq, list);
-	qp->noverflow++;
+	VTAILQ_INSERT_TAIL(&qp->queue, wrq, list);
 	qp->nqueue++;
+	qp->lqueue++;
 	Lck_Unlock(&qp->mtx);
 	AZ(pthread_cond_signal(&herder_cond));
 	return (0);
@@ -356,7 +356,7 @@
 		XXXAN(wq[u]);
 		wq[u]->magic = WQ_MAGIC;
 		Lck_New(&wq[u]->mtx, lck_wq);
-		VTAILQ_INIT(&wq[u]->overflow);
+		VTAILQ_INIT(&wq[u]->queue);
 		VTAILQ_INIT(&wq[u]->idle);
 	}
 	(void)owq;	/* XXX: avoid race, leak it. */
@@ -374,9 +374,9 @@
 
 	Lck_Lock(&qp->mtx);
 	vs->n_wrk += qp->nthr;
-	vs->n_wrk_queue += qp->nqueue;
+	vs->n_wrk_lqueue += qp->lqueue;
 	vs->n_wrk_drop += qp->ndrop;
-	vs->n_wrk_overflow += qp->noverflow;
+	vs->n_wrk_queued += qp->nqueue;
 
 	if (qp->nthr > params->wthread_min) {
 		w = VTAILQ_LAST(&qp->idle, workerhead);
@@ -442,21 +442,21 @@
 			u = params->wthread_min;
 		nthr_max = u;
 
-		ovfl_max = (nthr_max * params->overflow_max) / 100;
+		queue_max = (nthr_max * params->queue_max) / 100;
 
 		vs->n_wrk = 0;
-		vs->n_wrk_queue = 0;
+		vs->n_wrk_lqueue = 0;
 		vs->n_wrk_drop = 0;
-		vs->n_wrk_overflow = 0;
+		vs->n_wrk_queued = 0;
 
 		t_idle = TIM_real() - params->wthread_timeout;
 		for (u = 0; u < nwq; u++)
 			wrk_decimate_flock(wq[u], t_idle, vs);
 
 		VSC_main->n_wrk= vs->n_wrk;
-		VSC_main->n_wrk_queue = vs->n_wrk_queue;
+		VSC_main->n_wrk_lqueue = vs->n_wrk_lqueue;
 		VSC_main->n_wrk_drop = vs->n_wrk_drop;
-		VSC_main->n_wrk_overflow = vs->n_wrk_overflow;
+		VSC_main->n_wrk_queued = vs->n_wrk_queued;
 
 		TIM_sleep(params->wthread_purge_delay * 1e-3);
 	}
@@ -477,8 +477,8 @@
 	 * one more thread.
 	 */
 	if (qp->nthr < params->wthread_min ||	/* Not enough threads yet */
-	    (qp->nqueue > params->wthread_add_threshold && /* more needed */
-	    qp->nqueue > qp->lqueue)) {	/* not getting better since last */
+	    (qp->lqueue > params->wthread_add_threshold && /* more needed */
+	    qp->lqueue > qp->last_lqueue)) {	/* not getting better since last */
 		if (qp->nthr >= nthr_max) {
 			VSC_main->n_wrk_max++;
 		} else if (pthread_create(&tp, tp_attr, wrk_thread, qp)) {
@@ -492,11 +492,11 @@
 			TIM_sleep(params->wthread_add_delay * 1e-3);
 		}
 	}
-	qp->lqueue = qp->nqueue;
+	qp->last_lqueue = qp->lqueue;
 }
 
 /*--------------------------------------------------------------------
- * This thread wakes up whenever a pool overflows.
+ * This thread wakes up whenever a pool queues.
  *
  * The trick here is to not be too aggressive about creating threads.
  * We do this by only examining one pool at a time, and by sleeping

Modified: trunk/varnish-cache/bin/varnishd/heritage.h
===================================================================
--- trunk/varnish-cache/bin/varnishd/heritage.h	2011-01-03 09:17:47 UTC (rev 5672)
+++ trunk/varnish-cache/bin/varnishd/heritage.h	2011-01-03 09:40:09 UTC (rev 5673)
@@ -91,7 +91,7 @@
 	unsigned		wthread_stats_rate;
 	unsigned		wthread_stacksize;
 
-	unsigned		overflow_max;
+	unsigned		queue_max;
 
 	/* Memory allocation hints */
 	unsigned		sess_workspace;

Modified: trunk/varnish-cache/bin/varnishd/mgt_pool.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/mgt_pool.c	2011-01-03 09:17:47 UTC (rev 5672)
+++ trunk/varnish-cache/bin/varnishd/mgt_pool.c	2011-01-03 09:40:09 UTC (rev 5673)
@@ -205,8 +205,8 @@
 		"its accumulated stats into the global counters.\n",
 		EXPERIMENTAL,
 		"10", "requests" },
-	{ "overflow_max", tweak_uint, &master.overflow_max, 0, UINT_MAX,
-		"Percentage permitted overflow queue length.\n"
+	{ "queue_max", tweak_uint, &master.queue_max, 0, UINT_MAX,
+		"Percentage permitted queue length.\n"
 		"\n"
 		"This sets the ratio of queued requests to worker threads, "
 		"above which sessions will be dropped instead of queued.\n",

Modified: trunk/varnish-cache/include/vsc_fields.h
===================================================================
--- trunk/varnish-cache/include/vsc_fields.h	2011-01-03 09:17:47 UTC (rev 5672)
+++ trunk/varnish-cache/include/vsc_fields.h	2011-01-03 09:40:09 UTC (rev 5673)
@@ -83,8 +83,8 @@
 VSC_F(n_wrk_failed,	uint64_t, 0, 'a',
 					"N worker threads not created")
 VSC_F(n_wrk_max,		uint64_t, 0, 'a', "N worker threads limited")
-VSC_F(n_wrk_queue,		uint64_t, 0, 'a', "N queued work requests")
-VSC_F(n_wrk_overflow,	uint64_t, 0, 'a', "N overflowed work requests")
+VSC_F(n_wrk_lqueue,		uint64_t, 0, 'a', "work request queue length")
+VSC_F(n_wrk_queued,		uint64_t, 0, 'a', "N queued work requests")
 VSC_F(n_wrk_drop,		uint64_t, 0, 'a', "N dropped work requests")
 VSC_F(n_backend,		uint64_t, 0, 'i', "N backends")
 




From phk at varnish-cache.org  Mon Jan  3 10:37:36 2011
From: phk at varnish-cache.org (phk at varnish-cache.org)
Date: Mon,  3 Jan 2011 11:37:36 +0100
Subject: r5674 - trunk/varnish-cache/bin/varnishd
Message-ID: <E1PZhnE-00021w-4x@project.varnish-software.com>

Author: phk
Date: 2011-01-03 11:37:35 +0100 (Mon, 03 Jan 2011)
New Revision: 5674

Modified:
   trunk/varnish-cache/bin/varnishd/heritage.h
   trunk/varnish-cache/bin/varnishd/mgt_param.c
Log:
Add a parameter to control GZIP processing



Modified: trunk/varnish-cache/bin/varnishd/heritage.h
===================================================================
--- trunk/varnish-cache/bin/varnishd/heritage.h	2011-01-03 09:40:09 UTC (rev 5673)
+++ trunk/varnish-cache/bin/varnishd/heritage.h	2011-01-03 10:37:35 UTC (rev 5674)
@@ -199,6 +199,8 @@
 
 	unsigned		http_range_support;
 
+	unsigned		http_gzip_support;
+
 	double			critbit_cooloff;
 };
 

Modified: trunk/varnish-cache/bin/varnishd/mgt_param.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/mgt_param.c	2011-01-03 09:40:09 UTC (rev 5673)
+++ trunk/varnish-cache/bin/varnishd/mgt_param.c	2011-01-03 10:37:35 UTC (rev 5674)
@@ -812,6 +812,10 @@
 		"Enable support for HTTP Range headers.\n",
 		EXPERIMENTAL,
 		"off", "bool" },
+	{ "http_gzip_support", tweak_bool, &master.http_gzip_support, 0, 0,
+		"Enable support for HTTP GZIP compression.\n",
+		EXPERIMENTAL,
+		"off", "bool" },
 	{ "critbit_cooloff", tweak_timeout_double,
 		&master.critbit_cooloff, 60, 254,
 		"How long time the critbit hasher keeps deleted objheads "




From phk at varnish-cache.org  Mon Jan  3 11:08:19 2011
From: phk at varnish-cache.org (phk at varnish-cache.org)
Date: Mon,  3 Jan 2011 12:08:19 +0100
Subject: r5675 - trunk/varnish-cache/lib/libvgz
Message-ID: <E1PZiGx-00033P-HL@project.varnish-software.com>

Author: phk
Date: 2011-01-03 12:08:19 +0100 (Mon, 03 Jan 2011)
New Revision: 5675

Modified:
   trunk/varnish-cache/lib/libvgz/zconf.h
Log:
Prefix everything in libvgz with vgz



Modified: trunk/varnish-cache/lib/libvgz/zconf.h
===================================================================
--- trunk/varnish-cache/lib/libvgz/zconf.h	2011-01-03 10:37:35 UTC (rev 5674)
+++ trunk/varnish-cache/lib/libvgz/zconf.h	2011-01-03 11:08:19 UTC (rev 5675)
@@ -16,120 +16,120 @@
  * Even better than compiling with -DZ_PREFIX would be to use configure to set
  * this permanently in zconf.h using "./configure --zprefix".
  */
-#ifdef Z_PREFIX     /* may be set to #if 1 by ./configure */
+#if 1
 
 /* all linked symbols */
-#  define _dist_code            z__dist_code
-#  define _length_code          z__length_code
-#  define _tr_align             z__tr_align
-#  define _tr_flush_block       z__tr_flush_block
-#  define _tr_init              z__tr_init
-#  define _tr_stored_block      z__tr_stored_block
-#  define _tr_tally             z__tr_tally
-#  define adler32               z_adler32
-#  define adler32_combine       z_adler32_combine
-#  define adler32_combine64     z_adler32_combine64
-#  define compress              z_compress
-#  define compress2             z_compress2
-#  define compressBound         z_compressBound
-#  define crc32                 z_crc32
-#  define crc32_combine         z_crc32_combine
-#  define crc32_combine64       z_crc32_combine64
-#  define deflate               z_deflate
-#  define deflateBound          z_deflateBound
-#  define deflateCopy           z_deflateCopy
-#  define deflateEnd            z_deflateEnd
-#  define deflateInit2_         z_deflateInit2_
-#  define deflateInit_          z_deflateInit_
-#  define deflateParams         z_deflateParams
-#  define deflatePrime          z_deflatePrime
-#  define deflateReset          z_deflateReset
-#  define deflateSetDictionary  z_deflateSetDictionary
-#  define deflateSetHeader      z_deflateSetHeader
-#  define deflateTune           z_deflateTune
-#  define deflate_copyright     z_deflate_copyright
-#  define get_crc_table         z_get_crc_table
-#  define gz_error              z_gz_error
-#  define gz_intmax             z_gz_intmax
-#  define gz_strwinerror        z_gz_strwinerror
-#  define gzbuffer              z_gzbuffer
-#  define gzclearerr            z_gzclearerr
-#  define gzclose               z_gzclose
-#  define gzclose_r             z_gzclose_r
-#  define gzclose_w             z_gzclose_w
-#  define gzdirect              z_gzdirect
-#  define gzdopen               z_gzdopen
-#  define gzeof                 z_gzeof
-#  define gzerror               z_gzerror
-#  define gzflush               z_gzflush
-#  define gzgetc                z_gzgetc
-#  define gzgets                z_gzgets
-#  define gzoffset              z_gzoffset
-#  define gzoffset64            z_gzoffset64
-#  define gzopen                z_gzopen
-#  define gzopen64              z_gzopen64
-#  define gzprintf              z_gzprintf
-#  define gzputc                z_gzputc
-#  define gzputs                z_gzputs
-#  define gzread                z_gzread
-#  define gzrewind              z_gzrewind
-#  define gzseek                z_gzseek
-#  define gzseek64              z_gzseek64
-#  define gzsetparams           z_gzsetparams
-#  define gztell                z_gztell
-#  define gztell64              z_gztell64
-#  define gzungetc              z_gzungetc
-#  define gzwrite               z_gzwrite
-#  define inflate               z_inflate
-#  define inflateBack           z_inflateBack
-#  define inflateBackEnd        z_inflateBackEnd
-#  define inflateBackInit_      z_inflateBackInit_
-#  define inflateCopy           z_inflateCopy
-#  define inflateEnd            z_inflateEnd
-#  define inflateGetHeader      z_inflateGetHeader
-#  define inflateInit2_         z_inflateInit2_
-#  define inflateInit_          z_inflateInit_
-#  define inflateMark           z_inflateMark
-#  define inflatePrime          z_inflatePrime
-#  define inflateReset          z_inflateReset
-#  define inflateReset2         z_inflateReset2
-#  define inflateSetDictionary  z_inflateSetDictionary
-#  define inflateSync           z_inflateSync
-#  define inflateSyncPoint      z_inflateSyncPoint
-#  define inflateUndermine      z_inflateUndermine
-#  define inflate_copyright     z_inflate_copyright
-#  define inflate_fast          z_inflate_fast
-#  define inflate_table         z_inflate_table
-#  define uncompress            z_uncompress
-#  define zError                z_zError
-#  define zcalloc               z_zcalloc
-#  define zcfree                z_zcfree
-#  define zlibCompileFlags      z_zlibCompileFlags
-#  define zlibVersion           z_zlibVersion
+#  define _dist_code            vgz__dist_code
+#  define _length_code          vgz__length_code
+#  define _tr_align             vgz__tr_align
+#  define _tr_flush_block       vgz__tr_flush_block
+#  define _tr_init              vgz__tr_init
+#  define _tr_stored_block      vgz__tr_stored_block
+#  define _tr_tally             vgz__tr_tally
+#  define adler32               vgz_adler32
+#  define adler32_combine       vgz_adler32_combine
+#  define adler32_combine64     vgz_adler32_combine64
+#  define compress              vgz_compress
+#  define compress2             vgz_compress2
+#  define compressBound         vgz_compressBound
+#  define crc32                 vgz_crc32
+#  define crc32_combine         vgz_crc32_combine
+#  define crc32_combine64       vgz_crc32_combine64
+#  define deflate               vgz_deflate
+#  define deflateBound          vgz_deflateBound
+#  define deflateCopy           vgz_deflateCopy
+#  define deflateEnd            vgz_deflateEnd
+#  define deflateInit2_         vgz_deflateInit2_
+#  define deflateInit_          vgz_deflateInit_
+#  define deflateParams         vgz_deflateParams
+#  define deflatePrime          vgz_deflatePrime
+#  define deflateReset          vgz_deflateReset
+#  define deflateSetDictionary  vgz_deflateSetDictionary
+#  define deflateSetHeader      vgz_deflateSetHeader
+#  define deflateTune           vgz_deflateTune
+#  define deflate_copyright     vgz_deflate_copyright
+#  define get_crc_table         vgz_get_crc_table
+#  define gz_error              vgz_gz_error
+#  define gz_intmax             vgz_gz_intmax
+#  define gz_strwinerror        vgz_gz_strwinerror
+#  define gzbuffer              vgz_gzbuffer
+#  define gzclearerr            vgz_gzclearerr
+#  define gzclose               vgz_gzclose
+#  define gzclose_r             vgz_gzclose_r
+#  define gzclose_w             vgz_gzclose_w
+#  define gzdirect              vgz_gzdirect
+#  define gzdopen               vgz_gzdopen
+#  define gzeof                 vgz_gzeof
+#  define gzerror               vgz_gzerror
+#  define gzflush               vgz_gzflush
+#  define gzgetc                vgz_gzgetc
+#  define gzgets                vgz_gzgets
+#  define gzoffset              vgz_gzoffset
+#  define gzoffset64            vgz_gzoffset64
+#  define gzopen                vgz_gzopen
+#  define gzopen64              vgz_gzopen64
+#  define gzprintf              vgz_gzprintf
+#  define gzputc                vgz_gzputc
+#  define gzputs                vgz_gzputs
+#  define gzread                vgz_gzread
+#  define gzrewind              vgz_gzrewind
+#  define gzseek                vgz_gzseek
+#  define gzseek64              vgz_gzseek64
+#  define gzsetparams           vgz_gzsetparams
+#  define gztell                vgz_gztell
+#  define gztell64              vgz_gztell64
+#  define gzungetc              vgz_gzungetc
+#  define gzwrite               vgz_gzwrite
+#  define inflate               vgz_inflate
+#  define inflateBack           vgz_inflateBack
+#  define inflateBackEnd        vgz_inflateBackEnd
+#  define inflateBackInit_      vgz_inflateBackInit_
+#  define inflateCopy           vgz_inflateCopy
+#  define inflateEnd            vgz_inflateEnd
+#  define inflateGetHeader      vgz_inflateGetHeader
+#  define inflateInit2_         vgz_inflateInit2_
+#  define inflateInit_          vgz_inflateInit_
+#  define inflateMark           vgz_inflateMark
+#  define inflatePrime          vgz_inflatePrime
+#  define inflateReset          vgz_inflateReset
+#  define inflateReset2         vgz_inflateReset2
+#  define inflateSetDictionary  vgz_inflateSetDictionary
+#  define inflateSync           vgz_inflateSync
+#  define inflateSyncPoint      vgz_inflateSyncPoint
+#  define inflateUndermine      vgz_inflateUndermine
+#  define inflate_copyright     vgz_inflate_copyright
+#  define inflate_fast          vgz_inflate_fast
+#  define inflate_table         vgz_inflate_table
+#  define uncompress            vgz_uncompress
+#  define zError                vgz_zError
+#  define zcalloc               vgz_zcalloc
+#  define zcfree                vgz_zcfree
+#  define zlibCompileFlags      vgz_zlibCompileFlags
+#  define zlibVersion           vgz_zlibVersion
 
 /* all zlib typedefs in zlib.h and zconf.h */
-#  define Byte                  z_Byte
-#  define Bytef                 z_Bytef
-#  define alloc_func            z_alloc_func
-#  define charf                 z_charf
-#  define free_func             z_free_func
-#  define gzFile                z_gzFile
-#  define gz_header             z_gz_header
-#  define gz_headerp            z_gz_headerp
-#  define in_func               z_in_func
-#  define intf                  z_intf
-#  define out_func              z_out_func
-#  define uInt                  z_uInt
-#  define uIntf                 z_uIntf
-#  define uLong                 z_uLong
-#  define uLongf                z_uLongf
-#  define voidp                 z_voidp
-#  define voidpc                z_voidpc
-#  define voidpf                z_voidpf
+#  define Byte                  vgz_Byte
+#  define Bytef                 vgz_Bytef
+#  define alloc_func            vgz_alloc_func
+#  define charf                 vgz_charf
+#  define free_func             vgz_free_func
+#  define gzFile                vgz_gzFile
+#  define gz_header             vgz_gz_header
+#  define gz_headerp            vgz_gz_headerp
+#  define in_func               vgz_in_func
+#  define intf                  vgz_intf
+#  define out_func              vgz_out_func
+#  define uInt                  vgz_uInt
+#  define uIntf                 vgz_uIntf
+#  define uLong                 vgz_uLong
+#  define uLongf                vgz_uLongf
+#  define voidp                 vgz_voidp
+#  define voidpc                vgz_voidpc
+#  define voidpf                vgz_voidpf
 
 /* all zlib structs in zlib.h and zconf.h */
-#  define gz_header_s           z_gz_header_s
-#  define internal_state        z_internal_state
+#  define gz_header_s           vgz_gz_header_s
+#  define internal_state        vgz_internal_state
 
 #endif
 




From phk at varnish-cache.org  Mon Jan  3 14:56:17 2011
From: phk at varnish-cache.org (phk at varnish-cache.org)
Date: Mon,  3 Jan 2011 15:56:17 +0100
Subject: r5676 - in trunk/varnish-cache/bin/varnishtest: . tests
Message-ID: <E1PZlpZ-0000fg-Ka@project.varnish-software.com>

Author: phk
Date: 2011-01-03 15:56:17 +0100 (Mon, 03 Jan 2011)
New Revision: 5676

Added:
   trunk/varnish-cache/bin/varnishtest/tests/a00011.vtc
Modified:
   trunk/varnish-cache/bin/varnishtest/Makefile.am
   trunk/varnish-cache/bin/varnishtest/flint.sh
   trunk/varnish-cache/bin/varnishtest/vtc.c
   trunk/varnish-cache/bin/varnishtest/vtc.h
   trunk/varnish-cache/bin/varnishtest/vtc_http.c
   trunk/varnish-cache/bin/varnishtest/vtc_log.c
   trunk/varnish-cache/bin/varnishtest/vtc_varnish.c
Log:
First cut at gzip support in varnishtest.



Modified: trunk/varnish-cache/bin/varnishtest/Makefile.am
===================================================================
--- trunk/varnish-cache/bin/varnishtest/Makefile.am	2011-01-03 11:08:19 UTC (rev 5675)
+++ trunk/varnish-cache/bin/varnishtest/Makefile.am	2011-01-03 14:56:17 UTC (rev 5676)
@@ -9,7 +9,7 @@
 
 DISTCLEANFILES = _.ok
 
-INCLUDES = -I$(top_srcdir)/include
+INCLUDES = -I$(top_srcdir)/include -I$(top_srcdir)/lib/libvgz
 
 bin_PROGRAMS =	varnishtest
 
@@ -30,6 +30,7 @@
 		$(top_builddir)/lib/libvarnish/libvarnish.la \
 		$(top_builddir)/lib/libvarnishcompat/libvarnishcompat.la \
 		$(top_builddir)/lib/libvarnishapi/libvarnishapi.la \
+		$(top_builddir)/lib/libvgz/libvgz.la \
 		${LIBM} ${PTHREAD_LIBS}
 
 varnishtest_CFLAGS = \

Modified: trunk/varnish-cache/bin/varnishtest/flint.sh
===================================================================
--- trunk/varnish-cache/bin/varnishtest/flint.sh	2011-01-03 11:08:19 UTC (rev 5675)
+++ trunk/varnish-cache/bin/varnishtest/flint.sh	2011-01-03 14:56:17 UTC (rev 5676)
@@ -5,6 +5,7 @@
 	-I/usr/include \
 	-I. \
 	-I../../include \
+	-I../../lib/libvgz \
 	-I../.. \
 	../flint.lnt \
 	flint.lnt \

Added: trunk/varnish-cache/bin/varnishtest/tests/a00011.vtc
===================================================================
--- trunk/varnish-cache/bin/varnishtest/tests/a00011.vtc	                        (rev 0)
+++ trunk/varnish-cache/bin/varnishtest/tests/a00011.vtc	2011-01-03 14:56:17 UTC (rev 5676)
@@ -0,0 +1,17 @@
+# $Id$
+
+test "test vtc gzip support"
+
+server s1 {
+	rxreq
+	txresp -gzipbody FOO
+} -start
+
+client c1 -connect ${s1_sock} {
+	txreq 
+	rxresp
+	expect resp.bodylen == "26"
+	gunzip
+	expect resp.bodylen == "3"
+	expect resp.http.content-encoding == "gzip"
+} -run

Modified: trunk/varnish-cache/bin/varnishtest/vtc.c
===================================================================
--- trunk/varnish-cache/bin/varnishtest/vtc.c	2011-01-03 11:08:19 UTC (rev 5675)
+++ trunk/varnish-cache/bin/varnishtest/vtc.c	2011-01-03 14:56:17 UTC (rev 5676)
@@ -355,7 +355,7 @@
 		return;
 	AN(av[1]);
 	AZ(av[2]);
-	vtc_dump(vl, 4, "shell", av[1]);
+	vtc_dump(vl, 4, "shell", av[1], -1);
 	r = system(av[1]);
 	assert(WEXITSTATUS(r) == 0);
 }

Modified: trunk/varnish-cache/bin/varnishtest/vtc.h
===================================================================
--- trunk/varnish-cache/bin/varnishtest/vtc.h	2011-01-03 11:08:19 UTC (rev 5675)
+++ trunk/varnish-cache/bin/varnishtest/vtc.h	2011-01-03 14:56:17 UTC (rev 5676)
@@ -73,7 +73,7 @@
 void vtc_logclose(struct vtclog *vl);
 void vtc_log(struct vtclog *vl, unsigned lvl, const char *fmt, ...);
 void vtc_dump(struct vtclog *vl, unsigned lvl, const char *pfx,
-    const char *str);
+    const char *str, int len);
 
 int exec_file(const char *fn, const char *script, const char *tmpdir,
     char *logbuf, unsigned loglen);

Modified: trunk/varnish-cache/bin/varnishtest/vtc_http.c
===================================================================
--- trunk/varnish-cache/bin/varnishtest/vtc_http.c	2011-01-03 11:08:19 UTC (rev 5675)
+++ trunk/varnish-cache/bin/varnishtest/vtc_http.c	2011-01-03 14:56:17 UTC (rev 5676)
@@ -49,6 +49,8 @@
 
 #include "vtc.h"
 
+#include "zlib.h"
+
 #define MAX_HDR		50
 
 struct http {
@@ -65,6 +67,7 @@
 	char			*rxbuf;
 	int			prxbuf;
 	char			*body;
+	unsigned		bodyl;
 	char			bodylen[20];
 
 	char			*req[MAX_HDR];
@@ -135,7 +138,7 @@
 
 	vsb_finish(hp->vsb);
 	AZ(vsb_overflowed(hp->vsb));
-	vtc_dump(hp->vl, lvl, pfx, vsb_data(hp->vsb));
+	vtc_dump(hp->vl, lvl, pfx, vsb_data(hp->vsb), vsb_len(hp->vsb));
 	l = write(hp->fd, vsb_data(hp->vsb), vsb_len(hp->vsb));
 	if (l != vsb_len(hp->vsb))
 		vtc_log(hp->vl, 0, "Write failed: %s", strerror(errno));
@@ -315,7 +318,7 @@
 
 	for (n = 0; n < 3 || hh[n] != NULL; n++) {
 		sprintf(buf, "http[%2d] ", n);
-		vtc_dump(hp->vl, 4, buf, hh[n]);
+		vtc_dump(hp->vl, 4, buf, hh[n], -1);
 	}
 }
 
@@ -381,7 +384,8 @@
 		l = strtoul(p, NULL, 0);
 		hp->body = hp->rxbuf + hp->prxbuf;
 		http_rxchar(hp, l);
-		vtc_dump(hp->vl, 4, "body", hp->body);
+		vtc_dump(hp->vl, 4, "body", hp->body, l);
+		hp->bodyl = l;
 		sprintf(hp->bodylen, "%d", l);
 		return;
 	}
@@ -393,7 +397,7 @@
 			do
 				http_rxchar(hp, 1);
 			while (hp->rxbuf[hp->prxbuf - 1] != '\n');
-			vtc_dump(hp->vl, 4, "len", hp->rxbuf + l);
+			vtc_dump(hp->vl, 4, "len", hp->rxbuf + l, -1);
 			i = strtoul(hp->rxbuf + l, &q, 16);
 			assert(q != hp->rxbuf + l);
 			assert(*q == '\0' || vct_islws(*q));
@@ -401,7 +405,7 @@
 			if (i > 0) {
 				ll += i;
 				http_rxchar(hp, i);
-				vtc_dump(hp->vl, 4, "chunk", hp->rxbuf + l);
+				vtc_dump(hp->vl, 4, "chunk", hp->rxbuf + l, -1);
 			}
 			l = hp->prxbuf;
 			http_rxchar(hp, 2);
@@ -412,7 +416,8 @@
 			if (i == 0)
 				break;
 		}
-		vtc_dump(hp->vl, 4, "body", hp->body);
+		vtc_dump(hp->vl, 4, "body", hp->body, ll);
+		hp->bodyl = ll;
 		sprintf(hp->bodylen, "%d", ll);
 		return;
 	}
@@ -422,8 +427,9 @@
 			i = http_rxchar_eof(hp, 1);
 			ll += i;
 		} while (i > 0);
-		vtc_dump(hp->vl, 4, "rxeof", hp->body);
+		vtc_dump(hp->vl, 4, "rxeof", hp->body, ll);
 	}
+	hp->bodyl = ll;
 	sprintf(hp->bodylen, "%d", ll);
 }
 
@@ -454,7 +460,7 @@
 		if (i == 2)
 			break;
 	}
-	vtc_dump(hp->vl, 4, "rxhdr", hp->rxbuf);
+	vtc_dump(hp->vl, 4, "rxhdr", hp->rxbuf, -1);
 }
 
 
@@ -493,6 +499,82 @@
 }
 
 /**********************************************************************
+ * Ungzip rx'ed body
+ */
+
+#define TRUST_ME(ptr)   ((void*)(uintptr_t)(ptr))
+
+#define OVERHEAD 31
+
+
+static void
+cmd_http_gunzip_body(CMD_ARGS)
+{
+	int i;
+	z_stream vz;
+	struct http *hp;
+	char *p;
+	unsigned l;
+
+	(void)cmd;
+	(void)vl;
+	CAST_OBJ_NOTNULL(hp, priv, HTTP_MAGIC);
+	ONLY_CLIENT(hp, av);
+
+	memset(&vz, 0, sizeof vz);
+
+	vz.next_in = TRUST_ME(hp->body);
+	vz.avail_in = hp->bodyl;
+
+	l = hp->bodyl;
+	p = calloc(l, 1);
+	AN(p);
+
+	vz.next_out = TRUST_ME(p);
+	vz.avail_out = l;
+
+	assert(Z_OK == inflateInit2(&vz, 31));
+	i = inflate(&vz, Z_FINISH);
+	if (i != Z_STREAM_END)
+		vtc_log(hp->vl, 0, "Gunzip error = %d", i);
+	hp->bodyl = vz.total_out;
+	memcpy(hp->body, p, hp->bodyl);
+	free(p);
+	vtc_dump(hp->vl, 4, "body", hp->body, hp->bodyl);
+	bprintf(hp->bodylen, "%u", hp->bodyl);
+	assert(Z_OK == inflateEnd(&vz));
+}
+
+/**********************************************************************
+ * Create a gzip'ed body
+ */
+
+static void
+gzip_body(const char *txt, char **body, int *bodylen)
+{
+	int l;
+	z_stream vz;
+
+	memset(&vz, 0, sizeof vz);
+
+	l = strlen(txt);
+	*body = calloc(l + OVERHEAD, 1);
+	AN(*body);
+
+	vz.next_in = TRUST_ME(txt);
+	vz.avail_in = l;
+
+	vz.next_out = TRUST_ME(*body);
+	vz.avail_out = l + OVERHEAD;
+
+	assert(Z_OK == deflateInit2(&vz,
+	    Z_NO_COMPRESSION, Z_DEFLATED, 31, 9, Z_DEFAULT_STRATEGY));
+	assert(Z_STREAM_END == deflate(&vz, Z_FINISH));
+	*bodylen = vz.total_out;
+	assert(Z_OK == deflateEnd(&vz));
+}
+
+/**********************************************************************
  * Transmit a response
  */
 
@@ -571,6 +653,11 @@
 			body = synth_body(av[1]);
 			bodylen = strlen(body);
 			av++;
+		} else if (!strcmp(*av, "-gzipbody")) {
+			assert(body == nullbody);
+			gzip_body(av[1], &body, &bodylen);
+			vsb_printf(hp->vsb, "Content-Encoding: gzip%s", nl);
+			av++;
 		} else
 			break;
 	}
@@ -726,7 +813,7 @@
 	CAST_OBJ_NOTNULL(hp, priv, HTTP_MAGIC);
 	AN(av[1]);
 	AZ(av[2]);
-	vtc_dump(hp->vl, 4, "send", av[1]);
+	vtc_dump(hp->vl, 4, "send", av[1], -1);
 	i = write(hp->fd, av[1], strlen(av[1]));
 	assert(i == strlen(av[1]));
 
@@ -900,6 +987,7 @@
 
 	{ "txresp",		cmd_http_txresp },
 	{ "rxresp",		cmd_http_rxresp },
+	{ "gunzip",		cmd_http_gunzip_body },
 	{ "expect",		cmd_http_expect },
 	{ "send",		cmd_http_send },
 	{ "chunked",		cmd_http_chunked },

Modified: trunk/varnish-cache/bin/varnishtest/vtc_log.c
===================================================================
--- trunk/varnish-cache/bin/varnishtest/vtc_log.c	2011-01-03 11:08:19 UTC (rev 5675)
+++ trunk/varnish-cache/bin/varnishtest/vtc_log.c	2011-01-03 14:56:17 UTC (rev 5676)
@@ -155,7 +155,7 @@
 
 //lint -e{818}
 void
-vtc_dump(struct vtclog *vl, unsigned lvl, const char *pfx, const char *str)
+vtc_dump(struct vtclog *vl, unsigned lvl, const char *pfx, const char *str, int len)
 {
 	int nl = 1;
 	unsigned l;
@@ -170,9 +170,10 @@
 		vsb_printf(vl->vsb, "%s %-4s %s(null)\n",
 		    lead[lvl], vl->id, pfx);
 	else {
-		l = 0;
-		for(; *str != '\0'; str++) {
-			if (++l > 512) {
+		if (len == -1)
+			len = strlen(str);
+		for (l = 0; l < len; l++, str++) {
+			if (l > 512) {
 				vsb_printf(vl->vsb, "...");
 				break;
 			}
@@ -189,7 +190,7 @@
 				vsb_printf(vl->vsb, "\\n\n");
 				nl = 1;
 			} else if (*str < 0x20 || *str > 0x7e)
-				vsb_printf(vl->vsb, "\\x%02x", *str);
+				vsb_printf(vl->vsb, "\\x%02x", (*str) & 0xff);
 			else
 				vsb_printf(vl->vsb, "%c", *str);
 		}

Modified: trunk/varnish-cache/bin/varnishtest/vtc_varnish.c
===================================================================
--- trunk/varnish-cache/bin/varnishtest/vtc_varnish.c	2011-01-03 11:08:19 UTC (rev 5675)
+++ trunk/varnish-cache/bin/varnishtest/vtc_varnish.c	2011-01-03 14:56:17 UTC (rev 5676)
@@ -96,7 +96,7 @@
 	char *r;
 
 	if (cmd != NULL) {
-		vtc_dump(v->vl, 4, "CLI TX", cmd);
+		vtc_dump(v->vl, 4, "CLI TX", cmd, -1);
 		i = write(v->cli_fd, cmd, strlen(cmd));
 		assert(i == strlen(cmd));
 		i = write(v->cli_fd, "\n", 1);
@@ -110,7 +110,7 @@
 	}
 	assert(i == 0);
 	vtc_log(v->vl, 3, "CLI RX  %u", retval);
-	vtc_dump(v->vl, 4, "CLI RX", r);
+	vtc_dump(v->vl, 4, "CLI RX", r, -1);
 	if (repl != NULL)
 		*repl = r;
 	else
@@ -221,7 +221,7 @@
 		if (i <= 0)
 			break;
 		buf[i] = '\0';
-		vtc_dump(v->vl1, 3, "debug", buf);
+		vtc_dump(v->vl1, 3, "debug", buf, -1);
 	}
 	return (NULL);
 }




From phk at varnish-cache.org  Tue Jan  4 09:13:07 2011
From: phk at varnish-cache.org (phk at varnish-cache.org)
Date: Tue,  4 Jan 2011 10:13:07 +0100
Subject: r5677 - trunk/varnish-cache/bin/varnishd
Message-ID: <E1Pa2x1-0007ze-O5@project.varnish-software.com>

Author: phk
Date: 2011-01-04 10:13:07 +0100 (Tue, 04 Jan 2011)
New Revision: 5677

Added:
   trunk/varnish-cache/bin/varnishd/cache_gzip.c
Modified:
   trunk/varnish-cache/bin/varnishd/Makefile.am
Log:
Add a sourcefile to contain the libz namespace pollution



Modified: trunk/varnish-cache/bin/varnishd/Makefile.am
===================================================================
--- trunk/varnish-cache/bin/varnishd/Makefile.am	2011-01-03 14:56:17 UTC (rev 5676)
+++ trunk/varnish-cache/bin/varnishd/Makefile.am	2011-01-04 09:13:07 UTC (rev 5677)
@@ -25,6 +25,7 @@
 	cache_esi.c \
 	cache_expire.c \
 	cache_fetch.c \
+	cache_gzip.c \
 	cache_hash.c \
 	cache_http.c \
 	cache_httpconn.c \

Added: trunk/varnish-cache/bin/varnishd/cache_gzip.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_gzip.c	                        (rev 0)
+++ trunk/varnish-cache/bin/varnishd/cache_gzip.c	2011-01-04 09:13:07 UTC (rev 5677)
@@ -0,0 +1,42 @@
+/*-
+ * Copyright (c) 2011 Varnish Software AS
+ * All rights reserved.
+ *
+ * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * Interaction with the linvgz (libz) library.
+ *
+ * The libz library pollutes namespace a LOT when you include the "zlib.h"
+ * file so we contain the damage by vectoring all access to libz through
+ * this source file.
+ *
+ * The API defined by this file, will also insulate the rest of the code,
+ * should we find a better gzip library at a later date.
+ */
+
+#include "config.h"
+
+#include "svnid.h"
+SVNID("$Id$")
+


Property changes on: trunk/varnish-cache/bin/varnishd/cache_gzip.c
___________________________________________________________________
Added: svn:keywords
   + Id




From phk at varnish-cache.org  Tue Jan  4 10:49:09 2011
From: phk at varnish-cache.org (phk at varnish-cache.org)
Date: Tue,  4 Jan 2011 11:49:09 +0100
Subject: r5678 - trunk/varnish-cache/bin/varnishd
Message-ID: <E1Pa4Rx-0001yD-Bf@project.varnish-software.com>

Author: phk
Date: 2011-01-04 11:49:09 +0100 (Tue, 04 Jan 2011)
New Revision: 5678

Modified:
   trunk/varnish-cache/bin/varnishd/Makefile.am
Log:
Link against libvgz



Modified: trunk/varnish-cache/bin/varnishd/Makefile.am
===================================================================
--- trunk/varnish-cache/bin/varnishd/Makefile.am	2011-01-04 09:13:07 UTC (rev 5677)
+++ trunk/varnish-cache/bin/varnishd/Makefile.am	2011-01-04 10:49:09 UTC (rev 5678)
@@ -1,6 +1,9 @@
 # $Id$
 
-INCLUDES = -I$(top_srcdir)/include -I$(top_builddir)/include
+INCLUDES = \
+	-I$(top_srcdir)/include \
+	-I$(top_srcdir)/lib/libvgz \
+	-I$(top_builddir)/include
 
 sbin_PROGRAMS = varnishd
 
@@ -94,6 +97,7 @@
 	$(top_builddir)/lib/libvarnish/libvarnish.la \
 	$(top_builddir)/lib/libvarnishcompat/libvarnishcompat.la \
 	$(top_builddir)/lib/libvcl/libvcl.la \
+	$(top_builddir)/lib/libvgz/libvgz.la \
 	@JEMALLOC_LDADD@ \
 	${DL_LIBS} ${PTHREAD_LIBS} ${NET_LIBS} ${LIBM} ${LIBUMEM}
 




From phk at varnish-cache.org  Tue Jan  4 10:50:54 2011
From: phk at varnish-cache.org (phk at varnish-cache.org)
Date: Tue,  4 Jan 2011 11:50:54 +0100
Subject: r5679 - trunk/varnish-cache/bin/varnishd
Message-ID: <E1Pa4Te-00026e-Uw@project.varnish-software.com>

Author: phk
Date: 2011-01-04 11:50:53 +0100 (Tue, 04 Jan 2011)
New Revision: 5679

Modified:
   trunk/varnish-cache/bin/varnishd/cache.h
   trunk/varnish-cache/bin/varnishd/cache_gzip.c
Log:
A first order vgz API



Modified: trunk/varnish-cache/bin/varnishd/cache.h
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache.h	2011-01-04 10:49:09 UTC (rev 5678)
+++ trunk/varnish-cache/bin/varnishd/cache.h	2011-01-04 10:50:53 UTC (rev 5679)
@@ -584,6 +584,14 @@
 int FetchReqBody(struct sess *sp);
 void Fetch_Init(void);
 
+/* cache_gzip.c */
+struct vgz;
+
+struct vgz *VGZ_NewUnzip(struct sess *sp, struct ws *tmp, struct ws *buf);
+int VGZ_Feed(struct vgz *, const void *, size_t len);
+int VGZ_Produce(struct vgz *, const void **, size_t *len);
+int VGZ_Destroy(struct vgz **);
+
 /* cache_http.c */
 unsigned HTTP_estimate(unsigned nhttp);
 void HTTP_Copy(struct http *to, const struct http * const fm);

Modified: trunk/varnish-cache/bin/varnishd/cache_gzip.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_gzip.c	2011-01-04 10:49:09 UTC (rev 5678)
+++ trunk/varnish-cache/bin/varnishd/cache_gzip.c	2011-01-04 10:50:53 UTC (rev 5679)
@@ -25,7 +25,7 @@
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
  *
- * Interaction with the linvgz (libz) library.
+ * Interaction with the linvgz (zlib) library.
  *
  * The libz library pollutes namespace a LOT when you include the "zlib.h"
  * file so we contain the damage by vectoring all access to libz through
@@ -40,3 +40,132 @@
 #include "svnid.h"
 SVNID("$Id$")
 
+// #include <sys/types.h>
+// #include <sys/time.h>
+
+#include <stdio.h>
+// #include <stdlib.h>
+
+#include "cache.h"
+
+#include "zlib.h"
+
+struct vgz {
+	unsigned		magic;
+#define VGZ_MAGIC		0x162df0cb
+	struct sess 		*sp;
+	struct ws		*tmp;
+	char			*tmp_snapshot;
+
+	struct ws		*buf;
+	unsigned		bufsiz;
+
+	z_stream		vz;
+};
+
+/*--------------------------------------------------------------------*/
+
+static voidpf
+vgz_alloc(voidpf opaque, uInt items, uInt size)
+{
+	struct vgz *vg;
+
+	CAST_OBJ_NOTNULL(vg, opaque, VGZ_MAGIC);
+
+	return(WS_Alloc(vg->tmp, items * size));
+}
+
+static void
+vgz_free(voidpf opaque, voidpf address)
+{
+	struct vgz *vg;
+
+	CAST_OBJ_NOTNULL(vg, opaque, VGZ_MAGIC);
+	(void)address;
+}
+
+/*--------------------------------------------------------------------*/
+
+struct vgz *
+VGZ_NewUnzip(struct sess *sp, struct ws *tmp, struct ws *buf)
+{
+	struct vgz *vg;
+	char *s;
+
+	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+
+	WS_Assert(tmp);
+	WS_Assert(buf);
+
+	s = WS_Snapshot(tmp);
+	vg = (void*)WS_Alloc(tmp, sizeof *vg);
+	AN(vg);
+	memset(vg, 0, sizeof *vg);
+	vg->magic = VGZ_MAGIC;
+	vg->sp = sp;
+	vg->tmp = tmp;
+	vg->buf = buf;
+	vg->tmp_snapshot = s;
+
+	vg->vz.zalloc = vgz_alloc;
+	vg->vz.zfree = vgz_free;
+	vg->vz.opaque = vg;
+
+	vg->bufsiz = WS_Reserve(buf, 0);
+
+	assert(Z_OK == inflateInit2(&vg->vz, 31));
+	return (vg);
+}
+
+/*--------------------------------------------------------------------*/
+
+int
+VGZ_Feed(struct vgz *vg, const void *ptr, size_t len)
+{
+
+	CHECK_OBJ_NOTNULL(vg, VGZ_MAGIC);
+
+	vg->vz.next_in = TRUST_ME(ptr);
+	vg->vz.avail_in = len;
+
+	return (0);
+}
+
+/*--------------------------------------------------------------------*/
+
+int
+VGZ_Produce(struct vgz *vg, const void **pptr, size_t *plen)
+{
+	int i;
+
+	CHECK_OBJ_NOTNULL(vg, VGZ_MAGIC);
+
+	*pptr = NULL;
+	*plen = 0;
+	vg->vz.next_out = (void*)vg->buf->f;
+	vg->vz.avail_out = vg->bufsiz;
+
+	i = inflate(&vg->vz, 0);
+	if (i == Z_OK || i == Z_STREAM_END) {
+		*pptr = vg->buf->f;
+		*plen = vg->bufsiz - vg->vz.avail_out;
+	}
+	if (i == Z_OK)
+		return (0);
+	if (i == Z_STREAM_END)
+		return (1);
+	return (-1);
+}
+
+/*--------------------------------------------------------------------*/
+
+int
+VGZ_Destroy(struct vgz **vg)
+{
+
+	CHECK_OBJ_NOTNULL(*vg, VGZ_MAGIC);
+	WS_Release((*vg)->buf, 0);
+	WS_Reset((*vg)->tmp, (*vg)->tmp_snapshot);
+	*vg = NULL;
+	return (0);
+}




From phk at varnish-cache.org  Tue Jan  4 10:51:58 2011
From: phk at varnish-cache.org (phk at varnish-cache.org)
Date: Tue,  4 Jan 2011 11:51:58 +0100
Subject: r5680 - in trunk/varnish-cache/bin: varnishd varnishtest/tests
Message-ID: <E1Pa4Ug-00029w-Ar@project.varnish-software.com>

Author: phk
Date: 2011-01-04 11:51:57 +0100 (Tue, 04 Jan 2011)
New Revision: 5680

Added:
   trunk/varnish-cache/bin/varnishtest/tests/g00001.vtc
Modified:
   trunk/varnish-cache/bin/varnishd/cache.h
   trunk/varnish-cache/bin/varnishd/cache_center.c
   trunk/varnish-cache/bin/varnishd/cache_response.c
Log:
Beat a path through, where we can successfully ungzip an trivial object,
when the client does not support gzip.



Modified: trunk/varnish-cache/bin/varnishd/cache.h
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache.h	2011-01-04 10:50:53 UTC (rev 5679)
+++ trunk/varnish-cache/bin/varnishd/cache.h	2011-01-04 10:51:57 UTC (rev 5680)
@@ -741,6 +741,7 @@
 /* cache_response.c */
 void RES_BuildHttp(struct sess *sp);
 void RES_WriteObj(struct sess *sp);
+void RES_WriteGunzipObj(struct sess *sp);
 
 /* cache_vary.c */
 struct vsb *VRY_Create(const struct sess *sp, const struct http *hp);

Modified: trunk/varnish-cache/bin/varnishd/cache_center.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_center.c	2011-01-04 10:50:53 UTC (rev 5679)
+++ trunk/varnish-cache/bin/varnishd/cache_center.c	2011-01-04 10:51:57 UTC (rev 5680)
@@ -210,7 +210,14 @@
 	sp->director = NULL;
 	sp->restarts = 0;
 
-	RES_WriteObj(sp);
+	if (params->http_gzip_support &&
+	    http_HdrIs(sp->wrk->resp, H_Content_Encoding, "gzip") &&
+	    !RFC2616_Req_Gzip(sp) &&
+	    sp->wantbody)
+		RES_WriteGunzipObj(sp);
+	else 
+		RES_WriteObj(sp);
+
 	AZ(sp->wrk->wfd);
 	(void)HSH_Deref(sp->wrk, NULL, &sp->obj);
 	sp->wrk->resp = NULL;
@@ -879,6 +886,15 @@
 	http_Setup(sp->wrk->bereq, sp->wrk->ws);
 	http_FilterHeader(sp, HTTPH_R_FETCH);
 	http_ForceGet(sp->wrk->bereq);
+	if (params->http_gzip_support) {
+		/*
+		 * We always ask the backend for gzip, even if the
+		 * client doesn't grok it.  We will uncompress for
+		 * the minority of clients which don't.
+		 */
+		http_PrintfHeader(sp->wrk, sp->fd, sp->wrk->bereq,
+		    "Accept-Encoding: gzip");
+	}
 	sp->wrk->connect_timeout = 0;
 	sp->wrk->first_byte_timeout = 0;
 	sp->wrk->between_bytes_timeout = 0;

Modified: trunk/varnish-cache/bin/varnishd/cache_response.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_response.c	2011-01-04 10:50:53 UTC (rev 5679)
+++ trunk/varnish-cache/bin/varnishd/cache_response.c	2011-01-04 10:51:57 UTC (rev 5680)
@@ -246,7 +246,6 @@
 	unsigned low, high, ptr, off, len;
 
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
-	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 
 	WRW_Reserve(sp->wrk, &sp->fd);
 
@@ -274,7 +273,7 @@
 	high = sp->obj->len - 1;
 
 	if (sp->disable_esi || sp->esis == 0) {
-		/* For none ESI and non ESI-included objects, try Range */
+		/* For non-ESI and non ESI-included objects, try Range */
 		if (params->http_range_support &&
 		    (sp->disable_esi || sp->esis == 0) &&
 		    sp->obj->response == 200 &&
@@ -358,3 +357,109 @@
 	if (WRW_FlushRelease(sp->wrk))
 		vca_close_session(sp, "remote closed");
 }
+
+/*--------------------------------------------------------------------
+ * We have a gzip'ed object and need to ungzip it for a client which
+ * does not understand gzip.
+ */
+
+void
+RES_WriteGunzipObj(struct sess *sp)
+{
+	struct storage *st;
+	unsigned u = 0;
+#if 0
+	char lenbuf[20];
+#endif
+	struct vgz *vg;
+	const void *dp;
+	size_t dl;
+	int i;
+
+	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+	AN(sp->wantbody);
+
+	WRW_Reserve(sp->wrk, &sp->fd);
+
+	/* We don't know the length  (XXX: Cache once we do ?) */
+	http_Unset(sp->wrk->resp, H_Content_Length);
+	http_Unset(sp->wrk->resp, H_Content_Encoding);
+	http_Unset(sp->wrk->resp, H_Connection);
+
+	http_PrintfHeader(sp->wrk, sp->fd, sp->wrk->resp, "Connection: %s",
+	    "close");
+	sp->doclose = "Gunzip EOF";
+
+	/*
+	 * ESI objects get special delivery
+	 */
+	if (!sp->disable_esi && sp->obj->esidata != NULL) {
+		INCOMPL();
+#if 0
+		if (sp->esis == 0)
+			/* no headers for interior ESI includes */
+			sp->acct_tmp.hdrbytes +=
+			    http_Write(sp->wrk, sp->wrk->resp, 1);
+
+		if (WRW_FlushRelease(sp->wrk)) {
+			vca_close_session(sp, "remote closed");
+		} else 
+			ESI_Deliver(sp);
+		return;
+#endif
+	}
+
+	if (sp->disable_esi || sp->esis == 0) {
+		sp->acct_tmp.hdrbytes += http_Write(sp->wrk, sp->wrk->resp, 1);
+	} else if (!sp->disable_esi &&
+	    sp->esis > 0 &&
+	    sp->http->protover >= 1.1 &&
+	    sp->obj->len > 0) {
+		INCOMPL();
+#if 0
+		/*
+		 * Interior ESI includes (which are not themselves ESI
+		 * objects) use chunked encoding (here) or EOF (nothing)
+		 */
+		sprintf(lenbuf, "%x\r\n", sp->obj->len);
+		(void)WRW_Write(sp->wrk, lenbuf, -1);
+#endif
+	}
+
+	vg = VGZ_NewUnzip(sp, sp->ws, sp->wrk->ws);
+	AN(vg);
+
+	VTAILQ_FOREACH(st, &sp->obj->store, list) {
+		CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+		CHECK_OBJ_NOTNULL(st, STORAGE_MAGIC);
+		u += st->len;
+
+		sp->acct_tmp.bodybytes += st->len;
+		VSC_main->n_objwrite++;
+
+		VGZ_Feed(vg, st->ptr, st->len);
+		do {
+			i = VGZ_Produce(vg, &dp, &dl);
+			if (dl != 0) {
+				(void)WRW_Write(sp->wrk, dp, dl);
+				if (WRW_Flush(sp->wrk))
+					break;
+			}
+		} while (i == 0);
+	}
+	VGZ_Destroy(&vg);
+	assert(u == sp->obj->len);
+	if (!sp->disable_esi &&
+	    sp->esis > 0 &&
+	    sp->http->protover >= 1.1 &&
+	    sp->obj->len > 0) {
+		INCOMPL();
+#if 0
+		/* post-chunk new line */
+		(void)WRW_Write(sp->wrk, "\r\n", -1);
+#endif
+	}
+
+	if (WRW_FlushRelease(sp->wrk))
+		vca_close_session(sp, "remote closed");
+}

Added: trunk/varnish-cache/bin/varnishtest/tests/g00001.vtc
===================================================================
--- trunk/varnish-cache/bin/varnishtest/tests/g00001.vtc	                        (rev 0)
+++ trunk/varnish-cache/bin/varnishtest/tests/g00001.vtc	2011-01-04 10:51:57 UTC (rev 5680)
@@ -0,0 +1,27 @@
+# $Id$
+
+test "test basic gunzip for client"
+
+server s1 {
+	rxreq
+	expect req.http.accept-encoding == "gzip"
+	txresp -gzipbody FOO
+} -start
+
+varnish v1 -cliok "param.set http_gzip_support true" -vcl+backend {
+} -start
+
+client c1 {
+	txreq 
+	rxresp
+	expect resp.bodylen == "3"
+	expect resp.http.content-encoding == "resp.http.content-encoding"
+} -run
+
+client c1 {
+	txreq -hdr "Accept-encoding: gzip;q=0.1"
+	rxresp
+	expect resp.http.content-encoding == "gzip"
+	gunzip
+	expect resp.bodylen == "3"
+} -run


Property changes on: trunk/varnish-cache/bin/varnishtest/tests/g00001.vtc
___________________________________________________________________
Added: svn:keywords
   + Id




From phk at varnish-cache.org  Tue Jan  4 14:44:22 2011
From: phk at varnish-cache.org (phk at varnish-cache.org)
Date: Tue,  4 Jan 2011 15:44:22 +0100
Subject: r5681 - trunk/varnish-cache/bin/varnishd
Message-ID: <E1Pa87a-0008PK-4T@project.varnish-software.com>

Author: phk
Date: 2011-01-04 15:44:21 +0100 (Tue, 04 Jan 2011)
New Revision: 5681

Modified:
   trunk/varnish-cache/bin/varnishd/cache_gzip.c
Log:
Remove unnecessary #include



Modified: trunk/varnish-cache/bin/varnishd/cache_gzip.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_gzip.c	2011-01-04 10:51:57 UTC (rev 5680)
+++ trunk/varnish-cache/bin/varnishd/cache_gzip.c	2011-01-04 14:44:21 UTC (rev 5681)
@@ -40,12 +40,6 @@
 #include "svnid.h"
 SVNID("$Id$")
 
-// #include <sys/types.h>
-// #include <sys/time.h>
-
-#include <stdio.h>
-// #include <stdlib.h>
-
 #include "cache.h"
 
 #include "zlib.h"




From phk at varnish-cache.org  Tue Jan  4 14:56:05 2011
From: phk at varnish-cache.org (phk at varnish-cache.org)
Date: Tue,  4 Jan 2011 15:56:05 +0100
Subject: r5682 - trunk/varnish-cache/lib/libvgz
Message-ID: <E1Pa8Iv-0000KI-B5@project.varnish-software.com>

Author: phk
Date: 2011-01-04 15:56:04 +0100 (Tue, 04 Jan 2011)
New Revision: 5682

Modified:
   trunk/varnish-cache/lib/libvgz/zconf.h
Log:
We cannot use the zconf/PREFIX stuff to hide symbols, the #defines
clash with the version hacks near the end of zlib.h

Will revisit once I know which subset of this library I actually need,
it will probably be possible to remove the entire gz* family functions.



Modified: trunk/varnish-cache/lib/libvgz/zconf.h
===================================================================
--- trunk/varnish-cache/lib/libvgz/zconf.h	2011-01-04 14:44:21 UTC (rev 5681)
+++ trunk/varnish-cache/lib/libvgz/zconf.h	2011-01-04 14:56:04 UTC (rev 5682)
@@ -16,120 +16,120 @@
  * Even better than compiling with -DZ_PREFIX would be to use configure to set
  * this permanently in zconf.h using "./configure --zprefix".
  */
-#if 1
+#ifdef Z_PREFIX     /* may be set to #if 1 by ./configure */
 
 /* all linked symbols */
-#  define _dist_code            vgz__dist_code
-#  define _length_code          vgz__length_code
-#  define _tr_align             vgz__tr_align
-#  define _tr_flush_block       vgz__tr_flush_block
-#  define _tr_init              vgz__tr_init
-#  define _tr_stored_block      vgz__tr_stored_block
-#  define _tr_tally             vgz__tr_tally
-#  define adler32               vgz_adler32
-#  define adler32_combine       vgz_adler32_combine
-#  define adler32_combine64     vgz_adler32_combine64
-#  define compress              vgz_compress
-#  define compress2             vgz_compress2
-#  define compressBound         vgz_compressBound
-#  define crc32                 vgz_crc32
-#  define crc32_combine         vgz_crc32_combine
-#  define crc32_combine64       vgz_crc32_combine64
-#  define deflate               vgz_deflate
-#  define deflateBound          vgz_deflateBound
-#  define deflateCopy           vgz_deflateCopy
-#  define deflateEnd            vgz_deflateEnd
-#  define deflateInit2_         vgz_deflateInit2_
-#  define deflateInit_          vgz_deflateInit_
-#  define deflateParams         vgz_deflateParams
-#  define deflatePrime          vgz_deflatePrime
-#  define deflateReset          vgz_deflateReset
-#  define deflateSetDictionary  vgz_deflateSetDictionary
-#  define deflateSetHeader      vgz_deflateSetHeader
-#  define deflateTune           vgz_deflateTune
-#  define deflate_copyright     vgz_deflate_copyright
-#  define get_crc_table         vgz_get_crc_table
-#  define gz_error              vgz_gz_error
-#  define gz_intmax             vgz_gz_intmax
-#  define gz_strwinerror        vgz_gz_strwinerror
-#  define gzbuffer              vgz_gzbuffer
-#  define gzclearerr            vgz_gzclearerr
-#  define gzclose               vgz_gzclose
-#  define gzclose_r             vgz_gzclose_r
-#  define gzclose_w             vgz_gzclose_w
-#  define gzdirect              vgz_gzdirect
-#  define gzdopen               vgz_gzdopen
-#  define gzeof                 vgz_gzeof
-#  define gzerror               vgz_gzerror
-#  define gzflush               vgz_gzflush
-#  define gzgetc                vgz_gzgetc
-#  define gzgets                vgz_gzgets
-#  define gzoffset              vgz_gzoffset
-#  define gzoffset64            vgz_gzoffset64
-#  define gzopen                vgz_gzopen
-#  define gzopen64              vgz_gzopen64
-#  define gzprintf              vgz_gzprintf
-#  define gzputc                vgz_gzputc
-#  define gzputs                vgz_gzputs
-#  define gzread                vgz_gzread
-#  define gzrewind              vgz_gzrewind
-#  define gzseek                vgz_gzseek
-#  define gzseek64              vgz_gzseek64
-#  define gzsetparams           vgz_gzsetparams
-#  define gztell                vgz_gztell
-#  define gztell64              vgz_gztell64
-#  define gzungetc              vgz_gzungetc
-#  define gzwrite               vgz_gzwrite
-#  define inflate               vgz_inflate
-#  define inflateBack           vgz_inflateBack
-#  define inflateBackEnd        vgz_inflateBackEnd
-#  define inflateBackInit_      vgz_inflateBackInit_
-#  define inflateCopy           vgz_inflateCopy
-#  define inflateEnd            vgz_inflateEnd
-#  define inflateGetHeader      vgz_inflateGetHeader
-#  define inflateInit2_         vgz_inflateInit2_
-#  define inflateInit_          vgz_inflateInit_
-#  define inflateMark           vgz_inflateMark
-#  define inflatePrime          vgz_inflatePrime
-#  define inflateReset          vgz_inflateReset
-#  define inflateReset2         vgz_inflateReset2
-#  define inflateSetDictionary  vgz_inflateSetDictionary
-#  define inflateSync           vgz_inflateSync
-#  define inflateSyncPoint      vgz_inflateSyncPoint
-#  define inflateUndermine      vgz_inflateUndermine
-#  define inflate_copyright     vgz_inflate_copyright
-#  define inflate_fast          vgz_inflate_fast
-#  define inflate_table         vgz_inflate_table
-#  define uncompress            vgz_uncompress
-#  define zError                vgz_zError
-#  define zcalloc               vgz_zcalloc
-#  define zcfree                vgz_zcfree
-#  define zlibCompileFlags      vgz_zlibCompileFlags
-#  define zlibVersion           vgz_zlibVersion
+#  define _dist_code            z__dist_code
+#  define _length_code          z__length_code
+#  define _tr_align             z__tr_align
+#  define _tr_flush_block       z__tr_flush_block
+#  define _tr_init              z__tr_init
+#  define _tr_stored_block      z__tr_stored_block
+#  define _tr_tally             z__tr_tally
+#  define adler32               z_adler32
+#  define adler32_combine       z_adler32_combine
+#  define adler32_combine64     z_adler32_combine64
+#  define compress              z_compress
+#  define compress2             z_compress2
+#  define compressBound         z_compressBound
+#  define crc32                 z_crc32
+#  define crc32_combine         z_crc32_combine
+#  define crc32_combine64       z_crc32_combine64
+#  define deflate               z_deflate
+#  define deflateBound          z_deflateBound
+#  define deflateCopy           z_deflateCopy
+#  define deflateEnd            z_deflateEnd
+#  define deflateInit2_         z_deflateInit2_
+#  define deflateInit_          z_deflateInit_
+#  define deflateParams         z_deflateParams
+#  define deflatePrime          z_deflatePrime
+#  define deflateReset          z_deflateReset
+#  define deflateSetDictionary  z_deflateSetDictionary
+#  define deflateSetHeader      z_deflateSetHeader
+#  define deflateTune           z_deflateTune
+#  define deflate_copyright     z_deflate_copyright
+#  define get_crc_table         z_get_crc_table
+#  define gz_error              z_gz_error
+#  define gz_intmax             z_gz_intmax
+#  define gz_strwinerror        z_gz_strwinerror
+#  define gzbuffer              z_gzbuffer
+#  define gzclearerr            z_gzclearerr
+#  define gzclose               z_gzclose
+#  define gzclose_r             z_gzclose_r
+#  define gzclose_w             z_gzclose_w
+#  define gzdirect              z_gzdirect
+#  define gzdopen               z_gzdopen
+#  define gzeof                 z_gzeof
+#  define gzerror               z_gzerror
+#  define gzflush               z_gzflush
+#  define gzgetc                z_gzgetc
+#  define gzgets                z_gzgets
+#  define gzoffset              z_gzoffset
+#  define gzoffset64            z_gzoffset64
+#  define gzopen                z_gzopen
+#  define gzopen64              z_gzopen64
+#  define gzprintf              z_gzprintf
+#  define gzputc                z_gzputc
+#  define gzputs                z_gzputs
+#  define gzread                z_gzread
+#  define gzrewind              z_gzrewind
+#  define gzseek                z_gzseek
+#  define gzseek64              z_gzseek64
+#  define gzsetparams           z_gzsetparams
+#  define gztell                z_gztell
+#  define gztell64              z_gztell64
+#  define gzungetc              z_gzungetc
+#  define gzwrite               z_gzwrite
+#  define inflate               z_inflate
+#  define inflateBack           z_inflateBack
+#  define inflateBackEnd        z_inflateBackEnd
+#  define inflateBackInit_      z_inflateBackInit_
+#  define inflateCopy           z_inflateCopy
+#  define inflateEnd            z_inflateEnd
+#  define inflateGetHeader      z_inflateGetHeader
+#  define inflateInit2_         z_inflateInit2_
+#  define inflateInit_          z_inflateInit_
+#  define inflateMark           z_inflateMark
+#  define inflatePrime          z_inflatePrime
+#  define inflateReset          z_inflateReset
+#  define inflateReset2         z_inflateReset2
+#  define inflateSetDictionary  z_inflateSetDictionary
+#  define inflateSync           z_inflateSync
+#  define inflateSyncPoint      z_inflateSyncPoint
+#  define inflateUndermine      z_inflateUndermine
+#  define inflate_copyright     z_inflate_copyright
+#  define inflate_fast          z_inflate_fast
+#  define inflate_table         z_inflate_table
+#  define uncompress            z_uncompress
+#  define zError                z_zError
+#  define zcalloc               z_zcalloc
+#  define zcfree                z_zcfree
+#  define zlibCompileFlags      z_zlibCompileFlags
+#  define zlibVersion           z_zlibVersion
 
 /* all zlib typedefs in zlib.h and zconf.h */
-#  define Byte                  vgz_Byte
-#  define Bytef                 vgz_Bytef
-#  define alloc_func            vgz_alloc_func
-#  define charf                 vgz_charf
-#  define free_func             vgz_free_func
-#  define gzFile                vgz_gzFile
-#  define gz_header             vgz_gz_header
-#  define gz_headerp            vgz_gz_headerp
-#  define in_func               vgz_in_func
-#  define intf                  vgz_intf
-#  define out_func              vgz_out_func
-#  define uInt                  vgz_uInt
-#  define uIntf                 vgz_uIntf
-#  define uLong                 vgz_uLong
-#  define uLongf                vgz_uLongf
-#  define voidp                 vgz_voidp
-#  define voidpc                vgz_voidpc
-#  define voidpf                vgz_voidpf
+#  define Byte                  z_Byte
+#  define Bytef                 z_Bytef
+#  define alloc_func            z_alloc_func
+#  define charf                 z_charf
+#  define free_func             z_free_func
+#  define gzFile                z_gzFile
+#  define gz_header             z_gz_header
+#  define gz_headerp            z_gz_headerp
+#  define in_func               z_in_func
+#  define intf                  z_intf
+#  define out_func              z_out_func
+#  define uInt                  z_uInt
+#  define uIntf                 z_uIntf
+#  define uLong                 z_uLong
+#  define uLongf                z_uLongf
+#  define voidp                 z_voidp
+#  define voidpc                z_voidpc
+#  define voidpf                z_voidpf
 
 /* all zlib structs in zlib.h and zconf.h */
-#  define gz_header_s           vgz_gz_header_s
-#  define internal_state        vgz_internal_state
+#  define gz_header_s           z_gz_header_s
+#  define internal_state        z_internal_state
 
 #endif
 




From phk at varnish-cache.org  Tue Jan  4 15:44:40 2011
From: phk at varnish-cache.org (phk at varnish-cache.org)
Date: Tue,  4 Jan 2011 16:44:40 +0100
Subject: r5683 - trunk/varnish-cache/lib/libvgz
Message-ID: <E1Pa93x-0001Y7-2k@project.varnish-software.com>

Author: phk
Date: 2011-01-04 16:44:40 +0100 (Tue, 04 Jan 2011)
New Revision: 5683

Modified:
   trunk/varnish-cache/lib/libvgz/zlib.h
Log:
Try to work around Linux FILE_OFFSET_64 brain damage.



Modified: trunk/varnish-cache/lib/libvgz/zlib.h
===================================================================
--- trunk/varnish-cache/lib/libvgz/zlib.h	2011-01-04 14:56:04 UTC (rev 5682)
+++ trunk/varnish-cache/lib/libvgz/zlib.h	2011-01-04 15:44:40 UTC (rev 5683)
@@ -1569,9 +1569,7 @@
    ZEXTERN z_off64_t ZEXPORT gzoffset64 OF((gzFile));
    ZEXTERN uLong ZEXPORT adler32_combine64 OF((uLong, uLong, z_off64_t));
    ZEXTERN uLong ZEXPORT crc32_combine64 OF((uLong, uLong, z_off64_t));
-#endif
-
-#if !defined(ZLIB_INTERNAL) && _FILE_OFFSET_BITS-0 == 64 && _LFS64_LARGEFILE-0
+#elif !defined(ZLIB_INTERNAL) && _FILE_OFFSET_BITS-0 == 64 && _LFS64_LARGEFILE-0
 #  define gzopen gzopen64
 #  define gzseek gzseek64
 #  define gztell gztell64




From phk at varnish-cache.org  Wed Jan  5 10:26:24 2011
From: phk at varnish-cache.org (phk at varnish-cache.org)
Date: Wed,  5 Jan 2011 11:26:24 +0100
Subject: r5684 - trunk/varnish-cache/bin/varnishd
Message-ID: <E1PaQZV-00054m-0K@project.varnish-software.com>

Author: phk
Date: 2011-01-05 11:26:24 +0100 (Wed, 05 Jan 2011)
New Revision: 5684

Modified:
   trunk/varnish-cache/bin/varnishd/cache_center.c
Log:
Remove XXX comments OBE



Modified: trunk/varnish-cache/bin/varnishd/cache_center.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_center.c	2011-01-04 15:44:40 UTC (rev 5683)
+++ trunk/varnish-cache/bin/varnishd/cache_center.c	2011-01-05 10:26:24 UTC (rev 5684)
@@ -156,17 +156,6 @@
 DOT }
 DOT deliver2 -> DONE [style=bold,color=green,weight=4]
  *
- * XXX: Ideally we should make the req. available in vcl_deliver() but for
- * XXX: reasons of economy we don't, since that allows us to reuse the space
- * XXX: in sp->req for the response.
- *
- * XXX: Rather than allocate two http's and workspaces for all sessions to
- * XXX: address this deficiency, we could make the VCL compiler set a flag
- * XXX: if req. is used in vcl_deliver().  When the flag is set we would
- * XXX: take the memory overhead, for instance by borrowing a struct bereq
- * XXX: or similar.
- *
- * XXX: For now, wait until somebody asks for it.
  */
 
 static int




From phk at varnish-cache.org  Wed Jan  5 11:13:30 2011
From: phk at varnish-cache.org (phk at varnish-cache.org)
Date: Wed,  5 Jan 2011 12:13:30 +0100
Subject: r5685 - trunk/varnish-cache/bin/varnishd
Message-ID: <E1PaRJ4-00069v-Co@project.varnish-software.com>

Author: phk
Date: 2011-01-05 12:13:30 +0100 (Wed, 05 Jan 2011)
New Revision: 5685

Modified:
   trunk/varnish-cache/bin/varnishd/cache.h
   trunk/varnish-cache/bin/varnishd/cache_center.c
   trunk/varnish-cache/bin/varnishd/cache_response.c
Log:
Centralize the "how do we deliver this ?" magic into a
sp->wrk variable where everybody can find it.



Modified: trunk/varnish-cache/bin/varnishd/cache.h
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache.h	2011-01-05 10:26:24 UTC (rev 5684)
+++ trunk/varnish-cache/bin/varnishd/cache.h	2011-01-05 11:13:30 UTC (rev 5685)
@@ -265,6 +265,15 @@
 	double			first_byte_timeout;
 	double			between_bytes_timeout;
 
+	/* Delivery mode */
+	unsigned		res_mode;
+#define RES_LEN			(1<<1)
+#define RES_EOF			(1<<2)
+#define RES_CHUNKED		(1<<3)
+#define RES_ESI			(1<<4)
+#define RES_ESI_CHILD		(1<<5)
+#define RES_GUNZIP		(1<<6)
+
 };
 
 /* Work Request for worker thread ------------------------------------*/

Modified: trunk/varnish-cache/bin/varnishd/cache_center.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_center.c	2011-01-05 10:26:24 UTC (rev 5684)
+++ trunk/varnish-cache/bin/varnishd/cache_center.c	2011-01-05 11:13:30 UTC (rev 5685)
@@ -166,6 +166,35 @@
 	CHECK_OBJ_NOTNULL(sp->obj, OBJECT_MAGIC);
 	CHECK_OBJ_NOTNULL(sp->vcl, VCL_CONF_MAGIC);
 
+	sp->wrk->res_mode = RES_LEN;
+
+	if (!sp->disable_esi && sp->obj->esidata != NULL) {
+		/* In ESI mode, we don't know the aggregate length */
+		sp->wrk->res_mode &= ~RES_LEN;
+	}
+
+	if (params->http_gzip_support &&
+	    http_HdrIs(sp->obj->http, H_Content_Encoding, "gzip") &&
+	    !RFC2616_Req_Gzip(sp) &&
+	    sp->wantbody) {
+		/*
+		 * We don't know what it uncompresses to
+		 * XXX: we could cache that
+		 */
+		sp->wrk->res_mode &= ~RES_LEN;
+		sp->wrk->res_mode |= RES_EOF;		/* XXX */
+		sp->wrk->res_mode |= RES_GUNZIP;
+	}
+
+	if (!(sp->wrk->res_mode & (RES_LEN|RES_CHUNKED|RES_EOF))) {
+		if(sp->http->protover >= 1.1) {
+			sp->wrk->res_mode |= RES_CHUNKED;
+		} else {
+			sp->wrk->res_mode |= RES_EOF;
+			sp->doclose = "EOF mode";
+		}
+	}
+
 	sp->t_resp = TIM_real();
 	if (sp->obj->objcore != NULL) {
 		if ((sp->t_resp - sp->obj->last_lru) > params->lru_timeout)
@@ -199,13 +228,7 @@
 	sp->director = NULL;
 	sp->restarts = 0;
 
-	if (params->http_gzip_support &&
-	    http_HdrIs(sp->wrk->resp, H_Content_Encoding, "gzip") &&
-	    !RFC2616_Req_Gzip(sp) &&
-	    sp->wantbody)
-		RES_WriteGunzipObj(sp);
-	else 
-		RES_WriteObj(sp);
+	RES_WriteObj(sp);
 
 	AZ(sp->wrk->wfd);
 	(void)HSH_Deref(sp->wrk, NULL, &sp->obj);

Modified: trunk/varnish-cache/bin/varnishd/cache_response.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_response.c	2011-01-05 10:26:24 UTC (rev 5684)
+++ trunk/varnish-cache/bin/varnishd/cache_response.c	2011-01-05 11:13:30 UTC (rev 5685)
@@ -205,18 +205,17 @@
 	http_FilterFields(sp->wrk, sp->fd, sp->wrk->resp, sp->obj->http,
 	    HTTPH_A_DELIVER);
 
-	/* Only HTTP 1.1 can do Chunked encoding */
-	if (!sp->disable_esi && sp->obj->esidata != NULL) {
+	if (!(sp->wrk->res_mode & RES_LEN)) {
 		http_Unset(sp->wrk->resp, H_Content_Length);
-		if(sp->http->protover >= 1.1)
-			http_PrintfHeader(sp->wrk, sp->fd, sp->wrk->resp,
-			    "Transfer-Encoding: chunked");
-		else
-			sp->doclose = "ESI EOF";
-	} else if (params->http_range_support)
+	} else if (params->http_range_support) {
+		/* We only accept ranges if we know the length */
 		http_SetHeader(sp->wrk, sp->fd, sp->wrk->resp,
 		    "Accept-Ranges: bytes");
+	}
 
+	if (sp->wrk->res_mode & RES_CHUNKED)
+		http_PrintfHeader(sp->wrk, sp->fd, sp->wrk->resp,
+		    "Transfer-Encoding: chunked");
 
 	TIM_format(TIM_real(), time_str);
 	http_PrintfHeader(sp->wrk, sp->fd, sp->wrk->resp, "Date: %s", time_str);
@@ -247,6 +246,11 @@
 
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 
+	if (sp->wrk->res_mode & RES_GUNZIP) {
+		RES_WriteGunzipObj(sp);
+		return;
+	}
+
 	WRW_Reserve(sp->wrk, &sp->fd);
 
 	/*




From phk at varnish-cache.org  Wed Jan  5 12:10:13 2011
From: phk at varnish-cache.org (phk at varnish-cache.org)
Date: Wed,  5 Jan 2011 13:10:13 +0100
Subject: r5686 - trunk/varnish-cache/bin/varnishd
Message-ID: <E1PaSBx-0008TD-Mx@project.varnish-software.com>

Author: phk
Date: 2011-01-05 13:10:11 +0100 (Wed, 05 Jan 2011)
New Revision: 5686

Modified:
   trunk/varnish-cache/bin/varnishd/cache_center.c
   trunk/varnish-cache/bin/varnishd/cache_response.c
Log:
Use sp->wrk->res_mode more.



Modified: trunk/varnish-cache/bin/varnishd/cache_center.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_center.c	2011-01-05 11:13:30 UTC (rev 5685)
+++ trunk/varnish-cache/bin/varnishd/cache_center.c	2011-01-05 12:10:11 UTC (rev 5686)
@@ -171,8 +171,14 @@
 	if (!sp->disable_esi && sp->obj->esidata != NULL) {
 		/* In ESI mode, we don't know the aggregate length */
 		sp->wrk->res_mode &= ~RES_LEN;
+		sp->wrk->res_mode |= RES_ESI;
 	}
 
+	if (sp->esis > 0) {
+		sp->wrk->res_mode &= ~RES_LEN;
+		sp->wrk->res_mode |= RES_ESI_CHILD;
+	}
+
 	if (params->http_gzip_support &&
 	    http_HdrIs(sp->obj->http, H_Content_Encoding, "gzip") &&
 	    !RFC2616_Req_Gzip(sp) &&

Modified: trunk/varnish-cache/bin/varnishd/cache_response.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_response.c	2011-01-05 11:13:30 UTC (rev 5685)
+++ trunk/varnish-cache/bin/varnishd/cache_response.c	2011-01-05 12:10:11 UTC (rev 5686)
@@ -246,19 +246,13 @@
 
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 
-	if (sp->wrk->res_mode & RES_GUNZIP) {
-		RES_WriteGunzipObj(sp);
-		return;
-	}
-
 	WRW_Reserve(sp->wrk, &sp->fd);
 
 	/*
 	 * ESI objects get special delivery
 	 */
-	if (!sp->disable_esi && sp->obj->esidata != NULL) {
-
-		if (sp->esis == 0)
+	if (sp->wrk->res_mode & RES_ESI) {
+		if (!(sp->wrk->res_mode & RES_ESI_CHILD))
 			/* no headers for interior ESI includes */
 			sp->acct_tmp.hdrbytes +=
 			    http_Write(sp->wrk, sp->wrk->resp, 1);
@@ -270,13 +264,18 @@
 		return;
 	}
 
+	if (sp->wrk->res_mode & RES_GUNZIP) {
+		RES_WriteGunzipObj(sp);
+		return;
+	}
+
 	/*
 	 * How much of the object we want to deliver
 	 */
 	low = 0;
 	high = sp->obj->len - 1;
 
-	if (sp->disable_esi || sp->esis == 0) {
+	if (!(sp->wrk->res_mode & (RES_ESI|RES_ESI_CHILD))) {
 		/* For non-ESI and non ESI-included objects, try Range */
 		if (params->http_range_support &&
 		    (sp->disable_esi || sp->esis == 0) &&
@@ -286,14 +285,7 @@
 			res_dorange(sp, r, &low, &high);
 
 		sp->acct_tmp.hdrbytes += http_Write(sp->wrk, sp->wrk->resp, 1);
-	} else if (!sp->disable_esi &&
-	    sp->esis > 0 &&
-	    sp->http->protover >= 1.1 &&
-	    sp->obj->len > 0) {
-		/*
-		 * Interior ESI includes (which are not themselves ESI
-		 * objects) use chunked encoding (here) or EOF (nothing)
-		 */
+	} else if (sp->obj->len > 0 && (sp->wrk->res_mode & RES_CHUNKED)) {
 		assert(sp->wantbody);
 		sprintf(lenbuf, "%x\r\n", sp->obj->len);
 		(void)WRW_Write(sp->wrk, lenbuf, -1);
@@ -383,8 +375,6 @@
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 	AN(sp->wantbody);
 
-	WRW_Reserve(sp->wrk, &sp->fd);
-
 	/* We don't know the length  (XXX: Cache once we do ?) */
 	http_Unset(sp->wrk->resp, H_Content_Length);
 	http_Unset(sp->wrk->resp, H_Content_Encoding);




From phk at varnish-cache.org  Wed Jan  5 12:33:59 2011
From: phk at varnish-cache.org (phk at varnish-cache.org)
Date: Wed,  5 Jan 2011 13:33:59 +0100
Subject: r5687 - trunk/varnish-cache/bin/varnishtest
Message-ID: <E1PaSYx-0000kh-ON@project.varnish-software.com>

Author: phk
Date: 2011-01-05 13:33:58 +0100 (Wed, 05 Jan 2011)
New Revision: 5687

Modified:
   trunk/varnish-cache/bin/varnishtest/vtc_http.c
Log:
Don't assert, report chunk len trouble the usual way.



Modified: trunk/varnish-cache/bin/varnishtest/vtc_http.c
===================================================================
--- trunk/varnish-cache/bin/varnishtest/vtc_http.c	2011-01-05 12:10:11 UTC (rev 5686)
+++ trunk/varnish-cache/bin/varnishtest/vtc_http.c	2011-01-05 12:33:58 UTC (rev 5687)
@@ -399,13 +399,19 @@
 			while (hp->rxbuf[hp->prxbuf - 1] != '\n');
 			vtc_dump(hp->vl, 4, "len", hp->rxbuf + l, -1);
 			i = strtoul(hp->rxbuf + l, &q, 16);
+			if ((q == hp->rxbuf + l) ||
+				(*q != '\0' && !vct_islws(*q))) {
+				vtc_log(hp->vl, 0, "chunked fail %02x @ %d",
+				    *q, q - (hp->rxbuf + l));
+			}
 			assert(q != hp->rxbuf + l);
 			assert(*q == '\0' || vct_islws(*q));
 			hp->prxbuf = l;
 			if (i > 0) {
 				ll += i;
 				http_rxchar(hp, i);
-				vtc_dump(hp->vl, 4, "chunk", hp->rxbuf + l, -1);
+				vtc_dump(hp->vl, 4, "chunk",
+				    hp->rxbuf + l, -1);
 			}
 			l = hp->prxbuf;
 			http_rxchar(hp, 2);




From phk at varnish-cache.org  Wed Jan  5 13:07:50 2011
From: phk at varnish-cache.org (phk at varnish-cache.org)
Date: Wed,  5 Jan 2011 14:07:50 +0100
Subject: r5688 - trunk/varnish-cache/bin/varnishd
Message-ID: <E1PaT5j-0004U9-0X@project.varnish-software.com>

Author: phk
Date: 2011-01-05 14:07:48 +0100 (Wed, 05 Jan 2011)
New Revision: 5688

Modified:
   trunk/varnish-cache/bin/varnishd/cache_center.c
   trunk/varnish-cache/bin/varnishd/cache_esi.c
   trunk/varnish-cache/bin/varnishd/cache_response.c
Log:
Further unravel og unify the response path.



Modified: trunk/varnish-cache/bin/varnishd/cache_center.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_center.c	2011-01-05 12:33:58 UTC (rev 5687)
+++ trunk/varnish-cache/bin/varnishd/cache_center.c	2011-01-05 13:07:48 UTC (rev 5688)
@@ -190,6 +190,7 @@
 		sp->wrk->res_mode &= ~RES_LEN;
 		sp->wrk->res_mode |= RES_EOF;		/* XXX */
 		sp->wrk->res_mode |= RES_GUNZIP;
+		sp->doclose = "gunzip EOF";
 	}
 
 	if (!(sp->wrk->res_mode & (RES_LEN|RES_CHUNKED|RES_EOF))) {

Modified: trunk/varnish-cache/bin/varnishd/cache_esi.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_esi.c	2011-01-05 12:33:58 UTC (rev 5687)
+++ trunk/varnish-cache/bin/varnishd/cache_esi.c	2011-01-05 13:07:48 UTC (rev 5688)
@@ -867,7 +867,7 @@
 	CHECK_OBJ_NOTNULL(ed, ESIDATA_MAGIC);
 	VTAILQ_FOREACH(eb, &ed->esibits, list) {
 		if (Tlen(eb->verbatim)) {
-			if (sp->http->protover >= 1.1)
+			if (sp->wrk->res_mode & RES_CHUNKED)
 				(void)WRW_Write(w, eb->chunk_length, -1);
 			sp->acct_tmp.bodybytes += WRW_Write(w,
 			    eb->verbatim.b, Tlen(eb->verbatim));

Modified: trunk/varnish-cache/bin/varnishd/cache_response.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_response.c	2011-01-05 12:33:58 UTC (rev 5687)
+++ trunk/varnish-cache/bin/varnishd/cache_response.c	2011-01-05 13:07:48 UTC (rev 5688)
@@ -178,11 +178,11 @@
 	http_PrintfHeader(sp->wrk, sp->fd, sp->wrk->resp,
 	    "Content-Range: bytes %u-%u/%u", low, high, sp->obj->len);
 	http_Unset(sp->wrk->resp, H_Content_Length);
+	assert(sp->wrk->res_mode & RES_LEN);
 	http_PrintfHeader(sp->wrk, sp->fd, sp->wrk->resp,
 	    "Content-Length: %u", 1 + high - low);
 	http_SetResp(sp->wrk->resp, "HTTP/1.1", 206, "Partial Content");
 
-
 	*plow = low;
 	*phigh = high;
 }
@@ -249,14 +249,33 @@
 	WRW_Reserve(sp->wrk, &sp->fd);
 
 	/*
+	 * If nothing special planned, we can attempt Range support
+	 */
+	low = 0;
+	high = sp->obj->len - 1;
+	if (!(sp->wrk->res_mode & (RES_ESI|RES_ESI_CHILD|RES_GUNZIP)) &&
+	    params->http_range_support && sp->obj->response == 200 &&
+	    sp->wantbody && http_GetHdr(sp->http, H_Range, &r))
+		res_dorange(sp, r, &low, &high);
+
+	if (sp->wrk->res_mode & RES_GUNZIP) {
+		assert(sp->wrk->res_mode & RES_EOF);
+		assert(!(sp->wrk->res_mode & (RES_ESI|RES_ESI_CHILD)));
+		AN(sp->doclose);
+		http_Unset(sp->wrk->resp, H_Content_Encoding);
+	}
+
+	/*
+	 * no headers for interior ESI includes 
+	 */
+	if (!(sp->wrk->res_mode & RES_ESI_CHILD))
+		sp->acct_tmp.hdrbytes +=
+		    http_Write(sp->wrk, sp->wrk->resp, 1);
+
+	/*
 	 * ESI objects get special delivery
 	 */
 	if (sp->wrk->res_mode & RES_ESI) {
-		if (!(sp->wrk->res_mode & RES_ESI_CHILD))
-			/* no headers for interior ESI includes */
-			sp->acct_tmp.hdrbytes +=
-			    http_Write(sp->wrk, sp->wrk->resp, 1);
-
 		if (WRW_FlushRelease(sp->wrk)) {
 			vca_close_session(sp, "remote closed");
 		} else if (sp->wantbody)
@@ -264,41 +283,32 @@
 		return;
 	}
 
+	if (!sp->wantbody) {
+		/* This was a HEAD request */
+		assert(sp->esis == 0);
+		if (WRW_FlushRelease(sp->wrk))
+			vca_close_session(sp, "remote closed");
+		return;
+	}
+
+	if (sp->obj->len == 0) {
+		if (WRW_FlushRelease(sp->wrk))
+			vca_close_session(sp, "remote closed");
+		return;
+	}
+
 	if (sp->wrk->res_mode & RES_GUNZIP) {
 		RES_WriteGunzipObj(sp);
 		return;
 	}
 
-	/*
-	 * How much of the object we want to deliver
-	 */
-	low = 0;
-	high = sp->obj->len - 1;
 
-	if (!(sp->wrk->res_mode & (RES_ESI|RES_ESI_CHILD))) {
-		/* For non-ESI and non ESI-included objects, try Range */
-		if (params->http_range_support &&
-		    (sp->disable_esi || sp->esis == 0) &&
-		    sp->obj->response == 200 &&
-		    sp->wantbody &&
-		    http_GetHdr(sp->http, H_Range, &r))
-			res_dorange(sp, r, &low, &high);
-
-		sp->acct_tmp.hdrbytes += http_Write(sp->wrk, sp->wrk->resp, 1);
-	} else if (sp->obj->len > 0 && (sp->wrk->res_mode & RES_CHUNKED)) {
+	if (sp->wrk->res_mode & RES_CHUNKED) {
 		assert(sp->wantbody);
 		sprintf(lenbuf, "%x\r\n", sp->obj->len);
 		(void)WRW_Write(sp->wrk, lenbuf, -1);
 	}
 
-	if (!sp->wantbody) {
-		/* This was a HEAD request */
-		assert(sp->esis == 0);
-		if (WRW_FlushRelease(sp->wrk))
-			vca_close_session(sp, "remote closed");
-		return;
-	}
-
 	ptr = 0;
 	VTAILQ_FOREACH(st, &sp->obj->store, list) {
 		CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
@@ -342,13 +352,9 @@
 		(void)WRW_Write(sp->wrk, st->ptr + off, len);
 	}
 	assert(u == sp->obj->len);
-	if (!sp->disable_esi &&
-	    sp->esis > 0 &&
-	    sp->http->protover >= 1.1 &&
-	    sp->obj->len > 0) {
+	if ((sp->wrk->res_mode & RES_CHUNKED))
 		/* post-chunk new line */
 		(void)WRW_Write(sp->wrk, "\r\n", -1);
-	}
 
 	if (WRW_FlushRelease(sp->wrk))
 		vca_close_session(sp, "remote closed");
@@ -364,9 +370,6 @@
 {
 	struct storage *st;
 	unsigned u = 0;
-#if 0
-	char lenbuf[20];
-#endif
 	struct vgz *vg;
 	const void *dp;
 	size_t dl;
@@ -375,51 +378,6 @@
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 	AN(sp->wantbody);
 
-	/* We don't know the length  (XXX: Cache once we do ?) */
-	http_Unset(sp->wrk->resp, H_Content_Length);
-	http_Unset(sp->wrk->resp, H_Content_Encoding);
-	http_Unset(sp->wrk->resp, H_Connection);
-
-	http_PrintfHeader(sp->wrk, sp->fd, sp->wrk->resp, "Connection: %s",
-	    "close");
-	sp->doclose = "Gunzip EOF";
-
-	/*
-	 * ESI objects get special delivery
-	 */
-	if (!sp->disable_esi && sp->obj->esidata != NULL) {
-		INCOMPL();
-#if 0
-		if (sp->esis == 0)
-			/* no headers for interior ESI includes */
-			sp->acct_tmp.hdrbytes +=
-			    http_Write(sp->wrk, sp->wrk->resp, 1);
-
-		if (WRW_FlushRelease(sp->wrk)) {
-			vca_close_session(sp, "remote closed");
-		} else 
-			ESI_Deliver(sp);
-		return;
-#endif
-	}
-
-	if (sp->disable_esi || sp->esis == 0) {
-		sp->acct_tmp.hdrbytes += http_Write(sp->wrk, sp->wrk->resp, 1);
-	} else if (!sp->disable_esi &&
-	    sp->esis > 0 &&
-	    sp->http->protover >= 1.1 &&
-	    sp->obj->len > 0) {
-		INCOMPL();
-#if 0
-		/*
-		 * Interior ESI includes (which are not themselves ESI
-		 * objects) use chunked encoding (here) or EOF (nothing)
-		 */
-		sprintf(lenbuf, "%x\r\n", sp->obj->len);
-		(void)WRW_Write(sp->wrk, lenbuf, -1);
-#endif
-	}
-
 	vg = VGZ_NewUnzip(sp, sp->ws, sp->wrk->ws);
 	AN(vg);
 
@@ -443,16 +401,6 @@
 	}
 	VGZ_Destroy(&vg);
 	assert(u == sp->obj->len);
-	if (!sp->disable_esi &&
-	    sp->esis > 0 &&
-	    sp->http->protover >= 1.1 &&
-	    sp->obj->len > 0) {
-		INCOMPL();
-#if 0
-		/* post-chunk new line */
-		(void)WRW_Write(sp->wrk, "\r\n", -1);
-#endif
-	}
 
 	if (WRW_FlushRelease(sp->wrk))
 		vca_close_session(sp, "remote closed");




From phk at varnish-cache.org  Wed Jan  5 14:11:50 2011
From: phk at varnish-cache.org (phk at varnish-cache.org)
Date: Wed,  5 Jan 2011 15:11:50 +0100
Subject: r5689 - trunk/varnish-cache/bin/varnishd
Message-ID: <E1PaU5e-0006Gj-Ji@project.varnish-software.com>

Author: phk
Date: 2011-01-05 15:11:50 +0100 (Wed, 05 Jan 2011)
New Revision: 5689

Modified:
   trunk/varnish-cache/bin/varnishd/cache.h
   trunk/varnish-cache/bin/varnishd/cache_center.c
   trunk/varnish-cache/bin/varnishd/cache_esi.c
   trunk/varnish-cache/bin/varnishd/cache_response.c
Log:
Further unify delivery path so that gunzip, esi and direct object
don't replicate common code.



Modified: trunk/varnish-cache/bin/varnishd/cache.h
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache.h	2011-01-05 13:07:48 UTC (rev 5688)
+++ trunk/varnish-cache/bin/varnishd/cache.h	2011-01-05 14:11:50 UTC (rev 5689)
@@ -750,7 +750,6 @@
 /* cache_response.c */
 void RES_BuildHttp(struct sess *sp);
 void RES_WriteObj(struct sess *sp);
-void RES_WriteGunzipObj(struct sess *sp);
 
 /* cache_vary.c */
 struct vsb *VRY_Create(const struct sess *sp, const struct http *hp);

Modified: trunk/varnish-cache/bin/varnishd/cache_center.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_center.c	2011-01-05 13:07:48 UTC (rev 5688)
+++ trunk/varnish-cache/bin/varnishd/cache_center.c	2011-01-05 14:11:50 UTC (rev 5689)
@@ -181,20 +181,23 @@
 
 	if (params->http_gzip_support &&
 	    http_HdrIs(sp->obj->http, H_Content_Encoding, "gzip") &&
-	    !RFC2616_Req_Gzip(sp) &&
-	    sp->wantbody) {
+	    !RFC2616_Req_Gzip(sp)) {
 		/*
 		 * We don't know what it uncompresses to
 		 * XXX: we could cache that
 		 */
 		sp->wrk->res_mode &= ~RES_LEN;
-		sp->wrk->res_mode |= RES_EOF;		/* XXX */
 		sp->wrk->res_mode |= RES_GUNZIP;
-		sp->doclose = "gunzip EOF";
 	}
 
 	if (!(sp->wrk->res_mode & (RES_LEN|RES_CHUNKED|RES_EOF))) {
-		if(sp->http->protover >= 1.1) {
+		if (sp->obj->len == 0)
+			/*
+			 * If the object is empty, neither ESI nor GUNZIP
+			 * can make it any different size
+			 */
+			sp->wrk->res_mode |= RES_LEN;
+		else if (sp->http->protover >= 1.1) {
 			sp->wrk->res_mode |= RES_CHUNKED;
 		} else {
 			sp->wrk->res_mode |= RES_EOF;

Modified: trunk/varnish-cache/bin/varnishd/cache_esi.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_esi.c	2011-01-05 13:07:48 UTC (rev 5688)
+++ trunk/varnish-cache/bin/varnishd/cache_esi.c	2011-01-05 14:11:50 UTC (rev 5689)
@@ -858,10 +858,9 @@
 	char *ws_wm;
 	struct http http_save;
 	struct esidata *ed;
-	unsigned sxid;
+	unsigned sxid, res_mode;
 
 	w = sp->wrk;
-	WRW_Reserve(w, &sp->fd);
 	http_save.magic = 0;
 	ed = sp->obj->esidata;
 	CHECK_OBJ_NOTNULL(ed, ESIDATA_MAGIC);
@@ -878,14 +877,16 @@
 		    sp->esis >= params->max_esi_includes)
 			continue;
 
-		if (WRW_FlushRelease(w)) {
+		if (WRW_Flush(w)) {
 			vca_close_session(sp, "remote closed");
 			return;
 		}
+		AZ(WRW_FlushRelease(w));
 
 		sp->esis++;
 		obj = sp->obj;
 		sp->obj = NULL;
+		res_mode = sp->wrk->res_mode;
 
 		/* Save the master objects HTTP state, we may need it later */
 		if (http_save.magic == 0)
@@ -935,6 +936,7 @@
 		assert(sp->step == STP_DONE);
 		sp->esis--;
 		sp->obj = obj;
+		sp->wrk->res_mode = res_mode;
 
 		/* Reset the workspace */
 		WS_Reset(sp->ws, ws_wm);
@@ -943,13 +945,6 @@
 		if (sp->fd < 0)
 			break;
 	}
-	/* Restore master objects HTTP state */
-	if (http_save.magic)
-		*sp->http = http_save;
-	if (sp->esis == 0 && sp->http->protover >= 1.1)
-		(void)WRW_Write(sp->wrk, "0\r\n\r\n", -1);
-	if (WRW_FlushRelease(sp->wrk))
-		vca_close_session(sp, "remote closed");
 }
 
 /*--------------------------------------------------------------------*/

Modified: trunk/varnish-cache/bin/varnishd/cache_response.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_response.c	2011-01-05 13:07:48 UTC (rev 5688)
+++ trunk/varnish-cache/bin/varnishd/cache_response.c	2011-01-05 14:11:50 UTC (rev 5689)
@@ -233,79 +233,67 @@
 	    sp->doclose ? "close" : "keep-alive");
 }
 
-/*--------------------------------------------------------------------*/
+/*--------------------------------------------------------------------
+ * We have a gzip'ed object and need to ungzip it for a client which
+ * does not understand gzip.
+ */
 
-void
-RES_WriteObj(struct sess *sp)
+static void
+res_WriteGunzipObj(struct sess *sp, char lenbuf[20])
 {
 	struct storage *st;
 	unsigned u = 0;
-	char lenbuf[20];
-	char *r;
-	unsigned low, high, ptr, off, len;
+	struct vgz *vg;
+	const void *dp;
+	size_t dl;
+	int i;
 
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 
-	WRW_Reserve(sp->wrk, &sp->fd);
+	vg = VGZ_NewUnzip(sp, sp->ws, sp->wrk->ws);
+	AN(vg);
 
-	/*
-	 * If nothing special planned, we can attempt Range support
-	 */
-	low = 0;
-	high = sp->obj->len - 1;
-	if (!(sp->wrk->res_mode & (RES_ESI|RES_ESI_CHILD|RES_GUNZIP)) &&
-	    params->http_range_support && sp->obj->response == 200 &&
-	    sp->wantbody && http_GetHdr(sp->http, H_Range, &r))
-		res_dorange(sp, r, &low, &high);
+	VTAILQ_FOREACH(st, &sp->obj->store, list) {
+		CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+		CHECK_OBJ_NOTNULL(st, STORAGE_MAGIC);
+		u += st->len;
 
-	if (sp->wrk->res_mode & RES_GUNZIP) {
-		assert(sp->wrk->res_mode & RES_EOF);
-		assert(!(sp->wrk->res_mode & (RES_ESI|RES_ESI_CHILD)));
-		AN(sp->doclose);
-		http_Unset(sp->wrk->resp, H_Content_Encoding);
-	}
+		sp->acct_tmp.bodybytes += st->len;
+		VSC_main->n_objwrite++;
 
-	/*
-	 * no headers for interior ESI includes 
-	 */
-	if (!(sp->wrk->res_mode & RES_ESI_CHILD))
-		sp->acct_tmp.hdrbytes +=
-		    http_Write(sp->wrk, sp->wrk->resp, 1);
-
-	/*
-	 * ESI objects get special delivery
-	 */
-	if (sp->wrk->res_mode & RES_ESI) {
-		if (WRW_FlushRelease(sp->wrk)) {
-			vca_close_session(sp, "remote closed");
-		} else if (sp->wantbody)
-			ESI_Deliver(sp);
-		return;
+		VGZ_Feed(vg, st->ptr, st->len);
+		do {
+			i = VGZ_Produce(vg, &dp, &dl);
+			if (dl != 0) {
+				if (sp->wrk->res_mode & RES_CHUNKED) {
+					bprintf(lenbuf, "%x\r\n", (unsigned)dl);
+					(void)WRW_Write(sp->wrk, lenbuf, -1);
+				}
+				(void)WRW_Write(sp->wrk, dp, dl);
+				if (sp->wrk->res_mode & RES_CHUNKED) 
+					(void)WRW_Write(sp->wrk, "\r\n", -1);
+				if (WRW_Flush(sp->wrk))
+					break;
+			}
+		} while (i == 0);
 	}
+	VGZ_Destroy(&vg);
+	assert(u == sp->obj->len);
+}
 
-	if (!sp->wantbody) {
-		/* This was a HEAD request */
-		assert(sp->esis == 0);
-		if (WRW_FlushRelease(sp->wrk))
-			vca_close_session(sp, "remote closed");
-		return;
-	}
+/*--------------------------------------------------------------------*/
 
-	if (sp->obj->len == 0) {
-		if (WRW_FlushRelease(sp->wrk))
-			vca_close_session(sp, "remote closed");
-		return;
-	}
+static void
+res_WriteDirObj(struct sess *sp, char lenbuf[20], size_t low, size_t high)
+{
+	unsigned u = 0;
+	size_t ptr, off, len;
+	struct storage *st;
 
-	if (sp->wrk->res_mode & RES_GUNZIP) {
-		RES_WriteGunzipObj(sp);
-		return;
-	}
+	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 
-
 	if (sp->wrk->res_mode & RES_CHUNKED) {
-		assert(sp->wantbody);
-		sprintf(lenbuf, "%x\r\n", sp->obj->len);
+		bprintf(lenbuf, "%x\r\n", sp->obj->len);
 		(void)WRW_Write(sp->wrk, lenbuf, -1);
 	}
 
@@ -352,56 +340,62 @@
 		(void)WRW_Write(sp->wrk, st->ptr + off, len);
 	}
 	assert(u == sp->obj->len);
-	if ((sp->wrk->res_mode & RES_CHUNKED))
-		/* post-chunk new line */
+	if (sp->wrk->res_mode & RES_CHUNKED) 
 		(void)WRW_Write(sp->wrk, "\r\n", -1);
-
-	if (WRW_FlushRelease(sp->wrk))
-		vca_close_session(sp, "remote closed");
 }
 
-/*--------------------------------------------------------------------
- * We have a gzip'ed object and need to ungzip it for a client which
- * does not understand gzip.
- */
+/*--------------------------------------------------------------------*/
 
 void
-RES_WriteGunzipObj(struct sess *sp)
+RES_WriteObj(struct sess *sp)
 {
-	struct storage *st;
-	unsigned u = 0;
-	struct vgz *vg;
-	const void *dp;
-	size_t dl;
-	int i;
+	char *r;
+	unsigned low, high;
+	char lenbuf[20];
 
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
-	AN(sp->wantbody);
 
-	vg = VGZ_NewUnzip(sp, sp->ws, sp->wrk->ws);
-	AN(vg);
+	WRW_Reserve(sp->wrk, &sp->fd);
 
-	VTAILQ_FOREACH(st, &sp->obj->store, list) {
-		CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
-		CHECK_OBJ_NOTNULL(st, STORAGE_MAGIC);
-		u += st->len;
+	/*
+	 * If nothing special planned, we can attempt Range support
+	 */
+	low = 0;
+	high = sp->obj->len - 1;
+	if (!(sp->wrk->res_mode & (RES_ESI|RES_ESI_CHILD|RES_GUNZIP)) &&
+	    params->http_range_support && sp->obj->response == 200 &&
+	    sp->wantbody && http_GetHdr(sp->http, H_Range, &r))
+		res_dorange(sp, r, &low, &high);
 
-		sp->acct_tmp.bodybytes += st->len;
-		VSC_main->n_objwrite++;
+	/*
+	 * Always remove C-E if client don't grok it
+	 */
+	if (sp->wrk->res_mode & RES_GUNZIP)
+		http_Unset(sp->wrk->resp, H_Content_Encoding);
 
-		VGZ_Feed(vg, st->ptr, st->len);
-		do {
-			i = VGZ_Produce(vg, &dp, &dl);
-			if (dl != 0) {
-				(void)WRW_Write(sp->wrk, dp, dl);
-				if (WRW_Flush(sp->wrk))
-					break;
-			}
-		} while (i == 0);
+	/*
+	 * Send HTTP protocol header, unless interior ESI object
+	 */
+	if (!(sp->wrk->res_mode & RES_ESI_CHILD))
+		sp->acct_tmp.hdrbytes +=
+		    http_Write(sp->wrk, sp->wrk->resp, 1);
+
+	if (!sp->wantbody) {
+		/* This was a HEAD request */
+	} else if (sp->obj->len == 0) {
+		/* Nothing to do here */
+	} else if (sp->wrk->res_mode & RES_ESI) {
+		ESI_Deliver(sp);
+	} else if (sp->wrk->res_mode & RES_GUNZIP) {
+		res_WriteGunzipObj(sp, lenbuf); 
+	} else {
+		res_WriteDirObj(sp, lenbuf, low, high);
 	}
-	VGZ_Destroy(&vg);
-	assert(u == sp->obj->len);
 
+	if (sp->wrk->res_mode & RES_CHUNKED &&
+	    !(sp->wrk->res_mode & RES_ESI_CHILD))
+		(void)WRW_Write(sp->wrk, "0\r\n\r\n", -1);
+
 	if (WRW_FlushRelease(sp->wrk))
 		vca_close_session(sp, "remote closed");
 }




From phk at varnish-cache.org  Wed Jan  5 14:27:59 2011
From: phk at varnish-cache.org (phk at varnish-cache.org)
Date: Wed,  5 Jan 2011 15:27:59 +0100
Subject: r5690 - trunk/varnish-cache/bin/varnishd
Message-ID: <E1PaULH-0006tW-4h@project.varnish-software.com>

Author: phk
Date: 2011-01-05 15:27:58 +0100 (Wed, 05 Jan 2011)
New Revision: 5690

Modified:
   trunk/varnish-cache/bin/varnishd/cache_center.c
Log:
Only set chunked mode if we actually want the body (ie: Not HEAD)



Modified: trunk/varnish-cache/bin/varnishd/cache_center.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_center.c	2011-01-05 14:11:50 UTC (rev 5689)
+++ trunk/varnish-cache/bin/varnishd/cache_center.c	2011-01-05 14:27:58 UTC (rev 5690)
@@ -197,7 +197,9 @@
 			 * can make it any different size
 			 */
 			sp->wrk->res_mode |= RES_LEN;
-		else if (sp->http->protover >= 1.1) {
+		else if (!sp->wantbody) {
+			/* Nothing */
+		} else if (sp->http->protover >= 1.1) {
 			sp->wrk->res_mode |= RES_CHUNKED;
 		} else {
 			sp->wrk->res_mode |= RES_EOF;




From phk at varnish-cache.org  Wed Jan  5 14:28:16 2011
From: phk at varnish-cache.org (phk at varnish-cache.org)
Date: Wed,  5 Jan 2011 15:28:16 +0100
Subject: r5691 - trunk/varnish-cache/bin/varnishtest/tests
Message-ID: <E1PaULY-0006w2-PA@project.varnish-software.com>

Author: phk
Date: 2011-01-05 15:28:16 +0100 (Wed, 05 Jan 2011)
New Revision: 5691

Modified:
   trunk/varnish-cache/bin/varnishtest/tests/g00001.vtc
Log:
Test now more capable gunzip feature



Modified: trunk/varnish-cache/bin/varnishtest/tests/g00001.vtc
===================================================================
--- trunk/varnish-cache/bin/varnishtest/tests/g00001.vtc	2011-01-05 14:27:58 UTC (rev 5690)
+++ trunk/varnish-cache/bin/varnishtest/tests/g00001.vtc	2011-01-05 14:28:16 UTC (rev 5691)
@@ -16,12 +16,28 @@
 	rxresp
 	expect resp.bodylen == "3"
 	expect resp.http.content-encoding == "resp.http.content-encoding"
-} -run
 
-client c1 {
 	txreq -hdr "Accept-encoding: gzip;q=0.1"
 	rxresp
 	expect resp.http.content-encoding == "gzip"
 	gunzip
 	expect resp.bodylen == "3"
 } -run
+
+client c1 {
+	txreq -proto HTTP/1.0
+	rxresp
+	expect resp.bodylen == "3"
+	expect resp.http.content-encoding == "resp.http.content-encoding"
+} -run
+
+client c1 {
+	txreq -req HEAD
+	rxresp -no_obj
+	expect resp.http.content-encoding == "resp.http.content-encoding"
+
+	txreq -req HEAD -hdr "Accept-encoding: gzip;q=0.1"
+	rxresp -no_obj
+	expect resp.http.content-length == "26"
+	expect resp.http.content-encoding == "gzip"
+} -run




From phk at varnish-cache.org  Thu Jan  6 14:55:05 2011
From: phk at varnish-cache.org (phk at varnish-cache.org)
Date: Thu,  6 Jan 2011 15:55:05 +0100
Subject: r5692 - trunk/varnish-cache/lib/libvgz
Message-ID: <E1ParF3-0003yY-M3@project.varnish-software.com>

Author: phk
Date: 2011-01-06 15:55:05 +0100 (Thu, 06 Jan 2011)
New Revision: 5692

Modified:
   trunk/varnish-cache/lib/libvgz/README
   trunk/varnish-cache/lib/libvgz/inflate.c
   trunk/varnish-cache/lib/libvgz/zlib.h
Log:
Modify libvgz's inflate() function to record for us the bit positions of
the first deflate block, the 'last' bit and the first (padding) bit after
the laste deflate block.

We need to know these in order to do the multi-evil gzip surgery for
ESI compression.



Modified: trunk/varnish-cache/lib/libvgz/README
===================================================================
--- trunk/varnish-cache/lib/libvgz/README	2011-01-05 14:28:16 UTC (rev 5691)
+++ trunk/varnish-cache/lib/libvgz/README	2011-01-06 14:55:05 UTC (rev 5692)
@@ -1,7 +1,16 @@
-This is a cut down version of the ZLIB data compression library.
+This is a modified version of the ZLIB 1.2.5 data compression library.
 
 For the original and complete version of the library please visit:
 
 	http://www.zlib.net/
 
 For copyright and licensing information, please see the zlib.h file.
+
+Modifications include:
+
+* Unnecessary files not imported.
+
+* Extension to inflate() to record the bit positions of:
+	A) The first deflate block
+	B) The 'last' bit
+	C) The first (padding) bit after the last deflate block

Modified: trunk/varnish-cache/lib/libvgz/inflate.c
===================================================================
--- trunk/varnish-cache/lib/libvgz/inflate.c	2011-01-05 14:28:16 UTC (rev 5691)
+++ trunk/varnish-cache/lib/libvgz/inflate.c	2011-01-06 14:55:05 UTC (rev 5692)
@@ -810,13 +810,18 @@
         case TYPE:
             if (flush == Z_BLOCK || flush == Z_TREES) goto inf_leave;
         case TYPEDO:
+            if (strm->start_bit == 0) 
+                strm->start_bit = 8 * (strm->total_in + in - have) - bits;
             if (state->last) {
+                strm->stop_bit = 8 * (strm->total_in + in - have) - bits;
                 BYTEBITS();
                 state->mode = CHECK;
                 break;
             }
             NEEDBITS(3);
             state->last = BITS(1);
+            if (state->last)
+                strm->last_bit = 8 * (strm->total_in + in - have) - bits;
             DROPBITS(1);
             switch (BITS(2)) {
             case 0:                             /* stored block */

Modified: trunk/varnish-cache/lib/libvgz/zlib.h
===================================================================
--- trunk/varnish-cache/lib/libvgz/zlib.h	2011-01-05 14:28:16 UTC (rev 5691)
+++ trunk/varnish-cache/lib/libvgz/zlib.h	2011-01-06 14:55:05 UTC (rev 5692)
@@ -101,6 +101,10 @@
     int     data_type;  /* best guess about the data type: binary or text */
     uLong   adler;      /* adler32 value of the uncompressed data */
     uLong   reserved;   /* reserved for future use */
+
+    uLong   start_bit;	/* Bit pos of first deflate block */
+    uLong   stop_bit;	/* Bit pos after last deflate block */
+    uLong   last_bit;	/* Bit pos of 'last' bit */
 } z_stream;
 
 typedef z_stream FAR *z_streamp;




From phk at varnish-cache.org  Thu Jan  6 14:57:11 2011
From: phk at varnish-cache.org (phk at varnish-cache.org)
Date: Thu,  6 Jan 2011 15:57:11 +0100
Subject: r5693 - trunk/varnish-cache/bin/varnishtest
Message-ID: <E1ParH5-00041X-1u@project.varnish-software.com>

Author: phk
Date: 2011-01-06 15:57:10 +0100 (Thu, 06 Jan 2011)
New Revision: 5693

Modified:
   trunk/varnish-cache/bin/varnishtest/vtc.h
   trunk/varnish-cache/bin/varnishtest/vtc_log.c
Log:
Add a hexdump function, now that we fiddle binary stuff.



Modified: trunk/varnish-cache/bin/varnishtest/vtc.h
===================================================================
--- trunk/varnish-cache/bin/varnishtest/vtc.h	2011-01-06 14:55:05 UTC (rev 5692)
+++ trunk/varnish-cache/bin/varnishtest/vtc.h	2011-01-06 14:57:10 UTC (rev 5693)
@@ -74,6 +74,8 @@
 void vtc_log(struct vtclog *vl, unsigned lvl, const char *fmt, ...);
 void vtc_dump(struct vtclog *vl, unsigned lvl, const char *pfx,
     const char *str, int len);
+void vtc_hexdump(struct vtclog *vl, unsigned lvl, const char *pfx,
+    const unsigned char *str, int len);
 
 int exec_file(const char *fn, const char *script, const char *tmpdir,
     char *logbuf, unsigned loglen);

Modified: trunk/varnish-cache/bin/varnishtest/vtc_log.c
===================================================================
--- trunk/varnish-cache/bin/varnishtest/vtc_log.c	2011-01-06 14:55:05 UTC (rev 5692)
+++ trunk/varnish-cache/bin/varnishtest/vtc_log.c	2011-01-06 14:57:10 UTC (rev 5693)
@@ -162,6 +162,15 @@
 
 	CHECK_OBJ_NOTNULL(vl, VTCLOG_MAGIC);
 	assert(lvl < NLEAD);
+	if (0 && str != NULL && len > 0) {
+		for (l = 0; l < len; l++) {
+			if (str[l] & 0x80) {
+				vtc_hexdump(vl, lvl, pfx,
+				    (const void*)str, len);
+				return;
+			}
+		}
+	}
 	AZ(pthread_mutex_lock(&vl->mtx));
 	vsb_clear(vl->vsb);
 	if (pfx == NULL)
@@ -210,3 +219,58 @@
 			pthread_exit(NULL);
 	}
 }
+
+/**********************************************************************
+ * Hexdump
+ */
+
+//lint -e{818}
+void
+vtc_hexdump(struct vtclog *vl, unsigned lvl, const char *pfx, const unsigned char *str, int len)
+{
+	int nl = 1;
+	unsigned l;
+
+	CHECK_OBJ_NOTNULL(vl, VTCLOG_MAGIC);
+	assert(len >= 0);
+	assert(lvl < NLEAD);
+	AZ(pthread_mutex_lock(&vl->mtx));
+	vsb_clear(vl->vsb);
+	if (pfx == NULL)
+		pfx = "";
+	if (str == NULL)
+		vsb_printf(vl->vsb, "%s %-4s %s(null)\n",
+		    lead[lvl], vl->id, pfx);
+	else {
+		for (l = 0; l < len; l++, str++) {
+			if (l > 512) {
+				vsb_printf(vl->vsb, "...");
+				break;
+			}
+			if (nl) {
+				vsb_printf(vl->vsb, "%s %-4s %s| ",
+				    lead[lvl], vl->id, pfx);
+				nl = 0;
+			}
+			vsb_printf(vl->vsb, " %02x", *str);
+			if ((l & 0xf) == 0xf) {
+				vsb_printf(vl->vsb, "\n");
+				nl = 1;
+			}
+		}
+	}
+	if (!nl)
+		vsb_printf(vl->vsb, "\n");
+	vsb_finish(vl->vsb);
+	AZ(vsb_overflowed(vl->vsb));
+
+	vtc_log_emit(vl, lvl);
+
+	vsb_clear(vl->vsb);
+	AZ(pthread_mutex_unlock(&vl->mtx));
+	if (lvl == 0) {
+		vtc_error = 1;
+		if (pthread_self() != vtc_thread)
+			pthread_exit(NULL);
+	}
+}




From phk at varnish-cache.org  Thu Jan  6 15:32:23 2011
From: phk at varnish-cache.org (phk at varnish-cache.org)
Date: Thu,  6 Jan 2011 16:32:23 +0100
Subject: r5694 - trunk/varnish-cache/bin/varnishtest
Message-ID: <E1Parp9-00051Y-AD@project.varnish-software.com>

Author: phk
Date: 2011-01-06 16:32:23 +0100 (Thu, 06 Jan 2011)
New Revision: 5694

Modified:
   trunk/varnish-cache/bin/varnishtest/vtc_http.c
Log:
More gzip support



Modified: trunk/varnish-cache/bin/varnishtest/vtc_http.c
===================================================================
--- trunk/varnish-cache/bin/varnishtest/vtc_http.c	2011-01-06 14:57:10 UTC (rev 5693)
+++ trunk/varnish-cache/bin/varnishtest/vtc_http.c	2011-01-06 15:32:23 UTC (rev 5694)
@@ -97,7 +97,7 @@
  */
 
 static char *
-synth_body(const char *len)
+synth_body(const char *len, int rnd)
 {
 	int i, j, k, l;
 	char *b;
@@ -116,6 +116,8 @@
 			if (k == '~')
 				k = '!';
 			l = k;
+		} else if (rnd) {
+			b[j] = (random() % 95) + ' ';
 		} else {
 			b[j] = (char)l;
 			if (++l == '~')
@@ -548,6 +550,12 @@
 	free(p);
 	vtc_dump(hp->vl, 4, "body", hp->body, hp->bodyl);
 	bprintf(hp->bodylen, "%u", hp->bodyl);
+	vtc_log(hp->vl, 4, "startbit = %ju %ju/%ju",
+	    vz.start_bit, vz.start_bit >> 3, vz.start_bit & 7);
+	vtc_log(hp->vl, 4, "lastbit = %ju %ju/%ju",
+	    vz.last_bit, vz.last_bit >> 3, vz.last_bit & 7);
+	vtc_log(hp->vl, 4, "stopbit = %ju %ju/%ju",
+	    vz.stop_bit, vz.stop_bit >> 3, vz.stop_bit & 7);
 	assert(Z_OK == inflateEnd(&vz));
 }
 
@@ -656,13 +664,20 @@
 			}
 		} else if (!strcmp(*av, "-bodylen")) {
 			assert(body == nullbody);
-			body = synth_body(av[1]);
+			body = synth_body(av[1], 0);
 			bodylen = strlen(body);
 			av++;
+		} else if (!strcmp(*av, "-gziplen")) {
+			assert(body == nullbody);
+			b = synth_body(av[1], 1);
+			gzip_body(b, &body, &bodylen);
+			vtc_hexdump(hp->vl, 4, "gzip", (void*)body, bodylen);
+			av++;
 		} else if (!strcmp(*av, "-gzipbody")) {
 			assert(body == nullbody);
 			gzip_body(av[1], &body, &bodylen);
 			vsb_printf(hp->vsb, "Content-Encoding: gzip%s", nl);
+			vtc_hexdump(hp->vl, 4, "gzip", (void*)body, bodylen);
 			av++;
 		} else
 			break;
@@ -787,7 +802,7 @@
 			av++;
 		} else if (!strcmp(*av, "-bodylen")) {
 			AZ(body);
-			body = synth_body(av[1]);
+			body = synth_body(av[1], 0);
 			av++;
 		} else
 			break;




From ingvar at varnish-cache.org  Thu Jan  6 21:06:19 2011
From: ingvar at varnish-cache.org (ingvar at varnish-cache.org)
Date: Thu,  6 Jan 2011 22:06:19 +0100
Subject: r5695 - trunk/varnish-cache/redhat
Message-ID: <E1Pax2J-00052u-Cj@project.varnish-software.com>

Author: ingvar
Date: 2011-01-06 22:06:18 +0100 (Thu, 06 Jan 2011)
New Revision: 5695

Modified:
   trunk/varnish-cache/redhat/varnish.initrc
   trunk/varnish-cache/redhat/varnishlog.initrc
   trunk/varnish-cache/redhat/varnishncsa.initrc
Log:
cosmetic changes to redhat initrc scripts

Modified: trunk/varnish-cache/redhat/varnish.initrc
===================================================================
--- trunk/varnish-cache/redhat/varnish.initrc	2011-01-06 15:32:23 UTC (rev 5694)
+++ trunk/varnish-cache/redhat/varnish.initrc	2011-01-06 21:06:18 UTC (rev 5695)
@@ -6,7 +6,7 @@
 # description: Varnish is a high-perfomance HTTP accelerator
 # processname: varnishd
 # config: /etc/sysconfig/varnish
-# pidfile: /var/run/varnish/varnishd.pid
+# pidfile: /var/run/varnishd.pid
 
 ### BEGIN INIT INFO
 # Provides: varnish

Modified: trunk/varnish-cache/redhat/varnishlog.initrc
===================================================================
--- trunk/varnish-cache/redhat/varnishlog.initrc	2011-01-06 15:32:23 UTC (rev 5694)
+++ trunk/varnish-cache/redhat/varnishlog.initrc	2011-01-06 21:06:18 UTC (rev 5695)
@@ -6,7 +6,7 @@
 # description: Varnish Cache logging daemon
 # processname: varnishlog
 # config: 
-# pidfile: /var/run/varnish/varnishlog.pid
+# pidfile: /var/run/varnishlog.pid
 
 ### BEGIN INIT INFO
 # Provides: varnishlog
@@ -31,7 +31,7 @@
 
 DAEMON_OPTS="-a -w $logfile -D -P $pidfile"
 
-# Include varnish defaults
+# Include varnishlog defaults
 [ -e /etc/sysconfig/varnishlog ] && . /etc/sysconfig/varnishlog
 
 start() {

Modified: trunk/varnish-cache/redhat/varnishncsa.initrc
===================================================================
--- trunk/varnish-cache/redhat/varnishncsa.initrc	2011-01-06 15:32:23 UTC (rev 5694)
+++ trunk/varnish-cache/redhat/varnishncsa.initrc	2011-01-06 21:06:18 UTC (rev 5695)
@@ -6,7 +6,7 @@
 # description: Varnish Cache logging daemon
 # processname: varnishncsa
 # config: 
-# pidfile: /var/run/varnish/varnishncsa.pid
+# pidfile: /var/run/varnishncsa.pid
 
 ### BEGIN INIT INFO
 # Provides: varnishncsa
@@ -31,7 +31,7 @@
 
 DAEMON_OPTS="-a -w $logfile -D -P $pidfile"
 
-# Include defaults
+# Include varnishncsa defaults
 [ -e /etc/sysconfig/varnishncsa ] && . /etc/sysconfig/varnishncsa
 
 start() {




From phk at varnish-cache.org  Thu Jan  6 22:14:00 2011
From: phk at varnish-cache.org (phk at varnish-cache.org)
Date: Thu,  6 Jan 2011 23:14:00 +0100
Subject: r5696 - trunk/varnish-cache/bin/varnishtest
Message-ID: <E1Pay5o-0006g1-M9@project.varnish-software.com>

Author: phk
Date: 2011-01-06 23:14:00 +0100 (Thu, 06 Jan 2011)
New Revision: 5696

Modified:
   trunk/varnish-cache/bin/varnishtest/vtc_http.c
Log:
A little less verbose



Modified: trunk/varnish-cache/bin/varnishtest/vtc_http.c
===================================================================
--- trunk/varnish-cache/bin/varnishtest/vtc_http.c	2011-01-06 21:06:18 UTC (rev 5695)
+++ trunk/varnish-cache/bin/varnishtest/vtc_http.c	2011-01-06 22:14:00 UTC (rev 5696)
@@ -671,13 +671,14 @@
 			assert(body == nullbody);
 			b = synth_body(av[1], 1);
 			gzip_body(b, &body, &bodylen);
-			vtc_hexdump(hp->vl, 4, "gzip", (void*)body, bodylen);
+			vsb_printf(hp->vsb, "Content-Encoding: gzip%s", nl);
+			// vtc_hexdump(hp->vl, 4, "gzip", (void*)body, bodylen);
 			av++;
 		} else if (!strcmp(*av, "-gzipbody")) {
 			assert(body == nullbody);
 			gzip_body(av[1], &body, &bodylen);
 			vsb_printf(hp->vsb, "Content-Encoding: gzip%s", nl);
-			vtc_hexdump(hp->vl, 4, "gzip", (void*)body, bodylen);
+			// vtc_hexdump(hp->vl, 4, "gzip", (void*)body, bodylen);
 			av++;
 		} else
 			break;




From phk at varnish-cache.org  Thu Jan  6 22:14:54 2011
From: phk at varnish-cache.org (phk at varnish-cache.org)
Date: Thu,  6 Jan 2011 23:14:54 +0100
Subject: r5697 - in trunk/varnish-cache/bin: varnishd varnishtest/tests
Message-ID: <E1Pay6g-0006iV-Nn@project.varnish-software.com>

Author: phk
Date: 2011-01-06 23:14:54 +0100 (Thu, 06 Jan 2011)
New Revision: 5697

Added:
   trunk/varnish-cache/bin/varnishtest/tests/g00002.vtc
Modified:
   trunk/varnish-cache/bin/varnishd/cache_gzip.c
   trunk/varnish-cache/bin/varnishd/cache_response.c
Log:
Make sure we handle segmentet storage correctly when gunzip'ing
a response.



Modified: trunk/varnish-cache/bin/varnishd/cache_gzip.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_gzip.c	2011-01-06 22:14:00 UTC (rev 5696)
+++ trunk/varnish-cache/bin/varnishd/cache_gzip.c	2011-01-06 22:14:54 UTC (rev 5697)
@@ -119,6 +119,7 @@
 
 	CHECK_OBJ_NOTNULL(vg, VGZ_MAGIC);
 
+	AZ(vg->vz.avail_in);
 	vg->vz.next_in = TRUST_ME(ptr);
 	vg->vz.avail_in = len;
 
@@ -126,7 +127,7 @@
 }
 
 /*--------------------------------------------------------------------*/
-
+#include <stdio.h>
 int
 VGZ_Produce(struct vgz *vg, const void **pptr, size_t *plen)
 {
@@ -148,6 +149,9 @@
 		return (0);
 	if (i == Z_STREAM_END)
 		return (1);
+	if (i == Z_BUF_ERROR)
+		return (2);
+fprintf(stderr, "--------------------> GUNZIP = %d\n", i);
 	return (-1);
 }
 

Modified: trunk/varnish-cache/bin/varnishd/cache_response.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_response.c	2011-01-06 22:14:00 UTC (rev 5696)
+++ trunk/varnish-cache/bin/varnishd/cache_response.c	2011-01-06 22:14:54 UTC (rev 5697)
@@ -275,6 +275,7 @@
 				if (WRW_Flush(sp->wrk))
 					break;
 			}
+			assert(i >= 0);
 		} while (i == 0);
 	}
 	VGZ_Destroy(&vg);

Added: trunk/varnish-cache/bin/varnishtest/tests/g00002.vtc
===================================================================
--- trunk/varnish-cache/bin/varnishtest/tests/g00002.vtc	                        (rev 0)
+++ trunk/varnish-cache/bin/varnishtest/tests/g00002.vtc	2011-01-06 22:14:54 UTC (rev 5697)
@@ -0,0 +1,55 @@
+# $Id$
+
+test "test basic gunzip for client"
+
+server s1 {
+	rxreq
+	expect req.http.accept-encoding == "gzip"
+	expect req.url == "/foo"
+	txresp -nolen -gziplen 4100
+	accept
+	rxreq
+	expect req.url == "/bar"
+	txresp -body {<H1><esi:include src="/foo"/></H1>}
+} -start
+
+varnish v1 -cliok "param.set http_gzip_support true" -vcl+backend {
+
+	sub vcl_fetch {
+		esi;
+	}
+} -start
+
+varnish v1 -cliok "param.set fetch_chunksize 4"
+
+client c1 {
+	txreq -url /foo -hdr "Accept-Encoding: gzip"
+	rxresp
+	gunzip
+	expect resp.http.content-encoding == "gzip"
+	expect resp.bodylen == 4100
+} -run
+
+# If this fails, the multiple storage allocations did not happen
+varnish v1 -expect SMF.s0.nreq == 3
+
+client c1 {
+	# See varnish can gunzip it.
+	txreq -url /foo -hdr "Accept-Encoding: null"
+	rxresp
+	expect resp.http.content-encoding == "resp.http.content-encoding"
+	expect resp.bodylen == 4100
+
+	# See varnish can gunzip it, inside ESI
+	txreq -url /bar -hdr "Accept-Encoding: null"
+	rxresp
+	expect resp.http.content-encoding == "resp.http.content-encoding"
+	expect resp.bodylen == 4109
+
+	# See varnish can deliver gzip'ed ESI (NOTYET)
+	#txreq -url /bar -hdr "Accept-Encoding: gzip"
+	#rxresp
+	# expect resp.http.content-encoding == "gzip"
+	#gunzip
+	#expect resp.bodylen == 4109
+} -run




From phk at varnish-cache.org  Fri Jan  7 12:32:25 2011
From: phk at varnish-cache.org (phk at varnish-cache.org)
Date: Fri,  7 Jan 2011 13:32:25 +0100
Subject: r5698 - trunk/varnish-cache/bin/varnishd
Message-ID: <E1PbBUX-0003kc-IF@project.varnish-software.com>

Author: phk
Date: 2011-01-07 13:32:25 +0100 (Fri, 07 Jan 2011)
New Revision: 5698

Modified:
   trunk/varnish-cache/bin/varnishd/cache.h
   trunk/varnish-cache/bin/varnishd/cache_center.c
   trunk/varnish-cache/bin/varnishd/cache_pool.c
   trunk/varnish-cache/bin/varnishd/cache_vrt_var.c
Log:
Rename a variable to better reflect what it really is.



Modified: trunk/varnish-cache/bin/varnishd/cache.h
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache.h	2011-01-06 22:14:54 UTC (rev 5697)
+++ trunk/varnish-cache/bin/varnishd/cache.h	2011-01-07 12:32:25 UTC (rev 5698)
@@ -258,7 +258,8 @@
 	double			grace;
 	unsigned		do_esi;
 
-	char			*storage;
+	/* This is only here so VRT can find it */
+	char			*storage_hint;
 
 	/* Timeouts */
 	double			connect_timeout;

Modified: trunk/varnish-cache/bin/varnishd/cache_center.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_center.c	2011-01-06 22:14:54 UTC (rev 5697)
+++ trunk/varnish-cache/bin/varnishd/cache_center.c	2011-01-07 12:32:25 UTC (rev 5698)
@@ -556,7 +556,7 @@
 
 	sp->wrk->body_status = RFC2616_Body(sp);
 
-	AZ(sp->wrk->storage);
+	AZ(sp->wrk->storage_hint);
 
 	VCL_fetch_method(sp);
 
@@ -599,10 +599,11 @@
 	 * XXX: also.
 	 */
 
-	sp->obj = STV_NewObject(sp, sp->wrk->storage, l, sp->wrk->ttl, nhttp);
+	sp->obj = STV_NewObject(sp, sp->wrk->storage_hint, l,
+	    sp->wrk->ttl, nhttp);
 	CHECK_OBJ_NOTNULL(sp->obj, OBJECT_MAGIC);
 
-	sp->wrk->storage = NULL;
+	sp->wrk->storage_hint = NULL;
 
 	if (vary != NULL) {
 		sp->obj->vary =
@@ -1294,7 +1295,7 @@
 		CHECK_OBJ_NOTNULL(sp->wrk, WORKER_MAGIC);
 		CHECK_OBJ_ORNULL(w->nobjhead, OBJHEAD_MAGIC);
 		WS_Assert(w->ws);
-		AZ(sp->wrk->storage);
+		AZ(sp->wrk->storage_hint);
 
 		switch (sp->step) {
 #define STEP(l,u) \

Modified: trunk/varnish-cache/bin/varnishd/cache_pool.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_pool.c	2011-01-06 22:14:54 UTC (rev 5697)
+++ trunk/varnish-cache/bin/varnishd/cache_pool.c	2011-01-07 12:32:25 UTC (rev 5698)
@@ -181,7 +181,7 @@
 		w->beresp1 = NULL;
 		w->beresp = NULL;
 		w->resp = NULL;
-		w->storage = NULL;
+		w->storage_hint = NULL;
 		w->wrq->func(w, w->wrq->priv);
 		AZ(w->bereq);
 		AZ(w->beresp1);
@@ -189,7 +189,7 @@
 		AZ(w->resp);
 		WS_Assert(w->ws);
 		AZ(w->wfd);
-		AZ(w->storage);
+		AZ(w->storage_hint);
 		assert(w->wlp == w->wlb);
 		w->wrq = NULL;
 		if (params->diag_bitmap & 0x00040000) {

Modified: trunk/varnish-cache/bin/varnishd/cache_vrt_var.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_vrt_var.c	2011-01-06 22:14:54 UTC (rev 5697)
+++ trunk/varnish-cache/bin/varnishd/cache_vrt_var.c	2011-01-07 12:32:25 UTC (rev 5698)
@@ -313,8 +313,8 @@
 VRT_r_beresp_storage(struct sess *sp)
 {
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
-	if (sp->wrk->storage != NULL)
-		return (sp->wrk->storage);
+	if (sp->wrk->storage_hint != NULL)
+		return (sp->wrk->storage_hint);
 	else
 		return (NULL);
 }
@@ -328,7 +328,7 @@
 	va_start(ap, str);
 	b = VRT_String(sp->wrk->ws, NULL, str, ap);
 	va_end(ap);
-	sp->wrk->storage = b;
+	sp->wrk->storage_hint = b;
 }
 
 void




From phk at varnish-cache.org  Fri Jan  7 13:48:36 2011
From: phk at varnish-cache.org (phk at varnish-cache.org)
Date: Fri,  7 Jan 2011 14:48:36 +0100
Subject: r5699 - trunk/varnish-cache/include
Message-ID: <E1PbCgG-0005fz-7g@project.varnish-software.com>

Author: phk
Date: 2011-01-07 14:48:35 +0100 (Fri, 07 Jan 2011)
New Revision: 5699

Modified:
   trunk/varnish-cache/include/vct.h
Log:
Add vct_ishex()



Modified: trunk/varnish-cache/include/vct.h
===================================================================
--- trunk/varnish-cache/include/vct.h	2011-01-07 12:32:25 UTC (rev 5698)
+++ trunk/varnish-cache/include/vct.h	2011-01-07 13:48:35 UTC (rev 5699)
@@ -54,6 +54,7 @@
 }
 
 #define vct_issp(x) vct_is(x, VCT_SP)
+#define vct_ishex(x) vct_is(x, VCT_HEX)
 #define vct_iscrlf(x) vct_is(x, VCT_CRLF)
 #define vct_islws(x) vct_is(x, VCT_LWS)
 #define vct_isctl(x) vct_is(x, VCT_CTL)




From phk at varnish-cache.org  Fri Jan  7 13:49:05 2011
From: phk at varnish-cache.org (phk at varnish-cache.org)
Date: Fri,  7 Jan 2011 14:49:05 +0100
Subject: r5700 - trunk/varnish-cache/bin/varnishtest/tests
Message-ID: <E1PbCgj-0005jM-V9@project.varnish-software.com>

Author: phk
Date: 2011-01-07 14:49:05 +0100 (Fri, 07 Jan 2011)
New Revision: 5700

Modified:
   trunk/varnish-cache/bin/varnishtest/tests/r00387.vtc
Log:
Make this test more robust by making sure the hex number is bogus.



Modified: trunk/varnish-cache/bin/varnishtest/tests/r00387.vtc
===================================================================
--- trunk/varnish-cache/bin/varnishtest/tests/r00387.vtc	2011-01-07 13:48:35 UTC (rev 5699)
+++ trunk/varnish-cache/bin/varnishtest/tests/r00387.vtc	2011-01-07 13:49:05 UTC (rev 5700)
@@ -8,7 +8,7 @@
 	send "Transfer-encoding: chunked\r\n"
 	send "\r\n"
 	send "004\r\n1234\r\n"
-	send "000000000000000000001\r\n@\r\n"
+	send "1000000000000000000001\r\n@\r\n"
 	send "00000000\r\n"
 	send "\r\n"
 } -start




From phk at varnish-cache.org  Fri Jan  7 14:22:19 2011
From: phk at varnish-cache.org (phk at varnish-cache.org)
Date: Fri,  7 Jan 2011 15:22:19 +0100
Subject: r5701 - trunk/varnish-cache/bin/varnishtest/tests
Message-ID: <E1PbDCt-0006hA-R4@project.varnish-software.com>

Author: phk
Date: 2011-01-07 15:22:19 +0100 (Fri, 07 Jan 2011)
New Revision: 5701

Modified:
   trunk/varnish-cache/bin/varnishtest/tests/r00801.vtc
Log:
Don't rely on a syntactically invalid Content-Length for this test



Modified: trunk/varnish-cache/bin/varnishtest/tests/r00801.vtc
===================================================================
--- trunk/varnish-cache/bin/varnishtest/tests/r00801.vtc	2011-01-07 13:49:05 UTC (rev 5700)
+++ trunk/varnish-cache/bin/varnishtest/tests/r00801.vtc	2011-01-07 14:22:19 UTC (rev 5701)
@@ -6,7 +6,7 @@
 	rxreq
 	txresp \
 		-hdr "Date: Mon, 25 Oct 2010 06:34:06 GMT" \
-		-hdr "Content-length: 10xx" \
+		-hdr "Content-length: 000010" \
 		-nolen -bodylen 10
 } -start
 




From phk at varnish-cache.org  Fri Jan  7 15:01:18 2011
From: phk at varnish-cache.org (phk at varnish-cache.org)
Date: Fri,  7 Jan 2011 16:01:18 +0100
Subject: r5702 - trunk/varnish-cache/bin/varnishd
Message-ID: <E1PbDoc-0007in-LC@project.varnish-software.com>

Author: phk
Date: 2011-01-07 16:01:18 +0100 (Fri, 07 Jan 2011)
New Revision: 5702

Modified:
   trunk/varnish-cache/bin/varnishd/cache.h
   trunk/varnish-cache/bin/varnishd/cache_center.c
   trunk/varnish-cache/bin/varnishd/cache_fetch.c
   trunk/varnish-cache/bin/varnishd/cache_httpconn.c
   trunk/varnish-cache/bin/varnishd/cache_pool.c
Log:
Rework the fetch code, to use a common function for feeding bytes into
storage slabs.  Pretty soon we will not be doing that directly and
we want only one place to know about the gzip/gunzip magic.

Overhaul the chunked encoding fetcher code, to be much simpler and 
much more robust.

Sort out some 4GB issues while we where here anyway.



Modified: trunk/varnish-cache/bin/varnishd/cache.h
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache.h	2011-01-07 14:22:19 UTC (rev 5701)
+++ trunk/varnish-cache/bin/varnishd/cache.h	2011-01-07 15:01:18 UTC (rev 5702)
@@ -90,6 +90,7 @@
 struct object;
 struct objhead;
 struct objcore;
+struct storage;
 struct workreq;
 struct esidata;
 struct vrt_backend;
@@ -250,7 +251,6 @@
 	struct http		*beresp;
 	struct http		*resp;
 
-	enum body_status	body_status;
 	unsigned		cacheable;
 	double			age;
 	double			entered;
@@ -261,6 +261,10 @@
 	/* This is only here so VRT can find it */
 	char			*storage_hint;
 
+	/* Fetch stuff */
+	enum body_status	body_status;
+	struct storage		*storage;
+
 	/* Timeouts */
 	double			connect_timeout;
 	double			first_byte_timeout;
@@ -649,7 +653,7 @@
 void HTC_Init(struct http_conn *htc, struct ws *ws, int fd);
 int HTC_Reinit(struct http_conn *htc);
 int HTC_Rx(struct http_conn *htc);
-int HTC_Read(struct http_conn *htc, void *d, unsigned len);
+ssize_t HTC_Read(struct http_conn *htc, void *d, size_t len);
 int HTC_Complete(struct http_conn *htc);
 
 #define HTTPH(a, b, c, d, e, f, g) extern char b[];

Modified: trunk/varnish-cache/bin/varnishd/cache_center.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_center.c	2011-01-07 14:22:19 UTC (rev 5701)
+++ trunk/varnish-cache/bin/varnishd/cache_center.c	2011-01-07 15:01:18 UTC (rev 5702)
@@ -1296,6 +1296,7 @@
 		CHECK_OBJ_ORNULL(w->nobjhead, OBJHEAD_MAGIC);
 		WS_Assert(w->ws);
 		AZ(sp->wrk->storage_hint);
+		AZ(sp->wrk->storage);
 
 		switch (sp->step) {
 #define STEP(l,u) \

Modified: trunk/varnish-cache/bin/varnishd/cache_fetch.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_fetch.c	2011-01-07 14:22:19 UTC (rev 5701)
+++ trunk/varnish-cache/bin/varnishd/cache_fetch.c	2011-01-07 15:01:18 UTC (rev 5702)
@@ -41,49 +41,139 @@
 #include "cache.h"
 #include "stevedore.h"
 #include "cli_priv.h"
+#include "vct.h"
 
 static unsigned fetchfrag;
 
+/*--------------------------------------------------------------------
+ * If we have any idea how big this object might be, we try to allocate
+ * a slab of storage that big.
+ */
+
+static void
+fetch_estimate(const struct sess *sp, size_t estimate)
+{
+
+	AZ(sp->wrk->storage);
+	if (fetchfrag > 0) {
+		estimate = fetchfrag;
+		WSL(sp->wrk, SLT_Debug, sp->fd,
+		    "Fetch %d byte segments:", fetchfrag);
+	}
+	sp->wrk->storage = STV_alloc(sp, estimate);
+}
+
+/*--------------------------------------------------------------------
+ * Fetch this many bytes and feed them to the monster.
+ */
+
+static int
+fetch_bytes(const struct sess *sp, struct http_conn *htc, size_t bytes)
+{
+	ssize_t l, w;
+	struct storage *st;
+
+	while (bytes > 0) {
+		if (sp->wrk->storage == NULL) {
+			l = fetchfrag;
+			if (l == 0)
+				l = params->fetch_chunksize * 1024LL;
+			sp->wrk->storage = STV_alloc(sp, l);
+		}
+		if (sp->wrk->storage == NULL) {
+			errno = ENOMEM;
+			return (-1);
+		}
+		st = sp->wrk->storage;
+		l = st->space - st->len;
+		if (l > bytes)
+			l = bytes;
+		w = HTC_Read(htc, st->ptr + st->len, l);
+		if (w <= 0)
+			return (w);
+		st->len += w;
+		sp->obj->len += w;
+		if (st->len == st->space) {
+			VTAILQ_INSERT_TAIL(&sp->obj->store,
+			    sp->wrk->storage, list);
+			sp->wrk->storage = NULL;
+			st = NULL;
+		}
+		bytes -= w;
+	}
+	return (1);
+}
+
+/*--------------------------------------------------------------------
+ * Fetch is complete, dispose of cached storage slab
+ */
+
+static void
+fetch_end(const struct sess *sp)
+{
+	struct storage *st;
+
+	st = sp->wrk->storage;
+	sp->wrk->storage = NULL;
+	if (st == NULL)
+		return;
+
+	if (st->len == 0) {
+		STV_free(st);
+		return;
+	}
+	if (st->len < st->space)
+		STV_trim(st, st->len);
+	VTAILQ_INSERT_TAIL(&sp->obj->store, st, list);
+}
+
+/*--------------------------------------------------------------------
+ * Convert a string to a size_t safely
+ */
+
+static ssize_t
+fetch_number(const char *nbr, int radix)
+{
+	uintmax_t cll;
+	ssize_t cl;
+	char *q;
+
+	if (*nbr == '\0')
+		return (-1);
+	cll = strtoumax(nbr, &q, radix);
+	if (q == NULL || *q != '\0')
+		return (-1);
+
+	cl = (ssize_t)cll;
+	if((uintmax_t)cl != cll) /* Protect against bogusly large values */
+		return (-1);
+	return (cl);
+}
+
 /*--------------------------------------------------------------------*/
 
 static int
 fetch_straight(const struct sess *sp, struct http_conn *htc, const char *b)
 {
 	int i;
-	unsigned char *p;
-	uintmax_t cll;
-	unsigned cl, sl;
-	struct storage *st;
+	ssize_t cl;
 
 	assert(sp->wrk->body_status == BS_LENGTH);
-	cll = strtoumax(b, NULL, 0);
-	if (cll == 0)
+	cl = fetch_number(b, 10);
+	if (cl < 0) {
+		WSP(sp, SLT_FetchError, "straight length syntax");
+		return (-1);
+	}
+	if (cl == 0)
 		return (0);
 
-	cl = (unsigned)cll;
-	assert((uintmax_t)cl == cll); /* Protect against bogusly large values */
+	fetch_estimate(sp, cl);
 
-	while (cl > 0) {
-		st = STV_alloc(sp, cl);
-		VTAILQ_INSERT_TAIL(&sp->obj->store, st, list);
-		sl = st->space;
-		if (sl > cl)
-			sl = cl;
-		p = st->ptr;
-
-		while (sl > 0) {
-			i = HTC_Read(htc, p, sl);
-			if (i <= 0) {
-				WSP(sp, SLT_FetchError,
-				    "straight read_error: %d", errno);
-				return (-1);
-			}
-			p += i;
-			st->len += i;
-			sp->obj->len += i;
-			sl -= i;
-			cl -= i;
-		}
+	i = fetch_bytes(sp, htc, cl);
+	if (i <= 0) {
+		WSP(sp, SLT_FetchError, "straight read_error: %d (%s)",
+		    errno, strerror(errno));
+		return (-1);
 	}
 	return (0);
 }
@@ -91,205 +181,97 @@
 /*--------------------------------------------------------------------*/
 /* XXX: Cleanup.  It must be possible somehow :-( */
 
+#define CERR() do {						\
+		if (i != 1) {					\
+			WSP(sp, SLT_FetchError,			\
+			    "chunked read_error: %d (%s)",	\
+			    errno, strerror(errno));		\
+			return (-1);				\
+		}						\
+	} while (0)
+
 static int
 fetch_chunked(const struct sess *sp, struct http_conn *htc)
 {
 	int i;
-	char *q;
-	struct storage *st;
-	unsigned u, v, w;
-	char buf[20];		/* XXX: arbitrary */
-	char *bp, *be;
+	char buf[20];		/* XXX: 20 is arbitrary */
+	unsigned u;
+	ssize_t cl;
 
 	assert(sp->wrk->body_status == BS_CHUNKED);
-	be = buf + sizeof buf - 1;
-	bp = buf;
-	st = NULL;
-	u = 0;
-	while (1) {
-		/* Try to parse buf as a chunk length */
-		*bp = '\0';
-		u = strtoul(buf, &q, 16);
+	do {
+		/* Skip leading whitespace */
+		do {
+			i = HTC_Read(htc, buf, 1);
+			CERR();
+		} while (vct_islws(buf[0]));
 
-		/* Skip trailing whitespace */
-		if (q != NULL && q > buf) {
-			while (*q == '\t' || *q == ' ')
-				q++;
-			if (*q == '\r')
-				q++;
+		/* Collect hex digits, skipping leading zeros */
+		for (u = 1; u < sizeof buf; u++) {
+			do {
+				i = HTC_Read(htc, buf + u, 1);
+				CERR();
+			} while (u == 1 && buf[0] == '0' && buf[u] == '0');
+			if (!vct_ishex(buf[u]))
+				break;
 		}
 
-		/* If we didn't succeed, add to buffer, try again */
-		if (q == NULL || q == buf || *q != '\n') {
-			if (bp >= be) {
-				WSP(sp, SLT_FetchError,
-				    "chunked hex-line too long");
-				return (-1);
-			}
-			/*
-			 * The semantics we need here is "read until you have
-			 * received at least one character, but feel free to
-			 * return up to (be-bp) if they are available, but do
-			 * not wait for those extra characters.
-			 *
-			 * The canonical way to do that is to do a blocking
-			 * read(2) of one char, then change to nonblocking,
-			 * read as many as we find, then change back to
-			 * blocking reads again.
-			 *
-			 * Hardly much more efficient and certainly a good
-			 * deal more complex than reading a single character
-			 * at a time.
-			 */
-			i = HTC_Read(htc, bp, 1);
-			if (i <= 0) {
-				WSP(sp, SLT_FetchError,
-				    "chunked read_error: %d", errno);
-				return (-1);
-			}
-			bp += i;
-			continue;
+		if (u >= sizeof buf) {
+			WSP(sp, SLT_FetchError,	"chunked header too long");
+			return (-1);
 		}
 
-		/* Skip NL */
-		q++;
+		/* Skip trailing white space */
+		while(vct_islws(buf[u]) && buf[u] != '\n') {
+			i = HTC_Read(htc, buf + u, 1);
+			CERR();
+		}
 
-		/* Last chunk is zero bytes long */
-		if (u == 0)
-			break;
-
-		while (u > 0) {
-
-			/* Get some storage if we don't have any */
-			if (st == NULL || st->len == st->space) {
-				v = u;
-				if (u < params->fetch_chunksize * 1024)
-					v = params->fetch_chunksize * 1024;
-				st = STV_alloc(sp, v);
-				VTAILQ_INSERT_TAIL(&sp->obj->store, st, list);
-			}
-			v = st->space - st->len;
-			if (v > u)
-				v = u;
-
-			/* Handle anything left in our buffer first */
-			w = pdiff(q, bp);
-			if (w > v)
-				w = v;
-			if (w != 0) {
-				memcpy(st->ptr + st->len, q, w);
-				st->len += w;
-				sp->obj->len += w;
-				u -= w;
-				v -= w;
-				q += w;
-			}
-			if (u == 0)
-				break;
-			if (v == 0)
-				continue;
-
-			/* Pick up the rest of this chunk */
-			while (v > 0) {
-				i = HTC_Read(htc, st->ptr + st->len, v);
-				if (i <= 0)
-					return (-1);
-				st->len += i;
-				sp->obj->len += i;
-				u -= i;
-				v -= i;
-			}
+		if (buf[u] != '\n') {
+			WSP(sp, SLT_FetchError,	"chunked header char syntax");
+			return (-1);
 		}
-		assert(u == 0);
+		buf[u] = '\0';
 
-		/* We might still have stuff in our buffer */
-		v = pdiff(q, bp);
-		if (v > 0)
-			memcpy(buf, q, v);
-		q = bp = buf + v;
-	}
-
-	/* Expect a CRLF to trail the chunks */
-	i = HTC_Read(htc, buf, 1);
-	if (i == 1 && buf[0] == '\r')
+		cl = fetch_number(buf, 16);
+		if (cl < 0) {
+			WSP(sp, SLT_FetchError,	"chunked header nbr syntax");
+			return (-1);
+		} else if (cl > 0) {
+			i = fetch_bytes(sp, htc, cl);
+			CERR();
+		}
 		i = HTC_Read(htc, buf, 1);
-	if (i != 1 || buf[0] != '\n') {
-		WSP(sp, SLT_FetchError, "chunked missing trailing crlf");
-		return (1);	/* Accept fetch, but do not reuse connection */
-	}
-
-	if (st != NULL && st->len == 0) {
-		VTAILQ_REMOVE(&sp->obj->store, st, list);
-		STV_free(st);
-	} else if (st != NULL && st->len < st->space)
-		STV_trim(st, st->len);
+		CERR();
+		if (buf[0] == '\r') {
+			i = HTC_Read(htc, buf, 1);
+			CERR();
+		}
+		if (buf[0] != '\n') {
+			WSP(sp, SLT_FetchError,	"chunked tail syntax");
+			return (-1);
+		}
+	} while (cl > 0);
 	return (0);
 }
 
+#undef CERR
 
 /*--------------------------------------------------------------------*/
 
-static void
-dump_st(const struct sess *sp, const struct storage *st)
-{
-	txt t;
-
-	t.b = (void*)st->ptr;
-	t.e = (void*)(st->ptr + st->len);
-	WSLR(sp->wrk, SLT_Debug, sp->fd, t);
-}
-
 static int
 fetch_eof(const struct sess *sp, struct http_conn *htc)
 {
 	int i;
-	unsigned char *p;
-	struct storage *st;
-	unsigned v;
 
 	assert(sp->wrk->body_status == BS_EOF);
-	if (fetchfrag > 0)
-		WSL(sp->wrk, SLT_Debug, sp->fd,
-		    "Fetch %d byte segments:", fetchfrag);
-	p = NULL;
-	v = 0;
-	st = NULL;
-	while (1) {
-		if (v == 0) {
-			if (st != NULL && fetchfrag > 0)
-				dump_st(sp, st);
-			st = STV_alloc(sp, params->fetch_chunksize * 1024LL);
-			VTAILQ_INSERT_TAIL(&sp->obj->store, st, list);
-			p = st->ptr + st->len;
-			v = st->space - st->len;
-			if (fetchfrag > 0 && v > fetchfrag)
-				v = fetchfrag;
-		}
-		AN(p);
-		AN(st);
-		i = HTC_Read(htc, p, v);
-		if (i < 0) {
-			WSP(sp, SLT_FetchError,
-			    "eof read_error: %d", errno);
-			return (-1);
-		}
-		if (i == 0)
-			break;
-		p += i;
-		v -= i;
-		st->len += i;
-		sp->obj->len += i;
+	i = fetch_bytes(sp, htc, 1000000000000);	/* XXX ? */
+	if (i < 0) {
+		WSP(sp, SLT_FetchError, "eof read_error: %d (%s)",
+		    errno, strerror(errno));
+		return (-1);
 	}
-	if (fetchfrag > 0)
-		dump_st(sp, st);
-
-	if (st->len == 0) {
-		VTAILQ_REMOVE(&sp->obj->store, st, list);
-		STV_free(st);
-	} else
-		STV_trim(st, st->len);
-
-	return (1);
+	return (0);
 }
 
 /*--------------------------------------------------------------------
@@ -474,6 +456,7 @@
 	 * XXX: Missing:  RFC2616 sec. 4.4 in re 1xx, 204 & 304 responses
 	 */
 
+	AZ(sp->wrk->storage);
 	switch (sp->wrk->body_status) {
 	case BS_NONE:
 		cls = 0;
@@ -505,8 +488,10 @@
 		mklen = 0;
 		INCOMPL();
 	}
+	fetch_end(sp);
+	AZ(sp->wrk->storage);
 
-	WSL(sp->wrk, SLT_Fetch_Body, sp->vbc->fd, "%u %u %u",
+	WSL(sp->wrk, SLT_Fetch_Body, sp->vbc->fd, "%u %d %u",
 	    sp->wrk->body_status, cls, mklen);
 
 	if (sp->wrk->body_status == BS_ERROR) {

Modified: trunk/varnish-cache/bin/varnishd/cache_httpconn.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_httpconn.c	2011-01-07 14:22:19 UTC (rev 5701)
+++ trunk/varnish-cache/bin/varnishd/cache_httpconn.c	2011-01-07 15:01:18 UTC (rev 5702)
@@ -178,12 +178,12 @@
 	return (HTC_Complete(htc));
 }
 
-int
-HTC_Read(struct http_conn *htc, void *d, unsigned len)
+ssize_t
+HTC_Read(struct http_conn *htc, void *d, size_t len)
 {
-	unsigned l;
+	size_t l;
 	unsigned char *p;
-	int i;
+	ssize_t i;
 
 	l = 0;
 	p = d;

Modified: trunk/varnish-cache/bin/varnishd/cache_pool.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_pool.c	2011-01-07 14:22:19 UTC (rev 5701)
+++ trunk/varnish-cache/bin/varnishd/cache_pool.c	2011-01-07 15:01:18 UTC (rev 5702)
@@ -182,11 +182,13 @@
 		w->beresp = NULL;
 		w->resp = NULL;
 		w->storage_hint = NULL;
+		w->storage = NULL;
 		w->wrq->func(w, w->wrq->priv);
 		AZ(w->bereq);
 		AZ(w->beresp1);
 		AZ(w->beresp);
 		AZ(w->resp);
+		AZ(w->storage);
 		WS_Assert(w->ws);
 		AZ(w->wfd);
 		AZ(w->storage_hint);




From phk at varnish-cache.org  Mon Jan 10 11:29:22 2011
From: phk at varnish-cache.org (phk at varnish-cache.org)
Date: Mon, 10 Jan 2011 12:29:22 +0100
Subject: r5703 - trunk/varnish-cache/lib/libvgz
Message-ID: <E1PcFwA-0007ap-VX@project.varnish-software.com>

Author: phk
Date: 2011-01-10 12:29:22 +0100 (Mon, 10 Jan 2011)
New Revision: 5703

Modified:
   trunk/varnish-cache/lib/libvgz/deflate.c
   trunk/varnish-cache/lib/libvgz/trees.c
Log:
Also register the start, stop and last bits when compressing


PS: I'm getting faster at this, today I wrote 3 lines of code per hour



Modified: trunk/varnish-cache/lib/libvgz/deflate.c
===================================================================
--- trunk/varnish-cache/lib/libvgz/deflate.c	2011-01-07 15:01:18 UTC (rev 5702)
+++ trunk/varnish-cache/lib/libvgz/deflate.c	2011-01-10 11:29:22 UTC (rev 5703)
@@ -811,6 +811,9 @@
         ERR_RETURN(strm, Z_BUF_ERROR);
     }
 
+    if (strm->start_bit == 0)
+	strm->start_bit = (strm->total_out + s->pending) * 8 + s->bi_valid;
+
     /* Start a new block or continue the current one.
      */
     if (strm->avail_in != 0 || s->lookahead != 0 ||

Modified: trunk/varnish-cache/lib/libvgz/trees.c
===================================================================
--- trunk/varnish-cache/lib/libvgz/trees.c	2011-01-07 15:01:18 UTC (rev 5702)
+++ trunk/varnish-cache/lib/libvgz/trees.c	2011-01-10 11:29:22 UTC (rev 5703)
@@ -931,6 +931,10 @@
     ulg opt_lenb, static_lenb; /* opt_len and static_len in bytes */
     int max_blindex = 0;  /* index of last bit length code of non zero freq */
 
+    if (last)
+        s->strm->last_bit =
+	    (s->strm->total_out + s->pending) * 8 + s->bi_valid;
+
     /* Build the Huffman trees unless a stored block is forced */
     if (s->level > 0) {
 
@@ -1011,6 +1015,9 @@
     init_block(s);
 
     if (last) {
+        s->strm->stop_bit =
+	    (s->strm->total_out + s->pending) * 8 + s->bi_valid;
+
         bi_windup(s);
 #ifdef DEBUG
         s->compressed_len += 7;  /* align on byte boundary */




From phk at varnish-cache.org  Mon Jan 10 11:33:16 2011
From: phk at varnish-cache.org (phk at varnish-cache.org)
Date: Mon, 10 Jan 2011 12:33:16 +0100
Subject: r5704 - trunk/varnish-cache/bin/varnishtest
Message-ID: <E1PcFzw-0007j2-Iv@project.varnish-software.com>

Author: phk
Date: 2011-01-10 12:33:13 +0100 (Mon, 10 Jan 2011)
New Revision: 5704

Modified:
   trunk/varnish-cache/bin/varnishtest/vtc_http.c
Log:
Report magic bits during gzip



Modified: trunk/varnish-cache/bin/varnishtest/vtc_http.c
===================================================================
--- trunk/varnish-cache/bin/varnishtest/vtc_http.c	2011-01-10 11:29:22 UTC (rev 5703)
+++ trunk/varnish-cache/bin/varnishtest/vtc_http.c	2011-01-10 11:33:13 UTC (rev 5704)
@@ -534,7 +534,7 @@
 	vz.next_in = TRUST_ME(hp->body);
 	vz.avail_in = hp->bodyl;
 
-	l = hp->bodyl;
+	l = hp->bodyl * 10;
 	p = calloc(l, 1);
 	AN(p);
 
@@ -564,7 +564,7 @@
  */
 
 static void
-gzip_body(const char *txt, char **body, int *bodylen)
+gzip_body(struct http *hp, const char *txt, char **body, int *bodylen)
 {
 	int l;
 	z_stream vz;
@@ -585,6 +585,12 @@
 	    Z_NO_COMPRESSION, Z_DEFLATED, 31, 9, Z_DEFAULT_STRATEGY));
 	assert(Z_STREAM_END == deflate(&vz, Z_FINISH));
 	*bodylen = vz.total_out;
+	vtc_log(hp->vl, 4, "startbit = %ju %ju/%ju",
+	    vz.start_bit, vz.start_bit >> 3, vz.start_bit & 7);
+	vtc_log(hp->vl, 4, "lastbit = %ju %ju/%ju",
+	    vz.last_bit, vz.last_bit >> 3, vz.last_bit & 7);
+	vtc_log(hp->vl, 4, "stopbit = %ju %ju/%ju",
+	    vz.stop_bit, vz.stop_bit >> 3, vz.stop_bit & 7);
 	assert(Z_OK == deflateEnd(&vz));
 }
 
@@ -670,13 +676,13 @@
 		} else if (!strcmp(*av, "-gziplen")) {
 			assert(body == nullbody);
 			b = synth_body(av[1], 1);
-			gzip_body(b, &body, &bodylen);
+			gzip_body(hp, b, &body, &bodylen);
 			vsb_printf(hp->vsb, "Content-Encoding: gzip%s", nl);
 			// vtc_hexdump(hp->vl, 4, "gzip", (void*)body, bodylen);
 			av++;
 		} else if (!strcmp(*av, "-gzipbody")) {
 			assert(body == nullbody);
-			gzip_body(av[1], &body, &bodylen);
+			gzip_body(hp, av[1], &body, &bodylen);
 			vsb_printf(hp->vsb, "Content-Encoding: gzip%s", nl);
 			// vtc_hexdump(hp->vl, 4, "gzip", (void*)body, bodylen);
 			av++;




From phk at varnish-cache.org  Mon Jan 10 12:09:19 2011
From: phk at varnish-cache.org (phk at varnish-cache.org)
Date: Mon, 10 Jan 2011 13:09:19 +0100
Subject: r5705 - in trunk/varnish-cache: bin/varnishtest/tests lib/libvcl
Message-ID: <E1PcGYp-0000E0-4k@project.varnish-software.com>

Author: phk
Date: 2011-01-10 13:09:18 +0100 (Mon, 10 Jan 2011)
New Revision: 5705

Added:
   trunk/varnish-cache/bin/varnishtest/tests/r00832.vtc
Modified:
   trunk/varnish-cache/lib/libvcl/vcc_backend.c
Log:
Fix a parse bug in IPv6 addresses.

Fixes: #832

Patch by: bz (THANKS!)


Added: trunk/varnish-cache/bin/varnishtest/tests/r00832.vtc
===================================================================
--- trunk/varnish-cache/bin/varnishtest/tests/r00832.vtc	                        (rev 0)
+++ trunk/varnish-cache/bin/varnishtest/tests/r00832.vtc	2011-01-10 12:09:18 UTC (rev 5705)
@@ -0,0 +1,9 @@
+# $Id
+
+test "Regression #832 IPV6 parse bug"
+
+varnish v1 -vcl {
+	backend default {
+		.host = "[::]";
+	}
+}

Modified: trunk/varnish-cache/lib/libvcl/vcc_backend.c
===================================================================
--- trunk/varnish-cache/lib/libvcl/vcc_backend.c	2011-01-10 11:33:13 UTC (rev 5704)
+++ trunk/varnish-cache/lib/libvcl/vcc_backend.c	2011-01-10 12:09:18 UTC (rev 5705)
@@ -132,10 +132,10 @@
 		vcc_ErrWhere(tl, t_host);
 		return;
 	}
-	if (pop != NULL)
-		error = getaddrinfo(hop, pop, &hint, &res0);
-	else
-		error = getaddrinfo(t_host->dec, port, &hint, &res0);
+	error = getaddrinfo(
+	    hop != NULL ? hop : t_host->dec,
+	    pop != NULL ? pop : port,
+	    &hint, &res0);
 	free(hop);
 	free(pop);
 	if (error) {




From phk at varnish-cache.org  Mon Jan 10 12:37:48 2011
From: phk at varnish-cache.org (phk at varnish-cache.org)
Date: Mon, 10 Jan 2011 13:37:48 +0100
Subject: r5706 - in trunk/varnish-cache: bin/varnishd bin/varnishtest/tests
	lib/libvcl
Message-ID: <E1PcH0O-00010i-9q@project.varnish-software.com>

Author: phk
Date: 2011-01-10 13:37:47 +0100 (Mon, 10 Jan 2011)
New Revision: 5706

Modified:
   trunk/varnish-cache/bin/varnishd/cache.h
   trunk/varnish-cache/bin/varnishd/cache_center.c
   trunk/varnish-cache/bin/varnishd/cache_esi.c
   trunk/varnish-cache/bin/varnishd/cache_hash.c
   trunk/varnish-cache/bin/varnishd/cache_panic.c
   trunk/varnish-cache/bin/varnishd/cache_vrt_var.c
   trunk/varnish-cache/bin/varnishtest/tests/e00003.vtc
   trunk/varnish-cache/lib/libvcl/generate.py
Log:
Rename sp->esis to sp->esi_level and make it available in VCL as
req.vcl_level.

Fixes: #782


Modified: trunk/varnish-cache/bin/varnishd/cache.h
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache.h	2011-01-10 12:09:18 UTC (rev 5705)
+++ trunk/varnish-cache/bin/varnishd/cache.h	2011-01-10 12:37:47 UTC (rev 5706)
@@ -434,7 +434,7 @@
 	unsigned		xid;
 
 	int			restarts;
-	int			esis;
+	int			esi_level;
 	int			disable_esi;
 
 	uint8_t			hash_ignore_busy;

Modified: trunk/varnish-cache/bin/varnishd/cache_center.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_center.c	2011-01-10 12:09:18 UTC (rev 5705)
+++ trunk/varnish-cache/bin/varnishd/cache_center.c	2011-01-10 12:37:47 UTC (rev 5706)
@@ -174,7 +174,7 @@
 		sp->wrk->res_mode |= RES_ESI;
 	}
 
-	if (sp->esis > 0) {
+	if (sp->esi_level > 0) {
 		sp->wrk->res_mode &= ~RES_LEN;
 		sp->wrk->res_mode |= RES_ESI_CHILD;
 	}
@@ -273,7 +273,7 @@
 	sp->director = NULL;
 	sp->restarts = 0;
 
-	if (sp->vcl != NULL && sp->esis == 0) {
+	if (sp->vcl != NULL && sp->esi_level == 0) {
 		if (sp->wrk->vcl != NULL)
 			VCL_Rel(&sp->wrk->vcl);
 		sp->wrk->vcl = sp->vcl;
@@ -287,7 +287,7 @@
 	if (sp->xid == 0) {
 		sp->t_req = sp->t_end;
 		sp->t_resp = sp->t_end;
-	} else if (sp->esis == 0) {
+	} else if (sp->esi_level == 0) {
 		dp = sp->t_resp - sp->t_req;
 		da = sp->t_end - sp->t_resp;
 		dh = sp->t_req - sp->t_open;
@@ -302,7 +302,7 @@
 	WSL_Flush(sp->wrk, 0);
 
 	/* If we did an ESI include, don't mess up our state */
-	if (sp->esis > 0)
+	if (sp->esi_level > 0)
 		return (1);
 
 	memset(&sp->acct_req, 0, sizeof sp->acct_req);
@@ -1126,7 +1126,7 @@
 		sp->step = STP_LOOKUP;
 		return (0);
 	case VCL_RET_PIPE:
-		if (sp->esis > 0) {
+		if (sp->esi_level > 0) {
 			/* XXX: VSL something */
 			INCOMPL();
 			/* sp->step = STP_DONE; */

Modified: trunk/varnish-cache/bin/varnishd/cache_esi.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_esi.c	2011-01-10 12:09:18 UTC (rev 5705)
+++ trunk/varnish-cache/bin/varnishd/cache_esi.c	2011-01-10 12:37:47 UTC (rev 5706)
@@ -874,7 +874,7 @@
 				(void)WRW_Write(w, "\r\n", -1);
 		}
 		if (eb->include.b == NULL ||
-		    sp->esis >= params->max_esi_includes)
+		    sp->esi_level >= params->max_esi_includes)
 			continue;
 
 		if (WRW_Flush(w)) {
@@ -883,7 +883,7 @@
 		}
 		AZ(WRW_FlushRelease(w));
 
-		sp->esis++;
+		sp->esi_level++;
 		obj = sp->obj;
 		sp->obj = NULL;
 		res_mode = sp->wrk->res_mode;
@@ -934,7 +934,7 @@
 		sp->xid = sxid;
 		AN(sp->wrk);
 		assert(sp->step == STP_DONE);
-		sp->esis--;
+		sp->esi_level--;
 		sp->obj = obj;
 		sp->wrk->res_mode = res_mode;
 

Modified: trunk/varnish-cache/bin/varnishd/cache_hash.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_hash.c	2011-01-10 12:09:18 UTC (rev 5705)
+++ trunk/varnish-cache/bin/varnishd/cache_hash.c	2011-01-10 12:37:47 UTC (rev 5706)
@@ -429,7 +429,7 @@
 
 	if (busy_oc != NULL) {
 		/* There are one or more busy objects, wait for them */
-		if (sp->esis == 0) {
+		if (sp->esi_level == 0) {
 			CHECK_OBJ_NOTNULL(sp->wrk->nwaitinglist,
 			    WAITINGLIST_MAGIC);
 			if (oh->waitinglist == NULL) {

Modified: trunk/varnish-cache/bin/varnishd/cache_panic.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_panic.c	2011-01-10 12:09:18 UTC (rev 5705)
+++ trunk/varnish-cache/bin/varnishd/cache_panic.c	2011-01-10 12:37:47 UTC (rev 5706)
@@ -238,8 +238,8 @@
 		    "  err_code = %d, err_reason = %s,\n", sp->err_code,
 		    sp->err_reason ? sp->err_reason : "(null)");
 
-	vsb_printf(vsp, "  restarts = %d, esis = %d\n",
-	    sp->restarts, sp->esis);
+	vsb_printf(vsp, "  restarts = %d, esi_level = %d\n",
+	    sp->restarts, sp->esi_level);
 
 	pan_ws(sp->ws, 2);
 	pan_http("req", sp->http, 2);

Modified: trunk/varnish-cache/bin/varnishd/cache_vrt_var.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_vrt_var.c	2011-01-10 12:09:18 UTC (rev 5705)
+++ trunk/varnish-cache/bin/varnishd/cache_vrt_var.c	2011-01-10 12:37:47 UTC (rev 5706)
@@ -412,7 +412,7 @@
 	 * Only allow you to turn of esi in the main request
 	 * else everything gets confused
 	 */
-	if(sp->esis == 0)
+	if(sp->esi_level == 0)
 		sp->disable_esi = !process_esi;
 }
 
@@ -423,6 +423,14 @@
 	return (!sp->disable_esi);
 }
 
+int
+VRT_r_req_esi_level(const struct sess *sp)
+{
+
+	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+	return(sp->esi_level);
+}
+
 /*--------------------------------------------------------------------*/
 
 unsigned __match_proto__()

Modified: trunk/varnish-cache/bin/varnishtest/tests/e00003.vtc
===================================================================
--- trunk/varnish-cache/bin/varnishtest/tests/e00003.vtc	2011-01-10 12:09:18 UTC (rev 5705)
+++ trunk/varnish-cache/bin/varnishtest/tests/e00003.vtc	2011-01-10 12:37:47 UTC (rev 5706)
@@ -12,13 +12,18 @@
 		After include
 	}
 	rxreq 
-	expect req.url == "/body"
+	expect req.url == "/body1"
 	txresp -body {
 		Included file
 	}
 } -start
 
 varnish v1 -vcl+backend {
+	sub vcl_recv {
+		if (req.esi_level > 0) {
+			set req.url = req.url + req.esi_level;
+		}
+	}
 	sub vcl_fetch {
 		esi;
 	}

Modified: trunk/varnish-cache/lib/libvcl/generate.py
===================================================================
--- trunk/varnish-cache/lib/libvcl/generate.py	2011-01-10 12:09:18 UTC (rev 5705)
+++ trunk/varnish-cache/lib/libvcl/generate.py	2011-01-10 12:37:47 UTC (rev 5706)
@@ -172,6 +172,12 @@
 		( ),
 		'const struct sess *'
 	),
+	('req.esi_level',
+		'INT',
+		( 'all',),
+		( ),
+		'const struct sess *'
+	),
 	('req.grace',
 		'DURATION',
 		( 'all',),




From kristian at varnish-cache.org  Mon Jan 10 14:45:01 2011
From: kristian at varnish-cache.org (kristian at varnish-cache.org)
Date: Mon, 10 Jan 2011 15:45:01 +0100
Subject: r5707 - trunk/varnish-cache/include
Message-ID: <E1PcIzV-0004Rl-7F@project.varnish-software.com>

Author: kristian
Date: 2011-01-10 15:45:00 +0100 (Mon, 10 Jan 2011)
New Revision: 5707

Modified:
   trunk/varnish-cache/include/Makefile.am
Log:
include/Makefile.am: Declare vrt_stv_var.h dep

Modified: trunk/varnish-cache/include/Makefile.am
===================================================================
--- trunk/varnish-cache/include/Makefile.am	2011-01-10 12:37:47 UTC (rev 5706)
+++ trunk/varnish-cache/include/Makefile.am	2011-01-10 14:45:00 UTC (rev 5707)
@@ -53,7 +53,7 @@
 	vss.h \
 	vtypes.h
 
-vcl_returns.h vcl.h vrt_obj.h: $(top_srcdir)/lib/libvcl/generate.py $(top_srcdir)/include/vrt.h
+vrt_stv_var.h vcl_returns.h vcl.h vrt_obj.h: $(top_srcdir)/lib/libvcl/generate.py $(top_srcdir)/include/vrt.h
 	@PYTHON@ $(top_srcdir)/lib/libvcl/generate.py $(top_srcdir) $(top_builddir)
 
 CLEANFILES = vcl_returns.h \




From phk at varnish-cache.org  Tue Jan 11 09:18:20 2011
From: phk at varnish-cache.org (phk at varnish-cache.org)
Date: Tue, 11 Jan 2011 10:18:20 +0100
Subject: r5708 - trunk/varnish-cache/bin/varnishd
Message-ID: <E1PcaMu-0007ZP-QI@project.varnish-software.com>

Author: phk
Date: 2011-01-11 10:18:20 +0100 (Tue, 11 Jan 2011)
New Revision: 5708

Modified:
   trunk/varnish-cache/bin/varnishd/cache_center.c
Log:
Make sure we have space to align the Content-Length header we create.



Modified: trunk/varnish-cache/bin/varnishd/cache_center.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_center.c	2011-01-10 14:45:00 UTC (rev 5707)
+++ trunk/varnish-cache/bin/varnishd/cache_center.c	2011-01-11 09:18:20 UTC (rev 5708)
@@ -591,8 +591,11 @@
 	if (vary != NULL)
 		l += varyl;
 
-	/* Space for producing a Content-Length: header */
-	l += 30;
+	/*
+	 * Space for producing a Content-Length: header including padding 
+	 * A billion gigabytes is enough for anybody.
+	 */
+	l += strlen("Content-Encoding: XxxXxxXxxXxxXxxXxx" + sizeof(void *));
 
 	/*
 	 * XXX: If we have a Length: header, we should allocate the body




From phk at varnish-cache.org  Tue Jan 11 10:04:07 2011
From: phk at varnish-cache.org (phk at varnish-cache.org)
Date: Tue, 11 Jan 2011 11:04:07 +0100
Subject: r5709 - trunk/varnish-cache/bin/varnishd
Message-ID: <E1Pcb5D-0000mx-U3@project.varnish-software.com>

Author: phk
Date: 2011-01-11 11:04:07 +0100 (Tue, 11 Jan 2011)
New Revision: 5709

Modified:
   trunk/varnish-cache/bin/varnishd/cache_fetch.c
Log:
Always call the fetch_begin function, also for chunked and eof



Modified: trunk/varnish-cache/bin/varnishd/cache_fetch.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_fetch.c	2011-01-11 09:18:20 UTC (rev 5708)
+++ trunk/varnish-cache/bin/varnishd/cache_fetch.c	2011-01-11 10:04:07 UTC (rev 5709)
@@ -51,7 +51,7 @@
  */
 
 static void
-fetch_estimate(const struct sess *sp, size_t estimate)
+fetch_begin(const struct sess *sp, size_t estimate)
 {
 
 	AZ(sp->wrk->storage);
@@ -60,7 +60,8 @@
 		WSL(sp->wrk, SLT_Debug, sp->fd,
 		    "Fetch %d byte segments:", fetchfrag);
 	}
-	sp->wrk->storage = STV_alloc(sp, estimate);
+	if (estimate > 0)
+		sp->wrk->storage = STV_alloc(sp, estimate);
 }
 
 /*--------------------------------------------------------------------
@@ -167,7 +168,7 @@
 	if (cl == 0)
 		return (0);
 
-	fetch_estimate(sp, cl);
+	fetch_begin(sp, cl);
 
 	i = fetch_bytes(sp, htc, cl);
 	if (i <= 0) {
@@ -198,6 +199,7 @@
 	unsigned u;
 	ssize_t cl;
 
+	fetch_begin(sp, 0);
 	assert(sp->wrk->body_status == BS_CHUNKED);
 	do {
 		/* Skip leading whitespace */
@@ -265,6 +267,7 @@
 	int i;
 
 	assert(sp->wrk->body_status == BS_EOF);
+	fetch_begin(sp, 0);
 	i = fetch_bytes(sp, htc, 1000000000000);	/* XXX ? */
 	if (i < 0) {
 		WSP(sp, SLT_FetchError, "eof read_error: %d (%s)",




From phk at varnish-cache.org  Tue Jan 11 10:20:50 2011
From: phk at varnish-cache.org (phk at varnish-cache.org)
Date: Tue, 11 Jan 2011 11:20:50 +0100
Subject: r5710 - trunk/varnish-cache/lib/libvgz
Message-ID: <E1PcbLO-0001F1-7G@project.varnish-software.com>

Author: phk
Date: 2011-01-11 11:20:48 +0100 (Tue, 11 Jan 2011)
New Revision: 5710

Modified:
   trunk/varnish-cache/lib/libvgz/deflate.c
   trunk/varnish-cache/lib/libvgz/inflate.c
Log:
Initialize bit positions to zero during Reset



Modified: trunk/varnish-cache/lib/libvgz/deflate.c
===================================================================
--- trunk/varnish-cache/lib/libvgz/deflate.c	2011-01-11 10:04:07 UTC (rev 5709)
+++ trunk/varnish-cache/lib/libvgz/deflate.c	2011-01-11 10:20:48 UTC (rev 5710)
@@ -361,6 +361,7 @@
     }
 
     strm->total_in = strm->total_out = 0;
+    strm->start_bit = strm->stop_bit = strm->last_bit = 0;
     strm->msg = Z_NULL; /* use zfree if we ever allocate msg dynamically */
     strm->data_type = Z_UNKNOWN;
 

Modified: trunk/varnish-cache/lib/libvgz/inflate.c
===================================================================
--- trunk/varnish-cache/lib/libvgz/inflate.c	2011-01-11 10:04:07 UTC (rev 5709)
+++ trunk/varnish-cache/lib/libvgz/inflate.c	2011-01-11 10:20:48 UTC (rev 5710)
@@ -108,6 +108,7 @@
     if (strm == Z_NULL || strm->state == Z_NULL) return Z_STREAM_ERROR;
     state = (struct inflate_state FAR *)strm->state;
     strm->total_in = strm->total_out = state->total = 0;
+    strm->start_bit = strm->stop_bit = strm->last_bit = 0;
     strm->msg = Z_NULL;
     strm->adler = 1;        /* to support ill-conceived Java test suite */
     state->mode = HEAD;




From phk at varnish-cache.org  Tue Jan 11 10:49:47 2011
From: phk at varnish-cache.org (phk at varnish-cache.org)
Date: Tue, 11 Jan 2011 11:49:47 +0100
Subject: r5711 - trunk/varnish-cache/bin/varnishd
Message-ID: <E1PcbnP-00022J-4r@project.varnish-software.com>

Author: phk
Date: 2011-01-11 11:49:46 +0100 (Tue, 11 Jan 2011)
New Revision: 5711

Modified:
   trunk/varnish-cache/bin/varnishd/cache.h
   trunk/varnish-cache/bin/varnishd/cache_center.c
   trunk/varnish-cache/bin/varnishd/cache_fetch.c
Log:
VTLATD (VTLA Of The Day :-)

VFP, Varnish Fetch Processors, are code that does things with the bytes
we receive from the backend.  By default nothing.



Modified: trunk/varnish-cache/bin/varnishd/cache.h
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache.h	2011-01-11 10:20:48 UTC (rev 5710)
+++ trunk/varnish-cache/bin/varnishd/cache.h	2011-01-11 10:49:46 UTC (rev 5711)
@@ -211,6 +211,18 @@
 #undef L0
 #undef L1
 
+/* Fetch processors --------------------------------------------------*/
+
+typedef void vfp_begin_f(const struct sess *, size_t );
+typedef int vfp_bytes_f(const struct sess *, struct http_conn *, size_t);
+typedef int vfp_end_f(const struct sess *sp);
+
+struct vfp {
+	vfp_begin_f	*begin;
+	vfp_bytes_f	*bytes;
+	vfp_end_f	*end;
+};
+
 /*--------------------------------------------------------------------*/
 
 struct worker {
@@ -264,6 +276,7 @@
 	/* Fetch stuff */
 	enum body_status	body_status;
 	struct storage		*storage;
+	struct vfp		*vfp;
 
 	/* Timeouts */
 	double			connect_timeout;

Modified: trunk/varnish-cache/bin/varnishd/cache_center.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_center.c	2011-01-11 10:20:48 UTC (rev 5710)
+++ trunk/varnish-cache/bin/varnishd/cache_center.c	2011-01-11 10:49:46 UTC (rev 5711)
@@ -643,7 +643,9 @@
 	else
 		sp->obj->last_modified = sp->wrk->entered;
 
+	AZ(sp->wrk->vfp);
 	i = FetchBody(sp);
+	sp->wrk->vfp = NULL;
 	AZ(sp->wrk->wfd);
 	AZ(sp->vbc);
 	AN(sp->director);

Modified: trunk/varnish-cache/bin/varnishd/cache_fetch.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_fetch.c	2011-01-11 10:20:48 UTC (rev 5710)
+++ trunk/varnish-cache/bin/varnishd/cache_fetch.c	2011-01-11 10:49:46 UTC (rev 5711)
@@ -46,12 +46,22 @@
 static unsigned fetchfrag;
 
 /*--------------------------------------------------------------------
- * If we have any idea how big this object might be, we try to allocate
- * a slab of storage that big.
+ * VFP NOOP
+ *
+ * This fetch-processor does nothing but store the object.
+ * It also documents the API
  */
 
+/*--------------------------------------------------------------------
+ * VFP_BEGIN
+ *
+ * Called to set up stuff.
+ *
+ * 'estimate' is the estimate of the number of bytes we expect to receive,
+ * as seen on the socket, or zero if unknown.
+ */
 static void
-fetch_begin(const struct sess *sp, size_t estimate)
+vfp_nop_begin(const struct sess *sp, size_t estimate)
 {
 
 	AZ(sp->wrk->storage);
@@ -65,11 +75,17 @@
 }
 
 /*--------------------------------------------------------------------
- * Fetch this many bytes and feed them to the monster.
+ * VFP_BYTES
+ *
+ * Process (up to) 'bytes' from the socket.
+ *
+ * Return -1 on error
+ * Return 0 on EOF on socket even if bytes not reached.
+ * Return 1 when 'bytes' have been processed.
  */
 
 static int
-fetch_bytes(const struct sess *sp, struct http_conn *htc, size_t bytes)
+vfp_nop_bytes(const struct sess *sp, struct http_conn *htc, size_t bytes)
 {
 	ssize_t l, w;
 	struct storage *st;
@@ -106,28 +122,40 @@
 }
 
 /*--------------------------------------------------------------------
- * Fetch is complete, dispose of cached storage slab
+ * VFP_END
+ *
+ * Finish & cleanup
+ *
+ * Return -1 for error
+ * Return 0 for OK
  */
 
-static void
-fetch_end(const struct sess *sp)
+static int
+vfp_nop_end(const struct sess *sp)
 {
 	struct storage *st;
 
 	st = sp->wrk->storage;
 	sp->wrk->storage = NULL;
 	if (st == NULL)
-		return;
+		return (0);
 
 	if (st->len == 0) {
 		STV_free(st);
-		return;
+		return (0);
 	}
 	if (st->len < st->space)
 		STV_trim(st, st->len);
 	VTAILQ_INSERT_TAIL(&sp->obj->store, st, list);
+	return (0);
 }
 
+static struct vfp vfp_nop = {
+	.begin	=	vfp_nop_begin,
+	.bytes	=	vfp_nop_bytes,
+	.end	=	vfp_nop_end,
+};
+
 /*--------------------------------------------------------------------
  * Convert a string to a size_t safely
  */
@@ -168,9 +196,9 @@
 	if (cl == 0)
 		return (0);
 
-	fetch_begin(sp, cl);
+	sp->wrk->vfp->begin(sp, cl);
 
-	i = fetch_bytes(sp, htc, cl);
+	i = sp->wrk->vfp->bytes(sp, htc, cl);
 	if (i <= 0) {
 		WSP(sp, SLT_FetchError, "straight read_error: %d (%s)",
 		    errno, strerror(errno));
@@ -199,7 +227,7 @@
 	unsigned u;
 	ssize_t cl;
 
-	fetch_begin(sp, 0);
+	sp->wrk->vfp->begin(sp, 0);
 	assert(sp->wrk->body_status == BS_CHUNKED);
 	do {
 		/* Skip leading whitespace */
@@ -240,7 +268,7 @@
 			WSP(sp, SLT_FetchError,	"chunked header nbr syntax");
 			return (-1);
 		} else if (cl > 0) {
-			i = fetch_bytes(sp, htc, cl);
+			i = sp->wrk->vfp->bytes(sp, htc, cl);
 			CERR();
 		}
 		i = HTC_Read(htc, buf, 1);
@@ -267,8 +295,8 @@
 	int i;
 
 	assert(sp->wrk->body_status == BS_EOF);
-	fetch_begin(sp, 0);
-	i = fetch_bytes(sp, htc, 1000000000000);	/* XXX ? */
+	sp->wrk->vfp->begin(sp, 0);
+	i = sp->wrk->vfp->bytes(sp, htc, 1000000000000);	/* XXX ? */
 	if (i < 0) {
 		WSP(sp, SLT_FetchError, "eof read_error: %d (%s)",
 		    errno, strerror(errno));
@@ -449,6 +477,9 @@
 	CHECK_OBJ_NOTNULL(sp->obj, OBJECT_MAGIC);
 	CHECK_OBJ_NOTNULL(sp->obj->http, HTTP_MAGIC);
 
+	if (sp->wrk->vfp == NULL)
+		sp->wrk->vfp = &vfp_nop;
+
 	/* We use the unmodified headers */
 	hp = sp->wrk->beresp1;
 	AN(sp->director);
@@ -491,7 +522,7 @@
 		mklen = 0;
 		INCOMPL();
 	}
-	fetch_end(sp);
+	XXXAZ(sp->wrk->vfp->end(sp));
 	AZ(sp->wrk->storage);
 
 	WSL(sp->wrk, SLT_Fetch_Body, sp->vbc->fd, "%u %d %u",




From phk at varnish-cache.org  Tue Jan 11 11:01:48 2011
From: phk at varnish-cache.org (phk at varnish-cache.org)
Date: Tue, 11 Jan 2011 12:01:48 +0100
Subject: r5712 - trunk/varnish-cache/lib/libvcl
Message-ID: <E1Pcbz3-0002Vr-H9@project.varnish-software.com>

Author: phk
Date: 2011-01-11 12:01:47 +0100 (Tue, 11 Jan 2011)
New Revision: 5712

Modified:
   trunk/varnish-cache/lib/libvcl/generate.py
Log:
Add beresp.esi, beresp.gzip and beresp.gunzip variables



Modified: trunk/varnish-cache/lib/libvcl/generate.py
===================================================================
--- trunk/varnish-cache/lib/libvcl/generate.py	2011-01-11 10:49:46 UTC (rev 5711)
+++ trunk/varnish-cache/lib/libvcl/generate.py	2011-01-11 11:01:47 UTC (rev 5712)
@@ -298,6 +298,24 @@
 		( 'fetch',),
 		'const struct sess *'
 	),
+	('beresp.esi',
+		'BOOL',
+		( 'fetch',),
+		( 'fetch',),
+		'const struct sess *'
+	),
+	('beresp.gzip',
+		'BOOL',
+		( 'fetch',),
+		( 'fetch',),
+		'const struct sess *'
+	),
+	('beresp.gunzip',
+		'BOOL',
+		( 'fetch',),
+		( 'fetch',),
+		'const struct sess *'
+	),
 	('beresp.ttl',
 		'DURATION',
 		( 'fetch',),




From phk at varnish-cache.org  Tue Jan 11 11:24:39 2011
From: phk at varnish-cache.org (phk at varnish-cache.org)
Date: Tue, 11 Jan 2011 12:24:39 +0100
Subject: r5713 - in trunk/varnish-cache: bin/varnishd bin/varnishtest/tests
	lib/libvcl
Message-ID: <E1PccL9-00039g-JW@project.varnish-software.com>

Author: phk
Date: 2011-01-11 12:24:39 +0100 (Tue, 11 Jan 2011)
New Revision: 5713

Modified:
   trunk/varnish-cache/bin/varnishd/cache.h
   trunk/varnish-cache/bin/varnishd/cache_fetch.c
   trunk/varnish-cache/bin/varnishd/cache_gzip.c
   trunk/varnish-cache/bin/varnishd/cache_vrt.c
   trunk/varnish-cache/bin/varnishd/cache_vrt_var.c
   trunk/varnish-cache/bin/varnishtest/tests/e00000.vtc
   trunk/varnish-cache/bin/varnishtest/tests/e00001.vtc
   trunk/varnish-cache/bin/varnishtest/tests/e00002.vtc
   trunk/varnish-cache/bin/varnishtest/tests/e00003.vtc
   trunk/varnish-cache/bin/varnishtest/tests/e00004.vtc
   trunk/varnish-cache/bin/varnishtest/tests/e00005.vtc
   trunk/varnish-cache/bin/varnishtest/tests/e00006.vtc
   trunk/varnish-cache/bin/varnishtest/tests/e00007.vtc
   trunk/varnish-cache/bin/varnishtest/tests/e00008.vtc
   trunk/varnish-cache/bin/varnishtest/tests/e00009.vtc
   trunk/varnish-cache/bin/varnishtest/tests/e00010.vtc
   trunk/varnish-cache/bin/varnishtest/tests/e00011.vtc
   trunk/varnish-cache/bin/varnishtest/tests/e00012.vtc
   trunk/varnish-cache/bin/varnishtest/tests/e00013.vtc
   trunk/varnish-cache/bin/varnishtest/tests/e00014.vtc
   trunk/varnish-cache/bin/varnishtest/tests/e00015.vtc
   trunk/varnish-cache/bin/varnishtest/tests/e00016.vtc
   trunk/varnish-cache/bin/varnishtest/tests/e00017.vtc
   trunk/varnish-cache/bin/varnishtest/tests/e00018.vtc
   trunk/varnish-cache/bin/varnishtest/tests/g00002.vtc
   trunk/varnish-cache/bin/varnishtest/tests/r00318.vtc
   trunk/varnish-cache/bin/varnishtest/tests/r00326.vtc
   trunk/varnish-cache/bin/varnishtest/tests/r00345.vtc
   trunk/varnish-cache/bin/varnishtest/tests/r00386.vtc
   trunk/varnish-cache/bin/varnishtest/tests/r00427.vtc
   trunk/varnish-cache/bin/varnishtest/tests/r00433.vtc
   trunk/varnish-cache/bin/varnishtest/tests/r00445.vtc
   trunk/varnish-cache/bin/varnishtest/tests/r00476.vtc
   trunk/varnish-cache/bin/varnishtest/tests/r00524.vtc
   trunk/varnish-cache/bin/varnishtest/tests/r00590.vtc
   trunk/varnish-cache/bin/varnishtest/tests/r00612.vtc
   trunk/varnish-cache/bin/varnishtest/tests/r00641.vtc
   trunk/varnish-cache/lib/libvcl/generate.py
   trunk/varnish-cache/lib/libvcl/vcc_action.c
Log:
Rename the "esi" action to a BOOL variable named beresp.do_esi.

In other words, you need to change
	sub vcl_fetch {
		esi;
	}

to:

	sub vcl_fetch {
		set beresp.do_esi = true;
	}

Add VCL support and skeleton versions of the gzip (beresp.do_gzip)
and gunzip (beresp.do_gunzip) VFP's while here anyway.



Modified: trunk/varnish-cache/bin/varnishd/cache.h
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache.h	2011-01-11 11:01:47 UTC (rev 5712)
+++ trunk/varnish-cache/bin/varnishd/cache.h	2011-01-11 11:24:39 UTC (rev 5713)
@@ -223,6 +223,9 @@
 	vfp_end_f	*end;
 };
 
+extern struct vfp vfp_gunzip;
+extern struct vfp vfp_gzip;
+
 /*--------------------------------------------------------------------*/
 
 struct worker {
@@ -268,7 +271,6 @@
 	double			entered;
 	double			ttl;
 	double			grace;
-	unsigned		do_esi;
 
 	/* This is only here so VRT can find it */
 	char			*storage_hint;
@@ -277,6 +279,9 @@
 	enum body_status	body_status;
 	struct storage		*storage;
 	struct vfp		*vfp;
+	unsigned		do_esi;
+	unsigned		do_gzip;
+	unsigned		do_gunzip;
 
 	/* Timeouts */
 	double			connect_timeout;

Modified: trunk/varnish-cache/bin/varnishd/cache_fetch.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_fetch.c	2011-01-11 11:01:47 UTC (rev 5712)
+++ trunk/varnish-cache/bin/varnishd/cache_fetch.c	2011-01-11 11:24:39 UTC (rev 5713)
@@ -46,7 +46,7 @@
 static unsigned fetchfrag;
 
 /*--------------------------------------------------------------------
- * VFP NOOP
+ * VFP_NOP
  *
  * This fetch-processor does nothing but store the object.
  * It also documents the API

Modified: trunk/varnish-cache/bin/varnishd/cache_gzip.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_gzip.c	2011-01-11 11:01:47 UTC (rev 5712)
+++ trunk/varnish-cache/bin/varnishd/cache_gzip.c	2011-01-11 11:24:39 UTC (rev 5713)
@@ -167,3 +167,75 @@
 	*vg = NULL;
 	return (0);
 }
+
+/*--------------------------------------------------------------------
+ * VFP_GUNZIP
+ *
+ * A VFP for gunzip'ing an object as we receive it from the backend
+ */
+
+static void
+vfp_gunzip_begin(const struct sess *sp, size_t estimate)
+{
+	(void)sp;
+	(void)estimate;
+}
+
+static int
+vfp_gunzip_bytes(const struct sess *sp, struct http_conn *htc, size_t bytes)
+{
+	(void)sp;
+	(void)htc;
+	(void)bytes;
+	return (-1);
+}
+
+static int
+vfp_gunzip_end(const struct sess *sp)
+{
+	(void)sp;
+	return (-1);
+}
+
+struct vfp vfp_gunzip = {
+        .begin  =       vfp_gunzip_begin,
+        .bytes  =       vfp_gunzip_bytes,
+        .end    =       vfp_gunzip_end,
+};
+
+
+/*--------------------------------------------------------------------
+ * VFP_GZIP
+ *
+ * A VFP for gzip'ing an object as we receive it from the backend
+ */
+
+static void
+vfp_gzip_begin(const struct sess *sp, size_t estimate)
+{
+	(void)sp;
+	(void)estimate;
+}
+
+static int
+vfp_gzip_bytes(const struct sess *sp, struct http_conn *htc, size_t bytes)
+{
+	(void)sp;
+	(void)htc;
+	(void)bytes;
+	return (-1);
+}
+
+static int
+vfp_gzip_end(const struct sess *sp)
+{
+	(void)sp;
+	return (-1);
+}
+
+struct vfp vfp_gzip = {
+        .begin  =       vfp_gzip_begin,
+        .bytes  =       vfp_gzip_bytes,
+        .end    =       vfp_gzip_end,
+};
+

Modified: trunk/varnish-cache/bin/varnishd/cache_vrt.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_vrt.c	2011-01-11 11:01:47 UTC (rev 5712)
+++ trunk/varnish-cache/bin/varnishd/cache_vrt.c	2011-01-11 11:24:39 UTC (rev 5713)
@@ -378,22 +378,6 @@
 /*--------------------------------------------------------------------*/
 
 void
-VRT_ESI(const struct sess *sp)
-{
-
-	if (sp->cur_method != VCL_MET_FETCH) {
-		/* XXX: we should catch this at compile time */
-		WSP(sp, SLT_VCL_error,
-		    "esi can only be called from vcl_fetch");
-		return;
-	}
-
-	sp->wrk->do_esi = 1;
-}
-
-/*--------------------------------------------------------------------*/
-
-void
 VRT_panic(const struct sess *sp, const char *str, ...)
 {
 	va_list ap;

Modified: trunk/varnish-cache/bin/varnishd/cache_vrt_var.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_vrt_var.c	2011-01-11 11:01:47 UTC (rev 5712)
+++ trunk/varnish-cache/bin/varnishd/cache_vrt_var.c	2011-01-11 11:24:39 UTC (rev 5713)
@@ -196,6 +196,9 @@
 }
 
 VBERESP(beresp, unsigned, cacheable, cacheable)
+VBERESP(beresp, unsigned, do_esi, do_esi)
+VBERESP(beresp, unsigned, do_gzip, do_gzip)
+VBERESP(beresp, unsigned, do_gunzip, do_gunzip)
 
 /*--------------------------------------------------------------------*/
 

Modified: trunk/varnish-cache/bin/varnishtest/tests/e00000.vtc
===================================================================
--- trunk/varnish-cache/bin/varnishtest/tests/e00000.vtc	2011-01-11 11:01:47 UTC (rev 5712)
+++ trunk/varnish-cache/bin/varnishtest/tests/e00000.vtc	2011-01-11 11:24:39 UTC (rev 5713)
@@ -12,7 +12,7 @@
 
 varnish v1 -vcl+backend {
 	sub vcl_fetch {
-		esi;
+		set beresp.do_esi = true;
 	}
 } -start
 

Modified: trunk/varnish-cache/bin/varnishtest/tests/e00001.vtc
===================================================================
--- trunk/varnish-cache/bin/varnishtest/tests/e00001.vtc	2011-01-11 11:01:47 UTC (rev 5712)
+++ trunk/varnish-cache/bin/varnishtest/tests/e00001.vtc	2011-01-11 11:24:39 UTC (rev 5713)
@@ -15,7 +15,7 @@
 
 varnish v1 -vcl+backend {
 	sub vcl_fetch {
-		esi;
+		set beresp.do_esi = true;
 	}
 } -start -cliok "param.set esi_syntax 4"
 

Modified: trunk/varnish-cache/bin/varnishtest/tests/e00002.vtc
===================================================================
--- trunk/varnish-cache/bin/varnishtest/tests/e00002.vtc	2011-01-11 11:01:47 UTC (rev 5712)
+++ trunk/varnish-cache/bin/varnishtest/tests/e00002.vtc	2011-01-11 11:24:39 UTC (rev 5713)
@@ -16,7 +16,7 @@
 
 varnish v1 -vcl+backend {
 	sub vcl_fetch {
-		esi;
+		set beresp.do_esi = true;
 	}
 } -start
 

Modified: trunk/varnish-cache/bin/varnishtest/tests/e00003.vtc
===================================================================
--- trunk/varnish-cache/bin/varnishtest/tests/e00003.vtc	2011-01-11 11:01:47 UTC (rev 5712)
+++ trunk/varnish-cache/bin/varnishtest/tests/e00003.vtc	2011-01-11 11:24:39 UTC (rev 5713)
@@ -25,7 +25,7 @@
 		}
 	}
 	sub vcl_fetch {
-		esi;
+		set beresp.do_esi = true;
 	}
 } -start
 

Modified: trunk/varnish-cache/bin/varnishtest/tests/e00004.vtc
===================================================================
--- trunk/varnish-cache/bin/varnishtest/tests/e00004.vtc	2011-01-11 11:01:47 UTC (rev 5712)
+++ trunk/varnish-cache/bin/varnishtest/tests/e00004.vtc	2011-01-11 11:24:39 UTC (rev 5713)
@@ -20,7 +20,7 @@
 
 varnish v1 -vcl+backend {
 	sub vcl_fetch {
-		esi;
+		set beresp.do_esi = true;
 	}
 } -start
 

Modified: trunk/varnish-cache/bin/varnishtest/tests/e00005.vtc
===================================================================
--- trunk/varnish-cache/bin/varnishtest/tests/e00005.vtc	2011-01-11 11:01:47 UTC (rev 5712)
+++ trunk/varnish-cache/bin/varnishtest/tests/e00005.vtc	2011-01-11 11:24:39 UTC (rev 5713)
@@ -21,7 +21,7 @@
 
 varnish v1 -vcl+backend {
 	sub vcl_fetch {
-		esi;
+		set beresp.do_esi = true;
 	}
 } -start
 

Modified: trunk/varnish-cache/bin/varnishtest/tests/e00006.vtc
===================================================================
--- trunk/varnish-cache/bin/varnishtest/tests/e00006.vtc	2011-01-11 11:01:47 UTC (rev 5712)
+++ trunk/varnish-cache/bin/varnishtest/tests/e00006.vtc	2011-01-11 11:24:39 UTC (rev 5713)
@@ -32,7 +32,7 @@
 		}
 	}
 	sub vcl_fetch {
-		esi;
+		set beresp.do_esi = true;
 	}
 } -start
 

Modified: trunk/varnish-cache/bin/varnishtest/tests/e00007.vtc
===================================================================
--- trunk/varnish-cache/bin/varnishtest/tests/e00007.vtc	2011-01-11 11:01:47 UTC (rev 5712)
+++ trunk/varnish-cache/bin/varnishtest/tests/e00007.vtc	2011-01-11 11:24:39 UTC (rev 5713)
@@ -35,7 +35,7 @@
 
 varnish v1 -storage "-smalloc,2m" -vcl+backend {
 	sub vcl_fetch {
-		esi;
+		set beresp.do_esi = true;
 	}
 } -start
 

Modified: trunk/varnish-cache/bin/varnishtest/tests/e00008.vtc
===================================================================
--- trunk/varnish-cache/bin/varnishtest/tests/e00008.vtc	2011-01-11 11:01:47 UTC (rev 5712)
+++ trunk/varnish-cache/bin/varnishtest/tests/e00008.vtc	2011-01-11 11:24:39 UTC (rev 5713)
@@ -60,7 +60,7 @@
 
 varnish v1 -vcl+backend {
 	sub vcl_fetch {
-		esi;
+		set beresp.do_esi = true;
 	}
 } -start 
 

Modified: trunk/varnish-cache/bin/varnishtest/tests/e00009.vtc
===================================================================
--- trunk/varnish-cache/bin/varnishtest/tests/e00009.vtc	2011-01-11 11:01:47 UTC (rev 5712)
+++ trunk/varnish-cache/bin/varnishtest/tests/e00009.vtc	2011-01-11 11:24:39 UTC (rev 5713)
@@ -23,7 +23,7 @@
 
 varnish v1 -vcl+backend {
 	sub vcl_fetch {
-		esi;
+		set beresp.do_esi = true;
 	}
 } -start
 

Modified: trunk/varnish-cache/bin/varnishtest/tests/e00010.vtc
===================================================================
--- trunk/varnish-cache/bin/varnishtest/tests/e00010.vtc	2011-01-11 11:01:47 UTC (rev 5712)
+++ trunk/varnish-cache/bin/varnishtest/tests/e00010.vtc	2011-01-11 11:24:39 UTC (rev 5713)
@@ -15,7 +15,7 @@
 
 varnish v1 -vcl+backend {
 	sub vcl_fetch {
-		esi;
+		set beresp.do_esi = true;
 	}
 } -start -cli "param.set esi_syntax 2"
 

Modified: trunk/varnish-cache/bin/varnishtest/tests/e00011.vtc
===================================================================
--- trunk/varnish-cache/bin/varnishtest/tests/e00011.vtc	2011-01-11 11:01:47 UTC (rev 5712)
+++ trunk/varnish-cache/bin/varnishtest/tests/e00011.vtc	2011-01-11 11:24:39 UTC (rev 5713)
@@ -19,7 +19,7 @@
 
 varnish v1 -vcl+backend {
 	sub vcl_fetch {
-		esi;
+		set beresp.do_esi = true;
 		return(pass);
 	}
 } -start 

Modified: trunk/varnish-cache/bin/varnishtest/tests/e00012.vtc
===================================================================
--- trunk/varnish-cache/bin/varnishtest/tests/e00012.vtc	2011-01-11 11:01:47 UTC (rev 5712)
+++ trunk/varnish-cache/bin/varnishtest/tests/e00012.vtc	2011-01-11 11:24:39 UTC (rev 5713)
@@ -20,7 +20,7 @@
 
 varnish v1 -vcl+backend {
         sub vcl_fetch {
-                esi;
+                set beresp.do_esi = true;
         }
 } -start
 

Modified: trunk/varnish-cache/bin/varnishtest/tests/e00013.vtc
===================================================================
--- trunk/varnish-cache/bin/varnishtest/tests/e00013.vtc	2011-01-11 11:01:47 UTC (rev 5712)
+++ trunk/varnish-cache/bin/varnishtest/tests/e00013.vtc	2011-01-11 11:24:39 UTC (rev 5713)
@@ -11,7 +11,7 @@
 
 varnish v1 -vcl+backend {
         sub vcl_fetch {
-                esi;
+                set beresp.do_esi = true;
         }
 } -start
 

Modified: trunk/varnish-cache/bin/varnishtest/tests/e00014.vtc
===================================================================
--- trunk/varnish-cache/bin/varnishtest/tests/e00014.vtc	2011-01-11 11:01:47 UTC (rev 5712)
+++ trunk/varnish-cache/bin/varnishtest/tests/e00014.vtc	2011-01-11 11:24:39 UTC (rev 5713)
@@ -13,7 +13,7 @@
 
 varnish v1 -vcl+backend {
         sub vcl_fetch {
-                esi;
+                set beresp.do_esi = true;
         }
 } -start
 

Modified: trunk/varnish-cache/bin/varnishtest/tests/e00015.vtc
===================================================================
--- trunk/varnish-cache/bin/varnishtest/tests/e00015.vtc	2011-01-11 11:01:47 UTC (rev 5712)
+++ trunk/varnish-cache/bin/varnishtest/tests/e00015.vtc	2011-01-11 11:24:39 UTC (rev 5713)
@@ -29,7 +29,7 @@
 	    	if(req.url == "/") {
  	    	   set req.esi = false;
 		}
-		esi;
+		set beresp.do_esi = true;
 	}
 } -start
 

Modified: trunk/varnish-cache/bin/varnishtest/tests/e00016.vtc
===================================================================
--- trunk/varnish-cache/bin/varnishtest/tests/e00016.vtc	2011-01-11 11:01:47 UTC (rev 5712)
+++ trunk/varnish-cache/bin/varnishtest/tests/e00016.vtc	2011-01-11 11:24:39 UTC (rev 5713)
@@ -34,7 +34,7 @@
 	    	if(req.url == "/body") {
 		   set req.esi = false;
 		} 
-		esi;
+		set beresp.do_esi = true;
 	}
 } -start
 

Modified: trunk/varnish-cache/bin/varnishtest/tests/e00017.vtc
===================================================================
--- trunk/varnish-cache/bin/varnishtest/tests/e00017.vtc	2011-01-11 11:01:47 UTC (rev 5712)
+++ trunk/varnish-cache/bin/varnishtest/tests/e00017.vtc	2011-01-11 11:24:39 UTC (rev 5713)
@@ -74,7 +74,7 @@
 
 varnish v1 -vcl+backend {
 	sub vcl_fetch {
-		esi;
+		set beresp.do_esi = true;
 	}
 } -start
 

Modified: trunk/varnish-cache/bin/varnishtest/tests/e00018.vtc
===================================================================
--- trunk/varnish-cache/bin/varnishtest/tests/e00018.vtc	2011-01-11 11:01:47 UTC (rev 5712)
+++ trunk/varnish-cache/bin/varnishtest/tests/e00018.vtc	2011-01-11 11:24:39 UTC (rev 5713)
@@ -39,7 +39,7 @@
 		return (pass);
 	}
 	sub vcl_fetch {
-		esi;
+		set beresp.do_esi = true;
 	}
 } -start
 

Modified: trunk/varnish-cache/bin/varnishtest/tests/g00002.vtc
===================================================================
--- trunk/varnish-cache/bin/varnishtest/tests/g00002.vtc	2011-01-11 11:01:47 UTC (rev 5712)
+++ trunk/varnish-cache/bin/varnishtest/tests/g00002.vtc	2011-01-11 11:24:39 UTC (rev 5713)
@@ -16,7 +16,7 @@
 varnish v1 -cliok "param.set http_gzip_support true" -vcl+backend {
 
 	sub vcl_fetch {
-		esi;
+		set beresp.do_esi = true;
 	}
 } -start
 

Modified: trunk/varnish-cache/bin/varnishtest/tests/r00318.vtc
===================================================================
--- trunk/varnish-cache/bin/varnishtest/tests/r00318.vtc	2011-01-11 11:01:47 UTC (rev 5712)
+++ trunk/varnish-cache/bin/varnishtest/tests/r00318.vtc	2011-01-11 11:24:39 UTC (rev 5713)
@@ -9,7 +9,7 @@
 
 varnish v1 -vcl+backend {
 	sub vcl_fetch {
-		esi;
+		set beresp.do_esi = true;
 		return(pass);
 	}
 } -start

Modified: trunk/varnish-cache/bin/varnishtest/tests/r00326.vtc
===================================================================
--- trunk/varnish-cache/bin/varnishtest/tests/r00326.vtc	2011-01-11 11:01:47 UTC (rev 5712)
+++ trunk/varnish-cache/bin/varnishtest/tests/r00326.vtc	2011-01-11 11:24:39 UTC (rev 5713)
@@ -11,7 +11,7 @@
 
 varnish v1 -vcl+backend {
 	sub vcl_fetch {
-		esi;
+		set beresp.do_esi = true;
 	}
 } -start
 

Modified: trunk/varnish-cache/bin/varnishtest/tests/r00345.vtc
===================================================================
--- trunk/varnish-cache/bin/varnishtest/tests/r00345.vtc	2011-01-11 11:01:47 UTC (rev 5712)
+++ trunk/varnish-cache/bin/varnishtest/tests/r00345.vtc	2011-01-11 11:24:39 UTC (rev 5713)
@@ -14,7 +14,7 @@
 varnish v1 -arg "-p diag_bitmap=0x20" -vcl+backend {
 	sub vcl_fetch {
 		if (req.url == "/") {
-			esi;
+			set beresp.do_esi = true;
 		}
 	}
 } -start

Modified: trunk/varnish-cache/bin/varnishtest/tests/r00386.vtc
===================================================================
--- trunk/varnish-cache/bin/varnishtest/tests/r00386.vtc	2011-01-11 11:01:47 UTC (rev 5712)
+++ trunk/varnish-cache/bin/varnishtest/tests/r00386.vtc	2011-01-11 11:24:39 UTC (rev 5713)
@@ -14,7 +14,7 @@
 varnish v1 -arg "-p diag_bitmap=0x20" -vcl+backend {
 	sub vcl_fetch {
 		if (req.url == "/") {
-			esi;
+			set beresp.do_esi = true;
 		}
 	}
 } -start

Modified: trunk/varnish-cache/bin/varnishtest/tests/r00427.vtc
===================================================================
--- trunk/varnish-cache/bin/varnishtest/tests/r00427.vtc	2011-01-11 11:01:47 UTC (rev 5712)
+++ trunk/varnish-cache/bin/varnishtest/tests/r00427.vtc	2011-01-11 11:24:39 UTC (rev 5713)
@@ -27,7 +27,7 @@
 
 varnish v1 -vcl+backend {
 	sub vcl_fetch {
-		esi;
+		set beresp.do_esi = true;
 	}
 } -start
 

Modified: trunk/varnish-cache/bin/varnishtest/tests/r00433.vtc
===================================================================
--- trunk/varnish-cache/bin/varnishtest/tests/r00433.vtc	2011-01-11 11:01:47 UTC (rev 5712)
+++ trunk/varnish-cache/bin/varnishtest/tests/r00433.vtc	2011-01-11 11:24:39 UTC (rev 5713)
@@ -34,7 +34,7 @@
 	}
 
 	sub vcl_fetch {
-		esi;
+		set beresp.do_esi = true;
 	}
 } -start
 

Modified: trunk/varnish-cache/bin/varnishtest/tests/r00445.vtc
===================================================================
--- trunk/varnish-cache/bin/varnishtest/tests/r00445.vtc	2011-01-11 11:01:47 UTC (rev 5712)
+++ trunk/varnish-cache/bin/varnishtest/tests/r00445.vtc	2011-01-11 11:24:39 UTC (rev 5713)
@@ -13,7 +13,7 @@
 
 varnish v1 -vcl+backend {
 	sub vcl_fetch {
-		esi;
+		set beresp.do_esi = true;
 	}
 } -start
 

Modified: trunk/varnish-cache/bin/varnishtest/tests/r00476.vtc
===================================================================
--- trunk/varnish-cache/bin/varnishtest/tests/r00476.vtc	2011-01-11 11:01:47 UTC (rev 5712)
+++ trunk/varnish-cache/bin/varnishtest/tests/r00476.vtc	2011-01-11 11:24:39 UTC (rev 5713)
@@ -19,7 +19,7 @@
 
 varnish v1 -vcl+backend {
 	sub vcl_fetch {
-		esi;
+		set beresp.do_esi = true;
 	}
 } -start
 

Modified: trunk/varnish-cache/bin/varnishtest/tests/r00524.vtc
===================================================================
--- trunk/varnish-cache/bin/varnishtest/tests/r00524.vtc	2011-01-11 11:01:47 UTC (rev 5712)
+++ trunk/varnish-cache/bin/varnishtest/tests/r00524.vtc	2011-01-11 11:24:39 UTC (rev 5713)
@@ -21,7 +21,7 @@
 		// return (pass);
 	}
 	sub vcl_fetch {
-		esi;
+		set beresp.do_esi = true;
 	}
 } -cliok "param.set sess_timeout 60" -start
 

Modified: trunk/varnish-cache/bin/varnishtest/tests/r00590.vtc
===================================================================
--- trunk/varnish-cache/bin/varnishtest/tests/r00590.vtc	2011-01-11 11:01:47 UTC (rev 5712)
+++ trunk/varnish-cache/bin/varnishtest/tests/r00590.vtc	2011-01-11 11:24:39 UTC (rev 5713)
@@ -63,7 +63,7 @@
 		return (pass);
 	}
 	sub vcl_fetch {
-		esi;
+		set beresp.do_esi = true;
 	}
 } -start
 

Modified: trunk/varnish-cache/bin/varnishtest/tests/r00612.vtc
===================================================================
--- trunk/varnish-cache/bin/varnishtest/tests/r00612.vtc	2011-01-11 11:01:47 UTC (rev 5712)
+++ trunk/varnish-cache/bin/varnishtest/tests/r00612.vtc	2011-01-11 11:24:39 UTC (rev 5713)
@@ -39,7 +39,7 @@
 		return (pass);
 	}
 	sub vcl_fetch {
-		esi;
+		set beresp.do_esi = true;
 	}
 } -start
 

Modified: trunk/varnish-cache/bin/varnishtest/tests/r00641.vtc
===================================================================
--- trunk/varnish-cache/bin/varnishtest/tests/r00641.vtc	2011-01-11 11:01:47 UTC (rev 5712)
+++ trunk/varnish-cache/bin/varnishtest/tests/r00641.vtc	2011-01-11 11:24:39 UTC (rev 5713)
@@ -18,7 +18,7 @@
 		// return (pass);
 	}
 	sub vcl_fetch {
-		esi;
+		set beresp.do_esi = true;
 	}
 } -cliok "param.set sess_timeout 60" -start
 

Modified: trunk/varnish-cache/lib/libvcl/generate.py
===================================================================
--- trunk/varnish-cache/lib/libvcl/generate.py	2011-01-11 11:01:47 UTC (rev 5712)
+++ trunk/varnish-cache/lib/libvcl/generate.py	2011-01-11 11:24:39 UTC (rev 5713)
@@ -298,19 +298,19 @@
 		( 'fetch',),
 		'const struct sess *'
 	),
-	('beresp.esi',
+	('beresp.do_esi',
 		'BOOL',
 		( 'fetch',),
 		( 'fetch',),
 		'const struct sess *'
 	),
-	('beresp.gzip',
+	('beresp.do_gzip',
 		'BOOL',
 		( 'fetch',),
 		( 'fetch',),
 		'const struct sess *'
 	),
-	('beresp.gunzip',
+	('beresp.do_gunzip',
 		'BOOL',
 		( 'fetch',),
 		( 'fetch',),

Modified: trunk/varnish-cache/lib/libvcl/vcc_action.c
===================================================================
--- trunk/varnish-cache/lib/libvcl/vcc_action.c	2011-01-11 11:01:47 UTC (rev 5712)
+++ trunk/varnish-cache/lib/libvcl/vcc_action.c	2011-01-11 11:24:39 UTC (rev 5713)
@@ -283,16 +283,6 @@
 /*--------------------------------------------------------------------*/
 
 static void
-parse_esi(struct vcc *tl)
-{
-
-	vcc_NextToken(tl);
-	Fb(tl, 1, "VRT_ESI(sp);\n");
-}
-
-/*--------------------------------------------------------------------*/
-
-static void
 parse_new_syntax(struct vcc *tl)
 {
 
@@ -402,7 +392,6 @@
 
 	/* Keep list sorted from here */
 	{ "call",		parse_call },
-	{ "esi",		parse_esi, VCL_MET_FETCH },
 	{ "hash_data",		parse_hash_data, VCL_MET_HASH },
 	{ "panic",		parse_panic },
 	{ "ban",		parse_ban },




From phk at varnish-cache.org  Tue Jan 11 12:00:43 2011
From: phk at varnish-cache.org (phk at varnish-cache.org)
Date: Tue, 11 Jan 2011 13:00:43 +0100
Subject: r5714 - trunk/varnish-cache/bin/varnishd
Message-ID: <E1Pccu3-0004Wj-P2@project.varnish-software.com>

Author: phk
Date: 2011-01-11 13:00:42 +0100 (Tue, 11 Jan 2011)
New Revision: 5714

Modified:
   trunk/varnish-cache/bin/varnishd/cache.h
   trunk/varnish-cache/bin/varnishd/cache_center.c
   trunk/varnish-cache/bin/varnishd/cache_fetch.c
   trunk/varnish-cache/bin/varnishd/cache_gzip.c
Log:
Un-const the sp argument to vpf methods



Modified: trunk/varnish-cache/bin/varnishd/cache.h
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache.h	2011-01-11 11:24:39 UTC (rev 5713)
+++ trunk/varnish-cache/bin/varnishd/cache.h	2011-01-11 12:00:42 UTC (rev 5714)
@@ -213,9 +213,9 @@
 
 /* Fetch processors --------------------------------------------------*/
 
-typedef void vfp_begin_f(const struct sess *, size_t );
-typedef int vfp_bytes_f(const struct sess *, struct http_conn *, size_t);
-typedef int vfp_end_f(const struct sess *sp);
+typedef void vfp_begin_f(struct sess *, size_t );
+typedef int vfp_bytes_f(struct sess *, struct http_conn *, size_t);
+typedef int vfp_end_f(struct sess *sp);
 
 struct vfp {
 	vfp_begin_f	*begin;
@@ -279,6 +279,7 @@
 	enum body_status	body_status;
 	struct storage		*storage;
 	struct vfp		*vfp;
+	void			*vfp_private;
 	unsigned		do_esi;
 	unsigned		do_gzip;
 	unsigned		do_gunzip;

Modified: trunk/varnish-cache/bin/varnishd/cache_center.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_center.c	2011-01-11 11:24:39 UTC (rev 5713)
+++ trunk/varnish-cache/bin/varnishd/cache_center.c	2011-01-11 12:00:42 UTC (rev 5714)
@@ -598,8 +598,7 @@
 	l += strlen("Content-Encoding: XxxXxxXxxXxxXxxXxx" + sizeof(void *));
 
 	/*
-	 * XXX: If we have a Length: header, we should allocate the body
-	 * XXX: also.
+	 * XXX: VFP's may affect estimate
 	 */
 
 	sp->obj = STV_NewObject(sp, sp->wrk->storage_hint, l,
@@ -644,6 +643,11 @@
 		sp->obj->last_modified = sp->wrk->entered;
 
 	AZ(sp->wrk->vfp);
+	/* XXX: precedence, also: do_esi */
+	if (sp->wrk->do_gunzip)
+		sp->wrk->vfp = &vfp_gunzip;
+	else if (sp->wrk->do_gzip)
+		sp->wrk->vfp = &vfp_gzip;
 	i = FetchBody(sp);
 	sp->wrk->vfp = NULL;
 	AZ(sp->wrk->wfd);

Modified: trunk/varnish-cache/bin/varnishd/cache_fetch.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_fetch.c	2011-01-11 11:24:39 UTC (rev 5713)
+++ trunk/varnish-cache/bin/varnishd/cache_fetch.c	2011-01-11 12:00:42 UTC (rev 5714)
@@ -61,7 +61,7 @@
  * as seen on the socket, or zero if unknown.
  */
 static void
-vfp_nop_begin(const struct sess *sp, size_t estimate)
+vfp_nop_begin(struct sess *sp, size_t estimate)
 {
 
 	AZ(sp->wrk->storage);
@@ -85,7 +85,7 @@
  */
 
 static int
-vfp_nop_bytes(const struct sess *sp, struct http_conn *htc, size_t bytes)
+vfp_nop_bytes(struct sess *sp, struct http_conn *htc, size_t bytes)
 {
 	ssize_t l, w;
 	struct storage *st;
@@ -131,7 +131,7 @@
  */
 
 static int
-vfp_nop_end(const struct sess *sp)
+vfp_nop_end(struct sess *sp)
 {
 	struct storage *st;
 
@@ -182,7 +182,7 @@
 /*--------------------------------------------------------------------*/
 
 static int
-fetch_straight(const struct sess *sp, struct http_conn *htc, const char *b)
+fetch_straight(struct sess *sp, struct http_conn *htc, const char *b)
 {
 	int i;
 	ssize_t cl;
@@ -220,7 +220,7 @@
 	} while (0)
 
 static int
-fetch_chunked(const struct sess *sp, struct http_conn *htc)
+fetch_chunked(struct sess *sp, struct http_conn *htc)
 {
 	int i;
 	char buf[20];		/* XXX: 20 is arbitrary */
@@ -290,7 +290,7 @@
 /*--------------------------------------------------------------------*/
 
 static int
-fetch_eof(const struct sess *sp, struct http_conn *htc)
+fetch_eof(struct sess *sp, struct http_conn *htc)
 {
 	int i;
 

Modified: trunk/varnish-cache/bin/varnishd/cache_gzip.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_gzip.c	2011-01-11 11:24:39 UTC (rev 5713)
+++ trunk/varnish-cache/bin/varnishd/cache_gzip.c	2011-01-11 12:00:42 UTC (rev 5714)
@@ -127,7 +127,7 @@
 }
 
 /*--------------------------------------------------------------------*/
-#include <stdio.h>
+
 int
 VGZ_Produce(struct vgz *vg, const void **pptr, size_t *plen)
 {
@@ -151,7 +151,6 @@
 		return (1);
 	if (i == Z_BUF_ERROR)
 		return (2);
-fprintf(stderr, "--------------------> GUNZIP = %d\n", i);
 	return (-1);
 }
 
@@ -175,25 +174,29 @@
  */
 
 static void
-vfp_gunzip_begin(const struct sess *sp, size_t estimate)
+vfp_gunzip_begin(struct sess *sp, size_t estimate)
 {
-	(void)sp;
 	(void)estimate;
+	sp->wrk->vfp_private = VGZ_NewUnzip(sp, sp->ws, sp->wrk->ws);
 }
 
 static int
-vfp_gunzip_bytes(const struct sess *sp, struct http_conn *htc, size_t bytes)
+vfp_gunzip_bytes(struct sess *sp, struct http_conn *htc, size_t bytes)
 {
-	(void)sp;
+	struct vgz *vgz;
+
+	CAST_OBJ_NOTNULL(vgz, sp->wrk->vfp_private, VGZ_MAGIC);
 	(void)htc;
 	(void)bytes;
 	return (-1);
 }
 
 static int
-vfp_gunzip_end(const struct sess *sp)
+vfp_gunzip_end(struct sess *sp)
 {
-	(void)sp;
+	struct vgz *vgz;
+
+	CAST_OBJ_NOTNULL(vgz, sp->wrk->vfp_private, VGZ_MAGIC);
 	return (-1);
 }
 
@@ -211,14 +214,14 @@
  */
 
 static void
-vfp_gzip_begin(const struct sess *sp, size_t estimate)
+vfp_gzip_begin(struct sess *sp, size_t estimate)
 {
 	(void)sp;
 	(void)estimate;
 }
 
 static int
-vfp_gzip_bytes(const struct sess *sp, struct http_conn *htc, size_t bytes)
+vfp_gzip_bytes(struct sess *sp, struct http_conn *htc, size_t bytes)
 {
 	(void)sp;
 	(void)htc;
@@ -227,7 +230,7 @@
 }
 
 static int
-vfp_gzip_end(const struct sess *sp)
+vfp_gzip_end(struct sess *sp)
 {
 	(void)sp;
 	return (-1);




From phk at varnish-cache.org  Tue Jan 11 13:12:34 2011
From: phk at varnish-cache.org (phk at varnish-cache.org)
Date: Tue, 11 Jan 2011 14:12:34 +0100
Subject: r5715 - in trunk/varnish-cache/bin: varnishd varnishtest/tests
Message-ID: <E1Pce1a-0006aZ-Pb@project.varnish-software.com>

Author: phk
Date: 2011-01-11 14:12:34 +0100 (Tue, 11 Jan 2011)
New Revision: 5715

Added:
   trunk/varnish-cache/bin/varnishtest/tests/g00003.vtc
Modified:
   trunk/varnish-cache/bin/varnishd/cache_center.c
   trunk/varnish-cache/bin/varnishd/cache_fetch.c
   trunk/varnish-cache/bin/varnishd/cache_gzip.c
   trunk/varnish-cache/bin/varnishd/cache_response.c
Log:
Flesh out the gzip and gunzip fetch processors.

Now you can get varnish to gzip or gunzip objects as they are
received from the backend by setting

	sub vcl_fetch {
		beresp.do_gzip = true;
		beresp.do_guzip = true;
	}

They will only do something if the object is/isn't gzip'ed already
(according to the beresp.http.Content-Encoding header) so the above
silly-ish example will compress al uncompressed objects and decompress
all compressed objects.

NB: Lots of errorchecks not implemented yet.



Modified: trunk/varnish-cache/bin/varnishd/cache_center.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_center.c	2011-01-11 12:00:42 UTC (rev 5714)
+++ trunk/varnish-cache/bin/varnishd/cache_center.c	2011-01-11 13:12:34 UTC (rev 5715)
@@ -585,6 +585,22 @@
 		AZ(sp->objcore);
 	}
 
+	AZ(sp->wrk->vfp);
+	/* XXX: precedence, also: do_esi */
+
+	if (sp->wrk->do_gunzip &&
+	    http_HdrIs(sp->wrk->beresp, H_Content_Encoding, "gzip")) {
+		http_Unset(sp->wrk->beresp, H_Content_Encoding);
+		sp->wrk->vfp = &vfp_gunzip;
+	}
+
+	if (sp->wrk->do_gzip &&
+	    !http_HdrIs(sp->wrk->beresp, H_Content_Encoding, "gzip")) {
+		http_PrintfHeader(sp->wrk, sp->fd, sp->wrk->beresp,
+		    "Content-Encoding: %s", "gzip");
+		sp->wrk->vfp = &vfp_gzip;
+	}
+
 	l = http_EstimateWS(sp->wrk->beresp,
 	    sp->pass ? HTTPH_R_PASS : HTTPH_A_INS, &nhttp);
 
@@ -642,12 +658,6 @@
 	else
 		sp->obj->last_modified = sp->wrk->entered;
 
-	AZ(sp->wrk->vfp);
-	/* XXX: precedence, also: do_esi */
-	if (sp->wrk->do_gunzip)
-		sp->wrk->vfp = &vfp_gunzip;
-	else if (sp->wrk->do_gzip)
-		sp->wrk->vfp = &vfp_gzip;
 	i = FetchBody(sp);
 	sp->wrk->vfp = NULL;
 	AZ(sp->wrk->wfd);

Modified: trunk/varnish-cache/bin/varnishd/cache_fetch.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_fetch.c	2011-01-11 12:00:42 UTC (rev 5714)
+++ trunk/varnish-cache/bin/varnishd/cache_fetch.c	2011-01-11 13:12:34 UTC (rev 5715)
@@ -60,7 +60,7 @@
  * 'estimate' is the estimate of the number of bytes we expect to receive,
  * as seen on the socket, or zero if unknown.
  */
-static void
+static void __match_proto__()
 vfp_nop_begin(struct sess *sp, size_t estimate)
 {
 
@@ -84,7 +84,7 @@
  * Return 1 when 'bytes' have been processed.
  */
 
-static int
+static int __match_proto__()
 vfp_nop_bytes(struct sess *sp, struct http_conn *htc, size_t bytes)
 {
 	ssize_t l, w;
@@ -130,7 +130,7 @@
  * Return 0 for OK
  */
 
-static int
+static int __match_proto__()
 vfp_nop_end(struct sess *sp)
 {
 	struct storage *st;

Modified: trunk/varnish-cache/bin/varnishd/cache_gzip.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_gzip.c	2011-01-11 12:00:42 UTC (rev 5714)
+++ trunk/varnish-cache/bin/varnishd/cache_gzip.c	2011-01-11 13:12:34 UTC (rev 5715)
@@ -41,6 +41,7 @@
 SVNID("$Id$")
 
 #include "cache.h"
+#include "stevedore.h"
 
 #include "zlib.h"
 
@@ -52,7 +53,7 @@
 	char			*tmp_snapshot;
 
 	struct ws		*buf;
-	unsigned		bufsiz;
+	size_t			bufsiz;
 
 	z_stream		vz;
 };
@@ -173,31 +174,86 @@
  * A VFP for gunzip'ing an object as we receive it from the backend
  */
 
-static void
+static void __match_proto__()
 vfp_gunzip_begin(struct sess *sp, size_t estimate)
 {
 	(void)estimate;
 	sp->wrk->vfp_private = VGZ_NewUnzip(sp, sp->ws, sp->wrk->ws);
 }
 
-static int
+static int __match_proto__()
 vfp_gunzip_bytes(struct sess *sp, struct http_conn *htc, size_t bytes)
 {
-	struct vgz *vgz;
+	struct vgz *vg;
+	struct storage *st;
+	ssize_t l, w;
+	int i = -100;
 
-	CAST_OBJ_NOTNULL(vgz, sp->wrk->vfp_private, VGZ_MAGIC);
-	(void)htc;
-	(void)bytes;
+	CAST_OBJ_NOTNULL(vg, sp->wrk->vfp_private, VGZ_MAGIC);
+	AZ(vg->vz.avail_in);
+	while (bytes > 0 || vg->vz.avail_in > 0) {
+		if (sp->wrk->storage == NULL)
+			sp->wrk->storage = STV_alloc(sp,
+			    params->fetch_chunksize * 1024LL);
+		if (sp->wrk->storage == NULL) {
+			errno = ENOMEM;
+			return (-1);
+		}
+		st = sp->wrk->storage;
+
+		vg->vz.next_out = st->ptr + st->len;
+		vg->vz.avail_out = st->space - st->len;
+
+		if (vg->vz.avail_in == 0 && bytes > 0) {
+			l = vg->bufsiz;
+			if (l > bytes)
+				l = bytes;
+			w = HTC_Read(htc, vg->buf->f, l);
+			if (w <= 0)
+				return (w);
+			vg->vz.next_in = (void*)vg->buf->f;
+			vg->vz.avail_in = w;
+			bytes -= w;
+		}
+
+		i = inflate(&vg->vz, 0);
+		assert(i == Z_OK || i == Z_STREAM_END);
+		st->len = st->space - vg->vz.avail_out;
+		if (st->len == st->space) {
+			VTAILQ_INSERT_TAIL(&sp->obj->store,
+			    sp->wrk->storage, list);
+			sp->obj->len += st->len;
+			sp->wrk->storage = NULL;
+		}
+	}
+	if (i == Z_STREAM_END)
+		return (1);
 	return (-1);
 }
 
-static int
+static int __match_proto__()
 vfp_gunzip_end(struct sess *sp)
 {
-	struct vgz *vgz;
+	struct vgz *vg;
+	struct storage *st;
 
-	CAST_OBJ_NOTNULL(vgz, sp->wrk->vfp_private, VGZ_MAGIC);
-	return (-1);
+	CAST_OBJ_NOTNULL(vg, sp->wrk->vfp_private, VGZ_MAGIC);
+	VGZ_Destroy(&vg);
+
+	st = sp->wrk->storage;
+	sp->wrk->storage = NULL;
+	if (st == NULL)
+		return (0);
+
+	if (st->len == 0) {
+		STV_free(st);
+		return (0);
+	}
+	if (st->len < st->space)
+		STV_trim(st, st->len);
+	sp->obj->len += st->len;
+	VTAILQ_INSERT_TAIL(&sp->obj->store, st, list);
+	return (0);
 }
 
 struct vfp vfp_gunzip = {
@@ -213,27 +269,98 @@
  * A VFP for gzip'ing an object as we receive it from the backend
  */
 
-static void
+static void __match_proto__()
 vfp_gzip_begin(struct sess *sp, size_t estimate)
 {
-	(void)sp;
+	struct vgz *vg;
 	(void)estimate;
+
+	vg = VGZ_NewUnzip(sp, sp->ws, sp->wrk->ws);
+	/* XXX: hack */
+	memset(&vg->vz, 0, sizeof vg->vz);
+	assert(Z_OK == deflateInit2(&vg->vz,
+	    0,
+	    Z_DEFLATED,
+	    31,
+	    9,
+	    Z_DEFAULT_STRATEGY));
+
+	sp->wrk->vfp_private = vg;
 }
 
-static int
+static int __match_proto__()
 vfp_gzip_bytes(struct sess *sp, struct http_conn *htc, size_t bytes)
 {
-	(void)sp;
-	(void)htc;
-	(void)bytes;
+	struct vgz *vg;
+	struct storage *st;
+	ssize_t l, w;
+	int i = -100;
+
+	CAST_OBJ_NOTNULL(vg, sp->wrk->vfp_private, VGZ_MAGIC);
+	AZ(vg->vz.avail_in);
+	while (bytes > 0 || vg->vz.avail_in > 0) {
+		if (sp->wrk->storage == NULL)
+			sp->wrk->storage = STV_alloc(sp,
+			    params->fetch_chunksize * 1024LL);
+		if (sp->wrk->storage == NULL) {
+			errno = ENOMEM;
+			return (-1);
+		}
+		st = sp->wrk->storage;
+
+		vg->vz.next_out = st->ptr + st->len;
+		vg->vz.avail_out = st->space - st->len;
+
+		if (vg->vz.avail_in == 0 && bytes > 0) {
+			l = vg->bufsiz;
+			if (l > bytes)
+				l = bytes;
+			w = HTC_Read(htc, vg->buf->f, l);
+			if (w <= 0)
+				return (w);
+			vg->vz.next_in = (void*)vg->buf->f;
+			vg->vz.avail_in = w;
+			bytes -= w;
+		}
+
+		i = deflate(&vg->vz, bytes == 0 ? Z_FINISH : 0);
+		assert(i == Z_OK || i == Z_STREAM_END);
+		st->len = st->space - vg->vz.avail_out;
+		if (st->len == st->space) {
+			VTAILQ_INSERT_TAIL(&sp->obj->store,
+			    sp->wrk->storage, list);
+			sp->obj->len += st->len;
+			sp->wrk->storage = NULL;
+		}
+	}
+	if (i == Z_STREAM_END)
+		return (1);
 	return (-1);
 }
 
-static int
+static int __match_proto__()
 vfp_gzip_end(struct sess *sp)
 {
-	(void)sp;
-	return (-1);
+	struct vgz *vg;
+	struct storage *st;
+
+	CAST_OBJ_NOTNULL(vg, sp->wrk->vfp_private, VGZ_MAGIC);
+	VGZ_Destroy(&vg);
+
+	st = sp->wrk->storage;
+	sp->wrk->storage = NULL;
+	if (st == NULL)
+		return (0);
+
+	if (st->len == 0) {
+		STV_free(st);
+		return (0);
+	}
+	if (st->len < st->space)
+		STV_trim(st, st->len);
+	sp->obj->len += st->len;
+	VTAILQ_INSERT_TAIL(&sp->obj->store, st, list);
+	return (0);
 }
 
 struct vfp vfp_gzip = {
@@ -241,4 +368,3 @@
         .bytes  =       vfp_gzip_bytes,
         .end    =       vfp_gzip_end,
 };
-

Modified: trunk/varnish-cache/bin/varnishd/cache_response.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_response.c	2011-01-11 12:00:42 UTC (rev 5714)
+++ trunk/varnish-cache/bin/varnishd/cache_response.c	2011-01-11 13:12:34 UTC (rev 5715)
@@ -236,6 +236,7 @@
 /*--------------------------------------------------------------------
  * We have a gzip'ed object and need to ungzip it for a client which
  * does not understand gzip.
+ * XXX: handle invalid gzip data better (how ?)
  */
 
 static void

Added: trunk/varnish-cache/bin/varnishtest/tests/g00003.vtc
===================================================================
--- trunk/varnish-cache/bin/varnishtest/tests/g00003.vtc	                        (rev 0)
+++ trunk/varnish-cache/bin/varnishtest/tests/g00003.vtc	2011-01-11 13:12:34 UTC (rev 5715)
@@ -0,0 +1,54 @@
+# $Id$
+
+test "test gunzip on fetch"
+
+server s1 {
+	rxreq
+	expect req.url == "/foo"
+	expect req.http.accept-encoding == "gzip"
+	txresp -gziplen 41
+
+	rxreq
+	expect req.url == "/bar"
+	expect req.http.accept-encoding == "gzip"
+	txresp -bodylen 42
+
+	rxreq
+	expect req.url == "/foobar"
+	expect req.http.accept-encoding == "gzip"
+	txresp -bodylen 43
+} -start
+
+varnish v1 -cliok "param.set http_gzip_support true" -vcl+backend {
+
+	sub vcl_fetch {
+		set beresp.do_gunzip = true;
+		if (req.url == "/foobar") {
+			set beresp.do_gzip = true;
+		}
+	}
+} -start
+
+client c1 {
+	txreq -url /foo -hdr "Accept-Encoding: gzip"
+	rxresp
+	expect resp.http.content-encoding == "resp.http.content-encoding"
+	expect resp.bodylen == 41
+
+	txreq -url /bar -hdr "Accept-Encoding: gzip"
+	rxresp
+	expect resp.http.content-encoding == "resp.http.content-encoding"
+	expect resp.bodylen == 42
+
+	txreq -url /foobar -hdr "Accept-Encoding: gzip"
+	rxresp
+	expect resp.http.content-encoding == "gzip"
+	gunzip
+	expect resp.bodylen == 43
+
+	txreq -url /foobar 
+	rxresp
+	expect resp.http.content-encoding == "resp.http.content-encoding"
+	expect resp.bodylen == 43
+} -run
+




From phk at varnish-cache.org  Wed Jan 12 11:38:53 2011
From: phk at varnish-cache.org (phk at varnish-cache.org)
Date: Wed, 12 Jan 2011 12:38:53 +0100
Subject: r5716 - trunk/varnish-cache/bin/varnishd
Message-ID: <E1Pcz2T-0007EN-O7@project.varnish-software.com>

Author: phk
Date: 2011-01-12 12:38:53 +0100 (Wed, 12 Jan 2011)
New Revision: 5716

Added:
   trunk/varnish-cache/bin/varnishd/cache_esi_parse.c
Modified:
   trunk/varnish-cache/bin/varnishd/Makefile.am
   trunk/varnish-cache/bin/varnishd/cache.h
   trunk/varnish-cache/bin/varnishd/cache_center.c
   trunk/varnish-cache/bin/varnishd/cache_esi.c
   trunk/varnish-cache/bin/varnishd/cache_hash.c
   trunk/varnish-cache/bin/varnishd/cache_response.c
Log:
Gzip support for ESI is turning into a new implementation of ESI, so
bite the bullet and do so.

Add an OLD_ESI macro which defines which implementation we use and
set it by default.



Modified: trunk/varnish-cache/bin/varnishd/Makefile.am
===================================================================
--- trunk/varnish-cache/bin/varnishd/Makefile.am	2011-01-11 13:12:34 UTC (rev 5715)
+++ trunk/varnish-cache/bin/varnishd/Makefile.am	2011-01-12 11:38:53 UTC (rev 5716)
@@ -25,6 +25,7 @@
 	cache_dir_random.c \
 	cache_dir_dns.c \
 	cache_dir_round_robin.c \
+	cache_esi_parse.c \
 	cache_esi.c \
 	cache_expire.c \
 	cache_fetch.c \

Modified: trunk/varnish-cache/bin/varnishd/cache.h
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache.h	2011-01-11 13:12:34 UTC (rev 5715)
+++ trunk/varnish-cache/bin/varnishd/cache.h	2011-01-12 11:38:53 UTC (rev 5716)
@@ -35,6 +35,8 @@
  */
 #define VARNISH_CACHE_CHILD	1
 
+#define OLD_ESI 
+
 #include <sys/time.h>
 #include <sys/uio.h>
 #include <sys/socket.h>
@@ -225,6 +227,7 @@
 
 extern struct vfp vfp_gunzip;
 extern struct vfp vfp_gzip;
+extern struct vfp vfp_esi;
 
 /*--------------------------------------------------------------------*/
 
@@ -284,6 +287,9 @@
 	unsigned		do_gzip;
 	unsigned		do_gunzip;
 
+	/* ESI stuff */
+	struct vep_state	*vep;
+
 	/* Timeouts */
 	double			connect_timeout;
 	double			first_byte_timeout;
@@ -436,7 +442,11 @@
 
 	VTAILQ_HEAD(, storage)	store;
 
+#ifdef OLD_ESI
 	struct esidata		*esidata;
+#else
+	struct storage		*esidata;
+#endif
 
 	double			last_use;
 
@@ -795,11 +805,13 @@
 char *VRT_String(struct ws *ws, const char *h, const char *p, va_list ap);
 char *VRT_StringList(char *d, unsigned dl, const char *p, va_list ap);
 
+#ifdef OLD_ESI
 /* cache_vrt_esi.c */
 
 void ESI_Deliver(struct sess *);
 void ESI_Destroy(struct object *);
 void ESI_Parse(struct sess *);
+#endif /* OLD_ESI */
 
 /* cache_vrt_vmod.c */
 void VMOD_Init(void);

Modified: trunk/varnish-cache/bin/varnishd/cache_center.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_center.c	2011-01-11 13:12:34 UTC (rev 5715)
+++ trunk/varnish-cache/bin/varnishd/cache_center.c	2011-01-12 11:38:53 UTC (rev 5716)
@@ -587,14 +587,16 @@
 
 	AZ(sp->wrk->vfp);
 	/* XXX: precedence, also: do_esi */
-
+#ifndef OLD_ESI
+	if (sp->wrk->do_esi) {
+		sp->wrk->vfp = &vfp_esi;
+	} else
+#endif
 	if (sp->wrk->do_gunzip &&
 	    http_HdrIs(sp->wrk->beresp, H_Content_Encoding, "gzip")) {
 		http_Unset(sp->wrk->beresp, H_Content_Encoding);
 		sp->wrk->vfp = &vfp_gunzip;
-	}
-
-	if (sp->wrk->do_gzip &&
+	} else if (sp->wrk->do_gzip &&
 	    !http_HdrIs(sp->wrk->beresp, H_Content_Encoding, "gzip")) {
 		http_PrintfHeader(sp->wrk, sp->fd, sp->wrk->beresp,
 		    "Content-Encoding: %s", "gzip");
@@ -675,8 +677,10 @@
 		return (0);
 	}
 
+#ifdef OLD_ESI
 	if (sp->wrk->do_esi)
 		ESI_Parse(sp);
+#endif
 
 	switch (sp->handling) {
 	case VCL_RET_RESTART:

Modified: trunk/varnish-cache/bin/varnishd/cache_esi.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_esi.c	2011-01-11 13:12:34 UTC (rev 5715)
+++ trunk/varnish-cache/bin/varnishd/cache_esi.c	2011-01-12 11:38:53 UTC (rev 5716)
@@ -38,6 +38,7 @@
  * hanging, esi:include
  */
 
+
 #include "config.h"
 
 #include "svnid.h"
@@ -56,6 +57,8 @@
 #include "cache.h"
 #include "stevedore.h"
 
+#ifdef OLD_ESI
+
 /*--------------------------------------------------------------------*/
 
 struct esi_bit {
@@ -959,3 +962,5 @@
 	CHECK_OBJ_NOTNULL(ed, ESIDATA_MAGIC);
 	STV_free(ed->storage);
 }
+
+#endif /* OLD_ESI */

Added: trunk/varnish-cache/bin/varnishd/cache_esi_parse.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_esi_parse.c	                        (rev 0)
+++ trunk/varnish-cache/bin/varnishd/cache_esi_parse.c	2011-01-12 11:38:53 UTC (rev 5716)
@@ -0,0 +1,138 @@
+/*-
+ * Copyright (c) 2011 Varnish Software AS
+ * All rights reserved.
+ *
+ * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ */
+
+#ifndef OLD_ESI
+#include "config.h"
+
+#include "svnid.h"
+SVNID("$Id")
+
+#include "cache.h"
+#include "stevedore.h"
+
+struct vep_state {
+	unsigned		magic;
+#define VEP_MAGIC		0x55cb9b82
+	vfp_bytes_f		*bytes;
+	struct vsb		*vsb;
+};
+
+/*---------------------------------------------------------------------
+ * We receive a ungzip'ed object, and want to store it ungzip'ed.
+ */
+
+static int __match_proto__()
+vfp_esi_bytes_uu(struct sess *sp, struct http_conn *htc, size_t bytes)
+{
+	struct vep_state *vep;
+	ssize_t l, w;
+	struct storage *st;
+
+	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+	vep = sp->wrk->vep;
+	CHECK_OBJ_NOTNULL(vep, VEP_MAGIC);
+
+	while (bytes > 0) {
+		if (sp->wrk->storage == NULL) {
+			l = params->fetch_chunksize * 1024LL;
+			sp->wrk->storage = STV_alloc(sp, l);
+		}
+		if (sp->wrk->storage == NULL) {
+			errno = ENOMEM;
+			return (-1);
+		}
+		st = sp->wrk->storage;
+		l = st->space - st->len;
+		if (l > bytes)
+			l = bytes;
+		w = HTC_Read(htc, st->ptr + st->len, l);
+		if (w <= 0)
+			return (w);
+		st->len += w;
+		sp->obj->len += w;
+		if (st->len == st->space) {
+			VTAILQ_INSERT_TAIL(&sp->obj->store,
+			    sp->wrk->storage, list);
+			sp->wrk->storage = NULL;
+			st = NULL;
+		}
+		bytes -= w;
+	}
+	return (1);
+}
+
+/*---------------------------------------------------------------------*/
+
+static void __match_proto__()
+vfp_esi_begin(struct sess *sp, size_t estimate)
+{
+	struct vep_state *vep;
+
+	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+	AZ(sp->wrk->vep);
+	vep = (void*)WS_Alloc(sp->wrk->ws, sizeof *vep);
+	AN(vep);
+
+	memset(vep, 0, sizeof *vep);
+	vep->magic = VEP_MAGIC;
+	vep->bytes = vfp_esi_bytes_uu;
+	vep->vsb = vsb_newauto();
+	AN(vep->vsb);
+
+	sp->wrk->vep = vep;
+	(void)estimate;
+}
+
+static int __match_proto__()
+vfp_esi_bytes(struct sess *sp, struct http_conn *htc, size_t bytes)
+{
+	struct vep_state *vep;
+
+	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+	vep = sp->wrk->vep;
+	CHECK_OBJ_NOTNULL(vep, VEP_MAGIC);
+	AN(vep->bytes);
+	return (vep->bytes(sp, htc, bytes));
+}
+
+static int __match_proto__()
+vfp_esi_end(struct sess *sp)
+{
+
+	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+	return (-1);
+}
+
+struct vfp vfp_esi = {
+        .begin  =       vfp_esi_begin,
+        .bytes  =       vfp_esi_bytes,
+        .end    =       vfp_esi_end,
+};
+
+#endif /* OLD_ESI */

Modified: trunk/varnish-cache/bin/varnishd/cache_hash.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_hash.c	2011-01-11 13:12:34 UTC (rev 5715)
+++ trunk/varnish-cache/bin/varnishd/cache_hash.c	2011-01-12 11:38:53 UTC (rev 5716)
@@ -693,8 +693,15 @@
 		DSL(0x40, SLT_Debug, 0, "Object %u workspace min free %u",
 		    o->xid, WS_Free(o->ws_o));
 
+#ifdef OLD_ESI
 		if (o->esidata != NULL)
 			ESI_Destroy(o);
+#else
+		if (o->esidata != NULL) {
+			STV_free(o->esidata);
+			o->esidata = NULL;
+		}
+#endif
 		if (oc != NULL)
 			oc_freeobj(oc);
 		w->stats.n_object--;

Modified: trunk/varnish-cache/bin/varnishd/cache_response.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_response.c	2011-01-11 13:12:34 UTC (rev 5715)
+++ trunk/varnish-cache/bin/varnishd/cache_response.c	2011-01-12 11:38:53 UTC (rev 5716)
@@ -386,8 +386,10 @@
 		/* This was a HEAD request */
 	} else if (sp->obj->len == 0) {
 		/* Nothing to do here */
+#ifdef OLD_ESI
 	} else if (sp->wrk->res_mode & RES_ESI) {
 		ESI_Deliver(sp);
+#endif
 	} else if (sp->wrk->res_mode & RES_GUNZIP) {
 		res_WriteGunzipObj(sp, lenbuf); 
 	} else {




From phk at varnish-cache.org  Wed Jan 12 13:10:09 2011
From: phk at varnish-cache.org (phk at varnish-cache.org)
Date: Wed, 12 Jan 2011 14:10:09 +0100
Subject: r5717 - trunk/varnish-cache/bin/varnishd
Message-ID: <E1Pd0Sn-0001Nb-PV@project.varnish-software.com>

Author: phk
Date: 2011-01-12 14:10:08 +0100 (Wed, 12 Jan 2011)
New Revision: 5717

Added:
   trunk/varnish-cache/bin/varnishd/cache_esi.h
Modified:
   trunk/varnish-cache/bin/varnishd/Makefile.am
   trunk/varnish-cache/bin/varnishd/cache_esi_parse.c
Log:
Attempt to generate the correct VEC string for testcase e00001.vtc



Modified: trunk/varnish-cache/bin/varnishd/Makefile.am
===================================================================
--- trunk/varnish-cache/bin/varnishd/Makefile.am	2011-01-12 11:38:53 UTC (rev 5716)
+++ trunk/varnish-cache/bin/varnishd/Makefile.am	2011-01-12 13:10:08 UTC (rev 5717)
@@ -75,6 +75,7 @@
 	cache_ban.h \
 	cache_backend.h \
 	cache_backend_poll.h \
+	cache_esi.h \
 	cache_waiter.h \
 	common.h \
 	default_vcl.h \

Added: trunk/varnish-cache/bin/varnishd/cache_esi.h
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_esi.h	                        (rev 0)
+++ trunk/varnish-cache/bin/varnishd/cache_esi.h	2011-01-12 13:10:08 UTC (rev 5717)
@@ -0,0 +1,35 @@
+/*-
+ * Copyright (c) 2011 Varnish Software AS
+ * All rights reserved.
+ *
+ * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ */
+
+#define	VEC_V1	'B'
+#define	VEC_V2	'W'
+#define	VEC_V4	'L'
+#define	VEC_S1	'b'
+#define	VEC_S2	'w'
+#define	VEC_S4	'l'

Modified: trunk/varnish-cache/bin/varnishd/cache_esi_parse.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_esi_parse.c	2011-01-12 11:38:53 UTC (rev 5716)
+++ trunk/varnish-cache/bin/varnishd/cache_esi_parse.c	2011-01-12 13:10:08 UTC (rev 5717)
@@ -27,23 +27,293 @@
  *
  */
 
-#ifndef OLD_ESI
 #include "config.h"
 
 #include "svnid.h"
 SVNID("$Id")
 
 #include "cache.h"
+#include "cache_esi.h"
+#include "vend.h"
+#include "vct.h"
 #include "stevedore.h"
 
+#ifndef OLD_ESI
+
+struct vep_match {
+	const char	*match;
+	const char	**state;
+};
+
 struct vep_state {
 	unsigned		magic;
 #define VEP_MAGIC		0x55cb9b82
 	vfp_bytes_f		*bytes;
 	struct vsb		*vsb;
+
+	/* parser state */
+	const char		*state;
+
+	const char		*ver_p;
+
+	const char		*until;
+	const char		*until_p;
+	const char		*until_s;
+
+	struct vep_match	*match;
+	int			match_l;
+
+	char			tag[10];
+	int			tag_i;
+	
 };
 
+/*---------------------------------------------------------------------*/
+
+static const char *VEP_START =		"[Start]";
+static const char *VEP_NOTXML =	 	"[NotXml]";
+static const char *VEP_STARTTAG = 	"[StartTag]";
+static const char *VEP_MATCHBUF = 	"[MatchBuf]";
+static const char *VEP_NEXTTAG = 	"[NxtTag]";
+static const char *VEP_NOTMYTAG =	"[NotMyTag]";
+static const char *VEP_ESICMT =		"[EsiComment]";
+static const char *VEP_CMT =		"[Comment]";
+static const char *VEP_CDATA =		"[CDATA]";
+static const char *VEP_ESITAG =		"[ESITag]";
+static const char *VEP_ESIETAG =	"[ESIEndTag]";
+static const char *VEP_UNTIL =		"[Until]";
+static const char *VEP_ESIREMOVE =	"[ESI:Remove]";
+static const char *VEP_ESI_REMOVE =	"[ESI:/Remove]";
+static const char *VEP_ESIINCLUDE =	"[ESI:Include]";
+static const char *VEP_ESICOMMENT =	"[ESI:Comment]";
+static const char *VEP_MATCH =		"[Match]";
+static const char *VEP_XXX =		"[XXX]";
+
+/*---------------------------------------------------------------------*/
+
+static struct vep_match vep_match_tbl[] = {
+	{ "<!--esi",	&VEP_ESICMT },
+	{ "<!--",	&VEP_CMT },
+	{ "</esi:",	&VEP_ESIETAG },
+	{ "<esi:",	&VEP_ESITAG },
+	{ "<![CDATA[",	&VEP_CDATA },
+	{ NULL,		&VEP_NOTMYTAG }
+};
+
+static const int vep_match_tbl_len =
+    sizeof vep_match_tbl / sizeof vep_match_tbl[0];
+
+static struct vep_match vep_match_esi[] = {
+	{ "include",	&VEP_ESIINCLUDE },
+	{ "remove",	&VEP_ESIREMOVE },
+	{ "comment",	&VEP_ESICOMMENT },
+	{ NULL,		&VEP_XXX }
+};
+
+static const int vep_match_esi_len =
+    sizeof vep_match_esi / sizeof vep_match_esi[0];
+
 /*---------------------------------------------------------------------
+ * return match or NULL if more input needed.
+ */
+static struct vep_match *
+vep_match(struct vep_state *vep, const char *b, const char *e)
+{
+	struct vep_match *vm;
+	const char *q, *r;
+
+	for (vm = vep->match; vm->match; vm++) {
+		r = b;
+		for (q = vm->match; *q && r < e; q++, r++)
+			if (*q != *r)
+				break;
+		if (*q != '\0' && r == e) {
+			if (b != vep->tag) {
+				assert(e - b < sizeof vep->tag);
+				memcpy(vep->tag, b, e - b);
+				vep->tag_i = e - b;
+			}
+			return (NULL);
+		}
+		if (*q == '\0')
+			return (vm);
+	}
+	return (vm);
+}
+
+/*---------------------------------------------------------------------
+ *
+ */
+
+static void
+vep_end_verbatim(struct vep_state *vep, const char *p)
+{
+	ssize_t l;
+	uint8_t buf[5];
+
+	AN(vep->ver_p);
+	l = p - vep->ver_p;
+	if (l == 0)
+		return;
+	assert(l > 0);
+	if (l < 256) {
+		buf[0] = VEC_V1;
+		buf[1] = (uint8_t)l;
+		vsb_bcat(vep->vsb, buf, 2);
+	} else if (l < 65536) {
+		buf[0] = VEC_V2;
+		vbe16enc(buf + 1, (uint16_t)l);
+		vsb_bcat(vep->vsb, buf, 3);
+	} else {
+		/* XXX assert < 2^32 */
+		buf[0] = VEC_V4;
+		vbe32enc(buf + 1, (uint32_t)l);
+		vsb_bcat(vep->vsb, buf, 5);
+	}
+	vep->ver_p = p;
+} 
+
+static void
+vep_end_skip(struct vep_state *vep, const char *p)
+{
+	ssize_t l;
+	uint8_t buf[5];
+
+	AN(vep->ver_p);
+	l = p - vep->ver_p;
+	if (l == 0)
+		return;
+	assert(l > 0);
+	if (l < 256) {
+		buf[0] = VEC_S1;
+		buf[1] = (uint8_t)l;
+		vsb_bcat(vep->vsb, buf, 2);
+	} else if (l < 65536) {
+		buf[0] = VEC_S2;
+		vbe16enc(buf + 1, (uint16_t)l);
+		vsb_bcat(vep->vsb, buf, 3);
+	} else {
+		/* XXX assert < 2^32 */
+		buf[0] = VEC_S4;
+		vbe32enc(buf + 1, (uint32_t)l);
+		vsb_bcat(vep->vsb, buf, 5);
+	}
+	vep->ver_p = p;
+} 
+
+/*---------------------------------------------------------------------
+ * Parse object for ESI instructions
+ */
+
+#include <stdio.h>
+
+static void
+vep_parse(struct vep_state *vep, const char *b, size_t l)
+{
+	const char *e, *p;
+	struct vep_match *vm;
+
+	CHECK_OBJ_NOTNULL(vep, VEP_MAGIC);
+	assert(l > 0);
+
+	e = b + l;
+
+	vep->ver_p = b;
+	printf("EP Call %d [%.*s]\n", (int)l, (int)l, b);
+	p = b;
+	while (p < e) {
+		AN(vep->state);
+		printf("EP %s [%.*s]\n",
+		    vep->state,
+		    (int)(e - p), p);
+		if (vep->state == VEP_START) {
+			/*
+			 * Look for the first non-white char, and
+			 * abandon if it is not '<' under the assumption
+			 * that it is not an ESI file
+			 */
+			while (p < e && vct_islws(*p))
+				p++;
+			if (p < e) {
+				if (*p == '<')
+					vep->state = VEP_STARTTAG;
+				else
+					vep->state = VEP_NOTXML;
+			}
+		} else if (vep->state == VEP_NEXTTAG) {
+			/*
+			 * Hunt for start of next tag
+			 */
+			while (p < e && *p != '<') {
+#if 0
+				if (vep->incmt != NULL &&
+				    *p == *vep->incmt_p) {
+					if (*++vep->incmt_p == '\0') {
+						vep->incmt = NULL;
+						vep->incmt = NULL;
+					}
+				} else
+					vep->incmt_p = vep->incmt;
+#endif
+				p++;
+			}
+			if (p < e)
+				vep->state = VEP_STARTTAG;
+		} else if (vep->state == VEP_STARTTAG) {
+			/*
+			 * Start of tag, set up match table
+			 */
+			vep_end_verbatim(vep, p);
+			vep->match = vep_match_tbl;
+			vep->match_l = vep_match_tbl_len;
+			vep->state = VEP_MATCH;
+		} else if (vep->state == VEP_MATCH) {
+			/*
+			 * Match against a table
+			 */
+			vm = vep_match(vep, p, e);
+			if (vm != NULL) {
+				if (vm->match != NULL)
+					p += strlen(vm->match);
+				b = p;
+				vep->state = *vm->state;
+				vep->tag_i = 0;
+			} else {
+				vep->state = VEP_MATCHBUF;
+				return;
+			}
+		} else if (vep->state == VEP_UNTIL) {
+			/*
+			 * Skip until we see magic string
+			 */
+			while (p < e) {
+				if (*p++ != *vep->until_p++) {
+					vep->until_p = vep->until;
+				} else if (*vep->until_p == '\0') {
+					vep->state = vep->until_s;
+					break;
+				}
+			}
+		} else if (vep->state == VEP_ESITAG) {
+			vep->match = vep_match_esi;
+			vep->match_l = vep_match_esi_len;
+			vep->state = VEP_MATCH;
+		} else if (vep->state == VEP_ESIREMOVE) {
+			vep->until_p = vep->until = "</esi:remove>";
+			vep->until_s = VEP_ESI_REMOVE;
+			vep->state = VEP_UNTIL;;
+		} else if (vep->state == VEP_ESI_REMOVE) {
+			vep_end_skip(vep, p);
+			vep->state = VEP_NEXTTAG;
+		} else {
+			printf("*** Unknown state %s\n", vep->state);
+			break;
+		}
+	}
+	vep_end_verbatim(vep, p);
+}
+
+/*---------------------------------------------------------------------
  * We receive a ungzip'ed object, and want to store it ungzip'ed.
  */
 
@@ -74,6 +344,7 @@
 		w = HTC_Read(htc, st->ptr + st->len, l);
 		if (w <= 0)
 			return (w);
+		vep_parse(vep, (const char *)st->ptr + st->len, w);
 		st->len += w;
 		sp->obj->len += w;
 		if (st->len == st->space) {
@@ -99,10 +370,12 @@
 	vep = (void*)WS_Alloc(sp->wrk->ws, sizeof *vep);
 	AN(vep);
 
+
 	memset(vep, 0, sizeof *vep);
 	vep->magic = VEP_MAGIC;
 	vep->bytes = vfp_esi_bytes_uu;
 	vep->vsb = vsb_newauto();
+	vep->state = VEP_START;
 	AN(vep->vsb);
 
 	sp->wrk->vep = vep;
@@ -124,9 +397,34 @@
 static int __match_proto__()
 vfp_esi_end(struct sess *sp)
 {
+	struct storage *st;
+	struct vep_state *vep;
 
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
-	return (-1);
+	vep = sp->wrk->vep;
+	CHECK_OBJ_NOTNULL(vep, VEP_MAGIC);
+
+	vsb_finish(vep->vsb);
+	printf("ESI <%s>\n", vsb_data(vep->vsb));
+
+	/* XXX: This is a huge waste of storage... */
+	sp->obj->esidata = STV_alloc(sp, vsb_len(vep->vsb));
+	AN(sp->obj->esidata);
+	memcpy(sp->obj->esidata->ptr, vsb_data(vep->vsb), vsb_len(vep->vsb));
+
+	st = sp->wrk->storage;
+	sp->wrk->storage = NULL;
+	if (st == NULL)
+		return (0);
+
+	if (st->len == 0) {
+		STV_free(st);
+		return (0);
+	}
+	if (st->len < st->space)
+		STV_trim(st, st->len);
+	VTAILQ_INSERT_TAIL(&sp->obj->store, st, list);
+	return (0);
 }
 
 struct vfp vfp_esi = {




From phk at varnish-cache.org  Wed Jan 12 13:37:12 2011
From: phk at varnish-cache.org (phk at varnish-cache.org)
Date: Wed, 12 Jan 2011 14:37:12 +0100
Subject: r5718 - trunk/varnish-cache/bin/varnishd
Message-ID: <E1Pd0sy-0002Nj-2V@project.varnish-software.com>

Author: phk
Date: 2011-01-12 14:37:11 +0100 (Wed, 12 Jan 2011)
New Revision: 5718

Added:
   trunk/varnish-cache/bin/varnishd/cache_esi_deliver.c
Modified:
   trunk/varnish-cache/bin/varnishd/Makefile.am
   trunk/varnish-cache/bin/varnishd/cache.h
   trunk/varnish-cache/bin/varnishd/cache_esi_parse.c
   trunk/varnish-cache/bin/varnishd/cache_response.c
Log:
Add new esi delivery function which very noisily can pass e00001.vtc



Modified: trunk/varnish-cache/bin/varnishd/Makefile.am
===================================================================
--- trunk/varnish-cache/bin/varnishd/Makefile.am	2011-01-12 13:10:08 UTC (rev 5717)
+++ trunk/varnish-cache/bin/varnishd/Makefile.am	2011-01-12 13:37:11 UTC (rev 5718)
@@ -26,6 +26,7 @@
 	cache_dir_dns.c \
 	cache_dir_round_robin.c \
 	cache_esi_parse.c \
+	cache_esi_deliver.c \
 	cache_esi.c \
 	cache_expire.c \
 	cache_fetch.c \

Modified: trunk/varnish-cache/bin/varnishd/cache.h
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache.h	2011-01-12 13:10:08 UTC (rev 5717)
+++ trunk/varnish-cache/bin/varnishd/cache.h	2011-01-12 13:37:11 UTC (rev 5718)
@@ -811,6 +811,8 @@
 void ESI_Deliver(struct sess *);
 void ESI_Destroy(struct object *);
 void ESI_Parse(struct sess *);
+#else
+void ESI_Deliver(struct sess *);
 #endif /* OLD_ESI */
 
 /* cache_vrt_vmod.c */

Added: trunk/varnish-cache/bin/varnishd/cache_esi_deliver.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_esi_deliver.c	                        (rev 0)
+++ trunk/varnish-cache/bin/varnishd/cache_esi_deliver.c	2011-01-12 13:37:11 UTC (rev 5718)
@@ -0,0 +1,96 @@
+/*-
+ * Copyright (c) 2011 Varnish Software AS
+ * All rights reserved.
+ *
+ * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ */
+
+#include "config.h"
+
+#include "svnid.h"
+SVNID("$Id")
+
+#include "cache.h"
+#include "cache_esi.h"
+#include "vend.h"
+#include "vct.h"
+#include "stevedore.h"
+
+#include <stdio.h>
+
+#ifndef OLD_ESI
+
+void
+ESI_Deliver(struct sess *sp)
+{
+	struct storage *st;
+	uint8_t *p, *e, *q;
+	unsigned off;
+	size_t l;
+
+printf("DELIV\n");
+fflush(stdout);
+sleep(1);
+	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+	st = sp->obj->esidata;
+	AN(st);
+	p = st->ptr;
+	e = st->ptr + st->len;
+
+	st = VTAILQ_FIRST(&sp->obj->store);
+	off = 0;
+
+	while (p < e) {
+		switch (*p) {
+		case VEC_V1:
+			l = p[1];
+			p += 2;
+			q = (void*)strchr((const char*)p, '\0');
+			assert (q > p);
+			printf("VER(%d) %d\n", (int)l, (int)(q-p));
+			if (sp->wrk->res_mode & RES_CHUNKED)
+				WRW_Write(sp->wrk, p, q - p);
+			WRW_Write(sp->wrk, st->ptr + off, l);
+			if (sp->wrk->res_mode & RES_CHUNKED)
+				WRW_Write(sp->wrk, "\r\n", -1);
+			printf("[%.*s]", (int)l, st->ptr + off);
+			off += l;
+			p = q + 1;
+			break;
+		case VEC_S1:
+			l = p[1];
+			p += 2;
+			printf("SKIP(%d)\n", (int)l);
+			off += l;
+			break;
+		default:
+			printf("XXXX %02x [%c]\n", *p, *p);
+			return;
+		}
+	}
+	WRW_Flush(sp->wrk);
+}
+
+#endif /* OLD_ESI */

Modified: trunk/varnish-cache/bin/varnishd/cache_esi_parse.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_esi_parse.c	2011-01-12 13:10:08 UTC (rev 5717)
+++ trunk/varnish-cache/bin/varnishd/cache_esi_parse.c	2011-01-12 13:37:11 UTC (rev 5718)
@@ -170,6 +170,7 @@
 		vbe32enc(buf + 1, (uint32_t)l);
 		vsb_bcat(vep->vsb, buf, 5);
 	}
+	vsb_printf(vep->vsb, "%lx\r\n%c", l, 0);
 	vep->ver_p = p;
 } 
 
@@ -411,6 +412,7 @@
 	sp->obj->esidata = STV_alloc(sp, vsb_len(vep->vsb));
 	AN(sp->obj->esidata);
 	memcpy(sp->obj->esidata->ptr, vsb_data(vep->vsb), vsb_len(vep->vsb));
+	sp->obj->esidata->len = vsb_len(vep->vsb);
 
 	st = sp->wrk->storage;
 	sp->wrk->storage = NULL;

Modified: trunk/varnish-cache/bin/varnishd/cache_response.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_response.c	2011-01-12 13:10:08 UTC (rev 5717)
+++ trunk/varnish-cache/bin/varnishd/cache_response.c	2011-01-12 13:37:11 UTC (rev 5718)
@@ -386,10 +386,8 @@
 		/* This was a HEAD request */
 	} else if (sp->obj->len == 0) {
 		/* Nothing to do here */
-#ifdef OLD_ESI
 	} else if (sp->wrk->res_mode & RES_ESI) {
 		ESI_Deliver(sp);
-#endif
 	} else if (sp->wrk->res_mode & RES_GUNZIP) {
 		res_WriteGunzipObj(sp, lenbuf); 
 	} else {




From phk at varnish-cache.org  Wed Jan 12 13:41:18 2011
From: phk at varnish-cache.org (phk at varnish-cache.org)
Date: Wed, 12 Jan 2011 14:41:18 +0100
Subject: r5719 - trunk/varnish-cache/bin/varnishd
Message-ID: <E1Pd0ww-0002WP-8X@project.varnish-software.com>

Author: phk
Date: 2011-01-12 14:41:16 +0100 (Wed, 12 Jan 2011)
New Revision: 5719

Modified:
   trunk/varnish-cache/bin/varnishd/cache_esi_parse.c
Log:
Make the "not-xml" case work (e00000)



Modified: trunk/varnish-cache/bin/varnishd/cache_esi_parse.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_esi_parse.c	2011-01-12 13:37:11 UTC (rev 5718)
+++ trunk/varnish-cache/bin/varnishd/cache_esi_parse.c	2011-01-12 13:41:16 UTC (rev 5719)
@@ -241,6 +241,8 @@
 				else
 					vep->state = VEP_NOTXML;
 			}
+		} else if (vep->state == VEP_NOTXML) {
+			p = e;
 		} else if (vep->state == VEP_NEXTTAG) {
 			/*
 			 * Hunt for start of next tag
@@ -406,13 +408,17 @@
 	CHECK_OBJ_NOTNULL(vep, VEP_MAGIC);
 
 	vsb_finish(vep->vsb);
-	printf("ESI <%s>\n", vsb_data(vep->vsb));
+	if (vep->state != VEP_NOTXML) {
+		printf("ESI <%s>\n", vsb_data(vep->vsb));
 
-	/* XXX: This is a huge waste of storage... */
-	sp->obj->esidata = STV_alloc(sp, vsb_len(vep->vsb));
-	AN(sp->obj->esidata);
-	memcpy(sp->obj->esidata->ptr, vsb_data(vep->vsb), vsb_len(vep->vsb));
-	sp->obj->esidata->len = vsb_len(vep->vsb);
+		/* XXX: This is a huge waste of storage... */
+		sp->obj->esidata = STV_alloc(sp, vsb_len(vep->vsb));
+		AN(sp->obj->esidata);
+		memcpy(sp->obj->esidata->ptr,
+		    vsb_data(vep->vsb), vsb_len(vep->vsb));
+		sp->obj->esidata->len = vsb_len(vep->vsb);
+	}
+	vsb_delete(vep->vsb);
 
 	st = sp->wrk->storage;
 	sp->wrk->storage = NULL;




From phk at varnish-cache.org  Wed Jan 12 14:39:50 2011
From: phk at varnish-cache.org (phk at varnish-cache.org)
Date: Wed, 12 Jan 2011 15:39:50 +0100
Subject: r5720 - trunk/varnish-cache/bin/varnishd
Message-ID: <E1Pd1ra-0004A3-7p@project.varnish-software.com>

Author: phk
Date: 2011-01-12 15:39:49 +0100 (Wed, 12 Jan 2011)
New Revision: 5720

Modified:
   trunk/varnish-cache/bin/varnishd/cache_esi_deliver.c
   trunk/varnish-cache/bin/varnishd/cache_esi_parse.c
Log:
Reduce debug noise in deliver.

Get CDATA right in parser.

Add one byte at a time torture mode.



Modified: trunk/varnish-cache/bin/varnishd/cache_esi_deliver.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_esi_deliver.c	2011-01-12 13:41:16 UTC (rev 5719)
+++ trunk/varnish-cache/bin/varnishd/cache_esi_deliver.c	2011-01-12 14:39:49 UTC (rev 5720)
@@ -32,13 +32,15 @@
 #include "svnid.h"
 SVNID("$Id")
 
+#include <stdlib.h>
+#include <stdio.h>
+
 #include "cache.h"
 #include "cache_esi.h"
 #include "vend.h"
 #include "vct.h"
 #include "stevedore.h"
 
-#include <stdio.h>
 
 #ifndef OLD_ESI
 
@@ -51,8 +53,6 @@
 	size_t l;
 
 printf("DELIV\n");
-fflush(stdout);
-sleep(1);
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 	st = sp->obj->esidata;
 	AN(st);
@@ -75,7 +75,7 @@
 			WRW_Write(sp->wrk, st->ptr + off, l);
 			if (sp->wrk->res_mode & RES_CHUNKED)
 				WRW_Write(sp->wrk, "\r\n", -1);
-			printf("[%.*s]", (int)l, st->ptr + off);
+			// printf("[%.*s]", (int)l, st->ptr + off);
 			off += l;
 			p = q + 1;
 			break;
@@ -87,6 +87,7 @@
 			break;
 		default:
 			printf("XXXX %02x [%c]\n", *p, *p);
+			INCOMPL();
 			return;
 		}
 	}

Modified: trunk/varnish-cache/bin/varnishd/cache_esi_parse.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_esi_parse.c	2011-01-12 13:41:16 UTC (rev 5719)
+++ trunk/varnish-cache/bin/varnishd/cache_esi_parse.c	2011-01-12 14:39:49 UTC (rev 5720)
@@ -38,6 +38,8 @@
 #include "vct.h"
 #include "stevedore.h"
 
+#include <stdio.h>
+
 #ifndef OLD_ESI
 
 struct vep_match {
@@ -54,6 +56,12 @@
 	/* parser state */
 	const char		*state;
 
+	unsigned		remove;
+	unsigned		skip;
+
+	unsigned		o_verbatim;
+	unsigned		o_skip;
+
 	const char		*ver_p;
 
 	const char		*until;
@@ -73,6 +81,7 @@
 static const char *VEP_START =		"[Start]";
 static const char *VEP_NOTXML =	 	"[NotXml]";
 static const char *VEP_STARTTAG = 	"[StartTag]";
+static const char *VEP_ENDTAG = 	"[EndTag]";
 static const char *VEP_MATCHBUF = 	"[MatchBuf]";
 static const char *VEP_NEXTTAG = 	"[NxtTag]";
 static const char *VEP_NOTMYTAG =	"[NotMyTag]";
@@ -113,6 +122,14 @@
 static const int vep_match_esi_len =
     sizeof vep_match_esi / sizeof vep_match_esi[0];
 
+static struct vep_match vep_match_esie[] = {
+	{ "remove",	&VEP_ESI_REMOVE },
+	{ NULL,		&VEP_XXX }
+};
+
+static const int vep_match_esie_len =
+    sizeof vep_match_esie / sizeof vep_match_esie[0];
+
 /*---------------------------------------------------------------------
  * return match or NULL if more input needed.
  */
@@ -146,68 +163,94 @@
  */
 
 static void
-vep_end_verbatim(struct vep_state *vep, const char *p)
+vep_emit_skip(struct vep_state *vep)
 {
 	ssize_t l;
 	uint8_t buf[5];
 
-	AN(vep->ver_p);
-	l = p - vep->ver_p;
-	if (l == 0)
-		return;
+	l = vep->o_skip;
+	vep->o_skip = 0;
 	assert(l > 0);
 	if (l < 256) {
-		buf[0] = VEC_V1;
+		buf[0] = VEC_S1;
 		buf[1] = (uint8_t)l;
 		vsb_bcat(vep->vsb, buf, 2);
 	} else if (l < 65536) {
-		buf[0] = VEC_V2;
+		buf[0] = VEC_S2;
 		vbe16enc(buf + 1, (uint16_t)l);
 		vsb_bcat(vep->vsb, buf, 3);
 	} else {
 		/* XXX assert < 2^32 */
-		buf[0] = VEC_V4;
+		buf[0] = VEC_S4;
 		vbe32enc(buf + 1, (uint32_t)l);
 		vsb_bcat(vep->vsb, buf, 5);
 	}
-	vsb_printf(vep->vsb, "%lx\r\n%c", l, 0);
-	vep->ver_p = p;
 } 
 
 static void
-vep_end_skip(struct vep_state *vep, const char *p)
+vep_emit_verbatim(struct vep_state *vep)
 {
 	ssize_t l;
 	uint8_t buf[5];
 
-	AN(vep->ver_p);
-	l = p - vep->ver_p;
-	if (l == 0)
-		return;
+	l = vep->o_verbatim;
+	vep->o_verbatim = 0;
 	assert(l > 0);
 	if (l < 256) {
-		buf[0] = VEC_S1;
+		buf[0] = VEC_V1;
 		buf[1] = (uint8_t)l;
 		vsb_bcat(vep->vsb, buf, 2);
 	} else if (l < 65536) {
-		buf[0] = VEC_S2;
+		buf[0] = VEC_V2;
 		vbe16enc(buf + 1, (uint16_t)l);
 		vsb_bcat(vep->vsb, buf, 3);
 	} else {
 		/* XXX assert < 2^32 */
-		buf[0] = VEC_S4;
+		buf[0] = VEC_V4;
 		vbe32enc(buf + 1, (uint32_t)l);
 		vsb_bcat(vep->vsb, buf, 5);
 	}
+	vsb_printf(vep->vsb, "%lx\r\n%c", l, 0);
+} 
+
+static void
+vep_end_verbatim(struct vep_state *vep, const char *p)
+{
+	ssize_t l;
+
+	AN(vep->ver_p);
+	l = p - vep->ver_p;
+	if (l == 0)
+		return;
+	if (vep->o_skip > 0) 
+		vep_emit_skip(vep);
+	AZ(vep->o_skip);
+	printf("-->V(%d) [%.*s]\n", (int)l, (int)l, vep->ver_p);
+	vep->o_verbatim += l;
 	vep->ver_p = p;
 } 
 
+static void
+vep_end_skip(struct vep_state *vep, const char *p)
+{
+	ssize_t l;
+
+	AN(vep->ver_p);
+	l = p - vep->ver_p;
+	if (l == 0)
+		return;
+	if (vep->o_verbatim > 0) 
+		vep_emit_verbatim(vep);
+	AZ(vep->o_verbatim);
+	printf("-->S(%d) [%.*s]\n", (int)l, (int)l, vep->ver_p);
+	vep->o_skip += l;
+	vep->ver_p = p;
+} 
+
 /*---------------------------------------------------------------------
  * Parse object for ESI instructions
  */
 
-#include <stdio.h>
-
 static void
 vep_parse(struct vep_state *vep, const char *b, size_t l)
 {
@@ -266,10 +309,26 @@
 			/*
 			 * Start of tag, set up match table
 			 */
-			vep_end_verbatim(vep, p);
+			assert(*p == '<');
+			if (!vep->remove)
+				vep_end_verbatim(vep, p);
 			vep->match = vep_match_tbl;
 			vep->match_l = vep_match_tbl_len;
 			vep->state = VEP_MATCH;
+			vep->skip = 1;
+		} else if (vep->state == VEP_ENDTAG) {
+			while (p < e && *p != '>') 
+				p++;
+			if (*p == '>') {
+				p++;
+				vep->state = VEP_NEXTTAG;
+			}
+			vep_end_skip(vep, p);
+			vep->skip = 0;
+		} else if (vep->state == VEP_CDATA) {
+			vep->until_p = vep->until = "]]>";
+			vep->until_s = VEP_NEXTTAG;
+			vep->state = VEP_UNTIL;
 		} else if (vep->state == VEP_MATCH) {
 			/*
 			 * Match against a table
@@ -280,11 +339,46 @@
 					p += strlen(vm->match);
 				b = p;
 				vep->state = *vm->state;
+				vep->match = NULL;
 				vep->tag_i = 0;
 			} else {
+				memcpy(vep->tag, p, e - p);
+				vep->tag_i = e - p;
 				vep->state = VEP_MATCHBUF;
-				return;
+				p = e;
+				break;
 			}
+		} else if (vep->state == VEP_MATCHBUF) {
+			/*
+			 * Match against a table while split over input
+			 * sections.
+			 */
+			do {
+				if (*p == '>') {
+					vm = NULL;
+				} else {
+					vep->tag[vep->tag_i++] = *p++;
+					vm = vep_match(vep,
+					    vep->tag, vep->tag + vep->tag_i);
+				}
+			} while (vm == 0 && p < e);
+			if (vm == 0) {
+				b = e;
+				break;
+			}
+			if (vep->tag_i > strlen(vm->match)) {
+				/*
+				 * not generally safe but
+				 * works for the required
+				 * case of <--esi and <--
+				 */
+				p -= vep->tag_i -
+				    strlen(vm->match);
+			}
+			b = p;
+			vep->state = *vm->state;
+			vep->match = NULL;
+			vep->tag_i = 0;
 		} else if (vep->state == VEP_UNTIL) {
 			/*
 			 * Skip until we see magic string
@@ -298,22 +392,31 @@
 				}
 			}
 		} else if (vep->state == VEP_ESITAG) {
+			vep->skip = 1;
+			vep_end_skip(vep, p);
 			vep->match = vep_match_esi;
 			vep->match_l = vep_match_esi_len;
 			vep->state = VEP_MATCH;
+		} else if (vep->state == VEP_ESIETAG) {
+			vep->match = vep_match_esie;
+			vep->match_l = vep_match_esie_len;
+			vep->state = VEP_MATCH;
 		} else if (vep->state == VEP_ESIREMOVE) {
-			vep->until_p = vep->until = "</esi:remove>";
-			vep->until_s = VEP_ESI_REMOVE;
-			vep->state = VEP_UNTIL;;
-		} else if (vep->state == VEP_ESI_REMOVE) {
 			vep_end_skip(vep, p);
+			vep->remove = 1;
 			vep->state = VEP_NEXTTAG;
+		} else if (vep->state == VEP_ESI_REMOVE) {
+			vep->remove = 0;
+			vep->state = VEP_ENDTAG;
 		} else {
 			printf("*** Unknown state %s\n", vep->state);
 			break;
 		}
 	}
-	vep_end_verbatim(vep, p);
+	if (vep->remove || vep->skip)
+		vep_end_skip(vep, p);
+	else
+		vep_end_verbatim(vep, p);
 }
 
 /*---------------------------------------------------------------------
@@ -347,7 +450,14 @@
 		w = HTC_Read(htc, st->ptr + st->len, l);
 		if (w <= 0)
 			return (w);
+#if 0
+		{
+		for (l = 0; l < w; l++) 
+			vep_parse(vep, (const char *)st->ptr + st->len + l, 1);
+		}
+#else
 		vep_parse(vep, (const char *)st->ptr + st->len, w);
+#endif
 		st->len += w;
 		sp->obj->len += w;
 		if (st->len == st->space) {
@@ -402,14 +512,20 @@
 {
 	struct storage *st;
 	struct vep_state *vep;
+	size_t l;
 
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 	vep = sp->wrk->vep;
 	CHECK_OBJ_NOTNULL(vep, VEP_MAGIC);
 
+	if (vep->o_verbatim)
+		vep_emit_verbatim(vep);
+	if (vep->o_skip)
+		vep_emit_skip(vep);
 	vsb_finish(vep->vsb);
-	if (vep->state != VEP_NOTXML) {
-		printf("ESI <%s>\n", vsb_data(vep->vsb));
+	l = vsb_len(vep->vsb);
+	if (vep->state != VEP_NOTXML && l != 0) {
+		printf("ESI %d <%s>\n", (int)l, vsb_data(vep->vsb));
 
 		/* XXX: This is a huge waste of storage... */
 		sp->obj->esidata = STV_alloc(sp, vsb_len(vep->vsb));




From phk at varnish-cache.org  Wed Jan 12 15:13:21 2011
From: phk at varnish-cache.org (phk at varnish-cache.org)
Date: Wed, 12 Jan 2011 16:13:21 +0100
Subject: r5721 - trunk/varnish-cache/bin/varnishd
Message-ID: <E1Pd2O1-0005WA-Lg@project.varnish-software.com>

Author: phk
Date: 2011-01-12 16:13:20 +0100 (Wed, 12 Jan 2011)
New Revision: 5721

Modified:
   trunk/varnish-cache/bin/varnishd/cache_esi.h
   trunk/varnish-cache/bin/varnishd/cache_esi_deliver.c
   trunk/varnish-cache/bin/varnishd/cache_esi_parse.c
Log:
Add encoding for literals, for when the match buffer spans storage
segments.

Collapse consecutive verbatim/skip sequences.

Handling non-esi tags.




Modified: trunk/varnish-cache/bin/varnishd/cache_esi.h
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_esi.h	2011-01-12 14:39:49 UTC (rev 5720)
+++ trunk/varnish-cache/bin/varnishd/cache_esi.h	2011-01-12 15:13:20 UTC (rev 5721)
@@ -33,3 +33,6 @@
 #define	VEC_S1	'b'
 #define	VEC_S2	'w'
 #define	VEC_S4	'l'
+#define	VEC_L1	'c'
+#define	VEC_L2	'x'
+#define	VEC_L4	'm'

Modified: trunk/varnish-cache/bin/varnishd/cache_esi_deliver.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_esi_deliver.c	2011-01-12 14:39:49 UTC (rev 5720)
+++ trunk/varnish-cache/bin/varnishd/cache_esi_deliver.c	2011-01-12 15:13:20 UTC (rev 5721)
@@ -85,6 +85,20 @@
 			printf("SKIP(%d)\n", (int)l);
 			off += l;
 			break;
+		case VEC_L1:
+			l = p[1];
+			p += 2;
+			q = (void*)strchr((const char*)p, '\0');
+			assert (q > p);
+			printf("LIT(%d) %d\n", (int)l, (int)(q-p));
+			if (sp->wrk->res_mode & RES_CHUNKED)
+				WRW_Write(sp->wrk, p, q - p);
+			p = q + 1;
+			WRW_Write(sp->wrk, p, l);
+			if (sp->wrk->res_mode & RES_CHUNKED)
+				WRW_Write(sp->wrk, "\r\n", -1);
+			p = p + l;
+			break;
 		default:
 			printf("XXXX %02x [%c]\n", *p, *p);
 			INCOMPL();

Modified: trunk/varnish-cache/bin/varnishd/cache_esi_parse.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_esi_parse.c	2011-01-12 14:39:49 UTC (rev 5720)
+++ trunk/varnish-cache/bin/varnishd/cache_esi_parse.c	2011-01-12 15:13:20 UTC (rev 5721)
@@ -163,58 +163,52 @@
  */
 
 static void
-vep_emit_skip(struct vep_state *vep)
+vep_emit_len(struct vep_state *vep, ssize_t l, int m8, int m16, int m32)
 {
-	ssize_t l;
 	uint8_t buf[5];
 
-	l = vep->o_skip;
-	vep->o_skip = 0;
 	assert(l > 0);
 	if (l < 256) {
-		buf[0] = VEC_S1;
+		buf[0] = m8;
 		buf[1] = (uint8_t)l;
 		vsb_bcat(vep->vsb, buf, 2);
 	} else if (l < 65536) {
-		buf[0] = VEC_S2;
+		buf[0] = m16;
 		vbe16enc(buf + 1, (uint16_t)l);
 		vsb_bcat(vep->vsb, buf, 3);
 	} else {
 		/* XXX assert < 2^32 */
-		buf[0] = VEC_S4;
+		buf[0] = m32;
 		vbe32enc(buf + 1, (uint32_t)l);
 		vsb_bcat(vep->vsb, buf, 5);
 	}
 } 
 
 static void
+vep_emit_skip(struct vep_state *vep)
+{
+	ssize_t l;
+
+	l = vep->o_skip;
+	vep->o_skip = 0;
+	assert(l > 0);
+	vep_emit_len(vep, l, VEC_S1, VEC_S2, VEC_S4);
+} 
+
+static void
 vep_emit_verbatim(struct vep_state *vep)
 {
 	ssize_t l;
-	uint8_t buf[5];
 
 	l = vep->o_verbatim;
 	vep->o_verbatim = 0;
 	assert(l > 0);
-	if (l < 256) {
-		buf[0] = VEC_V1;
-		buf[1] = (uint8_t)l;
-		vsb_bcat(vep->vsb, buf, 2);
-	} else if (l < 65536) {
-		buf[0] = VEC_V2;
-		vbe16enc(buf + 1, (uint16_t)l);
-		vsb_bcat(vep->vsb, buf, 3);
-	} else {
-		/* XXX assert < 2^32 */
-		buf[0] = VEC_V4;
-		vbe32enc(buf + 1, (uint32_t)l);
-		vsb_bcat(vep->vsb, buf, 5);
-	}
+	vep_emit_len(vep, l, VEC_V1, VEC_V2, VEC_V4);
 	vsb_printf(vep->vsb, "%lx\r\n%c", l, 0);
 } 
 
 static void
-vep_end_verbatim(struct vep_state *vep, const char *p)
+vep_mark_verbatim(struct vep_state *vep, const char *p)
 {
 	ssize_t l;
 
@@ -231,7 +225,7 @@
 } 
 
 static void
-vep_end_skip(struct vep_state *vep, const char *p)
+vep_mark_skip(struct vep_state *vep, const char *p)
 {
 	ssize_t l;
 
@@ -247,6 +241,22 @@
 	vep->ver_p = p;
 } 
 
+static void
+vep_emit_literal(struct vep_state *vep, const char *p, const char *e)
+{
+	ssize_t l;
+
+	if (vep->o_verbatim > 0) 
+		vep_emit_verbatim(vep);
+	if (vep->o_skip > 0) 
+		vep_emit_skip(vep);
+	l = e - p;
+	printf("---->L(%d) [%.*s]\n", (int)l, (int)l, p);
+	vep_emit_len(vep, l, VEC_L1, VEC_L2, VEC_L4);
+	vsb_printf(vep->vsb, "%lx\r\n%c", l, 0);
+	vsb_bcat(vep->vsb, p, l);
+}
+
 /*---------------------------------------------------------------------
  * Parse object for ESI instructions
  */
@@ -311,7 +321,7 @@
 			 */
 			assert(*p == '<');
 			if (!vep->remove)
-				vep_end_verbatim(vep, p);
+				vep_mark_verbatim(vep, p);
 			vep->match = vep_match_tbl;
 			vep->match_l = vep_match_tbl_len;
 			vep->state = VEP_MATCH;
@@ -323,7 +333,7 @@
 				p++;
 				vep->state = VEP_NEXTTAG;
 			}
-			vep_end_skip(vep, p);
+			vep_mark_skip(vep, p);
 			vep->skip = 0;
 		} else if (vep->state == VEP_CDATA) {
 			vep->until_p = vep->until = "]]>";
@@ -366,7 +376,7 @@
 				b = e;
 				break;
 			}
-			if (vep->tag_i > strlen(vm->match)) {
+			if (vm->match && vep->tag_i > strlen(vm->match)) {
 				/*
 				 * not generally safe but
 				 * works for the required
@@ -374,7 +384,12 @@
 				 */
 				p -= vep->tag_i -
 				    strlen(vm->match);
+				vep->tag_i--;
 			}
+			if (vm->match == NULL) {
+				vep_emit_literal(vep,
+				    vep->tag, vep->tag + vep->tag_i);
+			}
 			b = p;
 			vep->state = *vm->state;
 			vep->match = NULL;
@@ -391,9 +406,17 @@
 					break;
 				}
 			}
+		} else if (vep->state == VEP_NOTMYTAG) {
+			vep->skip = 0;
+			while (p < e) {
+				if (*p++ == '>') {
+					vep->state = VEP_NEXTTAG;
+					break;
+				}
+			}
 		} else if (vep->state == VEP_ESITAG) {
 			vep->skip = 1;
-			vep_end_skip(vep, p);
+			vep_mark_skip(vep, p);
 			vep->match = vep_match_esi;
 			vep->match_l = vep_match_esi_len;
 			vep->state = VEP_MATCH;
@@ -402,7 +425,7 @@
 			vep->match_l = vep_match_esie_len;
 			vep->state = VEP_MATCH;
 		} else if (vep->state == VEP_ESIREMOVE) {
-			vep_end_skip(vep, p);
+			vep_mark_skip(vep, p);
 			vep->remove = 1;
 			vep->state = VEP_NEXTTAG;
 		} else if (vep->state == VEP_ESI_REMOVE) {
@@ -414,9 +437,9 @@
 		}
 	}
 	if (vep->remove || vep->skip)
-		vep_end_skip(vep, p);
+		vep_mark_skip(vep, p);
 	else
-		vep_end_verbatim(vep, p);
+		vep_mark_verbatim(vep, p);
 }
 
 /*---------------------------------------------------------------------
@@ -450,7 +473,7 @@
 		w = HTC_Read(htc, st->ptr + st->len, l);
 		if (w <= 0)
 			return (w);
-#if 0
+#if 1
 		{
 		for (l = 0; l < w; l++) 
 			vep_parse(vep, (const char *)st->ptr + st->len + l, 1);




From phk at varnish-cache.org  Wed Jan 12 15:13:35 2011
From: phk at varnish-cache.org (phk at varnish-cache.org)
Date: Wed, 12 Jan 2011 16:13:35 +0100
Subject: r5722 - trunk/varnish-cache/bin/varnishtest/tests
Message-ID: <E1Pd2OF-0005Z1-Jz@project.varnish-software.com>

Author: phk
Date: 2011-01-12 16:13:35 +0100 (Wed, 12 Jan 2011)
New Revision: 5722

Modified:
   trunk/varnish-cache/bin/varnishtest/tests/e00001.vtc
Log:
Make this test-case a tad more challenging



Modified: trunk/varnish-cache/bin/varnishtest/tests/e00001.vtc
===================================================================
--- trunk/varnish-cache/bin/varnishtest/tests/e00001.vtc	2011-01-12 15:13:20 UTC (rev 5721)
+++ trunk/varnish-cache/bin/varnishtest/tests/e00001.vtc	2011-01-12 15:13:35 UTC (rev 5722)
@@ -9,7 +9,7 @@
 		<esi:remove>
 		This is a test: Unseen University
 		</esi:remove>
-		This is a test: Hello world
+		<esX>This is a test: Hello world
 	}
 } -start
 
@@ -23,7 +23,7 @@
 	txreq 
 	rxresp
 	expect resp.status == 200
-	expect resp.bodylen == 35
+	expect resp.bodylen == 40
 }
 
 client c1 -run




From phk at varnish-cache.org  Thu Jan 13 08:41:07 2011
From: phk at varnish-cache.org (phk at varnish-cache.org)
Date: Thu, 13 Jan 2011 09:41:07 +0100
Subject: r5723 - trunk/varnish-cache/bin/varnishtest/tests
Message-ID: <E1PdIjz-0007oW-Bq@project.varnish-software.com>

Author: phk
Date: 2011-01-13 09:41:06 +0100 (Thu, 13 Jan 2011)
New Revision: 5723

Modified:
   trunk/varnish-cache/bin/varnishtest/tests/e00001.vtc
Log:
Make this test more challenging



Modified: trunk/varnish-cache/bin/varnishtest/tests/e00001.vtc
===================================================================
--- trunk/varnish-cache/bin/varnishtest/tests/e00001.vtc	2011-01-12 15:13:35 UTC (rev 5722)
+++ trunk/varnish-cache/bin/varnishtest/tests/e00001.vtc	2011-01-13 08:41:06 UTC (rev 5723)
@@ -8,6 +8,8 @@
 	txresp -body {
 		<esi:remove>
 		This is a test: Unseen University
+		<esi:include src="trick question">
+		<!--esi XXX -->
 		</esi:remove>
 		<esX>This is a test: Hello world
 	}




From phk at varnish-cache.org  Thu Jan 13 08:53:45 2011
From: phk at varnish-cache.org (phk at varnish-cache.org)
Date: Thu, 13 Jan 2011 09:53:45 +0100
Subject: r5724 - trunk/varnish-cache/bin/varnishtest/tests
Message-ID: <E1PdIwD-0008AT-Al@project.varnish-software.com>

Author: phk
Date: 2011-01-13 09:53:44 +0100 (Thu, 13 Jan 2011)
New Revision: 5724

Modified:
   trunk/varnish-cache/bin/varnishtest/tests/e00001.vtc
Log:
This is strictly speaking an ESI bug



Modified: trunk/varnish-cache/bin/varnishtest/tests/e00001.vtc
===================================================================
--- trunk/varnish-cache/bin/varnishtest/tests/e00001.vtc	2011-01-13 08:41:06 UTC (rev 5723)
+++ trunk/varnish-cache/bin/varnishtest/tests/e00001.vtc	2011-01-13 08:53:44 UTC (rev 5724)
@@ -29,4 +29,4 @@
 }
 
 client c1 -run
-varnish v1 -expect esi_errors == 0
+varnish v1 -expect esi_errors == 1




From phk at varnish-cache.org  Thu Jan 13 09:04:55 2011
From: phk at varnish-cache.org (phk at varnish-cache.org)
Date: Thu, 13 Jan 2011 10:04:55 +0100
Subject: r5725 - trunk/varnish-cache/bin/varnishd
Message-ID: <E1PdJ71-0008Qk-Vr@project.varnish-software.com>

Author: phk
Date: 2011-01-13 10:04:55 +0100 (Thu, 13 Jan 2011)
New Revision: 5725

Modified:
   trunk/varnish-cache/bin/varnishd/cache_esi_parse.c
Log:
Get comments and cdata right



Modified: trunk/varnish-cache/bin/varnishd/cache_esi_parse.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_esi_parse.c	2011-01-13 08:53:44 UTC (rev 5724)
+++ trunk/varnish-cache/bin/varnishd/cache_esi_parse.c	2011-01-13 09:04:55 UTC (rev 5725)
@@ -32,14 +32,15 @@
 #include "svnid.h"
 SVNID("$Id")
 
+#include <stdio.h>
+#include <stdlib.h>
+
 #include "cache.h"
 #include "cache_esi.h"
 #include "vend.h"
 #include "vct.h"
 #include "stevedore.h"
 
-#include <stdio.h>
-
 #ifndef OLD_ESI
 
 struct vep_match {
@@ -68,12 +69,14 @@
 	const char		*until_p;
 	const char		*until_s;
 
+	const char		*esicmt;
+	const char		*esicmt_p;
+
 	struct vep_match	*match;
 	int			match_l;
 
 	char			tag[10];
 	int			tag_i;
-	
 };
 
 /*---------------------------------------------------------------------*/
@@ -85,8 +88,7 @@
 static const char *VEP_MATCHBUF = 	"[MatchBuf]";
 static const char *VEP_NEXTTAG = 	"[NxtTag]";
 static const char *VEP_NOTMYTAG =	"[NotMyTag]";
-static const char *VEP_ESICMT =		"[EsiComment]";
-static const char *VEP_CMT =		"[Comment]";
+static const char *VEP_COMMENT =	"[Comment]";
 static const char *VEP_CDATA =		"[CDATA]";
 static const char *VEP_ESITAG =		"[ESITag]";
 static const char *VEP_ESIETAG =	"[ESIEndTag]";
@@ -100,18 +102,19 @@
 
 /*---------------------------------------------------------------------*/
 
-static struct vep_match vep_match_tbl[] = {
-	{ "<!--esi",	&VEP_ESICMT },
-	{ "<!--",	&VEP_CMT },
+static struct vep_match vep_match_starttag[] = {
+	{ "<!--",	&VEP_COMMENT },
 	{ "</esi:",	&VEP_ESIETAG },
 	{ "<esi:",	&VEP_ESITAG },
 	{ "<![CDATA[",	&VEP_CDATA },
 	{ NULL,		&VEP_NOTMYTAG }
 };
 
-static const int vep_match_tbl_len =
-    sizeof vep_match_tbl / sizeof vep_match_tbl[0];
+static const int vep_match_starttag_len =
+    sizeof vep_match_starttag / sizeof vep_match_starttag[0];
 
+/*---------------------------------------------------------------------*/
+
 static struct vep_match vep_match_esi[] = {
 	{ "include",	&VEP_ESIINCLUDE },
 	{ "remove",	&VEP_ESIREMOVE },
@@ -122,6 +125,8 @@
 static const int vep_match_esi_len =
     sizeof vep_match_esi / sizeof vep_match_esi[0];
 
+/*---------------------------------------------------------------------*/
+
 static struct vep_match vep_match_esie[] = {
 	{ "remove",	&VEP_ESI_REMOVE },
 	{ NULL,		&VEP_XXX }
@@ -208,6 +213,25 @@
 } 
 
 static void
+vep_emit_literal(struct vep_state *vep, const char *p, const char *e)
+{
+	ssize_t l;
+
+	if (e == NULL)
+		e = strchr(p, '\0');
+	if (vep->o_verbatim > 0) 
+		vep_emit_verbatim(vep);
+	if (vep->o_skip > 0) 
+		vep_emit_skip(vep);
+	l = e - p;
+	printf("---->L(%d) [%.*s]\n", (int)l, (int)l, p);
+	vep_emit_len(vep, l, VEC_L1, VEC_L2, VEC_L4);
+	vsb_printf(vep->vsb, "%lx\r\n%c", l, 0);
+	vsb_bcat(vep->vsb, p, l);
+}
+
+
+static void
 vep_mark_verbatim(struct vep_state *vep, const char *p)
 {
 	ssize_t l;
@@ -241,24 +265,12 @@
 	vep->ver_p = p;
 } 
 
-static void
-vep_emit_literal(struct vep_state *vep, const char *p, const char *e)
-{
-	ssize_t l;
-
-	if (vep->o_verbatim > 0) 
-		vep_emit_verbatim(vep);
-	if (vep->o_skip > 0) 
-		vep_emit_skip(vep);
-	l = e - p;
-	printf("---->L(%d) [%.*s]\n", (int)l, (int)l, p);
-	vep_emit_len(vep, l, VEC_L1, VEC_L2, VEC_L4);
-	vsb_printf(vep->vsb, "%lx\r\n%c", l, 0);
-	vsb_bcat(vep->vsb, p, l);
-}
-
 /*---------------------------------------------------------------------
- * Parse object for ESI instructions
+ * Lex/Parse object for ESI instructions
+ *
+ * This function is called with the input object piecemal so do not
+ * assume that we have more than one char available at at time, but
+ * optimize for getting huge chunks. 
  */
 
 static void
@@ -272,49 +284,98 @@
 
 	e = b + l;
 
-	vep->ver_p = b;
-	printf("EP Call %d [%.*s]\n", (int)l, (int)l, b);
+	if (0)
+		vep_emit_literal(vep, "A", "B");
+
 	p = b;
 	while (p < e) {
 		AN(vep->state);
-		printf("EP %s [%.*s]\n",
+		printf("EP %s %d %d (%.*s) [%.*s]\n",
 		    vep->state,
+		    vep->skip,
+		    vep->remove,
+		    vep->tag_i, vep->tag,
 		    (int)(e - p), p);
+		fflush(stdout);
+		usleep(10);
+
+		/******************************************************
+		 *
+		 */
+
 		if (vep->state == VEP_START) {
 			/*
-			 * Look for the first non-white char, and
-			 * abandon if it is not '<' under the assumption
-			 * that it is not an ESI file
+			 * If the first non-whitespace char is different
+			 * from '<' we assume this is not XML.
 			 */
-			while (p < e && vct_islws(*p))
+			while (p < e && vct_islws(*p)) {
 				p++;
+				vep_mark_verbatim(vep, p);
+			}
 			if (p < e) {
-				if (*p == '<')
+				if (*p == '<') {
 					vep->state = VEP_STARTTAG;
-				else
+				} else
 					vep->state = VEP_NOTXML;
 			}
 		} else if (vep->state == VEP_NOTXML) {
+			/*
+			 * This is not recognized as XML, just skip thru
+			 * vfp_esi_end() will handle the rest
+			 */
 			p = e;
+
+		/******************************************************
+		 *
+		 */
+
+		} else if (vep->state == VEP_NOTMYTAG) {
+			vep->tag_i = 0;
+			while (p < e) {
+				if (!vep->remove)
+					vep_mark_verbatim(vep, p + 1);
+				if (*p++ == '>') {
+					vep->state = VEP_NEXTTAG;
+					break;
+				}
+			}
 		} else if (vep->state == VEP_NEXTTAG) {
 			/*
-			 * Hunt for start of next tag
+			 * Hunt for start of next tag and keep an eye
+			 * out for end of EsiCmt if armed.
 			 */
 			while (p < e && *p != '<') {
-#if 0
-				if (vep->incmt != NULL &&
-				    *p == *vep->incmt_p) {
-					if (*++vep->incmt_p == '\0') {
-						vep->incmt = NULL;
-						vep->incmt = NULL;
+				if (vep->esicmt_p != NULL &&
+				    *p == *vep->esicmt_p++) {
+					p++;
+					if (*vep->esicmt_p == '\0') {
+						vep->esicmt = NULL;
+						vep->esicmt_p = NULL;
+						/*
+						 * The end of the esicmt
+						 * should not be emitted.
+						 * But the stuff before should
+						 */
+						if (!vep->remove)
+							vep_mark_verbatim(vep,
+							    p - 3);
+						vep_mark_skip(vep, p);
 					}
-				} else
-					vep->incmt_p = vep->incmt;
-#endif
-				p++;
+				} else {
+					p++;
+					vep->esicmt_p = vep->esicmt;
+					if (vep->esicmt_p == NULL &&
+					    !vep->remove)
+						vep_mark_verbatim(vep, p);
+				}
 			}
 			if (p < e)
 				vep->state = VEP_STARTTAG;
+
+		/******************************************************
+		 *
+		 */
+
 		} else if (vep->state == VEP_STARTTAG) {
 			/*
 			 * Start of tag, set up match table
@@ -322,10 +383,41 @@
 			assert(*p == '<');
 			if (!vep->remove)
 				vep_mark_verbatim(vep, p);
-			vep->match = vep_match_tbl;
-			vep->match_l = vep_match_tbl_len;
+			vep->match = vep_match_starttag;
+			vep->match_l = vep_match_starttag_len;
 			vep->state = VEP_MATCH;
-			vep->skip = 1;
+		} else if (vep->state == VEP_COMMENT) {
+			/*
+			 * We are in a comment, find out if it is an
+			 * ESI comment or a regular comment
+			 */
+			if (vep->esicmt == NULL)
+				vep->esicmt_p = vep->esicmt = "esi";
+			while (p < e) {
+				if (*p == *vep->esicmt_p) {
+					p++;
+					if (*++vep->esicmt_p == '\0') {
+						vep->esicmt_p =
+						    vep->esicmt = "-->";
+						vep->state = VEP_NEXTTAG;
+						vep_mark_skip(vep, p);
+						break;
+					}
+				} else {
+					vep->esicmt_p = vep->esicmt = NULL; 
+					vep->until_p = vep->until = "-->";
+					vep->until_s = VEP_NEXTTAG;
+					vep->state = VEP_UNTIL;
+					break;
+				}
+			}
+		} else if (vep->state == VEP_CDATA) {
+			/*
+			 * Easy: just look for the end of CDATA
+			 */
+			vep->until_p = vep->until = "]]>";
+			vep->until_s = VEP_NEXTTAG;
+			vep->state = VEP_UNTIL;
 		} else if (vep->state == VEP_ENDTAG) {
 			while (p < e && *p != '>') 
 				p++;
@@ -335,10 +427,38 @@
 			}
 			vep_mark_skip(vep, p);
 			vep->skip = 0;
-		} else if (vep->state == VEP_CDATA) {
-			vep->until_p = vep->until = "]]>";
-			vep->until_s = VEP_NEXTTAG;
-			vep->state = VEP_UNTIL;
+
+		/******************************************************
+		 *
+		 */
+
+		} else if (vep->state == VEP_ESITAG) {
+			if (vep->remove) {
+				VSC_main->esi_errors++;
+				vep->state = VEP_NOTMYTAG;
+				break;
+			}
+			vep->skip = 1;
+			vep_mark_skip(vep, p);
+			vep->match = vep_match_esi;
+			vep->match_l = vep_match_esi_len;
+			vep->state = VEP_MATCH;
+		} else if (vep->state == VEP_ESIETAG) {
+			vep->match = vep_match_esie;
+			vep->match_l = vep_match_esie_len;
+			vep->state = VEP_MATCH;
+		} else if (vep->state == VEP_ESIREMOVE) {
+			vep_mark_skip(vep, p);
+			vep->remove = 1;
+			vep->state = VEP_NEXTTAG;
+		} else if (vep->state == VEP_ESI_REMOVE) {
+			vep->remove = 0;
+			vep->state = VEP_ENDTAG;
+
+		/******************************************************
+		 *
+		 */
+
 		} else if (vep->state == VEP_MATCH) {
 			/*
 			 * Match against a table
@@ -376,24 +496,9 @@
 				b = e;
 				break;
 			}
-			if (vm->match && vep->tag_i > strlen(vm->match)) {
-				/*
-				 * not generally safe but
-				 * works for the required
-				 * case of <--esi and <--
-				 */
-				p -= vep->tag_i -
-				    strlen(vm->match);
-				vep->tag_i--;
-			}
-			if (vm->match == NULL) {
-				vep_emit_literal(vep,
-				    vep->tag, vep->tag + vep->tag_i);
-			}
 			b = p;
 			vep->state = *vm->state;
 			vep->match = NULL;
-			vep->tag_i = 0;
 		} else if (vep->state == VEP_UNTIL) {
 			/*
 			 * Skip until we see magic string
@@ -406,40 +511,11 @@
 					break;
 				}
 			}
-		} else if (vep->state == VEP_NOTMYTAG) {
-			vep->skip = 0;
-			while (p < e) {
-				if (*p++ == '>') {
-					vep->state = VEP_NEXTTAG;
-					break;
-				}
-			}
-		} else if (vep->state == VEP_ESITAG) {
-			vep->skip = 1;
-			vep_mark_skip(vep, p);
-			vep->match = vep_match_esi;
-			vep->match_l = vep_match_esi_len;
-			vep->state = VEP_MATCH;
-		} else if (vep->state == VEP_ESIETAG) {
-			vep->match = vep_match_esie;
-			vep->match_l = vep_match_esie_len;
-			vep->state = VEP_MATCH;
-		} else if (vep->state == VEP_ESIREMOVE) {
-			vep_mark_skip(vep, p);
-			vep->remove = 1;
-			vep->state = VEP_NEXTTAG;
-		} else if (vep->state == VEP_ESI_REMOVE) {
-			vep->remove = 0;
-			vep->state = VEP_ENDTAG;
 		} else {
 			printf("*** Unknown state %s\n", vep->state);
-			break;
+			INCOMPL();
 		}
 	}
-	if (vep->remove || vep->skip)
-		vep_mark_skip(vep, p);
-	else
-		vep_mark_verbatim(vep, p);
 }
 
 /*---------------------------------------------------------------------
@@ -473,6 +549,7 @@
 		w = HTC_Read(htc, st->ptr + st->len, l);
 		if (w <= 0)
 			return (w);
+		vep->ver_p = (const char *)st->ptr + st->len;
 #if 1
 		{
 		for (l = 0; l < w; l++) 




From phk at varnish-cache.org  Thu Jan 13 10:26:39 2011
From: phk at varnish-cache.org (phk at varnish-cache.org)
Date: Thu, 13 Jan 2011 11:26:39 +0100
Subject: r5726 - trunk/varnish-cache/bin/varnishd
Message-ID: <E1PdKO7-00024q-Q1@project.varnish-software.com>

Author: phk
Date: 2011-01-13 11:26:39 +0100 (Thu, 13 Jan 2011)
New Revision: 5726

Modified:
   trunk/varnish-cache/bin/varnishd/cache_esi_parse.c
Log:
Code to collect attributes we care about and ignore the rest.

This Is needlessly general for now, but will make it easier to
add more ESI features at a later date.



Modified: trunk/varnish-cache/bin/varnishd/cache_esi_parse.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_esi_parse.c	2011-01-13 09:04:55 UTC (rev 5725)
+++ trunk/varnish-cache/bin/varnishd/cache_esi_parse.c	2011-01-13 10:26:39 UTC (rev 5726)
@@ -57,6 +57,10 @@
 	/* parser state */
 	const char		*state;
 
+	unsigned		endtag;
+	unsigned		emptytag;
+	unsigned		canattr;
+
 	unsigned		remove;
 	unsigned		skip;
 
@@ -72,8 +76,14 @@
 	const char		*esicmt;
 	const char		*esicmt_p;
 
+	struct vep_match	*attr;
+	int			attr_l;
+	struct vsb		*attr_vsb;
+	int			attr_delim;
+
 	struct vep_match	*match;
 	int			match_l;
+	struct vep_match	*match_hit;
 
 	char			tag[10];
 	int			tag_i;
@@ -83,20 +93,30 @@
 
 static const char *VEP_START =		"[Start]";
 static const char *VEP_NOTXML =	 	"[NotXml]";
-static const char *VEP_STARTTAG = 	"[StartTag]";
-static const char *VEP_ENDTAG = 	"[EndTag]";
-static const char *VEP_MATCHBUF = 	"[MatchBuf]";
+
 static const char *VEP_NEXTTAG = 	"[NxtTag]";
 static const char *VEP_NOTMYTAG =	"[NotMyTag]";
+
+static const char *VEP_STARTTAG = 	"[StartTag]";
 static const char *VEP_COMMENT =	"[Comment]";
 static const char *VEP_CDATA =		"[CDATA]";
 static const char *VEP_ESITAG =		"[ESITag]";
 static const char *VEP_ESIETAG =	"[ESIEndTag]";
-static const char *VEP_UNTIL =		"[Until]";
+
 static const char *VEP_ESIREMOVE =	"[ESI:Remove]";
-static const char *VEP_ESI_REMOVE =	"[ESI:/Remove]";
 static const char *VEP_ESIINCLUDE =	"[ESI:Include]";
 static const char *VEP_ESICOMMENT =	"[ESI:Comment]";
+
+static const char *VEP_INTAG =		"[InTag]";
+
+static const char *VEP_ATTR =		"[Attribute]";
+static const char *VEP_SKIPATTR =	"[SkipAttribute]";
+static const char *VEP_SKIPATTR2 =	"[SkipAttribute2]";
+static const char *VEP_ATTRGETVAL =	"[AttrGetValue]";
+static const char *VEP_ATTRVAL =	"[AttrValue]";
+
+static const char *VEP_UNTIL =		"[Until]";
+static const char *VEP_MATCHBUF = 	"[MatchBuf]";
 static const char *VEP_MATCH =		"[Match]";
 static const char *VEP_XXX =		"[XXX]";
 
@@ -128,13 +148,23 @@
 /*---------------------------------------------------------------------*/
 
 static struct vep_match vep_match_esie[] = {
-	{ "remove",	&VEP_ESI_REMOVE },
+	{ "remove",	&VEP_ESIREMOVE },
 	{ NULL,		&VEP_XXX }
 };
 
 static const int vep_match_esie_len =
     sizeof vep_match_esie / sizeof vep_match_esie[0];
 
+/*---------------------------------------------------------------------*/
+
+static struct vep_match vep_match_attr_include[] = {
+	{ "src=",	&VEP_ATTRGETVAL },
+	{ NULL,		&VEP_SKIPATTR }
+};
+
+static const int vep_match_attr_include_len =
+    sizeof vep_match_attr_include / sizeof vep_match_attr_include[0];
+
 /*---------------------------------------------------------------------
  * return match or NULL if more input needed.
  */
@@ -278,6 +308,7 @@
 {
 	const char *e, *p;
 	struct vep_match *vm;
+	int i;
 
 	CHECK_OBJ_NOTNULL(vep, VEP_MAGIC);
 	assert(l > 0);
@@ -300,7 +331,7 @@
 		usleep(10);
 
 		/******************************************************
-		 *
+		 * SECTION A
 		 */
 
 		if (vep->state == VEP_START) {
@@ -326,7 +357,7 @@
 			p = e;
 
 		/******************************************************
-		 *
+		 * SECTION D
 		 */
 
 		} else if (vep->state == VEP_NOTMYTAG) {
@@ -373,7 +404,7 @@
 				vep->state = VEP_STARTTAG;
 
 		/******************************************************
-		 *
+		 * SECTION B
 		 */
 
 		} else if (vep->state == VEP_STARTTAG) {
@@ -418,21 +449,9 @@
 			vep->until_p = vep->until = "]]>";
 			vep->until_s = VEP_NEXTTAG;
 			vep->state = VEP_UNTIL;
-		} else if (vep->state == VEP_ENDTAG) {
-			while (p < e && *p != '>') 
-				p++;
-			if (*p == '>') {
-				p++;
-				vep->state = VEP_NEXTTAG;
-			}
-			vep_mark_skip(vep, p);
-			vep->skip = 0;
-
-		/******************************************************
-		 *
-		 */
-
 		} else if (vep->state == VEP_ESITAG) {
+			vep->tag_i = 0;
+			vep->endtag = 0;
 			if (vep->remove) {
 				VSC_main->esi_errors++;
 				vep->state = VEP_NOTMYTAG;
@@ -444,26 +463,122 @@
 			vep->match_l = vep_match_esi_len;
 			vep->state = VEP_MATCH;
 		} else if (vep->state == VEP_ESIETAG) {
+			vep->tag_i = 0;
+			vep->endtag = 1;
 			vep->match = vep_match_esie;
 			vep->match_l = vep_match_esie_len;
 			vep->state = VEP_MATCH;
+		} else if (vep->state == VEP_ESIINCLUDE) {
+			vep->state = VEP_INTAG;
+			vep->attr = vep_match_attr_include;
+			vep->attr_l = vep_match_attr_include_len;
 		} else if (vep->state == VEP_ESIREMOVE) {
-			vep_mark_skip(vep, p);
-			vep->remove = 1;
-			vep->state = VEP_NEXTTAG;
-		} else if (vep->state == VEP_ESI_REMOVE) {
-			vep->remove = 0;
-			vep->state = VEP_ENDTAG;
+			vep->remove = !vep->endtag;
 
 		/******************************************************
-		 *
+		 * SECTION F
 		 */
 
+		} else if (vep->state == VEP_INTAG) {
+			vep->tag_i = 0;
+			while (p < e && vct_islws(*p)) {
+				p++;	
+				vep->canattr = 1;
+			}
+			if (p < e && *p == '/' && !vep->emptytag) {
+				p++;
+				vep->emptytag = 1;
+				vep->canattr = 0;
+			}
+			if (p < e && *p == '>') {
+				p++;
+				/* XXX: processing */
+				vep->state = VEP_NEXTTAG;
+			} else if (p < e && vep->emptytag) {
+				INCOMPL();	/* ESI-SYNTAX ERROR */
+			} else if (p < e && vct_isxmlnamestart(*p)) {
+				vep->state = VEP_ATTR;
+			} else if (p < e) {
+				INCOMPL();	/* ESI-SYNTAX ERROR */
+			}
+
+		/******************************************************
+		 * SECTION G
+		 */
+
+		} else if (vep->state == VEP_ATTR) {
+			AZ(vep->attr_delim);
+			if (vep->attr == NULL) {
+				p++;
+				AZ(vep->attr_vsb);
+				vep->state = VEP_SKIPATTR;
+				break;
+			}
+			vep->match = vep->attr;
+			vep->match_l = vep->attr_l;
+			vep->state = VEP_MATCH;
+		} else if (vep->state == VEP_SKIPATTR) {
+			vep->state = VEP_SKIPATTR2;
+			for (i = 0; i < vep->tag_i; i++) {
+				if (vct_isxmlname(vep->tag[i]))
+					continue;
+				if (vep->tag[i] == '=') {
+					assert(i + 1 == vep->tag_i);
+					vep->state = VEP_ATTRVAL;
+				}
+			}
+			xxxassert(i == vep->tag_i);
+			vep->tag_i = 0;
+		} else if (vep->state == VEP_SKIPATTR2) {
+			while (p < e && vct_isxmlname(*p))
+				p++;
+			if (p < e && *p == '=') {
+				p++;
+				vep->state = VEP_ATTRVAL;
+				break;
+			}
+			if (p < e) {
+				INCOMPL();	/* ESI-SYNTAX ERROR */
+			}
+		} else if (vep->state == VEP_ATTRGETVAL) {
+			vep->attr_vsb = vsb_newauto();
+			vep->state = VEP_ATTRVAL;
+		} else if (vep->state == VEP_ATTRVAL) {
+			if (vep->attr_delim == 0)  {
+				if (*p != '"' && *p != '\'')
+					INCOMPL();	/* ESI-SYNTAX */
+				vep->attr_delim = *p++;
+			}
+			while (p < e && *p != vep->attr_delim) {
+				if (vep->attr_vsb != NULL)
+					vsb_bcat(vep->attr_vsb, p, 1);
+				p++;
+			}
+			if (p < e) {
+				if (vep->attr_vsb != NULL) {
+					vsb_finish(vep->attr_vsb);
+					printf("ATTR (%s) (%s)\n",
+						vep->match_hit->match,
+						vsb_data(vep->attr_vsb));
+					vsb_delete(vep->attr_vsb);
+					vep->attr_vsb = NULL;
+				}
+				p++;
+				vep->attr_delim = 0;
+				vep->state = VEP_INTAG;
+			}
+	
+
+		/******************************************************
+		 * Utility Section
+		 */
+
 		} else if (vep->state == VEP_MATCH) {
 			/*
 			 * Match against a table
 			 */
 			vm = vep_match(vep, p, e);
+			vep->match_hit = vm;
 			if (vm != NULL) {
 				if (vm->match != NULL)
 					p += strlen(vm->match);
@@ -492,6 +607,7 @@
 					    vep->tag, vep->tag + vep->tag_i);
 				}
 			} while (vm == 0 && p < e);
+			vep->match_hit = vm;
 			if (vm == 0) {
 				b = e;
 				break;




From phk at varnish-cache.org  Thu Jan 13 10:49:57 2011
From: phk at varnish-cache.org (phk at varnish-cache.org)
Date: Thu, 13 Jan 2011 11:49:57 +0100
Subject: r5727 - trunk/varnish-cache/bin/varnishd
Message-ID: <E1PdKkf-0002eV-G5@project.varnish-software.com>

Author: phk
Date: 2011-01-13 11:49:56 +0100 (Thu, 13 Jan 2011)
New Revision: 5727

Modified:
   trunk/varnish-cache/bin/varnishd/cache_esi.h
   trunk/varnish-cache/bin/varnishd/cache_esi_deliver.c
   trunk/varnish-cache/bin/varnishd/cache_esi_parse.c
Log:
Add call backfunction for esi attributes & tags.

Have esi:include emit the include VEC and debug it on delivery.



Modified: trunk/varnish-cache/bin/varnishd/cache_esi.h
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_esi.h	2011-01-13 10:26:39 UTC (rev 5726)
+++ trunk/varnish-cache/bin/varnishd/cache_esi.h	2011-01-13 10:49:56 UTC (rev 5727)
@@ -36,3 +36,4 @@
 #define	VEC_L1	'c'
 #define	VEC_L2	'x'
 #define	VEC_L4	'm'
+#define	VEC_INCL	'I'

Modified: trunk/varnish-cache/bin/varnishd/cache_esi_deliver.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_esi_deliver.c	2011-01-13 10:26:39 UTC (rev 5726)
+++ trunk/varnish-cache/bin/varnishd/cache_esi_deliver.c	2011-01-13 10:49:56 UTC (rev 5727)
@@ -99,6 +99,12 @@
 				WRW_Write(sp->wrk, "\r\n", -1);
 			p = p + l;
 			break;
+		case VEC_INCL:
+			p++;
+			q = (void*)strchr((const char*)p, '\0');
+			printf("INCL [%s]\n", p);
+			p = q + 1;
+			break;
 		default:
 			printf("XXXX %02x [%c]\n", *p, *p);
 			INCOMPL();

Modified: trunk/varnish-cache/bin/varnishd/cache_esi_parse.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_esi_parse.c	2011-01-13 10:26:39 UTC (rev 5726)
+++ trunk/varnish-cache/bin/varnishd/cache_esi_parse.c	2011-01-13 10:49:56 UTC (rev 5727)
@@ -43,6 +43,11 @@
 
 #ifndef OLD_ESI
 
+struct vep_state;
+
+enum dowhat {DO_ATTR, DO_TAG};
+typedef void dostuff_f(struct vep_state *, enum dowhat);
+
 struct vep_match {
 	const char	*match;
 	const char	**state;
@@ -87,6 +92,10 @@
 
 	char			tag[10];
 	int			tag_i;
+
+	dostuff_f		*dostuff;
+
+	struct vsb		*include_src;
 };
 
 /*---------------------------------------------------------------------*/
@@ -296,6 +305,35 @@
 } 
 
 /*---------------------------------------------------------------------
+ */
+
+static void
+vep_do_include(struct vep_state *vep, enum dowhat what)
+{
+
+	printf("DO_INCLUDE(%d)\n", what);
+	if (what == DO_ATTR) {
+		printf("ATTR (%s) (%s)\n", vep->match_hit->match,
+			vsb_data(vep->attr_vsb));
+		XXXAZ(vep->include_src);	/* multiple src= */
+		vep->include_src = vep->attr_vsb;
+	} else {
+		XXXAN(vep->include_src);
+		if (vep->o_skip > 0) 
+			vep_emit_skip(vep);
+		if (vep->o_verbatim > 0) 
+			vep_emit_verbatim(vep);
+		/* XXX: what if it contains NUL bytes ?? */
+		vsb_printf(vep->vsb, "%c%s%c",
+		    VEC_INCL,
+		    vsb_data(vep->include_src), 0);
+
+		vsb_delete(vep->include_src);
+		vep->include_src = NULL;
+	}
+}
+
+/*---------------------------------------------------------------------
  * Lex/Parse object for ESI instructions
  *
  * This function is called with the input object piecemal so do not
@@ -469,6 +507,7 @@
 			vep->match_l = vep_match_esie_len;
 			vep->state = VEP_MATCH;
 		} else if (vep->state == VEP_ESIINCLUDE) {
+			vep->dostuff = vep_do_include;
 			vep->state = VEP_INTAG;
 			vep->attr = vep_match_attr_include;
 			vep->attr_l = vep_match_attr_include_len;
@@ -492,7 +531,9 @@
 			}
 			if (p < e && *p == '>') {
 				p++;
-				/* XXX: processing */
+				vep_mark_skip(vep, p);
+				AN(vep->dostuff);
+				vep->dostuff(vep, DO_TAG);
 				vep->state = VEP_NEXTTAG;
 			} else if (p < e && vep->emptytag) {
 				INCOMPL();	/* ESI-SYNTAX ERROR */
@@ -535,9 +576,7 @@
 			if (p < e && *p == '=') {
 				p++;
 				vep->state = VEP_ATTRVAL;
-				break;
-			}
-			if (p < e) {
+			} else if (p < e) {
 				INCOMPL();	/* ESI-SYNTAX ERROR */
 			}
 		} else if (vep->state == VEP_ATTRGETVAL) {
@@ -557,10 +596,8 @@
 			if (p < e) {
 				if (vep->attr_vsb != NULL) {
 					vsb_finish(vep->attr_vsb);
-					printf("ATTR (%s) (%s)\n",
-						vep->match_hit->match,
-						vsb_data(vep->attr_vsb));
-					vsb_delete(vep->attr_vsb);
+					AN(vep->dostuff);
+					vep->dostuff(vep, DO_ATTR);
 					vep->attr_vsb = NULL;
 				}
 				p++;
@@ -591,7 +628,6 @@
 				vep->tag_i = e - p;
 				vep->state = VEP_MATCHBUF;
 				p = e;
-				break;
 			}
 		} else if (vep->state == VEP_MATCHBUF) {
 			/*
@@ -606,15 +642,14 @@
 					vm = vep_match(vep,
 					    vep->tag, vep->tag + vep->tag_i);
 				}
-			} while (vm == 0 && p < e);
+			} while (vm == NULL && p < e);
 			vep->match_hit = vm;
-			if (vm == 0) {
-				b = e;
-				break;
+			if (vm == NULL) {
+				assert(p == e);
+			} else {
+				vep->state = *vm->state;
+				vep->match = NULL;
 			}
-			b = p;
-			vep->state = *vm->state;
-			vep->match = NULL;
 		} else if (vep->state == VEP_UNTIL) {
 			/*
 			 * Skip until we see magic string




From phk at varnish-cache.org  Thu Jan 13 11:07:52 2011
From: phk at varnish-cache.org (phk at varnish-cache.org)
Date: Thu, 13 Jan 2011 12:07:52 +0100
Subject: r5728 - trunk/varnish-cache/bin/varnishd
Message-ID: <E1PdL20-00037X-UJ@project.varnish-software.com>

Author: phk
Date: 2011-01-13 12:07:52 +0100 (Thu, 13 Jan 2011)
New Revision: 5728

Modified:
   trunk/varnish-cache/bin/varnishd/cache_esi_parse.c
Log:
Make esi:remove discard any attributes



Modified: trunk/varnish-cache/bin/varnishd/cache_esi_parse.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_esi_parse.c	2011-01-13 10:49:56 UTC (rev 5727)
+++ trunk/varnish-cache/bin/varnishd/cache_esi_parse.c	2011-01-13 11:07:52 UTC (rev 5728)
@@ -308,6 +308,20 @@
  */
 
 static void
+vep_do_nothing(struct vep_state *vep, enum dowhat what)
+{
+	printf("DO_NOTHING(%d)\n", what);
+	if (what == DO_ATTR) {
+		printf("ATTR (%s) (%s)\n", vep->match_hit->match,
+			vsb_data(vep->attr_vsb));
+		vsb_delete(vep->attr_vsb);
+	}
+}
+
+/*---------------------------------------------------------------------
+ */
+
+static void
 vep_do_include(struct vep_state *vep, enum dowhat what)
 {
 
@@ -512,7 +526,9 @@
 			vep->attr = vep_match_attr_include;
 			vep->attr_l = vep_match_attr_include_len;
 		} else if (vep->state == VEP_ESIREMOVE) {
+			vep->dostuff = vep_do_nothing;
 			vep->remove = !vep->endtag;
+			vep->state = VEP_INTAG;
 
 		/******************************************************
 		 * SECTION F
@@ -731,6 +747,7 @@
 
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 	AZ(sp->wrk->vep);
+	/* XXX: snapshot WS ? We'll need the space */
 	vep = (void*)WS_Alloc(sp->wrk->ws, sizeof *vep);
 	AN(vep);
 
@@ -799,6 +816,7 @@
 	if (st->len < st->space)
 		STV_trim(st, st->len);
 	VTAILQ_INSERT_TAIL(&sp->obj->store, st, list);
+	sp->wrk->vep = NULL;
 	return (0);
 }
 




From phk at varnish-cache.org  Thu Jan 13 11:12:08 2011
From: phk at varnish-cache.org (phk at varnish-cache.org)
Date: Thu, 13 Jan 2011 12:12:08 +0100
Subject: r5729 - trunk/varnish-cache/bin/varnishd
Message-ID: <E1PdL68-0003GD-Ot@project.varnish-software.com>

Author: phk
Date: 2011-01-13 12:12:06 +0100 (Thu, 13 Jan 2011)
New Revision: 5729

Modified:
   trunk/varnish-cache/bin/varnishd/cache.h
   trunk/varnish-cache/bin/varnishd/cache_esi.c
   trunk/varnish-cache/bin/varnishd/cache_esi_deliver.c
Log:
Move the magic code which delivers an included object to cache_esi_deliver.c
where it will live in the future.  Call it there from the old OLD_ESI code.

Also call it from the new ESI code.

That makes e0000[45].vtc pass.



Modified: trunk/varnish-cache/bin/varnishd/cache.h
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache.h	2011-01-13 11:07:52 UTC (rev 5728)
+++ trunk/varnish-cache/bin/varnishd/cache.h	2011-01-13 11:12:06 UTC (rev 5729)
@@ -814,6 +814,7 @@
 #else
 void ESI_Deliver(struct sess *);
 #endif /* OLD_ESI */
+void ESI_Include(struct sess *sp, const char *src, const char *host);
 
 /* cache_vrt_vmod.c */
 void VMOD_Init(void);

Modified: trunk/varnish-cache/bin/varnishd/cache_esi.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_esi.c	2011-01-13 11:07:52 UTC (rev 5728)
+++ trunk/varnish-cache/bin/varnishd/cache_esi.c	2011-01-13 11:12:06 UTC (rev 5729)
@@ -856,15 +856,10 @@
 ESI_Deliver(struct sess *sp)
 {
 	struct esi_bit *eb;
-	struct object *obj;
 	struct worker *w;
-	char *ws_wm;
-	struct http http_save;
 	struct esidata *ed;
-	unsigned sxid, res_mode;
 
 	w = sp->wrk;
-	http_save.magic = 0;
 	ed = sp->obj->esidata;
 	CHECK_OBJ_NOTNULL(ed, ESIDATA_MAGIC);
 	VTAILQ_FOREACH(eb, &ed->esibits, list) {
@@ -880,71 +875,7 @@
 		    sp->esi_level >= params->max_esi_includes)
 			continue;
 
-		if (WRW_Flush(w)) {
-			vca_close_session(sp, "remote closed");
-			return;
-		}
-		AZ(WRW_FlushRelease(w));
-
-		sp->esi_level++;
-		obj = sp->obj;
-		sp->obj = NULL;
-		res_mode = sp->wrk->res_mode;
-
-		/* Save the master objects HTTP state, we may need it later */
-		if (http_save.magic == 0)
-			http_save = *sp->http;
-
-		/* Reset request to status before we started messing with it */
-		HTTP_Copy(sp->http, sp->http0);
-
-		/* Take a workspace snapshot */
-		ws_wm = WS_Snapshot(sp->ws);
-
-		http_SetH(sp->http, HTTP_HDR_URL, eb->include.b);
-		if (eb->host.b != NULL)  {
-			http_Unset(sp->http, H_Host);
-			http_Unset(sp->http, H_If_Modified_Since);
-			http_SetHeader(w, sp->fd, sp->http, eb->host.b);
-		}
-		/*
-		 * XXX: We should decide if we should cache the director
-		 * XXX: or not (for session/backend coupling).  Until then
-		 * XXX: make sure we don't trip up the check in vcl_recv.
-		 */
-		sp->director = NULL;
-		sp->step = STP_RECV;
-		http_ForceGet(sp->http);
-
-		/* Don't do conditionals */
-		sp->http->conds = 0;
-		http_Unset(sp->http, H_If_Modified_Since);
-
-		/* Client content already taken care of */
-		http_Unset(sp->http, H_Content_Length);
-
-		sxid = sp->xid;
-		while (1) {
-			sp->wrk = w;
-			CNT_Session(sp);
-			if (sp->step == STP_DONE)
-				break;
-			AZ(sp->wrk);
-			WSL_Flush(w, 0);
-			DSL(0x20, SLT_Debug, sp->id, "loop waiting for ESI");
-			(void)usleep(10000);
-		}
-		sp->xid = sxid;
-		AN(sp->wrk);
-		assert(sp->step == STP_DONE);
-		sp->esi_level--;
-		sp->obj = obj;
-		sp->wrk->res_mode = res_mode;
-
-		/* Reset the workspace */
-		WS_Reset(sp->ws, ws_wm);
-
-		WRW_Reserve(sp->wrk, &sp->fd);
+		ESI_Include(sp, eb->include.b, eb->host.b);
 		if (sp->fd < 0)
 			break;
 	}

Modified: trunk/varnish-cache/bin/varnishd/cache_esi_deliver.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_esi_deliver.c	2011-01-13 11:07:52 UTC (rev 5728)
+++ trunk/varnish-cache/bin/varnishd/cache_esi_deliver.c	2011-01-13 11:12:06 UTC (rev 5729)
@@ -41,7 +41,84 @@
 #include "vct.h"
 #include "stevedore.h"
 
+/*--------------------------------------------------------------------*/
 
+void
+ESI_Include(struct sess *sp, const char *src, const char *host)
+{
+	struct object *obj;
+	struct worker *w;
+	char *ws_wm;
+	unsigned sxid, res_mode;
+
+	w = sp->wrk;
+
+	if (WRW_Flush(w)) {
+		vca_close_session(sp, "remote closed");
+		return;
+	}
+
+	AZ(WRW_FlushRelease(w));
+
+	sp->esi_level++;
+	obj = sp->obj;
+	sp->obj = NULL;
+	res_mode = sp->wrk->res_mode;
+
+	/* Reset request to status before we started messing with it */
+	HTTP_Copy(sp->http, sp->http0);
+
+	/* Take a workspace snapshot */
+	ws_wm = WS_Snapshot(sp->ws);
+
+	http_SetH(sp->http, HTTP_HDR_URL, src);
+	if (host != NULL)  {
+		http_Unset(sp->http, H_Host);
+		http_Unset(sp->http, H_If_Modified_Since);
+		http_SetHeader(w, sp->fd, sp->http, host);
+	}
+	/*
+	 * XXX: We should decide if we should cache the director
+	 * XXX: or not (for session/backend coupling).  Until then
+	 * XXX: make sure we don't trip up the check in vcl_recv.
+	 */
+	sp->director = NULL;
+	sp->step = STP_RECV;
+	http_ForceGet(sp->http);
+
+	/* Don't do conditionals */
+	sp->http->conds = 0;
+	http_Unset(sp->http, H_If_Modified_Since);
+
+	/* Client content already taken care of */
+	http_Unset(sp->http, H_Content_Length);
+
+	sxid = sp->xid;
+	while (1) {
+		sp->wrk = w;
+		CNT_Session(sp);
+		if (sp->step == STP_DONE)
+			break;
+		AZ(sp->wrk);
+		WSL_Flush(w, 0);
+		DSL(0x20, SLT_Debug, sp->id, "loop waiting for ESI");
+		(void)usleep(10000);
+	}
+	sp->xid = sxid;
+	AN(sp->wrk);
+	assert(sp->step == STP_DONE);
+	sp->esi_level--;
+	sp->obj = obj;
+	sp->wrk->res_mode = res_mode;
+
+	/* Reset the workspace */
+	WS_Reset(sp->ws, ws_wm);
+
+	WRW_Reserve(sp->wrk, &sp->fd);
+}
+
+/*--------------------------------------------------------------------*/
+
 #ifndef OLD_ESI
 
 void
@@ -103,6 +180,7 @@
 			p++;
 			q = (void*)strchr((const char*)p, '\0');
 			printf("INCL [%s]\n", p);
+			ESI_Include(sp, (const char*)p, NULL);
 			p = q + 1;
 			break;
 		default:




From phk at varnish-cache.org  Thu Jan 13 11:55:11 2011
From: phk at varnish-cache.org (phk at varnish-cache.org)
Date: Thu, 13 Jan 2011 12:55:11 +0100
Subject: r5730 - trunk/varnish-cache/bin/varnishd
Message-ID: <E1PdLln-0004Q4-BA@project.varnish-software.com>

Author: phk
Date: 2011-01-13 12:55:11 +0100 (Thu, 13 Jan 2011)
New Revision: 5730

Modified:
   trunk/varnish-cache/bin/varnishd/cache_esi_deliver.c
   trunk/varnish-cache/bin/varnishd/cache_esi_parse.c
Log:
Munge the esi:include src attribute as required, to produce URL and
Host: header.

Correctly unpack Host: header in esi delivery.

Now e00008.vtc is the first case to fail.



Modified: trunk/varnish-cache/bin/varnishd/cache_esi_deliver.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_esi_deliver.c	2011-01-13 11:12:06 UTC (rev 5729)
+++ trunk/varnish-cache/bin/varnishd/cache_esi_deliver.c	2011-01-13 11:55:11 UTC (rev 5730)
@@ -72,7 +72,7 @@
 	ws_wm = WS_Snapshot(sp->ws);
 
 	http_SetH(sp->http, HTTP_HDR_URL, src);
-	if (host != NULL)  {
+	if (host != NULL && *host != '\0')  {
 		http_Unset(sp->http, H_Host);
 		http_Unset(sp->http, H_If_Modified_Since);
 		http_SetHeader(w, sp->fd, sp->http, host);
@@ -125,7 +125,7 @@
 ESI_Deliver(struct sess *sp)
 {
 	struct storage *st;
-	uint8_t *p, *e, *q;
+	uint8_t *p, *e, *q, *r;
 	unsigned off;
 	size_t l;
 
@@ -140,6 +140,8 @@
 	off = 0;
 
 	while (p < e) {
+		//usleep(10000);
+		//WRW_Flush(sp->wrk);
 		switch (*p) {
 		case VEC_V1:
 			l = p[1];
@@ -179,9 +181,14 @@
 		case VEC_INCL:
 			p++;
 			q = (void*)strchr((const char*)p, '\0');
-			printf("INCL [%s]\n", p);
-			ESI_Include(sp, (const char*)p, NULL);
-			p = q + 1;
+			AN(q);
+			q++;
+			r = (void*)strchr((const char*)q, '\0');
+			AN(r);
+			printf("INCL [%s][%s] BEGIN\n", p, q);
+			ESI_Include(sp, (const char*)p, (const char*)q);
+			printf("INCL [%s] END\n", p);
+			p = r + 1;
 			break;
 		default:
 			printf("XXXX %02x [%c]\n", *p, *p);
@@ -189,6 +196,7 @@
 			return;
 		}
 	}
+printf("DONE\n");
 	WRW_Flush(sp->wrk);
 }
 

Modified: trunk/varnish-cache/bin/varnishd/cache_esi_parse.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_esi_parse.c	2011-01-13 11:12:06 UTC (rev 5729)
+++ trunk/varnish-cache/bin/varnishd/cache_esi_parse.c	2011-01-13 11:55:11 UTC (rev 5730)
@@ -59,6 +59,8 @@
 	vfp_bytes_f		*bytes;
 	struct vsb		*vsb;
 
+	struct sess		*sp;
+
 	/* parser state */
 	const char		*state;
 
@@ -324,6 +326,9 @@
 static void
 vep_do_include(struct vep_state *vep, enum dowhat what)
 {
+	char *p, *q, *h;
+	ssize_t l;
+	txt url;
 
 	printf("DO_INCLUDE(%d)\n", what);
 	if (what == DO_ATTR) {
@@ -338,10 +343,39 @@
 		if (vep->o_verbatim > 0) 
 			vep_emit_verbatim(vep);
 		/* XXX: what if it contains NUL bytes ?? */
-		vsb_printf(vep->vsb, "%c%s%c",
-		    VEC_INCL,
-		    vsb_data(vep->include_src), 0);
+		p = vsb_data(vep->include_src);
+		l = vsb_len(vep->include_src);
+		h = 0;
 
+		if (l > 7 && !memcmp(p, "http://", 7)) {
+			h = p + 7;
+			p = strchr(h, '/');
+			AN(p);
+			printf("HOST <%.*s> PATH <%s>\n", (int)(p-h),h, p);
+			vsb_printf(vep->vsb, "%c%s%cHost: %.*s%c",
+			    VEC_INCL, p, 0,
+			    (int)(p-h), h, 0);
+		} else if (*p == '/') {
+			vsb_printf(vep->vsb, "%c%s%c%c",
+			    VEC_INCL, p, 0, 0);
+		} else {
+			url = vep->sp->wrk->bereq->hd[HTTP_HDR_URL];
+			/* Look for the last / before a '?' */
+			h = NULL;
+			for (q = url.b; q < url.e && *q != '?'; q++)
+				if (*q == '/')
+					h = q;
+			if (h == NULL)
+				h = q + 1;
+				
+			printf("INCL:: %.*s/%s\n",
+			    (int)(h - url.b), url.b, p);
+			vsb_printf(vep->vsb, "%c%.*s/%s%c%c",
+			    VEC_INCL, 
+			    (int)(h - url.b), url.b,
+			    p, 0, 0);
+		}
+
 		vsb_delete(vep->include_src);
 		vep->include_src = NULL;
 	}
@@ -373,12 +407,15 @@
 	p = b;
 	while (p < e) {
 		AN(vep->state);
+		i = e - p;
+		if (i > 10)
+			i = 10;
 		printf("EP %s %d %d (%.*s) [%.*s]\n",
 		    vep->state,
 		    vep->skip,
 		    vep->remove,
 		    vep->tag_i, vep->tag,
-		    (int)(e - p), p);
+		    i, p);
 		fflush(stdout);
 		usleep(10);
 
@@ -415,13 +452,13 @@
 		} else if (vep->state == VEP_NOTMYTAG) {
 			vep->tag_i = 0;
 			while (p < e) {
-				if (!vep->remove)
-					vep_mark_verbatim(vep, p + 1);
 				if (*p++ == '>') {
 					vep->state = VEP_NEXTTAG;
 					break;
 				}
 			}
+			if (!vep->remove)
+				vep_mark_verbatim(vep, p + 1);
 		} else if (vep->state == VEP_NEXTTAG) {
 			/*
 			 * Hunt for start of next tag and keep an eye
@@ -447,11 +484,10 @@
 				} else {
 					p++;
 					vep->esicmt_p = vep->esicmt;
-					if (vep->esicmt_p == NULL &&
-					    !vep->remove)
-						vep_mark_verbatim(vep, p);
 				}
 			}
+			if (vep->esicmt_p == NULL && !vep->remove)
+				vep_mark_verbatim(vep, p);
 			if (p < e)
 				vep->state = VEP_STARTTAG;
 
@@ -507,13 +543,13 @@
 			if (vep->remove) {
 				VSC_main->esi_errors++;
 				vep->state = VEP_NOTMYTAG;
-				break;
+			} else {
+				vep->skip = 1;
+				vep_mark_skip(vep, p);
+				vep->match = vep_match_esi;
+				vep->match_l = vep_match_esi_len;
+				vep->state = VEP_MATCH;
 			}
-			vep->skip = 1;
-			vep_mark_skip(vep, p);
-			vep->match = vep_match_esi;
-			vep->match_l = vep_match_esi_len;
-			vep->state = VEP_MATCH;
 		} else if (vep->state == VEP_ESIETAG) {
 			vep->tag_i = 0;
 			vep->endtag = 1;
@@ -528,6 +564,7 @@
 		} else if (vep->state == VEP_ESIREMOVE) {
 			vep->dostuff = vep_do_nothing;
 			vep->remove = !vep->endtag;
+printf(">>> REMOVE %d\n", vep->remove);
 			vep->state = VEP_INTAG;
 
 		/******************************************************
@@ -754,6 +791,7 @@
 
 	memset(vep, 0, sizeof *vep);
 	vep->magic = VEP_MAGIC;
+	vep->sp = sp;
 	vep->bytes = vfp_esi_bytes_uu;
 	vep->vsb = vsb_newauto();
 	vep->state = VEP_START;




From phk at varnish-cache.org  Thu Jan 13 16:29:34 2011
From: phk at varnish-cache.org (phk at varnish-cache.org)
Date: Thu, 13 Jan 2011 17:29:34 +0100
Subject: r5731 - trunk/varnish-cache/bin/varnishd
Message-ID: <E1PdQ3K-00034D-R9@project.varnish-software.com>

Author: phk
Date: 2011-01-13 17:29:34 +0100 (Thu, 13 Jan 2011)
New Revision: 5731

Modified:
   trunk/varnish-cache/bin/varnishd/cache_esi_deliver.c
Log:
Implement another VEC code



Modified: trunk/varnish-cache/bin/varnishd/cache_esi_deliver.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_esi_deliver.c	2011-01-13 11:55:11 UTC (rev 5730)
+++ trunk/varnish-cache/bin/varnishd/cache_esi_deliver.c	2011-01-13 16:29:34 UTC (rev 5731)
@@ -164,6 +164,12 @@
 			printf("SKIP(%d)\n", (int)l);
 			off += l;
 			break;
+		case VEC_S2:
+			l = vbe16dec(p + 1);
+			p += 3;
+			printf("SKIP(%d)\n", (int)l);
+			off += l;
+			break;
 		case VEC_L1:
 			l = p[1];
 			p += 2;
@@ -191,7 +197,7 @@
 			p = r + 1;
 			break;
 		default:
-			printf("XXXX %02x [%c]\n", *p, *p);
+			printf("XXXX 0x%02x [%s]\n", *p, p);
 			INCOMPL();
 			return;
 		}




From phk at varnish-cache.org  Thu Jan 13 16:30:28 2011
From: phk at varnish-cache.org (phk at varnish-cache.org)
Date: Thu, 13 Jan 2011 17:30:28 +0100
Subject: r5732 - trunk/varnish-cache/bin/varnishd
Message-ID: <E1PdQ4C-0003CI-DO@project.varnish-software.com>

Author: phk
Date: 2011-01-13 17:30:28 +0100 (Thu, 13 Jan 2011)
New Revision: 5732

Modified:
   trunk/varnish-cache/bin/varnishd/cache_esi_parse.c
Log:
Polish up error checking/reporting to pass e00008.vtc



Modified: trunk/varnish-cache/bin/varnishd/cache_esi_parse.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_esi_parse.c	2011-01-13 16:29:34 UTC (rev 5731)
+++ trunk/varnish-cache/bin/varnishd/cache_esi_parse.c	2011-01-13 16:30:28 UTC (rev 5732)
@@ -74,6 +74,7 @@
 	unsigned		o_verbatim;
 	unsigned		o_skip;
 
+const char		*hack_p;
 	const char		*ver_p;
 
 	const char		*until;
@@ -117,19 +118,21 @@
 static const char *VEP_ESIREMOVE =	"[ESI:Remove]";
 static const char *VEP_ESIINCLUDE =	"[ESI:Include]";
 static const char *VEP_ESICOMMENT =	"[ESI:Comment]";
+static const char *VEP_ESIBOGON =	"[ESI:Bogon]";
 
 static const char *VEP_INTAG =		"[InTag]";
+static const char *VEP_TAGERROR =	"[TagError]";
 
 static const char *VEP_ATTR =		"[Attribute]";
 static const char *VEP_SKIPATTR =	"[SkipAttribute]";
 static const char *VEP_SKIPATTR2 =	"[SkipAttribute2]";
+static const char *VEP_ATTRDELIM =	"[AttrDelim]";
 static const char *VEP_ATTRGETVAL =	"[AttrGetValue]";
 static const char *VEP_ATTRVAL =	"[AttrValue]";
 
 static const char *VEP_UNTIL =		"[Until]";
 static const char *VEP_MATCHBUF = 	"[MatchBuf]";
 static const char *VEP_MATCH =		"[Match]";
-static const char *VEP_XXX =		"[XXX]";
 
 /*---------------------------------------------------------------------*/
 
@@ -150,7 +153,7 @@
 	{ "include",	&VEP_ESIINCLUDE },
 	{ "remove",	&VEP_ESIREMOVE },
 	{ "comment",	&VEP_ESICOMMENT },
-	{ NULL,		&VEP_XXX }
+	{ NULL,		&VEP_ESIBOGON }
 };
 
 static const int vep_match_esi_len =
@@ -160,7 +163,7 @@
 
 static struct vep_match vep_match_esie[] = {
 	{ "remove",	&VEP_ESIREMOVE },
-	{ NULL,		&VEP_XXX }
+	{ NULL,		&VEP_ESIBOGON }
 };
 
 static const int vep_match_esie_len =
@@ -176,6 +179,20 @@
 static const int vep_match_attr_include_len =
     sizeof vep_match_attr_include / sizeof vep_match_attr_include[0];
 
+/*--------------------------------------------------------------------
+ * Report a parsing error
+ */
+
+static void
+vep_error(const struct vep_state *vep, const char *p)
+{
+
+	VSC_main->esi_errors++;
+	WSP(vep->sp, SLT_ESI_xmlerror, "at %ld %s",
+	    (vep->ver_p - vep->hack_p), p);
+	
+}
+
 /*---------------------------------------------------------------------
  * return match or NULL if more input needed.
  */
@@ -309,6 +326,7 @@
 /*---------------------------------------------------------------------
  */
 
+#if 0
 static void
 vep_do_nothing(struct vep_state *vep, enum dowhat what)
 {
@@ -319,11 +337,64 @@
 		vsb_delete(vep->attr_vsb);
 	}
 }
+#endif
 
 /*---------------------------------------------------------------------
  */
 
 static void
+vep_do_comment(struct vep_state *vep, enum dowhat what)
+{
+	printf("DO_COMMENT(%d)\n", what);
+	if (what == DO_ATTR) {
+		printf("ATTR (%s) (%s)\n", vep->match_hit->match,
+			vsb_data(vep->attr_vsb));
+		vsb_delete(vep->attr_vsb);
+	} else {
+		if (vep->endtag)
+			vep_error(vep,
+			    "ESI 1.0 </esi:comment> not allowed");
+		else if (!vep->emptytag) 
+			vep_error(vep,
+			    "ESI 1.0 <esi:comment> not allowed");
+	}
+}
+
+/*---------------------------------------------------------------------
+ */
+
+static void
+vep_do_remove(struct vep_state *vep, enum dowhat what)
+{
+	printf("DO_REMOVE(%d)\n", what);
+	if (what == DO_ATTR) {
+		printf("ATTR (%s) (%s)\n", vep->match_hit->match,
+			vsb_data(vep->attr_vsb));
+		vsb_delete(vep->attr_vsb);
+	} else {
+		if (vep->emptytag)
+#if 0
+			vep_error(vep,
+			    "ESI 1.0 <esi:remove/> not legal");
+#else
+			;
+#endif
+		else
+		if (vep->remove && !vep->endtag)
+			vep_error(vep,
+			    "ESI 1.0 <esi:remove> already open");
+		else if (!vep->remove && vep->endtag)
+			vep_error(vep,
+			    "ESI 1.0 <esi:remove> not open");
+		else 
+			vep->remove = !vep->endtag;
+	}
+}
+
+/*---------------------------------------------------------------------
+ */
+
+static void
 vep_do_include(struct vep_state *vep, enum dowhat what)
 {
 	char *p, *q, *h;
@@ -337,6 +408,11 @@
 		XXXAZ(vep->include_src);	/* multiple src= */
 		vep->include_src = vep->attr_vsb;
 	} else {
+		if (vep->include_src == NULL) {
+			vep_error(vep,
+			    "ESI 1.0 <esi:include> lacks src attr");
+			return;
+		}
 		XXXAN(vep->include_src);
 		if (vep->o_skip > 0) 
 			vep_emit_skip(vep);
@@ -435,8 +511,11 @@
 			if (p < e) {
 				if (*p == '<') {
 					vep->state = VEP_STARTTAG;
-				} else
+				} else {
+					WSP(vep->sp, SLT_ESI_xmlerror,
+					    "No ESI processing, first char not '<'");
 					vep->state = VEP_NOTXML;
+				}
 			}
 		} else if (vep->state == VEP_NOTXML) {
 			/*
@@ -464,6 +543,8 @@
 			 * Hunt for start of next tag and keep an eye
 			 * out for end of EsiCmt if armed.
 			 */
+			vep->emptytag = 0;
+			vep->endtag = 0;
 			while (p < e && *p != '<') {
 				if (vep->esicmt_p != NULL &&
 				    *p == *vep->esicmt_p++) {
@@ -541,7 +622,9 @@
 			vep->tag_i = 0;
 			vep->endtag = 0;
 			if (vep->remove) {
-				VSC_main->esi_errors++;
+				vep_mark_skip(vep, p);
+				vep_error(vep,
+				    "ESI 1.0 Nested ESI element in <esi:remove>");
 				vep->state = VEP_NOTMYTAG;
 			} else {
 				vep->skip = 1;
@@ -562,10 +645,16 @@
 			vep->attr = vep_match_attr_include;
 			vep->attr_l = vep_match_attr_include_len;
 		} else if (vep->state == VEP_ESIREMOVE) {
-			vep->dostuff = vep_do_nothing;
-			vep->remove = !vep->endtag;
-printf(">>> REMOVE %d\n", vep->remove);
+			vep->dostuff = vep_do_remove;
 			vep->state = VEP_INTAG;
+		} else if (vep->state == VEP_ESICOMMENT) {
+			vep->dostuff = vep_do_comment;
+			vep->state = VEP_INTAG;
+		} else if (vep->state == VEP_ESIBOGON) {
+			vep_mark_skip(vep, p);
+			vep_error(vep,
+			    "ESI 1.0 <esi:bogus> element");
+			vep->state = VEP_TAGERROR;
 
 		/******************************************************
 		 * SECTION F
@@ -573,6 +662,7 @@
 
 		} else if (vep->state == VEP_INTAG) {
 			vep->tag_i = 0;
+			vep_mark_skip(vep, p);
 			while (p < e && vct_islws(*p)) {
 				p++;	
 				vep->canattr = 1;
@@ -586,6 +676,7 @@
 				p++;
 				vep_mark_skip(vep, p);
 				AN(vep->dostuff);
+				vep_mark_skip(vep, p);
 				vep->dostuff(vep, DO_TAG);
 				vep->state = VEP_NEXTTAG;
 			} else if (p < e && vep->emptytag) {
@@ -593,8 +684,20 @@
 			} else if (p < e && vct_isxmlnamestart(*p)) {
 				vep->state = VEP_ATTR;
 			} else if (p < e) {
-				INCOMPL();	/* ESI-SYNTAX ERROR */
+				vep_mark_skip(vep, p);
+				vep_error(vep,
+				    "XML 1.0 Illegal attribute start char");
+printf("ERR %d [%.*s]\n", __LINE__, (int)(e-p), p);
+				vep->state = VEP_TAGERROR;
 			}
+		} else if (vep->state == VEP_TAGERROR) {
+			while (p < e && *p != '>')
+				p++;
+			if (p < e) {
+				p++;
+				vep_mark_skip(vep, p);
+				vep->state = VEP_NEXTTAG;
+			}
 
 		/******************************************************
 		 * SECTION G
@@ -618,7 +721,7 @@
 					continue;
 				if (vep->tag[i] == '=') {
 					assert(i + 1 == vep->tag_i);
-					vep->state = VEP_ATTRVAL;
+					vep->state = VEP_ATTRDELIM;
 				}
 			}
 			xxxassert(i == vep->tag_i);
@@ -628,33 +731,62 @@
 				p++;
 			if (p < e && *p == '=') {
 				p++;
-				vep->state = VEP_ATTRVAL;
+				vep->state = VEP_ATTRDELIM;
+			} else if (p < e && vct_issp(*p)) {
+				vep->state = VEP_INTAG;
 			} else if (p < e) {
-				INCOMPL();	/* ESI-SYNTAX ERROR */
+				vep_mark_skip(vep, p);
+				vep_error(vep,
+				    "XML 1.0 Illegal attr char");
+				vep->state = VEP_TAGERROR;
 			}
 		} else if (vep->state == VEP_ATTRGETVAL) {
 			vep->attr_vsb = vsb_newauto();
-			vep->state = VEP_ATTRVAL;
-		} else if (vep->state == VEP_ATTRVAL) {
-			if (vep->attr_delim == 0)  {
-				if (*p != '"' && *p != '\'')
-					INCOMPL();	/* ESI-SYNTAX */
+			vep->state = VEP_ATTRDELIM;
+		} else if (vep->state == VEP_ATTRDELIM) {
+			AZ(vep->attr_delim);
+			if (*p == '"' || *p == '\'') {
 				vep->attr_delim = *p++;
+				vep->state = VEP_ATTRVAL;
+			} else if (!vct_issp(*p)) {
+				vep->attr_delim = ' ';
+				vep->state = VEP_ATTRVAL;
+			} else {
+				vep_mark_skip(vep, p);
+				vep_error(vep,
+				    "XML 1.0 Illegal attribute delimiter");
+				vep->state = VEP_TAGERROR;
 			}
-			while (p < e && *p != vep->attr_delim) {
+			
+		} else if (vep->state == VEP_ATTRVAL) {
+			while (p < e && *p != '>' && *p != vep->attr_delim &&
+			   (vep->attr_delim != ' ' || !vct_issp(*p))) {
 				if (vep->attr_vsb != NULL)
 					vsb_bcat(vep->attr_vsb, p, 1);
 				p++;
 			}
-			if (p < e) {
+			if (p < e && *p == '>') {
+				vep_mark_skip(vep, p);
+				vep_error(vep,
+				    "XML 1.0 Missing end attribute delimiter");
+printf("ERR %d %c %c\n", __LINE__, *p, vep->attr_delim);
+				vep->state = VEP_TAGERROR;
+				vep->attr_delim = 0;
 				if (vep->attr_vsb != NULL) {
 					vsb_finish(vep->attr_vsb);
+					vsb_delete(vep->attr_vsb);
+					vep->attr_vsb = NULL;
+				}
+			} else if (p < e) {
+				vep->attr_delim = 0;
+				p++;
+				if (vep->attr_vsb != NULL) {
+					vsb_finish(vep->attr_vsb);
 					AN(vep->dostuff);
+					vep_mark_skip(vep, p);
 					vep->dostuff(vep, DO_ATTR);
 					vep->attr_vsb = NULL;
 				}
-				p++;
-				vep->attr_delim = 0;
 				vep->state = VEP_INTAG;
 			}
 	
@@ -753,8 +885,10 @@
 		w = HTC_Read(htc, st->ptr + st->len, l);
 		if (w <= 0)
 			return (w);
+if (vep->hack_p == NULL)
+	vep->hack_p = (const char *)st->ptr + st->len;
 		vep->ver_p = (const char *)st->ptr + st->len;
-#if 1
+#if 0
 		{
 		for (l = 0; l < w; l++) 
 			vep_parse(vep, (const char *)st->ptr + st->len + l, 1);




From phk at varnish-cache.org  Thu Jan 13 17:13:18 2011
From: phk at varnish-cache.org (phk at varnish-cache.org)
Date: Thu, 13 Jan 2011 18:13:18 +0100
Subject: r5733 - trunk/varnish-cache/bin/varnishtest/tests
Message-ID: <E1PdQje-0004E6-H3@project.varnish-software.com>

Author: phk
Date: 2011-01-13 18:13:18 +0100 (Thu, 13 Jan 2011)
New Revision: 5733

Modified:
   trunk/varnish-cache/bin/varnishtest/tests/e00013.vtc
Log:
Fix this test-case to do what it should do:  We have to supress
the Content-Length: header to push varnish into EOF mode.



Modified: trunk/varnish-cache/bin/varnishtest/tests/e00013.vtc
===================================================================
--- trunk/varnish-cache/bin/varnishtest/tests/e00013.vtc	2011-01-13 16:30:28 UTC (rev 5732)
+++ trunk/varnish-cache/bin/varnishtest/tests/e00013.vtc	2011-01-13 17:13:18 UTC (rev 5733)
@@ -5,7 +5,7 @@
 server s1 {
         rxreq
         expect req.url == "/foo"
-        txresp -hdr "Connection: close"
+        txresp -nolen -hdr "Connection: close"
         send {<x>                                      }
 } -start
 




From phk at varnish-cache.org  Thu Jan 13 17:14:30 2011
From: phk at varnish-cache.org (phk at varnish-cache.org)
Date: Thu, 13 Jan 2011 18:14:30 +0100
Subject: r5734 - trunk/varnish-cache/bin/varnishtest/tests
Message-ID: <E1PdQko-0004Go-5D@project.varnish-software.com>

Author: phk
Date: 2011-01-13 18:14:29 +0100 (Thu, 13 Jan 2011)
New Revision: 5734

Modified:
   trunk/varnish-cache/bin/varnishtest/tests/e00009.vtc
Log:
Remove incomplete XML tag, that's not what we are testing and
the behaviour in that case is undefined.



Modified: trunk/varnish-cache/bin/varnishtest/tests/e00009.vtc
===================================================================
--- trunk/varnish-cache/bin/varnishtest/tests/e00009.vtc	2011-01-13 17:13:18 UTC (rev 5733)
+++ trunk/varnish-cache/bin/varnishtest/tests/e00009.vtc	2011-01-13 17:14:29 UTC (rev 5734)
@@ -9,7 +9,6 @@
 		<esi:remove>
 			bar
 		</esi:remove>
-		<esi:include
 	}
 	rxreq 
 	txresp -body {
@@ -17,7 +16,6 @@
 		<esi:remove>
 			bar
 		</esi:remove>
-		<esi:include
 	}
 } -start
 
@@ -31,8 +29,8 @@
 	txreq 
 	rxresp
 	expect resp.status == 200
-	expect resp.bodylen == 72
-	expect resp.http.content-length == 72
+	expect resp.bodylen == 57
+	expect resp.http.content-length == 57
 } -run
 
 varnish v1  -cli "param.set esi_syntax 1" 
@@ -41,7 +39,7 @@
 	txreq -url bar
 	rxresp
 	expect resp.status == 200
-	expect resp.bodylen == 37
+	expect resp.bodylen == 22
 	expect resp.http.transfer-encoding == "chunked"
 } -run
-varnish v1 -expect esi_errors == 1
+varnish v1 -expect esi_errors == 0




From phk at varnish-cache.org  Thu Jan 13 17:14:46 2011
From: phk at varnish-cache.org (phk at varnish-cache.org)
Date: Thu, 13 Jan 2011 18:14:46 +0100
Subject: r5735 - trunk/varnish-cache/bin/varnishtest/tests
Message-ID: <E1PdQl4-0004JS-W8@project.varnish-software.com>

Author: phk
Date: 2011-01-13 18:14:46 +0100 (Thu, 13 Jan 2011)
New Revision: 5735

Modified:
   trunk/varnish-cache/bin/varnishtest/tests/e00003.vtc
Log:
Make this one a tad more challenging



Modified: trunk/varnish-cache/bin/varnishtest/tests/e00003.vtc
===================================================================
--- trunk/varnish-cache/bin/varnishtest/tests/e00003.vtc	2011-01-13 17:14:29 UTC (rev 5734)
+++ trunk/varnish-cache/bin/varnishtest/tests/e00003.vtc	2011-01-13 17:14:46 UTC (rev 5735)
@@ -8,7 +8,7 @@
 	txresp -body {
 		<html>
 		Before include
-		<esi:include src="/body"/>
+		<esi:include src="/body" sr="foo"/>
 		After include
 	}
 	rxreq 




From phk at varnish-cache.org  Thu Jan 13 17:15:42 2011
From: phk at varnish-cache.org (phk at varnish-cache.org)
Date: Thu, 13 Jan 2011 18:15:42 +0100
Subject: r5736 - trunk/varnish-cache/bin/varnishd
Message-ID: <E1PdQly-0004Rc-OY@project.varnish-software.com>

Author: phk
Date: 2011-01-13 18:15:36 +0100 (Thu, 13 Jan 2011)
New Revision: 5736

Modified:
   trunk/varnish-cache/bin/varnishd/cache_esi_deliver.c
   trunk/varnish-cache/bin/varnishd/cache_esi_parse.c
   trunk/varnish-cache/bin/varnishd/cache_fetch.c
Log:
Make it possible to supress the debugging, and do so.

Various other fixes, to get through testcases.

Now just six left.



Modified: trunk/varnish-cache/bin/varnishd/cache_esi_deliver.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_esi_deliver.c	2011-01-13 17:14:46 UTC (rev 5735)
+++ trunk/varnish-cache/bin/varnishd/cache_esi_deliver.c	2011-01-13 17:15:36 UTC (rev 5736)
@@ -121,6 +121,9 @@
 
 #ifndef OLD_ESI
 
+// #define Debug(fmt, ...) printf(fmt, __VA_ARGS__)
+#define Debug(fmt, ...) /**/
+
 void
 ESI_Deliver(struct sess *sp)
 {
@@ -129,7 +132,6 @@
 	unsigned off;
 	size_t l;
 
-printf("DELIV\n");
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 	st = sp->obj->esidata;
 	AN(st);
@@ -148,26 +150,26 @@
 			p += 2;
 			q = (void*)strchr((const char*)p, '\0');
 			assert (q > p);
-			printf("VER(%d) %d\n", (int)l, (int)(q-p));
+			Debug("VER(%d) %d\n", (int)l, (int)(q-p));
 			if (sp->wrk->res_mode & RES_CHUNKED)
 				WRW_Write(sp->wrk, p, q - p);
 			WRW_Write(sp->wrk, st->ptr + off, l);
 			if (sp->wrk->res_mode & RES_CHUNKED)
 				WRW_Write(sp->wrk, "\r\n", -1);
-			// printf("[%.*s]", (int)l, st->ptr + off);
+			// Debug("[%.*s]", (int)l, st->ptr + off);
 			off += l;
 			p = q + 1;
 			break;
 		case VEC_S1:
 			l = p[1];
 			p += 2;
-			printf("SKIP(%d)\n", (int)l);
+			Debug("SKIP(%d)\n", (int)l);
 			off += l;
 			break;
 		case VEC_S2:
 			l = vbe16dec(p + 1);
 			p += 3;
-			printf("SKIP(%d)\n", (int)l);
+			Debug("SKIP(%d)\n", (int)l);
 			off += l;
 			break;
 		case VEC_L1:
@@ -175,7 +177,7 @@
 			p += 2;
 			q = (void*)strchr((const char*)p, '\0');
 			assert (q > p);
-			printf("LIT(%d) %d\n", (int)l, (int)(q-p));
+			Debug("LIT(%d) %d\n", (int)l, (int)(q-p));
 			if (sp->wrk->res_mode & RES_CHUNKED)
 				WRW_Write(sp->wrk, p, q - p);
 			p = q + 1;
@@ -191,18 +193,17 @@
 			q++;
 			r = (void*)strchr((const char*)q, '\0');
 			AN(r);
-			printf("INCL [%s][%s] BEGIN\n", p, q);
+			Debug("INCL [%s][%s] BEGIN\n", p, q);
 			ESI_Include(sp, (const char*)p, (const char*)q);
-			printf("INCL [%s] END\n", p);
+			Debug("INCL [%s] END\n", p);
 			p = r + 1;
 			break;
 		default:
-			printf("XXXX 0x%02x [%s]\n", *p, p);
+			Debug("XXXX 0x%02x [%s]\n", *p, p);
 			INCOMPL();
 			return;
 		}
 	}
-printf("DONE\n");
 	WRW_Flush(sp->wrk);
 }
 

Modified: trunk/varnish-cache/bin/varnishd/cache_esi_parse.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_esi_parse.c	2011-01-13 17:14:46 UTC (rev 5735)
+++ trunk/varnish-cache/bin/varnishd/cache_esi_parse.c	2011-01-13 17:15:36 UTC (rev 5736)
@@ -43,6 +43,9 @@
 
 #ifndef OLD_ESI
 
+// #define Debug(fmt, ...) printf(fmt, __VA_ARGS__)
+#define Debug(fmt, ...) /**/
+
 struct vep_state;
 
 enum dowhat {DO_ATTR, DO_TAG};
@@ -104,6 +107,7 @@
 /*---------------------------------------------------------------------*/
 
 static const char *VEP_START =		"[Start]";
+static const char *VEP_TESTXML = 	"[TestXml]";
 static const char *VEP_NOTXML =	 	"[NotXml]";
 
 static const char *VEP_NEXTTAG = 	"[NxtTag]";
@@ -282,7 +286,7 @@
 	if (vep->o_skip > 0) 
 		vep_emit_skip(vep);
 	l = e - p;
-	printf("---->L(%d) [%.*s]\n", (int)l, (int)l, p);
+	Debug("---->L(%d) [%.*s]\n", (int)l, (int)l, p);
 	vep_emit_len(vep, l, VEC_L1, VEC_L2, VEC_L4);
 	vsb_printf(vep->vsb, "%lx\r\n%c", l, 0);
 	vsb_bcat(vep->vsb, p, l);
@@ -301,7 +305,7 @@
 	if (vep->o_skip > 0) 
 		vep_emit_skip(vep);
 	AZ(vep->o_skip);
-	printf("-->V(%d) [%.*s]\n", (int)l, (int)l, vep->ver_p);
+	Debug("-->V(%d) [%.*s]\n", (int)l, (int)l, vep->ver_p);
 	vep->o_verbatim += l;
 	vep->ver_p = p;
 } 
@@ -318,7 +322,7 @@
 	if (vep->o_verbatim > 0) 
 		vep_emit_verbatim(vep);
 	AZ(vep->o_verbatim);
-	printf("-->S(%d) [%.*s]\n", (int)l, (int)l, vep->ver_p);
+	Debug("-->S(%d) [%.*s]\n", (int)l, (int)l, vep->ver_p);
 	vep->o_skip += l;
 	vep->ver_p = p;
 } 
@@ -330,9 +334,9 @@
 static void
 vep_do_nothing(struct vep_state *vep, enum dowhat what)
 {
-	printf("DO_NOTHING(%d)\n", what);
+	Debug("DO_NOTHING(%d)\n", what);
 	if (what == DO_ATTR) {
-		printf("ATTR (%s) (%s)\n", vep->match_hit->match,
+		Debug("ATTR (%s) (%s)\n", vep->match_hit->match,
 			vsb_data(vep->attr_vsb));
 		vsb_delete(vep->attr_vsb);
 	}
@@ -345,9 +349,9 @@
 static void
 vep_do_comment(struct vep_state *vep, enum dowhat what)
 {
-	printf("DO_COMMENT(%d)\n", what);
+	Debug("DO_COMMENT(%d)\n", what);
 	if (what == DO_ATTR) {
-		printf("ATTR (%s) (%s)\n", vep->match_hit->match,
+		Debug("ATTR (%s) (%s)\n", vep->match_hit->match,
 			vsb_data(vep->attr_vsb));
 		vsb_delete(vep->attr_vsb);
 	} else {
@@ -366,9 +370,9 @@
 static void
 vep_do_remove(struct vep_state *vep, enum dowhat what)
 {
-	printf("DO_REMOVE(%d)\n", what);
+	Debug("DO_REMOVE(%d)\n", what);
 	if (what == DO_ATTR) {
-		printf("ATTR (%s) (%s)\n", vep->match_hit->match,
+		Debug("ATTR (%s) (%s)\n", vep->match_hit->match,
 			vsb_data(vep->attr_vsb));
 		vsb_delete(vep->attr_vsb);
 	} else {
@@ -401,9 +405,9 @@
 	ssize_t l;
 	txt url;
 
-	printf("DO_INCLUDE(%d)\n", what);
+	Debug("DO_INCLUDE(%d)\n", what);
 	if (what == DO_ATTR) {
-		printf("ATTR (%s) (%s)\n", vep->match_hit->match,
+		Debug("ATTR (%s) (%s)\n", vep->match_hit->match,
 			vsb_data(vep->attr_vsb));
 		XXXAZ(vep->include_src);	/* multiple src= */
 		vep->include_src = vep->attr_vsb;
@@ -427,7 +431,7 @@
 			h = p + 7;
 			p = strchr(h, '/');
 			AN(p);
-			printf("HOST <%.*s> PATH <%s>\n", (int)(p-h),h, p);
+			Debug("HOST <%.*s> PATH <%s>\n", (int)(p-h),h, p);
 			vsb_printf(vep->vsb, "%c%s%cHost: %.*s%c",
 			    VEC_INCL, p, 0,
 			    (int)(p-h), h, 0);
@@ -444,7 +448,7 @@
 			if (h == NULL)
 				h = q + 1;
 				
-			printf("INCL:: %.*s/%s\n",
+			Debug("INCL:: %.*s/%s\n",
 			    (int)(h - url.b), url.b, p);
 			vsb_printf(vep->vsb, "%c%.*s/%s%c%c",
 			    VEC_INCL, 
@@ -486,7 +490,7 @@
 		i = e - p;
 		if (i > 10)
 			i = 10;
-		printf("EP %s %d %d (%.*s) [%.*s]\n",
+		Debug("EP %s %d %d (%.*s) [%.*s]\n",
 		    vep->state,
 		    vep->skip,
 		    vep->remove,
@@ -500,6 +504,11 @@
 		 */
 
 		if (vep->state == VEP_START) {
+			if (params->esi_syntax & 0x1)
+				vep->state = VEP_NEXTTAG;
+			else
+				vep->state = VEP_TESTXML;
+		} else if (vep->state == VEP_TESTXML) {
 			/*
 			 * If the first non-whitespace char is different
 			 * from '<' we assume this is not XML.
@@ -529,15 +538,22 @@
 		 */
 
 		} else if (vep->state == VEP_NOTMYTAG) {
-			vep->tag_i = 0;
-			while (p < e) {
-				if (*p++ == '>') {
-					vep->state = VEP_NEXTTAG;
-					break;
+			if (params->esi_syntax & 0x2) {
+				p++;
+				vep->state = VEP_NEXTTAG;
+				if (!vep->remove)
+					vep_mark_verbatim(vep, p + 1);
+			} else {
+				vep->tag_i = 0;
+				while (p < e) {
+					if (*p++ == '>') {
+						vep->state = VEP_NEXTTAG;
+						break;
+					}
 				}
+				if (!vep->remove)
+					vep_mark_verbatim(vep, p + 1);
 			}
-			if (!vep->remove)
-				vep_mark_verbatim(vep, p + 1);
 		} else if (vep->state == VEP_NEXTTAG) {
 			/*
 			 * Hunt for start of next tag and keep an eye
@@ -687,7 +703,7 @@
 				vep_mark_skip(vep, p);
 				vep_error(vep,
 				    "XML 1.0 Illegal attribute start char");
-printf("ERR %d [%.*s]\n", __LINE__, (int)(e-p), p);
+Debug("ERR %d [%.*s]\n", __LINE__, (int)(e-p), p);
 				vep->state = VEP_TAGERROR;
 			}
 		} else if (vep->state == VEP_TAGERROR) {
@@ -769,7 +785,7 @@
 				vep_mark_skip(vep, p);
 				vep_error(vep,
 				    "XML 1.0 Missing end attribute delimiter");
-printf("ERR %d %c %c\n", __LINE__, *p, vep->attr_delim);
+Debug("ERR %d %c %c\n", __LINE__, *p, vep->attr_delim);
 				vep->state = VEP_TAGERROR;
 				vep->attr_delim = 0;
 				if (vep->attr_vsb != NULL) {
@@ -848,7 +864,7 @@
 				}
 			}
 		} else {
-			printf("*** Unknown state %s\n", vep->state);
+			Debug("*** Unknown state %s\n", vep->state);
 			INCOMPL();
 		}
 	}
@@ -922,6 +938,7 @@
 	vep = (void*)WS_Alloc(sp->wrk->ws, sizeof *vep);
 	AN(vep);
 
+	Debug("BEGIN %p\n", vep);
 
 	memset(vep, 0, sizeof *vep);
 	vep->magic = VEP_MAGIC;
@@ -942,6 +959,7 @@
 
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 	vep = sp->wrk->vep;
+	Debug("BYTES %p\n", vep);
 	CHECK_OBJ_NOTNULL(vep, VEP_MAGIC);
 	AN(vep->bytes);
 	return (vep->bytes(sp, htc, bytes));
@@ -956,16 +974,19 @@
 
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 	vep = sp->wrk->vep;
+	Debug("ENDING %p\n", vep);
 	CHECK_OBJ_NOTNULL(vep, VEP_MAGIC);
 
+	usleep(100);
+	Debug("ENDING STATE: %s\n", vep->state);
+	if (vep->o_skip)
+		vep_emit_skip(vep);
 	if (vep->o_verbatim)
 		vep_emit_verbatim(vep);
-	if (vep->o_skip)
-		vep_emit_skip(vep);
 	vsb_finish(vep->vsb);
 	l = vsb_len(vep->vsb);
 	if (vep->state != VEP_NOTXML && l != 0) {
-		printf("ESI %d <%s>\n", (int)l, vsb_data(vep->vsb));
+		Debug("ESI %d <%s>\n", (int)l, vsb_data(vep->vsb));
 
 		/* XXX: This is a huge waste of storage... */
 		sp->obj->esidata = STV_alloc(sp, vsb_len(vep->vsb));

Modified: trunk/varnish-cache/bin/varnishd/cache_fetch.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_fetch.c	2011-01-13 17:14:46 UTC (rev 5735)
+++ trunk/varnish-cache/bin/varnishd/cache_fetch.c	2011-01-13 17:15:36 UTC (rev 5736)
@@ -193,10 +193,14 @@
 		WSP(sp, SLT_FetchError, "straight length syntax");
 		return (-1);
 	}
+	/*
+	 * XXX: we shouldn't need this if we have cl==0
+	 * XXX: but we must also conditionalize the vfp->end()
+	 */
+	sp->wrk->vfp->begin(sp, cl);
 	if (cl == 0)
 		return (0);
 
-	sp->wrk->vfp->begin(sp, cl);
 
 	i = sp->wrk->vfp->bytes(sp, htc, cl);
 	if (i <= 0) {




From phk at varnish-cache.org  Thu Jan 13 18:44:07 2011
From: phk at varnish-cache.org (phk at varnish-cache.org)
Date: Thu, 13 Jan 2011 19:44:07 +0100
Subject: r5737 - trunk/varnish-cache/bin/varnishd
Message-ID: <E1PdS9X-0006ub-AL@project.varnish-software.com>

Author: phk
Date: 2011-01-13 19:44:06 +0100 (Thu, 13 Jan 2011)
New Revision: 5737

Modified:
   trunk/varnish-cache/bin/varnishd/cache_esi_parse.c
Log:
Fix an off by one in esi delivery.

Correctly deliver last bits of object.

One test-case left, and that the trivia e00018.vtc



Modified: trunk/varnish-cache/bin/varnishd/cache_esi_parse.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_esi_parse.c	2011-01-13 17:15:36 UTC (rev 5736)
+++ trunk/varnish-cache/bin/varnishd/cache_esi_parse.c	2011-01-13 18:44:06 UTC (rev 5737)
@@ -551,7 +551,7 @@
 						break;
 					}
 				}
-				if (!vep->remove)
+				if (p < e && !vep->remove)
 					vep_mark_verbatim(vep, p + 1);
 			}
 		} else if (vep->state == VEP_NEXTTAG) {
@@ -583,7 +583,7 @@
 					vep->esicmt_p = vep->esicmt;
 				}
 			}
-			if (vep->esicmt_p == NULL && !vep->remove)
+			if (p < e && vep->esicmt_p == NULL && !vep->remove)
 				vep_mark_verbatim(vep, p);
 			if (p < e)
 				vep->state = VEP_STARTTAG;
@@ -974,13 +974,20 @@
 
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 	vep = sp->wrk->vep;
+	sp->wrk->vep = NULL;
 	Debug("ENDING %p\n", vep);
 	CHECK_OBJ_NOTNULL(vep, VEP_MAGIC);
 
 	usleep(100);
-	Debug("ENDING STATE: %s\n", vep->state);
+	st = sp->wrk->storage;
+	if (st != NULL)
+		l = (const char *)(st->ptr + st->len) - (const char*)vep->ver_p;
+	else
+		l = 0;
+	Debug("ENDING STATE: %s (%ld)\n", vep->state, l);
 	if (vep->o_skip)
 		vep_emit_skip(vep);
+	vep->o_verbatim += l;
 	if (vep->o_verbatim)
 		vep_emit_verbatim(vep);
 	vsb_finish(vep->vsb);




From phk at varnish-cache.org  Thu Jan 13 18:50:07 2011
From: phk at varnish-cache.org (phk at varnish-cache.org)
Date: Thu, 13 Jan 2011 19:50:07 +0100
Subject: r5738 - trunk/varnish-cache/bin/varnishtest/tests
Message-ID: <E1PdSFL-00079b-Ms@project.varnish-software.com>

Author: phk
Date: 2011-01-13 19:50:05 +0100 (Thu, 13 Jan 2011)
New Revision: 5738

Modified:
   trunk/varnish-cache/bin/varnishtest/tests/r00476.vtc
Log:
Typo



Modified: trunk/varnish-cache/bin/varnishtest/tests/r00476.vtc
===================================================================
--- trunk/varnish-cache/bin/varnishtest/tests/r00476.vtc	2011-01-13 18:44:06 UTC (rev 5737)
+++ trunk/varnish-cache/bin/varnishtest/tests/r00476.vtc	2011-01-13 18:50:05 UTC (rev 5738)
@@ -1,6 +1,6 @@
 # $Id$
 
-test "zero length ESI include segmens with chunked encoding"
+test "zero length ESI include segments with chunked encoding"
 
 server s1 {
 	rxreq




From phk at varnish-cache.org  Thu Jan 13 22:43:07 2011
From: phk at varnish-cache.org (phk at varnish-cache.org)
Date: Thu, 13 Jan 2011 23:43:07 +0100
Subject: r5739 - trunk/varnish-cache/include
Message-ID: <E1PdVsq-0004Fy-Aa@project.varnish-software.com>

Author: phk
Date: 2011-01-13 23:43:05 +0100 (Thu, 13 Jan 2011)
New Revision: 5739

Modified:
   trunk/varnish-cache/include/vend.h
Log:
Uncomment some endianess functions we will need.



Modified: trunk/varnish-cache/include/vend.h
===================================================================
--- trunk/varnish-cache/include/vend.h	2011-01-13 18:50:05 UTC (rev 5738)
+++ trunk/varnish-cache/include/vend.h	2011-01-13 22:43:05 UTC (rev 5739)
@@ -35,7 +35,6 @@
 
 /* Alignment-agnostic encode/decode bytestream to/from little/big endian. */
 
-#if 0
 static __inline uint16_t
 vbe16dec(const void *pp)
 {
@@ -43,7 +42,6 @@
 
 	return ((p[0] << 8) | p[1]);
 }
-#endif
 
 static __inline uint32_t
 vbe32dec(const void *pp)
@@ -53,7 +51,6 @@
 	return (((unsigned)p[0] << 24) | (p[1] << 16) | (p[2] << 8) | p[3]);
 }
 
-#if 0
 static __inline uint64_t
 vbe64dec(const void *pp)
 {
@@ -62,6 +59,7 @@
 	return (((uint64_t)vbe32dec(p) << 32) | vbe32dec(p + 4));
 }
 
+#if 0
 static __inline uint16_t
 vle16dec(const void *pp)
 {
@@ -87,6 +85,7 @@
 
 	return (((uint64_t)vle32dec(p + 4) << 32) | vle32dec(p));
 }
+#endif
 
 static __inline void
 vbe16enc(void *pp, uint16_t u)
@@ -97,8 +96,6 @@
 	p[1] = u & 0xff;
 }
 
-#endif
-
 static __inline void
 vbe32enc(void *pp, uint32_t u)
 {




From phk at varnish-cache.org  Thu Jan 13 22:49:34 2011
From: phk at varnish-cache.org (phk at varnish-cache.org)
Date: Thu, 13 Jan 2011 23:49:34 +0100
Subject: r5740 - trunk/varnish-cache/bin/varnishd
Message-ID: <E1PdVz4-0004Pn-Bh@project.varnish-software.com>

Author: phk
Date: 2011-01-13 23:49:27 +0100 (Thu, 13 Jan 2011)
New Revision: 5740

Modified:
   trunk/varnish-cache/bin/varnishd/cache_esi.h
   trunk/varnish-cache/bin/varnishd/cache_esi_deliver.c
   trunk/varnish-cache/bin/varnishd/cache_esi_parse.c
Log:
Make the VEC codes be 1, 2 and 8 byte wide length fields.  It does not
make sense to add the overhead of a 4 byte code to save 0.0006% space.

Add standard entity expansion for URLs in <esi:include>

Now the new code passes all testcases :->

If you want to try it out, change
	#define OLD_ESI
to
	#undef OLD_ESI
at the top of bin/varnishd/cache.h

Once I have polished the new code a bit, it will become the default.



Modified: trunk/varnish-cache/bin/varnishd/cache_esi.h
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_esi.h	2011-01-13 22:43:05 UTC (rev 5739)
+++ trunk/varnish-cache/bin/varnishd/cache_esi.h	2011-01-13 22:49:27 UTC (rev 5740)
@@ -29,11 +29,11 @@
 
 #define	VEC_V1	'B'
 #define	VEC_V2	'W'
-#define	VEC_V4	'L'
+#define	VEC_V8	'L'
 #define	VEC_S1	'b'
 #define	VEC_S2	'w'
-#define	VEC_S4	'l'
+#define	VEC_S8	'l'
 #define	VEC_L1	'c'
 #define	VEC_L2	'x'
-#define	VEC_L4	'm'
+#define	VEC_L8	'm'
 #define	VEC_INCL	'I'

Modified: trunk/varnish-cache/bin/varnishd/cache_esi_deliver.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_esi_deliver.c	2011-01-13 22:43:05 UTC (rev 5739)
+++ trunk/varnish-cache/bin/varnishd/cache_esi_deliver.c	2011-01-13 22:49:27 UTC (rev 5740)
@@ -121,9 +121,22 @@
 
 #ifndef OLD_ESI
 
-// #define Debug(fmt, ...) printf(fmt, __VA_ARGS__)
+//#define Debug(fmt, ...) printf(fmt, __VA_ARGS__)
 #define Debug(fmt, ...) /**/
 
+static void
+esi_sendchunk(struct sess *sp, const void *cb, ssize_t cl,
+    const void *ptr, ssize_t l)
+{
+
+	Debug("VER(%d) %d\n", (int)l, (int)(cb-ce));
+	if (sp->wrk->res_mode & RES_CHUNKED)
+		WRW_Write(sp->wrk, cb, cl);
+	WRW_Write(sp->wrk, ptr, l);
+	if (sp->wrk->res_mode & RES_CHUNKED)
+		WRW_Write(sp->wrk, "\r\n", -1);
+}
+
 void
 ESI_Deliver(struct sess *sp)
 {
@@ -150,28 +163,46 @@
 			p += 2;
 			q = (void*)strchr((const char*)p, '\0');
 			assert (q > p);
-			Debug("VER(%d) %d\n", (int)l, (int)(q-p));
-			if (sp->wrk->res_mode & RES_CHUNKED)
-				WRW_Write(sp->wrk, p, q - p);
-			WRW_Write(sp->wrk, st->ptr + off, l);
-			if (sp->wrk->res_mode & RES_CHUNKED)
-				WRW_Write(sp->wrk, "\r\n", -1);
-			// Debug("[%.*s]", (int)l, st->ptr + off);
+			esi_sendchunk(sp, p, q - p, st->ptr + off, l);
 			off += l;
 			p = q + 1;
 			break;
+		case VEC_V2:
+			l = vbe16dec(p + 1);
+			p += 3;
+			q = (void*)strchr((const char*)p, '\0');
+			assert (q > p);
+			esi_sendchunk(sp, p, q - p, st->ptr + off, l);
+			off += l;
+			p = q + 1;
+			break;
+		case VEC_V8:
+			l = vbe64dec(p + 1);
+			p += 9;
+			q = (void*)strchr((const char*)p, '\0');
+			assert (q > p);
+			esi_sendchunk(sp, p, q - p, st->ptr + off, l);
+			off += l;
+			p = q + 1;
+			break;
 		case VEC_S1:
 			l = p[1];
 			p += 2;
-			Debug("SKIP(%d)\n", (int)l);
+			Debug("SKIP1(%d)\n", (int)l);
 			off += l;
 			break;
 		case VEC_S2:
 			l = vbe16dec(p + 1);
 			p += 3;
-			Debug("SKIP(%d)\n", (int)l);
+			Debug("SKIP2(%d)\n", (int)l);
 			off += l;
 			break;
+		case VEC_S8:
+			l = vbe64dec(p + 1);
+			p += 9;
+			Debug("SKIP8(%d)\n", (int)l);
+			off += l;
+			break;
 		case VEC_L1:
 			l = p[1];
 			p += 2;
@@ -193,9 +224,9 @@
 			q++;
 			r = (void*)strchr((const char*)q, '\0');
 			AN(r);
-			Debug("INCL [%s][%s] BEGIN\n", p, q);
-			ESI_Include(sp, (const char*)p, (const char*)q);
-			Debug("INCL [%s] END\n", p);
+			Debug("INCL [%s][%s] BEGIN\n", q, p);
+			ESI_Include(sp, (const char*)q, (const char*)p);
+			Debug("INCL [%s][%s] END\n", q, p);
 			p = r + 1;
 			break;
 		default:

Modified: trunk/varnish-cache/bin/varnishd/cache_esi_parse.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_esi_parse.c	2011-01-13 22:43:05 UTC (rev 5739)
+++ trunk/varnish-cache/bin/varnishd/cache_esi_parse.c	2011-01-13 22:49:27 UTC (rev 5740)
@@ -43,7 +43,7 @@
 
 #ifndef OLD_ESI
 
-// #define Debug(fmt, ...) printf(fmt, __VA_ARGS__)
+//#define Debug(fmt, ...) printf(fmt, __VA_ARGS__)
 #define Debug(fmt, ...) /**/
 
 struct vep_state;
@@ -230,24 +230,26 @@
  */
 
 static void
-vep_emit_len(struct vep_state *vep, ssize_t l, int m8, int m16, int m32)
+vep_emit_len(struct vep_state *vep, ssize_t l, int m8, int m16, int m64)
 {
-	uint8_t buf[5];
+	uint8_t buf[9];
 
 	assert(l > 0);
 	if (l < 256) {
 		buf[0] = m8;
 		buf[1] = (uint8_t)l;
+		assert((ssize_t)buf[1] == l);
 		vsb_bcat(vep->vsb, buf, 2);
 	} else if (l < 65536) {
 		buf[0] = m16;
 		vbe16enc(buf + 1, (uint16_t)l);
+		assert((ssize_t)vbe16dec(buf + 1) == l);
 		vsb_bcat(vep->vsb, buf, 3);
 	} else {
-		/* XXX assert < 2^32 */
-		buf[0] = m32;
-		vbe32enc(buf + 1, (uint32_t)l);
-		vsb_bcat(vep->vsb, buf, 5);
+		buf[0] = m64;
+		vbe64enc(buf + 1, l);
+		assert((ssize_t)vbe64dec(buf + 1) == l);
+		vsb_bcat(vep->vsb, buf, 9);
 	}
 } 
 
@@ -259,7 +261,7 @@
 	l = vep->o_skip;
 	vep->o_skip = 0;
 	assert(l > 0);
-	vep_emit_len(vep, l, VEC_S1, VEC_S2, VEC_S4);
+	vep_emit_len(vep, l, VEC_S1, VEC_S2, VEC_S8);
 } 
 
 static void
@@ -270,7 +272,7 @@
 	l = vep->o_verbatim;
 	vep->o_verbatim = 0;
 	assert(l > 0);
-	vep_emit_len(vep, l, VEC_V1, VEC_V2, VEC_V4);
+	vep_emit_len(vep, l, VEC_V1, VEC_V2, VEC_V8);
 	vsb_printf(vep->vsb, "%lx\r\n%c", l, 0);
 } 
 
@@ -287,7 +289,7 @@
 		vep_emit_skip(vep);
 	l = e - p;
 	Debug("---->L(%d) [%.*s]\n", (int)l, (int)l, p);
-	vep_emit_len(vep, l, VEC_L1, VEC_L2, VEC_L4);
+	vep_emit_len(vep, l, VEC_L1, VEC_L2, VEC_L8);
 	vsb_printf(vep->vsb, "%lx\r\n%c", l, 0);
 	vsb_bcat(vep->vsb, p, l);
 }
@@ -427,18 +429,18 @@
 		l = vsb_len(vep->include_src);
 		h = 0;
 
+		vsb_printf(vep->vsb, "%c", VEC_INCL);
 		if (l > 7 && !memcmp(p, "http://", 7)) {
 			h = p + 7;
 			p = strchr(h, '/');
 			AN(p);
 			Debug("HOST <%.*s> PATH <%s>\n", (int)(p-h),h, p);
-			vsb_printf(vep->vsb, "%c%s%cHost: %.*s%c",
-			    VEC_INCL, p, 0,
+			vsb_printf(vep->vsb, "Host: %.*s%c",
 			    (int)(p-h), h, 0);
 		} else if (*p == '/') {
-			vsb_printf(vep->vsb, "%c%s%c%c",
-			    VEC_INCL, p, 0, 0);
+			vsb_printf(vep->vsb, "%c", 0);
 		} else {
+			vsb_printf(vep->vsb, "%c", 0);
 			url = vep->sp->wrk->bereq->hd[HTTP_HDR_URL];
 			/* Look for the last / before a '?' */
 			h = NULL;
@@ -448,13 +450,29 @@
 			if (h == NULL)
 				h = q + 1;
 				
-			Debug("INCL:: %.*s/%s\n",
+			Debug("INCL:: [%.*s]/[%s]\n",
 			    (int)(h - url.b), url.b, p);
-			vsb_printf(vep->vsb, "%c%.*s/%s%c%c",
-			    VEC_INCL, 
-			    (int)(h - url.b), url.b,
-			    p, 0, 0);
+			vsb_printf(vep->vsb, "%.*s/", (int)(h - url.b), url.b);
 		}
+		l -= (p - vsb_data(vep->include_src));
+		for (q = p; *q != '\0'; ) {
+			if (*q == '&') {
+#define R(w,f,r)							\
+				if (q + w <= p + l && !memcmp(q, f, w)) { \
+					vsb_printf(vep->vsb, "%c", r);	\
+					q += l;				\
+					continue;			\
+				}
+				R(6, "&apos;", '\'');
+				R(6, "&quot;", '"');
+				R(4, "&lt;", '<');
+				R(4, "&gt;", '>');
+				R(5, "&amp;", '&');
+			}
+			vsb_printf(vep->vsb, "%c", *q++);
+		}
+#undef R
+		vsb_printf(vep->vsb, "%c", 0);
 
 		vsb_delete(vep->include_src);
 		vep->include_src = NULL;




From phk at varnish-cache.org  Thu Jan 13 23:25:56 2011
From: phk at varnish-cache.org (phk at varnish-cache.org)
Date: Fri, 14 Jan 2011 00:25:56 +0100
Subject: r5741 - trunk/varnish-cache/bin/varnishd
Message-ID: <E1PdWYG-0005UZ-H4@project.varnish-software.com>

Author: phk
Date: 2011-01-14 00:25:54 +0100 (Fri, 14 Jan 2011)
New Revision: 5741

Modified:
   trunk/varnish-cache/bin/varnishd/cache_fetch.c
Log:
Make big number a long long



Modified: trunk/varnish-cache/bin/varnishd/cache_fetch.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_fetch.c	2011-01-13 22:49:27 UTC (rev 5740)
+++ trunk/varnish-cache/bin/varnishd/cache_fetch.c	2011-01-13 23:25:54 UTC (rev 5741)
@@ -300,7 +300,7 @@
 
 	assert(sp->wrk->body_status == BS_EOF);
 	sp->wrk->vfp->begin(sp, 0);
-	i = sp->wrk->vfp->bytes(sp, htc, 1000000000000);	/* XXX ? */
+	i = sp->wrk->vfp->bytes(sp, htc, 1000000000000LL);	/* XXX ? */
 	if (i < 0) {
 		WSP(sp, SLT_FetchError, "eof read_error: %d (%s)",
 		    errno, strerror(errno));




From phk at varnish-cache.org  Thu Jan 13 23:51:35 2011
From: phk at varnish-cache.org (phk at varnish-cache.org)
Date: Fri, 14 Jan 2011 00:51:35 +0100
Subject: r5742 - trunk/varnish-cache/bin/varnishd
Message-ID: <E1PdWx6-00064w-0M@project.varnish-software.com>

Author: phk
Date: 2011-01-14 00:51:34 +0100 (Fri, 14 Jan 2011)
New Revision: 5742

Modified:
   trunk/varnish-cache/bin/varnishd/cache_fetch.c
Log:
Try to get centos/32bit compiling another way.



Modified: trunk/varnish-cache/bin/varnishd/cache_fetch.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_fetch.c	2011-01-13 23:25:54 UTC (rev 5741)
+++ trunk/varnish-cache/bin/varnishd/cache_fetch.c	2011-01-13 23:51:34 UTC (rev 5742)
@@ -300,7 +300,13 @@
 
 	assert(sp->wrk->body_status == BS_EOF);
 	sp->wrk->vfp->begin(sp, 0);
-	i = sp->wrk->vfp->bytes(sp, htc, 1000000000000LL);	/* XXX ? */
+	i = sp->wrk->vfp->bytes(sp, htc,
+#ifdef SIZE_T_MAX
+	    SIZE_T_MAX
+#else
+	    sizeof(void*) == 4 ? ((size_t)1<<31) : ((size_t)1 << 63)
+#endif
+	    ); 
 	if (i < 0) {
 		WSP(sp, SLT_FetchError, "eof read_error: %d (%s)",
 		    errno, strerror(errno));




From phk at varnish-cache.org  Thu Jan 13 23:52:41 2011
From: phk at varnish-cache.org (phk at varnish-cache.org)
Date: Fri, 14 Jan 2011 00:52:41 +0100
Subject: r5743 - trunk/varnish-cache/bin/varnishd
Message-ID: <E1PdWy9-00067V-Sy@project.varnish-software.com>

Author: phk
Date: 2011-01-14 00:52:40 +0100 (Fri, 14 Jan 2011)
New Revision: 5743

Modified:
   trunk/varnish-cache/bin/varnishd/cache_esi_deliver.c
   trunk/varnish-cache/bin/varnishd/cache_esi_parse.c
Log:
A quick round of FlexeLint silencing



Modified: trunk/varnish-cache/bin/varnishd/cache_esi_deliver.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_esi_deliver.c	2011-01-13 23:51:34 UTC (rev 5742)
+++ trunk/varnish-cache/bin/varnishd/cache_esi_deliver.c	2011-01-13 23:52:40 UTC (rev 5743)
@@ -125,16 +125,16 @@
 #define Debug(fmt, ...) /**/
 
 static void
-esi_sendchunk(struct sess *sp, const void *cb, ssize_t cl,
+esi_sendchunk(const struct sess *sp, const void *cb, ssize_t cl,
     const void *ptr, ssize_t l)
 {
 
 	Debug("VER(%d) %d\n", (int)l, (int)(cb-ce));
 	if (sp->wrk->res_mode & RES_CHUNKED)
-		WRW_Write(sp->wrk, cb, cl);
-	WRW_Write(sp->wrk, ptr, l);
+		(void)WRW_Write(sp->wrk, cb, cl);
+	(void)WRW_Write(sp->wrk, ptr, l);
 	if (sp->wrk->res_mode & RES_CHUNKED)
-		WRW_Write(sp->wrk, "\r\n", -1);
+		(void)WRW_Write(sp->wrk, "\r\n", -1);
 }
 
 void
@@ -209,12 +209,7 @@
 			q = (void*)strchr((const char*)p, '\0');
 			assert (q > p);
 			Debug("LIT(%d) %d\n", (int)l, (int)(q-p));
-			if (sp->wrk->res_mode & RES_CHUNKED)
-				WRW_Write(sp->wrk, p, q - p);
-			p = q + 1;
-			WRW_Write(sp->wrk, p, l);
-			if (sp->wrk->res_mode & RES_CHUNKED)
-				WRW_Write(sp->wrk, "\r\n", -1);
+			esi_sendchunk(sp, p, q - p, q + 1, l);
 			p = p + l;
 			break;
 		case VEC_INCL:
@@ -232,10 +227,9 @@
 		default:
 			Debug("XXXX 0x%02x [%s]\n", *p, p);
 			INCOMPL();
-			return;
 		}
 	}
-	WRW_Flush(sp->wrk);
+	(void)WRW_Flush(sp->wrk);
 }
 
 #endif /* OLD_ESI */

Modified: trunk/varnish-cache/bin/varnishd/cache_esi_parse.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_esi_parse.c	2011-01-13 23:51:34 UTC (rev 5742)
+++ trunk/varnish-cache/bin/varnishd/cache_esi_parse.c	2011-01-13 23:52:40 UTC (rev 5743)
@@ -53,7 +53,7 @@
 
 struct vep_match {
 	const char	*match;
-	const char	**state;
+	const char	* const *state;
 };
 
 struct vep_state {
@@ -106,37 +106,37 @@
 
 /*---------------------------------------------------------------------*/
 
-static const char *VEP_START =		"[Start]";
-static const char *VEP_TESTXML = 	"[TestXml]";
-static const char *VEP_NOTXML =	 	"[NotXml]";
+static const char * const VEP_START =		"[Start]";
+static const char * const VEP_TESTXML = 	"[TestXml]";
+static const char * const VEP_NOTXML =	 	"[NotXml]";
 
-static const char *VEP_NEXTTAG = 	"[NxtTag]";
-static const char *VEP_NOTMYTAG =	"[NotMyTag]";
+static const char * const VEP_NEXTTAG = 	"[NxtTag]";
+static const char * const VEP_NOTMYTAG =	"[NotMyTag]";
 
-static const char *VEP_STARTTAG = 	"[StartTag]";
-static const char *VEP_COMMENT =	"[Comment]";
-static const char *VEP_CDATA =		"[CDATA]";
-static const char *VEP_ESITAG =		"[ESITag]";
-static const char *VEP_ESIETAG =	"[ESIEndTag]";
+static const char * const VEP_STARTTAG = 	"[StartTag]";
+static const char * const VEP_COMMENT =		"[Comment]";
+static const char * const VEP_CDATA =		"[CDATA]";
+static const char * const VEP_ESITAG =		"[ESITag]";
+static const char * const VEP_ESIETAG =		"[ESIEndTag]";
 
-static const char *VEP_ESIREMOVE =	"[ESI:Remove]";
-static const char *VEP_ESIINCLUDE =	"[ESI:Include]";
-static const char *VEP_ESICOMMENT =	"[ESI:Comment]";
-static const char *VEP_ESIBOGON =	"[ESI:Bogon]";
+static const char * const VEP_ESIREMOVE =	"[ESI:Remove]";
+static const char * const VEP_ESIINCLUDE =	"[ESI:Include]";
+static const char * const VEP_ESICOMMENT =	"[ESI:Comment]";
+static const char * const VEP_ESIBOGON =	"[ESI:Bogon]";
 
-static const char *VEP_INTAG =		"[InTag]";
-static const char *VEP_TAGERROR =	"[TagError]";
+static const char * const VEP_INTAG =		"[InTag]";
+static const char * const VEP_TAGERROR =	"[TagError]";
 
-static const char *VEP_ATTR =		"[Attribute]";
-static const char *VEP_SKIPATTR =	"[SkipAttribute]";
-static const char *VEP_SKIPATTR2 =	"[SkipAttribute2]";
-static const char *VEP_ATTRDELIM =	"[AttrDelim]";
-static const char *VEP_ATTRGETVAL =	"[AttrGetValue]";
-static const char *VEP_ATTRVAL =	"[AttrValue]";
+static const char * const VEP_ATTR =		"[Attribute]";
+static const char * const VEP_SKIPATTR =	"[SkipAttribute]";
+static const char * const VEP_SKIPATTR2 =	"[SkipAttribute2]";
+static const char * const VEP_ATTRDELIM =	"[AttrDelim]";
+static const char * const VEP_ATTRGETVAL =	"[AttrGetValue]";
+static const char * const VEP_ATTRVAL =	"[AttrValue]";
 
-static const char *VEP_UNTIL =		"[Until]";
-static const char *VEP_MATCHBUF = 	"[MatchBuf]";
-static const char *VEP_MATCH =		"[Match]";
+static const char * const VEP_UNTIL =		"[Until]";
+static const char * const VEP_MATCHBUF = 	"[MatchBuf]";
+static const char * const VEP_MATCH =		"[Match]";
 
 /*---------------------------------------------------------------------*/
 
@@ -205,6 +205,7 @@
 {
 	struct vep_match *vm;
 	const char *q, *r;
+	ssize_t l;
 
 	for (vm = vep->match; vm->match; vm++) {
 		r = b;
@@ -213,9 +214,10 @@
 				break;
 		if (*q != '\0' && r == e) {
 			if (b != vep->tag) {
-				assert(e - b < sizeof vep->tag);
-				memcpy(vep->tag, b, e - b);
-				vep->tag_i = e - b;
+				l = e - b;
+				assert(l < sizeof vep->tag);
+				memmove(vep->tag, b, l);
+				vep->tag_i = l;
 			}
 			return (NULL);
 		}
@@ -230,23 +232,23 @@
  */
 
 static void
-vep_emit_len(struct vep_state *vep, ssize_t l, int m8, int m16, int m64)
+vep_emit_len(const struct vep_state *vep, ssize_t l, int m8, int m16, int m64)
 {
 	uint8_t buf[9];
 
 	assert(l > 0);
 	if (l < 256) {
-		buf[0] = m8;
+		buf[0] = (uint8_t)m8;
 		buf[1] = (uint8_t)l;
 		assert((ssize_t)buf[1] == l);
 		vsb_bcat(vep->vsb, buf, 2);
 	} else if (l < 65536) {
-		buf[0] = m16;
+		buf[0] = (uint8_t)m16;
 		vbe16enc(buf + 1, (uint16_t)l);
 		assert((ssize_t)vbe16dec(buf + 1) == l);
 		vsb_bcat(vep->vsb, buf, 3);
 	} else {
-		buf[0] = m64;
+		buf[0] = (uint8_t)m64;
 		vbe64enc(buf + 1, l);
 		assert((ssize_t)vbe64dec(buf + 1) == l);
 		vsb_bcat(vep->vsb, buf, 9);
@@ -276,6 +278,7 @@
 	vsb_printf(vep->vsb, "%lx\r\n%c", l, 0);
 } 
 
+#if 0
 static void
 vep_emit_literal(struct vep_state *vep, const char *p, const char *e)
 {
@@ -293,8 +296,8 @@
 	vsb_printf(vep->vsb, "%lx\r\n%c", l, 0);
 	vsb_bcat(vep->vsb, p, l);
 }
+#endif
 
-
 static void
 vep_mark_verbatim(struct vep_state *vep, const char *p)
 {
@@ -332,23 +335,7 @@
 /*---------------------------------------------------------------------
  */
 
-#if 0
-static void
-vep_do_nothing(struct vep_state *vep, enum dowhat what)
-{
-	Debug("DO_NOTHING(%d)\n", what);
-	if (what == DO_ATTR) {
-		Debug("ATTR (%s) (%s)\n", vep->match_hit->match,
-			vsb_data(vep->attr_vsb));
-		vsb_delete(vep->attr_vsb);
-	}
-}
-#endif
-
-/*---------------------------------------------------------------------
- */
-
-static void
+static void __match_proto__()
 vep_do_comment(struct vep_state *vep, enum dowhat what)
 {
 	Debug("DO_COMMENT(%d)\n", what);
@@ -369,7 +356,7 @@
 /*---------------------------------------------------------------------
  */
 
-static void
+static void __match_proto__()
 vep_do_remove(struct vep_state *vep, enum dowhat what)
 {
 	Debug("DO_REMOVE(%d)\n", what);
@@ -400,7 +387,7 @@
 /*---------------------------------------------------------------------
  */
 
-static void
+static void __match_proto__()
 vep_do_include(struct vep_state *vep, enum dowhat what)
 {
 	char *p, *q, *h;
@@ -488,21 +475,17 @@
  */
 
 static void
-vep_parse(struct vep_state *vep, const char *b, size_t l)
+vep_parse(struct vep_state *vep, const char *p, size_t l)
 {
-	const char *e, *p;
+	const char *e;
 	struct vep_match *vm;
 	int i;
 
 	CHECK_OBJ_NOTNULL(vep, VEP_MAGIC);
 	assert(l > 0);
 
-	e = b + l;
+	e = p + l;
 
-	if (0)
-		vep_emit_literal(vep, "A", "B");
-
-	p = b;
 	while (p < e) {
 		AN(vep->state);
 		i = e - p;
@@ -514,8 +497,6 @@
 		    vep->remove,
 		    vep->tag_i, vep->tag,
 		    i, p);
-		fflush(stdout);
-		usleep(10);
 
 		/******************************************************
 		 * SECTION A
@@ -721,7 +702,7 @@
 				vep_mark_skip(vep, p);
 				vep_error(vep,
 				    "XML 1.0 Illegal attribute start char");
-Debug("ERR %d [%.*s]\n", __LINE__, (int)(e-p), p);
+				Debug("ERR %d [%.*s]\n", __LINE__, (int)(e-p), p);
 				vep->state = VEP_TAGERROR;
 			}
 		} else if (vep->state == VEP_TAGERROR) {
@@ -803,7 +784,7 @@
 				vep_mark_skip(vep, p);
 				vep_error(vep,
 				    "XML 1.0 Missing end attribute delimiter");
-Debug("ERR %d %c %c\n", __LINE__, *p, vep->attr_delim);
+				Debug("ERR %d %c %c\n", __LINE__, *p, vep->attr_delim);
 				vep->state = VEP_TAGERROR;
 				vep->attr_delim = 0;
 				if (vep->attr_vsb != NULL) {
@@ -838,7 +819,6 @@
 			if (vm != NULL) {
 				if (vm->match != NULL)
 					p += strlen(vm->match);
-				b = p;
 				vep->state = *vm->state;
 				vep->match = NULL;
 				vep->tag_i = 0;
@@ -886,6 +866,7 @@
 			INCOMPL();
 		}
 	}
+	/* XXX: should we return p ? */
 }
 
 /*---------------------------------------------------------------------
@@ -919,8 +900,8 @@
 		w = HTC_Read(htc, st->ptr + st->len, l);
 		if (w <= 0)
 			return (w);
-if (vep->hack_p == NULL)
-	vep->hack_p = (const char *)st->ptr + st->len;
+		if (vep->hack_p == NULL)
+			vep->hack_p = (const char *)st->ptr + st->len;
 		vep->ver_p = (const char *)st->ptr + st->len;
 #if 0
 		{
@@ -996,7 +977,6 @@
 	Debug("ENDING %p\n", vep);
 	CHECK_OBJ_NOTNULL(vep, VEP_MAGIC);
 
-	usleep(100);
 	st = sp->wrk->storage;
 	if (st != NULL)
 		l = (const char *)(st->ptr + st->len) - (const char*)vep->ver_p;
@@ -1010,15 +990,14 @@
 		vep_emit_verbatim(vep);
 	vsb_finish(vep->vsb);
 	l = vsb_len(vep->vsb);
-	if (vep->state != VEP_NOTXML && l != 0) {
+	if (vep->state != VEP_NOTXML && l > 0) {
 		Debug("ESI %d <%s>\n", (int)l, vsb_data(vep->vsb));
 
 		/* XXX: This is a huge waste of storage... */
-		sp->obj->esidata = STV_alloc(sp, vsb_len(vep->vsb));
+		sp->obj->esidata = STV_alloc(sp, l);
 		AN(sp->obj->esidata);
-		memcpy(sp->obj->esidata->ptr,
-		    vsb_data(vep->vsb), vsb_len(vep->vsb));
-		sp->obj->esidata->len = vsb_len(vep->vsb);
+		memcpy(sp->obj->esidata->ptr, vsb_data(vep->vsb), l);
+		sp->obj->esidata->len = l;
 	}
 	vsb_delete(vep->vsb);
 




From phk at varnish-cache.org  Fri Jan 14 00:40:11 2011
From: phk at varnish-cache.org (phk at varnish-cache.org)
Date: Fri, 14 Jan 2011 01:40:11 +0100
Subject: r5744 - trunk/varnish-cache/bin/varnishd
Message-ID: <E1PdXi7-0007Kz-Nc@project.varnish-software.com>

Author: phk
Date: 2011-01-14 01:40:10 +0100 (Fri, 14 Jan 2011)
New Revision: 5744

Modified:
   trunk/varnish-cache/bin/varnishd/cache_esi.h
   trunk/varnish-cache/bin/varnishd/cache_esi_deliver.c
   trunk/varnish-cache/bin/varnishd/cache_esi_parse.c
Log:
When we fail a buffered match on '>' don't ask for more, fail it.

Drop the VEC literal stuff until we need it.



Modified: trunk/varnish-cache/bin/varnishd/cache_esi.h
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_esi.h	2011-01-13 23:52:40 UTC (rev 5743)
+++ trunk/varnish-cache/bin/varnishd/cache_esi.h	2011-01-14 00:40:10 UTC (rev 5744)
@@ -33,7 +33,4 @@
 #define	VEC_S1	'b'
 #define	VEC_S2	'w'
 #define	VEC_S8	'l'
-#define	VEC_L1	'c'
-#define	VEC_L2	'x'
-#define	VEC_L8	'm'
 #define	VEC_INCL	'I'

Modified: trunk/varnish-cache/bin/varnishd/cache_esi_deliver.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_esi_deliver.c	2011-01-13 23:52:40 UTC (rev 5743)
+++ trunk/varnish-cache/bin/varnishd/cache_esi_deliver.c	2011-01-14 00:40:10 UTC (rev 5744)
@@ -203,15 +203,6 @@
 			Debug("SKIP8(%d)\n", (int)l);
 			off += l;
 			break;
-		case VEC_L1:
-			l = p[1];
-			p += 2;
-			q = (void*)strchr((const char*)p, '\0');
-			assert (q > p);
-			Debug("LIT(%d) %d\n", (int)l, (int)(q-p));
-			esi_sendchunk(sp, p, q - p, q + 1, l);
-			p = p + l;
-			break;
 		case VEC_INCL:
 			p++;
 			q = (void*)strchr((const char*)p, '\0');

Modified: trunk/varnish-cache/bin/varnishd/cache_esi_parse.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_esi_parse.c	2011-01-13 23:52:40 UTC (rev 5743)
+++ trunk/varnish-cache/bin/varnishd/cache_esi_parse.c	2011-01-14 00:40:10 UTC (rev 5744)
@@ -200,6 +200,7 @@
 /*---------------------------------------------------------------------
  * return match or NULL if more input needed.
  */
+
 static struct vep_match *
 vep_match(struct vep_state *vep, const char *b, const char *e)
 {
@@ -278,27 +279,7 @@
 	vsb_printf(vep->vsb, "%lx\r\n%c", l, 0);
 } 
 
-#if 0
 static void
-vep_emit_literal(struct vep_state *vep, const char *p, const char *e)
-{
-	ssize_t l;
-
-	if (e == NULL)
-		e = strchr(p, '\0');
-	if (vep->o_verbatim > 0) 
-		vep_emit_verbatim(vep);
-	if (vep->o_skip > 0) 
-		vep_emit_skip(vep);
-	l = e - p;
-	Debug("---->L(%d) [%.*s]\n", (int)l, (int)l, p);
-	vep_emit_len(vep, l, VEC_L1, VEC_L2, VEC_L8);
-	vsb_printf(vep->vsb, "%lx\r\n%c", l, 0);
-	vsb_bcat(vep->vsb, p, l);
-}
-#endif
-
-static void
 vep_mark_verbatim(struct vep_state *vep, const char *p)
 {
 	ssize_t l;
@@ -516,14 +497,12 @@
 				p++;
 				vep_mark_verbatim(vep, p);
 			}
-			if (p < e) {
-				if (*p == '<') {
-					vep->state = VEP_STARTTAG;
-				} else {
-					WSP(vep->sp, SLT_ESI_xmlerror,
-					    "No ESI processing, first char not '<'");
-					vep->state = VEP_NOTXML;
-				}
+			if (p < e && *p == '<') {
+				vep->state = VEP_STARTTAG;
+			} else if (p < e) {
+				WSP(vep->sp, SLT_ESI_xmlerror,
+				    "No ESI processing, first char not '<'");
+				vep->state = VEP_NOTXML;
 			}
 		} else if (vep->state == VEP_NOTXML) {
 			/*
@@ -835,7 +814,7 @@
 			 */
 			do {
 				if (*p == '>') {
-					vm = NULL;
+					vm = vep->match+vep->match_l-1;
 				} else {
 					vep->tag[vep->tag_i++] = *p++;
 					vm = vep_match(vep,
@@ -905,9 +884,14 @@
 		vep->ver_p = (const char *)st->ptr + st->len;
 #if 0
 		{
-		for (l = 0; l < w; l++) 
-			vep_parse(vep, (const char *)st->ptr + st->len + l, 1);
+		ssize_t d;
+		for (l = 0; l < w; l += d)  {
+			d = 1;
+			if (l + d >= w)
+				d = 1;
+			vep_parse(vep, (const char *)st->ptr + st->len + l, d);
 		}
+		}
 #else
 		vep_parse(vep, (const char *)st->ptr + st->len, w);
 #endif




From phk at varnish-cache.org  Fri Jan 14 11:43:27 2011
From: phk at varnish-cache.org (phk at varnish-cache.org)
Date: Fri, 14 Jan 2011 12:43:27 +0100
Subject: r5745 - trunk/varnish-cache/bin/varnishd
Message-ID: <E1Pdi3z-0001ut-RK@project.varnish-software.com>

Author: phk
Date: 2011-01-14 12:43:27 +0100 (Fri, 14 Jan 2011)
New Revision: 5745

Modified:
   trunk/varnish-cache/bin/varnishd/cache_esi_parse.c
Log:
Append the dot-diagram for the state machine and synchronize the 
section letters between the dot and the C-source.

Flesh out a couple of error cases that would still panic.



Modified: trunk/varnish-cache/bin/varnishd/cache_esi_parse.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_esi_parse.c	2011-01-14 00:40:10 UTC (rev 5744)
+++ trunk/varnish-cache/bin/varnishd/cache_esi_parse.c	2011-01-14 11:43:27 UTC (rev 5745)
@@ -453,6 +453,9 @@
  * This function is called with the input object piecemal so do not
  * assume that we have more than one char available at at time, but
  * optimize for getting huge chunks. 
+ *
+ * NB: At the bottom of this source-file, there is a dot-diagram matching
+ * NB: the state-machine.  Please maintain it along with the code.
  */
 
 static void
@@ -512,7 +515,7 @@
 			p = e;
 
 		/******************************************************
-		 * SECTION D
+		 * SECTION B
 		 */
 
 		} else if (vep->state == VEP_NOTMYTAG) {
@@ -567,7 +570,7 @@
 				vep->state = VEP_STARTTAG;
 
 		/******************************************************
-		 * SECTION B
+		 * SECTION C
 		 */
 
 		} else if (vep->state == VEP_STARTTAG) {
@@ -651,7 +654,7 @@
 			vep->state = VEP_TAGERROR;
 
 		/******************************************************
-		 * SECTION F
+		 * SECTION D
 		 */
 
 		} else if (vep->state == VEP_INTAG) {
@@ -674,14 +677,15 @@
 				vep->dostuff(vep, DO_TAG);
 				vep->state = VEP_NEXTTAG;
 			} else if (p < e && vep->emptytag) {
-				INCOMPL();	/* ESI-SYNTAX ERROR */
-			} else if (p < e && vct_isxmlnamestart(*p)) {
+				vep_error(vep,
+				    "XML 1.0 '>' does not follow '/' in tag");
+				vep->state = VEP_TAGERROR;
+			} else if (p < e && vep->canattr &&
+			    vct_isxmlnamestart(*p)) {
 				vep->state = VEP_ATTR;
 			} else if (p < e) {
-				vep_mark_skip(vep, p);
 				vep_error(vep,
 				    "XML 1.0 Illegal attribute start char");
-				Debug("ERR %d [%.*s]\n", __LINE__, (int)(e-p), p);
 				vep->state = VEP_TAGERROR;
 			}
 		} else if (vep->state == VEP_TAGERROR) {
@@ -694,7 +698,7 @@
 			}
 
 		/******************************************************
-		 * SECTION G
+		 * SECTION E
 		 */
 
 		} else if (vep->state == VEP_ATTR) {
@@ -716,6 +720,10 @@
 				if (vep->tag[i] == '=') {
 					assert(i + 1 == vep->tag_i);
 					vep->state = VEP_ATTRDELIM;
+				} else {
+					vep_error(vep,
+					    "XML 1.0 Illegal attr char");
+					vep->state = VEP_TAGERROR;
 				}
 			}
 			xxxassert(i == vep->tag_i);
@@ -1008,3 +1016,130 @@
 };
 
 #endif /* OLD_ESI */
+
+#if 0
+
+digraph xml {
+	rankdir="LR"
+	size="7,10"
+#################################################################
+# SECTION A
+#
+
+START		[shape=ellipse]
+TESTXML		[shape=ellipse]
+NOTXML		[shape=ellipse]
+NEXTTAGa	[shape=hexagon, label="NEXTTAG"]
+STARTTAGa	[shape=hexagon, label="STARTTAG"]
+START		-> TESTXML
+START		-> NEXTTAGa	[style=dotted, label="syntax:1"]
+TESTXML		-> TESTXML	[label="lws"]
+TESTXML		-> NOTXML
+TESTXML		-> STARTTAGa	[label="'<'"]
+
+#################################################################
+# SECTION B
+
+NOTMYTAG	[shape=ellipse]
+NEXTTAG		[shape=ellipse]
+NOTMYTAG	-> NEXTTAG	[style=dotted, label="syntax:2"]
+STARTTAGb	[shape=hexagon, label="STARTTAG"]
+NOTMYTAG	-> NEXTTAG	[label="'>'"]
+NOTMYTAG	-> NOTMYTAG	[label="*"]
+NEXTTAG		-> NEXTTAG	[label="'-->'"]
+NEXTTAG		-> NEXTTAG	[label="*"]
+NEXTTAG		-> STARTTAGb	[label="'<'"]
+
+#################################################################
+# SECTION C
+
+STARTTAG	[shape=ellipse]
+COMMENT		[shape=ellipse]
+CDATA		[shape=ellipse]
+ESITAG		[shape=ellipse]
+ESIETAG		[shape=ellipse]
+ESIINCLUDE	[shape=ellipse]
+ESIREMOVE	[shape=ellipse]
+ESICOMMENT	[shape=ellipse]
+ESIBOGON	[shape=ellipse]
+INTAGc		[shape=hexagon, label="INTAG"]
+NOTMYTAGc	[shape=hexagon, label="NOTMYTAG"]
+NEXTTAGc	[shape=hexagon, label="NEXTTAG"]
+TAGERRORc	[shape=hexagon, label="TAGERROR"]
+C1		[shape=circle,label=""]
+STARTTAG	-> COMMENT	[label="'<!--'"]
+STARTTAG	-> ESIETAG	[label="'</esi'"]
+STARTTAG	-> ESITAG	[label="'<esi'"]
+STARTTAG	-> CDATA	[label="'<![CDATA['"]
+STARTTAG	-> NOTMYTAGc	[label="'*'"]
+COMMENT		-> NEXTTAGc	[label="'esi'"]
+COMMENT		-> C1		[label="*"]
+C1		-> C1		[label="*"]
+C1		-> NEXTTAGc	[label="-->"]
+CDATA		-> CDATA	[label="*"]
+CDATA		-> NEXTTAGc	[label="]]>"]
+ESITAG		-> ESIINCLUDE	[label="'include'"]
+ESITAG		-> ESIREMOVE	[label="'remove'"]
+ESITAG		-> ESICOMMENT	[label="'comment'"]
+ESITAG		-> ESIBOGON	[label="*"]
+ESITAG		-> NOTMYTAGc	[style=dotted, label="nested\nin\nremove"]
+ESIETAG		-> ESIREMOVE	[label="'remove'"]
+ESIETAG		-> ESIBOGON	[label="*"]
+ESICOMMENT	-> INTAGc
+ESIREMOVE	-> INTAGc
+ESIINCLUDE	-> INTAGc
+ESIBOGON	-> TAGERRORc
+
+#################################################################
+# SECTION D
+
+INTAG		[shape=ellipse]
+TAGERROR	[shape=ellipse]
+NEXTTAGd	[shape=hexagon, label="NEXTTAG"]
+ATTRd		[shape=hexagon, label="ATTR"]
+D1		[shape=circle, label=""]
+D2		[shape=circle, label=""]
+INTAG		-> D1		[label="lws"]
+D1		-> D2		[label="/"]
+INTAG		-> D2		[label="/"]
+INTAG		-> NEXTTAGd	[label=">"]
+D1		-> NEXTTAGd	[label=">"]
+D2		-> NEXTTAGd	[label=">"]
+D1		-> ATTRd	[label="XMLstartchar"]
+D1		-> TAGERROR	[label="*"]
+D2		-> TAGERROR	[label="*"]
+TAGERROR	-> TAGERROR	[label="*"]
+TAGERROR	-> NEXTTAGd	[label="'>'"]
+
+#################################################################
+# SECTION E
+
+ATTR		[shape=ellipse]
+SKIPATTR	[shape=ellipse]
+SKIPATTR2	[shape=ellipse]
+ATTRGETVAL	[shape=ellipse]
+ATTRDELIM	[shape=ellipse]
+ATTRVAL		[shape=ellipse]
+TAGERRORe	[shape=hexagon, label="TAGERROR"]
+INTAGe		[shape=hexagon, label="INTAG"]
+ATTR		-> SKIPATTR	[label="*"]
+ATTR		-> ATTRGETVAL	[label="wanted attr"]
+SKIPATTR	-> SKIPATTR	[label="XMLname"]
+SKIPATTR	-> ATTRDELIM	[label="'='"]
+SKIPATTR	-> SKIPATTR2	[label="(when tag exhaused)"]
+SKIPATTR	-> TAGERRORe	[label="*"]
+SKIPATTR2	-> SKIPATTR2	[label="XMLname"]
+SKIPATTR2	-> ATTRDELIM	[label="'='"]
+SKIPATTR2	-> TAGERRORe	[label="*"]
+ATTRGETVAL	-> ATTRDELIM
+ATTRDELIM	-> ATTRVAL	[label="\""]
+ATTRDELIM	-> ATTRVAL	[label="\'"]
+ATTRDELIM	-> ATTRVAL	[label="*"]
+ATTRDELIM	-> TAGERRORe	[label="lws"]
+ATTRVAL		-> TAGERRORe	[label="'>'"]
+ATTRVAL		-> INTAGe	[label="delim"]
+ATTRVAL		-> ATTRVAL	[label="*"]
+
+}
+
+#endif




From phk at varnish-cache.org  Fri Jan 14 12:06:32 2011
From: phk at varnish-cache.org (phk at varnish-cache.org)
Date: Fri, 14 Jan 2011 13:06:32 +0100
Subject: r5746 - trunk/varnish-cache/bin/varnishd
Message-ID: <E1PdiQK-0002fd-M9@project.varnish-software.com>

Author: phk
Date: 2011-01-14 13:06:32 +0100 (Fri, 14 Jan 2011)
New Revision: 5746

Modified:
   trunk/varnish-cache/bin/varnishd/cache.h
Log:
Throw the switch and use the new ESI code.

If it gives you trouble, I would appreciate if you can test both
before and after this commit, to see what the difference is.

The old ESI code will be removed before 3.0 is released.




Modified: trunk/varnish-cache/bin/varnishd/cache.h
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache.h	2011-01-14 11:43:27 UTC (rev 5745)
+++ trunk/varnish-cache/bin/varnishd/cache.h	2011-01-14 12:06:32 UTC (rev 5746)
@@ -35,7 +35,7 @@
  */
 #define VARNISH_CACHE_CHILD	1
 
-#define OLD_ESI 
+#undef OLD_ESI 
 
 #include <sys/time.h>
 #include <sys/uio.h>




From phk at varnish-cache.org  Fri Jan 14 12:51:10 2011
From: phk at varnish-cache.org (phk at varnish-cache.org)
Date: Fri, 14 Jan 2011 13:51:10 +0100
Subject: r5747 - trunk/varnish-cache/bin/varnishd
Message-ID: <E1Pdj7W-0003xd-BI@project.varnish-software.com>

Author: phk
Date: 2011-01-14 13:51:07 +0100 (Fri, 14 Jan 2011)
New Revision: 5747

Modified:
   trunk/varnish-cache/bin/varnishd/cache_esi_parse.c
   trunk/varnish-cache/bin/varnishd/mgt_param.c
Log:
Use esi_syntax bit 0x8 to control input block-length fuzzing for debugging
purposes.



Modified: trunk/varnish-cache/bin/varnishd/cache_esi_parse.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_esi_parse.c	2011-01-14 12:06:32 UTC (rev 5746)
+++ trunk/varnish-cache/bin/varnishd/cache_esi_parse.c	2011-01-14 12:51:07 UTC (rev 5747)
@@ -890,19 +890,17 @@
 		if (vep->hack_p == NULL)
 			vep->hack_p = (const char *)st->ptr + st->len;
 		vep->ver_p = (const char *)st->ptr + st->len;
-#if 0
-		{
-		ssize_t d;
-		for (l = 0; l < w; l += d)  {
-			d = 1;
-			if (l + d >= w)
-				d = 1;
-			vep_parse(vep, (const char *)st->ptr + st->len + l, d);
-		}
-		}
-#else
-		vep_parse(vep, (const char *)st->ptr + st->len, w);
-#endif
+		if (params->esi_syntax & 0x8) {
+			ssize_t d;
+			for (l = 0; l < w; l += d)  {
+				d = (random() & 3) + 1;
+				if (l + d >= w)
+					d = 1;
+				vep_parse(vep,
+				    (const char *)st->ptr + st->len + l, d);
+			}
+		} else
+			vep_parse(vep, (const char *)st->ptr + st->len, w);
 		st->len += w;
 		sp->obj->len += w;
 		if (st->len == st->space) {

Modified: trunk/varnish-cache/bin/varnishd/mgt_param.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/mgt_param.c	2011-01-14 12:06:32 UTC (rev 5746)
+++ trunk/varnish-cache/bin/varnishd/mgt_param.c	2011-01-14 12:51:07 UTC (rev 5747)
@@ -643,6 +643,7 @@
 		"  0x00000001 - Don't check if it looks like XML\n"
 		"  0x00000002 - Ignore non-esi elements\n"
 		"  0x00000004 - Emit parsing debug records\n"
+		"  0x00000008 - Force-dplit parser input (debugging)\n"
 		"Use 0x notation and do the bitor in your head :-)\n",
 		0,
 		"0", "bitmap" },




From phk at varnish-cache.org  Fri Jan 14 12:51:35 2011
From: phk at varnish-cache.org (phk at varnish-cache.org)
Date: Fri, 14 Jan 2011 13:51:35 +0100
Subject: r5748 - trunk/varnish-cache/bin/varnishtest/tests
Message-ID: <E1Pdj7v-000402-R6@project.varnish-software.com>

Author: phk
Date: 2011-01-14 13:51:33 +0100 (Fri, 14 Jan 2011)
New Revision: 5748

Modified:
   trunk/varnish-cache/bin/varnishtest/tests/e00008.vtc
Log:
Fuzz the input blocksize to the ESI-parser on this test



Modified: trunk/varnish-cache/bin/varnishtest/tests/e00008.vtc
===================================================================
--- trunk/varnish-cache/bin/varnishtest/tests/e00008.vtc	2011-01-14 12:51:07 UTC (rev 5747)
+++ trunk/varnish-cache/bin/varnishtest/tests/e00008.vtc	2011-01-14 12:51:33 UTC (rev 5748)
@@ -64,7 +64,7 @@
 	}
 } -start 
 
-varnish v1 -cliok "param.set esi_syntax 6" 
+varnish v1 -cliok "param.set esi_syntax 0xe" 
 
 varnish v1 -cliok "param.set diag_bitmap 0x10000"
 




From phk at varnish-cache.org  Fri Jan 14 14:22:59 2011
From: phk at varnish-cache.org (phk at varnish-cache.org)
Date: Fri, 14 Jan 2011 15:22:59 +0100
Subject: r5749 - trunk/varnish-cache/include
Message-ID: <E1PdkYN-0002Ui-PG@project.varnish-software.com>

Author: phk
Date: 2011-01-14 15:22:59 +0100 (Fri, 14 Jan 2011)
New Revision: 5749

Modified:
   trunk/varnish-cache/include/vsc_fields.h
Log:
Add a counter for ESI warnings



Modified: trunk/varnish-cache/include/vsc_fields.h
===================================================================
--- trunk/varnish-cache/include/vsc_fields.h	2011-01-14 12:51:33 UTC (rev 5748)
+++ trunk/varnish-cache/include/vsc_fields.h	2011-01-14 14:22:59 UTC (rev 5749)
@@ -144,6 +144,7 @@
 
 VSC_F(esi_parse,		uint64_t, 0, 'a', "Objects ESI parsed (unlock)")
 VSC_F(esi_errors,		uint64_t, 0, 'a', "ESI parse errors (unlock)")
+VSC_F(esi_warnings,		uint64_t, 0, 'a', "ESI parse warnings (unlock)")
 VSC_F(accept_fail,		uint64_t, 0, 'a', "Accept failures")
 VSC_F(client_drop_late,	uint64_t, 0, 'a', "Connection dropped late")
 VSC_F(uptime,		uint64_t, 0, 'a', "Client uptime")




From phk at varnish-cache.org  Fri Jan 14 14:42:09 2011
From: phk at varnish-cache.org (phk at varnish-cache.org)
Date: Fri, 14 Jan 2011 15:42:09 +0100
Subject: r5750 - in trunk/varnish-cache/bin: varnishd varnishtest
	varnishtest/tests
Message-ID: <E1Pdkqv-000343-3s@project.varnish-software.com>

Author: phk
Date: 2011-01-14 15:42:08 +0100 (Fri, 14 Jan 2011)
New Revision: 5750

Added:
   trunk/varnish-cache/bin/varnishtest/tests/e00019.vtc
Modified:
   trunk/varnish-cache/bin/varnishd/cache_esi_deliver.c
   trunk/varnish-cache/bin/varnishd/cache_esi_parse.c
   trunk/varnish-cache/bin/varnishtest/tests/e00001.vtc
   trunk/varnish-cache/bin/varnishtest/tests/e00008.vtc
   trunk/varnish-cache/bin/varnishtest/vtc_main.c
Log:
Improve error messages for problems found during ESI parsing.

Split diagnostics into errors and warnings.  Warnings are
issued when the intent is clear and we can carry it out,
everything else is an error.

Collapse tag/endtag procssing, it's simpler.

Various code cleanups.




Modified: trunk/varnish-cache/bin/varnishd/cache_esi_deliver.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_esi_deliver.c	2011-01-14 14:22:59 UTC (rev 5749)
+++ trunk/varnish-cache/bin/varnishd/cache_esi_deliver.c	2011-01-14 14:42:08 UTC (rev 5750)
@@ -130,6 +130,7 @@
 {
 
 	Debug("VER(%d) %d\n", (int)l, (int)(cb-ce));
+	assert(l > 0);
 	if (sp->wrk->res_mode & RES_CHUNKED)
 		(void)WRW_Write(sp->wrk, cb, cl);
 	(void)WRW_Write(sp->wrk, ptr, l);

Modified: trunk/varnish-cache/bin/varnishd/cache_esi_parse.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_esi_parse.c	2011-01-14 14:22:59 UTC (rev 5749)
+++ trunk/varnish-cache/bin/varnishd/cache_esi_parse.c	2011-01-14 14:42:08 UTC (rev 5750)
@@ -72,7 +72,6 @@
 	unsigned		canattr;
 
 	unsigned		remove;
-	unsigned		skip;
 
 	unsigned		o_verbatim;
 	unsigned		o_skip;
@@ -88,12 +87,10 @@
 	const char		*esicmt_p;
 
 	struct vep_match	*attr;
-	int			attr_l;
 	struct vsb		*attr_vsb;
 	int			attr_delim;
 
 	struct vep_match	*match;
-	int			match_l;
 	struct vep_match	*match_hit;
 
 	char			tag[10];
@@ -117,7 +114,6 @@
 static const char * const VEP_COMMENT =		"[Comment]";
 static const char * const VEP_CDATA =		"[CDATA]";
 static const char * const VEP_ESITAG =		"[ESITag]";
-static const char * const VEP_ESIETAG =		"[ESIEndTag]";
 
 static const char * const VEP_ESIREMOVE =	"[ESI:Remove]";
 static const char * const VEP_ESIINCLUDE =	"[ESI:Include]";
@@ -132,7 +128,7 @@
 static const char * const VEP_SKIPATTR2 =	"[SkipAttribute2]";
 static const char * const VEP_ATTRDELIM =	"[AttrDelim]";
 static const char * const VEP_ATTRGETVAL =	"[AttrGetValue]";
-static const char * const VEP_ATTRVAL =	"[AttrValue]";
+static const char * const VEP_ATTRVAL =		"[AttrValue]";
 
 static const char * const VEP_UNTIL =		"[Until]";
 static const char * const VEP_MATCHBUF = 	"[MatchBuf]";
@@ -141,16 +137,12 @@
 /*---------------------------------------------------------------------*/
 
 static struct vep_match vep_match_starttag[] = {
-	{ "<!--",	&VEP_COMMENT },
-	{ "</esi:",	&VEP_ESIETAG },
-	{ "<esi:",	&VEP_ESITAG },
-	{ "<![CDATA[",	&VEP_CDATA },
+	{ "!--",	&VEP_COMMENT },
+	{ "esi:",	&VEP_ESITAG },
+	{ "![CDATA[",	&VEP_CDATA },
 	{ NULL,		&VEP_NOTMYTAG }
 };
 
-static const int vep_match_starttag_len =
-    sizeof vep_match_starttag / sizeof vep_match_starttag[0];
-
 /*---------------------------------------------------------------------*/
 
 static struct vep_match vep_match_esi[] = {
@@ -160,29 +152,13 @@
 	{ NULL,		&VEP_ESIBOGON }
 };
 
-static const int vep_match_esi_len =
-    sizeof vep_match_esi / sizeof vep_match_esi[0];
-
 /*---------------------------------------------------------------------*/
 
-static struct vep_match vep_match_esie[] = {
-	{ "remove",	&VEP_ESIREMOVE },
-	{ NULL,		&VEP_ESIBOGON }
-};
-
-static const int vep_match_esie_len =
-    sizeof vep_match_esie / sizeof vep_match_esie[0];
-
-/*---------------------------------------------------------------------*/
-
 static struct vep_match vep_match_attr_include[] = {
 	{ "src=",	&VEP_ATTRGETVAL },
 	{ NULL,		&VEP_SKIPATTR }
 };
 
-static const int vep_match_attr_include_len =
-    sizeof vep_match_attr_include / sizeof vep_match_attr_include[0];
-
 /*--------------------------------------------------------------------
  * Report a parsing error
  */
@@ -190,13 +166,31 @@
 static void
 vep_error(const struct vep_state *vep, const char *p)
 {
+	intmax_t l;
 
 	VSC_main->esi_errors++;
-	WSP(vep->sp, SLT_ESI_xmlerror, "at %ld %s",
-	    (vep->ver_p - vep->hack_p), p);
+	l = (intmax_t)(vep->ver_p - vep->hack_p);
+	printf("ERROR at %jd %s\n", l , p);
+	WSP(vep->sp, SLT_ESI_xmlerror, "ERR at %jd %s", l, p);
 	
 }
 
+/*--------------------------------------------------------------------
+ * Report a parsing warning
+ */
+
+static void
+vep_warn(const struct vep_state *vep, const char *p)
+{
+	intmax_t l;
+
+	VSC_main->esi_warnings++;
+	l = (intmax_t)(vep->ver_p - vep->hack_p);
+	printf("WARNING at %jd %s\n", l, p);
+	WSP(vep->sp, SLT_ESI_xmlerror, "WARN at %jd %s", l, p);
+	
+}
+
 /*---------------------------------------------------------------------
  * return match or NULL if more input needed.
  */
@@ -320,18 +314,9 @@
 vep_do_comment(struct vep_state *vep, enum dowhat what)
 {
 	Debug("DO_COMMENT(%d)\n", what);
-	if (what == DO_ATTR) {
-		Debug("ATTR (%s) (%s)\n", vep->match_hit->match,
-			vsb_data(vep->attr_vsb));
-		vsb_delete(vep->attr_vsb);
-	} else {
-		if (vep->endtag)
-			vep_error(vep,
-			    "ESI 1.0 </esi:comment> not allowed");
-		else if (!vep->emptytag) 
-			vep_error(vep,
-			    "ESI 1.0 <esi:comment> not allowed");
-	}
+	assert(what == DO_TAG);
+	if (!vep->emptytag) 
+		vep_error(vep, "ESI 1.0 <esi:comment> needs final '/'");
 }
 
 /*---------------------------------------------------------------------
@@ -340,20 +325,13 @@
 static void __match_proto__()
 vep_do_remove(struct vep_state *vep, enum dowhat what)
 {
-	Debug("DO_REMOVE(%d)\n", what);
-	if (what == DO_ATTR) {
-		Debug("ATTR (%s) (%s)\n", vep->match_hit->match,
-			vsb_data(vep->attr_vsb));
-		vsb_delete(vep->attr_vsb);
+	Debug("DO_REMOVE(%d, end %d empty %d remove %d)\n",
+	    what, vep->endtag, vep->emptytag, vep->remove);
+	assert(what == DO_TAG);
+	if (vep->emptytag) {
+		vep_error(vep,
+		    "ESI 1.0 <esi:remove/> not legal");
 	} else {
-		if (vep->emptytag)
-#if 0
-			vep_error(vep,
-			    "ESI 1.0 <esi:remove/> not legal");
-#else
-			;
-#endif
-		else
 		if (vep->remove && !vep->endtag)
 			vep_error(vep,
 			    "ESI 1.0 <esi:remove> already open");
@@ -381,70 +359,74 @@
 			vsb_data(vep->attr_vsb));
 		XXXAZ(vep->include_src);	/* multiple src= */
 		vep->include_src = vep->attr_vsb;
+		return;
+	}
+	assert(what == DO_TAG);
+	if (!vep->emptytag) 
+		vep_warn(vep,
+		    "ESI 1.0 <esi:include> lacks final '/'");
+	if (vep->include_src == NULL) {
+		vep_error(vep,
+		    "ESI 1.0 <esi:include> lacks src attr");
+		return;
+	}
+	XXXAN(vep->include_src);
+	if (vep->o_skip > 0) 
+		vep_emit_skip(vep);
+	if (vep->o_verbatim > 0) 
+		vep_emit_verbatim(vep);
+	/* XXX: what if it contains NUL bytes ?? */
+	p = vsb_data(vep->include_src);
+	l = vsb_len(vep->include_src);
+	h = 0;
+
+	vsb_printf(vep->vsb, "%c", VEC_INCL);
+	if (l > 7 && !memcmp(p, "http://", 7)) {
+		h = p + 7;
+		p = strchr(h, '/');
+		AN(p);
+		Debug("HOST <%.*s> PATH <%s>\n", (int)(p-h),h, p);
+		vsb_printf(vep->vsb, "Host: %.*s%c",
+		    (int)(p-h), h, 0);
+	} else if (*p == '/') {
+		vsb_printf(vep->vsb, "%c", 0);
 	} else {
-		if (vep->include_src == NULL) {
-			vep_error(vep,
-			    "ESI 1.0 <esi:include> lacks src attr");
-			return;
-		}
-		XXXAN(vep->include_src);
-		if (vep->o_skip > 0) 
-			vep_emit_skip(vep);
-		if (vep->o_verbatim > 0) 
-			vep_emit_verbatim(vep);
-		/* XXX: what if it contains NUL bytes ?? */
-		p = vsb_data(vep->include_src);
-		l = vsb_len(vep->include_src);
-		h = 0;
-
-		vsb_printf(vep->vsb, "%c", VEC_INCL);
-		if (l > 7 && !memcmp(p, "http://", 7)) {
-			h = p + 7;
-			p = strchr(h, '/');
-			AN(p);
-			Debug("HOST <%.*s> PATH <%s>\n", (int)(p-h),h, p);
-			vsb_printf(vep->vsb, "Host: %.*s%c",
-			    (int)(p-h), h, 0);
-		} else if (*p == '/') {
-			vsb_printf(vep->vsb, "%c", 0);
-		} else {
-			vsb_printf(vep->vsb, "%c", 0);
-			url = vep->sp->wrk->bereq->hd[HTTP_HDR_URL];
-			/* Look for the last / before a '?' */
-			h = NULL;
-			for (q = url.b; q < url.e && *q != '?'; q++)
-				if (*q == '/')
-					h = q;
-			if (h == NULL)
-				h = q + 1;
-				
-			Debug("INCL:: [%.*s]/[%s]\n",
-			    (int)(h - url.b), url.b, p);
-			vsb_printf(vep->vsb, "%.*s/", (int)(h - url.b), url.b);
-		}
-		l -= (p - vsb_data(vep->include_src));
-		for (q = p; *q != '\0'; ) {
-			if (*q == '&') {
+		vsb_printf(vep->vsb, "%c", 0);
+		url = vep->sp->wrk->bereq->hd[HTTP_HDR_URL];
+		/* Look for the last / before a '?' */
+		h = NULL;
+		for (q = url.b; q < url.e && *q != '?'; q++)
+			if (*q == '/')
+				h = q;
+		if (h == NULL)
+			h = q + 1;
+			
+		Debug("INCL:: [%.*s]/[%s]\n",
+		    (int)(h - url.b), url.b, p);
+		vsb_printf(vep->vsb, "%.*s/", (int)(h - url.b), url.b);
+	}
+	l -= (p - vsb_data(vep->include_src));
+	for (q = p; *q != '\0'; ) {
+		if (*q == '&') {
 #define R(w,f,r)							\
-				if (q + w <= p + l && !memcmp(q, f, w)) { \
-					vsb_printf(vep->vsb, "%c", r);	\
-					q += l;				\
-					continue;			\
-				}
-				R(6, "&apos;", '\'');
-				R(6, "&quot;", '"');
-				R(4, "&lt;", '<');
-				R(4, "&gt;", '>');
-				R(5, "&amp;", '&');
+			if (q + w <= p + l && !memcmp(q, f, w)) { \
+				vsb_printf(vep->vsb, "%c", r);	\
+				q += l;				\
+				continue;			\
 			}
-			vsb_printf(vep->vsb, "%c", *q++);
+			R(6, "&apos;", '\'');
+			R(6, "&quot;", '"');
+			R(4, "&lt;", '<');
+			R(4, "&gt;", '>');
+			R(5, "&amp;", '&');
 		}
+		vsb_printf(vep->vsb, "%c", *q++);
+	}
 #undef R
-		vsb_printf(vep->vsb, "%c", 0);
+	vsb_printf(vep->vsb, "%c", 0);
 
-		vsb_delete(vep->include_src);
-		vep->include_src = NULL;
-	}
+	vsb_delete(vep->include_src);
+	vep->include_src = NULL;
 }
 
 /*---------------------------------------------------------------------
@@ -475,9 +457,8 @@
 		i = e - p;
 		if (i > 10)
 			i = 10;
-		Debug("EP %s %d %d (%.*s) [%.*s]\n",
+		Debug("EP %s %d (%.*s) [%.*s]\n",
 		    vep->state,
-		    vep->skip,
 		    vep->remove,
 		    vep->tag_i, vep->tag,
 		    i, p);
@@ -501,6 +482,9 @@
 				vep_mark_verbatim(vep, p);
 			}
 			if (p < e && *p == '<') {
+				if (!vep->remove)
+					vep_mark_verbatim(vep, p);
+				p++;
 				vep->state = VEP_STARTTAG;
 			} else if (p < e) {
 				WSP(vep->sp, SLT_ESI_xmlerror,
@@ -542,6 +526,7 @@
 			 */
 			vep->emptytag = 0;
 			vep->endtag = 0;
+			vep->attr = NULL;
 			while (p < e && *p != '<') {
 				if (vep->esicmt_p != NULL &&
 				    *p == *vep->esicmt_p++) {
@@ -566,8 +551,13 @@
 			}
 			if (p < e && vep->esicmt_p == NULL && !vep->remove)
 				vep_mark_verbatim(vep, p);
-			if (p < e)
+			if (p < e) {
+				assert(*p == '<');
+				if (!vep->remove)
+					vep_mark_verbatim(vep, p);
+				p++;
 				vep->state = VEP_STARTTAG;
+			}
 
 		/******************************************************
 		 * SECTION C
@@ -577,12 +567,14 @@
 			/*
 			 * Start of tag, set up match table
 			 */
-			assert(*p == '<');
-			if (!vep->remove)
-				vep_mark_verbatim(vep, p);
-			vep->match = vep_match_starttag;
-			vep->match_l = vep_match_starttag_len;
-			vep->state = VEP_MATCH;
+			if (p < e) {
+				if (*p == '/') {
+					vep->endtag = 1;
+					p++;
+				} 
+				vep->match = vep_match_starttag;
+				vep->state = VEP_MATCH;
+			}
 		} else if (vep->state == VEP_COMMENT) {
 			/*
 			 * We are in a comment, find out if it is an
@@ -591,22 +583,24 @@
 			if (vep->esicmt == NULL)
 				vep->esicmt_p = vep->esicmt = "esi";
 			while (p < e) {
-				if (*p == *vep->esicmt_p) {
-					p++;
-					if (*++vep->esicmt_p == '\0') {
-						vep->esicmt_p =
-						    vep->esicmt = "-->";
-						vep->state = VEP_NEXTTAG;
-						vep_mark_skip(vep, p);
-						break;
-					}
-				} else {
+				if (*p != *vep->esicmt_p) {
 					vep->esicmt_p = vep->esicmt = NULL; 
 					vep->until_p = vep->until = "-->";
 					vep->until_s = VEP_NEXTTAG;
 					vep->state = VEP_UNTIL;
 					break;
 				}
+				p++;
+				if (*++vep->esicmt_p != '\0')
+					continue;
+				if (vep->remove)
+					vep_error(vep,
+					    "ESI 1.0 Nested <!--esi"
+					    " element in <esi:remove>");
+				vep->esicmt_p = vep->esicmt = "-->";
+				vep->state = VEP_NEXTTAG;
+				vep_mark_skip(vep, p);
+				break;
 			}
 		} else if (vep->state == VEP_CDATA) {
 			/*
@@ -616,37 +610,43 @@
 			vep->until_s = VEP_NEXTTAG;
 			vep->state = VEP_UNTIL;
 		} else if (vep->state == VEP_ESITAG) {
-			vep->tag_i = 0;
-			vep->endtag = 0;
+			vep_mark_skip(vep, p);
+			vep->match = vep_match_esi;
+			vep->state = VEP_MATCH;
+		} else if (vep->state == VEP_ESIINCLUDE) {
 			if (vep->remove) {
 				vep_mark_skip(vep, p);
 				vep_error(vep,
-				    "ESI 1.0 Nested ESI element in <esi:remove>");
-				vep->state = VEP_NOTMYTAG;
+				    "ESI 1.0 <esi:include> element"
+				    " nested in <esi:remove>");
+				vep->state = VEP_TAGERROR;
+			} else if (vep->endtag) {
+				vep_error(vep,
+				    "ESI 1.0 </esi:include> illegal end-tag");
+				vep->state = VEP_TAGERROR;
 			} else {
-				vep->skip = 1;
-				vep_mark_skip(vep, p);
-				vep->match = vep_match_esi;
-				vep->match_l = vep_match_esi_len;
-				vep->state = VEP_MATCH;
+				vep->dostuff = vep_do_include;
+				vep->state = VEP_INTAG;
+				vep->attr = vep_match_attr_include;
 			}
-		} else if (vep->state == VEP_ESIETAG) {
-			vep->tag_i = 0;
-			vep->endtag = 1;
-			vep->match = vep_match_esie;
-			vep->match_l = vep_match_esie_len;
-			vep->state = VEP_MATCH;
-		} else if (vep->state == VEP_ESIINCLUDE) {
-			vep->dostuff = vep_do_include;
-			vep->state = VEP_INTAG;
-			vep->attr = vep_match_attr_include;
-			vep->attr_l = vep_match_attr_include_len;
 		} else if (vep->state == VEP_ESIREMOVE) {
 			vep->dostuff = vep_do_remove;
 			vep->state = VEP_INTAG;
 		} else if (vep->state == VEP_ESICOMMENT) {
-			vep->dostuff = vep_do_comment;
-			vep->state = VEP_INTAG;
+			if (vep->remove) {
+				vep_mark_skip(vep, p);
+				vep_error(vep,
+				    "ESI 1.0 <esi:comment> element"
+				    " nested in <esi:remove>");
+				vep->state = VEP_TAGERROR;
+			} else if (vep->endtag) {
+				vep_error(vep,
+				    "ESI 1.0 </esi:comment> illegal end-tag");
+				vep->state = VEP_TAGERROR;
+			} else {
+				vep->dostuff = vep_do_comment;
+				vep->state = VEP_INTAG;
+			}
 		} else if (vep->state == VEP_ESIBOGON) {
 			vep_mark_skip(vep, p);
 			vep_error(vep,
@@ -660,7 +660,7 @@
 		} else if (vep->state == VEP_INTAG) {
 			vep->tag_i = 0;
 			vep_mark_skip(vep, p);
-			while (p < e && vct_islws(*p)) {
+			while (p < e && vct_islws(*p) && !vep->emptytag) {
 				p++;	
 				vep->canattr = 1;
 			}
@@ -671,7 +671,6 @@
 			}
 			if (p < e && *p == '>') {
 				p++;
-				vep_mark_skip(vep, p);
 				AN(vep->dostuff);
 				vep_mark_skip(vep, p);
 				vep->dostuff(vep, DO_TAG);
@@ -707,11 +706,10 @@
 				p++;
 				AZ(vep->attr_vsb);
 				vep->state = VEP_SKIPATTR;
-				break;
+			} else {
+				vep->match = vep->attr;
+				vep->state = VEP_MATCH;
 			}
-			vep->match = vep->attr;
-			vep->match_l = vep->attr_l;
-			vep->state = VEP_MATCH;
 		} else if (vep->state == VEP_SKIPATTR) {
 			vep->state = VEP_SKIPATTR2;
 			for (i = 0; i < vep->tag_i; i++) {
@@ -722,7 +720,7 @@
 					vep->state = VEP_ATTRDELIM;
 				} else {
 					vep_error(vep,
-					    "XML 1.0 Illegal attr char");
+					    "XML 1.0 Illegal attr char (_i)");
 					vep->state = VEP_TAGERROR;
 				}
 			}
@@ -734,6 +732,10 @@
 			if (p < e && *p == '=') {
 				p++;
 				vep->state = VEP_ATTRDELIM;
+			} else if (p < e && *p == '>') {
+				vep->state = VEP_INTAG;
+			} else if (p < e && *p == '/') {
+				vep->state = VEP_INTAG;
 			} else if (p < e && vct_issp(*p)) {
 				vep->state = VEP_INTAG;
 			} else if (p < e) {
@@ -771,7 +773,6 @@
 				vep_mark_skip(vep, p);
 				vep_error(vep,
 				    "XML 1.0 Missing end attribute delimiter");
-				Debug("ERR %d %c %c\n", __LINE__, *p, vep->attr_delim);
 				vep->state = VEP_TAGERROR;
 				vep->attr_delim = 0;
 				if (vep->attr_vsb != NULL) {
@@ -822,11 +823,17 @@
 			 */
 			do {
 				if (*p == '>') {
-					vm = vep->match+vep->match_l-1;
+					for (vm = vep->match;
+					    vm->match != NULL; vm++)
+						break;
 				} else {
 					vep->tag[vep->tag_i++] = *p++;
 					vm = vep_match(vep,
 					    vep->tag, vep->tag + vep->tag_i);
+					if (vm && vm->match == NULL) {
+						vep->tag_i--;
+						p--;
+					}
 				}
 			} while (vm == NULL && p < e);
 			vep->match_hit = vm;
@@ -1066,7 +1073,6 @@
 TAGERRORc	[shape=hexagon, label="TAGERROR"]
 C1		[shape=circle,label=""]
 STARTTAG	-> COMMENT	[label="'<!--'"]
-STARTTAG	-> ESIETAG	[label="'</esi'"]
 STARTTAG	-> ESITAG	[label="'<esi'"]
 STARTTAG	-> CDATA	[label="'<![CDATA['"]
 STARTTAG	-> NOTMYTAGc	[label="'*'"]
@@ -1080,12 +1086,14 @@
 ESITAG		-> ESIREMOVE	[label="'remove'"]
 ESITAG		-> ESICOMMENT	[label="'comment'"]
 ESITAG		-> ESIBOGON	[label="*"]
-ESITAG		-> NOTMYTAGc	[style=dotted, label="nested\nin\nremove"]
-ESIETAG		-> ESIREMOVE	[label="'remove'"]
-ESIETAG		-> ESIBOGON	[label="*"]
 ESICOMMENT	-> INTAGc
+ESICOMMENT	-> TAGERRORc
+ESICOMMENT	-> TAGERRORc	[style=dotted, label="nested\nin\nremove"]
 ESIREMOVE	-> INTAGc
+ESIREMOVE	-> TAGERRORc
 ESIINCLUDE	-> INTAGc
+ESIINCLUDE	-> TAGERRORc
+ESIINCLUDE	-> TAGERRORc	[style=dotted, label="nested\nin\nremove"]
 ESIBOGON	-> TAGERRORc
 
 #################################################################

Modified: trunk/varnish-cache/bin/varnishtest/tests/e00001.vtc
===================================================================
--- trunk/varnish-cache/bin/varnishtest/tests/e00001.vtc	2011-01-14 14:22:59 UTC (rev 5749)
+++ trunk/varnish-cache/bin/varnishtest/tests/e00001.vtc	2011-01-14 14:42:08 UTC (rev 5750)
@@ -29,4 +29,4 @@
 }
 
 client c1 -run
-varnish v1 -expect esi_errors == 1
+varnish v1 -expect esi_errors == 2

Modified: trunk/varnish-cache/bin/varnishtest/tests/e00008.vtc
===================================================================
--- trunk/varnish-cache/bin/varnishtest/tests/e00008.vtc	2011-01-14 14:22:59 UTC (rev 5749)
+++ trunk/varnish-cache/bin/varnishtest/tests/e00008.vtc	2011-01-14 14:42:08 UTC (rev 5750)
@@ -76,4 +76,4 @@
 }
 
 client c1 -run
-varnish v1 -expect esi_errors == 14
+varnish v1 -expect esi_errors == 15

Added: trunk/varnish-cache/bin/varnishtest/tests/e00019.vtc
===================================================================
--- trunk/varnish-cache/bin/varnishtest/tests/e00019.vtc	                        (rev 0)
+++ trunk/varnish-cache/bin/varnishtest/tests/e00019.vtc	2011-01-14 14:42:08 UTC (rev 5750)
@@ -0,0 +1,36 @@
+# $Id$
+
+test "Push corners in new ESI parser"
+
+server s1 {
+	rxreq 
+	txresp -body {
+		</esi:comment foo>
+		<esi:comment / >
+		<esi:remove foo="bar"></esi:remove>
+		<esi:include src="foo">
+	}
+	rxreq
+	expect req.url == "bar/foo"
+	txresp -body {<H1></H1>}
+} -start
+
+varnish v1 -vcl+backend {
+	sub vcl_fetch {
+		set beresp.do_esi = true;
+	}
+} -start 
+
+varnish v1 -cliok "param.set esi_syntax 8" 
+
+varnish v1 -cliok "param.set diag_bitmap 0x10000"
+
+client c1 {
+	txreq  -url bar
+	rxresp
+	expect resp.status == 200
+	expect resp.bodylen == 23
+} -run
+
+varnish v1 -expect esi_errors == 2
+varnish v1 -expect esi_warnings == 1

Modified: trunk/varnish-cache/bin/varnishtest/vtc_main.c
===================================================================
--- trunk/varnish-cache/bin/varnishtest/vtc_main.c	2011-01-14 14:22:59 UTC (rev 5749)
+++ trunk/varnish-cache/bin/varnishtest/vtc_main.c	2011-01-14 14:42:08 UTC (rev 5750)
@@ -237,7 +237,7 @@
 	ALLOC_OBJ(jp, JOB_MAGIC);
 	AN(jp);
 
-	jp->bufsiz = 64*1024;		/* XXX */
+	jp->bufsiz = 256*1024;		/* XXX */
 
 	jp->buf = mmap(NULL, jp->bufsiz, PROT_READ|PROT_WRITE,
 	    MAP_ANON | MAP_SHARED, -1, 0);




From phk at varnish-cache.org  Fri Jan 14 15:41:43 2011
From: phk at varnish-cache.org (phk at varnish-cache.org)
Date: Fri, 14 Jan 2011 16:41:43 +0100
Subject: r5751 - trunk/varnish-cache/bin/varnishd
Message-ID: <E1PdlmZ-0004ef-Iv@project.varnish-software.com>

Author: phk
Date: 2011-01-14 16:41:43 +0100 (Fri, 14 Jan 2011)
New Revision: 5751

Modified:
   trunk/varnish-cache/bin/varnishd/cache_esi_parse.c
Log:
The SKIPATTR2 state is not needed.  The ->tag can only contain an
initial substring from one of the attributes we asked for with ->attr,
and those presumably only valid "isxmlname" characters in them.



Modified: trunk/varnish-cache/bin/varnishd/cache_esi_parse.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_esi_parse.c	2011-01-14 14:42:08 UTC (rev 5750)
+++ trunk/varnish-cache/bin/varnishd/cache_esi_parse.c	2011-01-14 15:41:43 UTC (rev 5751)
@@ -125,7 +125,6 @@
 
 static const char * const VEP_ATTR =		"[Attribute]";
 static const char * const VEP_SKIPATTR =	"[SkipAttribute]";
-static const char * const VEP_SKIPATTR2 =	"[SkipAttribute2]";
 static const char * const VEP_ATTRDELIM =	"[AttrDelim]";
 static const char * const VEP_ATTRGETVAL =	"[AttrGetValue]";
 static const char * const VEP_ATTRVAL =		"[AttrValue]";
@@ -684,7 +683,7 @@
 				vep->state = VEP_ATTR;
 			} else if (p < e) {
 				vep_error(vep,
-				    "XML 1.0 Illegal attribute start char");
+				    "XML 1.0 Illegal attribute tart char");
 				vep->state = VEP_TAGERROR;
 			}
 		} else if (vep->state == VEP_TAGERROR) {
@@ -711,22 +710,6 @@
 				vep->state = VEP_MATCH;
 			}
 		} else if (vep->state == VEP_SKIPATTR) {
-			vep->state = VEP_SKIPATTR2;
-			for (i = 0; i < vep->tag_i; i++) {
-				if (vct_isxmlname(vep->tag[i]))
-					continue;
-				if (vep->tag[i] == '=') {
-					assert(i + 1 == vep->tag_i);
-					vep->state = VEP_ATTRDELIM;
-				} else {
-					vep_error(vep,
-					    "XML 1.0 Illegal attr char (_i)");
-					vep->state = VEP_TAGERROR;
-				}
-			}
-			xxxassert(i == vep->tag_i);
-			vep->tag_i = 0;
-		} else if (vep->state == VEP_SKIPATTR2) {
 			while (p < e && vct_isxmlname(*p))
 				p++;
 			if (p < e && *p == '=') {
@@ -1122,7 +1105,6 @@
 
 ATTR		[shape=ellipse]
 SKIPATTR	[shape=ellipse]
-SKIPATTR2	[shape=ellipse]
 ATTRGETVAL	[shape=ellipse]
 ATTRDELIM	[shape=ellipse]
 ATTRVAL		[shape=ellipse]
@@ -1132,11 +1114,7 @@
 ATTR		-> ATTRGETVAL	[label="wanted attr"]
 SKIPATTR	-> SKIPATTR	[label="XMLname"]
 SKIPATTR	-> ATTRDELIM	[label="'='"]
-SKIPATTR	-> SKIPATTR2	[label="(when tag exhaused)"]
 SKIPATTR	-> TAGERRORe	[label="*"]
-SKIPATTR2	-> SKIPATTR2	[label="XMLname"]
-SKIPATTR2	-> ATTRDELIM	[label="'='"]
-SKIPATTR2	-> TAGERRORe	[label="*"]
 ATTRGETVAL	-> ATTRDELIM
 ATTRDELIM	-> ATTRVAL	[label="\""]
 ATTRDELIM	-> ATTRVAL	[label="\'"]




From phk at varnish-cache.org  Fri Jan 14 19:38:48 2011
From: phk at varnish-cache.org (phk at varnish-cache.org)
Date: Fri, 14 Jan 2011 20:38:48 +0100
Subject: r5752 - in trunk/varnish-cache/bin/varnishtest: . tests
Message-ID: <E1PdpU1-0001S8-TQ@project.varnish-software.com>

Author: phk
Date: 2011-01-14 20:38:13 +0100 (Fri, 14 Jan 2011)
New Revision: 5752

Modified:
   trunk/varnish-cache/bin/varnishtest/tests/r00694.vtc
   trunk/varnish-cache/bin/varnishtest/tests/r00776.vtc
   trunk/varnish-cache/bin/varnishtest/vtc_http.c
Log:
Make the "chunkedlen" word work like "chunked" and not send the
final zero-length chunk automatically, allowing us to easily
generate specifically constructed messages this way.



Modified: trunk/varnish-cache/bin/varnishtest/tests/r00694.vtc
===================================================================
--- trunk/varnish-cache/bin/varnishtest/tests/r00694.vtc	2011-01-14 15:41:43 UTC (rev 5751)
+++ trunk/varnish-cache/bin/varnishtest/tests/r00694.vtc	2011-01-14 19:38:13 UTC (rev 5752)
@@ -9,6 +9,7 @@
 	send "\r\n"
 	# This is chunksize (128k) + 2 to force to chunks to be allocated
 	chunkedlen 131074
+	chunkedlen 0
 } -start
 
 varnish v1 -vcl+backend {

Modified: trunk/varnish-cache/bin/varnishtest/tests/r00776.vtc
===================================================================
--- trunk/varnish-cache/bin/varnishtest/tests/r00776.vtc	2011-01-14 15:41:43 UTC (rev 5751)
+++ trunk/varnish-cache/bin/varnishtest/tests/r00776.vtc	2011-01-14 19:38:13 UTC (rev 5752)
@@ -6,6 +6,7 @@
 	rxreq
 	txresp -nolen -hdr "Transfer-encoding: chunked"
 	chunkedlen 4096
+	chunkedlen 0
 } -start
 
 varnish v1 \

Modified: trunk/varnish-cache/bin/varnishtest/vtc_http.c
===================================================================
--- trunk/varnish-cache/bin/varnishtest/vtc_http.c	2011-01-14 15:41:43 UTC (rev 5751)
+++ trunk/varnish-cache/bin/varnishtest/vtc_http.c	2011-01-14 19:38:13 UTC (rev 5752)
@@ -883,17 +883,21 @@
 
 	len = atoi(av[1]);
 
-	for (u = 0; u < sizeof buf; u++)
-		buf[u] = (u & 7) + '0';
+	if (len == 0) {
+		vsb_printf(hp->vsb, "0%s%s", nl, nl);
+	} else {
+		for (u = 0; u < sizeof buf; u++)
+			buf[u] = (u & 7) + '0';
 
-	for (u = 0; u < len; u += 16384) {
-		v = len - u;
-		if (v > sizeof buf)
-			v = sizeof buf;
-		vsb_printf(hp->vsb, "%x%s", v, nl);
-		vsb_printf(hp->vsb, "%*.*s%s", v, v, buf, nl);
+		vsb_printf(hp->vsb, "%x%s", len, nl);
+		for (u = 0; u < len; u += v) {
+			v = len - u;
+			if (v > sizeof buf)
+				v = sizeof buf;
+			vsb_bcat(hp->vsb, buf, v);
+		}
+		vsb_printf(hp->vsb, "%s", nl);
 	}
-	vsb_printf(hp->vsb, "%x%s%s", 0, nl, nl);
 	http_write(hp, 4, "chunked");
 }
 




From phk at varnish-cache.org  Fri Jan 14 19:45:50 2011
From: phk at varnish-cache.org (phk at varnish-cache.org)
Date: Fri, 14 Jan 2011 20:45:50 +0100
Subject: r5753 - in trunk/varnish-cache/bin: varnishd varnishtest/tests
Message-ID: <E1Pdpao-0001hC-HS@project.varnish-software.com>

Author: phk
Date: 2011-01-14 20:45:48 +0100 (Fri, 14 Jan 2011)
New Revision: 5753

Modified:
   trunk/varnish-cache/bin/varnishd/cache_esi_parse.c
   trunk/varnish-cache/bin/varnishd/mgt_param.c
   trunk/varnish-cache/bin/varnishtest/tests/e00019.vtc
Log:
Fix bugs relating to the space that may or may not be between received
chunks and multiple storage segments.

Along the way add a couple more debug options.



Modified: trunk/varnish-cache/bin/varnishd/cache_esi_parse.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_esi_parse.c	2011-01-14 19:38:13 UTC (rev 5752)
+++ trunk/varnish-cache/bin/varnishd/cache_esi_parse.c	2011-01-14 19:45:48 UTC (rev 5753)
@@ -73,8 +73,10 @@
 
 	unsigned		remove;
 
-	unsigned		o_verbatim;
-	unsigned		o_skip;
+	ssize_t			o_verbatim;
+	ssize_t			o_skip;
+	ssize_t			o_pending;
+	ssize_t			o_total;
 
 const char		*hack_p;
 	const char		*ver_p;
@@ -250,26 +252,30 @@
 } 
 
 static void
-vep_emit_skip(struct vep_state *vep)
+vep_emit_skip(struct vep_state *vep, ssize_t *l)
 {
-	ssize_t l;
 
-	l = vep->o_skip;
-	vep->o_skip = 0;
-	assert(l > 0);
-	vep_emit_len(vep, l, VEC_S1, VEC_S2, VEC_S8);
+	assert(*l > 0);
+	if (params->esi_syntax & 0x20) {
+		Debug("---> SKIP(%jd)\n", (intmax_t)*l);
+	}
+	vep_emit_len(vep, *l, VEC_S1, VEC_S2, VEC_S8);
+	vep->o_total += *l;
+	*l = 0;
 } 
 
 static void
-vep_emit_verbatim(struct vep_state *vep)
+vep_emit_verbatim(struct vep_state *vep, ssize_t *l)
 {
-	ssize_t l;
 
-	l = vep->o_verbatim;
-	vep->o_verbatim = 0;
-	assert(l > 0);
-	vep_emit_len(vep, l, VEC_V1, VEC_V2, VEC_V8);
-	vsb_printf(vep->vsb, "%lx\r\n%c", l, 0);
+	assert(*l > 0);
+	if (params->esi_syntax & 0x20) {
+		Debug("---> VERBATIM(%jd)\n", (intmax_t)*l);
+	}
+	vep_emit_len(vep, *l, VEC_V1, VEC_V2, VEC_V8);
+	vsb_printf(vep->vsb, "%lx\r\n%c", *l, 0);
+	vep->o_total += *l;
+	*l = 0;
 } 
 
 static void
@@ -279,14 +285,20 @@
 
 	AN(vep->ver_p);
 	l = p - vep->ver_p;
-	if (l == 0)
-		return;
-	if (vep->o_skip > 0) 
-		vep_emit_skip(vep);
-	AZ(vep->o_skip);
-	Debug("-->V(%d) [%.*s]\n", (int)l, (int)l, vep->ver_p);
-	vep->o_verbatim += l;
 	vep->ver_p = p;
+	assert(l >= 0);
+	if (params->esi_syntax & 0x10) {
+		if (vep->o_pending)
+			vep_emit_verbatim(vep, &vep->o_pending);
+		if (l)
+			vep_emit_verbatim(vep, &l);
+	} else {
+		if (vep->o_skip > 0) 
+			vep_emit_skip(vep, &vep->o_skip);
+		vep->o_verbatim += vep->o_pending;
+		vep->o_pending = 0;
+		vep->o_verbatim += l;
+	}
 } 
 
 static void
@@ -296,14 +308,20 @@
 
 	AN(vep->ver_p);
 	l = p - vep->ver_p;
-	if (l == 0)
-		return;
-	if (vep->o_verbatim > 0) 
-		vep_emit_verbatim(vep);
-	AZ(vep->o_verbatim);
-	Debug("-->S(%d) [%.*s]\n", (int)l, (int)l, vep->ver_p);
-	vep->o_skip += l;
 	vep->ver_p = p;
+	assert(l >= 0);
+	if (params->esi_syntax & 0x10) {
+		if (vep->o_pending) 
+			vep_emit_skip(vep, &vep->o_pending);
+		if (l)
+			vep_emit_skip(vep, &l);
+	} else {
+		if (vep->o_verbatim > 0) 
+			vep_emit_verbatim(vep, &vep->o_verbatim);
+		vep->o_skip += vep->o_pending;
+		vep->o_pending = 0;
+		vep->o_skip += l;
+	}
 } 
 
 /*---------------------------------------------------------------------
@@ -371,9 +389,9 @@
 	}
 	XXXAN(vep->include_src);
 	if (vep->o_skip > 0) 
-		vep_emit_skip(vep);
+		vep_emit_skip(vep, &vep->o_skip);
 	if (vep->o_verbatim > 0) 
-		vep_emit_verbatim(vep);
+		vep_emit_verbatim(vep, &vep->o_verbatim);
 	/* XXX: what if it contains NUL bytes ?? */
 	p = vsb_data(vep->include_src);
 	l = vsb_len(vep->include_src);
@@ -461,6 +479,7 @@
 		    vep->remove,
 		    vep->tag_i, vep->tag,
 		    i, p);
+		assert(p >= vep->ver_p);
 
 		/******************************************************
 		 * SECTION A
@@ -506,7 +525,7 @@
 				p++;
 				vep->state = VEP_NEXTTAG;
 				if (!vep->remove)
-					vep_mark_verbatim(vep, p + 1);
+					vep_mark_verbatim(vep, p);
 			} else {
 				vep->tag_i = 0;
 				while (p < e) {
@@ -516,7 +535,7 @@
 					}
 				}
 				if (p < e && !vep->remove)
-					vep_mark_verbatim(vep, p + 1);
+					vep_mark_verbatim(vep, p);
 			}
 		} else if (vep->state == VEP_NEXTTAG) {
 			/*
@@ -526,6 +545,7 @@
 			vep->emptytag = 0;
 			vep->endtag = 0;
 			vep->attr = NULL;
+			vep->dostuff = NULL;
 			while (p < e && *p != '<') {
 				if (vep->esicmt_p != NULL &&
 				    *p == *vep->esicmt_p++) {
@@ -538,13 +558,12 @@
 						 * should not be emitted.
 						 * But the stuff before should
 						 */
-						if (!vep->remove)
-							vep_mark_verbatim(vep,
-							    p - 3);
 						vep_mark_skip(vep, p);
 					}
 				} else {
 					p++;
+					if (!vep->remove) 
+						vep_mark_verbatim(vep, p);
 					vep->esicmt_p = vep->esicmt;
 				}
 			}
@@ -808,7 +827,8 @@
 				if (*p == '>') {
 					for (vm = vep->match;
 					    vm->match != NULL; vm++)
-						break;
+						continue;
+					AZ(vm->match);
 				} else {
 					vep->tag[vep->tag_i++] = *p++;
 					vm = vep_match(vep,
@@ -843,7 +863,8 @@
 			INCOMPL();
 		}
 	}
-	/* XXX: should we return p ? */
+	vep->o_pending +=  p - vep->ver_p;
+	vep->ver_p = p;
 }
 
 /*---------------------------------------------------------------------
@@ -938,7 +959,7 @@
 
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 	vep = sp->wrk->vep;
-	Debug("BYTES %p\n", vep);
+	Debug("BYTES %jd\n", (intmax_t)bytes);
 	CHECK_OBJ_NOTNULL(vep, VEP_MAGIC);
 	AN(vep->bytes);
 	return (vep->bytes(sp, htc, bytes));
@@ -949,7 +970,7 @@
 {
 	struct storage *st;
 	struct vep_state *vep;
-	size_t l;
+	ssize_t l;
 
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 	vep = sp->wrk->vep;
@@ -957,17 +978,21 @@
 	Debug("ENDING %p\n", vep);
 	CHECK_OBJ_NOTNULL(vep, VEP_MAGIC);
 
-	st = sp->wrk->storage;
-	if (st != NULL)
-		l = (const char *)(st->ptr + st->len) - (const char*)vep->ver_p;
-	else
-		l = 0;
-	Debug("ENDING STATE: %s (%ld)\n", vep->state, l);
+	l = sp->obj->len - vep->o_total;
+	Debug("ENDING STATE: %s (skip %jd) (verbatim %jd)"
+	    " (tot %jd) (len %jd) (diff %jd)\n",
+	    vep->state,
+	    (intmax_t)vep->o_skip, (intmax_t)vep->o_verbatim,
+	    (intmax_t)vep->o_total, (intmax_t)sp->obj->len, (intmax_t)l);
+	assert(l >= 0);
 	if (vep->o_skip)
-		vep_emit_skip(vep);
+		vep_emit_skip(vep, &vep->o_skip);
+	if (vep->o_verbatim)
+		vep_emit_verbatim(vep, &vep->o_verbatim);
+	l = sp->obj->len - vep->o_total;
+	if (l)
+		vep_emit_verbatim(vep, &l);
 	vep->o_verbatim += l;
-	if (vep->o_verbatim)
-		vep_emit_verbatim(vep);
 	vsb_finish(vep->vsb);
 	l = vsb_len(vep->vsb);
 	if (vep->state != VEP_NOTXML && l > 0) {

Modified: trunk/varnish-cache/bin/varnishd/mgt_param.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/mgt_param.c	2011-01-14 19:38:13 UTC (rev 5752)
+++ trunk/varnish-cache/bin/varnishd/mgt_param.c	2011-01-14 19:45:48 UTC (rev 5753)
@@ -643,7 +643,8 @@
 		"  0x00000001 - Don't check if it looks like XML\n"
 		"  0x00000002 - Ignore non-esi elements\n"
 		"  0x00000004 - Emit parsing debug records\n"
-		"  0x00000008 - Force-dplit parser input (debugging)\n"
+		"  0x00000008 - Force-split parser input (debugging)\n"
+		"  0x00000010 - Don't coalesce VEC string (debugging)\n"
 		"Use 0x notation and do the bitor in your head :-)\n",
 		0,
 		"0", "bitmap" },

Modified: trunk/varnish-cache/bin/varnishtest/tests/e00019.vtc
===================================================================
--- trunk/varnish-cache/bin/varnishtest/tests/e00019.vtc	2011-01-14 19:38:13 UTC (rev 5752)
+++ trunk/varnish-cache/bin/varnishtest/tests/e00019.vtc	2011-01-14 19:45:48 UTC (rev 5753)
@@ -4,20 +4,37 @@
 
 server s1 {
 	rxreq 
-	txresp -body {
-		</esi:comment foo>
-		<esi:comment / >
-		<esi:remove foo="bar"></esi:remove>
-		<esi:include src="foo">
-	}
+	txresp -nolen -hdr "Transfer-encoding: chunked"
+	chunked {<1></esi:comment foo><1>}
+	chunked {<2><esi:comment / ><esi:comment doo><2>}
+	chunked {<3><esi:remove foo="bar"></esi:remove><3>}
+	chunked {<4><esi:include src="foo"><esi:incl><4>}
+	chunked {<H1><esi:remove>}
+	chunkedlen 256
+	chunked {</esi:remove></H1>}
+
+	chunked {<H2><esi:remove>}
+	chunkedlen 65536
+	chunked {</esi:remove></H2>}
+
+	chunked {<esi:comment/>}
+	chunkedlen 256
+	chunked {<esi:comment/>}
+	chunkedlen 65536
+	chunked {<esi:comment/>}
+
+	chunkedlen 0
+
 	rxreq
 	expect req.url == "bar/foo"
-	txresp -body {<H1></H1>}
+	txresp -body {<INCL>}
 } -start
 
 varnish v1 -vcl+backend {
 	sub vcl_fetch {
-		set beresp.do_esi = true;
+		if (req.url == "bar") {
+			set beresp.do_esi = true;
+		}
 	}
 } -start 
 
@@ -29,8 +46,8 @@
 	txreq  -url bar
 	rxresp
 	expect resp.status == 200
-	expect resp.bodylen == 23
+	expect resp.bodylen == 65840
 } -run
 
-varnish v1 -expect esi_errors == 2
+varnish v1 -expect esi_errors == 4
 varnish v1 -expect esi_warnings == 1




From phk at varnish-cache.org  Fri Jan 14 20:07:06 2011
From: phk at varnish-cache.org (phk at varnish-cache.org)
Date: Fri, 14 Jan 2011 21:07:06 +0100
Subject: r5754 - trunk/varnish-cache/bin/varnishtest/tests
Message-ID: <E1PdpvO-0002JR-2y@project.varnish-software.com>

Author: phk
Date: 2011-01-14 21:07:05 +0100 (Fri, 14 Jan 2011)
New Revision: 5754

Modified:
   trunk/varnish-cache/bin/varnishtest/tests/e00008.vtc
Log:
Cover the 0x10 and 0x20 esi_syntax code also



Modified: trunk/varnish-cache/bin/varnishtest/tests/e00008.vtc
===================================================================
--- trunk/varnish-cache/bin/varnishtest/tests/e00008.vtc	2011-01-14 19:45:48 UTC (rev 5753)
+++ trunk/varnish-cache/bin/varnishtest/tests/e00008.vtc	2011-01-14 20:07:05 UTC (rev 5754)
@@ -64,7 +64,7 @@
 	}
 } -start 
 
-varnish v1 -cliok "param.set esi_syntax 0xe" 
+varnish v1 -cliok "param.set esi_syntax 0x3e" 
 
 varnish v1 -cliok "param.set diag_bitmap 0x10000"
 




From tfheen at varnish-cache.org  Tue Jan 18 08:00:54 2011
From: tfheen at varnish-cache.org (tfheen at varnish-cache.org)
Date: Tue, 18 Jan 2011 09:00:54 +0100
Subject: r5755 - trunk/varnish-cache/bin/varnishtest/tests
Message-ID: <E1Pf6Uo-0006BO-KR@project.varnish-software.com>

Author: tfheen
Date: 2011-01-18 09:00:53 +0100 (Tue, 18 Jan 2011)
New Revision: 5755

Modified:
   trunk/varnish-cache/bin/varnishtest/tests/c00003.vtc
Log:
Add comment about non-local binds in c00003.vtc


Modified: trunk/varnish-cache/bin/varnishtest/tests/c00003.vtc
===================================================================
--- trunk/varnish-cache/bin/varnishtest/tests/c00003.vtc	2011-01-14 20:07:05 UTC (rev 5754)
+++ trunk/varnish-cache/bin/varnishtest/tests/c00003.vtc	2011-01-18 08:00:53 UTC (rev 5755)
@@ -9,8 +9,11 @@
 	txresp -hdr "Connection: close" -body "012345\n"
 } -start
 
+# This requires non-local binds to be disabled.  If you see this fail
+# and are on Linux, ensure /proc/net/ipv4/ip_nonlocal_bind is set to 0.
 varnish v1 -cliok "param.set listen_address ${bad_ip}:0"
 varnish v1 -vcl+backend {} -clierr 300 start
+
 varnish v1 -cliok "param.set listen_address 127.0.0.1:0,${bad_ip}:9082"
 varnish v1 -start
 




From phk at varnish-cache.org  Tue Jan 18 09:06:45 2011
From: phk at varnish-cache.org (phk at varnish-cache.org)
Date: Tue, 18 Jan 2011 10:06:45 +0100
Subject: r5756 - trunk/varnish-cache/bin/varnishd
Message-ID: <E1Pf7WX-0007tT-Fp@project.varnish-software.com>

Author: phk
Date: 2011-01-18 10:06:43 +0100 (Tue, 18 Jan 2011)
New Revision: 5756

Modified:
   trunk/varnish-cache/bin/varnishd/cache_esi.h
   trunk/varnish-cache/bin/varnishd/cache_esi_deliver.c
Log:
Change VEC codes to make it possible to collapse length decoding
in ESI_Deliver()



Modified: trunk/varnish-cache/bin/varnishd/cache_esi.h
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_esi.h	2011-01-18 08:00:53 UTC (rev 5755)
+++ trunk/varnish-cache/bin/varnishd/cache_esi.h	2011-01-18 09:06:43 UTC (rev 5756)
@@ -27,10 +27,10 @@
  *
  */
 
-#define	VEC_V1	'B'
-#define	VEC_V2	'W'
-#define	VEC_V8	'L'
-#define	VEC_S1	'b'
-#define	VEC_S2	'w'
-#define	VEC_S8	'l'
+#define	VEC_V1	(0x40 + 1)
+#define	VEC_V2	(0x40 + 2)
+#define	VEC_V8	(0x40 + 8)
+#define	VEC_S1	(0x60 + 1)
+#define	VEC_S2	(0x60 + 2)
+#define	VEC_S8	(0x60 + 8)
 #define	VEC_INCL	'I'

Modified: trunk/varnish-cache/bin/varnishd/cache_esi_deliver.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_esi_deliver.c	2011-01-18 08:00:53 UTC (rev 5755)
+++ trunk/varnish-cache/bin/varnishd/cache_esi_deliver.c	2011-01-18 09:06:43 UTC (rev 5756)
@@ -138,13 +138,42 @@
 		(void)WRW_Write(sp->wrk, "\r\n", -1);
 }
 
+static ssize_t
+ved_decode_len(uint8_t **pp)
+{
+	uint8_t *p;
+	ssize_t l;
+
+	p = *pp;
+	switch (*p & 15) {
+	case 1:
+		l = p[1];
+		p += 2;
+		break;
+	case 2:
+		l = vbe16dec(p + 1);
+		p += 3;
+		break;
+	case 8:
+		l = vbe64dec(p + 1);
+		p += 9;
+		break;
+	default:
+		printf("%d\n",(*p & 15));
+		INCOMPL();
+	}
+	*pp = p;
+	assert(l > 0);
+	return (l);
+}
+
 void
 ESI_Deliver(struct sess *sp)
 {
 	struct storage *st;
 	uint8_t *p, *e, *q, *r;
 	unsigned off;
-	size_t l;
+	ssize_t l;
 
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 	st = sp->obj->esidata;
@@ -156,30 +185,11 @@
 	off = 0;
 
 	while (p < e) {
-		//usleep(10000);
-		//WRW_Flush(sp->wrk);
 		switch (*p) {
 		case VEC_V1:
-			l = p[1];
-			p += 2;
-			q = (void*)strchr((const char*)p, '\0');
-			assert (q > p);
-			esi_sendchunk(sp, p, q - p, st->ptr + off, l);
-			off += l;
-			p = q + 1;
-			break;
 		case VEC_V2:
-			l = vbe16dec(p + 1);
-			p += 3;
-			q = (void*)strchr((const char*)p, '\0');
-			assert (q > p);
-			esi_sendchunk(sp, p, q - p, st->ptr + off, l);
-			off += l;
-			p = q + 1;
-			break;
 		case VEC_V8:
-			l = vbe64dec(p + 1);
-			p += 9;
+			l = ved_decode_len(&p);
 			q = (void*)strchr((const char*)p, '\0');
 			assert (q > p);
 			esi_sendchunk(sp, p, q - p, st->ptr + off, l);
@@ -187,21 +197,10 @@
 			p = q + 1;
 			break;
 		case VEC_S1:
-			l = p[1];
-			p += 2;
-			Debug("SKIP1(%d)\n", (int)l);
-			off += l;
-			break;
 		case VEC_S2:
-			l = vbe16dec(p + 1);
-			p += 3;
-			Debug("SKIP2(%d)\n", (int)l);
-			off += l;
-			break;
 		case VEC_S8:
-			l = vbe64dec(p + 1);
-			p += 9;
-			Debug("SKIP8(%d)\n", (int)l);
+			l = ved_decode_len(&p);
+			Debug("SKIP1(%d)\n", (int)l);
 			off += l;
 			break;
 		case VEC_INCL:




From phk at varnish-cache.org  Tue Jan 18 11:07:58 2011
From: phk at varnish-cache.org (phk at varnish-cache.org)
Date: Tue, 18 Jan 2011 12:07:58 +0100
Subject: r5757 - trunk/varnish-cache/bin/varnishd
Message-ID: <E1Pf9Pq-0000iZ-5B@project.varnish-software.com>

Author: phk
Date: 2011-01-18 12:07:57 +0100 (Tue, 18 Jan 2011)
New Revision: 5757

Modified:
   trunk/varnish-cache/bin/varnishd/cache.h
Log:
struct esi_data is only used with OLD_ESI



Modified: trunk/varnish-cache/bin/varnishd/cache.h
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache.h	2011-01-18 09:06:43 UTC (rev 5756)
+++ trunk/varnish-cache/bin/varnishd/cache.h	2011-01-18 11:07:57 UTC (rev 5757)
@@ -94,7 +94,9 @@
 struct objcore;
 struct storage;
 struct workreq;
+#ifdef OLD_ESI
 struct esidata;
+#endif
 struct vrt_backend;
 struct cli_proto;
 struct ban;




From phk at varnish-cache.org  Tue Jan 18 11:08:46 2011
From: phk at varnish-cache.org (phk at varnish-cache.org)
Date: Tue, 18 Jan 2011 12:08:46 +0100
Subject: r5758 - trunk/varnish-cache/bin/varnishd
Message-ID: <E1Pf9Qc-0000lB-Lf@project.varnish-software.com>

Author: phk
Date: 2011-01-18 12:08:46 +0100 (Tue, 18 Jan 2011)
New Revision: 5758

Modified:
   trunk/varnish-cache/bin/varnishd/cache_esi_deliver.c
   trunk/varnish-cache/bin/varnishd/cache_esi_parse.c
Log:
Calculate and emit CRC32 checksum for all verbatim bites,
we need this to produce the correct final CRC32 for gzip'ed
ESI documents.



Modified: trunk/varnish-cache/bin/varnishd/cache_esi_deliver.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_esi_deliver.c	2011-01-18 11:07:57 UTC (rev 5757)
+++ trunk/varnish-cache/bin/varnishd/cache_esi_deliver.c	2011-01-18 11:08:46 UTC (rev 5758)
@@ -39,6 +39,7 @@
 #include "cache_esi.h"
 #include "vend.h"
 #include "vct.h"
+#include "zlib.h"
 #include "stevedore.h"
 
 /*--------------------------------------------------------------------*/
@@ -159,7 +160,7 @@
 		p += 9;
 		break;
 	default:
-		printf("%d\n",(*p & 15));
+		printf("Illegal Length %d %d\n", *p, (*p & 15));
 		INCOMPL();
 	}
 	*pp = p;
@@ -174,6 +175,7 @@
 	uint8_t *p, *e, *q, *r;
 	unsigned off;
 	ssize_t l;
+	uint32_t crc, crc_ref;
 
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 	st = sp->obj->esidata;
@@ -190,8 +192,16 @@
 		case VEC_V2:
 		case VEC_V8:
 			l = ved_decode_len(&p);
+			crc = vbe32dec(p);
+			p += 4;
 			q = (void*)strchr((const char*)p, '\0');
 			assert (q > p);
+			crc_ref = crc32(0L, Z_NULL, 0);
+			crc_ref = crc32(crc_ref, st->ptr + off, l);
+			if (crc_ref != crc) {
+				printf("CRC Mismatch %08x %08x\n", crc_ref, crc);
+			}
+			xxxassert(crc_ref == crc);
 			esi_sendchunk(sp, p, q - p, st->ptr + off, l);
 			off += l;
 			p = q + 1;

Modified: trunk/varnish-cache/bin/varnishd/cache_esi_parse.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_esi_parse.c	2011-01-18 11:07:57 UTC (rev 5757)
+++ trunk/varnish-cache/bin/varnishd/cache_esi_parse.c	2011-01-18 11:08:46 UTC (rev 5758)
@@ -39,6 +39,7 @@
 #include "cache_esi.h"
 #include "vend.h"
 #include "vct.h"
+#include "zlib.h"
 #include "stevedore.h"
 
 #ifndef OLD_ESI
@@ -77,6 +78,9 @@
 	ssize_t			o_skip;
 	ssize_t			o_pending;
 	ssize_t			o_total;
+	uint32_t		crc;
+	ssize_t			o_crc;
+	uint32_t		crcp;
 
 const char		*hack_p;
 	const char		*ver_p;
@@ -252,78 +256,138 @@
 } 
 
 static void
-vep_emit_skip(struct vep_state *vep, ssize_t *l)
+vep_emit_skip(struct vep_state *vep, ssize_t l)
 {
 
-	assert(*l > 0);
 	if (params->esi_syntax & 0x20) {
-		Debug("---> SKIP(%jd)\n", (intmax_t)*l);
+		Debug("---> SKIP(%jd)\n", (intmax_t)l);
 	}
-	vep_emit_len(vep, *l, VEC_S1, VEC_S2, VEC_S8);
-	vep->o_total += *l;
-	*l = 0;
+	vep_emit_len(vep, l, VEC_S1, VEC_S2, VEC_S8);
 } 
 
 static void
-vep_emit_verbatim(struct vep_state *vep, ssize_t *l)
+vep_emit_verbatim(struct vep_state *vep, ssize_t l)
 {
+	uint8_t buf[4];
 
-	assert(*l > 0);
 	if (params->esi_syntax & 0x20) {
-		Debug("---> VERBATIM(%jd)\n", (intmax_t)*l);
+		Debug("---> VERBATIM(%jd)\n", (intmax_t)l);
 	}
-	vep_emit_len(vep, *l, VEC_V1, VEC_V2, VEC_V8);
-	vsb_printf(vep->vsb, "%lx\r\n%c", *l, 0);
+	vep_emit_len(vep, l, VEC_V1, VEC_V2, VEC_V8);
+	vbe32enc(buf, vep->crc);
+	vsb_bcat(vep->vsb, buf, sizeof buf);
+	vsb_printf(vep->vsb, "%lx\r\n%c", l, 0);
+} 
+
+static void
+vep_emit_common(struct vep_state *vep, ssize_t *l, int skip)
+{
+	assert(*l > 0);
+	assert(*l == vep->o_crc);
+
+	if (skip)
+		vep_emit_skip(vep, *l);
+	else
+		vep_emit_verbatim(vep, *l);
+
+	vep->crc = crc32(0L, Z_NULL, 0);
+	vep->o_crc = 0;
 	vep->o_total += *l;
 	*l = 0;
-} 
+}
 
+/*---------------------------------------------------------------------
+ *
+ */
+
 static void
-vep_mark_verbatim(struct vep_state *vep, const char *p)
+vep_mark_common(struct vep_state *vep, const char *p, int skip)
 {
 	ssize_t l;
 
+	/*
+	 * If we changed mode, emit whatever the opposite mode
+	 * assembled before the pending bytes.
+	 */
+	if (skip && vep->o_verbatim > 0)
+		vep_emit_common(vep, &vep->o_verbatim, 0);
+	else if (!skip && vep->o_skip > 0)
+		vep_emit_common(vep, &vep->o_skip, 1);
+
+	/* In debug mode, the pending bytes are emitted separately.  */
+	if ((params->esi_syntax & 0x10) && vep->o_pending) {
+		vep->o_crc = vep->o_pending;
+		vep->crc = vep->crcp;
+		vep->crcp = crc32(0L, Z_NULL, 0);
+		vep_emit_common(vep, &vep->o_pending, skip);
+	}
+
+	/* Transfer pending bytes CRC into active mode.  */
+	if (vep->o_pending) {
+		if (vep->o_crc == 0) {
+			vep->crc = vep->crcp;
+			vep->o_crc = vep->o_pending;
+		} else {
+			vep->crc = crc32_combine(vep->crc,
+			    vep->crcp, vep->o_pending);
+			vep->o_crc += vep->o_pending;
+		}
+		vep->crcp = crc32(0L, Z_NULL, 0);
+	}
+
+	/*
+	 * Process this bit of input
+	 */
 	AN(vep->ver_p);
 	l = p - vep->ver_p;
+	assert(l >= 0);
+	vep->crc = crc32(vep->crc, (const void*)vep->ver_p, l);
+	vep->o_crc += l;
 	vep->ver_p = p;
-	assert(l >= 0);
-	if (params->esi_syntax & 0x10) {
-		if (vep->o_pending)
-			vep_emit_verbatim(vep, &vep->o_pending);
-		if (l)
-			vep_emit_verbatim(vep, &l);
+
+	if ((params->esi_syntax & 0x10) && l > 0) {
+		/* Emit right away if debug mode */
+		vep_emit_common(vep, &l, skip);
+	} else if (skip) {
+		vep->o_skip += vep->o_pending;
+		vep->o_skip += l;
 	} else {
-		if (vep->o_skip > 0) 
-			vep_emit_skip(vep, &vep->o_skip);
 		vep->o_verbatim += vep->o_pending;
-		vep->o_pending = 0;
 		vep->o_verbatim += l;
 	}
+	vep->o_pending = 0;
+}
+
+
+static void
+vep_mark_verbatim(struct vep_state *vep, const char *p)
+{
+	vep_mark_common(vep, p, 0);
 } 
 
 static void
 vep_mark_skip(struct vep_state *vep, const char *p)
 {
+	vep_mark_common(vep, p, 1);
+} 
+
+static void
+vep_mark_pending(struct vep_state *vep, const char *p)
+{
 	ssize_t l;
 
 	AN(vep->ver_p);
 	l = p - vep->ver_p;
+	if (l == 0)
+		return;
+	assert(l >= 0);
+	vep->crcp = crc32(vep->crcp, (const void *)vep->ver_p, l);
 	vep->ver_p = p;
-	assert(l >= 0);
-	if (params->esi_syntax & 0x10) {
-		if (vep->o_pending) 
-			vep_emit_skip(vep, &vep->o_pending);
-		if (l)
-			vep_emit_skip(vep, &l);
-	} else {
-		if (vep->o_verbatim > 0) 
-			vep_emit_verbatim(vep, &vep->o_verbatim);
-		vep->o_skip += vep->o_pending;
-		vep->o_pending = 0;
-		vep->o_skip += l;
-	}
-} 
 
+	vep->o_pending += l;
+	fflush(stdout);
+}
+
 /*---------------------------------------------------------------------
  */
 
@@ -389,9 +453,9 @@
 	}
 	XXXAN(vep->include_src);
 	if (vep->o_skip > 0) 
-		vep_emit_skip(vep, &vep->o_skip);
+		vep_emit_common(vep, &vep->o_skip, 1);
 	if (vep->o_verbatim > 0) 
-		vep_emit_verbatim(vep, &vep->o_verbatim);
+		vep_emit_common(vep, &vep->o_verbatim, 0);
 	/* XXX: what if it contains NUL bytes ?? */
 	p = vsb_data(vep->include_src);
 	l = vsb_len(vep->include_src);
@@ -863,8 +927,7 @@
 			INCOMPL();
 		}
 	}
-	vep->o_pending +=  p - vep->ver_p;
-	vep->ver_p = p;
+	vep_mark_pending(vep, p);
 }
 
 /*---------------------------------------------------------------------
@@ -944,9 +1007,11 @@
 	vep->magic = VEP_MAGIC;
 	vep->sp = sp;
 	vep->bytes = vfp_esi_bytes_uu;
+	vep->state = VEP_START;
 	vep->vsb = vsb_newauto();
-	vep->state = VEP_START;
 	AN(vep->vsb);
+	vep->crc = crc32(0L, Z_NULL, 0);
+	vep->crcp = crc32(0L, Z_NULL, 0);
 
 	sp->wrk->vep = vep;
 	(void)estimate;
@@ -985,14 +1050,12 @@
 	    (intmax_t)vep->o_skip, (intmax_t)vep->o_verbatim,
 	    (intmax_t)vep->o_total, (intmax_t)sp->obj->len, (intmax_t)l);
 	assert(l >= 0);
-	if (vep->o_skip)
-		vep_emit_skip(vep, &vep->o_skip);
-	if (vep->o_verbatim)
-		vep_emit_verbatim(vep, &vep->o_verbatim);
-	l = sp->obj->len - vep->o_total;
-	if (l)
-		vep_emit_verbatim(vep, &l);
-	vep->o_verbatim += l;
+	if (vep->o_pending)
+		vep_mark_verbatim(vep, vep->ver_p);
+	if (vep->o_skip > 0)
+		vep_emit_common(vep, &vep->o_skip, 1);
+	else if (vep->o_verbatim > 0)
+		vep_emit_common(vep, &vep->o_verbatim, 0);
 	vsb_finish(vep->vsb);
 	l = vsb_len(vep->vsb);
 	if (vep->state != VEP_NOTXML && l > 0) {




From martin at varnish-cache.org  Tue Jan 18 11:25:48 2011
From: martin at varnish-cache.org (martin at varnish-cache.org)
Date: Tue, 18 Jan 2011 12:25:48 +0100
Subject: r5759 - trunk/varnish-cache/lib/libvmod_std
Message-ID: <E1Pf9h6-0001HJ-HK@project.varnish-software.com>

Author: martin
Date: 2011-01-18 12:25:47 +0100 (Tue, 18 Jan 2011)
New Revision: 5759

Modified:
   trunk/varnish-cache/lib/libvmod_std/Makefile.am
Log:
Make std vmod install to pkglibdir/vmods



Modified: trunk/varnish-cache/lib/libvmod_std/Makefile.am
===================================================================
--- trunk/varnish-cache/lib/libvmod_std/Makefile.am	2011-01-18 11:08:46 UTC (rev 5758)
+++ trunk/varnish-cache/lib/libvmod_std/Makefile.am	2011-01-18 11:25:47 UTC (rev 5759)
@@ -2,7 +2,8 @@
 
 INCLUDES = -I$(top_srcdir)/include -I$(top_builddir)/include
 
-lib_LTLIBRARIES = libvmod_std.la
+vmoddir = $(pkglibdir)/vmods
+vmod_LTLIBRARIES = libvmod_std.la
 
 libvmod_std_la_LDFLAGS = -version-info 1:0:0
 




From phk at varnish-cache.org  Tue Jan 18 14:05:58 2011
From: phk at varnish-cache.org (phk at varnish-cache.org)
Date: Tue, 18 Jan 2011 15:05:58 +0100
Subject: r5760 - trunk/varnish-cache/bin/varnishd
Message-ID: <E1PfCC6-0000fO-Li@project.varnish-software.com>

Author: phk
Date: 2011-01-18 15:05:58 +0100 (Tue, 18 Jan 2011)
New Revision: 5760

Modified:
   trunk/varnish-cache/bin/varnishd/cache_esi_parse.c
Log:
Minimize use of "pending" buffer and reduce number of (effective)
calls to the 'mark' functions as much as possible.



Modified: trunk/varnish-cache/bin/varnishd/cache_esi_parse.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_esi_parse.c	2011-01-18 11:25:47 UTC (rev 5759)
+++ trunk/varnish-cache/bin/varnishd/cache_esi_parse.c	2011-01-18 14:05:58 UTC (rev 5760)
@@ -89,6 +89,8 @@
 	const char		*until_p;
 	const char		*until_s;
 
+	int			in_esi_tag;
+
 	const char		*esicmt;
 	const char		*esicmt_p;
 
@@ -105,6 +107,11 @@
 	dostuff_f		*dostuff;
 
 	struct vsb		*include_src;
+
+	unsigned		nm_skip;
+	unsigned		nm_verbatim;
+	unsigned		nm_pending;
+	int			last_mark;
 };
 
 /*---------------------------------------------------------------------*/
@@ -362,13 +369,23 @@
 static void
 vep_mark_verbatim(struct vep_state *vep, const char *p)
 {
+	if (vep->last_mark == 0 && p == vep->ver_p)
+		return;
+//printf("MARK VERB %d %s <%.*s>\n", vep->remove, vep->state, (int)(p - vep->ver_p), vep->ver_p);
 	vep_mark_common(vep, p, 0);
+	vep->nm_verbatim++;
+	vep->last_mark = 0;
 } 
 
 static void
 vep_mark_skip(struct vep_state *vep, const char *p)
 {
+	if (vep->last_mark == 1 && p == vep->ver_p)
+		return;
+//printf("MARK SKIP %d %s <%.*s>\n", vep->remove, vep->state, (int)(p - vep->ver_p), vep->ver_p);
 	vep_mark_common(vep, p, 1);
+	vep->nm_skip++;
+	vep->last_mark = 1;
 } 
 
 static void
@@ -376,16 +393,20 @@
 {
 	ssize_t l;
 
+	if (vep->last_mark == 2 && p == vep->ver_p)
+		return;
 	AN(vep->ver_p);
 	l = p - vep->ver_p;
 	if (l == 0)
 		return;
+// printf("MARK PEND %d %s <%.*s>\n", vep->remove, vep->state, (int)l, vep->ver_p);
 	assert(l >= 0);
 	vep->crcp = crc32(vep->crcp, (const void *)vep->ver_p, l);
 	vep->ver_p = p;
 
 	vep->o_pending += l;
-	fflush(stdout);
+	vep->nm_pending++;
+	vep->last_mark = 2;
 }
 
 /*---------------------------------------------------------------------
@@ -559,13 +580,10 @@
 			 * If the first non-whitespace char is different
 			 * from '<' we assume this is not XML.
 			 */
-			while (p < e && vct_islws(*p)) {
+			while (p < e && vct_islws(*p)) 
 				p++;
-				vep_mark_verbatim(vep, p);
-			}
+			vep_mark_verbatim(vep, p);
 			if (p < e && *p == '<') {
-				if (!vep->remove)
-					vep_mark_verbatim(vep, p);
 				p++;
 				vep->state = VEP_STARTTAG;
 			} else if (p < e) {
@@ -579,6 +597,7 @@
 			 * vfp_esi_end() will handle the rest
 			 */
 			p = e;
+			vep_mark_verbatim(vep, p);
 
 		/******************************************************
 		 * SECTION B
@@ -588,8 +607,6 @@
 			if (params->esi_syntax & 0x2) {
 				p++;
 				vep->state = VEP_NEXTTAG;
-				if (!vep->remove)
-					vep_mark_verbatim(vep, p);
 			} else {
 				vep->tag_i = 0;
 				while (p < e) {
@@ -598,9 +615,9 @@
 						break;
 					}
 				}
-				if (p < e && !vep->remove)
-					vep_mark_verbatim(vep, p);
 			}
+			if (p == e && !vep->remove)
+				vep_mark_verbatim(vep, p);
 		} else if (vep->state == VEP_NEXTTAG) {
 			/*
 			 * Hunt for start of next tag and keep an eye
@@ -611,35 +628,38 @@
 			vep->attr = NULL;
 			vep->dostuff = NULL;
 			while (p < e && *p != '<') {
-				if (vep->esicmt_p != NULL &&
-				    *p == *vep->esicmt_p++) {
+				if (vep->esicmt_p == NULL) {
 					p++;
-					if (*vep->esicmt_p == '\0') {
-						vep->esicmt = NULL;
-						vep->esicmt_p = NULL;
-						/*
-						 * The end of the esicmt
-						 * should not be emitted.
-						 * But the stuff before should
-						 */
-						vep_mark_skip(vep, p);
-					}
-				} else {
+					continue;
+				}
+				if (*p != *vep->esicmt_p) {
 					p++;
-					if (!vep->remove) 
-						vep_mark_verbatim(vep, p);
 					vep->esicmt_p = vep->esicmt;
+					continue;
 				}
+				if (!vep->remove &&
+				    vep->esicmt_p == vep->esicmt)
+					vep_mark_verbatim(vep, p);
+				p++;
+				if (*++vep->esicmt_p == '\0') {
+					vep->esicmt = NULL;
+					vep->esicmt_p = NULL;
+					/*
+					 * The end of the esicmt
+					 * should not be emitted.
+					 * But the stuff before should
+					 */
+					vep_mark_skip(vep, p);
+				}
 			}
-			if (p < e && vep->esicmt_p == NULL && !vep->remove)
-				vep_mark_verbatim(vep, p);
 			if (p < e) {
-				assert(*p == '<');
 				if (!vep->remove)
 					vep_mark_verbatim(vep, p);
+				assert(*p == '<');
 				p++;
 				vep->state = VEP_STARTTAG;
-			}
+			} else if (vep->esicmt_p == vep->esicmt && !vep->remove)
+				vep_mark_verbatim(vep, p);
 
 		/******************************************************
 		 * SECTION C
@@ -692,12 +712,12 @@
 			vep->until_s = VEP_NEXTTAG;
 			vep->state = VEP_UNTIL;
 		} else if (vep->state == VEP_ESITAG) {
+			vep->in_esi_tag = 1;
 			vep_mark_skip(vep, p);
 			vep->match = vep_match_esi;
 			vep->state = VEP_MATCH;
 		} else if (vep->state == VEP_ESIINCLUDE) {
 			if (vep->remove) {
-				vep_mark_skip(vep, p);
 				vep_error(vep,
 				    "ESI 1.0 <esi:include> element"
 				    " nested in <esi:remove>");
@@ -716,7 +736,6 @@
 			vep->state = VEP_INTAG;
 		} else if (vep->state == VEP_ESICOMMENT) {
 			if (vep->remove) {
-				vep_mark_skip(vep, p);
 				vep_error(vep,
 				    "ESI 1.0 <esi:comment> element"
 				    " nested in <esi:remove>");
@@ -730,7 +749,6 @@
 				vep->state = VEP_INTAG;
 			}
 		} else if (vep->state == VEP_ESIBOGON) {
-			vep_mark_skip(vep, p);
 			vep_error(vep,
 			    "ESI 1.0 <esi:bogus> element");
 			vep->state = VEP_TAGERROR;
@@ -741,7 +759,6 @@
 
 		} else if (vep->state == VEP_INTAG) {
 			vep->tag_i = 0;
-			vep_mark_skip(vep, p);
 			while (p < e && vct_islws(*p) && !vep->emptytag) {
 				p++;	
 				vep->canattr = 1;
@@ -756,6 +773,7 @@
 				AN(vep->dostuff);
 				vep_mark_skip(vep, p);
 				vep->dostuff(vep, DO_TAG);
+				vep->in_esi_tag = 0;
 				vep->state = VEP_NEXTTAG;
 			} else if (p < e && vep->emptytag) {
 				vep_error(vep,
@@ -775,6 +793,7 @@
 			if (p < e) {
 				p++;
 				vep_mark_skip(vep, p);
+				vep->in_esi_tag = 0;
 				vep->state = VEP_NEXTTAG;
 			}
 
@@ -805,7 +824,6 @@
 			} else if (p < e && vct_issp(*p)) {
 				vep->state = VEP_INTAG;
 			} else if (p < e) {
-				vep_mark_skip(vep, p);
 				vep_error(vep,
 				    "XML 1.0 Illegal attr char");
 				vep->state = VEP_TAGERROR;
@@ -822,7 +840,6 @@
 				vep->attr_delim = ' ';
 				vep->state = VEP_ATTRVAL;
 			} else {
-				vep_mark_skip(vep, p);
 				vep_error(vep,
 				    "XML 1.0 Illegal attribute delimiter");
 				vep->state = VEP_TAGERROR;
@@ -836,7 +853,6 @@
 				p++;
 			}
 			if (p < e && *p == '>') {
-				vep_mark_skip(vep, p);
 				vep_error(vep,
 				    "XML 1.0 Missing end attribute delimiter");
 				vep->state = VEP_TAGERROR;
@@ -852,7 +868,6 @@
 				if (vep->attr_vsb != NULL) {
 					vsb_finish(vep->attr_vsb);
 					AN(vep->dostuff);
-					vep_mark_skip(vep, p);
 					vep->dostuff(vep, DO_ATTR);
 					vep->attr_vsb = NULL;
 				}
@@ -921,13 +936,27 @@
 					vep->state = vep->until_s;
 					break;
 				}
+			if (p == e && !vep->remove)
+				vep_mark_verbatim(vep, p);
 			}
 		} else {
 			Debug("*** Unknown state %s\n", vep->state);
 			INCOMPL();
 		}
 	}
-	vep_mark_pending(vep, p);
+	/*
+	 * We must always mark up the storage we got, try to do so
+	 * in the most efficient way, in particular with respect to
+	 * minimizing and limiting use of pending.
+	 */
+	if (p == vep->ver_p) 
+		;
+	else if (vep->in_esi_tag)
+		vep_mark_skip(vep, p);
+	else if (vep->remove)
+		vep_mark_skip(vep, p);
+	else
+		vep_mark_pending(vep, p);
 }
 
 /*---------------------------------------------------------------------
@@ -1056,6 +1085,8 @@
 		vep_emit_common(vep, &vep->o_skip, 1);
 	else if (vep->o_verbatim > 0)
 		vep_emit_common(vep, &vep->o_verbatim, 0);
+
+printf("NMARK VER %u SKIP %u PEND %u\n", vep->nm_verbatim, vep->nm_skip, vep->nm_pending);
 	vsb_finish(vep->vsb);
 	l = vsb_len(vep->vsb);
 	if (vep->state != VEP_NOTXML && l > 0) {




From kristian at varnish-cache.org  Tue Jan 18 14:34:48 2011
From: kristian at varnish-cache.org (kristian at varnish-cache.org)
Date: Tue, 18 Jan 2011 15:34:48 +0100
Subject: r5761 - in branches/2.1/varnish-cache: . redhat
Message-ID: <E1PfCe0-0001ac-SM@project.varnish-software.com>

Author: kristian
Date: 2011-01-18 15:34:48 +0100 (Tue, 18 Jan 2011)
New Revision: 5761

Modified:
   branches/2.1/varnish-cache/configure.ac
   branches/2.1/varnish-cache/redhat/Makefile.am
   branches/2.1/varnish-cache/redhat/varnish.spec
Log:
2.1: Prepare for 2.1.5

- Bump version numbers to 2.1.5
- Redhat: Don't rebuild doc-stuff (it's done at make dist-time)
- Redhat: Include the varnish_reload_vcl script in extra dist

Modified: branches/2.1/varnish-cache/configure.ac
===================================================================
--- branches/2.1/varnish-cache/configure.ac	2011-01-18 14:05:58 UTC (rev 5760)
+++ branches/2.1/varnish-cache/configure.ac	2011-01-18 14:34:48 UTC (rev 5761)
@@ -5,7 +5,7 @@
 Copyright (c) 2006-2010 Redpill Linpro AS
 Copyright (c) 2010 Varnish Software AS])
 AC_REVISION([$Id$])
-AC_INIT([Varnish], [2.1.4], [varnish-dev at varnish-cache.org])
+AC_INIT([Varnish], [2.1.5], [varnish-dev at varnish-cache.org])
 AC_CONFIG_SRCDIR(include/varnishapi.h)
 AM_CONFIG_HEADER(config.h)
 

Modified: branches/2.1/varnish-cache/redhat/Makefile.am
===================================================================
--- branches/2.1/varnish-cache/redhat/Makefile.am	2011-01-18 14:05:58 UTC (rev 5760)
+++ branches/2.1/varnish-cache/redhat/Makefile.am	2011-01-18 14:34:48 UTC (rev 5761)
@@ -8,4 +8,5 @@
 	varnish.spec		\
 	varnish.sysconfig	\
 	varnishlog.initrc	\
+	varnish_reload_vcl	\
 	varnishncsa.initrc

Modified: branches/2.1/varnish-cache/redhat/varnish.spec
===================================================================
--- branches/2.1/varnish-cache/redhat/varnish.spec	2011-01-18 14:05:58 UTC (rev 5760)
+++ branches/2.1/varnish-cache/redhat/varnish.spec	2011-01-18 14:34:48 UTC (rev 5761)
@@ -1,6 +1,6 @@
 Summary: High-performance HTTP accelerator
 Name: varnish
-Version: 2.1.4
+Version: 2.1.5
 Release: 1%{?dist}
 License: BSD
 Group: System Environment/Daemons
@@ -136,9 +136,6 @@
 	redhat/varnish.initrc redhat/varnishlog.initrc redhat/varnishncsa.initrc
 %endif
 
-pushd doc/sphinx
-make html
-popd
 cp -r doc/sphinx/\=build/html doc
 
 %check




From phk at varnish-cache.org  Tue Jan 18 14:45:18 2011
From: phk at varnish-cache.org (phk at varnish-cache.org)
Date: Tue, 18 Jan 2011 15:45:18 +0100
Subject: r5762 - trunk/varnish-cache/bin/varnishd
Message-ID: <E1PfCoA-0001xJ-4l@project.varnish-software.com>

Author: phk
Date: 2011-01-18 15:45:15 +0100 (Tue, 18 Jan 2011)
New Revision: 5762

Modified:
   trunk/varnish-cache/bin/varnishd/cache_esi_parse.c
Log:
Optimize a corner case and add a couple of asserts.



Modified: trunk/varnish-cache/bin/varnishd/cache_esi_parse.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_esi_parse.c	2011-01-18 14:34:48 UTC (rev 5761)
+++ trunk/varnish-cache/bin/varnishd/cache_esi_parse.c	2011-01-18 14:45:15 UTC (rev 5762)
@@ -393,12 +393,10 @@
 {
 	ssize_t l;
 
-	if (vep->last_mark == 2 && p == vep->ver_p)
-		return;
+	assert (vep->last_mark != 2 || p != vep->ver_p);
 	AN(vep->ver_p);
 	l = p - vep->ver_p;
-	if (l == 0)
-		return;
+	assert(l > 0);
 // printf("MARK PEND %d %s <%.*s>\n", vep->remove, vep->state, (int)l, vep->ver_p);
 	assert(l >= 0);
 	vep->crcp = crc32(vep->crcp, (const void *)vep->ver_p, l);
@@ -473,10 +471,16 @@
 		return;
 	}
 	XXXAN(vep->include_src);
-	if (vep->o_skip > 0) 
-		vep_emit_common(vep, &vep->o_skip, 1);
-	if (vep->o_verbatim > 0) 
-		vep_emit_common(vep, &vep->o_verbatim, 0);
+	/*
+	 * Strictly speaking, we ought to spit out any piled up skip before
+	 * emitting the VEC for the include, but objectively that makes no
+	 * difference and robs us of a chance to collapse another skip into
+	 * this on so we don't do that.
+	 * However, we cannot tolerate any verbatim stuff piling up.
+	 * The mark_skip() before calling dostuff should have taken
+	 * care of that.  Make sure.
+	 */
+	assert(vep->o_verbatim == 0);
 	/* XXX: what if it contains NUL bytes ?? */
 	p = vsb_data(vep->include_src);
 	l = vsb_len(vep->include_src);




From phk at varnish-cache.org  Wed Jan 19 09:37:51 2011
From: phk at varnish-cache.org (phk at varnish-cache.org)
Date: Wed, 19 Jan 2011 10:37:51 +0100
Subject: r5763 - trunk/varnish-cache/bin/varnishd
Message-ID: <E1PfUUC-0004Hw-11@project.varnish-software.com>

Author: phk
Date: 2011-01-19 10:37:51 +0100 (Wed, 19 Jan 2011)
New Revision: 5763

Modified:
   trunk/varnish-cache/bin/varnishd/cache_esi_parse.c
   trunk/varnish-cache/bin/varnishd/mgt_param.c
Log:
Code polishing



Modified: trunk/varnish-cache/bin/varnishd/cache_esi_parse.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_esi_parse.c	2011-01-18 14:45:15 UTC (rev 5762)
+++ trunk/varnish-cache/bin/varnishd/cache_esi_parse.c	2011-01-19 09:37:51 UTC (rev 5763)
@@ -57,6 +57,8 @@
 	const char	* const *state;
 };
 
+enum vep_mark { VERBATIM = 0, SKIP };
+
 struct vep_state {
 	unsigned		magic;
 #define VEP_MAGIC		0x55cb9b82
@@ -74,8 +76,7 @@
 
 	unsigned		remove;
 
-	ssize_t			o_verbatim;
-	ssize_t			o_skip;
+	ssize_t			o_wait;
 	ssize_t			o_pending;
 	ssize_t			o_total;
 	uint32_t		crc;
@@ -111,7 +112,7 @@
 	unsigned		nm_skip;
 	unsigned		nm_verbatim;
 	unsigned		nm_pending;
-	int			last_mark;
+	enum vep_mark		last_mark;
 };
 
 /*---------------------------------------------------------------------*/
@@ -263,7 +264,7 @@
 } 
 
 static void
-vep_emit_skip(struct vep_state *vep, ssize_t l)
+vep_emit_skip(const struct vep_state *vep, ssize_t l)
 {
 
 	if (params->esi_syntax & 0x20) {
@@ -273,7 +274,7 @@
 } 
 
 static void
-vep_emit_verbatim(struct vep_state *vep, ssize_t l)
+vep_emit_verbatim(const struct vep_state *vep, ssize_t l)
 {
 	uint8_t buf[4];
 
@@ -287,12 +288,13 @@
 } 
 
 static void
-vep_emit_common(struct vep_state *vep, ssize_t *l, int skip)
+vep_emit_common(struct vep_state *vep, ssize_t *l, enum vep_mark mark)
 {
 	assert(*l > 0);
 	assert(*l == vep->o_crc);
 
-	if (skip)
+	assert(mark == SKIP || mark == VERBATIM);
+	if (mark == SKIP)
 		vep_emit_skip(vep, *l);
 	else
 		vep_emit_verbatim(vep, *l);
@@ -308,28 +310,25 @@
  */
 
 static void
-vep_mark_common(struct vep_state *vep, const char *p, int skip)
+vep_mark_common(struct vep_state *vep, const char *p, enum vep_mark mark)
 {
 	ssize_t l;
 
+	assert(mark == SKIP || mark == VERBATIM);
+
+	/* The NO-OP case, no data, no pending data & no change of mode */
+	if (vep->last_mark == mark && p == vep->ver_p && vep->o_pending == 0)
+		return;
+
 	/*
 	 * If we changed mode, emit whatever the opposite mode
 	 * assembled before the pending bytes.
 	 */
-	if (skip && vep->o_verbatim > 0)
-		vep_emit_common(vep, &vep->o_verbatim, 0);
-	else if (!skip && vep->o_skip > 0)
-		vep_emit_common(vep, &vep->o_skip, 1);
 
-	/* In debug mode, the pending bytes are emitted separately.  */
-	if ((params->esi_syntax & 0x10) && vep->o_pending) {
-		vep->o_crc = vep->o_pending;
-		vep->crc = vep->crcp;
-		vep->crcp = crc32(0L, Z_NULL, 0);
-		vep_emit_common(vep, &vep->o_pending, skip);
-	}
+	if (vep->last_mark != mark && vep->o_wait > 0)
+		vep_emit_common(vep, &vep->o_wait, vep->last_mark);
 
-	/* Transfer pending bytes CRC into active mode.  */
+	/* Transfer pending bytes CRC into active mode CRC */
 	if (vep->o_pending) {
 		if (vep->o_crc == 0) {
 			vep->crc = vep->crcp;
@@ -342,9 +341,7 @@
 		vep->crcp = crc32(0L, Z_NULL, 0);
 	}
 
-	/*
-	 * Process this bit of input
-	 */
+	/* * Process this bit of input */
 	AN(vep->ver_p);
 	l = p - vep->ver_p;
 	assert(l >= 0);
@@ -352,40 +349,26 @@
 	vep->o_crc += l;
 	vep->ver_p = p;
 
-	if ((params->esi_syntax & 0x10) && l > 0) {
-		/* Emit right away if debug mode */
-		vep_emit_common(vep, &l, skip);
-	} else if (skip) {
-		vep->o_skip += vep->o_pending;
-		vep->o_skip += l;
-	} else {
-		vep->o_verbatim += vep->o_pending;
-		vep->o_verbatim += l;
-	}
+	vep->o_wait += vep->o_pending;
+	vep->o_wait += l;
 	vep->o_pending = 0;
+	vep->last_mark = mark;
 }
 
-
 static void
 vep_mark_verbatim(struct vep_state *vep, const char *p)
 {
-	if (vep->last_mark == 0 && p == vep->ver_p)
-		return;
-//printf("MARK VERB %d %s <%.*s>\n", vep->remove, vep->state, (int)(p - vep->ver_p), vep->ver_p);
-	vep_mark_common(vep, p, 0);
+
+	vep_mark_common(vep, p, VERBATIM);
 	vep->nm_verbatim++;
-	vep->last_mark = 0;
 } 
 
 static void
 vep_mark_skip(struct vep_state *vep, const char *p)
 {
-	if (vep->last_mark == 1 && p == vep->ver_p)
-		return;
-//printf("MARK SKIP %d %s <%.*s>\n", vep->remove, vep->state, (int)(p - vep->ver_p), vep->ver_p);
-	vep_mark_common(vep, p, 1);
+
+	vep_mark_common(vep, p, SKIP);
 	vep->nm_skip++;
-	vep->last_mark = 1;
 } 
 
 static void
@@ -393,18 +376,15 @@
 {
 	ssize_t l;
 
-	assert (vep->last_mark != 2 || p != vep->ver_p);
 	AN(vep->ver_p);
 	l = p - vep->ver_p;
 	assert(l > 0);
-// printf("MARK PEND %d %s <%.*s>\n", vep->remove, vep->state, (int)l, vep->ver_p);
 	assert(l >= 0);
 	vep->crcp = crc32(vep->crcp, (const void *)vep->ver_p, l);
 	vep->ver_p = p;
 
 	vep->o_pending += l;
 	vep->nm_pending++;
-	vep->last_mark = 2;
 }
 
 /*---------------------------------------------------------------------
@@ -470,7 +450,7 @@
 		    "ESI 1.0 <esi:include> lacks src attr");
 		return;
 	}
-	XXXAN(vep->include_src);
+
 	/*
 	 * Strictly speaking, we ought to spit out any piled up skip before
 	 * emitting the VEC for the include, but objectively that makes no
@@ -480,7 +460,7 @@
 	 * The mark_skip() before calling dostuff should have taken
 	 * care of that.  Make sure.
 	 */
-	assert(vep->o_verbatim == 0);
+	assert(vep->o_wait == 0 || vep->last_mark == SKIP);
 	/* XXX: what if it contains NUL bytes ?? */
 	p = vsb_data(vep->include_src);
 	l = vsb_len(vep->include_src);
@@ -1077,20 +1057,12 @@
 	CHECK_OBJ_NOTNULL(vep, VEP_MAGIC);
 
 	l = sp->obj->len - vep->o_total;
-	Debug("ENDING STATE: %s (skip %jd) (verbatim %jd)"
-	    " (tot %jd) (len %jd) (diff %jd)\n",
-	    vep->state,
-	    (intmax_t)vep->o_skip, (intmax_t)vep->o_verbatim,
-	    (intmax_t)vep->o_total, (intmax_t)sp->obj->len, (intmax_t)l);
 	assert(l >= 0);
 	if (vep->o_pending)
-		vep_mark_verbatim(vep, vep->ver_p);
-	if (vep->o_skip > 0)
-		vep_emit_common(vep, &vep->o_skip, 1);
-	else if (vep->o_verbatim > 0)
-		vep_emit_common(vep, &vep->o_verbatim, 0);
+		vep_mark_common(vep, vep->ver_p, vep->last_mark);
+	if (vep->o_wait > 0)
+		vep_emit_common(vep, &vep->o_wait, vep->last_mark);
 
-printf("NMARK VER %u SKIP %u PEND %u\n", vep->nm_verbatim, vep->nm_skip, vep->nm_pending);
 	vsb_finish(vep->vsb);
 	l = vsb_len(vep->vsb);
 	if (vep->state != VEP_NOTXML && l > 0) {

Modified: trunk/varnish-cache/bin/varnishd/mgt_param.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/mgt_param.c	2011-01-18 14:45:15 UTC (rev 5762)
+++ trunk/varnish-cache/bin/varnishd/mgt_param.c	2011-01-19 09:37:51 UTC (rev 5763)
@@ -644,7 +644,6 @@
 		"  0x00000002 - Ignore non-esi elements\n"
 		"  0x00000004 - Emit parsing debug records\n"
 		"  0x00000008 - Force-split parser input (debugging)\n"
-		"  0x00000010 - Don't coalesce VEC string (debugging)\n"
 		"Use 0x notation and do the bitor in your head :-)\n",
 		0,
 		"0", "bitmap" },




From phk at varnish-cache.org  Wed Jan 19 09:43:21 2011
From: phk at varnish-cache.org (phk at varnish-cache.org)
Date: Wed, 19 Jan 2011 10:43:21 +0100
Subject: r5764 - trunk/varnish-cache/bin/varnishd
Message-ID: <E1PfUZV-0004S0-PF@project.varnish-software.com>

Author: phk
Date: 2011-01-19 10:43:19 +0100 (Wed, 19 Jan 2011)
New Revision: 5764

Removed:
   trunk/varnish-cache/bin/varnishd/cache_esi.c
Modified:
   trunk/varnish-cache/bin/varnishd/Makefile.am
   trunk/varnish-cache/bin/varnishd/cache.h
   trunk/varnish-cache/bin/varnishd/cache_center.c
   trunk/varnish-cache/bin/varnishd/cache_esi_deliver.c
   trunk/varnish-cache/bin/varnishd/cache_esi_parse.c
   trunk/varnish-cache/bin/varnishd/cache_hash.c
Log:
Retire the old ESI code.



Modified: trunk/varnish-cache/bin/varnishd/Makefile.am
===================================================================
--- trunk/varnish-cache/bin/varnishd/Makefile.am	2011-01-19 09:37:51 UTC (rev 5763)
+++ trunk/varnish-cache/bin/varnishd/Makefile.am	2011-01-19 09:43:19 UTC (rev 5764)
@@ -27,7 +27,6 @@
 	cache_dir_round_robin.c \
 	cache_esi_parse.c \
 	cache_esi_deliver.c \
-	cache_esi.c \
 	cache_expire.c \
 	cache_fetch.c \
 	cache_gzip.c \

Modified: trunk/varnish-cache/bin/varnishd/cache.h
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache.h	2011-01-19 09:37:51 UTC (rev 5763)
+++ trunk/varnish-cache/bin/varnishd/cache.h	2011-01-19 09:43:19 UTC (rev 5764)
@@ -35,8 +35,6 @@
  */
 #define VARNISH_CACHE_CHILD	1
 
-#undef OLD_ESI 
-
 #include <sys/time.h>
 #include <sys/uio.h>
 #include <sys/socket.h>
@@ -94,9 +92,6 @@
 struct objcore;
 struct storage;
 struct workreq;
-#ifdef OLD_ESI
-struct esidata;
-#endif
 struct vrt_backend;
 struct cli_proto;
 struct ban;
@@ -444,11 +439,7 @@
 
 	VTAILQ_HEAD(, storage)	store;
 
-#ifdef OLD_ESI
-	struct esidata		*esidata;
-#else
 	struct storage		*esidata;
-#endif
 
 	double			last_use;
 
@@ -807,16 +798,7 @@
 char *VRT_String(struct ws *ws, const char *h, const char *p, va_list ap);
 char *VRT_StringList(char *d, unsigned dl, const char *p, va_list ap);
 
-#ifdef OLD_ESI
-/* cache_vrt_esi.c */
-
 void ESI_Deliver(struct sess *);
-void ESI_Destroy(struct object *);
-void ESI_Parse(struct sess *);
-#else
-void ESI_Deliver(struct sess *);
-#endif /* OLD_ESI */
-void ESI_Include(struct sess *sp, const char *src, const char *host);
 
 /* cache_vrt_vmod.c */
 void VMOD_Init(void);

Modified: trunk/varnish-cache/bin/varnishd/cache_center.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_center.c	2011-01-19 09:37:51 UTC (rev 5763)
+++ trunk/varnish-cache/bin/varnishd/cache_center.c	2011-01-19 09:43:19 UTC (rev 5764)
@@ -587,11 +587,9 @@
 
 	AZ(sp->wrk->vfp);
 	/* XXX: precedence, also: do_esi */
-#ifndef OLD_ESI
 	if (sp->wrk->do_esi) {
 		sp->wrk->vfp = &vfp_esi;
 	} else
-#endif
 	if (sp->wrk->do_gunzip &&
 	    http_HdrIs(sp->wrk->beresp, H_Content_Encoding, "gzip")) {
 		http_Unset(sp->wrk->beresp, H_Content_Encoding);
@@ -677,11 +675,6 @@
 		return (0);
 	}
 
-#ifdef OLD_ESI
-	if (sp->wrk->do_esi)
-		ESI_Parse(sp);
-#endif
-
 	switch (sp->handling) {
 	case VCL_RET_RESTART:
 		HSH_Drop(sp);

Deleted: trunk/varnish-cache/bin/varnishd/cache_esi.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_esi.c	2011-01-19 09:37:51 UTC (rev 5763)
+++ trunk/varnish-cache/bin/varnishd/cache_esi.c	2011-01-19 09:43:19 UTC (rev 5764)
@@ -1,897 +0,0 @@
-/*-
- * Copyright (c) 2006 Verdens Gang AS
- * Copyright (c) 2006-2010 Linpro AS
- * All rights reserved.
- *
- * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- *
- * Runtime support for compiled VCL programs ESI processing.
- *
- * The basic ESI 1.0 is a very simple specification:
- *	http://www.w3.org/TR/esi-lang
- * But it seems that Oracle and Akamai has embrodiered it to be almost a new
- * layer of scripting language in HTTP transmission chains.
- *
- * It is not obvious how much help the "advanced" features of ESI really
- * are to users, so our aim is to pick the fruit starting with the lowest
- * hanging, esi:include
- */
-
-
-#include "config.h"
-
-#include "svnid.h"
-SVNID("$Id$")
-
-#include <stdio.h>
-#include <unistd.h>
-#include <ctype.h>
-#include <string.h>
-#include <stdlib.h>
-#include <stdarg.h>
-
-#include "vrt.h"
-#include "vcl.h"
-#include "vct.h"
-#include "cache.h"
-#include "stevedore.h"
-
-#ifdef OLD_ESI
-
-/*--------------------------------------------------------------------*/
-
-struct esi_bit {
-	VTAILQ_ENTRY(esi_bit)	list;
-	char			chunk_length[20];
-	txt			verbatim;
-	txt			host;
-	txt			include;
-};
-
-struct esi_ptr {
-	const char		*p;
-	const char              *e;
-	struct storage		*st;
-};
-
-struct esi_work {
-	struct sess		*sp;
-	size_t			off;
-
-	struct esi_ptr		s;
-	struct esi_ptr		p;
-
-	txt			tag;
-
-	txt			t;
-	struct esi_bit		*eb;
-	int			remflg;	/* inside <esi:remove> </esi:remove> */
-	int			incmt;	/* inside <!--esi ... --> comment */
-
-	unsigned		space;	/* ... needed */
-
-	VTAILQ_HEAD(, esi_bit)	esibits;
-
-};
-
-struct esidata	{
-	unsigned		magic;
-#define ESIDATA_MAGIC		0x7255277f
-	VTAILQ_HEAD(, esi_bit)	esibits;
-	struct storage		*storage;
-};
-
-/*--------------------------------------------------------------------
- * Move an esi_ptr one char forward
- */
-
-static void
-Nep(struct esi_ptr *ep)
-{
-	static const char * const finis = "";
-
-	if (ep->p == finis)
-		return;
-	ep->p++;
-	if (ep->p < (char*)ep->st->ptr + ep->st->len)
-		return;
-	ep->st = VTAILQ_NEXT(ep->st, list);
-	if (ep->st != NULL) {
-		ep->p = (char *)ep->st->ptr;
-		ep->e = ep->p + ep->st->len;
-		return;
-	}
-	ep->p = finis;
-	ep->e = finis;
-	return;
-}
-
-/*--------------------------------------------------------------------
- * Consume one input character.
- */
-
-static void
-N(struct esi_work *ew)
-{
-
-	if (ew->p.p < ew->p.e)
-		ew->off++;
-	Nep(&ew->p);
-}
-
-/*--------------------------------------------------------------------
- * Strcmp for objects pointers
- */
-
-static int
-CMP(const struct esi_ptr *ep, const char *str)
-{
-	struct esi_ptr p2;
-
-	for (p2 = *ep; *str == *p2.p; str++)
-		Nep(&p2);
-	return (*str);
-}
-
-
-/*--------------------------------------------------------------------
- * Replace the mandatory XML 1.0 entity references, in place.
- */
-
-static void
-XMLentity(txt *t)
-{
-	char *s, *d;
-
-	for (s = d = t->b; s < t->e; ) {
-		if (*s == '&') {
-#define R(l,f,r)							\
-			if (s + l <= t->e && !memcmp(s, f, l)) {	\
-				*d++ = r;				\
-				s += l;					\
-				continue;				\
-			}
-			R(6, "&apos;", '\'');
-			R(6, "&quot;", '"');
-			R(4, "&lt;", '<');
-			R(4, "&gt;", '>');
-			R(5, "&amp;", '&');
-		}
-#undef R
-		*d++ = *s++;
-	}
-	t->e = d;
-	t->e[0] = '\0';
-}
-
-
-/*--------------------------------------------------------------------
- * Report a parsing error
- *
- * XXX: The "at xxx" count is usually the tail of the sequence.  Since we
- * XXX: wander over the storage in an oderly manner now, we could keep
- * XXX: track of line+pos and record the beginning of the stuff that
- * XXX: offends os in the central dispatch loop.
- * XXX: This is left a an excercise for the reader.
- */
-
-static void
-esi_error(const struct esi_work *ew, const char *p, int i, const char *err)
-{
-	int ellipsis = 0;
-	char buf[256], *q;
-	txt t;
-
-	VSC_main->esi_errors++;
-	if (i == 0)
-		i = p - ew->t.b;
-	if (i > 20) {
-		i = 20;
-		ellipsis = 1;
-	}
-	q = buf;
-	q += sprintf(buf, "at %zu: %s \"", ew->off, err);
-	while (i > 0) {
-		if (*p >= ' ' && *p <= '~') {
-			*q++ = *p;
-		} else if (*p == '\n') {
-			*q++ = '\\';
-			*q++ = 'n';
-		} else if (*p == '\r') {
-			*q++ = '\\';
-			*q++ = 'r';
-		} else if (*p == '\t') {
-			*q++ = '\\';
-			*q++ = 't';
-		} else {
-			/* XXX: use %%%02x instead ? */
-			q += sprintf(q, "\\x%02x", *p & 0xff);
-		}
-		p++;
-		i--;
-	}
-	if (ellipsis) {
-		*q++ = '[';
-		*q++ = '.';
-		*q++ = '.';
-		*q++ = '.';
-		*q++ = ']';
-	}
-	*q++ = '"';
-	*q++ = '\0';
-	t.b = buf;
-	t.e = q;
-	WSPR(ew->sp, SLT_ESI_xmlerror, t);
-}
-
-/*--------------------------------------------------------------------
- * Add ESI bit to object
- */
-
-static void
-esi_addbit(struct esi_work *ew, const char *verbatim, unsigned len)
-{
-
-	ew->space += sizeof(*ew->eb);
-	ew->eb = (void*)WS_Alloc(ew->sp->wrk->ws, sizeof *ew->eb);
-	AN(ew->eb);
-	memset(ew->eb, 0, sizeof *ew->eb);
-
-	VTAILQ_INSERT_TAIL(&ew->esibits, ew->eb, list);
-	if (verbatim != NULL) {
-		ew->eb->verbatim.b = TRUST_ME(verbatim);
-		if (len > 0)
-			ew->eb->verbatim.e = TRUST_ME(verbatim + len);
-		sprintf(ew->eb->chunk_length, "%x\r\n", Tlen(ew->eb->verbatim));
-		if (params->esi_syntax & 0x4)
-			WSP(ew->sp, SLT_Debug, "AddBit: %d <%.*s>",
-			    Tlen(ew->eb->verbatim),
-			    Tlen(ew->eb->verbatim),
-			    ew->eb->verbatim.b);
-	} else {
-		AN(ew->s.p);
-		ew->eb->verbatim.b = ew->eb->verbatim.e = TRUST_ME(ew->s.p);
-	}
-}
-
-/*--------------------------------------------------------------------*/
-
-static void
-esi_addpfx(struct esi_work *ew)
-{
-	const char *ep;
-
-	if (ew->remflg) {
-		/* In <esi:remove...> don't add anything */
-		ew->s = ew->p;
-		return;
-	}
-	while (ew->s.st != ew->p.st) {
-		ep = (const char *)(ew->s.st->ptr + ew->s.st->len);
-		esi_addbit(ew, ew->s.p, ep - ew->s.p);
-		ew->s.p = ep;
-		Nep(&ew->s);
-	}
-	if (ew->s.st != NULL && ew->p.p != ew->s.p)
-		esi_addbit(ew, ew->s.p, ew->p.p - ew->s.p);
-	ew->s.p = ew->p.p;
-}
-
-/*--------------------------------------------------------------------
- * Tease the next attribute and value out of an XML element.
- *
- * XXX: is the syntax correct ?
- */
-
-static int
-esi_attrib(const struct esi_work *ew, txt *in, txt *attrib, txt *val)
-{
-
-	AN(*in->b);
-	/* Skip leading blanks */
-	while(in->b < in->e && isspace(*in->b))
-		in->b++;
-
-	/* Nothing found */
-	if (in->b >= in->e)
-		return (0);
-
-	if (!vct_isxmlnamestart(*in->b)) {
-		/* XXX error */
-		esi_error(ew, in->b, 1,
-		    "XML 1.0 Illegal attribute start character");
-		return (-1);
-	}
-
-	/* Attribute name until '=' or space */
-	*attrib = *in;
-	while(in->b < in->e && *in->b != '=' && !isspace(*in->b)) {
-		if (!vct_isxmlname(*in->b)) {
-			esi_error(ew, attrib->b, 1 + (in->b - attrib->b),
-			    "XML 1.0 Illegal attribute character");
-			return (-1);
-		}
-		in->b++;
-	}
-	attrib->e = in->b;
-
-	if (in->b >= in->e || isspace(*in->b)) {
-		/* Attribute without value */
-		val->b = val->e = in->b;
-		return (1);
-	}
-
-	/* skip '=' */
-	in->b++;
-
-	if (isspace(*in->b)) {
-		val->e = val->b = in->b;
-		in->b++;
-		return (1);
-	}
-
-	/* Value, if any ? */
-	*val = *in;
-	if (in->b >= in->e)
-		return (1);
-
-	if (*in->b == '"') {
-		/* Skip quote */
-		in->b++;
-		val->b++;
-
-		/* Anything goes, until next quote */
-		while(in->b < in->e && *in->b != '"')
-			in->b++;
-		val->e = in->b;
-
-		if (in->b >= in->e) {
-			esi_error(ew, val->b, in->e - val->b,
-			    "XML 1.0 missing ending quote");
-			return (-1);
-		}
-
-		/* Skip quote */
-		in->b++;
-	} else {
-		/* Anything until whitespace */
-		while(in->b < in->e && !isspace(*in->b))
-			in->b++;
-		val->e = in->b;
-		in->b++;
-	}
-	return (1);
-}
-
-/*--------------------------------------------------------------------
- * Add one piece to the output, either verbatim or include
- */
-
-static void
-esi_handle_include(struct esi_work *ew)
-{
-	struct esi_bit *eb;
-	char *p, *q, *c;
-	txt t = ew->tag;
-	txt tag;
-	txt val;
-	unsigned u, v, s;
-
-	if (ew->eb == NULL || ew->eb->include.b != NULL)
-		esi_addbit(ew, NULL, 0);
-	eb = ew->eb;
-	WSP(ew->sp, SLT_Debug, "Incl \"%.*s\"", t.e - t.b, t.b);
-	while (esi_attrib(ew, &t, &tag, &val) == 1) {
-		if (params->esi_syntax & 0x4)
-			WSP(ew->sp, SLT_Debug, "<%.*s> -> <%.*s>",
-			    tag.e - tag.b, tag.b, val.e - val.b, val.b);
-		if (Tlen(tag) != 3 || memcmp(tag.b, "src", 3))
-			continue;
-		if (Tlen(val) == 0) {
-			esi_error(ew, tag.b, Tlen(tag),
-			    "ESI esi:include src attribute without value");
-			continue;
-		}
-
-		/* We are saving the original string */
-		s = 0;
-
-		if (val.b != val.e) {
-			s = Tlen(val) + 1;
-			c = WS_Alloc(ew->sp->wrk->ws, s);
-			XXXAN(c);
-			memcpy(c, val.b, Tlen(val));
-			val.b = c;
-			val.e = val.b + s;
-			val.e[-1] = '\0';
-		}
-
-		if (strchr(val.b, '&'))
-			XMLentity(&val);
-
-		if (Tlen(val) > 7 && !memcmp(val.b, "http://", 7)) {
-			/*  Rewrite to Host: header inplace */
-			eb->host.b = val.b;
-			memcpy(eb->host.b, "Host: ", 6);
-			q = eb->host.b + 6;
-			for (p = eb->host.b + 7; p < val.e && *p != '/'; p++)
-				*q++ = *p;
-			*q++ = '\0';
-			eb->host.e = q;
-			assert(*p == '/');	/* XXX */
-			/* The rest is the URL */
-			eb->include.b = p;
-			eb->include.e = val.e;
-		} else if (Tlen(val) > 0 && *val.b == '/') {
-			/* Absolute on this host */
-			eb->include = val;
-		} else {
-
-			/*
-			 * Decision:  We interpret the relative URL against
-			 * the actual URL we asked the backend for.
-			 * The client's request URL may be entirely
-			 * different and have been rewritten underway.
-			 */
-			CHECK_OBJ_NOTNULL(ew->sp, SESS_MAGIC);
-			CHECK_OBJ_NOTNULL(ew->sp->wrk->bereq, HTTP_MAGIC);
-			tag = ew->sp->wrk->bereq->hd[HTTP_HDR_URL];
-
-			/* Use the objects WS to store the result */
-			CHECK_OBJ_NOTNULL(ew->sp->obj, OBJECT_MAGIC);
-
-			/* Look for the last '/' before a '?' */
-			q = NULL;
-			for (p = tag.b; p < tag.e && *p != '?'; p++)
-				if (*p == '/')
-					q = p;
-			if (q != NULL)
-				tag.e = q + 1;
-
-			u = WS_Reserve(ew->sp->wrk->ws, 0);
-			v = snprintf(ew->sp->wrk->ws->f, u - 1, "%.*s%.*s",
-			    pdiff(tag.b, tag.e), tag.b,
-			    pdiff(val.b, val.e), val.b);
-			v++;
-			xxxassert(v < u);
-			eb->include.b = ew->sp->wrk->ws->f;
-			eb->include.e = ew->sp->wrk->ws->f + v;
-			WS_Release(ew->sp->wrk->ws, v);
-		}
-		if (eb->include.b != NULL)
-			ew->space += Tlen(eb->include);
-		if (eb->host.b != NULL)
-			ew->space += Tlen(eb->host);
-	}
-}
-
-/*--------------------------------------------------------------------
- * See if this looks like XML: first non-white char must be '<'
- */
-
-static int
-looks_like_xml(const struct object *obj) {
-	struct storage *st;
-	unsigned u;
-
-	VTAILQ_FOREACH(st, &obj->store, list) {
-		AN(st);
-		for (u = 0; u < st->len; u++) {
-			if (isspace(st->ptr[u]))
-				continue;
-			if (st->ptr[u] == '<')
-				return (1);
-			else
-				return (0);
-		}
-	}
-	return (0);
-}
-
-/*--------------------------------------------------------------------
- * A quick stroll through the object, to find out if it contains any
- * esi sequences at all.
- */
-
-static int
-contain_esi(const struct object *obj) {
-	struct storage *st;
-	unsigned u;
-	const char *r, *r2;
-	static const char * const wanted = "<esi:";
-	static const char * const wanted2 = "<!--esi";
-
-	/*
-	 * Do a fast check to see if there is any '<esi:' sequences at all
-	 */
-	r = wanted;
-	r2 = wanted2;
-	VTAILQ_FOREACH(st, &obj->store, list) {
-		AN(st);
-		for (u = 0; u < st->len; u++) {
-			if (st->ptr[u] != *r) {
-				r = wanted;
-			} else if (*++r == '\0')
-				return (1);
-			if (st->ptr[u] != *r2) {
-				r2 = wanted2;
-			} else if (*++r2 == '\0')
-				return (1);
-		}
-	}
-	return (0);
-}
-
-/*--------------------------------------------------------------------*/
-
-static void
-parse_esi_comment(struct esi_work *ew)
-{
-
-	esi_addpfx(ew);
-
-	N(ew); N(ew); N(ew); N(ew); N(ew); N(ew); N(ew);
-	assert(!ew->incmt);
-	ew->incmt = 1;
-	ew->s.p = ew->p.p;
-}
-
-/*--------------------------------------------------------------------*/
-
-static void
-parse_comment(struct esi_work *ew)
-{
-
-	do {
-		N(ew);
-		if (*ew->p.p == '-' && !CMP(&ew->p, "-->")) {
-			N(ew);
-			N(ew);
-			N(ew);
-			break;
-		}
-	} while (ew->p.p < ew->p.e);
-}
-
-/*--------------------------------------------------------------------*/
-
-static void
-parse_cdata(struct esi_work *ew)
-{
-
-	esi_addpfx(ew);
-
-	do {
-		N(ew);
-		if (*ew->p.p == ']' && !CMP(&ew->p, "]]>")) {
-			N(ew);
-			N(ew);
-			N(ew);
-			break;
-		}
-	} while (ew->p.p < ew->p.e);
-}
-
-/*--------------------------------------------------------------------*/
-
-static void
-parse_esi_tag(struct esi_work *ew, int closing)
-{
-	int l, ll, empty;
-	struct esi_ptr px;
-	char *q;
-
-	esi_addpfx(ew);
-
-	do
-		N(ew);
-	while (*ew->p.p != '>' && ew->p.p < ew->p.e);
-	if (ew->p.p == ew->p.e) {
-		esi_addpfx(ew);
-		esi_error(ew, ew->s.p, 0,
-		    "XML 1.0 incomplete language element");
-		return;
-	}
-	N(ew);
-
-	if (ew->p.st == ew->s.st) {
-		ew->tag.b = TRUST_ME(ew->s.p);
-		ew->tag.e = TRUST_ME(ew->p.p);
-	} else {
-		/*
-		 * The element is spread over more than one storage
-		 * segment, pull it together in the object workspace
-		 * XXX: Ideally, we should only pull together the bits
-		 * XXX: we need, like the filename.
-		 */
-		ew->tag.b = ew->sp->wrk->ws->f;
-		ew->tag.e = ew->tag.b + WS_Reserve(ew->sp->wrk->ws, 0);
-		px = ew->s;
-		q = ew->tag.b;
-		while (px.p != ew->p.p) {
-			xxxassert(q < ew->tag.e);
-			*q++ = *px.p;
-			Nep(&px);
-		}
-		ew->tag.e = q;
-		WS_Release(ew->sp->wrk->ws, Tlen(ew->tag));
-	}
-	ll = Tlen(ew->tag);
-	ew->tag.b++;
-	ew->tag.e--;
-	empty = (ew->tag.e[-1] == '/') ? 1 : 0;
-	if (empty)
-		ew->tag.e--;
-
-	if (empty && closing)
-		esi_error(ew, ew->s.p, ll,
-		    "XML 1.0 empty and closing element");
-
-	ew->tag.b += 4 + (closing ? 1 : 0);
-	l = Tlen(ew->tag);
-	WSP(ew->sp, SLT_Debug,
-	    "tag {%.*s} %d %d %d", l, ew->tag.b, ew->remflg, empty, closing);
-	if (l >= 6 && !memcmp(ew->tag.b, "remove", 6)) {
-		if (empty) {
-			/* XXX ?? */
-		} else if (closing) {
-			if (!ew->remflg)
-				esi_error(ew, ew->s.p, ll,
-				    "ESI 1.0 esi:remove not opened");
-			ew->remflg = 0;
-		} else {
-			if (ew->remflg)
-				esi_error(ew, ew->s.p, ll,
-				    "ESI 1.0 forbids nested esi:remove");
-			ew->remflg = 1;
-		}
-	} else if (ew->remflg) {
-		esi_error(ew, ew->s.p, ll,
-		    "ESI 1.0 forbids esi: elements inside esi:remove");
-	} else if (l >= 7 && !memcmp(ew->tag.b, "comment", 7)) {
-		if (closing)
-			esi_error(ew, ew->s.p, ll,
-			    "ESI 1.0 closing esi:comment illegal");
-		else if (!empty)
-			esi_error(ew, ew->s.p, ll,
-			    "ESI 1.0 wants empty esi:comment");
-	} else if (l >= 7 && !memcmp(ew->tag.b, "include", 7)) {
-		if (closing) {
-			esi_error(ew, ew->s.p, ll,
-			    "ESI 1.0 closing esi:include illegal");
-		} else if (!empty) {
-			esi_error(ew, ew->s.p, ll,
-			    "ESI 1.0 wants empty esi:include");
-		}
-		ew->tag.b += 7;
-		esi_handle_include(ew);
-	} else {
-		esi_error(ew, ew->s.p, ll,
-		    "ESI 1.0 unimplemented element");
-	}
-	ew->s = ew->p;
-}
-
-/*--------------------------------------------------------------------*/
-
-void
-ESI_Parse(struct sess *sp)
-{
-	struct esi_work *ew, eww[1];
-	struct esi_bit *eb;
-	struct esidata *ed;
-	struct storage *st;
-	unsigned u;
-	char *hack;
-
-	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
-	CHECK_OBJ_NOTNULL(sp->obj, OBJECT_MAGIC);
-	AssertObjPassOrBusy(sp->obj);
-	if (VTAILQ_EMPTY(&sp->obj->store))
-		return;
-
-	if (!(params->esi_syntax & 0x00000001)) {
-		/*
-		 * By default, we will not ESI process an object where
-		 *  the first non-space character is different from '<'
-		 */
-		if (!looks_like_xml(sp->obj)) {
-			WSP(sp, SLT_ESI_xmlerror,
-			    "No ESI processing, first char not '<'");
-			return;
-		}
-	}
-
-	/*
-	 * Do a fast check to see if there is any '<esi:' sequences at all
-	 */
-	if (!contain_esi(sp->obj))
-		return;
-
-	/* XXX: debugging hack */
-	hack = sp->wrk->ws->f;
-
-	VSC_main->esi_parse++;
-	/* XXX: only if GET ? */
-	ew = eww;
-	memset(eww, 0, sizeof eww);
-	VTAILQ_INIT(&ew->esibits);
-	ew->sp = sp;
-	ew->off = 1;
-
-	ew->space += sizeof(struct esidata);
-
-	ew->p.st = VTAILQ_FIRST(&sp->obj->store);
-	AN(ew->p.st);
-	ew->p.p = (char *)ew->p.st->ptr;
-	ew->p.e = ew->p.p + ew->p.st->len;
-
-	/* ->s points to the first un-dealt-with byte */
-	ew->s = ew->p;
-
-	while (ew->p.p < ew->p.e) {
-
-		if (ew->incmt && *ew->p.p == '-' && !CMP(&ew->p, "-->")) {
-			/* End of ESI comment */
-			esi_addpfx(ew);
-			N(ew);
-			N(ew);
-			N(ew);
-			ew->s = ew->p;
-			ew->incmt = 0;
-			continue;
-		}
-		/* Skip forward to the first '<' */
-		if (*ew->p.p != '<') {
-			N(ew);
-			continue;
-		}
-
-		if (!CMP(&ew->p, "<!--esi")) {
-			parse_esi_comment(ew);
-		} else if (!CMP(&ew->p, "<!--")) {
-			parse_comment(ew);
-		} else if (!CMP(&ew->p, "</esi")) {
-			parse_esi_tag(ew, 1);
-		} else if (!CMP(&ew->p, "<esi:")) {
-			parse_esi_tag(ew, 0);
-		} else if (!CMP(&ew->p, "<![CDATA[")) {
-			parse_cdata(ew);
-		} else {
-			/*
-			 * Something we don't care about, just skip it.
-			 */
-			N(ew);
-			if (!(params->esi_syntax & 0x2)) {
-				/* XXX: drop this ? */
-				do {
-					N(ew);
-				} while (*ew->p.p != '>' && ew->p.p < ew->p.e);
-			}
-		}
-	}
-	esi_addpfx(ew);
-
-	/*
-	 * XXX: we could record the starting point of these elements
-	 * XXX: so that the char-index were more useful, but we are
-	 * XXX: not trivially able to print their contents, so leave
-	 * XXX: it like this for now, pending more thought about the
-	 * XXX: proper way to report these errors.
-	 */
-	if (ew->remflg)
-		esi_error(ew, ew->t.e, -1,
-		    "ESI 1.0 unterminated <esi:remove> element");
-	if (ew->incmt)
-		esi_error(ew, ew->t.e, -1,
-		    "ESI 1.0 unterminated <!--esi comment");
-
-	st = STV_alloc(sp, ew->space);
-	AN(st);
-	assert(st->space >= ew->space);
-
-	ed = (void*)st->ptr;
-	st->len += sizeof(*ed);
-	memset(ed, 0, sizeof *ed);
-	ed->magic = ESIDATA_MAGIC;
-	ed->storage = st;
-	VTAILQ_INIT(&ed->esibits);
-
-	while (!VTAILQ_EMPTY(&ew->esibits)) {
-		eb = VTAILQ_FIRST(&ew->esibits);
-		VTAILQ_REMOVE(&ew->esibits, eb, list);
-		memcpy(st->ptr + st->len, eb, sizeof *eb);
-		eb = (void*)(st->ptr + st->len);
-		st->len += sizeof(*eb);
-
-		if (eb->include.b != NULL) {
-			u = Tlen(eb->include);
-			memcpy(st->ptr + st->len, eb->include.b, u);
-			eb->include.b = (void*)(st->ptr + st->len);
-			eb->include.e = eb->include.b + u;
-			st->len += u;
-		}
-		if (eb->host.b != NULL) {
-			u = Tlen(eb->host);
-			memcpy(st->ptr + st->len, eb->host.b, u);
-			eb->host.b = (void*)(st->ptr + st->len);
-			eb->host.e = eb->host.b + u;
-			st->len += u;
-		}
-
-		VTAILQ_INSERT_TAIL(&ed->esibits, eb, list);
-	}
-
-	assert(st->len <= st->space);
-	assert(st->len == ew->space);
-	sp->obj->esidata = ed;
-
-	memset(hack, 0xaa, sp->wrk->ws->f - hack);
-}
-
-/*--------------------------------------------------------------------*/
-
-void
-ESI_Deliver(struct sess *sp)
-{
-	struct esi_bit *eb;
-	struct worker *w;
-	struct esidata *ed;
-
-	w = sp->wrk;
-	ed = sp->obj->esidata;
-	CHECK_OBJ_NOTNULL(ed, ESIDATA_MAGIC);
-	VTAILQ_FOREACH(eb, &ed->esibits, list) {
-		if (Tlen(eb->verbatim)) {
-			if (sp->wrk->res_mode & RES_CHUNKED)
-				(void)WRW_Write(w, eb->chunk_length, -1);
-			sp->acct_tmp.bodybytes += WRW_Write(w,
-			    eb->verbatim.b, Tlen(eb->verbatim));
-			if (sp->http->protover >= 1.1)
-				(void)WRW_Write(w, "\r\n", -1);
-		}
-		if (eb->include.b == NULL ||
-		    sp->esi_level >= params->max_esi_includes)
-			continue;
-
-		ESI_Include(sp, eb->include.b, eb->host.b);
-		if (sp->fd < 0)
-			break;
-	}
-}
-
-/*--------------------------------------------------------------------*/
-
-void
-ESI_Destroy(struct object *o)
-{
-	struct esidata *ed;
-
-	ed = o->esidata;
-	o->esidata = NULL;
-	CHECK_OBJ_NOTNULL(ed, ESIDATA_MAGIC);
-	STV_free(ed->storage);
-}
-
-#endif /* OLD_ESI */

Modified: trunk/varnish-cache/bin/varnishd/cache_esi_deliver.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_esi_deliver.c	2011-01-19 09:37:51 UTC (rev 5763)
+++ trunk/varnish-cache/bin/varnishd/cache_esi_deliver.c	2011-01-19 09:43:19 UTC (rev 5764)
@@ -44,7 +44,7 @@
 
 /*--------------------------------------------------------------------*/
 
-void
+static void
 ESI_Include(struct sess *sp, const char *src, const char *host)
 {
 	struct object *obj;
@@ -120,7 +120,6 @@
 
 /*--------------------------------------------------------------------*/
 
-#ifndef OLD_ESI
 
 //#define Debug(fmt, ...) printf(fmt, __VA_ARGS__)
 #define Debug(fmt, ...) /**/
@@ -233,4 +232,3 @@
 	(void)WRW_Flush(sp->wrk);
 }
 
-#endif /* OLD_ESI */

Modified: trunk/varnish-cache/bin/varnishd/cache_esi_parse.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_esi_parse.c	2011-01-19 09:37:51 UTC (rev 5763)
+++ trunk/varnish-cache/bin/varnishd/cache_esi_parse.c	2011-01-19 09:43:19 UTC (rev 5764)
@@ -42,8 +42,6 @@
 #include "zlib.h"
 #include "stevedore.h"
 
-#ifndef OLD_ESI
-
 //#define Debug(fmt, ...) printf(fmt, __VA_ARGS__)
 #define Debug(fmt, ...) /**/
 
@@ -1098,8 +1096,6 @@
         .end    =       vfp_esi_end,
 };
 
-#endif /* OLD_ESI */
-
 #if 0
 
 digraph xml {

Modified: trunk/varnish-cache/bin/varnishd/cache_hash.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_hash.c	2011-01-19 09:37:51 UTC (rev 5763)
+++ trunk/varnish-cache/bin/varnishd/cache_hash.c	2011-01-19 09:43:19 UTC (rev 5764)
@@ -693,15 +693,10 @@
 		DSL(0x40, SLT_Debug, 0, "Object %u workspace min free %u",
 		    o->xid, WS_Free(o->ws_o));
 
-#ifdef OLD_ESI
-		if (o->esidata != NULL)
-			ESI_Destroy(o);
-#else
 		if (o->esidata != NULL) {
 			STV_free(o->esidata);
 			o->esidata = NULL;
 		}
-#endif
 		if (oc != NULL)
 			oc_freeobj(oc);
 		w->stats.n_object--;




From phk at varnish-cache.org  Wed Jan 19 09:59:57 2011
From: phk at varnish-cache.org (phk at varnish-cache.org)
Date: Wed, 19 Jan 2011 10:59:57 +0100
Subject: r5765 - trunk/varnish-cache/bin/varnishd
Message-ID: <E1PfUpZ-0004yj-N6@project.varnish-software.com>

Author: phk
Date: 2011-01-19 10:59:57 +0100 (Wed, 19 Jan 2011)
New Revision: 5765

Modified:
   trunk/varnish-cache/bin/varnishd/cache_esi_deliver.c
   trunk/varnish-cache/bin/varnishd/cache_esi_parse.c
Log:
Minor cleanups



Modified: trunk/varnish-cache/bin/varnishd/cache_esi_deliver.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_esi_deliver.c	2011-01-19 09:43:19 UTC (rev 5764)
+++ trunk/varnish-cache/bin/varnishd/cache_esi_deliver.c	2011-01-19 09:59:57 UTC (rev 5765)
@@ -25,6 +25,7 @@
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
  *
+ * VED - Varnish Esi Delivery
  */
 
 #include "config.h"
@@ -45,7 +46,7 @@
 /*--------------------------------------------------------------------*/
 
 static void
-ESI_Include(struct sess *sp, const char *src, const char *host)
+ved_include(struct sess *sp, const char *src, const char *host)
 {
 	struct object *obj;
 	struct worker *w;
@@ -125,7 +126,7 @@
 #define Debug(fmt, ...) /**/
 
 static void
-esi_sendchunk(const struct sess *sp, const void *cb, ssize_t cl,
+ved_sendchunk(const struct sess *sp, const void *cb, ssize_t cl,
     const void *ptr, ssize_t l)
 {
 
@@ -197,11 +198,8 @@
 			assert (q > p);
 			crc_ref = crc32(0L, Z_NULL, 0);
 			crc_ref = crc32(crc_ref, st->ptr + off, l);
-			if (crc_ref != crc) {
-				printf("CRC Mismatch %08x %08x\n", crc_ref, crc);
-			}
 			xxxassert(crc_ref == crc);
-			esi_sendchunk(sp, p, q - p, st->ptr + off, l);
+			ved_sendchunk(sp, p, q - p, st->ptr + off, l);
 			off += l;
 			p = q + 1;
 			break;
@@ -220,7 +218,7 @@
 			r = (void*)strchr((const char*)q, '\0');
 			AN(r);
 			Debug("INCL [%s][%s] BEGIN\n", q, p);
-			ESI_Include(sp, (const char*)q, (const char*)p);
+			ved_include(sp, (const char*)q, (const char*)p);
 			Debug("INCL [%s][%s] END\n", q, p);
 			p = r + 1;
 			break;

Modified: trunk/varnish-cache/bin/varnishd/cache_esi_parse.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_esi_parse.c	2011-01-19 09:43:19 UTC (rev 5764)
+++ trunk/varnish-cache/bin/varnishd/cache_esi_parse.c	2011-01-19 09:59:57 UTC (rev 5765)
@@ -25,6 +25,7 @@
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
  *
+ * VEP Varnish Esi Parsing
  */
 
 #include "config.h"




From phk at varnish-cache.org  Wed Jan 19 10:16:24 2011
From: phk at varnish-cache.org (phk at varnish-cache.org)
Date: Wed, 19 Jan 2011 11:16:24 +0100
Subject: r5766 - trunk/varnish-cache/bin/varnishd
Message-ID: <E1PfV5U-0005Xs-VL@project.varnish-software.com>

Author: phk
Date: 2011-01-19 11:16:19 +0100 (Wed, 19 Jan 2011)
New Revision: 5766

Modified:
   trunk/varnish-cache/bin/varnishd/cache_center.c
Log:
Unify header handling for gzip, and make wrk->do_g[un]zip usable
as intention flags.



Modified: trunk/varnish-cache/bin/varnishd/cache_center.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_center.c	2011-01-19 09:59:57 UTC (rev 5765)
+++ trunk/varnish-cache/bin/varnishd/cache_center.c	2011-01-19 10:16:19 UTC (rev 5766)
@@ -586,20 +586,33 @@
 	}
 
 	AZ(sp->wrk->vfp);
-	/* XXX: precedence, also: do_esi */
-	if (sp->wrk->do_esi) {
-		sp->wrk->vfp = &vfp_esi;
-	} else
+
+	/* We won't gunzip unless it is gzip'ed, if we do remove C-E header */
 	if (sp->wrk->do_gunzip &&
-	    http_HdrIs(sp->wrk->beresp, H_Content_Encoding, "gzip")) {
+	     !http_HdrIs(sp->wrk->beresp, H_Content_Encoding, "gzip"))
+		sp->wrk->do_gunzip = 0;
+	if (sp->wrk->do_gunzip) 
 		http_Unset(sp->wrk->beresp, H_Content_Encoding);
-		sp->wrk->vfp = &vfp_gunzip;
-	} else if (sp->wrk->do_gzip &&
-	    !http_HdrIs(sp->wrk->beresp, H_Content_Encoding, "gzip")) {
+
+
+	/* And we wont gzip if it already has a C-E header, if we do add it */
+	if (sp->wrk->do_gzip &&
+	     http_GetHdr(sp->wrk->beresp, H_Content_Encoding, NULL))
+		sp->wrk->do_gzip = 0;
+	if (sp->wrk->do_gzip) 
 		http_PrintfHeader(sp->wrk, sp->fd, sp->wrk->beresp,
 		    "Content-Encoding: %s", "gzip");
+
+	/* But we can't do both */
+	assert(sp->wrk->do_gzip == 0 || sp->wrk->do_gunzip == 0);
+
+	/* ESI takes precedence and handles gzip/gunzip also */
+	if (sp->wrk->do_esi)
+		sp->wrk->vfp = &vfp_esi;
+	else if (sp->wrk->do_gunzip)
+		sp->wrk->vfp = &vfp_gunzip;
+	else if (sp->wrk->do_gzip)
 		sp->wrk->vfp = &vfp_gzip;
-	}
 
 	l = http_EstimateWS(sp->wrk->beresp,
 	    sp->pass ? HTTPH_R_PASS : HTTPH_A_INS, &nhttp);




From phk at varnish-cache.org  Wed Jan 19 10:47:12 2011
From: phk at varnish-cache.org (phk at varnish-cache.org)
Date: Wed, 19 Jan 2011 11:47:12 +0100
Subject: r5767 - trunk/varnish-cache/bin/varnishd
Message-ID: <E1PfVZI-0006LT-Q0@project.varnish-software.com>

Author: phk
Date: 2011-01-19 11:47:11 +0100 (Wed, 19 Jan 2011)
New Revision: 5767

Modified:
   trunk/varnish-cache/bin/varnishd/cache.h
   trunk/varnish-cache/bin/varnishd/cache_center.c
Log:
Roll out the gzip/gunzip logic some more



Modified: trunk/varnish-cache/bin/varnishd/cache.h
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache.h	2011-01-19 10:16:19 UTC (rev 5766)
+++ trunk/varnish-cache/bin/varnishd/cache.h	2011-01-19 10:47:11 UTC (rev 5767)
@@ -282,7 +282,9 @@
 	void			*vfp_private;
 	unsigned		do_esi;
 	unsigned		do_gzip;
+	unsigned		is_gzip;
 	unsigned		do_gunzip;
+	unsigned		is_gunzip;
 
 	/* ESI stuff */
 	struct vep_state	*vep;

Modified: trunk/varnish-cache/bin/varnishd/cache_center.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_center.c	2011-01-19 10:16:19 UTC (rev 5766)
+++ trunk/varnish-cache/bin/varnishd/cache_center.c	2011-01-19 10:47:11 UTC (rev 5767)
@@ -587,26 +587,53 @@
 
 	AZ(sp->wrk->vfp);
 
-	/* We won't gunzip unless it is gzip'ed, if we do remove C-E header */
-	if (sp->wrk->do_gunzip &&
-	     !http_HdrIs(sp->wrk->beresp, H_Content_Encoding, "gzip"))
+	/*
+	 * The VCL variables beresp.do_g[un]zip tells us how we want the
+	 * object stored.
+	 *
+ 	 * The backend Content-Encoding header tells us what we are going
+	 * to receive, which we classify in the following three classes:
+	 *
+	 * 	"Content-Encoding: gzip"	--> object is gzip'ed.
+	 * 	no Content-Encoding		--> object is not gzip'ed.
+	 * 	anything else			--> do nothing wrt gzip
+	 *
+	 */
+
+	/* We do nothing unless the param is set */
+	if (!params->http_gzip_support)
+		sp->wrk->do_gzip = sp->wrk->do_gunzip = 0;
+
+	sp->wrk->is_gzip = 
+	    http_HdrIs(sp->wrk->beresp, H_Content_Encoding, "gzip");
+
+	sp->wrk->is_gunzip = 
+	    !http_GetHdr(sp->wrk->beresp, H_Content_Encoding, NULL);
+
+	/* It can't be both */
+	assert(sp->wrk->is_gzip == 0 || sp->wrk->is_gunzip == 0);
+
+	/* We won't gunzip unless it is gzip'ed */
+	if (sp->wrk->do_gunzip && !sp->wrk->is_gzip)
 		sp->wrk->do_gunzip = 0;
-	if (sp->wrk->do_gunzip) 
+
+	/* If we do gunzip, remove the C-E header */
+	if (sp->wrk->do_gunzip)
 		http_Unset(sp->wrk->beresp, H_Content_Encoding);
 
+	/* We wont gzip unless it is ungziped */
+	if (sp->wrk->do_gzip && !sp->wrk->is_gunzip)
+		sp->wrk->do_gzip = 0;
 
-	/* And we wont gzip if it already has a C-E header, if we do add it */
-	if (sp->wrk->do_gzip &&
-	     http_GetHdr(sp->wrk->beresp, H_Content_Encoding, NULL))
-		sp->wrk->do_gzip = 0;
+	/* If we do gzip, add the C-E header */
 	if (sp->wrk->do_gzip) 
 		http_PrintfHeader(sp->wrk, sp->fd, sp->wrk->beresp,
 		    "Content-Encoding: %s", "gzip");
 
-	/* But we can't do both */
+	/* But we can't do both at the same time */
 	assert(sp->wrk->do_gzip == 0 || sp->wrk->do_gunzip == 0);
 
-	/* ESI takes precedence and handles gzip/gunzip also */
+	/* ESI takes precedence and handles gzip/gunzip itself */
 	if (sp->wrk->do_esi)
 		sp->wrk->vfp = &vfp_esi;
 	else if (sp->wrk->do_gunzip)




From phk at varnish-cache.org  Wed Jan 19 11:16:39 2011
From: phk at varnish-cache.org (phk at varnish-cache.org)
Date: Wed, 19 Jan 2011 12:16:39 +0100
Subject: r5768 - trunk/varnish-cache/bin/varnishd
Message-ID: <E1PfW1n-0007Ac-FD@project.varnish-software.com>

Author: phk
Date: 2011-01-19 12:16:37 +0100 (Wed, 19 Jan 2011)
New Revision: 5768

Added:
   trunk/varnish-cache/bin/varnishd/cache_esi_fetch.c
Modified:
   trunk/varnish-cache/bin/varnishd/Makefile.am
   trunk/varnish-cache/bin/varnishd/cache_esi.h
   trunk/varnish-cache/bin/varnishd/cache_esi_deliver.c
   trunk/varnish-cache/bin/varnishd/cache_esi_parse.c
Log:
Now that the going is staring to get tough, split the ESI parsing from
the ESI fetch processing to keep source file size manageable.



Modified: trunk/varnish-cache/bin/varnishd/Makefile.am
===================================================================
--- trunk/varnish-cache/bin/varnishd/Makefile.am	2011-01-19 10:47:11 UTC (rev 5767)
+++ trunk/varnish-cache/bin/varnishd/Makefile.am	2011-01-19 11:16:37 UTC (rev 5768)
@@ -25,6 +25,7 @@
 	cache_dir_random.c \
 	cache_dir_dns.c \
 	cache_dir_round_robin.c \
+	cache_esi_fetch.c \
 	cache_esi_parse.c \
 	cache_esi_deliver.c \
 	cache_expire.c \

Modified: trunk/varnish-cache/bin/varnishd/cache_esi.h
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_esi.h	2011-01-19 10:47:11 UTC (rev 5767)
+++ trunk/varnish-cache/bin/varnishd/cache_esi.h	2011-01-19 11:16:37 UTC (rev 5768)
@@ -34,3 +34,9 @@
 #define	VEC_S2	(0x60 + 2)
 #define	VEC_S8	(0x60 + 8)
 #define	VEC_INCL	'I'
+
+void VEP_Init(const struct sess *sp);
+void VEP_parse(const struct sess *sp, const char *p, size_t l);
+struct vsb *VEP_Finish(const struct sess *sp);
+
+

Modified: trunk/varnish-cache/bin/varnishd/cache_esi_deliver.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_esi_deliver.c	2011-01-19 10:47:11 UTC (rev 5767)
+++ trunk/varnish-cache/bin/varnishd/cache_esi_deliver.c	2011-01-19 11:16:37 UTC (rev 5768)
@@ -223,7 +223,7 @@
 			p = r + 1;
 			break;
 		default:
-			Debug("XXXX 0x%02x [%s]\n", *p, p);
+			printf("XXXX 0x%02x [%s]\n", *p, p);
 			INCOMPL();
 		}
 	}

Copied: trunk/varnish-cache/bin/varnishd/cache_esi_fetch.c (from rev 5765, trunk/varnish-cache/bin/varnishd/cache_esi_parse.c)
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_esi_fetch.c	                        (rev 0)
+++ trunk/varnish-cache/bin/varnishd/cache_esi_fetch.c	2011-01-19 11:16:37 UTC (rev 5768)
@@ -0,0 +1,200 @@
+/*-
+ * Copyright (c) 2011 Varnish Software AS
+ * All rights reserved.
+ *
+ * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * VEP Varnish Esi Parsing
+ */
+
+#include "config.h"
+
+#include "svnid.h"
+SVNID("$Id")
+
+#include <stdio.h>
+#include <stdlib.h>
+
+#include "cache.h"
+#include "cache_esi.h"
+#include "vct.h"
+#include "zlib.h"
+#include "stevedore.h"
+
+/*---------------------------------------------------------------------
+ * We receive a ungzip'ed object, and want to store it ungzip'ed.
+ */
+
+static int __match_proto__()
+vfp_esi_bytes_uu(struct sess *sp, struct http_conn *htc, size_t bytes)
+{
+	ssize_t l, w;
+	struct storage *st;
+
+	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+
+	while (bytes > 0) {
+		if (sp->wrk->storage == NULL) {
+			l = params->fetch_chunksize * 1024LL;
+			sp->wrk->storage = STV_alloc(sp, l);
+		}
+		if (sp->wrk->storage == NULL) {
+			errno = ENOMEM;
+			return (-1);
+		}
+		st = sp->wrk->storage;
+		l = st->space - st->len;
+		if (l > bytes)
+			l = bytes;
+		w = HTC_Read(htc, st->ptr + st->len, l);
+		if (w <= 0)
+			return (w);
+		if (params->esi_syntax & 0x8) {
+			ssize_t d;
+			for (l = 0; l < w; l += d)  {
+				d = (random() & 3) + 1;
+				if (l + d >= w)
+					d = 1;
+				VEP_parse(sp,
+				    (const char *)st->ptr + st->len + l, d);
+			}
+		} else
+			VEP_parse(sp, (const char *)st->ptr + st->len, w);
+		st->len += w;
+		sp->obj->len += w;
+		if (st->len == st->space) {
+			VTAILQ_INSERT_TAIL(&sp->obj->store,
+			    sp->wrk->storage, list);
+			sp->wrk->storage = NULL;
+			st = NULL;
+		}
+		bytes -= w;
+	}
+	return (1);
+}
+
+/*---------------------------------------------------------------------
+ * We receive a ungzip'ed object, and want to store it gzip'ed.
+ */
+
+static int __match_proto__()
+vfp_esi_bytes_ug(struct sess *sp, struct http_conn *htc, size_t bytes)
+{
+	return (vfp_esi_bytes_uu(sp, htc, bytes));
+}
+
+/*---------------------------------------------------------------------
+ * We receive a gzip'ed object, and want to store it ungzip'ed.
+ */
+
+static int __match_proto__()
+vfp_esi_bytes_gu(struct sess *sp, struct http_conn *htc, size_t bytes)
+{
+	return (vfp_esi_bytes_uu(sp, htc, bytes));
+}
+
+/*---------------------------------------------------------------------
+ * We receive a gzip'ed object, and want to store it gzip'ed.
+ */
+
+static int __match_proto__()
+vfp_esi_bytes_gg(struct sess *sp, struct http_conn *htc, size_t bytes)
+{
+	return (vfp_esi_bytes_uu(sp, htc, bytes));
+}
+
+
+/*---------------------------------------------------------------------*/
+
+static void __match_proto__()
+vfp_esi_begin(struct sess *sp, size_t estimate)
+{
+
+	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+	/* XXX: snapshot WS's ? We'll need the space */
+
+	VEP_Init(sp);
+
+	(void)estimate;
+}
+
+static int __match_proto__()
+vfp_esi_bytes(struct sess *sp, struct http_conn *htc, size_t bytes)
+{
+	int i;
+
+	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+	if (sp->wrk->is_gzip && sp->wrk->do_gunzip)
+		i = vfp_esi_bytes_gu(sp, htc, bytes);
+	else if (sp->wrk->is_gunzip && sp->wrk->do_gzip)
+		i = vfp_esi_bytes_ug(sp, htc, bytes);
+	else if (sp->wrk->is_gzip)
+		i = vfp_esi_bytes_gg(sp, htc, bytes);
+	else
+		i = vfp_esi_bytes_uu(sp, htc, bytes);
+	return (i);
+}
+
+static int __match_proto__()
+vfp_esi_end(struct sess *sp)
+{
+	struct storage *st;
+	struct vsb *vsb;
+	ssize_t l;
+
+	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+
+	vsb = VEP_Finish(sp);
+
+	if (vsb != NULL) {
+		l = vsb_len(vsb);
+		assert(l > 0);
+		/* XXX: This is a huge waste of storage... */
+		sp->obj->esidata = STV_alloc(sp, l);
+		AN(sp->obj->esidata);
+		memcpy(sp->obj->esidata->ptr, vsb_data(vsb), l);
+		sp->obj->esidata->len = l;
+		vsb_delete(vsb);
+	}
+
+	st = sp->wrk->storage;
+	sp->wrk->storage = NULL;
+	if (st == NULL)
+		return (0);
+
+	if (st->len == 0) {
+		STV_free(st);
+		return (0);
+	}
+	if (st->len < st->space)
+		STV_trim(st, st->len);
+	VTAILQ_INSERT_TAIL(&sp->obj->store, st, list);
+	return (0);
+}
+
+struct vfp vfp_esi = {
+        .begin  =       vfp_esi_begin,
+        .bytes  =       vfp_esi_bytes,
+        .end    =       vfp_esi_end,
+};

Modified: trunk/varnish-cache/bin/varnishd/cache_esi_parse.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_esi_parse.c	2011-01-19 10:47:11 UTC (rev 5767)
+++ trunk/varnish-cache/bin/varnishd/cache_esi_parse.c	2011-01-19 11:16:37 UTC (rev 5768)
@@ -61,10 +61,9 @@
 struct vep_state {
 	unsigned		magic;
 #define VEP_MAGIC		0x55cb9b82
-	vfp_bytes_f		*bytes;
 	struct vsb		*vsb;
 
-	struct sess		*sp;
+	const struct sess	*sp;
 
 	/* parser state */
 	const char		*state;
@@ -525,16 +524,25 @@
  * NB: the state-machine.  Please maintain it along with the code.
  */
 
-static void
-vep_parse(struct vep_state *vep, const char *p, size_t l)
+void
+VEP_parse(const struct sess *sp, const char *p, size_t l)
 {
+	struct vep_state *vep;
 	const char *e;
 	struct vep_match *vm;
 	int i;
 
+	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+	vep = sp->wrk->vep;
 	CHECK_OBJ_NOTNULL(vep, VEP_MAGIC);
 	assert(l > 0);
 
+	/* XXX: Really need to fix this */
+	if (vep->hack_p == NULL)
+		vep->hack_p = p;
+
+	vep->ver_p = p;
+
 	e = p + l;
 
 	while (p < e) {
@@ -942,121 +950,38 @@
 		vep_mark_pending(vep, p);
 }
 
-/*---------------------------------------------------------------------
- * We receive a ungzip'ed object, and want to store it ungzip'ed.
- */
-
-static int __match_proto__()
-vfp_esi_bytes_uu(struct sess *sp, struct http_conn *htc, size_t bytes)
+void
+VEP_Init(const struct sess *sp)
 {
 	struct vep_state *vep;
-	ssize_t l, w;
-	struct storage *st;
 
-	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
-	vep = sp->wrk->vep;
-	CHECK_OBJ_NOTNULL(vep, VEP_MAGIC);
-
-	while (bytes > 0) {
-		if (sp->wrk->storage == NULL) {
-			l = params->fetch_chunksize * 1024LL;
-			sp->wrk->storage = STV_alloc(sp, l);
-		}
-		if (sp->wrk->storage == NULL) {
-			errno = ENOMEM;
-			return (-1);
-		}
-		st = sp->wrk->storage;
-		l = st->space - st->len;
-		if (l > bytes)
-			l = bytes;
-		w = HTC_Read(htc, st->ptr + st->len, l);
-		if (w <= 0)
-			return (w);
-		if (vep->hack_p == NULL)
-			vep->hack_p = (const char *)st->ptr + st->len;
-		vep->ver_p = (const char *)st->ptr + st->len;
-		if (params->esi_syntax & 0x8) {
-			ssize_t d;
-			for (l = 0; l < w; l += d)  {
-				d = (random() & 3) + 1;
-				if (l + d >= w)
-					d = 1;
-				vep_parse(vep,
-				    (const char *)st->ptr + st->len + l, d);
-			}
-		} else
-			vep_parse(vep, (const char *)st->ptr + st->len, w);
-		st->len += w;
-		sp->obj->len += w;
-		if (st->len == st->space) {
-			VTAILQ_INSERT_TAIL(&sp->obj->store,
-			    sp->wrk->storage, list);
-			sp->wrk->storage = NULL;
-			st = NULL;
-		}
-		bytes -= w;
-	}
-	return (1);
-}
-
-/*---------------------------------------------------------------------*/
-
-static void __match_proto__()
-vfp_esi_begin(struct sess *sp, size_t estimate)
-{
-	struct vep_state *vep;
-
-	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 	AZ(sp->wrk->vep);
-	/* XXX: snapshot WS ? We'll need the space */
-	vep = (void*)WS_Alloc(sp->wrk->ws, sizeof *vep);
-	AN(vep);
+	sp->wrk->vep = (void*)WS_Alloc(sp->wrk->ws, sizeof *vep);
+	AN(sp->wrk->vep);
 
-	Debug("BEGIN %p\n", vep);
-
+	vep = sp->wrk->vep;
 	memset(vep, 0, sizeof *vep);
 	vep->magic = VEP_MAGIC;
 	vep->sp = sp;
-	vep->bytes = vfp_esi_bytes_uu;
+
 	vep->state = VEP_START;
 	vep->vsb = vsb_newauto();
 	AN(vep->vsb);
 	vep->crc = crc32(0L, Z_NULL, 0);
 	vep->crcp = crc32(0L, Z_NULL, 0);
-
-	sp->wrk->vep = vep;
-	(void)estimate;
 }
 
-static int __match_proto__()
-vfp_esi_bytes(struct sess *sp, struct http_conn *htc, size_t bytes)
+struct vsb *
+VEP_Finish(const struct sess *sp)
 {
 	struct vep_state *vep;
-
-	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
-	vep = sp->wrk->vep;
-	Debug("BYTES %jd\n", (intmax_t)bytes);
-	CHECK_OBJ_NOTNULL(vep, VEP_MAGIC);
-	AN(vep->bytes);
-	return (vep->bytes(sp, htc, bytes));
-}
-
-static int __match_proto__()
-vfp_esi_end(struct sess *sp)
-{
-	struct storage *st;
-	struct vep_state *vep;
 	ssize_t l;
 
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 	vep = sp->wrk->vep;
 	sp->wrk->vep = NULL;
-	Debug("ENDING %p\n", vep);
 	CHECK_OBJ_NOTNULL(vep, VEP_MAGIC);
 
-	l = sp->obj->len - vep->o_total;
-	assert(l >= 0);
 	if (vep->o_pending)
 		vep_mark_common(vep, vep->ver_p, vep->last_mark);
 	if (vep->o_wait > 0)
@@ -1064,39 +989,12 @@
 
 	vsb_finish(vep->vsb);
 	l = vsb_len(vep->vsb);
-	if (vep->state != VEP_NOTXML && l > 0) {
-		Debug("ESI %d <%s>\n", (int)l, vsb_data(vep->vsb));
-
-		/* XXX: This is a huge waste of storage... */
-		sp->obj->esidata = STV_alloc(sp, l);
-		AN(sp->obj->esidata);
-		memcpy(sp->obj->esidata->ptr, vsb_data(vep->vsb), l);
-		sp->obj->esidata->len = l;
-	}
+	if (vep->state != VEP_NOTXML && l > 0)
+		return (vep->vsb);
 	vsb_delete(vep->vsb);
-
-	st = sp->wrk->storage;
-	sp->wrk->storage = NULL;
-	if (st == NULL)
-		return (0);
-
-	if (st->len == 0) {
-		STV_free(st);
-		return (0);
-	}
-	if (st->len < st->space)
-		STV_trim(st, st->len);
-	VTAILQ_INSERT_TAIL(&sp->obj->store, st, list);
-	sp->wrk->vep = NULL;
-	return (0);
+	return (NULL);
 }
 
-struct vfp vfp_esi = {
-        .begin  =       vfp_esi_begin,
-        .bytes  =       vfp_esi_bytes,
-        .end    =       vfp_esi_end,
-};
-
 #if 0
 
 digraph xml {




From phk at varnish-cache.org  Wed Jan 19 11:32:42 2011
From: phk at varnish-cache.org (phk at varnish-cache.org)
Date: Wed, 19 Jan 2011 12:32:42 +0100
Subject: r5769 - trunk/varnish-cache/bin/varnishd
Message-ID: <E1PfWHK-0007hQ-Lp@project.varnish-software.com>

Author: phk
Date: 2011-01-19 12:32:42 +0100 (Wed, 19 Jan 2011)
New Revision: 5769

Modified:
   trunk/varnish-cache/bin/varnishd/cache_gzip.c
Log:
Add a comment with our worst case gzip scenario



Modified: trunk/varnish-cache/bin/varnishd/cache_gzip.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_gzip.c	2011-01-19 11:16:37 UTC (rev 5768)
+++ trunk/varnish-cache/bin/varnishd/cache_gzip.c	2011-01-19 11:32:42 UTC (rev 5769)
@@ -33,6 +33,35 @@
  *
  * The API defined by this file, will also insulate the rest of the code,
  * should we find a better gzip library at a later date.
+ *
+ * The absolutely worst case gzip processing path, once we have pipe-lining,
+ * will be the following, so we need to be a bit careful with the scratch
+ * space we use:
+ *
+ * 	Backend		Tmp	Input	Output
+ *         |		----------------------
+ *	   v
+ *	 gunzip		wrk	stack	?
+ *         |
+ *	   v
+ *	  esi
+ *         |
+ *	   v
+ *	  gzip		wrk	?	storage
+ *         |
+ *	   v
+ *	  cache
+ *         |
+ *	   v
+ *	 gunzip		wrk	storage	stack
+ *         |
+ *	   v
+ *	 client
+ *
+ * XXXX: The two '?' are obviously the same memory, but I have yet to decide
+ * where it goes.   As usual we try to avoid the session->ws if we can but
+ * I may have to use that.
+ *
  */
 
 #include "config.h"




From phk at varnish-cache.org  Wed Jan 19 12:59:00 2011
From: phk at varnish-cache.org (phk at varnish-cache.org)
Date: Wed, 19 Jan 2011 13:59:00 +0100
Subject: r5770 - trunk/varnish-cache/bin/varnishd
Message-ID: <E1PfXcq-0001Yq-3T@project.varnish-software.com>

Author: phk
Date: 2011-01-19 13:58:59 +0100 (Wed, 19 Jan 2011)
New Revision: 5770

Modified:
   trunk/varnish-cache/bin/varnishd/cache.h
   trunk/varnish-cache/bin/varnishd/cache_gzip.c
   trunk/varnish-cache/bin/varnishd/cache_response.c
Log:
Improve and clean up the VGZ allocation a bit.



Modified: trunk/varnish-cache/bin/varnishd/cache.h
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache.h	2011-01-19 11:32:42 UTC (rev 5769)
+++ trunk/varnish-cache/bin/varnishd/cache.h	2011-01-19 12:58:59 UTC (rev 5770)
@@ -625,10 +625,11 @@
 /* cache_gzip.c */
 struct vgz;
 
-struct vgz *VGZ_NewUnzip(struct sess *sp, struct ws *tmp, struct ws *buf);
+struct vgz *VGZ_NewUnzip(const struct sess *sp, struct ws *tmp,
+    struct ws *buf_ws, void *buf, ssize_t bufl);
 int VGZ_Feed(struct vgz *, const void *, size_t len);
 int VGZ_Produce(struct vgz *, const void **, size_t *len);
-int VGZ_Destroy(struct vgz **);
+void VGZ_Destroy(struct vgz **);
 
 /* cache_http.c */
 unsigned HTTP_estimate(unsigned nhttp);

Modified: trunk/varnish-cache/bin/varnishd/cache_gzip.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_gzip.c	2011-01-19 11:32:42 UTC (rev 5769)
+++ trunk/varnish-cache/bin/varnishd/cache_gzip.c	2011-01-19 12:58:59 UTC (rev 5770)
@@ -65,6 +65,7 @@
  */
 
 #include "config.h"
+#include <stdio.h>
 
 #include "svnid.h"
 SVNID("$Id$")
@@ -77,11 +78,11 @@
 struct vgz {
 	unsigned		magic;
 #define VGZ_MAGIC		0x162df0cb
-	struct sess 		*sp;
 	struct ws		*tmp;
 	char			*tmp_snapshot;
 
-	struct ws		*buf;
+	struct ws		*buf_ws;
+	void			*buf;
 	size_t			bufsiz;
 
 	z_stream		vz;
@@ -96,7 +97,7 @@
 
 	CAST_OBJ_NOTNULL(vg, opaque, VGZ_MAGIC);
 
-	return(WS_Alloc(vg->tmp, items * size));
+	return (WS_Alloc(vg->tmp, items * size));
 }
 
 static void
@@ -108,39 +109,97 @@
 	(void)address;
 }
 
-/*--------------------------------------------------------------------*/
+/*--------------------------------------------------------------------
+ * Set up a gunzip instance
+ */
 
-struct vgz *
-VGZ_NewUnzip(struct sess *sp, struct ws *tmp, struct ws *buf)
+static struct vgz *
+vgz_alloc_vgz(struct ws *ws, struct ws *buf_ws, void *buf, ssize_t bufl)
 {
+	char *s;
 	struct vgz *vg;
-	char *s;
 
-	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
-
-	WS_Assert(tmp);
-	WS_Assert(buf);
-
-	s = WS_Snapshot(tmp);
-	vg = (void*)WS_Alloc(tmp, sizeof *vg);
+	WS_Assert(ws);
+	s = WS_Snapshot(ws);
+	vg = (void*)WS_Alloc(ws, sizeof *vg);
 	AN(vg);
 	memset(vg, 0, sizeof *vg);
 	vg->magic = VGZ_MAGIC;
-	vg->sp = sp;
-	vg->tmp = tmp;
-	vg->buf = buf;
+	vg->tmp = ws;
 	vg->tmp_snapshot = s;
 
 	vg->vz.zalloc = vgz_alloc;
 	vg->vz.zfree = vgz_free;
 	vg->vz.opaque = vg;
 
-	vg->bufsiz = WS_Reserve(buf, 0);
+	assert(buf_ws == NULL || buf == NULL);
+	if (buf_ws != NULL) {
+		WS_Assert(buf_ws);
+		vg->buf_ws = buf_ws;
+		vg->bufsiz = WS_Reserve(buf_ws, 0);
+		vg->buf = buf_ws->f;
+	} else {
+		assert(bufl > 0);
+		vg->buf = buf;
+		vg->bufsiz = bufl;
+	}
 
+	return (vg);
+}
+
+struct vgz *
+VGZ_NewUnzip(const struct sess *sp, struct ws *tmp, struct ws *buf_ws,
+    void *buf, ssize_t bufl)
+{
+	struct vgz *vg;
+
+	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+	vg = vgz_alloc_vgz(tmp, buf_ws, buf, bufl);
+
+	/*
+	 * Max memory usage according to zonf.h:
+	 * 	mem_needed = "a few kb" + (1 << (windowBits))
+	 * Since we don't control windowBits, we have to assume
+	 * it is 15, so 34-35KB or so.
+	 */
 	assert(Z_OK == inflateInit2(&vg->vz, 31));
 	return (vg);
 }
 
+static struct vgz *
+VGZ_NewGzip(const struct sess *sp, struct ws *tmp, struct ws *buf_ws,
+    void *buf, ssize_t bufl)
+{
+	struct vgz *vg;
+	int i;
+
+	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+	vg = vgz_alloc_vgz(tmp, buf_ws, buf, bufl);
+
+	/*
+	 * From zconf.h:
+	 *
+	 * 	mem_needed = "a few kb" 
+	 *		+ (1 << (windowBits+2))
+	 *		+  (1 << (memLevel+9))
+	 *
+	 * windowBits [8..15] (-> 1K..128K)
+	 * memLevel [1..9] (-> 1K->256K)
+	 *
+	 * XXX: They probably needs to be params...
+	 */
+	i = deflateInit2(&vg->vz,
+	    0,				/* Level */
+	    Z_DEFLATED,			/* Method */
+	    16 + 8,			/* Window bits (16=gzip + 15) */
+	    1,				/* memLevel */
+	    Z_DEFAULT_STRATEGY);
+	if (i != Z_OK)
+		printf("deflateInit2() = %d\n", i);
+	assert(Z_OK == i);
+	return (vg);
+}
+
 /*--------------------------------------------------------------------*/
 
 int
@@ -167,12 +226,12 @@
 
 	*pptr = NULL;
 	*plen = 0;
-	vg->vz.next_out = (void*)vg->buf->f;
+	vg->vz.next_out = vg->buf;
 	vg->vz.avail_out = vg->bufsiz;
 
 	i = inflate(&vg->vz, 0);
 	if (i == Z_OK || i == Z_STREAM_END) {
-		*pptr = vg->buf->f;
+		*pptr = vg->buf;
 		*plen = vg->bufsiz - vg->vz.avail_out;
 	}
 	if (i == Z_OK)
@@ -186,15 +245,15 @@
 
 /*--------------------------------------------------------------------*/
 
-int
+void
 VGZ_Destroy(struct vgz **vg)
 {
 
 	CHECK_OBJ_NOTNULL(*vg, VGZ_MAGIC);
-	WS_Release((*vg)->buf, 0);
+	if ((*vg)->buf_ws != NULL)
+		WS_Release((*vg)->buf_ws, 0);
 	WS_Reset((*vg)->tmp, (*vg)->tmp_snapshot);
 	*vg = NULL;
-	return (0);
 }
 
 /*--------------------------------------------------------------------
@@ -207,7 +266,7 @@
 vfp_gunzip_begin(struct sess *sp, size_t estimate)
 {
 	(void)estimate;
-	sp->wrk->vfp_private = VGZ_NewUnzip(sp, sp->ws, sp->wrk->ws);
+	sp->wrk->vfp_private = VGZ_NewUnzip(sp, sp->ws, sp->wrk->ws, NULL, 0);
 }
 
 static int __match_proto__()
@@ -237,10 +296,10 @@
 			l = vg->bufsiz;
 			if (l > bytes)
 				l = bytes;
-			w = HTC_Read(htc, vg->buf->f, l);
+			w = HTC_Read(htc, vg->buf, l);
 			if (w <= 0)
 				return (w);
-			vg->vz.next_in = (void*)vg->buf->f;
+			vg->vz.next_in = vg->buf;
 			vg->vz.avail_in = w;
 			bytes -= w;
 		}
@@ -304,16 +363,7 @@
 	struct vgz *vg;
 	(void)estimate;
 
-	vg = VGZ_NewUnzip(sp, sp->ws, sp->wrk->ws);
-	/* XXX: hack */
-	memset(&vg->vz, 0, sizeof vg->vz);
-	assert(Z_OK == deflateInit2(&vg->vz,
-	    0,
-	    Z_DEFLATED,
-	    31,
-	    9,
-	    Z_DEFAULT_STRATEGY));
-
+	vg = VGZ_NewGzip(sp, sp->ws, sp->wrk->ws, NULL, 0);
 	sp->wrk->vfp_private = vg;
 }
 
@@ -344,10 +394,10 @@
 			l = vg->bufsiz;
 			if (l > bytes)
 				l = bytes;
-			w = HTC_Read(htc, vg->buf->f, l);
+			w = HTC_Read(htc, vg->buf, l);
 			if (w <= 0)
 				return (w);
-			vg->vz.next_in = (void*)vg->buf->f;
+			vg->vz.next_in = vg->buf;
 			vg->vz.avail_in = w;
 			bytes -= w;
 		}

Modified: trunk/varnish-cache/bin/varnishd/cache_response.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_response.c	2011-01-19 11:32:42 UTC (rev 5769)
+++ trunk/varnish-cache/bin/varnishd/cache_response.c	2011-01-19 12:58:59 UTC (rev 5770)
@@ -251,7 +251,7 @@
 
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 
-	vg = VGZ_NewUnzip(sp, sp->ws, sp->wrk->ws);
+	vg = VGZ_NewUnzip(sp, sp->ws, sp->wrk->ws, NULL, 0);
 	AN(vg);
 
 	VTAILQ_FOREACH(st, &sp->obj->store, list) {




From phk at varnish-cache.org  Wed Jan 19 13:12:38 2011
From: phk at varnish-cache.org (phk at varnish-cache.org)
Date: Wed, 19 Jan 2011 14:12:38 +0100
Subject: r5771 - trunk/varnish-cache/bin/varnishd
Message-ID: <E1PfXq2-0001xs-B0@project.varnish-software.com>

Author: phk
Date: 2011-01-19 14:12:38 +0100 (Wed, 19 Jan 2011)
New Revision: 5771

Modified:
   trunk/varnish-cache/bin/varnishd/cache_response.c
Log:
Move the delivery gunzip away from session workspace and use a
64k stackbuffer for bufferspace instead.



Modified: trunk/varnish-cache/bin/varnishd/cache_response.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_response.c	2011-01-19 12:58:59 UTC (rev 5770)
+++ trunk/varnish-cache/bin/varnishd/cache_response.c	2011-01-19 13:12:38 UTC (rev 5771)
@@ -240,18 +240,20 @@
  */
 
 static void
-res_WriteGunzipObj(struct sess *sp, char lenbuf[20])
+res_WriteGunzipObj(struct sess *sp)
 {
 	struct storage *st;
 	unsigned u = 0;
 	struct vgz *vg;
 	const void *dp;
+	char lenbuf[20];
+	char obuf[64*1024];	/* XXX: size? */
 	size_t dl;
 	int i;
 
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 
-	vg = VGZ_NewUnzip(sp, sp->ws, sp->wrk->ws, NULL, 0);
+	vg = VGZ_NewUnzip(sp, sp->wrk->ws, NULL, obuf, sizeof obuf);
 	AN(vg);
 
 	VTAILQ_FOREACH(st, &sp->obj->store, list) {
@@ -259,7 +261,7 @@
 		CHECK_OBJ_NOTNULL(st, STORAGE_MAGIC);
 		u += st->len;
 
-		sp->acct_tmp.bodybytes += st->len;
+		sp->acct_tmp.bodybytes += st->len;	/* XXX ? */
 		VSC_main->n_objwrite++;
 
 		VGZ_Feed(vg, st->ptr, st->len);
@@ -389,7 +391,7 @@
 	} else if (sp->wrk->res_mode & RES_ESI) {
 		ESI_Deliver(sp);
 	} else if (sp->wrk->res_mode & RES_GUNZIP) {
-		res_WriteGunzipObj(sp, lenbuf); 
+		res_WriteGunzipObj(sp); 
 	} else {
 		res_WriteDirObj(sp, lenbuf, low, high);
 	}




From phk at varnish-cache.org  Wed Jan 19 13:41:46 2011
From: phk at varnish-cache.org (phk at varnish-cache.org)
Date: Wed, 19 Jan 2011 14:41:46 +0100
Subject: r5772 - trunk/varnish-cache/bin/varnishd
Message-ID: <E1PfYIE-0002mr-Un@project.varnish-software.com>

Author: phk
Date: 2011-01-19 14:41:46 +0100 (Wed, 19 Jan 2011)
New Revision: 5772

Modified:
   trunk/varnish-cache/bin/varnishd/cache.h
   trunk/varnish-cache/bin/varnishd/cache_gzip.c
   trunk/varnish-cache/bin/varnishd/cache_response.c
Log:
Futher modelling of VGZ functions



Modified: trunk/varnish-cache/bin/varnishd/cache.h
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache.h	2011-01-19 13:12:38 UTC (rev 5771)
+++ trunk/varnish-cache/bin/varnishd/cache.h	2011-01-19 13:41:46 UTC (rev 5772)
@@ -625,10 +625,11 @@
 /* cache_gzip.c */
 struct vgz;
 
-struct vgz *VGZ_NewUnzip(const struct sess *sp, struct ws *tmp,
-    struct ws *buf_ws, void *buf, ssize_t bufl);
-int VGZ_Feed(struct vgz *, const void *, size_t len);
-int VGZ_Produce(struct vgz *, const void **, size_t *len);
+struct vgz *VGZ_NewUngzip(const struct sess *sp, struct ws *tmp);
+void VGZ_Ibuf(struct vgz *, const void *, size_t len);
+void VGZ_Obuf(struct vgz *, const void *, size_t len);
+int VGZ_Gzip(struct vgz *, const void **, size_t *len, int flag);
+int VGZ_Gunzip(struct vgz *, const void **, size_t *len);
 void VGZ_Destroy(struct vgz **);
 
 /* cache_http.c */

Modified: trunk/varnish-cache/bin/varnishd/cache_gzip.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_gzip.c	2011-01-19 13:12:38 UTC (rev 5771)
+++ trunk/varnish-cache/bin/varnishd/cache_gzip.c	2011-01-19 13:41:46 UTC (rev 5772)
@@ -81,9 +81,7 @@
 	struct ws		*tmp;
 	char			*tmp_snapshot;
 
-	struct ws		*buf_ws;
-	void			*buf;
-	size_t			bufsiz;
+	void			*before;
 
 	z_stream		vz;
 };
@@ -114,7 +112,7 @@
  */
 
 static struct vgz *
-vgz_alloc_vgz(struct ws *ws, struct ws *buf_ws, void *buf, ssize_t bufl)
+vgz_alloc_vgz(struct ws *ws)
 {
 	char *s;
 	struct vgz *vg;
@@ -132,29 +130,16 @@
 	vg->vz.zfree = vgz_free;
 	vg->vz.opaque = vg;
 
-	assert(buf_ws == NULL || buf == NULL);
-	if (buf_ws != NULL) {
-		WS_Assert(buf_ws);
-		vg->buf_ws = buf_ws;
-		vg->bufsiz = WS_Reserve(buf_ws, 0);
-		vg->buf = buf_ws->f;
-	} else {
-		assert(bufl > 0);
-		vg->buf = buf;
-		vg->bufsiz = bufl;
-	}
-
 	return (vg);
 }
 
 struct vgz *
-VGZ_NewUnzip(const struct sess *sp, struct ws *tmp, struct ws *buf_ws,
-    void *buf, ssize_t bufl)
+VGZ_NewUngzip(const struct sess *sp, struct ws *tmp)
 {
 	struct vgz *vg;
 
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
-	vg = vgz_alloc_vgz(tmp, buf_ws, buf, bufl);
+	vg = vgz_alloc_vgz(tmp);
 
 	/*
 	 * Max memory usage according to zonf.h:
@@ -167,14 +152,13 @@
 }
 
 static struct vgz *
-VGZ_NewGzip(const struct sess *sp, struct ws *tmp, struct ws *buf_ws,
-    void *buf, ssize_t bufl)
+VGZ_NewGzip(const struct sess *sp, struct ws *tmp)
 {
 	struct vgz *vg;
 	int i;
 
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
-	vg = vgz_alloc_vgz(tmp, buf_ws, buf, bufl);
+	vg = vgz_alloc_vgz(tmp);
 
 	/*
 	 * From zconf.h:
@@ -202,8 +186,8 @@
 
 /*--------------------------------------------------------------------*/
 
-int
-VGZ_Feed(struct vgz *vg, const void *ptr, size_t len)
+void
+VGZ_Ibuf(struct vgz *vg, const void *ptr, size_t len)
 {
 
 	CHECK_OBJ_NOTNULL(vg, VGZ_MAGIC);
@@ -211,14 +195,24 @@
 	AZ(vg->vz.avail_in);
 	vg->vz.next_in = TRUST_ME(ptr);
 	vg->vz.avail_in = len;
+}
 
-	return (0);
+/*--------------------------------------------------------------------*/
+
+void
+VGZ_Obuf(struct vgz *vg, const void *ptr, size_t len)
+{
+
+	CHECK_OBJ_NOTNULL(vg, VGZ_MAGIC);
+
+	vg->vz.next_out = TRUST_ME(ptr);
+	vg->vz.avail_out = len;
 }
 
 /*--------------------------------------------------------------------*/
 
 int
-VGZ_Produce(struct vgz *vg, const void **pptr, size_t *plen)
+VGZ_Gunzip(struct vgz *vg, const void **pptr, size_t *plen)
 {
 	int i;
 
@@ -226,13 +220,13 @@
 
 	*pptr = NULL;
 	*plen = 0;
-	vg->vz.next_out = vg->buf;
-	vg->vz.avail_out = vg->bufsiz;
-
+	AN(vg->vz.next_out);
+	AN(vg->vz.avail_out);
+	vg->before = vg->vz.next_out;
 	i = inflate(&vg->vz, 0);
 	if (i == Z_OK || i == Z_STREAM_END) {
-		*pptr = vg->buf;
-		*plen = vg->bufsiz - vg->vz.avail_out;
+		*pptr = vg->before;
+		*plen = (const uint8_t *)vg->vz.next_out - (const uint8_t*)vg->before;
 	}
 	if (i == Z_OK)
 		return (0);
@@ -245,13 +239,39 @@
 
 /*--------------------------------------------------------------------*/
 
+int
+VGZ_Gzip(struct vgz *vg, const void **pptr, size_t *plen, int flags)
+{
+	int i;
+
+	CHECK_OBJ_NOTNULL(vg, VGZ_MAGIC);
+
+	*pptr = NULL;
+	*plen = 0;
+	AN(vg->vz.next_out);
+	AN(vg->vz.avail_out);
+	vg->before = vg->vz.next_out;
+	i = deflate(&vg->vz, flags);
+	if (i == Z_OK || i == Z_STREAM_END) {
+		*pptr = vg->before;
+		*plen = (const uint8_t *)vg->vz.next_out - (const uint8_t*)vg->before;
+	}
+	if (i == Z_OK)
+		return (0);
+	if (i == Z_STREAM_END)
+		return (1);
+	if (i == Z_BUF_ERROR)
+		return (2);
+	return (-1);
+}
+
+/*--------------------------------------------------------------------*/
+
 void
 VGZ_Destroy(struct vgz **vg)
 {
 
 	CHECK_OBJ_NOTNULL(*vg, VGZ_MAGIC);
-	if ((*vg)->buf_ws != NULL)
-		WS_Release((*vg)->buf_ws, 0);
 	WS_Reset((*vg)->tmp, (*vg)->tmp_snapshot);
 	*vg = NULL;
 }
@@ -266,7 +286,7 @@
 vfp_gunzip_begin(struct sess *sp, size_t estimate)
 {
 	(void)estimate;
-	sp->wrk->vfp_private = VGZ_NewUnzip(sp, sp->ws, sp->wrk->ws, NULL, 0);
+	sp->wrk->vfp_private = VGZ_NewUngzip(sp, sp->ws);
 }
 
 static int __match_proto__()
@@ -276,6 +296,9 @@
 	struct storage *st;
 	ssize_t l, w;
 	int i = -100;
+	uint8_t	ibuf[64*1024];		/* XXX size ? */
+	size_t dl;
+	const void *dp;
 
 	CAST_OBJ_NOTNULL(vg, sp->wrk->vfp_private, VGZ_MAGIC);
 	AZ(vg->vz.avail_in);
@@ -289,24 +312,22 @@
 		}
 		st = sp->wrk->storage;
 
-		vg->vz.next_out = st->ptr + st->len;
-		vg->vz.avail_out = st->space - st->len;
+		VGZ_Obuf(vg, st->ptr + st->len, st->space - st->len);
 
 		if (vg->vz.avail_in == 0 && bytes > 0) {
-			l = vg->bufsiz;
+			l = sizeof ibuf;
 			if (l > bytes)
 				l = bytes;
-			w = HTC_Read(htc, vg->buf, l);
+			w = HTC_Read(htc, ibuf, l);
 			if (w <= 0)
 				return (w);
-			vg->vz.next_in = vg->buf;
-			vg->vz.avail_in = w;
+			VGZ_Ibuf(vg, ibuf, w);
 			bytes -= w;
 		}
 
-		i = inflate(&vg->vz, 0);
+		i = VGZ_Gunzip(vg, &dp, &dl);
 		assert(i == Z_OK || i == Z_STREAM_END);
-		st->len = st->space - vg->vz.avail_out;
+		st->len += dl;
 		if (st->len == st->space) {
 			VTAILQ_INSERT_TAIL(&sp->obj->store,
 			    sp->wrk->storage, list);
@@ -363,7 +384,7 @@
 	struct vgz *vg;
 	(void)estimate;
 
-	vg = VGZ_NewGzip(sp, sp->ws, sp->wrk->ws, NULL, 0);
+	vg = VGZ_NewGzip(sp, sp->ws);
 	sp->wrk->vfp_private = vg;
 }
 
@@ -374,6 +395,9 @@
 	struct storage *st;
 	ssize_t l, w;
 	int i = -100;
+	uint8_t ibuf[64*1024];		/* XXX size ? */
+	size_t dl;
+	const void *dp;
 
 	CAST_OBJ_NOTNULL(vg, sp->wrk->vfp_private, VGZ_MAGIC);
 	AZ(vg->vz.avail_in);
@@ -387,24 +411,22 @@
 		}
 		st = sp->wrk->storage;
 
-		vg->vz.next_out = st->ptr + st->len;
-		vg->vz.avail_out = st->space - st->len;
+		VGZ_Obuf(vg, st->ptr + st->len, st->space - st->len);
 
 		if (vg->vz.avail_in == 0 && bytes > 0) {
-			l = vg->bufsiz;
+			l = sizeof ibuf;
 			if (l > bytes)
 				l = bytes;
-			w = HTC_Read(htc, vg->buf, l);
+			w = HTC_Read(htc, ibuf, l);
 			if (w <= 0)
 				return (w);
-			vg->vz.next_in = vg->buf;
-			vg->vz.avail_in = w;
+			VGZ_Ibuf(vg, ibuf, w);
 			bytes -= w;
 		}
 
-		i = deflate(&vg->vz, bytes == 0 ? Z_FINISH : 0);
+		i = VGZ_Gzip(vg, &dp, &dl, bytes == 0 ? Z_FINISH : 0);
 		assert(i == Z_OK || i == Z_STREAM_END);
-		st->len = st->space - vg->vz.avail_out;
+		st->len = st->space - dl;
 		if (st->len == st->space) {
 			VTAILQ_INSERT_TAIL(&sp->obj->store,
 			    sp->wrk->storage, list);

Modified: trunk/varnish-cache/bin/varnishd/cache_response.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_response.c	2011-01-19 13:12:38 UTC (rev 5771)
+++ trunk/varnish-cache/bin/varnishd/cache_response.c	2011-01-19 13:41:46 UTC (rev 5772)
@@ -253,8 +253,7 @@
 
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 
-	vg = VGZ_NewUnzip(sp, sp->wrk->ws, NULL, obuf, sizeof obuf);
-	AN(vg);
+	vg = VGZ_NewUngzip(sp, sp->wrk->ws);
 
 	VTAILQ_FOREACH(st, &sp->obj->store, list) {
 		CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
@@ -264,9 +263,10 @@
 		sp->acct_tmp.bodybytes += st->len;	/* XXX ? */
 		VSC_main->n_objwrite++;
 
-		VGZ_Feed(vg, st->ptr, st->len);
+		VGZ_Ibuf(vg, st->ptr, st->len);
 		do {
-			i = VGZ_Produce(vg, &dp, &dl);
+			VGZ_Obuf(vg, obuf, sizeof obuf);
+			i = VGZ_Gunzip(vg, &dp, &dl);
 			if (dl != 0) {
 				if (sp->wrk->res_mode & RES_CHUNKED) {
 					bprintf(lenbuf, "%x\r\n", (unsigned)dl);




From phk at varnish-cache.org  Thu Jan 20 07:07:39 2011
From: phk at varnish-cache.org (phk at varnish-cache.org)
Date: Thu, 20 Jan 2011 08:07:39 +0100
Subject: r5773 - trunk/varnish-cache/bin/varnishd
Message-ID: <E1PfocN-0005oj-4p@project.varnish-software.com>

Author: phk
Date: 2011-01-20 08:07:38 +0100 (Thu, 20 Jan 2011)
New Revision: 5773

Modified:
   trunk/varnish-cache/bin/varnishd/cache.h
   trunk/varnish-cache/bin/varnishd/cache_esi_fetch.c
   trunk/varnish-cache/bin/varnishd/cache_gzip.c
Log:
Add separate anchor fields for vgz for reception and esi work.
(The response vgz is a local variable)



Modified: trunk/varnish-cache/bin/varnishd/cache.h
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache.h	2011-01-19 13:41:46 UTC (rev 5772)
+++ trunk/varnish-cache/bin/varnishd/cache.h	2011-01-20 07:07:38 UTC (rev 5773)
@@ -279,7 +279,8 @@
 	enum body_status	body_status;
 	struct storage		*storage;
 	struct vfp		*vfp;
-	void			*vfp_private;
+	struct vgz		*vgz_rx;
+	struct vgz		*vgz_esi;
 	unsigned		do_esi;
 	unsigned		do_gzip;
 	unsigned		is_gzip;

Modified: trunk/varnish-cache/bin/varnishd/cache_esi_fetch.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_esi_fetch.c	2011-01-19 13:41:46 UTC (rev 5772)
+++ trunk/varnish-cache/bin/varnishd/cache_esi_fetch.c	2011-01-20 07:07:38 UTC (rev 5773)
@@ -111,6 +111,7 @@
 static int __match_proto__()
 vfp_esi_bytes_gu(struct sess *sp, struct http_conn *htc, size_t bytes)
 {
+
 	return (vfp_esi_bytes_uu(sp, htc, bytes));
 }
 

Modified: trunk/varnish-cache/bin/varnishd/cache_gzip.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_gzip.c	2011-01-19 13:41:46 UTC (rev 5772)
+++ trunk/varnish-cache/bin/varnishd/cache_gzip.c	2011-01-20 07:07:38 UTC (rev 5773)
@@ -171,6 +171,9 @@
 	 * memLevel [1..9] (-> 1K->256K)
 	 *
 	 * XXX: They probably needs to be params...
+	 *
+	 * XXX: It may be more efficent to malloc them, rather than have
+	 * XXX: too many worker threads grow the stacks.
 	 */
 	i = deflateInit2(&vg->vz,
 	    0,				/* Level */
@@ -286,7 +289,7 @@
 vfp_gunzip_begin(struct sess *sp, size_t estimate)
 {
 	(void)estimate;
-	sp->wrk->vfp_private = VGZ_NewUngzip(sp, sp->ws);
+	sp->wrk->vgz_rx = VGZ_NewUngzip(sp, sp->ws);
 }
 
 static int __match_proto__()
@@ -300,7 +303,8 @@
 	size_t dl;
 	const void *dp;
 
-	CAST_OBJ_NOTNULL(vg, sp->wrk->vfp_private, VGZ_MAGIC);
+	vg = sp->wrk->vgz_rx;
+	CHECK_OBJ_NOTNULL(vg, VGZ_MAGIC);
 	AZ(vg->vz.avail_in);
 	while (bytes > 0 || vg->vz.avail_in > 0) {
 		if (sp->wrk->storage == NULL)
@@ -346,7 +350,8 @@
 	struct vgz *vg;
 	struct storage *st;
 
-	CAST_OBJ_NOTNULL(vg, sp->wrk->vfp_private, VGZ_MAGIC);
+	vg = sp->wrk->vgz_rx;
+	CHECK_OBJ_NOTNULL(vg, VGZ_MAGIC);
 	VGZ_Destroy(&vg);
 
 	st = sp->wrk->storage;
@@ -381,11 +386,9 @@
 static void __match_proto__()
 vfp_gzip_begin(struct sess *sp, size_t estimate)
 {
-	struct vgz *vg;
 	(void)estimate;
 
-	vg = VGZ_NewGzip(sp, sp->ws);
-	sp->wrk->vfp_private = vg;
+	sp->wrk->vgz_rx = VGZ_NewGzip(sp, sp->ws);
 }
 
 static int __match_proto__()
@@ -399,7 +402,8 @@
 	size_t dl;
 	const void *dp;
 
-	CAST_OBJ_NOTNULL(vg, sp->wrk->vfp_private, VGZ_MAGIC);
+	vg = sp->wrk->vgz_rx;
+	CHECK_OBJ_NOTNULL(vg, VGZ_MAGIC);
 	AZ(vg->vz.avail_in);
 	while (bytes > 0 || vg->vz.avail_in > 0) {
 		if (sp->wrk->storage == NULL)
@@ -445,7 +449,8 @@
 	struct vgz *vg;
 	struct storage *st;
 
-	CAST_OBJ_NOTNULL(vg, sp->wrk->vfp_private, VGZ_MAGIC);
+	vg = sp->wrk->vgz_rx;
+	CHECK_OBJ_NOTNULL(vg, VGZ_MAGIC);
 	VGZ_Destroy(&vg);
 
 	st = sp->wrk->storage;




From tfheen at project.varnish-software.com  Thu Jan 20 15:47:50 2011
From: tfheen at project.varnish-software.com (Tollef Fog Heen)
Date: Thu, 20 Jan 2011 16:47:50 +0100
Subject: [master] 71e1ee6 Fix typos in tutorial
Message-ID: <E1Pfwjm-0007Gr-Hr@project.varnish-software.com>

commit 71e1ee607dca1cf36c6654cd52d712ebb80f67ca
Author: Bj?rn Ruberg <bjorn.ruberg at redpill-linpro.com>
Date:   Thu Jan 20 14:56:05 2011 +0100

    Fix typos in tutorial

diff --git a/doc/sphinx/tutorial/increasing_your_hitrate.rst b/doc/sphinx/tutorial/increasing_your_hitrate.rst
index c84f390..b8fe6c2 100644
--- a/doc/sphinx/tutorial/increasing_your_hitrate.rst
+++ b/doc/sphinx/tutorial/increasing_your_hitrate.rst
@@ -19,11 +19,11 @@ the web server. If you have Varnish the easiest is to use varnishlog
 and varnishtop but sometimes a client-side tool makes sense. Here are
 the ones I use.
 
-Tool: varnistop
-~~~~~~~~~~~~~~~
+Tool: varnishtop
+~~~~~~~~~~~~~~~~
 
 You can use varnishtop to identify what URLs are hitting the backend
-the most. ``varnishtop -i txurl`` is a essential command. You can see
+the most. ``varnishtop -i txurl`` is an essential command. You can see
 some other examples of varnishtop usage in :ref:`tutorial-statistics`.
 
 
@@ -39,13 +39,13 @@ from the client (-c) matching /foo/bar.
 Tool: lwp-request
 ~~~~~~~~~~~~~~~~~
 
-lwp-request is part of The World-Wide Web library for Perl. It's
-couple of really basic programs that can execute a HTTP request and
+lwp-request is part of The World-Wide Web library for Perl. It's a
+couple of really basic programs that can execute an HTTP request and
 give you the result. I mostly use two programs, GET and HEAD.
 
 vg.no was the first site to use Varnish and the people running Varnish
-there are quite clue-full. So its interesting to look at their HTTP
-Headers. Lets send a GET request for their home page.::
+there are quite clueful. So it's interesting to look at their HTTP
+Headers. Let's send a GET request for their home page::
 
   $ GET -H 'Host: www.vg.no' -Used http://vg.no/
   GET http://vg.no/
@@ -66,16 +66,16 @@ Headers. Lets send a GET request for their home page.::
 
 OK. Let me explain what it does. GET usually send off HTTP 0.9
 requests, which lack the Host header. So I add a Host header with the
--H option. -U print request headers, -s prints response status -e
-prints repsonse headers and -d discards the actual content. We dont
+-H option. -U print request headers, -s prints response status, -e
+prints response headers and -d discards the actual content. We don't
 really care about the content, only the headers.
 
-As you can see VG ads quite a bit of information in their
+As you can see, VG adds quite a bit of information in their
 headers. Some of the headers, like the X-Rick-Would-Never are specific
 to vg.no and their somewhat odd sense of humour. Others, like the
 X-VG-Webcache are for debugging purposes. 
 
-So, to check whether a site sets cookies for a specific URL just do::
+So, to check whether a site sets cookies for a specific URL, just do::
 
   GET -Used http://example.com/ |grep ^Set-Cookie
 
@@ -83,7 +83,7 @@ Tool: Live HTTP Headers
 ~~~~~~~~~~~~~~~~~~~~~~~
 
 There is also a plugin for Firefox. *Live HTTP Headers* can show you
-what headers are beeing sent and recieved. Live HTTP Headers can be
+what headers are being sent and recieved. Live HTTP Headers can be
 found at https://addons.mozilla.org/en-US/firefox/addon/3829/ or by
 googling "Live HTTP Headers".
 
@@ -97,14 +97,14 @@ it is appropriate to cache the contents and how long Varnish can keep
 the content.
 
 Please note that when considering these headers Varnish actually
-considers itself *part of* the actual webserver. The ratinonale being
+considers itself *part of* the actual webserver. The rationale being
 that both are under your control. 
 
 The term *surrogate origin cache* is not really well defined by the
 IETF so RFC 2616 so the various ways Varnish works might differ from
 your expectations.
 
-Lets take a look at the importent headers you should be aware of:
+Let's take a look at the important headers you should be aware of:
 
 Cache-Control
 ~~~~~~~~~~~~~
@@ -114,11 +114,11 @@ cares about the *max-age* parameter and uses it to calculate the TTL
 for an object. 
 
 "Cache-Control: nocache" is ignored but if you need this you can
-easyli add support for it.
+easily add support for it.
 
 So make sure use issue a Cache-Control header with a max-age
-header. You can have a look at what Varnish Softwares drupal server
-issues:::
+header. You can have a look at what Varnish Software's drupal server
+issues::
 
   $ GET -Used http://www.varnish-software.com/|grep ^Cache-Control
   Cache-Control: public, max-age=600
@@ -126,7 +126,7 @@ issues:::
 Age
 ~~~
 
-Varnish adds a Age header to indicate how long the object has been
+Varnish adds an Age header to indicate how long the object has been
 kept inside Varnish. You can grep out Age from varnishlog like this::
 
   varnishlog -i TxHeader -I ^Age
@@ -134,9 +134,8 @@ kept inside Varnish. You can grep out Age from varnishlog like this::
 Pragma
 ~~~~~~
 
-
 HTTP 1.0 server might send "Pragma: nocache". Varnish ignores this
-header. You could easly add support for this header in VCL.
+header. You could easily add support for this header in VCL.
 
 In vcl_fetch::
 
@@ -147,18 +146,18 @@ In vcl_fetch::
 Authorization
 ~~~~~~~~~~~~~
 
-If Varnish sees a Authorization header it will pass the request. If
+If Varnish sees an Authorization header it will pass the request. If
 this is not what you want you can unset the header.
 
 Overriding the time-to-live (ttl)
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 Sometimes your backend will misbehave. It might, depending on your
-setup, be easier to override the ttl in Varnish then to fix your
+setup, be easier to override the ttl in Varnish than to fix your
 somewhat cumbersome backend. 
 
 You need VCL to identify the objects you want and then you set the
-beresp.ttl to whatever you want.::
+beresp.ttl to whatever you want::
 
   sub vcl_fetch {
       if (req.url ~ "^/legacy_broken_cms/") {
@@ -171,12 +170,12 @@ Normalizing your namespace
 ~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 Some sites are accessed via lots of
-hostnames. http://www.varnish-software.com ,
-http://varnish-software.com and http://varnishsoftware.com/ all point
-at the same site. Since Varnish doesn't know they are different
+hostnames. http://www.varnish-software.com/,
+http://varnish-software.com/ and http://varnishsoftware.com/ all point
+at the same site. Since Varnish doesn't know they are different,
 Varnish will cache different versions of every page for every
 hostname. You can mitigate this in your web server configuration by
-setting up redirects or by using the following VCL:::
+setting up redirects or by using the following VCL::
 
   if (req.http.host ~ "^(www.)?varnish-?software.com") {
     set req.http.host = "varnish-software.com";



From phk at project.varnish-software.com  Fri Jan 21 11:17:04 2011
From: phk at project.varnish-software.com (Poul-Henning Kamp)
Date: Fri, 21 Jan 2011 12:17:04 +0100
Subject: [master] 17c076b Replay of SVN r5773
Message-ID: <E1PgEzI-0003J1-1A@project.varnish-software.com>

commit 17c076b3f12cdc24a2e6c5100d24fe374b8f224a
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Fri Jan 21 11:15:58 2011 +0000

    Replay of SVN r5773
    
    Add separate anchor fields for vgz for reception and esi work.
    (The response vgz is a local variable)

diff --git a/bin/varnishd/cache.h b/bin/varnishd/cache.h
index 0577fd9..8d1696e 100644
--- a/bin/varnishd/cache.h
+++ b/bin/varnishd/cache.h
@@ -279,7 +279,8 @@ struct worker {
 	enum body_status	body_status;
 	struct storage		*storage;
 	struct vfp		*vfp;
-	void			*vfp_private;
+	struct vgz		*vgz_rx;
+	struct vgz		*vgz_esi;
 	unsigned		do_esi;
 	unsigned		do_gzip;
 	unsigned		is_gzip;
diff --git a/bin/varnishd/cache_esi_fetch.c b/bin/varnishd/cache_esi_fetch.c
index b69dc65..d36e694 100644
--- a/bin/varnishd/cache_esi_fetch.c
+++ b/bin/varnishd/cache_esi_fetch.c
@@ -111,6 +111,7 @@ vfp_esi_bytes_ug(struct sess *sp, struct http_conn *htc, size_t bytes)
 static int __match_proto__()
 vfp_esi_bytes_gu(struct sess *sp, struct http_conn *htc, size_t bytes)
 {
+
 	return (vfp_esi_bytes_uu(sp, htc, bytes));
 }
 
diff --git a/bin/varnishd/cache_gzip.c b/bin/varnishd/cache_gzip.c
index acb6606..9ac3d6c 100644
--- a/bin/varnishd/cache_gzip.c
+++ b/bin/varnishd/cache_gzip.c
@@ -171,6 +171,9 @@ VGZ_NewGzip(const struct sess *sp, struct ws *tmp)
 	 * memLevel [1..9] (-> 1K->256K)
 	 *
 	 * XXX: They probably needs to be params...
+	 *
+	 * XXX: It may be more efficent to malloc them, rather than have
+	 * XXX: too many worker threads grow the stacks.
 	 */
 	i = deflateInit2(&vg->vz,
 	    0,				/* Level */
@@ -286,7 +289,7 @@ static void __match_proto__()
 vfp_gunzip_begin(struct sess *sp, size_t estimate)
 {
 	(void)estimate;
-	sp->wrk->vfp_private = VGZ_NewUngzip(sp, sp->ws);
+	sp->wrk->vgz_rx = VGZ_NewUngzip(sp, sp->ws);
 }
 
 static int __match_proto__()
@@ -300,7 +303,8 @@ vfp_gunzip_bytes(struct sess *sp, struct http_conn *htc, size_t bytes)
 	size_t dl;
 	const void *dp;
 
-	CAST_OBJ_NOTNULL(vg, sp->wrk->vfp_private, VGZ_MAGIC);
+	vg = sp->wrk->vgz_rx;
+	CHECK_OBJ_NOTNULL(vg, VGZ_MAGIC);
 	AZ(vg->vz.avail_in);
 	while (bytes > 0 || vg->vz.avail_in > 0) {
 		if (sp->wrk->storage == NULL)
@@ -346,7 +350,8 @@ vfp_gunzip_end(struct sess *sp)
 	struct vgz *vg;
 	struct storage *st;
 
-	CAST_OBJ_NOTNULL(vg, sp->wrk->vfp_private, VGZ_MAGIC);
+	vg = sp->wrk->vgz_rx;
+	CHECK_OBJ_NOTNULL(vg, VGZ_MAGIC);
 	VGZ_Destroy(&vg);
 
 	st = sp->wrk->storage;
@@ -381,11 +386,9 @@ struct vfp vfp_gunzip = {
 static void __match_proto__()
 vfp_gzip_begin(struct sess *sp, size_t estimate)
 {
-	struct vgz *vg;
 	(void)estimate;
 
-	vg = VGZ_NewGzip(sp, sp->ws);
-	sp->wrk->vfp_private = vg;
+	sp->wrk->vgz_rx = VGZ_NewGzip(sp, sp->ws);
 }
 
 static int __match_proto__()
@@ -399,7 +402,8 @@ vfp_gzip_bytes(struct sess *sp, struct http_conn *htc, size_t bytes)
 	size_t dl;
 	const void *dp;
 
-	CAST_OBJ_NOTNULL(vg, sp->wrk->vfp_private, VGZ_MAGIC);
+	vg = sp->wrk->vgz_rx;
+	CHECK_OBJ_NOTNULL(vg, VGZ_MAGIC);
 	AZ(vg->vz.avail_in);
 	while (bytes > 0 || vg->vz.avail_in > 0) {
 		if (sp->wrk->storage == NULL)
@@ -445,7 +449,8 @@ vfp_gzip_end(struct sess *sp)
 	struct vgz *vg;
 	struct storage *st;
 
-	CAST_OBJ_NOTNULL(vg, sp->wrk->vfp_private, VGZ_MAGIC);
+	vg = sp->wrk->vgz_rx;
+	CHECK_OBJ_NOTNULL(vg, VGZ_MAGIC);
 	VGZ_Destroy(&vg);
 
 	st = sp->wrk->storage;



From phk at project.varnish-software.com  Fri Jan 21 11:17:06 2011
From: phk at project.varnish-software.com (Poul-Henning Kamp)
Date: Fri, 21 Jan 2011 12:17:06 +0100
Subject: [master] 13a49a4 Replay SVN r5771
Message-ID: <E1PgEzK-0003JI-3F@project.varnish-software.com>

commit 13a49a4d840c3cb2c7e69b07f7f838b2c82673ae
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Fri Jan 21 11:09:50 2011 +0000

    Replay SVN r5771
    
    Move the delivery gunzip away from session workspace and use a
    64k stackbuffer for bufferspace instead.

diff --git a/bin/varnishd/cache_response.c b/bin/varnishd/cache_response.c
index 45a9748..bbea517 100644
--- a/bin/varnishd/cache_response.c
+++ b/bin/varnishd/cache_response.c
@@ -240,18 +240,20 @@ RES_BuildHttp(struct sess *sp)
  */
 
 static void
-res_WriteGunzipObj(struct sess *sp, char lenbuf[20])
+res_WriteGunzipObj(struct sess *sp)
 {
 	struct storage *st;
 	unsigned u = 0;
 	struct vgz *vg;
 	const void *dp;
+	char lenbuf[20];
+	char obuf[64*1024];	/* XXX: size? */
 	size_t dl;
 	int i;
 
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 
-	vg = VGZ_NewUnzip(sp, sp->ws, sp->wrk->ws, NULL, 0);
+	vg = VGZ_NewUnzip(sp, sp->wrk->ws, NULL, obuf, sizeof obuf);
 	AN(vg);
 
 	VTAILQ_FOREACH(st, &sp->obj->store, list) {
@@ -259,7 +261,7 @@ res_WriteGunzipObj(struct sess *sp, char lenbuf[20])
 		CHECK_OBJ_NOTNULL(st, STORAGE_MAGIC);
 		u += st->len;
 
-		sp->acct_tmp.bodybytes += st->len;
+		sp->acct_tmp.bodybytes += st->len;	/* XXX ? */
 		VSC_main->n_objwrite++;
 
 		VGZ_Feed(vg, st->ptr, st->len);
@@ -389,7 +391,7 @@ RES_WriteObj(struct sess *sp)
 	} else if (sp->wrk->res_mode & RES_ESI) {
 		ESI_Deliver(sp);
 	} else if (sp->wrk->res_mode & RES_GUNZIP) {
-		res_WriteGunzipObj(sp, lenbuf); 
+		res_WriteGunzipObj(sp); 
 	} else {
 		res_WriteDirObj(sp, lenbuf, low, high);
 	}



From phk at project.varnish-software.com  Fri Jan 21 11:17:05 2011
From: phk at project.varnish-software.com (Poul-Henning Kamp)
Date: Fri, 21 Jan 2011 12:17:05 +0100
Subject: [master] 70f943a Replay of SVN r5772
Message-ID: <E1PgEzJ-0003J8-4O@project.varnish-software.com>

commit 70f943a46fc42858f9ab4caad3f4a6d88635f2c1
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Fri Jan 21 11:13:59 2011 +0000

    Replay of SVN r5772
    
    Futher modelling of VGZ functions

diff --git a/bin/varnishd/cache.h b/bin/varnishd/cache.h
index a2b6353..0577fd9 100644
--- a/bin/varnishd/cache.h
+++ b/bin/varnishd/cache.h
@@ -625,10 +625,11 @@ void Fetch_Init(void);
 /* cache_gzip.c */
 struct vgz;
 
-struct vgz *VGZ_NewUnzip(const struct sess *sp, struct ws *tmp,
-    struct ws *buf_ws, void *buf, ssize_t bufl);
-int VGZ_Feed(struct vgz *, const void *, size_t len);
-int VGZ_Produce(struct vgz *, const void **, size_t *len);
+struct vgz *VGZ_NewUngzip(const struct sess *sp, struct ws *tmp);
+void VGZ_Ibuf(struct vgz *, const void *, size_t len);
+void VGZ_Obuf(struct vgz *, const void *, size_t len);
+int VGZ_Gzip(struct vgz *, const void **, size_t *len, int flag);
+int VGZ_Gunzip(struct vgz *, const void **, size_t *len);
 void VGZ_Destroy(struct vgz **);
 
 /* cache_http.c */
diff --git a/bin/varnishd/cache_gzip.c b/bin/varnishd/cache_gzip.c
index 0bf6f2d..acb6606 100644
--- a/bin/varnishd/cache_gzip.c
+++ b/bin/varnishd/cache_gzip.c
@@ -81,9 +81,7 @@ struct vgz {
 	struct ws		*tmp;
 	char			*tmp_snapshot;
 
-	struct ws		*buf_ws;
-	void			*buf;
-	size_t			bufsiz;
+	void			*before;
 
 	z_stream		vz;
 };
@@ -114,7 +112,7 @@ vgz_free(voidpf opaque, voidpf address)
  */
 
 static struct vgz *
-vgz_alloc_vgz(struct ws *ws, struct ws *buf_ws, void *buf, ssize_t bufl)
+vgz_alloc_vgz(struct ws *ws)
 {
 	char *s;
 	struct vgz *vg;
@@ -132,29 +130,16 @@ vgz_alloc_vgz(struct ws *ws, struct ws *buf_ws, void *buf, ssize_t bufl)
 	vg->vz.zfree = vgz_free;
 	vg->vz.opaque = vg;
 
-	assert(buf_ws == NULL || buf == NULL);
-	if (buf_ws != NULL) {
-		WS_Assert(buf_ws);
-		vg->buf_ws = buf_ws;
-		vg->bufsiz = WS_Reserve(buf_ws, 0);
-		vg->buf = buf_ws->f;
-	} else {
-		assert(bufl > 0);
-		vg->buf = buf;
-		vg->bufsiz = bufl;
-	}
-
 	return (vg);
 }
 
 struct vgz *
-VGZ_NewUnzip(const struct sess *sp, struct ws *tmp, struct ws *buf_ws,
-    void *buf, ssize_t bufl)
+VGZ_NewUngzip(const struct sess *sp, struct ws *tmp)
 {
 	struct vgz *vg;
 
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
-	vg = vgz_alloc_vgz(tmp, buf_ws, buf, bufl);
+	vg = vgz_alloc_vgz(tmp);
 
 	/*
 	 * Max memory usage according to zonf.h:
@@ -167,14 +152,13 @@ VGZ_NewUnzip(const struct sess *sp, struct ws *tmp, struct ws *buf_ws,
 }
 
 static struct vgz *
-VGZ_NewGzip(const struct sess *sp, struct ws *tmp, struct ws *buf_ws,
-    void *buf, ssize_t bufl)
+VGZ_NewGzip(const struct sess *sp, struct ws *tmp)
 {
 	struct vgz *vg;
 	int i;
 
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
-	vg = vgz_alloc_vgz(tmp, buf_ws, buf, bufl);
+	vg = vgz_alloc_vgz(tmp);
 
 	/*
 	 * From zconf.h:
@@ -202,8 +186,8 @@ VGZ_NewGzip(const struct sess *sp, struct ws *tmp, struct ws *buf_ws,
 
 /*--------------------------------------------------------------------*/
 
-int
-VGZ_Feed(struct vgz *vg, const void *ptr, size_t len)
+void
+VGZ_Ibuf(struct vgz *vg, const void *ptr, size_t len)
 {
 
 	CHECK_OBJ_NOTNULL(vg, VGZ_MAGIC);
@@ -211,14 +195,24 @@ VGZ_Feed(struct vgz *vg, const void *ptr, size_t len)
 	AZ(vg->vz.avail_in);
 	vg->vz.next_in = TRUST_ME(ptr);
 	vg->vz.avail_in = len;
+}
 
-	return (0);
+/*--------------------------------------------------------------------*/
+
+void
+VGZ_Obuf(struct vgz *vg, const void *ptr, size_t len)
+{
+
+	CHECK_OBJ_NOTNULL(vg, VGZ_MAGIC);
+
+	vg->vz.next_out = TRUST_ME(ptr);
+	vg->vz.avail_out = len;
 }
 
 /*--------------------------------------------------------------------*/
 
 int
-VGZ_Produce(struct vgz *vg, const void **pptr, size_t *plen)
+VGZ_Gunzip(struct vgz *vg, const void **pptr, size_t *plen)
 {
 	int i;
 
@@ -226,13 +220,41 @@ VGZ_Produce(struct vgz *vg, const void **pptr, size_t *plen)
 
 	*pptr = NULL;
 	*plen = 0;
-	vg->vz.next_out = vg->buf;
-	vg->vz.avail_out = vg->bufsiz;
-
+	AN(vg->vz.next_out);
+	AN(vg->vz.avail_out);
+	vg->before = vg->vz.next_out;
 	i = inflate(&vg->vz, 0);
 	if (i == Z_OK || i == Z_STREAM_END) {
-		*pptr = vg->buf;
-		*plen = vg->bufsiz - vg->vz.avail_out;
+		*pptr = vg->before;
+		*plen = (const uint8_t *)vg->vz.next_out - (const uint8_t*)vg->before;
+	}
+	if (i == Z_OK)
+		return (0);
+	if (i == Z_STREAM_END)
+		return (1);
+	if (i == Z_BUF_ERROR)
+		return (2);
+	return (-1);
+}
+
+/*--------------------------------------------------------------------*/
+
+int
+VGZ_Gzip(struct vgz *vg, const void **pptr, size_t *plen, int flags)
+{
+	int i;
+
+	CHECK_OBJ_NOTNULL(vg, VGZ_MAGIC);
+
+	*pptr = NULL;
+	*plen = 0;
+	AN(vg->vz.next_out);
+	AN(vg->vz.avail_out);
+	vg->before = vg->vz.next_out;
+	i = deflate(&vg->vz, flags);
+	if (i == Z_OK || i == Z_STREAM_END) {
+		*pptr = vg->before;
+		*plen = (const uint8_t *)vg->vz.next_out - (const uint8_t*)vg->before;
 	}
 	if (i == Z_OK)
 		return (0);
@@ -250,8 +272,6 @@ VGZ_Destroy(struct vgz **vg)
 {
 
 	CHECK_OBJ_NOTNULL(*vg, VGZ_MAGIC);
-	if ((*vg)->buf_ws != NULL)
-		WS_Release((*vg)->buf_ws, 0);
 	WS_Reset((*vg)->tmp, (*vg)->tmp_snapshot);
 	*vg = NULL;
 }
@@ -266,7 +286,7 @@ static void __match_proto__()
 vfp_gunzip_begin(struct sess *sp, size_t estimate)
 {
 	(void)estimate;
-	sp->wrk->vfp_private = VGZ_NewUnzip(sp, sp->ws, sp->wrk->ws, NULL, 0);
+	sp->wrk->vfp_private = VGZ_NewUngzip(sp, sp->ws);
 }
 
 static int __match_proto__()
@@ -276,6 +296,9 @@ vfp_gunzip_bytes(struct sess *sp, struct http_conn *htc, size_t bytes)
 	struct storage *st;
 	ssize_t l, w;
 	int i = -100;
+	uint8_t	ibuf[64*1024];		/* XXX size ? */
+	size_t dl;
+	const void *dp;
 
 	CAST_OBJ_NOTNULL(vg, sp->wrk->vfp_private, VGZ_MAGIC);
 	AZ(vg->vz.avail_in);
@@ -289,24 +312,22 @@ vfp_gunzip_bytes(struct sess *sp, struct http_conn *htc, size_t bytes)
 		}
 		st = sp->wrk->storage;
 
-		vg->vz.next_out = st->ptr + st->len;
-		vg->vz.avail_out = st->space - st->len;
+		VGZ_Obuf(vg, st->ptr + st->len, st->space - st->len);
 
 		if (vg->vz.avail_in == 0 && bytes > 0) {
-			l = vg->bufsiz;
+			l = sizeof ibuf;
 			if (l > bytes)
 				l = bytes;
-			w = HTC_Read(htc, vg->buf, l);
+			w = HTC_Read(htc, ibuf, l);
 			if (w <= 0)
 				return (w);
-			vg->vz.next_in = vg->buf;
-			vg->vz.avail_in = w;
+			VGZ_Ibuf(vg, ibuf, w);
 			bytes -= w;
 		}
 
-		i = inflate(&vg->vz, 0);
+		i = VGZ_Gunzip(vg, &dp, &dl);
 		assert(i == Z_OK || i == Z_STREAM_END);
-		st->len = st->space - vg->vz.avail_out;
+		st->len += dl;
 		if (st->len == st->space) {
 			VTAILQ_INSERT_TAIL(&sp->obj->store,
 			    sp->wrk->storage, list);
@@ -363,7 +384,7 @@ vfp_gzip_begin(struct sess *sp, size_t estimate)
 	struct vgz *vg;
 	(void)estimate;
 
-	vg = VGZ_NewGzip(sp, sp->ws, sp->wrk->ws, NULL, 0);
+	vg = VGZ_NewGzip(sp, sp->ws);
 	sp->wrk->vfp_private = vg;
 }
 
@@ -374,6 +395,9 @@ vfp_gzip_bytes(struct sess *sp, struct http_conn *htc, size_t bytes)
 	struct storage *st;
 	ssize_t l, w;
 	int i = -100;
+	uint8_t ibuf[64*1024];		/* XXX size ? */
+	size_t dl;
+	const void *dp;
 
 	CAST_OBJ_NOTNULL(vg, sp->wrk->vfp_private, VGZ_MAGIC);
 	AZ(vg->vz.avail_in);
@@ -387,24 +411,22 @@ vfp_gzip_bytes(struct sess *sp, struct http_conn *htc, size_t bytes)
 		}
 		st = sp->wrk->storage;
 
-		vg->vz.next_out = st->ptr + st->len;
-		vg->vz.avail_out = st->space - st->len;
+		VGZ_Obuf(vg, st->ptr + st->len, st->space - st->len);
 
 		if (vg->vz.avail_in == 0 && bytes > 0) {
-			l = vg->bufsiz;
+			l = sizeof ibuf;
 			if (l > bytes)
 				l = bytes;
-			w = HTC_Read(htc, vg->buf, l);
+			w = HTC_Read(htc, ibuf, l);
 			if (w <= 0)
 				return (w);
-			vg->vz.next_in = vg->buf;
-			vg->vz.avail_in = w;
+			VGZ_Ibuf(vg, ibuf, w);
 			bytes -= w;
 		}
 
-		i = deflate(&vg->vz, bytes == 0 ? Z_FINISH : 0);
+		i = VGZ_Gzip(vg, &dp, &dl, bytes == 0 ? Z_FINISH : 0);
 		assert(i == Z_OK || i == Z_STREAM_END);
-		st->len = st->space - vg->vz.avail_out;
+		st->len = st->space - dl;
 		if (st->len == st->space) {
 			VTAILQ_INSERT_TAIL(&sp->obj->store,
 			    sp->wrk->storage, list);
diff --git a/bin/varnishd/cache_response.c b/bin/varnishd/cache_response.c
index bbea517..dbfb520 100644
--- a/bin/varnishd/cache_response.c
+++ b/bin/varnishd/cache_response.c
@@ -253,8 +253,7 @@ res_WriteGunzipObj(struct sess *sp)
 
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 
-	vg = VGZ_NewUnzip(sp, sp->wrk->ws, NULL, obuf, sizeof obuf);
-	AN(vg);
+	vg = VGZ_NewUngzip(sp, sp->wrk->ws);
 
 	VTAILQ_FOREACH(st, &sp->obj->store, list) {
 		CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
@@ -264,9 +263,10 @@ res_WriteGunzipObj(struct sess *sp)
 		sp->acct_tmp.bodybytes += st->len;	/* XXX ? */
 		VSC_main->n_objwrite++;
 
-		VGZ_Feed(vg, st->ptr, st->len);
+		VGZ_Ibuf(vg, st->ptr, st->len);
 		do {
-			i = VGZ_Produce(vg, &dp, &dl);
+			VGZ_Obuf(vg, obuf, sizeof obuf);
+			i = VGZ_Gunzip(vg, &dp, &dl);
 			if (dl != 0) {
 				if (sp->wrk->res_mode & RES_CHUNKED) {
 					bprintf(lenbuf, "%x\r\n", (unsigned)dl);



From phk at project.varnish-software.com  Fri Jan 21 11:17:06 2011
From: phk at project.varnish-software.com (Poul-Henning Kamp)
Date: Fri, 21 Jan 2011 12:17:06 +0100
Subject: [master] 661e05e Replay SVN r5770
Message-ID: <E1PgEzK-0003JZ-Ve@project.varnish-software.com>

commit 661e05ec51036bbe8c9dae401ec09d06ecd5b41a
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Fri Jan 21 11:08:20 2011 +0000

    Replay SVN r5770
    
    Improve and clean up the VGZ allocation a bit.

diff --git a/bin/varnishd/cache.h b/bin/varnishd/cache.h
index cdec155..a2b6353 100644
--- a/bin/varnishd/cache.h
+++ b/bin/varnishd/cache.h
@@ -625,10 +625,11 @@ void Fetch_Init(void);
 /* cache_gzip.c */
 struct vgz;
 
-struct vgz *VGZ_NewUnzip(struct sess *sp, struct ws *tmp, struct ws *buf);
+struct vgz *VGZ_NewUnzip(const struct sess *sp, struct ws *tmp,
+    struct ws *buf_ws, void *buf, ssize_t bufl);
 int VGZ_Feed(struct vgz *, const void *, size_t len);
 int VGZ_Produce(struct vgz *, const void **, size_t *len);
-int VGZ_Destroy(struct vgz **);
+void VGZ_Destroy(struct vgz **);
 
 /* cache_http.c */
 unsigned HTTP_estimate(unsigned nhttp);
diff --git a/bin/varnishd/cache_gzip.c b/bin/varnishd/cache_gzip.c
index d3925ad..0bf6f2d 100644
--- a/bin/varnishd/cache_gzip.c
+++ b/bin/varnishd/cache_gzip.c
@@ -65,6 +65,7 @@
  */
 
 #include "config.h"
+#include <stdio.h>
 
 #include "svnid.h"
 SVNID("$Id$")
@@ -77,11 +78,11 @@ SVNID("$Id$")
 struct vgz {
 	unsigned		magic;
 #define VGZ_MAGIC		0x162df0cb
-	struct sess 		*sp;
 	struct ws		*tmp;
 	char			*tmp_snapshot;
 
-	struct ws		*buf;
+	struct ws		*buf_ws;
+	void			*buf;
 	size_t			bufsiz;
 
 	z_stream		vz;
@@ -96,7 +97,7 @@ vgz_alloc(voidpf opaque, uInt items, uInt size)
 
 	CAST_OBJ_NOTNULL(vg, opaque, VGZ_MAGIC);
 
-	return(WS_Alloc(vg->tmp, items * size));
+	return (WS_Alloc(vg->tmp, items * size));
 }
 
 static void
@@ -108,39 +109,97 @@ vgz_free(voidpf opaque, voidpf address)
 	(void)address;
 }
 
-/*--------------------------------------------------------------------*/
+/*--------------------------------------------------------------------
+ * Set up a gunzip instance
+ */
 
-struct vgz *
-VGZ_NewUnzip(struct sess *sp, struct ws *tmp, struct ws *buf)
+static struct vgz *
+vgz_alloc_vgz(struct ws *ws, struct ws *buf_ws, void *buf, ssize_t bufl)
 {
-	struct vgz *vg;
 	char *s;
+	struct vgz *vg;
 
-	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
-
-	WS_Assert(tmp);
-	WS_Assert(buf);
-
-	s = WS_Snapshot(tmp);
-	vg = (void*)WS_Alloc(tmp, sizeof *vg);
+	WS_Assert(ws);
+	s = WS_Snapshot(ws);
+	vg = (void*)WS_Alloc(ws, sizeof *vg);
 	AN(vg);
 	memset(vg, 0, sizeof *vg);
 	vg->magic = VGZ_MAGIC;
-	vg->sp = sp;
-	vg->tmp = tmp;
-	vg->buf = buf;
+	vg->tmp = ws;
 	vg->tmp_snapshot = s;
 
 	vg->vz.zalloc = vgz_alloc;
 	vg->vz.zfree = vgz_free;
 	vg->vz.opaque = vg;
 
-	vg->bufsiz = WS_Reserve(buf, 0);
+	assert(buf_ws == NULL || buf == NULL);
+	if (buf_ws != NULL) {
+		WS_Assert(buf_ws);
+		vg->buf_ws = buf_ws;
+		vg->bufsiz = WS_Reserve(buf_ws, 0);
+		vg->buf = buf_ws->f;
+	} else {
+		assert(bufl > 0);
+		vg->buf = buf;
+		vg->bufsiz = bufl;
+	}
 
+	return (vg);
+}
+
+struct vgz *
+VGZ_NewUnzip(const struct sess *sp, struct ws *tmp, struct ws *buf_ws,
+    void *buf, ssize_t bufl)
+{
+	struct vgz *vg;
+
+	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+	vg = vgz_alloc_vgz(tmp, buf_ws, buf, bufl);
+
+	/*
+	 * Max memory usage according to zonf.h:
+	 * 	mem_needed = "a few kb" + (1 << (windowBits))
+	 * Since we don't control windowBits, we have to assume
+	 * it is 15, so 34-35KB or so.
+	 */
 	assert(Z_OK == inflateInit2(&vg->vz, 31));
 	return (vg);
 }
 
+static struct vgz *
+VGZ_NewGzip(const struct sess *sp, struct ws *tmp, struct ws *buf_ws,
+    void *buf, ssize_t bufl)
+{
+	struct vgz *vg;
+	int i;
+
+	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+	vg = vgz_alloc_vgz(tmp, buf_ws, buf, bufl);
+
+	/*
+	 * From zconf.h:
+	 *
+	 * 	mem_needed = "a few kb" 
+	 *		+ (1 << (windowBits+2))
+	 *		+  (1 << (memLevel+9))
+	 *
+	 * windowBits [8..15] (-> 1K..128K)
+	 * memLevel [1..9] (-> 1K->256K)
+	 *
+	 * XXX: They probably needs to be params...
+	 */
+	i = deflateInit2(&vg->vz,
+	    0,				/* Level */
+	    Z_DEFLATED,			/* Method */
+	    16 + 8,			/* Window bits (16=gzip + 15) */
+	    1,				/* memLevel */
+	    Z_DEFAULT_STRATEGY);
+	if (i != Z_OK)
+		printf("deflateInit2() = %d\n", i);
+	assert(Z_OK == i);
+	return (vg);
+}
+
 /*--------------------------------------------------------------------*/
 
 int
@@ -167,12 +226,12 @@ VGZ_Produce(struct vgz *vg, const void **pptr, size_t *plen)
 
 	*pptr = NULL;
 	*plen = 0;
-	vg->vz.next_out = (void*)vg->buf->f;
+	vg->vz.next_out = vg->buf;
 	vg->vz.avail_out = vg->bufsiz;
 
 	i = inflate(&vg->vz, 0);
 	if (i == Z_OK || i == Z_STREAM_END) {
-		*pptr = vg->buf->f;
+		*pptr = vg->buf;
 		*plen = vg->bufsiz - vg->vz.avail_out;
 	}
 	if (i == Z_OK)
@@ -186,15 +245,15 @@ VGZ_Produce(struct vgz *vg, const void **pptr, size_t *plen)
 
 /*--------------------------------------------------------------------*/
 
-int
+void
 VGZ_Destroy(struct vgz **vg)
 {
 
 	CHECK_OBJ_NOTNULL(*vg, VGZ_MAGIC);
-	WS_Release((*vg)->buf, 0);
+	if ((*vg)->buf_ws != NULL)
+		WS_Release((*vg)->buf_ws, 0);
 	WS_Reset((*vg)->tmp, (*vg)->tmp_snapshot);
 	*vg = NULL;
-	return (0);
 }
 
 /*--------------------------------------------------------------------
@@ -207,7 +266,7 @@ static void __match_proto__()
 vfp_gunzip_begin(struct sess *sp, size_t estimate)
 {
 	(void)estimate;
-	sp->wrk->vfp_private = VGZ_NewUnzip(sp, sp->ws, sp->wrk->ws);
+	sp->wrk->vfp_private = VGZ_NewUnzip(sp, sp->ws, sp->wrk->ws, NULL, 0);
 }
 
 static int __match_proto__()
@@ -237,10 +296,10 @@ vfp_gunzip_bytes(struct sess *sp, struct http_conn *htc, size_t bytes)
 			l = vg->bufsiz;
 			if (l > bytes)
 				l = bytes;
-			w = HTC_Read(htc, vg->buf->f, l);
+			w = HTC_Read(htc, vg->buf, l);
 			if (w <= 0)
 				return (w);
-			vg->vz.next_in = (void*)vg->buf->f;
+			vg->vz.next_in = vg->buf;
 			vg->vz.avail_in = w;
 			bytes -= w;
 		}
@@ -304,16 +363,7 @@ vfp_gzip_begin(struct sess *sp, size_t estimate)
 	struct vgz *vg;
 	(void)estimate;
 
-	vg = VGZ_NewUnzip(sp, sp->ws, sp->wrk->ws);
-	/* XXX: hack */
-	memset(&vg->vz, 0, sizeof vg->vz);
-	assert(Z_OK == deflateInit2(&vg->vz,
-	    0,
-	    Z_DEFLATED,
-	    31,
-	    9,
-	    Z_DEFAULT_STRATEGY));
-
+	vg = VGZ_NewGzip(sp, sp->ws, sp->wrk->ws, NULL, 0);
 	sp->wrk->vfp_private = vg;
 }
 
@@ -344,10 +394,10 @@ vfp_gzip_bytes(struct sess *sp, struct http_conn *htc, size_t bytes)
 			l = vg->bufsiz;
 			if (l > bytes)
 				l = bytes;
-			w = HTC_Read(htc, vg->buf->f, l);
+			w = HTC_Read(htc, vg->buf, l);
 			if (w <= 0)
 				return (w);
-			vg->vz.next_in = (void*)vg->buf->f;
+			vg->vz.next_in = vg->buf;
 			vg->vz.avail_in = w;
 			bytes -= w;
 		}
diff --git a/bin/varnishd/cache_response.c b/bin/varnishd/cache_response.c
index 0899562..45a9748 100644
--- a/bin/varnishd/cache_response.c
+++ b/bin/varnishd/cache_response.c
@@ -251,7 +251,7 @@ res_WriteGunzipObj(struct sess *sp, char lenbuf[20])
 
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 
-	vg = VGZ_NewUnzip(sp, sp->ws, sp->wrk->ws);
+	vg = VGZ_NewUnzip(sp, sp->ws, sp->wrk->ws, NULL, 0);
 	AN(vg);
 
 	VTAILQ_FOREACH(st, &sp->obj->store, list) {



From phk at project.varnish-software.com  Fri Jan 21 11:17:08 2011
From: phk at project.varnish-software.com (Poul-Henning Kamp)
Date: Fri, 21 Jan 2011 12:17:08 +0100
Subject: [master] 87072ee Replay SVN r5769
Message-ID: <E1PgEzM-0003K8-RL@project.varnish-software.com>

commit 87072eedab8d96f29db2c456a3251cda4ff85afd
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Fri Jan 21 11:06:43 2011 +0000

    Replay SVN r5769
    
    Add a comment with our worst case gzip scenario

diff --git a/bin/varnishd/cache_gzip.c b/bin/varnishd/cache_gzip.c
index 4529364..d3925ad 100644
--- a/bin/varnishd/cache_gzip.c
+++ b/bin/varnishd/cache_gzip.c
@@ -33,6 +33,35 @@
  *
  * The API defined by this file, will also insulate the rest of the code,
  * should we find a better gzip library at a later date.
+ *
+ * The absolutely worst case gzip processing path, once we have pipe-lining,
+ * will be the following, so we need to be a bit careful with the scratch
+ * space we use:
+ *
+ * 	Backend		Tmp	Input	Output
+ *         |		----------------------
+ *	   v
+ *	 gunzip		wrk	stack	?
+ *         |
+ *	   v
+ *	  esi
+ *         |
+ *	   v
+ *	  gzip		wrk	?	storage
+ *         |
+ *	   v
+ *	  cache
+ *         |
+ *	   v
+ *	 gunzip		wrk	storage	stack
+ *         |
+ *	   v
+ *	 client
+ *
+ * XXXX: The two '?' are obviously the same memory, but I have yet to decide
+ * where it goes.   As usual we try to avoid the session->ws if we can but
+ * I may have to use that.
+ *
  */
 
 #include "config.h"



From phk at project.varnish-software.com  Fri Jan 21 11:17:09 2011
From: phk at project.varnish-software.com (Poul-Henning Kamp)
Date: Fri, 21 Jan 2011 12:17:09 +0100
Subject: [master] 1c0a69c Replay SVN r5768
Message-ID: <E1PgEzN-0003KV-Pj@project.varnish-software.com>

commit 1c0a69cba03190f92fab79e690c291c31fcdb316
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Fri Jan 21 11:05:00 2011 +0000

    Replay SVN r5768
    
    Now that the going is staring to get tough, split the ESI parsing from
    the ESI fetch processing to keep source file size manageable.

diff --git a/bin/varnishd/Makefile.am b/bin/varnishd/Makefile.am
index 9beac80..7948977 100644
--- a/bin/varnishd/Makefile.am
+++ b/bin/varnishd/Makefile.am
@@ -25,6 +25,7 @@ varnishd_SOURCES = \
 	cache_dir_random.c \
 	cache_dir_dns.c \
 	cache_dir_round_robin.c \
+	cache_esi_fetch.c \
 	cache_esi_parse.c \
 	cache_esi_deliver.c \
 	cache_expire.c \
diff --git a/bin/varnishd/cache_esi.h b/bin/varnishd/cache_esi.h
index d04a2f9..ad97b46 100644
--- a/bin/varnishd/cache_esi.h
+++ b/bin/varnishd/cache_esi.h
@@ -34,3 +34,9 @@
 #define	VEC_S2	(0x60 + 2)
 #define	VEC_S8	(0x60 + 8)
 #define	VEC_INCL	'I'
+
+void VEP_Init(const struct sess *sp);
+void VEP_parse(const struct sess *sp, const char *p, size_t l);
+struct vsb *VEP_Finish(const struct sess *sp);
+
+
diff --git a/bin/varnishd/cache_esi_deliver.c b/bin/varnishd/cache_esi_deliver.c
index 336715d..e4016c6 100644
--- a/bin/varnishd/cache_esi_deliver.c
+++ b/bin/varnishd/cache_esi_deliver.c
@@ -223,7 +223,7 @@ ESI_Deliver(struct sess *sp)
 			p = r + 1;
 			break;
 		default:
-			Debug("XXXX 0x%02x [%s]\n", *p, p);
+			printf("XXXX 0x%02x [%s]\n", *p, p);
 			INCOMPL();
 		}
 	}
diff --git a/bin/varnishd/cache_esi_fetch.c b/bin/varnishd/cache_esi_fetch.c
new file mode 100644
index 0000000..b69dc65
--- /dev/null
+++ b/bin/varnishd/cache_esi_fetch.c
@@ -0,0 +1,200 @@
+/*-
+ * Copyright (c) 2011 Varnish Software AS
+ * All rights reserved.
+ *
+ * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * VEP Varnish Esi Parsing
+ */
+
+#include "config.h"
+
+#include "svnid.h"
+SVNID("$Id")
+
+#include <stdio.h>
+#include <stdlib.h>
+
+#include "cache.h"
+#include "cache_esi.h"
+#include "vct.h"
+#include "zlib.h"
+#include "stevedore.h"
+
+/*---------------------------------------------------------------------
+ * We receive a ungzip'ed object, and want to store it ungzip'ed.
+ */
+
+static int __match_proto__()
+vfp_esi_bytes_uu(struct sess *sp, struct http_conn *htc, size_t bytes)
+{
+	ssize_t l, w;
+	struct storage *st;
+
+	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+
+	while (bytes > 0) {
+		if (sp->wrk->storage == NULL) {
+			l = params->fetch_chunksize * 1024LL;
+			sp->wrk->storage = STV_alloc(sp, l);
+		}
+		if (sp->wrk->storage == NULL) {
+			errno = ENOMEM;
+			return (-1);
+		}
+		st = sp->wrk->storage;
+		l = st->space - st->len;
+		if (l > bytes)
+			l = bytes;
+		w = HTC_Read(htc, st->ptr + st->len, l);
+		if (w <= 0)
+			return (w);
+		if (params->esi_syntax & 0x8) {
+			ssize_t d;
+			for (l = 0; l < w; l += d)  {
+				d = (random() & 3) + 1;
+				if (l + d >= w)
+					d = 1;
+				VEP_parse(sp,
+				    (const char *)st->ptr + st->len + l, d);
+			}
+		} else
+			VEP_parse(sp, (const char *)st->ptr + st->len, w);
+		st->len += w;
+		sp->obj->len += w;
+		if (st->len == st->space) {
+			VTAILQ_INSERT_TAIL(&sp->obj->store,
+			    sp->wrk->storage, list);
+			sp->wrk->storage = NULL;
+			st = NULL;
+		}
+		bytes -= w;
+	}
+	return (1);
+}
+
+/*---------------------------------------------------------------------
+ * We receive a ungzip'ed object, and want to store it gzip'ed.
+ */
+
+static int __match_proto__()
+vfp_esi_bytes_ug(struct sess *sp, struct http_conn *htc, size_t bytes)
+{
+	return (vfp_esi_bytes_uu(sp, htc, bytes));
+}
+
+/*---------------------------------------------------------------------
+ * We receive a gzip'ed object, and want to store it ungzip'ed.
+ */
+
+static int __match_proto__()
+vfp_esi_bytes_gu(struct sess *sp, struct http_conn *htc, size_t bytes)
+{
+	return (vfp_esi_bytes_uu(sp, htc, bytes));
+}
+
+/*---------------------------------------------------------------------
+ * We receive a gzip'ed object, and want to store it gzip'ed.
+ */
+
+static int __match_proto__()
+vfp_esi_bytes_gg(struct sess *sp, struct http_conn *htc, size_t bytes)
+{
+	return (vfp_esi_bytes_uu(sp, htc, bytes));
+}
+
+
+/*---------------------------------------------------------------------*/
+
+static void __match_proto__()
+vfp_esi_begin(struct sess *sp, size_t estimate)
+{
+
+	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+	/* XXX: snapshot WS's ? We'll need the space */
+
+	VEP_Init(sp);
+
+	(void)estimate;
+}
+
+static int __match_proto__()
+vfp_esi_bytes(struct sess *sp, struct http_conn *htc, size_t bytes)
+{
+	int i;
+
+	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+	if (sp->wrk->is_gzip && sp->wrk->do_gunzip)
+		i = vfp_esi_bytes_gu(sp, htc, bytes);
+	else if (sp->wrk->is_gunzip && sp->wrk->do_gzip)
+		i = vfp_esi_bytes_ug(sp, htc, bytes);
+	else if (sp->wrk->is_gzip)
+		i = vfp_esi_bytes_gg(sp, htc, bytes);
+	else
+		i = vfp_esi_bytes_uu(sp, htc, bytes);
+	return (i);
+}
+
+static int __match_proto__()
+vfp_esi_end(struct sess *sp)
+{
+	struct storage *st;
+	struct vsb *vsb;
+	ssize_t l;
+
+	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+
+	vsb = VEP_Finish(sp);
+
+	if (vsb != NULL) {
+		l = vsb_len(vsb);
+		assert(l > 0);
+		/* XXX: This is a huge waste of storage... */
+		sp->obj->esidata = STV_alloc(sp, l);
+		AN(sp->obj->esidata);
+		memcpy(sp->obj->esidata->ptr, vsb_data(vsb), l);
+		sp->obj->esidata->len = l;
+		vsb_delete(vsb);
+	}
+
+	st = sp->wrk->storage;
+	sp->wrk->storage = NULL;
+	if (st == NULL)
+		return (0);
+
+	if (st->len == 0) {
+		STV_free(st);
+		return (0);
+	}
+	if (st->len < st->space)
+		STV_trim(st, st->len);
+	VTAILQ_INSERT_TAIL(&sp->obj->store, st, list);
+	return (0);
+}
+
+struct vfp vfp_esi = {
+        .begin  =       vfp_esi_begin,
+        .bytes  =       vfp_esi_bytes,
+        .end    =       vfp_esi_end,
+};
diff --git a/bin/varnishd/cache_esi_parse.c b/bin/varnishd/cache_esi_parse.c
index 660c0ce..1325a74 100644
--- a/bin/varnishd/cache_esi_parse.c
+++ b/bin/varnishd/cache_esi_parse.c
@@ -61,10 +61,9 @@ enum vep_mark { VERBATIM = 0, SKIP };
 struct vep_state {
 	unsigned		magic;
 #define VEP_MAGIC		0x55cb9b82
-	vfp_bytes_f		*bytes;
 	struct vsb		*vsb;
 
-	struct sess		*sp;
+	const struct sess	*sp;
 
 	/* parser state */
 	const char		*state;
@@ -525,16 +524,25 @@ vep_do_include(struct vep_state *vep, enum dowhat what)
  * NB: the state-machine.  Please maintain it along with the code.
  */
 
-static void
-vep_parse(struct vep_state *vep, const char *p, size_t l)
+void
+VEP_parse(const struct sess *sp, const char *p, size_t l)
 {
+	struct vep_state *vep;
 	const char *e;
 	struct vep_match *vm;
 	int i;
 
+	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+	vep = sp->wrk->vep;
 	CHECK_OBJ_NOTNULL(vep, VEP_MAGIC);
 	assert(l > 0);
 
+	/* XXX: Really need to fix this */
+	if (vep->hack_p == NULL)
+		vep->hack_p = p;
+
+	vep->ver_p = p;
+
 	e = p + l;
 
 	while (p < e) {
@@ -942,121 +950,38 @@ vep_parse(struct vep_state *vep, const char *p, size_t l)
 		vep_mark_pending(vep, p);
 }
 
-/*---------------------------------------------------------------------
- * We receive a ungzip'ed object, and want to store it ungzip'ed.
- */
-
-static int __match_proto__()
-vfp_esi_bytes_uu(struct sess *sp, struct http_conn *htc, size_t bytes)
-{
-	struct vep_state *vep;
-	ssize_t l, w;
-	struct storage *st;
-
-	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
-	vep = sp->wrk->vep;
-	CHECK_OBJ_NOTNULL(vep, VEP_MAGIC);
-
-	while (bytes > 0) {
-		if (sp->wrk->storage == NULL) {
-			l = params->fetch_chunksize * 1024LL;
-			sp->wrk->storage = STV_alloc(sp, l);
-		}
-		if (sp->wrk->storage == NULL) {
-			errno = ENOMEM;
-			return (-1);
-		}
-		st = sp->wrk->storage;
-		l = st->space - st->len;
-		if (l > bytes)
-			l = bytes;
-		w = HTC_Read(htc, st->ptr + st->len, l);
-		if (w <= 0)
-			return (w);
-		if (vep->hack_p == NULL)
-			vep->hack_p = (const char *)st->ptr + st->len;
-		vep->ver_p = (const char *)st->ptr + st->len;
-		if (params->esi_syntax & 0x8) {
-			ssize_t d;
-			for (l = 0; l < w; l += d)  {
-				d = (random() & 3) + 1;
-				if (l + d >= w)
-					d = 1;
-				vep_parse(vep,
-				    (const char *)st->ptr + st->len + l, d);
-			}
-		} else
-			vep_parse(vep, (const char *)st->ptr + st->len, w);
-		st->len += w;
-		sp->obj->len += w;
-		if (st->len == st->space) {
-			VTAILQ_INSERT_TAIL(&sp->obj->store,
-			    sp->wrk->storage, list);
-			sp->wrk->storage = NULL;
-			st = NULL;
-		}
-		bytes -= w;
-	}
-	return (1);
-}
-
-/*---------------------------------------------------------------------*/
-
-static void __match_proto__()
-vfp_esi_begin(struct sess *sp, size_t estimate)
+void
+VEP_Init(const struct sess *sp)
 {
 	struct vep_state *vep;
 
-	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 	AZ(sp->wrk->vep);
-	/* XXX: snapshot WS ? We'll need the space */
-	vep = (void*)WS_Alloc(sp->wrk->ws, sizeof *vep);
-	AN(vep);
-
-	Debug("BEGIN %p\n", vep);
+	sp->wrk->vep = (void*)WS_Alloc(sp->wrk->ws, sizeof *vep);
+	AN(sp->wrk->vep);
 
+	vep = sp->wrk->vep;
 	memset(vep, 0, sizeof *vep);
 	vep->magic = VEP_MAGIC;
 	vep->sp = sp;
-	vep->bytes = vfp_esi_bytes_uu;
+
 	vep->state = VEP_START;
 	vep->vsb = vsb_newauto();
 	AN(vep->vsb);
 	vep->crc = crc32(0L, Z_NULL, 0);
 	vep->crcp = crc32(0L, Z_NULL, 0);
-
-	sp->wrk->vep = vep;
-	(void)estimate;
-}
-
-static int __match_proto__()
-vfp_esi_bytes(struct sess *sp, struct http_conn *htc, size_t bytes)
-{
-	struct vep_state *vep;
-
-	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
-	vep = sp->wrk->vep;
-	Debug("BYTES %jd\n", (intmax_t)bytes);
-	CHECK_OBJ_NOTNULL(vep, VEP_MAGIC);
-	AN(vep->bytes);
-	return (vep->bytes(sp, htc, bytes));
 }
 
-static int __match_proto__()
-vfp_esi_end(struct sess *sp)
+struct vsb *
+VEP_Finish(const struct sess *sp)
 {
-	struct storage *st;
 	struct vep_state *vep;
 	ssize_t l;
 
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 	vep = sp->wrk->vep;
 	sp->wrk->vep = NULL;
-	Debug("ENDING %p\n", vep);
 	CHECK_OBJ_NOTNULL(vep, VEP_MAGIC);
 
-	l = sp->obj->len - vep->o_total;
-	assert(l >= 0);
 	if (vep->o_pending)
 		vep_mark_common(vep, vep->ver_p, vep->last_mark);
 	if (vep->o_wait > 0)
@@ -1064,39 +989,12 @@ vfp_esi_end(struct sess *sp)
 
 	vsb_finish(vep->vsb);
 	l = vsb_len(vep->vsb);
-	if (vep->state != VEP_NOTXML && l > 0) {
-		Debug("ESI %d <%s>\n", (int)l, vsb_data(vep->vsb));
-
-		/* XXX: This is a huge waste of storage... */
-		sp->obj->esidata = STV_alloc(sp, l);
-		AN(sp->obj->esidata);
-		memcpy(sp->obj->esidata->ptr, vsb_data(vep->vsb), l);
-		sp->obj->esidata->len = l;
-	}
+	if (vep->state != VEP_NOTXML && l > 0)
+		return (vep->vsb);
 	vsb_delete(vep->vsb);
-
-	st = sp->wrk->storage;
-	sp->wrk->storage = NULL;
-	if (st == NULL)
-		return (0);
-
-	if (st->len == 0) {
-		STV_free(st);
-		return (0);
-	}
-	if (st->len < st->space)
-		STV_trim(st, st->len);
-	VTAILQ_INSERT_TAIL(&sp->obj->store, st, list);
-	sp->wrk->vep = NULL;
-	return (0);
+	return (NULL);
 }
 
-struct vfp vfp_esi = {
-        .begin  =       vfp_esi_begin,
-        .bytes  =       vfp_esi_bytes,
-        .end    =       vfp_esi_end,
-};
-
 #if 0
 
 digraph xml {



From phk at project.varnish-software.com  Fri Jan 21 11:17:10 2011
From: phk at project.varnish-software.com (Poul-Henning Kamp)
Date: Fri, 21 Jan 2011 12:17:10 +0100
Subject: [master] a97f5ff Replay SVN r5767
Message-ID: <E1PgEzO-0003Kl-GK@project.varnish-software.com>

commit a97f5ffd8914ed00a1eebea9cfe498bc068859bf
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Fri Jan 21 11:02:11 2011 +0000

    Replay SVN r5767
    
    Roll out the gzip/gunzip logic some more

diff --git a/bin/varnishd/cache.h b/bin/varnishd/cache.h
index 0426188..cdec155 100644
--- a/bin/varnishd/cache.h
+++ b/bin/varnishd/cache.h
@@ -282,7 +282,9 @@ struct worker {
 	void			*vfp_private;
 	unsigned		do_esi;
 	unsigned		do_gzip;
+	unsigned		is_gzip;
 	unsigned		do_gunzip;
+	unsigned		is_gunzip;
 
 	/* ESI stuff */
 	struct vep_state	*vep;
diff --git a/bin/varnishd/cache_center.c b/bin/varnishd/cache_center.c
index 1d82650..b55b903 100644
--- a/bin/varnishd/cache_center.c
+++ b/bin/varnishd/cache_center.c
@@ -587,26 +587,53 @@ cnt_fetch(struct sess *sp)
 
 	AZ(sp->wrk->vfp);
 
-	/* We won't gunzip unless it is gzip'ed, if we do remove C-E header */
-	if (sp->wrk->do_gunzip &&
-	     !http_HdrIs(sp->wrk->beresp, H_Content_Encoding, "gzip"))
+	/*
+	 * The VCL variables beresp.do_g[un]zip tells us how we want the
+	 * object stored.
+	 *
+ 	 * The backend Content-Encoding header tells us what we are going
+	 * to receive, which we classify in the following three classes:
+	 *
+	 * 	"Content-Encoding: gzip"	--> object is gzip'ed.
+	 * 	no Content-Encoding		--> object is not gzip'ed.
+	 * 	anything else			--> do nothing wrt gzip
+	 *
+	 */
+
+	/* We do nothing unless the param is set */
+	if (!params->http_gzip_support)
+		sp->wrk->do_gzip = sp->wrk->do_gunzip = 0;
+
+	sp->wrk->is_gzip = 
+	    http_HdrIs(sp->wrk->beresp, H_Content_Encoding, "gzip");
+
+	sp->wrk->is_gunzip = 
+	    !http_GetHdr(sp->wrk->beresp, H_Content_Encoding, NULL);
+
+	/* It can't be both */
+	assert(sp->wrk->is_gzip == 0 || sp->wrk->is_gunzip == 0);
+
+	/* We won't gunzip unless it is gzip'ed */
+	if (sp->wrk->do_gunzip && !sp->wrk->is_gzip)
 		sp->wrk->do_gunzip = 0;
-	if (sp->wrk->do_gunzip) 
-		http_Unset(sp->wrk->beresp, H_Content_Encoding);
 
+	/* If we do gunzip, remove the C-E header */
+	if (sp->wrk->do_gunzip)
+		http_Unset(sp->wrk->beresp, H_Content_Encoding);
 
-	/* And we wont gzip if it already has a C-E header, if we do add it */
-	if (sp->wrk->do_gzip &&
-	     http_GetHdr(sp->wrk->beresp, H_Content_Encoding, NULL))
+	/* We wont gzip unless it is ungziped */
+	if (sp->wrk->do_gzip && !sp->wrk->is_gunzip)
 		sp->wrk->do_gzip = 0;
+
+	/* If we do gzip, add the C-E header */
 	if (sp->wrk->do_gzip) 
 		http_PrintfHeader(sp->wrk, sp->fd, sp->wrk->beresp,
 		    "Content-Encoding: %s", "gzip");
 
-	/* But we can't do both */
+	/* But we can't do both at the same time */
 	assert(sp->wrk->do_gzip == 0 || sp->wrk->do_gunzip == 0);
 
-	/* ESI takes precedence and handles gzip/gunzip also */
+	/* ESI takes precedence and handles gzip/gunzip itself */
 	if (sp->wrk->do_esi)
 		sp->wrk->vfp = &vfp_esi;
 	else if (sp->wrk->do_gunzip)



From phk at project.varnish-software.com  Fri Jan 21 11:17:11 2011
From: phk at project.varnish-software.com (Poul-Henning Kamp)
Date: Fri, 21 Jan 2011 12:17:11 +0100
Subject: [master] da30ce1 Replay of SVN r5766
Message-ID: <E1PgEzP-0003L4-1y@project.varnish-software.com>

commit da30ce1d1cf387c1cdcfddd3777f924d4b551828
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Fri Jan 21 11:00:27 2011 +0000

    Replay of SVN r5766
    
    Unify header handling for gzip, and make wrk->do_g[un]zip usable
    as intention flags.

diff --git a/bin/varnishd/cache_center.c b/bin/varnishd/cache_center.c
index 53c653c..1d82650 100644
--- a/bin/varnishd/cache_center.c
+++ b/bin/varnishd/cache_center.c
@@ -586,20 +586,33 @@ cnt_fetch(struct sess *sp)
 	}
 
 	AZ(sp->wrk->vfp);
-	/* XXX: precedence, also: do_esi */
-	if (sp->wrk->do_esi) {
-		sp->wrk->vfp = &vfp_esi;
-	} else
+
+	/* We won't gunzip unless it is gzip'ed, if we do remove C-E header */
 	if (sp->wrk->do_gunzip &&
-	    http_HdrIs(sp->wrk->beresp, H_Content_Encoding, "gzip")) {
+	     !http_HdrIs(sp->wrk->beresp, H_Content_Encoding, "gzip"))
+		sp->wrk->do_gunzip = 0;
+	if (sp->wrk->do_gunzip) 
 		http_Unset(sp->wrk->beresp, H_Content_Encoding);
-		sp->wrk->vfp = &vfp_gunzip;
-	} else if (sp->wrk->do_gzip &&
-	    !http_HdrIs(sp->wrk->beresp, H_Content_Encoding, "gzip")) {
+
+
+	/* And we wont gzip if it already has a C-E header, if we do add it */
+	if (sp->wrk->do_gzip &&
+	     http_GetHdr(sp->wrk->beresp, H_Content_Encoding, NULL))
+		sp->wrk->do_gzip = 0;
+	if (sp->wrk->do_gzip) 
 		http_PrintfHeader(sp->wrk, sp->fd, sp->wrk->beresp,
 		    "Content-Encoding: %s", "gzip");
+
+	/* But we can't do both */
+	assert(sp->wrk->do_gzip == 0 || sp->wrk->do_gunzip == 0);
+
+	/* ESI takes precedence and handles gzip/gunzip also */
+	if (sp->wrk->do_esi)
+		sp->wrk->vfp = &vfp_esi;
+	else if (sp->wrk->do_gunzip)
+		sp->wrk->vfp = &vfp_gunzip;
+	else if (sp->wrk->do_gzip)
 		sp->wrk->vfp = &vfp_gzip;
-	}
 
 	l = http_EstimateWS(sp->wrk->beresp,
 	    sp->pass ? HTTPH_R_PASS : HTTPH_A_INS, &nhttp);



From phk at project.varnish-software.com  Fri Jan 21 11:17:11 2011
From: phk at project.varnish-software.com (Poul-Henning Kamp)
Date: Fri, 21 Jan 2011 12:17:11 +0100
Subject: [master] 8e00aa8 Replay of SVN r5765
Message-ID: <E1PgEzP-0003LD-Ae@project.varnish-software.com>

commit 8e00aa82fc2ab8209e0d420487b34ae9da65b997
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Fri Jan 21 10:58:25 2011 +0000

    Replay of SVN r5765
    
    Minor cleanups

diff --git a/bin/varnishd/cache_esi_deliver.c b/bin/varnishd/cache_esi_deliver.c
index 4a75ce1..336715d 100644
--- a/bin/varnishd/cache_esi_deliver.c
+++ b/bin/varnishd/cache_esi_deliver.c
@@ -25,6 +25,7 @@
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
  *
+ * VED - Varnish Esi Delivery
  */
 
 #include "config.h"
@@ -45,7 +46,7 @@ SVNID("$Id")
 /*--------------------------------------------------------------------*/
 
 static void
-ESI_Include(struct sess *sp, const char *src, const char *host)
+ved_include(struct sess *sp, const char *src, const char *host)
 {
 	struct object *obj;
 	struct worker *w;
@@ -125,7 +126,7 @@ ESI_Include(struct sess *sp, const char *src, const char *host)
 #define Debug(fmt, ...) /**/
 
 static void
-esi_sendchunk(const struct sess *sp, const void *cb, ssize_t cl,
+ved_sendchunk(const struct sess *sp, const void *cb, ssize_t cl,
     const void *ptr, ssize_t l)
 {
 
@@ -197,11 +198,8 @@ ESI_Deliver(struct sess *sp)
 			assert (q > p);
 			crc_ref = crc32(0L, Z_NULL, 0);
 			crc_ref = crc32(crc_ref, st->ptr + off, l);
-			if (crc_ref != crc) {
-				printf("CRC Mismatch %08x %08x\n", crc_ref, crc);
-			}
 			xxxassert(crc_ref == crc);
-			esi_sendchunk(sp, p, q - p, st->ptr + off, l);
+			ved_sendchunk(sp, p, q - p, st->ptr + off, l);
 			off += l;
 			p = q + 1;
 			break;
@@ -220,7 +218,7 @@ ESI_Deliver(struct sess *sp)
 			r = (void*)strchr((const char*)q, '\0');
 			AN(r);
 			Debug("INCL [%s][%s] BEGIN\n", q, p);
-			ESI_Include(sp, (const char*)q, (const char*)p);
+			ved_include(sp, (const char*)q, (const char*)p);
 			Debug("INCL [%s][%s] END\n", q, p);
 			p = r + 1;
 			break;
diff --git a/bin/varnishd/cache_esi_parse.c b/bin/varnishd/cache_esi_parse.c
index f7f1a82..660c0ce 100644
--- a/bin/varnishd/cache_esi_parse.c
+++ b/bin/varnishd/cache_esi_parse.c
@@ -25,6 +25,7 @@
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
  *
+ * VEP Varnish Esi Parsing
  */
 
 #include "config.h"



From phk at project.varnish-software.com  Fri Jan 21 11:17:11 2011
From: phk at project.varnish-software.com (Poul-Henning Kamp)
Date: Fri, 21 Jan 2011 12:17:11 +0100
Subject: [master] 0b06c85 Replay of SVN r5764:
Message-ID: <E1PgEzP-0003LV-JL@project.varnish-software.com>

commit 0b06c851e22d75df4ae061d9e5ea6f29a74580a7
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Fri Jan 21 10:56:26 2011 +0000

    Replay of SVN r5764:
    
    Retire the old ESI code.

diff --git a/bin/varnishd/Makefile.am b/bin/varnishd/Makefile.am
index d2ccd01..9beac80 100644
--- a/bin/varnishd/Makefile.am
+++ b/bin/varnishd/Makefile.am
@@ -27,7 +27,6 @@ varnishd_SOURCES = \
 	cache_dir_round_robin.c \
 	cache_esi_parse.c \
 	cache_esi_deliver.c \
-	cache_esi.c \
 	cache_expire.c \
 	cache_fetch.c \
 	cache_gzip.c \
diff --git a/bin/varnishd/cache.h b/bin/varnishd/cache.h
index 028d6bf..0426188 100644
--- a/bin/varnishd/cache.h
+++ b/bin/varnishd/cache.h
@@ -35,8 +35,6 @@
  */
 #define VARNISH_CACHE_CHILD	1
 
-#undef OLD_ESI 
-
 #include <sys/time.h>
 #include <sys/uio.h>
 #include <sys/socket.h>
@@ -94,9 +92,6 @@ struct objhead;
 struct objcore;
 struct storage;
 struct workreq;
-#ifdef OLD_ESI
-struct esidata;
-#endif
 struct vrt_backend;
 struct cli_proto;
 struct ban;
@@ -444,11 +439,7 @@ struct object {
 
 	VTAILQ_HEAD(, storage)	store;
 
-#ifdef OLD_ESI
-	struct esidata		*esidata;
-#else
 	struct storage		*esidata;
-#endif
 
 	double			last_use;
 
@@ -807,16 +798,7 @@ void VCL_Poll(void);
 char *VRT_String(struct ws *ws, const char *h, const char *p, va_list ap);
 char *VRT_StringList(char *d, unsigned dl, const char *p, va_list ap);
 
-#ifdef OLD_ESI
-/* cache_vrt_esi.c */
-
-void ESI_Deliver(struct sess *);
-void ESI_Destroy(struct object *);
-void ESI_Parse(struct sess *);
-#else
 void ESI_Deliver(struct sess *);
-#endif /* OLD_ESI */
-void ESI_Include(struct sess *sp, const char *src, const char *host);
 
 /* cache_vrt_vmod.c */
 void VMOD_Init(void);
diff --git a/bin/varnishd/cache_center.c b/bin/varnishd/cache_center.c
index ac5351f..53c653c 100644
--- a/bin/varnishd/cache_center.c
+++ b/bin/varnishd/cache_center.c
@@ -587,11 +587,9 @@ cnt_fetch(struct sess *sp)
 
 	AZ(sp->wrk->vfp);
 	/* XXX: precedence, also: do_esi */
-#ifndef OLD_ESI
 	if (sp->wrk->do_esi) {
 		sp->wrk->vfp = &vfp_esi;
 	} else
-#endif
 	if (sp->wrk->do_gunzip &&
 	    http_HdrIs(sp->wrk->beresp, H_Content_Encoding, "gzip")) {
 		http_Unset(sp->wrk->beresp, H_Content_Encoding);
@@ -677,11 +675,6 @@ cnt_fetch(struct sess *sp)
 		return (0);
 	}
 
-#ifdef OLD_ESI
-	if (sp->wrk->do_esi)
-		ESI_Parse(sp);
-#endif
-
 	switch (sp->handling) {
 	case VCL_RET_RESTART:
 		HSH_Drop(sp);
diff --git a/bin/varnishd/cache_esi.c b/bin/varnishd/cache_esi.c
index 9118a01..e69de29 100644
--- a/bin/varnishd/cache_esi.c
+++ b/bin/varnishd/cache_esi.c
@@ -1,897 +0,0 @@
-/*-
- * Copyright (c) 2006 Verdens Gang AS
- * Copyright (c) 2006-2010 Linpro AS
- * All rights reserved.
- *
- * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- *
- * Runtime support for compiled VCL programs ESI processing.
- *
- * The basic ESI 1.0 is a very simple specification:
- *	http://www.w3.org/TR/esi-lang
- * But it seems that Oracle and Akamai has embrodiered it to be almost a new
- * layer of scripting language in HTTP transmission chains.
- *
- * It is not obvious how much help the "advanced" features of ESI really
- * are to users, so our aim is to pick the fruit starting with the lowest
- * hanging, esi:include
- */
-
-
-#include "config.h"
-
-#include "svnid.h"
-SVNID("$Id$")
-
-#include <stdio.h>
-#include <unistd.h>
-#include <ctype.h>
-#include <string.h>
-#include <stdlib.h>
-#include <stdarg.h>
-
-#include "vrt.h"
-#include "vcl.h"
-#include "vct.h"
-#include "cache.h"
-#include "stevedore.h"
-
-#ifdef OLD_ESI
-
-/*--------------------------------------------------------------------*/
-
-struct esi_bit {
-	VTAILQ_ENTRY(esi_bit)	list;
-	char			chunk_length[20];
-	txt			verbatim;
-	txt			host;
-	txt			include;
-};
-
-struct esi_ptr {
-	const char		*p;
-	const char              *e;
-	struct storage		*st;
-};
-
-struct esi_work {
-	struct sess		*sp;
-	size_t			off;
-
-	struct esi_ptr		s;
-	struct esi_ptr		p;
-
-	txt			tag;
-
-	txt			t;
-	struct esi_bit		*eb;
-	int			remflg;	/* inside <esi:remove> </esi:remove> */
-	int			incmt;	/* inside <!--esi ... --> comment */
-
-	unsigned		space;	/* ... needed */
-
-	VTAILQ_HEAD(, esi_bit)	esibits;
-
-};
-
-struct esidata	{
-	unsigned		magic;
-#define ESIDATA_MAGIC		0x7255277f
-	VTAILQ_HEAD(, esi_bit)	esibits;
-	struct storage		*storage;
-};
-
-/*--------------------------------------------------------------------
- * Move an esi_ptr one char forward
- */
-
-static void
-Nep(struct esi_ptr *ep)
-{
-	static const char * const finis = "";
-
-	if (ep->p == finis)
-		return;
-	ep->p++;
-	if (ep->p < (char*)ep->st->ptr + ep->st->len)
-		return;
-	ep->st = VTAILQ_NEXT(ep->st, list);
-	if (ep->st != NULL) {
-		ep->p = (char *)ep->st->ptr;
-		ep->e = ep->p + ep->st->len;
-		return;
-	}
-	ep->p = finis;
-	ep->e = finis;
-	return;
-}
-
-/*--------------------------------------------------------------------
- * Consume one input character.
- */
-
-static void
-N(struct esi_work *ew)
-{
-
-	if (ew->p.p < ew->p.e)
-		ew->off++;
-	Nep(&ew->p);
-}
-
-/*--------------------------------------------------------------------
- * Strcmp for objects pointers
- */
-
-static int
-CMP(const struct esi_ptr *ep, const char *str)
-{
-	struct esi_ptr p2;
-
-	for (p2 = *ep; *str == *p2.p; str++)
-		Nep(&p2);
-	return (*str);
-}
-
-
-/*--------------------------------------------------------------------
- * Replace the mandatory XML 1.0 entity references, in place.
- */
-
-static void
-XMLentity(txt *t)
-{
-	char *s, *d;
-
-	for (s = d = t->b; s < t->e; ) {
-		if (*s == '&') {
-#define R(l,f,r)							\
-			if (s + l <= t->e && !memcmp(s, f, l)) {	\
-				*d++ = r;				\
-				s += l;					\
-				continue;				\
-			}
-			R(6, "&apos;", '\'');
-			R(6, "&quot;", '"');
-			R(4, "&lt;", '<');
-			R(4, "&gt;", '>');
-			R(5, "&amp;", '&');
-		}
-#undef R
-		*d++ = *s++;
-	}
-	t->e = d;
-	t->e[0] = '\0';
-}
-
-
-/*--------------------------------------------------------------------
- * Report a parsing error
- *
- * XXX: The "at xxx" count is usually the tail of the sequence.  Since we
- * XXX: wander over the storage in an oderly manner now, we could keep
- * XXX: track of line+pos and record the beginning of the stuff that
- * XXX: offends os in the central dispatch loop.
- * XXX: This is left a an excercise for the reader.
- */
-
-static void
-esi_error(const struct esi_work *ew, const char *p, int i, const char *err)
-{
-	int ellipsis = 0;
-	char buf[256], *q;
-	txt t;
-
-	VSC_main->esi_errors++;
-	if (i == 0)
-		i = p - ew->t.b;
-	if (i > 20) {
-		i = 20;
-		ellipsis = 1;
-	}
-	q = buf;
-	q += sprintf(buf, "at %zu: %s \"", ew->off, err);
-	while (i > 0) {
-		if (*p >= ' ' && *p <= '~') {
-			*q++ = *p;
-		} else if (*p == '\n') {
-			*q++ = '\\';
-			*q++ = 'n';
-		} else if (*p == '\r') {
-			*q++ = '\\';
-			*q++ = 'r';
-		} else if (*p == '\t') {
-			*q++ = '\\';
-			*q++ = 't';
-		} else {
-			/* XXX: use %%%02x instead ? */
-			q += sprintf(q, "\\x%02x", *p & 0xff);
-		}
-		p++;
-		i--;
-	}
-	if (ellipsis) {
-		*q++ = '[';
-		*q++ = '.';
-		*q++ = '.';
-		*q++ = '.';
-		*q++ = ']';
-	}
-	*q++ = '"';
-	*q++ = '\0';
-	t.b = buf;
-	t.e = q;
-	WSPR(ew->sp, SLT_ESI_xmlerror, t);
-}
-
-/*--------------------------------------------------------------------
- * Add ESI bit to object
- */
-
-static void
-esi_addbit(struct esi_work *ew, const char *verbatim, unsigned len)
-{
-
-	ew->space += sizeof(*ew->eb);
-	ew->eb = (void*)WS_Alloc(ew->sp->wrk->ws, sizeof *ew->eb);
-	AN(ew->eb);
-	memset(ew->eb, 0, sizeof *ew->eb);
-
-	VTAILQ_INSERT_TAIL(&ew->esibits, ew->eb, list);
-	if (verbatim != NULL) {
-		ew->eb->verbatim.b = TRUST_ME(verbatim);
-		if (len > 0)
-			ew->eb->verbatim.e = TRUST_ME(verbatim + len);
-		sprintf(ew->eb->chunk_length, "%x\r\n", Tlen(ew->eb->verbatim));
-		if (params->esi_syntax & 0x4)
-			WSP(ew->sp, SLT_Debug, "AddBit: %d <%.*s>",
-			    Tlen(ew->eb->verbatim),
-			    Tlen(ew->eb->verbatim),
-			    ew->eb->verbatim.b);
-	} else {
-		AN(ew->s.p);
-		ew->eb->verbatim.b = ew->eb->verbatim.e = TRUST_ME(ew->s.p);
-	}
-}
-
-/*--------------------------------------------------------------------*/
-
-static void
-esi_addpfx(struct esi_work *ew)
-{
-	const char *ep;
-
-	if (ew->remflg) {
-		/* In <esi:remove...> don't add anything */
-		ew->s = ew->p;
-		return;
-	}
-	while (ew->s.st != ew->p.st) {
-		ep = (const char *)(ew->s.st->ptr + ew->s.st->len);
-		esi_addbit(ew, ew->s.p, ep - ew->s.p);
-		ew->s.p = ep;
-		Nep(&ew->s);
-	}
-	if (ew->s.st != NULL && ew->p.p != ew->s.p)
-		esi_addbit(ew, ew->s.p, ew->p.p - ew->s.p);
-	ew->s.p = ew->p.p;
-}
-
-/*--------------------------------------------------------------------
- * Tease the next attribute and value out of an XML element.
- *
- * XXX: is the syntax correct ?
- */
-
-static int
-esi_attrib(const struct esi_work *ew, txt *in, txt *attrib, txt *val)
-{
-
-	AN(*in->b);
-	/* Skip leading blanks */
-	while(in->b < in->e && isspace(*in->b))
-		in->b++;
-
-	/* Nothing found */
-	if (in->b >= in->e)
-		return (0);
-
-	if (!vct_isxmlnamestart(*in->b)) {
-		/* XXX error */
-		esi_error(ew, in->b, 1,
-		    "XML 1.0 Illegal attribute start character");
-		return (-1);
-	}
-
-	/* Attribute name until '=' or space */
-	*attrib = *in;
-	while(in->b < in->e && *in->b != '=' && !isspace(*in->b)) {
-		if (!vct_isxmlname(*in->b)) {
-			esi_error(ew, attrib->b, 1 + (in->b - attrib->b),
-			    "XML 1.0 Illegal attribute character");
-			return (-1);
-		}
-		in->b++;
-	}
-	attrib->e = in->b;
-
-	if (in->b >= in->e || isspace(*in->b)) {
-		/* Attribute without value */
-		val->b = val->e = in->b;
-		return (1);
-	}
-
-	/* skip '=' */
-	in->b++;
-
-	if (isspace(*in->b)) {
-		val->e = val->b = in->b;
-		in->b++;
-		return (1);
-	}
-
-	/* Value, if any ? */
-	*val = *in;
-	if (in->b >= in->e)
-		return (1);
-
-	if (*in->b == '"') {
-		/* Skip quote */
-		in->b++;
-		val->b++;
-
-		/* Anything goes, until next quote */
-		while(in->b < in->e && *in->b != '"')
-			in->b++;
-		val->e = in->b;
-
-		if (in->b >= in->e) {
-			esi_error(ew, val->b, in->e - val->b,
-			    "XML 1.0 missing ending quote");
-			return (-1);
-		}
-
-		/* Skip quote */
-		in->b++;
-	} else {
-		/* Anything until whitespace */
-		while(in->b < in->e && !isspace(*in->b))
-			in->b++;
-		val->e = in->b;
-		in->b++;
-	}
-	return (1);
-}
-
-/*--------------------------------------------------------------------
- * Add one piece to the output, either verbatim or include
- */
-
-static void
-esi_handle_include(struct esi_work *ew)
-{
-	struct esi_bit *eb;
-	char *p, *q, *c;
-	txt t = ew->tag;
-	txt tag;
-	txt val;
-	unsigned u, v, s;
-
-	if (ew->eb == NULL || ew->eb->include.b != NULL)
-		esi_addbit(ew, NULL, 0);
-	eb = ew->eb;
-	WSP(ew->sp, SLT_Debug, "Incl \"%.*s\"", t.e - t.b, t.b);
-	while (esi_attrib(ew, &t, &tag, &val) == 1) {
-		if (params->esi_syntax & 0x4)
-			WSP(ew->sp, SLT_Debug, "<%.*s> -> <%.*s>",
-			    tag.e - tag.b, tag.b, val.e - val.b, val.b);
-		if (Tlen(tag) != 3 || memcmp(tag.b, "src", 3))
-			continue;
-		if (Tlen(val) == 0) {
-			esi_error(ew, tag.b, Tlen(tag),
-			    "ESI esi:include src attribute without value");
-			continue;
-		}
-
-		/* We are saving the original string */
-		s = 0;
-
-		if (val.b != val.e) {
-			s = Tlen(val) + 1;
-			c = WS_Alloc(ew->sp->wrk->ws, s);
-			XXXAN(c);
-			memcpy(c, val.b, Tlen(val));
-			val.b = c;
-			val.e = val.b + s;
-			val.e[-1] = '\0';
-		}
-
-		if (strchr(val.b, '&'))
-			XMLentity(&val);
-
-		if (Tlen(val) > 7 && !memcmp(val.b, "http://", 7)) {
-			/*  Rewrite to Host: header inplace */
-			eb->host.b = val.b;
-			memcpy(eb->host.b, "Host: ", 6);
-			q = eb->host.b + 6;
-			for (p = eb->host.b + 7; p < val.e && *p != '/'; p++)
-				*q++ = *p;
-			*q++ = '\0';
-			eb->host.e = q;
-			assert(*p == '/');	/* XXX */
-			/* The rest is the URL */
-			eb->include.b = p;
-			eb->include.e = val.e;
-		} else if (Tlen(val) > 0 && *val.b == '/') {
-			/* Absolute on this host */
-			eb->include = val;
-		} else {
-
-			/*
-			 * Decision:  We interpret the relative URL against
-			 * the actual URL we asked the backend for.
-			 * The client's request URL may be entirely
-			 * different and have been rewritten underway.
-			 */
-			CHECK_OBJ_NOTNULL(ew->sp, SESS_MAGIC);
-			CHECK_OBJ_NOTNULL(ew->sp->wrk->bereq, HTTP_MAGIC);
-			tag = ew->sp->wrk->bereq->hd[HTTP_HDR_URL];
-
-			/* Use the objects WS to store the result */
-			CHECK_OBJ_NOTNULL(ew->sp->obj, OBJECT_MAGIC);
-
-			/* Look for the last '/' before a '?' */
-			q = NULL;
-			for (p = tag.b; p < tag.e && *p != '?'; p++)
-				if (*p == '/')
-					q = p;
-			if (q != NULL)
-				tag.e = q + 1;
-
-			u = WS_Reserve(ew->sp->wrk->ws, 0);
-			v = snprintf(ew->sp->wrk->ws->f, u - 1, "%.*s%.*s",
-			    pdiff(tag.b, tag.e), tag.b,
-			    pdiff(val.b, val.e), val.b);
-			v++;
-			xxxassert(v < u);
-			eb->include.b = ew->sp->wrk->ws->f;
-			eb->include.e = ew->sp->wrk->ws->f + v;
-			WS_Release(ew->sp->wrk->ws, v);
-		}
-		if (eb->include.b != NULL)
-			ew->space += Tlen(eb->include);
-		if (eb->host.b != NULL)
-			ew->space += Tlen(eb->host);
-	}
-}
-
-/*--------------------------------------------------------------------
- * See if this looks like XML: first non-white char must be '<'
- */
-
-static int
-looks_like_xml(const struct object *obj) {
-	struct storage *st;
-	unsigned u;
-
-	VTAILQ_FOREACH(st, &obj->store, list) {
-		AN(st);
-		for (u = 0; u < st->len; u++) {
-			if (isspace(st->ptr[u]))
-				continue;
-			if (st->ptr[u] == '<')
-				return (1);
-			else
-				return (0);
-		}
-	}
-	return (0);
-}
-
-/*--------------------------------------------------------------------
- * A quick stroll through the object, to find out if it contains any
- * esi sequences at all.
- */
-
-static int
-contain_esi(const struct object *obj) {
-	struct storage *st;
-	unsigned u;
-	const char *r, *r2;
-	static const char * const wanted = "<esi:";
-	static const char * const wanted2 = "<!--esi";
-
-	/*
-	 * Do a fast check to see if there is any '<esi:' sequences at all
-	 */
-	r = wanted;
-	r2 = wanted2;
-	VTAILQ_FOREACH(st, &obj->store, list) {
-		AN(st);
-		for (u = 0; u < st->len; u++) {
-			if (st->ptr[u] != *r) {
-				r = wanted;
-			} else if (*++r == '\0')
-				return (1);
-			if (st->ptr[u] != *r2) {
-				r2 = wanted2;
-			} else if (*++r2 == '\0')
-				return (1);
-		}
-	}
-	return (0);
-}
-
-/*--------------------------------------------------------------------*/
-
-static void
-parse_esi_comment(struct esi_work *ew)
-{
-
-	esi_addpfx(ew);
-
-	N(ew); N(ew); N(ew); N(ew); N(ew); N(ew); N(ew);
-	assert(!ew->incmt);
-	ew->incmt = 1;
-	ew->s.p = ew->p.p;
-}
-
-/*--------------------------------------------------------------------*/
-
-static void
-parse_comment(struct esi_work *ew)
-{
-
-	do {
-		N(ew);
-		if (*ew->p.p == '-' && !CMP(&ew->p, "-->")) {
-			N(ew);
-			N(ew);
-			N(ew);
-			break;
-		}
-	} while (ew->p.p < ew->p.e);
-}
-
-/*--------------------------------------------------------------------*/
-
-static void
-parse_cdata(struct esi_work *ew)
-{
-
-	esi_addpfx(ew);
-
-	do {
-		N(ew);
-		if (*ew->p.p == ']' && !CMP(&ew->p, "]]>")) {
-			N(ew);
-			N(ew);
-			N(ew);
-			break;
-		}
-	} while (ew->p.p < ew->p.e);
-}
-
-/*--------------------------------------------------------------------*/
-
-static void
-parse_esi_tag(struct esi_work *ew, int closing)
-{
-	int l, ll, empty;
-	struct esi_ptr px;
-	char *q;
-
-	esi_addpfx(ew);
-
-	do
-		N(ew);
-	while (*ew->p.p != '>' && ew->p.p < ew->p.e);
-	if (ew->p.p == ew->p.e) {
-		esi_addpfx(ew);
-		esi_error(ew, ew->s.p, 0,
-		    "XML 1.0 incomplete language element");
-		return;
-	}
-	N(ew);
-
-	if (ew->p.st == ew->s.st) {
-		ew->tag.b = TRUST_ME(ew->s.p);
-		ew->tag.e = TRUST_ME(ew->p.p);
-	} else {
-		/*
-		 * The element is spread over more than one storage
-		 * segment, pull it together in the object workspace
-		 * XXX: Ideally, we should only pull together the bits
-		 * XXX: we need, like the filename.
-		 */
-		ew->tag.b = ew->sp->wrk->ws->f;
-		ew->tag.e = ew->tag.b + WS_Reserve(ew->sp->wrk->ws, 0);
-		px = ew->s;
-		q = ew->tag.b;
-		while (px.p != ew->p.p) {
-			xxxassert(q < ew->tag.e);
-			*q++ = *px.p;
-			Nep(&px);
-		}
-		ew->tag.e = q;
-		WS_Release(ew->sp->wrk->ws, Tlen(ew->tag));
-	}
-	ll = Tlen(ew->tag);
-	ew->tag.b++;
-	ew->tag.e--;
-	empty = (ew->tag.e[-1] == '/') ? 1 : 0;
-	if (empty)
-		ew->tag.e--;
-
-	if (empty && closing)
-		esi_error(ew, ew->s.p, ll,
-		    "XML 1.0 empty and closing element");
-
-	ew->tag.b += 4 + (closing ? 1 : 0);
-	l = Tlen(ew->tag);
-	WSP(ew->sp, SLT_Debug,
-	    "tag {%.*s} %d %d %d", l, ew->tag.b, ew->remflg, empty, closing);
-	if (l >= 6 && !memcmp(ew->tag.b, "remove", 6)) {
-		if (empty) {
-			/* XXX ?? */
-		} else if (closing) {
-			if (!ew->remflg)
-				esi_error(ew, ew->s.p, ll,
-				    "ESI 1.0 esi:remove not opened");
-			ew->remflg = 0;
-		} else {
-			if (ew->remflg)
-				esi_error(ew, ew->s.p, ll,
-				    "ESI 1.0 forbids nested esi:remove");
-			ew->remflg = 1;
-		}
-	} else if (ew->remflg) {
-		esi_error(ew, ew->s.p, ll,
-		    "ESI 1.0 forbids esi: elements inside esi:remove");
-	} else if (l >= 7 && !memcmp(ew->tag.b, "comment", 7)) {
-		if (closing)
-			esi_error(ew, ew->s.p, ll,
-			    "ESI 1.0 closing esi:comment illegal");
-		else if (!empty)
-			esi_error(ew, ew->s.p, ll,
-			    "ESI 1.0 wants empty esi:comment");
-	} else if (l >= 7 && !memcmp(ew->tag.b, "include", 7)) {
-		if (closing) {
-			esi_error(ew, ew->s.p, ll,
-			    "ESI 1.0 closing esi:include illegal");
-		} else if (!empty) {
-			esi_error(ew, ew->s.p, ll,
-			    "ESI 1.0 wants empty esi:include");
-		}
-		ew->tag.b += 7;
-		esi_handle_include(ew);
-	} else {
-		esi_error(ew, ew->s.p, ll,
-		    "ESI 1.0 unimplemented element");
-	}
-	ew->s = ew->p;
-}
-
-/*--------------------------------------------------------------------*/
-
-void
-ESI_Parse(struct sess *sp)
-{
-	struct esi_work *ew, eww[1];
-	struct esi_bit *eb;
-	struct esidata *ed;
-	struct storage *st;
-	unsigned u;
-	char *hack;
-
-	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
-	CHECK_OBJ_NOTNULL(sp->obj, OBJECT_MAGIC);
-	AssertObjPassOrBusy(sp->obj);
-	if (VTAILQ_EMPTY(&sp->obj->store))
-		return;
-
-	if (!(params->esi_syntax & 0x00000001)) {
-		/*
-		 * By default, we will not ESI process an object where
-		 *  the first non-space character is different from '<'
-		 */
-		if (!looks_like_xml(sp->obj)) {
-			WSP(sp, SLT_ESI_xmlerror,
-			    "No ESI processing, first char not '<'");
-			return;
-		}
-	}
-
-	/*
-	 * Do a fast check to see if there is any '<esi:' sequences at all
-	 */
-	if (!contain_esi(sp->obj))
-		return;
-
-	/* XXX: debugging hack */
-	hack = sp->wrk->ws->f;
-
-	VSC_main->esi_parse++;
-	/* XXX: only if GET ? */
-	ew = eww;
-	memset(eww, 0, sizeof eww);
-	VTAILQ_INIT(&ew->esibits);
-	ew->sp = sp;
-	ew->off = 1;
-
-	ew->space += sizeof(struct esidata);
-
-	ew->p.st = VTAILQ_FIRST(&sp->obj->store);
-	AN(ew->p.st);
-	ew->p.p = (char *)ew->p.st->ptr;
-	ew->p.e = ew->p.p + ew->p.st->len;
-
-	/* ->s points to the first un-dealt-with byte */
-	ew->s = ew->p;
-
-	while (ew->p.p < ew->p.e) {
-
-		if (ew->incmt && *ew->p.p == '-' && !CMP(&ew->p, "-->")) {
-			/* End of ESI comment */
-			esi_addpfx(ew);
-			N(ew);
-			N(ew);
-			N(ew);
-			ew->s = ew->p;
-			ew->incmt = 0;
-			continue;
-		}
-		/* Skip forward to the first '<' */
-		if (*ew->p.p != '<') {
-			N(ew);
-			continue;
-		}
-
-		if (!CMP(&ew->p, "<!--esi")) {
-			parse_esi_comment(ew);
-		} else if (!CMP(&ew->p, "<!--")) {
-			parse_comment(ew);
-		} else if (!CMP(&ew->p, "</esi")) {
-			parse_esi_tag(ew, 1);
-		} else if (!CMP(&ew->p, "<esi:")) {
-			parse_esi_tag(ew, 0);
-		} else if (!CMP(&ew->p, "<![CDATA[")) {
-			parse_cdata(ew);
-		} else {
-			/*
-			 * Something we don't care about, just skip it.
-			 */
-			N(ew);
-			if (!(params->esi_syntax & 0x2)) {
-				/* XXX: drop this ? */
-				do {
-					N(ew);
-				} while (*ew->p.p != '>' && ew->p.p < ew->p.e);
-			}
-		}
-	}
-	esi_addpfx(ew);
-
-	/*
-	 * XXX: we could record the starting point of these elements
-	 * XXX: so that the char-index were more useful, but we are
-	 * XXX: not trivially able to print their contents, so leave
-	 * XXX: it like this for now, pending more thought about the
-	 * XXX: proper way to report these errors.
-	 */
-	if (ew->remflg)
-		esi_error(ew, ew->t.e, -1,
-		    "ESI 1.0 unterminated <esi:remove> element");
-	if (ew->incmt)
-		esi_error(ew, ew->t.e, -1,
-		    "ESI 1.0 unterminated <!--esi comment");
-
-	st = STV_alloc(sp, ew->space);
-	AN(st);
-	assert(st->space >= ew->space);
-
-	ed = (void*)st->ptr;
-	st->len += sizeof(*ed);
-	memset(ed, 0, sizeof *ed);
-	ed->magic = ESIDATA_MAGIC;
-	ed->storage = st;
-	VTAILQ_INIT(&ed->esibits);
-
-	while (!VTAILQ_EMPTY(&ew->esibits)) {
-		eb = VTAILQ_FIRST(&ew->esibits);
-		VTAILQ_REMOVE(&ew->esibits, eb, list);
-		memcpy(st->ptr + st->len, eb, sizeof *eb);
-		eb = (void*)(st->ptr + st->len);
-		st->len += sizeof(*eb);
-
-		if (eb->include.b != NULL) {
-			u = Tlen(eb->include);
-			memcpy(st->ptr + st->len, eb->include.b, u);
-			eb->include.b = (void*)(st->ptr + st->len);
-			eb->include.e = eb->include.b + u;
-			st->len += u;
-		}
-		if (eb->host.b != NULL) {
-			u = Tlen(eb->host);
-			memcpy(st->ptr + st->len, eb->host.b, u);
-			eb->host.b = (void*)(st->ptr + st->len);
-			eb->host.e = eb->host.b + u;
-			st->len += u;
-		}
-
-		VTAILQ_INSERT_TAIL(&ed->esibits, eb, list);
-	}
-
-	assert(st->len <= st->space);
-	assert(st->len == ew->space);
-	sp->obj->esidata = ed;
-
-	memset(hack, 0xaa, sp->wrk->ws->f - hack);
-}
-
-/*--------------------------------------------------------------------*/
-
-void
-ESI_Deliver(struct sess *sp)
-{
-	struct esi_bit *eb;
-	struct worker *w;
-	struct esidata *ed;
-
-	w = sp->wrk;
-	ed = sp->obj->esidata;
-	CHECK_OBJ_NOTNULL(ed, ESIDATA_MAGIC);
-	VTAILQ_FOREACH(eb, &ed->esibits, list) {
-		if (Tlen(eb->verbatim)) {
-			if (sp->wrk->res_mode & RES_CHUNKED)
-				(void)WRW_Write(w, eb->chunk_length, -1);
-			sp->acct_tmp.bodybytes += WRW_Write(w,
-			    eb->verbatim.b, Tlen(eb->verbatim));
-			if (sp->http->protover >= 1.1)
-				(void)WRW_Write(w, "\r\n", -1);
-		}
-		if (eb->include.b == NULL ||
-		    sp->esi_level >= params->max_esi_includes)
-			continue;
-
-		ESI_Include(sp, eb->include.b, eb->host.b);
-		if (sp->fd < 0)
-			break;
-	}
-}
-
-/*--------------------------------------------------------------------*/
-
-void
-ESI_Destroy(struct object *o)
-{
-	struct esidata *ed;
-
-	ed = o->esidata;
-	o->esidata = NULL;
-	CHECK_OBJ_NOTNULL(ed, ESIDATA_MAGIC);
-	STV_free(ed->storage);
-}
-
-#endif /* OLD_ESI */
diff --git a/bin/varnishd/cache_esi_deliver.c b/bin/varnishd/cache_esi_deliver.c
index 7a36e59..4a75ce1 100644
--- a/bin/varnishd/cache_esi_deliver.c
+++ b/bin/varnishd/cache_esi_deliver.c
@@ -44,7 +44,7 @@ SVNID("$Id")
 
 /*--------------------------------------------------------------------*/
 
-void
+static void
 ESI_Include(struct sess *sp, const char *src, const char *host)
 {
 	struct object *obj;
@@ -120,7 +120,6 @@ ESI_Include(struct sess *sp, const char *src, const char *host)
 
 /*--------------------------------------------------------------------*/
 
-#ifndef OLD_ESI
 
 //#define Debug(fmt, ...) printf(fmt, __VA_ARGS__)
 #define Debug(fmt, ...) /**/
@@ -233,4 +232,3 @@ ESI_Deliver(struct sess *sp)
 	(void)WRW_Flush(sp->wrk);
 }
 
-#endif /* OLD_ESI */
diff --git a/bin/varnishd/cache_esi_parse.c b/bin/varnishd/cache_esi_parse.c
index e4a2c8b..f7f1a82 100644
--- a/bin/varnishd/cache_esi_parse.c
+++ b/bin/varnishd/cache_esi_parse.c
@@ -42,8 +42,6 @@ SVNID("$Id")
 #include "zlib.h"
 #include "stevedore.h"
 
-#ifndef OLD_ESI
-
 //#define Debug(fmt, ...) printf(fmt, __VA_ARGS__)
 #define Debug(fmt, ...) /**/
 
@@ -1098,8 +1096,6 @@ struct vfp vfp_esi = {
         .end    =       vfp_esi_end,
 };
 
-#endif /* OLD_ESI */
-
 #if 0
 
 digraph xml {
diff --git a/bin/varnishd/cache_hash.c b/bin/varnishd/cache_hash.c
index ed70404..39d416f 100644
--- a/bin/varnishd/cache_hash.c
+++ b/bin/varnishd/cache_hash.c
@@ -693,15 +693,10 @@ HSH_Deref(struct worker *w, struct objcore *oc, struct object **oo)
 		DSL(0x40, SLT_Debug, 0, "Object %u workspace min free %u",
 		    o->xid, WS_Free(o->ws_o));
 
-#ifdef OLD_ESI
-		if (o->esidata != NULL)
-			ESI_Destroy(o);
-#else
 		if (o->esidata != NULL) {
 			STV_free(o->esidata);
 			o->esidata = NULL;
 		}
-#endif
 		if (oc != NULL)
 			oc_freeobj(oc);
 		w->stats.n_object--;



From phk at project.varnish-software.com  Fri Jan 21 11:17:11 2011
From: phk at project.varnish-software.com (Poul-Henning Kamp)
Date: Fri, 21 Jan 2011 12:17:11 +0100
Subject: [master] a48f5ed Replay of SVN 5763:
Message-ID: <E1PgEzP-0003Li-Sz@project.varnish-software.com>

commit a48f5ed3d39b01d5203f146c7c422cff3ee708bc
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Fri Jan 21 10:48:35 2011 +0000

    Replay of SVN 5763:
    
    Code polishing

diff --git a/bin/varnishd/cache_esi_parse.c b/bin/varnishd/cache_esi_parse.c
index dafba72..e4a2c8b 100644
--- a/bin/varnishd/cache_esi_parse.c
+++ b/bin/varnishd/cache_esi_parse.c
@@ -57,6 +57,8 @@ struct vep_match {
 	const char	* const *state;
 };
 
+enum vep_mark { VERBATIM = 0, SKIP };
+
 struct vep_state {
 	unsigned		magic;
 #define VEP_MAGIC		0x55cb9b82
@@ -74,8 +76,7 @@ struct vep_state {
 
 	unsigned		remove;
 
-	ssize_t			o_verbatim;
-	ssize_t			o_skip;
+	ssize_t			o_wait;
 	ssize_t			o_pending;
 	ssize_t			o_total;
 	uint32_t		crc;
@@ -111,7 +112,7 @@ const char		*hack_p;
 	unsigned		nm_skip;
 	unsigned		nm_verbatim;
 	unsigned		nm_pending;
-	int			last_mark;
+	enum vep_mark		last_mark;
 };
 
 /*---------------------------------------------------------------------*/
@@ -263,7 +264,7 @@ vep_emit_len(const struct vep_state *vep, ssize_t l, int m8, int m16, int m64)
 } 
 
 static void
-vep_emit_skip(struct vep_state *vep, ssize_t l)
+vep_emit_skip(const struct vep_state *vep, ssize_t l)
 {
 
 	if (params->esi_syntax & 0x20) {
@@ -273,7 +274,7 @@ vep_emit_skip(struct vep_state *vep, ssize_t l)
 } 
 
 static void
-vep_emit_verbatim(struct vep_state *vep, ssize_t l)
+vep_emit_verbatim(const struct vep_state *vep, ssize_t l)
 {
 	uint8_t buf[4];
 
@@ -287,12 +288,13 @@ vep_emit_verbatim(struct vep_state *vep, ssize_t l)
 } 
 
 static void
-vep_emit_common(struct vep_state *vep, ssize_t *l, int skip)
+vep_emit_common(struct vep_state *vep, ssize_t *l, enum vep_mark mark)
 {
 	assert(*l > 0);
 	assert(*l == vep->o_crc);
 
-	if (skip)
+	assert(mark == SKIP || mark == VERBATIM);
+	if (mark == SKIP)
 		vep_emit_skip(vep, *l);
 	else
 		vep_emit_verbatim(vep, *l);
@@ -308,28 +310,25 @@ vep_emit_common(struct vep_state *vep, ssize_t *l, int skip)
  */
 
 static void
-vep_mark_common(struct vep_state *vep, const char *p, int skip)
+vep_mark_common(struct vep_state *vep, const char *p, enum vep_mark mark)
 {
 	ssize_t l;
 
+	assert(mark == SKIP || mark == VERBATIM);
+
+	/* The NO-OP case, no data, no pending data & no change of mode */
+	if (vep->last_mark == mark && p == vep->ver_p && vep->o_pending == 0)
+		return;
+
 	/*
 	 * If we changed mode, emit whatever the opposite mode
 	 * assembled before the pending bytes.
 	 */
-	if (skip && vep->o_verbatim > 0)
-		vep_emit_common(vep, &vep->o_verbatim, 0);
-	else if (!skip && vep->o_skip > 0)
-		vep_emit_common(vep, &vep->o_skip, 1);
-
-	/* In debug mode, the pending bytes are emitted separately.  */
-	if ((params->esi_syntax & 0x10) && vep->o_pending) {
-		vep->o_crc = vep->o_pending;
-		vep->crc = vep->crcp;
-		vep->crcp = crc32(0L, Z_NULL, 0);
-		vep_emit_common(vep, &vep->o_pending, skip);
-	}
 
-	/* Transfer pending bytes CRC into active mode.  */
+	if (vep->last_mark != mark && vep->o_wait > 0)
+		vep_emit_common(vep, &vep->o_wait, vep->last_mark);
+
+	/* Transfer pending bytes CRC into active mode CRC */
 	if (vep->o_pending) {
 		if (vep->o_crc == 0) {
 			vep->crc = vep->crcp;
@@ -342,9 +341,7 @@ vep_mark_common(struct vep_state *vep, const char *p, int skip)
 		vep->crcp = crc32(0L, Z_NULL, 0);
 	}
 
-	/*
-	 * Process this bit of input
-	 */
+	/* * Process this bit of input */
 	AN(vep->ver_p);
 	l = p - vep->ver_p;
 	assert(l >= 0);
@@ -352,40 +349,26 @@ vep_mark_common(struct vep_state *vep, const char *p, int skip)
 	vep->o_crc += l;
 	vep->ver_p = p;
 
-	if ((params->esi_syntax & 0x10) && l > 0) {
-		/* Emit right away if debug mode */
-		vep_emit_common(vep, &l, skip);
-	} else if (skip) {
-		vep->o_skip += vep->o_pending;
-		vep->o_skip += l;
-	} else {
-		vep->o_verbatim += vep->o_pending;
-		vep->o_verbatim += l;
-	}
+	vep->o_wait += vep->o_pending;
+	vep->o_wait += l;
 	vep->o_pending = 0;
+	vep->last_mark = mark;
 }
 
-
 static void
 vep_mark_verbatim(struct vep_state *vep, const char *p)
 {
-	if (vep->last_mark == 0 && p == vep->ver_p)
-		return;
-//printf("MARK VERB %d %s <%.*s>\n", vep->remove, vep->state, (int)(p - vep->ver_p), vep->ver_p);
-	vep_mark_common(vep, p, 0);
+
+	vep_mark_common(vep, p, VERBATIM);
 	vep->nm_verbatim++;
-	vep->last_mark = 0;
 } 
 
 static void
 vep_mark_skip(struct vep_state *vep, const char *p)
 {
-	if (vep->last_mark == 1 && p == vep->ver_p)
-		return;
-//printf("MARK SKIP %d %s <%.*s>\n", vep->remove, vep->state, (int)(p - vep->ver_p), vep->ver_p);
-	vep_mark_common(vep, p, 1);
+
+	vep_mark_common(vep, p, SKIP);
 	vep->nm_skip++;
-	vep->last_mark = 1;
 } 
 
 static void
@@ -393,18 +376,15 @@ vep_mark_pending(struct vep_state *vep, const char *p)
 {
 	ssize_t l;
 
-	assert (vep->last_mark != 2 || p != vep->ver_p);
 	AN(vep->ver_p);
 	l = p - vep->ver_p;
 	assert(l > 0);
-// printf("MARK PEND %d %s <%.*s>\n", vep->remove, vep->state, (int)l, vep->ver_p);
 	assert(l >= 0);
 	vep->crcp = crc32(vep->crcp, (const void *)vep->ver_p, l);
 	vep->ver_p = p;
 
 	vep->o_pending += l;
 	vep->nm_pending++;
-	vep->last_mark = 2;
 }
 
 /*---------------------------------------------------------------------
@@ -470,7 +450,7 @@ vep_do_include(struct vep_state *vep, enum dowhat what)
 		    "ESI 1.0 <esi:include> lacks src attr");
 		return;
 	}
-	XXXAN(vep->include_src);
+
 	/*
 	 * Strictly speaking, we ought to spit out any piled up skip before
 	 * emitting the VEC for the include, but objectively that makes no
@@ -480,7 +460,7 @@ vep_do_include(struct vep_state *vep, enum dowhat what)
 	 * The mark_skip() before calling dostuff should have taken
 	 * care of that.  Make sure.
 	 */
-	assert(vep->o_verbatim == 0);
+	assert(vep->o_wait == 0 || vep->last_mark == SKIP);
 	/* XXX: what if it contains NUL bytes ?? */
 	p = vsb_data(vep->include_src);
 	l = vsb_len(vep->include_src);
@@ -1077,20 +1057,12 @@ vfp_esi_end(struct sess *sp)
 	CHECK_OBJ_NOTNULL(vep, VEP_MAGIC);
 
 	l = sp->obj->len - vep->o_total;
-	Debug("ENDING STATE: %s (skip %jd) (verbatim %jd)"
-	    " (tot %jd) (len %jd) (diff %jd)\n",
-	    vep->state,
-	    (intmax_t)vep->o_skip, (intmax_t)vep->o_verbatim,
-	    (intmax_t)vep->o_total, (intmax_t)sp->obj->len, (intmax_t)l);
 	assert(l >= 0);
 	if (vep->o_pending)
-		vep_mark_verbatim(vep, vep->ver_p);
-	if (vep->o_skip > 0)
-		vep_emit_common(vep, &vep->o_skip, 1);
-	else if (vep->o_verbatim > 0)
-		vep_emit_common(vep, &vep->o_verbatim, 0);
+		vep_mark_common(vep, vep->ver_p, vep->last_mark);
+	if (vep->o_wait > 0)
+		vep_emit_common(vep, &vep->o_wait, vep->last_mark);
 
-printf("NMARK VER %u SKIP %u PEND %u\n", vep->nm_verbatim, vep->nm_skip, vep->nm_pending);
 	vsb_finish(vep->vsb);
 	l = vsb_len(vep->vsb);
 	if (vep->state != VEP_NOTXML && l > 0) {
diff --git a/bin/varnishd/mgt_param.c b/bin/varnishd/mgt_param.c
index 7718797..26b4d1a 100644
--- a/bin/varnishd/mgt_param.c
+++ b/bin/varnishd/mgt_param.c
@@ -644,7 +644,6 @@ static const struct parspec input_parspec[] = {
 		"  0x00000002 - Ignore non-esi elements\n"
 		"  0x00000004 - Emit parsing debug records\n"
 		"  0x00000008 - Force-split parser input (debugging)\n"
-		"  0x00000010 - Don't coalesce VEC string (debugging)\n"
 		"Use 0x notation and do the bitor in your head :-)\n",
 		0,
 		"0", "bitmap" },



From phk at project.varnish-software.com  Fri Jan 21 11:48:56 2011
From: phk at project.varnish-software.com (Poul-Henning Kamp)
Date: Fri, 21 Jan 2011 12:48:56 +0100
Subject: [master] 1165edd Add a parameter to set the size of the stack buffers
	we use for gzip processing.
Message-ID: <E1PgFU8-0005LD-NQ@project.varnish-software.com>

commit 1165eddd80563b58713384555cadeb3893cd9f16
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Fri Jan 21 11:48:38 2011 +0000

    Add a parameter to set the size of the stack buffers we use for gzip
    processing.

diff --git a/bin/varnishd/heritage.h b/bin/varnishd/heritage.h
index d272d29..8eab742 100644
--- a/bin/varnishd/heritage.h
+++ b/bin/varnishd/heritage.h
@@ -200,6 +200,7 @@ struct params {
 	unsigned		http_range_support;
 
 	unsigned		http_gzip_support;
+	unsigned		gzip_stack_buffer;
 
 	double			critbit_cooloff;
 };
diff --git a/bin/varnishd/mgt_param.c b/bin/varnishd/mgt_param.c
index 26b4d1a..fe6805c 100644
--- a/bin/varnishd/mgt_param.c
+++ b/bin/varnishd/mgt_param.c
@@ -817,6 +817,11 @@ static const struct parspec input_parspec[] = {
 		"Enable support for HTTP GZIP compression.\n",
 		EXPERIMENTAL,
 		"off", "bool" },
+	{ "gzip_stack_buffer", tweak_uint, &master.gzip_stack_buffer,
+	        4, UINT_MAX,
+		"Size of stack buffer used for gzip processing.\n",
+		EXPERIMENTAL,
+		"64", "Kilobytes" },
 	{ "critbit_cooloff", tweak_timeout_double,
 		&master.critbit_cooloff, 60, 254,
 		"How long time the critbit hasher keeps deleted objheads "



From phk at project.varnish-software.com  Fri Jan 21 12:01:13 2011
From: phk at project.varnish-software.com (Poul-Henning Kamp)
Date: Fri, 21 Jan 2011 13:01:13 +0100
Subject: [master] af87aa4 Implement the easy case of gzip/ESI interaction:
	Receiving a gzip'ed object but wanting to store it ungzip'ed.
Message-ID: <E1PgFg1-00060V-Cd@project.varnish-software.com>

commit af87aa4c0bb6fc67038cd351c4f3e9c800defe77
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Fri Jan 21 12:00:41 2011 +0000

    Implement the easy case of gzip/ESI interaction: Receiving a gzip'ed
    object but wanting to store it ungzip'ed.

diff --git a/bin/varnishd/cache_esi_fetch.c b/bin/varnishd/cache_esi_fetch.c
index d36e694..b9cc5fc 100644
--- a/bin/varnishd/cache_esi_fetch.c
+++ b/bin/varnishd/cache_esi_fetch.c
@@ -111,8 +111,52 @@ vfp_esi_bytes_ug(struct sess *sp, struct http_conn *htc, size_t bytes)
 static int __match_proto__()
 vfp_esi_bytes_gu(struct sess *sp, struct http_conn *htc, size_t bytes)
 {
+	struct vgz *vg;
+	ssize_t l, w;
+	struct storage *st;
+	uint8_t	ibuf[1024 * params->gzip_stack_buffer];
+	int i;
+	size_t dl;
+	const void *dp;
 
-	return (vfp_esi_bytes_uu(sp, htc, bytes));
+	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+	vg = sp->wrk->vgz_rx;
+
+	while (bytes > 0) {
+		if (sp->wrk->storage == NULL) {
+			l = params->fetch_chunksize * 1024LL;
+			sp->wrk->storage = STV_alloc(sp, l);
+		}
+		if (sp->wrk->storage == NULL) {
+			errno = ENOMEM;
+			return (-1);
+		}
+		st = sp->wrk->storage;
+
+		VGZ_Obuf(vg, st->ptr + st->len, st->space - st->len);
+
+		if (VGZ_IbufEmpty(vg) && bytes > 0) {
+			l = sizeof ibuf;
+			if (l > bytes)
+				l = bytes;
+			w = HTC_Read(htc, ibuf, l);
+			if (w <= 0)
+				return (w);
+			VGZ_Ibuf(vg, ibuf, w);
+			bytes -= w;
+		}
+		i = VGZ_Gunzip(vg, &dp, &dl);
+		VEP_parse(sp, dp, dl);
+		st->len += dl;
+		sp->obj->len += dl;
+		if (st->len == st->space) {
+			VTAILQ_INSERT_TAIL(&sp->obj->store,
+			    sp->wrk->storage, list);
+			sp->wrk->storage = NULL;
+			st = NULL;
+		}
+	}
+	return (1);
 }
 
 /*---------------------------------------------------------------------
@@ -137,6 +181,9 @@ vfp_esi_begin(struct sess *sp, size_t estimate)
 
 	VEP_Init(sp);
 
+	if (sp->wrk->is_gzip && sp->wrk->do_gunzip)
+		sp->wrk->vgz_rx = VGZ_NewUngzip(sp, sp->ws);
+
 	(void)estimate;
 }
 
@@ -165,6 +212,8 @@ vfp_esi_end(struct sess *sp)
 	ssize_t l;
 
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+	if (sp->wrk->is_gzip && sp->wrk->do_gunzip)
+		VGZ_Destroy(&sp->wrk->vgz_rx);
 
 	vsb = VEP_Finish(sp);
 
diff --git a/bin/varnishtest/tests/e00020.vtc b/bin/varnishtest/tests/e00020.vtc
new file mode 100644
index 0000000..3fe9239
--- /dev/null
+++ b/bin/varnishtest/tests/e00020.vtc
@@ -0,0 +1,36 @@
+# $Id$
+
+test "ESI:remove"
+
+
+server s1 {
+	rxreq 
+	txresp -gzipbody {
+		<esi:remove>
+		This is a test: Unseen University
+		<esi:include src="trick question">
+		<!--esi XXX -->
+		</esi:remove>
+		<esX>This is a test: Hello world
+	}
+} -start
+
+varnish v1 -vcl+backend {
+	sub vcl_fetch {
+		set beresp.do_esi = true;
+		set beresp.do_gunzip = true;
+	}
+} -start 
+
+varnish v1 -cliok "param.set esi_syntax 4"
+varnish v1 -cliok "param.set http_gzip_support true"
+
+client c1 {
+	txreq 
+	rxresp
+	expect resp.status == 200
+	expect resp.bodylen == 40
+}
+
+client c1 -run
+varnish v1 -expect esi_errors == 2



From phk at project.varnish-software.com  Fri Jan 21 12:01:13 2011
From: phk at project.varnish-software.com (Poul-Henning Kamp)
Date: Fri, 21 Jan 2011 13:01:13 +0100
Subject: [master] 06e1b51 Add a function to test if the input gzip buffer is
	empty.
Message-ID: <E1PgFg1-00060d-Mu@project.varnish-software.com>

commit 06e1b51d7495852c56a596fd7611e5b8f7c01075
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Fri Jan 21 11:56:56 2011 +0000

    Add a function to test if the input gzip buffer is empty.

diff --git a/bin/varnishd/cache.h b/bin/varnishd/cache.h
index 8d1696e..3e2fe64 100644
--- a/bin/varnishd/cache.h
+++ b/bin/varnishd/cache.h
@@ -627,8 +627,9 @@ void Fetch_Init(void);
 struct vgz;
 
 struct vgz *VGZ_NewUngzip(const struct sess *sp, struct ws *tmp);
-void VGZ_Ibuf(struct vgz *, const void *, size_t len);
-void VGZ_Obuf(struct vgz *, const void *, size_t len);
+void VGZ_Ibuf(struct vgz *, const void *, ssize_t len);
+int VGZ_IbufEmpty(struct vgz *vg);
+void VGZ_Obuf(struct vgz *, const void *, ssize_t len);
 int VGZ_Gzip(struct vgz *, const void **, size_t *len, int flag);
 int VGZ_Gunzip(struct vgz *, const void **, size_t *len);
 void VGZ_Destroy(struct vgz **);
diff --git a/bin/varnishd/cache_gzip.c b/bin/varnishd/cache_gzip.c
index 9ac3d6c..be34261 100644
--- a/bin/varnishd/cache_gzip.c
+++ b/bin/varnishd/cache_gzip.c
@@ -190,7 +190,7 @@ VGZ_NewGzip(const struct sess *sp, struct ws *tmp)
 /*--------------------------------------------------------------------*/
 
 void
-VGZ_Ibuf(struct vgz *vg, const void *ptr, size_t len)
+VGZ_Ibuf(struct vgz *vg, const void *ptr, ssize_t len)
 {
 
 	CHECK_OBJ_NOTNULL(vg, VGZ_MAGIC);
@@ -200,10 +200,18 @@ VGZ_Ibuf(struct vgz *vg, const void *ptr, size_t len)
 	vg->vz.avail_in = len;
 }
 
+int
+VGZ_IbufEmpty(struct vgz *vg)
+{
+
+	CHECK_OBJ_NOTNULL(vg, VGZ_MAGIC);
+	return (vg->vz.avail_in == 0);
+}
+
 /*--------------------------------------------------------------------*/
 
 void
-VGZ_Obuf(struct vgz *vg, const void *ptr, size_t len)
+VGZ_Obuf(struct vgz *vg, const void *ptr, ssize_t len)
 {
 
 	CHECK_OBJ_NOTNULL(vg, VGZ_MAGIC);



From phk at project.varnish-software.com  Fri Jan 21 14:30:23 2011
From: phk at project.varnish-software.com (Poul-Henning Kamp)
Date: Fri, 21 Jan 2011 15:30:23 +0100
Subject: [master] 1054aed Add the ungzip'ed length of verbatim sequences to
	the VEC string so we can combine the CRC32 values.
Message-ID: <E1PgI0N-0006EZ-Qz@project.varnish-software.com>

commit 1054aede3663867189dd8c4160bb54dc9b475a52
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Fri Jan 21 14:29:44 2011 +0000

    Add the ungzip'ed length of verbatim sequences to the VEC string
    so we can combine the CRC32 values.

diff --git a/bin/varnishd/cache_esi.h b/bin/varnishd/cache_esi.h
index ad97b46..1e15d6b 100644
--- a/bin/varnishd/cache_esi.h
+++ b/bin/varnishd/cache_esi.h
@@ -30,6 +30,9 @@
 #define	VEC_V1	(0x40 + 1)
 #define	VEC_V2	(0x40 + 2)
 #define	VEC_V8	(0x40 + 8)
+#define	VEC_C1	(0x50 + 1)
+#define	VEC_C2	(0x50 + 2)
+#define	VEC_C8	(0x50 + 8)
 #define	VEC_S1	(0x60 + 1)
 #define	VEC_S2	(0x60 + 2)
 #define	VEC_S8	(0x60 + 8)
diff --git a/bin/varnishd/cache_esi_deliver.c b/bin/varnishd/cache_esi_deliver.c
index e4016c6..6a344ec 100644
--- a/bin/varnishd/cache_esi_deliver.c
+++ b/bin/varnishd/cache_esi_deliver.c
@@ -174,7 +174,7 @@ ESI_Deliver(struct sess *sp)
 	struct storage *st;
 	uint8_t *p, *e, *q, *r;
 	unsigned off;
-	ssize_t l;
+	ssize_t l, l_crc;
 	uint32_t crc, crc_ref;
 
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
@@ -192,6 +192,8 @@ ESI_Deliver(struct sess *sp)
 		case VEC_V2:
 		case VEC_V8:
 			l = ved_decode_len(&p);
+			assert(*p == VEC_C1 || *p == VEC_C2 || *p == VEC_C8);
+			l_crc = ved_decode_len(&p);
 			crc = vbe32dec(p);
 			p += 4;
 			q = (void*)strchr((const char*)p, '\0');
diff --git a/bin/varnishd/cache_esi_parse.c b/bin/varnishd/cache_esi_parse.c
index 1325a74..2a3edea 100644
--- a/bin/varnishd/cache_esi_parse.c
+++ b/bin/varnishd/cache_esi_parse.c
@@ -280,6 +280,7 @@ vep_emit_verbatim(const struct vep_state *vep, ssize_t l)
 		Debug("---> VERBATIM(%jd)\n", (intmax_t)l);
 	}
 	vep_emit_len(vep, l, VEC_V1, VEC_V2, VEC_V8);
+	vep_emit_len(vep, l, VEC_C1, VEC_C2, VEC_C8);
 	vbe32enc(buf, vep->crc);
 	vsb_bcat(vep->vsb, buf, sizeof buf);
 	vsb_printf(vep->vsb, "%lx\r\n%c", l, 0);



From phk at project.varnish-software.com  Fri Jan 21 14:30:24 2011
From: phk at project.varnish-software.com (Poul-Henning Kamp)
Date: Fri, 21 Jan 2011 15:30:24 +0100
Subject: [master] 031976a Complain loudly if we try to gunzip and the magic
	bytes are missing
Message-ID: <E1PgI0O-0006Eh-91@project.varnish-software.com>

commit 031976a0a7765e215c00af14c0057b40b1a749cc
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Fri Jan 21 12:50:23 2011 +0000

    Complain loudly if we try to gunzip and the magic bytes are missing

diff --git a/bin/varnishtest/vtc_http.c b/bin/varnishtest/vtc_http.c
index a534fdd..2b05c38 100644
--- a/bin/varnishtest/vtc_http.c
+++ b/bin/varnishtest/vtc_http.c
@@ -531,6 +531,8 @@ cmd_http_gunzip_body(CMD_ARGS)
 
 	memset(&vz, 0, sizeof vz);
 
+	if (hp->body[0] != (char)0x1f || hp->body[1] != (char)0x8b)
+		vtc_log(hp->vl, 0, "Gunzip error: Body lacks gzip magics");
 	vz.next_in = TRUST_ME(hp->body);
 	vz.avail_in = hp->bodyl;
 



From phk at project.varnish-software.com  Fri Jan 21 14:30:25 2011
From: phk at project.varnish-software.com (Poul-Henning Kamp)
Date: Fri, 21 Jan 2011 15:30:25 +0100
Subject: [master] 7c19b3c Expose VGZ_NewGzip()
Message-ID: <E1PgI0P-0006Et-7x@project.varnish-software.com>

commit 7c19b3c5822e55fb98bd2c56393b26d7f94d84c0
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Fri Jan 21 12:36:38 2011 +0000

    Expose VGZ_NewGzip()

diff --git a/bin/varnishd/cache.h b/bin/varnishd/cache.h
index 3e2fe64..510e7ab 100644
--- a/bin/varnishd/cache.h
+++ b/bin/varnishd/cache.h
@@ -627,6 +627,7 @@ void Fetch_Init(void);
 struct vgz;
 
 struct vgz *VGZ_NewUngzip(const struct sess *sp, struct ws *tmp);
+struct vgz *VGZ_NewGzip(const struct sess *sp, struct ws *tmp);
 void VGZ_Ibuf(struct vgz *, const void *, ssize_t len);
 int VGZ_IbufEmpty(struct vgz *vg);
 void VGZ_Obuf(struct vgz *, const void *, ssize_t len);
diff --git a/bin/varnishd/cache_gzip.c b/bin/varnishd/cache_gzip.c
index be64895..d532afc 100644
--- a/bin/varnishd/cache_gzip.c
+++ b/bin/varnishd/cache_gzip.c
@@ -151,7 +151,7 @@ VGZ_NewUngzip(const struct sess *sp, struct ws *tmp)
 	return (vg);
 }
 
-static struct vgz *
+struct vgz *
 VGZ_NewGzip(const struct sess *sp, struct ws *tmp)
 {
 	struct vgz *vg;



From phk at project.varnish-software.com  Fri Jan 21 14:30:25 2011
From: phk at project.varnish-software.com (Poul-Henning Kamp)
Date: Fri, 21 Jan 2011 15:30:25 +0100
Subject: [master] c640301 Use the gzip_stack_buffer param throughout.
Message-ID: <E1PgI0P-0006F1-Jb@project.varnish-software.com>

commit c640301185b328c37bf382e1fd650099fa391029
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Fri Jan 21 12:11:10 2011 +0000

    Use the gzip_stack_buffer param throughout.

diff --git a/bin/varnishd/cache_gzip.c b/bin/varnishd/cache_gzip.c
index be34261..be64895 100644
--- a/bin/varnishd/cache_gzip.c
+++ b/bin/varnishd/cache_gzip.c
@@ -307,7 +307,7 @@ vfp_gunzip_bytes(struct sess *sp, struct http_conn *htc, size_t bytes)
 	struct storage *st;
 	ssize_t l, w;
 	int i = -100;
-	uint8_t	ibuf[64*1024];		/* XXX size ? */
+	uint8_t	ibuf[1024 * params->gzip_stack_buffer];
 	size_t dl;
 	const void *dp;
 
@@ -406,7 +406,7 @@ vfp_gzip_bytes(struct sess *sp, struct http_conn *htc, size_t bytes)
 	struct storage *st;
 	ssize_t l, w;
 	int i = -100;
-	uint8_t ibuf[64*1024];		/* XXX size ? */
+	uint8_t ibuf[1024 * params->gzip_stack_buffer];
 	size_t dl;
 	const void *dp;
 
diff --git a/bin/varnishd/cache_response.c b/bin/varnishd/cache_response.c
index dbfb520..0b7e713 100644
--- a/bin/varnishd/cache_response.c
+++ b/bin/varnishd/cache_response.c
@@ -247,7 +247,7 @@ res_WriteGunzipObj(struct sess *sp)
 	struct vgz *vg;
 	const void *dp;
 	char lenbuf[20];
-	char obuf[64*1024];	/* XXX: size? */
+	char obuf[1024 * params->gzip_stack_buffer];
 	size_t dl;
 	int i;
 



From perbu at project.varnish-software.com  Fri Jan 21 14:38:08 2011
From: perbu at project.varnish-software.com (Per Andreas Buer)
Date: Fri, 21 Jan 2011 15:38:08 +0100
Subject: [master] 6350bfb Merge branch 'master' of
	ssh://git.varnish-cache.org/git/varnish-cache
Message-ID: <E1PgI7s-0006hJ-Iy@project.varnish-software.com>

commit 6350bfb4dc0c34cbfbca90e5b44552104f24bb3e
Merge: 4fcd5b8 1054aed
Author: Per Buer <perbu at varnish-software.com>
Date:   Fri Jan 21 15:38:05 2011 +0100

    Merge branch 'master' of ssh://git.varnish-cache.org/git/varnish-cache



From perbu at project.varnish-software.com  Fri Jan 21 14:38:08 2011
From: perbu at project.varnish-software.com (Per Andreas Buer)
Date: Fri, 21 Jan 2011 15:38:08 +0100
Subject: [master] 4fcd5b8 Clarifications on where varnish should run
Message-ID: <E1PgI7s-0006hR-RD@project.varnish-software.com>

commit 4fcd5b8f2290938a34c08bdd8ea00668ea938399
Author: Per Buer <perbu at varnish-software.com>
Date:   Fri Jan 21 15:37:48 2011 +0100

    Clarifications on where varnish should run

diff --git a/doc/sphinx/tutorial/putting_varnish_on_port_80.rst b/doc/sphinx/tutorial/putting_varnish_on_port_80.rst
index d3ceb26..5742806 100644
--- a/doc/sphinx/tutorial/putting_varnish_on_port_80.rst
+++ b/doc/sphinx/tutorial/putting_varnish_on_port_80.rst
@@ -2,19 +2,24 @@
 Put Varnish on port 80
 ----------------------
 
-If your application works OK we can now switch the ports so Varnish
-will listen to port 80. Kill varnish.::
+Until now we've been running with Varnish on a high port, for testing
+purposes. You should test your application and if it works OK we can
+switch, so Varnish will be running on port 80 and your web server on a
+high port.
+
+First we kill off varnishd.::
 
      # pkill varnishd
 
 and stop your web server. Edit the configuration for your web server
 and make it bind to port 8080 instead of 80. Now open the Varnish
-default.vcl and change the port of the default backend to 8080.
+default.vcl and change the port of the *default* backend to 8080.
 
 Start up your web server and then start varnish.::
 
       # varnishd -f /usr/local/etc/varnish/default.vcl -s malloc,1G -T 127.0.0.1:2000
 
-We're removed the -a option. Now Varnish will bind to the http port as
-it is its default. Now try your web application and see if it works
-OK.
+Note that we've removed the -a option. Now Varnish, as its default
+setting dictates, will bind to the http port (80). Now everyone thats
+accessing your site will be accessing through Varnish.
+



From phk at project.varnish-software.com  Fri Jan 21 14:58:39 2011
From: phk at project.varnish-software.com (Poul-Henning Kamp)
Date: Fri, 21 Jan 2011 15:58:39 +0100
Subject: [master] 5ef8238 Merge branch 'master' of
	ssh://git.varnish-cache.org/git/varnish-cache
Message-ID: <E1PgIRj-0000Sk-87@project.varnish-software.com>

commit 5ef823824008ab2533d37dca6b99fafae02834cd
Merge: 6ee9fa3 6350bfb
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Fri Jan 21 14:58:33 2011 +0000

    Merge branch 'master' of ssh://git.varnish-cache.org/git/varnish-cache



From phk at project.varnish-software.com  Fri Jan 21 14:58:39 2011
From: phk at project.varnish-software.com (Poul-Henning Kamp)
Date: Fri, 21 Jan 2011 15:58:39 +0100
Subject: [master] 6ee9fa3 Add the VEP callback function, which,
	which can do the gzip compression and return the offsets we need to
	compose the VEC string.
Message-ID: <E1PgIRj-0000Sv-IB@project.varnish-software.com>

commit 6ee9fa333669afa3156c91d0a0ebe54b9e5c116f
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Fri Jan 21 14:57:00 2011 +0000

    Add the VEP callback function, which, which can do the gzip compression
    and return the offsets we need to compose the VEC string.
    
    The default callback function is a 1:1 mapping.

diff --git a/bin/varnishd/cache_esi.h b/bin/varnishd/cache_esi.h
index 1e15d6b..455e351 100644
--- a/bin/varnishd/cache_esi.h
+++ b/bin/varnishd/cache_esi.h
@@ -38,7 +38,11 @@
 #define	VEC_S8	(0x60 + 8)
 #define	VEC_INCL	'I'
 
-void VEP_Init(const struct sess *sp);
+enum vep_flg { VEP_NORMAL, VEP_ALIGN, VEP_RESET, VEP_FINISH };
+typedef ssize_t vep_callback_t(const struct sess *sp,
+    ssize_t l, enum vep_flg flg);
+
+void VEP_Init(const struct sess *sp, vep_callback_t *cb);
 void VEP_parse(const struct sess *sp, const char *p, size_t l);
 struct vsb *VEP_Finish(const struct sess *sp);
 
diff --git a/bin/varnishd/cache_esi_fetch.c b/bin/varnishd/cache_esi_fetch.c
index b9cc5fc..690f41f 100644
--- a/bin/varnishd/cache_esi_fetch.c
+++ b/bin/varnishd/cache_esi_fetch.c
@@ -179,7 +179,7 @@ vfp_esi_begin(struct sess *sp, size_t estimate)
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 	/* XXX: snapshot WS's ? We'll need the space */
 
-	VEP_Init(sp);
+	VEP_Init(sp, NULL);
 
 	if (sp->wrk->is_gzip && sp->wrk->do_gunzip)
 		sp->wrk->vgz_rx = VGZ_NewUngzip(sp, sp->ws);
diff --git a/bin/varnishd/cache_esi_parse.c b/bin/varnishd/cache_esi_parse.c
index 2a3edea..347fc6e 100644
--- a/bin/varnishd/cache_esi_parse.c
+++ b/bin/varnishd/cache_esi_parse.c
@@ -64,6 +64,10 @@ struct vep_state {
 	struct vsb		*vsb;
 
 	const struct sess	*sp;
+	vep_callback_t		*cb;
+
+	/* Internal Counter for default call-back function */
+	ssize_t			cb_x;
 
 	/* parser state */
 	const char		*state;
@@ -80,6 +84,7 @@ struct vep_state {
 	uint32_t		crc;
 	ssize_t			o_crc;
 	uint32_t		crcp;
+	ssize_t			o_last;
 
 const char		*hack_p;
 	const char		*ver_p;
@@ -272,7 +277,7 @@ vep_emit_skip(const struct vep_state *vep, ssize_t l)
 } 
 
 static void
-vep_emit_verbatim(const struct vep_state *vep, ssize_t l)
+vep_emit_verbatim(const struct vep_state *vep, ssize_t l, ssize_t l_crc)
 {
 	uint8_t buf[4];
 
@@ -280,28 +285,26 @@ vep_emit_verbatim(const struct vep_state *vep, ssize_t l)
 		Debug("---> VERBATIM(%jd)\n", (intmax_t)l);
 	}
 	vep_emit_len(vep, l, VEC_V1, VEC_V2, VEC_V8);
-	vep_emit_len(vep, l, VEC_C1, VEC_C2, VEC_C8);
+	vep_emit_len(vep, l_crc, VEC_C1, VEC_C2, VEC_C8);
 	vbe32enc(buf, vep->crc);
 	vsb_bcat(vep->vsb, buf, sizeof buf);
 	vsb_printf(vep->vsb, "%lx\r\n%c", l, 0);
 } 
 
 static void
-vep_emit_common(struct vep_state *vep, ssize_t *l, enum vep_mark mark)
+vep_emit_common(struct vep_state *vep, ssize_t l, enum vep_mark mark)
 {
-	assert(*l > 0);
-	assert(*l == vep->o_crc);
+	assert(l > 0);
 
 	assert(mark == SKIP || mark == VERBATIM);
 	if (mark == SKIP)
-		vep_emit_skip(vep, *l);
+		vep_emit_skip(vep, l);
 	else
-		vep_emit_verbatim(vep, *l);
+		vep_emit_verbatim(vep, l, vep->o_crc);
 
 	vep->crc = crc32(0L, Z_NULL, 0);
 	vep->o_crc = 0;
-	vep->o_total += *l;
-	*l = 0;
+	vep->o_total += l;
 }
 
 /*---------------------------------------------------------------------
@@ -311,7 +314,7 @@ vep_emit_common(struct vep_state *vep, ssize_t *l, enum vep_mark mark)
 static void
 vep_mark_common(struct vep_state *vep, const char *p, enum vep_mark mark)
 {
-	ssize_t l;
+	ssize_t l, lcb;
 
 	assert(mark == SKIP || mark == VERBATIM);
 
@@ -324,11 +327,17 @@ vep_mark_common(struct vep_state *vep, const char *p, enum vep_mark mark)
 	 * assembled before the pending bytes.
 	 */
 
-	if (vep->last_mark != mark && vep->o_wait > 0)
-		vep_emit_common(vep, &vep->o_wait, vep->last_mark);
+	if (vep->last_mark != mark && vep->o_wait > 0) {
+		lcb = vep->cb(vep->sp, 0,
+		    mark == VERBATIM ? VEP_RESET : VEP_ALIGN);
+		vep_emit_common(vep, lcb - vep->o_last, vep->last_mark);
+		vep->o_last = lcb;
+		vep->o_wait = 0;
+	}
 
 	/* Transfer pending bytes CRC into active mode CRC */
 	if (vep->o_pending) {
+		(void)vep->cb(vep->sp, vep->o_pending, VEP_NORMAL);
 		if (vep->o_crc == 0) {
 			vep->crc = vep->crcp;
 			vep->o_crc = vep->o_pending;
@@ -338,6 +347,8 @@ vep_mark_common(struct vep_state *vep, const char *p, enum vep_mark mark)
 			vep->o_crc += vep->o_pending;
 		}
 		vep->crcp = crc32(0L, Z_NULL, 0);
+		vep->o_wait += vep->o_pending;
+		vep->o_pending = 0;
 	}
 
 	/* * Process this bit of input */
@@ -348,10 +359,9 @@ vep_mark_common(struct vep_state *vep, const char *p, enum vep_mark mark)
 	vep->o_crc += l;
 	vep->ver_p = p;
 
-	vep->o_wait += vep->o_pending;
 	vep->o_wait += l;
-	vep->o_pending = 0;
 	vep->last_mark = mark;
+	(void)vep->cb(vep->sp, l, VEP_NORMAL);
 }
 
 static void
@@ -951,8 +961,26 @@ VEP_parse(const struct sess *sp, const char *p, size_t l)
 		vep_mark_pending(vep, p);
 }
 
+
+/*---------------------------------------------------------------------
+ */
+
+static ssize_t
+vep_default_cb(const struct sess *sp, ssize_t l, enum vep_flg flg)
+{
+
+	(void)flg;
+	AN(sp->wrk->vep);
+	sp->wrk->vep->cb_x += l;
+Debug("CB(%jd,%d) = %jd\n", (intmax_t)l, flg, (intmax_t)sp->wrk->vep->cb_x);
+	return (sp->wrk->vep->cb_x);
+}
+
+/*---------------------------------------------------------------------
+ */
+
 void
-VEP_Init(const struct sess *sp)
+VEP_Init(const struct sess *sp, vep_callback_t *cb)
 {
 	struct vep_state *vep;
 
@@ -964,6 +992,10 @@ VEP_Init(const struct sess *sp)
 	memset(vep, 0, sizeof *vep);
 	vep->magic = VEP_MAGIC;
 	vep->sp = sp;
+	if (cb != NULL)
+		vep->cb = cb;
+	else
+		vep->cb = vep_default_cb;
 
 	vep->state = VEP_START;
 	vep->vsb = vsb_newauto();
@@ -972,21 +1004,27 @@ VEP_Init(const struct sess *sp)
 	vep->crcp = crc32(0L, Z_NULL, 0);
 }
 
+/*---------------------------------------------------------------------
+ */
+
 struct vsb *
 VEP_Finish(const struct sess *sp)
 {
 	struct vep_state *vep;
-	ssize_t l;
+	ssize_t l, lcb;
 
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 	vep = sp->wrk->vep;
-	sp->wrk->vep = NULL;
 	CHECK_OBJ_NOTNULL(vep, VEP_MAGIC);
 
 	if (vep->o_pending)
 		vep_mark_common(vep, vep->ver_p, vep->last_mark);
-	if (vep->o_wait > 0)
-		vep_emit_common(vep, &vep->o_wait, vep->last_mark);
+	if (vep->o_wait > 0) {
+		lcb = vep->cb(vep->sp, 0, VEP_FINISH);
+		vep_emit_common(vep, lcb - vep->o_last, vep->last_mark);
+	}
+
+	sp->wrk->vep = NULL;
 
 	vsb_finish(vep->vsb);
 	l = vsb_len(vep->vsb);



From phk at project.varnish-software.com  Fri Jan 21 18:14:47 2011
From: phk at project.varnish-software.com (Poul-Henning Kamp)
Date: Fri, 21 Jan 2011 19:14:47 +0100
Subject: [master] 35febe9 Yah! Enable gzip-file stitching and pass the first
	trivial ESI+GZIP testcase
Message-ID: <E1PgLVX-00041R-Gu@project.varnish-software.com>

commit 35febe95ae77ba13c4da970d3cc26468abb1e517
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Fri Jan 21 18:13:50 2011 +0000

    Yah! Enable gzip-file stitching and pass the first trivial
    ESI+GZIP testcase

diff --git a/bin/varnishd/cache.h b/bin/varnishd/cache.h
index b0497fb..9ee361d 100644
--- a/bin/varnishd/cache.h
+++ b/bin/varnishd/cache.h
@@ -98,6 +98,7 @@ struct ban;
 struct SHA256Context;
 struct vsc_lck;
 struct waitinglist;
+struct vef_priv;
 
 struct lock { void *priv; };		// Opaque
 
@@ -280,7 +281,7 @@ struct worker {
 	struct storage		*storage;
 	struct vfp		*vfp;
 	struct vgz		*vgz_rx;
-	struct vgz		*vgz_esi;
+	struct vef_priv		*vef_priv;
 	unsigned		do_esi;
 	unsigned		do_gzip;
 	unsigned		is_gzip;
diff --git a/bin/varnishd/cache_esi.h b/bin/varnishd/cache_esi.h
index b57cda9..4110687 100644
--- a/bin/varnishd/cache_esi.h
+++ b/bin/varnishd/cache_esi.h
@@ -27,6 +27,7 @@
  *
  */
 
+#define	VEC_GZ	(0x21)
 #define	VEC_V1	(0x40 + 1)
 #define	VEC_V2	(0x40 + 2)
 #define	VEC_V8	(0x40 + 8)
diff --git a/bin/varnishd/cache_esi_deliver.c b/bin/varnishd/cache_esi_deliver.c
index 6a344ec..a17d101 100644
--- a/bin/varnishd/cache_esi_deliver.c
+++ b/bin/varnishd/cache_esi_deliver.c
@@ -174,8 +174,10 @@ ESI_Deliver(struct sess *sp)
 	struct storage *st;
 	uint8_t *p, *e, *q, *r;
 	unsigned off;
-	ssize_t l, l_crc;
-	uint32_t crc, crc_ref;
+	ssize_t l, l_icrc, l_crc;
+	uint32_t crc, icrc;
+	uint8_t tailbuf[8 + 5];
+	int dogzip;
 
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 	st = sp->obj->esidata;
@@ -183,6 +185,14 @@ ESI_Deliver(struct sess *sp)
 	p = st->ptr;
 	e = st->ptr + st->len;
 
+	if (*p == VEC_GZ) {
+		p++;	
+		dogzip = 1;
+		crc = crc32(0L, Z_NULL, 0);
+		l_crc = 0;
+	} else 
+		dogzip = 0;
+
 	st = VTAILQ_FIRST(&sp->obj->store);
 	off = 0;
 
@@ -192,15 +202,19 @@ ESI_Deliver(struct sess *sp)
 		case VEC_V2:
 		case VEC_V8:
 			l = ved_decode_len(&p);
-			assert(*p == VEC_C1 || *p == VEC_C2 || *p == VEC_C8);
-			l_crc = ved_decode_len(&p);
-			crc = vbe32dec(p);
-			p += 4;
+			if (dogzip) {
+				assert(*p == VEC_C1 || *p == VEC_C2 ||
+				    *p == VEC_C8);
+				l_icrc = ved_decode_len(&p);
+				icrc = vbe32dec(p);
+				p += 4;
+			}
 			q = (void*)strchr((const char*)p, '\0');
 			assert (q > p);
-			crc_ref = crc32(0L, Z_NULL, 0);
-			crc_ref = crc32(crc_ref, st->ptr + off, l);
-			xxxassert(crc_ref == crc);
+			if (dogzip) {
+				crc = crc32_combine(crc, icrc, l_icrc);
+				l_crc += l_icrc;
+			}
 			ved_sendchunk(sp, p, q - p, st->ptr + off, l);
 			off += l;
 			p = q + 1;
@@ -229,6 +243,22 @@ ESI_Deliver(struct sess *sp)
 			INCOMPL();
 		}
 	}
+	if (dogzip) {
+		/* Emit a gzip literal block with finish bit set */
+		tailbuf[0] = 0x01;
+		tailbuf[1] = 0x00;
+		tailbuf[2] = 0x00;
+		tailbuf[3] = 0xff;
+		tailbuf[4] = 0xff;
+
+		/* Emit CRC32 */
+		vle32enc(tailbuf + 5, crc);
+
+		/* MOD(2^32) length */
+		vle32enc(tailbuf + 9, l_crc);
+
+		ved_sendchunk(sp, "d\r\n", 3, tailbuf, 13);
+	}
 	(void)WRW_Flush(sp->wrk);
 }
 
diff --git a/bin/varnishd/cache_esi_fetch.c b/bin/varnishd/cache_esi_fetch.c
index 690f41f..68643d2 100644
--- a/bin/varnishd/cache_esi_fetch.c
+++ b/bin/varnishd/cache_esi_fetch.c
@@ -25,7 +25,7 @@
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
  *
- * VEP Varnish Esi Parsing
+ * VEF Varnish Esi Fetching
  */
 
 #include "config.h"
@@ -95,16 +95,6 @@ vfp_esi_bytes_uu(struct sess *sp, struct http_conn *htc, size_t bytes)
 }
 
 /*---------------------------------------------------------------------
- * We receive a ungzip'ed object, and want to store it gzip'ed.
- */
-
-static int __match_proto__()
-vfp_esi_bytes_ug(struct sess *sp, struct http_conn *htc, size_t bytes)
-{
-	return (vfp_esi_bytes_uu(sp, htc, bytes));
-}
-
-/*---------------------------------------------------------------------
  * We receive a gzip'ed object, and want to store it ungzip'ed.
  */
 
@@ -160,6 +150,123 @@ vfp_esi_bytes_gu(struct sess *sp, struct http_conn *htc, size_t bytes)
 }
 
 /*---------------------------------------------------------------------
+ */
+
+struct vef_priv {
+	unsigned		magic;
+#define VEF_MAGIC		0xf104b51f
+	struct vgz		*vgz;
+
+	char			*bufp;
+	ssize_t			tot;
+	int			error;
+};
+
+/*---------------------------------------------------------------------
+ * We receive a ungzip'ed object, and want to store it gzip'ed.
+ */
+
+#include "vend.h"
+
+static ssize_t
+vfp_vep_callback(const struct sess *sp, ssize_t l, enum vgz_flag flg)
+{
+	struct vef_priv *vef;
+	struct storage *st;
+	size_t dl;
+	const void *dp;
+	int i;
+	char *p;
+
+printf("ZCB(%jd, %d)\n", l, flg);
+
+	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+	vef = sp->wrk->vef_priv;
+	CHECK_OBJ_NOTNULL(vef, VEF_MAGIC);
+	assert(l >= 0);
+
+	if (vef->error) {
+		vef->tot += l;
+		return (vef->tot);
+	}
+
+	VGZ_Ibuf(vef->vgz, vef->bufp, l);
+	do {
+		if (sp->wrk->storage == NULL) {
+			l = params->fetch_chunksize * 1024LL;
+			sp->wrk->storage = STV_alloc(sp, l);
+		}
+		if (sp->wrk->storage == NULL) {
+			vef->error = ENOMEM;
+			vef->tot += l;
+			return (vef->tot);
+		}
+
+		st = sp->wrk->storage;
+		VGZ_Obuf(vef->vgz, st->ptr + st->len, st->space - st->len);
+		p = (void*)(st->ptr + st->len);
+		i = VGZ_Gzip(vef->vgz, &dp, &dl, flg);
+printf("GZI = %d %jd\n", i, dl);
+		vef->tot += dl;
+		st->len += dl;
+		if (st->len == st->space) {
+			VTAILQ_INSERT_TAIL(&sp->obj->store,
+			    sp->wrk->storage, list);
+			sp->wrk->storage = NULL;
+			st = NULL;
+		}
+	} while (!VGZ_IbufEmpty(vef->vgz));
+	vef->bufp += l;
+if (flg == VGZ_FINISH)
+	assert(i == 1);			/* XXX */
+else
+	assert(i == 0);			/* XXX */
+printf("ZCB = %jd\n", vef->tot);
+fflush(stdout);
+usleep(100);
+	return (vef->tot);
+}
+
+static int __match_proto__()
+vfp_esi_bytes_ug(struct sess *sp, struct http_conn *htc, size_t bytes)
+{
+	ssize_t l, w;
+	char ibuf[1024 * params->gzip_stack_buffer];
+	struct vef_priv *vef;
+
+	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+	vef = sp->wrk->vef_priv;
+	CHECK_OBJ_NOTNULL(vef, VEF_MAGIC);
+
+	while (bytes > 0) {
+		if (sp->wrk->storage == NULL) {
+			l = params->fetch_chunksize * 1024LL;
+			sp->wrk->storage = STV_alloc(sp, l);
+		}
+		if (sp->wrk->storage == NULL) {
+			errno = ENOMEM;
+			return (-1);
+		}
+		l = sizeof ibuf;
+		if (l > bytes)
+			l = bytes;
+		w = HTC_Read(htc, ibuf, l);
+		if (w <= 0) {
+printf("RT %jd\n", w);
+			return (w);
+}
+		bytes -= w;
+		vef->bufp = ibuf;
+		VEP_parse(sp, ibuf, l);
+		if (vef->error) {
+			// errno = vef->error;
+			return (-1);
+		}
+	}
+	return (1);
+}
+
+/*---------------------------------------------------------------------
  * We receive a gzip'ed object, and want to store it gzip'ed.
  */
 
@@ -175,14 +282,26 @@ vfp_esi_bytes_gg(struct sess *sp, struct http_conn *htc, size_t bytes)
 static void __match_proto__()
 vfp_esi_begin(struct sess *sp, size_t estimate)
 {
+	struct vef_priv *vef;
 
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 	/* XXX: snapshot WS's ? We'll need the space */
 
-	VEP_Init(sp, NULL);
-
-	if (sp->wrk->is_gzip && sp->wrk->do_gunzip)
+	if (sp->wrk->is_gzip && sp->wrk->do_gunzip) {
 		sp->wrk->vgz_rx = VGZ_NewUngzip(sp, sp->ws);
+		VEP_Init(sp, NULL);
+	} else if (sp->wrk->is_gunzip && sp->wrk->do_gzip) {
+		VEP_Init(sp, vfp_vep_callback);
+		vef = (void*)WS_Alloc(sp->ws, sizeof *vef);
+		AN(vef);
+		memset(vef, 0, sizeof *vef);
+		vef->magic = VEF_MAGIC;
+		vef->vgz = VGZ_NewGzip(sp, sp->ws);
+		AZ(sp->wrk->vef_priv);
+		sp->wrk->vef_priv = vef;
+	} else {
+		VEP_Init(sp, NULL);
+	}
 
 	(void)estimate;
 }
@@ -193,6 +312,7 @@ vfp_esi_bytes(struct sess *sp, struct http_conn *htc, size_t bytes)
 	int i;
 
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+printf("BYTES = %jd\n", bytes);
 	if (sp->wrk->is_gzip && sp->wrk->do_gunzip)
 		i = vfp_esi_bytes_gu(sp, htc, bytes);
 	else if (sp->wrk->is_gunzip && sp->wrk->do_gzip)
@@ -201,6 +321,7 @@ vfp_esi_bytes(struct sess *sp, struct http_conn *htc, size_t bytes)
 		i = vfp_esi_bytes_gg(sp, htc, bytes);
 	else
 		i = vfp_esi_bytes_uu(sp, htc, bytes);
+printf("BYTES = %d\n", i);
 	return (i);
 }
 
@@ -209,8 +330,10 @@ vfp_esi_end(struct sess *sp)
 {
 	struct storage *st;
 	struct vsb *vsb;
+	struct vef_priv *vef;
 	ssize_t l;
 
+printf("END\n");
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 	if (sp->wrk->is_gzip && sp->wrk->do_gunzip)
 		VGZ_Destroy(&sp->wrk->vgz_rx);
@@ -230,6 +353,16 @@ vfp_esi_end(struct sess *sp)
 
 	st = sp->wrk->storage;
 	sp->wrk->storage = NULL;
+
+	if (sp->wrk->vef_priv != NULL) {
+		vef = sp->wrk->vef_priv;
+		sp->wrk->vef_priv = NULL;
+		CHECK_OBJ_NOTNULL(vef, VEF_MAGIC);
+		XXXAZ(vef->error);
+printf("TOT %jd\n", vef->tot);
+sp->obj->len = vef->tot;
+	}
+
 	if (st == NULL)
 		return (0);
 
diff --git a/bin/varnishd/cache_esi_parse.c b/bin/varnishd/cache_esi_parse.c
index 0be7faa..5e85eff 100644
--- a/bin/varnishd/cache_esi_parse.c
+++ b/bin/varnishd/cache_esi_parse.c
@@ -64,6 +64,7 @@ struct vep_state {
 	struct vsb		*vsb;
 
 	const struct sess	*sp;
+	int			dogzip;
 	vep_callback_t		*cb;
 
 	/* Internal Counter for default call-back function */
@@ -285,9 +286,12 @@ vep_emit_verbatim(const struct vep_state *vep, ssize_t l, ssize_t l_crc)
 		Debug("---> VERBATIM(%jd)\n", (intmax_t)l);
 	}
 	vep_emit_len(vep, l, VEC_V1, VEC_V2, VEC_V8);
-	vep_emit_len(vep, l_crc, VEC_C1, VEC_C2, VEC_C8);
-	vbe32enc(buf, vep->crc);
-	vsb_bcat(vep->vsb, buf, sizeof buf);
+	if (vep->dogzip) {
+		vep_emit_len(vep, l_crc, VEC_C1, VEC_C2, VEC_C8);
+		vbe32enc(buf, vep->crc);
+		vsb_bcat(vep->vsb, buf, sizeof buf);
+	}
+	/* Emit Chunked header */
 	vsb_printf(vep->vsb, "%lx\r\n%c", l, 0);
 } 
 
@@ -992,14 +996,19 @@ VEP_Init(const struct sess *sp, vep_callback_t *cb)
 	memset(vep, 0, sizeof *vep);
 	vep->magic = VEP_MAGIC;
 	vep->sp = sp;
-	if (cb != NULL)
+	vep->vsb = vsb_newauto();
+	AN(vep->vsb);
+
+	if (cb != NULL) {
+		vep->dogzip = 1;
+		/* XXX */
+		vsb_printf(vep->vsb, "%c", VEC_GZ);
 		vep->cb = cb;
-	else
+	} else {
 		vep->cb = vep_default_cb;
+	}
 
 	vep->state = VEP_START;
-	vep->vsb = vsb_newauto();
-	AN(vep->vsb);
 	vep->crc = crc32(0L, Z_NULL, 0);
 	vep->crcp = crc32(0L, Z_NULL, 0);
 }
@@ -1020,8 +1029,9 @@ VEP_Finish(const struct sess *sp)
 	if (vep->o_pending)
 		vep_mark_common(vep, vep->ver_p, vep->last_mark);
 	if (vep->o_wait > 0) {
-		lcb = vep->cb(vep->sp, 0, VGZ_FINISH);
+		lcb = vep->cb(vep->sp, 0, VGZ_ALIGN);
 		vep_emit_common(vep, lcb - vep->o_last, vep->last_mark);
+		(void)vep->cb(vep->sp, 0, VGZ_FINISH);
 	}
 
 	sp->wrk->vep = NULL;
diff --git a/bin/varnishd/cache_fetch.c b/bin/varnishd/cache_fetch.c
index 7de0fb7..8f113f2 100644
--- a/bin/varnishd/cache_fetch.c
+++ b/bin/varnishd/cache_fetch.c
@@ -204,8 +204,8 @@ fetch_straight(struct sess *sp, struct http_conn *htc, const char *b)
 
 	i = sp->wrk->vfp->bytes(sp, htc, cl);
 	if (i <= 0) {
-		WSP(sp, SLT_FetchError, "straight read_error: %d (%s)",
-		    errno, strerror(errno));
+		WSP(sp, SLT_FetchError, "straight read_error: %d %d (%s)",
+		    i, errno, strerror(errno));
 		return (-1);
 	}
 	return (0);
diff --git a/bin/varnishtest/tests/e00021.vtc b/bin/varnishtest/tests/e00021.vtc
new file mode 100644
index 0000000..1debf38
--- /dev/null
+++ b/bin/varnishtest/tests/e00021.vtc
@@ -0,0 +1,41 @@
+# $Id$
+
+test "ESI ability to stitch gzip files together"
+
+
+server s1 {
+	rxreq 
+	txresp -body {
+		<esi:remove>
+		This is a test: Unseen University
+		<esi:include src="trick question">
+		<!--esi XXX -->
+		</esi:remove>
+		<esX>This is a test: Hello world
+	}
+} -start
+
+varnish v1 -vcl+backend {
+	sub vcl_recv {
+		set req.esi = true;
+	}
+	sub vcl_fetch {
+		set beresp.do_esi = true;
+		set beresp.do_gzip = true;
+	}
+} -start 
+
+varnish v1 -cliok "param.set esi_syntax 4"
+varnish v1 -cliok "param.set http_gzip_support true"
+
+client c1 {
+	txreq  -hdr "Accept-Encoding: gzip"
+	rxresp
+	expect resp.http.content-encoding == gzip
+	gunzip
+	expect resp.status == 200
+	expect resp.bodylen == 40
+}
+
+client c1 -run
+varnish v1 -expect esi_errors == 2



From phk at project.varnish-software.com  Fri Jan 21 18:14:47 2011
From: phk at project.varnish-software.com (Poul-Henning Kamp)
Date: Fri, 21 Jan 2011 19:14:47 +0100
Subject: [master] 8a79da2 enable the vle32enc(),
	we need it for writing gzip trailers
Message-ID: <E1PgLVX-00041c-Tb@project.varnish-software.com>

commit 8a79da2afedec7b42b06be94f2ea1001584cd6d8
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Fri Jan 21 17:39:06 2011 +0000

    enable the vle32enc(), we need it for writing gzip trailers

diff --git a/include/vend.h b/include/vend.h
index 15fcb70..01f0dfc 100644
--- a/include/vend.h
+++ b/include/vend.h
@@ -126,6 +126,7 @@ vle16enc(void *pp, uint16_t u)
 	p[0] = u & 0xff;
 	p[1] = (u >> 8) & 0xff;
 }
+#endif
 
 static __inline void
 vle32enc(void *pp, uint32_t u)
@@ -138,6 +139,7 @@ vle32enc(void *pp, uint32_t u)
 	p[3] = (u >> 24) & 0xff;
 }
 
+#if 0
 static __inline void
 vle64enc(void *pp, uint64_t u)
 {



From phk at project.varnish-software.com  Fri Jan 21 18:14:48 2011
From: phk at project.varnish-software.com (Poul-Henning Kamp)
Date: Fri, 21 Jan 2011 19:14:48 +0100
Subject: [master] ab76cb9 Add VGZ_ flags to pass to Gzip function to insulate
	from zlib
Message-ID: <E1PgLVY-00041m-4z@project.varnish-software.com>

commit ab76cb9a091d1dda23afe8ab0279d9e682f4c689
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Fri Jan 21 16:11:30 2011 +0000

    Add VGZ_ flags to pass to Gzip function to insulate from zlib

diff --git a/bin/varnishd/cache.h b/bin/varnishd/cache.h
index 510e7ab..b0497fb 100644
--- a/bin/varnishd/cache.h
+++ b/bin/varnishd/cache.h
@@ -626,12 +626,13 @@ void Fetch_Init(void);
 /* cache_gzip.c */
 struct vgz;
 
+enum vgz_flag { VGZ_NORMAL, VGZ_ALIGN, VGZ_RESET, VGZ_FINISH };
 struct vgz *VGZ_NewUngzip(const struct sess *sp, struct ws *tmp);
 struct vgz *VGZ_NewGzip(const struct sess *sp, struct ws *tmp);
 void VGZ_Ibuf(struct vgz *, const void *, ssize_t len);
 int VGZ_IbufEmpty(struct vgz *vg);
 void VGZ_Obuf(struct vgz *, const void *, ssize_t len);
-int VGZ_Gzip(struct vgz *, const void **, size_t *len, int flag);
+int VGZ_Gzip(struct vgz *, const void **, size_t *len, enum vgz_flag);
 int VGZ_Gunzip(struct vgz *, const void **, size_t *len);
 void VGZ_Destroy(struct vgz **);
 
diff --git a/bin/varnishd/cache_esi.h b/bin/varnishd/cache_esi.h
index 455e351..b57cda9 100644
--- a/bin/varnishd/cache_esi.h
+++ b/bin/varnishd/cache_esi.h
@@ -38,9 +38,8 @@
 #define	VEC_S8	(0x60 + 8)
 #define	VEC_INCL	'I'
 
-enum vep_flg { VEP_NORMAL, VEP_ALIGN, VEP_RESET, VEP_FINISH };
 typedef ssize_t vep_callback_t(const struct sess *sp,
-    ssize_t l, enum vep_flg flg);
+    ssize_t l, enum vgz_flag flg);
 
 void VEP_Init(const struct sess *sp, vep_callback_t *cb);
 void VEP_parse(const struct sess *sp, const char *p, size_t l);
diff --git a/bin/varnishd/cache_esi_parse.c b/bin/varnishd/cache_esi_parse.c
index 347fc6e..0be7faa 100644
--- a/bin/varnishd/cache_esi_parse.c
+++ b/bin/varnishd/cache_esi_parse.c
@@ -329,7 +329,7 @@ vep_mark_common(struct vep_state *vep, const char *p, enum vep_mark mark)
 
 	if (vep->last_mark != mark && vep->o_wait > 0) {
 		lcb = vep->cb(vep->sp, 0,
-		    mark == VERBATIM ? VEP_RESET : VEP_ALIGN);
+		    mark == VERBATIM ? VGZ_RESET : VGZ_ALIGN);
 		vep_emit_common(vep, lcb - vep->o_last, vep->last_mark);
 		vep->o_last = lcb;
 		vep->o_wait = 0;
@@ -337,7 +337,7 @@ vep_mark_common(struct vep_state *vep, const char *p, enum vep_mark mark)
 
 	/* Transfer pending bytes CRC into active mode CRC */
 	if (vep->o_pending) {
-		(void)vep->cb(vep->sp, vep->o_pending, VEP_NORMAL);
+		(void)vep->cb(vep->sp, vep->o_pending, VGZ_NORMAL);
 		if (vep->o_crc == 0) {
 			vep->crc = vep->crcp;
 			vep->o_crc = vep->o_pending;
@@ -361,7 +361,7 @@ vep_mark_common(struct vep_state *vep, const char *p, enum vep_mark mark)
 
 	vep->o_wait += l;
 	vep->last_mark = mark;
-	(void)vep->cb(vep->sp, l, VEP_NORMAL);
+	(void)vep->cb(vep->sp, l, VGZ_NORMAL);
 }
 
 static void
@@ -966,7 +966,7 @@ VEP_parse(const struct sess *sp, const char *p, size_t l)
  */
 
 static ssize_t
-vep_default_cb(const struct sess *sp, ssize_t l, enum vep_flg flg)
+vep_default_cb(const struct sess *sp, ssize_t l, enum vgz_flag flg)
 {
 
 	(void)flg;
@@ -1020,7 +1020,7 @@ VEP_Finish(const struct sess *sp)
 	if (vep->o_pending)
 		vep_mark_common(vep, vep->ver_p, vep->last_mark);
 	if (vep->o_wait > 0) {
-		lcb = vep->cb(vep->sp, 0, VEP_FINISH);
+		lcb = vep->cb(vep->sp, 0, VGZ_FINISH);
 		vep_emit_common(vep, lcb - vep->o_last, vep->last_mark);
 	}
 
diff --git a/bin/varnishd/cache_gzip.c b/bin/varnishd/cache_gzip.c
index d532afc..dd4f8e7 100644
--- a/bin/varnishd/cache_gzip.c
+++ b/bin/varnishd/cache_gzip.c
@@ -66,6 +66,7 @@
 
 #include "config.h"
 #include <stdio.h>
+#include <stdlib.h>
 
 #include "svnid.h"
 SVNID("$Id$")
@@ -251,9 +252,10 @@ VGZ_Gunzip(struct vgz *vg, const void **pptr, size_t *plen)
 /*--------------------------------------------------------------------*/
 
 int
-VGZ_Gzip(struct vgz *vg, const void **pptr, size_t *plen, int flags)
+VGZ_Gzip(struct vgz *vg, const void **pptr, size_t *plen, enum vgz_flag flags)
 {
 	int i;
+	int zflg;
 
 	CHECK_OBJ_NOTNULL(vg, VGZ_MAGIC);
 
@@ -262,7 +264,14 @@ VGZ_Gzip(struct vgz *vg, const void **pptr, size_t *plen, int flags)
 	AN(vg->vz.next_out);
 	AN(vg->vz.avail_out);
 	vg->before = vg->vz.next_out;
-	i = deflate(&vg->vz, flags);
+	switch(flags) {
+	case VGZ_NORMAL:	zflg = Z_NO_FLUSH; break;
+	case VGZ_ALIGN:		zflg = Z_SYNC_FLUSH; break;
+	case VGZ_RESET:		zflg = Z_FULL_FLUSH; break;
+	case VGZ_FINISH:	zflg = Z_FINISH; break;
+	default:		INCOMPL();
+	}
+	i = deflate(&vg->vz, zflg);
 	if (i == Z_OK || i == Z_STREAM_END) {
 		*pptr = vg->before;
 		*plen = (const uint8_t *)vg->vz.next_out - (const uint8_t*)vg->before;
@@ -436,7 +445,8 @@ vfp_gzip_bytes(struct sess *sp, struct http_conn *htc, size_t bytes)
 			bytes -= w;
 		}
 
-		i = VGZ_Gzip(vg, &dp, &dl, bytes == 0 ? Z_FINISH : 0);
+		i = VGZ_Gzip(vg, &dp, &dl,
+		    bytes == 0 ? VGZ_FINISH : VGZ_NORMAL);
 		assert(i == Z_OK || i == Z_STREAM_END);
 		st->len = st->space - dl;
 		if (st->len == st->space) {



From phk at project.varnish-software.com  Fri Jan 21 21:42:02 2011
From: phk at project.varnish-software.com (Poul-Henning Kamp)
Date: Fri, 21 Jan 2011 22:42:02 +0100
Subject: [master] 1c9dafa Fix a problem with proper termination of object when
	it is gzip'ed on reception
Message-ID: <E1PgOk6-0002dR-Vl@project.varnish-software.com>

commit 1c9dafaddfdb115f8c3d96277c7f9c2c84dc712f
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Fri Jan 21 21:41:26 2011 +0000

    Fix a problem with proper termination of object when it is gzip'ed
    on reception

diff --git a/bin/varnishd/cache_gzip.c b/bin/varnishd/cache_gzip.c
index dd4f8e7..e852ba3 100644
--- a/bin/varnishd/cache_gzip.c
+++ b/bin/varnishd/cache_gzip.c
@@ -422,7 +422,7 @@ vfp_gzip_bytes(struct sess *sp, struct http_conn *htc, size_t bytes)
 	vg = sp->wrk->vgz_rx;
 	CHECK_OBJ_NOTNULL(vg, VGZ_MAGIC);
 	AZ(vg->vz.avail_in);
-	while (bytes > 0 || vg->vz.avail_in > 0) {
+	while (bytes > 0 || !VGZ_IbufEmpty(vg)) {
 		if (sp->wrk->storage == NULL)
 			sp->wrk->storage = STV_alloc(sp,
 			    params->fetch_chunksize * 1024LL);
@@ -434,7 +434,7 @@ vfp_gzip_bytes(struct sess *sp, struct http_conn *htc, size_t bytes)
 
 		VGZ_Obuf(vg, st->ptr + st->len, st->space - st->len);
 
-		if (vg->vz.avail_in == 0 && bytes > 0) {
+		if (VGZ_IbufEmpty(vg) && bytes > 0) {
 			l = sizeof ibuf;
 			if (l > bytes)
 				l = bytes;
@@ -445,20 +445,17 @@ vfp_gzip_bytes(struct sess *sp, struct http_conn *htc, size_t bytes)
 			bytes -= w;
 		}
 
-		i = VGZ_Gzip(vg, &dp, &dl,
-		    bytes == 0 ? VGZ_FINISH : VGZ_NORMAL);
-		assert(i == Z_OK || i == Z_STREAM_END);
-		st->len = st->space - dl;
+		i = VGZ_Gzip(vg, &dp, &dl, VGZ_NORMAL);
+		assert(i == Z_OK);
+		st->len += dl;
+		sp->obj->len += dl;
 		if (st->len == st->space) {
 			VTAILQ_INSERT_TAIL(&sp->obj->store,
 			    sp->wrk->storage, list);
-			sp->obj->len += st->len;
 			sp->wrk->storage = NULL;
 		}
 	}
-	if (i == Z_STREAM_END)
-		return (1);
-	return (-1);
+	return (1);
 }
 
 static int __match_proto__()
@@ -466,24 +463,43 @@ vfp_gzip_end(struct sess *sp)
 {
 	struct vgz *vg;
 	struct storage *st;
+	size_t dl;
+	const void *dp;
+	int i;
 
 	vg = sp->wrk->vgz_rx;
 	CHECK_OBJ_NOTNULL(vg, VGZ_MAGIC);
+	do {
+		if (sp->wrk->storage == NULL)
+			sp->wrk->storage = STV_alloc(sp,
+			    params->fetch_chunksize * 1024LL);
+		if (sp->wrk->storage == NULL) {
+			errno = ENOMEM;
+			return (-1);
+		}
+		st = sp->wrk->storage;
+
+		VGZ_Ibuf(vg, "", 0);
+		VGZ_Obuf(vg, st->ptr + st->len, st->space - st->len);
+		i = VGZ_Gzip(vg, &dp, &dl, VGZ_FINISH);
+		st->len += dl;
+		sp->obj->len += dl;
+		if (st->len == st->space) {
+			VTAILQ_INSERT_TAIL(&sp->obj->store,
+			    sp->wrk->storage, list);
+			sp->wrk->storage = NULL;
+		}
+	} while (i != Z_STREAM_END);
 	VGZ_Destroy(&vg);
 
 	st = sp->wrk->storage;
 	sp->wrk->storage = NULL;
-	if (st == NULL)
-		return (0);
-
-	if (st->len == 0) {
+	if (st != NULL && st->len == 0) {
 		STV_free(st);
-		return (0);
-	}
-	if (st->len < st->space)
+	} else if (st != NULL && st->len < st->space) {
 		STV_trim(st, st->len);
-	sp->obj->len += st->len;
-	VTAILQ_INSERT_TAIL(&sp->obj->store, st, list);
+		VTAILQ_INSERT_TAIL(&sp->obj->store, st, list);
+	}
 	return (0);
 }
 



From phk at project.varnish-software.com  Fri Jan 21 22:26:34 2011
From: phk at project.varnish-software.com (Poul-Henning Kamp)
Date: Fri, 21 Jan 2011 23:26:34 +0100
Subject: [master] ba29375 Create a procedure to centralize the housekeeping of
	feeding a VGZ instance into sp->obj.
Message-ID: <E1PgPRC-0007I8-KH@project.varnish-software.com>

commit ba293755d0b3b62dad1629f458a6ffbc661d380a
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Fri Jan 21 22:25:02 2011 +0000

    Create a procedure to centralize the housekeeping of feeding a VGZ
    instance into sp->obj.

diff --git a/bin/varnishd/cache.h b/bin/varnishd/cache.h
index 9ee361d..9cacf45 100644
--- a/bin/varnishd/cache.h
+++ b/bin/varnishd/cache.h
@@ -633,6 +633,7 @@ struct vgz *VGZ_NewGzip(const struct sess *sp, struct ws *tmp);
 void VGZ_Ibuf(struct vgz *, const void *, ssize_t len);
 int VGZ_IbufEmpty(struct vgz *vg);
 void VGZ_Obuf(struct vgz *, const void *, ssize_t len);
+int VGZ_ObufStorage(const struct sess *sp, struct vgz *vg);
 int VGZ_Gzip(struct vgz *, const void **, size_t *len, enum vgz_flag);
 int VGZ_Gunzip(struct vgz *, const void **, size_t *len);
 void VGZ_Destroy(struct vgz **);
diff --git a/bin/varnishd/cache_esi_fetch.c b/bin/varnishd/cache_esi_fetch.c
index 68643d2..e2ebc1d 100644
--- a/bin/varnishd/cache_esi_fetch.c
+++ b/bin/varnishd/cache_esi_fetch.c
@@ -103,7 +103,6 @@ vfp_esi_bytes_gu(struct sess *sp, struct http_conn *htc, size_t bytes)
 {
 	struct vgz *vg;
 	ssize_t l, w;
-	struct storage *st;
 	uint8_t	ibuf[1024 * params->gzip_stack_buffer];
 	int i;
 	size_t dl;
@@ -113,18 +112,6 @@ vfp_esi_bytes_gu(struct sess *sp, struct http_conn *htc, size_t bytes)
 	vg = sp->wrk->vgz_rx;
 
 	while (bytes > 0) {
-		if (sp->wrk->storage == NULL) {
-			l = params->fetch_chunksize * 1024LL;
-			sp->wrk->storage = STV_alloc(sp, l);
-		}
-		if (sp->wrk->storage == NULL) {
-			errno = ENOMEM;
-			return (-1);
-		}
-		st = sp->wrk->storage;
-
-		VGZ_Obuf(vg, st->ptr + st->len, st->space - st->len);
-
 		if (VGZ_IbufEmpty(vg) && bytes > 0) {
 			l = sizeof ibuf;
 			if (l > bytes)
@@ -135,16 +122,11 @@ vfp_esi_bytes_gu(struct sess *sp, struct http_conn *htc, size_t bytes)
 			VGZ_Ibuf(vg, ibuf, w);
 			bytes -= w;
 		}
+		if (VGZ_ObufStorage(sp, vg))
+			return (-1);
 		i = VGZ_Gunzip(vg, &dp, &dl);
 		VEP_parse(sp, dp, dl);
-		st->len += dl;
 		sp->obj->len += dl;
-		if (st->len == st->space) {
-			VTAILQ_INSERT_TAIL(&sp->obj->store,
-			    sp->wrk->storage, list);
-			sp->wrk->storage = NULL;
-			st = NULL;
-		}
 	}
 	return (1);
 }
@@ -172,11 +154,9 @@ static ssize_t
 vfp_vep_callback(const struct sess *sp, ssize_t l, enum vgz_flag flg)
 {
 	struct vef_priv *vef;
-	struct storage *st;
 	size_t dl;
 	const void *dp;
 	int i;
-	char *p;
 
 printf("ZCB(%jd, %d)\n", l, flg);
 
@@ -192,29 +172,15 @@ printf("ZCB(%jd, %d)\n", l, flg);
 
 	VGZ_Ibuf(vef->vgz, vef->bufp, l);
 	do {
-		if (sp->wrk->storage == NULL) {
-			l = params->fetch_chunksize * 1024LL;
-			sp->wrk->storage = STV_alloc(sp, l);
-		}
-		if (sp->wrk->storage == NULL) {
+		if (VGZ_ObufStorage(sp, vef->vgz)) {
 			vef->error = ENOMEM;
 			vef->tot += l;
 			return (vef->tot);
 		}
-
-		st = sp->wrk->storage;
-		VGZ_Obuf(vef->vgz, st->ptr + st->len, st->space - st->len);
-		p = (void*)(st->ptr + st->len);
 		i = VGZ_Gzip(vef->vgz, &dp, &dl, flg);
 printf("GZI = %d %jd\n", i, dl);
 		vef->tot += dl;
-		st->len += dl;
-		if (st->len == st->space) {
-			VTAILQ_INSERT_TAIL(&sp->obj->store,
-			    sp->wrk->storage, list);
-			sp->wrk->storage = NULL;
-			st = NULL;
-		}
+		sp->obj->len += dl;
 	} while (!VGZ_IbufEmpty(vef->vgz));
 	vef->bufp += l;
 if (flg == VGZ_FINISH)
@@ -251,15 +217,13 @@ vfp_esi_bytes_ug(struct sess *sp, struct http_conn *htc, size_t bytes)
 		if (l > bytes)
 			l = bytes;
 		w = HTC_Read(htc, ibuf, l);
-		if (w <= 0) {
-printf("RT %jd\n", w);
+		if (w <= 0)
 			return (w);
-}
 		bytes -= w;
 		vef->bufp = ibuf;
 		VEP_parse(sp, ibuf, l);
 		if (vef->error) {
-			// errno = vef->error;
+			errno = vef->error;
 			return (-1);
 		}
 	}
diff --git a/bin/varnishd/cache_fetch.c b/bin/varnishd/cache_fetch.c
index 8f113f2..d28dd5d 100644
--- a/bin/varnishd/cache_fetch.c
+++ b/bin/varnishd/cache_fetch.c
@@ -567,7 +567,7 @@ FetchBody(struct sess *sp)
 
 	{
 	/* Sanity check fetch methods accounting */
-		unsigned uu;
+		ssize_t uu;
 
 		uu = 0;
 		VTAILQ_FOREACH(st, &sp->obj->store, list)
diff --git a/bin/varnishd/cache_gzip.c b/bin/varnishd/cache_gzip.c
index e852ba3..8d0980c 100644
--- a/bin/varnishd/cache_gzip.c
+++ b/bin/varnishd/cache_gzip.c
@@ -82,7 +82,7 @@ struct vgz {
 	struct ws		*tmp;
 	char			*tmp_snapshot;
 
-	void			*before;
+	struct storage		*obuf;
 
 	z_stream		vz;
 };
@@ -221,12 +221,45 @@ VGZ_Obuf(struct vgz *vg, const void *ptr, ssize_t len)
 	vg->vz.avail_out = len;
 }
 
+/*--------------------------------------------------------------------
+ * Keep the outbuffer supplied with storage and file it under the
+ * sp->obj as it fills.
+ */
+
+int
+VGZ_ObufStorage(const struct sess *sp, struct vgz *vg)
+{
+	struct storage *st;
+
+	st = sp->wrk->storage;
+	if (st != NULL && st->len == st->space) {
+		VTAILQ_INSERT_TAIL(&sp->obj->store, st, list);
+		sp->wrk->storage = NULL;
+		st = NULL;
+		vg->obuf = NULL;
+	}
+	if (st == NULL) { 
+		st = STV_alloc(sp, params->fetch_chunksize * 1024LL);
+		if (st == NULL) {
+			errno = ENOMEM;
+			return (-1);
+		}
+		sp->wrk->storage = st;
+	}
+	vg->obuf = st;
+	VGZ_Obuf(vg, st->ptr + st->len, st->space - st->len);
+
+	return (0);
+}
+
 /*--------------------------------------------------------------------*/
 
 int
 VGZ_Gunzip(struct vgz *vg, const void **pptr, size_t *plen)
 {
 	int i;
+	ssize_t l;
+	const uint8_t *before;
 
 	CHECK_OBJ_NOTNULL(vg, VGZ_MAGIC);
 
@@ -234,11 +267,14 @@ VGZ_Gunzip(struct vgz *vg, const void **pptr, size_t *plen)
 	*plen = 0;
 	AN(vg->vz.next_out);
 	AN(vg->vz.avail_out);
-	vg->before = vg->vz.next_out;
+	before = vg->vz.next_out;
 	i = inflate(&vg->vz, 0);
 	if (i == Z_OK || i == Z_STREAM_END) {
-		*pptr = vg->before;
-		*plen = (const uint8_t *)vg->vz.next_out - (const uint8_t*)vg->before;
+		*pptr = before;
+		l = (const uint8_t *)vg->vz.next_out - before;
+		*plen = l;
+		if (vg->obuf != NULL)
+			vg->obuf->len += l;
 	}
 	if (i == Z_OK)
 		return (0);
@@ -256,6 +292,8 @@ VGZ_Gzip(struct vgz *vg, const void **pptr, size_t *plen, enum vgz_flag flags)
 {
 	int i;
 	int zflg;
+	ssize_t l;
+	const uint8_t *before;
 
 	CHECK_OBJ_NOTNULL(vg, VGZ_MAGIC);
 
@@ -263,7 +301,7 @@ VGZ_Gzip(struct vgz *vg, const void **pptr, size_t *plen, enum vgz_flag flags)
 	*plen = 0;
 	AN(vg->vz.next_out);
 	AN(vg->vz.avail_out);
-	vg->before = vg->vz.next_out;
+	before = vg->vz.next_out;
 	switch(flags) {
 	case VGZ_NORMAL:	zflg = Z_NO_FLUSH; break;
 	case VGZ_ALIGN:		zflg = Z_SYNC_FLUSH; break;
@@ -273,8 +311,11 @@ VGZ_Gzip(struct vgz *vg, const void **pptr, size_t *plen, enum vgz_flag flags)
 	}
 	i = deflate(&vg->vz, zflg);
 	if (i == Z_OK || i == Z_STREAM_END) {
-		*pptr = vg->before;
-		*plen = (const uint8_t *)vg->vz.next_out - (const uint8_t*)vg->before;
+		*pptr = before;
+		l = (const uint8_t *)vg->vz.next_out - before;
+		*plen = l;
+		if (vg->obuf != NULL)
+			vg->obuf->len += l;
 	}
 	if (i == Z_OK)
 		return (0);
@@ -313,7 +354,6 @@ static int __match_proto__()
 vfp_gunzip_bytes(struct sess *sp, struct http_conn *htc, size_t bytes)
 {
 	struct vgz *vg;
-	struct storage *st;
 	ssize_t l, w;
 	int i = -100;
 	uint8_t	ibuf[1024 * params->gzip_stack_buffer];
@@ -324,17 +364,6 @@ vfp_gunzip_bytes(struct sess *sp, struct http_conn *htc, size_t bytes)
 	CHECK_OBJ_NOTNULL(vg, VGZ_MAGIC);
 	AZ(vg->vz.avail_in);
 	while (bytes > 0 || vg->vz.avail_in > 0) {
-		if (sp->wrk->storage == NULL)
-			sp->wrk->storage = STV_alloc(sp,
-			    params->fetch_chunksize * 1024LL);
-		if (sp->wrk->storage == NULL) {
-			errno = ENOMEM;
-			return (-1);
-		}
-		st = sp->wrk->storage;
-
-		VGZ_Obuf(vg, st->ptr + st->len, st->space - st->len);
-
 		if (vg->vz.avail_in == 0 && bytes > 0) {
 			l = sizeof ibuf;
 			if (l > bytes)
@@ -346,15 +375,11 @@ vfp_gunzip_bytes(struct sess *sp, struct http_conn *htc, size_t bytes)
 			bytes -= w;
 		}
 
+		if (VGZ_ObufStorage(sp, vg))
+			return (-1);
 		i = VGZ_Gunzip(vg, &dp, &dl);
 		assert(i == Z_OK || i == Z_STREAM_END);
-		st->len += dl;
-		if (st->len == st->space) {
-			VTAILQ_INSERT_TAIL(&sp->obj->store,
-			    sp->wrk->storage, list);
-			sp->obj->len += st->len;
-			sp->wrk->storage = NULL;
-		}
+		sp->obj->len += dl;
 	}
 	if (i == Z_STREAM_END)
 		return (1);
@@ -382,7 +407,6 @@ vfp_gunzip_end(struct sess *sp)
 	}
 	if (st->len < st->space)
 		STV_trim(st, st->len);
-	sp->obj->len += st->len;
 	VTAILQ_INSERT_TAIL(&sp->obj->store, st, list);
 	return (0);
 }
@@ -412,7 +436,6 @@ static int __match_proto__()
 vfp_gzip_bytes(struct sess *sp, struct http_conn *htc, size_t bytes)
 {
 	struct vgz *vg;
-	struct storage *st;
 	ssize_t l, w;
 	int i = -100;
 	uint8_t ibuf[1024 * params->gzip_stack_buffer];
@@ -423,17 +446,6 @@ vfp_gzip_bytes(struct sess *sp, struct http_conn *htc, size_t bytes)
 	CHECK_OBJ_NOTNULL(vg, VGZ_MAGIC);
 	AZ(vg->vz.avail_in);
 	while (bytes > 0 || !VGZ_IbufEmpty(vg)) {
-		if (sp->wrk->storage == NULL)
-			sp->wrk->storage = STV_alloc(sp,
-			    params->fetch_chunksize * 1024LL);
-		if (sp->wrk->storage == NULL) {
-			errno = ENOMEM;
-			return (-1);
-		}
-		st = sp->wrk->storage;
-
-		VGZ_Obuf(vg, st->ptr + st->len, st->space - st->len);
-
 		if (VGZ_IbufEmpty(vg) && bytes > 0) {
 			l = sizeof ibuf;
 			if (l > bytes)
@@ -444,16 +456,11 @@ vfp_gzip_bytes(struct sess *sp, struct http_conn *htc, size_t bytes)
 			VGZ_Ibuf(vg, ibuf, w);
 			bytes -= w;
 		}
-
+		if (VGZ_ObufStorage(sp, vg))
+			return (-1);
 		i = VGZ_Gzip(vg, &dp, &dl, VGZ_NORMAL);
 		assert(i == Z_OK);
-		st->len += dl;
 		sp->obj->len += dl;
-		if (st->len == st->space) {
-			VTAILQ_INSERT_TAIL(&sp->obj->store,
-			    sp->wrk->storage, list);
-			sp->wrk->storage = NULL;
-		}
 	}
 	return (1);
 }
@@ -462,33 +469,19 @@ static int __match_proto__()
 vfp_gzip_end(struct sess *sp)
 {
 	struct vgz *vg;
-	struct storage *st;
 	size_t dl;
 	const void *dp;
 	int i;
+	struct storage *st;
 
 	vg = sp->wrk->vgz_rx;
 	CHECK_OBJ_NOTNULL(vg, VGZ_MAGIC);
 	do {
-		if (sp->wrk->storage == NULL)
-			sp->wrk->storage = STV_alloc(sp,
-			    params->fetch_chunksize * 1024LL);
-		if (sp->wrk->storage == NULL) {
-			errno = ENOMEM;
-			return (-1);
-		}
-		st = sp->wrk->storage;
-
 		VGZ_Ibuf(vg, "", 0);
-		VGZ_Obuf(vg, st->ptr + st->len, st->space - st->len);
+		if (VGZ_ObufStorage(sp, vg))
+			return (-1);
 		i = VGZ_Gzip(vg, &dp, &dl, VGZ_FINISH);
-		st->len += dl;
 		sp->obj->len += dl;
-		if (st->len == st->space) {
-			VTAILQ_INSERT_TAIL(&sp->obj->store,
-			    sp->wrk->storage, list);
-			sp->wrk->storage = NULL;
-		}
 	} while (i != Z_STREAM_END);
 	VGZ_Destroy(&vg);
 



From phk at project.varnish-software.com  Fri Jan 21 22:38:09 2011
From: phk at project.varnish-software.com (Poul-Henning Kamp)
Date: Fri, 21 Jan 2011 23:38:09 +0100
Subject: [master] 76c392c Always explicitly call vfp_nop_end() after the
	"real" VFP->end() to tuck in the last storage segment sitting
	on sp->work->storage,
	rather than have this code duplicated througout all the VFP->end()'s
Message-ID: <E1PgPcP-0008Nn-9g@project.varnish-software.com>

commit 76c392cc6670774d6d9fa30b88fcaae2e13122ff
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Fri Jan 21 22:36:47 2011 +0000

    Always explicitly call vfp_nop_end() after the "real" VFP->end()
    to tuck in the last storage segment sitting on sp->work->storage,
    rather than have this code duplicated througout all the VFP->end()'s

diff --git a/bin/varnishd/cache_esi_fetch.c b/bin/varnishd/cache_esi_fetch.c
index e2ebc1d..116b96e 100644
--- a/bin/varnishd/cache_esi_fetch.c
+++ b/bin/varnishd/cache_esi_fetch.c
@@ -292,7 +292,6 @@ printf("BYTES = %d\n", i);
 static int __match_proto__()
 vfp_esi_end(struct sess *sp)
 {
-	struct storage *st;
 	struct vsb *vsb;
 	struct vef_priv *vef;
 	ssize_t l;
@@ -315,9 +314,6 @@ printf("END\n");
 		vsb_delete(vsb);
 	}
 
-	st = sp->wrk->storage;
-	sp->wrk->storage = NULL;
-
 	if (sp->wrk->vef_priv != NULL) {
 		vef = sp->wrk->vef_priv;
 		sp->wrk->vef_priv = NULL;
@@ -326,17 +322,6 @@ printf("END\n");
 printf("TOT %jd\n", vef->tot);
 sp->obj->len = vef->tot;
 	}
-
-	if (st == NULL)
-		return (0);
-
-	if (st->len == 0) {
-		STV_free(st);
-		return (0);
-	}
-	if (st->len < st->space)
-		STV_trim(st, st->len);
-	VTAILQ_INSERT_TAIL(&sp->obj->store, st, list);
 	return (0);
 }
 
diff --git a/bin/varnishd/cache_fetch.c b/bin/varnishd/cache_fetch.c
index d28dd5d..1e8c337 100644
--- a/bin/varnishd/cache_fetch.c
+++ b/bin/varnishd/cache_fetch.c
@@ -533,6 +533,12 @@ FetchBody(struct sess *sp)
 		INCOMPL();
 	}
 	XXXAZ(sp->wrk->vfp->end(sp));
+	/*
+	 * It is OK for ->end to just leave the last storage segment
+	 * sitting on sp->wrk->storage, we will always call vfp_nop_end()
+	 * to get it trimmed and added to the object.
+	 */
+	XXXAZ(vfp_nop_end(sp));
 	AZ(sp->wrk->storage);
 
 	WSL(sp->wrk, SLT_Fetch_Body, sp->vbc->fd, "%u %d %u",
diff --git a/bin/varnishd/cache_gzip.c b/bin/varnishd/cache_gzip.c
index 8d0980c..08df961 100644
--- a/bin/varnishd/cache_gzip.c
+++ b/bin/varnishd/cache_gzip.c
@@ -390,24 +390,10 @@ static int __match_proto__()
 vfp_gunzip_end(struct sess *sp)
 {
 	struct vgz *vg;
-	struct storage *st;
 
 	vg = sp->wrk->vgz_rx;
 	CHECK_OBJ_NOTNULL(vg, VGZ_MAGIC);
 	VGZ_Destroy(&vg);
-
-	st = sp->wrk->storage;
-	sp->wrk->storage = NULL;
-	if (st == NULL)
-		return (0);
-
-	if (st->len == 0) {
-		STV_free(st);
-		return (0);
-	}
-	if (st->len < st->space)
-		STV_trim(st, st->len);
-	VTAILQ_INSERT_TAIL(&sp->obj->store, st, list);
 	return (0);
 }
 
@@ -472,7 +458,6 @@ vfp_gzip_end(struct sess *sp)
 	size_t dl;
 	const void *dp;
 	int i;
-	struct storage *st;
 
 	vg = sp->wrk->vgz_rx;
 	CHECK_OBJ_NOTNULL(vg, VGZ_MAGIC);
@@ -484,15 +469,6 @@ vfp_gzip_end(struct sess *sp)
 		sp->obj->len += dl;
 	} while (i != Z_STREAM_END);
 	VGZ_Destroy(&vg);
-
-	st = sp->wrk->storage;
-	sp->wrk->storage = NULL;
-	if (st != NULL && st->len == 0) {
-		STV_free(st);
-	} else if (st != NULL && st->len < st->space) {
-		STV_trim(st, st->len);
-		VTAILQ_INSERT_TAIL(&sp->obj->store, st, list);
-	}
 	return (0);
 }
 



From phk at project.varnish-software.com  Sat Jan 22 07:09:49 2011
From: phk at project.varnish-software.com (Poul-Henning Kamp)
Date: Sat, 22 Jan 2011 08:09:49 +0100
Subject: [master] 92d43d1 Add FetchStorage() which will feed storage segments
	into objects via sp->wrk->storage.
Message-ID: <E1PgXbZ-0002uM-99@project.varnish-software.com>

commit 92d43d17feadf595077c78d4b20623758e12b7af
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sat Jan 22 07:09:03 2011 +0000

    Add FetchStorage() which will feed storage segments into objects
    via sp->wrk->storage.

diff --git a/bin/varnishd/cache.h b/bin/varnishd/cache.h
index 9cacf45..c5359e0 100644
--- a/bin/varnishd/cache.h
+++ b/bin/varnishd/cache.h
@@ -619,6 +619,7 @@ void EXP_Touch(struct object *o, double tnow);
 int EXP_NukeOne(const struct sess *sp, const struct lru *lru);
 
 /* cache_fetch.c */
+int FetchStorage(const struct sess *sp);
 int FetchHdr(struct sess *sp);
 int FetchBody(struct sess *sp);
 int FetchReqBody(struct sess *sp);
diff --git a/bin/varnishd/cache_esi_fetch.c b/bin/varnishd/cache_esi_fetch.c
index 116b96e..ef99413 100644
--- a/bin/varnishd/cache_esi_fetch.c
+++ b/bin/varnishd/cache_esi_fetch.c
@@ -55,14 +55,8 @@ vfp_esi_bytes_uu(struct sess *sp, struct http_conn *htc, size_t bytes)
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 
 	while (bytes > 0) {
-		if (sp->wrk->storage == NULL) {
-			l = params->fetch_chunksize * 1024LL;
-			sp->wrk->storage = STV_alloc(sp, l);
-		}
-		if (sp->wrk->storage == NULL) {
-			errno = ENOMEM;
+		if (FetchStorage(sp))
 			return (-1);
-		}
 		st = sp->wrk->storage;
 		l = st->space - st->len;
 		if (l > bytes)
@@ -83,12 +77,6 @@ vfp_esi_bytes_uu(struct sess *sp, struct http_conn *htc, size_t bytes)
 			VEP_parse(sp, (const char *)st->ptr + st->len, w);
 		st->len += w;
 		sp->obj->len += w;
-		if (st->len == st->space) {
-			VTAILQ_INSERT_TAIL(&sp->obj->store,
-			    sp->wrk->storage, list);
-			sp->wrk->storage = NULL;
-			st = NULL;
-		}
 		bytes -= w;
 	}
 	return (1);
@@ -205,14 +193,6 @@ vfp_esi_bytes_ug(struct sess *sp, struct http_conn *htc, size_t bytes)
 	CHECK_OBJ_NOTNULL(vef, VEF_MAGIC);
 
 	while (bytes > 0) {
-		if (sp->wrk->storage == NULL) {
-			l = params->fetch_chunksize * 1024LL;
-			sp->wrk->storage = STV_alloc(sp, l);
-		}
-		if (sp->wrk->storage == NULL) {
-			errno = ENOMEM;
-			return (-1);
-		}
 		l = sizeof ibuf;
 		if (l > bytes)
 			l = bytes;
diff --git a/bin/varnishd/cache_fetch.c b/bin/varnishd/cache_fetch.c
index 1e8c337..b7f798f 100644
--- a/bin/varnishd/cache_fetch.c
+++ b/bin/varnishd/cache_fetch.c
@@ -91,16 +91,8 @@ vfp_nop_bytes(struct sess *sp, struct http_conn *htc, size_t bytes)
 	struct storage *st;
 
 	while (bytes > 0) {
-		if (sp->wrk->storage == NULL) {
-			l = fetchfrag;
-			if (l == 0)
-				l = params->fetch_chunksize * 1024LL;
-			sp->wrk->storage = STV_alloc(sp, l);
-		}
-		if (sp->wrk->storage == NULL) {
-			errno = ENOMEM;
+		if (FetchStorage(sp)) 
 			return (-1);
-		}
 		st = sp->wrk->storage;
 		l = st->space - st->len;
 		if (l > bytes)
@@ -110,12 +102,6 @@ vfp_nop_bytes(struct sess *sp, struct http_conn *htc, size_t bytes)
 			return (w);
 		st->len += w;
 		sp->obj->len += w;
-		if (st->len == st->space) {
-			VTAILQ_INSERT_TAIL(&sp->obj->store,
-			    sp->wrk->storage, list);
-			sp->wrk->storage = NULL;
-			st = NULL;
-		}
 		bytes -= w;
 	}
 	return (1);
@@ -157,6 +143,33 @@ static struct vfp vfp_nop = {
 };
 
 /*--------------------------------------------------------------------
+ * Fetch Storage
+ */
+
+int
+FetchStorage(const struct sess *sp)
+{
+	ssize_t l;
+
+	if (sp->wrk->storage != NULL &&
+	    sp->wrk->storage->len == sp->wrk->storage->space) {
+		VTAILQ_INSERT_TAIL(&sp->obj->store, sp->wrk->storage, list);
+		sp->wrk->storage = NULL;
+	}
+	if (sp->wrk->storage == NULL) {
+		l = fetchfrag;
+		if (l == 0)
+			l = params->fetch_chunksize * 1024LL;
+		sp->wrk->storage = STV_alloc(sp, l);
+	}
+	if (sp->wrk->storage == NULL) {
+		errno = ENOMEM;
+		return (-1);
+	}
+	return (0);
+}
+
+/*--------------------------------------------------------------------
  * Convert a string to a size_t safely
  */
 
diff --git a/bin/varnishd/cache_gzip.c b/bin/varnishd/cache_gzip.c
index 08df961..f77157c 100644
--- a/bin/varnishd/cache_gzip.c
+++ b/bin/varnishd/cache_gzip.c
@@ -231,21 +231,10 @@ VGZ_ObufStorage(const struct sess *sp, struct vgz *vg)
 {
 	struct storage *st;
 
+	if (FetchStorage(sp)) 
+		return (-1);
+
 	st = sp->wrk->storage;
-	if (st != NULL && st->len == st->space) {
-		VTAILQ_INSERT_TAIL(&sp->obj->store, st, list);
-		sp->wrk->storage = NULL;
-		st = NULL;
-		vg->obuf = NULL;
-	}
-	if (st == NULL) { 
-		st = STV_alloc(sp, params->fetch_chunksize * 1024LL);
-		if (st == NULL) {
-			errno = ENOMEM;
-			return (-1);
-		}
-		sp->wrk->storage = st;
-	}
 	vg->obuf = st;
 	VGZ_Obuf(vg, st->ptr + st->len, st->space - st->len);
 



From phk at project.varnish-software.com  Sat Jan 22 07:37:34 2011
From: phk at project.varnish-software.com (Poul-Henning Kamp)
Date: Sat, 22 Jan 2011 08:37:34 +0100
Subject: [master] 2700f6c A bit of cleanup
Message-ID: <E1PgY2Q-0008Gn-AR@project.varnish-software.com>

commit 2700f6cd8488f8c81c1ae740818a22e728c2cfa3
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sat Jan 22 07:37:23 2011 +0000

    A bit of cleanup

diff --git a/bin/varnishd/cache.h b/bin/varnishd/cache.h
index c5359e0..466b0aa 100644
--- a/bin/varnishd/cache.h
+++ b/bin/varnishd/cache.h
@@ -632,7 +632,7 @@ enum vgz_flag { VGZ_NORMAL, VGZ_ALIGN, VGZ_RESET, VGZ_FINISH };
 struct vgz *VGZ_NewUngzip(const struct sess *sp, struct ws *tmp);
 struct vgz *VGZ_NewGzip(const struct sess *sp, struct ws *tmp);
 void VGZ_Ibuf(struct vgz *, const void *, ssize_t len);
-int VGZ_IbufEmpty(struct vgz *vg);
+int VGZ_IbufEmpty(const struct vgz *vg);
 void VGZ_Obuf(struct vgz *, const void *, ssize_t len);
 int VGZ_ObufStorage(const struct sess *sp, struct vgz *vg);
 int VGZ_Gzip(struct vgz *, const void **, size_t *len, enum vgz_flag);
diff --git a/bin/varnishd/cache_esi_deliver.c b/bin/varnishd/cache_esi_deliver.c
index a17d101..ffd37ea 100644
--- a/bin/varnishd/cache_esi_deliver.c
+++ b/bin/varnishd/cache_esi_deliver.c
@@ -174,8 +174,8 @@ ESI_Deliver(struct sess *sp)
 	struct storage *st;
 	uint8_t *p, *e, *q, *r;
 	unsigned off;
-	ssize_t l, l_icrc, l_crc;
-	uint32_t crc, icrc;
+	ssize_t l, l_icrc, l_crc = 0;
+	uint32_t crc = 0, icrc;
 	uint8_t tailbuf[8 + 5];
 	int dogzip;
 
@@ -208,13 +208,11 @@ ESI_Deliver(struct sess *sp)
 				l_icrc = ved_decode_len(&p);
 				icrc = vbe32dec(p);
 				p += 4;
-			}
-			q = (void*)strchr((const char*)p, '\0');
-			assert (q > p);
-			if (dogzip) {
 				crc = crc32_combine(crc, icrc, l_icrc);
 				l_crc += l_icrc;
 			}
+			q = (void*)strchr((const char*)p, '\0');
+			assert (q > p);
 			ved_sendchunk(sp, p, q - p, st->ptr + off, l);
 			off += l;
 			p = q + 1;
diff --git a/bin/varnishd/cache_esi_fetch.c b/bin/varnishd/cache_esi_fetch.c
index ef99413..20287ef 100644
--- a/bin/varnishd/cache_esi_fetch.c
+++ b/bin/varnishd/cache_esi_fetch.c
@@ -113,6 +113,7 @@ vfp_esi_bytes_gu(struct sess *sp, struct http_conn *htc, size_t bytes)
 		if (VGZ_ObufStorage(sp, vg))
 			return (-1);
 		i = VGZ_Gunzip(vg, &dp, &dl);
+		xxxassert(i == Z_OK || i == Z_STREAM_END);
 		VEP_parse(sp, dp, dl);
 		sp->obj->len += dl;
 	}
@@ -136,8 +137,6 @@ struct vef_priv {
  * We receive a ungzip'ed object, and want to store it gzip'ed.
  */
 
-#include "vend.h"
-
 static ssize_t
 vfp_vep_callback(const struct sess *sp, ssize_t l, enum vgz_flag flg)
 {
@@ -146,8 +145,6 @@ vfp_vep_callback(const struct sess *sp, ssize_t l, enum vgz_flag flg)
 	const void *dp;
 	int i;
 
-printf("ZCB(%jd, %d)\n", l, flg);
-
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 	vef = sp->wrk->vef_priv;
 	CHECK_OBJ_NOTNULL(vef, VEF_MAGIC);
@@ -166,18 +163,14 @@ printf("ZCB(%jd, %d)\n", l, flg);
 			return (vef->tot);
 		}
 		i = VGZ_Gzip(vef->vgz, &dp, &dl, flg);
-printf("GZI = %d %jd\n", i, dl);
 		vef->tot += dl;
 		sp->obj->len += dl;
 	} while (!VGZ_IbufEmpty(vef->vgz));
 	vef->bufp += l;
-if (flg == VGZ_FINISH)
-	assert(i == 1);			/* XXX */
-else
-	assert(i == 0);			/* XXX */
-printf("ZCB = %jd\n", vef->tot);
-fflush(stdout);
-usleep(100);
+	if (flg == VGZ_FINISH)
+		assert(i == 1);			/* XXX */
+	else
+		assert(i == 0);			/* XXX */
 	return (vef->tot);
 }
 
@@ -256,7 +249,6 @@ vfp_esi_bytes(struct sess *sp, struct http_conn *htc, size_t bytes)
 	int i;
 
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
-printf("BYTES = %jd\n", bytes);
 	if (sp->wrk->is_gzip && sp->wrk->do_gunzip)
 		i = vfp_esi_bytes_gu(sp, htc, bytes);
 	else if (sp->wrk->is_gunzip && sp->wrk->do_gzip)
@@ -265,7 +257,6 @@ printf("BYTES = %jd\n", bytes);
 		i = vfp_esi_bytes_gg(sp, htc, bytes);
 	else
 		i = vfp_esi_bytes_uu(sp, htc, bytes);
-printf("BYTES = %d\n", i);
 	return (i);
 }
 
@@ -276,7 +267,6 @@ vfp_esi_end(struct sess *sp)
 	struct vef_priv *vef;
 	ssize_t l;
 
-printf("END\n");
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 	if (sp->wrk->is_gzip && sp->wrk->do_gunzip)
 		VGZ_Destroy(&sp->wrk->vgz_rx);
@@ -299,8 +289,7 @@ printf("END\n");
 		sp->wrk->vef_priv = NULL;
 		CHECK_OBJ_NOTNULL(vef, VEF_MAGIC);
 		XXXAZ(vef->error);
-printf("TOT %jd\n", vef->tot);
-sp->obj->len = vef->tot;
+		// sp->obj->len = vef->tot;
 	}
 	return (0);
 }
diff --git a/bin/varnishd/cache_gzip.c b/bin/varnishd/cache_gzip.c
index f77157c..be05a68 100644
--- a/bin/varnishd/cache_gzip.c
+++ b/bin/varnishd/cache_gzip.c
@@ -202,7 +202,7 @@ VGZ_Ibuf(struct vgz *vg, const void *ptr, ssize_t len)
 }
 
 int
-VGZ_IbufEmpty(struct vgz *vg)
+VGZ_IbufEmpty(const struct vgz *vg)
 {
 
 	CHECK_OBJ_NOTNULL(vg, VGZ_MAGIC);



From phk at project.varnish-software.com  Sat Jan 22 08:04:29 2011
From: phk at project.varnish-software.com (Poul-Henning Kamp)
Date: Sat, 22 Jan 2011 09:04:29 +0100
Subject: [master] a39ba3d Make the esi-parse torture logic part of the
	esi_parser code so it applies to all fetch methods.
Message-ID: <E1PgYST-00030X-U2@project.varnish-software.com>

commit a39ba3d88902233040e6dd67f4ca0e81b6aed263
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sat Jan 22 08:03:50 2011 +0000

    Make the esi-parse torture logic part of the esi_parser code
    so it applies to all fetch methods.

diff --git a/bin/varnishd/cache_esi_deliver.c b/bin/varnishd/cache_esi_deliver.c
index ffd37ea..c60f6da 100644
--- a/bin/varnishd/cache_esi_deliver.c
+++ b/bin/varnishd/cache_esi_deliver.c
@@ -259,4 +259,3 @@ ESI_Deliver(struct sess *sp)
 	}
 	(void)WRW_Flush(sp->wrk);
 }
-
diff --git a/bin/varnishd/cache_esi_fetch.c b/bin/varnishd/cache_esi_fetch.c
index 20287ef..38102bf 100644
--- a/bin/varnishd/cache_esi_fetch.c
+++ b/bin/varnishd/cache_esi_fetch.c
@@ -64,17 +64,7 @@ vfp_esi_bytes_uu(struct sess *sp, struct http_conn *htc, size_t bytes)
 		w = HTC_Read(htc, st->ptr + st->len, l);
 		if (w <= 0)
 			return (w);
-		if (params->esi_syntax & 0x8) {
-			ssize_t d;
-			for (l = 0; l < w; l += d)  {
-				d = (random() & 3) + 1;
-				if (l + d >= w)
-					d = 1;
-				VEP_parse(sp,
-				    (const char *)st->ptr + st->len + l, d);
-			}
-		} else
-			VEP_parse(sp, (const char *)st->ptr + st->len, w);
+		VEP_parse(sp, (const char *)st->ptr + st->len, w);
 		st->len += w;
 		sp->obj->len += w;
 		bytes -= w;
diff --git a/bin/varnishd/cache_esi_parse.c b/bin/varnishd/cache_esi_parse.c
index 5e85eff..ed04ec1 100644
--- a/bin/varnishd/cache_esi_parse.c
+++ b/bin/varnishd/cache_esi_parse.c
@@ -539,8 +539,8 @@ vep_do_include(struct vep_state *vep, enum dowhat what)
  * NB: the state-machine.  Please maintain it along with the code.
  */
 
-void
-VEP_parse(const struct sess *sp, const char *p, size_t l)
+static void
+vep_parse_int(const struct sess *sp, const char *p, size_t l)
 {
 	struct vep_state *vep;
 	const char *e;
@@ -965,6 +965,21 @@ VEP_parse(const struct sess *sp, const char *p, size_t l)
 		vep_mark_pending(vep, p);
 }
 
+void
+VEP_parse(const struct sess *sp, const char *p, size_t w)
+{
+	ssize_t l, d;
+
+	if (params->esi_syntax & 0x8) {
+		for (l = 0; l < w; l += d)  {
+			d = (random() & 3) + 1;
+			if (l + d >= w)
+				d = 1;
+			vep_parse_int(sp, p + l, d);
+		}
+	} else
+		vep_parse_int(sp, p, w);
+}
 
 /*---------------------------------------------------------------------
  */



From phk at project.varnish-software.com  Sat Jan 22 08:57:03 2011
From: phk at project.varnish-software.com (Poul-Henning Kamp)
Date: Sat, 22 Jan 2011 09:57:03 +0100
Subject: [master] ea19d46 Fix the max-value of the vfp->bytes() length
	argument.
Message-ID: <E1PgZHL-0001Qe-HY@project.varnish-software.com>

commit ea19d46387e62ef3bb4015580bee99d5e75a3930
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sat Jan 22 08:55:51 2011 +0000

    Fix the max-value of the vfp->bytes() length argument.
    
    Add a vef_read() function which implements the esi_syntax&8 bit,
    it is too late to do it when we call VEP_parse().

diff --git a/bin/varnishd/cache.h b/bin/varnishd/cache.h
index 466b0aa..b3b49d9 100644
--- a/bin/varnishd/cache.h
+++ b/bin/varnishd/cache.h
@@ -214,7 +214,7 @@ struct dstat {
 /* Fetch processors --------------------------------------------------*/
 
 typedef void vfp_begin_f(struct sess *, size_t );
-typedef int vfp_bytes_f(struct sess *, struct http_conn *, size_t);
+typedef int vfp_bytes_f(struct sess *, struct http_conn *, ssize_t);
 typedef int vfp_end_f(struct sess *sp);
 
 struct vfp {
diff --git a/bin/varnishd/cache_esi_fetch.c b/bin/varnishd/cache_esi_fetch.c
index 38102bf..96ae56f 100644
--- a/bin/varnishd/cache_esi_fetch.c
+++ b/bin/varnishd/cache_esi_fetch.c
@@ -43,13 +43,35 @@ SVNID("$Id")
 #include "stevedore.h"
 
 /*---------------------------------------------------------------------
+ * Read some bytes.
+ *
+ * If the esi_syntax&8 bit is set, we read only a couple of bytes at
+ * a time, in order to stress the parse/pending/callback code.
+ */
+
+static ssize_t
+vef_read(struct http_conn *htc, void *buf, ssize_t buflen, ssize_t bytes)
+{
+	ssize_t d;
+
+	if (buflen < bytes)
+		bytes = buflen;
+	if (params->esi_syntax & 0x8) {
+		d = (random() & 3) + 1;
+		if (d < bytes)
+			bytes = d;
+	}
+	return (HTC_Read(htc, buf, bytes));
+}
+
+/*---------------------------------------------------------------------
  * We receive a ungzip'ed object, and want to store it ungzip'ed.
  */
 
 static int __match_proto__()
-vfp_esi_bytes_uu(struct sess *sp, struct http_conn *htc, size_t bytes)
+vfp_esi_bytes_uu(struct sess *sp, struct http_conn *htc, ssize_t bytes)
 {
-	ssize_t l, w;
+	ssize_t w;
 	struct storage *st;
 
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
@@ -58,10 +80,8 @@ vfp_esi_bytes_uu(struct sess *sp, struct http_conn *htc, size_t bytes)
 		if (FetchStorage(sp))
 			return (-1);
 		st = sp->wrk->storage;
-		l = st->space - st->len;
-		if (l > bytes)
-			l = bytes;
-		w = HTC_Read(htc, st->ptr + st->len, l);
+		w = vef_read(htc,
+		    st->ptr + st->len, st->space - st->len, bytes);
 		if (w <= 0)
 			return (w);
 		VEP_parse(sp, (const char *)st->ptr + st->len, w);
@@ -77,10 +97,10 @@ vfp_esi_bytes_uu(struct sess *sp, struct http_conn *htc, size_t bytes)
  */
 
 static int __match_proto__()
-vfp_esi_bytes_gu(struct sess *sp, struct http_conn *htc, size_t bytes)
+vfp_esi_bytes_gu(struct sess *sp, struct http_conn *htc, ssize_t bytes)
 {
 	struct vgz *vg;
-	ssize_t l, w;
+	ssize_t w;
 	uint8_t	ibuf[1024 * params->gzip_stack_buffer];
 	int i;
 	size_t dl;
@@ -91,10 +111,7 @@ vfp_esi_bytes_gu(struct sess *sp, struct http_conn *htc, size_t bytes)
 
 	while (bytes > 0) {
 		if (VGZ_IbufEmpty(vg) && bytes > 0) {
-			l = sizeof ibuf;
-			if (l > bytes)
-				l = bytes;
-			w = HTC_Read(htc, ibuf, l);
+			w = vef_read(htc, ibuf, sizeof ibuf, bytes);
 			if (w <= 0)
 				return (w);
 			VGZ_Ibuf(vg, ibuf, w);
@@ -145,6 +162,11 @@ vfp_vep_callback(const struct sess *sp, ssize_t l, enum vgz_flag flg)
 		return (vef->tot);
 	}
 
+	/* This would just give Z_BUF_ERROR anyway */
+	if (l == 0 && flg == VGZ_NORMAL)
+		return (vef->tot);
+
+printf("xxC %jd\n", l);
 	VGZ_Ibuf(vef->vgz, vef->bufp, l);
 	do {
 		if (VGZ_ObufStorage(sp, vef->vgz)) {
@@ -165,9 +187,9 @@ vfp_vep_callback(const struct sess *sp, ssize_t l, enum vgz_flag flg)
 }
 
 static int __match_proto__()
-vfp_esi_bytes_ug(struct sess *sp, struct http_conn *htc, size_t bytes)
+vfp_esi_bytes_ug(struct sess *sp, struct http_conn *htc, ssize_t bytes)
 {
-	ssize_t l, w;
+	ssize_t w;
 	char ibuf[1024 * params->gzip_stack_buffer];
 	struct vef_priv *vef;
 
@@ -176,19 +198,18 @@ vfp_esi_bytes_ug(struct sess *sp, struct http_conn *htc, size_t bytes)
 	CHECK_OBJ_NOTNULL(vef, VEF_MAGIC);
 
 	while (bytes > 0) {
-		l = sizeof ibuf;
-		if (l > bytes)
-			l = bytes;
-		w = HTC_Read(htc, ibuf, l);
+		w = vef_read(htc, ibuf, sizeof ibuf, bytes);
 		if (w <= 0)
 			return (w);
 		bytes -= w;
 		vef->bufp = ibuf;
-		VEP_parse(sp, ibuf, l);
+printf("xxP %jd\n", w);
+		VEP_parse(sp, ibuf, w);
 		if (vef->error) {
 			errno = vef->error;
 			return (-1);
 		}
+		assert(vef->bufp == ibuf + w);
 	}
 	return (1);
 }
@@ -234,7 +255,7 @@ vfp_esi_begin(struct sess *sp, size_t estimate)
 }
 
 static int __match_proto__()
-vfp_esi_bytes(struct sess *sp, struct http_conn *htc, size_t bytes)
+vfp_esi_bytes(struct sess *sp, struct http_conn *htc, ssize_t bytes)
 {
 	int i;
 
diff --git a/bin/varnishd/cache_esi_parse.c b/bin/varnishd/cache_esi_parse.c
index ed04ec1..93522d2 100644
--- a/bin/varnishd/cache_esi_parse.c
+++ b/bin/varnishd/cache_esi_parse.c
@@ -539,8 +539,8 @@ vep_do_include(struct vep_state *vep, enum dowhat what)
  * NB: the state-machine.  Please maintain it along with the code.
  */
 
-static void
-vep_parse_int(const struct sess *sp, const char *p, size_t l)
+void
+VEP_parse(const struct sess *sp, const char *p, size_t l)
 {
 	struct vep_state *vep;
 	const char *e;
@@ -965,22 +965,6 @@ vep_parse_int(const struct sess *sp, const char *p, size_t l)
 		vep_mark_pending(vep, p);
 }
 
-void
-VEP_parse(const struct sess *sp, const char *p, size_t w)
-{
-	ssize_t l, d;
-
-	if (params->esi_syntax & 0x8) {
-		for (l = 0; l < w; l += d)  {
-			d = (random() & 3) + 1;
-			if (l + d >= w)
-				d = 1;
-			vep_parse_int(sp, p + l, d);
-		}
-	} else
-		vep_parse_int(sp, p, w);
-}
-
 /*---------------------------------------------------------------------
  */
 
diff --git a/bin/varnishd/cache_fetch.c b/bin/varnishd/cache_fetch.c
index b7f798f..a2a5ba0 100644
--- a/bin/varnishd/cache_fetch.c
+++ b/bin/varnishd/cache_fetch.c
@@ -85,7 +85,7 @@ vfp_nop_begin(struct sess *sp, size_t estimate)
  */
 
 static int __match_proto__()
-vfp_nop_bytes(struct sess *sp, struct http_conn *htc, size_t bytes)
+vfp_nop_bytes(struct sess *sp, struct http_conn *htc, ssize_t bytes)
 {
 	ssize_t l, w;
 	struct storage *st;
@@ -314,10 +314,10 @@ fetch_eof(struct sess *sp, struct http_conn *htc)
 	assert(sp->wrk->body_status == BS_EOF);
 	sp->wrk->vfp->begin(sp, 0);
 	i = sp->wrk->vfp->bytes(sp, htc,
-#ifdef SIZE_T_MAX
-	    SIZE_T_MAX
+#ifdef SSIZE_T_MAX
+	    SSIZE_T_MAX
 #else
-	    sizeof(void*) == 4 ? ((size_t)1<<31) : ((size_t)1 << 63)
+	    sizeof(size_t) == 4 ? ((size_t)1<<30) : ((size_t)1 << 62)
 #endif
 	    ); 
 	if (i < 0) {
diff --git a/bin/varnishd/cache_gzip.c b/bin/varnishd/cache_gzip.c
index be05a68..747a41a 100644
--- a/bin/varnishd/cache_gzip.c
+++ b/bin/varnishd/cache_gzip.c
@@ -340,7 +340,7 @@ vfp_gunzip_begin(struct sess *sp, size_t estimate)
 }
 
 static int __match_proto__()
-vfp_gunzip_bytes(struct sess *sp, struct http_conn *htc, size_t bytes)
+vfp_gunzip_bytes(struct sess *sp, struct http_conn *htc, ssize_t bytes)
 {
 	struct vgz *vg;
 	ssize_t l, w;
@@ -408,7 +408,7 @@ vfp_gzip_begin(struct sess *sp, size_t estimate)
 }
 
 static int __match_proto__()
-vfp_gzip_bytes(struct sess *sp, struct http_conn *htc, size_t bytes)
+vfp_gzip_bytes(struct sess *sp, struct http_conn *htc, ssize_t bytes)
 {
 	struct vgz *vg;
 	ssize_t l, w;



From phk at project.varnish-software.com  Sat Jan 22 09:33:57 2011
From: phk at project.varnish-software.com (Poul-Henning Kamp)
Date: Sat, 22 Jan 2011 10:33:57 +0100
Subject: [master] 6648371 Buffer the pending bytes until we need them.
Message-ID: <E1PgZr3-00051z-65@project.varnish-software.com>

commit 66483710fa6d59a01c4064e874564f10c1c0ae14
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sat Jan 22 09:33:41 2011 +0000

    Buffer the pending bytes until we need them.

diff --git a/bin/varnishd/cache_esi_fetch.c b/bin/varnishd/cache_esi_fetch.c
index 96ae56f..7a76313 100644
--- a/bin/varnishd/cache_esi_fetch.c
+++ b/bin/varnishd/cache_esi_fetch.c
@@ -138,6 +138,8 @@ struct vef_priv {
 	char			*bufp;
 	ssize_t			tot;
 	int			error;
+	char			pending[20];
+	ssize_t			npend;
 };
 
 /*---------------------------------------------------------------------
@@ -148,7 +150,7 @@ static ssize_t
 vfp_vep_callback(const struct sess *sp, ssize_t l, enum vgz_flag flg)
 {
 	struct vef_priv *vef;
-	size_t dl;
+	size_t dl, px;
 	const void *dp;
 	int i;
 
@@ -162,23 +164,42 @@ vfp_vep_callback(const struct sess *sp, ssize_t l, enum vgz_flag flg)
 		return (vef->tot);
 	}
 
-	/* This would just give Z_BUF_ERROR anyway */
+	/*
+	 * l == 0 is valid when 'flg' calls for action, but in the
+	 * normal case we can just ignore a l==0 request.
+	 * (It would cause Z_BUF_ERROR anyway)
+	 */
 	if (l == 0 && flg == VGZ_NORMAL)
 		return (vef->tot);
 
-printf("xxC %jd\n", l);
-	VGZ_Ibuf(vef->vgz, vef->bufp, l);
 	do {
-		if (VGZ_ObufStorage(sp, vef->vgz)) {
-			vef->error = ENOMEM;
-			vef->tot += l;
-			return (vef->tot);
+		px = vef->npend;
+		if (l < px)
+			px = l;
+		if (px != 0) {
+			VGZ_Ibuf(vef->vgz, vef->pending, px);
+			l -= px;
+		} else {
+			VGZ_Ibuf(vef->vgz, vef->bufp, l);
+			vef->bufp += l;
+			l = 0;
 		}
-		i = VGZ_Gzip(vef->vgz, &dp, &dl, flg);
-		vef->tot += dl;
-		sp->obj->len += dl;
-	} while (!VGZ_IbufEmpty(vef->vgz));
-	vef->bufp += l;
+		do {
+			if (VGZ_ObufStorage(sp, vef->vgz)) {
+				vef->error = errno;
+				vef->tot += l;
+				return (vef->tot);
+			}
+			i = VGZ_Gzip(vef->vgz, &dp, &dl, flg);
+			vef->tot += dl;
+			sp->obj->len += dl;
+		} while (!VGZ_IbufEmpty(vef->vgz));
+		if (px != 0) {
+			memmove(vef->pending, vef->pending + px,
+			    vef->npend - px);
+			vef->npend -= px;
+		}
+	} while (l > 0);
 	if (flg == VGZ_FINISH)
 		assert(i == 1);			/* XXX */
 	else
@@ -203,13 +224,17 @@ vfp_esi_bytes_ug(struct sess *sp, struct http_conn *htc, ssize_t bytes)
 			return (w);
 		bytes -= w;
 		vef->bufp = ibuf;
-printf("xxP %jd\n", w);
 		VEP_parse(sp, ibuf, w);
 		if (vef->error) {
 			errno = vef->error;
 			return (-1);
 		}
-		assert(vef->bufp == ibuf + w);
+		if (vef->bufp < ibuf + w) {
+			w = (ibuf + w) - vef->bufp;
+			assert(w + vef->npend < sizeof vef->pending);
+			memcpy(vef->pending + vef->npend, vef->bufp, w);
+			vef->npend += w;
+		}
 	}
 	return (1);
 }



From phk at project.varnish-software.com  Sat Jan 22 09:33:57 2011
From: phk at project.varnish-software.com (Poul-Henning Kamp)
Date: Sat, 22 Jan 2011 10:33:57 +0100
Subject: [master] ee4a4ee We should be able to rely on POSIX' SSIZE_MAX here.
Message-ID: <E1PgZr3-000527-Iv@project.varnish-software.com>

commit ee4a4eecd86acaa4d39341c8c1592c68734c6400
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sat Jan 22 09:33:18 2011 +0000

    We should be able to rely on POSIX' SSIZE_MAX here.

diff --git a/bin/varnishd/cache_fetch.c b/bin/varnishd/cache_fetch.c
index a2a5ba0..35fed5f 100644
--- a/bin/varnishd/cache_fetch.c
+++ b/bin/varnishd/cache_fetch.c
@@ -313,13 +313,7 @@ fetch_eof(struct sess *sp, struct http_conn *htc)
 
 	assert(sp->wrk->body_status == BS_EOF);
 	sp->wrk->vfp->begin(sp, 0);
-	i = sp->wrk->vfp->bytes(sp, htc,
-#ifdef SSIZE_T_MAX
-	    SSIZE_T_MAX
-#else
-	    sizeof(size_t) == 4 ? ((size_t)1<<30) : ((size_t)1 << 62)
-#endif
-	    ); 
+	i = sp->wrk->vfp->bytes(sp, htc, SSIZE_MAX);
 	if (i < 0) {
 		WSP(sp, SLT_FetchError, "eof read_error: %d (%s)",
 		    errno, strerror(errno));



From phk at project.varnish-software.com  Sat Jan 22 09:33:57 2011
From: phk at project.varnish-software.com (Poul-Henning Kamp)
Date: Sat, 22 Jan 2011 10:33:57 +0100
Subject: [master] f84f6a8 Report the zlib message if we have gunzip trouble
Message-ID: <E1PgZr3-00052K-RG@project.varnish-software.com>

commit f84f6a80979392deaea0a3b7d2327fff8945c4a7
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sat Jan 22 09:32:56 2011 +0000

    Report the zlib message if we have gunzip trouble

diff --git a/bin/varnishtest/vtc_http.c b/bin/varnishtest/vtc_http.c
index c4c4ca9..cf34a4e 100644
--- a/bin/varnishtest/vtc_http.c
+++ b/bin/varnishtest/vtc_http.c
@@ -546,7 +546,7 @@ cmd_http_gunzip_body(CMD_ARGS)
 	assert(Z_OK == inflateInit2(&vz, 31));
 	i = inflate(&vz, Z_FINISH);
 	if (i != Z_STREAM_END)
-		vtc_log(hp->vl, 0, "Gunzip error = %d", i);
+		vtc_log(hp->vl, 0, "Gunzip error = %d (%s)", i, vz.msg);
 	hp->bodyl = vz.total_out;
 	memcpy(hp->body, p, hp->bodyl);
 	free(p);



From phk at project.varnish-software.com  Sat Jan 22 09:33:58 2011
From: phk at project.varnish-software.com (Poul-Henning Kamp)
Date: Sat, 22 Jan 2011 10:33:58 +0100
Subject: [master] 1e10a0d Dump the received chunk by length,
	rather than until NUL
Message-ID: <E1PgZr4-00052X-6i@project.varnish-software.com>

commit 1e10a0df6424d554d9cc0c7f7547cfc07b1ea5ba
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sat Jan 22 09:17:50 2011 +0000

    Dump the received chunk by length, rather than until NUL

diff --git a/bin/varnishtest/vtc_http.c b/bin/varnishtest/vtc_http.c
index 2b05c38..c4c4ca9 100644
--- a/bin/varnishtest/vtc_http.c
+++ b/bin/varnishtest/vtc_http.c
@@ -413,7 +413,7 @@ http_swallow_body(struct http *hp, char * const *hh, int body)
 				ll += i;
 				http_rxchar(hp, i);
 				vtc_dump(hp->vl, 4, "chunk",
-				    hp->rxbuf + l, -1);
+				    hp->rxbuf + l, i);
 			}
 			l = hp->prxbuf;
 			http_rxchar(hp, 2);



From phk at project.varnish-software.com  Sat Jan 22 09:35:39 2011
From: phk at project.varnish-software.com (Poul-Henning Kamp)
Date: Sat, 22 Jan 2011 10:35:39 +0100
Subject: [master] 9c542e1 Force this testcase into the parser/pending/callback
	code
Message-ID: <E1PgZsh-0005KV-6F@project.varnish-software.com>

commit 9c542e1a8d8dfc2265d1698ada571d713a2248af
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sat Jan 22 09:35:19 2011 +0000

    Force this testcase into the parser/pending/callback code

diff --git a/bin/varnishtest/tests/e00021.vtc b/bin/varnishtest/tests/e00021.vtc
index 1debf38..5c9d848 100644
--- a/bin/varnishtest/tests/e00021.vtc
+++ b/bin/varnishtest/tests/e00021.vtc
@@ -25,7 +25,7 @@ varnish v1 -vcl+backend {
 	}
 } -start 
 
-varnish v1 -cliok "param.set esi_syntax 4"
+varnish v1 -cliok "param.set esi_syntax 0xc"
 varnish v1 -cliok "param.set http_gzip_support true"
 
 client c1 {



From phk at project.varnish-software.com  Sat Jan 22 13:01:53 2011
From: phk at project.varnish-software.com (Poul-Henning Kamp)
Date: Sat, 22 Jan 2011 14:01:53 +0100
Subject: [master] 79cfb9c Mark objects with a "gziped" bit if they are.
Message-ID: <E1Pgd6H-0008VR-Vl@project.varnish-software.com>

commit 79cfb9cbf462a8c7ec5b544e54431ca82ad1d4c8
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sat Jan 22 13:01:31 2011 +0000

    Mark objects with a "gziped" bit if they are.

diff --git a/bin/varnishd/cache.h b/bin/varnishd/cache.h
index 0d6b63f..3ea26ca 100644
--- a/bin/varnishd/cache.h
+++ b/bin/varnishd/cache.h
@@ -430,7 +430,9 @@ struct object {
 	double			ban_t;
 	unsigned		response;
 
+	/* XXX: make bitmap */
 	unsigned		cacheable;
+	unsigned		gziped;
 
 	ssize_t			len;
 
diff --git a/bin/varnishd/cache_center.c b/bin/varnishd/cache_center.c
index b55b903..99e5d81 100644
--- a/bin/varnishd/cache_center.c
+++ b/bin/varnishd/cache_center.c
@@ -179,8 +179,7 @@ cnt_deliver(struct sess *sp)
 		sp->wrk->res_mode |= RES_ESI_CHILD;
 	}
 
-	if (params->http_gzip_support &&
-	    http_HdrIs(sp->obj->http, H_Content_Encoding, "gzip") &&
+	if (params->http_gzip_support && sp->obj->gziped &&
 	    !RFC2616_Req_Gzip(sp)) {
 		/*
 		 * We don't know what it uncompresses to
@@ -662,6 +661,7 @@ cnt_fetch(struct sess *sp)
 	CHECK_OBJ_NOTNULL(sp->obj, OBJECT_MAGIC);
 
 	sp->wrk->storage_hint = NULL;
+	sp->obj->gziped = sp->wrk->is_gzip;
 
 	if (vary != NULL) {
 		sp->obj->vary =
diff --git a/bin/varnishd/cache_esi_fetch.c b/bin/varnishd/cache_esi_fetch.c
index e8ccdc8..fe802f1 100644
--- a/bin/varnishd/cache_esi_fetch.c
+++ b/bin/varnishd/cache_esi_fetch.c
@@ -269,6 +269,7 @@ vfp_esi_bytes_gg(struct sess *sp, struct http_conn *htc, size_t bytes)
 		do {
 			VGZ_Obuf(sp->wrk->vgz_rx, ibuf2, sizeof ibuf2);
 			i = VGZ_Gunzip(sp->wrk->vgz_rx, &dp, &dl);
+			/* XXX: check i */
 			vef->bufp = ibuf2;
 			if (dl > 0)
 				VEP_parse(sp, ibuf2, dl);
@@ -373,6 +374,9 @@ vfp_esi_end(struct sess *sp)
 		sp->wrk->vef_priv = NULL;
 		CHECK_OBJ_NOTNULL(vef, VEF_MAGIC);
 		XXXAZ(vef->error);
+		sp->obj->gziped = 1;
+	} else {
+		sp->obj->gziped = 0;
 	}
 	return (0);
 }
diff --git a/bin/varnishd/cache_gzip.c b/bin/varnishd/cache_gzip.c
index 747a41a..189b62e 100644
--- a/bin/varnishd/cache_gzip.c
+++ b/bin/varnishd/cache_gzip.c
@@ -383,6 +383,7 @@ vfp_gunzip_end(struct sess *sp)
 	vg = sp->wrk->vgz_rx;
 	CHECK_OBJ_NOTNULL(vg, VGZ_MAGIC);
 	VGZ_Destroy(&vg);
+	sp->obj->gziped = 0;
 	return (0);
 }
 
@@ -458,6 +459,7 @@ vfp_gzip_end(struct sess *sp)
 		sp->obj->len += dl;
 	} while (i != Z_STREAM_END);
 	VGZ_Destroy(&vg);
+	sp->obj->gziped = 1;
 	return (0);
 }
 



From phk at project.varnish-software.com  Sat Jan 22 13:01:54 2011
From: phk at project.varnish-software.com (Poul-Henning Kamp)
Date: Sat, 22 Jan 2011 14:01:54 +0100
Subject: [master] 181a049 Make the object length a ssize_t
Message-ID: <E1Pgd6I-0008Ve-Qn@project.varnish-software.com>

commit 181a049692fa6cb8340b09e0a853d34cdac53d66
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sat Jan 22 12:32:11 2011 +0000

    Make the object length a ssize_t

diff --git a/bin/varnishd/cache.h b/bin/varnishd/cache.h
index c378f85..0d6b63f 100644
--- a/bin/varnishd/cache.h
+++ b/bin/varnishd/cache.h
@@ -432,7 +432,7 @@ struct object {
 
 	unsigned		cacheable;
 
-	unsigned		len;
+	ssize_t			len;
 
 	double			ttl;
 	double			age;
diff --git a/bin/varnishd/cache_response.c b/bin/varnishd/cache_response.c
index 0b7e713..31818e8 100644
--- a/bin/varnishd/cache_response.c
+++ b/bin/varnishd/cache_response.c
@@ -297,7 +297,7 @@ res_WriteDirObj(struct sess *sp, char lenbuf[20], size_t low, size_t high)
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 
 	if (sp->wrk->res_mode & RES_CHUNKED) {
-		bprintf(lenbuf, "%x\r\n", sp->obj->len);
+		bprintf(lenbuf, "%jx\r\n", (intmax_t)sp->obj->len);
 		(void)WRW_Write(sp->wrk, lenbuf, -1);
 	}
 



From phk at project.varnish-software.com  Sat Jan 22 13:01:55 2011
From: phk at project.varnish-software.com (Poul-Henning Kamp)
Date: Sat, 22 Jan 2011 14:01:55 +0100
Subject: [master] ae57a6b Make the variables that control gzip'ed ESI deliver
	to sp->wrk so included files can get at them.
Message-ID: <E1Pgd6J-0008Vw-8a@project.varnish-software.com>

commit ae57a6bd17549425ce847b4c3ec7012803e71289
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sat Jan 22 12:26:57 2011 +0000

    Make the variables that control gzip'ed ESI deliver to sp->wrk so
    included files can get at them.

diff --git a/bin/varnishd/cache.h b/bin/varnishd/cache.h
index b3b49d9..c378f85 100644
--- a/bin/varnishd/cache.h
+++ b/bin/varnishd/cache.h
@@ -290,6 +290,9 @@ struct worker {
 
 	/* ESI stuff */
 	struct vep_state	*vep;
+	int			gzip_resp;
+	ssize_t			l_crc;
+	uint32_t		crc;
 
 	/* Timeouts */
 	double			connect_timeout;
diff --git a/bin/varnishd/cache_esi_deliver.c b/bin/varnishd/cache_esi_deliver.c
index c60f6da..e5209e3 100644
--- a/bin/varnishd/cache_esi_deliver.c
+++ b/bin/varnishd/cache_esi_deliver.c
@@ -174,10 +174,9 @@ ESI_Deliver(struct sess *sp)
 	struct storage *st;
 	uint8_t *p, *e, *q, *r;
 	unsigned off;
-	ssize_t l, l_icrc, l_crc = 0;
-	uint32_t crc = 0, icrc;
+	ssize_t l, l_icrc;
+	uint32_t icrc;
 	uint8_t tailbuf[8 + 5];
-	int dogzip;
 
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 	st = sp->obj->esidata;
@@ -185,13 +184,14 @@ ESI_Deliver(struct sess *sp)
 	p = st->ptr;
 	e = st->ptr + st->len;
 
-	if (*p == VEC_GZ) {
-		p++;	
-		dogzip = 1;
-		crc = crc32(0L, Z_NULL, 0);
-		l_crc = 0;
-	} else 
-		dogzip = 0;
+	if (sp->esi_level == 0) {
+		if (*p == VEC_GZ) {
+			p++;	
+			sp->wrk->gzip_resp = 1;
+			sp->wrk->crc = crc32(0L, Z_NULL, 0);
+		} else 
+			sp->wrk->gzip_resp = 0;
+	}
 
 	st = VTAILQ_FIRST(&sp->obj->store);
 	off = 0;
@@ -202,14 +202,15 @@ ESI_Deliver(struct sess *sp)
 		case VEC_V2:
 		case VEC_V8:
 			l = ved_decode_len(&p);
-			if (dogzip) {
+			if (sp->wrk->gzip_resp) {
 				assert(*p == VEC_C1 || *p == VEC_C2 ||
 				    *p == VEC_C8);
 				l_icrc = ved_decode_len(&p);
 				icrc = vbe32dec(p);
 				p += 4;
-				crc = crc32_combine(crc, icrc, l_icrc);
-				l_crc += l_icrc;
+				sp->wrk->crc =
+				    crc32_combine(sp->wrk->crc, icrc, l_icrc);
+				sp->wrk->l_crc += l_icrc;
 			}
 			q = (void*)strchr((const char*)p, '\0');
 			assert (q > p);
@@ -241,7 +242,7 @@ ESI_Deliver(struct sess *sp)
 			INCOMPL();
 		}
 	}
-	if (dogzip) {
+	if (sp->wrk->gzip_resp && sp->esi_level == 0) {
 		/* Emit a gzip literal block with finish bit set */
 		tailbuf[0] = 0x01;
 		tailbuf[1] = 0x00;
@@ -250,10 +251,10 @@ ESI_Deliver(struct sess *sp)
 		tailbuf[4] = 0xff;
 
 		/* Emit CRC32 */
-		vle32enc(tailbuf + 5, crc);
+		vle32enc(tailbuf + 5, sp->wrk->crc);
 
 		/* MOD(2^32) length */
-		vle32enc(tailbuf + 9, l_crc);
+		vle32enc(tailbuf + 9, sp->wrk->l_crc);
 
 		ved_sendchunk(sp, "d\r\n", 3, tailbuf, 13);
 	}



From phk at project.varnish-software.com  Sat Jan 22 13:01:55 2011
From: phk at project.varnish-software.com (Poul-Henning Kamp)
Date: Sat, 22 Jan 2011 14:01:55 +0100
Subject: [master] f36b3ca Add a gunzip-esi-gzip testcase
Message-ID: <E1Pgd6J-0008WB-Og@project.varnish-software.com>

commit f36b3ca617b3e6ce8dff2906f978bf3376db738b
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sat Jan 22 10:49:00 2011 +0000

    Add a gunzip-esi-gzip testcase

diff --git a/bin/varnishtest/tests/e00022.vtc b/bin/varnishtest/tests/e00022.vtc
new file mode 100644
index 0000000..5634797
--- /dev/null
+++ b/bin/varnishtest/tests/e00022.vtc
@@ -0,0 +1,41 @@
+# $Id$
+
+test "ESI ability to stitch gzip files together"
+
+
+server s1 {
+	rxreq 
+	expect req.http.accept-encoding == gzip
+	txresp -gzipbody {
+		<esi:remove>
+		This is a test: Unseen University
+		<esi:include src="trick question">
+		<!--esi XXX -->
+		</esi:remove>
+		<esX>This is a test: Hello world
+	}
+} -start
+
+varnish v1 -vcl+backend {
+	sub vcl_recv {
+		set req.esi = true;
+	}
+	sub vcl_fetch {
+		set beresp.do_esi = true;
+	}
+} -start 
+
+varnish v1 -cliok "param.set esi_syntax 0xc"
+varnish v1 -cliok "param.set http_gzip_support true"
+
+client c1 {
+	txreq  -hdr "Accept-Encoding: gzip"
+	rxresp
+	expect resp.http.content-encoding == gzip
+	gunzip
+	expect resp.status == 200
+	expect resp.bodylen == 40
+}
+
+client c1 -run
+varnish v1 -expect esi_errors == 2



From phk at project.varnish-software.com  Sat Jan 22 13:01:56 2011
From: phk at project.varnish-software.com (Poul-Henning Kamp)
Date: Sat, 22 Jan 2011 14:01:56 +0100
Subject: [master] 94459d4 Handle the case where we receive gziped from backend,
	ungzip so we can do ESI, then gzip again to store it.
Message-ID: <E1Pgd6K-0008WO-6q@project.varnish-software.com>

commit 94459d4ab4cb9c33626b377f45dcbbec578a25c5
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sat Jan 22 10:48:10 2011 +0000

    Handle the case where we receive gziped from backend, ungzip
    so we can do ESI, then gzip again to store it.

diff --git a/bin/varnishd/cache_esi_fetch.c b/bin/varnishd/cache_esi_fetch.c
index 7a76313..e8ccdc8 100644
--- a/bin/varnishd/cache_esi_fetch.c
+++ b/bin/varnishd/cache_esi_fetch.c
@@ -232,7 +232,7 @@ vfp_esi_bytes_ug(struct sess *sp, struct http_conn *htc, ssize_t bytes)
 		if (vef->bufp < ibuf + w) {
 			w = (ibuf + w) - vef->bufp;
 			assert(w + vef->npend < sizeof vef->pending);
-			memcpy(vef->pending + vef->npend, vef->bufp, w);
+			memmove(vef->pending + vef->npend, vef->bufp, w);
 			vef->npend += w;
 		}
 	}
@@ -246,7 +246,45 @@ vfp_esi_bytes_ug(struct sess *sp, struct http_conn *htc, ssize_t bytes)
 static int __match_proto__()
 vfp_esi_bytes_gg(struct sess *sp, struct http_conn *htc, size_t bytes)
 {
-	return (vfp_esi_bytes_uu(sp, htc, bytes));
+	ssize_t w;
+	char ibuf[1024 * params->gzip_stack_buffer];
+	char ibuf2[1024 * params->gzip_stack_buffer];
+	struct vef_priv *vef;
+	size_t dl;
+	const void *dp;
+	int i;
+
+	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+	vef = sp->wrk->vef_priv;
+	CHECK_OBJ_NOTNULL(vef, VEF_MAGIC);
+
+	while (bytes > 0) {
+		w = vef_read(htc, ibuf, sizeof ibuf, bytes);
+		if (w <= 0)
+			return (w);
+		bytes -= w;
+		
+		vef->bufp = ibuf;
+		VGZ_Ibuf(sp->wrk->vgz_rx, ibuf, w);
+		do {
+			VGZ_Obuf(sp->wrk->vgz_rx, ibuf2, sizeof ibuf2);
+			i = VGZ_Gunzip(sp->wrk->vgz_rx, &dp, &dl);
+			vef->bufp = ibuf2;
+			if (dl > 0)
+				VEP_parse(sp, ibuf2, dl);
+			if (vef->error) {
+				errno = vef->error;
+				return (-1);
+			}
+			if (vef->bufp < ibuf2 + dl) {
+				dl = (ibuf2 + dl) - vef->bufp;
+				assert(dl + vef->npend < sizeof vef->pending);
+				memmove(vef->pending + vef->npend, vef->bufp, dl);
+				vef->npend += dl;
+			}
+		} while (!VGZ_IbufEmpty(sp->wrk->vgz_rx));
+	}
+	return (1);
 }
 
 
@@ -264,7 +302,16 @@ vfp_esi_begin(struct sess *sp, size_t estimate)
 		sp->wrk->vgz_rx = VGZ_NewUngzip(sp, sp->ws);
 		VEP_Init(sp, NULL);
 	} else if (sp->wrk->is_gunzip && sp->wrk->do_gzip) {
+		vef = (void*)WS_Alloc(sp->ws, sizeof *vef);
+		AN(vef);
+		memset(vef, 0, sizeof *vef);
+		vef->magic = VEF_MAGIC;
+		vef->vgz = VGZ_NewGzip(sp, sp->ws);
+		AZ(sp->wrk->vef_priv);
+		sp->wrk->vef_priv = vef;
 		VEP_Init(sp, vfp_vep_callback);
+	} else if (sp->wrk->is_gzip) {
+		sp->wrk->vgz_rx = VGZ_NewUngzip(sp, sp->ws);
 		vef = (void*)WS_Alloc(sp->ws, sizeof *vef);
 		AN(vef);
 		memset(vef, 0, sizeof *vef);
@@ -272,6 +319,7 @@ vfp_esi_begin(struct sess *sp, size_t estimate)
 		vef->vgz = VGZ_NewGzip(sp, sp->ws);
 		AZ(sp->wrk->vef_priv);
 		sp->wrk->vef_priv = vef;
+		VEP_Init(sp, vfp_vep_callback);
 	} else {
 		VEP_Init(sp, NULL);
 	}
@@ -325,7 +373,6 @@ vfp_esi_end(struct sess *sp)
 		sp->wrk->vef_priv = NULL;
 		CHECK_OBJ_NOTNULL(vef, VEF_MAGIC);
 		XXXAZ(vef->error);
-		// sp->obj->len = vef->tot;
 	}
 	return (0);
 }



From phk at project.varnish-software.com  Sat Jan 22 13:55:38 2011
From: phk at project.varnish-software.com (Poul-Henning Kamp)
Date: Sat, 22 Jan 2011 14:55:38 +0100
Subject: [master] fd08146 Add another easy bit of code: including a ungzip'ed
	non-ESI object in an gzip'ed ESI object.
Message-ID: <E1PgdwI-00062X-Vt@project.varnish-software.com>

commit fd081460f887db22ae90f091010a179744c3bf71
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sat Jan 22 13:55:07 2011 +0000

    Add another easy bit of code:  including a ungzip'ed non-ESI object
    in an gzip'ed ESI object.

diff --git a/bin/varnishd/cache.h b/bin/varnishd/cache.h
index 3ea26ca..a2ed4cc 100644
--- a/bin/varnishd/cache.h
+++ b/bin/varnishd/cache.h
@@ -815,6 +815,7 @@ char *VRT_String(struct ws *ws, const char *h, const char *p, va_list ap);
 char *VRT_StringList(char *d, unsigned dl, const char *p, va_list ap);
 
 void ESI_Deliver(struct sess *);
+void ESI_DeliverChild(struct sess *);
 
 /* cache_vrt_vmod.c */
 void VMOD_Init(void);
diff --git a/bin/varnishd/cache_esi_deliver.c b/bin/varnishd/cache_esi_deliver.c
index ca581f8..a3c9dc5 100644
--- a/bin/varnishd/cache_esi_deliver.c
+++ b/bin/varnishd/cache_esi_deliver.c
@@ -333,3 +333,20 @@ ESI_Deliver(struct sess *sp)
 	}
 	(void)WRW_Flush(sp->wrk);
 }
+
+/*---------------------------------------------------------------------
+ * Include an object in a gzip'ed ESI object delivery
+ */
+
+void
+ESI_DeliverChild(struct sess *sp)
+{
+	struct storage *st;
+
+	if (sp->obj->gziped) {
+		INCOMPL();
+	} else {
+		VTAILQ_FOREACH(st, &sp->obj->store, list)
+			ved_pretend_gzip(sp, st->ptr, st->len);
+	}
+}
diff --git a/bin/varnishd/cache_response.c b/bin/varnishd/cache_response.c
index 31818e8..6347d82 100644
--- a/bin/varnishd/cache_response.c
+++ b/bin/varnishd/cache_response.c
@@ -390,6 +390,8 @@ RES_WriteObj(struct sess *sp)
 		/* Nothing to do here */
 	} else if (sp->wrk->res_mode & RES_ESI) {
 		ESI_Deliver(sp);
+	} else if (sp->wrk->res_mode & RES_ESI_CHILD && sp->wrk->gzip_resp) {
+		ESI_DeliverChild(sp);
 	} else if (sp->wrk->res_mode & RES_GUNZIP) {
 		res_WriteGunzipObj(sp); 
 	} else {
diff --git a/bin/varnishtest/tests/e00023.vtc b/bin/varnishtest/tests/e00023.vtc
index c493fa6..0d4457c 100644
--- a/bin/varnishtest/tests/e00023.vtc
+++ b/bin/varnishtest/tests/e00023.vtc
@@ -1,19 +1,24 @@
 # $Id$
 
-test "Include an ungzip'ed ESI object in a gzip'ed ESI object"
+test "Include an ungzip'ed (ESI) object in a gzip'ed ESI object"
 
 server s1 {
 	rxreq 
 	expect req.http.accept-encoding == gzip
 	txresp -gzipbody {
 		<html>					1
-		Before include				2
+		Before includes				2
 		<esi:include src="/foo"/>		3
-		After include				4
+		Between includes			4
+		<esi:include src="/bar"/>		5
+		After includes				6
 	}
 	rxreq
 	expect req.url == "/foo"
 	txresp -body {<h1>/////<h2>}
+	rxreq
+	expect req.url == "/bar"
+	txresp -body {*****}
 } -start
 
 varnish v1 -vcl+backend {
@@ -33,7 +38,7 @@ client c1 {
 	expect resp.http.content-encoding == gzip
 	gunzip
 	expect resp.status == 200
-	expect resp.bodylen == 79
+	expect resp.bodylen == 115
 }
 
 client c1 -run



From phk at project.varnish-software.com  Sat Jan 22 13:55:39 2011
From: phk at project.varnish-software.com (Poul-Henning Kamp)
Date: Sat, 22 Jan 2011 14:55:39 +0100
Subject: [master] 66e7a25 Add a testcase for including a non-gzip'ed ESI
	object in a gzip'ed ESI object.
Message-ID: <E1PgdwJ-00062j-Pm@project.varnish-software.com>

commit 66e7a250a79e9b26aa64df05dd039401298f216f
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sat Jan 22 13:42:26 2011 +0000

    Add a testcase for including a non-gzip'ed ESI object in a gzip'ed
    ESI object.

diff --git a/bin/varnishtest/tests/e00023.vtc b/bin/varnishtest/tests/e00023.vtc
new file mode 100644
index 0000000..c493fa6
--- /dev/null
+++ b/bin/varnishtest/tests/e00023.vtc
@@ -0,0 +1,40 @@
+# $Id$
+
+test "Include an ungzip'ed ESI object in a gzip'ed ESI object"
+
+server s1 {
+	rxreq 
+	expect req.http.accept-encoding == gzip
+	txresp -gzipbody {
+		<html>					1
+		Before include				2
+		<esi:include src="/foo"/>		3
+		After include				4
+	}
+	rxreq
+	expect req.url == "/foo"
+	txresp -body {<h1>/////<h2>}
+} -start
+
+varnish v1 -vcl+backend {
+	sub vcl_fetch {
+		set beresp.do_esi = true;
+	}
+} -start 
+
+varnish v1 -cliok "param.set http_gzip_support true"
+varnish v1 -cliok "param.set esi_syntax 0x3e" 
+
+varnish v1 -cliok "param.set diag_bitmap 0x10000"
+
+client c1 {
+	txreq -hdr "Accept-Encoding: gzip"
+	rxresp
+	expect resp.http.content-encoding == gzip
+	gunzip
+	expect resp.status == 200
+	expect resp.bodylen == 79
+}
+
+client c1 -run
+varnish v1 -expect esi_errors == 0



From phk at project.varnish-software.com  Sat Jan 22 13:55:41 2011
From: phk at project.varnish-software.com (Poul-Henning Kamp)
Date: Sat, 22 Jan 2011 14:55:41 +0100
Subject: [master] 2899bb9 And now the going gets ugly: Add code to include a
	non-gzip'ed ESI-object in a gzip'ed ESI object.
Message-ID: <E1PgdwL-00062w-0p@project.varnish-software.com>

commit 2899bb924ddd4fa45c1b7235a6439acef7abffdf
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sat Jan 22 13:41:41 2011 +0000

    And now the going gets ugly:  Add code to include a non-gzip'ed ESI-object
    in a gzip'ed ESI object.

diff --git a/bin/varnishd/cache_esi_deliver.c b/bin/varnishd/cache_esi_deliver.c
index e5209e3..ca581f8 100644
--- a/bin/varnishd/cache_esi_deliver.c
+++ b/bin/varnishd/cache_esi_deliver.c
@@ -168,6 +168,57 @@ ved_decode_len(uint8_t **pp)
 	return (l);
 }
 
+/*---------------------------------------------------------------------
+ * If a gzip'ed ESI object includes a ungzip'ed object, we need to make
+ * it looked like a gzip'ed data stream.  The official way to do so would
+ * be to fire up libvgz and gzip it, but we don't, we fake it.
+ *
+ * First, we cannot know if it is ungzip'ed on purpose, the admin may
+ * know something we don't.
+ *
+ * What do you mean "BS ?"
+ *
+ * All right then...
+ *
+ * The matter of the fact is that we simply will not fire up a gzip in
+ * the output path because it costs too much memory and CPU, so we simply
+ * wrap the data in very convenient "gzip copy-blocks" and send it down
+ * the stream with a bit more overhead.
+ */
+
+static void
+ved_pretend_gzip(struct sess *sp, uint8_t *p, ssize_t l)
+{
+	ssize_t ll;
+	uint8_t buf[5];
+	char chunk[20];
+
+	while (l > 0) {
+		ll = l;
+		if (ll > 65535)
+			ll = 65535;
+		buf[0] = 0;
+		vle16enc(buf + 1, ll);
+		vle16enc(buf + 3, ~ll);
+		if (sp->wrk->res_mode & RES_CHUNKED) {
+			bprintf(chunk, "%jx\r\n", (intmax_t)ll + 5);
+			(void)WRW_Write(sp->wrk, chunk, -1);
+		}
+		(void)WRW_Write(sp->wrk, buf, sizeof buf);
+		(void)WRW_Write(sp->wrk, p, ll);
+		if (sp->wrk->res_mode & RES_CHUNKED) 
+			(void)WRW_Write(sp->wrk, "\r\n", -1);
+		(void)WRW_Flush(sp->wrk);
+		sp->wrk->crc = crc32(sp->wrk->crc, p, ll);
+		sp->wrk->l_crc += ll;
+		l -= ll;
+		p += ll;
+	}
+}
+
+/*---------------------------------------------------------------------
+ */
+
 void
 ESI_Deliver(struct sess *sp)
 {
@@ -177,6 +228,7 @@ ESI_Deliver(struct sess *sp)
 	ssize_t l, l_icrc;
 	uint32_t icrc;
 	uint8_t tailbuf[8 + 5];
+	int isgzip;
 
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 	st = sp->obj->esidata;
@@ -184,9 +236,15 @@ ESI_Deliver(struct sess *sp)
 	p = st->ptr;
 	e = st->ptr + st->len;
 
+	if (*p == VEC_GZ) {
+		isgzip = 1;
+		p++;
+	} else {
+		isgzip = 0;
+	}
+
 	if (sp->esi_level == 0) {
-		if (*p == VEC_GZ) {
-			p++;	
+		if (isgzip) {
 			sp->wrk->gzip_resp = 1;
 			sp->wrk->crc = crc32(0L, Z_NULL, 0);
 		} else 
@@ -202,21 +260,36 @@ ESI_Deliver(struct sess *sp)
 		case VEC_V2:
 		case VEC_V8:
 			l = ved_decode_len(&p);
-			if (sp->wrk->gzip_resp) {
-				assert(*p == VEC_C1 || *p == VEC_C2 ||
-				    *p == VEC_C8);
-				l_icrc = ved_decode_len(&p);
-				icrc = vbe32dec(p);
-				p += 4;
-				sp->wrk->crc =
-				    crc32_combine(sp->wrk->crc, icrc, l_icrc);
-				sp->wrk->l_crc += l_icrc;
-			}
+			r = p;
 			q = (void*)strchr((const char*)p, '\0');
-			assert (q > p);
-			ved_sendchunk(sp, p, q - p, st->ptr + off, l);
-			off += l;
 			p = q + 1;
+			if (sp->wrk->gzip_resp) {
+				if (isgzip) {
+					assert(*p == VEC_C1 || *p == VEC_C2 ||
+					    *p == VEC_C8);
+					l_icrc = ved_decode_len(&p);
+					icrc = vbe32dec(p);
+					p += 4;
+					sp->wrk->crc = crc32_combine(
+					    sp->wrk->crc, icrc, l_icrc);
+					sp->wrk->l_crc += l_icrc;
+					ved_sendchunk(sp, r, q - r,
+					    st->ptr + off, l);
+					off += l;
+				} else {
+					ved_pretend_gzip(sp,
+					    st->ptr + off, l);
+					off += l;
+				}
+			} else {
+				if (isgzip) {
+					INCOMPL();
+				} else {
+					ved_sendchunk(sp, r, q - r,
+					    st->ptr + off, l);
+					off += l;
+				}
+			}
 			break;
 		case VEC_S1:
 		case VEC_S2:
diff --git a/bin/varnishd/cache_esi_parse.c b/bin/varnishd/cache_esi_parse.c
index 93522d2..6baeb13 100644
--- a/bin/varnishd/cache_esi_parse.c
+++ b/bin/varnishd/cache_esi_parse.c
@@ -286,13 +286,13 @@ vep_emit_verbatim(const struct vep_state *vep, ssize_t l, ssize_t l_crc)
 		Debug("---> VERBATIM(%jd)\n", (intmax_t)l);
 	}
 	vep_emit_len(vep, l, VEC_V1, VEC_V2, VEC_V8);
+	/* Emit Chunked header */
+	vsb_printf(vep->vsb, "%lx\r\n%c", l, 0);
 	if (vep->dogzip) {
 		vep_emit_len(vep, l_crc, VEC_C1, VEC_C2, VEC_C8);
 		vbe32enc(buf, vep->crc);
 		vsb_bcat(vep->vsb, buf, sizeof buf);
 	}
-	/* Emit Chunked header */
-	vsb_printf(vep->vsb, "%lx\r\n%c", l, 0);
 } 
 
 static void



From phk at project.varnish-software.com  Sat Jan 22 13:55:42 2011
From: phk at project.varnish-software.com (Poul-Henning Kamp)
Date: Sat, 22 Jan 2011 14:55:42 +0100
Subject: [master] 4996592 and now we also need vle16enc()
Message-ID: <E1PgdwM-00063H-Fl@project.varnish-software.com>

commit 49965920392ae0b18d001f950b54b70a28a4a22c
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sat Jan 22 13:40:46 2011 +0000

    and now we also need vle16enc()

diff --git a/include/vend.h b/include/vend.h
index 01f0dfc..5ffe268 100644
--- a/include/vend.h
+++ b/include/vend.h
@@ -116,8 +116,6 @@ vbe64enc(void *pp, uint64_t u)
 	vbe32enc(p + 4, (uint32_t)(u & 0xffffffffU));
 }
 
-#if 0
-
 static __inline void
 vle16enc(void *pp, uint16_t u)
 {
@@ -126,7 +124,6 @@ vle16enc(void *pp, uint16_t u)
 	p[0] = u & 0xff;
 	p[1] = (u >> 8) & 0xff;
 }
-#endif
 
 static __inline void
 vle32enc(void *pp, uint32_t u)



From phk at project.varnish-software.com  Sat Jan 22 16:53:24 2011
From: phk at project.varnish-software.com (Poul-Henning Kamp)
Date: Sat, 22 Jan 2011 17:53:24 +0100
Subject: [master] 12632f2 Qualify for the "evilphk" monicker:
Message-ID: <E1PggiK-0007aR-9v@project.varnish-software.com>

commit 12632f242303debe3ba8adafea54ab8e35ad910f
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sat Jan 22 16:51:17 2011 +0000

    Qualify for the "evilphk" monicker:
    
    For a limited number of cases we can now esi:include a gzip'ed
    object into a gzip'ed ESI object.
    
    There is less than 1 chance in hell this will work for you in
    practice, I need to handle the N-1 other cases of last bit
    and stop bit combinations in the gziped object, but that is
    just a small (but tedious) matter of programming.

diff --git a/bin/varnishd/cache_esi_deliver.c b/bin/varnishd/cache_esi_deliver.c
index a3c9dc5..2be22e3 100644
--- a/bin/varnishd/cache_esi_deliver.c
+++ b/bin/varnishd/cache_esi_deliver.c
@@ -129,14 +129,22 @@ static void
 ved_sendchunk(const struct sess *sp, const void *cb, ssize_t cl,
     const void *ptr, ssize_t l)
 {
+	char chunk[20];
 
-	Debug("VER(%d) %d\n", (int)l, (int)(cb-ce));
 	assert(l > 0);
-	if (sp->wrk->res_mode & RES_CHUNKED)
-		(void)WRW_Write(sp->wrk, cb, cl);
+	if (sp->wrk->res_mode & RES_CHUNKED) {
+		if (cb == NULL) {
+			bprintf(chunk, "%jx\r\n", l);
+			(void)WRW_Write(sp->wrk, chunk, -1);
+		} else
+			(void)WRW_Write(sp->wrk, cb, cl);
+	}
 	(void)WRW_Write(sp->wrk, ptr, l);
-	if (sp->wrk->res_mode & RES_CHUNKED)
+	if (sp->wrk->res_mode & RES_CHUNKED) {
 		(void)WRW_Write(sp->wrk, "\r\n", -1);
+		if (cb == NULL) 
+			(void)WRW_Flush(sp->wrk);
+	}
 }
 
 static ssize_t
@@ -273,8 +281,14 @@ ESI_Deliver(struct sess *sp)
 					sp->wrk->crc = crc32_combine(
 					    sp->wrk->crc, icrc, l_icrc);
 					sp->wrk->l_crc += l_icrc;
+if (sp->esi_level > 0 && off == 0) {
+assert(l > 10);
+					ved_sendchunk(sp, NULL, 0,
+					    st->ptr + off + 10, l - 10);
+} else {
 					ved_sendchunk(sp, r, q - r,
 					    st->ptr + off, l);
+}
 					off += l;
 				} else {
 					ved_pretend_gzip(sp,
@@ -342,11 +356,82 @@ void
 ESI_DeliverChild(struct sess *sp)
 {
 	struct storage *st;
+	struct object *obj;
+	ssize_t start, last, stop, l, lx, dl;
+	u_char *p;
+	uint32_t icrc;
+	uint32_t ilen;
 
-	if (sp->obj->gziped) {
-		INCOMPL();
-	} else {
+	if (!sp->obj->gziped) {
 		VTAILQ_FOREACH(st, &sp->obj->store, list)
 			ved_pretend_gzip(sp, st->ptr, st->len);
+		return;
 	}
+	/*
+	 * This is the interesting case: Deliver all the deflate
+	 * blocks, stripping the "LAST" bit of the last one and
+	 * padding it, as necessary, to a byte boundary.
+	 */
+	obj = sp->obj;
+	CHECK_OBJ_NOTNULL(obj, OBJECT_MAGIC);
+	start = obj->gzip_start;
+	last = obj->gzip_last;
+	stop = obj->gzip_stop;
+	assert(last >= start);
+	assert(last < stop);
+	assert(start > 0 && start <= obj->len * 8);
+	assert(last > 0 && last <= obj->len * 8);
+	assert(stop > 0 && stop <= obj->len * 8);
+printf("BITS %jd %jd %jd\n", start, last, stop);
+
+	/* The start bit must be byte aligned. */
+	AZ(start & 7);
+
+	lx = 0;
+	VTAILQ_FOREACH(st, &sp->obj->store, list) {
+		p = st->ptr;
+		l = st->len;
+		xxxassert(start/8 < l);
+		if (start/8 > 0) {
+			l -= start/8;
+			p += start/8;
+			lx += start/8;
+			start = 0;
+		}
+		assert(l >= 0);
+		if (l == 0)
+			continue;
+		printf("XXXX: %jd %jd %jd\n", l, lx, last / 8);
+		dl = last/8 - lx;
+		if (dl > 0) {
+			if (dl > l)
+				dl = l;
+printf("CH1 %jd\n", dl);
+			ved_sendchunk(sp, NULL, 0, p, dl);
+			lx += dl;
+			p += dl;
+			l -= dl;
+		}
+		assert(l >= 0);
+		if (l == 0)
+			continue;
+		printf("XXXX: %jd %jd %jd %02x\n", l, lx, last / 8, *p);
+		/*
+		 * If we are lucky, the last bit is aligned and in a copy
+		 * block, detect and be happy
+		 */
+		if (l >= 3 && (last & 7) == 0 &&
+		    p[0] == 0x01 && p[1] == 0 && p[2] == 0)
+			break;
+		INCOMPL();
+	}
+	AZ(VTAILQ_NEXT(st, list));
+	assert(st->len > 8);
+	p = st->ptr + st->len - 8;
+	icrc = vle32dec(p);
+	ilen = vle32dec(p + 4);
+printf("CRC %08x LEN %d\n", icrc, ilen);
+	sp->wrk->crc = crc32_combine(sp->wrk->crc, icrc, ilen);
+	sp->wrk->l_crc += ilen;
+
 }
diff --git a/bin/varnishtest/tests/e00023.vtc b/bin/varnishtest/tests/e00023.vtc
index 0d4457c..0bb17f8 100644
--- a/bin/varnishtest/tests/e00023.vtc
+++ b/bin/varnishtest/tests/e00023.vtc
@@ -1,6 +1,6 @@
 # $Id$
 
-test "Include an ungzip'ed (ESI) object in a gzip'ed ESI object"
+test "Include gzip'ed and ungzip'ed (ESI) objects in a gzip'ed ESI object"
 
 server s1 {
 	rxreq 
@@ -11,14 +11,30 @@ server s1 {
 		<esi:include src="/foo"/>		3
 		Between includes			4
 		<esi:include src="/bar"/>		5
-		After includes				6
+		Between includes			6
+		<esi:include src="/fooz"/>		7
+		Between includes			8
+		<esi:include src="/barz"/>		9
+		After includes				10
 	}
+
 	rxreq
 	expect req.url == "/foo"
 	txresp -body {<h1>/////<h2>}
+
 	rxreq
 	expect req.url == "/bar"
 	txresp -body {*****}
+
+	rxreq
+	expect req.http.accept-encoding == gzip
+	expect req.url == "/fooz"
+	txresp -gzipbody {<h1>-----<h2>}
+
+	rxreq
+	expect req.http.accept-encoding == gzip
+	expect req.url == "/barz"
+	txresp -gzipbody {#####}
 } -start
 
 varnish v1 -vcl+backend {
@@ -38,7 +54,7 @@ client c1 {
 	expect resp.http.content-encoding == gzip
 	gunzip
 	expect resp.status == 200
-	expect resp.bodylen == 115
+	expect resp.bodylen == 192
 }
 
 client c1 -run
diff --git a/bin/varnishtest/vtc_http.c b/bin/varnishtest/vtc_http.c
index cf34a4e..9c9609d 100644
--- a/bin/varnishtest/vtc_http.c
+++ b/bin/varnishtest/vtc_http.c
@@ -417,8 +417,14 @@ http_swallow_body(struct http *hp, char * const *hh, int body)
 			}
 			l = hp->prxbuf;
 			http_rxchar(hp, 2);
-			assert(vct_iscrlf(hp->rxbuf[l]));
-			assert(vct_iscrlf(hp->rxbuf[l + 1]));
+			if(!vct_iscrlf(hp->rxbuf[l]))
+				vtc_log(hp->vl, 0,
+				    "Wrong chunk tail[0] = %02x",
+				    hp->rxbuf[l] & 0xff);
+			if(!vct_iscrlf(hp->rxbuf[l + 1]))
+				vtc_log(hp->vl, 0,
+				    "Wrong chunk tail[1] = %02x",
+				    hp->rxbuf[l + 1] & 0xff);
 			hp->prxbuf = l;
 			hp->rxbuf[l] = '\0';
 			if (i == 0)



From phk at project.varnish-software.com  Sat Jan 22 16:53:24 2011
From: phk at project.varnish-software.com (Poul-Henning Kamp)
Date: Sat, 22 Jan 2011 17:53:24 +0100
Subject: [master] 01b6669 Record the positions of gzip start,
	last and stop bits in objects when we know them.
Message-ID: <E1PggiK-0007aa-J4@project.varnish-software.com>

commit 01b66692d2cb7b2be5e587afb3062ed86e0fb1f7
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sat Jan 22 15:57:17 2011 +0000

    Record the positions of gzip start, last and stop bits in
    objects when we know them.
    
    For now: use malloc for gzip instead of running out of workspace.

diff --git a/bin/varnishd/cache.h b/bin/varnishd/cache.h
index a2ed4cc..2779a00 100644
--- a/bin/varnishd/cache.h
+++ b/bin/varnishd/cache.h
@@ -433,6 +433,10 @@ struct object {
 	/* XXX: make bitmap */
 	unsigned		cacheable;
 	unsigned		gziped;
+	/* Bit positions in the gzip stream */
+	ssize_t			gzip_start;
+	ssize_t			gzip_last;
+	ssize_t			gzip_stop;
 
 	ssize_t			len;
 
@@ -643,6 +647,7 @@ int VGZ_ObufStorage(const struct sess *sp, struct vgz *vg);
 int VGZ_Gzip(struct vgz *, const void **, size_t *len, enum vgz_flag);
 int VGZ_Gunzip(struct vgz *, const void **, size_t *len);
 void VGZ_Destroy(struct vgz **);
+void VGZ_UpdateObj(const struct vgz*, struct object *);
 
 /* cache_http.c */
 unsigned HTTP_estimate(unsigned nhttp);
diff --git a/bin/varnishd/cache_esi_fetch.c b/bin/varnishd/cache_esi_fetch.c
index fe802f1..3e05d7c 100644
--- a/bin/varnishd/cache_esi_fetch.c
+++ b/bin/varnishd/cache_esi_fetch.c
@@ -270,6 +270,7 @@ vfp_esi_bytes_gg(struct sess *sp, struct http_conn *htc, size_t bytes)
 			VGZ_Obuf(sp->wrk->vgz_rx, ibuf2, sizeof ibuf2);
 			i = VGZ_Gunzip(sp->wrk->vgz_rx, &dp, &dl);
 			/* XXX: check i */
+			assert(i >= 0);
 			vef->bufp = ibuf2;
 			if (dl > 0)
 				VEP_parse(sp, ibuf2, dl);
@@ -371,6 +372,7 @@ vfp_esi_end(struct sess *sp)
 
 	if (sp->wrk->vef_priv != NULL) {
 		vef = sp->wrk->vef_priv;
+		VGZ_UpdateObj(vef->vgz, sp->obj);
 		sp->wrk->vef_priv = NULL;
 		CHECK_OBJ_NOTNULL(vef, VEF_MAGIC);
 		XXXAZ(vef->error);
diff --git a/bin/varnishd/cache_gzip.c b/bin/varnishd/cache_gzip.c
index 189b62e..6e929e9 100644
--- a/bin/varnishd/cache_gzip.c
+++ b/bin/varnishd/cache_gzip.c
@@ -148,6 +148,11 @@ VGZ_NewUngzip(const struct sess *sp, struct ws *tmp)
 	 * Since we don't control windowBits, we have to assume
 	 * it is 15, so 34-35KB or so.
 	 */
+#if 1
+	vg->vz.zalloc = NULL;
+	vg->vz.zfree = NULL;
+	vg->vz.opaque = NULL;
+#endif
 	assert(Z_OK == inflateInit2(&vg->vz, 31));
 	return (vg);
 }
@@ -176,6 +181,11 @@ VGZ_NewGzip(const struct sess *sp, struct ws *tmp)
 	 * XXX: It may be more efficent to malloc them, rather than have
 	 * XXX: too many worker threads grow the stacks.
 	 */
+#if 1
+	vg->vz.zalloc = NULL;
+	vg->vz.zfree = NULL;
+	vg->vz.opaque = NULL;
+#endif
 	i = deflateInit2(&vg->vz,
 	    0,				/* Level */
 	    Z_DEFLATED,			/* Method */
@@ -271,6 +281,7 @@ VGZ_Gunzip(struct vgz *vg, const void **pptr, size_t *plen)
 		return (1);
 	if (i == Z_BUF_ERROR)
 		return (2);
+printf("INFLATE=%d\n", i);
 	return (-1);
 }
 
@@ -318,6 +329,18 @@ VGZ_Gzip(struct vgz *vg, const void **pptr, size_t *plen, enum vgz_flag flags)
 /*--------------------------------------------------------------------*/
 
 void
+VGZ_UpdateObj(const struct vgz *vg, struct object *obj)
+{
+
+	CHECK_OBJ_NOTNULL(vg, VGZ_MAGIC);
+	obj->gzip_start	= vg->vz.start_bit;
+	obj->gzip_last	= vg->vz.last_bit;
+	obj->gzip_stop	= vg->vz.stop_bit;
+}
+
+/*--------------------------------------------------------------------*/
+
+void
 VGZ_Destroy(struct vgz **vg)
 {
 
@@ -458,6 +481,7 @@ vfp_gzip_end(struct sess *sp)
 		i = VGZ_Gzip(vg, &dp, &dl, VGZ_FINISH);
 		sp->obj->len += dl;
 	} while (i != Z_STREAM_END);
+	VGZ_UpdateObj(vg, sp->obj);
 	VGZ_Destroy(&vg);
 	sp->obj->gziped = 1;
 	return (0);



From phk at project.varnish-software.com  Sat Jan 22 20:08:43 2011
From: phk at project.varnish-software.com (Poul-Henning Kamp)
Date: Sat, 22 Jan 2011 21:08:43 +0100
Subject: [master] 6996fab Make it possible to tweak the gzip compressoin level
Message-ID: <E1PgjlL-0000eW-KL@project.varnish-software.com>

commit 6996fab96f1207129ef43c338703d027116f900b
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sat Jan 22 20:08:28 2011 +0000

    Make it possible to tweak the gzip compressoin level

diff --git a/bin/varnishtest/vtc_http.c b/bin/varnishtest/vtc_http.c
index 9c9609d..00a8a71 100644
--- a/bin/varnishtest/vtc_http.c
+++ b/bin/varnishtest/vtc_http.c
@@ -72,6 +72,8 @@ struct http {
 
 	char			*req[MAX_HDR];
 	char			*resp[MAX_HDR];
+
+	int			gzip_level;
 };
 
 #define ONLY_CLIENT(hp, av)						\
@@ -590,7 +592,7 @@ gzip_body(struct http *hp, const char *txt, char **body, int *bodylen)
 	vz.avail_out = l + OVERHEAD;
 
 	assert(Z_OK == deflateInit2(&vz,
-	    Z_NO_COMPRESSION, Z_DEFLATED, 31, 9, Z_DEFAULT_STRATEGY));
+	    hp->gziplevel, Z_DEFLATED, 31, 9, Z_DEFAULT_STRATEGY));
 	assert(Z_STREAM_END == deflate(&vz, Z_FINISH));
 	*bodylen = vz.total_out;
 	vtc_log(hp->vl, 4, "startbit = %ju %ju/%ju",
@@ -681,6 +683,9 @@ cmd_http_txresp(CMD_ARGS)
 			body = synth_body(av[1], 0);
 			bodylen = strlen(body);
 			av++;
+		} else if (!strcmp(*av, "-gziplevel")) {
+			hp->gzip_level = strtoul(av[1], NULL, 0);
+			av++;
 		} else if (!strcmp(*av, "-gziplen")) {
 			assert(body == nullbody);
 			b = synth_body(av[1], 1);



From phk at project.varnish-software.com  Sat Jan 22 20:09:32 2011
From: phk at project.varnish-software.com (Poul-Henning Kamp)
Date: Sat, 22 Jan 2011 21:09:32 +0100
Subject: [master] f701705 Typo
Message-ID: <E1Pgjm8-0000kb-Ex@project.varnish-software.com>

commit f7017059b795716894abe58d3f4d9491a54fcf93
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sat Jan 22 20:09:25 2011 +0000

    Typo

diff --git a/bin/varnishtest/vtc_http.c b/bin/varnishtest/vtc_http.c
index 00a8a71..8a0bd8a 100644
--- a/bin/varnishtest/vtc_http.c
+++ b/bin/varnishtest/vtc_http.c
@@ -73,7 +73,7 @@ struct http {
 	char			*req[MAX_HDR];
 	char			*resp[MAX_HDR];
 
-	int			gzip_level;
+	int			gziplevel;
 };
 
 #define ONLY_CLIENT(hp, av)						\
@@ -684,7 +684,7 @@ cmd_http_txresp(CMD_ARGS)
 			bodylen = strlen(body);
 			av++;
 		} else if (!strcmp(*av, "-gziplevel")) {
-			hp->gzip_level = strtoul(av[1], NULL, 0);
+			hp->gziplevel = strtoul(av[1], NULL, 0);
 			av++;
 		} else if (!strcmp(*av, "-gziplen")) {
 			assert(body == nullbody);



From phk at project.varnish-software.com  Sat Jan 22 22:58:26 2011
From: phk at project.varnish-software.com (Poul-Henning Kamp)
Date: Sat, 22 Jan 2011 23:58:26 +0100
Subject: [master] 70dacf2 Add the necessary code (& a testcase) to pad out an
	included gzip file,
	no matter which of the 8 bits of a byte the last bit lands in.
Message-ID: <E1PgmPa-00052T-6E@project.varnish-software.com>

commit 70dacf26f31659353adbf9d287b52a6fc6c818cb
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sat Jan 22 22:57:19 2011 +0000

    Add the necessary code (& a testcase) to pad out an included
    gzip file, no matter which of the 8 bits of a byte the last bit
    lands in.

diff --git a/bin/varnishtest/tests/e00024.vtc b/bin/varnishtest/tests/e00024.vtc
new file mode 100644
index 0000000..ad3ecd4
--- /dev/null
+++ b/bin/varnishtest/tests/e00024.vtc
@@ -0,0 +1,85 @@
+# $Id$
+
+test "Test all 8 gzip stopbit residuals"
+
+server s1 {
+	rxreq 
+	expect req.http.accept-encoding == gzip
+	txresp -gzipbody {
+		<html>					1
+		Before includes				2
+		{<esi:include src="/bit0"/>} bit0	3
+		{<esi:include src="/bit1"/>} bit1	4
+		{<esi:include src="/bit2"/>} bit2	5
+		{<esi:include src="/bit3"/>} bit3	6
+		{<esi:include src="/bit4"/>} bit4	7
+		{<esi:include src="/bit5"/>} bit5	8
+		{<esi:include src="/bit6"/>} bit6	9
+		{<esi:include src="/bit7"/>} bit7	10
+		After includes				11
+	}
+
+	rxreq
+	expect req.url == "/bit0"
+	expect req.http.accept-encoding == gzip
+	txresp -gzipresidual 0 -gziplevel 9 -gzipbody {e04c8d0fd604c}
+
+	rxreq
+	expect req.url == "/bit1"
+	expect req.http.accept-encoding == gzip
+	txresp -gzipresidual 1 -gziplevel 9 -gzipbody {1ea86e6cf31bf4ec3d7a86}
+
+	rxreq
+	expect req.url == "/bit2"
+	expect req.http.accept-encoding == gzip
+	txresp -gzipresidual 2 -gziplevel 9 -gzipbody {10}
+
+	rxreq
+	expect req.url == "/bit3"
+	expect req.http.accept-encoding == gzip
+	txresp -gzipresidual 3 -gziplevel 9 -gzipbody {a5e2e2e1c2e2}
+
+	rxreq
+	expect req.url == "/bit4"
+	expect req.http.accept-encoding == gzip
+	txresp -gzipresidual 4 -gziplevel 9 -gzipbody {71c5d18ec5d5d1}
+
+	rxreq
+	expect req.url == "/bit5"
+	expect req.http.accept-encoding == gzip
+	txresp -gzipresidual 5 -gziplevel 9 -gzipbody {39886d28a6d2988}
+
+	rxreq
+	expect req.url == "/bit6"
+	expect req.http.accept-encoding == gzip
+	txresp -gzipresidual 6 -gziplevel 9 -gzipbody {80000}
+
+	rxreq
+	expect req.url == "/bit7"
+	expect req.http.accept-encoding == gzip
+	txresp -gzipresidual 7 -gziplevel 9 -gzipbody {386811868}
+
+} -start
+
+varnish v1 -vcl+backend {
+	sub vcl_fetch {
+		if (req.url == "/") {
+			set beresp.do_esi = true;
+		}
+	}
+} -start 
+
+varnish v1 -cliok "param.set http_gzip_support true"
+varnish v1 -cliok "param.set esi_syntax 0x3e" 
+varnish v1 -cliok "param.set diag_bitmap 0x10000"
+
+client c1 {
+	txreq -hdr "Accept-Encoding: gzip"
+	rxresp
+	expect resp.http.content-encoding == gzip
+	gunzip
+	expect resp.status == 200
+	expect resp.bodylen == 252
+} -run
+
+varnish v1 -expect esi_errors == 0



From phk at project.varnish-software.com  Sat Jan 22 22:58:26 2011
From: phk at project.varnish-software.com (Poul-Henning Kamp)
Date: Sat, 22 Jan 2011 23:58:26 +0100
Subject: [master] d0cd23f Make it possible to call VTAILQ_TAIL on objects
Message-ID: <E1PgmPa-00052a-GV@project.varnish-software.com>

commit d0cd23f5f7cb989e9b637bcce31527c95490cde2
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sat Jan 22 22:55:15 2011 +0000

    Make it possible to call VTAILQ_TAIL on objects

diff --git a/bin/varnishd/cache.h b/bin/varnishd/cache.h
index af1bfd2..599ef75 100644
--- a/bin/varnishd/cache.h
+++ b/bin/varnishd/cache.h
@@ -417,6 +417,7 @@ struct lru {
 };
 
 /* Object structure --------------------------------------------------*/
+VTAILQ_HEAD(objecthead, storage);
 
 struct object {
 	unsigned		magic;
@@ -451,7 +452,7 @@ struct object {
 
 	struct http		*http;
 
-	VTAILQ_HEAD(, storage)	store;
+	struct objecthead	store;
 
 	struct storage		*esidata;
 



From phk at project.varnish-software.com  Sat Jan 22 22:58:26 2011
From: phk at project.varnish-software.com (Poul-Henning Kamp)
Date: Sat, 22 Jan 2011 23:58:26 +0100
Subject: [master] a42afbb Add a -gzipresidual %d argument to make sure we get
	the residual stopbit we expect.
Message-ID: <E1PgmPa-00052k-Ux@project.varnish-software.com>

commit a42afbbc199da0119630f44bd93174488f9b8f6c
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sat Jan 22 22:44:18 2011 +0000

    Add a -gzipresidual %d argument to make sure we get the
    residual stopbit we expect.

diff --git a/bin/varnishtest/vtc_http.c b/bin/varnishtest/vtc_http.c
index 631225f..f8aa59b 100644
--- a/bin/varnishtest/vtc_http.c
+++ b/bin/varnishtest/vtc_http.c
@@ -74,6 +74,7 @@ struct http {
 	char			*resp[MAX_HDR];
 
 	int			gziplevel;
+	int			gzipresidual;
 };
 
 #define ONLY_CLIENT(hp, av)						\
@@ -576,7 +577,7 @@ cmd_http_gunzip_body(CMD_ARGS)
 static void
 gzip_body(struct http *hp, const char *txt, char **body, int *bodylen)
 {
-	int l;
+	int l, i;
 	z_stream vz;
 
 	memset(&vz, 0, sizeof vz);
@@ -594,6 +595,10 @@ gzip_body(struct http *hp, const char *txt, char **body, int *bodylen)
 	assert(Z_OK == deflateInit2(&vz,
 	    hp->gziplevel, Z_DEFLATED, 31, 9, Z_DEFAULT_STRATEGY));
 	assert(Z_STREAM_END == deflate(&vz, Z_FINISH));
+	i = vz.stop_bit & 7;
+	if (hp->gzipresidual >= 0 && hp->gzipresidual != i) 
+		vtc_log(hp->vl, 0, "Wrong gzip residual got %d wanted %d",
+		    i, hp->gzipresidual);
 	*bodylen = vz.total_out;
 	vtc_log(hp->vl, 4, "startbit = %ju %ju/%ju",
 	    vz.start_bit, vz.start_bit >> 3, vz.start_bit & 7);
@@ -683,6 +688,9 @@ cmd_http_txresp(CMD_ARGS)
 			body = synth_body(av[1], 0);
 			bodylen = strlen(body);
 			av++;
+		} else if (!strcmp(*av, "-gzipresidual")) {
+			hp->gzipresidual = strtoul(av[1], NULL, 0);
+			av++;
 		} else if (!strcmp(*av, "-gziplevel")) {
 			hp->gziplevel = strtoul(av[1], NULL, 0);
 			av++;
@@ -1061,6 +1069,8 @@ http_process(struct vtclog *vl, const char *spec, int sock, int sfd)
 	hp->rxbuf = malloc(hp->nrxbuf);		/* XXX */
 	hp->sfd = sfd;
 	hp->vl = vl;
+	hp->gziplevel = 0;
+	hp->gzipresidual = -1;
 	AN(hp->rxbuf);
 	AN(hp->vsb);
 



From phk at project.varnish-software.com  Sat Jan 22 22:58:27 2011
From: phk at project.varnish-software.com (Poul-Henning Kamp)
Date: Sat, 22 Jan 2011 23:58:27 +0100
Subject: [master] 5145486 Add some #ifdef'd out code to find magic gzip inputs
	for tests still to come.
Message-ID: <E1PgmPb-00053A-PF@project.varnish-software.com>

commit 5145486efbe6317dcd8d13780b56bf291e8f27d7
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sat Jan 22 22:36:41 2011 +0000

    Add some #ifdef'd out code to find magic gzip inputs for tests still
    to come.

diff --git a/bin/varnishtest/vtc_http.c b/bin/varnishtest/vtc_http.c
index 8a0bd8a..631225f 100644
--- a/bin/varnishtest/vtc_http.c
+++ b/bin/varnishtest/vtc_http.c
@@ -520,7 +520,7 @@ cmd_http_rxresp(CMD_ARGS)
 
 #define TRUST_ME(ptr)   ((void*)(uintptr_t)(ptr))
 
-#define OVERHEAD 31
+#define OVERHEAD 64
 
 
 static void
@@ -1073,3 +1073,65 @@ http_process(struct vtclog *vl, const char *spec, int sock, int sfd)
 	free(hp->rxbuf);
 	free(hp);
 }
+
+/**********************************************************************
+ * Magic test routine
+ *
+ * This function brute-forces some short strings through gzip(9) to
+ * find candidates for all possible 8 bit positions of the stopbit.
+ *
+ * Here is some good short output strings:
+ *
+ *	0 184 <e04c8d0fd604c>
+ *	1 257 <1ea86e6cf31bf4ec3d7a86>
+ *	2 106 <10>
+ *	3 163 <a5e2e2e1c2e2>
+ *	4 180 <71c5d18ec5d5d1>
+ *	5 189 <39886d28a6d2988>
+ *	6 118 <80000>
+ *	7 151 <386811868>
+ *
+ */
+
+#if 0
+void xxx(void);
+
+void
+xxx(void)
+{
+	z_stream vz;
+	int n;
+	char ibuf[200];
+	char obuf[200];
+	int fl[8];
+	int i, j;
+
+	for (n = 0; n < 8; n++)
+		fl[n] = 9999;
+
+	memset(&vz, 0, sizeof vz);
+
+	for(n = 0;  n < 999999999; n++) {
+		*ibuf = 0;
+		for (j = 0; j < 7; j++) {
+			sprintf(strchr(ibuf, 0), "%x",
+			    (unsigned)random() & 0xffff);
+			vz.next_in = TRUST_ME(ibuf);
+			vz.avail_in = strlen(ibuf);
+			vz.next_out = TRUST_ME(obuf);
+			vz.avail_out = sizeof obuf;
+			assert(Z_OK == deflateInit2(&vz,
+			    9, Z_DEFLATED, 31, 9, Z_DEFAULT_STRATEGY));
+			assert(Z_STREAM_END == deflate(&vz, Z_FINISH));
+			i = vz.stop_bit & 7;
+			if (fl[i] > strlen(ibuf)) {
+				printf("%d %jd <%s>\n", i, vz.stop_bit, ibuf);
+				fl[i] = strlen(ibuf);
+			}
+			assert(Z_OK == deflateEnd(&vz));
+		}
+	}
+
+	printf("FOO\n");
+}
+#endif



From phk at project.varnish-software.com  Sat Jan 22 22:58:28 2011
From: phk at project.varnish-software.com (Poul-Henning Kamp)
Date: Sat, 22 Jan 2011 23:58:28 +0100
Subject: [master] df48e70 Add the vfp_testgzip which verifies that gzip'ed
	data are indeed gzip'ed data and extracts the magic bits should
	it ever be included from ESI.
Message-ID: <E1PgmPc-00053P-RH@project.varnish-software.com>

commit df48e70aa1077b935783ceac0d4d0f5aba06b02e
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sat Jan 22 20:12:25 2011 +0000

    Add the vfp_testgzip which verifies that gzip'ed data are
    indeed gzip'ed data and extracts the magic bits should it ever
    be included from ESI.
    
    In general Varnish is not in the business of inspecting content
    and with the exception of ESI we still do not do so.
    
    What we do here is make sure that we do not serve corrupt data,
    in the same sense as we bail out on chunked encoding trouble or
    HTTP header trouble from the backend.

diff --git a/bin/varnishd/cache.h b/bin/varnishd/cache.h
index 2779a00..af1bfd2 100644
--- a/bin/varnishd/cache.h
+++ b/bin/varnishd/cache.h
@@ -225,6 +225,7 @@ struct vfp {
 
 extern struct vfp vfp_gunzip;
 extern struct vfp vfp_gzip;
+extern struct vfp vfp_testgzip;
 extern struct vfp vfp_esi;
 
 /*--------------------------------------------------------------------*/
diff --git a/bin/varnishd/cache_center.c b/bin/varnishd/cache_center.c
index 99e5d81..ec945d2 100644
--- a/bin/varnishd/cache_center.c
+++ b/bin/varnishd/cache_center.c
@@ -639,6 +639,8 @@ cnt_fetch(struct sess *sp)
 		sp->wrk->vfp = &vfp_gunzip;
 	else if (sp->wrk->do_gzip)
 		sp->wrk->vfp = &vfp_gzip;
+	else if (sp->wrk->is_gzip)
+		sp->wrk->vfp = &vfp_testgzip;
 
 	l = http_EstimateWS(sp->wrk->beresp,
 	    sp->pass ? HTTPH_R_PASS : HTTPH_A_INS, &nhttp);
diff --git a/bin/varnishd/cache_gzip.c b/bin/varnishd/cache_gzip.c
index 6e929e9..a4ea296 100644
--- a/bin/varnishd/cache_gzip.c
+++ b/bin/varnishd/cache_gzip.c
@@ -492,3 +492,78 @@ struct vfp vfp_gzip = {
         .bytes  =       vfp_gzip_bytes,
         .end    =       vfp_gzip_end,
 };
+
+/*--------------------------------------------------------------------
+ * VFP_TESTGZIP
+ *
+ * A VFP for testing that received gzip data is valid, and for
+ * collecting the magic bits while we're at it.
+ */
+
+static void __match_proto__()
+vfp_testgzip_begin(struct sess *sp, size_t estimate)
+{
+	(void)estimate;
+	sp->wrk->vgz_rx = VGZ_NewUngzip(sp, sp->ws);
+}
+
+static int __match_proto__()
+vfp_testgzip_bytes(struct sess *sp, struct http_conn *htc, ssize_t bytes)
+{
+	struct vgz *vg;
+	ssize_t l, w;
+	int i = -100;
+	uint8_t	ibuf[1024 * params->gzip_stack_buffer];
+	size_t dl;
+	const void *dp;
+	struct storage *st;
+
+	vg = sp->wrk->vgz_rx;
+	CHECK_OBJ_NOTNULL(vg, VGZ_MAGIC);
+	AZ(vg->vz.avail_in);
+	while (bytes > 0) {
+		if (FetchStorage(sp))
+			return (-1);
+		st = sp->wrk->storage;
+		l = st->space - st->len;
+		if (l > bytes)
+			l = bytes;
+		w = HTC_Read(htc, st->ptr + st->len, l);
+		if (w <= 0)
+			return (w);
+		bytes -= w;
+		VGZ_Ibuf(vg, st->ptr + st->len, w);
+		st->len += w;
+		sp->obj->len += w;
+
+		while (!VGZ_IbufEmpty(vg)) {
+			VGZ_Obuf(vg, ibuf, sizeof ibuf);
+			i = VGZ_Gunzip(vg, &dp, &dl);
+			assert(i == Z_OK || i == Z_STREAM_END);
+		}
+	}
+	if (i == Z_STREAM_END)
+		return (1);
+	return (-1);
+}
+
+static int __match_proto__()
+vfp_testgzip_end(struct sess *sp)
+{
+	struct vgz *vg;
+
+	vg = sp->wrk->vgz_rx;
+	CHECK_OBJ_NOTNULL(vg, VGZ_MAGIC);
+	VGZ_UpdateObj(vg, sp->obj);
+	VGZ_Destroy(&vg);
+	sp->obj->gziped = 1;
+	return (0);
+}
+
+struct vfp vfp_testgzip = {
+        .begin  =       vfp_testgzip_begin,
+        .bytes  =       vfp_testgzip_bytes,
+        .end    =       vfp_testgzip_end,
+};
+
+



From phk at project.varnish-software.com  Sat Jan 22 23:11:19 2011
From: phk at project.varnish-software.com (Poul-Henning Kamp)
Date: Sun, 23 Jan 2011 00:11:19 +0100
Subject: [master] 1b9e19c For reasons not understood,
	last commit missed this file. Try again:
Message-ID: <E1Pgmc3-0006e1-2N@project.varnish-software.com>

commit 1b9e19cbd589637c6e4cfa63188932dcfcd721d4
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sat Jan 22 23:10:11 2011 +0000

    For reasons not understood, last commit missed this file.
    Try again:
    
        Add the necessary code (& a testcase) to pad out an included
        gzip file, no matter which of the 8 bits of a byte the last bit
        lands in.

diff --git a/bin/varnishd/cache_esi_deliver.c b/bin/varnishd/cache_esi_deliver.c
index 2be22e3..ca1c517 100644
--- a/bin/varnishd/cache_esi_deliver.c
+++ b/bin/varnishd/cache_esi_deliver.c
@@ -352,15 +352,55 @@ assert(l > 10);
  * Include an object in a gzip'ed ESI object delivery
  */
 
+static unsigned
+ved_deliver_byterange(struct sess *sp, ssize_t low, ssize_t high)
+{
+	struct storage *st;
+	ssize_t l, lx;
+	u_char *p;
+
+//printf("BR %jd %jd\n", low, high);
+	lx = 0;
+	VTAILQ_FOREACH(st, &sp->obj->store, list) {
+		p = st->ptr;
+		l = st->len;
+//printf("[0-] %jd %jd\n", lx, lx + l);
+		if (lx + l < low) {
+			lx += l;
+			continue;
+		}
+		if (lx == high)
+			return (p[0]);
+		assert(lx < high);
+		if (lx < low) {
+			p += (low - lx);
+			l -= (low - lx);
+			lx = low;
+		}
+//printf("[1-] %jd %jd\n", lx, lx + l);
+		if (lx + l >= high) 
+			l = high - lx;
+//printf("[2-] %jd %jd\n", lx, lx + l);
+		assert(lx >= low && lx + l <= high);
+		if (l != 0)
+			ved_sendchunk(sp, NULL, 0, p, l);
+		if (lx + st->len > high)
+			return(p[l]);
+		lx += st->len;
+	}
+	INCOMPL();
+}
+
 void
 ESI_DeliverChild(struct sess *sp)
 {
 	struct storage *st;
 	struct object *obj;
-	ssize_t start, last, stop, l, lx, dl;
-	u_char *p;
+	ssize_t start, last, stop, lpad;
+	u_char *p, cc;
 	uint32_t icrc;
 	uint32_t ilen;
+	uint8_t pad[6];
 
 	if (!sp->obj->gziped) {
 		VTAILQ_FOREACH(st, &sp->obj->store, list)
@@ -377,61 +417,74 @@ ESI_DeliverChild(struct sess *sp)
 	start = obj->gzip_start;
 	last = obj->gzip_last;
 	stop = obj->gzip_stop;
+	assert(start > 0 && start < obj->len * 8);
+	assert(last > 0 && last < obj->len * 8);
+	assert(stop > 0 && stop < obj->len * 8);
 	assert(last >= start);
 	assert(last < stop);
-	assert(start > 0 && start <= obj->len * 8);
-	assert(last > 0 && last <= obj->len * 8);
-	assert(stop > 0 && stop <= obj->len * 8);
-printf("BITS %jd %jd %jd\n", start, last, stop);
+//printf("BITS %jd %jd %jd\n", start, last, stop);
 
 	/* The start bit must be byte aligned. */
 	AZ(start & 7);
 
-	lx = 0;
-	VTAILQ_FOREACH(st, &sp->obj->store, list) {
-		p = st->ptr;
-		l = st->len;
-		xxxassert(start/8 < l);
-		if (start/8 > 0) {
-			l -= start/8;
-			p += start/8;
-			lx += start/8;
-			start = 0;
-		}
-		assert(l >= 0);
-		if (l == 0)
-			continue;
-		printf("XXXX: %jd %jd %jd\n", l, lx, last / 8);
-		dl = last/8 - lx;
-		if (dl > 0) {
-			if (dl > l)
-				dl = l;
-printf("CH1 %jd\n", dl);
-			ved_sendchunk(sp, NULL, 0, p, dl);
-			lx += dl;
-			p += dl;
-			l -= dl;
-		}
-		assert(l >= 0);
-		if (l == 0)
-			continue;
-		printf("XXXX: %jd %jd %jd %02x\n", l, lx, last / 8, *p);
-		/*
-		 * If we are lucky, the last bit is aligned and in a copy
-		 * block, detect and be happy
-		 */
-		if (l >= 3 && (last & 7) == 0 &&
-		    p[0] == 0x01 && p[1] == 0 && p[2] == 0)
-			break;
-		INCOMPL();
+	/*
+	 * XXX: optimize for the case where the 'last'
+	 * XXX: bit is in a empty copy block
+	 */
+	cc = ved_deliver_byterange(sp, start/8, last/8);
+//printf("CC_LAST %x\n", cc);
+	cc &= ~(1 << (start & 7));
+	ved_sendchunk(sp, NULL, 0, &cc, 1);
+	cc = ved_deliver_byterange(sp, 1 + last/8, stop/8);
+//printf("CC_STOP %x (%d)\n", cc, (int)(stop & 7));
+	switch((int)(stop & 7)) {
+	case 0: /* xxxxxxxx */
+		/* I think we have an off by one here, but that's OK */
+		lpad = 0;
+		break;
+	case 1: /* x000.... 00000000 00000000 11111111 11111111 */
+	case 3: /* xxx000.. 00000000 00000000 11111111 11111111 */
+	case 5: /* xxxxx000 00000000 00000000 11111111 11111111 */
+		pad[0] = cc | 0x00;
+		pad[1] = 0x00; pad[2] = 0x00; pad[3] = 0xff; pad[4] = 0xff;
+		lpad = 5;
+		break;
+	case 2: /* xx010000 00000100 00000001 00000000 */
+		pad[0] = cc | 0x08;
+		pad[1] = 0x20;
+		pad[2] = 0x80;
+		pad[3] = 0x00;
+		lpad = 4;
+		break;
+	case 4: /* xxxx0100 00000001 00000000 */
+		pad[0] = cc | 0x20;
+		pad[1] = 0x80;
+		pad[2] = 0x00;
+		lpad = 3;
+		break;
+	case 6: /* xxxxxx01 00000000 */
+		pad[0] = cc | 0x80;
+		pad[1] = 0x00;
+		lpad = 2;
+		break;
+	case 7:	/* xxxxxxx0 00...... 00000000 00000000 11111111 11111111 */
+		pad[0] = cc | 0x00;
+		pad[1] = 0x00;
+		pad[2] = 0x00; pad[3] = 0x00; pad[4] = 0xff; pad[5] = 0xff;
+		lpad = 6;
+		break;
+	default:
+		AZ(stop & 7);
 	}
-	AZ(VTAILQ_NEXT(st, list));
+	if (lpad > 0)
+		ved_sendchunk(sp, NULL, 0, pad, lpad);
+	st = VTAILQ_LAST(&sp->obj->store, objecthead);
 	assert(st->len > 8);
+
 	p = st->ptr + st->len - 8;
 	icrc = vle32dec(p);
 	ilen = vle32dec(p + 4);
-printf("CRC %08x LEN %d\n", icrc, ilen);
+//printf("CRC %08x LEN %d\n", icrc, ilen);
 	sp->wrk->crc = crc32_combine(sp->wrk->crc, icrc, ilen);
 	sp->wrk->l_crc += ilen;
-
 }



From phk at varnish-cache.org  Mon Jan 24 08:53:05 2011
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Mon, 24 Jan 2011 09:53:05 +0100
Subject: [master] 5846168 Add code to gunzip an ESI VEC string,
	if a gzip'ed ESI object is included from a non-gzip ESI object or
	during gunzip delivery.
Message-ID: <E1PhIAb-00028L-9n@project.varnish-software.com>

commit 584616884c9468a04395fa89d712e20eb29d1fd1
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Jan 24 08:52:16 2011 +0000

    Add code to gunzip an ESI VEC string, if a gzip'ed ESI object is included
    from a non-gzip ESI object or during gunzip delivery.

diff --git a/bin/varnishd/cache.h b/bin/varnishd/cache.h
index 599ef75..2a5ff93 100644
--- a/bin/varnishd/cache.h
+++ b/bin/varnishd/cache.h
@@ -822,7 +822,7 @@ char *VRT_String(struct ws *ws, const char *h, const char *p, va_list ap);
 char *VRT_StringList(char *d, unsigned dl, const char *p, va_list ap);
 
 void ESI_Deliver(struct sess *);
-void ESI_DeliverChild(struct sess *);
+void ESI_DeliverChild(const struct sess *);
 
 /* cache_vrt_vmod.c */
 void VMOD_Init(void);
diff --git a/bin/varnishd/cache_esi_deliver.c b/bin/varnishd/cache_esi_deliver.c
index ca1c517..d5e7c90 100644
--- a/bin/varnishd/cache_esi_deliver.c
+++ b/bin/varnishd/cache_esi_deliver.c
@@ -195,32 +195,33 @@ ved_decode_len(uint8_t **pp)
  */
 
 static void
-ved_pretend_gzip(struct sess *sp, uint8_t *p, ssize_t l)
+ved_pretend_gzip(const struct sess *sp, const uint8_t *p, ssize_t l)
 {
-	ssize_t ll;
 	uint8_t buf[5];
 	char chunk[20];
+	uint16_t lx;
 
 	while (l > 0) {
-		ll = l;
-		if (ll > 65535)
-			ll = 65535;
+		if (l > 65535)
+			lx = 65535;
+		else
+			lx = (uint16_t)l;
 		buf[0] = 0;
-		vle16enc(buf + 1, ll);
-		vle16enc(buf + 3, ~ll);
+		vle16enc(buf + 1, lx);
+		vle16enc(buf + 3, ~lx);
 		if (sp->wrk->res_mode & RES_CHUNKED) {
-			bprintf(chunk, "%jx\r\n", (intmax_t)ll + 5);
+			bprintf(chunk, "%x\r\n", lx + 5);
 			(void)WRW_Write(sp->wrk, chunk, -1);
 		}
 		(void)WRW_Write(sp->wrk, buf, sizeof buf);
-		(void)WRW_Write(sp->wrk, p, ll);
+		(void)WRW_Write(sp->wrk, p, lx);
 		if (sp->wrk->res_mode & RES_CHUNKED) 
 			(void)WRW_Write(sp->wrk, "\r\n", -1);
 		(void)WRW_Flush(sp->wrk);
-		sp->wrk->crc = crc32(sp->wrk->crc, p, ll);
-		sp->wrk->l_crc += ll;
-		l -= ll;
-		p += ll;
+		sp->wrk->crc = crc32(sp->wrk->crc, p, lx);
+		sp->wrk->l_crc += lx;
+		l -= lx;
+		p += lx;
 	}
 }
 
@@ -233,14 +234,24 @@ ESI_Deliver(struct sess *sp)
 	struct storage *st;
 	uint8_t *p, *e, *q, *r;
 	unsigned off;
-	ssize_t l, l_icrc;
-	uint32_t icrc;
+	ssize_t l, l_icrc = 0;
+	uint32_t icrc = 0;
 	uint8_t tailbuf[8 + 5];
 	int isgzip;
+	struct vgz *vgz = NULL;
+	char obuf[1024 * params->gzip_stack_buffer];
+	ssize_t obufl = 0;
+	size_t dl;
+	const void *dp;
+	int i;
 
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 	st = sp->obj->esidata;
 	AN(st);
+	assert(sizeof obuf >= 1024);
+
+	obuf[0] = 0;	/* For flexelint */
+
 	p = st->ptr;
 	e = st->ptr + st->len;
 
@@ -252,16 +263,24 @@ ESI_Deliver(struct sess *sp)
 	}
 
 	if (sp->esi_level == 0) {
-		if (isgzip) {
+		/*
+		 * Only the top level document gets to decide this.
+		 */
+		if (isgzip && !(sp->wrk->res_mode & RES_GUNZIP)) {
 			sp->wrk->gzip_resp = 1;
 			sp->wrk->crc = crc32(0L, Z_NULL, 0);
 		} else 
 			sp->wrk->gzip_resp = 0;
 	}
 
+	if (isgzip && !sp->wrk->gzip_resp) {
+		vgz = VGZ_NewUngzip(sp, sp->wrk->ws);
+		obufl = 0;
+	}
+
 	st = VTAILQ_FIRST(&sp->obj->store);
 	off = 0;
-
+	
 	while (p < e) {
 		switch (*p) {
 		case VEC_V1:
@@ -271,39 +290,65 @@ ESI_Deliver(struct sess *sp)
 			r = p;
 			q = (void*)strchr((const char*)p, '\0');
 			p = q + 1;
-			if (sp->wrk->gzip_resp) {
-				if (isgzip) {
-					assert(*p == VEC_C1 || *p == VEC_C2 ||
-					    *p == VEC_C8);
-					l_icrc = ved_decode_len(&p);
-					icrc = vbe32dec(p);
-					p += 4;
-					sp->wrk->crc = crc32_combine(
-					    sp->wrk->crc, icrc, l_icrc);
-					sp->wrk->l_crc += l_icrc;
-if (sp->esi_level > 0 && off == 0) {
-assert(l > 10);
+			if (isgzip) {
+				assert(*p == VEC_C1 || *p == VEC_C2 ||
+				    *p == VEC_C8);
+				l_icrc = ved_decode_len(&p);
+				icrc = vbe32dec(p);
+				p += 4;
+			}
+			if (sp->wrk->gzip_resp && isgzip) {
+				/*
+				 * We have a gzip'ed VEC and delivers a
+				 * gzip'ed ESI response.
+				 */
+				sp->wrk->crc = crc32_combine(
+				    sp->wrk->crc, icrc, l_icrc);
+				sp->wrk->l_crc += l_icrc;
+				if (sp->esi_level > 0 && off == 0) {
+					/*
+					 * Skip the GZ header, we know it is
+					 * 10 bytes: we made it ourself.
+					 */
+					assert(l > 10);
 					ved_sendchunk(sp, NULL, 0,
-					    st->ptr + off + 10, l - 10);
-} else {
-					ved_sendchunk(sp, r, q - r,
-					    st->ptr + off, l);
-}
-					off += l;
-				} else {
-					ved_pretend_gzip(sp,
-					    st->ptr + off, l);
-					off += l;
-				}
-			} else {
-				if (isgzip) {
-					INCOMPL();
+					    st->ptr + 10, l - 10);
 				} else {
 					ved_sendchunk(sp, r, q - r,
 					    st->ptr + off, l);
-					off += l;
 				}
+			} else if (sp->wrk->gzip_resp) {
+				/*
+				 * A gzip'ed ESI response, but the VEC was
+				 * not gzip'ed.
+				 */
+				ved_pretend_gzip(sp, st->ptr + off, l);
+				off += l;
+			} else if (isgzip) {
+				/*
+				 * A gzip'ed VEC, but ungzip'ed ESI response
+				 */
+				AN(vgz);
+				VGZ_Ibuf(vgz, st->ptr + off, l);
+				do {
+					if (obufl == sizeof obuf) {
+						ved_sendchunk(sp, NULL, 0,
+						    obuf, obufl);
+						obufl = 0;
+					}
+					VGZ_Obuf(vgz, obuf + obufl,
+					    sizeof obuf - obufl);
+					i = VGZ_Gunzip(vgz, &dp, &dl);
+					assert(i == Z_OK || i == Z_STREAM_END);
+					obufl += dl;
+				} while (!VGZ_IbufEmpty(vgz));
+			} else {
+				/*
+				 * Ungzip'ed VEC, ungzip'ed ESI response
+				 */
+				ved_sendchunk(sp, r, q - r, st->ptr + off, l);
 			}
+			off += l;
 			break;
 		case VEC_S1:
 		case VEC_S2:
@@ -329,6 +374,11 @@ assert(l > 10);
 			INCOMPL();
 		}
 	}
+	if (vgz != NULL) {
+		if (obufl > 0) 
+			ved_sendchunk(sp, NULL, 0, obuf, obufl);
+		VGZ_Destroy(&vgz);
+	}
 	if (sp->wrk->gzip_resp && sp->esi_level == 0) {
 		/* Emit a gzip literal block with finish bit set */
 		tailbuf[0] = 0x01;
@@ -352,8 +402,8 @@ assert(l > 10);
  * Include an object in a gzip'ed ESI object delivery
  */
 
-static unsigned
-ved_deliver_byterange(struct sess *sp, ssize_t low, ssize_t high)
+static uint8_t
+ved_deliver_byterange(const struct sess *sp, ssize_t low, ssize_t high)
 {
 	struct storage *st;
 	ssize_t l, lx;
@@ -392,7 +442,7 @@ ved_deliver_byterange(struct sess *sp, ssize_t low, ssize_t high)
 }
 
 void
-ESI_DeliverChild(struct sess *sp)
+ESI_DeliverChild(const struct sess *sp)
 {
 	struct storage *st;
 	struct object *obj;
@@ -400,7 +450,7 @@ ESI_DeliverChild(struct sess *sp)
 	u_char *p, cc;
 	uint32_t icrc;
 	uint32_t ilen;
-	uint8_t pad[6];
+	uint8_t pad[6] = { 0 };
 
 	if (!sp->obj->gziped) {
 		VTAILQ_FOREACH(st, &sp->obj->store, list)
@@ -433,7 +483,7 @@ ESI_DeliverChild(struct sess *sp)
 	 */
 	cc = ved_deliver_byterange(sp, start/8, last/8);
 //printf("CC_LAST %x\n", cc);
-	cc &= ~(1 << (start & 7));
+	cc &= ~(1U << (start & 7));
 	ved_sendchunk(sp, NULL, 0, &cc, 1);
 	cc = ved_deliver_byterange(sp, 1 + last/8, stop/8);
 //printf("CC_STOP %x (%d)\n", cc, (int)(stop & 7));
@@ -474,7 +524,7 @@ ESI_DeliverChild(struct sess *sp)
 		lpad = 6;
 		break;
 	default:
-		AZ(stop & 7);
+		INCOMPL();
 	}
 	if (lpad > 0)
 		ved_sendchunk(sp, NULL, 0, pad, lpad);
diff --git a/bin/varnishd/cache_esi_fetch.c b/bin/varnishd/cache_esi_fetch.c
index 3e05d7c..0bf2f7c 100644
--- a/bin/varnishd/cache_esi_fetch.c
+++ b/bin/varnishd/cache_esi_fetch.c
@@ -257,6 +257,8 @@ vfp_esi_bytes_gg(struct sess *sp, struct http_conn *htc, size_t bytes)
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 	vef = sp->wrk->vef_priv;
 	CHECK_OBJ_NOTNULL(vef, VEF_MAGIC);
+	assert(sizeof ibuf >= 1024);
+	ibuf2[0] = 0; /* For Flexelint */
 
 	while (bytes > 0) {
 		w = vef_read(htc, ibuf, sizeof ibuf, bytes);
diff --git a/bin/varnishd/flint.sh b/bin/varnishd/flint.sh
index 3116420..d163374 100755
--- a/bin/varnishd/flint.sh
+++ b/bin/varnishd/flint.sh
@@ -5,6 +5,7 @@ flexelint \
 	flint.lnt \
 	-I. \
 	-I../../include \
+	-I../../lib/libvgz \
 	-I../.. \
 	-I/usr/local/include \
 	-DVARNISH_STATE_DIR=\"foo\" \
diff --git a/bin/varnishtest/tests/e00024.vtc b/bin/varnishtest/tests/e00024.vtc
index ad3ecd4..e884ef3 100644
--- a/bin/varnishtest/tests/e00024.vtc
+++ b/bin/varnishtest/tests/e00024.vtc
@@ -80,6 +80,12 @@ client c1 {
 	gunzip
 	expect resp.status == 200
 	expect resp.bodylen == 252
+
+	txreq 
+	rxresp
+	expect resp.http.content-encoding == resp.http.content-encoding
+	expect resp.status == 200
+	expect resp.bodylen == 252
 } -run
 
 varnish v1 -expect esi_errors == 0



From phk at varnish-cache.org  Mon Jan 24 08:58:20 2011
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Mon, 24 Jan 2011 09:58:20 +0100
Subject: [master] b5f08fd Rename our version og zlib.h to vgz.h to prevent
	contamination from host system.
Message-ID: <E1PhIFg-0002U9-BT@project.varnish-software.com>

commit b5f08fdcdeeb6f4cc118b399832d0fe66d3c2139
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Jan 24 08:57:59 2011 +0000

    Rename our version og zlib.h to vgz.h to prevent contamination from
    host system.

diff --git a/lib/libvgz/vgz.h b/lib/libvgz/vgz.h
new file mode 100644
index 0000000..d6855be
--- /dev/null
+++ b/lib/libvgz/vgz.h
@@ -0,0 +1,1615 @@
+/* zlib.h -- interface of the 'zlib' general purpose compression library
+  version 1.2.5, April 19th, 2010
+
+  Copyright (C) 1995-2010 Jean-loup Gailly and Mark Adler
+
+  This software is provided 'as-is', without any express or implied
+  warranty.  In no event will the authors be held liable for any damages
+  arising from the use of this software.
+
+  Permission is granted to anyone to use this software for any purpose,
+  including commercial applications, and to alter it and redistribute it
+  freely, subject to the following restrictions:
+
+  1. The origin of this software must not be misrepresented; you must not
+     claim that you wrote the original software. If you use this software
+     in a product, an acknowledgment in the product documentation would be
+     appreciated but is not required.
+  2. Altered source versions must be plainly marked as such, and must not be
+     misrepresented as being the original software.
+  3. This notice may not be removed or altered from any source distribution.
+
+  Jean-loup Gailly        Mark Adler
+  jloup at gzip.org          madler at alumni.caltech.edu
+
+
+  The data format used by the zlib library is described by RFCs (Request for
+  Comments) 1950 to 1952 in the files http://www.ietf.org/rfc/rfc1950.txt
+  (zlib format), rfc1951.txt (deflate format) and rfc1952.txt (gzip format).
+*/
+
+#ifndef ZLIB_H
+#define ZLIB_H
+
+#include "zconf.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#define ZLIB_VERSION "1.2.5"
+#define ZLIB_VERNUM 0x1250
+#define ZLIB_VER_MAJOR 1
+#define ZLIB_VER_MINOR 2
+#define ZLIB_VER_REVISION 5
+#define ZLIB_VER_SUBREVISION 0
+
+/*
+    The 'zlib' compression library provides in-memory compression and
+  decompression functions, including integrity checks of the uncompressed data.
+  This version of the library supports only one compression method (deflation)
+  but other algorithms will be added later and will have the same stream
+  interface.
+
+    Compression can be done in a single step if the buffers are large enough,
+  or can be done by repeated calls of the compression function.  In the latter
+  case, the application must provide more input and/or consume the output
+  (providing more output space) before each call.
+
+    The compressed data format used by default by the in-memory functions is
+  the zlib format, which is a zlib wrapper documented in RFC 1950, wrapped
+  around a deflate stream, which is itself documented in RFC 1951.
+
+    The library also supports reading and writing files in gzip (.gz) format
+  with an interface similar to that of stdio using the functions that start
+  with "gz".  The gzip format is different from the zlib format.  gzip is a
+  gzip wrapper, documented in RFC 1952, wrapped around a deflate stream.
+
+    This library can optionally read and write gzip streams in memory as well.
+
+    The zlib format was designed to be compact and fast for use in memory
+  and on communications channels.  The gzip format was designed for single-
+  file compression on file systems, has a larger header than zlib to maintain
+  directory information, and uses a different, slower check method than zlib.
+
+    The library does not install any signal handler.  The decoder checks
+  the consistency of the compressed data, so the library should never crash
+  even in case of corrupted input.
+*/
+
+typedef voidpf (*alloc_func) OF((voidpf opaque, uInt items, uInt size));
+typedef void   (*free_func)  OF((voidpf opaque, voidpf address));
+
+struct internal_state;
+
+typedef struct z_stream_s {
+    Bytef    *next_in;  /* next input byte */
+    uInt     avail_in;  /* number of bytes available at next_in */
+    uLong    total_in;  /* total nb of input bytes read so far */
+
+    Bytef    *next_out; /* next output byte should be put there */
+    uInt     avail_out; /* remaining free space at next_out */
+    uLong    total_out; /* total nb of bytes output so far */
+
+    const char *msg;    /* last error message, NULL if no error */
+    struct internal_state FAR *state; /* not visible by applications */
+
+    alloc_func zalloc;  /* used to allocate the internal state */
+    free_func  zfree;   /* used to free the internal state */
+    voidpf     opaque;  /* private data object passed to zalloc and zfree */
+
+    int     data_type;  /* best guess about the data type: binary or text */
+    uLong   adler;      /* adler32 value of the uncompressed data */
+    uLong   reserved;   /* reserved for future use */
+
+    uLong   start_bit;	/* Bit pos of first deflate block */
+    uLong   stop_bit;	/* Bit pos after last deflate block */
+    uLong   last_bit;	/* Bit pos of 'last' bit */
+} z_stream;
+
+typedef z_stream FAR *z_streamp;
+
+/*
+     gzip header information passed to and from zlib routines.  See RFC 1952
+  for more details on the meanings of these fields.
+*/
+typedef struct gz_header_s {
+    int     text;       /* true if compressed data believed to be text */
+    uLong   time;       /* modification time */
+    int     xflags;     /* extra flags (not used when writing a gzip file) */
+    int     os;         /* operating system */
+    Bytef   *extra;     /* pointer to extra field or Z_NULL if none */
+    uInt    extra_len;  /* extra field length (valid if extra != Z_NULL) */
+    uInt    extra_max;  /* space at extra (only when reading header) */
+    Bytef   *name;      /* pointer to zero-terminated file name or Z_NULL */
+    uInt    name_max;   /* space at name (only when reading header) */
+    Bytef   *comment;   /* pointer to zero-terminated comment or Z_NULL */
+    uInt    comm_max;   /* space at comment (only when reading header) */
+    int     hcrc;       /* true if there was or will be a header crc */
+    int     done;       /* true when done reading gzip header (not used
+                           when writing a gzip file) */
+} gz_header;
+
+typedef gz_header FAR *gz_headerp;
+
+/*
+     The application must update next_in and avail_in when avail_in has dropped
+   to zero.  It must update next_out and avail_out when avail_out has dropped
+   to zero.  The application must initialize zalloc, zfree and opaque before
+   calling the init function.  All other fields are set by the compression
+   library and must not be updated by the application.
+
+     The opaque value provided by the application will be passed as the first
+   parameter for calls of zalloc and zfree.  This can be useful for custom
+   memory management.  The compression library attaches no meaning to the
+   opaque value.
+
+     zalloc must return Z_NULL if there is not enough memory for the object.
+   If zlib is used in a multi-threaded application, zalloc and zfree must be
+   thread safe.
+
+     On 16-bit systems, the functions zalloc and zfree must be able to allocate
+   exactly 65536 bytes, but will not be required to allocate more than this if
+   the symbol MAXSEG_64K is defined (see zconf.h).  WARNING: On MSDOS, pointers
+   returned by zalloc for objects of exactly 65536 bytes *must* have their
+   offset normalized to zero.  The default allocation function provided by this
+   library ensures this (see zutil.c).  To reduce memory requirements and avoid
+   any allocation of 64K objects, at the expense of compression ratio, compile
+   the library with -DMAX_WBITS=14 (see zconf.h).
+
+     The fields total_in and total_out can be used for statistics or progress
+   reports.  After compression, total_in holds the total size of the
+   uncompressed data and may be saved for use in the decompressor (particularly
+   if the decompressor wants to decompress everything in a single step).
+*/
+
+                        /* constants */
+
+#define Z_NO_FLUSH      0
+#define Z_PARTIAL_FLUSH 1
+#define Z_SYNC_FLUSH    2
+#define Z_FULL_FLUSH    3
+#define Z_FINISH        4
+#define Z_BLOCK         5
+#define Z_TREES         6
+/* Allowed flush values; see deflate() and inflate() below for details */
+
+#define Z_OK            0
+#define Z_STREAM_END    1
+#define Z_NEED_DICT     2
+#define Z_ERRNO        (-1)
+#define Z_STREAM_ERROR (-2)
+#define Z_DATA_ERROR   (-3)
+#define Z_MEM_ERROR    (-4)
+#define Z_BUF_ERROR    (-5)
+#define Z_VERSION_ERROR (-6)
+/* Return codes for the compression/decompression functions. Negative values
+ * are errors, positive values are used for special but normal events.
+ */
+
+#define Z_NO_COMPRESSION         0
+#define Z_BEST_SPEED             1
+#define Z_BEST_COMPRESSION       9
+#define Z_DEFAULT_COMPRESSION  (-1)
+/* compression levels */
+
+#define Z_FILTERED            1
+#define Z_HUFFMAN_ONLY        2
+#define Z_RLE                 3
+#define Z_FIXED               4
+#define Z_DEFAULT_STRATEGY    0
+/* compression strategy; see deflateInit2() below for details */
+
+#define Z_BINARY   0
+#define Z_TEXT     1
+#define Z_ASCII    Z_TEXT   /* for compatibility with 1.2.2 and earlier */
+#define Z_UNKNOWN  2
+/* Possible values of the data_type field (though see inflate()) */
+
+#define Z_DEFLATED   8
+/* The deflate compression method (the only one supported in this version) */
+
+#define Z_NULL  0  /* for initializing zalloc, zfree, opaque */
+
+#define zlib_version zlibVersion()
+/* for compatibility with versions < 1.0.2 */
+
+
+                        /* basic functions */
+
+ZEXTERN const char * ZEXPORT zlibVersion OF((void));
+/* The application can compare zlibVersion and ZLIB_VERSION for consistency.
+   If the first character differs, the library code actually used is not
+   compatible with the zlib.h header file used by the application.  This check
+   is automatically made by deflateInit and inflateInit.
+ */
+
+/*
+ZEXTERN int ZEXPORT deflateInit OF((z_streamp strm, int level));
+
+     Initializes the internal stream state for compression.  The fields
+   zalloc, zfree and opaque must be initialized before by the caller.  If
+   zalloc and zfree are set to Z_NULL, deflateInit updates them to use default
+   allocation functions.
+
+     The compression level must be Z_DEFAULT_COMPRESSION, or between 0 and 9:
+   1 gives best speed, 9 gives best compression, 0 gives no compression at all
+   (the input data is simply copied a block at a time).  Z_DEFAULT_COMPRESSION
+   requests a default compromise between speed and compression (currently
+   equivalent to level 6).
+
+     deflateInit returns Z_OK if success, Z_MEM_ERROR if there was not enough
+   memory, Z_STREAM_ERROR if level is not a valid compression level, or
+   Z_VERSION_ERROR if the zlib library version (zlib_version) is incompatible
+   with the version assumed by the caller (ZLIB_VERSION).  msg is set to null
+   if there is no error message.  deflateInit does not perform any compression:
+   this will be done by deflate().
+*/
+
+
+ZEXTERN int ZEXPORT deflate OF((z_streamp strm, int flush));
+/*
+    deflate compresses as much data as possible, and stops when the input
+  buffer becomes empty or the output buffer becomes full.  It may introduce
+  some output latency (reading input without producing any output) except when
+  forced to flush.
+
+    The detailed semantics are as follows.  deflate performs one or both of the
+  following actions:
+
+  - Compress more input starting at next_in and update next_in and avail_in
+    accordingly.  If not all input can be processed (because there is not
+    enough room in the output buffer), next_in and avail_in are updated and
+    processing will resume at this point for the next call of deflate().
+
+  - Provide more output starting at next_out and update next_out and avail_out
+    accordingly.  This action is forced if the parameter flush is non zero.
+    Forcing flush frequently degrades the compression ratio, so this parameter
+    should be set only when necessary (in interactive applications).  Some
+    output may be provided even if flush is not set.
+
+    Before the call of deflate(), the application should ensure that at least
+  one of the actions is possible, by providing more input and/or consuming more
+  output, and updating avail_in or avail_out accordingly; avail_out should
+  never be zero before the call.  The application can consume the compressed
+  output when it wants, for example when the output buffer is full (avail_out
+  == 0), or after each call of deflate().  If deflate returns Z_OK and with
+  zero avail_out, it must be called again after making room in the output
+  buffer because there might be more output pending.
+
+    Normally the parameter flush is set to Z_NO_FLUSH, which allows deflate to
+  decide how much data to accumulate before producing output, in order to
+  maximize compression.
+
+    If the parameter flush is set to Z_SYNC_FLUSH, all pending output is
+  flushed to the output buffer and the output is aligned on a byte boundary, so
+  that the decompressor can get all input data available so far.  (In
+  particular avail_in is zero after the call if enough output space has been
+  provided before the call.) Flushing may degrade compression for some
+  compression algorithms and so it should be used only when necessary.  This
+  completes the current deflate block and follows it with an empty stored block
+  that is three bits plus filler bits to the next byte, followed by four bytes
+  (00 00 ff ff).
+
+    If flush is set to Z_PARTIAL_FLUSH, all pending output is flushed to the
+  output buffer, but the output is not aligned to a byte boundary.  All of the
+  input data so far will be available to the decompressor, as for Z_SYNC_FLUSH.
+  This completes the current deflate block and follows it with an empty fixed
+  codes block that is 10 bits long.  This assures that enough bytes are output
+  in order for the decompressor to finish the block before the empty fixed code
+  block.
+
+    If flush is set to Z_BLOCK, a deflate block is completed and emitted, as
+  for Z_SYNC_FLUSH, but the output is not aligned on a byte boundary, and up to
+  seven bits of the current block are held to be written as the next byte after
+  the next deflate block is completed.  In this case, the decompressor may not
+  be provided enough bits at this point in order to complete decompression of
+  the data provided so far to the compressor.  It may need to wait for the next
+  block to be emitted.  This is for advanced applications that need to control
+  the emission of deflate blocks.
+
+    If flush is set to Z_FULL_FLUSH, all output is flushed as with
+  Z_SYNC_FLUSH, and the compression state is reset so that decompression can
+  restart from this point if previous compressed data has been damaged or if
+  random access is desired.  Using Z_FULL_FLUSH too often can seriously degrade
+  compression.
+
+    If deflate returns with avail_out == 0, this function must be called again
+  with the same value of the flush parameter and more output space (updated
+  avail_out), until the flush is complete (deflate returns with non-zero
+  avail_out).  In the case of a Z_FULL_FLUSH or Z_SYNC_FLUSH, make sure that
+  avail_out is greater than six to avoid repeated flush markers due to
+  avail_out == 0 on return.
+
+    If the parameter flush is set to Z_FINISH, pending input is processed,
+  pending output is flushed and deflate returns with Z_STREAM_END if there was
+  enough output space; if deflate returns with Z_OK, this function must be
+  called again with Z_FINISH and more output space (updated avail_out) but no
+  more input data, until it returns with Z_STREAM_END or an error.  After
+  deflate has returned Z_STREAM_END, the only possible operations on the stream
+  are deflateReset or deflateEnd.
+
+    Z_FINISH can be used immediately after deflateInit if all the compression
+  is to be done in a single step.  In this case, avail_out must be at least the
+  value returned by deflateBound (see below).  If deflate does not return
+  Z_STREAM_END, then it must be called again as described above.
+
+    deflate() sets strm->adler to the adler32 checksum of all input read
+  so far (that is, total_in bytes).
+
+    deflate() may update strm->data_type if it can make a good guess about
+  the input data type (Z_BINARY or Z_TEXT).  In doubt, the data is considered
+  binary.  This field is only for information purposes and does not affect the
+  compression algorithm in any manner.
+
+    deflate() returns Z_OK if some progress has been made (more input
+  processed or more output produced), Z_STREAM_END if all input has been
+  consumed and all output has been produced (only when flush is set to
+  Z_FINISH), Z_STREAM_ERROR if the stream state was inconsistent (for example
+  if next_in or next_out was Z_NULL), Z_BUF_ERROR if no progress is possible
+  (for example avail_in or avail_out was zero).  Note that Z_BUF_ERROR is not
+  fatal, and deflate() can be called again with more input and more output
+  space to continue compressing.
+*/
+
+
+ZEXTERN int ZEXPORT deflateEnd OF((z_streamp strm));
+/*
+     All dynamically allocated data structures for this stream are freed.
+   This function discards any unprocessed input and does not flush any pending
+   output.
+
+     deflateEnd returns Z_OK if success, Z_STREAM_ERROR if the
+   stream state was inconsistent, Z_DATA_ERROR if the stream was freed
+   prematurely (some input or output was discarded).  In the error case, msg
+   may be set but then points to a static string (which must not be
+   deallocated).
+*/
+
+
+/*
+ZEXTERN int ZEXPORT inflateInit OF((z_streamp strm));
+
+     Initializes the internal stream state for decompression.  The fields
+   next_in, avail_in, zalloc, zfree and opaque must be initialized before by
+   the caller.  If next_in is not Z_NULL and avail_in is large enough (the
+   exact value depends on the compression method), inflateInit determines the
+   compression method from the zlib header and allocates all data structures
+   accordingly; otherwise the allocation will be deferred to the first call of
+   inflate.  If zalloc and zfree are set to Z_NULL, inflateInit updates them to
+   use default allocation functions.
+
+     inflateInit returns Z_OK if success, Z_MEM_ERROR if there was not enough
+   memory, Z_VERSION_ERROR if the zlib library version is incompatible with the
+   version assumed by the caller, or Z_STREAM_ERROR if the parameters are
+   invalid, such as a null pointer to the structure.  msg is set to null if
+   there is no error message.  inflateInit does not perform any decompression
+   apart from possibly reading the zlib header if present: actual decompression
+   will be done by inflate().  (So next_in and avail_in may be modified, but
+   next_out and avail_out are unused and unchanged.) The current implementation
+   of inflateInit() does not process any header information -- that is deferred
+   until inflate() is called.
+*/
+
+
+ZEXTERN int ZEXPORT inflate OF((z_streamp strm, int flush));
+/*
+    inflate decompresses as much data as possible, and stops when the input
+  buffer becomes empty or the output buffer becomes full.  It may introduce
+  some output latency (reading input without producing any output) except when
+  forced to flush.
+
+  The detailed semantics are as follows.  inflate performs one or both of the
+  following actions:
+
+  - Decompress more input starting at next_in and update next_in and avail_in
+    accordingly.  If not all input can be processed (because there is not
+    enough room in the output buffer), next_in is updated and processing will
+    resume at this point for the next call of inflate().
+
+  - Provide more output starting at next_out and update next_out and avail_out
+    accordingly.  inflate() provides as much output as possible, until there is
+    no more input data or no more space in the output buffer (see below about
+    the flush parameter).
+
+    Before the call of inflate(), the application should ensure that at least
+  one of the actions is possible, by providing more input and/or consuming more
+  output, and updating the next_* and avail_* values accordingly.  The
+  application can consume the uncompressed output when it wants, for example
+  when the output buffer is full (avail_out == 0), or after each call of
+  inflate().  If inflate returns Z_OK and with zero avail_out, it must be
+  called again after making room in the output buffer because there might be
+  more output pending.
+
+    The flush parameter of inflate() can be Z_NO_FLUSH, Z_SYNC_FLUSH, Z_FINISH,
+  Z_BLOCK, or Z_TREES.  Z_SYNC_FLUSH requests that inflate() flush as much
+  output as possible to the output buffer.  Z_BLOCK requests that inflate()
+  stop if and when it gets to the next deflate block boundary.  When decoding
+  the zlib or gzip format, this will cause inflate() to return immediately
+  after the header and before the first block.  When doing a raw inflate,
+  inflate() will go ahead and process the first block, and will return when it
+  gets to the end of that block, or when it runs out of data.
+
+    The Z_BLOCK option assists in appending to or combining deflate streams.
+  Also to assist in this, on return inflate() will set strm->data_type to the
+  number of unused bits in the last byte taken from strm->next_in, plus 64 if
+  inflate() is currently decoding the last block in the deflate stream, plus
+  128 if inflate() returned immediately after decoding an end-of-block code or
+  decoding the complete header up to just before the first byte of the deflate
+  stream.  The end-of-block will not be indicated until all of the uncompressed
+  data from that block has been written to strm->next_out.  The number of
+  unused bits may in general be greater than seven, except when bit 7 of
+  data_type is set, in which case the number of unused bits will be less than
+  eight.  data_type is set as noted here every time inflate() returns for all
+  flush options, and so can be used to determine the amount of currently
+  consumed input in bits.
+
+    The Z_TREES option behaves as Z_BLOCK does, but it also returns when the
+  end of each deflate block header is reached, before any actual data in that
+  block is decoded.  This allows the caller to determine the length of the
+  deflate block header for later use in random access within a deflate block.
+  256 is added to the value of strm->data_type when inflate() returns
+  immediately after reaching the end of the deflate block header.
+
+    inflate() should normally be called until it returns Z_STREAM_END or an
+  error.  However if all decompression is to be performed in a single step (a
+  single call of inflate), the parameter flush should be set to Z_FINISH.  In
+  this case all pending input is processed and all pending output is flushed;
+  avail_out must be large enough to hold all the uncompressed data.  (The size
+  of the uncompressed data may have been saved by the compressor for this
+  purpose.) The next operation on this stream must be inflateEnd to deallocate
+  the decompression state.  The use of Z_FINISH is never required, but can be
+  used to inform inflate that a faster approach may be used for the single
+  inflate() call.
+
+     In this implementation, inflate() always flushes as much output as
+  possible to the output buffer, and always uses the faster approach on the
+  first call.  So the only effect of the flush parameter in this implementation
+  is on the return value of inflate(), as noted below, or when it returns early
+  because Z_BLOCK or Z_TREES is used.
+
+     If a preset dictionary is needed after this call (see inflateSetDictionary
+  below), inflate sets strm->adler to the adler32 checksum of the dictionary
+  chosen by the compressor and returns Z_NEED_DICT; otherwise it sets
+  strm->adler to the adler32 checksum of all output produced so far (that is,
+  total_out bytes) and returns Z_OK, Z_STREAM_END or an error code as described
+  below.  At the end of the stream, inflate() checks that its computed adler32
+  checksum is equal to that saved by the compressor and returns Z_STREAM_END
+  only if the checksum is correct.
+
+    inflate() can decompress and check either zlib-wrapped or gzip-wrapped
+  deflate data.  The header type is detected automatically, if requested when
+  initializing with inflateInit2().  Any information contained in the gzip
+  header is not retained, so applications that need that information should
+  instead use raw inflate, see inflateInit2() below, or inflateBack() and
+  perform their own processing of the gzip header and trailer.
+
+    inflate() returns Z_OK if some progress has been made (more input processed
+  or more output produced), Z_STREAM_END if the end of the compressed data has
+  been reached and all uncompressed output has been produced, Z_NEED_DICT if a
+  preset dictionary is needed at this point, Z_DATA_ERROR if the input data was
+  corrupted (input stream not conforming to the zlib format or incorrect check
+  value), Z_STREAM_ERROR if the stream structure was inconsistent (for example
+  next_in or next_out was Z_NULL), Z_MEM_ERROR if there was not enough memory,
+  Z_BUF_ERROR if no progress is possible or if there was not enough room in the
+  output buffer when Z_FINISH is used.  Note that Z_BUF_ERROR is not fatal, and
+  inflate() can be called again with more input and more output space to
+  continue decompressing.  If Z_DATA_ERROR is returned, the application may
+  then call inflateSync() to look for a good compression block if a partial
+  recovery of the data is desired.
+*/
+
+
+ZEXTERN int ZEXPORT inflateEnd OF((z_streamp strm));
+/*
+     All dynamically allocated data structures for this stream are freed.
+   This function discards any unprocessed input and does not flush any pending
+   output.
+
+     inflateEnd returns Z_OK if success, Z_STREAM_ERROR if the stream state
+   was inconsistent.  In the error case, msg may be set but then points to a
+   static string (which must not be deallocated).
+*/
+
+
+                        /* Advanced functions */
+
+/*
+    The following functions are needed only in some special applications.
+*/
+
+/*
+ZEXTERN int ZEXPORT deflateInit2 OF((z_streamp strm,
+                                     int  level,
+                                     int  method,
+                                     int  windowBits,
+                                     int  memLevel,
+                                     int  strategy));
+
+     This is another version of deflateInit with more compression options.  The
+   fields next_in, zalloc, zfree and opaque must be initialized before by the
+   caller.
+
+     The method parameter is the compression method.  It must be Z_DEFLATED in
+   this version of the library.
+
+     The windowBits parameter is the base two logarithm of the window size
+   (the size of the history buffer).  It should be in the range 8..15 for this
+   version of the library.  Larger values of this parameter result in better
+   compression at the expense of memory usage.  The default value is 15 if
+   deflateInit is used instead.
+
+     windowBits can also be -8..-15 for raw deflate.  In this case, -windowBits
+   determines the window size.  deflate() will then generate raw deflate data
+   with no zlib header or trailer, and will not compute an adler32 check value.
+
+     windowBits can also be greater than 15 for optional gzip encoding.  Add
+   16 to windowBits to write a simple gzip header and trailer around the
+   compressed data instead of a zlib wrapper.  The gzip header will have no
+   file name, no extra data, no comment, no modification time (set to zero), no
+   header crc, and the operating system will be set to 255 (unknown).  If a
+   gzip stream is being written, strm->adler is a crc32 instead of an adler32.
+
+     The memLevel parameter specifies how much memory should be allocated
+   for the internal compression state.  memLevel=1 uses minimum memory but is
+   slow and reduces compression ratio; memLevel=9 uses maximum memory for
+   optimal speed.  The default value is 8.  See zconf.h for total memory usage
+   as a function of windowBits and memLevel.
+
+     The strategy parameter is used to tune the compression algorithm.  Use the
+   value Z_DEFAULT_STRATEGY for normal data, Z_FILTERED for data produced by a
+   filter (or predictor), Z_HUFFMAN_ONLY to force Huffman encoding only (no
+   string match), or Z_RLE to limit match distances to one (run-length
+   encoding).  Filtered data consists mostly of small values with a somewhat
+   random distribution.  In this case, the compression algorithm is tuned to
+   compress them better.  The effect of Z_FILTERED is to force more Huffman
+   coding and less string matching; it is somewhat intermediate between
+   Z_DEFAULT_STRATEGY and Z_HUFFMAN_ONLY.  Z_RLE is designed to be almost as
+   fast as Z_HUFFMAN_ONLY, but give better compression for PNG image data.  The
+   strategy parameter only affects the compression ratio but not the
+   correctness of the compressed output even if it is not set appropriately.
+   Z_FIXED prevents the use of dynamic Huffman codes, allowing for a simpler
+   decoder for special applications.
+
+     deflateInit2 returns Z_OK if success, Z_MEM_ERROR if there was not enough
+   memory, Z_STREAM_ERROR if any parameter is invalid (such as an invalid
+   method), or Z_VERSION_ERROR if the zlib library version (zlib_version) is
+   incompatible with the version assumed by the caller (ZLIB_VERSION).  msg is
+   set to null if there is no error message.  deflateInit2 does not perform any
+   compression: this will be done by deflate().
+*/
+
+ZEXTERN int ZEXPORT deflateSetDictionary OF((z_streamp strm,
+                                             const Bytef *dictionary,
+                                             uInt  dictLength));
+/*
+     Initializes the compression dictionary from the given byte sequence
+   without producing any compressed output.  This function must be called
+   immediately after deflateInit, deflateInit2 or deflateReset, before any call
+   of deflate.  The compressor and decompressor must use exactly the same
+   dictionary (see inflateSetDictionary).
+
+     The dictionary should consist of strings (byte sequences) that are likely
+   to be encountered later in the data to be compressed, with the most commonly
+   used strings preferably put towards the end of the dictionary.  Using a
+   dictionary is most useful when the data to be compressed is short and can be
+   predicted with good accuracy; the data can then be compressed better than
+   with the default empty dictionary.
+
+     Depending on the size of the compression data structures selected by
+   deflateInit or deflateInit2, a part of the dictionary may in effect be
+   discarded, for example if the dictionary is larger than the window size
+   provided in deflateInit or deflateInit2.  Thus the strings most likely to be
+   useful should be put at the end of the dictionary, not at the front.  In
+   addition, the current implementation of deflate will use at most the window
+   size minus 262 bytes of the provided dictionary.
+
+     Upon return of this function, strm->adler is set to the adler32 value
+   of the dictionary; the decompressor may later use this value to determine
+   which dictionary has been used by the compressor.  (The adler32 value
+   applies to the whole dictionary even if only a subset of the dictionary is
+   actually used by the compressor.) If a raw deflate was requested, then the
+   adler32 value is not computed and strm->adler is not set.
+
+     deflateSetDictionary returns Z_OK if success, or Z_STREAM_ERROR if a
+   parameter is invalid (e.g.  dictionary being Z_NULL) or the stream state is
+   inconsistent (for example if deflate has already been called for this stream
+   or if the compression method is bsort).  deflateSetDictionary does not
+   perform any compression: this will be done by deflate().
+*/
+
+ZEXTERN int ZEXPORT deflateCopy OF((z_streamp dest,
+                                    z_streamp source));
+/*
+     Sets the destination stream as a complete copy of the source stream.
+
+     This function can be useful when several compression strategies will be
+   tried, for example when there are several ways of pre-processing the input
+   data with a filter.  The streams that will be discarded should then be freed
+   by calling deflateEnd.  Note that deflateCopy duplicates the internal
+   compression state which can be quite large, so this strategy is slow and can
+   consume lots of memory.
+
+     deflateCopy returns Z_OK if success, Z_MEM_ERROR if there was not
+   enough memory, Z_STREAM_ERROR if the source stream state was inconsistent
+   (such as zalloc being Z_NULL).  msg is left unchanged in both source and
+   destination.
+*/
+
+ZEXTERN int ZEXPORT deflateReset OF((z_streamp strm));
+/*
+     This function is equivalent to deflateEnd followed by deflateInit,
+   but does not free and reallocate all the internal compression state.  The
+   stream will keep the same compression level and any other attributes that
+   may have been set by deflateInit2.
+
+     deflateReset returns Z_OK if success, or Z_STREAM_ERROR if the source
+   stream state was inconsistent (such as zalloc or state being Z_NULL).
+*/
+
+ZEXTERN int ZEXPORT deflateParams OF((z_streamp strm,
+                                      int level,
+                                      int strategy));
+/*
+     Dynamically update the compression level and compression strategy.  The
+   interpretation of level and strategy is as in deflateInit2.  This can be
+   used to switch between compression and straight copy of the input data, or
+   to switch to a different kind of input data requiring a different strategy.
+   If the compression level is changed, the input available so far is
+   compressed with the old level (and may be flushed); the new level will take
+   effect only at the next call of deflate().
+
+     Before the call of deflateParams, the stream state must be set as for
+   a call of deflate(), since the currently available input may have to be
+   compressed and flushed.  In particular, strm->avail_out must be non-zero.
+
+     deflateParams returns Z_OK if success, Z_STREAM_ERROR if the source
+   stream state was inconsistent or if a parameter was invalid, Z_BUF_ERROR if
+   strm->avail_out was zero.
+*/
+
+ZEXTERN int ZEXPORT deflateTune OF((z_streamp strm,
+                                    int good_length,
+                                    int max_lazy,
+                                    int nice_length,
+                                    int max_chain));
+/*
+     Fine tune deflate's internal compression parameters.  This should only be
+   used by someone who understands the algorithm used by zlib's deflate for
+   searching for the best matching string, and even then only by the most
+   fanatic optimizer trying to squeeze out the last compressed bit for their
+   specific input data.  Read the deflate.c source code for the meaning of the
+   max_lazy, good_length, nice_length, and max_chain parameters.
+
+     deflateTune() can be called after deflateInit() or deflateInit2(), and
+   returns Z_OK on success, or Z_STREAM_ERROR for an invalid deflate stream.
+ */
+
+ZEXTERN uLong ZEXPORT deflateBound OF((z_streamp strm,
+                                       uLong sourceLen));
+/*
+     deflateBound() returns an upper bound on the compressed size after
+   deflation of sourceLen bytes.  It must be called after deflateInit() or
+   deflateInit2(), and after deflateSetHeader(), if used.  This would be used
+   to allocate an output buffer for deflation in a single pass, and so would be
+   called before deflate().
+*/
+
+ZEXTERN int ZEXPORT deflatePrime OF((z_streamp strm,
+                                     int bits,
+                                     int value));
+/*
+     deflatePrime() inserts bits in the deflate output stream.  The intent
+   is that this function is used to start off the deflate output with the bits
+   leftover from a previous deflate stream when appending to it.  As such, this
+   function can only be used for raw deflate, and must be used before the first
+   deflate() call after a deflateInit2() or deflateReset().  bits must be less
+   than or equal to 16, and that many of the least significant bits of value
+   will be inserted in the output.
+
+     deflatePrime returns Z_OK if success, or Z_STREAM_ERROR if the source
+   stream state was inconsistent.
+*/
+
+ZEXTERN int ZEXPORT deflateSetHeader OF((z_streamp strm,
+                                         gz_headerp head));
+/*
+     deflateSetHeader() provides gzip header information for when a gzip
+   stream is requested by deflateInit2().  deflateSetHeader() may be called
+   after deflateInit2() or deflateReset() and before the first call of
+   deflate().  The text, time, os, extra field, name, and comment information
+   in the provided gz_header structure are written to the gzip header (xflag is
+   ignored -- the extra flags are set according to the compression level).  The
+   caller must assure that, if not Z_NULL, name and comment are terminated with
+   a zero byte, and that if extra is not Z_NULL, that extra_len bytes are
+   available there.  If hcrc is true, a gzip header crc is included.  Note that
+   the current versions of the command-line version of gzip (up through version
+   1.3.x) do not support header crc's, and will report that it is a "multi-part
+   gzip file" and give up.
+
+     If deflateSetHeader is not used, the default gzip header has text false,
+   the time set to zero, and os set to 255, with no extra, name, or comment
+   fields.  The gzip header is returned to the default state by deflateReset().
+
+     deflateSetHeader returns Z_OK if success, or Z_STREAM_ERROR if the source
+   stream state was inconsistent.
+*/
+
+/*
+ZEXTERN int ZEXPORT inflateInit2 OF((z_streamp strm,
+                                     int  windowBits));
+
+     This is another version of inflateInit with an extra parameter.  The
+   fields next_in, avail_in, zalloc, zfree and opaque must be initialized
+   before by the caller.
+
+     The windowBits parameter is the base two logarithm of the maximum window
+   size (the size of the history buffer).  It should be in the range 8..15 for
+   this version of the library.  The default value is 15 if inflateInit is used
+   instead.  windowBits must be greater than or equal to the windowBits value
+   provided to deflateInit2() while compressing, or it must be equal to 15 if
+   deflateInit2() was not used.  If a compressed stream with a larger window
+   size is given as input, inflate() will return with the error code
+   Z_DATA_ERROR instead of trying to allocate a larger window.
+
+     windowBits can also be zero to request that inflate use the window size in
+   the zlib header of the compressed stream.
+
+     windowBits can also be -8..-15 for raw inflate.  In this case, -windowBits
+   determines the window size.  inflate() will then process raw deflate data,
+   not looking for a zlib or gzip header, not generating a check value, and not
+   looking for any check values for comparison at the end of the stream.  This
+   is for use with other formats that use the deflate compressed data format
+   such as zip.  Those formats provide their own check values.  If a custom
+   format is developed using the raw deflate format for compressed data, it is
+   recommended that a check value such as an adler32 or a crc32 be applied to
+   the uncompressed data as is done in the zlib, gzip, and zip formats.  For
+   most applications, the zlib format should be used as is.  Note that comments
+   above on the use in deflateInit2() applies to the magnitude of windowBits.
+
+     windowBits can also be greater than 15 for optional gzip decoding.  Add
+   32 to windowBits to enable zlib and gzip decoding with automatic header
+   detection, or add 16 to decode only the gzip format (the zlib format will
+   return a Z_DATA_ERROR).  If a gzip stream is being decoded, strm->adler is a
+   crc32 instead of an adler32.
+
+     inflateInit2 returns Z_OK if success, Z_MEM_ERROR if there was not enough
+   memory, Z_VERSION_ERROR if the zlib library version is incompatible with the
+   version assumed by the caller, or Z_STREAM_ERROR if the parameters are
+   invalid, such as a null pointer to the structure.  msg is set to null if
+   there is no error message.  inflateInit2 does not perform any decompression
+   apart from possibly reading the zlib header if present: actual decompression
+   will be done by inflate().  (So next_in and avail_in may be modified, but
+   next_out and avail_out are unused and unchanged.) The current implementation
+   of inflateInit2() does not process any header information -- that is
+   deferred until inflate() is called.
+*/
+
+ZEXTERN int ZEXPORT inflateSetDictionary OF((z_streamp strm,
+                                             const Bytef *dictionary,
+                                             uInt  dictLength));
+/*
+     Initializes the decompression dictionary from the given uncompressed byte
+   sequence.  This function must be called immediately after a call of inflate,
+   if that call returned Z_NEED_DICT.  The dictionary chosen by the compressor
+   can be determined from the adler32 value returned by that call of inflate.
+   The compressor and decompressor must use exactly the same dictionary (see
+   deflateSetDictionary).  For raw inflate, this function can be called
+   immediately after inflateInit2() or inflateReset() and before any call of
+   inflate() to set the dictionary.  The application must insure that the
+   dictionary that was used for compression is provided.
+
+     inflateSetDictionary returns Z_OK if success, Z_STREAM_ERROR if a
+   parameter is invalid (e.g.  dictionary being Z_NULL) or the stream state is
+   inconsistent, Z_DATA_ERROR if the given dictionary doesn't match the
+   expected one (incorrect adler32 value).  inflateSetDictionary does not
+   perform any decompression: this will be done by subsequent calls of
+   inflate().
+*/
+
+ZEXTERN int ZEXPORT inflateSync OF((z_streamp strm));
+/*
+     Skips invalid compressed data until a full flush point (see above the
+   description of deflate with Z_FULL_FLUSH) can be found, or until all
+   available input is skipped.  No output is provided.
+
+     inflateSync returns Z_OK if a full flush point has been found, Z_BUF_ERROR
+   if no more input was provided, Z_DATA_ERROR if no flush point has been
+   found, or Z_STREAM_ERROR if the stream structure was inconsistent.  In the
+   success case, the application may save the current current value of total_in
+   which indicates where valid compressed data was found.  In the error case,
+   the application may repeatedly call inflateSync, providing more input each
+   time, until success or end of the input data.
+*/
+
+ZEXTERN int ZEXPORT inflateCopy OF((z_streamp dest,
+                                    z_streamp source));
+/*
+     Sets the destination stream as a complete copy of the source stream.
+
+     This function can be useful when randomly accessing a large stream.  The
+   first pass through the stream can periodically record the inflate state,
+   allowing restarting inflate at those points when randomly accessing the
+   stream.
+
+     inflateCopy returns Z_OK if success, Z_MEM_ERROR if there was not
+   enough memory, Z_STREAM_ERROR if the source stream state was inconsistent
+   (such as zalloc being Z_NULL).  msg is left unchanged in both source and
+   destination.
+*/
+
+ZEXTERN int ZEXPORT inflateReset OF((z_streamp strm));
+/*
+     This function is equivalent to inflateEnd followed by inflateInit,
+   but does not free and reallocate all the internal decompression state.  The
+   stream will keep attributes that may have been set by inflateInit2.
+
+     inflateReset returns Z_OK if success, or Z_STREAM_ERROR if the source
+   stream state was inconsistent (such as zalloc or state being Z_NULL).
+*/
+
+ZEXTERN int ZEXPORT inflateReset2 OF((z_streamp strm,
+                                      int windowBits));
+/*
+     This function is the same as inflateReset, but it also permits changing
+   the wrap and window size requests.  The windowBits parameter is interpreted
+   the same as it is for inflateInit2.
+
+     inflateReset2 returns Z_OK if success, or Z_STREAM_ERROR if the source
+   stream state was inconsistent (such as zalloc or state being Z_NULL), or if
+   the windowBits parameter is invalid.
+*/
+
+ZEXTERN int ZEXPORT inflatePrime OF((z_streamp strm,
+                                     int bits,
+                                     int value));
+/*
+     This function inserts bits in the inflate input stream.  The intent is
+   that this function is used to start inflating at a bit position in the
+   middle of a byte.  The provided bits will be used before any bytes are used
+   from next_in.  This function should only be used with raw inflate, and
+   should be used before the first inflate() call after inflateInit2() or
+   inflateReset().  bits must be less than or equal to 16, and that many of the
+   least significant bits of value will be inserted in the input.
+
+     If bits is negative, then the input stream bit buffer is emptied.  Then
+   inflatePrime() can be called again to put bits in the buffer.  This is used
+   to clear out bits leftover after feeding inflate a block description prior
+   to feeding inflate codes.
+
+     inflatePrime returns Z_OK if success, or Z_STREAM_ERROR if the source
+   stream state was inconsistent.
+*/
+
+ZEXTERN long ZEXPORT inflateMark OF((z_streamp strm));
+/*
+     This function returns two values, one in the lower 16 bits of the return
+   value, and the other in the remaining upper bits, obtained by shifting the
+   return value down 16 bits.  If the upper value is -1 and the lower value is
+   zero, then inflate() is currently decoding information outside of a block.
+   If the upper value is -1 and the lower value is non-zero, then inflate is in
+   the middle of a stored block, with the lower value equaling the number of
+   bytes from the input remaining to copy.  If the upper value is not -1, then
+   it is the number of bits back from the current bit position in the input of
+   the code (literal or length/distance pair) currently being processed.  In
+   that case the lower value is the number of bytes already emitted for that
+   code.
+
+     A code is being processed if inflate is waiting for more input to complete
+   decoding of the code, or if it has completed decoding but is waiting for
+   more output space to write the literal or match data.
+
+     inflateMark() is used to mark locations in the input data for random
+   access, which may be at bit positions, and to note those cases where the
+   output of a code may span boundaries of random access blocks.  The current
+   location in the input stream can be determined from avail_in and data_type
+   as noted in the description for the Z_BLOCK flush parameter for inflate.
+
+     inflateMark returns the value noted above or -1 << 16 if the provided
+   source stream state was inconsistent.
+*/
+
+ZEXTERN int ZEXPORT inflateGetHeader OF((z_streamp strm,
+                                         gz_headerp head));
+/*
+     inflateGetHeader() requests that gzip header information be stored in the
+   provided gz_header structure.  inflateGetHeader() may be called after
+   inflateInit2() or inflateReset(), and before the first call of inflate().
+   As inflate() processes the gzip stream, head->done is zero until the header
+   is completed, at which time head->done is set to one.  If a zlib stream is
+   being decoded, then head->done is set to -1 to indicate that there will be
+   no gzip header information forthcoming.  Note that Z_BLOCK or Z_TREES can be
+   used to force inflate() to return immediately after header processing is
+   complete and before any actual data is decompressed.
+
+     The text, time, xflags, and os fields are filled in with the gzip header
+   contents.  hcrc is set to true if there is a header CRC.  (The header CRC
+   was valid if done is set to one.) If extra is not Z_NULL, then extra_max
+   contains the maximum number of bytes to write to extra.  Once done is true,
+   extra_len contains the actual extra field length, and extra contains the
+   extra field, or that field truncated if extra_max is less than extra_len.
+   If name is not Z_NULL, then up to name_max characters are written there,
+   terminated with a zero unless the length is greater than name_max.  If
+   comment is not Z_NULL, then up to comm_max characters are written there,
+   terminated with a zero unless the length is greater than comm_max.  When any
+   of extra, name, or comment are not Z_NULL and the respective field is not
+   present in the header, then that field is set to Z_NULL to signal its
+   absence.  This allows the use of deflateSetHeader() with the returned
+   structure to duplicate the header.  However if those fields are set to
+   allocated memory, then the application will need to save those pointers
+   elsewhere so that they can be eventually freed.
+
+     If inflateGetHeader is not used, then the header information is simply
+   discarded.  The header is always checked for validity, including the header
+   CRC if present.  inflateReset() will reset the process to discard the header
+   information.  The application would need to call inflateGetHeader() again to
+   retrieve the header from the next gzip stream.
+
+     inflateGetHeader returns Z_OK if success, or Z_STREAM_ERROR if the source
+   stream state was inconsistent.
+*/
+
+/*
+ZEXTERN int ZEXPORT inflateBackInit OF((z_streamp strm, int windowBits,
+                                        unsigned char FAR *window));
+
+     Initialize the internal stream state for decompression using inflateBack()
+   calls.  The fields zalloc, zfree and opaque in strm must be initialized
+   before the call.  If zalloc and zfree are Z_NULL, then the default library-
+   derived memory allocation routines are used.  windowBits is the base two
+   logarithm of the window size, in the range 8..15.  window is a caller
+   supplied buffer of that size.  Except for special applications where it is
+   assured that deflate was used with small window sizes, windowBits must be 15
+   and a 32K byte window must be supplied to be able to decompress general
+   deflate streams.
+
+     See inflateBack() for the usage of these routines.
+
+     inflateBackInit will return Z_OK on success, Z_STREAM_ERROR if any of
+   the paramaters are invalid, Z_MEM_ERROR if the internal state could not be
+   allocated, or Z_VERSION_ERROR if the version of the library does not match
+   the version of the header file.
+*/
+
+typedef unsigned (*in_func) OF((void FAR *, unsigned char FAR * FAR *));
+typedef int (*out_func) OF((void FAR *, unsigned char FAR *, unsigned));
+
+ZEXTERN int ZEXPORT inflateBack OF((z_streamp strm,
+                                    in_func in, void FAR *in_desc,
+                                    out_func out, void FAR *out_desc));
+/*
+     inflateBack() does a raw inflate with a single call using a call-back
+   interface for input and output.  This is more efficient than inflate() for
+   file i/o applications in that it avoids copying between the output and the
+   sliding window by simply making the window itself the output buffer.  This
+   function trusts the application to not change the output buffer passed by
+   the output function, at least until inflateBack() returns.
+
+     inflateBackInit() must be called first to allocate the internal state
+   and to initialize the state with the user-provided window buffer.
+   inflateBack() may then be used multiple times to inflate a complete, raw
+   deflate stream with each call.  inflateBackEnd() is then called to free the
+   allocated state.
+
+     A raw deflate stream is one with no zlib or gzip header or trailer.
+   This routine would normally be used in a utility that reads zip or gzip
+   files and writes out uncompressed files.  The utility would decode the
+   header and process the trailer on its own, hence this routine expects only
+   the raw deflate stream to decompress.  This is different from the normal
+   behavior of inflate(), which expects either a zlib or gzip header and
+   trailer around the deflate stream.
+
+     inflateBack() uses two subroutines supplied by the caller that are then
+   called by inflateBack() for input and output.  inflateBack() calls those
+   routines until it reads a complete deflate stream and writes out all of the
+   uncompressed data, or until it encounters an error.  The function's
+   parameters and return types are defined above in the in_func and out_func
+   typedefs.  inflateBack() will call in(in_desc, &buf) which should return the
+   number of bytes of provided input, and a pointer to that input in buf.  If
+   there is no input available, in() must return zero--buf is ignored in that
+   case--and inflateBack() will return a buffer error.  inflateBack() will call
+   out(out_desc, buf, len) to write the uncompressed data buf[0..len-1].  out()
+   should return zero on success, or non-zero on failure.  If out() returns
+   non-zero, inflateBack() will return with an error.  Neither in() nor out()
+   are permitted to change the contents of the window provided to
+   inflateBackInit(), which is also the buffer that out() uses to write from.
+   The length written by out() will be at most the window size.  Any non-zero
+   amount of input may be provided by in().
+
+     For convenience, inflateBack() can be provided input on the first call by
+   setting strm->next_in and strm->avail_in.  If that input is exhausted, then
+   in() will be called.  Therefore strm->next_in must be initialized before
+   calling inflateBack().  If strm->next_in is Z_NULL, then in() will be called
+   immediately for input.  If strm->next_in is not Z_NULL, then strm->avail_in
+   must also be initialized, and then if strm->avail_in is not zero, input will
+   initially be taken from strm->next_in[0 ..  strm->avail_in - 1].
+
+     The in_desc and out_desc parameters of inflateBack() is passed as the
+   first parameter of in() and out() respectively when they are called.  These
+   descriptors can be optionally used to pass any information that the caller-
+   supplied in() and out() functions need to do their job.
+
+     On return, inflateBack() will set strm->next_in and strm->avail_in to
+   pass back any unused input that was provided by the last in() call.  The
+   return values of inflateBack() can be Z_STREAM_END on success, Z_BUF_ERROR
+   if in() or out() returned an error, Z_DATA_ERROR if there was a format error
+   in the deflate stream (in which case strm->msg is set to indicate the nature
+   of the error), or Z_STREAM_ERROR if the stream was not properly initialized.
+   In the case of Z_BUF_ERROR, an input or output error can be distinguished
+   using strm->next_in which will be Z_NULL only if in() returned an error.  If
+   strm->next_in is not Z_NULL, then the Z_BUF_ERROR was due to out() returning
+   non-zero.  (in() will always be called before out(), so strm->next_in is
+   assured to be defined if out() returns non-zero.) Note that inflateBack()
+   cannot return Z_OK.
+*/
+
+ZEXTERN int ZEXPORT inflateBackEnd OF((z_streamp strm));
+/*
+     All memory allocated by inflateBackInit() is freed.
+
+     inflateBackEnd() returns Z_OK on success, or Z_STREAM_ERROR if the stream
+   state was inconsistent.
+*/
+
+ZEXTERN uLong ZEXPORT zlibCompileFlags OF((void));
+/* Return flags indicating compile-time options.
+
+    Type sizes, two bits each, 00 = 16 bits, 01 = 32, 10 = 64, 11 = other:
+     1.0: size of uInt
+     3.2: size of uLong
+     5.4: size of voidpf (pointer)
+     7.6: size of z_off_t
+
+    Compiler, assembler, and debug options:
+     8: DEBUG
+     9: ASMV or ASMINF -- use ASM code
+     10: ZLIB_WINAPI -- exported functions use the WINAPI calling convention
+     11: 0 (reserved)
+
+    One-time table building (smaller code, but not thread-safe if true):
+     12: BUILDFIXED -- build static block decoding tables when needed
+     13: DYNAMIC_CRC_TABLE -- build CRC calculation tables when needed
+     14,15: 0 (reserved)
+
+    Library content (indicates missing functionality):
+     16: NO_GZCOMPRESS -- gz* functions cannot compress (to avoid linking
+                          deflate code when not needed)
+     17: NO_GZIP -- deflate can't write gzip streams, and inflate can't detect
+                    and decode gzip streams (to avoid linking crc code)
+     18-19: 0 (reserved)
+
+    Operation variations (changes in library functionality):
+     20: PKZIP_BUG_WORKAROUND -- slightly more permissive inflate
+     21: FASTEST -- deflate algorithm with only one, lowest compression level
+     22,23: 0 (reserved)
+
+    The sprintf variant used by gzprintf (zero is best):
+     24: 0 = vs*, 1 = s* -- 1 means limited to 20 arguments after the format
+     25: 0 = *nprintf, 1 = *printf -- 1 means gzprintf() not secure!
+     26: 0 = returns value, 1 = void -- 1 means inferred string length returned
+
+    Remainder:
+     27-31: 0 (reserved)
+ */
+
+
+                        /* utility functions */
+
+/*
+     The following utility functions are implemented on top of the basic
+   stream-oriented functions.  To simplify the interface, some default options
+   are assumed (compression level and memory usage, standard memory allocation
+   functions).  The source code of these utility functions can be modified if
+   you need special options.
+*/
+
+ZEXTERN int ZEXPORT compress OF((Bytef *dest,   uLongf *destLen,
+                                 const Bytef *source, uLong sourceLen));
+/*
+     Compresses the source buffer into the destination buffer.  sourceLen is
+   the byte length of the source buffer.  Upon entry, destLen is the total size
+   of the destination buffer, which must be at least the value returned by
+   compressBound(sourceLen).  Upon exit, destLen is the actual size of the
+   compressed buffer.
+
+     compress returns Z_OK if success, Z_MEM_ERROR if there was not
+   enough memory, Z_BUF_ERROR if there was not enough room in the output
+   buffer.
+*/
+
+ZEXTERN int ZEXPORT compress2 OF((Bytef *dest,   uLongf *destLen,
+                                  const Bytef *source, uLong sourceLen,
+                                  int level));
+/*
+     Compresses the source buffer into the destination buffer.  The level
+   parameter has the same meaning as in deflateInit.  sourceLen is the byte
+   length of the source buffer.  Upon entry, destLen is the total size of the
+   destination buffer, which must be at least the value returned by
+   compressBound(sourceLen).  Upon exit, destLen is the actual size of the
+   compressed buffer.
+
+     compress2 returns Z_OK if success, Z_MEM_ERROR if there was not enough
+   memory, Z_BUF_ERROR if there was not enough room in the output buffer,
+   Z_STREAM_ERROR if the level parameter is invalid.
+*/
+
+ZEXTERN uLong ZEXPORT compressBound OF((uLong sourceLen));
+/*
+     compressBound() returns an upper bound on the compressed size after
+   compress() or compress2() on sourceLen bytes.  It would be used before a
+   compress() or compress2() call to allocate the destination buffer.
+*/
+
+ZEXTERN int ZEXPORT uncompress OF((Bytef *dest,   uLongf *destLen,
+                                   const Bytef *source, uLong sourceLen));
+/*
+     Decompresses the source buffer into the destination buffer.  sourceLen is
+   the byte length of the source buffer.  Upon entry, destLen is the total size
+   of the destination buffer, which must be large enough to hold the entire
+   uncompressed data.  (The size of the uncompressed data must have been saved
+   previously by the compressor and transmitted to the decompressor by some
+   mechanism outside the scope of this compression library.) Upon exit, destLen
+   is the actual size of the uncompressed buffer.
+
+     uncompress returns Z_OK if success, Z_MEM_ERROR if there was not
+   enough memory, Z_BUF_ERROR if there was not enough room in the output
+   buffer, or Z_DATA_ERROR if the input data was corrupted or incomplete.
+*/
+
+
+                        /* gzip file access functions */
+
+/*
+     This library supports reading and writing files in gzip (.gz) format with
+   an interface similar to that of stdio, using the functions that start with
+   "gz".  The gzip format is different from the zlib format.  gzip is a gzip
+   wrapper, documented in RFC 1952, wrapped around a deflate stream.
+*/
+
+typedef voidp gzFile;       /* opaque gzip file descriptor */
+
+/*
+ZEXTERN gzFile ZEXPORT gzopen OF((const char *path, const char *mode));
+
+     Opens a gzip (.gz) file for reading or writing.  The mode parameter is as
+   in fopen ("rb" or "wb") but can also include a compression level ("wb9") or
+   a strategy: 'f' for filtered data as in "wb6f", 'h' for Huffman-only
+   compression as in "wb1h", 'R' for run-length encoding as in "wb1R", or 'F'
+   for fixed code compression as in "wb9F".  (See the description of
+   deflateInit2 for more information about the strategy parameter.) Also "a"
+   can be used instead of "w" to request that the gzip stream that will be
+   written be appended to the file.  "+" will result in an error, since reading
+   and writing to the same gzip file is not supported.
+
+     gzopen can be used to read a file which is not in gzip format; in this
+   case gzread will directly read from the file without decompression.
+
+     gzopen returns NULL if the file could not be opened, if there was
+   insufficient memory to allocate the gzFile state, or if an invalid mode was
+   specified (an 'r', 'w', or 'a' was not provided, or '+' was provided).
+   errno can be checked to determine if the reason gzopen failed was that the
+   file could not be opened.
+*/
+
+ZEXTERN gzFile ZEXPORT gzdopen OF((int fd, const char *mode));
+/*
+     gzdopen associates a gzFile with the file descriptor fd.  File descriptors
+   are obtained from calls like open, dup, creat, pipe or fileno (if the file
+   has been previously opened with fopen).  The mode parameter is as in gzopen.
+
+     The next call of gzclose on the returned gzFile will also close the file
+   descriptor fd, just like fclose(fdopen(fd, mode)) closes the file descriptor
+   fd.  If you want to keep fd open, use fd = dup(fd_keep); gz = gzdopen(fd,
+   mode);.  The duplicated descriptor should be saved to avoid a leak, since
+   gzdopen does not close fd if it fails.
+
+     gzdopen returns NULL if there was insufficient memory to allocate the
+   gzFile state, if an invalid mode was specified (an 'r', 'w', or 'a' was not
+   provided, or '+' was provided), or if fd is -1.  The file descriptor is not
+   used until the next gz* read, write, seek, or close operation, so gzdopen
+   will not detect if fd is invalid (unless fd is -1).
+*/
+
+ZEXTERN int ZEXPORT gzbuffer OF((gzFile file, unsigned size));
+/*
+     Set the internal buffer size used by this library's functions.  The
+   default buffer size is 8192 bytes.  This function must be called after
+   gzopen() or gzdopen(), and before any other calls that read or write the
+   file.  The buffer memory allocation is always deferred to the first read or
+   write.  Two buffers are allocated, either both of the specified size when
+   writing, or one of the specified size and the other twice that size when
+   reading.  A larger buffer size of, for example, 64K or 128K bytes will
+   noticeably increase the speed of decompression (reading).
+
+     The new buffer size also affects the maximum length for gzprintf().
+
+     gzbuffer() returns 0 on success, or -1 on failure, such as being called
+   too late.
+*/
+
+ZEXTERN int ZEXPORT gzsetparams OF((gzFile file, int level, int strategy));
+/*
+     Dynamically update the compression level or strategy.  See the description
+   of deflateInit2 for the meaning of these parameters.
+
+     gzsetparams returns Z_OK if success, or Z_STREAM_ERROR if the file was not
+   opened for writing.
+*/
+
+ZEXTERN int ZEXPORT gzread OF((gzFile file, voidp buf, unsigned len));
+/*
+     Reads the given number of uncompressed bytes from the compressed file.  If
+   the input file was not in gzip format, gzread copies the given number of
+   bytes into the buffer.
+
+     After reaching the end of a gzip stream in the input, gzread will continue
+   to read, looking for another gzip stream, or failing that, reading the rest
+   of the input file directly without decompression.  The entire input file
+   will be read if gzread is called until it returns less than the requested
+   len.
+
+     gzread returns the number of uncompressed bytes actually read, less than
+   len for end of file, or -1 for error.
+*/
+
+ZEXTERN int ZEXPORT gzwrite OF((gzFile file,
+                                voidpc buf, unsigned len));
+/*
+     Writes the given number of uncompressed bytes into the compressed file.
+   gzwrite returns the number of uncompressed bytes written or 0 in case of
+   error.
+*/
+
+ZEXTERN int ZEXPORTVA gzprintf OF((gzFile file, const char *format, ...));
+/*
+     Converts, formats, and writes the arguments to the compressed file under
+   control of the format string, as in fprintf.  gzprintf returns the number of
+   uncompressed bytes actually written, or 0 in case of error.  The number of
+   uncompressed bytes written is limited to 8191, or one less than the buffer
+   size given to gzbuffer().  The caller should assure that this limit is not
+   exceeded.  If it is exceeded, then gzprintf() will return an error (0) with
+   nothing written.  In this case, there may also be a buffer overflow with
+   unpredictable consequences, which is possible only if zlib was compiled with
+   the insecure functions sprintf() or vsprintf() because the secure snprintf()
+   or vsnprintf() functions were not available.  This can be determined using
+   zlibCompileFlags().
+*/
+
+ZEXTERN int ZEXPORT gzputs OF((gzFile file, const char *s));
+/*
+     Writes the given null-terminated string to the compressed file, excluding
+   the terminating null character.
+
+     gzputs returns the number of characters written, or -1 in case of error.
+*/
+
+ZEXTERN char * ZEXPORT gzgets OF((gzFile file, char *buf, int len));
+/*
+     Reads bytes from the compressed file until len-1 characters are read, or a
+   newline character is read and transferred to buf, or an end-of-file
+   condition is encountered.  If any characters are read or if len == 1, the
+   string is terminated with a null character.  If no characters are read due
+   to an end-of-file or len < 1, then the buffer is left untouched.
+
+     gzgets returns buf which is a null-terminated string, or it returns NULL
+   for end-of-file or in case of error.  If there was an error, the contents at
+   buf are indeterminate.
+*/
+
+ZEXTERN int ZEXPORT gzputc OF((gzFile file, int c));
+/*
+     Writes c, converted to an unsigned char, into the compressed file.  gzputc
+   returns the value that was written, or -1 in case of error.
+*/
+
+ZEXTERN int ZEXPORT gzgetc OF((gzFile file));
+/*
+     Reads one byte from the compressed file.  gzgetc returns this byte or -1
+   in case of end of file or error.
+*/
+
+ZEXTERN int ZEXPORT gzungetc OF((int c, gzFile file));
+/*
+     Push one character back onto the stream to be read as the first character
+   on the next read.  At least one character of push-back is allowed.
+   gzungetc() returns the character pushed, or -1 on failure.  gzungetc() will
+   fail if c is -1, and may fail if a character has been pushed but not read
+   yet.  If gzungetc is used immediately after gzopen or gzdopen, at least the
+   output buffer size of pushed characters is allowed.  (See gzbuffer above.)
+   The pushed character will be discarded if the stream is repositioned with
+   gzseek() or gzrewind().
+*/
+
+ZEXTERN int ZEXPORT gzflush OF((gzFile file, int flush));
+/*
+     Flushes all pending output into the compressed file.  The parameter flush
+   is as in the deflate() function.  The return value is the zlib error number
+   (see function gzerror below).  gzflush is only permitted when writing.
+
+     If the flush parameter is Z_FINISH, the remaining data is written and the
+   gzip stream is completed in the output.  If gzwrite() is called again, a new
+   gzip stream will be started in the output.  gzread() is able to read such
+   concatented gzip streams.
+
+     gzflush should be called only when strictly necessary because it will
+   degrade compression if called too often.
+*/
+
+/*
+ZEXTERN z_off_t ZEXPORT gzseek OF((gzFile file,
+                                   z_off_t offset, int whence));
+
+     Sets the starting position for the next gzread or gzwrite on the given
+   compressed file.  The offset represents a number of bytes in the
+   uncompressed data stream.  The whence parameter is defined as in lseek(2);
+   the value SEEK_END is not supported.
+
+     If the file is opened for reading, this function is emulated but can be
+   extremely slow.  If the file is opened for writing, only forward seeks are
+   supported; gzseek then compresses a sequence of zeroes up to the new
+   starting position.
+
+     gzseek returns the resulting offset location as measured in bytes from
+   the beginning of the uncompressed stream, or -1 in case of error, in
+   particular if the file is opened for writing and the new starting position
+   would be before the current position.
+*/
+
+ZEXTERN int ZEXPORT    gzrewind OF((gzFile file));
+/*
+     Rewinds the given file. This function is supported only for reading.
+
+     gzrewind(file) is equivalent to (int)gzseek(file, 0L, SEEK_SET)
+*/
+
+/*
+ZEXTERN z_off_t ZEXPORT    gztell OF((gzFile file));
+
+     Returns the starting position for the next gzread or gzwrite on the given
+   compressed file.  This position represents a number of bytes in the
+   uncompressed data stream, and is zero when starting, even if appending or
+   reading a gzip stream from the middle of a file using gzdopen().
+
+     gztell(file) is equivalent to gzseek(file, 0L, SEEK_CUR)
+*/
+
+/*
+ZEXTERN z_off_t ZEXPORT gzoffset OF((gzFile file));
+
+     Returns the current offset in the file being read or written.  This offset
+   includes the count of bytes that precede the gzip stream, for example when
+   appending or when using gzdopen() for reading.  When reading, the offset
+   does not include as yet unused buffered input.  This information can be used
+   for a progress indicator.  On error, gzoffset() returns -1.
+*/
+
+ZEXTERN int ZEXPORT gzeof OF((gzFile file));
+/*
+     Returns true (1) if the end-of-file indicator has been set while reading,
+   false (0) otherwise.  Note that the end-of-file indicator is set only if the
+   read tried to go past the end of the input, but came up short.  Therefore,
+   just like feof(), gzeof() may return false even if there is no more data to
+   read, in the event that the last read request was for the exact number of
+   bytes remaining in the input file.  This will happen if the input file size
+   is an exact multiple of the buffer size.
+
+     If gzeof() returns true, then the read functions will return no more data,
+   unless the end-of-file indicator is reset by gzclearerr() and the input file
+   has grown since the previous end of file was detected.
+*/
+
+ZEXTERN int ZEXPORT gzdirect OF((gzFile file));
+/*
+     Returns true (1) if file is being copied directly while reading, or false
+   (0) if file is a gzip stream being decompressed.  This state can change from
+   false to true while reading the input file if the end of a gzip stream is
+   reached, but is followed by data that is not another gzip stream.
+
+     If the input file is empty, gzdirect() will return true, since the input
+   does not contain a gzip stream.
+
+     If gzdirect() is used immediately after gzopen() or gzdopen() it will
+   cause buffers to be allocated to allow reading the file to determine if it
+   is a gzip file.  Therefore if gzbuffer() is used, it should be called before
+   gzdirect().
+*/
+
+ZEXTERN int ZEXPORT    gzclose OF((gzFile file));
+/*
+     Flushes all pending output if necessary, closes the compressed file and
+   deallocates the (de)compression state.  Note that once file is closed, you
+   cannot call gzerror with file, since its structures have been deallocated.
+   gzclose must not be called more than once on the same file, just as free
+   must not be called more than once on the same allocation.
+
+     gzclose will return Z_STREAM_ERROR if file is not valid, Z_ERRNO on a
+   file operation error, or Z_OK on success.
+*/
+
+ZEXTERN int ZEXPORT gzclose_r OF((gzFile file));
+ZEXTERN int ZEXPORT gzclose_w OF((gzFile file));
+/*
+     Same as gzclose(), but gzclose_r() is only for use when reading, and
+   gzclose_w() is only for use when writing or appending.  The advantage to
+   using these instead of gzclose() is that they avoid linking in zlib
+   compression or decompression code that is not used when only reading or only
+   writing respectively.  If gzclose() is used, then both compression and
+   decompression code will be included the application when linking to a static
+   zlib library.
+*/
+
+ZEXTERN const char * ZEXPORT gzerror OF((gzFile file, int *errnum));
+/*
+     Returns the error message for the last error which occurred on the given
+   compressed file.  errnum is set to zlib error number.  If an error occurred
+   in the file system and not in the compression library, errnum is set to
+   Z_ERRNO and the application may consult errno to get the exact error code.
+
+     The application must not modify the returned string.  Future calls to
+   this function may invalidate the previously returned string.  If file is
+   closed, then the string previously returned by gzerror will no longer be
+   available.
+
+     gzerror() should be used to distinguish errors from end-of-file for those
+   functions above that do not distinguish those cases in their return values.
+*/
+
+ZEXTERN void ZEXPORT gzclearerr OF((gzFile file));
+/*
+     Clears the error and end-of-file flags for file.  This is analogous to the
+   clearerr() function in stdio.  This is useful for continuing to read a gzip
+   file that is being written concurrently.
+*/
+
+
+                        /* checksum functions */
+
+/*
+     These functions are not related to compression but are exported
+   anyway because they might be useful in applications using the compression
+   library.
+*/
+
+ZEXTERN uLong ZEXPORT adler32 OF((uLong adler, const Bytef *buf, uInt len));
+/*
+     Update a running Adler-32 checksum with the bytes buf[0..len-1] and
+   return the updated checksum.  If buf is Z_NULL, this function returns the
+   required initial value for the checksum.
+
+     An Adler-32 checksum is almost as reliable as a CRC32 but can be computed
+   much faster.
+
+   Usage example:
+
+     uLong adler = adler32(0L, Z_NULL, 0);
+
+     while (read_buffer(buffer, length) != EOF) {
+       adler = adler32(adler, buffer, length);
+     }
+     if (adler != original_adler) error();
+*/
+
+/*
+ZEXTERN uLong ZEXPORT adler32_combine OF((uLong adler1, uLong adler2,
+                                          z_off_t len2));
+
+     Combine two Adler-32 checksums into one.  For two sequences of bytes, seq1
+   and seq2 with lengths len1 and len2, Adler-32 checksums were calculated for
+   each, adler1 and adler2.  adler32_combine() returns the Adler-32 checksum of
+   seq1 and seq2 concatenated, requiring only adler1, adler2, and len2.
+*/
+
+ZEXTERN uLong ZEXPORT crc32   OF((uLong crc, const Bytef *buf, uInt len));
+/*
+     Update a running CRC-32 with the bytes buf[0..len-1] and return the
+   updated CRC-32.  If buf is Z_NULL, this function returns the required
+   initial value for the for the crc.  Pre- and post-conditioning (one's
+   complement) is performed within this function so it shouldn't be done by the
+   application.
+
+   Usage example:
+
+     uLong crc = crc32(0L, Z_NULL, 0);
+
+     while (read_buffer(buffer, length) != EOF) {
+       crc = crc32(crc, buffer, length);
+     }
+     if (crc != original_crc) error();
+*/
+
+/*
+ZEXTERN uLong ZEXPORT crc32_combine OF((uLong crc1, uLong crc2, z_off_t len2));
+
+     Combine two CRC-32 check values into one.  For two sequences of bytes,
+   seq1 and seq2 with lengths len1 and len2, CRC-32 check values were
+   calculated for each, crc1 and crc2.  crc32_combine() returns the CRC-32
+   check value of seq1 and seq2 concatenated, requiring only crc1, crc2, and
+   len2.
+*/
+
+
+                        /* various hacks, don't look :) */
+
+/* deflateInit and inflateInit are macros to allow checking the zlib version
+ * and the compiler's view of z_stream:
+ */
+ZEXTERN int ZEXPORT deflateInit_ OF((z_streamp strm, int level,
+                                     const char *version, int stream_size));
+ZEXTERN int ZEXPORT inflateInit_ OF((z_streamp strm,
+                                     const char *version, int stream_size));
+ZEXTERN int ZEXPORT deflateInit2_ OF((z_streamp strm, int  level, int  method,
+                                      int windowBits, int memLevel,
+                                      int strategy, const char *version,
+                                      int stream_size));
+ZEXTERN int ZEXPORT inflateInit2_ OF((z_streamp strm, int  windowBits,
+                                      const char *version, int stream_size));
+ZEXTERN int ZEXPORT inflateBackInit_ OF((z_streamp strm, int windowBits,
+                                         unsigned char FAR *window,
+                                         const char *version,
+                                         int stream_size));
+#define deflateInit(strm, level) \
+        deflateInit_((strm), (level),       ZLIB_VERSION, sizeof(z_stream))
+#define inflateInit(strm) \
+        inflateInit_((strm),                ZLIB_VERSION, sizeof(z_stream))
+#define deflateInit2(strm, level, method, windowBits, memLevel, strategy) \
+        deflateInit2_((strm),(level),(method),(windowBits),(memLevel),\
+                      (strategy),           ZLIB_VERSION, sizeof(z_stream))
+#define inflateInit2(strm, windowBits) \
+        inflateInit2_((strm), (windowBits), ZLIB_VERSION, sizeof(z_stream))
+#define inflateBackInit(strm, windowBits, window) \
+        inflateBackInit_((strm), (windowBits), (window), \
+                                            ZLIB_VERSION, sizeof(z_stream))
+
+/* provide 64-bit offset functions if _LARGEFILE64_SOURCE defined, and/or
+ * change the regular functions to 64 bits if _FILE_OFFSET_BITS is 64 (if
+ * both are true, the application gets the *64 functions, and the regular
+ * functions are changed to 64 bits) -- in case these are set on systems
+ * without large file support, _LFS64_LARGEFILE must also be true
+ */
+#if defined(_LARGEFILE64_SOURCE) && _LFS64_LARGEFILE-0
+   ZEXTERN gzFile ZEXPORT gzopen64 OF((const char *, const char *));
+   ZEXTERN z_off64_t ZEXPORT gzseek64 OF((gzFile, z_off64_t, int));
+   ZEXTERN z_off64_t ZEXPORT gztell64 OF((gzFile));
+   ZEXTERN z_off64_t ZEXPORT gzoffset64 OF((gzFile));
+   ZEXTERN uLong ZEXPORT adler32_combine64 OF((uLong, uLong, z_off64_t));
+   ZEXTERN uLong ZEXPORT crc32_combine64 OF((uLong, uLong, z_off64_t));
+#elif !defined(ZLIB_INTERNAL) && _FILE_OFFSET_BITS-0 == 64 && _LFS64_LARGEFILE-0
+#  define gzopen gzopen64
+#  define gzseek gzseek64
+#  define gztell gztell64
+#  define gzoffset gzoffset64
+#  define adler32_combine adler32_combine64
+#  define crc32_combine crc32_combine64
+#  ifdef _LARGEFILE64_SOURCE
+     ZEXTERN gzFile ZEXPORT gzopen64 OF((const char *, const char *));
+     ZEXTERN z_off_t ZEXPORT gzseek64 OF((gzFile, z_off_t, int));
+     ZEXTERN z_off_t ZEXPORT gztell64 OF((gzFile));
+     ZEXTERN z_off_t ZEXPORT gzoffset64 OF((gzFile));
+     ZEXTERN uLong ZEXPORT adler32_combine64 OF((uLong, uLong, z_off_t));
+     ZEXTERN uLong ZEXPORT crc32_combine64 OF((uLong, uLong, z_off_t));
+#  endif
+#else
+   ZEXTERN gzFile ZEXPORT gzopen OF((const char *, const char *));
+   ZEXTERN z_off_t ZEXPORT gzseek OF((gzFile, z_off_t, int));
+   ZEXTERN z_off_t ZEXPORT gztell OF((gzFile));
+   ZEXTERN z_off_t ZEXPORT gzoffset OF((gzFile));
+   ZEXTERN uLong ZEXPORT adler32_combine OF((uLong, uLong, z_off_t));
+   ZEXTERN uLong ZEXPORT crc32_combine OF((uLong, uLong, z_off_t));
+#endif
+
+/* hack for buggy compilers */
+#if !defined(ZUTIL_H) && !defined(NO_DUMMY_DECL)
+    struct internal_state {int dummy;};
+#endif
+
+/* undocumented functions */
+ZEXTERN const char   * ZEXPORT zError           OF((int));
+ZEXTERN int            ZEXPORT inflateSyncPoint OF((z_streamp));
+ZEXTERN const uLongf * ZEXPORT get_crc_table    OF((void));
+ZEXTERN int            ZEXPORT inflateUndermine OF((z_streamp, int));
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* ZLIB_H */
diff --git a/lib/libvgz/zlib.h b/lib/libvgz/zlib.h
deleted file mode 100644
index d6855be..0000000
--- a/lib/libvgz/zlib.h
+++ /dev/null
@@ -1,1615 +0,0 @@
-/* zlib.h -- interface of the 'zlib' general purpose compression library
-  version 1.2.5, April 19th, 2010
-
-  Copyright (C) 1995-2010 Jean-loup Gailly and Mark Adler
-
-  This software is provided 'as-is', without any express or implied
-  warranty.  In no event will the authors be held liable for any damages
-  arising from the use of this software.
-
-  Permission is granted to anyone to use this software for any purpose,
-  including commercial applications, and to alter it and redistribute it
-  freely, subject to the following restrictions:
-
-  1. The origin of this software must not be misrepresented; you must not
-     claim that you wrote the original software. If you use this software
-     in a product, an acknowledgment in the product documentation would be
-     appreciated but is not required.
-  2. Altered source versions must be plainly marked as such, and must not be
-     misrepresented as being the original software.
-  3. This notice may not be removed or altered from any source distribution.
-
-  Jean-loup Gailly        Mark Adler
-  jloup at gzip.org          madler at alumni.caltech.edu
-
-
-  The data format used by the zlib library is described by RFCs (Request for
-  Comments) 1950 to 1952 in the files http://www.ietf.org/rfc/rfc1950.txt
-  (zlib format), rfc1951.txt (deflate format) and rfc1952.txt (gzip format).
-*/
-
-#ifndef ZLIB_H
-#define ZLIB_H
-
-#include "zconf.h"
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-#define ZLIB_VERSION "1.2.5"
-#define ZLIB_VERNUM 0x1250
-#define ZLIB_VER_MAJOR 1
-#define ZLIB_VER_MINOR 2
-#define ZLIB_VER_REVISION 5
-#define ZLIB_VER_SUBREVISION 0
-
-/*
-    The 'zlib' compression library provides in-memory compression and
-  decompression functions, including integrity checks of the uncompressed data.
-  This version of the library supports only one compression method (deflation)
-  but other algorithms will be added later and will have the same stream
-  interface.
-
-    Compression can be done in a single step if the buffers are large enough,
-  or can be done by repeated calls of the compression function.  In the latter
-  case, the application must provide more input and/or consume the output
-  (providing more output space) before each call.
-
-    The compressed data format used by default by the in-memory functions is
-  the zlib format, which is a zlib wrapper documented in RFC 1950, wrapped
-  around a deflate stream, which is itself documented in RFC 1951.
-
-    The library also supports reading and writing files in gzip (.gz) format
-  with an interface similar to that of stdio using the functions that start
-  with "gz".  The gzip format is different from the zlib format.  gzip is a
-  gzip wrapper, documented in RFC 1952, wrapped around a deflate stream.
-
-    This library can optionally read and write gzip streams in memory as well.
-
-    The zlib format was designed to be compact and fast for use in memory
-  and on communications channels.  The gzip format was designed for single-
-  file compression on file systems, has a larger header than zlib to maintain
-  directory information, and uses a different, slower check method than zlib.
-
-    The library does not install any signal handler.  The decoder checks
-  the consistency of the compressed data, so the library should never crash
-  even in case of corrupted input.
-*/
-
-typedef voidpf (*alloc_func) OF((voidpf opaque, uInt items, uInt size));
-typedef void   (*free_func)  OF((voidpf opaque, voidpf address));
-
-struct internal_state;
-
-typedef struct z_stream_s {
-    Bytef    *next_in;  /* next input byte */
-    uInt     avail_in;  /* number of bytes available at next_in */
-    uLong    total_in;  /* total nb of input bytes read so far */
-
-    Bytef    *next_out; /* next output byte should be put there */
-    uInt     avail_out; /* remaining free space at next_out */
-    uLong    total_out; /* total nb of bytes output so far */
-
-    const char *msg;    /* last error message, NULL if no error */
-    struct internal_state FAR *state; /* not visible by applications */
-
-    alloc_func zalloc;  /* used to allocate the internal state */
-    free_func  zfree;   /* used to free the internal state */
-    voidpf     opaque;  /* private data object passed to zalloc and zfree */
-
-    int     data_type;  /* best guess about the data type: binary or text */
-    uLong   adler;      /* adler32 value of the uncompressed data */
-    uLong   reserved;   /* reserved for future use */
-
-    uLong   start_bit;	/* Bit pos of first deflate block */
-    uLong   stop_bit;	/* Bit pos after last deflate block */
-    uLong   last_bit;	/* Bit pos of 'last' bit */
-} z_stream;
-
-typedef z_stream FAR *z_streamp;
-
-/*
-     gzip header information passed to and from zlib routines.  See RFC 1952
-  for more details on the meanings of these fields.
-*/
-typedef struct gz_header_s {
-    int     text;       /* true if compressed data believed to be text */
-    uLong   time;       /* modification time */
-    int     xflags;     /* extra flags (not used when writing a gzip file) */
-    int     os;         /* operating system */
-    Bytef   *extra;     /* pointer to extra field or Z_NULL if none */
-    uInt    extra_len;  /* extra field length (valid if extra != Z_NULL) */
-    uInt    extra_max;  /* space at extra (only when reading header) */
-    Bytef   *name;      /* pointer to zero-terminated file name or Z_NULL */
-    uInt    name_max;   /* space at name (only when reading header) */
-    Bytef   *comment;   /* pointer to zero-terminated comment or Z_NULL */
-    uInt    comm_max;   /* space at comment (only when reading header) */
-    int     hcrc;       /* true if there was or will be a header crc */
-    int     done;       /* true when done reading gzip header (not used
-                           when writing a gzip file) */
-} gz_header;
-
-typedef gz_header FAR *gz_headerp;
-
-/*
-     The application must update next_in and avail_in when avail_in has dropped
-   to zero.  It must update next_out and avail_out when avail_out has dropped
-   to zero.  The application must initialize zalloc, zfree and opaque before
-   calling the init function.  All other fields are set by the compression
-   library and must not be updated by the application.
-
-     The opaque value provided by the application will be passed as the first
-   parameter for calls of zalloc and zfree.  This can be useful for custom
-   memory management.  The compression library attaches no meaning to the
-   opaque value.
-
-     zalloc must return Z_NULL if there is not enough memory for the object.
-   If zlib is used in a multi-threaded application, zalloc and zfree must be
-   thread safe.
-
-     On 16-bit systems, the functions zalloc and zfree must be able to allocate
-   exactly 65536 bytes, but will not be required to allocate more than this if
-   the symbol MAXSEG_64K is defined (see zconf.h).  WARNING: On MSDOS, pointers
-   returned by zalloc for objects of exactly 65536 bytes *must* have their
-   offset normalized to zero.  The default allocation function provided by this
-   library ensures this (see zutil.c).  To reduce memory requirements and avoid
-   any allocation of 64K objects, at the expense of compression ratio, compile
-   the library with -DMAX_WBITS=14 (see zconf.h).
-
-     The fields total_in and total_out can be used for statistics or progress
-   reports.  After compression, total_in holds the total size of the
-   uncompressed data and may be saved for use in the decompressor (particularly
-   if the decompressor wants to decompress everything in a single step).
-*/
-
-                        /* constants */
-
-#define Z_NO_FLUSH      0
-#define Z_PARTIAL_FLUSH 1
-#define Z_SYNC_FLUSH    2
-#define Z_FULL_FLUSH    3
-#define Z_FINISH        4
-#define Z_BLOCK         5
-#define Z_TREES         6
-/* Allowed flush values; see deflate() and inflate() below for details */
-
-#define Z_OK            0
-#define Z_STREAM_END    1
-#define Z_NEED_DICT     2
-#define Z_ERRNO        (-1)
-#define Z_STREAM_ERROR (-2)
-#define Z_DATA_ERROR   (-3)
-#define Z_MEM_ERROR    (-4)
-#define Z_BUF_ERROR    (-5)
-#define Z_VERSION_ERROR (-6)
-/* Return codes for the compression/decompression functions. Negative values
- * are errors, positive values are used for special but normal events.
- */
-
-#define Z_NO_COMPRESSION         0
-#define Z_BEST_SPEED             1
-#define Z_BEST_COMPRESSION       9
-#define Z_DEFAULT_COMPRESSION  (-1)
-/* compression levels */
-
-#define Z_FILTERED            1
-#define Z_HUFFMAN_ONLY        2
-#define Z_RLE                 3
-#define Z_FIXED               4
-#define Z_DEFAULT_STRATEGY    0
-/* compression strategy; see deflateInit2() below for details */
-
-#define Z_BINARY   0
-#define Z_TEXT     1
-#define Z_ASCII    Z_TEXT   /* for compatibility with 1.2.2 and earlier */
-#define Z_UNKNOWN  2
-/* Possible values of the data_type field (though see inflate()) */
-
-#define Z_DEFLATED   8
-/* The deflate compression method (the only one supported in this version) */
-
-#define Z_NULL  0  /* for initializing zalloc, zfree, opaque */
-
-#define zlib_version zlibVersion()
-/* for compatibility with versions < 1.0.2 */
-
-
-                        /* basic functions */
-
-ZEXTERN const char * ZEXPORT zlibVersion OF((void));
-/* The application can compare zlibVersion and ZLIB_VERSION for consistency.
-   If the first character differs, the library code actually used is not
-   compatible with the zlib.h header file used by the application.  This check
-   is automatically made by deflateInit and inflateInit.
- */
-
-/*
-ZEXTERN int ZEXPORT deflateInit OF((z_streamp strm, int level));
-
-     Initializes the internal stream state for compression.  The fields
-   zalloc, zfree and opaque must be initialized before by the caller.  If
-   zalloc and zfree are set to Z_NULL, deflateInit updates them to use default
-   allocation functions.
-
-     The compression level must be Z_DEFAULT_COMPRESSION, or between 0 and 9:
-   1 gives best speed, 9 gives best compression, 0 gives no compression at all
-   (the input data is simply copied a block at a time).  Z_DEFAULT_COMPRESSION
-   requests a default compromise between speed and compression (currently
-   equivalent to level 6).
-
-     deflateInit returns Z_OK if success, Z_MEM_ERROR if there was not enough
-   memory, Z_STREAM_ERROR if level is not a valid compression level, or
-   Z_VERSION_ERROR if the zlib library version (zlib_version) is incompatible
-   with the version assumed by the caller (ZLIB_VERSION).  msg is set to null
-   if there is no error message.  deflateInit does not perform any compression:
-   this will be done by deflate().
-*/
-
-
-ZEXTERN int ZEXPORT deflate OF((z_streamp strm, int flush));
-/*
-    deflate compresses as much data as possible, and stops when the input
-  buffer becomes empty or the output buffer becomes full.  It may introduce
-  some output latency (reading input without producing any output) except when
-  forced to flush.
-
-    The detailed semantics are as follows.  deflate performs one or both of the
-  following actions:
-
-  - Compress more input starting at next_in and update next_in and avail_in
-    accordingly.  If not all input can be processed (because there is not
-    enough room in the output buffer), next_in and avail_in are updated and
-    processing will resume at this point for the next call of deflate().
-
-  - Provide more output starting at next_out and update next_out and avail_out
-    accordingly.  This action is forced if the parameter flush is non zero.
-    Forcing flush frequently degrades the compression ratio, so this parameter
-    should be set only when necessary (in interactive applications).  Some
-    output may be provided even if flush is not set.
-
-    Before the call of deflate(), the application should ensure that at least
-  one of the actions is possible, by providing more input and/or consuming more
-  output, and updating avail_in or avail_out accordingly; avail_out should
-  never be zero before the call.  The application can consume the compressed
-  output when it wants, for example when the output buffer is full (avail_out
-  == 0), or after each call of deflate().  If deflate returns Z_OK and with
-  zero avail_out, it must be called again after making room in the output
-  buffer because there might be more output pending.
-
-    Normally the parameter flush is set to Z_NO_FLUSH, which allows deflate to
-  decide how much data to accumulate before producing output, in order to
-  maximize compression.
-
-    If the parameter flush is set to Z_SYNC_FLUSH, all pending output is
-  flushed to the output buffer and the output is aligned on a byte boundary, so
-  that the decompressor can get all input data available so far.  (In
-  particular avail_in is zero after the call if enough output space has been
-  provided before the call.) Flushing may degrade compression for some
-  compression algorithms and so it should be used only when necessary.  This
-  completes the current deflate block and follows it with an empty stored block
-  that is three bits plus filler bits to the next byte, followed by four bytes
-  (00 00 ff ff).
-
-    If flush is set to Z_PARTIAL_FLUSH, all pending output is flushed to the
-  output buffer, but the output is not aligned to a byte boundary.  All of the
-  input data so far will be available to the decompressor, as for Z_SYNC_FLUSH.
-  This completes the current deflate block and follows it with an empty fixed
-  codes block that is 10 bits long.  This assures that enough bytes are output
-  in order for the decompressor to finish the block before the empty fixed code
-  block.
-
-    If flush is set to Z_BLOCK, a deflate block is completed and emitted, as
-  for Z_SYNC_FLUSH, but the output is not aligned on a byte boundary, and up to
-  seven bits of the current block are held to be written as the next byte after
-  the next deflate block is completed.  In this case, the decompressor may not
-  be provided enough bits at this point in order to complete decompression of
-  the data provided so far to the compressor.  It may need to wait for the next
-  block to be emitted.  This is for advanced applications that need to control
-  the emission of deflate blocks.
-
-    If flush is set to Z_FULL_FLUSH, all output is flushed as with
-  Z_SYNC_FLUSH, and the compression state is reset so that decompression can
-  restart from this point if previous compressed data has been damaged or if
-  random access is desired.  Using Z_FULL_FLUSH too often can seriously degrade
-  compression.
-
-    If deflate returns with avail_out == 0, this function must be called again
-  with the same value of the flush parameter and more output space (updated
-  avail_out), until the flush is complete (deflate returns with non-zero
-  avail_out).  In the case of a Z_FULL_FLUSH or Z_SYNC_FLUSH, make sure that
-  avail_out is greater than six to avoid repeated flush markers due to
-  avail_out == 0 on return.
-
-    If the parameter flush is set to Z_FINISH, pending input is processed,
-  pending output is flushed and deflate returns with Z_STREAM_END if there was
-  enough output space; if deflate returns with Z_OK, this function must be
-  called again with Z_FINISH and more output space (updated avail_out) but no
-  more input data, until it returns with Z_STREAM_END or an error.  After
-  deflate has returned Z_STREAM_END, the only possible operations on the stream
-  are deflateReset or deflateEnd.
-
-    Z_FINISH can be used immediately after deflateInit if all the compression
-  is to be done in a single step.  In this case, avail_out must be at least the
-  value returned by deflateBound (see below).  If deflate does not return
-  Z_STREAM_END, then it must be called again as described above.
-
-    deflate() sets strm->adler to the adler32 checksum of all input read
-  so far (that is, total_in bytes).
-
-    deflate() may update strm->data_type if it can make a good guess about
-  the input data type (Z_BINARY or Z_TEXT).  In doubt, the data is considered
-  binary.  This field is only for information purposes and does not affect the
-  compression algorithm in any manner.
-
-    deflate() returns Z_OK if some progress has been made (more input
-  processed or more output produced), Z_STREAM_END if all input has been
-  consumed and all output has been produced (only when flush is set to
-  Z_FINISH), Z_STREAM_ERROR if the stream state was inconsistent (for example
-  if next_in or next_out was Z_NULL), Z_BUF_ERROR if no progress is possible
-  (for example avail_in or avail_out was zero).  Note that Z_BUF_ERROR is not
-  fatal, and deflate() can be called again with more input and more output
-  space to continue compressing.
-*/
-
-
-ZEXTERN int ZEXPORT deflateEnd OF((z_streamp strm));
-/*
-     All dynamically allocated data structures for this stream are freed.
-   This function discards any unprocessed input and does not flush any pending
-   output.
-
-     deflateEnd returns Z_OK if success, Z_STREAM_ERROR if the
-   stream state was inconsistent, Z_DATA_ERROR if the stream was freed
-   prematurely (some input or output was discarded).  In the error case, msg
-   may be set but then points to a static string (which must not be
-   deallocated).
-*/
-
-
-/*
-ZEXTERN int ZEXPORT inflateInit OF((z_streamp strm));
-
-     Initializes the internal stream state for decompression.  The fields
-   next_in, avail_in, zalloc, zfree and opaque must be initialized before by
-   the caller.  If next_in is not Z_NULL and avail_in is large enough (the
-   exact value depends on the compression method), inflateInit determines the
-   compression method from the zlib header and allocates all data structures
-   accordingly; otherwise the allocation will be deferred to the first call of
-   inflate.  If zalloc and zfree are set to Z_NULL, inflateInit updates them to
-   use default allocation functions.
-
-     inflateInit returns Z_OK if success, Z_MEM_ERROR if there was not enough
-   memory, Z_VERSION_ERROR if the zlib library version is incompatible with the
-   version assumed by the caller, or Z_STREAM_ERROR if the parameters are
-   invalid, such as a null pointer to the structure.  msg is set to null if
-   there is no error message.  inflateInit does not perform any decompression
-   apart from possibly reading the zlib header if present: actual decompression
-   will be done by inflate().  (So next_in and avail_in may be modified, but
-   next_out and avail_out are unused and unchanged.) The current implementation
-   of inflateInit() does not process any header information -- that is deferred
-   until inflate() is called.
-*/
-
-
-ZEXTERN int ZEXPORT inflate OF((z_streamp strm, int flush));
-/*
-    inflate decompresses as much data as possible, and stops when the input
-  buffer becomes empty or the output buffer becomes full.  It may introduce
-  some output latency (reading input without producing any output) except when
-  forced to flush.
-
-  The detailed semantics are as follows.  inflate performs one or both of the
-  following actions:
-
-  - Decompress more input starting at next_in and update next_in and avail_in
-    accordingly.  If not all input can be processed (because there is not
-    enough room in the output buffer), next_in is updated and processing will
-    resume at this point for the next call of inflate().
-
-  - Provide more output starting at next_out and update next_out and avail_out
-    accordingly.  inflate() provides as much output as possible, until there is
-    no more input data or no more space in the output buffer (see below about
-    the flush parameter).
-
-    Before the call of inflate(), the application should ensure that at least
-  one of the actions is possible, by providing more input and/or consuming more
-  output, and updating the next_* and avail_* values accordingly.  The
-  application can consume the uncompressed output when it wants, for example
-  when the output buffer is full (avail_out == 0), or after each call of
-  inflate().  If inflate returns Z_OK and with zero avail_out, it must be
-  called again after making room in the output buffer because there might be
-  more output pending.
-
-    The flush parameter of inflate() can be Z_NO_FLUSH, Z_SYNC_FLUSH, Z_FINISH,
-  Z_BLOCK, or Z_TREES.  Z_SYNC_FLUSH requests that inflate() flush as much
-  output as possible to the output buffer.  Z_BLOCK requests that inflate()
-  stop if and when it gets to the next deflate block boundary.  When decoding
-  the zlib or gzip format, this will cause inflate() to return immediately
-  after the header and before the first block.  When doing a raw inflate,
-  inflate() will go ahead and process the first block, and will return when it
-  gets to the end of that block, or when it runs out of data.
-
-    The Z_BLOCK option assists in appending to or combining deflate streams.
-  Also to assist in this, on return inflate() will set strm->data_type to the
-  number of unused bits in the last byte taken from strm->next_in, plus 64 if
-  inflate() is currently decoding the last block in the deflate stream, plus
-  128 if inflate() returned immediately after decoding an end-of-block code or
-  decoding the complete header up to just before the first byte of the deflate
-  stream.  The end-of-block will not be indicated until all of the uncompressed
-  data from that block has been written to strm->next_out.  The number of
-  unused bits may in general be greater than seven, except when bit 7 of
-  data_type is set, in which case the number of unused bits will be less than
-  eight.  data_type is set as noted here every time inflate() returns for all
-  flush options, and so can be used to determine the amount of currently
-  consumed input in bits.
-
-    The Z_TREES option behaves as Z_BLOCK does, but it also returns when the
-  end of each deflate block header is reached, before any actual data in that
-  block is decoded.  This allows the caller to determine the length of the
-  deflate block header for later use in random access within a deflate block.
-  256 is added to the value of strm->data_type when inflate() returns
-  immediately after reaching the end of the deflate block header.
-
-    inflate() should normally be called until it returns Z_STREAM_END or an
-  error.  However if all decompression is to be performed in a single step (a
-  single call of inflate), the parameter flush should be set to Z_FINISH.  In
-  this case all pending input is processed and all pending output is flushed;
-  avail_out must be large enough to hold all the uncompressed data.  (The size
-  of the uncompressed data may have been saved by the compressor for this
-  purpose.) The next operation on this stream must be inflateEnd to deallocate
-  the decompression state.  The use of Z_FINISH is never required, but can be
-  used to inform inflate that a faster approach may be used for the single
-  inflate() call.
-
-     In this implementation, inflate() always flushes as much output as
-  possible to the output buffer, and always uses the faster approach on the
-  first call.  So the only effect of the flush parameter in this implementation
-  is on the return value of inflate(), as noted below, or when it returns early
-  because Z_BLOCK or Z_TREES is used.
-
-     If a preset dictionary is needed after this call (see inflateSetDictionary
-  below), inflate sets strm->adler to the adler32 checksum of the dictionary
-  chosen by the compressor and returns Z_NEED_DICT; otherwise it sets
-  strm->adler to the adler32 checksum of all output produced so far (that is,
-  total_out bytes) and returns Z_OK, Z_STREAM_END or an error code as described
-  below.  At the end of the stream, inflate() checks that its computed adler32
-  checksum is equal to that saved by the compressor and returns Z_STREAM_END
-  only if the checksum is correct.
-
-    inflate() can decompress and check either zlib-wrapped or gzip-wrapped
-  deflate data.  The header type is detected automatically, if requested when
-  initializing with inflateInit2().  Any information contained in the gzip
-  header is not retained, so applications that need that information should
-  instead use raw inflate, see inflateInit2() below, or inflateBack() and
-  perform their own processing of the gzip header and trailer.
-
-    inflate() returns Z_OK if some progress has been made (more input processed
-  or more output produced), Z_STREAM_END if the end of the compressed data has
-  been reached and all uncompressed output has been produced, Z_NEED_DICT if a
-  preset dictionary is needed at this point, Z_DATA_ERROR if the input data was
-  corrupted (input stream not conforming to the zlib format or incorrect check
-  value), Z_STREAM_ERROR if the stream structure was inconsistent (for example
-  next_in or next_out was Z_NULL), Z_MEM_ERROR if there was not enough memory,
-  Z_BUF_ERROR if no progress is possible or if there was not enough room in the
-  output buffer when Z_FINISH is used.  Note that Z_BUF_ERROR is not fatal, and
-  inflate() can be called again with more input and more output space to
-  continue decompressing.  If Z_DATA_ERROR is returned, the application may
-  then call inflateSync() to look for a good compression block if a partial
-  recovery of the data is desired.
-*/
-
-
-ZEXTERN int ZEXPORT inflateEnd OF((z_streamp strm));
-/*
-     All dynamically allocated data structures for this stream are freed.
-   This function discards any unprocessed input and does not flush any pending
-   output.
-
-     inflateEnd returns Z_OK if success, Z_STREAM_ERROR if the stream state
-   was inconsistent.  In the error case, msg may be set but then points to a
-   static string (which must not be deallocated).
-*/
-
-
-                        /* Advanced functions */
-
-/*
-    The following functions are needed only in some special applications.
-*/
-
-/*
-ZEXTERN int ZEXPORT deflateInit2 OF((z_streamp strm,
-                                     int  level,
-                                     int  method,
-                                     int  windowBits,
-                                     int  memLevel,
-                                     int  strategy));
-
-     This is another version of deflateInit with more compression options.  The
-   fields next_in, zalloc, zfree and opaque must be initialized before by the
-   caller.
-
-     The method parameter is the compression method.  It must be Z_DEFLATED in
-   this version of the library.
-
-     The windowBits parameter is the base two logarithm of the window size
-   (the size of the history buffer).  It should be in the range 8..15 for this
-   version of the library.  Larger values of this parameter result in better
-   compression at the expense of memory usage.  The default value is 15 if
-   deflateInit is used instead.
-
-     windowBits can also be -8..-15 for raw deflate.  In this case, -windowBits
-   determines the window size.  deflate() will then generate raw deflate data
-   with no zlib header or trailer, and will not compute an adler32 check value.
-
-     windowBits can also be greater than 15 for optional gzip encoding.  Add
-   16 to windowBits to write a simple gzip header and trailer around the
-   compressed data instead of a zlib wrapper.  The gzip header will have no
-   file name, no extra data, no comment, no modification time (set to zero), no
-   header crc, and the operating system will be set to 255 (unknown).  If a
-   gzip stream is being written, strm->adler is a crc32 instead of an adler32.
-
-     The memLevel parameter specifies how much memory should be allocated
-   for the internal compression state.  memLevel=1 uses minimum memory but is
-   slow and reduces compression ratio; memLevel=9 uses maximum memory for
-   optimal speed.  The default value is 8.  See zconf.h for total memory usage
-   as a function of windowBits and memLevel.
-
-     The strategy parameter is used to tune the compression algorithm.  Use the
-   value Z_DEFAULT_STRATEGY for normal data, Z_FILTERED for data produced by a
-   filter (or predictor), Z_HUFFMAN_ONLY to force Huffman encoding only (no
-   string match), or Z_RLE to limit match distances to one (run-length
-   encoding).  Filtered data consists mostly of small values with a somewhat
-   random distribution.  In this case, the compression algorithm is tuned to
-   compress them better.  The effect of Z_FILTERED is to force more Huffman
-   coding and less string matching; it is somewhat intermediate between
-   Z_DEFAULT_STRATEGY and Z_HUFFMAN_ONLY.  Z_RLE is designed to be almost as
-   fast as Z_HUFFMAN_ONLY, but give better compression for PNG image data.  The
-   strategy parameter only affects the compression ratio but not the
-   correctness of the compressed output even if it is not set appropriately.
-   Z_FIXED prevents the use of dynamic Huffman codes, allowing for a simpler
-   decoder for special applications.
-
-     deflateInit2 returns Z_OK if success, Z_MEM_ERROR if there was not enough
-   memory, Z_STREAM_ERROR if any parameter is invalid (such as an invalid
-   method), or Z_VERSION_ERROR if the zlib library version (zlib_version) is
-   incompatible with the version assumed by the caller (ZLIB_VERSION).  msg is
-   set to null if there is no error message.  deflateInit2 does not perform any
-   compression: this will be done by deflate().
-*/
-
-ZEXTERN int ZEXPORT deflateSetDictionary OF((z_streamp strm,
-                                             const Bytef *dictionary,
-                                             uInt  dictLength));
-/*
-     Initializes the compression dictionary from the given byte sequence
-   without producing any compressed output.  This function must be called
-   immediately after deflateInit, deflateInit2 or deflateReset, before any call
-   of deflate.  The compressor and decompressor must use exactly the same
-   dictionary (see inflateSetDictionary).
-
-     The dictionary should consist of strings (byte sequences) that are likely
-   to be encountered later in the data to be compressed, with the most commonly
-   used strings preferably put towards the end of the dictionary.  Using a
-   dictionary is most useful when the data to be compressed is short and can be
-   predicted with good accuracy; the data can then be compressed better than
-   with the default empty dictionary.
-
-     Depending on the size of the compression data structures selected by
-   deflateInit or deflateInit2, a part of the dictionary may in effect be
-   discarded, for example if the dictionary is larger than the window size
-   provided in deflateInit or deflateInit2.  Thus the strings most likely to be
-   useful should be put at the end of the dictionary, not at the front.  In
-   addition, the current implementation of deflate will use at most the window
-   size minus 262 bytes of the provided dictionary.
-
-     Upon return of this function, strm->adler is set to the adler32 value
-   of the dictionary; the decompressor may later use this value to determine
-   which dictionary has been used by the compressor.  (The adler32 value
-   applies to the whole dictionary even if only a subset of the dictionary is
-   actually used by the compressor.) If a raw deflate was requested, then the
-   adler32 value is not computed and strm->adler is not set.
-
-     deflateSetDictionary returns Z_OK if success, or Z_STREAM_ERROR if a
-   parameter is invalid (e.g.  dictionary being Z_NULL) or the stream state is
-   inconsistent (for example if deflate has already been called for this stream
-   or if the compression method is bsort).  deflateSetDictionary does not
-   perform any compression: this will be done by deflate().
-*/
-
-ZEXTERN int ZEXPORT deflateCopy OF((z_streamp dest,
-                                    z_streamp source));
-/*
-     Sets the destination stream as a complete copy of the source stream.
-
-     This function can be useful when several compression strategies will be
-   tried, for example when there are several ways of pre-processing the input
-   data with a filter.  The streams that will be discarded should then be freed
-   by calling deflateEnd.  Note that deflateCopy duplicates the internal
-   compression state which can be quite large, so this strategy is slow and can
-   consume lots of memory.
-
-     deflateCopy returns Z_OK if success, Z_MEM_ERROR if there was not
-   enough memory, Z_STREAM_ERROR if the source stream state was inconsistent
-   (such as zalloc being Z_NULL).  msg is left unchanged in both source and
-   destination.
-*/
-
-ZEXTERN int ZEXPORT deflateReset OF((z_streamp strm));
-/*
-     This function is equivalent to deflateEnd followed by deflateInit,
-   but does not free and reallocate all the internal compression state.  The
-   stream will keep the same compression level and any other attributes that
-   may have been set by deflateInit2.
-
-     deflateReset returns Z_OK if success, or Z_STREAM_ERROR if the source
-   stream state was inconsistent (such as zalloc or state being Z_NULL).
-*/
-
-ZEXTERN int ZEXPORT deflateParams OF((z_streamp strm,
-                                      int level,
-                                      int strategy));
-/*
-     Dynamically update the compression level and compression strategy.  The
-   interpretation of level and strategy is as in deflateInit2.  This can be
-   used to switch between compression and straight copy of the input data, or
-   to switch to a different kind of input data requiring a different strategy.
-   If the compression level is changed, the input available so far is
-   compressed with the old level (and may be flushed); the new level will take
-   effect only at the next call of deflate().
-
-     Before the call of deflateParams, the stream state must be set as for
-   a call of deflate(), since the currently available input may have to be
-   compressed and flushed.  In particular, strm->avail_out must be non-zero.
-
-     deflateParams returns Z_OK if success, Z_STREAM_ERROR if the source
-   stream state was inconsistent or if a parameter was invalid, Z_BUF_ERROR if
-   strm->avail_out was zero.
-*/
-
-ZEXTERN int ZEXPORT deflateTune OF((z_streamp strm,
-                                    int good_length,
-                                    int max_lazy,
-                                    int nice_length,
-                                    int max_chain));
-/*
-     Fine tune deflate's internal compression parameters.  This should only be
-   used by someone who understands the algorithm used by zlib's deflate for
-   searching for the best matching string, and even then only by the most
-   fanatic optimizer trying to squeeze out the last compressed bit for their
-   specific input data.  Read the deflate.c source code for the meaning of the
-   max_lazy, good_length, nice_length, and max_chain parameters.
-
-     deflateTune() can be called after deflateInit() or deflateInit2(), and
-   returns Z_OK on success, or Z_STREAM_ERROR for an invalid deflate stream.
- */
-
-ZEXTERN uLong ZEXPORT deflateBound OF((z_streamp strm,
-                                       uLong sourceLen));
-/*
-     deflateBound() returns an upper bound on the compressed size after
-   deflation of sourceLen bytes.  It must be called after deflateInit() or
-   deflateInit2(), and after deflateSetHeader(), if used.  This would be used
-   to allocate an output buffer for deflation in a single pass, and so would be
-   called before deflate().
-*/
-
-ZEXTERN int ZEXPORT deflatePrime OF((z_streamp strm,
-                                     int bits,
-                                     int value));
-/*
-     deflatePrime() inserts bits in the deflate output stream.  The intent
-   is that this function is used to start off the deflate output with the bits
-   leftover from a previous deflate stream when appending to it.  As such, this
-   function can only be used for raw deflate, and must be used before the first
-   deflate() call after a deflateInit2() or deflateReset().  bits must be less
-   than or equal to 16, and that many of the least significant bits of value
-   will be inserted in the output.
-
-     deflatePrime returns Z_OK if success, or Z_STREAM_ERROR if the source
-   stream state was inconsistent.
-*/
-
-ZEXTERN int ZEXPORT deflateSetHeader OF((z_streamp strm,
-                                         gz_headerp head));
-/*
-     deflateSetHeader() provides gzip header information for when a gzip
-   stream is requested by deflateInit2().  deflateSetHeader() may be called
-   after deflateInit2() or deflateReset() and before the first call of
-   deflate().  The text, time, os, extra field, name, and comment information
-   in the provided gz_header structure are written to the gzip header (xflag is
-   ignored -- the extra flags are set according to the compression level).  The
-   caller must assure that, if not Z_NULL, name and comment are terminated with
-   a zero byte, and that if extra is not Z_NULL, that extra_len bytes are
-   available there.  If hcrc is true, a gzip header crc is included.  Note that
-   the current versions of the command-line version of gzip (up through version
-   1.3.x) do not support header crc's, and will report that it is a "multi-part
-   gzip file" and give up.
-
-     If deflateSetHeader is not used, the default gzip header has text false,
-   the time set to zero, and os set to 255, with no extra, name, or comment
-   fields.  The gzip header is returned to the default state by deflateReset().
-
-     deflateSetHeader returns Z_OK if success, or Z_STREAM_ERROR if the source
-   stream state was inconsistent.
-*/
-
-/*
-ZEXTERN int ZEXPORT inflateInit2 OF((z_streamp strm,
-                                     int  windowBits));
-
-     This is another version of inflateInit with an extra parameter.  The
-   fields next_in, avail_in, zalloc, zfree and opaque must be initialized
-   before by the caller.
-
-     The windowBits parameter is the base two logarithm of the maximum window
-   size (the size of the history buffer).  It should be in the range 8..15 for
-   this version of the library.  The default value is 15 if inflateInit is used
-   instead.  windowBits must be greater than or equal to the windowBits value
-   provided to deflateInit2() while compressing, or it must be equal to 15 if
-   deflateInit2() was not used.  If a compressed stream with a larger window
-   size is given as input, inflate() will return with the error code
-   Z_DATA_ERROR instead of trying to allocate a larger window.
-
-     windowBits can also be zero to request that inflate use the window size in
-   the zlib header of the compressed stream.
-
-     windowBits can also be -8..-15 for raw inflate.  In this case, -windowBits
-   determines the window size.  inflate() will then process raw deflate data,
-   not looking for a zlib or gzip header, not generating a check value, and not
-   looking for any check values for comparison at the end of the stream.  This
-   is for use with other formats that use the deflate compressed data format
-   such as zip.  Those formats provide their own check values.  If a custom
-   format is developed using the raw deflate format for compressed data, it is
-   recommended that a check value such as an adler32 or a crc32 be applied to
-   the uncompressed data as is done in the zlib, gzip, and zip formats.  For
-   most applications, the zlib format should be used as is.  Note that comments
-   above on the use in deflateInit2() applies to the magnitude of windowBits.
-
-     windowBits can also be greater than 15 for optional gzip decoding.  Add
-   32 to windowBits to enable zlib and gzip decoding with automatic header
-   detection, or add 16 to decode only the gzip format (the zlib format will
-   return a Z_DATA_ERROR).  If a gzip stream is being decoded, strm->adler is a
-   crc32 instead of an adler32.
-
-     inflateInit2 returns Z_OK if success, Z_MEM_ERROR if there was not enough
-   memory, Z_VERSION_ERROR if the zlib library version is incompatible with the
-   version assumed by the caller, or Z_STREAM_ERROR if the parameters are
-   invalid, such as a null pointer to the structure.  msg is set to null if
-   there is no error message.  inflateInit2 does not perform any decompression
-   apart from possibly reading the zlib header if present: actual decompression
-   will be done by inflate().  (So next_in and avail_in may be modified, but
-   next_out and avail_out are unused and unchanged.) The current implementation
-   of inflateInit2() does not process any header information -- that is
-   deferred until inflate() is called.
-*/
-
-ZEXTERN int ZEXPORT inflateSetDictionary OF((z_streamp strm,
-                                             const Bytef *dictionary,
-                                             uInt  dictLength));
-/*
-     Initializes the decompression dictionary from the given uncompressed byte
-   sequence.  This function must be called immediately after a call of inflate,
-   if that call returned Z_NEED_DICT.  The dictionary chosen by the compressor
-   can be determined from the adler32 value returned by that call of inflate.
-   The compressor and decompressor must use exactly the same dictionary (see
-   deflateSetDictionary).  For raw inflate, this function can be called
-   immediately after inflateInit2() or inflateReset() and before any call of
-   inflate() to set the dictionary.  The application must insure that the
-   dictionary that was used for compression is provided.
-
-     inflateSetDictionary returns Z_OK if success, Z_STREAM_ERROR if a
-   parameter is invalid (e.g.  dictionary being Z_NULL) or the stream state is
-   inconsistent, Z_DATA_ERROR if the given dictionary doesn't match the
-   expected one (incorrect adler32 value).  inflateSetDictionary does not
-   perform any decompression: this will be done by subsequent calls of
-   inflate().
-*/
-
-ZEXTERN int ZEXPORT inflateSync OF((z_streamp strm));
-/*
-     Skips invalid compressed data until a full flush point (see above the
-   description of deflate with Z_FULL_FLUSH) can be found, or until all
-   available input is skipped.  No output is provided.
-
-     inflateSync returns Z_OK if a full flush point has been found, Z_BUF_ERROR
-   if no more input was provided, Z_DATA_ERROR if no flush point has been
-   found, or Z_STREAM_ERROR if the stream structure was inconsistent.  In the
-   success case, the application may save the current current value of total_in
-   which indicates where valid compressed data was found.  In the error case,
-   the application may repeatedly call inflateSync, providing more input each
-   time, until success or end of the input data.
-*/
-
-ZEXTERN int ZEXPORT inflateCopy OF((z_streamp dest,
-                                    z_streamp source));
-/*
-     Sets the destination stream as a complete copy of the source stream.
-
-     This function can be useful when randomly accessing a large stream.  The
-   first pass through the stream can periodically record the inflate state,
-   allowing restarting inflate at those points when randomly accessing the
-   stream.
-
-     inflateCopy returns Z_OK if success, Z_MEM_ERROR if there was not
-   enough memory, Z_STREAM_ERROR if the source stream state was inconsistent
-   (such as zalloc being Z_NULL).  msg is left unchanged in both source and
-   destination.
-*/
-
-ZEXTERN int ZEXPORT inflateReset OF((z_streamp strm));
-/*
-     This function is equivalent to inflateEnd followed by inflateInit,
-   but does not free and reallocate all the internal decompression state.  The
-   stream will keep attributes that may have been set by inflateInit2.
-
-     inflateReset returns Z_OK if success, or Z_STREAM_ERROR if the source
-   stream state was inconsistent (such as zalloc or state being Z_NULL).
-*/
-
-ZEXTERN int ZEXPORT inflateReset2 OF((z_streamp strm,
-                                      int windowBits));
-/*
-     This function is the same as inflateReset, but it also permits changing
-   the wrap and window size requests.  The windowBits parameter is interpreted
-   the same as it is for inflateInit2.
-
-     inflateReset2 returns Z_OK if success, or Z_STREAM_ERROR if the source
-   stream state was inconsistent (such as zalloc or state being Z_NULL), or if
-   the windowBits parameter is invalid.
-*/
-
-ZEXTERN int ZEXPORT inflatePrime OF((z_streamp strm,
-                                     int bits,
-                                     int value));
-/*
-     This function inserts bits in the inflate input stream.  The intent is
-   that this function is used to start inflating at a bit position in the
-   middle of a byte.  The provided bits will be used before any bytes are used
-   from next_in.  This function should only be used with raw inflate, and
-   should be used before the first inflate() call after inflateInit2() or
-   inflateReset().  bits must be less than or equal to 16, and that many of the
-   least significant bits of value will be inserted in the input.
-
-     If bits is negative, then the input stream bit buffer is emptied.  Then
-   inflatePrime() can be called again to put bits in the buffer.  This is used
-   to clear out bits leftover after feeding inflate a block description prior
-   to feeding inflate codes.
-
-     inflatePrime returns Z_OK if success, or Z_STREAM_ERROR if the source
-   stream state was inconsistent.
-*/
-
-ZEXTERN long ZEXPORT inflateMark OF((z_streamp strm));
-/*
-     This function returns two values, one in the lower 16 bits of the return
-   value, and the other in the remaining upper bits, obtained by shifting the
-   return value down 16 bits.  If the upper value is -1 and the lower value is
-   zero, then inflate() is currently decoding information outside of a block.
-   If the upper value is -1 and the lower value is non-zero, then inflate is in
-   the middle of a stored block, with the lower value equaling the number of
-   bytes from the input remaining to copy.  If the upper value is not -1, then
-   it is the number of bits back from the current bit position in the input of
-   the code (literal or length/distance pair) currently being processed.  In
-   that case the lower value is the number of bytes already emitted for that
-   code.
-
-     A code is being processed if inflate is waiting for more input to complete
-   decoding of the code, or if it has completed decoding but is waiting for
-   more output space to write the literal or match data.
-
-     inflateMark() is used to mark locations in the input data for random
-   access, which may be at bit positions, and to note those cases where the
-   output of a code may span boundaries of random access blocks.  The current
-   location in the input stream can be determined from avail_in and data_type
-   as noted in the description for the Z_BLOCK flush parameter for inflate.
-
-     inflateMark returns the value noted above or -1 << 16 if the provided
-   source stream state was inconsistent.
-*/
-
-ZEXTERN int ZEXPORT inflateGetHeader OF((z_streamp strm,
-                                         gz_headerp head));
-/*
-     inflateGetHeader() requests that gzip header information be stored in the
-   provided gz_header structure.  inflateGetHeader() may be called after
-   inflateInit2() or inflateReset(), and before the first call of inflate().
-   As inflate() processes the gzip stream, head->done is zero until the header
-   is completed, at which time head->done is set to one.  If a zlib stream is
-   being decoded, then head->done is set to -1 to indicate that there will be
-   no gzip header information forthcoming.  Note that Z_BLOCK or Z_TREES can be
-   used to force inflate() to return immediately after header processing is
-   complete and before any actual data is decompressed.
-
-     The text, time, xflags, and os fields are filled in with the gzip header
-   contents.  hcrc is set to true if there is a header CRC.  (The header CRC
-   was valid if done is set to one.) If extra is not Z_NULL, then extra_max
-   contains the maximum number of bytes to write to extra.  Once done is true,
-   extra_len contains the actual extra field length, and extra contains the
-   extra field, or that field truncated if extra_max is less than extra_len.
-   If name is not Z_NULL, then up to name_max characters are written there,
-   terminated with a zero unless the length is greater than name_max.  If
-   comment is not Z_NULL, then up to comm_max characters are written there,
-   terminated with a zero unless the length is greater than comm_max.  When any
-   of extra, name, or comment are not Z_NULL and the respective field is not
-   present in the header, then that field is set to Z_NULL to signal its
-   absence.  This allows the use of deflateSetHeader() with the returned
-   structure to duplicate the header.  However if those fields are set to
-   allocated memory, then the application will need to save those pointers
-   elsewhere so that they can be eventually freed.
-
-     If inflateGetHeader is not used, then the header information is simply
-   discarded.  The header is always checked for validity, including the header
-   CRC if present.  inflateReset() will reset the process to discard the header
-   information.  The application would need to call inflateGetHeader() again to
-   retrieve the header from the next gzip stream.
-
-     inflateGetHeader returns Z_OK if success, or Z_STREAM_ERROR if the source
-   stream state was inconsistent.
-*/
-
-/*
-ZEXTERN int ZEXPORT inflateBackInit OF((z_streamp strm, int windowBits,
-                                        unsigned char FAR *window));
-
-     Initialize the internal stream state for decompression using inflateBack()
-   calls.  The fields zalloc, zfree and opaque in strm must be initialized
-   before the call.  If zalloc and zfree are Z_NULL, then the default library-
-   derived memory allocation routines are used.  windowBits is the base two
-   logarithm of the window size, in the range 8..15.  window is a caller
-   supplied buffer of that size.  Except for special applications where it is
-   assured that deflate was used with small window sizes, windowBits must be 15
-   and a 32K byte window must be supplied to be able to decompress general
-   deflate streams.
-
-     See inflateBack() for the usage of these routines.
-
-     inflateBackInit will return Z_OK on success, Z_STREAM_ERROR if any of
-   the paramaters are invalid, Z_MEM_ERROR if the internal state could not be
-   allocated, or Z_VERSION_ERROR if the version of the library does not match
-   the version of the header file.
-*/
-
-typedef unsigned (*in_func) OF((void FAR *, unsigned char FAR * FAR *));
-typedef int (*out_func) OF((void FAR *, unsigned char FAR *, unsigned));
-
-ZEXTERN int ZEXPORT inflateBack OF((z_streamp strm,
-                                    in_func in, void FAR *in_desc,
-                                    out_func out, void FAR *out_desc));
-/*
-     inflateBack() does a raw inflate with a single call using a call-back
-   interface for input and output.  This is more efficient than inflate() for
-   file i/o applications in that it avoids copying between the output and the
-   sliding window by simply making the window itself the output buffer.  This
-   function trusts the application to not change the output buffer passed by
-   the output function, at least until inflateBack() returns.
-
-     inflateBackInit() must be called first to allocate the internal state
-   and to initialize the state with the user-provided window buffer.
-   inflateBack() may then be used multiple times to inflate a complete, raw
-   deflate stream with each call.  inflateBackEnd() is then called to free the
-   allocated state.
-
-     A raw deflate stream is one with no zlib or gzip header or trailer.
-   This routine would normally be used in a utility that reads zip or gzip
-   files and writes out uncompressed files.  The utility would decode the
-   header and process the trailer on its own, hence this routine expects only
-   the raw deflate stream to decompress.  This is different from the normal
-   behavior of inflate(), which expects either a zlib or gzip header and
-   trailer around the deflate stream.
-
-     inflateBack() uses two subroutines supplied by the caller that are then
-   called by inflateBack() for input and output.  inflateBack() calls those
-   routines until it reads a complete deflate stream and writes out all of the
-   uncompressed data, or until it encounters an error.  The function's
-   parameters and return types are defined above in the in_func and out_func
-   typedefs.  inflateBack() will call in(in_desc, &buf) which should return the
-   number of bytes of provided input, and a pointer to that input in buf.  If
-   there is no input available, in() must return zero--buf is ignored in that
-   case--and inflateBack() will return a buffer error.  inflateBack() will call
-   out(out_desc, buf, len) to write the uncompressed data buf[0..len-1].  out()
-   should return zero on success, or non-zero on failure.  If out() returns
-   non-zero, inflateBack() will return with an error.  Neither in() nor out()
-   are permitted to change the contents of the window provided to
-   inflateBackInit(), which is also the buffer that out() uses to write from.
-   The length written by out() will be at most the window size.  Any non-zero
-   amount of input may be provided by in().
-
-     For convenience, inflateBack() can be provided input on the first call by
-   setting strm->next_in and strm->avail_in.  If that input is exhausted, then
-   in() will be called.  Therefore strm->next_in must be initialized before
-   calling inflateBack().  If strm->next_in is Z_NULL, then in() will be called
-   immediately for input.  If strm->next_in is not Z_NULL, then strm->avail_in
-   must also be initialized, and then if strm->avail_in is not zero, input will
-   initially be taken from strm->next_in[0 ..  strm->avail_in - 1].
-
-     The in_desc and out_desc parameters of inflateBack() is passed as the
-   first parameter of in() and out() respectively when they are called.  These
-   descriptors can be optionally used to pass any information that the caller-
-   supplied in() and out() functions need to do their job.
-
-     On return, inflateBack() will set strm->next_in and strm->avail_in to
-   pass back any unused input that was provided by the last in() call.  The
-   return values of inflateBack() can be Z_STREAM_END on success, Z_BUF_ERROR
-   if in() or out() returned an error, Z_DATA_ERROR if there was a format error
-   in the deflate stream (in which case strm->msg is set to indicate the nature
-   of the error), or Z_STREAM_ERROR if the stream was not properly initialized.
-   In the case of Z_BUF_ERROR, an input or output error can be distinguished
-   using strm->next_in which will be Z_NULL only if in() returned an error.  If
-   strm->next_in is not Z_NULL, then the Z_BUF_ERROR was due to out() returning
-   non-zero.  (in() will always be called before out(), so strm->next_in is
-   assured to be defined if out() returns non-zero.) Note that inflateBack()
-   cannot return Z_OK.
-*/
-
-ZEXTERN int ZEXPORT inflateBackEnd OF((z_streamp strm));
-/*
-     All memory allocated by inflateBackInit() is freed.
-
-     inflateBackEnd() returns Z_OK on success, or Z_STREAM_ERROR if the stream
-   state was inconsistent.
-*/
-
-ZEXTERN uLong ZEXPORT zlibCompileFlags OF((void));
-/* Return flags indicating compile-time options.
-
-    Type sizes, two bits each, 00 = 16 bits, 01 = 32, 10 = 64, 11 = other:
-     1.0: size of uInt
-     3.2: size of uLong
-     5.4: size of voidpf (pointer)
-     7.6: size of z_off_t
-
-    Compiler, assembler, and debug options:
-     8: DEBUG
-     9: ASMV or ASMINF -- use ASM code
-     10: ZLIB_WINAPI -- exported functions use the WINAPI calling convention
-     11: 0 (reserved)
-
-    One-time table building (smaller code, but not thread-safe if true):
-     12: BUILDFIXED -- build static block decoding tables when needed
-     13: DYNAMIC_CRC_TABLE -- build CRC calculation tables when needed
-     14,15: 0 (reserved)
-
-    Library content (indicates missing functionality):
-     16: NO_GZCOMPRESS -- gz* functions cannot compress (to avoid linking
-                          deflate code when not needed)
-     17: NO_GZIP -- deflate can't write gzip streams, and inflate can't detect
-                    and decode gzip streams (to avoid linking crc code)
-     18-19: 0 (reserved)
-
-    Operation variations (changes in library functionality):
-     20: PKZIP_BUG_WORKAROUND -- slightly more permissive inflate
-     21: FASTEST -- deflate algorithm with only one, lowest compression level
-     22,23: 0 (reserved)
-
-    The sprintf variant used by gzprintf (zero is best):
-     24: 0 = vs*, 1 = s* -- 1 means limited to 20 arguments after the format
-     25: 0 = *nprintf, 1 = *printf -- 1 means gzprintf() not secure!
-     26: 0 = returns value, 1 = void -- 1 means inferred string length returned
-
-    Remainder:
-     27-31: 0 (reserved)
- */
-
-
-                        /* utility functions */
-
-/*
-     The following utility functions are implemented on top of the basic
-   stream-oriented functions.  To simplify the interface, some default options
-   are assumed (compression level and memory usage, standard memory allocation
-   functions).  The source code of these utility functions can be modified if
-   you need special options.
-*/
-
-ZEXTERN int ZEXPORT compress OF((Bytef *dest,   uLongf *destLen,
-                                 const Bytef *source, uLong sourceLen));
-/*
-     Compresses the source buffer into the destination buffer.  sourceLen is
-   the byte length of the source buffer.  Upon entry, destLen is the total size
-   of the destination buffer, which must be at least the value returned by
-   compressBound(sourceLen).  Upon exit, destLen is the actual size of the
-   compressed buffer.
-
-     compress returns Z_OK if success, Z_MEM_ERROR if there was not
-   enough memory, Z_BUF_ERROR if there was not enough room in the output
-   buffer.
-*/
-
-ZEXTERN int ZEXPORT compress2 OF((Bytef *dest,   uLongf *destLen,
-                                  const Bytef *source, uLong sourceLen,
-                                  int level));
-/*
-     Compresses the source buffer into the destination buffer.  The level
-   parameter has the same meaning as in deflateInit.  sourceLen is the byte
-   length of the source buffer.  Upon entry, destLen is the total size of the
-   destination buffer, which must be at least the value returned by
-   compressBound(sourceLen).  Upon exit, destLen is the actual size of the
-   compressed buffer.
-
-     compress2 returns Z_OK if success, Z_MEM_ERROR if there was not enough
-   memory, Z_BUF_ERROR if there was not enough room in the output buffer,
-   Z_STREAM_ERROR if the level parameter is invalid.
-*/
-
-ZEXTERN uLong ZEXPORT compressBound OF((uLong sourceLen));
-/*
-     compressBound() returns an upper bound on the compressed size after
-   compress() or compress2() on sourceLen bytes.  It would be used before a
-   compress() or compress2() call to allocate the destination buffer.
-*/
-
-ZEXTERN int ZEXPORT uncompress OF((Bytef *dest,   uLongf *destLen,
-                                   const Bytef *source, uLong sourceLen));
-/*
-     Decompresses the source buffer into the destination buffer.  sourceLen is
-   the byte length of the source buffer.  Upon entry, destLen is the total size
-   of the destination buffer, which must be large enough to hold the entire
-   uncompressed data.  (The size of the uncompressed data must have been saved
-   previously by the compressor and transmitted to the decompressor by some
-   mechanism outside the scope of this compression library.) Upon exit, destLen
-   is the actual size of the uncompressed buffer.
-
-     uncompress returns Z_OK if success, Z_MEM_ERROR if there was not
-   enough memory, Z_BUF_ERROR if there was not enough room in the output
-   buffer, or Z_DATA_ERROR if the input data was corrupted or incomplete.
-*/
-
-
-                        /* gzip file access functions */
-
-/*
-     This library supports reading and writing files in gzip (.gz) format with
-   an interface similar to that of stdio, using the functions that start with
-   "gz".  The gzip format is different from the zlib format.  gzip is a gzip
-   wrapper, documented in RFC 1952, wrapped around a deflate stream.
-*/
-
-typedef voidp gzFile;       /* opaque gzip file descriptor */
-
-/*
-ZEXTERN gzFile ZEXPORT gzopen OF((const char *path, const char *mode));
-
-     Opens a gzip (.gz) file for reading or writing.  The mode parameter is as
-   in fopen ("rb" or "wb") but can also include a compression level ("wb9") or
-   a strategy: 'f' for filtered data as in "wb6f", 'h' for Huffman-only
-   compression as in "wb1h", 'R' for run-length encoding as in "wb1R", or 'F'
-   for fixed code compression as in "wb9F".  (See the description of
-   deflateInit2 for more information about the strategy parameter.) Also "a"
-   can be used instead of "w" to request that the gzip stream that will be
-   written be appended to the file.  "+" will result in an error, since reading
-   and writing to the same gzip file is not supported.
-
-     gzopen can be used to read a file which is not in gzip format; in this
-   case gzread will directly read from the file without decompression.
-
-     gzopen returns NULL if the file could not be opened, if there was
-   insufficient memory to allocate the gzFile state, or if an invalid mode was
-   specified (an 'r', 'w', or 'a' was not provided, or '+' was provided).
-   errno can be checked to determine if the reason gzopen failed was that the
-   file could not be opened.
-*/
-
-ZEXTERN gzFile ZEXPORT gzdopen OF((int fd, const char *mode));
-/*
-     gzdopen associates a gzFile with the file descriptor fd.  File descriptors
-   are obtained from calls like open, dup, creat, pipe or fileno (if the file
-   has been previously opened with fopen).  The mode parameter is as in gzopen.
-
-     The next call of gzclose on the returned gzFile will also close the file
-   descriptor fd, just like fclose(fdopen(fd, mode)) closes the file descriptor
-   fd.  If you want to keep fd open, use fd = dup(fd_keep); gz = gzdopen(fd,
-   mode);.  The duplicated descriptor should be saved to avoid a leak, since
-   gzdopen does not close fd if it fails.
-
-     gzdopen returns NULL if there was insufficient memory to allocate the
-   gzFile state, if an invalid mode was specified (an 'r', 'w', or 'a' was not
-   provided, or '+' was provided), or if fd is -1.  The file descriptor is not
-   used until the next gz* read, write, seek, or close operation, so gzdopen
-   will not detect if fd is invalid (unless fd is -1).
-*/
-
-ZEXTERN int ZEXPORT gzbuffer OF((gzFile file, unsigned size));
-/*
-     Set the internal buffer size used by this library's functions.  The
-   default buffer size is 8192 bytes.  This function must be called after
-   gzopen() or gzdopen(), and before any other calls that read or write the
-   file.  The buffer memory allocation is always deferred to the first read or
-   write.  Two buffers are allocated, either both of the specified size when
-   writing, or one of the specified size and the other twice that size when
-   reading.  A larger buffer size of, for example, 64K or 128K bytes will
-   noticeably increase the speed of decompression (reading).
-
-     The new buffer size also affects the maximum length for gzprintf().
-
-     gzbuffer() returns 0 on success, or -1 on failure, such as being called
-   too late.
-*/
-
-ZEXTERN int ZEXPORT gzsetparams OF((gzFile file, int level, int strategy));
-/*
-     Dynamically update the compression level or strategy.  See the description
-   of deflateInit2 for the meaning of these parameters.
-
-     gzsetparams returns Z_OK if success, or Z_STREAM_ERROR if the file was not
-   opened for writing.
-*/
-
-ZEXTERN int ZEXPORT gzread OF((gzFile file, voidp buf, unsigned len));
-/*
-     Reads the given number of uncompressed bytes from the compressed file.  If
-   the input file was not in gzip format, gzread copies the given number of
-   bytes into the buffer.
-
-     After reaching the end of a gzip stream in the input, gzread will continue
-   to read, looking for another gzip stream, or failing that, reading the rest
-   of the input file directly without decompression.  The entire input file
-   will be read if gzread is called until it returns less than the requested
-   len.
-
-     gzread returns the number of uncompressed bytes actually read, less than
-   len for end of file, or -1 for error.
-*/
-
-ZEXTERN int ZEXPORT gzwrite OF((gzFile file,
-                                voidpc buf, unsigned len));
-/*
-     Writes the given number of uncompressed bytes into the compressed file.
-   gzwrite returns the number of uncompressed bytes written or 0 in case of
-   error.
-*/
-
-ZEXTERN int ZEXPORTVA gzprintf OF((gzFile file, const char *format, ...));
-/*
-     Converts, formats, and writes the arguments to the compressed file under
-   control of the format string, as in fprintf.  gzprintf returns the number of
-   uncompressed bytes actually written, or 0 in case of error.  The number of
-   uncompressed bytes written is limited to 8191, or one less than the buffer
-   size given to gzbuffer().  The caller should assure that this limit is not
-   exceeded.  If it is exceeded, then gzprintf() will return an error (0) with
-   nothing written.  In this case, there may also be a buffer overflow with
-   unpredictable consequences, which is possible only if zlib was compiled with
-   the insecure functions sprintf() or vsprintf() because the secure snprintf()
-   or vsnprintf() functions were not available.  This can be determined using
-   zlibCompileFlags().
-*/
-
-ZEXTERN int ZEXPORT gzputs OF((gzFile file, const char *s));
-/*
-     Writes the given null-terminated string to the compressed file, excluding
-   the terminating null character.
-
-     gzputs returns the number of characters written, or -1 in case of error.
-*/
-
-ZEXTERN char * ZEXPORT gzgets OF((gzFile file, char *buf, int len));
-/*
-     Reads bytes from the compressed file until len-1 characters are read, or a
-   newline character is read and transferred to buf, or an end-of-file
-   condition is encountered.  If any characters are read or if len == 1, the
-   string is terminated with a null character.  If no characters are read due
-   to an end-of-file or len < 1, then the buffer is left untouched.
-
-     gzgets returns buf which is a null-terminated string, or it returns NULL
-   for end-of-file or in case of error.  If there was an error, the contents at
-   buf are indeterminate.
-*/
-
-ZEXTERN int ZEXPORT gzputc OF((gzFile file, int c));
-/*
-     Writes c, converted to an unsigned char, into the compressed file.  gzputc
-   returns the value that was written, or -1 in case of error.
-*/
-
-ZEXTERN int ZEXPORT gzgetc OF((gzFile file));
-/*
-     Reads one byte from the compressed file.  gzgetc returns this byte or -1
-   in case of end of file or error.
-*/
-
-ZEXTERN int ZEXPORT gzungetc OF((int c, gzFile file));
-/*
-     Push one character back onto the stream to be read as the first character
-   on the next read.  At least one character of push-back is allowed.
-   gzungetc() returns the character pushed, or -1 on failure.  gzungetc() will
-   fail if c is -1, and may fail if a character has been pushed but not read
-   yet.  If gzungetc is used immediately after gzopen or gzdopen, at least the
-   output buffer size of pushed characters is allowed.  (See gzbuffer above.)
-   The pushed character will be discarded if the stream is repositioned with
-   gzseek() or gzrewind().
-*/
-
-ZEXTERN int ZEXPORT gzflush OF((gzFile file, int flush));
-/*
-     Flushes all pending output into the compressed file.  The parameter flush
-   is as in the deflate() function.  The return value is the zlib error number
-   (see function gzerror below).  gzflush is only permitted when writing.
-
-     If the flush parameter is Z_FINISH, the remaining data is written and the
-   gzip stream is completed in the output.  If gzwrite() is called again, a new
-   gzip stream will be started in the output.  gzread() is able to read such
-   concatented gzip streams.
-
-     gzflush should be called only when strictly necessary because it will
-   degrade compression if called too often.
-*/
-
-/*
-ZEXTERN z_off_t ZEXPORT gzseek OF((gzFile file,
-                                   z_off_t offset, int whence));
-
-     Sets the starting position for the next gzread or gzwrite on the given
-   compressed file.  The offset represents a number of bytes in the
-   uncompressed data stream.  The whence parameter is defined as in lseek(2);
-   the value SEEK_END is not supported.
-
-     If the file is opened for reading, this function is emulated but can be
-   extremely slow.  If the file is opened for writing, only forward seeks are
-   supported; gzseek then compresses a sequence of zeroes up to the new
-   starting position.
-
-     gzseek returns the resulting offset location as measured in bytes from
-   the beginning of the uncompressed stream, or -1 in case of error, in
-   particular if the file is opened for writing and the new starting position
-   would be before the current position.
-*/
-
-ZEXTERN int ZEXPORT    gzrewind OF((gzFile file));
-/*
-     Rewinds the given file. This function is supported only for reading.
-
-     gzrewind(file) is equivalent to (int)gzseek(file, 0L, SEEK_SET)
-*/
-
-/*
-ZEXTERN z_off_t ZEXPORT    gztell OF((gzFile file));
-
-     Returns the starting position for the next gzread or gzwrite on the given
-   compressed file.  This position represents a number of bytes in the
-   uncompressed data stream, and is zero when starting, even if appending or
-   reading a gzip stream from the middle of a file using gzdopen().
-
-     gztell(file) is equivalent to gzseek(file, 0L, SEEK_CUR)
-*/
-
-/*
-ZEXTERN z_off_t ZEXPORT gzoffset OF((gzFile file));
-
-     Returns the current offset in the file being read or written.  This offset
-   includes the count of bytes that precede the gzip stream, for example when
-   appending or when using gzdopen() for reading.  When reading, the offset
-   does not include as yet unused buffered input.  This information can be used
-   for a progress indicator.  On error, gzoffset() returns -1.
-*/
-
-ZEXTERN int ZEXPORT gzeof OF((gzFile file));
-/*
-     Returns true (1) if the end-of-file indicator has been set while reading,
-   false (0) otherwise.  Note that the end-of-file indicator is set only if the
-   read tried to go past the end of the input, but came up short.  Therefore,
-   just like feof(), gzeof() may return false even if there is no more data to
-   read, in the event that the last read request was for the exact number of
-   bytes remaining in the input file.  This will happen if the input file size
-   is an exact multiple of the buffer size.
-
-     If gzeof() returns true, then the read functions will return no more data,
-   unless the end-of-file indicator is reset by gzclearerr() and the input file
-   has grown since the previous end of file was detected.
-*/
-
-ZEXTERN int ZEXPORT gzdirect OF((gzFile file));
-/*
-     Returns true (1) if file is being copied directly while reading, or false
-   (0) if file is a gzip stream being decompressed.  This state can change from
-   false to true while reading the input file if the end of a gzip stream is
-   reached, but is followed by data that is not another gzip stream.
-
-     If the input file is empty, gzdirect() will return true, since the input
-   does not contain a gzip stream.
-
-     If gzdirect() is used immediately after gzopen() or gzdopen() it will
-   cause buffers to be allocated to allow reading the file to determine if it
-   is a gzip file.  Therefore if gzbuffer() is used, it should be called before
-   gzdirect().
-*/
-
-ZEXTERN int ZEXPORT    gzclose OF((gzFile file));
-/*
-     Flushes all pending output if necessary, closes the compressed file and
-   deallocates the (de)compression state.  Note that once file is closed, you
-   cannot call gzerror with file, since its structures have been deallocated.
-   gzclose must not be called more than once on the same file, just as free
-   must not be called more than once on the same allocation.
-
-     gzclose will return Z_STREAM_ERROR if file is not valid, Z_ERRNO on a
-   file operation error, or Z_OK on success.
-*/
-
-ZEXTERN int ZEXPORT gzclose_r OF((gzFile file));
-ZEXTERN int ZEXPORT gzclose_w OF((gzFile file));
-/*
-     Same as gzclose(), but gzclose_r() is only for use when reading, and
-   gzclose_w() is only for use when writing or appending.  The advantage to
-   using these instead of gzclose() is that they avoid linking in zlib
-   compression or decompression code that is not used when only reading or only
-   writing respectively.  If gzclose() is used, then both compression and
-   decompression code will be included the application when linking to a static
-   zlib library.
-*/
-
-ZEXTERN const char * ZEXPORT gzerror OF((gzFile file, int *errnum));
-/*
-     Returns the error message for the last error which occurred on the given
-   compressed file.  errnum is set to zlib error number.  If an error occurred
-   in the file system and not in the compression library, errnum is set to
-   Z_ERRNO and the application may consult errno to get the exact error code.
-
-     The application must not modify the returned string.  Future calls to
-   this function may invalidate the previously returned string.  If file is
-   closed, then the string previously returned by gzerror will no longer be
-   available.
-
-     gzerror() should be used to distinguish errors from end-of-file for those
-   functions above that do not distinguish those cases in their return values.
-*/
-
-ZEXTERN void ZEXPORT gzclearerr OF((gzFile file));
-/*
-     Clears the error and end-of-file flags for file.  This is analogous to the
-   clearerr() function in stdio.  This is useful for continuing to read a gzip
-   file that is being written concurrently.
-*/
-
-
-                        /* checksum functions */
-
-/*
-     These functions are not related to compression but are exported
-   anyway because they might be useful in applications using the compression
-   library.
-*/
-
-ZEXTERN uLong ZEXPORT adler32 OF((uLong adler, const Bytef *buf, uInt len));
-/*
-     Update a running Adler-32 checksum with the bytes buf[0..len-1] and
-   return the updated checksum.  If buf is Z_NULL, this function returns the
-   required initial value for the checksum.
-
-     An Adler-32 checksum is almost as reliable as a CRC32 but can be computed
-   much faster.
-
-   Usage example:
-
-     uLong adler = adler32(0L, Z_NULL, 0);
-
-     while (read_buffer(buffer, length) != EOF) {
-       adler = adler32(adler, buffer, length);
-     }
-     if (adler != original_adler) error();
-*/
-
-/*
-ZEXTERN uLong ZEXPORT adler32_combine OF((uLong adler1, uLong adler2,
-                                          z_off_t len2));
-
-     Combine two Adler-32 checksums into one.  For two sequences of bytes, seq1
-   and seq2 with lengths len1 and len2, Adler-32 checksums were calculated for
-   each, adler1 and adler2.  adler32_combine() returns the Adler-32 checksum of
-   seq1 and seq2 concatenated, requiring only adler1, adler2, and len2.
-*/
-
-ZEXTERN uLong ZEXPORT crc32   OF((uLong crc, const Bytef *buf, uInt len));
-/*
-     Update a running CRC-32 with the bytes buf[0..len-1] and return the
-   updated CRC-32.  If buf is Z_NULL, this function returns the required
-   initial value for the for the crc.  Pre- and post-conditioning (one's
-   complement) is performed within this function so it shouldn't be done by the
-   application.
-
-   Usage example:
-
-     uLong crc = crc32(0L, Z_NULL, 0);
-
-     while (read_buffer(buffer, length) != EOF) {
-       crc = crc32(crc, buffer, length);
-     }
-     if (crc != original_crc) error();
-*/
-
-/*
-ZEXTERN uLong ZEXPORT crc32_combine OF((uLong crc1, uLong crc2, z_off_t len2));
-
-     Combine two CRC-32 check values into one.  For two sequences of bytes,
-   seq1 and seq2 with lengths len1 and len2, CRC-32 check values were
-   calculated for each, crc1 and crc2.  crc32_combine() returns the CRC-32
-   check value of seq1 and seq2 concatenated, requiring only crc1, crc2, and
-   len2.
-*/
-
-
-                        /* various hacks, don't look :) */
-
-/* deflateInit and inflateInit are macros to allow checking the zlib version
- * and the compiler's view of z_stream:
- */
-ZEXTERN int ZEXPORT deflateInit_ OF((z_streamp strm, int level,
-                                     const char *version, int stream_size));
-ZEXTERN int ZEXPORT inflateInit_ OF((z_streamp strm,
-                                     const char *version, int stream_size));
-ZEXTERN int ZEXPORT deflateInit2_ OF((z_streamp strm, int  level, int  method,
-                                      int windowBits, int memLevel,
-                                      int strategy, const char *version,
-                                      int stream_size));
-ZEXTERN int ZEXPORT inflateInit2_ OF((z_streamp strm, int  windowBits,
-                                      const char *version, int stream_size));
-ZEXTERN int ZEXPORT inflateBackInit_ OF((z_streamp strm, int windowBits,
-                                         unsigned char FAR *window,
-                                         const char *version,
-                                         int stream_size));
-#define deflateInit(strm, level) \
-        deflateInit_((strm), (level),       ZLIB_VERSION, sizeof(z_stream))
-#define inflateInit(strm) \
-        inflateInit_((strm),                ZLIB_VERSION, sizeof(z_stream))
-#define deflateInit2(strm, level, method, windowBits, memLevel, strategy) \
-        deflateInit2_((strm),(level),(method),(windowBits),(memLevel),\
-                      (strategy),           ZLIB_VERSION, sizeof(z_stream))
-#define inflateInit2(strm, windowBits) \
-        inflateInit2_((strm), (windowBits), ZLIB_VERSION, sizeof(z_stream))
-#define inflateBackInit(strm, windowBits, window) \
-        inflateBackInit_((strm), (windowBits), (window), \
-                                            ZLIB_VERSION, sizeof(z_stream))
-
-/* provide 64-bit offset functions if _LARGEFILE64_SOURCE defined, and/or
- * change the regular functions to 64 bits if _FILE_OFFSET_BITS is 64 (if
- * both are true, the application gets the *64 functions, and the regular
- * functions are changed to 64 bits) -- in case these are set on systems
- * without large file support, _LFS64_LARGEFILE must also be true
- */
-#if defined(_LARGEFILE64_SOURCE) && _LFS64_LARGEFILE-0
-   ZEXTERN gzFile ZEXPORT gzopen64 OF((const char *, const char *));
-   ZEXTERN z_off64_t ZEXPORT gzseek64 OF((gzFile, z_off64_t, int));
-   ZEXTERN z_off64_t ZEXPORT gztell64 OF((gzFile));
-   ZEXTERN z_off64_t ZEXPORT gzoffset64 OF((gzFile));
-   ZEXTERN uLong ZEXPORT adler32_combine64 OF((uLong, uLong, z_off64_t));
-   ZEXTERN uLong ZEXPORT crc32_combine64 OF((uLong, uLong, z_off64_t));
-#elif !defined(ZLIB_INTERNAL) && _FILE_OFFSET_BITS-0 == 64 && _LFS64_LARGEFILE-0
-#  define gzopen gzopen64
-#  define gzseek gzseek64
-#  define gztell gztell64
-#  define gzoffset gzoffset64
-#  define adler32_combine adler32_combine64
-#  define crc32_combine crc32_combine64
-#  ifdef _LARGEFILE64_SOURCE
-     ZEXTERN gzFile ZEXPORT gzopen64 OF((const char *, const char *));
-     ZEXTERN z_off_t ZEXPORT gzseek64 OF((gzFile, z_off_t, int));
-     ZEXTERN z_off_t ZEXPORT gztell64 OF((gzFile));
-     ZEXTERN z_off_t ZEXPORT gzoffset64 OF((gzFile));
-     ZEXTERN uLong ZEXPORT adler32_combine64 OF((uLong, uLong, z_off_t));
-     ZEXTERN uLong ZEXPORT crc32_combine64 OF((uLong, uLong, z_off_t));
-#  endif
-#else
-   ZEXTERN gzFile ZEXPORT gzopen OF((const char *, const char *));
-   ZEXTERN z_off_t ZEXPORT gzseek OF((gzFile, z_off_t, int));
-   ZEXTERN z_off_t ZEXPORT gztell OF((gzFile));
-   ZEXTERN z_off_t ZEXPORT gzoffset OF((gzFile));
-   ZEXTERN uLong ZEXPORT adler32_combine OF((uLong, uLong, z_off_t));
-   ZEXTERN uLong ZEXPORT crc32_combine OF((uLong, uLong, z_off_t));
-#endif
-
-/* hack for buggy compilers */
-#if !defined(ZUTIL_H) && !defined(NO_DUMMY_DECL)
-    struct internal_state {int dummy;};
-#endif
-
-/* undocumented functions */
-ZEXTERN const char   * ZEXPORT zError           OF((int));
-ZEXTERN int            ZEXPORT inflateSyncPoint OF((z_streamp));
-ZEXTERN const uLongf * ZEXPORT get_crc_table    OF((void));
-ZEXTERN int            ZEXPORT inflateUndermine OF((z_streamp, int));
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif /* ZLIB_H */



From phk at varnish-cache.org  Mon Jan 24 09:13:03 2011
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Mon, 24 Jan 2011 10:13:03 +0100
Subject: [master] 1f8a74a Rename zlib.h to vgz.h, take 2
Message-ID: <E1PhITv-0003Rg-CB@project.varnish-software.com>

commit 1f8a74ac389d8b59c362e95ad1a64af7e1829498
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Jan 24 09:12:45 2011 +0000

    Rename zlib.h to vgz.h, take 2

diff --git a/bin/varnishd/cache_esi_deliver.c b/bin/varnishd/cache_esi_deliver.c
index d5e7c90..6695dee 100644
--- a/bin/varnishd/cache_esi_deliver.c
+++ b/bin/varnishd/cache_esi_deliver.c
@@ -40,7 +40,7 @@ SVNID("$Id")
 #include "cache_esi.h"
 #include "vend.h"
 #include "vct.h"
-#include "zlib.h"
+#include "vgz.h"
 #include "stevedore.h"
 
 /*--------------------------------------------------------------------*/
diff --git a/bin/varnishd/cache_esi_fetch.c b/bin/varnishd/cache_esi_fetch.c
index 0bf2f7c..909d77b 100644
--- a/bin/varnishd/cache_esi_fetch.c
+++ b/bin/varnishd/cache_esi_fetch.c
@@ -39,7 +39,7 @@ SVNID("$Id")
 #include "cache.h"
 #include "cache_esi.h"
 #include "vct.h"
-#include "zlib.h"
+#include "vgz.h"
 #include "stevedore.h"
 
 /*---------------------------------------------------------------------
diff --git a/bin/varnishd/cache_esi_parse.c b/bin/varnishd/cache_esi_parse.c
index 6baeb13..6cc48fa 100644
--- a/bin/varnishd/cache_esi_parse.c
+++ b/bin/varnishd/cache_esi_parse.c
@@ -40,7 +40,7 @@ SVNID("$Id")
 #include "cache_esi.h"
 #include "vend.h"
 #include "vct.h"
-#include "zlib.h"
+#include "vgz.h"
 #include "stevedore.h"
 
 //#define Debug(fmt, ...) printf(fmt, __VA_ARGS__)
diff --git a/bin/varnishd/cache_gzip.c b/bin/varnishd/cache_gzip.c
index a4ea296..9d83462 100644
--- a/bin/varnishd/cache_gzip.c
+++ b/bin/varnishd/cache_gzip.c
@@ -27,9 +27,9 @@
  *
  * Interaction with the linvgz (zlib) library.
  *
- * The libz library pollutes namespace a LOT when you include the "zlib.h"
- * file so we contain the damage by vectoring all access to libz through
- * this source file.
+ * The zlib library pollutes namespace a LOT when you include the "vgz.h"
+ * (aka (zlib.h") file so we contain the damage by vectoring all access
+ * to libz through this source file.
  *
  * The API defined by this file, will also insulate the rest of the code,
  * should we find a better gzip library at a later date.
@@ -74,7 +74,7 @@ SVNID("$Id$")
 #include "cache.h"
 #include "stevedore.h"
 
-#include "zlib.h"
+#include "vgz.h"
 
 struct vgz {
 	unsigned		magic;
diff --git a/bin/varnishtest/vtc_http.c b/bin/varnishtest/vtc_http.c
index f8aa59b..52eb287 100644
--- a/bin/varnishtest/vtc_http.c
+++ b/bin/varnishtest/vtc_http.c
@@ -49,7 +49,7 @@ SVNID("$Id$")
 
 #include "vtc.h"
 
-#include "zlib.h"
+#include "vgz.h"
 
 #define MAX_HDR		50
 
diff --git a/lib/libvgz/Makefile.am b/lib/libvgz/Makefile.am
index bb6010d..446dc64 100644
--- a/lib/libvgz/Makefile.am
+++ b/lib/libvgz/Makefile.am
@@ -28,6 +28,6 @@ libvgz_la_SOURCES = \
 	trees.h \
 	uncompr.c \
 	zconf.h \
-	zlib.h \
+	vgz.h \
 	zutil.c \
 	zutil.h
diff --git a/lib/libvgz/compress.c b/lib/libvgz/compress.c
index fce2020..b369014 100644
--- a/lib/libvgz/compress.c
+++ b/lib/libvgz/compress.c
@@ -6,7 +6,7 @@
 /* @(#) $Id$ */
 
 #define ZLIB_INTERNAL
-#include "zlib.h"
+#include "vgz.h"
 
 /* ===========================================================================
      Compresses the source buffer into the destination buffer. The level
diff --git a/lib/libvgz/gzguts.h b/lib/libvgz/gzguts.h
index 0f8fb79..2f4bfec 100644
--- a/lib/libvgz/gzguts.h
+++ b/lib/libvgz/gzguts.h
@@ -19,7 +19,7 @@
 #endif
 
 #include <stdio.h>
-#include "zlib.h"
+#include "vgz.h"
 #ifdef STDC
 #  include <string.h>
 #  include <stdlib.h>
diff --git a/lib/libvgz/uncompr.c b/lib/libvgz/uncompr.c
index b8c7dcf..78324b1 100644
--- a/lib/libvgz/uncompr.c
+++ b/lib/libvgz/uncompr.c
@@ -6,7 +6,7 @@
 /* @(#) $Id$ */
 
 #define ZLIB_INTERNAL
-#include "zlib.h"
+#include "vgz.h"
 
 /* ===========================================================================
      Decompresses the source buffer into the destination buffer.  sourceLen is
diff --git a/lib/libvgz/zutil.h b/lib/libvgz/zutil.h
index c2727dc..0c7456e 100644
--- a/lib/libvgz/zutil.h
+++ b/lib/libvgz/zutil.h
@@ -19,7 +19,7 @@
 #  define ZLIB_INTERNAL
 #endif
 
-#include "zlib.h"
+#include "vgz.h"
 
 #ifdef STDC
 #  if !(defined(_WIN32_WCE) && defined(_MSC_VER))



From phk at varnish-cache.org  Mon Jan 24 09:46:17 2011
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Mon, 24 Jan 2011 10:46:17 +0100
Subject: [master] f31ed40 Remove the clients Accept-Encoding header when we
	add our own.
Message-ID: <E1PhJ05-0006Hl-RU@project.varnish-software.com>

commit f31ed40c2198d58ef5fa3cff9a22990285a0acae
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Jan 24 09:45:54 2011 +0000

    Remove the clients Accept-Encoding header when we add our own.

diff --git a/bin/varnishd/cache_center.c b/bin/varnishd/cache_center.c
index ec945d2..9ec86fa 100644
--- a/bin/varnishd/cache_center.c
+++ b/bin/varnishd/cache_center.c
@@ -975,6 +975,7 @@ cnt_miss(struct sess *sp)
 		 * client doesn't grok it.  We will uncompress for
 		 * the minority of clients which don't.
 		 */
+		http_Unset(sp->wrk->bereq, H_Accept_Encoding);
 		http_PrintfHeader(sp->wrk, sp->fd, sp->wrk->bereq,
 		    "Accept-Encoding: gzip");
 	}



From phk at varnish-cache.org  Mon Jan 24 10:17:42 2011
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Mon, 24 Jan 2011 11:17:42 +0100
Subject: [master] 0c80b0a Normalize requests Accept-Encoding header if we are
	not pipe or pass after vcl_recv{}. This is necessary to get
	consistent Vary: header processing.
Message-ID: <E1PhJUU-00005V-Ap@project.varnish-software.com>

commit 0c80b0a675c4fcf307a31ce3849e3de28beff568
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Jan 24 10:16:58 2011 +0000

    Normalize requests Accept-Encoding header if we are not pipe or pass
    after vcl_recv{}.  This is necessary to get consistent Vary: header
    processing.

diff --git a/bin/varnishd/cache_center.c b/bin/varnishd/cache_center.c
index 9ec86fa..31adc80 100644
--- a/bin/varnishd/cache_center.c
+++ b/bin/varnishd/cache_center.c
@@ -1168,6 +1168,18 @@ cnt_recv(struct sess *sp)
 		return (0);
 	}
 
+	if (params->http_gzip_support &&
+	     (recv_handling != VCL_RET_PIPE) &&
+	     (recv_handling != VCL_RET_PASS)) {
+		if (RFC2616_Req_Gzip(sp)) {
+			http_Unset(sp->http, H_Accept_Encoding);
+			http_PrintfHeader(sp->wrk, sp->fd, sp->http,
+			    "Accept-Encoding: gzip");
+		} else {
+			http_Unset(sp->http, H_Accept_Encoding);
+		}
+	}
+
 	SHA256_Init(sp->wrk->sha256ctx);
 	VCL_hash_method(sp);
 	assert(sp->handling == VCL_RET_HASH);



From phk at varnish-cache.org  Mon Jan 24 10:20:14 2011
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Mon, 24 Jan 2011 11:20:14 +0100
Subject: [master] 9a57367 Add sphinx doc about GZIP,
	ESI part still to be written
Message-ID: <E1PhJWw-0000K3-0Z@project.varnish-software.com>

commit 9a57367ff5aa05929ca04c6fb5ebbc6fb2f7bd07
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Jan 24 10:19:52 2011 +0000

    Add sphinx doc about GZIP, ESI part still to be written

diff --git a/doc/sphinx/phk/gzip.rst b/doc/sphinx/phk/gzip.rst
new file mode 100644
index 0000000..77ba94f
--- /dev/null
+++ b/doc/sphinx/phk/gzip.rst
@@ -0,0 +1,92 @@
+.. _phk_gzip:
+
+=======================================
+How GZIP, and GZIP+ESI works in Varnish
+=======================================
+
+First of all, everything you read about GZIP here, is controlled by the
+parameter:
+
+	http_gzip_support
+
+Which defaults to "on" if you do not want Varnish to try to be smart
+about compression, set it to "off" instead.
+
+What does http_gzip_support do
+------------------------------
+
+A request which is sent into 'pipe' or 'pass' mode from vcl_recv{}
+will not experience any difference, this processing only affects
+cache hit/miss requests.
+
+Unless vcl_recv{} results in "pipe" or "pass", we determine if the
+client is capable of receiving gzip'ed content.  The test amounts to:
+
+	Is there a Accept-Encoding header that mentions gzip, and if
+	is has a q=# number, is it larger than zero.
+
+Clients which can do gzip, gets their header rewritten to:
+
+	Accept-Encoding: gzip
+
+And clients which do not support gzip gets their Accept-Encoding header
+removed.  This ensures conformity with respect to Vary: strings during
+hash lookup.
+
+Varnish will not do any other types of compressions than gzip, in particular
+we will not do deflate, as there are browser bugs in that case.
+
+Before vcl_hit{} is called, the backend requests Accept-Encoding is
+always set to:
+
+	Accept-Encoding: gzip
+
+To always entice the backend into sending us gzip'ed content.
+
+Varnish will not gzip any content on its own (but see below), we trust
+the backend to know what content can be sensibly gzip'ed (html) and what
+can not (jpeg)
+
+If in vcl_fetch{} we find out that we are trying to deliver a gzip'ed object
+to a client that has not indicated willingness to receive gzip, we will
+ungzip the object during deliver.
+
+Tuning, tweaking and frobbing
+-----------------------------
+
+In vcl_recv{} you have a chance t modify the clients Accept-Encoding: header
+before anything else happens.
+
+In vcl_pass{} the clients Accept-Encoding header is copied to the
+backend request unchanged.
+Even if the client does not support gzip, you can force the A-C header
+to "gzip" to save bandwidth between the backend and varnish, varnish will
+gunzip the object before delivering to the client.
+
+In vcl_fetch{} two new variables allow you to modify the gzip-ness of
+objects during fetch:
+
+	set beresp.do_gunzip = true;
+
+Will make varnish gunzip an already gzip'ed object from the backend during
+fetch.  (I have no idea why/when you would use this...)
+
+	set beresp.do_gzip = true;
+
+Will make varnish gzip the object during fetch from the backend, provided
+the backend didn't send us a gzip'ed object.
+
+Remember that a lot of content types cannot sensibly be gziped, most
+notably compressed image formats like jpeg, png and similar, so a
+typical use would be:
+
+	sub vcl_fetch {
+		if (req.url ~ "html$") {
+			set beresp.do_gzip = true;
+		}
+	}
+
+GZIP and ESI
+------------
+
+TBD.
diff --git a/doc/sphinx/phk/index.rst b/doc/sphinx/phk/index.rst
index 9eecca8..e07e50e 100644
--- a/doc/sphinx/phk/index.rst
+++ b/doc/sphinx/phk/index.rst
@@ -8,6 +8,7 @@ You may or may not want to know what Poul-Henning think.
 
 .. toctree::
 
+	gzip.rst
 	vcl_expr.rst
 	ipv6suckage.rst
 	backends.rst



From kristian at varnish-cache.org  Mon Jan 24 11:03:05 2011
From: kristian at varnish-cache.org (=?UTF-8?Q?Kristian_Lyngst=C3=B8l?=)
Date: Mon, 24 Jan 2011 12:03:05 +0100
Subject: [2.1] a6ba7bc Fix copy/paste blunder in Changes XML for 2.1.4
Message-ID: <E1PhKCP-00034Q-Mv@project.varnish-software.com>

commit a6ba7bcdfe4f4bd0e0818e5b8835ae76626e4a36
Author: Kristian Lyngstol <kristian at bohemians.org>
Date:   Mon Jan 24 11:56:13 2011 +0100

    Fix copy/paste blunder in Changes XML for 2.1.4
    
    Also removed the $Id$, which was both incorrect and now obsolete.

diff --git a/doc/changes-2.1.3-2.1.4.xml b/doc/changes-2.1.3-2.1.4.xml
index cccfb42..775b6bd 100644
--- a/doc/changes-2.1.3-2.1.4.xml
+++ b/doc/changes-2.1.3-2.1.4.xml
@@ -2,8 +2,7 @@
 <!DOCTYPE group [
 <!ENTITY mdash "&#8212;">
 ]>
-<!-- $Id: changes-2.1.2-2.1.3.xml 4672 2010-04-19 10:26:23Z tfheen $ -->
-<group from="2.1.2" to="2.1.3">
+<group from="2.1.3" to="2.1.4">
   <subsystem>
     <name>varnishd</name>
     <change type="bug">



From kristian at varnish-cache.org  Mon Jan 24 12:24:15 2011
From: kristian at varnish-cache.org (=?UTF-8?Q?Kristian_Lyngst=C3=B8l?=)
Date: Mon, 24 Jan 2011 13:24:15 +0100
Subject: [2.1] 0843d7a Add changes for 2.1.5 to the build system
Message-ID: <E1PhLSx-0008Pg-QE@project.varnish-software.com>

commit 0843d7a0cd17a705bfeea1e3b229ec349117bab3
Author: Kristian Lyngstol <kristian at bohemians.org>
Date:   Mon Jan 24 13:13:55 2011 +0100

    Add changes for 2.1.5 to the build system

diff --git a/doc/Makefile.am b/doc/Makefile.am
index d3bfec9..78dd681 100644
--- a/doc/Makefile.am
+++ b/doc/Makefile.am
@@ -1,6 +1,7 @@
 # $Id$
 
 CHANGELOGS = \
+	changes-2.1.5.html \
 	changes-2.1.4.html \
 	changes-2.1.3.html \
 	changes-2.1.2.html \
@@ -19,6 +20,7 @@ CHANGELOGS = \
 	changes-1.0.4.html
 
 XML = \
+	changes-2.1.4-2.1.5.xml \
 	changes-2.1.3-2.1.4.xml \
 	changes-2.1.2-2.1.3.xml \
 	changes-2.1.1-2.1.2.xml \
diff --git a/doc/changes-2.1.4-2.1.5.xml b/doc/changes-2.1.4-2.1.5.xml
index a2c482a..9b7e8ae 100644
--- a/doc/changes-2.1.4-2.1.5.xml
+++ b/doc/changes-2.1.4-2.1.5.xml
@@ -102,7 +102,7 @@
       <para>Add bourn-shell like "here" documents on the CLI. Typical
       usage:
       <code>
-      vcl.inline vcl_new << 42
+      vcl.inline vcl_new &lt;&lt; 42
       backend foo {...}
       sub vcl_recv {...}
       42
diff --git a/doc/changes-2.1.5.xml b/doc/changes-2.1.5.xml
new file mode 100644
index 0000000..7e37731
--- /dev/null
+++ b/doc/changes-2.1.5.xml
@@ -0,0 +1,11 @@
+<?xml version="1.0" encoding="utf-8"?>
+<?xml-stylesheet type="text/xml" href="changes-html.xsl"?>
+<!DOCTYPE changelog [
+ <!ENTITY mdash "&#8212;">
+]>
+<changelog xmlns:xi="http://www.w3.org/2001/XInclude">
+  <package>Varnish</package>
+  <version>2.1.5</version>
+
+  <xi:include href="changes-2.1.4-2.1.5.xml"/>
+</changelog>



From kristian at varnish-cache.org  Mon Jan 24 12:24:16 2011
From: kristian at varnish-cache.org (=?UTF-8?Q?Kristian_Lyngst=C3=B8l?=)
Date: Mon, 24 Jan 2011 13:24:16 +0100
Subject: [2.1] 8ef5fb0 Changes for 2.1.5
Message-ID: <E1PhLSy-0008Pp-3M@project.varnish-software.com>

commit 8ef5fb0a50eb7c351b48b30788cd80001bf1a32d
Author: Kristian Lyngstol <kristian at bohemians.org>
Date:   Mon Jan 24 13:09:21 2011 +0100

    Changes for 2.1.5

diff --git a/doc/changes-2.1.4-2.1.5.xml b/doc/changes-2.1.4-2.1.5.xml
new file mode 100644
index 0000000..a2c482a
--- /dev/null
+++ b/doc/changes-2.1.4-2.1.5.xml
@@ -0,0 +1,132 @@
+<?xml version="1.0" encoding="utf-8"?>
+<!DOCTYPE group [
+<!ENTITY mdash "&#8212;">
+]>
+<group from="2.1.4" to="2.1.5">
+  <subsystem>
+    <name>varnishd</name>
+    <change type="bug" ref="801">
+      <para>On pass from vcl_recv, we did not remove the backends Content-Length
+      header before adding our own. This could cause confusion for
+      browsers and has been fixed.</para>
+    </change>
+   
+    <change type="bug" ref="806">
+      <para>Make pass with content-length work again. An issue with regards
+      to 304, Content-Length and pass has been resolved.</para>
+    </change>
+
+    <change type="bug" ref="803">
+      <para>An issue relating to passed requests with If-Modified-Since
+      headers has been fixed. Varnish did not recognize that the
+      304-response did not have a body.</para>
+    </change>
+   
+    <change type="bug">
+      <para>A potential lock-inversion with the ban lurker thread has been
+      resolved.</para>
+    </change>
+
+    <change type="bug">
+      <para>Several build-dependency issues relating to rst2man have been
+      fixed. Varnish should now build from source without rst2man if you
+      are using tar-balls.</para>
+    </change>
+
+    <change type="bug" ref="780">
+      <para>Ensure Varnish reads the expected last CRLF after chunked data
+      from the backend. This allows re-use of the connection.</para>
+    </change>
+
+    <change type="bug">
+      <para>Remove a GNU Make-ism during <code>make dist</code> to make BSD
+      happier.</para>
+    </change>
+
+    <change type="enh">
+      <para>Document the log, set, unset, return and restart statements in
+      the VCL documentation.</para>
+    </change>
+
+    <change type="bug" ref="808">
+      <para>Fix an embarrassingly old bug where Varnish would run out of
+      workspace when requests come in fast over a single connection,
+      typically during synthetic benchmarks.</para>
+    </change>
+
+    <change type="bug" ref="795">
+      <para>Varnish will now allow <code>If-Modified-Since</code> requests
+      to objects without a <code>Last-Modified</code>-header, and instead
+      use the time the object was cached instead.</para>
+    </change>
+
+    <change type="bug" ref="789">
+      <para>Do not filter out <code>Content-Range</code> headers in
+      pass.</para>
+    </change>
+
+    <change type="enh">
+      <para>Require -d, -b, -f, -S or -T when starting Varnishd. In human
+      terms, this means that it is legal to start varnishd without a Vcl or
+      backend, but only if you have a CLI channel of some kind.</para>
+    </change>
+
+    <change type="bug">
+      <para>Don't suppress <code>Cache-Control</code> headers in pass
+      responses.</para>
+    </change>
+
+    <change type="enh">
+      <para>Merge multi-line Cache-Control and Vary header fields. Until
+      now, no browsers have needed this, but Chromium seems to find it
+      necessary to spread its Cache-Control across two lines, and we get to
+      deal with it.</para>
+    </change>
+
+    <change type="bug" ref="812">
+      <para>Make new-purge not touch busy objects. This fixes a potential
+      crash when calling <code>VRT_purge</code>.</para>
+    </change>
+    
+    <change type="enh">
+      <para>If there are everal grace-able objects, pick the least expired
+      one.</para>
+    </change>
+
+    <change type="bug" ref="817">
+      <para>Fix an issue with <code>varnishadm -T :6082</code>
+      shorthand.</para>
+    </change>
+
+    <change type="enh">
+      <para>Add bourn-shell like "here" documents on the CLI. Typical
+      usage:
+      <code>
+      vcl.inline vcl_new << 42
+      backend foo {...}
+      sub vcl_recv {...}
+      42
+      </code></para>
+    </change>
+
+    <change type="enh">
+      <para>Add CLI version to the CLI-banner, starting with version 1.0 to
+      mark here-documents.</para>
+    </change>
+
+    <change type="bug">
+      <para>Fix a problem with the expiry thread slacking off during high
+      load.</para>
+    </change>
+
+  </subsystem>
+
+  <subsystem>
+    <name>varnishtest</name>
+    <change type="bug" ref="804">
+      <para>Remove no longer existing <code>-L</code> option.</para>
+    </change>
+  </subsystem>
+
+
+</group>



From phk at varnish-cache.org  Mon Jan 24 13:48:33 2011
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Mon, 24 Jan 2011 14:48:33 +0100
Subject: [master] 9469d95 Update with ESI+GZIP section
Message-ID: <E1PhMmX-0006SW-Bo@project.varnish-software.com>

commit 9469d9524c32a137852196600d62874401d5f19c
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Jan 24 13:48:18 2011 +0000

    Update with ESI+GZIP section

diff --git a/doc/sphinx/phk/gzip.rst b/doc/sphinx/phk/gzip.rst
index 77ba94f..5d00b21 100644
--- a/doc/sphinx/phk/gzip.rst
+++ b/doc/sphinx/phk/gzip.rst
@@ -36,11 +36,13 @@ hash lookup.
 Varnish will not do any other types of compressions than gzip, in particular
 we will not do deflate, as there are browser bugs in that case.
 
-Before vcl_hit{} is called, the backend requests Accept-Encoding is
+Before vcl_miss{} is called, the backend requests Accept-Encoding is
 always set to:
 
 	Accept-Encoding: gzip
 
+Even if this particular client does not support 
+
 To always entice the backend into sending us gzip'ed content.
 
 Varnish will not gzip any content on its own (but see below), we trust
@@ -63,6 +65,9 @@ Even if the client does not support gzip, you can force the A-C header
 to "gzip" to save bandwidth between the backend and varnish, varnish will
 gunzip the object before delivering to the client.
 
+In vcl_miss{} you can remove the "Accept-Encoding: gzip" header, if you
+do not want the backend to gzip this object.
+
 In vcl_fetch{} two new variables allow you to modify the gzip-ness of
 objects during fetch:
 
@@ -89,4 +94,70 @@ typical use would be:
 GZIP and ESI
 ------------
 
-TBD.
+First, note the new syntax for activating ESI:
+
+	sub vcl_fetch {
+		set beresp.do_esi = true;
+	}
+
+In theory, and hopefully in practice, all you read above should apply also
+when you enable ESI, if not it is a bug you should report.
+
+But things are vastly more complicated now.  What happens for
+instance, when the backend sends a gzip'ed object we ESI process
+it and it includes another object which is not gzip'ed, and we want
+to send the result gziped to the client ?
+
+Things can get really hairy here, so let me explain it in stages.
+
+Assume we have a ungzipped object we want to ESI process.
+
+The ESI parser will run through the object looking for the various
+magic strings and produce a byte-stream we call the "VEC" for Varnish
+ESI Codes.
+
+The VEC contains instructions like "skip 234 bytes", "deliver 12919 bytes",
+"include /foobar", "deliver 122 bytes" etc and it is stored with the
+object.
+
+When we deliver an object, and it has a VEC, special esi-delivery code
+interprets the VEC string and sends the output to the client as ordered.
+
+When the VEC says "include /foobar" we do what amounts to a restart with
+the new URL and possibly Host: header, and call vcl_recv{} etc.  You
+can tell that you are in an ESI include by examining the 'req.esi_level'
+variable in VCL.
+
+The ESI-parsed object is stored gzip'ed under the same conditions as
+above:  If the backend sends gzip'ed and VCL did not ask for do_gunzip,
+or if the backend sends ungzip'ed and VCL asked for do_gzip.
+
+Please note that since we need to insert flush and reset points in
+the gzip file, it will be slightly larger than a normal gzip file of
+the same object.
+
+When we encounter gzip'ed include objects which should not be, we
+gunzip them, but when we encounter gunzip'ed objects which should
+be, we gzip them, but only at compression level zero.
+
+So in order to avoid unnecessary work, and in order to get maximum
+compression efficiency, you should:
+
+	sub vcl_miss {
+		if (object needs ESI processing) {
+			unset bereq.http.accept-encoding;
+		}
+	}
+
+	sub vcl_fetch {
+		if (object needs ESI processing) {
+			set beresp.do_esi = true;
+			set beresp.do_gzip = true;
+		}
+	}
+
+So that the backend sends these objects uncompressed to varnish.
+
+You should also attempt to make sure that all objects which are
+esi:included are gziped, either by making the backend do it or
+by making varnish do it.



From phk at varnish-cache.org  Mon Jan 24 20:58:20 2011
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Mon, 24 Jan 2011 21:58:20 +0100
Subject: [master] b470a65 When running with gzip support enabled,
	ignore "accept-encoding" in vary-matching. Varnish will adapt
	whatever we find to the clients abilities as necessary.
Message-ID: <E1PhTUS-0000HN-2v@project.varnish-software.com>

commit b470a6524ac0917b20889740dc4c276c1a0d8d55
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Jan 24 20:57:15 2011 +0000

    When running with gzip support enabled, ignore "accept-encoding" in
    vary-matching.  Varnish will adapt whatever we find to the clients
    abilities as necessary.

diff --git a/bin/varnishd/cache_vary.c b/bin/varnishd/cache_vary.c
index 24770a3..9149df0 100644
--- a/bin/varnishd/cache_vary.c
+++ b/bin/varnishd/cache_vary.c
@@ -144,6 +144,24 @@ VRY_Match(const struct sess *sp, const unsigned char *vary)
 
 	while (*vary) {
 
+		if (params->http_gzip_support &&
+		    !strcasecmp(H_Accept_Encoding, (const char*)vary)) {
+			/*
+			 * If we do gzip processing, we do not vary on
+			 * Accept-Encoding, because we want everybody to
+			 * get the gzip'ed object, and varnish will gunzip
+			 * as necessary.  We implement the skip at check
+			 * time, rather than create time, so that object
+			 * in persistent storage can be used with either
+			 *  setting of http_gzip_support.
+			 */
+			vary += *vary + 2;
+			l = vary[0] * 256 + vary[1];
+			vary += 2;
+			if (l != 0xffff)
+				vary += l;
+			continue;
+		}
 		/* Look for header */
 		i = http_GetHdr(sp->http, (const char*)vary, &h);
 		vary += *vary + 2;
diff --git a/bin/varnishtest/tests/e00025.vtc b/bin/varnishtest/tests/e00025.vtc
new file mode 100644
index 0000000..6d2e90c
--- /dev/null
+++ b/bin/varnishtest/tests/e00025.vtc
@@ -0,0 +1,26 @@
+# $Id$
+
+test "Test that esi+gzip correctly bypasses Vary: accept-encoding"
+
+server s1 {
+	rxreq 
+	expect req.http.accept-encoding == gzip
+	txresp -hdr "Vary: Accept-encoding" -gzipbody {FOO}
+} -start
+
+varnish v1 -vcl+backend { } -start
+
+client c1 {
+	txreq -hdr "Accept-encoding: gzip"
+	rxresp
+	expect resp.http.content-encoding == gzip
+	expect resp.status == 200
+	gunzip
+	expect resp.bodylen == 3
+
+	txreq 
+	rxresp
+	expect resp.http.content-encoding == resp.http.content-encoding
+	expect resp.status == 200
+	expect resp.bodylen == 3
+} -run



From kristian at varnish-cache.org  Mon Jan 24 21:27:56 2011
From: kristian at varnish-cache.org (=?UTF-8?Q?Kristian_Lyngst=C3=B8l?=)
Date: Mon, 24 Jan 2011 22:27:56 +0100
Subject: [master] 8588a5a Add an extensive .gitignore
Message-ID: <E1PhTx6-0004ut-P1@project.varnish-software.com>

commit 8588a5a19f74ee40f758de9241fd7126ae87d9c7
Author: Kristian Lyngstol <kristian at bohemians.org>
Date:   Mon Jan 24 22:26:47 2011 +0100

    Add an extensive .gitignore
    
    Essentially ignores everything git status would list after a normal build.

diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000..9136788
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,78 @@
+# Matches ALL Makefile and Makefile.in occurrences 
+Makefile
+Makefile.in
+
+# ...
+.deps/
+.libs/
+*.o
+*.lo
+*.la
+
+# Various auto-tools artifacts
+/aclocal.m4
+/autom4te.cache/
+/compile
+/config.guess
+/config.h
+/config.h.in
+/config.h.in~
+/config.log
+/config.status
+/config.sub
+/configure
+/depcomp
+/install-sh
+/libtool
+/ltmain.sh
+/m4/libtool.m4
+/m4/ltoptions.m4
+/m4/ltsugar.m4
+/m4/ltversion.m4
+/m4/lt~obsolete.m4
+/missing
+/stamp-h1
+/varnishapi.pc
+
+# Default vcl made from bin/varnishd/default.vcl
+/bin/varnishd/default_vcl.h
+/etc/default.vcl
+
+# Various auto-generated code snippets
+/include/vcl.h
+/include/vcl_returns.h
+/include/vrt_obj.h
+/include/vrt_stv_var.h
+/lib/libvarnish/svn_version.c
+/lib/libvcl/vcc_fixed_token.c
+/lib/libvcl/vcc_obj.c
+/lib/libvcl/vcc_token_defs.h
+/lib/libvmod_std/vcc_if.c
+/lib/libvmod_std/vcc_if.h
+
+# Man-files and binaries
+/man/vcl.7
+/bin/varnishadm/varnishadm
+/bin/varnishadm/varnishadm.1
+/bin/varnishd/varnishd
+/bin/varnishd/varnishd.1
+/bin/varnishhist/varnishhist
+/bin/varnishhist/varnishhist.1
+/bin/varnishlog/varnishlog
+/bin/varnishlog/varnishlog.1
+/bin/varnishncsa/varnishncsa
+/bin/varnishncsa/varnishncsa.1
+/bin/varnishreplay/varnishreplay
+/bin/varnishreplay/varnishreplay.1
+/bin/varnishsizes/varnishsizes
+/bin/varnishsizes/varnishsizes.1
+/bin/varnishstat/varnishstat
+/bin/varnishstat/varnishstat.1
+/bin/varnishtest/varnishtest
+/bin/varnishtest/varnishtest.1
+/bin/varnishtop/varnishtop
+/bin/varnishtop/varnishtop.1
+
+# Doc-stuff generated from xml
+/doc/*.html
+/doc/sphinx/=build/



From kristian at varnish-cache.org  Mon Jan 24 21:27:57 2011
From: kristian at varnish-cache.org (=?UTF-8?Q?Kristian_Lyngst=C3=B8l?=)
Date: Mon, 24 Jan 2011 22:27:57 +0100
Subject: [master] 51ab1a7 Merge branch 'master' of
	varnish-cache.org:/git/varnish-cache
Message-ID: <E1PhTx7-0004v1-3j@project.varnish-software.com>

commit 51ab1a73cf9e30dfc9a4e01047454532d62374ee
Merge: 8588a5a b470a65
Author: Kristian Lyngstol <kristian at bohemians.org>
Date:   Mon Jan 24 22:27:42 2011 +0100

    Merge branch 'master' of varnish-cache.org:/git/varnish-cache



From phk at varnish-cache.org  Mon Jan 24 22:03:24 2011
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Mon, 24 Jan 2011 23:03:24 +0100
Subject: [master] 993021b Merge branch 'master' of
	ssh://git.varnish-cache.org/git/varnish-cache
Message-ID: <E1PhUVQ-0003gx-Rg@project.varnish-software.com>

commit 993021b5632f9b0caefcacd206f43fbb7abaf0c0
Merge: e4a26ad 51ab1a7
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Jan 24 21:31:09 2011 +0000

    Merge branch 'master' of ssh://git.varnish-cache.org/git/varnish-cache



From phk at varnish-cache.org  Mon Jan 24 22:03:23 2011
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Mon, 24 Jan 2011 23:03:23 +0100
Subject: [master] e4a26ad Update with the Vary/Accept-Encoding functionality.
Message-ID: <E1PhUVP-0003gh-Vk@project.varnish-software.com>

commit e4a26ad1a0efeafe0b62ecea0332f5b58169c41e
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Jan 24 21:22:31 2011 +0000

    Update with the Vary/Accept-Encoding functionality.

diff --git a/doc/sphinx/phk/gzip.rst b/doc/sphinx/phk/gzip.rst
index 5d00b21..a87034a 100644
--- a/doc/sphinx/phk/gzip.rst
+++ b/doc/sphinx/phk/gzip.rst
@@ -29,9 +29,15 @@ Clients which can do gzip, gets their header rewritten to:
 
 	Accept-Encoding: gzip
 
-And clients which do not support gzip gets their Accept-Encoding header
-removed.  This ensures conformity with respect to Vary: strings during
-hash lookup.
+And clients which do not support gzip gets their Accept-Encoding
+header removed.  This ensures conformity with respect to creating
+Vary: strings during object creation.
+
+During lookup, we ignore any "Accept-encoding" in objects Vary: strings,
+to avoid having a gzip and gunzip'ed version of the object, varnish
+can gunzip on demand.  (We implement this bit of magic at lookup time,
+so that any objects stored in persistent storage can be used with
+or without gzip support enabled.)
 
 Varnish will not do any other types of compressions than gzip, in particular
 we will not do deflate, as there are browser bugs in that case.



From phk at varnish-cache.org  Mon Jan 24 22:05:45 2011
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Mon, 24 Jan 2011 23:05:45 +0100
Subject: [master] f85567f Try to solve Linux compile issue. There is a special
	place reserved in my section of hell for the dimwits who
	decided that switching off_t from 32 to 64 bits was too scary
	to contemplate.
Message-ID: <E1PhUXh-000434-F4@project.varnish-software.com>

commit f85567f5d7d402d644193d8a2d339c6778f78736
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Jan 24 22:03:43 2011 +0000

    Try to solve Linux compile issue.  There is a special place reserved in
    my section of hell for the dimwits who decided that switching off_t from
    32 to 64 bits was too scary to contemplate.

diff --git a/lib/libvgz/Makefile.am b/lib/libvgz/Makefile.am
index 446dc64..573e033 100644
--- a/lib/libvgz/Makefile.am
+++ b/lib/libvgz/Makefile.am
@@ -3,6 +3,7 @@
 lib_LTLIBRARIES = libvgz.la
 
 libvgz_la_LDFLAGS = -version-info 1:0:0
+libvgz_la_CFLAGS = -D_LARGEFILE64_SOURCE=1
 
 libvgz_la_SOURCES = \
 	adler32.c \
diff --git a/lib/libvgz/vgz.h b/lib/libvgz/vgz.h
index d6855be..3f6e8cb 100644
--- a/lib/libvgz/vgz.h
+++ b/lib/libvgz/vgz.h
@@ -1573,7 +1573,9 @@ ZEXTERN int ZEXPORT inflateBackInit_ OF((z_streamp strm, int windowBits,
    ZEXTERN z_off64_t ZEXPORT gzoffset64 OF((gzFile));
    ZEXTERN uLong ZEXPORT adler32_combine64 OF((uLong, uLong, z_off64_t));
    ZEXTERN uLong ZEXPORT crc32_combine64 OF((uLong, uLong, z_off64_t));
-#elif !defined(ZLIB_INTERNAL) && _FILE_OFFSET_BITS-0 == 64 && _LFS64_LARGEFILE-0
+#endif
+
+#if !defined(ZLIB_INTERNAL) && _FILE_OFFSET_BITS-0 == 64 && _LFS64_LARGEFILE-0
 #  define gzopen gzopen64
 #  define gzseek gzseek64
 #  define gztell gztell64



From tfheen at varnish-cache.org  Tue Jan 25 08:16:18 2011
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Tue, 25 Jan 2011 09:16:18 +0100
Subject: [master] 23ab89b Adjust Source path in spec file
Message-ID: <E1Phe4Y-0005PA-J9@project.varnish-software.com>

commit 23ab89b72852e7a75b0277ab04fac89c94d4e636
Author: Tollef Fog Heen <tfheen at err.no>
Date:   Tue Jan 25 09:15:37 2011 +0100

    Adjust Source path in spec file
    
    The tarballs live on repo.varnish-cache.org, not
    www.varnish-software.com any longer, so adjust spec file to match.

diff --git a/redhat/varnish.spec b/redhat/varnish.spec
index 723891a..29200a9 100644
--- a/redhat/varnish.spec
+++ b/redhat/varnish.spec
@@ -5,7 +5,7 @@ Release: 0.svn20101115r5543%{?dist}
 License: BSD
 Group: System Environment/Daemons
 URL: http://www.varnish-cache.org/
-Source0: http://www.varnish-software.com/sites/default/files/%{name}-%{version}.tar.gz
+Source0: http://repo.varnish-cache.org/source/%{name}-%{version}.tar.gz
 BuildRoot: %{_tmppath}/%{name}-%{version}-%{release}-root-%(%{__id_u} -n)
 # The svn sources needs autoconf, automake and libtool to generate a suitable
 # configure script. Release tarballs would not need this



From phk at varnish-cache.org  Tue Jan 25 09:00:14 2011
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Tue, 25 Jan 2011 10:00:14 +0100
Subject: [master] 8b9e46f Cast to intmax_t for printf
Message-ID: <E1Phel4-0004Gx-3Q@project.varnish-software.com>

commit 8b9e46f8ae4a0a733cf2cb86a5f48414844d930d
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Jan 25 08:59:47 2011 +0000

    Cast to intmax_t for printf

diff --git a/bin/varnishd/cache_esi_deliver.c b/bin/varnishd/cache_esi_deliver.c
index 6695dee..b16c345 100644
--- a/bin/varnishd/cache_esi_deliver.c
+++ b/bin/varnishd/cache_esi_deliver.c
@@ -134,7 +134,7 @@ ved_sendchunk(const struct sess *sp, const void *cb, ssize_t cl,
 	assert(l > 0);
 	if (sp->wrk->res_mode & RES_CHUNKED) {
 		if (cb == NULL) {
-			bprintf(chunk, "%jx\r\n", l);
+			bprintf(chunk, "%jx\r\n", (intmax_t)l);
 			(void)WRW_Write(sp->wrk, chunk, -1);
 		} else
 			(void)WRW_Write(sp->wrk, cb, cl);



From phk at varnish-cache.org  Tue Jan 25 10:15:39 2011
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Tue, 25 Jan 2011 11:15:39 +0100
Subject: [master] 30ef1f0 Don't kill varnishd process before it has a chance
	to tell us its trouble
Message-ID: <E1Phfw3-0002a8-0K@project.varnish-software.com>

commit 30ef1f0a6f448376bc36aa61de51a1d0f24293ed
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Jan 25 09:54:50 2011 +0000

    Don't kill varnishd process before it has a chance to tell us its trouble

diff --git a/bin/varnishtest/vtc_varnish.c b/bin/varnishtest/vtc_varnish.c
index 1daf6dc..dd59251 100644
--- a/bin/varnishtest/vtc_varnish.c
+++ b/bin/varnishtest/vtc_varnish.c
@@ -305,15 +305,11 @@ varnish_launch(struct varnish *v)
 	fd.events = POLLIN;
 	i = poll(&fd, 1, 10000);
 	if (i != 1) {
-		AZ(close(v->fds[1]));
-		(void)kill(v->pid, SIGKILL);
 		vtc_log(v->vl, 0, "FAIL timeout waiting for CLI connection");
 		return;
 	}
 	nfd = accept(v->cli_fd, NULL, NULL);
 	if (nfd < 0) {
-		AZ(close(v->fds[1]));
-		(void)kill(v->pid, SIGKILL);
 		vtc_log(v->vl, 0, "FAIL no CLI connection accepted");
 		return;
 	}



From phk at varnish-cache.org  Tue Jan 25 10:15:40 2011
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Tue, 25 Jan 2011 11:15:40 +0100
Subject: [master] 711b83d Use more reasonalby sized stack buffers: 8k instead
	of 64k
Message-ID: <E1Phfw4-0002aL-QB@project.varnish-software.com>

commit 711b83de00d1c4931fe3a30306b4d93e61a81af8
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Jan 25 10:04:16 2011 +0000

    Use more reasonalby sized stack buffers: 8k instead of 64k

diff --git a/lib/libvmod_std/vmod_std.c b/lib/libvmod_std/vmod_std.c
index 9210e26..e032905 100644
--- a/lib/libvmod_std/vmod_std.c
+++ b/lib/libvmod_std/vmod_std.c
@@ -138,7 +138,7 @@ vmod_random(struct sess *sp, double lo, double hi)
 void
 vmod_log(struct sess *sp, const char *fmt, ...)
 {
-	char buf[64*1024], *p;
+	char buf[8192], *p;
 	va_list ap;
 
 	va_start(ap, fmt);
@@ -151,7 +151,7 @@ vmod_log(struct sess *sp, const char *fmt, ...)
 void
 vmod_syslog(struct sess *sp, int fac, const char *fmt, ...)
 {
-	char buf[64*1024], *p;
+	char buf[8192], *p;
 	va_list ap;
 
 	(void)sp;



From phk at varnish-cache.org  Tue Jan 25 10:15:42 2011
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Tue, 25 Jan 2011 11:15:42 +0100
Subject: [master] fba1456 Report which param setting failed,
	this is not obvious during startup.
Message-ID: <E1Phfw6-0002aX-1F@project.varnish-software.com>

commit fba14565d17114a446ad6a9c85bd6d63156e75c8
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Jan 25 10:07:04 2011 +0000

    Report which param setting failed, this is not obvious during startup.

diff --git a/bin/varnishd/mgt_param.c b/bin/varnishd/mgt_param.c
index f126736..2f71eef 100644
--- a/bin/varnishd/mgt_param.c
+++ b/bin/varnishd/mgt_param.c
@@ -954,6 +954,8 @@ MCF_ParamSet(struct cli *cli, const char *param, const char *val)
 	if (pp != NULL) {
 		pp->func(cli, pp, val);
 		if (cli->result != CLIS_OK) {
+			cli_out(cli, "(attempting to set param %s to %s)\n",
+			    pp->name, val);
 		} else if (child_pid >= 0 && pp->flags & MUST_RESTART) {
 			cli_out(cli, "Change will take effect"
 			    " when child is restarted");



From phk at varnish-cache.org  Tue Jan 25 10:15:44 2011
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Tue, 25 Jan 2011 11:15:44 +0100
Subject: [master] be21c60 Make the gzip_stack_buffer parameter have units of
	bytes for consistency.
Message-ID: <E1Phfw8-0002bF-Cm@project.varnish-software.com>

commit be21c60c8b4ab104d4237f2b5b66abfbd6eca6ba
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Jan 25 10:09:39 2011 +0000

    Make the gzip_stack_buffer parameter have units of bytes for consistency.

diff --git a/bin/varnishd/cache_esi_deliver.c b/bin/varnishd/cache_esi_deliver.c
index b16c345..c687dd7 100644
--- a/bin/varnishd/cache_esi_deliver.c
+++ b/bin/varnishd/cache_esi_deliver.c
@@ -239,7 +239,7 @@ ESI_Deliver(struct sess *sp)
 	uint8_t tailbuf[8 + 5];
 	int isgzip;
 	struct vgz *vgz = NULL;
-	char obuf[1024 * params->gzip_stack_buffer];
+	char obuf[params->gzip_stack_buffer];
 	ssize_t obufl = 0;
 	size_t dl;
 	const void *dp;
diff --git a/bin/varnishd/cache_esi_fetch.c b/bin/varnishd/cache_esi_fetch.c
index 909d77b..499f6b6 100644
--- a/bin/varnishd/cache_esi_fetch.c
+++ b/bin/varnishd/cache_esi_fetch.c
@@ -101,7 +101,7 @@ vfp_esi_bytes_gu(struct sess *sp, struct http_conn *htc, ssize_t bytes)
 {
 	struct vgz *vg;
 	ssize_t w;
-	uint8_t	ibuf[1024 * params->gzip_stack_buffer];
+	uint8_t	ibuf[params->gzip_stack_buffer];
 	int i;
 	size_t dl;
 	const void *dp;
@@ -211,7 +211,7 @@ static int __match_proto__()
 vfp_esi_bytes_ug(struct sess *sp, struct http_conn *htc, ssize_t bytes)
 {
 	ssize_t w;
-	char ibuf[1024 * params->gzip_stack_buffer];
+	char ibuf[params->gzip_stack_buffer];
 	struct vef_priv *vef;
 
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
@@ -247,8 +247,8 @@ static int __match_proto__()
 vfp_esi_bytes_gg(struct sess *sp, struct http_conn *htc, size_t bytes)
 {
 	ssize_t w;
-	char ibuf[1024 * params->gzip_stack_buffer];
-	char ibuf2[1024 * params->gzip_stack_buffer];
+	char ibuf[params->gzip_stack_buffer];
+	char ibuf2[params->gzip_stack_buffer];
 	struct vef_priv *vef;
 	size_t dl;
 	const void *dp;
diff --git a/bin/varnishd/cache_gzip.c b/bin/varnishd/cache_gzip.c
index 9d83462..4b62d24 100644
--- a/bin/varnishd/cache_gzip.c
+++ b/bin/varnishd/cache_gzip.c
@@ -368,7 +368,7 @@ vfp_gunzip_bytes(struct sess *sp, struct http_conn *htc, ssize_t bytes)
 	struct vgz *vg;
 	ssize_t l, w;
 	int i = -100;
-	uint8_t	ibuf[1024 * params->gzip_stack_buffer];
+	uint8_t	ibuf[params->gzip_stack_buffer];
 	size_t dl;
 	const void *dp;
 
@@ -437,7 +437,7 @@ vfp_gzip_bytes(struct sess *sp, struct http_conn *htc, ssize_t bytes)
 	struct vgz *vg;
 	ssize_t l, w;
 	int i = -100;
-	uint8_t ibuf[1024 * params->gzip_stack_buffer];
+	uint8_t ibuf[params->gzip_stack_buffer];
 	size_t dl;
 	const void *dp;
 
@@ -513,7 +513,7 @@ vfp_testgzip_bytes(struct sess *sp, struct http_conn *htc, ssize_t bytes)
 	struct vgz *vg;
 	ssize_t l, w;
 	int i = -100;
-	uint8_t	ibuf[1024 * params->gzip_stack_buffer];
+	uint8_t	ibuf[params->gzip_stack_buffer];
 	size_t dl;
 	const void *dp;
 	struct storage *st;
diff --git a/bin/varnishd/cache_response.c b/bin/varnishd/cache_response.c
index 6347d82..c210482 100644
--- a/bin/varnishd/cache_response.c
+++ b/bin/varnishd/cache_response.c
@@ -247,7 +247,7 @@ res_WriteGunzipObj(struct sess *sp)
 	struct vgz *vg;
 	const void *dp;
 	char lenbuf[20];
-	char obuf[1024 * params->gzip_stack_buffer];
+	char obuf[params->gzip_stack_buffer];
 	size_t dl;
 	int i;
 
diff --git a/bin/varnishd/mgt_param.c b/bin/varnishd/mgt_param.c
index 2f71eef..e2151ad 100644
--- a/bin/varnishd/mgt_param.c
+++ b/bin/varnishd/mgt_param.c
@@ -818,10 +818,10 @@ static const struct parspec input_parspec[] = {
 		EXPERIMENTAL,
 		"on", "bool" },
 	{ "gzip_stack_buffer", tweak_uint, &master.gzip_stack_buffer,
-	        4, UINT_MAX,
+	        2048, UINT_MAX,
 		"Size of stack buffer used for gzip processing.\n",
 		EXPERIMENTAL,
-		"64", "Kilobytes" },
+		"32768", "Bytes" },
 	{ "critbit_cooloff", tweak_timeout_double,
 		&master.critbit_cooloff, 60, 254,
 		"How long time the critbit hasher keeps deleted objheads "



From phk at varnish-cache.org  Tue Jan 25 10:15:46 2011
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Tue, 25 Jan 2011 11:15:46 +0100
Subject: [master] 5a3808b Tweak the 32bit mode parameters: We need more stack
	and cannot use all of it for gzip buffers.
Message-ID: <E1PhfwA-0002bl-5t@project.varnish-software.com>

commit 5a3808beb1b0df2849a84b8cecad5b9750bb368d
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Jan 25 10:10:34 2011 +0000

    Tweak the 32bit mode parameters:  We need more stack and cannot use
    all of it for gzip buffers.

diff --git a/bin/varnishd/varnishd.c b/bin/varnishd/varnishd.c
index 6c40e8d..d175c49 100644
--- a/bin/varnishd/varnishd.c
+++ b/bin/varnishd/varnishd.c
@@ -418,7 +418,10 @@ main(int argc, char * const *argv)
 		MCF_ParamSet(cli, "sess_workspace", "16384");
 		cli_check(cli);
 
-		MCF_ParamSet(cli, "thread_pool_stack", "32");
+		MCF_ParamSet(cli, "thread_pool_stack", "65536");
+		cli_check(cli);
+
+		MCF_ParamSet(cli, "gzip_stack_buffer", "4096");
 		cli_check(cli);
 	}
 



From phk at varnish-cache.org  Tue Jan 25 10:15:51 2011
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Tue, 25 Jan 2011 11:15:51 +0100
Subject: [master] ba64981 You are in a creepy maze of #ifdefs, all confusting.
Message-ID: <E1PhfwF-0002cO-MY@project.varnish-software.com>

commit ba64981d54f824705ff4cb546762424d29d97452
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Jan 25 10:11:05 2011 +0000

    You are in a creepy maze of #ifdefs, all confusting.

diff --git a/lib/libvgz/vgz.h b/lib/libvgz/vgz.h
index 3f6e8cb..1b8da5e 100644
--- a/lib/libvgz/vgz.h
+++ b/lib/libvgz/vgz.h
@@ -1575,7 +1575,7 @@ ZEXTERN int ZEXPORT inflateBackInit_ OF((z_streamp strm, int windowBits,
    ZEXTERN uLong ZEXPORT crc32_combine64 OF((uLong, uLong, z_off64_t));
 #endif
 
-#if !defined(ZLIB_INTERNAL) && _FILE_OFFSET_BITS-0 == 64 && _LFS64_LARGEFILE-0
+#if defined(NOTDEF) && !defined(ZLIB_INTERNAL) && _FILE_OFFSET_BITS-0 == 64 && _LFS64_LARGEFILE-0
 #  define gzopen gzopen64
 #  define gzseek gzseek64
 #  define gztell gztell64



From phk at varnish-cache.org  Tue Jan 25 10:33:28 2011
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Tue, 25 Jan 2011 11:33:28 +0100
Subject: [master] 9c932e4 Don't take sizeof a char[x] parameter,
	trust it to be long enough.
Message-ID: <E1PhgDI-0005gO-2k@project.varnish-software.com>

commit 9c932e459e0a59ee84f9bb35d060c1c869e88851
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Jan 25 10:33:04 2011 +0000

    Don't take sizeof a char[x] parameter, trust it to be long enough.

diff --git a/bin/varnishd/cache_response.c b/bin/varnishd/cache_response.c
index c210482..7065e44 100644
--- a/bin/varnishd/cache_response.c
+++ b/bin/varnishd/cache_response.c
@@ -288,7 +288,7 @@ res_WriteGunzipObj(struct sess *sp)
 /*--------------------------------------------------------------------*/
 
 static void
-res_WriteDirObj(struct sess *sp, char lenbuf[20], size_t low, size_t high)
+res_WriteDirObj(struct sess *sp, char *lenbuf, size_t low, size_t high)
 {
 	unsigned u = 0;
 	size_t ptr, off, len;
@@ -297,7 +297,7 @@ res_WriteDirObj(struct sess *sp, char lenbuf[20], size_t low, size_t high)
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 
 	if (sp->wrk->res_mode & RES_CHUNKED) {
-		bprintf(lenbuf, "%jx\r\n", (intmax_t)sp->obj->len);
+		sprintf(lenbuf, "%jx\r\n", (intmax_t)sp->obj->len);
 		(void)WRW_Write(sp->wrk, lenbuf, -1);
 	}
 



From perbu at varnish-cache.org  Tue Jan 25 11:54:02 2011
From: perbu at varnish-cache.org (Per Andreas Buer)
Date: Tue, 25 Jan 2011 12:54:02 +0100
Subject: [master] c958305 Version specifics on client.identity
Message-ID: <E1PhhTG-00024L-54@project.varnish-software.com>

commit c95830510e94dc280b038d5e7dd9c1bed2a537c9
Author: Per Buer <perbu at varnish-software.com>
Date:   Tue Jan 25 12:53:59 2011 +0100

    Version specifics on client.identity

diff --git a/doc/sphinx/reference/vcl.rst b/doc/sphinx/reference/vcl.rst
index 54d66a0..e8da875 100644
--- a/doc/sphinx/reference/vcl.rst
+++ b/doc/sphinx/reference/vcl.rst
@@ -174,8 +174,9 @@ The client director picks a backend based on the clients
 *identity*. You can set the VCL variable *client.identity* to identify
 the client by picking up the value of a session cookie or similar.
 
-Note: in 2.1 *client.identity* isn't available and the director will
-use client.ip to distribute clients across backends.
+Note: from 2.1.0 to 2.1.2 *client.identity* isn't available and the
+director will use automatically set the idenity based on client.ip In
+2.1.3 and onwards you can set client.identity to any string available.
 
 The client director takes one option - *retries* which set the number
 of retries the director should take in order to find a healthy



From tfheen at varnish-cache.org  Tue Jan 25 13:51:23 2011
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Tue, 25 Jan 2011 14:51:23 +0100
Subject: [master] 74f9b69 Add -o option to varnishncsa synopsis
Message-ID: <E1PhjIp-0004nK-RB@project.varnish-software.com>

commit 74f9b6972eebc29b6272f2517d7541eda2dbb1dd
Author: Tollef Fog Heen <tfheen at err.no>
Date:   Tue Jan 25 14:51:09 2011 +0100

    Add -o option to varnishncsa synopsis

diff --git a/doc/sphinx/reference/varnishncsa.rst b/doc/sphinx/reference/varnishncsa.rst
index 15a8cee..9c7a012 100644
--- a/doc/sphinx/reference/varnishncsa.rst
+++ b/doc/sphinx/reference/varnishncsa.rst
@@ -16,7 +16,7 @@ SYNOPSIS
 ========
 
 varnishncsa [-a] [-b] [-C] [-c] [-D] [-d] [-f] [-I regex] 
-[-i tag] [-n varnish_name] [-P file] [-r file] [-V] 
+[-i tag] [-n varnish_name] [-o tag regex] [-P file] [-r file] [-V] 
 [-w file] [-X regex] [-x tag]
 
 DESCRIPTION



From phk at varnish-cache.org  Wed Jan 26 10:48:54 2011
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Wed, 26 Jan 2011 11:48:54 +0100
Subject: [master] 0197fd4 Add two parameters more: One to control gzip level
	(it was zero until now,
	nobody seems to have noticed ?) and one to control where we put the
	temporary allocations for gzip/gunzip.
Message-ID: <E1Pi2vm-00029e-Fh@project.varnish-software.com>

commit 0197fd48c22da43382e2617c5832cfdfc14affba
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Jan 26 10:48:04 2011 +0000

    Add two parameters more:  One to control gzip level (it was zero
    until now, nobody seems to have noticed ?) and one to control where
    we put the temporary allocations for gzip/gunzip.

diff --git a/bin/varnishd/cache.h b/bin/varnishd/cache.h
index 2a5ff93..ead98c1 100644
--- a/bin/varnishd/cache.h
+++ b/bin/varnishd/cache.h
@@ -640,8 +640,8 @@ void Fetch_Init(void);
 struct vgz;
 
 enum vgz_flag { VGZ_NORMAL, VGZ_ALIGN, VGZ_RESET, VGZ_FINISH };
-struct vgz *VGZ_NewUngzip(const struct sess *sp, struct ws *tmp);
-struct vgz *VGZ_NewGzip(const struct sess *sp, struct ws *tmp);
+struct vgz *VGZ_NewUngzip(struct sess *sp);
+struct vgz *VGZ_NewGzip(struct sess *sp);
 void VGZ_Ibuf(struct vgz *, const void *, ssize_t len);
 int VGZ_IbufEmpty(const struct vgz *vg);
 void VGZ_Obuf(struct vgz *, const void *, ssize_t len);
diff --git a/bin/varnishd/cache_esi_deliver.c b/bin/varnishd/cache_esi_deliver.c
index c687dd7..589feeb 100644
--- a/bin/varnishd/cache_esi_deliver.c
+++ b/bin/varnishd/cache_esi_deliver.c
@@ -274,7 +274,7 @@ ESI_Deliver(struct sess *sp)
 	}
 
 	if (isgzip && !sp->wrk->gzip_resp) {
-		vgz = VGZ_NewUngzip(sp, sp->wrk->ws);
+		vgz = VGZ_NewUngzip(sp);
 		obufl = 0;
 	}
 
diff --git a/bin/varnishd/cache_esi_fetch.c b/bin/varnishd/cache_esi_fetch.c
index 499f6b6..840be7b 100644
--- a/bin/varnishd/cache_esi_fetch.c
+++ b/bin/varnishd/cache_esi_fetch.c
@@ -303,24 +303,24 @@ vfp_esi_begin(struct sess *sp, size_t estimate)
 	/* XXX: snapshot WS's ? We'll need the space */
 
 	if (sp->wrk->is_gzip && sp->wrk->do_gunzip) {
-		sp->wrk->vgz_rx = VGZ_NewUngzip(sp, sp->ws);
+		sp->wrk->vgz_rx = VGZ_NewUngzip(sp);
 		VEP_Init(sp, NULL);
 	} else if (sp->wrk->is_gunzip && sp->wrk->do_gzip) {
 		vef = (void*)WS_Alloc(sp->ws, sizeof *vef);
 		AN(vef);
 		memset(vef, 0, sizeof *vef);
 		vef->magic = VEF_MAGIC;
-		vef->vgz = VGZ_NewGzip(sp, sp->ws);
+		vef->vgz = VGZ_NewGzip(sp);
 		AZ(sp->wrk->vef_priv);
 		sp->wrk->vef_priv = vef;
 		VEP_Init(sp, vfp_vep_callback);
 	} else if (sp->wrk->is_gzip) {
-		sp->wrk->vgz_rx = VGZ_NewUngzip(sp, sp->ws);
+		sp->wrk->vgz_rx = VGZ_NewUngzip(sp);
 		vef = (void*)WS_Alloc(sp->ws, sizeof *vef);
 		AN(vef);
 		memset(vef, 0, sizeof *vef);
 		vef->magic = VEF_MAGIC;
-		vef->vgz = VGZ_NewGzip(sp, sp->ws);
+		vef->vgz = VGZ_NewGzip(sp);
 		AZ(sp->wrk->vef_priv);
 		sp->wrk->vef_priv = vef;
 		VEP_Init(sp, vfp_vep_callback);
diff --git a/bin/varnishd/cache_gzip.c b/bin/varnishd/cache_gzip.c
index 4b62d24..4bdd37c 100644
--- a/bin/varnishd/cache_gzip.c
+++ b/bin/varnishd/cache_gzip.c
@@ -113,34 +113,48 @@ vgz_free(voidpf opaque, voidpf address)
  */
 
 static struct vgz *
-vgz_alloc_vgz(struct ws *ws)
+vgz_alloc_vgz(struct sess *sp)
 {
-	char *s;
 	struct vgz *vg;
+	struct ws *ws = sp->wrk->ws;
 
 	WS_Assert(ws);
-	s = WS_Snapshot(ws);
 	vg = (void*)WS_Alloc(ws, sizeof *vg);
 	AN(vg);
 	memset(vg, 0, sizeof *vg);
 	vg->magic = VGZ_MAGIC;
-	vg->tmp = ws;
-	vg->tmp_snapshot = s;
-
-	vg->vz.zalloc = vgz_alloc;
-	vg->vz.zfree = vgz_free;
-	vg->vz.opaque = vg;
 
+	switch (params->gzip_tmp_space) {
+	case 0:
+		/* malloc, the default */
+		break;
+	case 1:
+		vg->tmp = sp->ws;
+		vg->tmp_snapshot = WS_Snapshot(vg->tmp);
+		vg->vz.zalloc = vgz_alloc;
+		vg->vz.zfree = vgz_free;
+		vg->vz.opaque = vg;
+		break;
+	case 2:
+		vg->tmp = sp->wrk->ws;
+		vg->tmp_snapshot = WS_Snapshot(vg->tmp);
+		vg->vz.zalloc = vgz_alloc;
+		vg->vz.zfree = vgz_free;
+		vg->vz.opaque = vg;
+		break;
+	default:
+		assert(0 == __LINE__);
+	} 
 	return (vg);
 }
 
 struct vgz *
-VGZ_NewUngzip(const struct sess *sp, struct ws *tmp)
+VGZ_NewUngzip(struct sess *sp)
 {
 	struct vgz *vg;
 
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
-	vg = vgz_alloc_vgz(tmp);
+	vg = vgz_alloc_vgz(sp);
 
 	/*
 	 * Max memory usage according to zonf.h:
@@ -148,23 +162,18 @@ VGZ_NewUngzip(const struct sess *sp, struct ws *tmp)
 	 * Since we don't control windowBits, we have to assume
 	 * it is 15, so 34-35KB or so.
 	 */
-#if 1
-	vg->vz.zalloc = NULL;
-	vg->vz.zfree = NULL;
-	vg->vz.opaque = NULL;
-#endif
 	assert(Z_OK == inflateInit2(&vg->vz, 31));
 	return (vg);
 }
 
 struct vgz *
-VGZ_NewGzip(const struct sess *sp, struct ws *tmp)
+VGZ_NewGzip(struct sess *sp)
 {
 	struct vgz *vg;
 	int i;
 
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
-	vg = vgz_alloc_vgz(tmp);
+	vg = vgz_alloc_vgz(sp);
 
 	/*
 	 * From zconf.h:
@@ -181,13 +190,8 @@ VGZ_NewGzip(const struct sess *sp, struct ws *tmp)
 	 * XXX: It may be more efficent to malloc them, rather than have
 	 * XXX: too many worker threads grow the stacks.
 	 */
-#if 1
-	vg->vz.zalloc = NULL;
-	vg->vz.zfree = NULL;
-	vg->vz.opaque = NULL;
-#endif
 	i = deflateInit2(&vg->vz,
-	    0,				/* Level */
+	    params->gzip_level,		/* Level */
 	    Z_DEFLATED,			/* Method */
 	    16 + 8,			/* Window bits (16=gzip + 15) */
 	    1,				/* memLevel */
@@ -345,7 +349,8 @@ VGZ_Destroy(struct vgz **vg)
 {
 
 	CHECK_OBJ_NOTNULL(*vg, VGZ_MAGIC);
-	WS_Reset((*vg)->tmp, (*vg)->tmp_snapshot);
+	if ((*vg)->tmp != NULL) 
+		WS_Reset((*vg)->tmp, (*vg)->tmp_snapshot);
 	*vg = NULL;
 }
 
@@ -359,7 +364,7 @@ static void __match_proto__()
 vfp_gunzip_begin(struct sess *sp, size_t estimate)
 {
 	(void)estimate;
-	sp->wrk->vgz_rx = VGZ_NewUngzip(sp, sp->ws);
+	sp->wrk->vgz_rx = VGZ_NewUngzip(sp);
 }
 
 static int __match_proto__()
@@ -428,7 +433,7 @@ vfp_gzip_begin(struct sess *sp, size_t estimate)
 {
 	(void)estimate;
 
-	sp->wrk->vgz_rx = VGZ_NewGzip(sp, sp->ws);
+	sp->wrk->vgz_rx = VGZ_NewGzip(sp);
 }
 
 static int __match_proto__()
@@ -504,7 +509,7 @@ static void __match_proto__()
 vfp_testgzip_begin(struct sess *sp, size_t estimate)
 {
 	(void)estimate;
-	sp->wrk->vgz_rx = VGZ_NewUngzip(sp, sp->ws);
+	sp->wrk->vgz_rx = VGZ_NewUngzip(sp);
 }
 
 static int __match_proto__()
diff --git a/bin/varnishd/cache_response.c b/bin/varnishd/cache_response.c
index 7065e44..7d44eff 100644
--- a/bin/varnishd/cache_response.c
+++ b/bin/varnishd/cache_response.c
@@ -253,7 +253,7 @@ res_WriteGunzipObj(struct sess *sp)
 
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 
-	vg = VGZ_NewUngzip(sp, sp->wrk->ws);
+	vg = VGZ_NewUngzip(sp);
 
 	VTAILQ_FOREACH(st, &sp->obj->store, list) {
 		CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
diff --git a/bin/varnishd/heritage.h b/bin/varnishd/heritage.h
index 8eab742..9c5a328 100644
--- a/bin/varnishd/heritage.h
+++ b/bin/varnishd/heritage.h
@@ -201,6 +201,8 @@ struct params {
 
 	unsigned		http_gzip_support;
 	unsigned		gzip_stack_buffer;
+	unsigned		gzip_tmp_space;
+	unsigned		gzip_level;
 
 	double			critbit_cooloff;
 };
diff --git a/bin/varnishd/mgt_param.c b/bin/varnishd/mgt_param.c
index e2151ad..f46d3ca 100644
--- a/bin/varnishd/mgt_param.c
+++ b/bin/varnishd/mgt_param.c
@@ -817,9 +817,29 @@ static const struct parspec input_parspec[] = {
 		"Enable support for HTTP GZIP compression.\n",
 		EXPERIMENTAL,
 		"on", "bool" },
+	{ "gzip_tmp_space", tweak_uint, &master.gzip_tmp_space, 0, 2,
+		"Where temporary space for gzip/gunzip is allocated.\n"
+		"  0 - malloc\n"
+		"  1 - session workspace\n"
+		"  2 - thread workspace\n"
+		"If you have much gzip/gunzip activity, it may be an"
+		" advantage to use workspace for these allocations to reduce"
+		" malloc activity.  Be aware that gzip needs 256+KB and gunzip"
+		" needs 32+KB of workspace (64+KB if ESI processing).",
+		EXPERIMENTAL,
+		"0", "" },
+	{ "gzip_level", tweak_uint, &master.gzip_level, 0, 9,
+		"Gzip compression level: 0=debug, 1=fast, 9=best",
+		0,
+		"6", ""},
 	{ "gzip_stack_buffer", tweak_uint, &master.gzip_stack_buffer,
 	        2048, UINT_MAX,
-		"Size of stack buffer used for gzip processing.\n",
+		"Size of stack buffer used for gzip processing.\n"
+		"The stack buffers are used for in-transit data,"
+		" for instance gunzip'ed data being sent to a client."
+		"Making this space to small results in more overhead,"
+		" writes to sockets etc, making it too big is probably"
+		" just a waste of memory.",
 		EXPERIMENTAL,
 		"32768", "Bytes" },
 	{ "critbit_cooloff", tweak_timeout_double,
diff --git a/bin/varnishtest/tests/e00020.vtc b/bin/varnishtest/tests/e00020.vtc
index 3fe9239..b654cf1 100644
--- a/bin/varnishtest/tests/e00020.vtc
+++ b/bin/varnishtest/tests/e00020.vtc
@@ -24,6 +24,7 @@ varnish v1 -vcl+backend {
 
 varnish v1 -cliok "param.set esi_syntax 4"
 varnish v1 -cliok "param.set http_gzip_support true"
+varnish v1 -cliok "param.set gzip_tmp_space 2"
 
 client c1 {
 	txreq 
diff --git a/bin/varnishtest/tests/e00022.vtc b/bin/varnishtest/tests/e00022.vtc
index 5634797..871b2c0 100644
--- a/bin/varnishtest/tests/e00022.vtc
+++ b/bin/varnishtest/tests/e00022.vtc
@@ -27,6 +27,7 @@ varnish v1 -vcl+backend {
 
 varnish v1 -cliok "param.set esi_syntax 0xc"
 varnish v1 -cliok "param.set http_gzip_support true"
+varnish v1 -cliok "param.set gzip_tmp_space 1"
 
 client c1 {
 	txreq  -hdr "Accept-Encoding: gzip"



From phk at varnish-cache.org  Wed Jan 26 10:48:55 2011
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Wed, 26 Jan 2011 11:48:55 +0100
Subject: [master] 54ddaec Merge branch 'master' of
	ssh://git.varnish-cache.org/git/varnish-cache
Message-ID: <E1Pi2vn-00029n-DM@project.varnish-software.com>

commit 54ddaece7bee30b46ff12d00177c060edf21833b
Merge: 0197fd4 74f9b69
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Jan 26 10:48:51 2011 +0000

    Merge branch 'master' of ssh://git.varnish-cache.org/git/varnish-cache



From tfheen at varnish-cache.org  Wed Jan 26 11:36:08 2011
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Wed, 26 Jan 2011 12:36:08 +0100
Subject: [master] 07c4432 varnishtest: setuid(nobody) if running as root
Message-ID: <E1Pi3fU-0006AT-DL@project.varnish-software.com>

commit 07c4432ba55b50b6640c5eeb24155a3618224f61
Author: Tollef Fog Heen <tfheen at err.no>
Date:   Wed Jan 26 12:18:21 2011 +0100

    varnishtest: setuid(nobody) if running as root
    
    If we were running as root, m00004 would fail as it would not be able
    to read the files from the temporary directory.  To avoid this
    problem, we now setuid to nobody if we are running as root.

diff --git a/bin/varnishtest/vtc_main.c b/bin/varnishtest/vtc_main.c
index 3db3bd9..ac3436e 100644
--- a/bin/varnishtest/vtc_main.c
+++ b/bin/varnishtest/vtc_main.c
@@ -42,6 +42,11 @@ SVNID("$Id$")
 #include <sys/stat.h>
 #include <sys/types.h>
 #include <sys/wait.h>
+#include <pwd.h>
+
+#ifdef __linux__
+#include <sys/prctl.h>
+#endif
 
 #include "libvarnish.h"
 #include "vev.h"
@@ -364,6 +369,21 @@ main(int argc, char * const *argv)
 
 	vb = vev_new_base();
 
+	if (geteuid() == 0) {
+		struct passwd *pw;
+		pw = getpwnam("nobody");
+		assert(setgid(pw->pw_gid) == 0);
+		assert(setuid(pw->pw_uid) == 0);
+		/* On Linux >= 2.4, you need to set the dumpable flag
+		   to get core dumps after you have done a setuid. */
+#ifdef __linux__
+		if (prctl(PR_SET_DUMPABLE, 1) != 0) {
+		  printf("Could not set dumpable bit.  Core dumps turned "
+			 "off\n");
+		}
+#endif
+	}
+
 	i = 0;
 	while(!VTAILQ_EMPTY(&tst_head) || i) {
 		if (!VTAILQ_EMPTY(&tst_head) && njob < npar) {



From tfheen at varnish-cache.org  Wed Jan 26 11:36:08 2011
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Wed, 26 Jan 2011 12:36:08 +0100
Subject: [master] 78c5c42 Adjust LD_LIBRARY_PATH in spec file
Message-ID: <E1Pi3fU-0006Ad-Lv@project.varnish-software.com>

commit 78c5c42fdb45aaa415d5ddc2c8fbe1f3af9c24da
Author: Tollef Fog Heen <tfheen at err.no>
Date:   Wed Jan 26 12:20:18 2011 +0100

    Adjust LD_LIBRARY_PATH in spec file
    
    Add libvgz to LD_LIBRARY_PATH in spec file and drop running varnishd
    -C as that is done as part of the test suite.

diff --git a/redhat/varnish.spec b/redhat/varnish.spec
index 29200a9..0f3f778 100644
--- a/redhat/varnish.spec
+++ b/redhat/varnish.spec
@@ -166,8 +166,7 @@ mv doc/sphinx/\=build/html doc
 	%endif
 %endif
 
-LD_LIBRARY_PATH="lib/libvarnish/.libs:lib/libvarnishcompat/.libs:lib/libvarnishapi/.libs:lib/libvcl/.libs" bin/varnishd/varnishd -b 127.0.0.1:80 -C -n /tmp/foo
-%{__make} check LD_LIBRARY_PATH="../../lib/libvarnish/.libs:../../lib/libvarnishcompat/.libs:../../lib/libvarnishapi/.libs:../../lib/libvcl/.libs"
+%{__make} check LD_LIBRARY_PATH="../../lib/libvarnish/.libs:../../lib/libvarnishcompat/.libs:../../lib/libvarnishapi/.libs:../../lib/libvcl/.libs:../../lib/libvgz/.libs"
 
 %install
 rm -rf %{buildroot}



From tfheen at varnish-cache.org  Wed Jan 26 11:36:09 2011
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Wed, 26 Jan 2011 12:36:09 +0100
Subject: [master] f214deb Drop varnish-cache prefix from spec file
Message-ID: <E1Pi3fV-0006As-47@project.varnish-software.com>

commit f214debc25102bf3fc13c3a21761aac3b68660fb
Author: Tollef Fog Heen <tfheen at err.no>
Date:   Wed Jan 26 12:21:30 2011 +0100

    Drop varnish-cache prefix from spec file
    
    git doesn't have the varnish-cache directory prefix, so drop that from
    the spec file too.

diff --git a/redhat/varnish.spec b/redhat/varnish.spec
index 0f3f778..8f04637 100644
--- a/redhat/varnish.spec
+++ b/redhat/varnish.spec
@@ -71,8 +71,7 @@ Documentation files for %name
 #Varnish Cache is a high-performance HTTP accelerator
 
 %prep
-#%setup -q
-%setup -q -n varnish-cache
+%setup -q
 
 # The svn sources needs to generate a suitable configure script
 # Release tarballs would not need this



From perbu at varnish-cache.org  Wed Jan 26 11:53:03 2011
From: perbu at varnish-cache.org (Per Andreas Buer)
Date: Wed, 26 Jan 2011 12:53:03 +0100
Subject: [master] ce20081 add -F to synopsis
Message-ID: <E1Pi3vr-0007Le-F0@project.varnish-software.com>

commit ce20081957dfd9e10e417dea9cb2ccc49a7138bf
Author: Per Buer <perbu at varnish-software.com>
Date:   Wed Jan 26 12:18:25 2011 +0100

    add -F to synopsis

diff --git a/doc/sphinx/reference/varnishncsa.rst b/doc/sphinx/reference/varnishncsa.rst
index 15a8cee..42fb09a 100644
--- a/doc/sphinx/reference/varnishncsa.rst
+++ b/doc/sphinx/reference/varnishncsa.rst
@@ -15,9 +15,9 @@ Display Varnish logs in Apache / NCSA combined log format
 SYNOPSIS
 ========
 
-varnishncsa [-a] [-b] [-C] [-c] [-D] [-d] [-f] [-I regex] 
-[-i tag] [-n varnish_name] [-P file] [-r file] [-V] 
-[-w file] [-X regex] [-x tag]
+varnishncsa [-a] [-b] [-C] [-c] [-D] [-d] [-f] [-F format] [-I regex]
+[-i tag] [-n varnish_name] [-P file] [-r file] [-V] [-w file] 
+[-X regex] [-x tag]
 
 DESCRIPTION
 ===========



From perbu at varnish-cache.org  Wed Jan 26 11:53:03 2011
From: perbu at varnish-cache.org (Per Andreas Buer)
Date: Wed, 26 Jan 2011 12:53:03 +0100
Subject: [master] 6affa47 Add default format to -F explaination
Message-ID: <E1Pi3vr-0007Ll-Nk@project.varnish-software.com>

commit 6affa47bc4aa7008ea56e81699bf99ec7cea63ec
Author: Per Buer <perbu at varnish-software.com>
Date:   Wed Jan 26 12:29:42 2011 +0100

    Add default format to -F explaination

diff --git a/doc/sphinx/reference/varnishncsa.rst b/doc/sphinx/reference/varnishncsa.rst
index 42fb09a..13fc82a 100644
--- a/doc/sphinx/reference/varnishncsa.rst
+++ b/doc/sphinx/reference/varnishncsa.rst
@@ -48,7 +48,12 @@ The following options are available:
 -f          Prefer the X-Forwarded-For HTTP header over client.ip in 
 	    the log output.
 
--F format   Specify the log format used.  Supported formatters are:
+-F format   Specify the log format used. If no format is specified the  
+   	    default log format is used. Currently it is:
+
+            %h %l %u %t "%r" %>s %b "%{Referer}i" "%{User-agent}i"
+
+	    Supported formatters are:
 
 	      %b 
 	         Size of response in bytes, excluding HTTP headers.



From perbu at varnish-cache.org  Wed Jan 26 11:53:03 2011
From: perbu at varnish-cache.org (Per Andreas Buer)
Date: Wed, 26 Jan 2011 12:53:03 +0100
Subject: [master] bf1f446 doc http_gzip_support
Message-ID: <E1Pi3vr-0007Lz-Vb@project.varnish-software.com>

commit bf1f446ef8610ee89a451b5712c779e56b76c45a
Author: Per Buer <perbu at varnish-software.com>
Date:   Wed Jan 26 12:35:53 2011 +0100

    doc http_gzip_support

diff --git a/doc/sphinx/reference/varnishd.rst b/doc/sphinx/reference/varnishd.rst
index 283d2f7..2c183ac 100644
--- a/doc/sphinx/reference/varnishd.rst
+++ b/doc/sphinx/reference/varnishd.rst
@@ -484,6 +484,21 @@ http_headers
       This space is preallocated in sessions and workthreads only objects allocate only space for the headers
       they store.
 
+http_gzip_support
+      Default: on
+
+      Enable gzip support. When enabled Varnish will compress
+      uncompressed objects before they are stored in the cache. If a
+      client does not support gzip encoding Varnish will uncompress
+      compressed objects on demand.
+      Varnish will also rewrite the Accept-Encoding header of clients indicating support for gzip to:
+
+      Accept-Encoding: gzip
+
+      Clients that do not support gzip will have their Accept-Encoding
+      header removed. For more information no how gzip is implemted
+      please see the chapter on gzip in the Varnish reference.
+
 http_range
       Default: off
       



From perbu at varnish-cache.org  Wed Jan 26 11:53:04 2011
From: perbu at varnish-cache.org (Per Andreas Buer)
Date: Wed, 26 Jan 2011 12:53:04 +0100
Subject: [master] 55d6151 vcl string concat updated to reflect trunk
Message-ID: <E1Pi3vs-0007M8-Pq@project.varnish-software.com>

commit 55d61517c7e2b8c84b55ee33a117ecd2abbf887c
Author: Per Buer <perbu at varnish-software.com>
Date:   Wed Jan 26 12:36:48 2011 +0100

    vcl string concat updated to reflect trunk

diff --git a/doc/sphinx/reference/vcl.rst b/doc/sphinx/reference/vcl.rst
index e8da875..6a93ccf 100644
--- a/doc/sphinx/reference/vcl.rst
+++ b/doc/sphinx/reference/vcl.rst
@@ -44,7 +44,7 @@ in strings in VCL, so it can be freely used in regular expressions
 without doubling.
 
 Strings are concatenated by putting them one after each other
-without a '+' operator between.
+with a '+' operator between.
 
 Assignments are introduced with the *set* keyword.  There are no
 user-defined variables; values can only be assigned to variables



From perbu at varnish-cache.org  Wed Jan 26 11:53:05 2011
From: perbu at varnish-cache.org (Per Andreas Buer)
Date: Wed, 26 Jan 2011 12:53:05 +0100
Subject: [master] 3d660ea gzip level
Message-ID: <E1Pi3vt-0007MN-5t@project.varnish-software.com>

commit 3d660ead4173552377e389fa717f5486c9a96820
Author: Per Buer <perbu at varnish-software.com>
Date:   Wed Jan 26 12:41:05 2011 +0100

    gzip level

diff --git a/doc/sphinx/reference/varnishd.rst b/doc/sphinx/reference/varnishd.rst
index 2c183ac..fb8c61c 100644
--- a/doc/sphinx/reference/varnishd.rst
+++ b/doc/sphinx/reference/varnishd.rst
@@ -476,6 +476,12 @@ group
 
       The unprivileged group to run as.
 
+gzip_level
+      Default: 6
+
+      Gzip compression level ranging from 1 (the fastest) to 9 (the
+      smallest possible output). 0 indicates "debug mode".
+
 http_headers
       Units: header lines
       Default: 64



From perbu at varnish-cache.org  Wed Jan 26 11:53:05 2011
From: perbu at varnish-cache.org (Per Andreas Buer)
Date: Wed, 26 Jan 2011 12:53:05 +0100
Subject: [master] 40e2168 gzip tmp space
Message-ID: <E1Pi3vt-0007MX-O7@project.varnish-software.com>

commit 40e216857418a420e6b8f330d5268957473d6509
Author: Per Buer <perbu at varnish-software.com>
Date:   Wed Jan 26 12:46:33 2011 +0100

    gzip tmp space

diff --git a/doc/sphinx/reference/varnishd.rst b/doc/sphinx/reference/varnishd.rst
index fb8c61c..f2e0ef4 100644
--- a/doc/sphinx/reference/varnishd.rst
+++ b/doc/sphinx/reference/varnishd.rst
@@ -482,6 +482,26 @@ gzip_level
       Gzip compression level ranging from 1 (the fastest) to 9 (the
       smallest possible output). 0 indicates "debug mode".
 
+gzip_stack_buffer
+      Unit: kilobytes
+      Default: 2048
+
+      Size of stack buffer used for in transit gzip processing, like
+      on the fly decompression.
+
+gzip_tmp_space
+      Default: 0
+
+      Where temporary space for gzip/gunzip is allocated.
+        0 - malloc
+        1 - session workspace
+        2 - thread workspace
+      
+      If you have much gzip/gunzip activity, it may be an advantage to
+      use workspace for these allocations to reduce malloc activity.
+      Be aware that gzip needs 256+KB and gunzip" needs 32+KB of
+      workspace (64+KB if ESI processing).
+
 http_headers
       Units: header lines
       Default: 64



From perbu at varnish-cache.org  Wed Jan 26 11:53:06 2011
From: perbu at varnish-cache.org (Per Andreas Buer)
Date: Wed, 26 Jan 2011 12:53:06 +0100
Subject: [master] a95a36f Merge branch 'master' of
	ssh://git.varnish-cache.org/git/varnish-cache
Message-ID: <E1Pi3vu-0007Mn-Ok@project.varnish-software.com>

commit a95a36f77713c8c91bef491f9136b4fe57ae5500
Merge: 40e2168 f214deb
Author: Per Buer <perbu at varnish-software.com>
Date:   Wed Jan 26 12:51:43 2011 +0100

    Merge branch 'master' of ssh://git.varnish-cache.org/git/varnish-cache
    
    Conflicts:
    	doc/sphinx/reference/varnishncsa.rst

diff --cc doc/sphinx/reference/varnishncsa.rst
index 13fc82a,9c7a012..0500d96
--- a/doc/sphinx/reference/varnishncsa.rst
+++ b/doc/sphinx/reference/varnishncsa.rst
@@@ -15,10 -15,10 +15,11 @@@ Display Varnish logs in Apache / NCSA c
  SYNOPSIS
  ========
  
 -varnishncsa [-a] [-b] [-C] [-c] [-D] [-d] [-f] [-I regex] 
 -[-i tag] [-n varnish_name] [-o tag regex] [-P file] [-r file] [-V] 
 -[-w file] [-X regex] [-x tag]
 +varnishncsa [-a] [-b] [-C] [-c] [-D] [-d] [-f] [-F format] [-I regex]
 +[-i tag] [-n varnish_name] [-P file] [-r file] [-V] [-w file] 
 +[-X regex] [-x tag]
 +
+ 
  DESCRIPTION
  ===========
  



From phk at varnish-cache.org  Wed Jan 26 20:06:40 2011
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Wed, 26 Jan 2011 21:06:40 +0100
Subject: [master] cb016fe White-space cleanup
Message-ID: <E1PiBdY-0001Ug-Hr@project.varnish-software.com>

commit cb016fe02841721b9fe37f5d48b66fb7e413fba9
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Jan 26 20:06:28 2011 +0000

    White-space cleanup

diff --git a/bin/varnishd/cache_center.c b/bin/varnishd/cache_center.c
index 31adc80..1085fa0 100644
--- a/bin/varnishd/cache_center.c
+++ b/bin/varnishd/cache_center.c
@@ -590,12 +590,12 @@ cnt_fetch(struct sess *sp)
 	 * The VCL variables beresp.do_g[un]zip tells us how we want the
 	 * object stored.
 	 *
- 	 * The backend Content-Encoding header tells us what we are going
+	 * The backend Content-Encoding header tells us what we are going
 	 * to receive, which we classify in the following three classes:
 	 *
-	 * 	"Content-Encoding: gzip"	--> object is gzip'ed.
-	 * 	no Content-Encoding		--> object is not gzip'ed.
-	 * 	anything else			--> do nothing wrt gzip
+	 *	"Content-Encoding: gzip"	--> object is gzip'ed.
+	 *	no Content-Encoding		--> object is not gzip'ed.
+	 *	anything else			--> do nothing wrt gzip
 	 *
 	 */
 
@@ -603,10 +603,10 @@ cnt_fetch(struct sess *sp)
 	if (!params->http_gzip_support)
 		sp->wrk->do_gzip = sp->wrk->do_gunzip = 0;
 
-	sp->wrk->is_gzip = 
+	sp->wrk->is_gzip =
 	    http_HdrIs(sp->wrk->beresp, H_Content_Encoding, "gzip");
 
-	sp->wrk->is_gunzip = 
+	sp->wrk->is_gunzip =
 	    !http_GetHdr(sp->wrk->beresp, H_Content_Encoding, NULL);
 
 	/* It can't be both */
@@ -625,7 +625,7 @@ cnt_fetch(struct sess *sp)
 		sp->wrk->do_gzip = 0;
 
 	/* If we do gzip, add the C-E header */
-	if (sp->wrk->do_gzip) 
+	if (sp->wrk->do_gzip)
 		http_PrintfHeader(sp->wrk, sp->fd, sp->wrk->beresp,
 		    "Content-Encoding: %s", "gzip");
 
@@ -649,7 +649,7 @@ cnt_fetch(struct sess *sp)
 		l += varyl;
 
 	/*
-	 * Space for producing a Content-Length: header including padding 
+	 * Space for producing a Content-Length: header including padding
 	 * A billion gigabytes is enough for anybody.
 	 */
 	l += strlen("Content-Encoding: XxxXxxXxxXxxXxxXxx" + sizeof(void *));
diff --git a/bin/varnishd/cache_esi_deliver.c b/bin/varnishd/cache_esi_deliver.c
index 589feeb..5a5b1a6 100644
--- a/bin/varnishd/cache_esi_deliver.c
+++ b/bin/varnishd/cache_esi_deliver.c
@@ -142,7 +142,7 @@ ved_sendchunk(const struct sess *sp, const void *cb, ssize_t cl,
 	(void)WRW_Write(sp->wrk, ptr, l);
 	if (sp->wrk->res_mode & RES_CHUNKED) {
 		(void)WRW_Write(sp->wrk, "\r\n", -1);
-		if (cb == NULL) 
+		if (cb == NULL)
 			(void)WRW_Flush(sp->wrk);
 	}
 }
@@ -215,7 +215,7 @@ ved_pretend_gzip(const struct sess *sp, const uint8_t *p, ssize_t l)
 		}
 		(void)WRW_Write(sp->wrk, buf, sizeof buf);
 		(void)WRW_Write(sp->wrk, p, lx);
-		if (sp->wrk->res_mode & RES_CHUNKED) 
+		if (sp->wrk->res_mode & RES_CHUNKED)
 			(void)WRW_Write(sp->wrk, "\r\n", -1);
 		(void)WRW_Flush(sp->wrk);
 		sp->wrk->crc = crc32(sp->wrk->crc, p, lx);
@@ -269,7 +269,7 @@ ESI_Deliver(struct sess *sp)
 		if (isgzip && !(sp->wrk->res_mode & RES_GUNZIP)) {
 			sp->wrk->gzip_resp = 1;
 			sp->wrk->crc = crc32(0L, Z_NULL, 0);
-		} else 
+		} else
 			sp->wrk->gzip_resp = 0;
 	}
 
@@ -280,7 +280,7 @@ ESI_Deliver(struct sess *sp)
 
 	st = VTAILQ_FIRST(&sp->obj->store);
 	off = 0;
-	
+
 	while (p < e) {
 		switch (*p) {
 		case VEC_V1:
@@ -375,7 +375,7 @@ ESI_Deliver(struct sess *sp)
 		}
 	}
 	if (vgz != NULL) {
-		if (obufl > 0) 
+		if (obufl > 0)
 			ved_sendchunk(sp, NULL, 0, obuf, obufl);
 		VGZ_Destroy(&vgz);
 	}
@@ -428,7 +428,7 @@ ved_deliver_byterange(const struct sess *sp, ssize_t low, ssize_t high)
 			lx = low;
 		}
 //printf("[1-] %jd %jd\n", lx, lx + l);
-		if (lx + l >= high) 
+		if (lx + l >= high)
 			l = high - lx;
 //printf("[2-] %jd %jd\n", lx, lx + l);
 		assert(lx >= low && lx + l <= high);
diff --git a/bin/varnishd/cache_esi_fetch.c b/bin/varnishd/cache_esi_fetch.c
index 840be7b..533fc72 100644
--- a/bin/varnishd/cache_esi_fetch.c
+++ b/bin/varnishd/cache_esi_fetch.c
@@ -265,7 +265,7 @@ vfp_esi_bytes_gg(struct sess *sp, struct http_conn *htc, size_t bytes)
 		if (w <= 0)
 			return (w);
 		bytes -= w;
-		
+
 		vef->bufp = ibuf;
 		VGZ_Ibuf(sp->wrk->vgz_rx, ibuf, w);
 		do {
diff --git a/bin/varnishd/cache_esi_parse.c b/bin/varnishd/cache_esi_parse.c
index 6cc48fa..8a84311 100644
--- a/bin/varnishd/cache_esi_parse.c
+++ b/bin/varnishd/cache_esi_parse.c
@@ -122,13 +122,13 @@ const char		*hack_p;
 /*---------------------------------------------------------------------*/
 
 static const char * const VEP_START =		"[Start]";
-static const char * const VEP_TESTXML = 	"[TestXml]";
-static const char * const VEP_NOTXML =	 	"[NotXml]";
+static const char * const VEP_TESTXML =		"[TestXml]";
+static const char * const VEP_NOTXML =		"[NotXml]";
 
-static const char * const VEP_NEXTTAG = 	"[NxtTag]";
+static const char * const VEP_NEXTTAG =		"[NxtTag]";
 static const char * const VEP_NOTMYTAG =	"[NotMyTag]";
 
-static const char * const VEP_STARTTAG = 	"[StartTag]";
+static const char * const VEP_STARTTAG =	"[StartTag]";
 static const char * const VEP_COMMENT =		"[Comment]";
 static const char * const VEP_CDATA =		"[CDATA]";
 static const char * const VEP_ESITAG =		"[ESITag]";
@@ -148,7 +148,7 @@ static const char * const VEP_ATTRGETVAL =	"[AttrGetValue]";
 static const char * const VEP_ATTRVAL =		"[AttrValue]";
 
 static const char * const VEP_UNTIL =		"[Until]";
-static const char * const VEP_MATCHBUF = 	"[MatchBuf]";
+static const char * const VEP_MATCHBUF =	"[MatchBuf]";
 static const char * const VEP_MATCH =		"[Match]";
 
 /*---------------------------------------------------------------------*/
@@ -189,7 +189,7 @@ vep_error(const struct vep_state *vep, const char *p)
 	l = (intmax_t)(vep->ver_p - vep->hack_p);
 	printf("ERROR at %jd %s\n", l , p);
 	WSP(vep->sp, SLT_ESI_xmlerror, "ERR at %jd %s", l, p);
-	
+
 }
 
 /*--------------------------------------------------------------------
@@ -205,7 +205,7 @@ vep_warn(const struct vep_state *vep, const char *p)
 	l = (intmax_t)(vep->ver_p - vep->hack_p);
 	printf("WARNING at %jd %s\n", l, p);
 	WSP(vep->sp, SLT_ESI_xmlerror, "WARN at %jd %s", l, p);
-	
+
 }
 
 /*---------------------------------------------------------------------
@@ -265,7 +265,7 @@ vep_emit_len(const struct vep_state *vep, ssize_t l, int m8, int m16, int m64)
 		assert((ssize_t)vbe64dec(buf + 1) == l);
 		vsb_bcat(vep->vsb, buf, 9);
 	}
-} 
+}
 
 static void
 vep_emit_skip(const struct vep_state *vep, ssize_t l)
@@ -275,7 +275,7 @@ vep_emit_skip(const struct vep_state *vep, ssize_t l)
 		Debug("---> SKIP(%jd)\n", (intmax_t)l);
 	}
 	vep_emit_len(vep, l, VEC_S1, VEC_S2, VEC_S8);
-} 
+}
 
 static void
 vep_emit_verbatim(const struct vep_state *vep, ssize_t l, ssize_t l_crc)
@@ -293,7 +293,7 @@ vep_emit_verbatim(const struct vep_state *vep, ssize_t l, ssize_t l_crc)
 		vbe32enc(buf, vep->crc);
 		vsb_bcat(vep->vsb, buf, sizeof buf);
 	}
-} 
+}
 
 static void
 vep_emit_common(struct vep_state *vep, ssize_t l, enum vep_mark mark)
@@ -374,7 +374,7 @@ vep_mark_verbatim(struct vep_state *vep, const char *p)
 
 	vep_mark_common(vep, p, VERBATIM);
 	vep->nm_verbatim++;
-} 
+}
 
 static void
 vep_mark_skip(struct vep_state *vep, const char *p)
@@ -382,7 +382,7 @@ vep_mark_skip(struct vep_state *vep, const char *p)
 
 	vep_mark_common(vep, p, SKIP);
 	vep->nm_skip++;
-} 
+}
 
 static void
 vep_mark_pending(struct vep_state *vep, const char *p)
@@ -408,7 +408,7 @@ vep_do_comment(struct vep_state *vep, enum dowhat what)
 {
 	Debug("DO_COMMENT(%d)\n", what);
 	assert(what == DO_TAG);
-	if (!vep->emptytag) 
+	if (!vep->emptytag)
 		vep_error(vep, "ESI 1.0 <esi:comment> needs final '/'");
 }
 
@@ -431,7 +431,7 @@ vep_do_remove(struct vep_state *vep, enum dowhat what)
 		else if (!vep->remove && vep->endtag)
 			vep_error(vep,
 			    "ESI 1.0 <esi:remove> not open");
-		else 
+		else
 			vep->remove = !vep->endtag;
 	}
 }
@@ -455,7 +455,7 @@ vep_do_include(struct vep_state *vep, enum dowhat what)
 		return;
 	}
 	assert(what == DO_TAG);
-	if (!vep->emptytag) 
+	if (!vep->emptytag)
 		vep_warn(vep,
 		    "ESI 1.0 <esi:include> lacks final '/'");
 	if (vep->include_src == NULL) {
@@ -499,7 +499,7 @@ vep_do_include(struct vep_state *vep, enum dowhat what)
 				h = q;
 		if (h == NULL)
 			h = q + 1;
-			
+
 		Debug("INCL:: [%.*s]/[%s]\n",
 		    (int)(h - url.b), url.b, p);
 		vsb_printf(vep->vsb, "%.*s/", (int)(h - url.b), url.b);
@@ -533,7 +533,7 @@ vep_do_include(struct vep_state *vep, enum dowhat what)
  *
  * This function is called with the input object piecemal so do not
  * assume that we have more than one char available at at time, but
- * optimize for getting huge chunks. 
+ * optimize for getting huge chunks.
  *
  * NB: At the bottom of this source-file, there is a dot-diagram matching
  * NB: the state-machine.  Please maintain it along with the code.
@@ -586,7 +586,7 @@ VEP_parse(const struct sess *sp, const char *p, size_t l)
 			 * If the first non-whitespace char is different
 			 * from '<' we assume this is not XML.
 			 */
-			while (p < e && vct_islws(*p)) 
+			while (p < e && vct_islws(*p))
 				p++;
 			vep_mark_verbatim(vep, p);
 			if (p < e && *p == '<') {
@@ -679,7 +679,7 @@ VEP_parse(const struct sess *sp, const char *p, size_t l)
 				if (*p == '/') {
 					vep->endtag = 1;
 					p++;
-				} 
+				}
 				vep->match = vep_match_starttag;
 				vep->state = VEP_MATCH;
 			}
@@ -692,7 +692,7 @@ VEP_parse(const struct sess *sp, const char *p, size_t l)
 				vep->esicmt_p = vep->esicmt = "esi";
 			while (p < e) {
 				if (*p != *vep->esicmt_p) {
-					vep->esicmt_p = vep->esicmt = NULL; 
+					vep->esicmt_p = vep->esicmt = NULL;
 					vep->until_p = vep->until = "-->";
 					vep->until_s = VEP_NEXTTAG;
 					vep->state = VEP_UNTIL;
@@ -766,7 +766,7 @@ VEP_parse(const struct sess *sp, const char *p, size_t l)
 		} else if (vep->state == VEP_INTAG) {
 			vep->tag_i = 0;
 			while (p < e && vct_islws(*p) && !vep->emptytag) {
-				p++;	
+				p++;
 				vep->canattr = 1;
 			}
 			if (p < e && *p == '/' && !vep->emptytag) {
@@ -850,7 +850,7 @@ VEP_parse(const struct sess *sp, const char *p, size_t l)
 				    "XML 1.0 Illegal attribute delimiter");
 				vep->state = VEP_TAGERROR;
 			}
-			
+
 		} else if (vep->state == VEP_ATTRVAL) {
 			while (p < e && *p != '>' && *p != vep->attr_delim &&
 			   (vep->attr_delim != ' ' || !vct_issp(*p))) {
@@ -879,7 +879,6 @@ VEP_parse(const struct sess *sp, const char *p, size_t l)
 				}
 				vep->state = VEP_INTAG;
 			}
-	
 
 		/******************************************************
 		 * Utility Section
@@ -955,7 +954,7 @@ VEP_parse(const struct sess *sp, const char *p, size_t l)
 	 * in the most efficient way, in particular with respect to
 	 * minimizing and limiting use of pending.
 	 */
-	if (p == vep->ver_p) 
+	if (p == vep->ver_p)
 		;
 	else if (vep->in_esi_tag)
 		vep_mark_skip(vep, p);
diff --git a/bin/varnishd/cache_fetch.c b/bin/varnishd/cache_fetch.c
index 35fed5f..6b123f2 100644
--- a/bin/varnishd/cache_fetch.c
+++ b/bin/varnishd/cache_fetch.c
@@ -91,7 +91,7 @@ vfp_nop_bytes(struct sess *sp, struct http_conn *htc, ssize_t bytes)
 	struct storage *st;
 
 	while (bytes > 0) {
-		if (FetchStorage(sp)) 
+		if (FetchStorage(sp))
 			return (-1);
 		st = sp->wrk->storage;
 		l = st->space - st->len;
diff --git a/bin/varnishd/cache_gzip.c b/bin/varnishd/cache_gzip.c
index 4bdd37c..f49e7bd 100644
--- a/bin/varnishd/cache_gzip.c
+++ b/bin/varnishd/cache_gzip.c
@@ -38,7 +38,7 @@
  * will be the following, so we need to be a bit careful with the scratch
  * space we use:
  *
- * 	Backend		Tmp	Input	Output
+ *	Backend		Tmp	Input	Output
  *         |		----------------------
  *	   v
  *	 gunzip		wrk	stack	?
@@ -144,7 +144,7 @@ vgz_alloc_vgz(struct sess *sp)
 		break;
 	default:
 		assert(0 == __LINE__);
-	} 
+	}
 	return (vg);
 }
 
@@ -158,7 +158,7 @@ VGZ_NewUngzip(struct sess *sp)
 
 	/*
 	 * Max memory usage according to zonf.h:
-	 * 	mem_needed = "a few kb" + (1 << (windowBits))
+	 *	mem_needed = "a few kb" + (1 << (windowBits))
 	 * Since we don't control windowBits, we have to assume
 	 * it is 15, so 34-35KB or so.
 	 */
@@ -178,7 +178,7 @@ VGZ_NewGzip(struct sess *sp)
 	/*
 	 * From zconf.h:
 	 *
-	 * 	mem_needed = "a few kb" 
+	 *	mem_needed = "a few kb"
 	 *		+ (1 << (windowBits+2))
 	 *		+  (1 << (memLevel+9))
 	 *
@@ -245,7 +245,7 @@ VGZ_ObufStorage(const struct sess *sp, struct vgz *vg)
 {
 	struct storage *st;
 
-	if (FetchStorage(sp)) 
+	if (FetchStorage(sp))
 		return (-1);
 
 	st = sp->wrk->storage;
@@ -349,7 +349,7 @@ VGZ_Destroy(struct vgz **vg)
 {
 
 	CHECK_OBJ_NOTNULL(*vg, VGZ_MAGIC);
-	if ((*vg)->tmp != NULL) 
+	if ((*vg)->tmp != NULL)
 		WS_Reset((*vg)->tmp, (*vg)->tmp_snapshot);
 	*vg = NULL;
 }
diff --git a/bin/varnishd/cache_response.c b/bin/varnishd/cache_response.c
index 7d44eff..18c2612 100644
--- a/bin/varnishd/cache_response.c
+++ b/bin/varnishd/cache_response.c
@@ -273,7 +273,7 @@ res_WriteGunzipObj(struct sess *sp)
 					(void)WRW_Write(sp->wrk, lenbuf, -1);
 				}
 				(void)WRW_Write(sp->wrk, dp, dl);
-				if (sp->wrk->res_mode & RES_CHUNKED) 
+				if (sp->wrk->res_mode & RES_CHUNKED)
 					(void)WRW_Write(sp->wrk, "\r\n", -1);
 				if (WRW_Flush(sp->wrk))
 					break;
@@ -344,7 +344,7 @@ res_WriteDirObj(struct sess *sp, char *lenbuf, size_t low, size_t high)
 		(void)WRW_Write(sp->wrk, st->ptr + off, len);
 	}
 	assert(u == sp->obj->len);
-	if (sp->wrk->res_mode & RES_CHUNKED) 
+	if (sp->wrk->res_mode & RES_CHUNKED)
 		(void)WRW_Write(sp->wrk, "\r\n", -1);
 }
 
@@ -393,7 +393,7 @@ RES_WriteObj(struct sess *sp)
 	} else if (sp->wrk->res_mode & RES_ESI_CHILD && sp->wrk->gzip_resp) {
 		ESI_DeliverChild(sp);
 	} else if (sp->wrk->res_mode & RES_GUNZIP) {
-		res_WriteGunzipObj(sp); 
+		res_WriteGunzipObj(sp);
 	} else {
 		res_WriteDirObj(sp, lenbuf, low, high);
 	}
diff --git a/bin/varnishtest/vtc_http.c b/bin/varnishtest/vtc_http.c
index 52eb287..c9eb09a 100644
--- a/bin/varnishtest/vtc_http.c
+++ b/bin/varnishtest/vtc_http.c
@@ -596,7 +596,7 @@ gzip_body(struct http *hp, const char *txt, char **body, int *bodylen)
 	    hp->gziplevel, Z_DEFLATED, 31, 9, Z_DEFAULT_STRATEGY));
 	assert(Z_STREAM_END == deflate(&vz, Z_FINISH));
 	i = vz.stop_bit & 7;
-	if (hp->gzipresidual >= 0 && hp->gzipresidual != i) 
+	if (hp->gzipresidual >= 0 && hp->gzipresidual != i)
 		vtc_log(hp->vl, 0, "Wrong gzip residual got %d wanted %d",
 		    i, hp->gzipresidual);
 	*bodylen = vz.total_out;
diff --git a/lib/libvgz/inflate.c b/lib/libvgz/inflate.c
index 66b9cd5..0abb681 100644
--- a/lib/libvgz/inflate.c
+++ b/lib/libvgz/inflate.c
@@ -811,7 +811,7 @@ int flush;
         case TYPE:
             if (flush == Z_BLOCK || flush == Z_TREES) goto inf_leave;
         case TYPEDO:
-            if (strm->start_bit == 0) 
+            if (strm->start_bit == 0)
                 strm->start_bit = 8 * (strm->total_in + in - have) - bits;
             if (state->last) {
                 strm->stop_bit = 8 * (strm->total_in + in - have) - bits;



From perbu at varnish-cache.org  Wed Jan 26 20:07:00 2011
From: perbu at varnish-cache.org (Per Andreas Buer)
Date: Wed, 26 Jan 2011 21:07:00 +0100
Subject: [master] 11fd9bf try to explain beresp gzip stuff
Message-ID: <E1PiBds-0001cg-5v@project.varnish-software.com>

commit 11fd9bf3272eafeeed98ca5f7592a5f8eef7b871
Author: Per Buer <perbu at fiskepinne.(none)>
Date:   Wed Jan 26 18:40:41 2011 +0100

    try to explain beresp gzip stuff

diff --git a/doc/sphinx/reference/vcl.rst b/doc/sphinx/reference/vcl.rst
index 6a93ccf..1e74a8b 100644
--- a/doc/sphinx/reference/vcl.rst
+++ b/doc/sphinx/reference/vcl.rst
@@ -470,9 +470,6 @@ vcl_fetch
   error code [reason]
     Return the specified error code to the client and abandon the request.
 
-  esi
-     ESI-process the document which has just been fetched.
-
   pass
     Switch to pass mode.  Control will eventually pass to vcl_pass.
 
@@ -646,6 +643,22 @@ The following variables are available after the requested object has
 been retrieved from the backend, before it is entered into the cache. In
 other words, they are available in vcl_fetch:
 
+beresp.do_esi
+  ESI-process the object after fetching it. Defaults to 0. Set it to 1 to 
+  activate ESI.
+
+beresp.do_gzip
+  Gzip the object before storing it. Defaults to 1, 
+
+beresp.is_gzip
+  True if the object is compressed.
+
+beresp.do_gunzip
+  Unzip the object before storing it. 
+
+beresp.is_gunzip 
+  True if the object is not compressed.
+
 beresp.proto
   The HTTP protocol version used when the object was retrieved.
 



From perbu at varnish-cache.org  Wed Jan 26 20:07:01 2011
From: perbu at varnish-cache.org (Per Andreas Buer)
Date: Wed, 26 Jan 2011 21:07:01 +0100
Subject: [master] 7072538 PHK did some QA on the VCL syntax
Message-ID: <E1PiBdt-0001co-9T@project.varnish-software.com>

commit 7072538253db2715b1aa635204e1d9f8810b8989
Author: Per Buer <perbu at fiskepinne.(none)>
Date:   Wed Jan 26 21:01:14 2011 +0100

    PHK did some QA on the VCL syntax

diff --git a/doc/sphinx/reference/vcl.rst b/doc/sphinx/reference/vcl.rst
index 1e74a8b..2eb7023 100644
--- a/doc/sphinx/reference/vcl.rst
+++ b/doc/sphinx/reference/vcl.rst
@@ -644,23 +644,18 @@ been retrieved from the backend, before it is entered into the cache. In
 other words, they are available in vcl_fetch:
 
 beresp.do_esi
-  ESI-process the object after fetching it. Defaults to 0. Set it to 1 to 
-  activate ESI.
+  Boolean. ESI-process the object after fetching it. Defaults to false. Set it
+  to true to parse the object for ESI directives.
 
 beresp.do_gzip
-  Gzip the object before storing it. Defaults to 1, 
-
-beresp.is_gzip
-  True if the object is compressed.
+  Boolean. Gzip the object before storing it. Defaults to true. 
 
 beresp.do_gunzip
-  Unzip the object before storing it. 
-
-beresp.is_gunzip 
-  True if the object is not compressed.
+  Boolean. Unzip the object before storing it in the cache.  Defaults
+  to false.
 
 beresp.proto
-  The HTTP protocol version used when the object was retrieved.
+  The HTTP protocol version used the backend replied with.
 
 beresp.status
   The HTTP status code returned by the server.



From perbu at varnish-cache.org  Wed Jan 26 20:07:02 2011
From: perbu at varnish-cache.org (Per Andreas Buer)
Date: Wed, 26 Jan 2011 21:07:02 +0100
Subject: [master] b3745fc req.can_gzip
Message-ID: <E1PiBdu-0001d5-Mj@project.varnish-software.com>

commit b3745fc4ed669c13abd61c03a8ab67c9a69e1554
Author: Per Buer <perbu at varnish-software.com>
Date:   Wed Jan 26 21:06:50 2011 +0100

    req.can_gzip

diff --git a/doc/sphinx/reference/vcl.rst b/doc/sphinx/reference/vcl.rst
index 2eb7023..82efc96 100644
--- a/doc/sphinx/reference/vcl.rst
+++ b/doc/sphinx/reference/vcl.rst
@@ -613,6 +613,9 @@ req.hash_ignore_busy
   this if you have two server looking up content from each other to 
   avoid potential deadlocks.
 
+req.can_gzip
+  Does the client accept the gzip transfer encoding.
+
 The following variables are available while preparing a backend
 request (either for a cache miss or for pass or pipe mode):
 



From perbu at varnish-cache.org  Wed Jan 26 20:07:14 2011
From: perbu at varnish-cache.org (Per Andreas Buer)
Date: Wed, 26 Jan 2011 21:07:14 +0100
Subject: [master] d877e0e Merge branch 'master' of
	ssh://git.varnish-cache.org/git/varnish-cache
Message-ID: <E1PiBe6-0001e0-0t@project.varnish-software.com>

commit d877e0e0920fa4544f60f9ad4d6adbaa09821fab
Merge: b3745fc cb016fe
Author: Per Buer <perbu at varnish-software.com>
Date:   Wed Jan 26 21:07:04 2011 +0100

    Merge branch 'master' of ssh://git.varnish-cache.org/git/varnish-cache



From phk at varnish-cache.org  Wed Jan 26 21:31:48 2011
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Wed, 26 Jan 2011 22:31:48 +0100
Subject: [master] b965780 Hang the currently-being-filled storage segment off
	the object also.
Message-ID: <E1PiCxw-0000rZ-OC@project.varnish-software.com>

commit b965780e267b24e841809162eb180d706f4613f2
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Jan 26 21:31:24 2011 +0000

    Hang the currently-being-filled storage segment off the object also.

diff --git a/bin/varnishd/cache.h b/bin/varnishd/cache.h
index ead98c1..d2de03e 100644
--- a/bin/varnishd/cache.h
+++ b/bin/varnishd/cache.h
@@ -417,7 +417,7 @@ struct lru {
 };
 
 /* Object structure --------------------------------------------------*/
-VTAILQ_HEAD(objecthead, storage);
+VTAILQ_HEAD(storagehead, storage);
 
 struct object {
 	unsigned		magic;
@@ -452,7 +452,7 @@ struct object {
 
 	struct http		*http;
 
-	struct objecthead	store;
+	struct storagehead	store;
 
 	struct storage		*esidata;
 
@@ -630,7 +630,7 @@ void EXP_Touch(struct object *o, double tnow);
 int EXP_NukeOne(const struct sess *sp, const struct lru *lru);
 
 /* cache_fetch.c */
-int FetchStorage(const struct sess *sp);
+int FetchStorage(const struct sess *sp, ssize_t sz);
 int FetchHdr(struct sess *sp);
 int FetchBody(struct sess *sp);
 int FetchReqBody(struct sess *sp);
diff --git a/bin/varnishd/cache_esi_deliver.c b/bin/varnishd/cache_esi_deliver.c
index 5a5b1a6..9c5b0cf 100644
--- a/bin/varnishd/cache_esi_deliver.c
+++ b/bin/varnishd/cache_esi_deliver.c
@@ -528,7 +528,7 @@ ESI_DeliverChild(const struct sess *sp)
 	}
 	if (lpad > 0)
 		ved_sendchunk(sp, NULL, 0, pad, lpad);
-	st = VTAILQ_LAST(&sp->obj->store, objecthead);
+	st = VTAILQ_LAST(&sp->obj->store, storagehead);
 	assert(st->len > 8);
 
 	p = st->ptr + st->len - 8;
diff --git a/bin/varnishd/cache_esi_fetch.c b/bin/varnishd/cache_esi_fetch.c
index 533fc72..24aefeb 100644
--- a/bin/varnishd/cache_esi_fetch.c
+++ b/bin/varnishd/cache_esi_fetch.c
@@ -77,7 +77,7 @@ vfp_esi_bytes_uu(struct sess *sp, struct http_conn *htc, ssize_t bytes)
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 
 	while (bytes > 0) {
-		if (FetchStorage(sp))
+		if (FetchStorage(sp, 0))
 			return (-1);
 		st = sp->wrk->storage;
 		w = vef_read(htc,
diff --git a/bin/varnishd/cache_fetch.c b/bin/varnishd/cache_fetch.c
index 6b123f2..e84219f 100644
--- a/bin/varnishd/cache_fetch.c
+++ b/bin/varnishd/cache_fetch.c
@@ -71,7 +71,7 @@ vfp_nop_begin(struct sess *sp, size_t estimate)
 		    "Fetch %d byte segments:", fetchfrag);
 	}
 	if (estimate > 0)
-		sp->wrk->storage = STV_alloc(sp, estimate);
+		(void)FetchStorage(sp, estimate);
 }
 
 /*--------------------------------------------------------------------
@@ -91,7 +91,7 @@ vfp_nop_bytes(struct sess *sp, struct http_conn *htc, ssize_t bytes)
 	struct storage *st;
 
 	while (bytes > 0) {
-		if (FetchStorage(sp))
+		if (FetchStorage(sp, 0))
 			return (-1);
 		st = sp->wrk->storage;
 		l = st->space - st->len;
@@ -126,13 +126,15 @@ vfp_nop_end(struct sess *sp)
 	if (st == NULL)
 		return (0);
 
+	assert(st == VTAILQ_LAST(&sp->obj->store, storagehead));
+
 	if (st->len == 0) {
+		VTAILQ_REMOVE(&sp->obj->store, st, list);
 		STV_free(st);
 		return (0);
 	}
 	if (st->len < st->space)
 		STV_trim(st, st->len);
-	VTAILQ_INSERT_TAIL(&sp->obj->store, st, list);
 	return (0);
 }
 
@@ -147,25 +149,30 @@ static struct vfp vfp_nop = {
  */
 
 int
-FetchStorage(const struct sess *sp)
+FetchStorage(const struct sess *sp, ssize_t sz)
 {
 	ssize_t l;
 
 	if (sp->wrk->storage != NULL &&
-	    sp->wrk->storage->len == sp->wrk->storage->space) {
-		VTAILQ_INSERT_TAIL(&sp->obj->store, sp->wrk->storage, list);
+	    sp->wrk->storage->len == sp->wrk->storage->space)
 		sp->wrk->storage = NULL;
+	if (sp->wrk->storage != NULL) {
+		assert(sp->wrk->storage == VTAILQ_LAST(&sp->obj->store, storagehead));
+		return (0);
 	}
-	if (sp->wrk->storage == NULL) {
-		l = fetchfrag;
-		if (l == 0)
-			l = params->fetch_chunksize * 1024LL;
-		sp->wrk->storage = STV_alloc(sp, l);
-	}
+
+	l = fetchfrag;
+	if (l == 0)
+		l = sz;
+	if (l == 0)
+		l = params->fetch_chunksize * 1024LL;
+	sp->wrk->storage = STV_alloc(sp, l);
 	if (sp->wrk->storage == NULL) {
 		errno = ENOMEM;
 		return (-1);
 	}
+	AZ(sp->wrk->storage->len);
+	VTAILQ_INSERT_TAIL(&sp->obj->store, sp->wrk->storage, list);
 	return (0);
 }
 
diff --git a/bin/varnishd/cache_gzip.c b/bin/varnishd/cache_gzip.c
index f49e7bd..8a3908f 100644
--- a/bin/varnishd/cache_gzip.c
+++ b/bin/varnishd/cache_gzip.c
@@ -245,7 +245,7 @@ VGZ_ObufStorage(const struct sess *sp, struct vgz *vg)
 {
 	struct storage *st;
 
-	if (FetchStorage(sp))
+	if (FetchStorage(sp, 0))
 		return (-1);
 
 	st = sp->wrk->storage;
@@ -527,7 +527,7 @@ vfp_testgzip_bytes(struct sess *sp, struct http_conn *htc, ssize_t bytes)
 	CHECK_OBJ_NOTNULL(vg, VGZ_MAGIC);
 	AZ(vg->vz.avail_in);
 	while (bytes > 0) {
-		if (FetchStorage(sp))
+		if (FetchStorage(sp, 0))
 			return (-1);
 		st = sp->wrk->storage;
 		l = st->space - st->len;



From phk at varnish-cache.org  Wed Jan 26 21:50:00 2011
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Wed, 26 Jan 2011 22:50:00 +0100
Subject: [master] c5e69db Eliminate the cached the wrk->storage pointer,
	it is unnecesary.
Message-ID: <E1PiDFY-00026S-Am@project.varnish-software.com>

commit c5e69db8e2cfb03e294679eaf5bef4ba394e6587
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Jan 26 21:49:15 2011 +0000

    Eliminate the cached the wrk->storage pointer, it is unnecesary.

diff --git a/bin/varnishd/cache.h b/bin/varnishd/cache.h
index d2de03e..115ccf7 100644
--- a/bin/varnishd/cache.h
+++ b/bin/varnishd/cache.h
@@ -279,7 +279,6 @@ struct worker {
 
 	/* Fetch stuff */
 	enum body_status	body_status;
-	struct storage		*storage;
 	struct vfp		*vfp;
 	struct vgz		*vgz_rx;
 	struct vef_priv		*vef_priv;
@@ -630,7 +629,7 @@ void EXP_Touch(struct object *o, double tnow);
 int EXP_NukeOne(const struct sess *sp, const struct lru *lru);
 
 /* cache_fetch.c */
-int FetchStorage(const struct sess *sp, ssize_t sz);
+struct storage *FetchStorage(const struct sess *sp, ssize_t sz);
 int FetchHdr(struct sess *sp);
 int FetchBody(struct sess *sp);
 int FetchReqBody(struct sess *sp);
diff --git a/bin/varnishd/cache_center.c b/bin/varnishd/cache_center.c
index 1085fa0..16d1a7e 100644
--- a/bin/varnishd/cache_center.c
+++ b/bin/varnishd/cache_center.c
@@ -1367,7 +1367,6 @@ CNT_Session(struct sess *sp)
 		CHECK_OBJ_ORNULL(w->nobjhead, OBJHEAD_MAGIC);
 		WS_Assert(w->ws);
 		AZ(sp->wrk->storage_hint);
-		AZ(sp->wrk->storage);
 
 		switch (sp->step) {
 #define STEP(l,u) \
diff --git a/bin/varnishd/cache_esi_fetch.c b/bin/varnishd/cache_esi_fetch.c
index 24aefeb..ff36f28 100644
--- a/bin/varnishd/cache_esi_fetch.c
+++ b/bin/varnishd/cache_esi_fetch.c
@@ -77,9 +77,9 @@ vfp_esi_bytes_uu(struct sess *sp, struct http_conn *htc, ssize_t bytes)
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 
 	while (bytes > 0) {
-		if (FetchStorage(sp, 0))
+		st = FetchStorage(sp, 0);
+		if (st == NULL)
 			return (-1);
-		st = sp->wrk->storage;
 		w = vef_read(htc,
 		    st->ptr + st->len, st->space - st->len, bytes);
 		if (w <= 0)
diff --git a/bin/varnishd/cache_fetch.c b/bin/varnishd/cache_fetch.c
index e84219f..22cce8c 100644
--- a/bin/varnishd/cache_fetch.c
+++ b/bin/varnishd/cache_fetch.c
@@ -64,7 +64,6 @@ static void __match_proto__()
 vfp_nop_begin(struct sess *sp, size_t estimate)
 {
 
-	AZ(sp->wrk->storage);
 	if (fetchfrag > 0) {
 		estimate = fetchfrag;
 		WSL(sp->wrk, SLT_Debug, sp->fd,
@@ -91,9 +90,9 @@ vfp_nop_bytes(struct sess *sp, struct http_conn *htc, ssize_t bytes)
 	struct storage *st;
 
 	while (bytes > 0) {
-		if (FetchStorage(sp, 0))
+		st = FetchStorage(sp, 0);
+		if (st == NULL)
 			return (-1);
-		st = sp->wrk->storage;
 		l = st->space - st->len;
 		if (l > bytes)
 			l = bytes;
@@ -121,13 +120,10 @@ vfp_nop_end(struct sess *sp)
 {
 	struct storage *st;
 
-	st = sp->wrk->storage;
-	sp->wrk->storage = NULL;
+	st = VTAILQ_LAST(&sp->obj->store, storagehead);
 	if (st == NULL)
 		return (0);
 
-	assert(st == VTAILQ_LAST(&sp->obj->store, storagehead));
-
 	if (st->len == 0) {
 		VTAILQ_REMOVE(&sp->obj->store, st, list);
 		STV_free(st);
@@ -148,32 +144,29 @@ static struct vfp vfp_nop = {
  * Fetch Storage
  */
 
-int
+struct storage *
 FetchStorage(const struct sess *sp, ssize_t sz)
 {
 	ssize_t l;
+	struct storage *st;
 
-	if (sp->wrk->storage != NULL &&
-	    sp->wrk->storage->len == sp->wrk->storage->space)
-		sp->wrk->storage = NULL;
-	if (sp->wrk->storage != NULL) {
-		assert(sp->wrk->storage == VTAILQ_LAST(&sp->obj->store, storagehead));
-		return (0);
-	}
+	st = VTAILQ_LAST(&sp->obj->store, storagehead);
+	if (st != NULL && st->len < st->space)
+		return (st);
 
 	l = fetchfrag;
 	if (l == 0)
 		l = sz;
 	if (l == 0)
 		l = params->fetch_chunksize * 1024LL;
-	sp->wrk->storage = STV_alloc(sp, l);
-	if (sp->wrk->storage == NULL) {
+	st = STV_alloc(sp, l);
+	if (st == NULL) {
 		errno = ENOMEM;
-		return (-1);
+		return (NULL);
 	}
-	AZ(sp->wrk->storage->len);
-	VTAILQ_INSERT_TAIL(&sp->obj->store, sp->wrk->storage, list);
-	return (0);
+	AZ(st->len);
+	VTAILQ_INSERT_TAIL(&sp->obj->store, st, list);
+	return (st);
 }
 
 /*--------------------------------------------------------------------
@@ -514,7 +507,7 @@ FetchBody(struct sess *sp)
 	 * XXX: Missing:  RFC2616 sec. 4.4 in re 1xx, 204 & 304 responses
 	 */
 
-	AZ(sp->wrk->storage);
+	AZ(VTAILQ_FIRST(&sp->obj->store));
 	switch (sp->wrk->body_status) {
 	case BS_NONE:
 		cls = 0;
@@ -553,7 +546,6 @@ FetchBody(struct sess *sp)
 	 * to get it trimmed and added to the object.
 	 */
 	XXXAZ(vfp_nop_end(sp));
-	AZ(sp->wrk->storage);
 
 	WSL(sp->wrk, SLT_Fetch_Body, sp->vbc->fd, "%u %d %u",
 	    sp->wrk->body_status, cls, mklen);
diff --git a/bin/varnishd/cache_gzip.c b/bin/varnishd/cache_gzip.c
index 8a3908f..930d7f1 100644
--- a/bin/varnishd/cache_gzip.c
+++ b/bin/varnishd/cache_gzip.c
@@ -245,10 +245,10 @@ VGZ_ObufStorage(const struct sess *sp, struct vgz *vg)
 {
 	struct storage *st;
 
-	if (FetchStorage(sp, 0))
+	st = FetchStorage(sp, 0);
+	if (st == NULL)
 		return (-1);
 
-	st = sp->wrk->storage;
 	vg->obuf = st;
 	VGZ_Obuf(vg, st->ptr + st->len, st->space - st->len);
 
@@ -527,9 +527,9 @@ vfp_testgzip_bytes(struct sess *sp, struct http_conn *htc, ssize_t bytes)
 	CHECK_OBJ_NOTNULL(vg, VGZ_MAGIC);
 	AZ(vg->vz.avail_in);
 	while (bytes > 0) {
-		if (FetchStorage(sp, 0))
+		st = FetchStorage(sp, 0);
+		if (st == NULL)
 			return (-1);
-		st = sp->wrk->storage;
 		l = st->space - st->len;
 		if (l > bytes)
 			l = bytes;
diff --git a/bin/varnishd/cache_pool.c b/bin/varnishd/cache_pool.c
index 4654002..27b7253 100644
--- a/bin/varnishd/cache_pool.c
+++ b/bin/varnishd/cache_pool.c
@@ -182,13 +182,11 @@ wrk_thread_real(struct wq *qp, unsigned shm_workspace, unsigned sess_workspace,
 		w->beresp = NULL;
 		w->resp = NULL;
 		w->storage_hint = NULL;
-		w->storage = NULL;
 		w->wrq->func(w, w->wrq->priv);
 		AZ(w->bereq);
 		AZ(w->beresp1);
 		AZ(w->beresp);
 		AZ(w->resp);
-		AZ(w->storage);
 		WS_Assert(w->ws);
 		AZ(w->wfd);
 		AZ(w->storage_hint);



From tfheen at varnish-cache.org  Thu Jan 27 09:36:23 2011
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Thu, 27 Jan 2011 10:36:23 +0100
Subject: [master] 8bdd3d1 Revert "varnishtest: setuid(nobody) if running as
	root"
Message-ID: <E1PiOH9-0007nl-Hq@project.varnish-software.com>

commit 8bdd3d11c7c37053fcbf43e36a325b88813b7d32
Author: Tollef Fog Heen <tfheen at varnish-software.com>
Date:   Wed Jan 26 13:42:27 2011 +0100

    Revert "varnishtest: setuid(nobody) if running as root"
    
    This had the unfortunate side effect of making builds in
    non-world-readable directories not work correctly when run as root.
    
    This reverts commit 07c4432ba55b50b6640c5eeb24155a3618224f61.

diff --git a/bin/varnishtest/vtc_main.c b/bin/varnishtest/vtc_main.c
index ac3436e..3db3bd9 100644
--- a/bin/varnishtest/vtc_main.c
+++ b/bin/varnishtest/vtc_main.c
@@ -42,11 +42,6 @@ SVNID("$Id$")
 #include <sys/stat.h>
 #include <sys/types.h>
 #include <sys/wait.h>
-#include <pwd.h>
-
-#ifdef __linux__
-#include <sys/prctl.h>
-#endif
 
 #include "libvarnish.h"
 #include "vev.h"
@@ -369,21 +364,6 @@ main(int argc, char * const *argv)
 
 	vb = vev_new_base();
 
-	if (geteuid() == 0) {
-		struct passwd *pw;
-		pw = getpwnam("nobody");
-		assert(setgid(pw->pw_gid) == 0);
-		assert(setuid(pw->pw_uid) == 0);
-		/* On Linux >= 2.4, you need to set the dumpable flag
-		   to get core dumps after you have done a setuid. */
-#ifdef __linux__
-		if (prctl(PR_SET_DUMPABLE, 1) != 0) {
-		  printf("Could not set dumpable bit.  Core dumps turned "
-			 "off\n");
-		}
-#endif
-	}
-
 	i = 0;
 	while(!VTAILQ_EMPTY(&tst_head) || i) {
 		if (!VTAILQ_EMPTY(&tst_head) && njob < npar) {



From tfheen at varnish-cache.org  Thu Jan 27 09:36:24 2011
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Thu, 27 Jan 2011 10:36:24 +0100
Subject: [master] fefe93d Install vmods in the varnish RPM package
Message-ID: <E1PiOHA-0007ny-4E@project.varnish-software.com>

commit fefe93d0a681879d353930d26dac07f882c513ab
Author: Tollef Fog Heen <tfheen at varnish-software.com>
Date:   Wed Jan 26 14:20:03 2011 +0100

    Install vmods in the varnish RPM package
    
    Make sure the standard vmod ends up in the RPM

diff --git a/bin/varnishtest/vtc_main.c b/bin/varnishtest/vtc_main.c
index 3db3bd9..3afc448 100644
--- a/bin/varnishtest/vtc_main.c
+++ b/bin/varnishtest/vtc_main.c
@@ -246,7 +246,7 @@ start_test(void)
 
 	srandomdev();
 	bprintf(tmpdir, "/tmp/vtc.%d.%08x", getpid(), (unsigned)random());
-	AZ(mkdir(tmpdir, 0700));
+	AZ(mkdir(tmpdir, 0711));
 
 	tp = VTAILQ_FIRST(&tst_head);
 	CHECK_OBJ_NOTNULL(tp, TST_MAGIC);
diff --git a/redhat/varnish.spec b/redhat/varnish.spec
index 8f04637..031f0a9 100644
--- a/redhat/varnish.spec
+++ b/redhat/varnish.spec
@@ -195,6 +195,7 @@ rm -rf %{buildroot}
 %defattr(-,root,root,-)
 %{_sbindir}/*
 %{_bindir}/*
+%{_libdir}/varnish
 %{_var}/lib/varnish
 %{_var}/log/varnish
 %{_mandir}/man1/*.1*



From tfheen at varnish-cache.org  Thu Jan 27 09:36:25 2011
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Thu, 27 Jan 2011 10:36:25 +0100
Subject: [master] 5f6871e Merge branch 'master' of
	ssh://git.varnish-cache.org/git/varnish-cache
Message-ID: <E1PiOHB-0007oB-4s@project.varnish-software.com>

commit 5f6871eca000ce80d27c720c45b1d9c56370833b
Merge: fefe93d c5e69db
Author: Tollef Fog Heen <tfheen at varnish-software.com>
Date:   Thu Jan 27 09:42:31 2011 +0100

    Merge branch 'master' of ssh://git.varnish-cache.org/git/varnish-cache



From tfheen at varnish-cache.org  Thu Jan 27 09:36:25 2011
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Thu, 27 Jan 2011 10:36:25 +0100
Subject: [master] 713b44f Adjust spec file for tarball building
Message-ID: <E1PiOHB-0007oO-Gc@project.varnish-software.com>

commit 713b44ff73b88ac0e4ab3ffcafe0329a26587f40
Author: Tollef Fog Heen <tfheen at varnish-software.com>
Date:   Thu Jan 27 10:34:34 2011 +0100

    Adjust spec file for tarball building
    
    We support building RPMs from a tarball, not directly from git (RPM
    needs a tarball anyway, so requiring people to run make dist doesn't
    seem onerous).
    
    Remove autogen.sh invocation and remove explicit building of
    documentation.  Both of these are done as part of make dist

diff --git a/redhat/varnish.spec b/redhat/varnish.spec
index 031f0a9..cc4296a 100644
--- a/redhat/varnish.spec
+++ b/redhat/varnish.spec
@@ -75,7 +75,7 @@ Documentation files for %name
 
 # The svn sources needs to generate a suitable configure script
 # Release tarballs would not need this
-./autogen.sh
+#./autogen.sh
 
 # Hack to get 32- and 64-bits tests run concurrently on the same build machine
 case `uname -m` in
@@ -141,12 +141,7 @@ tail -n +11 etc/default.vcl >> redhat/default.vcl
 	redhat/varnish.initrc redhat/varnishlog.initrc redhat/varnishncsa.initrc
 %endif
 
-%if 0%{?rhel} > 4 || 0%{?fedora} > 6
-pushd doc/sphinx
-%{__make} html
-popd
-mv doc/sphinx/\=build/html doc
-%endif
+cp -r doc/sphinx/\=build/html doc
 
 %check
 # rhel5 on ppc64 is just too strange



From tfheen at varnish-cache.org  Thu Jan 27 09:57:03 2011
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Thu, 27 Jan 2011 10:57:03 +0100
Subject: [master] 0506b11 Drop sed hack for 32/64 bit machines from spec file
Message-ID: <E1PiOb9-00016V-Ro@project.varnish-software.com>

commit 0506b118aab82b9b9940cce65d1447a53e3db338
Author: Tollef Fog Heen <tfheen at varnish-software.com>
Date:   Thu Jan 27 10:56:23 2011 +0100

    Drop sed hack for 32/64 bit machines from spec file
    
    varnishtest now allocates ports dynamically, so it should no longer be
    a problem to have two varnishtest instances running concurrently.

diff --git a/redhat/varnish.spec b/redhat/varnish.spec
index cc4296a..0a7c99f 100644
--- a/redhat/varnish.spec
+++ b/redhat/varnish.spec
@@ -77,21 +77,6 @@ Documentation files for %name
 # Release tarballs would not need this
 #./autogen.sh
 
-# Hack to get 32- and 64-bits tests run concurrently on the same build machine
-case `uname -m` in
-	ppc64 | s390x | x86_64 | sparc64 )
-		sed -i ' 
-			s,9001,9011,g;
-			s,9080,9090,g; 
-			s,9081,9091,g; 
-			s,9082,9092,g; 
-			s,9180,9190,g;
-		' bin/varnishtest/*.c bin/varnishtest/tests/*vtc
-		;;
-	*)
-		;;
-esac
-
 mkdir examples
 cp bin/varnishd/default.vcl etc/zope-plone.vcl examples
 



From phk at varnish-cache.org  Thu Jan 27 10:44:22 2011
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Thu, 27 Jan 2011 11:44:22 +0100
Subject: [master] eb9eca8 Further clarify fetch() processing.
Message-ID: <E1PiPKw-0004bR-HL@project.varnish-software.com>

commit eb9eca8cd4c67556b9361ff2684e105f6b48f9b8
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu Jan 27 09:19:42 2011 +0000

    Further clarify fetch() processing.

diff --git a/bin/varnishd/cache.h b/bin/varnishd/cache.h
index 78c33e7..9116946 100644
--- a/bin/varnishd/cache.h
+++ b/bin/varnishd/cache.h
@@ -514,7 +514,6 @@ struct sess {
 	enum step		step;
 	unsigned		cur_method;
 	unsigned		handling;
-	unsigned char		pass;
 	unsigned char		sendbody;
 	unsigned char		wantbody;
 	int			err_code;
@@ -631,7 +630,7 @@ int EXP_NukeOne(const struct sess *sp, const struct lru *lru);
 /* cache_fetch.c */
 struct storage *FetchStorage(const struct sess *sp, ssize_t sz);
 int FetchHdr(struct sess *sp);
-int FetchBody(struct sess *sp, struct http *hp);
+int FetchBody(struct sess *sp, const struct http *hp);
 int FetchReqBody(struct sess *sp);
 void Fetch_Init(void);
 
diff --git a/bin/varnishd/cache_center.c b/bin/varnishd/cache_center.c
index 82df85d..08458c6 100644
--- a/bin/varnishd/cache_center.c
+++ b/bin/varnishd/cache_center.c
@@ -468,7 +468,7 @@ cnt_fetch(struct sess *sp)
 	struct http *hp, *hp2;
 	char *b;
 	unsigned l, nhttp;
-	int varyl = 0;
+	int varyl = 0, pass;
 	struct vsb *vary = NULL;
 
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
@@ -562,20 +562,16 @@ cnt_fetch(struct sess *sp)
 	if (sp->objcore == NULL) {
 		/* This is a pass from vcl_recv */
 		sp->wrk->cacheable = 0;
-	} else if (!sp->wrk->cacheable && sp->objcore != NULL) {
-		AZ(HSH_Deref(sp->wrk, sp->objcore, NULL));
-		sp->objcore = NULL;
+		pass = 1;
+	} else if (sp->handling == VCL_RET_PASS || !sp->wrk->cacheable) {
+		/* pass from vcl_fetch{} -> hit-for-pass */
+		pass = 1;
+	} else {
+		/* regular object */
+		pass = 0;
 	}
 
 	/*
-	 * At this point we are either committed to flesh out the busy
-	 * object we have in the hash or we have let go of it, if we ever
-	 * had one.
-	 */
-
-	AZ(sp->wrk->vfp);
-
-	/*
 	 * The VCL variables beresp.do_g[un]zip tells us how we want the
 	 * object processed before it is stored.
 	 *
@@ -588,6 +584,8 @@ cnt_fetch(struct sess *sp)
 	 *
 	 */
 
+	AZ(sp->wrk->vfp);
+
 	/* We do nothing unless the param is set */
 	if (!params->http_gzip_support)
 		sp->wrk->do_gzip = sp->wrk->do_gunzip = 0;
@@ -632,7 +630,7 @@ cnt_fetch(struct sess *sp)
 		sp->wrk->vfp = &vfp_testgzip;
 
 	l = http_EstimateWS(sp->wrk->beresp,
-	    sp->pass ? HTTPH_R_PASS : HTTPH_A_INS, &nhttp);
+	    pass ? HTTPH_R_PASS : HTTPH_A_INS, &nhttp);
 
 	/* Create Vary instructions */
 	if (sp->wrk->cacheable) {
@@ -677,7 +675,6 @@ cnt_fetch(struct sess *sp)
 	sp->obj->entered = sp->wrk->entered;
 	WS_Assert(sp->obj->ws_o);
 
-
 	/* Filter into object */
 	hp = sp->wrk->beresp;
 	hp2 = sp->obj->http;
@@ -685,7 +682,7 @@ cnt_fetch(struct sess *sp)
 	hp2->logtag = HTTP_Obj;
 	http_CopyResp(hp2, hp);
 	http_FilterFields(sp->wrk, sp->fd, hp2, hp,
-	    sp->pass ? HTTPH_R_PASS : HTTPH_A_INS);
+	    pass ? HTTPH_R_PASS : HTTPH_A_INS);
 	http_CopyHome(sp->wrk, sp->fd, hp2);
 
 	if (http_GetHdr(hp, H_Last_Modified, &b))
@@ -1050,7 +1047,6 @@ cnt_pass(struct sess *sp)
 	sp->acct_tmp.pass++;
 	sp->sendbody = 1;
 	sp->step = STP_FETCH;
-	sp->pass = 1;
 	return (0);
 }
 
@@ -1138,7 +1134,6 @@ cnt_recv(struct sess *sp)
 	AN(sp->director);
 
 	sp->disable_esi = 0;
-	sp->pass = 0;
 	sp->hash_always_miss = 0;
 	sp->hash_ignore_busy = 0;
 	sp->client_identity = NULL;
diff --git a/bin/varnishd/cache_fetch.c b/bin/varnishd/cache_fetch.c
index a1db19f..05e013f 100644
--- a/bin/varnishd/cache_fetch.c
+++ b/bin/varnishd/cache_fetch.c
@@ -481,7 +481,7 @@ FetchHdr(struct sess *sp)
 /*--------------------------------------------------------------------*/
 
 int
-FetchBody(struct sess *sp, struct http *hp)
+FetchBody(struct sess *sp, const struct http *hp)
 {
 	char *b;
 	int cls;



From phk at varnish-cache.org  Thu Jan 27 10:44:22 2011
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Thu, 27 Jan 2011 11:44:22 +0100
Subject: [master] 62e111b Tighten up the object fetch code a little bit.
Message-ID: <E1PiPKw-0004bJ-3q@project.varnish-software.com>

commit 62e111bef71dae24a25530e16dd7953a16781309
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu Jan 27 08:40:13 2011 +0000

    Tighten up the object fetch code a little bit.

diff --git a/bin/varnishd/cache.h b/bin/varnishd/cache.h
index 115ccf7..78c33e7 100644
--- a/bin/varnishd/cache.h
+++ b/bin/varnishd/cache.h
@@ -631,7 +631,7 @@ int EXP_NukeOne(const struct sess *sp, const struct lru *lru);
 /* cache_fetch.c */
 struct storage *FetchStorage(const struct sess *sp, ssize_t sz);
 int FetchHdr(struct sess *sp);
-int FetchBody(struct sess *sp);
+int FetchBody(struct sess *sp, struct http *hp);
 int FetchReqBody(struct sess *sp);
 void Fetch_Init(void);
 
diff --git a/bin/varnishd/cache_center.c b/bin/varnishd/cache_center.c
index 16d1a7e..82df85d 100644
--- a/bin/varnishd/cache_center.c
+++ b/bin/varnishd/cache_center.c
@@ -573,22 +573,11 @@ cnt_fetch(struct sess *sp)
 	 * had one.
 	 */
 
-	if (sp->wrk->cacheable) {
-		CHECK_OBJ_NOTNULL(sp->objcore, OBJCORE_MAGIC);
-		vary = VRY_Create(sp, sp->wrk->beresp);
-		if (vary != NULL) {
-			varyl = vsb_len(vary);
-			assert(varyl > 0);
-		}
-	} else {
-		AZ(sp->objcore);
-	}
-
 	AZ(sp->wrk->vfp);
 
 	/*
 	 * The VCL variables beresp.do_g[un]zip tells us how we want the
-	 * object stored.
+	 * object processed before it is stored.
 	 *
 	 * The backend Content-Encoding header tells us what we are going
 	 * to receive, which we classify in the following three classes:
@@ -645,8 +634,16 @@ cnt_fetch(struct sess *sp)
 	l = http_EstimateWS(sp->wrk->beresp,
 	    sp->pass ? HTTPH_R_PASS : HTTPH_A_INS, &nhttp);
 
-	if (vary != NULL)
-		l += varyl;
+	/* Create Vary instructions */
+	if (sp->wrk->cacheable) {
+		CHECK_OBJ_NOTNULL(sp->objcore, OBJCORE_MAGIC);
+		vary = VRY_Create(sp, sp->wrk->beresp);
+		if (vary != NULL) {
+			varyl = vsb_len(vary);
+			assert(varyl > 0);
+			l += varyl;
+		}
+	}
 
 	/*
 	 * Space for producing a Content-Length: header including padding
@@ -654,10 +651,6 @@ cnt_fetch(struct sess *sp)
 	 */
 	l += strlen("Content-Encoding: XxxXxxXxxXxxXxxXxx" + sizeof(void *));
 
-	/*
-	 * XXX: VFP's may affect estimate
-	 */
-
 	sp->obj = STV_NewObject(sp, sp->wrk->storage_hint, l,
 	    sp->wrk->ttl, nhttp);
 	CHECK_OBJ_NOTNULL(sp->obj, OBJECT_MAGIC);
@@ -700,7 +693,13 @@ cnt_fetch(struct sess *sp)
 	else
 		sp->obj->last_modified = sp->wrk->entered;
 
-	i = FetchBody(sp);
+
+	/* Use unmodified headers*/
+	i = FetchBody(sp, sp->wrk->beresp1);
+
+	sp->wrk->bereq = NULL;
+	sp->wrk->beresp = NULL;
+	sp->wrk->beresp1 = NULL;
 	sp->wrk->vfp = NULL;
 	AZ(sp->wrk->wfd);
 	AZ(sp->vbc);
@@ -709,9 +708,6 @@ cnt_fetch(struct sess *sp)
 	if (i) {
 		HSH_Drop(sp);
 		AZ(sp->obj);
-		sp->wrk->bereq = NULL;
-		sp->wrk->beresp = NULL;
-		sp->wrk->beresp1 = NULL;
 		sp->err_code = 503;
 		sp->step = STP_ERROR;
 		return (0);
@@ -722,9 +718,6 @@ cnt_fetch(struct sess *sp)
 		HSH_Drop(sp);
 		sp->director = NULL;
 		sp->restarts++;
-		sp->wrk->bereq = NULL;
-		sp->wrk->beresp = NULL;
-		sp->wrk->beresp1 = NULL;
 		sp->step = STP_RECV;
 		return (0);
 	case VCL_RET_PASS:
@@ -737,9 +730,6 @@ cnt_fetch(struct sess *sp)
 		break;
 	case VCL_RET_ERROR:
 		HSH_Drop(sp);
-		sp->wrk->bereq = NULL;
-		sp->wrk->beresp = NULL;
-		sp->wrk->beresp1 = NULL;
 		sp->step = STP_ERROR;
 		return (0);
 	default:
@@ -754,9 +744,6 @@ cnt_fetch(struct sess *sp)
 		HSH_Unbusy(sp);
 	}
 	sp->acct_tmp.fetch++;
-	sp->wrk->bereq = NULL;
-	sp->wrk->beresp = NULL;
-	sp->wrk->beresp1 = NULL;
 	sp->step = STP_DELIVER;
 	return (0);
 }
diff --git a/bin/varnishd/cache_fetch.c b/bin/varnishd/cache_fetch.c
index 22cce8c..a1db19f 100644
--- a/bin/varnishd/cache_fetch.c
+++ b/bin/varnishd/cache_fetch.c
@@ -481,11 +481,10 @@ FetchHdr(struct sess *sp)
 /*--------------------------------------------------------------------*/
 
 int
-FetchBody(struct sess *sp)
+FetchBody(struct sess *sp, struct http *hp)
 {
 	char *b;
 	int cls;
-	struct http *hp;
 	struct storage *st;
 	int mklen;
 
@@ -497,8 +496,6 @@ FetchBody(struct sess *sp)
 	if (sp->wrk->vfp == NULL)
 		sp->wrk->vfp = &vfp_nop;
 
-	/* We use the unmodified headers */
-	hp = sp->wrk->beresp1;
 	AN(sp->director);
 	AssertObjPassOrBusy(sp->obj);
 



From phk at varnish-cache.org  Thu Jan 27 10:44:22 2011
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Thu, 27 Jan 2011 11:44:22 +0100
Subject: [master] 81b3dd6 Merge branch 'master' of
	ssh://git.varnish-cache.org/git/varnish-cache
Message-ID: <E1PiPKw-0004bZ-Pp@project.varnish-software.com>

commit 81b3dd6527964c81dcbaec5dd06a40e31cbd0d0d
Merge: eb9eca8 0506b11
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu Jan 27 10:13:07 2011 +0000

    Merge branch 'master' of ssh://git.varnish-cache.org/git/varnish-cache



From tfheen at varnish-cache.org  Thu Jan 27 11:15:05 2011
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Thu, 27 Jan 2011 12:15:05 +0100
Subject: [master] 5bad62e Add -o flag to varnishncsa synposis
Message-ID: <E1PiPof-0007ZM-VY@project.varnish-software.com>

commit 5bad62e8f1c38182937405bb5a853820e57ef1e4
Author: Tollef Fog Heen <tfheen at varnish-software.com>
Date:   Thu Jan 27 12:13:52 2011 +0100

    Add -o flag to varnishncsa synposis
    
    This is 74f9b6972eebc29b6272f2517d7541eda2dbb1dd again, since it got
    lost in a merge conflict.

diff --git a/doc/sphinx/reference/varnishncsa.rst b/doc/sphinx/reference/varnishncsa.rst
index 0500d96..2cb64af 100644
--- a/doc/sphinx/reference/varnishncsa.rst
+++ b/doc/sphinx/reference/varnishncsa.rst
@@ -16,7 +16,7 @@ SYNOPSIS
 ========
 
 varnishncsa [-a] [-b] [-C] [-c] [-D] [-d] [-f] [-F format] [-I regex]
-[-i tag] [-n varnish_name] [-P file] [-r file] [-V] [-w file] 
+[-i tag] [-n varnish_name] [-o tag regex] [-P file] [-r file] [-V] [-w file] 
 [-X regex] [-x tag]
 
 



From tfheen at varnish-cache.org  Thu Jan 27 11:15:06 2011
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Thu, 27 Jan 2011 12:15:06 +0100
Subject: [master] d346cd9 Drop rst2man magic in spec file; unneeded
Message-ID: <E1PiPog-0007Zs-VK@project.varnish-software.com>

commit d346cd956427e3a4984e523957a454f6364e40b2
Author: Tollef Fog Heen <tfheen at varnish-software.com>
Date:   Thu Jan 27 12:14:39 2011 +0100

    Drop rst2man magic in spec file; unneeded

diff --git a/redhat/varnish.spec b/redhat/varnish.spec
index 0a7c99f..39748e0 100644
--- a/redhat/varnish.spec
+++ b/redhat/varnish.spec
@@ -81,11 +81,6 @@ mkdir examples
 cp bin/varnishd/default.vcl etc/zope-plone.vcl examples
 
 %build
-# No rst2man in rhel4 or rhel5 (use pregenerated manpages)
-#%if 0%{?rhel} < 6 && 0%{?fedora} < 12
-#	export RST2MAN=true
-#%endif
-
 # No pkgconfig/libpcre.pc in rhel4
 %if 0%{?rhel} == 4
 	export PCRE_CFLAGS=`pcre-config --cflags`



From tfheen at varnish-cache.org  Thu Jan 27 11:15:07 2011
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Thu, 27 Jan 2011 12:15:07 +0100
Subject: [master] 064f429 Merge branch 'master' of
	ssh://git.varnish-cache.org/git/varnish-cache
Message-ID: <E1PiPoh-0007a4-Nw@project.varnish-software.com>

commit 064f4292fb1166a86f7b95c5abb0d435e882252b
Merge: d346cd9 81b3dd6
Author: Tollef Fog Heen <tfheen at varnish-software.com>
Date:   Thu Jan 27 12:14:58 2011 +0100

    Merge branch 'master' of ssh://git.varnish-cache.org/git/varnish-cache



From tfheen at varnish-cache.org  Thu Jan 27 11:30:22 2011
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Thu, 27 Jan 2011 12:30:22 +0100
Subject: [master] 3f9a134 Fix rst warnings (markup)
Message-ID: <E1PiQ3S-0001xI-KK@project.varnish-software.com>

commit 3f9a13417658da18db318dce49048513c3f603b3
Author: Tollef Fog Heen <tfheen at varnish-software.com>
Date:   Thu Jan 27 12:29:07 2011 +0100

    Fix rst warnings (markup)
    
    Various markup problems as reported by sphinx.

diff --git a/doc/sphinx/phk/gzip.rst b/doc/sphinx/phk/gzip.rst
index a87034a..53bb1c9 100644
--- a/doc/sphinx/phk/gzip.rst
+++ b/doc/sphinx/phk/gzip.rst
@@ -89,7 +89,7 @@ the backend didn't send us a gzip'ed object.
 
 Remember that a lot of content types cannot sensibly be gziped, most
 notably compressed image formats like jpeg, png and similar, so a
-typical use would be:
+typical use would be::
 
 	sub vcl_fetch {
 		if (req.url ~ "html$") {
@@ -100,7 +100,7 @@ typical use would be:
 GZIP and ESI
 ------------
 
-First, note the new syntax for activating ESI:
+First, note the new syntax for activating ESI::
 
 	sub vcl_fetch {
 		set beresp.do_esi = true;
@@ -147,7 +147,7 @@ gunzip them, but when we encounter gunzip'ed objects which should
 be, we gzip them, but only at compression level zero.
 
 So in order to avoid unnecessary work, and in order to get maximum
-compression efficiency, you should:
+compression efficiency, you should::
 
 	sub vcl_miss {
 		if (object needs ESI processing) {
diff --git a/doc/sphinx/phk/ipv6suckage.rst b/doc/sphinx/phk/ipv6suckage.rst
index 63bdb22..dac68be 100644
--- a/doc/sphinx/phk/ipv6suckage.rst
+++ b/doc/sphinx/phk/ipv6suckage.rst
@@ -53,7 +53,9 @@ way (the same as RFC3986), you need to inspect the version field
 in the sockaddr to see if you should do
 
 	"%s:%s", host, port
+
 or
+
 	"[%s]:%s", host, port
 
 Careless standardization costs code, have I mentioned this before ?
diff --git a/doc/sphinx/reference/vmod.rst b/doc/sphinx/reference/vmod.rst
index 3813bf6..2b515c1 100644
--- a/doc/sphinx/reference/vmod.rst
+++ b/doc/sphinx/reference/vmod.rst
@@ -85,7 +85,7 @@ For the std VMOD, the compiled vcc_if.h file looks like this::
 	const char * vmod_tolower(struct sess *, struct vmod_priv *, const char *, ...);
 	int meta_function(void **, const struct VCL_conf *);
 
-Those are your C prototypes.  Notice the "vmod\_" prefix on the function
+Those are your C prototypes.  Notice the ``vmod_`` prefix on the function
 names and the C-types as return types and arguments.
 
 VCL and C data types
@@ -97,36 +97,36 @@ language representation.  Here is a description of them, from simple
 to nasty.
 
 INT
-	C-type: int
+	C-type: ``int``
 
 	An integer as we know and love them.
 
 REAL
-	C-type: double
+	C-type: ``double``
 
 	A floating point value
 
 DURATION
-	C-type: double
+	C-type: ``double``
 
 	Units: seconds
 
 	A time interval, as in "25 minutes".
 
 TIME
-	C-type: double
+	C-type: ``double``
 
 	Units: seconds since UNIX epoch
 
 	An absolute time, as in "Mon Sep 13 19:06:01 UTC 2010".
 
 STRING
-	C-type: const char *
+	C-type: ``const char *``
 
 	A NUL-terminated text-string.
 
 	Can be NULL to indicate that the nonexistent string, for
-	instance:
+	instance::
 
 		mymod.foo(req.http.foobar);
 
@@ -140,7 +140,7 @@ STRING
 	workspaces.
 
 STRING_LIST
-	C-type: const char *, ...
+	C-type: ``const char *, ...``
 
 	A multi-component text-string.  We try very hard to avoid
 	doing text-processing in Varnish, and this is one way we
@@ -152,15 +152,15 @@ STRING_LIST
 		set bereq.http.foo = std.toupper(req.http.foo + req.http.bar);
 
 	The usual way to do this, would be be to allocate memory for
-	the concatenated string, then pass that to toupper() which in
+	the concatenated string, then pass that to ``toupper()`` which in
 	turn would return another freshly allocated string with the
-	modified result.  Remember: strings in VCL are "const", we
+	modified result.  Remember: strings in VCL are ``const``, we
 	cannot just modify the string in place.
 
-	What we do instead, is declare that toupper() takes a "STRING_LIST"
-	as argument.  This makes the C function implementing toupper()
+	What we do instead, is declare that ``toupper()`` takes a "STRING_LIST"
+	as argument.  This makes the C function implementing ``toupper()``
 	a vararg function (see the prototype above) and responsible for
-	considering all the "const char *" arguments it finds, until the
+	considering all the ``const char *`` arguments it finds, until the
 	magic marker "vrt_magic_string_end" is encountered.
 
 	Bear in mind that the individual strings in a STRING_LIST can be
@@ -180,7 +180,7 @@ PRIV_CALL
 	See below
 
 VOID
-	C-type: void
+	C-type: ``void``
 
 	Can only be used for return-value, which makes the function a VCL
 	procedure.
@@ -208,7 +208,7 @@ last output of some expensive lookup.
 applies to all calls in this VCL, for instance flags that determine
 if regular expressions are case-sensitive in this vmod or similar.
 
-The way it works in the vmod code, is that a "struct vmod_priv *" is
+The way it works in the vmod code, is that a ``struct vmod_priv *`` is
 passed to the functions where argument type PRIV_VCL or PRIV_CALL
 is specified.
 
diff --git a/doc/sphinx/tutorial/advanced_topics.rst b/doc/sphinx/tutorial/advanced_topics.rst
index d140faa..5bdb7b4 100644
--- a/doc/sphinx/tutorial/advanced_topics.rst
+++ b/doc/sphinx/tutorial/advanced_topics.rst
@@ -17,7 +17,7 @@ discussed. For a complete(ish) guide to VCL have a look at the VCL man
 page - ref:`reference-vcl`.
 
 Using In-line C to extend Varnish
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 You can use *in-line C* to extend Varnish. Please note that you can
 seriously mess up Varnish this way. The C code runs within the Varnish



From tfheen at varnish-cache.org  Thu Jan 27 11:39:10 2011
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Thu, 27 Jan 2011 12:39:10 +0100
Subject: [master] 031c39e More markup fixing
Message-ID: <E1PiQBy-0002qT-Er@project.varnish-software.com>

commit 031c39eacbce52ea5c3368f16586f341543d5f49
Author: Tollef Fog Heen <tfheen at varnish-software.com>
Date:   Thu Jan 27 12:39:08 2011 +0100

    More markup fixing

diff --git a/doc/sphinx/reference/vcl.rst b/doc/sphinx/reference/vcl.rst
index 82efc96..2614d4b 100644
--- a/doc/sphinx/reference/vcl.rst
+++ b/doc/sphinx/reference/vcl.rst
@@ -56,7 +56,7 @@ You can use the *set* keyword to arbitrary HTTP headers. You can
 remove headers with the *remove* or *unset* keywords, which are
 synonym.
 
-The return(action) keyword terminates the subroutine. *action* can be,
+The ``return(action)`` keyword terminates the subroutine. *action* can be,
 depending on context one of
 
 * deliver



From tfheen at varnish-cache.org  Thu Jan 27 11:44:27 2011
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Thu, 27 Jan 2011 12:44:27 +0100
Subject: [master] 1ed1f4d Even more markup fixing
Message-ID: <E1PiQH5-0003R9-Je@project.varnish-software.com>

commit 1ed1f4dfca9ccfe133c0f4081be35b325b4e9e9f
Author: Tollef Fog Heen <tfheen at varnish-software.com>
Date:   Thu Jan 27 12:44:24 2011 +0100

    Even more markup fixing

diff --git a/doc/sphinx/reference/varnishtop.rst b/doc/sphinx/reference/varnishtop.rst
index 9d42b70..4c4899a 100644
--- a/doc/sphinx/reference/varnishtop.rst
+++ b/doc/sphinx/reference/varnishtop.rst
@@ -23,26 +23,26 @@ varnishtop [-1] [-b] [-C] [-c] [-d] [-f] [-I regex]
 DESCRIPTION
 ===========
 
-The varnishtop utility reads varnishd(1) shared memory logs and
+The varnishtop utility reads ``varnishd(1)`` shared memory logs and
 presents a continuously updated list of the most commonly occurring
-log entries.  With suitable filtering using the -I, -i, -X and -x
-options, it can be used to display a ranking of requested documents,
-clients, user agents, or any other information which is recorded in
-the log.
+log entries.  With suitable filtering using the ``-I``, ``-i``, ``-X``
+and ``-x`` options, it can be used to display a ranking of requested
+documents, clients, user agents, or any other information which is
+recorded in the log.
 
 The following options are available:
 
 -1          Instead of presenting of a continuously updated display, 
-	    print the statistics once and exit. Implies -d.
+	    print the statistics once and exit. Implies ``-d``.
 
 -b          Include log entries which result from communication 
-	    with a backend server.  If neither -b nor -c is
+	    with a backend server.  If neither ``-b`` nor ``-c`` is
 	    specified, varnishtop acts as if they both were.
 
 -C          Ignore case when matching regular expressions.
 
 -c          Include log entries which result from communication 
-	    with a client.  If neither -b nor -c is specified, 
+	    with a client.  If neither ``-b`` nor ``-c`` is specified, 
 	    varnishtop acts as if they both were.
 
 -d          Process old log entries on startup.  Normally, varnishtop 
@@ -54,14 +54,14 @@ The following options are available:
 	    where the first field is the client IP address.
 
 -I regex    Include log entries which match the specified regular 
-   	    expression.  If neither -I nor -i is specified, all log 
+   	    expression.  If neither ``-I`` nor ``-i`` is specified, all log 
 	    entries are included.
 
--i tag      Include log entries with the specified tag.  If neither -I 
-   	    nor -i is specified, all log entries are included.
+-i tag      Include log entries with the specified tag.  If neither ``-I``
+   	    nor ``-i`` is specified, all log entries are included.
 
 -n          Specifies the name of the varnishd instance to get logs from.  
-	    If -n is not specified, the host name is used.
+	    If ``-n`` is not specified, the host name is used.
 
 -r file     Read log entries from file instead of shared memory.
 



From phk at varnish-cache.org  Thu Jan 27 16:09:41 2011
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Thu, 27 Jan 2011 17:09:41 +0100
Subject: [master] 176190e Also retire the waitinglist structure if the number
	of sessions on it happened to be a multiplum of rush_exponent.
Message-ID: <E1PiUPl-0001aH-2W@project.varnish-software.com>

commit 176190e67641de430bbf0ddffe31d5b49cf670d6
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu Jan 27 16:08:49 2011 +0000

    Also retire the waitinglist structure if the number of sessions on
    it happened to be a multiplum of rush_exponent.
    
    Fixes:	#842

diff --git a/bin/varnishd/cache_hash.c b/bin/varnishd/cache_hash.c
index 39d416f..7266a56 100644
--- a/bin/varnishd/cache_hash.c
+++ b/bin/varnishd/cache_hash.c
@@ -485,11 +485,8 @@ hsh_rush(struct objhead *oh)
 	CHECK_OBJ_NOTNULL(wl, WAITINGLIST_MAGIC);
 	for (u = 0; u < params->rush_exponent; u++) {
 		sp = VTAILQ_FIRST(&wl->list);
-		if (sp == NULL) {
-			oh->waitinglist = NULL;
-			FREE_OBJ(wl);
-			return;
-		}
+		if (sp == NULL) 
+			break;
 		CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 		AZ(sp->wrk);
 		VTAILQ_REMOVE(&wl->list, sp, list);
@@ -503,6 +500,10 @@ hsh_rush(struct objhead *oh)
 			break;
 		}
 	}
+	if (VTAILQ_EMPTY(&wl->list)) {
+		oh->waitinglist = NULL;
+		FREE_OBJ(wl);
+	}
 }
 
 /*---------------------------------------------------------------------



From phk at varnish-cache.org  Thu Jan 27 17:03:20 2011
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Thu, 27 Jan 2011 18:03:20 +0100
Subject: [master] 61e8ab9 Various polishing
Message-ID: <E1PiVFg-0006ZD-D7@project.varnish-software.com>

commit 61e8ab914ab075c673ad8c5fb04f8ec1679b460b
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu Jan 27 17:02:53 2011 +0000

    Various polishing

diff --git a/bin/varnishd/cache_vrt_var.c b/bin/varnishd/cache_vrt_var.c
index 3a50f29..d0af3f3 100644
--- a/bin/varnishd/cache_vrt_var.c
+++ b/bin/varnishd/cache_vrt_var.c
@@ -322,7 +322,7 @@ VRT_r_beresp_storage(struct sess *sp)
 		return (NULL);
 }
 
-void
+void __match_proto__()
 VRT_l_beresp_storage(struct sess *sp, const char *str, ...)
 {
 	va_list ap;
diff --git a/bin/varnishd/mgt.h b/bin/varnishd/mgt.h
index 0abf533..2dd07bd 100644
--- a/bin/varnishd/mgt.h
+++ b/bin/varnishd/mgt.h
@@ -81,8 +81,8 @@ int mgt_vcc_default(const char *bflag, const char *f_arg, char *vcl, int Cflag);
 int mgt_push_vcls_and_start(unsigned *status, char **p);
 int mgt_has_vcl(void);
 extern char *mgt_cc_cmd;
-extern char *mgt_vcl_dir;
-extern char *mgt_vmod_dir;
+extern const char *mgt_vcl_dir;
+extern const char *mgt_vmod_dir;
 extern unsigned mgt_vcc_err_unref;
 
 #define REPORT0(pri, fmt)				\
diff --git a/bin/varnishd/mgt_vcc.c b/bin/varnishd/mgt_vcc.c
index 2ddec17..9c42393 100644
--- a/bin/varnishd/mgt_vcc.c
+++ b/bin/varnishd/mgt_vcc.c
@@ -67,8 +67,8 @@ struct vclprog {
 static VTAILQ_HEAD(, vclprog) vclhead = VTAILQ_HEAD_INITIALIZER(vclhead);
 
 char *mgt_cc_cmd;
-char *mgt_vcl_dir;
-char *mgt_vmod_dir;
+const char *mgt_vcl_dir;
+const char *mgt_vmod_dir;
 unsigned mgt_vcc_err_unref;
 
 static struct vcc *vcc;
diff --git a/bin/varnishd/varnishd.c b/bin/varnishd/varnishd.c
index d175c49..a6f1769 100644
--- a/bin/varnishd/varnishd.c
+++ b/bin/varnishd/varnishd.c
@@ -645,7 +645,7 @@ main(int argc, char * const *argv)
 	if (T_arg != NULL)
 		mgt_cli_telnet(T_arg);
 
-	VSM_Alloc(0, VSM_CLASS_MARK, "", "");
+	AN(VSM_Alloc(0, VSM_CLASS_MARK, "", ""));
 
 	MGT_Run();
 
diff --git a/include/binary_heap.h b/include/binary_heap.h
index 8876b9c..869ebf9 100644
--- a/include/binary_heap.h
+++ b/include/binary_heap.h
@@ -63,7 +63,7 @@ void binheap_insert(struct binheap *, void *);
 	 * Insert an item
 	 */
 
-void binheap_reorder(struct binheap *, unsigned idx);
+void binheap_reorder(const struct binheap *, unsigned idx);
 	/*
 	 * Move an order after changing its key value.
 	 */
diff --git a/lib/libvarnish/binary_heap.c b/lib/libvarnish/binary_heap.c
index 74310c3..53ff738 100644
--- a/lib/libvarnish/binary_heap.c
+++ b/lib/libvarnish/binary_heap.c
@@ -379,7 +379,7 @@ binheap_delete(struct binheap *bh, unsigned idx)
  */
 
 void
-binheap_reorder(struct binheap *bh, unsigned idx)
+binheap_reorder(const struct binheap *bh, unsigned idx)
 {
 
 	assert(bh != NULL);
diff --git a/lib/libvmod_std/vmod.py b/lib/libvmod_std/vmod.py
index 9798e7c..32fb536 100755
--- a/lib/libvmod_std/vmod.py
+++ b/lib/libvmod_std/vmod.py
@@ -230,7 +230,7 @@ dumps(pstruct + "} Vmod_Func_" + modname + ";\n")
 fc.write('\t;\n')
 fc.write("\n");
 
-fc.write('const char *Vmod_Spec[] = {\n' + slist + '\t0\n};\n')
+fc.write('const char * const Vmod_Spec[] = {\n' + slist + '\t0\n};\n')
 
 fc.write("\n")
 
diff --git a/lib/libvmod_std/vmod_std.c b/lib/libvmod_std/vmod_std.c
index e032905..c6a51ea 100644
--- a/lib/libvmod_std/vmod_std.c
+++ b/lib/libvmod_std/vmod_std.c
@@ -36,14 +36,14 @@
 
 #include "vcc_if.h"
 
-void
+void __match_proto__()
 vmod_set_ip_tos(struct sess *sp, int tos)
 {
 
 	TCP_Assert(setsockopt(sp->fd, IPPROTO_IP, IP_TOS, &tos, sizeof(tos)));
 }
 
-static const char *
+static const char * __match_proto__()
 vmod_updown(struct sess *sp, int up, const char *s, va_list ap)
 {
 	unsigned u;
@@ -79,7 +79,7 @@ vmod_updown(struct sess *sp, int up, const char *s, va_list ap)
 	}
 }
 
-const char *
+const char * __match_proto__()
 vmod_toupper(struct sess *sp, struct vmod_priv *priv, const char *s, ...)
 {
 	const char *p;
@@ -98,7 +98,7 @@ vmod_toupper(struct sess *sp, struct vmod_priv *priv, const char *s, ...)
 	return (p);
 }
 
-const char *
+const char * __match_proto__()
 vmod_tolower(struct sess *sp, struct vmod_priv *priv, const char *s, ...)
 {
 	const char *p;
@@ -135,7 +135,7 @@ vmod_random(struct sess *sp, double lo, double hi)
 	return (a);
 }
 
-void
+void __match_proto__()
 vmod_log(struct sess *sp, const char *fmt, ...)
 {
 	char buf[8192], *p;



From phk at varnish-cache.org  Thu Jan 27 18:19:30 2011
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Thu, 27 Jan 2011 19:19:30 +0100
Subject: [master] 5385a47 Report where in the gzip stream we found trouble
Message-ID: <E1PiWRO-0004jU-0o@project.varnish-software.com>

commit 5385a47c1620f98e9a2972f5f4b6dbb84786b5bd
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu Jan 27 18:16:30 2011 +0000

    Report where in the gzip stream we found trouble

diff --git a/bin/varnishtest/vtc_http.c b/bin/varnishtest/vtc_http.c
index c9eb09a..ec50a33 100644
--- a/bin/varnishtest/vtc_http.c
+++ b/bin/varnishtest/vtc_http.c
@@ -555,7 +555,8 @@ cmd_http_gunzip_body(CMD_ARGS)
 	assert(Z_OK == inflateInit2(&vz, 31));
 	i = inflate(&vz, Z_FINISH);
 	if (i != Z_STREAM_END)
-		vtc_log(hp->vl, 0, "Gunzip error = %d (%s)", i, vz.msg);
+		vtc_log(hp->vl, 0, "Gunzip error = %d (%s) in:%jd out:%jd",
+		    i, vz.msg, (intmax_t)vz.total_in, (intmax_t)vz.total_out);
 	hp->bodyl = vz.total_out;
 	memcpy(hp->body, p, hp->bodyl);
 	free(p);



From phk at varnish-cache.org  Thu Jan 27 18:19:30 2011
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Thu, 27 Jan 2011 19:19:30 +0100
Subject: [master] 9e69e5b In the absence of ordering comparisons,
	use unequality to make sure the number is 3 or higher
Message-ID: <E1PiWRO-0004jc-EL@project.varnish-software.com>

commit 9e69e5bdcbcabbd09ad89f7aea6f280c57a764be
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu Jan 27 18:16:49 2011 +0000

    In the absence of ordering comparisons, use unequality to make sure
    the number is 3 or higher

diff --git a/bin/varnishtest/tests/g00002.vtc b/bin/varnishtest/tests/g00002.vtc
index 4ba8ffa..b12e8db 100644
--- a/bin/varnishtest/tests/g00002.vtc
+++ b/bin/varnishtest/tests/g00002.vtc
@@ -31,7 +31,9 @@ client c1 {
 } -run
 
 # If this fails, the multiple storage allocations did not happen
-varnish v1 -expect SMF.s0.nreq == 3
+varnish v1 -expect SMF.s0.nreq != 0
+varnish v1 -expect SMF.s0.nreq != 1
+varnish v1 -expect SMF.s0.nreq != 2
 
 client c1 {
 	# See varnish can gunzip it.



From phk at varnish-cache.org  Thu Jan 27 18:19:30 2011
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Thu, 27 Jan 2011 19:19:30 +0100
Subject: [master] 068412d Add a VGZ_ObufFull() function
Message-ID: <E1PiWRO-0004jp-Mt@project.varnish-software.com>

commit 068412de9a427229879e553b23052650f997333b
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu Jan 27 18:17:57 2011 +0000

    Add a VGZ_ObufFull() function

diff --git a/bin/varnishd/cache.h b/bin/varnishd/cache.h
index 9116946..980ab6f 100644
--- a/bin/varnishd/cache.h
+++ b/bin/varnishd/cache.h
@@ -643,6 +643,7 @@ struct vgz *VGZ_NewGzip(struct sess *sp);
 void VGZ_Ibuf(struct vgz *, const void *, ssize_t len);
 int VGZ_IbufEmpty(const struct vgz *vg);
 void VGZ_Obuf(struct vgz *, const void *, ssize_t len);
+int VGZ_ObufFull(const struct vgz *vg);
 int VGZ_ObufStorage(const struct sess *sp, struct vgz *vg);
 int VGZ_Gzip(struct vgz *, const void **, size_t *len, enum vgz_flag);
 int VGZ_Gunzip(struct vgz *, const void **, size_t *len);
diff --git a/bin/varnishd/cache_gzip.c b/bin/varnishd/cache_gzip.c
index 930d7f1..5177d01 100644
--- a/bin/varnishd/cache_gzip.c
+++ b/bin/varnishd/cache_gzip.c
@@ -235,6 +235,14 @@ VGZ_Obuf(struct vgz *vg, const void *ptr, ssize_t len)
 	vg->vz.avail_out = len;
 }
 
+int
+VGZ_ObufFull(const struct vgz *vg)
+{
+
+	CHECK_OBJ_NOTNULL(vg, VGZ_MAGIC);
+	return (vg->vz.avail_out == 0);
+}
+
 /*--------------------------------------------------------------------
  * Keep the outbuffer supplied with storage and file it under the
  * sp->obj as it fills.



From phk at varnish-cache.org  Thu Jan 27 18:19:31 2011
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Thu, 27 Jan 2011 19:19:31 +0100
Subject: [master] 8d1d603 If we call VGZ_Gzip() with flags different from
	VGZ_NORMAL and it fills the output buffer,
	we have to call it again with same flags and more output space. (see
	somewhere in vgz.h)
Message-ID: <E1PiWRP-0004jz-0G@project.varnish-software.com>

commit 8d1d603dd686878e4a54c55b876cead575b1e402
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu Jan 27 18:18:15 2011 +0000

    If we call VGZ_Gzip() with flags different from VGZ_NORMAL and it
    fills the output buffer, we have to call it again with same flags
    and more output space. (see somewhere in vgz.h)

diff --git a/bin/varnishd/cache_esi_fetch.c b/bin/varnishd/cache_esi_fetch.c
index ff36f28..9369d0a 100644
--- a/bin/varnishd/cache_esi_fetch.c
+++ b/bin/varnishd/cache_esi_fetch.c
@@ -193,7 +193,8 @@ vfp_vep_callback(const struct sess *sp, ssize_t l, enum vgz_flag flg)
 			i = VGZ_Gzip(vef->vgz, &dp, &dl, flg);
 			vef->tot += dl;
 			sp->obj->len += dl;
-		} while (!VGZ_IbufEmpty(vef->vgz));
+		} while (!VGZ_IbufEmpty(vef->vgz) ||
+		    (flg != VGZ_NORMAL && VGZ_ObufFull(vef->vgz)));
 		if (px != 0) {
 			memmove(vef->pending, vef->pending + px,
 			    vef->npend - px);
@@ -283,7 +284,8 @@ vfp_esi_bytes_gg(struct sess *sp, struct http_conn *htc, size_t bytes)
 			if (vef->bufp < ibuf2 + dl) {
 				dl = (ibuf2 + dl) - vef->bufp;
 				assert(dl + vef->npend < sizeof vef->pending);
-				memmove(vef->pending + vef->npend, vef->bufp, dl);
+				memmove(vef->pending + vef->npend,
+				    vef->bufp, dl);
 				vef->npend += dl;
 			}
 		} while (!VGZ_IbufEmpty(sp->wrk->vgz_rx));



From tfheen at varnish-cache.org  Fri Jan 28 13:52:32 2011
From: tfheen at varnish-cache.org (Tollef Fog Heen)
Date: Fri, 28 Jan 2011 14:52:32 +0100
Subject: [master] a776169 Check for __attribute__((visibility)) support
Message-ID: <E1Pioka-0000iC-Fb@project.varnish-software.com>

commit a776169645967061d22ed87bf26236ca6d6f9488
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Fri Jan 28 14:50:41 2011 +0100

    Check for __attribute__((visibility)) support
    
    The solaris compiler does not support the GNU C extension
    __attribute__((visibility("hidden"))), leading to build failures.
    This commit adds a check for that support and defines NO_VIZ if it's
    visibility setting is not supported.
    
    Fixes #852

diff --git a/configure.ac b/configure.ac
index 2e79d93..7b17ba6 100644
--- a/configure.ac
+++ b/configure.ac
@@ -209,6 +209,30 @@ if test "$ac_cv_so_sendfile_works" = yes; then
 	AC_DEFINE([SENDFILE_WORKS], [1], [Define if SENDFILE works])
 fi
 
+# Support for visibility attribute 
+save_CFLAGS="${CFLAGS}" 
+CFLAGS="${CFLAGS} -Werror" 
+
+AC_CACHE_CHECK([whether we have support for visibility attributes], 
+	[ac_cv_have_viz], 
+	[AC_RUN_IFELSE( 
+		[AC_LANG_PROGRAM([[ 
+			#if ((__GNUC__-0) * 10 + __GNUC_MINOR__-0 >= 33) 
+			#  define ZLIB_INTERNAL __attribute__((visibility ("hidden"))) 
+			#else 
+			#  define ZLIB_INTERNAL 
+			#endif 
+			int ZLIB_INTERNAL foo; 
+		]],[])], 
+	[ac_cv_have_viz=yes], 
+	[ac_cv_have_viz=no]) 
+])
+if test "$ac_cv_have_viz" = no; then 
+	libvgz_extra_cflags="-DNO_VIZ" 
+	AC_SUBST(libvgz_extra_cflags) 
+fi 
+CFLAGS="${save_CFLAGS}" 
+
 # Userland slab allocator, available only on Solaris
 case $target in
 *-*-solaris*)
diff --git a/lib/libvgz/Makefile.am b/lib/libvgz/Makefile.am
index 573e033..ab9b561 100644
--- a/lib/libvgz/Makefile.am
+++ b/lib/libvgz/Makefile.am
@@ -3,7 +3,7 @@
 lib_LTLIBRARIES = libvgz.la
 
 libvgz_la_LDFLAGS = -version-info 1:0:0
-libvgz_la_CFLAGS = -D_LARGEFILE64_SOURCE=1
+libvgz_la_CFLAGS = -D_LARGEFILE64_SOURCE=1 $(libvgz_extra_cflags)
 
 libvgz_la_SOURCES = \
 	adler32.c \



From phk at varnish-cache.org  Fri Jan 28 14:53:24 2011
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Fri, 28 Jan 2011 15:53:24 +0100
Subject: [master] 30f4123 Add a assert handler which tries to assign and
	report the assert to the failing threads vtclog.
Message-ID: <E1PiphU-0007aL-Tf@project.varnish-software.com>

commit 30f4123c25d60f76737580ab08dee51be549b659
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Fri Jan 28 14:37:24 2011 +0000

    Add a assert handler which tries to assign and report the assert
    to the failing threads vtclog.

diff --git a/bin/varnishtest/vtc_log.c b/bin/varnishtest/vtc_log.c
index 01320cc..08e683e 100644
--- a/bin/varnishtest/vtc_log.c
+++ b/bin/varnishtest/vtc_log.c
@@ -41,6 +41,7 @@ SVNID("$Id$")
 #include "libvarnish.h"
 #include "vsb.h"
 #include "miniobj.h"
+#include "vas.h"
 
 #include "vtc.h"
 
@@ -56,6 +57,8 @@ struct vtclog {
 	pthread_mutex_t	mtx;
 };
 
+static pthread_key_t log_key;
+
 /**********************************************************************/
 
 void
@@ -65,6 +68,7 @@ vtc_loginit(char *buf, unsigned buflen)
 	vtclog_buf = buf;
 	vtclog_left = buflen;
 	AZ(pthread_mutex_init(&vtclog_mtx, NULL));
+	AZ(pthread_key_create(&log_key, NULL));
 }
 
 /**********************************************************************/
@@ -80,6 +84,7 @@ vtc_logopen(const char *id)
 	vl->id = id;
 	vl->vsb = vsb_newauto();
 	AZ(pthread_mutex_init(&vl->mtx, NULL));
+	AZ(pthread_setspecific(log_key, vl));
 	return (vl);
 }
 
@@ -274,3 +279,27 @@ vtc_hexdump(struct vtclog *vl, unsigned lvl, const char *pfx, const unsigned cha
 			pthread_exit(NULL);
 	}
 }
+
+/**********************************************************************/
+
+static void
+vtc_log_vas_fail(const char *func, const char *file, int line,
+    const char *cond, int err, int xxx)
+{
+	struct vtclog *vl;
+
+	(void)err;
+	(void)xxx;
+	vl = pthread_getspecific(log_key);
+	if (vl == NULL) {
+		fprintf(stderr,
+		    "Assert error in %s(), %s line %d:\n"
+		    "  Condition(%s) not true.\n",
+		    func, file, line, cond);
+	} else {
+		vtc_log(vl, 0, "Assert error in %s(), %s line %d:"
+		    "  Condition(%s) not true.\n", func, file, line, cond);
+	}
+}
+
+vas_f *vas_fail = vtc_log_vas_fail;



From phk at varnish-cache.org  Fri Jan 28 14:53:25 2011
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Fri, 28 Jan 2011 15:53:25 +0100
Subject: [master] c50858f Wait 10 seconds for a client connection
Message-ID: <E1PiphV-0007aT-5L@project.varnish-software.com>

commit c50858f2b73292a62e92fe77c4b68dc3051de775
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Fri Jan 28 14:53:11 2011 +0000

    Wait 10 seconds for a client connection

diff --git a/bin/varnishtest/vtc_client.c b/bin/varnishtest/vtc_client.c
index abdd279..b93dcd3 100644
--- a/bin/varnishtest/vtc_client.c
+++ b/bin/varnishtest/vtc_client.c
@@ -77,7 +77,6 @@ client_thread(void *priv)
 	struct client *c;
 	struct vtclog *vl;
 	int fd;
-	int i;
 	unsigned u;
 	struct vsb *vsb;
 	char *p;
@@ -99,14 +98,11 @@ client_thread(void *priv)
 		vtc_log(vl, 2, "Started (%u iterations)", c->repeat);
 	for (u = 0; u < c->repeat; u++) {
 		vtc_log(vl, 3, "Connect to %s", vsb_data(vsb));
-		fd = VSS_open(vsb_data(vsb), 0);
-		for (i = 0; fd < 0 && i < 3; i++) {
-			(void)sleep(1);
-			fd = VSS_open(vsb_data(vsb), 0);
-		}
+		fd = VSS_open(vsb_data(vsb), 10.);
 		if (fd < 0)
 			vtc_log(c->vl, 0, "Failed to open %s", vsb_data(vsb));
 		assert(fd >= 0);
+		TCP_blocking(fd);
 		TCP_myname(fd, mabuf, sizeof mabuf, mpbuf, sizeof mpbuf);
 		vtc_log(vl, 3, "connected fd %d from %s %s to %s",
 		    fd, mabuf, mpbuf, vsb_data(vsb));



From phk at varnish-cache.org  Fri Jan 28 21:38:37 2011
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Fri, 28 Jan 2011 22:38:37 +0100
Subject: [master] e0db8b0 Retire the obj.cacheable VCL variable. Whatever our
	thinking at the time might have been, it clearly was woollen,
	and all it did was confuse people.
Message-ID: <E1Piw1d-0000n1-0P@project.varnish-software.com>

commit e0db8b06a229058bb759962cb3a3db5e14828e25
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Fri Jan 28 21:37:38 2011 +0000

    Retire the obj.cacheable VCL variable.  Whatever our thinking at the
    time might have been, it clearly was woollen, and all it did was
    confuse people.

diff --git a/bin/varnishd/cache.h b/bin/varnishd/cache.h
index 980ab6f..c2051b3 100644
--- a/bin/varnishd/cache.h
+++ b/bin/varnishd/cache.h
@@ -432,7 +432,6 @@ struct object {
 	unsigned		response;
 
 	/* XXX: make bitmap */
-	unsigned		cacheable;
 	unsigned		gziped;
 	/* Bit positions in the gzip stream */
 	ssize_t			gzip_start;
diff --git a/bin/varnishd/cache_ban.c b/bin/varnishd/cache_ban.c
index ab06261..50d29e3 100644
--- a/bin/varnishd/cache_ban.c
+++ b/bin/varnishd/cache_ban.c
@@ -476,7 +476,6 @@ ban_check_object(struct object *o, const struct sess *sp, int has_req)
 		return (0);
 	} else {
 		o->ttl = 0;
-		o->cacheable = 0;
 		oc->ban = NULL;
 		oc_updatemeta(oc);
 		/* BAN also changed, but that is not important any more */
diff --git a/bin/varnishd/cache_center.c b/bin/varnishd/cache_center.c
index 08458c6..fc8de44 100644
--- a/bin/varnishd/cache_center.c
+++ b/bin/varnishd/cache_center.c
@@ -667,10 +667,7 @@ cnt_fetch(struct sess *sp)
 
 	sp->obj->xid = sp->xid;
 	sp->obj->response = sp->err_code;
-	sp->obj->cacheable = sp->wrk->cacheable;
 	sp->obj->grace = sp->wrk->grace;
-	if (sp->obj->ttl == 0. && sp->obj->grace == 0.)
-		sp->obj->cacheable = 0;
 	sp->obj->age = sp->wrk->age;
 	sp->obj->entered = sp->wrk->entered;
 	WS_Assert(sp->obj->ws_o);
@@ -733,7 +730,6 @@ cnt_fetch(struct sess *sp)
 		WRONG("Illegal action in vcl_fetch{}");
 	}
 
-	sp->obj->cacheable = 1;
 	if (sp->wrk->cacheable) {
 		EXP_Insert(sp->obj);
 		AN(sp->obj->objcore);
diff --git a/bin/varnishd/cache_expire.c b/bin/varnishd/cache_expire.c
index 19ec70c..3afadea 100644
--- a/bin/varnishd/cache_expire.c
+++ b/bin/varnishd/cache_expire.c
@@ -138,7 +138,6 @@ EXP_Insert(struct object *o)
 	oc = o->objcore;
 	CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
 	AssertObjBusy(o);
-	assert(o->cacheable);
 	HSH_Ref(oc);
 
 	assert(o->entered != 0 && !isnan(o->entered));
diff --git a/bin/varnishd/cache_hash.c b/bin/varnishd/cache_hash.c
index 7266a56..3694a75 100644
--- a/bin/varnishd/cache_hash.c
+++ b/bin/varnishd/cache_hash.c
@@ -362,8 +362,6 @@ HSH_Lookup(struct sess *sp, struct objhead **poh)
 		o = oc_getobj(sp->wrk, oc);
 		CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC);
 
-		if (!o->cacheable)
-			continue;
 		if (o->ttl == 0)
 			continue;
 		if (BAN_CheckObject(o, sp))
@@ -579,7 +577,6 @@ HSH_Drop(struct sess *sp)
 	o = sp->obj;
 	CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC);
 	AssertObjPassOrBusy(o);
-	o->cacheable = 0;
 	o->ttl = 0;
 	if (o->objcore != NULL)		/* Pass has no objcore */
 		HSH_Unbusy(sp);
diff --git a/bin/varnishd/cache_vrt_var.c b/bin/varnishd/cache_vrt_var.c
index d0af3f3..44f7ec3 100644
--- a/bin/varnishd/cache_vrt_var.c
+++ b/bin/varnishd/cache_vrt_var.c
@@ -367,28 +367,6 @@ VRT_r_obj_ttl(const struct sess *sp)
 	return (sp->obj->ttl - sp->t_req);
 }
 
-
-/*--------------------------------------------------------------------*/
-
-#define VOBJ(type,onm,field)						\
-void									\
-VRT_l_obj_##onm(const struct sess *sp, type a)				\
-{									\
-	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);				\
-	CHECK_OBJ_NOTNULL(sp->obj, OBJECT_MAGIC);	/* XXX */	\
-	sp->obj->field = a;						\
-}									\
-									\
-type									\
-VRT_r_obj_##onm(const struct sess *sp)					\
-{									\
-	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);				\
-	CHECK_OBJ_NOTNULL(sp->obj, OBJECT_MAGIC);	/* XXX */	\
-	return (sp->obj->field);					\
-}
-
-VOBJ(unsigned, cacheable, cacheable)
-
 /*--------------------------------------------------------------------*/
 
 void
diff --git a/bin/varnishd/default.vcl b/bin/varnishd/default.vcl
index a1a7587..350e0a7 100644
--- a/bin/varnishd/default.vcl
+++ b/bin/varnishd/default.vcl
@@ -94,9 +94,6 @@ sub vcl_hash {
 }
 
 sub vcl_hit {
-    if (!obj.cacheable) {
-        return (pass);
-    }
     return (deliver);
 }
 
diff --git a/bin/varnishtest/tests/r00365.vtc b/bin/varnishtest/tests/r00365.vtc
index f7daf85..33ad869 100644
--- a/bin/varnishtest/tests/r00365.vtc
+++ b/bin/varnishtest/tests/r00365.vtc
@@ -13,7 +13,7 @@ server s1 {
 
 varnish v1 -vcl+backend { 
 	sub vcl_hit {
-		set obj.cacheable = false;
+		set obj.ttl = 0s;
 		return (restart);
 	}
 } -start
diff --git a/bin/varnishtest/tests/r00444.vtc b/bin/varnishtest/tests/r00444.vtc
index 2ee3456..c6f3b3b 100644
--- a/bin/varnishtest/tests/r00444.vtc
+++ b/bin/varnishtest/tests/r00444.vtc
@@ -30,7 +30,6 @@ varnish v1 -vcl+backend {
 	sub vcl_hit {
 		if (req.request == "POST") {
 			/* Get rid of this object */
-			set obj.cacheable = false;
 			set obj.ttl = 0s;
 			return (pass);
 		}
diff --git a/bin/varnishtest/tests/v00018.vtc b/bin/varnishtest/tests/v00018.vtc
index 1ec304a..13e39e0 100644
--- a/bin/varnishtest/tests/v00018.vtc
+++ b/bin/varnishtest/tests/v00018.vtc
@@ -63,27 +63,27 @@ varnish v1 -vcl {
 
 varnish v1 -badvcl {
 	backend b { .host = "127.0.0.1"; }
-	sub vcl_fetch { set obj.cacheable += 1; }
+	sub vcl_fetch { set beresp.do_gzip += 1; }
 }
 
 varnish v1 -vcl {
 	backend b { .host = "127.0.0.1"; }
-	sub vcl_fetch { set beresp.cacheable = true; }
+	sub vcl_fetch { set beresp.do_gzip = true; }
 }
 
 varnish v1 -vcl {
 	backend b { .host = "127.0.0.1"; }
-	sub vcl_fetch { set beresp.cacheable = false; }
+	sub vcl_fetch { set beresp.do_gzip = false; }
 }
 
 varnish v1 -badvcl {
 	backend b { .host = "127.0.0.1"; }
-	sub vcl_fetch { set beresp.cacheable = mu; }
+	sub vcl_fetch { set beresp.do_gzip = mu; }
 }
 
 varnish v1 -badvcl {
 	backend b { .host = "127.0.0.1"; }
-	sub vcl_fetch { unset beresp.cacheable; }
+	sub vcl_fetch { unset beresp.do_gzip; }
 }
 
 varnish v1 -badvcl {
diff --git a/bin/varnishtest/vtc_varnish.c b/bin/varnishtest/vtc_varnish.c
index dd59251..56c6f4d 100644
--- a/bin/varnishtest/vtc_varnish.c
+++ b/bin/varnishtest/vtc_varnish.c
@@ -416,7 +416,9 @@ varnish_stop(struct varnish *v)
 	vtc_log(v->vl, 2, "Stop");
 	(void)varnish_ask_cli(v, "stop", NULL);
 	while (1) {
+		r = NULL;
 		(void)varnish_ask_cli(v, "status", &r);
+		AN(r);
 		if (!strcmp(r, "Child in state stopped"))
 			break;
 		free(r);
diff --git a/lib/libvcl/generate.py b/lib/libvcl/generate.py
index 7b0e9fa..922ca94 100755
--- a/lib/libvcl/generate.py
+++ b/lib/libvcl/generate.py
@@ -382,12 +382,6 @@ sp_variables = (
 		( 'error',),		# XXX ?
 		'const struct sess *'
 	),
-	('obj.cacheable',
-		'BOOL',
-		( 'hit',),
-		( 'hit',),
-		'const struct sess *'
-	),
 	('obj.ttl',
 		'DURATION',
 		( 'hit', 'error',),



From phk at varnish-cache.org  Fri Jan 28 21:46:56 2011
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Fri, 28 Jan 2011 22:46:56 +0100
Subject: [master] 71c18e5 Use the Grace-defaulting function we already have.
Message-ID: <E1Piw9g-0001vy-E1@project.varnish-software.com>

commit 71c18e53daf8673a8f678bb041c3e895e650d7ef
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Fri Jan 28 21:46:40 2011 +0000

    Use the Grace-defaulting function we already have.

diff --git a/bin/varnishd/cache_vrt_var.c b/bin/varnishd/cache_vrt_var.c
index 44f7ec3..a111e0b 100644
--- a/bin/varnishd/cache_vrt_var.c
+++ b/bin/varnishd/cache_vrt_var.c
@@ -450,9 +450,7 @@ VRT_r_##which##_grace(struct sess *sp)				\
 {								\
 								\
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);			\
-	if (isnan(fld))						\
-		 return ((double)params->default_grace);	\
-	return(fld);						\
+	return(HSH_Grace(fld));					\
 }
 
 VRT_DO_GRACE(req, sp->grace, )



From phk at varnish-cache.org  Fri Jan 28 22:44:19 2011
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Fri, 28 Jan 2011 23:44:19 +0100
Subject: [master] 00b606f Eliminate beresp.cachable VCL variable. Cacheability
	is now controlled solely through the .ttl and .grace variables.
Message-ID: <E1Pix3D-0000ji-JY@project.varnish-software.com>

commit 00b606f3f9704eaeb4fcd408a547864b41c753b1
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Fri Jan 28 22:42:07 2011 +0000

    Eliminate beresp.cachable VCL variable.  Cacheability is now controlled
    solely through the .ttl and .grace variables.
    
    Add a parameter "shortlived", objects with ttl shorter than this
    always go into transient (malloc) storage, just like pass objects do.
    
    Enforce a minimum 1sec TTL for hit-for-pass.

diff --git a/bin/varnishd/cache.h b/bin/varnishd/cache.h
index c2051b3..f997341 100644
--- a/bin/varnishd/cache.h
+++ b/bin/varnishd/cache.h
@@ -104,6 +104,8 @@ struct lock { void *priv; };		// Opaque
 
 #define DIGEST_LEN		32
 
+/* Name of transient storage */
+#define TRANSIENT_STORAGE	"Transient"
 
 /*--------------------------------------------------------------------
  * Pointer aligment magic
@@ -268,14 +270,13 @@ struct worker {
 	struct http		*beresp;
 	struct http		*resp;
 
-	unsigned		cacheable;
 	double			age;
 	double			entered;
 	double			ttl;
 	double			grace;
 
 	/* This is only here so VRT can find it */
-	char			*storage_hint;
+	const char		*storage_hint;
 
 	/* Fetch stuff */
 	enum body_status	body_status;
diff --git a/bin/varnishd/cache_center.c b/bin/varnishd/cache_center.c
index fc8de44..6595a98 100644
--- a/bin/varnishd/cache_center.c
+++ b/bin/varnishd/cache_center.c
@@ -379,7 +379,6 @@ cnt_error(struct sess *sp)
 	w = sp->wrk;
 	if (sp->obj == NULL) {
 		HSH_Prealloc(sp);
-		sp->wrk->cacheable = 0;
 		/* XXX: 1024 is a pure guess */
 		sp->obj = STV_NewObject(sp, NULL, 1024, 0,
 		     params->http_headers);
@@ -525,6 +524,13 @@ cnt_fetch(struct sess *sp)
 	sp->err_code = http_GetStatus(sp->wrk->beresp);
 
 	/*
+	 * What does RFC2616 think about TTL ?
+	 */
+	sp->wrk->entered = TIM_real();
+	sp->wrk->age = 0;
+	sp->wrk->ttl = RFC2616_Ttl(sp);
+
+	/*
 	 * Initial cacheability determination per [RFC2616, 13.4]
 	 * We do not support ranges yet, so 206 is out.
 	 */
@@ -536,19 +542,15 @@ cnt_fetch(struct sess *sp)
 	case 302: /* Moved Temporarily */
 	case 410: /* Gone */
 	case 404: /* Not Found */
-		sp->wrk->cacheable = 1;
 		break;
 	default:
-		sp->wrk->cacheable = 0;
+		sp->wrk->ttl = sp->t_req - 1.;
 		break;
 	}
 
-	sp->wrk->entered = TIM_real();
-	sp->wrk->age = 0;
-	sp->wrk->ttl = RFC2616_Ttl(sp);
-
+	/* pass from vclrecv{} has negative TTL */
 	if (sp->objcore == NULL)
-		sp->wrk->cacheable = 0;
+		sp->wrk->ttl = sp->t_req - 1.;
 
 	sp->wrk->do_esi = 0;
 	sp->wrk->grace = NAN;
@@ -561,11 +563,17 @@ cnt_fetch(struct sess *sp)
 
 	if (sp->objcore == NULL) {
 		/* This is a pass from vcl_recv */
-		sp->wrk->cacheable = 0;
 		pass = 1;
-	} else if (sp->handling == VCL_RET_PASS || !sp->wrk->cacheable) {
+		/* VCL may have fiddled this, but that doesn't help */
+		sp->wrk->ttl = sp->t_req - 1.;
+	} else if (sp->handling == VCL_RET_PASS) {
 		/* pass from vcl_fetch{} -> hit-for-pass */
+		/* XXX: the bereq was not filtered pass... */
 		pass = 1;
+		sp->objcore->flags |= OC_F_PASS;
+		/* Enforce a minimum TTL of 1 sec (if set from VCL) */
+		if (sp->wrk->ttl <= sp->t_req)
+			sp->wrk->ttl = sp->wrk->entered + params->default_ttl;
 	} else {
 		/* regular object */
 		pass = 0;
@@ -633,7 +641,7 @@ cnt_fetch(struct sess *sp)
 	    pass ? HTTPH_R_PASS : HTTPH_A_INS, &nhttp);
 
 	/* Create Vary instructions */
-	if (sp->wrk->cacheable) {
+	if (sp->objcore != NULL) {
 		CHECK_OBJ_NOTNULL(sp->objcore, OBJCORE_MAGIC);
 		vary = VRY_Create(sp, sp->wrk->beresp);
 		if (vary != NULL) {
@@ -649,11 +657,18 @@ cnt_fetch(struct sess *sp)
 	 */
 	l += strlen("Content-Encoding: XxxXxxXxxXxxXxxXxx" + sizeof(void *));
 
+	if (sp->wrk->ttl < sp->t_req + params->shortlived ||
+	    sp->objcore == NULL) 
+		sp->wrk->storage_hint = TRANSIENT_STORAGE;
+
 	sp->obj = STV_NewObject(sp, sp->wrk->storage_hint, l,
 	    sp->wrk->ttl, nhttp);
+	/* XXX: -> 513 */
 	CHECK_OBJ_NOTNULL(sp->obj, OBJECT_MAGIC);
 
 	sp->wrk->storage_hint = NULL;
+
+	/* VFP will update as needed */
 	sp->obj->gziped = sp->wrk->is_gzip;
 
 	if (vary != NULL) {
@@ -687,7 +702,6 @@ cnt_fetch(struct sess *sp)
 	else
 		sp->obj->last_modified = sp->wrk->entered;
 
-
 	/* Use unmodified headers*/
 	i = FetchBody(sp, sp->wrk->beresp1);
 
@@ -715,11 +729,6 @@ cnt_fetch(struct sess *sp)
 		sp->step = STP_RECV;
 		return (0);
 	case VCL_RET_PASS:
-		if (sp->obj->objcore != NULL)
-			sp->obj->objcore->flags |= OC_F_PASS;
-		if (sp->obj->ttl - sp->t_req < params->default_ttl)
-			sp->obj->ttl = sp->t_req + params->default_ttl;
-		break;
 	case VCL_RET_DELIVER:
 		break;
 	case VCL_RET_ERROR:
@@ -730,7 +739,7 @@ cnt_fetch(struct sess *sp)
 		WRONG("Illegal action in vcl_fetch{}");
 	}
 
-	if (sp->wrk->cacheable) {
+	if (sp->obj->objcore != NULL) {
 		EXP_Insert(sp->obj);
 		AN(sp->obj->objcore);
 		AN(sp->obj->objcore->ban);
diff --git a/bin/varnishd/cache_vrt_var.c b/bin/varnishd/cache_vrt_var.c
index a111e0b..cc0d11e 100644
--- a/bin/varnishd/cache_vrt_var.c
+++ b/bin/varnishd/cache_vrt_var.c
@@ -195,7 +195,6 @@ VRT_r_##dir##_##onm(const struct sess *sp)				\
 	return (sp->wrk->field);					\
 }
 
-VBERESP(beresp, unsigned, cacheable, cacheable)
 VBERESP(beresp, unsigned, do_esi, do_esi)
 VBERESP(beresp, unsigned, do_gzip, do_gzip)
 VBERESP(beresp, unsigned, do_gunzip, do_gunzip)
diff --git a/bin/varnishd/default.vcl b/bin/varnishd/default.vcl
index 350e0a7..6093780 100644
--- a/bin/varnishd/default.vcl
+++ b/bin/varnishd/default.vcl
@@ -102,7 +102,7 @@ sub vcl_miss {
 }
 
 sub vcl_fetch {
-    if (!beresp.cacheable) {
+    if (beresp.ttl <= 0s) {
         return (pass);
     }
     if (beresp.http.Set-Cookie) {
diff --git a/bin/varnishd/heritage.h b/bin/varnishd/heritage.h
index 9c5a328..4b902a5 100644
--- a/bin/varnishd/heritage.h
+++ b/bin/varnishd/heritage.h
@@ -205,6 +205,8 @@ struct params {
 	unsigned		gzip_level;
 
 	double			critbit_cooloff;
+
+	double			shortlived;
 };
 
 /*
diff --git a/bin/varnishd/mgt_param.c b/bin/varnishd/mgt_param.c
index f46d3ca..4ba6554 100644
--- a/bin/varnishd/mgt_param.c
+++ b/bin/varnishd/mgt_param.c
@@ -842,6 +842,12 @@ static const struct parspec input_parspec[] = {
 		" just a waste of memory.",
 		EXPERIMENTAL,
 		"32768", "Bytes" },
+	{ "shortlived", tweak_timeout_double,
+		&master.shortlived, 0, UINT_MAX,
+		"Objects created with TTL shorter than this are always "
+		"put in transient storage.\n",
+		0,
+		"10.0", "s" },
 	{ "critbit_cooloff", tweak_timeout_double,
 		&master.critbit_cooloff, 60, 254,
 		"How long time the critbit hasher keeps deleted objheads "
diff --git a/bin/varnishd/stevedore.c b/bin/varnishd/stevedore.c
index 81ce7a4..8e4d0c5 100644
--- a/bin/varnishd/stevedore.c
+++ b/bin/varnishd/stevedore.c
@@ -47,8 +47,6 @@ SVNID("$Id$")
 #include "cli_priv.h"
 #include "vrt_obj.h"
 
-#define TRANSIENT_NAME	"Transient"
-
 static VTAILQ_HEAD(, stevedore)	stevedores =
     VTAILQ_HEAD_INITIALIZER(stevedores);
 
@@ -91,7 +89,7 @@ stv_pick_stevedore(const char *hint)
 			if (!strcmp(stv->ident, hint))
 				return (stv);
 		}
-		if (!strcmp(TRANSIENT_NAME, hint))
+		if (!strcmp(TRANSIENT_STORAGE, hint))
 			return (stv_transient);
 	}
 	/* pick a stevedore and bump the head along */
@@ -268,10 +266,7 @@ STV_NewObject(struct sess *sp, const char *hint, unsigned wsl, double ttl,
 
 	ltot = sizeof *o + wsl + lhttp;
 
-	if (!sp->wrk->cacheable)
-		stv = stv_transient;
-	else
-		stv = stv_pick_stevedore(hint);
+	stv = stv_pick_stevedore(hint);
 	AN(stv->allocobj);
 	o = stv->allocobj(stv, sp, ltot, &soc);
 	CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC);
@@ -477,7 +472,7 @@ STV_Config(const char *spec)
 	else if (ac != 0)
 		ARGV_ERR("(-s%s) too many arguments\n", stv->name);
 
-	if (!strcmp(stv->ident, TRANSIENT_NAME)) {
+	if (!strcmp(stv->ident, TRANSIENT_STORAGE)) {
 		stv->transient = 1;
 		AZ(stv_transient);
 		stv_transient = stv;
@@ -497,9 +492,9 @@ STV_Config_Transient(void)
 
 	ASSERT_MGT();
 	VTAILQ_FOREACH(stv, &stevedores, list)
-		if (!strcmp(stv->ident, TRANSIENT_NAME))
+		if (!strcmp(stv->ident, TRANSIENT_STORAGE))
 			return;
-	STV_Config(TRANSIENT_NAME "=malloc");
+	STV_Config(TRANSIENT_STORAGE "=malloc");
 }
 
 /*--------------------------------------------------------------------*/
@@ -539,7 +534,7 @@ stv_find(const char *nm)
 	VTAILQ_FOREACH(stv, &stevedores, list)
 		if (!strcmp(stv->ident, nm))
 			return (stv);
-	if (!strcmp(TRANSIENT_NAME, nm))
+	if (!strcmp(TRANSIENT_STORAGE, nm))
 		return (stv_transient);
 	return (NULL);
 }
diff --git a/bin/varnishd/storage_persistent.c b/bin/varnishd/storage_persistent.c
index f9739bb..a4ba7a3 100644
--- a/bin/varnishd/storage_persistent.c
+++ b/bin/varnishd/storage_persistent.c
@@ -1436,7 +1436,7 @@ smp_allocobj(struct stevedore *stv, struct sess *sp, unsigned ltot,
 
 	/* XXX: temporary sanity */
 	AN(sp->objcore);
-	AN(sp->wrk->cacheable);
+	AN(sp->wrk->ttl >= 0);
 
 	sg = NULL;
 	st = smp_allocx(stv, ltot, &sg);
diff --git a/bin/varnishtest/tests/b00015.vtc b/bin/varnishtest/tests/b00015.vtc
index 4cc8360..d5c9aeb 100644
--- a/bin/varnishtest/tests/b00015.vtc
+++ b/bin/varnishtest/tests/b00015.vtc
@@ -57,7 +57,6 @@ server s1 {
 varnish v1 -vcl+backend {
 	sub vcl_fetch {
 		if (beresp.status == 502) {
-			set beresp.cacheable = true;
 			set beresp.ttl = 10m;
 		}
 	}
diff --git a/bin/varnishtest/tests/r00411.vtc b/bin/varnishtest/tests/r00411.vtc
index edef375..a61e5a9 100644
--- a/bin/varnishtest/tests/r00411.vtc
+++ b/bin/varnishtest/tests/r00411.vtc
@@ -21,7 +21,7 @@ varnish v1 -vcl+backend {
 	}
 	sub vcl_fetch {
 		if (beresp.status == 303) {
-			set beresp.cacheable = true;
+			set beresp.ttl = 3m;
 			set beresp.http.X-Magic-Redirect = "1";
 		}
 	}
diff --git a/bin/varnishtest/tests/r00412.vtc b/bin/varnishtest/tests/r00412.vtc
index cb2983e..5fcd43a 100644
--- a/bin/varnishtest/tests/r00412.vtc
+++ b/bin/varnishtest/tests/r00412.vtc
@@ -14,7 +14,6 @@ server s1 {
 varnish v1 -vcl+backend {
 	sub vcl_fetch {
 		if (beresp.status == 303) {
-			set beresp.cacheable = true;
 			set beresp.ttl = 60 s;
 			set beresp.http.X-Magic-Redirect = "1";
 			set req.url = beresp.http.Location;
diff --git a/bin/varnishtest/tests/r00667.vtc b/bin/varnishtest/tests/r00667.vtc
index 926c5bb..19d5cb9 100644
--- a/bin/varnishtest/tests/r00667.vtc
+++ b/bin/varnishtest/tests/r00667.vtc
@@ -19,7 +19,7 @@ server s1 {
 
 varnish v1 -vcl+backend {
 	sub vcl_fetch {
-		set beresp.cacheable = false;
+		set beresp.ttl = 0s;
 	}
 } -start
 
diff --git a/bin/varnishtest/tests/s00002.vtc b/bin/varnishtest/tests/s00002.vtc
index c0ef1f8..b0bc60a 100644
--- a/bin/varnishtest/tests/s00002.vtc
+++ b/bin/varnishtest/tests/s00002.vtc
@@ -52,7 +52,6 @@ varnish v1 -vcl {
 	sub vcl_fetch { 
 		set beresp.ttl = 1s; 
 		set beresp.grace = 1m; 
-		set beresp.cacheable = true; 
 	}
 } -start
 
diff --git a/bin/varnishtest/tests/s00003.vtc b/bin/varnishtest/tests/s00003.vtc
index 0d97a42..162f1b6 100644
--- a/bin/varnishtest/tests/s00003.vtc
+++ b/bin/varnishtest/tests/s00003.vtc
@@ -20,7 +20,6 @@ varnish v1 -vcl+backend {
 	sub vcl_fetch {
 		set beresp.ttl = 1s;
 		set beresp.grace = 10m;
-		set beresp.cacheable = true;
 		if (beresp.http.foo == "2")
 		{
 			set beresp.saintmode = 2s;
diff --git a/lib/libvcl/generate.py b/lib/libvcl/generate.py
index 922ca94..d6f75e4 100755
--- a/lib/libvcl/generate.py
+++ b/lib/libvcl/generate.py
@@ -292,12 +292,6 @@ sp_variables = (
 		( 'fetch',),
 		'const struct sess *'
 	),
-	('beresp.cacheable',
-		'BOOL',
-		( 'fetch',),
-		( 'fetch',),
-		'const struct sess *'
-	),
 	('beresp.do_esi',
 		'BOOL',
 		( 'fetch',),



From phk at varnish-cache.org  Fri Jan 28 22:58:44 2011
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Fri, 28 Jan 2011 23:58:44 +0100
Subject: [master] eaa6454 When obj.ttl or beresp.ttl is set to zero,
	also set the corresponding grace to zero.
Message-ID: <E1PixHA-0001mj-DF@project.varnish-software.com>

commit eaa6454e006f5844834cc2c02ef3ca7de1a87a42
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Fri Jan 28 22:57:45 2011 +0000

    When obj.ttl or beresp.ttl is set to zero, also set the corresponding
    grace to zero.
    
    if you want ttl=0 with a non-zero grace, set ttl first, then set grace.

diff --git a/bin/varnishd/cache_vrt_var.c b/bin/varnishd/cache_vrt_var.c
index cc0d11e..9fa56ba 100644
--- a/bin/varnishd/cache_vrt_var.c
+++ b/bin/varnishd/cache_vrt_var.c
@@ -246,9 +246,10 @@ VRT_l_beresp_ttl(const struct sess *sp, double a)
 	 * of the rounding to seconds.
 	 * We special case and make sure that rounding does not surprise.
 	 */
-	if (a <= 0)
+	if (a <= 0) {
 		sp->wrk->ttl = sp->t_req - 1;
-	else
+		sp->wrk->grace = 0.;
+	} else
 		sp->wrk->ttl = sp->t_req + a;
 }
 
@@ -349,9 +350,10 @@ VRT_l_obj_ttl(const struct sess *sp, double a)
 	 * of the rounding to seconds.
 	 * We special case and make sure that rounding does not surprise.
 	 */
-	if (a <= 0)
+	if (a <= 0) {
 		sp->obj->ttl = sp->t_req - 1;
-	else
+		sp->obj->grace = 0;
+	} else
 		sp->obj->ttl = sp->t_req + a;
 	EXP_Rearm(sp->obj);
 }



From phk at varnish-cache.org  Sat Jan 29 11:08:00 2011
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Sat, 29 Jan 2011 12:08:00 +0100
Subject: [master] e1a2c7f Define a ${pwd} macro, it's usefull for ad-hoc tests
Message-ID: <E1Pj8eu-0008KK-Md@project.varnish-software.com>

commit e1a2c7feff4e2952bd7a54919569690aa4139295
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sat Jan 29 11:07:18 2011 +0000

    Define a ${pwd} macro, it's usefull for ad-hoc tests

diff --git a/bin/varnishtest/vtc.c b/bin/varnishtest/vtc.c
index 66990e4..f601dbc 100644
--- a/bin/varnishtest/vtc.c
+++ b/bin/varnishtest/vtc.c
@@ -492,6 +492,9 @@ exec_file(const char *fn, const char *script, const char *tmpdir,
 	bprintf(topbuild, "%s/%s", cwd, TOP_BUILDDIR);
 	macro_def(vltop, NULL, "topbuild", topbuild);
 
+	AN(getcwd(topbuild, sizeof topbuild));
+	macro_def(vltop, NULL, "pwd", topbuild);
+
 	macro_def(vltop, NULL, "bad_ip", "10.255.255.255");
 
 	/* Move into our tmpdir */



From phk at varnish-cache.org  Mon Jan 31 09:07:03 2011
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Mon, 31 Jan 2011 10:07:03 +0100
Subject: [master] e2a3fcc Emit a "Gzip" VSL record on each gzip operation.
Message-ID: <E1Pjpix-0000ve-LX@project.varnish-software.com>

commit e2a3fcc6927e61a98d136d853bd7306f2335c4fb
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Jan 31 09:06:05 2011 +0000

    Emit a "Gzip" VSL record on each gzip operation.
    
    Remember to free vgz structures again in ESI fetching

diff --git a/bin/varnishd/cache.h b/bin/varnishd/cache.h
index f997341..fdd0308 100644
--- a/bin/varnishd/cache.h
+++ b/bin/varnishd/cache.h
@@ -638,8 +638,8 @@ void Fetch_Init(void);
 struct vgz;
 
 enum vgz_flag { VGZ_NORMAL, VGZ_ALIGN, VGZ_RESET, VGZ_FINISH };
-struct vgz *VGZ_NewUngzip(struct sess *sp);
-struct vgz *VGZ_NewGzip(struct sess *sp);
+struct vgz *VGZ_NewUngzip(struct sess *sp, const char *id);
+struct vgz *VGZ_NewGzip(struct sess *sp, const char *id);
 void VGZ_Ibuf(struct vgz *, const void *, ssize_t len);
 int VGZ_IbufEmpty(const struct vgz *vg);
 void VGZ_Obuf(struct vgz *, const void *, ssize_t len);
diff --git a/bin/varnishd/cache_center.c b/bin/varnishd/cache_center.c
index 6595a98..25d57cb 100644
--- a/bin/varnishd/cache_center.c
+++ b/bin/varnishd/cache_center.c
@@ -431,8 +431,7 @@ cnt_error(struct sess *sp)
 }
 
 /*--------------------------------------------------------------------
- * We have fetched the headers from the backend, ask the VCL code what
- * to do next, then head off in that direction.
+ * Fetch an object from the backend, either for pass or for caching.
  *
 DOT subgraph xcluster_fetch {
 DOT	fetch [
diff --git a/bin/varnishd/cache_esi_deliver.c b/bin/varnishd/cache_esi_deliver.c
index 9c5b0cf..a8050de 100644
--- a/bin/varnishd/cache_esi_deliver.c
+++ b/bin/varnishd/cache_esi_deliver.c
@@ -274,7 +274,7 @@ ESI_Deliver(struct sess *sp)
 	}
 
 	if (isgzip && !sp->wrk->gzip_resp) {
-		vgz = VGZ_NewUngzip(sp);
+		vgz = VGZ_NewUngzip(sp, "U D E");
 		obufl = 0;
 	}
 
diff --git a/bin/varnishd/cache_esi_fetch.c b/bin/varnishd/cache_esi_fetch.c
index 9369d0a..34d98a4 100644
--- a/bin/varnishd/cache_esi_fetch.c
+++ b/bin/varnishd/cache_esi_fetch.c
@@ -305,24 +305,24 @@ vfp_esi_begin(struct sess *sp, size_t estimate)
 	/* XXX: snapshot WS's ? We'll need the space */
 
 	if (sp->wrk->is_gzip && sp->wrk->do_gunzip) {
-		sp->wrk->vgz_rx = VGZ_NewUngzip(sp);
+		sp->wrk->vgz_rx = VGZ_NewUngzip(sp, "U F E");
 		VEP_Init(sp, NULL);
 	} else if (sp->wrk->is_gunzip && sp->wrk->do_gzip) {
 		vef = (void*)WS_Alloc(sp->ws, sizeof *vef);
 		AN(vef);
 		memset(vef, 0, sizeof *vef);
 		vef->magic = VEF_MAGIC;
-		vef->vgz = VGZ_NewGzip(sp);
+		vef->vgz = VGZ_NewGzip(sp, "G F E");
 		AZ(sp->wrk->vef_priv);
 		sp->wrk->vef_priv = vef;
 		VEP_Init(sp, vfp_vep_callback);
 	} else if (sp->wrk->is_gzip) {
-		sp->wrk->vgz_rx = VGZ_NewUngzip(sp);
+		sp->wrk->vgz_rx = VGZ_NewUngzip(sp, "U F E");
 		vef = (void*)WS_Alloc(sp->ws, sizeof *vef);
 		AN(vef);
 		memset(vef, 0, sizeof *vef);
 		vef->magic = VEF_MAGIC;
-		vef->vgz = VGZ_NewGzip(sp);
+		vef->vgz = VGZ_NewGzip(sp, "G F E");
 		AZ(sp->wrk->vef_priv);
 		sp->wrk->vef_priv = vef;
 		VEP_Init(sp, vfp_vep_callback);
@@ -373,12 +373,15 @@ vfp_esi_end(struct sess *sp)
 		sp->obj->esidata->len = l;
 		vsb_delete(vsb);
 	}
+	if (sp->wrk->vgz_rx != NULL)
+		VGZ_Destroy(&sp->wrk->vgz_rx);
 
 	if (sp->wrk->vef_priv != NULL) {
 		vef = sp->wrk->vef_priv;
-		VGZ_UpdateObj(vef->vgz, sp->obj);
-		sp->wrk->vef_priv = NULL;
 		CHECK_OBJ_NOTNULL(vef, VEF_MAGIC);
+		sp->wrk->vef_priv = NULL;
+		VGZ_UpdateObj(vef->vgz, sp->obj);
+		VGZ_Destroy(&vef->vgz);
 		XXXAZ(vef->error);
 		sp->obj->gziped = 1;
 	} else {
diff --git a/bin/varnishd/cache_gzip.c b/bin/varnishd/cache_gzip.c
index 5177d01..7ff3895 100644
--- a/bin/varnishd/cache_gzip.c
+++ b/bin/varnishd/cache_gzip.c
@@ -71,6 +71,7 @@
 #include "svnid.h"
 SVNID("$Id$")
 
+#include "vsl.h"
 #include "cache.h"
 #include "stevedore.h"
 
@@ -79,6 +80,8 @@ SVNID("$Id$")
 struct vgz {
 	unsigned		magic;
 #define VGZ_MAGIC		0x162df0cb
+	struct sess		*sess;
+	const char		*id;
 	struct ws		*tmp;
 	char			*tmp_snapshot;
 
@@ -113,7 +116,7 @@ vgz_free(voidpf opaque, voidpf address)
  */
 
 static struct vgz *
-vgz_alloc_vgz(struct sess *sp)
+vgz_alloc_vgz(struct sess *sp, const char *id)
 {
 	struct vgz *vg;
 	struct ws *ws = sp->wrk->ws;
@@ -123,6 +126,8 @@ vgz_alloc_vgz(struct sess *sp)
 	AN(vg);
 	memset(vg, 0, sizeof *vg);
 	vg->magic = VGZ_MAGIC;
+	vg->sess = sp;
+	vg->id = id;
 
 	switch (params->gzip_tmp_space) {
 	case 0:
@@ -149,12 +154,12 @@ vgz_alloc_vgz(struct sess *sp)
 }
 
 struct vgz *
-VGZ_NewUngzip(struct sess *sp)
+VGZ_NewUngzip(struct sess *sp, const char *id)
 {
 	struct vgz *vg;
 
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
-	vg = vgz_alloc_vgz(sp);
+	vg = vgz_alloc_vgz(sp, id);
 
 	/*
 	 * Max memory usage according to zonf.h:
@@ -167,13 +172,13 @@ VGZ_NewUngzip(struct sess *sp)
 }
 
 struct vgz *
-VGZ_NewGzip(struct sess *sp)
+VGZ_NewGzip(struct sess *sp, const char *id)
 {
 	struct vgz *vg;
 	int i;
 
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
-	vg = vgz_alloc_vgz(sp);
+	vg = vgz_alloc_vgz(sp, id);
 
 	/*
 	 * From zconf.h:
@@ -353,13 +358,23 @@ VGZ_UpdateObj(const struct vgz *vg, struct object *obj)
 /*--------------------------------------------------------------------*/
 
 void
-VGZ_Destroy(struct vgz **vg)
+VGZ_Destroy(struct vgz **vgp)
 {
+	struct vgz *vg;
 
-	CHECK_OBJ_NOTNULL(*vg, VGZ_MAGIC);
-	if ((*vg)->tmp != NULL)
-		WS_Reset((*vg)->tmp, (*vg)->tmp_snapshot);
-	*vg = NULL;
+	vg = *vgp;
+	CHECK_OBJ_NOTNULL(vg, VGZ_MAGIC);
+	*vgp = NULL;
+
+	WSP(vg->sess, SLT_Gzip, "%s %jd %jd %jd %jd %jd",
+	    vg->id,
+	    (intmax_t)vg->vz.total_in,
+	    (intmax_t)vg->vz.total_out,
+	    (intmax_t)vg->vz.start_bit,
+	    (intmax_t)vg->vz.last_bit,
+	    (intmax_t)vg->vz.stop_bit);
+	if (vg->tmp != NULL)
+		WS_Reset(vg->tmp, vg->tmp_snapshot);
 }
 
 /*--------------------------------------------------------------------
@@ -372,7 +387,7 @@ static void __match_proto__()
 vfp_gunzip_begin(struct sess *sp, size_t estimate)
 {
 	(void)estimate;
-	sp->wrk->vgz_rx = VGZ_NewUngzip(sp);
+	sp->wrk->vgz_rx = VGZ_NewUngzip(sp, "U F -");
 }
 
 static int __match_proto__()
@@ -441,7 +456,7 @@ vfp_gzip_begin(struct sess *sp, size_t estimate)
 {
 	(void)estimate;
 
-	sp->wrk->vgz_rx = VGZ_NewGzip(sp);
+	sp->wrk->vgz_rx = VGZ_NewGzip(sp, "G F -");
 }
 
 static int __match_proto__()
@@ -517,7 +532,7 @@ static void __match_proto__()
 vfp_testgzip_begin(struct sess *sp, size_t estimate)
 {
 	(void)estimate;
-	sp->wrk->vgz_rx = VGZ_NewUngzip(sp);
+	sp->wrk->vgz_rx = VGZ_NewUngzip(sp, "u F -");
 }
 
 static int __match_proto__()
diff --git a/bin/varnishd/cache_response.c b/bin/varnishd/cache_response.c
index 18c2612..94f2dd4 100644
--- a/bin/varnishd/cache_response.c
+++ b/bin/varnishd/cache_response.c
@@ -253,7 +253,7 @@ res_WriteGunzipObj(struct sess *sp)
 
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 
-	vg = VGZ_NewUngzip(sp);
+	vg = VGZ_NewUngzip(sp, "U R -");
 
 	VTAILQ_FOREACH(st, &sp->obj->store, list) {
 		CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
diff --git a/include/vsl_tags.h b/include/vsl_tags.h
index 7a59d55..6a28832 100644
--- a/include/vsl_tags.h
+++ b/include/vsl_tags.h
@@ -98,3 +98,5 @@ SLTM(Hash)
 
 SLTM(Backend_health)
 SLTM(VCL_Log)
+
+SLTM(Gzip)



From phk at varnish-cache.org  Mon Jan 31 09:19:08 2011
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Mon, 31 Jan 2011 10:19:08 +0100
Subject: [master] e9b315e Document gzip VSL records
Message-ID: <E1Pjpue-0002l6-3u@project.varnish-software.com>

commit e9b315e3dfd6362d50eac83196a2cb0efda864af
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Jan 31 09:18:57 2011 +0000

    Document gzip VSL records

diff --git a/doc/sphinx/reference/index.rst b/doc/sphinx/reference/index.rst
index 2dd8c1b..38dbd4a 100644
--- a/doc/sphinx/reference/index.rst
+++ b/doc/sphinx/reference/index.rst
@@ -19,6 +19,7 @@ The Varnish Reference Manual
 	varnishtop.rst
 	shmem.rst
 	vmod.rst
+	vsl.rst
 
 .. todo::
         The programs:
diff --git a/doc/sphinx/reference/vsl.rst b/doc/sphinx/reference/vsl.rst
new file mode 100644
index 0000000..ac1026a
--- /dev/null
+++ b/doc/sphinx/reference/vsl.rst
@@ -0,0 +1,32 @@
+.. _reference-vsl:
+
+=====================
+Shared Memory Logging
+=====================
+
+Gzip records
+~~~~~~~~~~~~
+
+A Gzip record is emitted for each instance of gzip or gunzip work
+performed.
+Worst case, an ESI transaction stored in gzip'ed objects but delivered
+gunziped, will run into many of these.
+
+The format is::
+
+
+	%c %c %c %d %d %d %d %d
+	|  |  |  |  |  |  |  |
+	|  |  |  |  |  |  |  +- Bit length of compressed data
+	|  |  |  |  |  |  +---- Bit location of 'last' bit
+	|  |  |  |  |  +------- Bit location of first deflate block
+	|  |  |  |  +---------- Bytes output
+	|  |  |  +------------- Bytes input
+	|  |  +---------------- 'E' = ESI, '-' = Plain object
+	|  +------------------- 'F' = Fetch, 'D' = Deliver
+	+---------------------- 'G' = Gzip, 'U' = Gunzip, 'u' = Gunzip-test
+
+Which in practice could look like::
+
+	U F E 182 159 80 80 1392
+	G F E 159 173 80 1304 1314



From phk at varnish-cache.org  Mon Jan 31 09:38:31 2011
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Mon, 31 Jan 2011 10:38:31 +0100
Subject: [master] 99cff3e Add two stats counters,
	counting number of gzip and gunzip operations.
Message-ID: <E1PjqDP-0006M3-1l@project.varnish-software.com>

commit 99cff3e9b7685c3361e467f14c013b8595cf0c3f
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Jan 31 09:37:38 2011 +0000

    Add two stats counters, counting number of gzip and gunzip operations.
    
    (fix a typo in previous commit)

diff --git a/bin/varnishd/cache_gzip.c b/bin/varnishd/cache_gzip.c
index 7ff3895..4522afd 100644
--- a/bin/varnishd/cache_gzip.c
+++ b/bin/varnishd/cache_gzip.c
@@ -160,6 +160,7 @@ VGZ_NewUngzip(struct sess *sp, const char *id)
 
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 	vg = vgz_alloc_vgz(sp, id);
+	VSC_main->n_gunzip++;
 
 	/*
 	 * Max memory usage according to zonf.h:
@@ -179,6 +180,7 @@ VGZ_NewGzip(struct sess *sp, const char *id)
 
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 	vg = vgz_alloc_vgz(sp, id);
+	VSC_main->n_gzip++;
 
 	/*
 	 * From zconf.h:
diff --git a/bin/varnishd/cache_response.c b/bin/varnishd/cache_response.c
index 94f2dd4..96ff344 100644
--- a/bin/varnishd/cache_response.c
+++ b/bin/varnishd/cache_response.c
@@ -253,7 +253,7 @@ res_WriteGunzipObj(struct sess *sp)
 
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 
-	vg = VGZ_NewUngzip(sp, "U R -");
+	vg = VGZ_NewUngzip(sp, "U D -");
 
 	VTAILQ_FOREACH(st, &sp->obj->store, list) {
 		CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
diff --git a/include/vsc_fields.h b/include/vsc_fields.h
index dd68151..ffb56db 100644
--- a/include/vsc_fields.h
+++ b/include/vsc_fields.h
@@ -154,9 +154,11 @@ VSC_F(dir_dns_failed,	uint64_t, 0, 'a', "DNS director failed lookups")
 VSC_F(dir_dns_hit,		uint64_t, 0, 'a', "DNS director cached lookups hit")
 VSC_F(dir_dns_cache_full,	uint64_t, 0, 'a', "DNS director full dnscache")
 
-
 VSC_F(vmods,		uint64_t, 0, 'i', "Loaded VMODs")
 
+VSC_F(n_gzip,			uint64_t, 0, 'a', "Gzip operations")
+VSC_F(n_gunzip,			uint64_t, 0, 'a', "Gunzip operations")
+
 #endif
 
 /**********************************************************************/



From phk at varnish-cache.org  Mon Jan 31 18:28:03 2011
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Mon, 31 Jan 2011 19:28:03 +0100
Subject: [master] 5da592f Use printf(1) instead of echo(1) with -n option,
	it should be more portable, says Nils Goroll
Message-ID: <E1PjyTr-0007kP-SO@project.varnish-software.com>

commit 5da592f57d3bbb13e0fecdef6397049b5043e8aa
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Jan 31 18:27:23 2011 +0000

    Use printf(1) instead of echo(1) with -n option, it should be
    more portable, says Nils Goroll

diff --git a/bin/varnishtest/tests/m00004.vtc b/bin/varnishtest/tests/m00004.vtc
index 1c629b2..2561b45 100644
--- a/bin/varnishtest/tests/m00004.vtc
+++ b/bin/varnishtest/tests/m00004.vtc
@@ -3,9 +3,9 @@
 test "Test fileread for std VMOD"
 
 shell {
-	echo -n "File One" > "${tmpdir}/m00004_file_one"
-	echo -n "File Two" > "${tmpdir}/m00004_file_two"
-	echo -n "File Three" > "${tmpdir}/m00004_file_three"
+	printf "File One" > "${tmpdir}/m00004_file_one"
+	printf "File Two" > "${tmpdir}/m00004_file_two"
+	printf "File Three" > "${tmpdir}/m00004_file_three"
 }
 
 server s1 {



From phk at varnish-cache.org  Mon Jan 31 20:03:01 2011
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Mon, 31 Jan 2011 21:03:01 +0100
Subject: [master] 34b365c Wash and improve the .vcc file parser a little bit
Message-ID: <E1Pjzxl-0008Va-J9@project.varnish-software.com>

commit 34b365c5ad4b750accf7738d09ae424e5277e17c
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Jan 31 20:02:44 2011 +0000

    Wash and improve the .vcc file parser a little bit

diff --git a/lib/libvmod_std/vmod.py b/lib/libvmod_std/vmod.py
index 32fb536..d750c99 100755
--- a/lib/libvmod_std/vmod.py
+++ b/lib/libvmod_std/vmod.py
@@ -40,14 +40,13 @@
 # $Id$
 
 import sys
+import re
 
 if len(sys.argv) == 2:
 	specfile = sys.argv[1]
 else:
 	specfile = "vmod.vcc"
 
-type_tab = dict()
-
 ctypes = {
 	'IP':		"struct sockaddr_storage *",
 	'STRING':	"const char *",
@@ -114,49 +113,89 @@ def partition(string, separator):
 		return (string[:i],separator,string[i+len(separator):])
 	return (string, '', '')
 
+#######################################################################
+
 f = open(specfile, "r")
 
-for l0 in f:
-	# print("# " + l0)
-	l0=l0.strip()
+def nextline():
+	while True:
+		l0 = f.readline()
+		if l0 == "":
+			return l0
+		l0 = re.sub("#.*$", "", l0)
+		l0 = re.sub("\s\s*", " ", l0.strip())
+		if l0 != "":
+			return l0
+
+def is_c_name(s):
+	return None != re.match("^[a-z][a-z0-9_]*$", s)
+
+while True:
+	l0 = nextline()
 	if l0 == "":
-		continue
-	i = l0.find("#")
-	if i == 0:
-		continue
-	if i >= 0:
-		line = l0[:i-1]
-	else:
-		line = l0
-	line = line.expandtabs().strip()
-	l = partition(line, " ")
+		break;
+	l = partition(l0, " ")
 
 	if l[0] == "Module":
 		modname = l[2].strip();
+		if not is_c_name(modname):
+			raise Exception("Module name '%s' is illegal" % modname)
 		continue
 
 	if l[0] == "Init":
 		initname = l[2].strip();
+		if not is_c_name(initname):
+			raise Exception("Init name '%s' is illegal" % initname)
 		continue
 
 	if l[0] != "Function":
-		assert False
+		raise Exception("Expected 'Function' line, got '%s'" % l[0])
 
+	# Find the return type of the function
 	l = partition(l[2].strip(), " ")
 	rt_type = l[0]
+	if rt_type not in ctypes:
+		raise Exception("Return type '%s' not a valid type" % rt_type)
 
+	# Find the function name
 	l = partition(l[2].strip(), "(")
+
 	fname = l[0].strip()
+	if not is_c_name(fname):
+		raise Exception("Function name '%s' is illegal" % fname)
+
+	if l[1] != '(':
+		raise Exception("Missing '('")
+
+	l = l[2]
+
+	while -1 == l.find(")"):
+		l1 = nextline()
+		if l1 == "":	
+			raise Exception("End Of Input looking for ')'")
+		l = l + l1
+
+	if -1 != l.find("("):
+		raise Exception("Nesting trouble with '(...)' ")
+
+	if l[-1:] != ')':
+		raise Exception("Junk after ')'")
+
+	l = l[:-1]
 
 	args = list()
 
 	while True:
-		l = partition(l[2].strip(), ",")
-		if len(l[2]) == 0:
+		l = partition(l, ",")
+		t = l[0].strip()
+		if t not in ctypes:
+			raise Exception(
+			    "Argument type '%s' not a valid type" % t)
+		args.append(t)
+		l = l[2]
+		if l == "":
 			break
-		args.append(l[0])
-	l = partition(l[0].strip(), ")")
-	args.append(l[0])
+
 	do_func(fname, rt_type, args)
 
 #######################################################################



From phk at varnish-cache.org  Mon Jan 31 20:23:15 2011
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Mon, 31 Jan 2011 21:23:15 +0100
Subject: [master] 35db7bb Use a regexp iterator (nice concept!) to separate
	the arguments and add support for argument qualifiers in {...}
Message-ID: <E1Pk0HL-0001sW-I2@project.varnish-software.com>

commit 35db7bbb8d24a0779db99b88591216f3e3cc494c
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Jan 31 20:22:42 2011 +0000

    Use a regexp iterator (nice concept!) to separate the arguments
    and add support for argument qualifiers in {...}

diff --git a/lib/libvmod_std/vmod.py b/lib/libvmod_std/vmod.py
index d750c99..f751e82 100755
--- a/lib/libvmod_std/vmod.py
+++ b/lib/libvmod_std/vmod.py
@@ -185,16 +185,13 @@ while True:
 
 	args = list()
 
-	while True:
-		l = partition(l, ",")
-		t = l[0].strip()
-		if t not in ctypes:
+	for i in re.finditer("([A-Z_]+)\s*({[^}]+})?(,|$)", l):
+		at = i.group(1)
+		tq = i.group(2)
+		if at not in ctypes:
 			raise Exception(
-			    "Argument type '%s' not a valid type" % t)
-		args.append(t)
-		l = l[2]
-		if l == "":
-			break
+			    "Argument type '%s' not a valid type" % at)
+		args.append(at)
 
 	do_func(fname, rt_type, args)
 



From phk at varnish-cache.org  Mon Jan 31 21:50:42 2011
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Mon, 31 Jan 2011 22:50:42 +0100
Subject: [master] dccd281 Add an "ENUM" type as a possible argument to VMOD
	functions.
Message-ID: <E1Pk1dy-00025m-JZ@project.varnish-software.com>

commit dccd2814066fc9c228b3e44887062c3d6c54ee2b
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Jan 31 21:49:03 2011 +0000

    Add an "ENUM" type as a possible argument to VMOD functions.
    
    VCC expects an ID, and complains if it is not on the list of
    valid values.  The value is passed as a "const char *"
    
    More and better error messages from vmod.py

diff --git a/bin/varnishtest/tests/m00000.vtc b/bin/varnishtest/tests/m00000.vtc
index 7ad2054..7874836 100644
--- a/bin/varnishtest/tests/m00000.vtc
+++ b/bin/varnishtest/tests/m00000.vtc
@@ -13,6 +13,7 @@ varnish v1 -vcl+backend {
 	sub vcl_deliver {
 		set resp.http.foo = std.toupper(resp.http.foo);
 		set resp.http.bar = std.tolower(resp.http.bar);
+		set resp.http.who = std.author(phk);
 		std.log("VCL initiated log");
 		std.syslog(8 + 7, "Somebody runs varnishtest");
 	}
@@ -26,3 +27,10 @@ client c1 {
 	expect resp.http.foo == "BAR"
 	expect resp.http.bar == "foo"
 } -run
+
+varnish v1 -badvcl {
+	import std from "${topbuild}/lib/libvmod_std/.libs/libvmod_std.so.1" ;
+	sub vcl_deliver {
+		set resp.http.who = std.author(jfk);
+	}
+}
diff --git a/lib/libvcl/vcc_expr.c b/lib/libvcl/vcc_expr.c
index ff3fb48..1a5b074 100644
--- a/lib/libvcl/vcc_expr.c
+++ b/lib/libvcl/vcc_expr.c
@@ -550,6 +550,32 @@ vcc_Eval_Func(struct vcc *tl, struct expr **e, const struct symbol *sym)
 			Fh(tl, 0, "struct vmod_priv %s;\n", buf);
 			e1 = vcc_mk_expr(VOID, "&%s", buf);
 			p += strlen(p) + 1;
+		} else if (fmt == ENUM) {
+			ExpectErr(tl, ID);
+			ERRCHK(tl);
+			r = p;
+			do {
+				if (vcc_IdIs(tl->t, p))
+					break;
+				p += strlen(p) + 1;
+			} while (*p != '\0');
+			if (*p == '\0') {
+				vsb_printf(tl->sb, "Wrong enum value.");
+				vsb_printf(tl->sb, "  Expected one of:\n");
+				do {
+					vsb_printf(tl->sb, "\t%s\n", r);
+					r += strlen(r) + 1;
+				} while (*r != '\0');
+				vcc_ErrWhere(tl, tl->t);
+				return;
+			}
+			e1 = vcc_mk_expr(VOID, "\"%.*s\"", PF(tl->t));
+			while (*p != '\0')
+				p += strlen(p) + 1;
+			p++;
+			SkipToken(tl, ID);
+			if (*p != '\0')
+				SkipToken(tl, ',');
 		} else {
 			vcc_expr0(tl, &e1, fmt);
 			ERRCHK(tl);
diff --git a/lib/libvcl/vcc_types.h b/lib/libvcl/vcc_types.h
index 94399e7..aeaec90 100644
--- a/lib/libvcl/vcc_types.h
+++ b/lib/libvcl/vcc_types.h
@@ -41,4 +41,5 @@ VCC_TYPE(IP)
 VCC_TYPE(HEADER)
 VCC_TYPE(BYTES)
 VCC_TYPE(REAL)
+VCC_TYPE(ENUM)
 /*lint -restore */
diff --git a/lib/libvmod_std/vmod.py b/lib/libvmod_std/vmod.py
index f751e82..7d751cb 100755
--- a/lib/libvmod_std/vmod.py
+++ b/lib/libvmod_std/vmod.py
@@ -53,6 +53,7 @@ ctypes = {
 	'STRING_LIST':	"const char *, ...",
 	'BOOL':		"unsigned",
 	'BACKEND':	"struct director *",
+	'ENUM':		"const char *",
 	'TIME':		"double",
 	'REAL':		"double",
 	'DURATION':	"double",
@@ -73,7 +74,7 @@ tdl = ""
 plist = ""
 slist = ""
 
-def do_func(fname, rval, args):
+def do_func(fname, rval, args, vargs):
 	global pstruct
 	global pinit
 	global plist
@@ -81,27 +82,33 @@ def do_func(fname, rval, args):
 	global tdl
 	#print(fname, rval, args)
 
-	proto = ctypes[rval] + " vmod_" + fname + "(struct sess *"
-	sproto = ctypes[rval] + " td_" + modname + "_" + fname + "(struct sess *"
-	s=", "
+	# C argument list
+	cargs = "(struct sess *"
 	for i in args:
-		proto += s + ctypes[i]
-		sproto += s + ctypes[i]
-	proto += ")"
-	sproto += ")"
+		cargs += ", " + i
+	cargs += ")"
 
+	# Prototypes for vmod implementation and interface typedef
+	proto = ctypes[rval] + " vmod_" + fname + cargs
+	sproto = ctypes[rval] + " td_" + modname + "_" + fname + cargs
+
+	# append to lists of prototypes
 	plist += proto + ";\n"
 	tdl += "typedef " + sproto + ";\n"
 
+	# Append to struct members
 	pstruct += "\ttd_" + modname + "_" + fname + "\t*" + fname + ";\n"
+
+	# Append to struct initializer
 	pinit += "\tvmod_" + fname + ",\n"
 
+	# Compose the vmod spec-string
 	s = modname + '.' + fname + "\\0"
 	s += "Vmod_Func_" + modname + "." + fname + "\\0"
-	s += rval
-	for i in args:
-		s += '\\0' + i
-	slist += '\t"' + s + '\\0",\n'
+	s += rval + '\\0'
+	for i in vargs:
+		s +=  i + '\\0'
+	slist += '\t"' + s + '",\n'
 
 #######################################################################
 
@@ -115,6 +122,30 @@ def partition(string, separator):
 
 #######################################################################
 
+def is_c_name(s):
+	return None != re.match("^[a-z][a-z0-9_]*$", s)
+
+#######################################################################
+
+def parse_enum(tq):
+	assert tq[0] == '{'
+	assert tq[-1] == '}'
+	f = tq[1:-1].split(',')
+	s="ENUM\\0"
+	b=dict()
+	for i in f:
+		i = i.strip()
+		if not is_c_name(i):
+			raise Exception("Enum value '%s' is illegal" % i)
+		if i in b:
+			raise Exception("Duplicate Enum value '%s'" % i)
+		b[i] = True
+		s = s + i.strip() + '\\0'
+	s = s + '\\0'
+	return s
+
+#######################################################################
+
 f = open(specfile, "r")
 
 def nextline():
@@ -127,9 +158,6 @@ def nextline():
 		if l0 != "":
 			return l0
 
-def is_c_name(s):
-	return None != re.match("^[a-z][a-z0-9_]*$", s)
-
 while True:
 	l0 = nextline()
 	if l0 == "":
@@ -184,6 +212,7 @@ while True:
 	l = l[:-1]
 
 	args = list()
+	vargs = list()
 
 	for i in re.finditer("([A-Z_]+)\s*({[^}]+})?(,|$)", l):
 		at = i.group(1)
@@ -191,9 +220,22 @@ while True:
 		if at not in ctypes:
 			raise Exception(
 			    "Argument type '%s' not a valid type" % at)
-		args.append(at)
 
-	do_func(fname, rt_type, args)
+		args.append(ctypes[at])
+
+		if at == "ENUM":
+			if tq == None:
+				raise Exception(
+				    "Argument type '%s' needs qualifier {...}" % at)
+			at=parse_enum(tq)
+
+		elif tq != None:
+			raise Exception(
+			    "Argument type '%s' cannot be qualified with {...}" % at)
+		
+		vargs.append(at)
+
+	do_func(fname, rt_type, args, vargs)
 
 #######################################################################
 def dumps(s):
diff --git a/lib/libvmod_std/vmod.vcc b/lib/libvmod_std/vmod.vcc
index ffb99b0..b2923dc 100644
--- a/lib/libvmod_std/vmod.vcc
+++ b/lib/libvmod_std/vmod.vcc
@@ -34,3 +34,9 @@ Function REAL random(REAL, REAL)
 Function VOID log(STRING_LIST)
 Function VOID syslog(INT, STRING_LIST)
 Function STRING fileread(PRIV_CALL, STRING)
+Function STRING author(ENUM {
+	phk,
+	des,
+	kristian,
+	mithrandir
+})
diff --git a/lib/libvmod_std/vmod_std.c b/lib/libvmod_std/vmod_std.c
index c6a51ea..2807af1 100644
--- a/lib/libvmod_std/vmod_std.c
+++ b/lib/libvmod_std/vmod_std.c
@@ -161,3 +161,18 @@ vmod_syslog(struct sess *sp, int fac, const char *fmt, ...)
 	if (p != NULL)
 		syslog(fac, "%s", buf);
 }
+
+const char * __match_proto__()
+vmod_author(struct sess *sp, const char *id)
+{
+	(void)sp;
+	if (!strcmp(id, "phk"))
+		return ("Poul-Henning");
+	if (!strcmp(id, "des"))
+		return ("Dag-Erling");
+	if (!strcmp(id, "kristian"))
+		return ("Kristian");
+	if (!strcmp(id, "mithrandir"))
+		return ("Tollef");
+	WRONG("Illegal VMOD enum");
+}



From phk at varnish-cache.org  Mon Jan 31 22:46:57 2011
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Mon, 31 Jan 2011 23:46:57 +0100
Subject: [master] db72da1 Better test coverage of VCC error detection code
Message-ID: <E1Pk2WP-0008QA-Op@project.varnish-software.com>

commit db72da163c401302dd80dd50df0dcf8251b7f29a
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Jan 31 22:28:43 2011 +0000

    Better test coverage of VCC error detection code

diff --git a/bin/varnishtest/tests/m00000.vtc b/bin/varnishtest/tests/m00000.vtc
index 7874836..e44dbb2 100644
--- a/bin/varnishtest/tests/m00000.vtc
+++ b/bin/varnishtest/tests/m00000.vtc
@@ -34,3 +34,10 @@ varnish v1 -badvcl {
 		set resp.http.who = std.author(jfk);
 	}
 }
+
+varnish v1 -badvcl {
+	import std from "${topbuild}/lib/libvmod_std/.libs/libvmod_std.so.1" ;
+	sub vcl_deliver {
+		set resp.http.who = std.random("foo", "bar");
+	}
+}
diff --git a/bin/varnishtest/tests/v00020.vtc b/bin/varnishtest/tests/v00020.vtc
index 057dfa7..e584150 100644
--- a/bin/varnishtest/tests/v00020.vtc
+++ b/bin/varnishtest/tests/v00020.vtc
@@ -27,7 +27,7 @@ varnish v1 -vcl {
 
 varnish v1 -badvcl {
 	sub vcl_recv {
-		if (req.restarts == req.url) {
+		if (!req.restarts != req.url) {
 			set req.http.foo = "foo" + 3;
 		}
 
@@ -39,3 +39,31 @@ varnish v1 -badvcl {
 		set req.http.foo = vcl_recv;
 	}
 }
+
+varnish v1 -badvcl {
+	sub vcl_recv {
+		if (req.grace < 3s || req.grace) {
+			set req.http.foo = vcl_recv;
+		}
+	}
+}
+
+varnish v1 -badvcl {
+	sub vcl_recv {
+		if (req.grace < 3s && req.grace) {
+			set req.http.foo = vcl_recv;
+		}
+	}
+}
+
+varnish v1 -badvcl {
+	sub vcl_recv {
+		set req.http.foo = "bla" * "foo";
+	}
+}
+
+varnish v1 -badvcl {
+	sub vcl_fetch {
+		set req.http.foo = req.grace + beresp.status;
+	}
+}
diff --git a/bin/varnishtest/tests/v00033.vtc b/bin/varnishtest/tests/v00033.vtc
index e6e342b..6939855 100644
--- a/bin/varnishtest/tests/v00033.vtc
+++ b/bin/varnishtest/tests/v00033.vtc
@@ -38,3 +38,16 @@ client c1 {
 	expect resp.http.foo == 1100586419201.000
 	expect resp.http.bar == true
 } -run
+
+varnish v1 -badvcl {
+	sub vcl_recv {
+		if (storage.nowhere.free_space > 4 "X") {
+		}
+	}
+}
+varnish v1 -badvcl {
+	sub vcl_recv {
+		if (storage.nowhere.free_space > 4 X) {
+		}
+	}
+}



From phk at varnish-cache.org  Mon Jan 31 23:11:10 2011
From: phk at varnish-cache.org (Poul-Henning Kamp)
Date: Tue, 01 Feb 2011 00:11:10 +0100
Subject: [master] d6210a9 A little bit more coverage of vcc_expr.c
Message-ID: <E1Pk2tq-0003WQ-U0@project.varnish-software.com>

commit d6210a9f5b821a4cddbc7bb6434f697265c91aab
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Jan 31 23:10:55 2011 +0000

    A little bit more coverage of vcc_expr.c

diff --git a/bin/varnishtest/tests/v00020.vtc b/bin/varnishtest/tests/v00020.vtc
index e584150..abba9b7 100644
--- a/bin/varnishtest/tests/v00020.vtc
+++ b/bin/varnishtest/tests/v00020.vtc
@@ -42,7 +42,7 @@ varnish v1 -badvcl {
 
 varnish v1 -badvcl {
 	sub vcl_recv {
-		if (req.grace < 3s || req.grace) {
+		if (req.grace < -3s || req.grace) {
 			set req.http.foo = vcl_recv;
 		}
 	}
@@ -67,3 +67,17 @@ varnish v1 -badvcl {
 		set req.http.foo = req.grace + beresp.status;
 	}
 }
+
+varnish v1 -badvcl {
+	sub vcl_fetch {
+		if (! req.grace) {
+		}
+	}
+}
+
+varnish v1 -badvcl {
+	sub vcl_fetch {
+		if (beresp.do_gzip + beresp.do_gunzip) {
+		}
+	}
+}