From phk at FreeBSD.org  Mon Jun  2 08:16:05 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Mon, 02 Jun 2014 10:16:05 +0200
Subject: [master] 046d8d1 Some more style polishing of vmodtool
Message-ID: <E1WrNPZ-0007Ho-Ec@project.varnish-software.com>


commit 046d8d1c6514e7987604e8ee28d9ec860938a4b0
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Jun 2 08:15:36 2014 +0000

    Some more style polishing of vmodtool

diff --git a/lib/libvcc/vmodtool.py b/lib/libvcc/vmodtool.py
index ea01391..3299074 100755
--- a/lib/libvcc/vmodtool.py
+++ b/lib/libvcc/vmodtool.py
@@ -25,17 +25,13 @@
 # LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
 # OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 # SUCH DAMAGE.
-#
-# Read the vmod.vcc file (inputvcc) and produce the vmod.h and vmod.c files.
-#
-# vmod.h contains the prototypes for the published functions, the module
-# C-code should include this file to ensure type-consistency.
-#
-# vmod.c contains the symbols which VCC and varnishd will use to access
-# the module:  A structure of properly typed function pointers, the
-# size of this structure in bytes, and the definition of the structure
-# as a string, suitable for inclusion in the C-source of the compile VCL
-# program.
+
+"""
+Read the vmod.vcc file (inputvcc) and produce:
+	vmod_if.h -- Prototypes for the implementation
+	vmod_if.c -- Magic glue & datastructures to make things a VMOD.
+	vmod_${name}.rst -- Extracted documentation
+"""
 
 # This script should work with both Python 2 and Python 3.
 from __future__ import print_function
@@ -45,7 +41,7 @@ import re
 import optparse
 import unittest
 from os import unlink
-from os.path import dirname, basename, realpath, exists
+from os.path import dirname, realpath, exists
 from pprint import pprint, pformat
 
 ctypes = {
@@ -88,7 +84,7 @@ def lwrap(s, w=72):
 	Wrap a c-prototype like string into a number of lines
 	"""
 	l = []
-	p=""
+	p = ""
 	while len(s) > w:
 		y = s[:w].rfind(',')
 		if y == -1:
@@ -101,7 +97,7 @@ def lwrap(s, w=72):
 	if len(s) > 0:
 		l.append(p + s)
 	return l
-		
+
 #######################################################################
 
 def is_c_name(s):
@@ -125,7 +121,7 @@ class FormatError(Exception):
 
 #######################################################################
 
-class token(object):
+class Token(object):
 	def __init__(self, ln, ch, tokstr):
 		self.ln = ln
 		self.ch = ch
@@ -136,7 +132,7 @@ class token(object):
 
 #######################################################################
 
-class vmod(object):
+class Vmod(object):
 	def __init__(self, nam, dnam, sec):
 		if not is_c_name(nam):
 			raise ParseError("Module name '%s' is illegal", nam)
@@ -160,9 +156,9 @@ class vmod(object):
 		self.funcs.append(fn)
 		self.doc_order.append(fn)
 
-	def add_obj(self, obj):
-		self.objs.append(obj)
-		self.doc_order.append(obj)
+	def add_obj(self, o):
+		self.objs.append(o)
+		self.doc_order.append(o)
 
 	def c_proto(self, fo):
 		for o in self.objs:
@@ -332,7 +328,7 @@ class vmod(object):
 
 #######################################################################
 
-class func(object):
+class Func(object):
 	def __init__(self, nam, retval, al):
 		#if not is_c_name(nam):
 		#	raise Exception("Func name '%s' is illegal" % nam)
@@ -399,7 +395,7 @@ class func(object):
 	def c_initializer(self):
 		return "\tvmod_" + self.cnam + ",\n"
 
-	def c_strspec(self, modnam, pfx = "\t"):
+	def c_strspec(self, modnam, pfx="\t"):
 		s = pfx + '"' + modnam + "." + self.nam + '\\0"\n'
 		s += pfx + '"'
 		s += "Vmod_" + modnam + "_Func." + self.cnam + '\\0"\n'
@@ -448,7 +444,7 @@ class func(object):
 
 #######################################################################
 
-class obj(object):
+class Obj(object):
 	def __init__(self, nam):
 		self.nam = nam
 		self.init = None
@@ -467,7 +463,7 @@ class obj(object):
 
 	def set_init(self, f):
 		self.init = f
-		self.fini = func(f.nam, "VOID", [])
+		self.fini = Func(f.nam, "VOID", [])
 		self.init.cnam += "__init"
 		self.fini.cnam += "__fini"
 
@@ -549,7 +545,7 @@ class obj(object):
 
 #######################################################################
 
-class arg(object):
+class Arg(object):
 	def __init__(self, typ, nam=None, det=None):
 		self.nam = nam
 		self.typ = typ
@@ -589,7 +585,7 @@ def parse_enum2(tl):
 			raise ParseError(
 			    "Expected \"}\" or \",\" not \"%s\"" % t.str)
 	s += "\\0"
-	return arg("ENUM", det=s)
+	return Arg("ENUM", det=s)
 
 #######################################################################
 #
@@ -602,13 +598,13 @@ def parse_module(tl):
 	while len(tl.tl) > 0:
 		s += " " + tl.get_token().str
 	dnm = s[1:]
-	return vmod(nm, dnm, sec)
+	return Vmod(nm, dnm, sec)
 
 #######################################################################
 #
 #
 
-def parse_func(tl, rt_type=None, obj=None):
+def parse_func(tl, rt_type=None, pobj=None):
 	al = list()
 	if rt_type == None:
 		t = tl.get_token()
@@ -619,8 +615,8 @@ def parse_func(tl, rt_type=None, obj=None):
 
 	t = tl.get_token()
 	fname = t.str
-	if obj != None and fname[0] == "." and is_c_name(fname[1:]):
-		fname = obj + fname
+	if pobj != None and fname[0] == "." and is_c_name(fname[1:]):
+		fname = pobj + fname
 	elif not is_c_name(fname):
 		raise ParseError("Function name '%s' is illegal", fname)
 
@@ -637,7 +633,7 @@ def parse_func(tl, rt_type=None, obj=None):
 		if t.str == "ENUM":
 			al.append(parse_enum2(tl))
 		elif t.str in ctypes:
-			al.append(arg(t.str))
+			al.append(Arg(t.str))
 		elif t.str == ")":
 			break
 		else:
@@ -655,7 +651,7 @@ def parse_func(tl, rt_type=None, obj=None):
 			    "Expected \")\" or \",\" not \"%s\"" % t.str)
 	if t.str != ")":
 		raise ParseError("End Of Input looking for ')'")
-	f = func(fname, rt_type, al)
+	f = Func(fname, rt_type, al)
 
 	return f
 
@@ -665,7 +661,7 @@ def parse_func(tl, rt_type=None, obj=None):
 
 def parse_obj(tl):
 	f = parse_func(tl, "VOID")
-	o = obj(f.nam)
+	o = Obj(f.nam)
 	o.set_init(f)
 	return o
 
@@ -673,7 +669,7 @@ def parse_obj(tl):
 #######################################################################
 # A section of the inputvcc, starting at a keyword
 
-class file_section(object):
+class FileSection(object):
 	def __init__(self):
 		self.l = []
 		self.tl = []
@@ -701,7 +697,7 @@ class file_section(object):
 		if l == "":
 			return
 		for j in l.split():
-			self.tl.append(token(ln, 0, j))
+			self.tl.append(Token(ln, 0, j))
 
 	def parse(self, vx):
 		t = self.get_token()
@@ -729,7 +725,7 @@ class file_section(object):
 		elif t.str == "$Method":
 			if len(vx) != 2:
 				raise FormatError("$Method outside $Object", "")
-			o = parse_func(self, obj=vx[1].nam)
+			o = parse_func(self, pobj=vx[1].nam)
 			vx[1].add_method(o)
 		else:
 			raise FormatError("Unknown keyword: %s" % t.str, "")
@@ -792,20 +788,20 @@ def runmain(inputvcc, outputname="vcc_if"):
 	#######################################################################
 	# First collect the copyright:  All initial lines starting with '#'
 
-	copyright = []
+	copy_right = []
 	while len(lines[0]) > 0 and lines[0][0] == "#":
 		ln += 1
-		copyright.append(lines.pop(0))
+		copy_right.append(lines.pop(0))
 
-	if len(copyright) > 0:
-		if copyright[0] == "#-":
-			copyright = []
+	if len(copy_right) > 0:
+		if copy_right[0] == "#-":
+			copy_right = []
 		else:
-			while polish(copyright):
+			while polish(copy_right):
 				continue
 
 	if False:
-		for i in copyright:
+		for i in copy_right:
 			print("(C)\t", i)
 
 	#######################################################################
@@ -820,14 +816,14 @@ def runmain(inputvcc, outputname="vcc_if"):
 	}
 
 	sl = []
-	sc = file_section()
+	sc = FileSection()
 	sl.append(sc)
 	while len(lines) > 0:
 		ln += 1
 		l = lines.pop(0)
 		j = l.split()
 		if len(j) > 0 and j[0] in keywords:
-			sc = file_section()
+			sc = FileSection()
 			sl.append(sc)
 		sc.add_line(ln, l)
 
@@ -844,7 +840,8 @@ def runmain(inputvcc, outputname="vcc_if"):
 		pprint(str(e))
 		exit(-1)
 	except FormatError as e:
-		print("ERROR: Format error reading \"%s\": %s" % (inputvcc, pformat(e.msg)))
+		print("ERROR: Format error reading \"%s\": %s" %
+		    (inputvcc, pformat(e.msg)))
 		print(e.details)
 		exit(-2)
 
@@ -865,14 +862,11 @@ def runmain(inputvcc, outputname="vcc_if"):
 
 	vx[0].c_proto(fh)
 
-	fc.write("""#include "config.h"
-
-#include "vrt.h"
-#include "vcc_if.h"
-#include "vmod_abi.h"
-
-
-""")
+	fc.write('#include "config.h"\n')
+	fc.write('#include "vrt.h"\n')
+	fc.write('#include "vcc_if.h"\n')
+	fc.write('#include "vmod_abi.h"\n')
+	fc.write('\n')
 
 	vx[0].c_typedefs(fc)
 	vx[0].c_vmod(fc)
@@ -886,12 +880,12 @@ def runmain(inputvcc, outputname="vcc_if"):
 
 		vx[0].doc_dump(fp, suf)
 
-		if len(copyright) > 0:
+		if len(copy_right) > 0:
 			fp.write("\n")
 			fp.write("COPYRIGHT\n")
 			fp.write("=========\n")
 			fp.write("\n::\n\n")
-			for i in copyright:
+			for i in copy_right:
 				fp.write("  %s\n" % i)
 			fp.write("\n")
 
@@ -901,25 +895,27 @@ if __name__ == "__main__":
 	oparser = optparse.OptionParser(usage=usagetext)
 
 	oparser.add_option('-N', '--strict', action='store_true', default=False,
-			    help="Be strict when parsing input file. (vmod.vcc)")
+	    help="Be strict when parsing input file. (vmod.vcc)")
 	oparser.add_option('', '--runtests', action='store_true', default=False,
-			    dest="runtests", help=optparse.SUPPRESS_HELP)
+	    dest="runtests", help=optparse.SUPPRESS_HELP)
 	(opts, args) = oparser.parse_args()
 
 	if opts.runtests:
-		del sys.argv[1]  # Pop off --runtests, pass remaining to unittest.
+		# Pop off --runtests, pass remaining to unittest.
+		del sys.argv[1]
 		unittest.main()
 		exit()
 
-	inputvcc = None
+	i_vcc = None
 	if len(args) == 1 and exists(args[0]):
-		inputvcc = args[0]
+		i_vcc = args[0]
 	elif exists("vmod.vcc"):
-		if not inputvcc:
-		    inputvcc = "vmod.vcc"
+		if not i_vcc:
+			i_vcc = "vmod.vcc"
 	else:
-		print("ERROR: No vmod.vcc file supplied or found.", file=sys.stderr)
+		print("ERROR: No vmod.vcc file supplied or found.",
+		    file=sys.stderr)
 		oparser.print_help()
 		exit(-1)
 
-	runmain(inputvcc)
+	runmain(i_vcc)



From phk at FreeBSD.org  Mon Jun  2 08:26:12 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Mon, 02 Jun 2014 10:26:12 +0200
Subject: [master] 5f67a2e Remove some unused forward struct declarations
Message-ID: <E1WrNZM-0007f1-O5@project.varnish-software.com>


commit 5f67a2e41562ea78ef8c0fd270defcc3dca050ca
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Jun 2 08:25:56 2014 +0000

    Remove some unused forward struct declarations

diff --git a/include/vrt.h b/include/vrt.h
index bfbbb2d..5884b00 100644
--- a/include/vrt.h
+++ b/include/vrt.h
@@ -33,15 +33,12 @@
 
 struct req;
 struct busyobj;
-struct worker;
 struct vsl_log;
 struct http;
 struct ws;
-struct vsb;
 struct cli;
 struct director;
 struct VCL_conf;
-struct sockaddr_storage;
 struct suckaddr;
 
 /***********************************************************************



From phk at FreeBSD.org  Mon Jun  2 09:35:38 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Mon, 02 Jun 2014 11:35:38 +0200
Subject: [master] 51a5d55 Beautify the compiled VCL a bit
Message-ID: <E1WrOeY-0001Xr-JG@project.varnish-software.com>


commit 51a5d55314401d142ccd5c1cae0a9262d4dabefa
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Jun 2 08:56:55 2014 +0000

    Beautify the compiled VCL a bit

diff --git a/lib/libvcc/generate.py b/lib/libvcc/generate.py
index c3b51ff..75649db 100755
--- a/lib/libvcc/generate.py
+++ b/lib/libvcc/generate.py
@@ -726,7 +726,7 @@ def emit_file(fo, fd, bn):
 	x = 0
 	l = 0
 	fo.write("\n\t/* %s */\n\n" % fn)
-	fo.write('\tVSB_cat(sb, "/* ---===### %s ###===--- */\\n");\n' % bn)
+	fo.write('\tVSB_cat(sb, "/* ---===### %s ###===--- */\\n\\n");\n' % bn)
 	for c in fc:
 		if l == 0:
 			fo.write("\tVSB_cat(sb, \"")
@@ -764,9 +764,10 @@ def emit_file(fo, fd, bn):
 			fo.write("\"\n")
 			x = 0
 	if x != 0:
-		fo.write("\"")
+		fo.write("\"\n")
 	if l != 0:
 		fo.write("\t);\n")
+	fo.write('\tVSB_cat(sb, "\\n");\n')
 
 #######################################################################
 
diff --git a/lib/libvcc/vcc_compile.c b/lib/libvcc/vcc_compile.c
index 81a7620..31212aa 100644
--- a/lib/libvcc/vcc_compile.c
+++ b/lib/libvcc/vcc_compile.c
@@ -597,7 +597,7 @@ vcc_CompileSource(const struct vcc *tl0, struct vsb *sb, struct source *sp)
 	sym->wildcard = vcc_Stv_Wildcard;
 
 	vcl_output_lang_h(tl->fh);
-	Fh(tl, 0, "\n/* ---===### VCC generated code ###===---*/\n");
+	Fh(tl, 0, "/* ---===### VCC generated code ###===---*/\n");
 	Fh(tl, 0, "\nextern const struct VCL_conf VCL_conf;\n");
 
 	/* Macro for accessing directors */
diff --git a/lib/libvcc/vcc_vmod.c b/lib/libvcc/vcc_vmod.c
index 044cd24..1723007 100644
--- a/lib/libvcc/vcc_vmod.c
+++ b/lib/libvcc/vcc_vmod.c
@@ -104,8 +104,6 @@ vcc_ParseImport(struct vcc *tl)
 		bprintf(fn, "%s/libvmod_%.*s.so", tl->vmod_dir, PF(mod));
 	}
 
-	Fh(tl, 0, "static void *VGC_vmod_%.*s;\n", PF(mod));
-
 	ifp = New_IniFin(tl);
 
 	VSB_printf(ifp->ini, "\tif (VRT_Vmod_Init(&VGC_vmod_%.*s,\n", PF(mod));
@@ -177,7 +175,6 @@ vcc_ParseImport(struct vcc *tl)
 		vcc_ErrWhere(tl, mod);
 		return;
 	}
-	Fh(tl, 0, "static struct vmod_priv vmod_priv_%.*s;\n", PF(mod));
 	for (; *spec != NULL; spec++) {
 		p = *spec;
 		if (!strcmp(p, "OBJ")) {
@@ -207,5 +204,10 @@ vcc_ParseImport(struct vcc *tl)
 				sym->kind = SYM_PROC;
 		}
 	}
+
+	Fh(tl, 0, "\n/* --- BEGIN VMOD %.*s --- */\n\n", PF(mod));
+	Fh(tl, 0, "static void *VGC_vmod_%.*s;\n", PF(mod));
+	Fh(tl, 0, "static struct vmod_priv vmod_priv_%.*s;\n", PF(mod));
 	Fh(tl, 0, "\n%s\n", proto);
+	Fh(tl, 0, "\n/* --- END VMOD %.*s --- */\n\n", PF(mod));
 }



From martin at varnish-software.com  Mon Jun  2 11:14:44 2014
From: martin at varnish-software.com (Martin Blix Grydeland)
Date: Mon, 02 Jun 2014 13:14:44 +0200
Subject: [3.0] 1b6cc9d Fix v-c.o bug 1327 (http_max_hdr not multiple of 4)
 also for 3.0
Message-ID: <E1WrQCS-000198-36@project.varnish-software.com>


commit 1b6cc9d28eb3741292d8b87c033c08f1590dd166
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Mon Jun 2 13:11:47 2014 +0200

    Fix v-c.o bug 1327 (http_max_hdr not multiple of 4) also for 3.0
    
    Round up the header space with PRNDUP in session space allocation to
    fix alignment issue.
    
    Fixes: #1327

diff --git a/bin/varnishd/cache_session.c b/bin/varnishd/cache_session.c
index d94ac7b..9ea5333 100644
--- a/bin/varnishd/cache_session.c
+++ b/bin/varnishd/cache_session.c
@@ -117,6 +117,7 @@ ses_sm_alloc(void)
 	nws = params->sess_workspace;
 	nhttp = (uint16_t)params->http_max_hdr;
 	hl = HTTP_estimate(nhttp);
+	hl = PRNDUP(hl);
 	l = sizeof *sm + nws + 2 * hl;
 	p = malloc(l);
 	if (p == NULL)
diff --git a/bin/varnishtest/tests/r01327.vtc b/bin/varnishtest/tests/r01327.vtc
new file mode 100644
index 0000000..241b65d
--- /dev/null
+++ b/bin/varnishtest/tests/r01327.vtc
@@ -0,0 +1,14 @@
+varnishtest "#1327 http_max_hdr not multiple of 4"
+
+server s1 {
+	rxreq
+	txresp
+} -start
+
+varnish v1 -arg "-p http_max_hdr=65" -vcl+backend {
+} -start
+
+client c1 {
+	txreq
+	rxresp
+} -run



From martin at varnish-software.com  Mon Jun  2 14:02:24 2014
From: martin at varnish-software.com (Martin Blix Grydeland)
Date: Mon, 02 Jun 2014 16:02:24 +0200
Subject: [master] aadf302 Document the storage.<name>.* VCL variables.
Message-ID: <E1WrSoi-0006bP-5W@project.varnish-software.com>


commit aadf302e4b7d856070dbf92ab1e7a6571003812d
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Mon Jun 2 16:01:03 2014 +0200

    Document the storage.<name>.* VCL variables.
    
    Fixes: #1514

diff --git a/lib/libvcc/generate.py b/lib/libvcc/generate.py
index 75649db..a1fd2d7 100755
--- a/lib/libvcc/generate.py
+++ b/lib/libvcc/generate.py
@@ -596,9 +596,18 @@ aliases = [
 ]
 
 stv_variables = (
-	('free_space',	'BYTES',	"0."),
-	('used_space',	'BYTES',	"0."),
-	('happy',	'BOOL',		"0"),
+	('free_space',	'BYTES',	"0.", 'storage.<name>.free_space', """
+        Free space available in the named stevedore. Only available for
+        the malloc stevedore.
+        """),
+	('used_space',	'BYTES',	"0.", 'storage.<name>.used_space', """
+        Used space in the named stevedore. Only available for the malloc
+        stevedore.
+        """),
+	('happy',	'BOOL',		"0", 'storage.<name>.happy', """
+        Health status for the named stevedore. Not available in any of the
+        current stevedores.
+        """),
 )
 
 #######################################################################
@@ -1175,4 +1184,15 @@ for i in l:
 	for j in i[4].split("\n"):
 		fp_vclvar.write("\t%s\n" % j.strip())
 
+hdr="storage"
+fp_vclvar.write("\n" + hdr + "\n");
+fp_vclvar.write("~" * len(hdr) + "\n");
+for i in stv_variables:
+        fp_vclvar.write("\n" + i[3] + "\n\n")
+        fp_vclvar.write("\tType: " + i[1] + "\n\n")
+        fp_vclvar.write("\tReadable from: client, backend\n\n")
+        for j in i[4].split("\n"):
+                fp_vclvar.write("\t%s\n" % j.strip())
+
+
 fp_vclvar.close()



From martin at varnish-software.com  Mon Jun  2 14:02:33 2014
From: martin at varnish-software.com (Martin Blix Grydeland)
Date: Mon, 02 Jun 2014 16:02:33 +0200
Subject: [3.0] 73efe81 Document the storage.<name>.* VCL variables.
Message-ID: <E1WrSor-0006hy-EC@project.varnish-software.com>


commit 73efe8191aa764e3084d2a2bd738a60d7af964bc
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Mon Jun 2 14:46:36 2014 +0200

    Document the storage.<name>.* VCL variables.
    
    Fixes: #1514

diff --git a/doc/sphinx/reference/vcl.rst b/doc/sphinx/reference/vcl.rst
index ffd7c0c..b348e9d 100644
--- a/doc/sphinx/reference/vcl.rst
+++ b/doc/sphinx/reference/vcl.rst
@@ -903,6 +903,21 @@ resp.response
 resp.http.header
   The corresponding HTTP header.
 
+The following read-only variables report on the state of a named
+storage stevedore. Not all stevedores implement all of the variables:
+
+storage.<name>.free_space
+  Free space in bytes on the named stevedore. Only the malloc
+  stevedore implements this.
+
+storage.<name>.used_space
+  Used space in bytes on the named stevedore. Only the malloc
+  stevedore implements this.
+
+storage.<name>.happy
+  Health status for the named stevedore. None of the stevedores
+  implements this.
+
 Values may be assigned to variables using the set keyword:
 ::
 



From phk at FreeBSD.org  Tue Jun  3 09:28:17 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 03 Jun 2014 11:28:17 +0200
Subject: [master] b25cfdc Simplify http_Teardown() a bit.
Message-ID: <E1Wrl0z-0003sL-6G@project.varnish-software.com>


commit b25cfdc974aec31cd82d9ff3238fad8b25458c08
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Jun 3 09:28:01 2014 +0000

    Simplify http_Teardown() a bit.

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index da77ecb..a82ea69 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -186,16 +186,19 @@ struct http {
 	unsigned		magic;
 #define HTTP_MAGIC		0x6428b5c9
 
-	enum VSL_tag_e		logtag;		/* Must be SLT_*Method */
-	struct vsl_log		*vsl;
-
-	struct ws		*ws;
+	uint16_t		shd;		/* Size of hd space */
 	txt			*hd;
 	unsigned char		*hdf;
 #define HDF_FILTER		(1 << 0)	/* Filtered by Connection */
 #define HDF_MARKER		(1 << 1)	/* Marker bit */
-	uint16_t		shd;		/* Size of hd space */
+
+	/* NB: ->nhd and below zeroed/initialized by http_Teardown */
 	uint16_t		nhd;		/* Next free hd */
+
+	enum VSL_tag_e		logtag;		/* Must be SLT_*Method */
+	struct vsl_log		*vsl;
+
+	struct ws		*ws;
 	uint16_t		status;
 	uint8_t			protover;
 	uint8_t			conds;		/* If-* headers present */
diff --git a/bin/varnishd/cache/cache_http.c b/bin/varnishd/cache/cache_http.c
index 7fe6c9f..88ff577 100644
--- a/bin/varnishd/cache/cache_http.c
+++ b/bin/varnishd/cache/cache_http.c
@@ -32,6 +32,7 @@
 #include "config.h"
 
 #include <stdio.h>
+#include <stddef.h>
 
 #include "cache.h"
 
@@ -155,22 +156,13 @@ HTTP_Setup(struct http *hp, struct ws *ws, struct vsl_log *vsl,
 void
 http_Teardown(struct http *hp)
 {
-	uint16_t shd;
-	txt *hd;
-	unsigned char *hdf;
-
-	/* XXX: This is not elegant, is it efficient ? */
-	shd = hp->shd;
-	hd = hp->hd;
-	hdf = hp->hdf;
-	memset(hp, 0, sizeof *hp);
-	memset(hd, 0, sizeof *hd * shd);
-	memset(hdf, 0, sizeof *hdf * shd);
-	hp->magic = HTTP_MAGIC;
+
+	CHECK_OBJ_NOTNULL(hp, HTTP_MAGIC);
+	AN(hp->shd);
+	memset(&hp->nhd, 0, sizeof *hp - offsetof(struct http, nhd));
+	memset(hp->hd, 0, sizeof *hp->hd * hp->shd);
+	memset(hp->hdf, 0, sizeof *hp->hdf * hp->shd);
 	hp->nhd = HTTP_HDR_FIRST;
-	hp->shd = shd;
-	hp->hd = hd;
-	hp->hdf = hdf;
 }
 
 /*--------------------------------------------------------------------*/



From phk at FreeBSD.org  Tue Jun  3 12:42:37 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 03 Jun 2014 14:42:37 +0200
Subject: [master] 47d66bb Introduce http->failed to mark struct http's
 suffering from trouble.
Message-ID: <E1Wro33-000482-0U@project.varnish-software.com>


commit 47d66bb472669e4def4c180cd349e0552d3b4451
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Jun 3 11:55:18 2014 +0000

    Introduce http->failed to mark struct http's suffering from trouble.
    
    Usually, (probably always) this means failure to get workspace for
    modifications of the http.
    
    The intent is that this flag will have the same "latching" behaviour
    as error handling in VSB's:  Once set, it stays set and nobody
    reads or writes the struct http any more.
    
    Setting the flag causes a SLT_Error message.
    
    Rewrite http_CollectHdr() to respect failed and optimize it
    slightly while were here.

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index a82ea69..979eedd 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -202,6 +202,7 @@ struct http {
 	uint16_t		status;
 	uint8_t			protover;
 	uint8_t			conds;		/* If-* headers present */
+	uint8_t			failed;		/* usually: ws-alloc failed */
 };
 
 /*--------------------------------------------------------------------
diff --git a/bin/varnishd/cache/cache_http.c b/bin/varnishd/cache/cache_http.c
index 88ff577..49522cd 100644
--- a/bin/varnishd/cache/cache_http.c
+++ b/bin/varnishd/cache/cache_http.c
@@ -93,6 +93,15 @@ http_VSL_log(const struct http *hp)
 }
 
 /*--------------------------------------------------------------------*/
+
+static void
+http_fail(struct http *hp)
+{
+	VSLb(hp->vsl, SLT_Error, "out of workspace");
+	hp->failed = 1;
+}
+
+/*--------------------------------------------------------------------*/
 /* List of canonical HTTP response code names from RFC2616 */
 
 static struct http_msg {
@@ -146,12 +155,17 @@ HTTP_Setup(struct http *hp, struct ws *ws, struct vsl_log *vsl,
     enum VSL_tag_e  whence)
 {
 	http_Teardown(hp);
+	hp->nhd = HTTP_HDR_FIRST;
 	hp->logtag = whence;
 	hp->ws = ws;
 	hp->vsl = vsl;
 }
 
-/*--------------------------------------------------------------------*/
+/*--------------------------------------------------------------------
+ * http_Teardown() is a safety feature, we use it to zap all http
+ * structs once we're done with them, to minimize the risk that
+ * old stale pointers exist to no longer valid stuff.
+ */
 
 void
 http_Teardown(struct http *hp)
@@ -162,7 +176,6 @@ http_Teardown(struct http *hp)
 	memset(&hp->nhd, 0, sizeof *hp - offsetof(struct http, nhd));
 	memset(hp->hd, 0, sizeof *hp->hd * hp->shd);
 	memset(hp->hdf, 0, sizeof *hp->hdf * hp->shd);
-	hp->nhd = HTTP_HDR_FIRST;
 }
 
 /*--------------------------------------------------------------------*/
@@ -181,6 +194,26 @@ http_IsHdr(const txt *hh, const char *hdr)
 	return (!strncasecmp(hdr, hh->b, l));
 }
 
+/*--------------------------------------------------------------------*/
+
+static unsigned
+http_findhdr(const struct http *hp, unsigned l, const char *hdr)
+{
+	unsigned u;
+
+	for (u = HTTP_HDR_FIRST; u < hp->nhd; u++) {
+		Tcheck(hp->hd[u]);
+		if (hp->hd[u].e < hp->hd[u].b + l + 1)
+			continue;
+		if (hp->hd[u].b[l] != ':')
+			continue;
+		if (strncasecmp(hdr, hp->hd[u].b, l))
+			continue;
+		return (u);
+	}
+	return (0);
+}
+
 /*--------------------------------------------------------------------
  * This function collapses multiple headerlines of the same name.
  * The lines are joined with a comma, according to [rfc2616, 4.2bot, p32]
@@ -189,57 +222,60 @@ http_IsHdr(const txt *hh, const char *hdr)
 void
 http_CollectHdr(struct http *hp, const char *hdr)
 {
-	unsigned u, v, ml, f = 0, x;
+	unsigned u, l, ml, f, x, d;
 	char *b = NULL, *e = NULL;
 
-	for (u = HTTP_HDR_FIRST; u < hp->nhd; u++) {
-		while (u < hp->nhd && http_IsHdr(&hp->hd[u], hdr)) {
-			Tcheck(hp->hd[u]);
-			if (f == 0) {
-				/* Found first header, just record the fact */
-				f = u;
-				break;
-			}
-			if (b == NULL) {
-				/* Found second header, start our collection */
-				ml = WS_Reserve(hp->ws, 0);
-				b = hp->ws->f;
-				e = b + ml;
-				x = Tlen(hp->hd[f]);
-				if (b + x < e) {
-					memcpy(b, hp->hd[f].b, x);
-					b += x;
-				} else
-					b = e;
-			}
+	if (hp->failed)
+		return;
+	l = hdr[0];
+	assert(l == strlen(hdr + 1));
+	assert(hdr[l] == ':');
+	f = http_findhdr(hp, l - 1, hdr + 1);
+	if (f == 0)
+		return;
 
-			AN(b);
-			AN(e);
-
-			/* Append the Nth header we found */
-			if (b < e)
-				*b++ = ',';
-			x = Tlen(hp->hd[u]) - *hdr;
-			if (b + x < e) {
-				memcpy(b, hp->hd[u].b + *hdr, x);
-				b += x;
-			} else
-				b = e;
-
-			/* Shift remaining headers up one slot */
-			for (v = u; v < hp->nhd - 1; v++)
-				hp->hd[v] = hp->hd[v + 1];
-			hp->nhd--;
+	for (d = u = f + 1; u < hp->nhd; u++) {
+		Tcheck(hp->hd[u]);
+		if (!http_IsHdr(&hp->hd[u], hdr)) {
+			if (d != u)
+				hp->hd[d] = hp->hd[u];
+			d++;
+			continue;
+		}
+		if (b == NULL) {
+			/* Found second header, start our collection */
+			ml = WS_Reserve(hp->ws, 0);
+			b = hp->ws->f;
+			e = b + ml;
+			x = Tlen(hp->hd[f]);
+			if (b + x >= e) {
+				http_fail(hp);
+				WS_Release(hp->ws, 0);
+				return;
+			}
+			memcpy(b, hp->hd[f].b, x);
+			b += x;
 		}
 
+		AN(b);
+		AN(e);
+
+		/* Append the Nth header we found */
+		if (b < e)
+			*b++ = ',';
+		x = Tlen(hp->hd[u]) - l;
+		if (b + x >= e) {
+			http_fail(hp);
+			WS_Release(hp->ws, 0);
+			return;
+		}
+		memcpy(b, hp->hd[u].b + *hdr, x);
+		b += x;
 	}
 	if (b == NULL)
 		return;
+	hp->nhd = (uint16_t)d;
 	AN(e);
-	if (b >= e) {
-		WS_Release(hp->ws, 0);
-		return;
-	}
 	*b = '\0';
 	hp->hd[f].b = hp->ws->f;
 	hp->hd[f].e = b;
@@ -248,24 +284,6 @@ http_CollectHdr(struct http *hp, const char *hdr)
 
 /*--------------------------------------------------------------------*/
 
-static unsigned
-http_findhdr(const struct http *hp, unsigned l, const char *hdr)
-{
-	unsigned u;
-
-	for (u = HTTP_HDR_FIRST; u < hp->nhd; u++) {
-		Tcheck(hp->hd[u]);
-		if (hp->hd[u].e < hp->hd[u].b + l + 1)
-			continue;
-		if (hp->hd[u].b[l] != ':')
-			continue;
-		if (strncasecmp(hdr, hp->hd[u].b, l))
-			continue;
-		return (u);
-	}
-	return (0);
-}
-
 int
 http_GetHdr(const struct http *hp, const char *hdr, char **ptr)
 {



From phk at FreeBSD.org  Tue Jun  3 12:42:37 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 03 Jun 2014 14:42:37 +0200
Subject: [master] 48bac01 More http->failed infusion and associated cleanups.
Message-ID: <E1Wro33-000489-1z@project.varnish-software.com>


commit 48bac014d3f51b72505afdde661a8e24a307564e
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Jun 3 12:42:06 2014 +0000

    More http->failed infusion and associated cleanups.

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index 979eedd..1c2732b 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -987,15 +987,14 @@ struct http *HTTP_create(void *p, uint16_t nhttp);
 const char *http_StatusMessage(unsigned);
 unsigned http_EstimateWS(const struct http *fm, unsigned how, uint16_t *nhd);
 void HTTP_Init(void);
-void http_ClrHeader(struct http *to);
 void http_SetResp(struct http *to, const char *proto, uint16_t status,
     const char *response);
 void http_FilterReq(struct http *to, const struct http *fm, unsigned how);
 void http_FilterResp(const struct http *fm, struct http *to, unsigned how);
-void http_PutProtocol(const struct http *to, const char *protocol);
+void http_PutProtocol(struct http *to, const char *protocol);
 void http_PutStatus(struct http *to, uint16_t status);
 void http_ForceHeader(struct http *to, const char *hdr, const char *val);
-void http_PutResponse(const struct http *to, const char *response);
+void http_PutResponse(struct http *to, const char *response);
 void http_PrintfHeader(struct http *to, const char *fmt, ...)
     __printflike(2, 3);
 void http_SetHeader(struct http *to, const char *hdr);
@@ -1014,7 +1013,7 @@ const char *http_GetReq(const struct http *hp);
 int http_HdrIs(const struct http *hp, const char *hdr, const char *val);
 int http_IsHdr(const txt *hh, const char *hdr);
 enum sess_close http_DoConnection(const struct http *);
-void http_CopyHome(const struct http *hp);
+void http_CopyHome(struct http *hp);
 void http_Unset(struct http *hp, const char *hdr);
 void http_CollectHdr(struct http *hp, const char *hdr);
 void http_VSL_log(const struct http *hp);
diff --git a/bin/varnishd/cache/cache_http.c b/bin/varnishd/cache/cache_http.c
index 49522cd..032ac79 100644
--- a/bin/varnishd/cache/cache_http.c
+++ b/bin/varnishd/cache/cache_http.c
@@ -97,6 +97,8 @@ http_VSL_log(const struct http *hp)
 static void
 http_fail(struct http *hp)
 {
+
+	VSC_C_main->losthdr++;
 	VSLb(hp->vsl, SLT_Error, "out of workspace");
 	hp->failed = 1;
 }
@@ -180,6 +182,19 @@ http_Teardown(struct http *hp)
 
 /*--------------------------------------------------------------------*/
 
+void
+HTTP_Copy(struct http *to, const struct http * const fm)
+{
+
+	assert(fm->nhd <= to->shd);
+	memcpy(&to->nhd, &fm->nhd, sizeof *to - offsetof(struct http, nhd));
+	memcpy(to->hd, fm->hd, fm->nhd * sizeof *to->hd);
+	memcpy(to->hdf, fm->hdf, fm->nhd * sizeof *to->hdf);
+}
+
+
+/*--------------------------------------------------------------------*/
+
 int
 http_IsHdr(const txt *hh, const char *hdr)
 {
@@ -237,8 +252,10 @@ http_CollectHdr(struct http *hp, const char *hdr)
 	for (d = u = f + 1; u < hp->nhd; u++) {
 		Tcheck(hp->hd[u]);
 		if (!http_IsHdr(&hp->hd[u], hdr)) {
-			if (d != u)
+			if (d != u) {
 				hp->hd[d] = hp->hd[u];
+				hp->hdf[d] = hp->hdf[u];
+			}
 			d++;
 			continue;
 		}
@@ -250,6 +267,7 @@ http_CollectHdr(struct http *hp, const char *hdr)
 			x = Tlen(hp->hd[f]);
 			if (b + x >= e) {
 				http_fail(hp);
+				VSLb(hp->vsl, SLT_LostHeader, "%s", hdr + 1);
 				WS_Release(hp->ws, 0);
 				return;
 			}
@@ -266,6 +284,7 @@ http_CollectHdr(struct http *hp, const char *hdr)
 		x = Tlen(hp->hd[u]) - l;
 		if (b + x >= e) {
 			http_fail(hp);
+			VSLb(hp->vsl, SLT_LostHeader, "%s", hdr + 1);
 			WS_Release(hp->ws, 0);
 			return;
 		}
@@ -520,17 +539,6 @@ http_SetH(const struct http *to, unsigned n, const char *fm)
 	http_VSLH(to, n);
 }
 
-static void
-http_linkh(const struct http *to, const struct http *fm, unsigned n)
-{
-
-	assert(n < HTTP_HDR_FIRST);
-	Tcheck(fm->hd[n]);
-	to->hd[n] = fm->hd[n];
-	to->hdf[n] = fm->hdf[n];
-	http_VSLH(to, n);
-}
-
 void
 http_ForceGet(const struct http *to)
 {
@@ -596,26 +604,35 @@ http_filterfields(struct http *to, const struct http *fm, unsigned how)
 			continue;
 		if (fm->hdf[u] & HDF_FILTER)
 			continue;
+		Tcheck(fm->hd[u]);
 #define HTTPH(a, b, c) \
 		if (((c) & how) && http_IsHdr(&fm->hd[u], (b))) \
 			continue;
 #include "tbl/http_headers.h"
 #undef HTTPH
-		Tcheck(fm->hd[u]);
-		if (to->nhd < to->shd) {
-			to->hd[to->nhd] = fm->hd[u];
-			to->hdf[to->nhd] = 0;
-			http_VSLH(to, to->nhd);
-			to->nhd++;
-		} else  {
-			VSC_C_main->losthdr++;
-			VSLbt(to->vsl, SLT_LostHeader, fm->hd[u]);
-		}
+		assert (to->nhd < to->shd);
+		to->hd[to->nhd] = fm->hd[u];
+		to->hdf[to->nhd] = 0;
+		http_VSLH(to, to->nhd);
+		to->nhd++;
 	}
 }
 
 /*--------------------------------------------------------------------*/
 
+static void
+http_linkh(const struct http *to, const struct http *fm, unsigned n)
+{
+
+	assert(n < HTTP_HDR_FIRST);
+	Tcheck(fm->hd[n]);
+	to->hd[n] = fm->hd[n];
+	to->hdf[n] = fm->hdf[n];
+	http_VSLH(to, n);
+}
+
+/*--------------------------------------------------------------------*/
+
 void
 http_FilterReq(struct http *to, const struct http *fm, unsigned how)
 {
@@ -689,7 +706,7 @@ http_Merge(const struct http *fm, struct http *to, int not_ce)
  */
 
 void
-http_CopyHome(const struct http *hp)
+http_CopyHome(struct http *hp)
 {
 	unsigned u, l;
 	char *p;
@@ -697,48 +714,31 @@ http_CopyHome(const struct http *hp)
 	for (u = 0; u < hp->nhd; u++) {
 		if (hp->hd[u].b == NULL)
 			continue;
-		if (hp->hd[u].b >= hp->ws->s && hp->hd[u].e <= hp->ws->e) {
+		if (hp->hd[u].b >= hp->ws->s && hp->hd[u].e <= hp->ws->e)
 			continue;
-		}
+
 		l = Tlen(hp->hd[u]);
 		p = WS_Copy(hp->ws, hp->hd[u].b, l + 1L);
-		if (p != NULL) {
-			hp->hd[u].b = p;
-			hp->hd[u].e = p + l;
-		} else {
-			/* XXX This leaves a slot empty */
-			VSC_C_main->losthdr++;
-			VSLbt(hp->vsl, SLT_LostHeader, hp->hd[u]);
-			hp->hd[u].b = NULL;
-			hp->hd[u].e = NULL;
+		if (p == NULL) {
+			http_fail(hp);
+			VSLb(hp->vsl, SLT_LostHeader, "%s", hp->hd[u].b);
+			return;
 		}
+		hp->hd[u].b = p;
+		hp->hd[u].e = p + l;
 	}
 }
 
 /*--------------------------------------------------------------------*/
 
 void
-http_ClrHeader(struct http *to)
-{
-
-	CHECK_OBJ_NOTNULL(to, HTTP_MAGIC);
-	to->nhd = HTTP_HDR_FIRST;
-	to->status = 0;
-	to->protover = 0;
-	to->conds = 0;
-	memset(to->hd, 0, sizeof *to->hd * to->shd);
-}
-
-/*--------------------------------------------------------------------*/
-
-void
 http_SetHeader(struct http *to, const char *hdr)
 {
 
 	CHECK_OBJ_NOTNULL(to, HTTP_MAGIC);
 	if (to->nhd >= to->shd) {
-		VSC_C_main->losthdr++;
 		VSLb(to->vsl, SLT_LostHeader, "%s", hdr);
+		http_fail(to);
 		return;
 	}
 	http_SetH(to, to->nhd++, hdr);
@@ -760,32 +760,29 @@ http_ForceHeader(struct http *to, const char *hdr, const char *val)
 /*--------------------------------------------------------------------*/
 
 static void
-http_PutField(const struct http *to, int field, const char *string)
+http_PutField(struct http *to, int field, const char *string)
 {
 	char *p;
 
 	CHECK_OBJ_NOTNULL(to, HTTP_MAGIC);
 	p = WS_Copy(to->ws, string, -1);
 	if (p == NULL) {
+		http_fail(to);
 		VSLb(to->vsl, SLT_LostHeader, "%s", string);
-		to->hd[field].b = NULL;
-		to->hd[field].e = NULL;
-		to->hdf[field] = 0;
-	} else {
-		to->hd[field].b = p;
-		to->hd[field].e = strchr(p, '\0');
-		to->hdf[field] = 0;
-		http_VSLH(to, field);
+		return;
 	}
+	to->hd[field].b = p;
+	to->hd[field].e = strchr(p, '\0');
+	to->hdf[field] = 0;
+	http_VSLH(to, field);
 }
 
 void
-http_PutProtocol(const struct http *to, const char *protocol)
+http_PutProtocol(struct http *to, const char *protocol)
 {
 
+	AN(protocol);
 	http_PutField(to, HTTP_HDR_PROTO, protocol);
-	if (to->hd[HTTP_HDR_PROTO].b == NULL)
-		http_SetH(to, HTTP_HDR_PROTO, "HTTP/1.1");
 	Tcheck(to->hd[HTTP_HDR_PROTO]);
 }
 
@@ -801,12 +798,11 @@ http_PutStatus(struct http *to, uint16_t status)
 }
 
 void
-http_PutResponse(const struct http *to, const char *response)
+http_PutResponse(struct http *to, const char *response)
 {
 
+	AN(response);
 	http_PutField(to, HTTP_HDR_RESPONSE, response);
-	if (to->hd[HTTP_HDR_RESPONSE].b == NULL)
-		http_SetH(to, HTTP_HDR_RESPONSE, "Lost Response");
 	Tcheck(to->hd[HTTP_HDR_RESPONSE]);
 }
 
@@ -822,17 +818,17 @@ http_PrintfHeader(struct http *to, const char *fmt, ...)
 	n = vsnprintf(to->ws->f, l, fmt, ap);
 	va_end(ap);
 	if (n + 1 >= l || to->nhd >= to->shd) {
-		VSC_C_main->losthdr++;
+		http_fail(to);
 		VSLb(to->vsl, SLT_LostHeader, "%s", to->ws->f);
 		WS_Release(to->ws, 0);
-	} else {
-		to->hd[to->nhd].b = to->ws->f;
-		to->hd[to->nhd].e = to->ws->f + n;
-		to->hdf[to->nhd] = 0;
-		WS_Release(to->ws, n + 1);
-		http_VSLH(to, to->nhd);
-		to->nhd++;
-	}
+		return;
+	} 
+	to->hd[to->nhd].b = to->ws->f;
+	to->hd[to->nhd].e = to->ws->f + n;
+	to->hdf[to->nhd] = 0;
+	WS_Release(to->ws, n + 1);
+	http_VSLH(to, to->nhd);
+	to->nhd++;
 }
 
 /*--------------------------------------------------------------------*/
@@ -861,22 +857,6 @@ http_Unset(struct http *hp, const char *hdr)
 /*--------------------------------------------------------------------*/
 
 void
-HTTP_Copy(struct http *to, const struct http * const fm)
-{
-
-	to->conds = fm->conds;
-	to->logtag = fm->logtag;
-	to->status = fm->status;
-	to->protover = fm->protover;
-	to->nhd = fm->nhd;
-	assert(fm->nhd <= to->shd);
-	memcpy(to->hd, fm->hd, fm->nhd * sizeof *to->hd);
-	memcpy(to->hdf, fm->hdf, fm->nhd * sizeof *to->hdf);
-}
-
-/*--------------------------------------------------------------------*/
-
-void
 HTTP_Init(void)
 {
 
diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c
index 86fed3e..0eed42b 100644
--- a/bin/varnishd/cache/cache_req_fsm.c
+++ b/bin/varnishd/cache/cache_req_fsm.c
@@ -102,8 +102,6 @@ cnt_deliver(struct worker *wrk, struct req *req)
 		EXP_Touch(req->obj->objcore, req->t_prev);
 
 	HTTP_Setup(req->resp, req->ws, req->vsl, SLT_RespMethod);
-
-	http_ClrHeader(req->resp);
 	http_FilterResp(req->obj->http, req->resp, 0);
 
 	if (req->wrk->stats.cache_hit)
@@ -202,8 +200,6 @@ cnt_synth(struct worker *wrk, struct req *req)
 
 	wrk->stats.s_synth++;
 
-	HTTP_Setup(req->resp, req->ws, req->vsl, SLT_RespMethod);
-	h = req->resp;
 
 	now = W_TIM_real(wrk);
 	VSLb_ts_req(req, "Process", now);
@@ -211,7 +207,8 @@ cnt_synth(struct worker *wrk, struct req *req)
 	if (req->err_code < 100 || req->err_code > 999)
 		req->err_code = 501;
 
-	http_ClrHeader(h);
+	HTTP_Setup(req->resp, req->ws, req->vsl, SLT_RespMethod);
+	h = req->resp;
 	http_PutProtocol(h, "HTTP/1.1");
 	http_PutStatus(h, req->err_code);
 	VTIM_format(now, date);
@@ -235,7 +232,7 @@ cnt_synth(struct worker *wrk, struct req *req)
 	AZ(VSB_finish(req->synth_body));
 
 	if (wrk->handling == VCL_RET_RESTART) {
-		http_ClrHeader(h);
+		HTTP_Setup(h, req->ws, req->vsl, SLT_RespMethod);
 		VSB_delete(req->synth_body);
 		req->synth_body = NULL;
 		req->req_step = R_STP_RESTART;



From phk at FreeBSD.org  Wed Jun  4 07:49:42 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Wed, 04 Jun 2014 09:49:42 +0200
Subject: [master] 22b8488 Cut the thicket for setting a HTTP response's first
 line down to just one function which does what you'd expect.
Message-ID: <E1Ws5x8-0006rO-DQ@project.varnish-software.com>


commit 22b8488592d5b76af6ac0636e070861be3e892d9
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Jun 4 07:42:25 2014 +0000

    Cut the thicket for setting a HTTP response's first line down to just
    one function which does what you'd expect.
    
    Start allowing internal VCL use of *resp.status > 999.  The idea
    is that we ignore the higher bits but it still has to end up >=
    100, so 10404 becomes 404 and 20099 is illegal.   (This change
    is not yet complete it will panic if you use it yet.)
    
    More http->failed stuff.

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index 1c2732b..2429ccd 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -987,14 +987,11 @@ struct http *HTTP_create(void *p, uint16_t nhttp);
 const char *http_StatusMessage(unsigned);
 unsigned http_EstimateWS(const struct http *fm, unsigned how, uint16_t *nhd);
 void HTTP_Init(void);
-void http_SetResp(struct http *to, const char *proto, uint16_t status,
+void http_PutResponse(struct http *to, const char *proto, uint16_t status,
     const char *response);
 void http_FilterReq(struct http *to, const struct http *fm, unsigned how);
 void http_FilterResp(const struct http *fm, struct http *to, unsigned how);
-void http_PutProtocol(struct http *to, const char *protocol);
-void http_PutStatus(struct http *to, uint16_t status);
 void http_ForceHeader(struct http *to, const char *hdr, const char *val);
-void http_PutResponse(struct http *to, const char *response);
 void http_PrintfHeader(struct http *to, const char *fmt, ...)
     __printflike(2, 3);
 void http_SetHeader(struct http *to, const char *hdr);
diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c
index 7ad709d..450560c 100644
--- a/bin/varnishd/cache/cache_fetch.c
+++ b/bin/varnishd/cache/cache_fetch.c
@@ -686,7 +686,7 @@ vbf_stp_error(struct worker *wrk, struct busyobj *bo)
 	// XXX: reset all beresp flags ?
 
 	HTTP_Setup(bo->beresp, bo->ws, bo->vsl, SLT_BerespMethod);
-	http_SetResp(bo->beresp, "HTTP/1.1", 503, "Backend fetch failed");
+	http_PutResponse(bo->beresp, "HTTP/1.1", 503, "Backend fetch failed");
 	VTIM_format(now, time_str);
 	http_PrintfHeader(bo->beresp, "Date: %s", time_str);
 	http_SetHeader(bo->beresp, "Server: Varnish");
diff --git a/bin/varnishd/cache/cache_http.c b/bin/varnishd/cache/cache_http.c
index 032ac79..d4296cd 100644
--- a/bin/varnishd/cache/cache_http.c
+++ b/bin/varnishd/cache/cache_http.c
@@ -120,11 +120,12 @@ http_StatusMessage(unsigned status)
 {
 	struct http_msg *mp;
 
-	assert(status >= 100 && status <= 999);
+	status %= 1000;
+	assert(status >= 100);
 	for (mp = http_msg; mp->nbr != 0 && mp->nbr <= status; mp++)
 		if (mp->nbr == status)
 			return (mp->txt);
-	return ("Unknown Error");
+	return ("Unknown HTTP Status");
 }
 
 /*--------------------------------------------------------------------*/
@@ -527,6 +528,25 @@ http_GetReq(const struct http *hp)
 
 /*--------------------------------------------------------------------*/
 
+static void
+http_PutField(struct http *to, int field, const char *string)
+{
+	char *p;
+
+	CHECK_OBJ_NOTNULL(to, HTTP_MAGIC);
+	p = WS_Copy(to->ws, string, -1);
+	if (p == NULL) {
+		http_fail(to);
+		VSLb(to->vsl, SLT_LostHeader, "%s", string);
+		return;
+	}
+	to->hd[field].b = p;
+	to->hd[field].e = strchr(p, '\0');
+	to->hdf[field] = 0;
+	http_VSLH(to, field);
+}
+/*--------------------------------------------------------------------*/
+
 void
 http_SetH(const struct http *to, unsigned n, const char *fm)
 {
@@ -547,14 +567,24 @@ http_ForceGet(const struct http *to)
 }
 
 void
-http_SetResp(struct http *to, const char *proto, uint16_t status,
+http_PutResponse(struct http *to, const char *proto, uint16_t status,
     const char *response)
 {
+	char buf[4];
 
 	CHECK_OBJ_NOTNULL(to, HTTP_MAGIC);
 	http_SetH(to, HTTP_HDR_PROTO, proto);
-	assert(status >= 100 && status <= 999);
-	http_PutStatus(to, status);
+	/*
+	 * We allow people to use top digits for internal VCL
+	 * signalling, strip them here.
+	 */
+	status %= 1000;
+	assert(status >= 100);
+	to->status = status;
+	bprintf(buf, "%03d", status % 1000);
+	http_PutField(to, HTTP_HDR_STATUS, buf);
+	if (response == NULL)
+		response = http_StatusMessage(status);
 	http_SetH(to, HTTP_HDR_RESPONSE, response);
 }
 
@@ -757,55 +787,6 @@ http_ForceHeader(struct http *to, const char *hdr, const char *val)
 	http_PrintfHeader(to, "%s %s", hdr + 1, val);
 }
 
-/*--------------------------------------------------------------------*/
-
-static void
-http_PutField(struct http *to, int field, const char *string)
-{
-	char *p;
-
-	CHECK_OBJ_NOTNULL(to, HTTP_MAGIC);
-	p = WS_Copy(to->ws, string, -1);
-	if (p == NULL) {
-		http_fail(to);
-		VSLb(to->vsl, SLT_LostHeader, "%s", string);
-		return;
-	}
-	to->hd[field].b = p;
-	to->hd[field].e = strchr(p, '\0');
-	to->hdf[field] = 0;
-	http_VSLH(to, field);
-}
-
-void
-http_PutProtocol(struct http *to, const char *protocol)
-{
-
-	AN(protocol);
-	http_PutField(to, HTTP_HDR_PROTO, protocol);
-	Tcheck(to->hd[HTTP_HDR_PROTO]);
-}
-
-void
-http_PutStatus(struct http *to, uint16_t status)
-{
-	char buf[4];
-
-	assert(status >= 100 && status <= 999);
-	to->status = status;
-	bprintf(buf, "%03d", status % 1000);
-	http_PutField(to, HTTP_HDR_STATUS, buf);
-}
-
-void
-http_PutResponse(struct http *to, const char *response)
-{
-
-	AN(response);
-	http_PutField(to, HTTP_HDR_RESPONSE, response);
-	Tcheck(to->hd[HTTP_HDR_RESPONSE]);
-}
-
 void
 http_PrintfHeader(struct http *to, const char *fmt, ...)
 {
@@ -822,7 +803,7 @@ http_PrintfHeader(struct http *to, const char *fmt, ...)
 		VSLb(to->vsl, SLT_LostHeader, "%s", to->ws->f);
 		WS_Release(to->ws, 0);
 		return;
-	} 
+	}
 	to->hd[to->nhd].b = to->ws->f;
 	to->hd[to->nhd].e = to->ws->f + n;
 	to->hdf[to->nhd] = 0;
diff --git a/bin/varnishd/cache/cache_http1_deliver.c b/bin/varnishd/cache/cache_http1_deliver.c
index 0b0e138..06b4a32 100644
--- a/bin/varnishd/cache/cache_http1_deliver.c
+++ b/bin/varnishd/cache/cache_http1_deliver.c
@@ -154,7 +154,7 @@ v1d_dorange(struct req *req, const char *r)
 	if (req->res_mode & RES_LEN)
 		http_PrintfHeader(req->resp, "Content-Length: %jd",
 		    (intmax_t)(1 + high - low));
-	http_SetResp(req->resp, "HTTP/1.1", 206, "Partial Content");
+	http_PutResponse(req->resp, "HTTP/1.1", 206, NULL);
 
 	req->range_off = 0;
 	req->range_low = low;
diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c
index 0eed42b..d14f398 100644
--- a/bin/varnishd/cache/cache_req_fsm.c
+++ b/bin/varnishd/cache/cache_req_fsm.c
@@ -149,7 +149,7 @@ cnt_deliver(struct worker *wrk, struct req *req)
 	    && req->esi_level == 0
 	    && http_GetStatus(req->obj->http) == 200
 	    && req->http->conds && RFC2616_Do_Cond(req))
-		http_SetResp(req->resp, "HTTP/1.1", 304, "Not Modified");
+		http_PutResponse(req->resp, "HTTP/1.1", 304, NULL);
 
 	V1D_Deliver(req);
 	VSLb_ts_req(req, "Resp", W_TIM_real(wrk));
@@ -209,17 +209,12 @@ cnt_synth(struct worker *wrk, struct req *req)
 
 	HTTP_Setup(req->resp, req->ws, req->vsl, SLT_RespMethod);
 	h = req->resp;
-	http_PutProtocol(h, "HTTP/1.1");
-	http_PutStatus(h, req->err_code);
 	VTIM_format(now, date);
 	http_PrintfHeader(h, "Date: %s", date);
 	http_SetHeader(h, "Server: Varnish");
 	http_PrintfHeader(req->resp,
 	    "X-Varnish: %u", req->vsl->wid & VSL_IDENTMASK);
-	if (req->err_reason != NULL)
-		http_PutResponse(h, req->err_reason);
-	else
-		http_PutResponse(h, http_StatusMessage(req->err_code));
+	http_PutResponse(h, "HTTP/1.1", req->err_code, req->err_reason);
 
 	AZ(req->synth_body);
 	req->synth_body = VSB_new_auto();
diff --git a/bin/varnishd/cache/cache_vrt_var.c b/bin/varnishd/cache/cache_vrt_var.c
index 65c72db..d2e6647 100644
--- a/bin/varnishd/cache/cache_vrt_var.c
+++ b/bin/varnishd/cache/cache_vrt_var.c
@@ -48,19 +48,20 @@ static char vrt_hostname[255] = "";
 /*--------------------------------------------------------------------*/
 
 static void
-vrt_do_string(const struct http *hp, int fld,
+vrt_do_string(struct http *hp, int fld,
     const char *err, const char *p, va_list ap)
 {
 	const char *b;
 
-	AN(hp);
+	CHECK_OBJ_NOTNULL(hp, HTTP_MAGIC);
+
 	b = VRT_String(hp->ws, NULL, p, ap);
 	if (b == NULL || *b == '\0') {
 		VSLb(hp->vsl, SLT_LostHeader, "%s", err);
+		hp->failed = 1;
 	} else {
 		http_SetH(hp, fld, b);
 	}
-	va_end(ap);
 }
 
 #define VRT_HDR_L(obj, hdr, fld)					\
@@ -95,8 +96,15 @@ VRT_l_##obj##_status(const struct vrt_ctx *ctx, long num)		\
 									\
 	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);				\
 	CHECK_OBJ_NOTNULL(ctx->http_##obj, HTTP_MAGIC);			\
-	assert(num >= 100 && num <= 999);				\
-	ctx->http_##obj->status = (uint16_t)num;			\
+	if (num > 65535) {						\
+		VSLb(ctx->vsl, SLT_VCL_Error, "%s.status > 65535", #obj); \
+		ctx->http_##obj->failed = 1;				\
+	} else if ((num % 1000) < 100) {				\
+		VSLb(ctx->vsl, SLT_VCL_Error, "illegal %s.status (..0##)", \
+		    #obj);						\
+		ctx->http_##obj->failed = 1;				\
+	} else								\
+		ctx->http_##obj->status = (uint16_t)num;		\
 }
 
 #define VRT_STATUS_R(obj)						\



From phk at FreeBSD.org  Wed Jun  4 08:06:25 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Wed, 04 Jun 2014 10:06:25 +0200
Subject: [master] d5affe5 Having wasted half an hour on it (again) it's time
 to kill the "response/reason" confusion for good.
Message-ID: <E1Ws6DJ-0008Re-5j@project.varnish-software.com>


commit d5affe570d38cee48dcbdca911aed28399a4771d
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Jun 4 08:05:48 2014 +0000

    Having wasted half an hour on it (again) it's time to kill the
    "response/reason" confusion for good.

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index 2429ccd..681c857 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -984,7 +984,7 @@ void VGZ_WrwFlush(struct req *, struct vgz *vg);
 unsigned HTTP_estimate(unsigned nhttp);
 void HTTP_Copy(struct http *to, const struct http * const fm);
 struct http *HTTP_create(void *p, uint16_t nhttp);
-const char *http_StatusMessage(unsigned);
+const char *http_Status2Reason(unsigned);
 unsigned http_EstimateWS(const struct http *fm, unsigned how, uint16_t *nhd);
 void HTTP_Init(void);
 void http_PutResponse(struct http *to, const char *proto, uint16_t status,
diff --git a/bin/varnishd/cache/cache_http.c b/bin/varnishd/cache/cache_http.c
index d4296cd..1b17cac 100644
--- a/bin/varnishd/cache/cache_http.c
+++ b/bin/varnishd/cache/cache_http.c
@@ -116,7 +116,7 @@ static struct http_msg {
 };
 
 const char *
-http_StatusMessage(unsigned status)
+http_Status2Reason(unsigned status)
 {
 	struct http_msg *mp;
 
@@ -568,7 +568,7 @@ http_ForceGet(const struct http *to)
 
 void
 http_PutResponse(struct http *to, const char *proto, uint16_t status,
-    const char *response)
+    const char *reason)
 {
 	char buf[4];
 
@@ -583,9 +583,9 @@ http_PutResponse(struct http *to, const char *proto, uint16_t status,
 	to->status = status;
 	bprintf(buf, "%03d", status % 1000);
 	http_PutField(to, HTTP_HDR_STATUS, buf);
-	if (response == NULL)
-		response = http_StatusMessage(status);
-	http_SetH(to, HTTP_HDR_RESPONSE, response);
+	if (reason == NULL)
+		reason = http_Status2Reason(status);
+	http_SetH(to, HTTP_HDR_REASON, reason);
 }
 
 /*--------------------------------------------------------------------
@@ -689,7 +689,7 @@ http_FilterResp(const struct http *fm, struct http *to, unsigned how)
 	http_SetH(to, HTTP_HDR_PROTO, "HTTP/1.1");
 	to->status = fm->status;
 	http_linkh(to, fm, HTTP_HDR_STATUS);
-	http_linkh(to, fm, HTTP_HDR_RESPONSE);
+	http_linkh(to, fm, HTTP_HDR_REASON);
 	http_filterfields(to, fm, how);
 }
 
@@ -708,7 +708,7 @@ http_Merge(const struct http *fm, struct http *to, int not_ce)
 	to->status = fm->status;
 	http_SetH(to, HTTP_HDR_PROTO, fm->hd[HTTP_HDR_PROTO].b);
 	http_SetH(to, HTTP_HDR_STATUS, fm->hd[HTTP_HDR_STATUS].b);
-	http_SetH(to, HTTP_HDR_RESPONSE, fm->hd[HTTP_HDR_RESPONSE].b);
+	http_SetH(to, HTTP_HDR_REASON, fm->hd[HTTP_HDR_REASON].b);
 
 	for (u = HTTP_HDR_FIRST; u < fm->nhd; u++)
 		fm->hdf[u] |= HDF_MARKER;
diff --git a/bin/varnishd/cache/cache_http1_proto.c b/bin/varnishd/cache/cache_http1_proto.c
index a896546..4daa814 100644
--- a/bin/varnishd/cache/cache_http1_proto.c
+++ b/bin/varnishd/cache/cache_http1_proto.c
@@ -310,7 +310,7 @@ htc_splitline(struct http *hp, const struct http_conn *htc, int req)
 	} else {
 		h1 = HTTP_HDR_PROTO;
 		h2 = HTTP_HDR_STATUS;
-		h3 = HTTP_HDR_RESPONSE;
+		h3 = HTTP_HDR_REASON;
 	}
 
 	/* Skip leading LWS */
@@ -493,13 +493,13 @@ HTTP1_DissectResponse(struct http *hp, const struct http_conn *htc)
 	} else
 		htc_proto_ver(hp);
 
-	if (hp->hd[HTTP_HDR_RESPONSE].b == NULL ||
-	    !Tlen(hp->hd[HTTP_HDR_RESPONSE])) {
+	if (hp->hd[HTTP_HDR_REASON].b == NULL ||
+	    !Tlen(hp->hd[HTTP_HDR_REASON])) {
 		/* Backend didn't send a response string, use the standard */
-		hp->hd[HTTP_HDR_RESPONSE].b =
-		    TRUST_ME(http_StatusMessage(hp->status));
-		hp->hd[HTTP_HDR_RESPONSE].e =
-		    strchr(hp->hd[HTTP_HDR_RESPONSE].b, '\0');
+		hp->hd[HTTP_HDR_REASON].b =
+		    TRUST_ME(http_Status2Reason(hp->status));
+		hp->hd[HTTP_HDR_REASON].e =
+		    strchr(hp->hd[HTTP_HDR_REASON].b, '\0');
 	}
 	return (retval);
 }
@@ -523,7 +523,7 @@ HTTP1_Write(const struct worker *w, const struct http *hp, int resp)
 
 		l += WRW_WriteH(w, &hp->hd[HTTP_HDR_STATUS], " ");
 
-		l += WRW_WriteH(w, &hp->hd[HTTP_HDR_RESPONSE], "\r\n");
+		l += WRW_WriteH(w, &hp->hd[HTTP_HDR_REASON], "\r\n");
 	} else {
 		AN(hp->hd[HTTP_HDR_URL].b);
 		l = WRW_WriteH(w, &hp->hd[HTTP_HDR_METHOD], " ");
diff --git a/bin/varnishd/cache/cache_vrt.c b/bin/varnishd/cache/cache_vrt.c
index 08e3b77..4038693 100644
--- a/bin/varnishd/cache/cache_vrt.c
+++ b/bin/varnishd/cache/cache_vrt.c
@@ -63,7 +63,7 @@ VRT_error(const struct vrt_ctx *ctx, unsigned code, const char *reason)
 		code = 503;
 	ctx->req->err_code = (uint16_t)code;
 	ctx->req->err_reason =
-	    reason ? reason : http_StatusMessage(ctx->req->err_code);
+	    reason ? reason : http_Status2Reason(ctx->req->err_code);
 }
 
 /*--------------------------------------------------------------------*/
diff --git a/bin/varnishd/cache/cache_vrt_var.c b/bin/varnishd/cache/cache_vrt_var.c
index d2e6647..293b029 100644
--- a/bin/varnishd/cache/cache_vrt_var.c
+++ b/bin/varnishd/cache/cache_vrt_var.c
@@ -122,11 +122,11 @@ VRT_HDR_LR(req,    url,		HTTP_HDR_URL)
 VRT_HDR_LR(req,    proto,	HTTP_HDR_PROTO)
 
 VRT_HDR_R(obj,    proto,	HTTP_HDR_PROTO)
-VRT_HDR_R(obj,    reason,	HTTP_HDR_RESPONSE)
+VRT_HDR_R(obj,    reason,	HTTP_HDR_REASON)
 VRT_STATUS_R(obj)
 
 VRT_HDR_LR(resp,   proto,	HTTP_HDR_PROTO)
-VRT_HDR_LR(resp,   reason,	HTTP_HDR_RESPONSE)
+VRT_HDR_LR(resp,   reason,	HTTP_HDR_REASON)
 VRT_STATUS_L(resp)
 VRT_STATUS_R(resp)
 
@@ -134,7 +134,7 @@ VRT_HDR_LR(bereq,  method,	HTTP_HDR_METHOD)
 VRT_HDR_LR(bereq,  url,		HTTP_HDR_URL)
 VRT_HDR_LR(bereq,  proto,	HTTP_HDR_PROTO)
 VRT_HDR_LR(beresp, proto,	HTTP_HDR_PROTO)
-VRT_HDR_LR(beresp, reason,	HTTP_HDR_RESPONSE)
+VRT_HDR_LR(beresp, reason,	HTTP_HDR_REASON)
 VRT_STATUS_L(beresp)
 VRT_STATUS_R(beresp)
 
diff --git a/include/tbl/vsl_tags_http.h b/include/tbl/vsl_tags_http.h
index 3a3ae50..ab09d70 100644
--- a/include/tbl/vsl_tags_http.h
+++ b/include/tbl/vsl_tags_http.h
@@ -55,7 +55,7 @@ SLTH(Protocol,	HTTP_HDR_PROTO,		1, 1, "protocol",
 SLTH(Status,	HTTP_HDR_STATUS,	0, 1, "status",
 	"The HTTP status code received.\n\n"
 )
-SLTH(Response,	HTTP_HDR_RESPONSE,	0, 1, "response",
+SLTH(Reason,	HTTP_HDR_REASON,	0, 1, "response",
 	"The HTTP response string received.\n\n"
 )
 SLTH(Header,	HTTP_HDR_FIRST,		1, 1, "header",



From phk at FreeBSD.org  Wed Jun  4 08:53:35 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Wed, 04 Jun 2014 10:53:35 +0200
Subject: [master] 9f1222e More http_ cleanup:
Message-ID: <E1Ws6wx-00041i-5O@project.varnish-software.com>


commit 9f1222ed48c3105ca395d88dc2d75ef77223da81
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Jun 4 08:51:10 2014 +0000

    More http_ cleanup:
    
    Move policy decisions out where they belong and let the http_Filter
    function send the first like through unmolested.
    
    Rather than value specific functions, make a generic
    http_ForceField() function, because the name is cool.

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index 681c857..22e0362 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -996,7 +996,7 @@ void http_PrintfHeader(struct http *to, const char *fmt, ...)
     __printflike(2, 3);
 void http_SetHeader(struct http *to, const char *hdr);
 void http_SetH(const struct http *to, unsigned n, const char *fm);
-void http_ForceGet(const struct http *to);
+void http_ForceField(const struct http *to, unsigned n, const char *t);
 void HTTP_Setup(struct http *, struct ws *, struct vsl_log *, enum VSL_tag_e);
 void http_Teardown(struct http *ht);
 int http_GetHdr(const struct http *hp, const char *hdr, char **ptr);
diff --git a/bin/varnishd/cache/cache_esi_deliver.c b/bin/varnishd/cache/cache_esi_deliver.c
index a246985..ee1b0e5 100644
--- a/bin/varnishd/cache/cache_esi_deliver.c
+++ b/bin/varnishd/cache/cache_esi_deliver.c
@@ -79,8 +79,12 @@ ved_include(struct req *preq, const char *src, const char *host)
 		http_SetHeader(req->http0, host);
 	}
 
-	http_ForceGet(req->http0);
+	http_ForceField(req->http0, HTTP_HDR_METHOD, "GET");
+	http_ForceField(req->http0, HTTP_HDR_PROTO, "HTTP/1.1");
+
+	/* Don't allow Conditions, we can't use a 304 */
 	http_Unset(req->http0, H_If_Modified_Since);
+	http_Unset(req->http0, H_If_None_Match);
 
 	/* Client content already taken care of */
 	http_Unset(req->http0, H_Content_Length);
diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c
index 450560c..c175e74 100644
--- a/bin/varnishd/cache/cache_fetch.c
+++ b/bin/varnishd/cache/cache_fetch.c
@@ -190,7 +190,8 @@ vbf_stp_mkbereq(const struct worker *wrk, struct busyobj *bo)
 	    bo->do_pass ? HTTPH_R_PASS : HTTPH_R_FETCH);
 
 	if (!bo->do_pass) {
-		http_ForceGet(bo->bereq0);
+		http_ForceField(bo->bereq0, HTTP_HDR_METHOD, "GET");
+		http_ForceField(bo->bereq0, HTTP_HDR_PROTO, "HTTP/1.1");
 		if (cache_param->http_gzip_support)
 			http_ForceHeader(bo->bereq0, H_Accept_Encoding, "gzip");
 		AN(bo->req);
diff --git a/bin/varnishd/cache/cache_http.c b/bin/varnishd/cache/cache_http.c
index 1b17cac..1d1f872 100644
--- a/bin/varnishd/cache/cache_http.c
+++ b/bin/varnishd/cache/cache_http.c
@@ -545,6 +545,7 @@ http_PutField(struct http *to, int field, const char *string)
 	to->hdf[field] = 0;
 	http_VSLH(to, field);
 }
+
 /*--------------------------------------------------------------------*/
 
 void
@@ -559,13 +560,22 @@ http_SetH(const struct http *to, unsigned n, const char *fm)
 	http_VSLH(to, n);
 }
 
+/*--------------------------------------------------------------------
+ * Force a particular header field to a particular value
+ */
+
 void
-http_ForceGet(const struct http *to)
+http_ForceField(const struct http *to, unsigned n, const char *t)
 {
-	if (strcmp(http_GetReq(to), "GET"))
-		http_SetH(to, HTTP_HDR_METHOD, "GET");
+	CHECK_OBJ_NOTNULL(to, HTTP_MAGIC);
+	assert(n < HTTP_HDR_FIRST);
+	AN(t);
+	if (to->hd[n].b == NULL || strcmp(to->hd[n].b, t))
+		http_SetH(to, n, t);
 }
 
+/*--------------------------------------------------------------------*/
+
 void
 http_PutResponse(struct http *to, const char *proto, uint16_t status,
     const char *reason)
@@ -671,10 +681,7 @@ http_FilterReq(struct http *to, const struct http *fm, unsigned how)
 
 	http_linkh(to, fm, HTTP_HDR_METHOD);
 	http_linkh(to, fm, HTTP_HDR_URL);
-	if (how == HTTPH_R_FETCH)
-		http_SetH(to, HTTP_HDR_PROTO, "HTTP/1.1");
-	else
-		http_linkh(to, fm, HTTP_HDR_PROTO);
+	http_linkh(to, fm, HTTP_HDR_PROTO);
 	http_filterfields(to, fm, how);
 }
 
@@ -686,8 +693,8 @@ http_FilterResp(const struct http *fm, struct http *to, unsigned how)
 
 	CHECK_OBJ_NOTNULL(fm, HTTP_MAGIC);
 	CHECK_OBJ_NOTNULL(to, HTTP_MAGIC);
-	http_SetH(to, HTTP_HDR_PROTO, "HTTP/1.1");
 	to->status = fm->status;
+	http_linkh(to, fm, HTTP_HDR_PROTO);
 	http_linkh(to, fm, HTTP_HDR_STATUS);
 	http_linkh(to, fm, HTTP_HDR_REASON);
 	http_filterfields(to, fm, how);
@@ -706,9 +713,9 @@ http_Merge(const struct http *fm, struct http *to, int not_ce)
 	const char *p;
 
 	to->status = fm->status;
-	http_SetH(to, HTTP_HDR_PROTO, fm->hd[HTTP_HDR_PROTO].b);
-	http_SetH(to, HTTP_HDR_STATUS, fm->hd[HTTP_HDR_STATUS].b);
-	http_SetH(to, HTTP_HDR_REASON, fm->hd[HTTP_HDR_REASON].b);
+	http_linkh(to, fm, HTTP_HDR_PROTO);
+	http_linkh(to, fm, HTTP_HDR_STATUS);
+	http_linkh(to, fm, HTTP_HDR_REASON);
 
 	for (u = HTTP_HDR_FIRST; u < fm->nhd; u++)
 		fm->hdf[u] |= HDF_MARKER;
diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c
index d14f398..4c8ca8e 100644
--- a/bin/varnishd/cache/cache_req_fsm.c
+++ b/bin/varnishd/cache/cache_req_fsm.c
@@ -103,6 +103,7 @@ cnt_deliver(struct worker *wrk, struct req *req)
 
 	HTTP_Setup(req->resp, req->ws, req->vsl, SLT_RespMethod);
 	http_FilterResp(req->obj->http, req->resp, 0);
+	http_ForceField(req->resp, HTTP_HDR_PROTO, "HTTP/1.1");
 
 	if (req->wrk->stats.cache_hit)
 		http_PrintfHeader(req->resp,



From phk at FreeBSD.org  Wed Jun  4 12:22:01 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Wed, 04 Jun 2014 14:22:01 +0200
Subject: [master] 1cf2d3c Reduce default thread pool size to 10 threads,
 no need to stress the pthread implementation in *every* test case.
Message-ID: <E1WsACf-0005VT-6r@project.varnish-software.com>


commit 1cf2d3c4dbd9923d41dde296252a51b727b74188
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Jun 4 12:20:54 2014 +0000

    Reduce default thread pool size to 10 threads, no need to stress
    the pthread implementation in *every* test case.

diff --git a/bin/varnishtest/vtc_varnish.c b/bin/varnishtest/vtc_varnish.c
index 14b94be..c559810 100644
--- a/bin/varnishtest/vtc_varnish.c
+++ b/bin/varnishtest/vtc_varnish.c
@@ -386,6 +386,7 @@ varnish_launch(struct varnish *v)
 	VSB_printf(vsb, " -p auto_restart=off");
 	VSB_printf(vsb, " -p syslog_cli_traffic=off");
 	VSB_printf(vsb, " -p sigsegv_handler=on");
+	VSB_printf(vsb, " -p thread_pool_min=10");
 	VSB_printf(vsb, " -a '%s'", "127.0.0.1:0");
 	VSB_printf(vsb, " -M '%s %s'", abuf, pbuf);
 	VSB_printf(vsb, " -P %s/varnishd.pid", v->workdir);



From phk at FreeBSD.org  Wed Jun  4 12:22:01 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Wed, 04 Jun 2014 14:22:01 +0200
Subject: [master] e51f32d More struct http polishing
Message-ID: <E1WsACf-0005VZ-8a@project.varnish-software.com>


commit e51f32d1db9e8c1ec813ef2c7fdf825979e3435c
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Jun 4 12:21:38 2014 +0000

    More struct http polishing

diff --git a/bin/varnishd/cache/cache_vrt_var.c b/bin/varnishd/cache/cache_vrt_var.c
index 293b029..3ce5817 100644
--- a/bin/varnishd/cache/cache_vrt_var.c
+++ b/bin/varnishd/cache/cache_vrt_var.c
@@ -45,7 +45,9 @@
 
 static char vrt_hostname[255] = "";
 
-/*--------------------------------------------------------------------*/
+/*--------------------------------------------------------------------
+ * VRT variables relating to first line of HTTP/1.1 req/resp
+ */
 
 static void
 vrt_do_string(struct http *hp, int fld,
@@ -59,9 +61,9 @@ vrt_do_string(struct http *hp, int fld,
 	if (b == NULL || *b == '\0') {
 		VSLb(hp->vsl, SLT_LostHeader, "%s", err);
 		hp->failed = 1;
-	} else {
-		http_SetH(hp, fld, b);
+		return;
 	}
+	http_SetH(hp, fld, b);
 }
 
 #define VRT_HDR_L(obj, hdr, fld)					\
@@ -138,7 +140,9 @@ VRT_HDR_LR(beresp, reason,	HTTP_HDR_REASON)
 VRT_STATUS_L(beresp)
 VRT_STATUS_R(beresp)
 
-/*--------------------------------------------------------------------*/
+/*--------------------------------------------------------------------
+ * bool-fields (.do_*)
+ */
 
 #define VBERESPW0(field)
 #define VBERESPW1(field)						\
@@ -235,9 +239,13 @@ VRT_l_client_identity(const struct vrt_ctx *ctx, const char *str, ...)
 	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
 	CHECK_OBJ_NOTNULL(ctx->req, REQ_MAGIC);
 	va_start(ap, str);
-	// XXX ?
 	b = VRT_String(ctx->req->http->ws, NULL, str, ap);
 	va_end(ap);
+	if (b == NULL) {
+		VSLb(ctx->vsl, SLT_LostHeader, "client.identity");
+		ctx->req->http->failed = 1;
+		return;
+	}
 	ctx->req->client_identity = b;
 }
 
@@ -296,6 +304,8 @@ VRT_r_beresp_backend_ip(const struct vrt_ctx *ctx)
 		return (NULL);
 }
 
+/*--------------------------------------------------------------------*/
+
 const char *
 VRT_r_beresp_storage_hint(const struct vrt_ctx *ctx)
 {
@@ -318,6 +328,11 @@ VRT_l_beresp_storage_hint(const struct vrt_ctx *ctx, const char *str, ...)
 	va_start(ap, str);
 	b = VRT_String(ctx->bo->ws, NULL, str, ap);	// XXX: ctx->ws ?
 	va_end(ap);
+	if (b == NULL) {
+		VSLb(ctx->vsl, SLT_LostHeader, "storage.hint");
+		ctx->bo->beresp->failed = 1;
+		return;
+	}
 	ctx->bo->storage_hint = b;
 }
 
@@ -414,7 +429,6 @@ VRT_r_req_can_gzip(const struct vrt_ctx *ctx)
 	return (RFC2616_Req_Gzip(ctx->req->http));	// XXX ?
 }
 
-
 /*--------------------------------------------------------------------*/
 
 long



From lkarsten at varnish-software.com  Thu Jun  5 14:12:34 2014
From: lkarsten at varnish-software.com (Lasse Karstensen)
Date: Thu, 05 Jun 2014 16:12:34 +0200
Subject: [master] 80e9e0c Add version requirement for docutils.
Message-ID: <E1WsYPC-0008J3-Sl@project.varnish-software.com>


commit 80e9e0cbf9e7aea064ca566a177669ae7b84fe09
Author: Lasse Karstensen <lkarsten at varnish-software.com>
Date:   Thu Jun 5 16:11:18 2014 +0200

    Add version requirement for docutils.
    
    The rst2man utility exists only in "newer" versions of python-docutils.
    
    Require at least 0.7, released July 2010.

diff --git a/redhat/varnish.spec b/redhat/varnish.spec
index 09752b8..82c17a7 100644
--- a/redhat/varnish.spec
+++ b/redhat/varnish.spec
@@ -17,7 +17,8 @@ BuildRoot: %{_tmppath}/%{name}-%{version}-%{release}-root-%(%{__id_u} -n)
 # To build from git, start with a make dist, see redhat/README.redhat 
 # You will need at least automake autoconf libtool
 #BuildRequires: automake autoconf libtool
-BuildRequires: ncurses-devel groff pcre-devel pkgconfig python-docutils libedit-devel jemalloc-devel
+BuildRequires: ncurses-devel groff pcre-devel pkgconfig libedit-devel jemalloc-devel
+BuildRequires: python-docutils <= 0.7
 Requires: varnish-libs = %{version}-%{release}
 Requires: logrotate
 Requires: ncurses



From lkarsten at varnish-software.com  Thu Jun  5 14:17:43 2014
From: lkarsten at varnish-software.com (Lasse Karstensen)
Date: Thu, 05 Jun 2014 16:17:43 +0200
Subject: [master] b356206 Correct wrong comparator from previous commit.
Message-ID: <E1WsYUB-0000Yc-T5@project.varnish-software.com>


commit b356206814a7b5f403ef65e1beb9cc381c3f2ab3
Author: Lasse Karstensen <lkarsten at varnish-software.com>
Date:   Thu Jun 5 16:17:28 2014 +0200

    Correct wrong comparator from previous commit.

diff --git a/redhat/varnish.spec b/redhat/varnish.spec
index 82c17a7..b85217c 100644
--- a/redhat/varnish.spec
+++ b/redhat/varnish.spec
@@ -18,7 +18,7 @@ BuildRoot: %{_tmppath}/%{name}-%{version}-%{release}-root-%(%{__id_u} -n)
 # You will need at least automake autoconf libtool
 #BuildRequires: automake autoconf libtool
 BuildRequires: ncurses-devel groff pcre-devel pkgconfig libedit-devel jemalloc-devel
-BuildRequires: python-docutils <= 0.7
+BuildRequires: python-docutils >= 0.7
 Requires: varnish-libs = %{version}-%{release}
 Requires: logrotate
 Requires: ncurses



From phk at FreeBSD.org  Tue Jun 10 07:20:10 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 10 Jun 2014 09:20:10 +0200
Subject: [master] 6960078 Minor polishing in http space,
 to get them out of substantial change to follow shortly.
Message-ID: <E1WuGLq-00057y-HV@project.varnish-software.com>


commit 69600783b911cb6b15f3fb07f6dbd595697a0933
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Jun 10 07:19:41 2014 +0000

    Minor polishing in http space, to get them out of substantial change
    to follow shortly.

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index 22e0362..732bf9f 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -89,15 +89,10 @@ enum sess_close {
 #undef SESS_CLOSE
 };
 
-/*--------------------------------------------------------------------*/
-
-/*
- * NB: HDR_STATUS is only used in cache_http.c, everybody else uses the
- * http->status integer field.
+/*--------------------------------------------------------------------
+ * Indicies into http->hd[]
  */
-
 enum {
-	/* Fields from the first line of HTTP proto */
 #define SLTH(tag, ind, req, resp, sdesc, ldesc)	ind,
 #include "tbl/vsl_tags_http.h"
 #undef SLTH
diff --git a/bin/varnishd/cache/cache_http.c b/bin/varnishd/cache/cache_http.c
index 1d1f872..8411e1f 100644
--- a/bin/varnishd/cache/cache_http.c
+++ b/bin/varnishd/cache/cache_http.c
@@ -193,6 +193,39 @@ HTTP_Copy(struct http *to, const struct http * const fm)
 	memcpy(to->hdf, fm->hdf, fm->nhd * sizeof *to->hdf);
 }
 
+/*--------------------------------------------------------------------*/
+
+void
+http_SetH(const struct http *to, unsigned n, const char *fm)
+{
+
+	assert(n < to->shd);
+	AN(fm);
+	to->hd[n].b = TRUST_ME(fm);
+	to->hd[n].e = strchr(to->hd[n].b, '\0');
+	to->hdf[n] = 0;
+	http_VSLH(to, n);
+}
+
+/*--------------------------------------------------------------------*/
+
+static void
+http_PutField(struct http *to, int field, const char *string)
+{
+	char *p;
+
+	CHECK_OBJ_NOTNULL(to, HTTP_MAGIC);
+	p = WS_Copy(to->ws, string, -1);
+	if (p == NULL) {
+		http_fail(to);
+		VSLb(to->vsl, SLT_LostHeader, "%s", string);
+		return;
+	}
+	to->hd[field].b = p;
+	to->hd[field].e = strchr(p, '\0');
+	to->hdf[field] = 0;
+	http_VSLH(to, field);
+}
 
 /*--------------------------------------------------------------------*/
 
@@ -515,6 +548,7 @@ uint16_t
 http_GetStatus(const struct http *hp)
 {
 
+	CHECK_OBJ_NOTNULL(hp, HTTP_MAGIC);
 	return (hp->status);
 }
 
@@ -522,44 +556,11 @@ const char *
 http_GetReq(const struct http *hp)
 {
 
+	CHECK_OBJ_NOTNULL(hp, HTTP_MAGIC);
 	Tcheck(hp->hd[HTTP_HDR_METHOD]);
 	return (hp->hd[HTTP_HDR_METHOD].b);
 }
 
-/*--------------------------------------------------------------------*/
-
-static void
-http_PutField(struct http *to, int field, const char *string)
-{
-	char *p;
-
-	CHECK_OBJ_NOTNULL(to, HTTP_MAGIC);
-	p = WS_Copy(to->ws, string, -1);
-	if (p == NULL) {
-		http_fail(to);
-		VSLb(to->vsl, SLT_LostHeader, "%s", string);
-		return;
-	}
-	to->hd[field].b = p;
-	to->hd[field].e = strchr(p, '\0');
-	to->hdf[field] = 0;
-	http_VSLH(to, field);
-}
-
-/*--------------------------------------------------------------------*/
-
-void
-http_SetH(const struct http *to, unsigned n, const char *fm)
-{
-
-	assert(n < to->shd);
-	AN(fm);
-	to->hd[n].b = TRUST_ME(fm);
-	to->hd[n].e = strchr(to->hd[n].b, '\0');
-	to->hdf[n] = 0;
-	http_VSLH(to, n);
-}
-
 /*--------------------------------------------------------------------
  * Force a particular header field to a particular value
  */
diff --git a/bin/varnishd/cache/cache_http1_proto.c b/bin/varnishd/cache/cache_http1_proto.c
index 4daa814..66323a9 100644
--- a/bin/varnishd/cache/cache_http1_proto.c
+++ b/bin/varnishd/cache/cache_http1_proto.c
@@ -312,6 +312,9 @@ htc_splitline(struct http *hp, const struct http_conn *htc, int req)
 		h2 = HTTP_HDR_STATUS;
 		h3 = HTTP_HDR_REASON;
 	}
+	AZ(hp->hd[h1].b);
+	AZ(hp->hd[h2].b);
+	AZ(hp->hd[h3].b);
 
 	/* Skip leading LWS */
 	for (p = htc->rxbuf.b ; vct_islws(*p); p++)
@@ -339,7 +342,6 @@ htc_splitline(struct http *hp, const struct http_conn *htc, int req)
 			return (400);
 	}
 	hp->hd[h2].e = p;
-
 	if (!Tlen(hp->hd[h2]))
 		return (400);
 
@@ -364,9 +366,7 @@ htc_splitline(struct http *hp, const struct http_conn *htc, int req)
 
 	*hp->hd[h1].e = '\0';
 	*hp->hd[h2].e = '\0';
-
-	if (hp->hd[h3].e != NULL)
-		*hp->hd[h3].e = '\0';
+	*hp->hd[h3].e = '\0';
 
 	return (htc_dissect_hdrs(hp, p, htc));
 }



From phk at FreeBSD.org  Tue Jun 10 07:30:04 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 10 Jun 2014 09:30:04 +0200
Subject: [master] bb603b3 Introduce a dedicated http_SetStatus() function.
Message-ID: <E1WuGVQ-0005v8-0I@project.varnish-software.com>


commit bb603b3d1f9c76906bea167cab1c6cadd4f0df54
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Jun 10 07:29:44 2014 +0000

    Introduce a dedicated http_SetStatus() function.

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index 732bf9f..a79b224 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -1001,6 +1001,7 @@ int http_GetHdrField(const struct http *hp, const char *hdr,
     const char *field, char **ptr);
 double http_GetHdrQ(const struct http *hp, const char *hdr, const char *field);
 uint16_t http_GetStatus(const struct http *hp);
+void http_SetStatus(struct http *to, uint16_t status);
 const char *http_GetReq(const struct http *hp);
 int http_HdrIs(const struct http *hp, const char *hdr, const char *val);
 int http_IsHdr(const txt *hh, const char *hdr);
diff --git a/bin/varnishd/cache/cache_http.c b/bin/varnishd/cache/cache_http.c
index 8411e1f..8313007 100644
--- a/bin/varnishd/cache/cache_http.c
+++ b/bin/varnishd/cache/cache_http.c
@@ -552,6 +552,27 @@ http_GetStatus(const struct http *hp)
 	return (hp->status);
 }
 
+/*--------------------------------------------------------------------*/
+
+void
+http_SetStatus(struct http *to, uint16_t status)
+{
+	char buf[4];
+
+	CHECK_OBJ_NOTNULL(to, HTTP_MAGIC);
+	/*
+	 * We allow people to use top digits for internal VCL
+	 * signalling, but strip them from the ASCII version.
+	 */
+	to->status = status;
+	status %= 1000;
+	assert(status >= 100);
+	bprintf(buf, "%03d", status);
+	http_PutField(to, HTTP_HDR_STATUS, buf);
+}
+
+/*--------------------------------------------------------------------*/
+
 const char *
 http_GetReq(const struct http *hp)
 {
@@ -581,19 +602,11 @@ void
 http_PutResponse(struct http *to, const char *proto, uint16_t status,
     const char *reason)
 {
-	char buf[4];
 
 	CHECK_OBJ_NOTNULL(to, HTTP_MAGIC);
-	http_SetH(to, HTTP_HDR_PROTO, proto);
-	/*
-	 * We allow people to use top digits for internal VCL
-	 * signalling, strip them here.
-	 */
-	status %= 1000;
-	assert(status >= 100);
-	to->status = status;
-	bprintf(buf, "%03d", status % 1000);
-	http_PutField(to, HTTP_HDR_STATUS, buf);
+	if (proto != NULL)
+		http_SetH(to, HTTP_HDR_PROTO, proto);
+	http_SetStatus(to, status);
 	if (reason == NULL)
 		reason = http_Status2Reason(status);
 	http_SetH(to, HTTP_HDR_REASON, reason);
diff --git a/bin/varnishd/cache/cache_vrt_var.c b/bin/varnishd/cache/cache_vrt_var.c
index 3ce5817..b74e101 100644
--- a/bin/varnishd/cache/cache_vrt_var.c
+++ b/bin/varnishd/cache/cache_vrt_var.c
@@ -105,8 +105,9 @@ VRT_l_##obj##_status(const struct vrt_ctx *ctx, long num)		\
 		VSLb(ctx->vsl, SLT_VCL_Error, "illegal %s.status (..0##)", \
 		    #obj);						\
 		ctx->http_##obj->failed = 1;				\
-	} else								\
-		ctx->http_##obj->status = (uint16_t)num;		\
+	} else {							\
+		http_SetStatus(ctx->http_##obj, (uint16_t)num);		\
+	}								\
 }
 
 #define VRT_STATUS_R(obj)						\



From phk at FreeBSD.org  Tue Jun 10 07:50:55 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 10 Jun 2014 09:50:55 +0200
Subject: [master] dbd98ce Polish up HTTP1_DissectResponse()
Message-ID: <E1WuGpb-0000gt-IM@project.varnish-software.com>


commit dbd98ce246e834936a044b2d04085c384a444ce2
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Jun 10 07:50:43 2014 +0000

    Polish up HTTP1_DissectResponse()

diff --git a/bin/varnishd/cache/cache_http1_proto.c b/bin/varnishd/cache/cache_http1_proto.c
index 66323a9..7316f0b 100644
--- a/bin/varnishd/cache/cache_http1_proto.c
+++ b/bin/varnishd/cache/cache_http1_proto.c
@@ -468,8 +468,11 @@ HTTP1_DissectResponse(struct http *hp, const struct http_conn *htc)
 	if (htc_splitline(hp, htc, 0))
 		retval = 503;
 
-	if (retval == 0 && memcmp(hp->hd[HTTP_HDR_PROTO].b, "HTTP/1.", 7))
-		retval = 503;
+	if (retval == 0) {
+		htc_proto_ver(hp);
+		if (hp->protover != 10 && hp->protover != 11)
+			retval = 503;
+	}
 
 	if (retval == 0 && Tlen(hp->hd[HTTP_HDR_STATUS]) != 3)
 		retval = 503;
@@ -489,18 +492,15 @@ HTTP1_DissectResponse(struct http *hp, const struct http_conn *htc)
 	if (retval != 0) {
 		VSLbt(hp->vsl, SLT_HttpGarbage, htc->rxbuf);
 		assert(retval >= 100 && retval <= 999);
+		assert(retval == 503);
 		hp->status = retval;
-	} else
-		htc_proto_ver(hp);
+		http_SetH(hp, HTTP_HDR_STATUS, "503");
+	}
 
 	if (hp->hd[HTTP_HDR_REASON].b == NULL ||
-	    !Tlen(hp->hd[HTTP_HDR_REASON])) {
-		/* Backend didn't send a response string, use the standard */
-		hp->hd[HTTP_HDR_REASON].b =
-		    TRUST_ME(http_Status2Reason(hp->status));
-		hp->hd[HTTP_HDR_REASON].e =
-		    strchr(hp->hd[HTTP_HDR_REASON].b, '\0');
-	}
+	    !Tlen(hp->hd[HTTP_HDR_REASON]))
+		http_SetH(hp, HTTP_HDR_REASON, http_Status2Reason(hp->status));
+
 	return (retval);
 }
 



From phk at FreeBSD.org  Tue Jun 10 08:34:06 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 10 Jun 2014 10:34:06 +0200
Subject: [master] e99e8fb Always set the reason headerfield when the status
 field is set, to keep them in sync.  If you want a custom reason field,
 set status first, then reason.
Message-ID: <E1WuHVO-0004E6-LY@project.varnish-software.com>


commit e99e8fbaa85aafa36abaa0aa9f508e98fd93dfc8
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Jun 10 08:31:16 2014 +0000

    Always set the reason headerfield when the status field is set, to
    keep them in sync.  If you want a custom reason field, set status
    first, then reason.
    
    With this in place, HTTP1_Write() doesn't need to worry about them
    being OK, and we can polish that code a fair bit.
    
    Turn a couple of boolean arguments into more readable magics.

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index a79b224..38e597b 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -857,6 +857,8 @@ void HTTP1_Session(struct worker *, struct req *);
 int HTTP1_DiscardReqBody(struct req *req);
 int HTTP1_CacheReqBody(struct req *req, ssize_t maxsize);
 int HTTP1_IterateReqBody(struct req *req, req_body_iter_f *func, void *priv);
+extern const int HTTP1_Req[3];
+extern const int HTTP1_Resp[3];
 
 /* cache_http1_deliver.c */
 unsigned V1D_FlushReleaseAcct(struct req *req);
@@ -1030,7 +1032,7 @@ ssize_t HTTP1_Read(struct http_conn *htc, void *d, size_t len);
 enum htc_status_e HTTP1_Complete(struct http_conn *htc);
 uint16_t HTTP1_DissectRequest(struct req *);
 uint16_t HTTP1_DissectResponse(struct http *sp, const struct http_conn *htc);
-unsigned HTTP1_Write(const struct worker *w, const struct http *hp, int resp);
+unsigned HTTP1_Write(const struct worker *w, const struct http *hp, const int*);
 
 #define HTTPH(a, b, c) extern char b[];
 #include "tbl/http_headers.h"
diff --git a/bin/varnishd/cache/cache_http.c b/bin/varnishd/cache/cache_http.c
index 8313007..159751b 100644
--- a/bin/varnishd/cache/cache_http.c
+++ b/bin/varnishd/cache/cache_http.c
@@ -552,7 +552,9 @@ http_GetStatus(const struct http *hp)
 	return (hp->status);
 }
 
-/*--------------------------------------------------------------------*/
+/*--------------------------------------------------------------------
+ * Setting the status will also set the Reason appropriately
+ */
 
 void
 http_SetStatus(struct http *to, uint16_t status)
@@ -569,6 +571,7 @@ http_SetStatus(struct http *to, uint16_t status)
 	assert(status >= 100);
 	bprintf(buf, "%03d", status);
 	http_PutField(to, HTTP_HDR_STATUS, buf);
+	http_SetH(to, HTTP_HDR_REASON, http_Status2Reason(status));
 }
 
 /*--------------------------------------------------------------------*/
diff --git a/bin/varnishd/cache/cache_http1_deliver.c b/bin/varnishd/cache/cache_http1_deliver.c
index 06b4a32..8778303 100644
--- a/bin/varnishd/cache/cache_http1_deliver.c
+++ b/bin/varnishd/cache/cache_http1_deliver.c
@@ -322,7 +322,8 @@ V1D_Deliver(struct req *req)
 	 * Send HTTP protocol header, unless interior ESI object
 	 */
 	if (!(req->res_mode & RES_ESI_CHILD))
-		req->resp_hdrbytes += HTTP1_Write(req->wrk, req->resp, 1);
+		req->resp_hdrbytes +=
+		    HTTP1_Write(req->wrk, req->resp, HTTP1_Resp);
 
 	if (req->res_mode & RES_CHUNKED)
 		WRW_Chunked(req->wrk);
@@ -429,7 +430,8 @@ V1D_Deliver_Synth(struct req *req)
 	 * Send HTTP protocol header, unless interior ESI object
 	 */
 	if (!(req->res_mode & RES_ESI_CHILD))
-		req->resp_hdrbytes += HTTP1_Write(req->wrk, req->resp, 1);
+		req->resp_hdrbytes +=
+		    HTTP1_Write(req->wrk, req->resp, HTTP1_Resp);
 
 	if (req->res_mode & RES_CHUNKED)
 		WRW_Chunked(req->wrk);
diff --git a/bin/varnishd/cache/cache_http1_fetch.c b/bin/varnishd/cache/cache_http1_fetch.c
index 11c712e..1c3ef53 100644
--- a/bin/varnishd/cache/cache_http1_fetch.c
+++ b/bin/varnishd/cache/cache_http1_fetch.c
@@ -327,7 +327,7 @@ V1F_fetch_hdr(struct worker *wrk, struct busyobj *bo, struct req *req)
 
 	(void)VTCP_blocking(vc->fd);	/* XXX: we should timeout instead */
 	WRW_Reserve(wrk, &vc->fd, bo->vsl, bo->t_prev);
-	hdrbytes = HTTP1_Write(wrk, hp, 0);
+	hdrbytes = HTTP1_Write(wrk, hp, HTTP1_Req);
 
 	/* Deal with any message-body the request might (still) have */
 	i = 0;
diff --git a/bin/varnishd/cache/cache_http1_proto.c b/bin/varnishd/cache/cache_http1_proto.c
index 7316f0b..7f7852c 100644
--- a/bin/varnishd/cache/cache_http1_proto.c
+++ b/bin/varnishd/cache/cache_http1_proto.c
@@ -49,6 +49,14 @@
 
 #include "vct.h"
 
+const int HTTP1_Req[3] = {
+	HTTP_HDR_METHOD, HTTP_HDR_URL, HTTP_HDR_PROTO
+};
+
+const int HTTP1_Resp[3] = {
+	HTTP_HDR_PROTO, HTTP_HDR_STATUS, HTTP_HDR_REASON
+};
+
 /*--------------------------------------------------------------------*/
 
 void
@@ -294,55 +302,46 @@ htc_dissect_hdrs(struct http *hp, char *p, const struct http_conn *htc)
  */
 
 static uint16_t
-htc_splitline(struct http *hp, const struct http_conn *htc, int req)
+htc_splitline(struct http *hp, const struct http_conn *htc, const int *hf)
 {
 	char *p;
-	int h1, h2, h3;
 
+	assert(hf == HTTP1_Req || hf == HTTP1_Resp);
 	CHECK_OBJ_NOTNULL(htc, HTTP_CONN_MAGIC);
 	CHECK_OBJ_NOTNULL(hp, HTTP_MAGIC);
 	Tcheck(htc->rxbuf);
 
-	if (req) {
-		h1 = HTTP_HDR_METHOD;
-		h2 = HTTP_HDR_URL;
-		h3 = HTTP_HDR_PROTO;
-	} else {
-		h1 = HTTP_HDR_PROTO;
-		h2 = HTTP_HDR_STATUS;
-		h3 = HTTP_HDR_REASON;
-	}
-	AZ(hp->hd[h1].b);
-	AZ(hp->hd[h2].b);
-	AZ(hp->hd[h3].b);
+	AZ(hp->hd[hf[0]].b);
+	AZ(hp->hd[hf[1]].b);
+	AZ(hp->hd[hf[2]].b);
 
 	/* Skip leading LWS */
 	for (p = htc->rxbuf.b ; vct_islws(*p); p++)
 		continue;
-	hp->hd[h1].b = p;
+	hp->hd[hf[0]].b = p;
 
 	/* First field cannot contain SP or CTL */
 	for (; !vct_issp(*p); p++) {
 		if (vct_isctl(*p))
 			return (400);
 	}
-	hp->hd[h1].e = p;
-	assert(Tlen(hp->hd[h1]));
+	hp->hd[hf[0]].e = p;
+	assert(Tlen(hp->hd[hf[0]]));
 
 	/* Skip SP */
 	for (; vct_issp(*p); p++) {
 		if (vct_isctl(*p))
 			return (400);
 	}
-	hp->hd[h2].b = p;
+	hp->hd[hf[1]].b = p;
 
 	/* Second field cannot contain LWS or CTL */
 	for (; !vct_islws(*p); p++) {
 		if (vct_isctl(*p))
 			return (400);
 	}
-	hp->hd[h2].e = p;
-	if (!Tlen(hp->hd[h2]))
+	hp->hd[hf[1]].e = p;
+	if (!Tlen(hp->hd[hf[1]]))
 		return (400);
 
 	/* Skip SP */
@@ -350,23 +349,23 @@ htc_splitline(struct http *hp, const struct http_conn *htc, int req)
 		if (vct_isctl(*p))
 			return (400);
 	}
-	hp->hd[h3].b = p;
+	hp->hd[hf[2]].b = p;
 
 	/* Third field is optional and cannot contain CTL except TAB */
 	for (; !vct_iscrlf(p); p++) {
 		if (vct_isctl(*p) && !vct_issp(*p)) {
-			hp->hd[h3].b = NULL;
+			hp->hd[hf[2]].b = NULL;
 			return (400);
 		}
 	}
-	hp->hd[h3].e = p;
+	hp->hd[hf[2]].e = p;
 
 	/* Skip CRLF */
 	p += vct_skipcrlf(p);
 
-	*hp->hd[h1].e = '\0';
-	*hp->hd[h2].e = '\0';
-	*hp->hd[h3].e = '\0';
+	*hp->hd[hf[0]].e = '\0';
+	*hp->hd[hf[1]].e = '\0';
+	*hp->hd[hf[2]].e = '\0';
 
 	return (htc_dissect_hdrs(hp, p, htc));
 }
@@ -426,7 +425,7 @@ HTTP1_DissectRequest(struct req *req)
 	hp = req->http;
 	CHECK_OBJ_NOTNULL(hp, HTTP_MAGIC);
 
-	retval = htc_splitline(hp, htc, 1);
+	retval = htc_splitline(hp, htc, HTTP1_Req);
 	if (retval != 0) {
 		VSLbt(req->vsl, SLT_HttpGarbage, htc->rxbuf);
 		return (retval);
@@ -465,7 +464,7 @@ HTTP1_DissectResponse(struct http *hp, const struct http_conn *htc)
 	CHECK_OBJ_NOTNULL(htc, HTTP_CONN_MAGIC);
 	CHECK_OBJ_NOTNULL(hp, HTTP_MAGIC);
 
-	if (htc_splitline(hp, htc, 0))
+	if (htc_splitline(hp, htc, HTTP1_Resp))
 		retval = 503;
 
 	if (retval == 0) {
@@ -495,6 +494,7 @@ HTTP1_DissectResponse(struct http *hp, const struct http_conn *htc)
 		assert(retval == 503);
 		hp->status = retval;
 		http_SetH(hp, HTTP_HDR_STATUS, "503");
+		http_SetH(hp, HTTP_HDR_REASON, http_Status2Reason(retval));
 	}
 
 	if (hp->hd[HTTP_HDR_REASON].b == NULL ||
@@ -507,36 +507,20 @@ HTTP1_DissectResponse(struct http *hp, const struct http_conn *htc)
 /*--------------------------------------------------------------------*/
 
 unsigned
-HTTP1_Write(const struct worker *w, const struct http *hp, int resp)
+HTTP1_Write(const struct worker *w, const struct http *hp, const int *hf)
 {
 	unsigned u, l;
 
-	if (resp) {
-		l = WRW_WriteH(w, &hp->hd[HTTP_HDR_PROTO], " ");
-
-		hp->hd[HTTP_HDR_STATUS].b = WS_Alloc(hp->ws, 4);
-		AN(hp->hd[HTTP_HDR_STATUS].b);
+	assert(hf == HTTP1_Req || hf == HTTP1_Resp);
+	AN(hp->hd[hf[0]].b);
+	AN(hp->hd[hf[1]].b);
+	AN(hp->hd[hf[2]].b);
+	l = WRW_WriteH(w, &hp->hd[hf[0]], " ");
+	l += WRW_WriteH(w, &hp->hd[hf[1]], " ");
+	l += WRW_WriteH(w, &hp->hd[hf[2]], "\r\n");
 
-		assert(hp->status >= 100 && hp->status <= 999);
-		sprintf(hp->hd[HTTP_HDR_STATUS].b, "%3d", hp->status);
-		hp->hd[HTTP_HDR_STATUS].e = hp->hd[HTTP_HDR_STATUS].b + 3;
-
-		l += WRW_WriteH(w, &hp->hd[HTTP_HDR_STATUS], " ");
-
-		l += WRW_WriteH(w, &hp->hd[HTTP_HDR_REASON], "\r\n");
-	} else {
-		AN(hp->hd[HTTP_HDR_URL].b);
-		l = WRW_WriteH(w, &hp->hd[HTTP_HDR_METHOD], " ");
-		l += WRW_WriteH(w, &hp->hd[HTTP_HDR_URL], " ");
-		l += WRW_WriteH(w, &hp->hd[HTTP_HDR_PROTO], "\r\n");
-	}
-	for (u = HTTP_HDR_FIRST; u < hp->nhd; u++) {
-		if (hp->hd[u].b == NULL)
-			continue;
-		AN(hp->hd[u].b);
-		AN(hp->hd[u].e);
+	for (u = HTTP_HDR_FIRST; u < hp->nhd; u++)
 		l += WRW_WriteH(w, &hp->hd[u], "\r\n");
-	}
 	l += WRW_Write(w, "\r\n", -1);
 	return (l);
 }
diff --git a/bin/varnishd/cache/cache_pipe.c b/bin/varnishd/cache/cache_pipe.c
index bb666a1..27ea964 100644
--- a/bin/varnishd/cache/cache_pipe.c
+++ b/bin/varnishd/cache/cache_pipe.c
@@ -124,7 +124,7 @@ PipeRequest(struct req *req, struct busyobj *bo)
 	(void)VTCP_blocking(vc->fd);
 
 	WRW_Reserve(wrk, &vc->fd, bo->vsl, req->t_req);
-	hdrbytes = HTTP1_Write(wrk, bo->bereq, 0);
+	hdrbytes = HTTP1_Write(wrk, bo->bereq, HTTP1_Req);
 
 	if (req->htc->pipeline.b != NULL)
 		(void)WRW_Write(wrk, req->htc->pipeline.b,



From daghf at varnish-software.com  Tue Jun 10 10:12:43 2014
From: daghf at varnish-software.com (Dag Haavi Finstad)
Date: Tue, 10 Jun 2014 12:12:43 +0200
Subject: [master] 8a1a59a Docfix an erroneous reference to r-r director having
 weight.
Message-ID: <E1WuJ2p-0007pl-Lz@project.varnish-software.com>


commit 8a1a59ab1547366bae942c087eef974aaba6fb5e
Author: Dag Haavi Finstad <daghf at varnish-software.com>
Date:   Tue Jun 10 12:10:32 2014 +0200

    Docfix an erroneous reference to r-r director having weight.
    
    Fixes: #1519

diff --git a/lib/libvmod_directors/vmod.vcc b/lib/libvmod_directors/vmod.vcc
index f0d9424..cc0bb6c 100644
--- a/lib/libvmod_directors/vmod.vcc
+++ b/lib/libvmod_directors/vmod.vcc
@@ -153,7 +153,7 @@ $Method VOID .add_backend(BACKEND, REAL)
 Description
 	Adds a backend to the director with a certain weight.
 
-	Weight is used as in the round_robin director. Recommended value is
+	Weight is used as in the random director. Recommended value is
 	1.0 unless you have special needs.
 
 Example



From lkarsten at varnish-software.com  Tue Jun 10 11:12:41 2014
From: lkarsten at varnish-software.com (Lasse Karstensen)
Date: Tue, 10 Jun 2014 13:12:41 +0200
Subject: [master] c0098d3 Fix minor double wording.
Message-ID: <E1WuJyr-0005zr-7n@project.varnish-software.com>


commit c0098d30058144c67243143fab16aaa8df386d90
Author: Lasse Karstensen <lkarsten at varnish-software.com>
Date:   Tue Jun 10 12:04:28 2014 +0200

    Fix minor double wording.

diff --git a/lib/libvmod_directors/vmod.vcc b/lib/libvmod_directors/vmod.vcc
index cc0bb6c..f7f3330 100644
--- a/lib/libvmod_directors/vmod.vcc
+++ b/lib/libvmod_directors/vmod.vcc
@@ -98,8 +98,8 @@ $Method VOID .add_backend(BACKEND)
 Description
         Add a backend to the director.
 
-        Note that the order in which this is done matters in the for the
-        fallback director.
+	Note that the order in which this is done matters for the fallback
+	director.
 
 Example
        vdir.add_backend(backend1);



From lkarsten at varnish-software.com  Tue Jun 10 11:12:41 2014
From: lkarsten at varnish-software.com (Lasse Karstensen)
Date: Tue, 10 Jun 2014 13:12:41 +0200
Subject: [master] ae54c1b Name directors consistently for clarity.
Message-ID: <E1WuJyr-0005zx-9M@project.varnish-software.com>


commit ae54c1b0b275d96404c41a25e200c806a6e75c9c
Author: Lasse Karstensen <lkarsten at varnish-software.com>
Date:   Tue Jun 10 12:42:21 2014 +0200

    Name directors consistently for clarity.
    
    Using multiple names for the instanciateed directors in this document
    just makes it more confusing, since .add_backend() and .backend() is
    mostly identical for all of them.

diff --git a/lib/libvmod_directors/vmod.vcc b/lib/libvmod_directors/vmod.vcc
index f7f3330..4900d4e 100644
--- a/lib/libvmod_directors/vmod.vcc
+++ b/lib/libvmod_directors/vmod.vcc
@@ -47,9 +47,9 @@ want to emulate the previous behavior of Varnish 3.0 you can just
 initialize the directors in vcl_init, like this:::
 
     sub vcl_init {
-        new bar = directors.round_robin();
-        bar.add_backend(server1);
-        bar.add_backend(server2);
+        new vdir = directors.round_robin();
+        vdir.add_backend(backend1);
+        vdir.add_backend(backend2);
     }
 
 As you can see there is nothing keeping you from manipulating the
@@ -65,21 +65,22 @@ Description
 
 	This director will pick backends in a round robin fashion.
 Example
-       new bar = directors.round_robin();
+	new vdir = directors.round_robin();
 
 $Method VOID .add_backend(BACKEND)
 
 Description
        Adds a backend to the director.
 Example
-       rrdir.add_backend(backend1);
+       vdir.add_backend(backend1);
+       vdir.add_backend(backend2);
 
 $Method BACKEND .backend()
 
 Description
        Picks a backend from the director.
 Example
-       set req.backend_hint = rrdir.backend();
+       set req.backend_hint = vdir.backend();
 
 
 $Object fallback()
@@ -122,21 +123,24 @@ Description
        more or less 1% of the traffic of a backend in the same
        director with a weight of 100.
 Example
-       new rand_dir = directors.random();
+	new vdir = directors.random();
 
 $Method VOID .add_backend(BACKEND, REAL)
 
 Description
        Adds a backend to the director with weight.
 Example
-       bar.add_backend(backend1, 3.14);
+	# 2/3 to backend1, 1/3 to backend2.
+	vdir.add_backend(backend1, 10);
+	vdir.add_backend(backend2, 5);
+
 
 $Method BACKEND .backend()
 
 Description
        Picks a backend from the director.
 Example
-       set req.backend_hint = rrdir.backend();
+	set req.backend_hint = vdir.backend();
 
 $Object hash()
 



From lkarsten at varnish-software.com  Tue Jun 10 11:12:41 2014
From: lkarsten at varnish-software.com (Lasse Karstensen)
Date: Tue, 10 Jun 2014 13:12:41 +0200
Subject: [master] f49967a Rewrite director descriptions.
Message-ID: <E1WuJyr-000609-Bi@project.varnish-software.com>


commit f49967a722206d089e920f5bcc3a29a69d71b765
Author: Lasse Karstensen <lkarsten at varnish-software.com>
Date:   Tue Jun 10 12:55:03 2014 +0200

    Rewrite director descriptions.
    
    Add a bit more detail while cleaning up the language.
    
    Reformat for tabs instead of 7 spaces.

diff --git a/lib/libvmod_directors/vmod.vcc b/lib/libvmod_directors/vmod.vcc
index 4900d4e..94b9349 100644
--- a/lib/libvmod_directors/vmod.vcc
+++ b/lib/libvmod_directors/vmod.vcc
@@ -70,7 +70,7 @@ Example
 $Method VOID .add_backend(BACKEND)
 
 Description
-       Adds a backend to the director.
+       Add a backend to the round-robin director.
 Example
        vdir.add_backend(backend1);
        vdir.add_backend(backend2);
@@ -78,7 +78,7 @@ Example
 $Method BACKEND .backend()
 
 Description
-       Picks a backend from the director.
+       Pick a backend from the director.
 Example
        set req.backend_hint = vdir.backend();
 
@@ -103,7 +103,8 @@ Description
 	director.
 
 Example
-       vdir.add_backend(backend1);
+	vdir.add_backend(backend1);
+	vdir.add_backend(backend2);
 
 $Method BACKEND .backend()
 
@@ -116,59 +117,69 @@ Example
 $Object random()
 
 Description
-       Adds a random director. This director chooses backend based on
-       a random number. As you add backends to the director each
-       backends gets a weight, which is used to when requests are
-       being distributed. So, a backend with a weight of 1 would get
-       more or less 1% of the traffic of a backend in the same
-       director with a weight of 100.
+	Create a random backend director.
+
+	The random director distributes load over the backends using
+	a weighted random probability distribution.
+
 Example
 	new vdir = directors.random();
 
 $Method VOID .add_backend(BACKEND, REAL)
 
 Description
-       Adds a backend to the director with weight.
+	Add a backend to the director with a given weight.
+
+	Each backend backend will receive approximately
+	100 * (weight / (sum(all_added_weights))) per cent of the traffic sent
+	to this backend.
+
 Example
-	# 2/3 to backend1, 1/3 to backend2.
 	vdir.add_backend(backend1, 10);
 	vdir.add_backend(backend2, 5);
+	# 2/3 to backend1, 1/3 to backend2.
 
 
 $Method BACKEND .backend()
 
 Description
-       Picks a backend from the director.
+	Pick a backend from the director.
 Example
 	set req.backend_hint = vdir.backend();
 
 $Object hash()
 
 Description
-       Creates a hash director. The hash director chooses the backend
-       based on hashing an arbitrary string. If you provide it with a
-       session cookie, you'll have the client connecting to the same
-       backend every time.
+	Create a hashing backend director.
+
+	The director chooses the backend server by computing a hash/digest of
+	the string given to .backend().
+
+	Commonly used with ``client.identity`` or a session cookie to get
+	sticky sessions.
+
 Example
-       new hdir = directors.hash();
+	new vdir = directors.hash();
 
 $Method VOID .add_backend(BACKEND, REAL)
 
 Description
-	Adds a backend to the director with a certain weight.
+	Add a backend to the director with a certain weight.
 
 	Weight is used as in the random director. Recommended value is
 	1.0 unless you have special needs.
 
 Example
-       hdir.add_backend(backend1, 1.0);
+	vdir.add_backend(backend1, 1.0);
+	vdir.add_backend(backend2, 1.0);
+
 
 $Method BACKEND .backend(STRING_LIST)
 
 Description
-       Picks a backend from the director. Use the string or list of
-       strings provided to pick the backend.
+	Pick a backend from the backend director.
+
+	Use the string or list of strings provided to pick the backend.
 Example
-       # pick a backend based on the cookie header from the client
-       set req.backend_hint = hdir.backend(req.http.cookie);
+	set req.backend_hint = vdir.backend(req.http.cookie);  # pick a backend based on the cookie header from the client
 



From phk at FreeBSD.org  Tue Jun 10 11:16:01 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 10 Jun 2014 13:16:01 +0200
Subject: [master] 8061a7c Rememeber to supress body if we do 304 conditional
 response.
Message-ID: <E1WuK25-0006Zw-Oe@project.varnish-software.com>


commit 8061a7c804be67b2b93bf8d7b6de7f5988452c48
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Jun 10 11:15:32 2014 +0000

    Rememeber to supress body if we do 304 conditional response.
    
    Fixes	#1518

diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c
index 4c8ca8e..c586ca4 100644
--- a/bin/varnishd/cache/cache_req_fsm.c
+++ b/bin/varnishd/cache/cache_req_fsm.c
@@ -149,8 +149,10 @@ cnt_deliver(struct worker *wrk, struct req *req)
 	if (!(req->obj->objcore->flags & OC_F_PASS)
 	    && req->esi_level == 0
 	    && http_GetStatus(req->obj->http) == 200
-	    && req->http->conds && RFC2616_Do_Cond(req))
+	    && req->http->conds && RFC2616_Do_Cond(req)) {
 		http_PutResponse(req->resp, "HTTP/1.1", 304, NULL);
+		req->wantbody = 0;
+	}
 
 	V1D_Deliver(req);
 	VSLb_ts_req(req, "Resp", W_TIM_real(wrk));



From phk at FreeBSD.org  Tue Jun 10 11:16:01 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 10 Jun 2014 13:16:01 +0200
Subject: [master] 7a6a3d7 Merge branch 'master' of
 git.varnish-cache.org:varnish-cache
Message-ID: <E1WuK25-0006a2-QZ@project.varnish-software.com>


commit 7a6a3d71f70e92c82c860c29e2ff645b7cced5bc
Merge: 8061a7c f49967a
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Jun 10 11:15:57 2014 +0000

    Merge branch 'master' of git.varnish-cache.org:varnish-cache



From daghf at varnish-software.com  Thu Jun 12 13:50:39 2014
From: daghf at varnish-software.com (Dag Haavi Finstad)
Date: Thu, 12 Jun 2014 15:50:39 +0200
Subject: [master] baad5e7 Clean up mis-pasted(?) information in description of
 .threshold.
Message-ID: <E1Wv5Op-0003gU-Qm@project.varnish-software.com>


commit baad5e714c31cb5dd5b15d8b87aa282b90edfc29
Author: Dag Haavi Finstad <daghf at varnish-software.com>
Date:   Thu Jun 12 15:50:37 2014 +0200

    Clean up mis-pasted(?) information in description of .threshold.

diff --git a/doc/sphinx/reference/vcl.rst b/doc/sphinx/reference/vcl.rst
index 1268920..9ae2aa7 100644
--- a/doc/sphinx/reference/vcl.rst
+++ b/doc/sphinx/reference/vcl.rst
@@ -252,11 +252,7 @@ There are no mandatory options. These are the options you can set:
 
   threshold
     How many of the polls in .window must have succeeded for us to
-    consider the backend healthy. If this is set to more than or equal
-    to the threshold, the backend starts as healthy. Defaults to the
-    value of threshold - 1. In this case, the backend starts as sick
-    and requires one poll to pass to become healthy. Defaults to
-    threshold - 1.
+    consider the backend healthy. Defaults to 3.
 
 
 Access Control List (ACL)



From nils.goroll at uplex.de  Tue Jun 17 09:45:39 2014
From: nils.goroll at uplex.de (Nils Goroll)
Date: Tue, 17 Jun 2014 11:45:39 +0200
Subject: [master] dcccd5a add a commercial support reference
Message-ID: <E1WwpxT-0000hA-Ea@project.varnish-software.com>


commit dcccd5a010a8c912a614c9fd518f0c508ec376c6
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Tue Jun 17 11:45:26 2014 +0200

    add a commercial support reference

diff --git a/doc/sphinx/installation/help.rst b/doc/sphinx/installation/help.rst
index 66433de..897324a 100644
--- a/doc/sphinx/installation/help.rst
+++ b/doc/sphinx/installation/help.rst
@@ -97,6 +97,8 @@ an email to phk at FreeBSD.org.
 	Varnish Software
 	sales at varnish-software.com
 
+	UPLEX
+	info at uplex.de
 
 .. _mailman: http://lists.varnish-cache.org/mailman/listinfo
 .. _pastebin: http://gist.github.com/



From martin at varnish-software.com  Tue Jun 17 10:15:18 2014
From: martin at varnish-software.com (Martin Blix Grydeland)
Date: Tue, 17 Jun 2014 12:15:18 +0200
Subject: [master] a318399 Add license to vxp_test.c
Message-ID: <E1WwqQA-0006z6-9O@project.varnish-software.com>


commit a318399f83d4e8be8bd82058d02657818ef82969
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Tue Jun 17 12:15:02 2014 +0200

    Add license to vxp_test.c

diff --git a/lib/libvarnishapi/vxp_test.c b/lib/libvarnishapi/vxp_test.c
index 89437f1..a4abf4a 100644
--- a/lib/libvarnishapi/vxp_test.c
+++ b/lib/libvarnishapi/vxp_test.c
@@ -1,3 +1,31 @@
+/*-
+ * Copyright (c) 2014 Varnish Software AS
+ * All rights reserved.
+ *
+ * Author: Martin Blix Grydeland <martin at varnish-software.com>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
 #include <stdlib.h>
 #include <stdio.h>
 #include <errno.h>



From fgsch at lodoss.net  Tue Jun 17 11:55:31 2014
From: fgsch at lodoss.net (Federico G. Schwindt)
Date: Tue, 17 Jun 2014 13:55:31 +0200
Subject: [master] e54c8aa Document y (years) time unit
Message-ID: <E1Wwrz9-0005ve-9k@project.varnish-software.com>


commit e54c8aaee5c7215954762dc3b2aeca09aefd26ad
Author: Federico G. Schwindt <fgsch at lodoss.net>
Date:   Tue Jun 17 12:52:52 2014 +0100

    Document y (years) time unit

diff --git a/lib/libvmod_std/vmod.vcc b/lib/libvmod_std/vmod.vcc
index b1b7010..7a22156 100644
--- a/lib/libvmod_std/vmod.vcc
+++ b/lib/libvmod_std/vmod.vcc
@@ -113,7 +113,7 @@ $Function DURATION duration(STRING, DURATION)
 Description
 	Converts the string *s* to seconds. *s* must be quantified
 	with ms (milliseconds), s (seconds), m (minutes), h (hours),
-	d (days) or w (weeks) units. If *s* fails to parse,
+	d (days), w (weeks) or y (years) units. If *s* fails to parse,
 	*fallback* will be returned.
 Example
 	set beresp.ttl = std.duration("1w", 3600s);



From fgsch at lodoss.net  Tue Jun 17 12:40:52 2014
From: fgsch at lodoss.net (Federico G. Schwindt)
Date: Tue, 17 Jun 2014 14:40:52 +0200
Subject: [master] 0913f09 Add y (years) time unit support to VCL
Message-ID: <E1Wwsh2-0003gx-LX@project.varnish-software.com>


commit 0913f097cdd55ae2b49e090c04f355d0dd9b1664
Author: Federico G. Schwindt <fgsch at lodoss.net>
Date:   Tue Jun 17 13:37:27 2014 +0100

    Add y (years) time unit support to VCL
    
    Follows the change in the std vmod a while ago.

diff --git a/bin/varnishtest/tests/v00016.vtc b/bin/varnishtest/tests/v00016.vtc
index 1107f86..dfe69ae 100644
--- a/bin/varnishtest/tests/v00016.vtc
+++ b/bin/varnishtest/tests/v00016.vtc
@@ -73,7 +73,7 @@ varnish v1 -vcl {
 
 }
 
-varnish v1 -errvcl {Unknown time unit 'k'.  Legal are 'ms', 's', 'm', 'h', 'd' and 'w'} {
+varnish v1 -errvcl {Unknown time unit 'k'.  Legal are 'ms', 's', 'm', 'h', 'd', 'w' and 'y'} {
 	backend b { .host = "127.0.0.1"; }
 	sub vcl_backend_response { set beresp.ttl = 1. k; }
 }
diff --git a/doc/sphinx/reference/vcl.rst b/doc/sphinx/reference/vcl.rst
index 9ae2aa7..3f8a831 100644
--- a/doc/sphinx/reference/vcl.rst
+++ b/doc/sphinx/reference/vcl.rst
@@ -115,6 +115,8 @@ so 1.5w is allowed.
   w
     weeks
 
+  y
+    years
 
 Integers
 --------
diff --git a/lib/libvcc/vcc_expr.c b/lib/libvcc/vcc_expr.c
index c143cf1..8e4e234 100644
--- a/lib/libvcc/vcc_expr.c
+++ b/lib/libvcc/vcc_expr.c
@@ -74,10 +74,12 @@ vcc_TimeUnit(struct vcc *tl)
 		sc = 60.0 * 60.0 * 24.0;
 	else if (vcc_IdIs(tl->t, "w"))
 		sc = 60.0 * 60.0 * 24.0 * 7.0;
+	else if (vcc_IdIs(tl->t, "y"))
+		sc = 60.0 * 60.0 * 24.0 * 365.0;
 	else {
 		VSB_printf(tl->sb, "Unknown time unit ");
 		vcc_ErrToken(tl, tl->t);
-		VSB_printf(tl->sb, ".  Legal are 'ms', 's', 'm', 'h', 'd' and 'w'\n");
+		VSB_printf(tl->sb, ".  Legal are 'ms', 's', 'm', 'h', 'd', 'w' and 'y'\n");
 		vcc_ErrWhere(tl, tl->t);
 		return (1.0);
 	}



From phk at FreeBSD.org  Thu Jun 19 07:56:23 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Thu, 19 Jun 2014 09:56:23 +0200
Subject: [master] 5f58e96 White space nits
Message-ID: <E1WxXCp-0002BU-Kd@project.varnish-software.com>


commit 5f58e9621e9f5fa915ebc4c2cdc74a558f4c76cd
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu Jun 19 07:56:10 2014 +0000

    White space nits

diff --git a/bin/varnishtest/tests/r01510.vtc b/bin/varnishtest/tests/r01510.vtc
index e8933be..47b87cb 100644
--- a/bin/varnishtest/tests/r01510.vtc
+++ b/bin/varnishtest/tests/r01510.vtc
@@ -6,7 +6,7 @@ varnish v1 -errvcl {Object name 'first' already used.} {
 		new first = debug.obj("FOO");
 		new first = debug.obj("BAH");
 	}
-} 
+}
 
 varnish v1 -errvcl {Object name 'first' already used.} {
 	import ${vmod_debug};
@@ -16,4 +16,4 @@ varnish v1 -errvcl {Object name 'first' already used.} {
 	sub vcl_init {
 		new first = debug.obj("FOO");
 	}
-} 
+}
diff --git a/lib/libvcc/vmodtool.py b/lib/libvcc/vmodtool.py
index 3299074..315b46b 100755
--- a/lib/libvcc/vmodtool.py
+++ b/lib/libvcc/vmodtool.py
@@ -66,16 +66,16 @@ ctypes = {
 
 def write_file_warning(fo, a, b, c):
 	fo.write(a + "\n")
-	fo.write(b + "NB:  This file is machine generated, DO NOT EDIT!\n")
+	fo.write(b + " NB:  This file is machine generated, DO NOT EDIT!\n")
 	fo.write(b + "\n")
-	fo.write(b + "Edit vmod.vcc and run make instead\n")
+	fo.write(b + " Edit vmod.vcc and run make instead\n")
 	fo.write(c + "\n\n")
 
 def write_c_file_warning(fo):
-	write_file_warning(fo, "/*", " * ", " */")
+	write_file_warning(fo, "/*", " *", " */")
 
 def write_rst_file_warning(fo):
-	write_file_warning(fo, "..", ".. ", "..")
+	write_file_warning(fo, "..", "..", "..")
 
 #######################################################################
 



From phk at FreeBSD.org  Thu Jun 19 07:57:08 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Thu, 19 Jun 2014 09:57:08 +0200
Subject: [master] e5c7502 Make sure attempts to talk (draft-)HTTP/2.0 don't
 confuse anybody.
Message-ID: <E1WxXDY-0002VI-1M@project.varnish-software.com>


commit e5c7502defb50e8e0b5113b583c1a7e7a1365d1b
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu Jun 19 07:56:37 2014 +0000

    Make sure attempts to talk (draft-)HTTP/2.0 don't confuse anybody.

diff --git a/bin/varnishd/builtin.vcl b/bin/varnishd/builtin.vcl
index f9d3527..30fcaea 100644
--- a/bin/varnishd/builtin.vcl
+++ b/bin/varnishd/builtin.vcl
@@ -45,6 +45,10 @@ vcl 4.0;
 # Client side
 
 sub vcl_recv {
+    if (req.method == "PRI") {
+	/* We do not support SPDY or HTTP/2.0 */
+	return (synth(405));
+    }
     if (req.method != "GET" &&
       req.method != "HEAD" &&
       req.method != "PUT" &&
diff --git a/include/tbl/http_headers.h b/include/tbl/http_headers.h
index 1db7784..26443ca 100644
--- a/include/tbl/http_headers.h
+++ b/include/tbl/http_headers.h
@@ -69,6 +69,7 @@ HTTPH("Expect",			H_Expect,		0					  )	/* RFC2616 14.20 */
 HTTPH("Expires",		H_Expires,		0					  )	/* RFC2616 14.21 */
 HTTPH("From",			H_From,			0					  )	/* RFC2616 14.22 */
 HTTPH("Host",			H_Host,			0					  )	/* RFC2616 14.23 */
+HTTPH("HTTP2-Settings",		H_HTTP2_Settings,	HTTPH_R_PASS | HTTPH_R_FETCH | HTTPH_A_INS)	/* draft-ietf-httpbis-http2-12.txt */
 HTTPH("If-Match",		H_If_Match,			       HTTPH_R_FETCH		  )	/* RFC2616 14.24 */
 HTTPH("If-Modified-Since",	H_If_Modified_Since,		       HTTPH_R_FETCH		  )	/* RFC2616 14.25 */
 HTTPH("If-None-Match",		H_If_None_Match,		       HTTPH_R_FETCH		  )	/* RFC2616 14.26 */



From phk at FreeBSD.org  Thu Jun 19 09:52:49 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Thu, 19 Jun 2014 11:52:49 +0200
Subject: [master] ee92182 Add VRT_MAJOR_VERSION and VRT_MINOR_VERSION as
 discussed on the Stockholm VDD.
Message-ID: <E1WxZ1V-0005Wl-CC@project.varnish-software.com>


commit ee92182fe25177c392c3be2d1c3955096d10d2e6
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu Jun 19 09:51:19 2014 +0000

    Add VRT_MAJOR_VERSION and VRT_MINOR_VERSION as discussed on the Stockholm VDD.
    
    Renovate and simplify the compile-time binding between VMODs, VCC
    and varnishd.

diff --git a/bin/varnishd/cache/cache_vrt_vmod.c b/bin/varnishd/cache/cache_vrt_vmod.c
index 70b8d95..a2dc7c0 100644
--- a/bin/varnishd/cache/cache_vrt_vmod.c
+++ b/bin/varnishd/cache/cache_vrt_vmod.c
@@ -39,7 +39,6 @@
 
 #include "vcli_priv.h"
 #include "vrt.h"
-#include "vmod_abi.h"
 
 /*--------------------------------------------------------------------
  * Modules stuff
@@ -64,10 +63,10 @@ static VTAILQ_HEAD(,vmod)	vmods = VTAILQ_HEAD_INITIALIZER(vmods);
 
 int
 VRT_Vmod_Init(void **hdl, void *ptr, int len, const char *nm,
-    const char *path, struct cli *cli)
+    const char *path, const char *file_id, struct cli *cli)
 {
 	struct vmod *v;
-	void *x, *y, *z, *w;
+	const struct vmod_data *d;
 	char buf[256];
 	void *dlhdl;
 
@@ -90,50 +89,37 @@ VRT_Vmod_Init(void **hdl, void *ptr, int len, const char *nm,
 
 		v->hdl = dlhdl;
 
-		bprintf(buf, "Vmod_%s_Name", nm);
-		x = dlsym(v->hdl, buf);
-		bprintf(buf, "Vmod_%s_Len", nm);
-		y = dlsym(v->hdl, buf);
-		bprintf(buf, "Vmod_%s_Func", nm);
-		z = dlsym(v->hdl, buf);
-		bprintf(buf, "Vmod_%s_ABI", nm);
-		w = dlsym(v->hdl, buf);
-		if (x == NULL || y == NULL || z == NULL || w == NULL) {
+		bprintf(buf, "Vmod_%s_Data", nm);
+		d = dlsym(v->hdl, buf);
+		if (d == NULL ||
+		    d->file_id == NULL ||
+		    strcmp(d->file_id, file_id)) {
 			VCLI_Out(cli, "Loading VMOD %s from %s:\n", nm, path);
-			VCLI_Out(cli, "VMOD symbols not found\n");
-			VCLI_Out(cli, "Check relative pathnames.\n");
+			VCLI_Out(cli,
+			    "This is no longer the same file seen by"
+			    " the VCL-compiler.\n");
 			(void)dlclose(v->hdl);
 			FREE_OBJ(v);
 			return (1);
 		}
-		AN(x);
-		AN(y);
-		AN(z);
-		AN(w);
-		if (strcmp(x, nm)) {
+		if (d->vrt_major != VRT_MAJOR_VERSION ||
+		    d->vrt_minor > VRT_MINOR_VERSION ||
+		    d->name == NULL ||
+		    strcmp(d->name, nm) ||
+		    d->func == NULL ||
+		    d->func_len <= 0 ||
+		    d->proto == NULL ||
+		    d->spec == NULL ||
+		    d->abi == NULL) {
 			VCLI_Out(cli, "Loading VMOD %s from %s:\n", nm, path);
-			VCLI_Out(cli, "File contain wrong VMOD (\"%s\")\n",
-			    (char *) x);
-			VCLI_Out(cli, "Check relative pathnames ?.\n");
+			VCLI_Out(cli, "VMOD data is mangled.\n");
 			(void)dlclose(v->hdl);
 			FREE_OBJ(v);
 			return (1);
 		}
 
-		if (strcmp(w, VMOD_ABI_Version)) {
-			VCLI_Out(cli, "Loading VMOD %s from %s:\n", nm, path);
-			VCLI_Out(cli, "VMOD ABI (%s)", (char*)w);
-			VCLI_Out(cli, " incompatible with varnish ABI (%s)\n",
-			    VMOD_ABI_Version);
-			(void)dlclose(v->hdl);
-			FREE_OBJ(v);
-			return (1);
-		}
-
-		// XXX: Check w for ABI version compatibility
-
-		v->funclen = *(const int *)y;
-		v->funcs = z;
+		v->funclen = d->func_len;
+		v->funcs = d->func;
 
 		REPLACE(v->nm, nm);
 		REPLACE(v->path, path);
diff --git a/bin/varnishd/flint.lnt b/bin/varnishd/flint.lnt
index 0c8c7d6..9ad0543 100644
--- a/bin/varnishd/flint.lnt
+++ b/bin/varnishd/flint.lnt
@@ -58,18 +58,8 @@
 
 // Stuff in VMODs which is used through dl*(3) functions
 -esym(754, Vmod_*_Func::*)
--esym(765, Vmod_*_Func)
--esym(552, Vmod_*_Func)
--esym(765, Vmod_*_Len)
--esym(714, Vmod_*_Len)
--esym(765, Vmod_*_Name)
--esym(714, Vmod_*_Name)
--esym(765, Vmod_*_Proto)
--esym(714, Vmod_*_Proto)
--esym(765, Vmod_*_Spec)
--esym(714, Vmod_*_Spec)
--esym(765, Vmod_*_ABI)
--esym(714, Vmod_*_ABI)
+-esym(714, Vmod_*_Data)
+-esym(765, Vmod_*_Data)
 
 
 //-sem (pthread_mutex_lock, thread_lock)
diff --git a/include/vrt.h b/include/vrt.h
index 5884b00..066d6c8 100644
--- a/include/vrt.h
+++ b/include/vrt.h
@@ -26,11 +26,26 @@
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
  *
- * Runtime support for compiled VCL programs.
+ * Runtime support for compiled VCL programs and VMODs.
  *
- * XXX: When this file is changed, lib/libvcc/generate.py *MUST* be rerun.
+ * NB: When this file is changed, lib/libvcc/generate.py *MUST* be rerun.
  */
 
+/***********************************************************************
+ * Major and minor VRT API versions.
+ *
+ * Whenever something is added, increment MINOR version
+ * Whenever something is deleted or changed in a way which is not
+ * binary/load-time compatible, increment MAJOR version
+ */
+
+#define VRT_MAJOR_VERSION	1U
+
+#define VRT_MINOR_VERSION	1U
+
+
+/***********************************************************************/
+
 struct req;
 struct busyobj;
 struct vsl_log;
@@ -89,6 +104,22 @@ struct vrt_ctx {
 
 /***********************************************************************/
 
+struct vmod_data {
+	/* The version/id fields must be first, they protect the rest */
+	unsigned			vrt_major;
+	unsigned			vrt_minor;
+	const char			*file_id;
+
+	const char			*name;
+	const void			*func;
+	int				func_len;
+	const char			*proto;
+	const char			* const *spec;
+	const char			*abi;
+};
+
+/***********************************************************************/
+
 enum gethdr_e { HDR_REQ, HDR_RESP, HDR_OBJ, HDR_BEREQ, HDR_BERESP };
 
 struct gethdr_s {
@@ -230,7 +261,7 @@ int VRT_VSA_GetPtr(const struct suckaddr *sua, const unsigned char ** dst);
 
 /* VMOD/Modules related */
 int VRT_Vmod_Init(void **hdl, void *ptr, int len, const char *nm,
-    const char *path, struct cli *cli);
+    const char *path, const char *file_id, struct cli *cli);
 void VRT_Vmod_Fini(void **hdl);
 
 struct vmod_priv;
diff --git a/lib/libvcc/vcc_vmod.c b/lib/libvcc/vcc_vmod.c
index 1723007..4f139de 100644
--- a/lib/libvcc/vcc_vmod.c
+++ b/lib/libvcc/vcc_vmod.c
@@ -35,6 +35,7 @@
 #include "vcc_compile.h"
 
 #include "vmod_abi.h"
+#include "vrt.h"
 
 void
 vcc_ParseImport(struct vcc *tl)
@@ -44,14 +45,12 @@ vcc_ParseImport(struct vcc *tl)
 	char buf[256];
 	struct token *mod, *t1;
 	struct inifin *ifp;
-	const char *modname;
-	const char *proto;
-	const char *abi;
-	const char **spec;
+	const char * const *spec;
 	struct symbol *sym;
 	const struct symbol *osym;
 	const char *p;
 	// int *modlen;
+	const struct vmod_data *vmd;
 
 	t1 = tl->t;
 	SkipToken(tl, ID);		/* "import" */
@@ -62,7 +61,7 @@ vcc_ParseImport(struct vcc *tl)
 
 	osym = VCC_FindSymbol(tl, mod, SYM_NONE);
 	if (osym != NULL && osym->kind != SYM_VMOD) {
-		VSB_printf(tl->sb, "Module %.*s conflics with other symbol.\n",
+		VSB_printf(tl->sb, "Module %.*s conflicts with other symbol.\n",
 		    PF(mod));
 		vcc_ErrWhere2(tl, t1, tl->t);
 		return;
@@ -104,77 +103,88 @@ vcc_ParseImport(struct vcc *tl)
 		bprintf(fn, "%s/libvmod_%.*s.so", tl->vmod_dir, PF(mod));
 	}
 
-	ifp = New_IniFin(tl);
-
-	VSB_printf(ifp->ini, "\tif (VRT_Vmod_Init(&VGC_vmod_%.*s,\n", PF(mod));
-	VSB_printf(ifp->ini, "\t    &Vmod_%.*s_Func,\n", PF(mod));
-	VSB_printf(ifp->ini, "\t    sizeof(Vmod_%.*s_Func),\n", PF(mod));
-	VSB_printf(ifp->ini, "\t    \"%.*s\",\n", PF(mod));
-	VSB_printf(ifp->ini, "\t    ");
-	EncString(ifp->ini, fn, NULL, 0);
-	VSB_printf(ifp->ini, ",\n\t    ");
-	VSB_printf(ifp->ini, "cli))\n");
-	VSB_printf(ifp->ini, "\t\treturn(1);");
-
-	/* XXX: zero the function pointer structure ?*/
-	VSB_printf(ifp->fin, "\tVRT_priv_fini(&vmod_priv_%.*s);", PF(mod));
-	VSB_printf(ifp->fin, "\n\tVRT_Vmod_Fini(&VGC_vmod_%.*s);", PF(mod));
-
-	ifp = NULL;
-
 	SkipToken(tl, ';');
 
 	hdl = dlopen(fn, RTLD_NOW | RTLD_LOCAL);
 	if (hdl == NULL) {
-		VSB_printf(tl->sb, "Could not load module %.*s\n\t%s\n\t%s\n",
-		    PF(mod), fn, dlerror());
+		VSB_printf(tl->sb, "Could not load VMOD %.*s\n", PF(mod));
+		VSB_printf(tl->sb, "\tFile name: %s\n", fn);
+		VSB_printf(tl->sb, "\tdlerror:: %s\n", dlerror());
 		vcc_ErrWhere(tl, mod);
 		return;
 	}
 
-	bprintf(buf, "Vmod_%.*s_Name", PF(mod));
-	modname = dlsym(hdl, buf);
-	if (modname == NULL) {
-		VSB_printf(tl->sb, "Could not load module %.*s\n\t%s\n\t%s\n",
-		    PF(mod), fn, "Symbol Vmod_Name not found");
+	bprintf(buf, "Vmod_%.*s_Data", PF(mod));
+	vmd = dlsym(hdl, buf);
+	if (vmd == NULL) {
+		VSB_printf(tl->sb, "Malformed VMOD %.*s\n", PF(mod));
+		VSB_printf(tl->sb, "\tFile name: %s\n", fn);
+		VSB_printf(tl->sb, "\t(no Vmod_Data symbol)\n");
 		vcc_ErrWhere(tl, mod);
 		return;
 	}
-	if (!vcc_IdIs(mod, modname)) {
-		VSB_printf(tl->sb, "Could not load module %.*s\n\t%s\n",
-		    PF(mod), fn);
-		VSB_printf(tl->sb, "\tModule has wrong name: <%s>\n", modname);
+	if (vmd->vrt_major != VRT_MAJOR_VERSION ||
+	    vmd->vrt_minor > VRT_MINOR_VERSION) {
+		VSB_printf(tl->sb, "Incompatible VMOD %.*s\n", PF(mod));
+		VSB_printf(tl->sb, "\tFile name: %s\n", fn);
+		VSB_printf(tl->sb, "\tVMOD version %u.%u\n",
+		    vmd->vrt_major, vmd->vrt_minor);
+		VSB_printf(tl->sb, "\tvarnishd version %u.%u\n",
+		    VRT_MAJOR_VERSION, VRT_MINOR_VERSION);
 		vcc_ErrWhere(tl, mod);
 		return;
 	}
-
-	bprintf(buf, "Vmod_%.*s_ABI", PF(mod));
-	abi = dlsym(hdl, buf);
-	if (abi == NULL || strcmp(abi, VMOD_ABI_Version) != 0) {
-		VSB_printf(tl->sb, "Could not load module %.*s\n\t%s\n",
-		    PF(mod), fn);
-		VSB_printf(tl->sb, "\tABI mismatch, expected <%s>, got <%s>\n",
-			   VMOD_ABI_Version, abi);
+	if (vmd->name == NULL ||
+	    vmd->func == NULL ||
+	    vmd->func_len <= 0 ||
+	    vmd->proto == NULL ||
+	    vmd->abi == NULL) {
+		VSB_printf(tl->sb, "Mangled VMOD %.*s\n", PF(mod));
+		VSB_printf(tl->sb, "\tFile name: %s\n", fn);
+		VSB_printf(tl->sb, "\tInconsistent metadata\n");
 		vcc_ErrWhere(tl, mod);
 		return;
 	}
 
-	bprintf(buf, "Vmod_%.*s_Proto", PF(mod));
-	proto = dlsym(hdl, buf);
-	if (proto == NULL) {
-		VSB_printf(tl->sb, "Could not load module %.*s\n\t%s\n\t%s\n",
-		    PF(mod), fn, "Symbol Vmod_Proto not found");
+	if (!vcc_IdIs(mod, vmd->name)) {
+		VSB_printf(tl->sb, "Wrong VMOD file %.*s\n", PF(mod));
+		VSB_printf(tl->sb, "\tFile name: %s\n", fn);
+		VSB_printf(tl->sb, "\tContains vmod \"%s\"\n", vmd->name);
 		vcc_ErrWhere(tl, mod);
 		return;
 	}
-	bprintf(buf, "Vmod_%.*s_Spec", PF(mod));
-	spec = dlsym(hdl, buf);
-	if (spec == NULL) {
-		VSB_printf(tl->sb, "Could not load module %.*s\n\t%s\n\t%s\n",
-		    PF(mod), fn, "Symbol Vmod_Spec not found");
+
+	if (strcmp(vmd->abi, VMOD_ABI_Version) != 0) {
+		VSB_printf(tl->sb, "Incompatible VMOD %.*s\n", PF(mod));
+		VSB_printf(tl->sb, "\tFile name: %s\n", fn);
+		VSB_printf(tl->sb, "\tABI mismatch, expected <%s>, got <%s>\n",
+			   VMOD_ABI_Version, vmd->abi);
 		vcc_ErrWhere(tl, mod);
 		return;
 	}
+
+	ifp = New_IniFin(tl);
+
+	VSB_printf(ifp->ini, "\tif (VRT_Vmod_Init(&VGC_vmod_%.*s,\n", PF(mod));
+	VSB_printf(ifp->ini, "\t    &Vmod_%.*s_Func,\n", PF(mod));
+	VSB_printf(ifp->ini, "\t    sizeof(Vmod_%.*s_Func),\n", PF(mod));
+	VSB_printf(ifp->ini, "\t    \"%.*s\",\n", PF(mod));
+	VSB_printf(ifp->ini, "\t    ");
+	EncString(ifp->ini, fn, NULL, 0);
+	VSB_printf(ifp->ini, ",\n");
+	AN(vmd);
+	AN(vmd->file_id);
+	VSB_printf(ifp->ini, "\t    \"%s\",\n", vmd->file_id);
+	VSB_printf(ifp->ini, "\t    cli))\n");
+	VSB_printf(ifp->ini, "\t\treturn(1);");
+
+	/* XXX: zero the function pointer structure ?*/
+	VSB_printf(ifp->fin, "\tVRT_priv_fini(&vmod_priv_%.*s);", PF(mod));
+	VSB_printf(ifp->fin, "\n\tVRT_Vmod_Fini(&VGC_vmod_%.*s);", PF(mod));
+
+	ifp = NULL;
+
+	spec = vmd->spec;
 	for (; *spec != NULL; spec++) {
 		p = *spec;
 		if (!strcmp(p, "OBJ")) {
@@ -208,6 +218,6 @@ vcc_ParseImport(struct vcc *tl)
 	Fh(tl, 0, "\n/* --- BEGIN VMOD %.*s --- */\n\n", PF(mod));
 	Fh(tl, 0, "static void *VGC_vmod_%.*s;\n", PF(mod));
 	Fh(tl, 0, "static struct vmod_priv vmod_priv_%.*s;\n", PF(mod));
-	Fh(tl, 0, "\n%s\n", proto);
+	Fh(tl, 0, "\n%s\n", vmd->proto);
 	Fh(tl, 0, "\n/* --- END VMOD %.*s --- */\n\n", PF(mod));
 }
diff --git a/lib/libvcc/vmodtool.py b/lib/libvcc/vmodtool.py
index 315b46b..21c6c6d 100755
--- a/lib/libvcc/vmodtool.py
+++ b/lib/libvcc/vmodtool.py
@@ -40,6 +40,7 @@ import sys
 import re
 import optparse
 import unittest
+import random
 from os import unlink
 from os.path import dirname, realpath, exists
 from pprint import pprint, pformat
@@ -196,29 +197,17 @@ class Vmod(object):
 				fo.write(j + "\n")
 
 	def c_vmod(self, fo):
-		fo.write('extern const char Vmod_' + self.nam + '_Name[];\n')
-		fo.write('const char Vmod_' + self.nam + '_Name[] =')
-		fo.write(' \"' + self.nam + '";\n')
-		fo.write("\n")
 
 		cs = self.c_struct()
 		fo.write(cs + ';\n')
 
-		vfn = 'Vmod_' + self.nam + '_Func'
+		vfn = 'Vmod_%s_Func' % self.nam
 
-		fo.write("extern const struct " + vfn + " " + vfn + ';\n')
-		fo.write("const struct " + vfn + " " + vfn + ' =')
+		fo.write("\nstatic const struct %s Vmod_Func =" % vfn)
 		fo.write(self.c_initializer())
 		fo.write("\n")
 
-		fo.write("\n")
-		fo.write("extern const int Vmod_" + self.nam + '_Len;\n')
-		fo.write("const int Vmod_" + self.nam + '_Len =')
-		fo.write(" sizeof(Vmod_" + self.nam + "_Func);\n")
-		fo.write("\n")
-
-		fo.write("extern const char Vmod_" + self.nam + "_Proto[];\n")
-		fo.write("const char Vmod_" + self.nam + "_Proto[] =\n")
+		fo.write("\nstatic const char Vmod_Proto[] =\n")
 		for t in self.c_typedefs_():
 			for i in lwrap(t, w=64):
 				fo.write('\t"' + i + '\\n"\n')
@@ -230,12 +219,28 @@ class Vmod(object):
 		fo.write(self.c_strspec())
 
 		fo.write("\n")
-		fo.write('extern const char Vmod_' + self.nam + '_ABI[];\n')
-		fo.write('const char Vmod_' + self.nam + '_ABI[] =')
-		fo.write(' VMOD_ABI_Version;\n')
-		#fo.write("\n")
-		#fo.write('const void * const Vmod_' + self.nam + '_Id =')
-		#fo.write(' &Vmod_' + self.nam + '_Id;\n')
+
+		nm = "Vmod_" + self.nam + "_Data"
+		fo.write("extern const struct vmod_data " + nm + ";\n\n")
+		fo.write("const struct vmod_data " + nm + " = {\n")
+		fo.write("\t.vrt_major = VRT_MAJOR_VERSION,\n");
+		fo.write("\t.vrt_minor = VRT_MINOR_VERSION,\n");
+		fo.write("\t.name = \"%s\",\n" % self.nam)
+		fo.write("\t.func = &Vmod_Func,\n")
+		fo.write("\t.func_len = sizeof(Vmod_Func),\n")
+		fo.write("\t.proto = Vmod_Proto,\n")
+		fo.write("\t.spec = Vmod_Spec,\n")
+		fo.write("\t.abi = VMOD_ABI_Version,\n")
+
+		# NB: Sort of hackish:
+		# Fill file_id with random stuff, so we can tell if
+		# VCC and VRT_Vmod_Init() dlopens the same file
+		#
+		fo.write("\t.file_id = \"")
+		for i in range(32):
+			fo.write("%c" % random.randint(0x23,0x5b))
+		fo.write("\",\n")
+		fo.write("};\n")
 
 	def c_initializer(self):
 		s = '{\n'
@@ -269,8 +274,7 @@ class Vmod(object):
 		return s
 
 	def c_strspec(self):
-		s = "const char * const Vmod_" + self.nam + "_Spec[]"
-		s = "extern " + s + ";\n" + s + " = {\n"
+		s = "static const char * const Vmod_Spec[] = {\n"
 
 		for o in self.objs:
 			s += o.c_strspec(self.nam) + ",\n\n"



From lkarsten at varnish-software.com  Fri Jun 20 14:26:31 2014
From: lkarsten at varnish-software.com (Lasse Karstensen)
Date: Fri, 20 Jun 2014 16:26:31 +0200
Subject: [master] 98923ba Fix use of backend where it should say director.
Message-ID: <E1Wxzlv-0005Tp-B6@project.varnish-software.com>


commit 98923ba4b07bd5dbb0349e38e8d2ad711a2e67a4
Author: Lasse Karstensen <lkarsten at varnish-software.com>
Date:   Tue Jun 10 13:33:50 2014 +0200

    Fix use of backend where it should say director.

diff --git a/lib/libvmod_directors/vmod.vcc b/lib/libvmod_directors/vmod.vcc
index 94b9349..bbfa0eb 100644
--- a/lib/libvmod_directors/vmod.vcc
+++ b/lib/libvmod_directors/vmod.vcc
@@ -132,7 +132,7 @@ Description
 
 	Each backend backend will receive approximately
 	100 * (weight / (sum(all_added_weights))) per cent of the traffic sent
-	to this backend.
+	to this director.
 
 Example
 	vdir.add_backend(backend1, 10);



From lkarsten at varnish-software.com  Fri Jun 20 14:26:31 2014
From: lkarsten at varnish-software.com (Lasse Karstensen)
Date: Fri, 20 Jun 2014 16:26:31 +0200
Subject: [master] 96d0ba1 Write changelog and do groundwork for 4.0.1-rc1.
Message-ID: <E1Wxzlv-0005Tv-Cu@project.varnish-software.com>


commit 96d0ba1df5d7002927caf358e14245e40251520e
Author: Lasse Karstensen <lkarsten at varnish-software.com>
Date:   Fri Jun 20 16:19:12 2014 +0200

    Write changelog and do groundwork for 4.0.1-rc1.
    
    We still have a few bugs left to fix, but start out with this so the
    Jenkins machinery can be fine tuned in advance.

diff --git a/configure.ac b/configure.ac
index 36cc542..eed48bc 100644
--- a/configure.ac
+++ b/configure.ac
@@ -2,7 +2,7 @@ AC_PREREQ(2.59)
 AC_COPYRIGHT([Copyright (c) 2006 Verdens Gang AS
 Copyright (c) 2006-2014 Varnish Software AS])
 AC_REVISION([$Id$])
-AC_INIT([Varnish], [4.0.0], [varnish-dev at varnish-cache.org])
+AC_INIT([Varnish], [4.0.1-rc1], [varnish-dev at varnish-cache.org])
 AC_CONFIG_SRCDIR(include/miniobj.h)
 AC_CONFIG_HEADERS([config.h])
 AC_CONFIG_MACRO_DIR([m4])
diff --git a/doc/changes.rst b/doc/changes.rst
index 837266d..2ee4aeb 100644
--- a/doc/changes.rst
+++ b/doc/changes.rst
@@ -1,3 +1,84 @@
+
+============================================
+Changes from 4.0.0 to 4.0.1-rc1 (2014-06-20)
+============================================
+
+New since 4.0.0:
+
+- Backend IMS is now only attempted when last status was 200.
+- Packaging now uses find-provides instead of find-requires. [redhat]
+- Two new counters: n_purges and n_obj_purged.
+- Core size can now be set from /etc/sysconfig/varnish [redhat]
+- Via header set is now RFC compliant.
+- Removed "purge" keyword in VCL. Use return(purge) instead.
+- fallback director is now documented.
+- %D format flag in varnishncsa is now truncated to an integer value.
+- persistent storage backend is now deprecated.
+  https://www.varnish-cache.org/docs/trunk/phk/persistent.html
+- Added format flags %I (total bytes received) and %O (total bytes sent) for
+  varnishncsa.
+- python-docutils >= 0.7 is now required.
+- Support year (y) as a duration in VCL.
+- VMOD ABI requirements are relaxed, a VMOD no longer have to be run on the
+  same git revision as it was compiled for. Replaced by a major/minor ABI counter.
+
+
+Bugs fixed
+----------
+
+* 1475_ - time-to-first-byte in varnishncsa was potentially dishonest.
+* 1480_ - Porting guide for 4.0 is incomplete.
+* 1482_ - Inherit group memberships of -u specified user.
+* 1473_ - Fail correctly in configure when rst2man is not found.
+* 1486_ - Truncate negative Age values to zero.
+* 1488_ - Don't panic on high request rates.
+* 1489_ - req.esi should only be available in client threads.
+* 1490_ - Fix thread leak when reducing number of threads.
+* 1491_ - Reorder backend connection close procedure to help test cases.
+* 1498_ - Prefix translated VCL names to avoid name clashes.
+* 1499_ - Don't leak an objcore when HSH_Lookup returns expired object.
+* 1493_ - vcl_purge can return synth or restart.
+* 1476_ - Cope with systems having sys/endian.h and endian.h.
+* 1496_ - varnishadm should be consistent in argv ordering.
+* 1494_ - Don't panic on VCL-initiated retry after a backend 500 error.
+* 1139_ - Also reset keep (for IMS) time when purging.
+* 1478_ - Avoid panic when delivering an object that expires during delivery.
+* 1504_ - ACLs can be unreferenced with vcc_err_unref=off set.
+* 1501_ - Handle that a director couldn't pick a backend.
+* 1495_ - Reduce WRK_SumStat contention.
+* 1510_ - Complain on symbol reuse in VCL.
+* 1514_ - Document storage.NAME.free_space and .used_space [docs]
+* 1518_ - Suppress body on 304 response when using ESI.
+* 1519_ - Round-robin director does not support weight. [docs]
+
+
+.. _1475: https://www.varnish-cache.org/trac/ticket/1475
+.. _1480: https://www.varnish-cache.org/trac/ticket/1480
+.. _1482: https://www.varnish-cache.org/trac/ticket/1482
+.. _1473: https://www.varnish-cache.org/trac/ticket/1473
+.. _1486: https://www.varnish-cache.org/trac/ticket/1486
+.. _1488: https://www.varnish-cache.org/trac/ticket/1488
+.. _1489: https://www.varnish-cache.org/trac/ticket/1489
+.. _1490: https://www.varnish-cache.org/trac/ticket/1490
+.. _1491: https://www.varnish-cache.org/trac/ticket/1491
+.. _1498: https://www.varnish-cache.org/trac/ticket/1498
+.. _1499: https://www.varnish-cache.org/trac/ticket/1499
+.. _1493: https://www.varnish-cache.org/trac/ticket/1493
+.. _1476: https://www.varnish-cache.org/trac/ticket/1476
+.. _1496: https://www.varnish-cache.org/trac/ticket/1496
+.. _1494: https://www.varnish-cache.org/trac/ticket/1494
+.. _1139: https://www.varnish-cache.org/trac/ticket/1139
+.. _1478: https://www.varnish-cache.org/trac/ticket/1478
+.. _1504: https://www.varnish-cache.org/trac/ticket/1504
+.. _1501: https://www.varnish-cache.org/trac/ticket/1501
+.. _1495: https://www.varnish-cache.org/trac/ticket/1495
+.. _1510: https://www.varnish-cache.org/trac/ticket/1510
+.. _1514: https://www.varnish-cache.org/trac/ticket/1514
+.. _1518: https://www.varnish-cache.org/trac/ticket/1518
+.. _1519: https://www.varnish-cache.org/trac/ticket/1519
+
+
+
 ==============================================
 Changes from 4.0.0 beta1 to 4.0.0 (2014-04-10)
 ==============================================
diff --git a/redhat/varnish.spec b/redhat/varnish.spec
index b85217c..ec81229 100644
--- a/redhat/varnish.spec
+++ b/redhat/varnish.spec
@@ -1,10 +1,10 @@
-%define v_rc beta1
+%define v_rc rc1
 %define vd_rc %{?v_rc:-%{?v_rc}}
 %define    _use_internal_dependency_generator 0
 %define __find_provides %{_builddir}/varnish-%{version}%{?v_rc:-%{?v_rc}}/redhat/find-provides
 Summary: High-performance HTTP accelerator
 Name: varnish
-Version: 4.0.0
+Version: 4.0.1
 #Release: 0.20140328%{?v_rc}%{?dist}
 Release: 1%{?v_rc}%{?dist}
 License: BSD



From lkarsten at varnish-software.com  Fri Jun 20 14:26:31 2014
From: lkarsten at varnish-software.com (Lasse Karstensen)
Date: Fri, 20 Jun 2014 16:26:31 +0200
Subject: [master] 5ef67f5 RST warnings in changes.rst is a hard error.
Message-ID: <E1Wxzlv-0005U5-EQ@project.varnish-software.com>


commit 5ef67f5ba14b99a9e08651aa83abfe99219df532
Author: Lasse Karstensen <lkarsten at varnish-software.com>
Date:   Fri Jun 20 16:26:27 2014 +0200

    RST warnings in changes.rst is a hard error.

diff --git a/doc/Makefile.am b/doc/Makefile.am
index cfd0fb5..5da0749 100644
--- a/doc/Makefile.am
+++ b/doc/Makefile.am
@@ -1,10 +1,12 @@
 #
+#
+RST2ANY_FLAGS = --halt=2
 
 EXTRA_DIST = changes.rst changes.html
 
 changes.html: changes.rst
 if HAVE_RST2HTML
-	${RST2HTML} $? $@
+	${RST2HTML} ${RST2ANY_FLAGS} $? $@
 else
 	@echo "========================================"
 	@echo "You need rst2html installed to make dist"



From lkarsten at varnish-software.com  Fri Jun 20 15:05:37 2014
From: lkarsten at varnish-software.com (Lasse Karstensen)
Date: Fri, 20 Jun 2014 17:05:37 +0200
Subject: [master] 228055e python-docutils v0.6 will have to do.
Message-ID: <E1Wy0Nl-0004XG-Dy@project.varnish-software.com>


commit 228055ea74aa509b4379025317b560066714a839
Author: Lasse Karstensen <lkarsten at varnish-software.com>
Date:   Fri Jun 20 17:03:25 2014 +0200

    python-docutils v0.6 will have to do.
    
    Seems like requiring a version from 2010 was way optimistic.
    
    EL6 has v0.6 which was the first version rst2man appeared in.

diff --git a/doc/changes.rst b/doc/changes.rst
index 2ee4aeb..a66bda1 100644
--- a/doc/changes.rst
+++ b/doc/changes.rst
@@ -17,7 +17,7 @@ New since 4.0.0:
   https://www.varnish-cache.org/docs/trunk/phk/persistent.html
 - Added format flags %I (total bytes received) and %O (total bytes sent) for
   varnishncsa.
-- python-docutils >= 0.7 is now required.
+- python-docutils >= 0.6 is now required.
 - Support year (y) as a duration in VCL.
 - VMOD ABI requirements are relaxed, a VMOD no longer have to be run on the
   same git revision as it was compiled for. Replaced by a major/minor ABI counter.
diff --git a/redhat/varnish.spec b/redhat/varnish.spec
index ec81229..b50c859 100644
--- a/redhat/varnish.spec
+++ b/redhat/varnish.spec
@@ -18,7 +18,7 @@ BuildRoot: %{_tmppath}/%{name}-%{version}-%{release}-root-%(%{__id_u} -n)
 # You will need at least automake autoconf libtool
 #BuildRequires: automake autoconf libtool
 BuildRequires: ncurses-devel groff pcre-devel pkgconfig libedit-devel jemalloc-devel
-BuildRequires: python-docutils >= 0.7
+BuildRequires: python-docutils >= 0.6
 Requires: varnish-libs = %{version}-%{release}
 Requires: logrotate
 Requires: ncurses



From lkarsten at varnish-software.com  Fri Jun 20 15:07:48 2014
From: lkarsten at varnish-software.com (Lasse Karstensen)
Date: Fri, 20 Jun 2014 17:07:48 +0200
Subject: [master] 78b34d9 Forgot to mention the 200 Not Modified fix.
Message-ID: <E1Wy0Ps-0006wh-TU@project.varnish-software.com>


commit 78b34d9eb05b4afa98daa8bf6851a1ff98913e9b
Author: Lasse Karstensen <lkarsten at varnish-software.com>
Date:   Fri Jun 20 17:08:21 2014 +0200

    Forgot to mention the 200 Not Modified fix.
    
    Reminded by: daghf

diff --git a/doc/changes.rst b/doc/changes.rst
index a66bda1..c5e1425 100644
--- a/doc/changes.rst
+++ b/doc/changes.rst
@@ -5,6 +5,7 @@ Changes from 4.0.0 to 4.0.1-rc1 (2014-06-20)
 
 New since 4.0.0:
 
+- Varnish will no longer reply with "200 Not Modified".
 - Backend IMS is now only attempted when last status was 200.
 - Packaging now uses find-provides instead of find-requires. [redhat]
 - Two new counters: n_purges and n_obj_purged.



From perbu at varnish-software.com  Sat Jun 21 16:42:24 2014
From: perbu at varnish-software.com (Per Buer)
Date: Sat, 21 Jun 2014 18:42:24 +0200
Subject: [master] 489460a Ported the boltsort vmod to 4.0 and included it in
 the std vmod as discussed on the latest vdd.
Message-ID: <E1WyOMy-0002JA-SA@project.varnish-software.com>


commit 489460a7ab218f1dc4303b0baeec1bdfa9b4da38
Author: Per Buer <perbu at varnish-software.com>
Date:   Sat Jun 21 18:21:04 2014 +0200

    Ported the boltsort vmod to 4.0 and included it
    in the std vmod as discussed on the latest vdd.
    
    Renamed it to std.querysort. Two tests where
    included. They both pass on my machine.

diff --git a/bin/varnishtest/tests/m00014.vtc b/bin/varnishtest/tests/m00014.vtc
new file mode 100644
index 0000000..db2a62e
--- /dev/null
+++ b/bin/varnishtest/tests/m00014.vtc
@@ -0,0 +1,68 @@
+varnishtest "Test querysort in std vmod"
+
+server s1 {
+
+    rxreq
+    txresp
+
+    rxreq
+    txresp
+
+    rxreq
+    txresp
+
+    rxreq
+    txresp
+
+    rxreq
+    txresp
+
+    rxreq
+    txresp
+
+    rxreq
+    txresp
+
+    rxreq
+    txresp
+
+} -start
+
+varnish v1 -vcl+backend {
+
+    import ${vmod_std};
+    sub vcl_deliver {
+        set resp.http.naren = std.querysort(req.url);
+    }
+
+} -start
+
+client c1 {
+
+    txreq -url "/video/44505073?title=0&byline=0&portrait=0&color=51a516"
+    rxresp
+    expect resp.http.naren == "/video/44505073?byline=0&color=51a516&portrait=0&title=0"
+
+    txreq -url "/video/44505073?byline=0&&&&&"
+    rxresp
+    expect resp.http.naren == "/video/44505073?byline=0"
+
+    txreq -url "/video/2?&"
+    rxresp
+    expect resp.http.naren == "/video/2?"
+
+    txreq -url "/video/2"
+    rxresp
+    expect resp.http.naren == "/video/2"
+
+    txreq -url "/video/2?cod=cape&cape=cod"
+    rxresp
+    expect resp.http.naren == "/video/2?cape=cod&cod=cape"
+
+    txreq -url "/"
+    rxresp
+    expect resp.http.naren == "/"
+
+}
+
+client c1 -run
diff --git a/bin/varnishtest/tests/m00015.vtc b/bin/varnishtest/tests/m00015.vtc
new file mode 100644
index 0000000..47109c8
--- /dev/null
+++ b/bin/varnishtest/tests/m00015.vtc
@@ -0,0 +1,75 @@
+varnishtest "Test querysort of req.url in vcl_hash"
+
+server s1 {
+
+    rxreq
+    txresp
+    rxreq
+    txresp
+
+} -start
+
+varnish v1 -vcl+backend {
+
+    import ${vmod_std};
+
+    sub vcl_hash {
+        set req.url = std.querysort(req.url);
+    }
+
+    sub vcl_deliver {
+        if (obj.hits > 0) {
+            set resp.http.X-Cache = "HIT";
+        }
+        else {
+            set resp.http.X-Cache = "MISS";
+        }
+    }
+
+} -start
+
+client c1 {
+
+    txreq -url "/video/47013255?title=0&byline=0&portrait=0&autoplay=1"
+    rxresp
+    expect resp.status == 200
+    expect resp.http.X-Cache == "MISS"
+
+    txreq -url "/video/47013255?title=0&byline=0&portrait=0&autoplay=1"
+    rxresp
+    expect resp.status == 200
+    expect resp.http.X-Cache == "HIT"
+
+}
+
+client c2 {
+
+    txreq -url "/video/47013255?autoplay=1&title=0&byline=0&portrait=0"
+    rxresp
+    expect resp.status == 200
+    expect resp.http.X-Cache == "HIT"
+
+    txreq -url "autoplay=1&title=0&byline=0&portrait=0&&&&"
+    rxresp
+    expect resp.status == 200
+    expect resp.http.X-Cache == "HIT"
+
+}
+
+client c2 {
+
+    txreq -url "/video/47013255?autoplay=1&title=0&byline=0&portrait=0"
+    rxresp
+    expect resp.status == 200
+    expect resp.http.X-Cache == "HIT"
+
+    txreq -url "/video/47013255?autoplay=1&title=0&byline=0&portrait=0&&&&"
+    rxresp
+    expect resp.status == 200
+    expect resp.http.X-Cache == "HIT"
+
+}
+
+
+client c1 -run
+client c2 -run
diff --git a/lib/libvmod_std/vmod.vcc b/lib/libvmod_std/vmod.vcc
index 7a22156..0be272e 100644
--- a/lib/libvmod_std/vmod.vcc
+++ b/lib/libvmod_std/vmod.vcc
@@ -155,6 +155,16 @@ Description
 Example
 	std.timestamp("curl-request");
 
+$Function STRING querysort(STRING)
+
+Description
+        Sorts the querystring for cache normalization purposes.
+
+Example
+        set req.url = std.querysort(req.url);
+
+
+
 SEE ALSO
 ========
 
@@ -173,3 +183,4 @@ COPYRIGHT
 
 This document is licensed under the same licence as Varnish
 itself. See LICENCE for details.
+
diff --git a/lib/libvmod_std/vmod_std.c b/lib/libvmod_std/vmod_std.c
index 233c2da..bedbd7d 100644
--- a/lib/libvmod_std/vmod_std.c
+++ b/lib/libvmod_std/vmod_std.c
@@ -225,3 +225,140 @@ vmod_timestamp(const struct vrt_ctx *ctx, VCL_STRING label)
 		VSLb_ts_req(ctx->req, label, VTIM_real());
 	}
 }
+
+
+/* Boltsort
+   Author: Naren Venkataraman of Vimeo Inc.
+
+   Included here with permission.
+*/
+
+
+#define QS_MAX_PARAM_COUNT 32
+#define QS_EQUALS(c, h) ((c == h) || (c == '\0' && h == '&') || (c == '&' && h == '\0'))
+#define QS_ENDS(s) (s == '&' || s == '\0') 
+
+static const char QS_TERMINATORS[2] = {'\0', '&'};
+
+//since we dont store param length, we have to evaluate everytime
+static inline int param_compare (char *s, char *t)
+{
+
+    for ( ;QS_EQUALS(*s, *t); s++, t++) {
+        if (QS_ENDS(*s)) {
+            return 0;
+        }
+    }
+    return *s - *t;
+
+}
+
+//end of param is either first occurance of & or '\0'
+static inline int param_copy(char *dst, char *src, char *last_param)
+{
+
+    int len = strchr(src, QS_TERMINATORS[(src != last_param)]) - src;
+    memcpy(dst, src, len);
+    return len;
+
+}
+
+//Varnish vmod requires this
+int init_function(struct vmod_priv *priv, const struct VCL_conf *conf)
+{
+    return 0;
+
+}
+
+//sort query string
+VCL_STRING vmod_querysort(const struct vrt_ctx * ctx, VCL_STRING url)
+{
+
+    if (url == NULL) {
+        return NULL;
+    }
+
+    int qs_index = 0;
+    int param_count = 0;
+
+    char *dst_url = NULL;
+    char *qs = NULL;
+
+    //To avoid 1 pass for count calculations, assuming MAX_PARAM_COUNT as max
+    char* params[QS_MAX_PARAM_COUNT];
+
+    int i, p;
+    char *param = NULL;
+
+    qs = strchr(url, '?');
+    if(!qs) {
+        return url;
+    }
+
+    //add first param and increment count
+    params[param_count++] = ++qs;
+    qs_index = qs - url;
+
+    //Continue to find query string
+    while((qs = strchr(qs, '&')) != NULL) {
+        param = ++qs;
+
+        for(p = 0; p < param_count; p++) {
+            //if incoming param is < param at position then place it at p and then move up rest
+            if(param[0] < params[p][0] || param_compare(param, params[p]) < 0) {
+                for(i = param_count; i > p; i--) {
+                    params[i] = params[i-1];
+                }
+                break;
+            }
+        }
+        params[p] = param;
+        param_count++;
+
+        //if it exceed max params return as is
+        if (param_count == QS_MAX_PARAM_COUNT) {
+            return url;
+        }
+    }
+
+    //there is nothing after & 
+    //eg: http://127.0.0.1/?me=1&
+    if (param_count == 1) {
+        return url;
+    }
+
+    //allocate space for sorted url
+    //    struct ws *ws = sp->wrk->ws; 
+    struct ws *ws = ctx->ws;
+    dst_url = WS_Alloc(ws, strchr(param, '\0') - url + 1);
+    WS_Assert(ws);
+
+    //if alloc fails return as is
+    if(dst_url == NULL) {
+        return url;
+    }
+
+    //copy data before query string
+    char* cp = memcpy(dst_url, url, qs_index) + qs_index;
+
+    //get rid of all empty params /test/?a&&&
+    for(p = 0; p < param_count - 1; p++) { 
+        if (params[p][0] != '\0' && params[p][0] != '&') {
+            break;
+        }
+    }
+
+    //copy sorted params
+    for(; p < param_count - 1; p++) {
+        //copy and increment
+        cp += param_copy(cp, params[p], param);
+        *cp++ = '&';
+    }
+
+    //copy the last param
+    cp += param_copy(cp, params[p], param);
+    *cp = '\0';
+
+    return dst_url;
+
+}



From fgsch at lodoss.net  Sun Jun 22 08:36:44 2014
From: fgsch at lodoss.net (Federico G. Schwindt)
Date: Sun, 22 Jun 2014 10:36:44 +0200
Subject: [master] ed8b091 Reorganise querysort a bit
Message-ID: <E1WydGW-0001ec-1Y@project.varnish-software.com>


commit ed8b09192612cf22e85be226b966994e96cbdb72
Author: Federico G. Schwindt <fgsch at lodoss.net>
Date:   Sun Jun 22 09:31:02 2014 +0100

    Reorganise querysort a bit
    
    Also kill superflous function and test. Is this a candidate for its
    own file?

diff --git a/bin/varnishtest/tests/m00014.vtc b/bin/varnishtest/tests/m00014.vtc
index db2a62e..260280f 100644
--- a/bin/varnishtest/tests/m00014.vtc
+++ b/bin/varnishtest/tests/m00014.vtc
@@ -1,68 +1,36 @@
-varnishtest "Test querysort in std vmod"
-
-server s1 {
-
-    rxreq
-    txresp
-
-    rxreq
-    txresp
-
-    rxreq
-    txresp
-
-    rxreq
-    txresp
-
-    rxreq
-    txresp
-
-    rxreq
-    txresp
-
-    rxreq
-    txresp
-
-    rxreq
-    txresp
+varnishtest "Test std.querysort"
 
+server s1 -repeat 5 {
+	rxreq
+	txresp
 } -start
 
 varnish v1 -vcl+backend {
+	import ${vmod_std};
 
-    import ${vmod_std};
-    sub vcl_deliver {
-        set resp.http.naren = std.querysort(req.url);
-    }
-
+	sub vcl_deliver {
+		set resp.http.url = std.querysort(req.url);
+	}
 } -start
 
 client c1 {
-
-    txreq -url "/video/44505073?title=0&byline=0&portrait=0&color=51a516"
-    rxresp
-    expect resp.http.naren == "/video/44505073?byline=0&color=51a516&portrait=0&title=0"
-
-    txreq -url "/video/44505073?byline=0&&&&&"
-    rxresp
-    expect resp.http.naren == "/video/44505073?byline=0"
-
-    txreq -url "/video/2?&"
-    rxresp
-    expect resp.http.naren == "/video/2?"
-
-    txreq -url "/video/2"
-    rxresp
-    expect resp.http.naren == "/video/2"
-
-    txreq -url "/video/2?cod=cape&cape=cod"
-    rxresp
-    expect resp.http.naren == "/video/2?cape=cod&cod=cape"
-
-    txreq -url "/"
-    rxresp
-    expect resp.http.naren == "/"
-
-}
-
-client c1 -run
+	txreq -url "/foo/bar?t=0&b=0&p=0&c=5"
+	rxresp
+	expect resp.http.url == "/foo/bar?b=0&c=5&p=0&t=0"
+
+	txreq -url "/foo/bar?coa=0&co=0"
+	rxresp
+	expect resp.http.url == "/foo/bar?co=0&coa=0"
+
+	txreq -url "/foo/bar?a=0&&&&&"
+	rxresp
+	expect resp.http.url == "/foo/bar?a=0"
+
+	txreq -url "/foo/bar?&"
+	rxresp
+	expect resp.http.url == "/foo/bar?"
+
+	txreq -url "/foo/bar"
+	rxresp
+	expect resp.http.url == "/foo/bar"
+} -run
diff --git a/bin/varnishtest/tests/m00015.vtc b/bin/varnishtest/tests/m00015.vtc
deleted file mode 100644
index 47109c8..0000000
--- a/bin/varnishtest/tests/m00015.vtc
+++ /dev/null
@@ -1,75 +0,0 @@
-varnishtest "Test querysort of req.url in vcl_hash"
-
-server s1 {
-
-    rxreq
-    txresp
-    rxreq
-    txresp
-
-} -start
-
-varnish v1 -vcl+backend {
-
-    import ${vmod_std};
-
-    sub vcl_hash {
-        set req.url = std.querysort(req.url);
-    }
-
-    sub vcl_deliver {
-        if (obj.hits > 0) {
-            set resp.http.X-Cache = "HIT";
-        }
-        else {
-            set resp.http.X-Cache = "MISS";
-        }
-    }
-
-} -start
-
-client c1 {
-
-    txreq -url "/video/47013255?title=0&byline=0&portrait=0&autoplay=1"
-    rxresp
-    expect resp.status == 200
-    expect resp.http.X-Cache == "MISS"
-
-    txreq -url "/video/47013255?title=0&byline=0&portrait=0&autoplay=1"
-    rxresp
-    expect resp.status == 200
-    expect resp.http.X-Cache == "HIT"
-
-}
-
-client c2 {
-
-    txreq -url "/video/47013255?autoplay=1&title=0&byline=0&portrait=0"
-    rxresp
-    expect resp.status == 200
-    expect resp.http.X-Cache == "HIT"
-
-    txreq -url "autoplay=1&title=0&byline=0&portrait=0&&&&"
-    rxresp
-    expect resp.status == 200
-    expect resp.http.X-Cache == "HIT"
-
-}
-
-client c2 {
-
-    txreq -url "/video/47013255?autoplay=1&title=0&byline=0&portrait=0"
-    rxresp
-    expect resp.status == 200
-    expect resp.http.X-Cache == "HIT"
-
-    txreq -url "/video/47013255?autoplay=1&title=0&byline=0&portrait=0&&&&"
-    rxresp
-    expect resp.status == 200
-    expect resp.http.X-Cache == "HIT"
-
-}
-
-
-client c1 -run
-client c2 -run
diff --git a/lib/libvmod_std/vmod_std.c b/lib/libvmod_std/vmod_std.c
index bedbd7d..60852b6 100644
--- a/lib/libvmod_std/vmod_std.c
+++ b/lib/libvmod_std/vmod_std.c
@@ -226,139 +226,97 @@ vmod_timestamp(const struct vrt_ctx *ctx, VCL_STRING label)
 	}
 }
 
+/*
+ * Boltsort
+ * Author: Naren Venkataraman of Vimeo Inc.
+ * Included here with permission.
+ */
 
-/* Boltsort
-   Author: Naren Venkataraman of Vimeo Inc.
+#define QS_MAX_PARAM_COUNT	32
+#define QS_EQUALS(a, b)	\
+    ((a) == (b) || ((a) == '\0' && (b) == '&') || ((a) == '&' && (b) == '\0'))
 
-   Included here with permission.
-*/
+static ssize_t
+param_compare(char *s, char *t)
+{
+	for (; QS_EQUALS(*s, *t); s++, t++) {
+		if (*s == '&' || s == '\0')
+			return (0);
+	}
+	return (*s - *t);
+}
 
+static size_t
+param_copy(char *dst, char *src)
+{
+	size_t len;
+	len = strcspn(src, "&");
+	memcpy(dst, src, len);
+	return (len);
+}
 
-#define QS_MAX_PARAM_COUNT 32
-#define QS_EQUALS(c, h) ((c == h) || (c == '\0' && h == '&') || (c == '&' && h == '\0'))
-#define QS_ENDS(s) (s == '&' || s == '\0') 
+VCL_STRING __match_proto__(td_std_querysort)
+vmod_querysort(const struct vrt_ctx *ctx, VCL_STRING url)
+{
+	char *param, *params[QS_MAX_PARAM_COUNT];
+	char *p, *r;
+	size_t len;
+	int param_count;
+	int i, n;
 
-static const char QS_TERMINATORS[2] = {'\0', '&'};
+	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
 
-//since we dont store param length, we have to evaluate everytime
-static inline int param_compare (char *s, char *t)
-{
+	if (url == NULL)
+		return (NULL);
 
-    for ( ;QS_EQUALS(*s, *t); s++, t++) {
-        if (QS_ENDS(*s)) {
-            return 0;
-        }
-    }
-    return *s - *t;
+	p = strchr(url, '?');
+	if (p == NULL)
+		return (url);
 
-}
+	param_count = 0;
+	params[param_count++] = ++p;
+	len = p - url;
 
-//end of param is either first occurance of & or '\0'
-static inline int param_copy(char *dst, char *src, char *last_param)
-{
+	while ((p = strchr(p, '&')) != NULL) {
+		param = ++p;
 
-    int len = strchr(src, QS_TERMINATORS[(src != last_param)]) - src;
-    memcpy(dst, src, len);
-    return len;
+		for (i = 0; i < param_count; i++) {
+			if (param[0] < params[i][0] ||
+			    param_compare(param, params[i]) < 0) {
+				for (n = param_count; n > i; n--)
+					params[n] = params[n - 1];
+				break;
+			}
+		}
+		params[i] = param;
+		param_count++;
 
-}
+		if (param_count == QS_MAX_PARAM_COUNT)
+			return (url);
+	}
 
-//Varnish vmod requires this
-int init_function(struct vmod_priv *priv, const struct VCL_conf *conf)
-{
-    return 0;
+	if (param_count == 1)
+		return (url);
 
-}
+	r = WS_Alloc(ctx->ws, strchr(param, '\0') - url + 1);
+	if (r == NULL)
+		return (url);
 
-//sort query string
-VCL_STRING vmod_querysort(const struct vrt_ctx * ctx, VCL_STRING url)
-{
+	p = memcpy(r, url, len);
+	p += len;
+
+	for (i = 0; i < param_count - 1; i++) {
+		if (params[i][0] != '\0' && params[i][0] != '&')
+			break;
+	}
+
+	for (; i < param_count - 1; i++) {
+		p += param_copy(p, params[i]);
+		*p++ = '&';
+	}
 
-    if (url == NULL) {
-        return NULL;
-    }
-
-    int qs_index = 0;
-    int param_count = 0;
-
-    char *dst_url = NULL;
-    char *qs = NULL;
-
-    //To avoid 1 pass for count calculations, assuming MAX_PARAM_COUNT as max
-    char* params[QS_MAX_PARAM_COUNT];
-
-    int i, p;
-    char *param = NULL;
-
-    qs = strchr(url, '?');
-    if(!qs) {
-        return url;
-    }
-
-    //add first param and increment count
-    params[param_count++] = ++qs;
-    qs_index = qs - url;
-
-    //Continue to find query string
-    while((qs = strchr(qs, '&')) != NULL) {
-        param = ++qs;
-
-        for(p = 0; p < param_count; p++) {
-            //if incoming param is < param at position then place it at p and then move up rest
-            if(param[0] < params[p][0] || param_compare(param, params[p]) < 0) {
-                for(i = param_count; i > p; i--) {
-                    params[i] = params[i-1];
-                }
-                break;
-            }
-        }
-        params[p] = param;
-        param_count++;
-
-        //if it exceed max params return as is
-        if (param_count == QS_MAX_PARAM_COUNT) {
-            return url;
-        }
-    }
-
-    //there is nothing after & 
-    //eg: http://127.0.0.1/?me=1&
-    if (param_count == 1) {
-        return url;
-    }
-
-    //allocate space for sorted url
-    //    struct ws *ws = sp->wrk->ws; 
-    struct ws *ws = ctx->ws;
-    dst_url = WS_Alloc(ws, strchr(param, '\0') - url + 1);
-    WS_Assert(ws);
-
-    //if alloc fails return as is
-    if(dst_url == NULL) {
-        return url;
-    }
-
-    //copy data before query string
-    char* cp = memcpy(dst_url, url, qs_index) + qs_index;
-
-    //get rid of all empty params /test/?a&&&
-    for(p = 0; p < param_count - 1; p++) { 
-        if (params[p][0] != '\0' && params[p][0] != '&') {
-            break;
-        }
-    }
-
-    //copy sorted params
-    for(; p < param_count - 1; p++) {
-        //copy and increment
-        cp += param_copy(cp, params[p], param);
-        *cp++ = '&';
-    }
-
-    //copy the last param
-    cp += param_copy(cp, params[p], param);
-    *cp = '\0';
-
-    return dst_url;
+	p += param_copy(p, params[i]);
+	*p = '\0';
 
+	return (r);
 }



From fgsch at lodoss.net  Sun Jun 22 08:54:10 2014
From: fgsch at lodoss.net (Federico G. Schwindt)
Date: Sun, 22 Jun 2014 10:54:10 +0200
Subject: [master] b61800b Add a string to real conversion function
Message-ID: <E1WydXO-0003ZL-3j@project.varnish-software.com>


commit b61800b639eb853faeebb3587592eb37006df199
Author: Federico G. Schwindt <fgsch at lodoss.net>
Date:   Sun Jun 22 09:49:35 2014 +0100

    Add a string to real conversion function
    
    As discussed on the latest vdd.

diff --git a/bin/varnishtest/tests/m00015.vtc b/bin/varnishtest/tests/m00015.vtc
new file mode 100644
index 0000000..077bbaf
--- /dev/null
+++ b/bin/varnishtest/tests/m00015.vtc
@@ -0,0 +1,23 @@
+varnishtest "Test vmod_std.real conversion"
+
+server s1 {
+	rxreq
+	txresp
+} -start
+
+varnish v1 -vcl+backend {
+	import ${vmod_std};
+
+	sub vcl_deliver {
+		set resp.http.x-real = std.real(req.http.foo, 0.0);
+		set resp.http.x-fallback = std.real(req.http.bar, 0.0);
+	}
+} -start
+
+client c1 {
+	txreq -hdr "foo: 1.00"
+	rxresp
+	expect resp.status == 200
+	expect resp.http.x-real == 1.000
+	expect resp.http.x-fallback == 0.000
+} -run
diff --git a/lib/libvmod_std/vmod.vcc b/lib/libvmod_std/vmod.vcc
index 0be272e..cc3eb43 100644
--- a/lib/libvmod_std/vmod.vcc
+++ b/lib/libvmod_std/vmod.vcc
@@ -135,6 +135,14 @@ Description
 Example
 	if (std.ip(req.http.X-forwarded-for, "0.0.0.0") ~ my_acl) { ... }
 
+$Function REAL real(STRING, REAL)
+
+Description
+	Converts the string *s* to a real.  If *s* fails to parse,
+	*fallback* will be returned.
+Example
+	set req.http.x-real = std.real(req.http.x-foo, 0.0);
+
 $Function BOOL healthy(BACKEND)
 
 Description
diff --git a/lib/libvmod_std/vmod_std_conversions.c b/lib/libvmod_std/vmod_std_conversions.c
index f98b5fb..7c1ac08 100644
--- a/lib/libvmod_std/vmod_std_conversions.c
+++ b/lib/libvmod_std/vmod_std_conversions.c
@@ -168,3 +168,33 @@ vmod_ip(const struct vrt_ctx *ctx, VCL_STRING s, VCL_IP d)
 		freeaddrinfo(res0);
 	return (r);
 }
+
+VCL_REAL __match_proto__()
+vmod_real(const struct vrt_ctx *ctx, VCL_STRING p, VCL_REAL d)
+{
+	char *e;
+	double r;
+
+	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
+
+	if (p == NULL)
+		return (d);
+
+	while (isspace(*p))
+		p++;
+
+	if (*p != '+' && *p != '-' && !isdigit(*p))
+		return (d);
+
+	e = NULL;
+
+	r = strtod(p, &e);
+
+	if (!isfinite(r))
+		return (d);
+
+	if (e == NULL || *e != '\0')
+		return (d);
+
+	return (r);
+}



From phk at FreeBSD.org  Mon Jun 23 06:58:03 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Mon, 23 Jun 2014 08:58:03 +0200
Subject: [master] 07d0bc4 Add a couple of 'const' and a missing star
Message-ID: <E1WyyCZ-0002W3-8n@project.varnish-software.com>


commit 07d0bc40e4a1d5d3e8754cafa65cf86ecfdb3de2
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Jun 23 06:57:47 2014 +0000

    Add a couple of 'const' and a missing star

diff --git a/lib/libvmod_std/vmod_std.c b/lib/libvmod_std/vmod_std.c
index 60852b6..98ed54d 100644
--- a/lib/libvmod_std/vmod_std.c
+++ b/lib/libvmod_std/vmod_std.c
@@ -237,17 +237,17 @@ vmod_timestamp(const struct vrt_ctx *ctx, VCL_STRING label)
     ((a) == (b) || ((a) == '\0' && (b) == '&') || ((a) == '&' && (b) == '\0'))
 
 static ssize_t
-param_compare(char *s, char *t)
+param_compare(const char *s, const char *t)
 {
 	for (; QS_EQUALS(*s, *t); s++, t++) {
-		if (*s == '&' || s == '\0')
+		if (*s == '&' || *s == '\0')
 			return (0);
 	}
 	return (*s - *t);
 }
 
 static size_t
-param_copy(char *dst, char *src)
+param_copy(char *dst, const char *src)
 {
 	size_t len;
 	len = strcspn(src, "&");



From phk at FreeBSD.org  Mon Jun 23 07:36:50 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Mon, 23 Jun 2014 09:36:50 +0200
Subject: [master] b61c230 Always expire the ims-candidate object on a
 successful fetch
Message-ID: <E1Wyyo6-0004W8-Pb@project.varnish-software.com>


commit b61c2305462c9428225a9531e664907db408fd7f
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Jun 23 07:36:23 2014 +0000

    Always expire the ims-candidate object on a successful fetch
    
    Fixes	#1530

diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c
index c175e74..a768170 100644
--- a/bin/varnishd/cache/cache_fetch.c
+++ b/bin/varnishd/cache/cache_fetch.c
@@ -557,6 +557,8 @@ vbf_stp_fetch(struct worker *wrk, struct busyobj *bo)
 
 	VBO_setstate(bo, BOS_FINISHED);
 	VSLb_ts_busyobj(bo, "BerespBody", W_TIM_real(wrk));
+	if (bo->ims_obj != NULL)
+		EXP_Rearm(bo->ims_obj, bo->ims_obj->exp.t_origin, 0, 0, 0);
 	return (F_STP_DONE);
 }
 



From phk at FreeBSD.org  Mon Jun 23 09:14:05 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Mon, 23 Jun 2014 11:14:05 +0200
Subject: [master] affd96d Generalize v1f_pull_chunked() into HTTP1_Chunked()
 which can also be used on requests.
Message-ID: <E1Wz0KD-000844-7f@project.varnish-software.com>


commit affd96d46f7be3e11b2db6747b13271ea9f63246
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Jun 23 09:13:25 2014 +0000

    Generalize v1f_pull_chunked() into HTTP1_Chunked() which can also be
    used on requests.

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index 38e597b..2b10f59 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -859,6 +859,14 @@ int HTTP1_CacheReqBody(struct req *req, ssize_t maxsize);
 int HTTP1_IterateReqBody(struct req *req, req_body_iter_f *func, void *priv);
 extern const int HTTP1_Req[3];
 extern const int HTTP1_Resp[3];
+enum http1_chunked_ret {
+	H1CR_ERROR,
+	H1CR_MORE,
+	H1CR_END,
+};
+enum http1_chunked_ret
+HTTP1_Chunked(struct http_conn *htc, intptr_t *priv, const char **error,
+    int64_t *statp, void *ptr, ssize_t *lp);
 
 /* cache_http1_deliver.c */
 unsigned V1D_FlushReleaseAcct(struct req *req);
diff --git a/bin/varnishd/cache/cache_http1_fetch.c b/bin/varnishd/cache/cache_http1_fetch.c
index 1c3ef53..aa009f2 100644
--- a/bin/varnishd/cache/cache_http1_fetch.c
+++ b/bin/varnishd/cache/cache_http1_fetch.c
@@ -40,7 +40,6 @@
 
 #include "cache_backend.h"
 #include "vcli_priv.h"
-#include "vct.h"
 #include "vtcp.h"
 #include "vtim.h"
 
@@ -110,10 +109,7 @@ v1f_pull_straight(struct busyobj *bo, void *p, ssize_t *lp, intptr_t *priv)
 static enum vfp_status __match_proto__(vfp_pull_f)
 v1f_pull_chunked(struct busyobj *bo, void *p, ssize_t *lp, intptr_t *priv)
 {
-	int i;
-	char buf[20];		/* XXX: 20 is arbitrary */
-	unsigned u;
-	ssize_t cl, l, lr;
+	const char *err;
 
 	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
 	if (p == vfp_init)
@@ -123,77 +119,18 @@ v1f_pull_chunked(struct busyobj *bo, void *p, ssize_t *lp, intptr_t *priv)
 	AN(p);
 	AN(lp);
 	AN(priv);
-	l = *lp;
-	*lp = 0;
-	if (*priv == -1) {
-		/* Skip leading whitespace */
-		do {
-			lr = HTTP1_Read(&bo->htc, buf, 1);
-			if (lr <= 0)
-				return (VFP_Error(bo, "chunked read err"));
-			bo->acct.beresp_bodybytes += lr;
-		} while (vct_islws(buf[0]));
-
-		if (!vct_ishex(buf[0]))
-			 return (VFP_Error(bo, "chunked header non-hex"));
-
-		/* Collect hex digits, skipping leading zeros */
-		for (u = 1; u < sizeof buf; u++) {
-			do {
-				lr = HTTP1_Read(&bo->htc, buf + u, 1);
-				if (lr <= 0)
-					return (VFP_Error(bo,
-					    "chunked read err"));
-				bo->acct.beresp_bodybytes += lr;
-			} while (u == 1 && buf[0] == '0' && buf[u] == '0');
-			if (!vct_ishex(buf[u]))
-				break;
-		}
-
-		if (u >= sizeof buf)
-			return (VFP_Error(bo,"chunked header too long"));
-
-		/* Skip trailing white space */
-		while(vct_islws(buf[u]) && buf[u] != '\n') {
-			lr = HTTP1_Read(&bo->htc, buf + u, 1);
-			if (lr <= 0)
-				return (VFP_Error(bo, "chunked read err"));
-			bo->acct.beresp_bodybytes += lr;
-		}
 
-		if (buf[u] != '\n')
-			return (VFP_Error(bo,"chunked header no NL"));
-
-		buf[u] = '\0';
-
-		cl = vbf_fetch_number(buf, 16);
-		if (cl < 0)
-			return (VFP_Error(bo,"chunked header number syntax"));
-		*priv = cl;
-	}
-	if (*priv > 0) {
-		if (*priv < l)
-			l = *priv;
-		lr = HTTP1_Read(&bo->htc, p, l);
-		if (lr <= 0)
-			return (VFP_Error(bo, "straight insufficient bytes"));
-		bo->acct.beresp_bodybytes += lr;
-		*lp = lr;
-		*priv -= lr;
-		if (*priv == 0)
-			*priv = -1;
+	switch (HTTP1_Chunked(&bo->htc, priv, &err,
+	    &bo->acct.beresp_bodybytes, p, lp)) {
+	case H1CR_ERROR:
+		return (VFP_Error(bo, "%s", err));
+	case H1CR_MORE:
 		return (VFP_OK);
+	case H1CR_END:
+		return (VFP_END);
+	default:
+		WRONG("invalid HTTP1_Chunked return");
 	}
-	AZ(*priv);
-	i = HTTP1_Read(&bo->htc, buf, 1);
-	if (i <= 0)
-		return (VFP_Error(bo, "chunked read err"));
-	bo->acct.beresp_bodybytes += i;
-	if (buf[0] == '\r' && HTTP1_Read(&bo->htc, buf, 1) <= 0)
-		return (VFP_Error(bo, "chunked read err"));
-	if (buf[0] != '\n')
-		return (VFP_Error(bo,"chunked tail no NL"));
-	return (VFP_END);
 }
 
 /*--------------------------------------------------------------------*/
diff --git a/bin/varnishd/cache/cache_http1_proto.c b/bin/varnishd/cache/cache_http1_proto.c
index 7f7852c..0dc5fe8 100644
--- a/bin/varnishd/cache/cache_http1_proto.c
+++ b/bin/varnishd/cache/cache_http1_proto.c
@@ -45,6 +45,8 @@
 
 #include "config.h"
 
+#include <inttypes.h>
+
 #include "cache.h"
 
 #include "vct.h"
@@ -524,3 +526,106 @@ HTTP1_Write(const struct worker *w, const struct http *hp, const int *hf)
 	l += WRW_Write(w, "\r\n", -1);
 	return (l);
 }
+
+/*--------------------------------------------------------------------
+ * Read a chunked body.
+ *
+ * XXX: Reading one byte at a time is pretty pessimal.
+ */
+
+enum http1_chunked_ret
+HTTP1_Chunked(struct http_conn *htc, intptr_t *priv, const char **error,
+    int64_t *statp, void *ptr, ssize_t *lp)
+{
+	int i;
+	char buf[20];		/* XXX: 20 is arbitrary */
+	char *q;
+	unsigned u;
+	uintmax_t cll;
+	ssize_t cl, l, lr;
+
+#define  ERR(x) do { *error = (x); return (H1CR_ERROR); } while (0)
+
+	CHECK_OBJ_NOTNULL(htc, HTTP_CONN_MAGIC);
+	AN(priv);
+	AN(error);
+	AN(statp);
+	AN(ptr);
+	AN(lp);
+	l = *lp;
+	*lp = 0;
+	if (*priv == -1) {
+		/* Skip leading whitespace */
+		do {
+			lr = HTTP1_Read(htc, buf, 1);
+			if (lr <= 0)
+				ERR("chunked read err");
+			*statp += lr;
+		} while (vct_islws(buf[0]));
+
+		if (!vct_ishex(buf[0]))
+			 ERR("chunked header non-hex");
+
+		/* Collect hex digits, skipping leading zeros */
+		for (u = 1; u < sizeof buf; u++) {
+			do {
+				lr = HTTP1_Read(htc, buf + u, 1);
+				if (lr <= 0)
+					ERR("chunked read err");
+				*statp += lr;
+			} while (u == 1 && buf[0] == '0' && buf[u] == '0');
+			if (!vct_ishex(buf[u]))
+				break;
+		}
+
+		if (u >= sizeof buf)
+			ERR("chunked header too long");
+
+		/* Skip trailing white space */
+		while(vct_islws(buf[u]) && buf[u] != '\n') {
+			lr = HTTP1_Read(htc, buf + u, 1);
+			if (lr <= 0)
+				ERR("chunked read err");
+			*statp += lr;
+		}
+
+		if (buf[u] != '\n')
+			ERR("chunked header no NL");
+
+		buf[u] = '\0';
+
+		cll = strtoumax(buf, &q, 16);
+		if (q == NULL || *q != '\0')
+			ERR("chunked header number syntax");
+		cl = (ssize_t)cll;
+		if((uintmax_t)cl != cll)
+			ERR("bogusly large chunk size");
+
+		*priv = cl;
+	}
+	if (*priv > 0) {
+		if (*priv < l)
+			l = *priv;
+		lr = HTTP1_Read(htc, ptr, l);
+		if (lr <= 0)
+			ERR("straight insufficient bytes");
+		*statp += lr;
+		*lp = lr;
+		*priv -= lr;
+		if (*priv == 0)
+			*priv = -1;
+		return (H1CR_MORE);
+	}
+	AZ(*priv);
+	i = HTTP1_Read(htc, buf, 1);
+	if (i <= 0)
+		ERR("chunked read err");
+	*statp += i;
+	if (buf[0] == '\r' && HTTP1_Read(htc, buf, 1) <= 0)
+		ERR("chunked read err");
+	if (buf[0] != '\n')
+		ERR("chunked tail no NL");
+	return (H1CR_END);
+#undef ERR
+}
+



From phk at FreeBSD.org  Mon Jun 23 09:53:58 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Mon, 23 Jun 2014 11:53:58 +0200
Subject: [master] b4b628f Implement proper handling of chunked encoding of
 req.body.
Message-ID: <E1Wz0wo-0004GU-Tv@project.varnish-software.com>


commit b4b628fa01591d50b2ce09e570ecd5470477cc65
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Jun 23 09:53:21 2014 +0000

    Implement proper handling of chunked encoding of req.body.
    
    Fixes	#1524

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index 2b10f59..dc7e6f1 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -236,7 +236,7 @@ struct acct_req {
 /*--------------------------------------------------------------------*/
 
 struct acct_bereq {
-#define ACCT(foo)	ssize_t		foo;
+#define ACCT(foo)	uint64_t	foo;
 #include "tbl/acct_fields_bereq.h"
 #undef ACCT
 };
@@ -665,6 +665,7 @@ struct req {
 
 	unsigned char		wantbody;
 	uint64_t		req_bodybytes;	/* Parsed req bodybytes */
+	intptr_t		chunk_ctr;	/* Parsed req bodybytes */
 
 	uint64_t		resp_hdrbytes;	/* Scheduled resp hdrbytes */
 	uint64_t		resp_bodybytes; /* Scheduled resp bodybytes */
@@ -866,7 +867,7 @@ enum http1_chunked_ret {
 };
 enum http1_chunked_ret
 HTTP1_Chunked(struct http_conn *htc, intptr_t *priv, const char **error,
-    int64_t *statp, void *ptr, ssize_t *lp);
+    uint64_t *statp, void *ptr, ssize_t *lp);
 
 /* cache_http1_deliver.c */
 unsigned V1D_FlushReleaseAcct(struct req *req);
@@ -1123,7 +1124,7 @@ void WRW_Chunked(const struct worker *w);
 void WRW_EndChunk(const struct worker *w);
 void WRW_Reserve(struct worker *w, int *fd, struct vsl_log *, double t0);
 unsigned WRW_Flush(const struct worker *w);
-unsigned WRW_FlushRelease(struct worker *w, ssize_t *pacc);
+unsigned WRW_FlushRelease(struct worker *w, uint64_t *pacc);
 unsigned WRW_Write(const struct worker *w, const void *ptr, int len);
 unsigned WRW_WriteH(const struct worker *w, const txt *hh, const char *suf);
 
diff --git a/bin/varnishd/cache/cache_http1_deliver.c b/bin/varnishd/cache/cache_http1_deliver.c
index 8778303..1bdcef7 100644
--- a/bin/varnishd/cache/cache_http1_deliver.c
+++ b/bin/varnishd/cache/cache_http1_deliver.c
@@ -211,7 +211,7 @@ unsigned
 V1D_FlushReleaseAcct(struct req *req)
 {
 	unsigned u;
-	ssize_t txcnt = 0, hdrbytes;
+	uint64_t txcnt = 0, hdrbytes;
 
 	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
 	CHECK_OBJ_NOTNULL(req->wrk, WORKER_MAGIC);
diff --git a/bin/varnishd/cache/cache_http1_fsm.c b/bin/varnishd/cache/cache_http1_fsm.c
index 9191d1c..dac34e0 100644
--- a/bin/varnishd/cache/cache_http1_fsm.c
+++ b/bin/varnishd/cache/cache_http1_fsm.c
@@ -285,6 +285,8 @@ http1_req_body_status(struct req *req)
 		req->h1.bytes_yet = req->req_bodybytes - req->h1.bytes_done;
 		return (REQ_BODY_PRESENT);
 	}
+	if (http_HdrIs(req->http, H_Transfer_Encoding, "chunked"))
+		return (REQ_BODY_CHUNKED);
 	if (http_GetHdr(req->http, H_Transfer_Encoding, NULL))
 		return (REQ_BODY_FAIL);
 	return (REQ_BODY_NONE);
@@ -480,28 +482,46 @@ HTTP1_Session(struct worker *wrk, struct req *req)
 }
 
 static ssize_t
-http1_iter_req_body(struct req *req, void *buf, ssize_t len)
+http1_iter_req_body(struct req *req, enum req_body_state_e bs,
+    void *buf, ssize_t len)
 {
+	const char *err;
+
 	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
 
-	AN(req->req_bodybytes);
 	AN(len);
 	AN(buf);
-	if (len > req->req_bodybytes - req->h1.bytes_done)
-		len = req->req_bodybytes - req->h1.bytes_done;
-	if (len == 0) {
-		req->req_body_status = REQ_BODY_DONE;
-		return (0);
-	}
-	len = HTTP1_Read(req->htc, buf, len);
-	if (len <= 0) {
-		req->req_body_status = REQ_BODY_FAIL;
-		return (-1);
-	}
-	req->h1.bytes_done += len;
-	req->h1.bytes_yet = req->req_bodybytes - req->h1.bytes_done;
-	req->acct.req_bodybytes += len;
-	return (len);
+	if (bs == REQ_BODY_PRESENT) {
+		AN(req->req_bodybytes);
+		if (len > req->req_bodybytes - req->h1.bytes_done)
+			len = req->req_bodybytes - req->h1.bytes_done;
+		if (len == 0) {
+			req->req_body_status = REQ_BODY_DONE;
+			return (0);
+		}
+		len = HTTP1_Read(req->htc, buf, len);
+		if (len <= 0) {
+			req->req_body_status = REQ_BODY_FAIL;
+			return (-1);
+		}
+		req->h1.bytes_done += len;
+		req->h1.bytes_yet = req->req_bodybytes - req->h1.bytes_done;
+		req->acct.req_bodybytes += len;
+		return (len);
+	} else if (bs == REQ_BODY_CHUNKED) {
+		switch (HTTP1_Chunked(req->htc, &req->chunk_ctr, &err,
+		    &req->acct.req_bodybytes, buf, &len)) {
+		case H1CR_ERROR:
+			return (-1);
+		case H1CR_MORE:
+			return (len);
+		case H1CR_END:
+			return (0);
+		default:
+			WRONG("invalid HTTP1_Chunked return");
+		}
+	} else
+		WRONG("Illegal req_bodystatus");
 }
 
 /*----------------------------------------------------------------------
@@ -516,6 +536,7 @@ HTTP1_IterateReqBody(struct req *req, req_body_iter_f *func, void *priv)
 {
 	char buf[8192];
 	struct storage *st;
+	enum req_body_state_e bs;
 	ssize_t l;
 	int i;
 
@@ -533,6 +554,7 @@ HTTP1_IterateReqBody(struct req *req, req_body_iter_f *func, void *priv)
 	case REQ_BODY_NONE:
 		return (0);
 	case REQ_BODY_PRESENT:
+	case REQ_BODY_CHUNKED:
 		break;
 	case REQ_BODY_DONE:
 	case REQ_BODY_TAKEN:
@@ -543,7 +565,9 @@ HTTP1_IterateReqBody(struct req *req, req_body_iter_f *func, void *priv)
 		WRONG("Wrong req_body_status in HTTP1_IterateReqBody()");
 	}
 	Lck_Lock(&req->sp->mtx);
-	if (req->req_body_status == REQ_BODY_PRESENT) {
+	bs = req->req_body_status;
+	if (req->req_body_status == REQ_BODY_PRESENT ||
+	    req->req_body_status == REQ_BODY_CHUNKED) {
 		req->req_body_status = REQ_BODY_TAKEN;
 		i = 0;
 	} else
@@ -556,7 +580,7 @@ HTTP1_IterateReqBody(struct req *req, req_body_iter_f *func, void *priv)
 	}
 
 	do {
-		l = http1_iter_req_body(req, buf, sizeof buf);
+		l = http1_iter_req_body(req, bs, buf, sizeof buf);
 		if (l < 0) {
 			req->doclose = SC_RX_BODY;
 			break;
@@ -626,6 +650,7 @@ HTTP1_CacheReqBody(struct req *req, ssize_t maxsize)
 		return (-1);
 	case REQ_BODY_NONE:
 		return (0);
+	case REQ_BODY_CHUNKED:
 	case REQ_BODY_PRESENT:
 		break;
 	default:
@@ -653,7 +678,8 @@ HTTP1_CacheReqBody(struct req *req, ssize_t maxsize)
 		}
 
 		l = st->space - st->len;
-		l = http1_iter_req_body(req, st->ptr + st->len, l);
+		l = http1_iter_req_body(req, req->req_body_status,
+		    st->ptr + st->len, l);
 		if (l < 0) {
 			req->doclose = SC_RX_BODY;
 			break;
diff --git a/bin/varnishd/cache/cache_http1_proto.c b/bin/varnishd/cache/cache_http1_proto.c
index 0dc5fe8..a3e2005 100644
--- a/bin/varnishd/cache/cache_http1_proto.c
+++ b/bin/varnishd/cache/cache_http1_proto.c
@@ -535,7 +535,7 @@ HTTP1_Write(const struct worker *w, const struct http *hp, const int *hf)
 
 enum http1_chunked_ret
 HTTP1_Chunked(struct http_conn *htc, intptr_t *priv, const char **error,
-    int64_t *statp, void *ptr, ssize_t *lp)
+    uint64_t *statp, void *ptr, ssize_t *lp)
 {
 	int i;
 	char buf[20];		/* XXX: 20 is arbitrary */
diff --git a/bin/varnishd/cache/cache_pipe.c b/bin/varnishd/cache/cache_pipe.c
index 27ea964..36ef425 100644
--- a/bin/varnishd/cache/cache_pipe.c
+++ b/bin/varnishd/cache/cache_pipe.c
@@ -43,14 +43,14 @@
 static struct lock pipestat_mtx;
 
 struct acct_pipe {
-	ssize_t		req;
-	ssize_t		bereq;
-	ssize_t		in;
-	ssize_t		out;
+	uint64_t	req;
+	uint64_t	bereq;
+	uint64_t	in;
+	uint64_t	out;
 };
 
 static int
-rdf(int fd0, int fd1, ssize_t *pcnt)
+rdf(int fd0, int fd1, uint64_t *pcnt)
 {
 	int i, j;
 	char buf[BUFSIZ], *p;
diff --git a/bin/varnishd/cache/cache_wrw.c b/bin/varnishd/cache/cache_wrw.c
index 09c7e26..cc02bbf 100644
--- a/bin/varnishd/cache/cache_wrw.c
+++ b/bin/varnishd/cache/cache_wrw.c
@@ -102,7 +102,7 @@ WRW_Reserve(struct worker *wrk, int *fd, struct vsl_log *vsl, double t0)
 }
 
 static void
-wrw_release(struct worker *wrk, ssize_t *pacc)
+wrw_release(struct worker *wrk, uint64_t *pacc)
 {
 	struct wrw *wrw;
 
@@ -219,7 +219,7 @@ WRW_Flush(const struct worker *wrk)
 }
 
 unsigned
-WRW_FlushRelease(struct worker *wrk, ssize_t *pacc)
+WRW_FlushRelease(struct worker *wrk, uint64_t *pacc)
 {
 	unsigned u;
 
diff --git a/bin/varnishtest/tests/c00067.vtc b/bin/varnishtest/tests/c00067.vtc
new file mode 100644
index 0000000..b57f904
--- /dev/null
+++ b/bin/varnishtest/tests/c00067.vtc
@@ -0,0 +1,24 @@
+varnishtest "chunked req.body"
+
+server s1 {
+	rxreq
+	expect req.bodylen == 106
+	txresp -body "ABCD"
+} -start
+
+varnish v1 -vcl+backend {
+} -start
+
+client c1 {
+	txreq -req POST -nolen -hdr "Transfer-encoding: chunked"
+	chunked {BLA}
+	delay .2
+	chunkedlen 100
+	delay .2
+	chunked {FOO}
+	delay .2
+	chunkedlen 0
+	rxresp
+	expect resp.status == 200
+	expect resp.bodylen == 4
+} -run
diff --git a/bin/varnishtest/tests/r01524.vtc b/bin/varnishtest/tests/r01524.vtc
new file mode 100644
index 0000000..a845da3
--- /dev/null
+++ b/bin/varnishtest/tests/r01524.vtc
@@ -0,0 +1,19 @@
+varnishtest "pipe of chunked request"
+
+server s1 {
+	rxreq
+	expect req.bodylen == 3
+	txresp -body "ABCD"
+} -start
+
+varnish v1 -vcl+backend {
+} -start
+
+client c1 {
+	txreq -req MAGIC -nolen -hdr "Transfer-encoding: chunked"
+	chunked {BLA}
+	chunkedlen 0
+	rxresp
+	expect resp.status == 200
+	expect resp.bodylen == 4
+} -run



From phk at FreeBSD.org  Mon Jun 23 10:16:28 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Mon, 23 Jun 2014 12:16:28 +0200
Subject: [master] 0e9bd49 Opps, forgot to commit from top of tree
Message-ID: <E1Wz1Ia-0007Fk-UY@project.varnish-software.com>


commit 0e9bd4998cd8f8abda85e45d697af16d8dbbe125
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Jun 23 10:16:14 2014 +0000

    Opps, forgot to commit from top of tree

diff --git a/include/tbl/req_body.h b/include/tbl/req_body.h
index a649bf2..83744cc 100644
--- a/include/tbl/req_body.h
+++ b/include/tbl/req_body.h
@@ -31,6 +31,7 @@
 
 REQ_BODY(INIT)
 REQ_BODY(PRESENT)
+REQ_BODY(CHUNKED)
 REQ_BODY(TAKEN)
 REQ_BODY(CACHED)
 REQ_BODY(DONE)



From lkarsten at varnish-software.com  Mon Jun 23 10:25:43 2014
From: lkarsten at varnish-software.com (Lasse Karstensen)
Date: Mon, 23 Jun 2014 12:25:43 +0200
Subject: [master] df2446b Add libedit dependency.
Message-ID: <E1Wz1RX-0000VR-PI@project.varnish-software.com>


commit df2446b0f6b7be06f775244746ae16ad6b2e50dd
Author: Lasse Karstensen <lkarsten at varnish-software.com>
Date:   Mon Jun 23 12:24:19 2014 +0200

    Add libedit dependency.
    
    Fixes: 	#1509

diff --git a/redhat/varnish.spec b/redhat/varnish.spec
index b50c859..338aa96 100644
--- a/redhat/varnish.spec
+++ b/redhat/varnish.spec
@@ -23,6 +23,7 @@ Requires: varnish-libs = %{version}-%{release}
 Requires: logrotate
 Requires: ncurses
 Requires: pcre
+Requires: libedit
 Requires: jemalloc
 Requires(pre): shadow-utils
 Requires(post): /sbin/chkconfig, /usr/bin/uuidgen



From phk at FreeBSD.org  Mon Jun 23 12:30:24 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Mon, 23 Jun 2014 14:30:24 +0200
Subject: [master] ac94f16 Now really make chunked req.body work with pass.
Message-ID: <E1Wz3OC-0008UB-Rw@project.varnish-software.com>


commit ac94f1662761f9d9b7e3f527a08476a5cddb7dfc
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Jun 23 12:29:33 2014 +0000

    Now really make chunked req.body work with pass.
    
    Please test this ASAP (before 4.0.1)

diff --git a/bin/varnishd/builtin.vcl b/bin/varnishd/builtin.vcl
index 30fcaea..cfb2238 100644
--- a/bin/varnishd/builtin.vcl
+++ b/bin/varnishd/builtin.vcl
@@ -60,12 +60,6 @@ sub vcl_recv {
         return (pipe);
     }
 
-    /* We don't support chunked uploads, except when piping. */
-    if ((req.method == "POST" || req.method == "PUT") &&
-      req.http.transfer-encoding ~ "chunked") {
-        return(pipe);
-    }
-
     if (req.method != "GET" && req.method != "HEAD") {
         /* We only deal with GET and HEAD by default */
         return (pass);
diff --git a/bin/varnishd/cache/cache_http1_fetch.c b/bin/varnishd/cache/cache_http1_fetch.c
index aa009f2..b6d72dd 100644
--- a/bin/varnishd/cache/cache_http1_fetch.c
+++ b/bin/varnishd/cache/cache_http1_fetch.c
@@ -192,6 +192,31 @@ V1F_Setup_Fetch(struct busyobj *bo)
 }
 
 /*--------------------------------------------------------------------
+ * Pass the request body to the backend with chunks
+ */
+
+static int __match_proto__(req_body_iter_f)
+vbf_iter_req_body_chunked(struct req *req, void *priv, void *ptr, size_t l)
+{
+	struct worker *wrk;
+	char buf[20];
+
+	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
+	CAST_OBJ_NOTNULL(wrk, priv, WORKER_MAGIC);
+
+	if (l > 0) {
+		bprintf(buf, "%jx\r\n", (uintmax_t)l);
+		VSLb(req->vsl, SLT_Debug, "WWWW: %s", buf);
+		(void)WRW_Write(wrk, buf, strlen(buf));
+		(void)WRW_Write(wrk, ptr, l);
+		(void)WRW_Write(wrk, "\r\n", 2);
+		if (WRW_Flush(wrk))
+			return (-1);
+	}
+	return (0);
+}
+
+/*--------------------------------------------------------------------
  * Pass the request body to the backend
  */
 
@@ -231,6 +256,7 @@ V1F_fetch_hdr(struct worker *wrk, struct busyobj *bo, struct req *req)
 	int i, j, first;
 	struct http_conn *htc;
 	ssize_t hdrbytes;
+	int do_chunked = 0;
 
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 	CHECK_OBJ_ORNULL(req, REQ_MAGIC);
@@ -262,6 +288,11 @@ V1F_fetch_hdr(struct worker *wrk, struct busyobj *bo, struct req *req)
 	if (!http_GetHdr(bo->bereq, H_Host, NULL))
 		VDI_AddHostHeader(bo->bereq, vc);
 
+	if (req != NULL && req->req_body_status == REQ_BODY_CHUNKED) {
+		http_PrintfHeader(hp, "Transfer-Encoding: chunked");
+		do_chunked = 1;
+	}
+
 	(void)VTCP_blocking(vc->fd);	/* XXX: we should timeout instead */
 	WRW_Reserve(wrk, &vc->fd, bo->vsl, bo->t_prev);
 	hdrbytes = HTTP1_Write(wrk, hp, HTTP1_Req);
@@ -270,10 +301,16 @@ V1F_fetch_hdr(struct worker *wrk, struct busyobj *bo, struct req *req)
 	i = 0;
 
 	if (req != NULL) {
-		i = HTTP1_IterateReqBody(req, vbf_iter_req_body, wrk);
-		if (req->req_body_status == REQ_BODY_DONE)
+		if (do_chunked) {
+			i = HTTP1_IterateReqBody(req,
+			    vbf_iter_req_body_chunked, wrk);
+			(void)WRW_Write(wrk, "0\r\n\r\n", 5);
+		} else {
+			i = HTTP1_IterateReqBody(req, vbf_iter_req_body, wrk);
+		}
+		if (req->req_body_status == REQ_BODY_DONE) {
 			retry = -1;
-		if (req->req_body_status == REQ_BODY_FAIL) {
+		} else if (req->req_body_status == REQ_BODY_FAIL) {
 			VSLb(bo->vsl, SLT_FetchError,
 			    "req.body read error: %d (%s)",
 			    errno, strerror(errno));
diff --git a/bin/varnishd/cache/cache_http1_fsm.c b/bin/varnishd/cache/cache_http1_fsm.c
index dac34e0..454ec28 100644
--- a/bin/varnishd/cache/cache_http1_fsm.c
+++ b/bin/varnishd/cache/cache_http1_fsm.c
@@ -285,8 +285,10 @@ http1_req_body_status(struct req *req)
 		req->h1.bytes_yet = req->req_bodybytes - req->h1.bytes_done;
 		return (REQ_BODY_PRESENT);
 	}
-	if (http_HdrIs(req->http, H_Transfer_Encoding, "chunked"))
+	if (http_HdrIs(req->http, H_Transfer_Encoding, "chunked")) {
+		req->chunk_ctr = -1;
 		return (REQ_BODY_CHUNKED);
+	}
 	if (http_GetHdr(req->http, H_Transfer_Encoding, NULL))
 		return (REQ_BODY_FAIL);
 	return (REQ_BODY_NONE);
@@ -512,6 +514,7 @@ http1_iter_req_body(struct req *req, enum req_body_state_e bs,
 		switch (HTTP1_Chunked(req->htc, &req->chunk_ctr, &err,
 		    &req->acct.req_bodybytes, buf, &len)) {
 		case H1CR_ERROR:
+			VSLb(req->vsl, SLT_Debug, "CHUNKERR: %s", err);
 			return (-1);
 		case H1CR_MORE:
 			return (len);
@@ -639,7 +642,7 @@ int
 HTTP1_CacheReqBody(struct req *req, ssize_t maxsize)
 {
 	struct storage *st;
-	ssize_t l;
+	ssize_t l, l2;
 
 	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
 
@@ -661,6 +664,7 @@ HTTP1_CacheReqBody(struct req *req, ssize_t maxsize)
 		req->req_body_status = REQ_BODY_FAIL;
 		return (-1);
 	}
+	l2 = 0;
 
 	st = NULL;
 	do {
@@ -690,13 +694,28 @@ HTTP1_CacheReqBody(struct req *req, ssize_t maxsize)
 			break;
 		}
 		if (l > 0) {
+			l2 += l;
 			st->len += l;
 			if (st->space == st->len)
 				st = NULL;
 		}
 	} while (l > 0);
-	if (l == 0)
+	if (l == 0) {
+		req->req_bodybytes = l2;
+		/* We must update also the "pristine" req.* copy */
+
+		http_Unset(req->http0, H_Content_Length);
+		http_Unset(req->http0, H_Transfer_Encoding);
+		http_PrintfHeader(req->http0, "Content-Length: %ju",
+		    req->req_bodybytes);
+
+		http_Unset(req->http, H_Content_Length);
+		http_Unset(req->http, H_Transfer_Encoding);
+		http_PrintfHeader(req->http, "Content-Length: %ju",
+		    req->req_bodybytes);
+
 		req->req_body_status = REQ_BODY_CACHED;
+	}
 	VSLb_ts_req(req, "ReqBody", VTIM_real());
 	return (l);
 }
diff --git a/bin/varnishd/cache/cache_http1_proto.c b/bin/varnishd/cache/cache_http1_proto.c
index a3e2005..f0a1abc 100644
--- a/bin/varnishd/cache/cache_http1_proto.c
+++ b/bin/varnishd/cache/cache_http1_proto.c
@@ -196,7 +196,7 @@ HTTP1_Read(struct http_conn *htc, void *d, size_t len)
 {
 	size_t l;
 	unsigned char *p;
-	ssize_t i;
+	ssize_t i = 0;
 
 	CHECK_OBJ_NOTNULL(htc, HTTP_CONN_MAGIC);
 	l = 0;
@@ -212,12 +212,12 @@ HTTP1_Read(struct http_conn *htc, void *d, size_t len)
 		if (htc->pipeline.b == htc->pipeline.e)
 			htc->pipeline.b = htc->pipeline.e = NULL;
 	}
-	if (len == 0)
-		return (l);
-	i = read(htc->fd, p, len);
-	if (i < 0) {
-		VSLb(htc->vsl, SLT_FetchError, "%s", strerror(errno));
-		return (i);
+	if (len > 0) {
+		i = read(htc->fd, p, len);
+		if (i < 0) {
+			VSLb(htc->vsl, SLT_FetchError, "%s", strerror(errno));
+			return (i);
+		}
 	}
 	return (i + l);
 }
diff --git a/bin/varnishtest/tests/c00055.vtc b/bin/varnishtest/tests/c00055.vtc
index e02bc47..0344d02 100644
--- a/bin/varnishtest/tests/c00055.vtc
+++ b/bin/varnishtest/tests/c00055.vtc
@@ -25,7 +25,7 @@ varnish v1 -cliok "param.set vcc_allow_inline_c true" -vcl+backend {
 varnish v1 -cliok "param.set debug +syncvsl"
 
 client c1 {
-	txreq -body "FOO"
+	txreq -req "POST" -body "FOO"
 	rxresp
 	expect resp.http.Foo == "Foo"
 	expect resp.bodylen == 2
diff --git a/bin/varnishtest/tests/c00067.vtc b/bin/varnishtest/tests/c00067.vtc
index b57f904..73bcb34 100644
--- a/bin/varnishtest/tests/c00067.vtc
+++ b/bin/varnishtest/tests/c00067.vtc
@@ -4,11 +4,16 @@ server s1 {
 	rxreq
 	expect req.bodylen == 106
 	txresp -body "ABCD"
+	rxreq
+	expect req.bodylen == 108
+	txresp -body "ABCDE"
 } -start
 
 varnish v1 -vcl+backend {
 } -start
 
+varnish v1 -cliok "param.set debug +syncvsl"
+
 client c1 {
 	txreq -req POST -nolen -hdr "Transfer-encoding: chunked"
 	chunked {BLA}
@@ -22,3 +27,25 @@ client c1 {
 	expect resp.status == 200
 	expect resp.bodylen == 4
 } -run
+
+delay .2
+
+varnish v1 -cliok "param.set vcc_allow_inline_c true" -vcl+backend {
+	sub vcl_recv {
+		C{ VRT_CacheReqBody(ctx, 1000); }C
+	}
+} 
+
+client c1 {
+	txreq -req POST -nolen -hdr "Transfer-encoding: chunked"
+	chunked {BLAS}
+	delay .2
+	chunkedlen 100
+	delay .2
+	chunked {TFOO}
+	delay .2
+	chunkedlen 0
+	rxresp
+	expect resp.status == 200
+	expect resp.bodylen == 5
+} -run
diff --git a/bin/varnishtest/vtc_http.c b/bin/varnishtest/vtc_http.c
index 966f043..28b7340 100644
--- a/bin/varnishtest/vtc_http.c
+++ b/bin/varnishtest/vtc_http.c
@@ -458,7 +458,7 @@ http_swallow_body(struct http *hp, char * const *hh, int body)
 		return;
 	}
 	p = http_find_header(hh, "transfer-encoding");
-	if (p != NULL && !strcmp(p, "chunked")) {
+	if (p != NULL && !strcasecmp(p, "chunked")) {
 		while (http_rxchunk(hp) != 0)
 			continue;
 		vtc_dump(hp->vl, 4, "body", hp->body, ll);



From martin at varnish-software.com  Mon Jun 23 13:10:51 2014
From: martin at varnish-software.com (Martin Blix Grydeland)
Date: Mon, 23 Jun 2014 15:10:51 +0200
Subject: [master] 542e5c7 Treat missing t_end in varnishncsa as being the same
 as t_start
Message-ID: <E1Wz41L-0004oB-0p@project.varnish-software.com>


commit 542e5c70fa394a8c6ddf13d3be66536bfd92f957
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Mon Jun 23 15:09:46 2014 +0200

    Treat missing t_end in varnishncsa as being the same as t_start
    
    This makes varnishncsa continue to log if the end timestamp should be missing for some reason.

diff --git a/bin/varnishncsa/varnishncsa.c b/bin/varnishncsa/varnishncsa.c
index 2ff1fc3..d26808e 100644
--- a/bin/varnishncsa/varnishncsa.c
+++ b/bin/varnishncsa/varnishncsa.c
@@ -50,6 +50,7 @@
 #include <stdint.h>
 #include <ctype.h>
 #include <time.h>
+#include <math.h>
 
 #include "base64.h"
 #include "vapi/vsm.h"
@@ -243,20 +244,27 @@ format_time(const struct format *format)
 	struct tm tm;
 
 	CHECK_OBJ_NOTNULL(format, FORMAT_MAGIC);
-	if (CTX.frag[F_tstart].gen != CTX.gen ||
-	    CTX.frag[F_tend].gen != CTX.gen) {
+	if (CTX.frag[F_tstart].gen == CTX.gen) {
+		t_start = strtod(CTX.frag[F_tstart].b, &p);
+		if (p != CTX.frag[F_tstart].e)
+			t_start = NAN;
+	} else
+		t_start = NAN;
+	if (isnan(t_start)) {
+		/* Missing t_start is a no go */
 		if (format->string == NULL)
 			return (-1);
 		AZ(VSB_cat(CTX.vsb, format->string));
 		return (0);
 	}
 
-	t_start = strtod(CTX.frag[F_tstart].b, &p);
-	if (p != CTX.frag[F_tstart].e)
-		return (-1);
-	t_end = strtod(CTX.frag[F_tend].b, &p);
-	if (p != CTX.frag[F_tend].e)
-		return (-1);
+	/* Missing t_end defaults to t_start */
+	if (CTX.frag[F_tend].gen == CTX.gen) {
+		t_end = strtod(CTX.frag[F_tend].b, &p);
+		if (p != CTX.frag[F_tend].e)
+			t_end = t_start;
+	} else
+		t_end = t_start;
 
 	switch (format->time_type) {
 	case 'D':



From apj at mutt.dk  Mon Jun 23 13:32:52 2014
From: apj at mutt.dk (Andreas Plesner)
Date: Mon, 23 Jun 2014 15:32:52 +0200
Subject: [master] 0b1fa10 Add varnishncsa logging of pipe requests
Message-ID: <E1Wz4Me-0000WR-JH@project.varnish-software.com>


commit 0b1fa10ab261b6c0301d5b20fd13193a739bd978
Author: Andreas Plesner <apj at mutt.dk>
Date:   Mon Jun 23 15:32:11 2014 +0200

    Add varnishncsa logging of pipe requests
    
    Fixes #1269

diff --git a/bin/varnishncsa/varnishncsa.c b/bin/varnishncsa/varnishncsa.c
index d26808e..97cc5ea 100644
--- a/bin/varnishncsa/varnishncsa.c
+++ b/bin/varnishncsa/varnishncsa.c
@@ -558,7 +558,7 @@ parse_format(const char *format)
 			addf_requestline();
 			break;
 		case 's':	/* Status code */
-			addf_fragment(&CTX.frag[F_s], "");
+			addf_fragment(&CTX.frag[F_s], "-");
 			break;
 		case 't':	/* strftime */
 			addf_time(*p, TIME_FMT, NULL);
@@ -791,11 +791,13 @@ dispatch_f(struct VSL_data *vsl, struct VSL_transaction * const pt[],
 					frag_fields(p, e, 1,
 					    &CTX.frag[F_tstart], 0, NULL);
 
-				} else if (isprefix(b, "Resp:", e, &p)) {
+				} else if (isprefix(b, "Resp:", e, &p) ||
+					isprefix(b, "PipeSess:", e, &p)) {
 					frag_fields(p, e, 1,
 					    &CTX.frag[F_tend], 0, NULL);
 
-				} else if (isprefix(b, "Process:", e, &p)) {
+				} else if (isprefix(b, "Process:", e, &p) ||
+					isprefix(b, "Pipe:", e, &p)) {
 					frag_fields(p, e, 2,
 					    &CTX.frag[F_ttfb], 0, NULL);
 				}
@@ -826,14 +828,15 @@ dispatch_f(struct VSL_data *vsl, struct VSL_transaction * const pt[],
 					   wrong */
 					CTX.hitmiss = "miss";
 					CTX.handling = "error";
-				} else if (!strcasecmp(b, "pipe")) {
-					CTX.hitmiss = "miss";
-					CTX.handling = "pipe";
 				}
 				break;
 			case SLT_VCL_return:
-				if (!strcasecmp(b, "restart"))
+				if (!strcasecmp(b, "restart")) {
 					skip = 1;
+				} else if (!strcasecmp(b, "pipe")) {
+					CTX.hitmiss = "miss";
+					CTX.handling = "pipe";
+				}
 				break;
 			default:
 				break;



From lkarsten at varnish-software.com  Mon Jun 23 13:53:01 2014
From: lkarsten at varnish-software.com (Lasse Karstensen)
Date: Mon, 23 Jun 2014 15:53:01 +0200
Subject: [master] 8ae27a9 Add Begin and Link VSL records for piped requests.
Message-ID: <E1Wz4g9-0002j2-MZ@project.varnish-software.com>


commit 8ae27a9c73bbbd7ef360a5c241cf98d69dbd89a5
Author: Lasse Karstensen <lkarsten at varnish-software.com>
Date:   Mon Jun 23 15:43:28 2014 +0200

    Add Begin and Link VSL records for piped requests.
    
    This allows varnishlog and the other tools to group it automatically
    with the client request.

diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c
index c586ca4..401ebf1 100644
--- a/bin/varnishd/cache/cache_req_fsm.c
+++ b/bin/varnishd/cache/cache_req_fsm.c
@@ -593,6 +593,7 @@ cnt_pipe(struct worker *wrk, struct req *req)
 	wrk->stats.s_pipe++;
 	bo = VBO_GetBusyObj(wrk, req);
 	HTTP_Setup(bo->bereq, bo->ws, bo->vsl, SLT_BereqMethod);
+	VSLb(bo->vsl, SLT_Begin, "bereq %u pipe", req->vsl->wid & VSL_IDENTMASK);
 	http_FilterReq(bo->bereq, req->http, 0);	// XXX: 0 ?
 	http_PrintfHeader(bo->bereq,
 	    "X-Varnish: %u", req->vsl->wid & VSL_IDENTMASK);
@@ -604,6 +605,7 @@ cnt_pipe(struct worker *wrk, struct req *req)
 		INCOMPL();
 	assert(wrk->handling == VCL_RET_PIPE);
 
+	VSLb(req->vsl, SLT_Link, "bereq %u pipe", bo->vsl->wid & VSL_IDENTMASK);
 	PipeRequest(req, bo);
 	assert(WRW_IsReleased(wrk));
 	http_Teardown(bo->bereq);
diff --git a/include/vapi/vsl.h b/include/vapi/vsl.h
index a6e0839..45c1aaf 100644
--- a/include/vapi/vsl.h
+++ b/include/vapi/vsl.h
@@ -103,6 +103,7 @@ enum VSL_reason_e {
 	VSL_r_pass,
 	VSL_r_fetch,
 	VSL_r_bgfetch,
+	VSL_r_pipe,
 	VSL_r__MAX,
 };
 
diff --git a/lib/libvarnishapi/vsl_dispatch.c b/lib/libvarnishapi/vsl_dispatch.c
index 0502c36..7f393b3 100644
--- a/lib/libvarnishapi/vsl_dispatch.c
+++ b/lib/libvarnishapi/vsl_dispatch.c
@@ -65,6 +65,7 @@ static const char * const vsl_r_names[VSL_r__MAX] = {
 	[VSL_r_pass]	= "pass",
 	[VSL_r_fetch]	= "fetch",
 	[VSL_r_bgfetch]	= "bgfetch",
+	[VSL_r_pipe]	= "pipe",
 };
 
 struct vtx;



From martin at varnish-software.com  Mon Jun 23 14:54:37 2014
From: martin at varnish-software.com (Martin Blix Grydeland)
Date: Mon, 23 Jun 2014 16:54:37 +0200
Subject: [master] 44b2c1d Parse byte counters for piped requests in varnishncsa
Message-ID: <E1Wz5dl-0002Ow-Bi@project.varnish-software.com>


commit 44b2c1d0d210faa29158e0ef774676724e47ac4b
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Mon Jun 23 16:54:00 2014 +0200

    Parse byte counters for piped requests in varnishncsa

diff --git a/bin/varnishncsa/varnishncsa.c b/bin/varnishncsa/varnishncsa.c
index 97cc5ea..685d869 100644
--- a/bin/varnishncsa/varnishncsa.c
+++ b/bin/varnishncsa/varnishncsa.c
@@ -74,7 +74,7 @@ enum e_frag {
 	F_H,			/* %H Proto */
 	F_U,			/* %U URL path */
 	F_q,			/* %q Query string */
-	F_b,			/* %b Bytes */
+	F_b,			/* %b Body bytes sent */
 	F_h,			/* %h Host name / IP Address */
 	F_m,			/* %m Method */
 	F_s,			/* %s Status */
@@ -527,7 +527,7 @@ parse_format(const char *format)
 
 		p++;
 		switch (*p) {
-		case 'b':	/* Bytes */
+		case 'b':	/* Body bytes sent */
 			addf_fragment(&CTX.frag[F_b], "-");
 			break;
 		case 'D':	/* Float request time */
@@ -760,6 +760,12 @@ dispatch_f(struct VSL_data *vsl, struct VSL_transaction * const pt[],
 				e--;
 
 			switch (tag) {
+			case SLT_PipeAcct:
+				frag_fields(b, e,
+				    3, &CTX.frag[F_I],
+				    4, &CTX.frag[F_O],
+				    0, NULL);
+				break;
 			case SLT_ReqStart:
 				frag_fields(b, e, 1, &CTX.frag[F_h], 0, NULL);
 				break;



From fgsch at lodoss.net  Mon Jun 23 23:35:28 2014
From: fgsch at lodoss.net (Federico G. Schwindt)
Date: Tue, 24 Jun 2014 01:35:28 +0200
Subject: [master] 9e5b06f Move querysort into its own file
Message-ID: <E1WzDlo-0001Mk-CW@project.varnish-software.com>


commit 9e5b06f94dab9c52089587724ae42d26d5fb6a30
Author: Federico G. Schwindt <fgsch at lodoss.net>
Date:   Mon Jun 23 23:33:19 2014 +0100

    Move querysort into its own file

diff --git a/lib/libvmod_std/Makefile.am b/lib/libvmod_std/Makefile.am
index c005b85..47dd5dc 100644
--- a/lib/libvmod_std/Makefile.am
+++ b/lib/libvmod_std/Makefile.am
@@ -17,20 +17,20 @@ libvmod_std_la_LDFLAGS = $(AM_LDFLAGS) -module -export-dynamic -avoid-version -s
 
 libvmod_std_la_SOURCES = \
 	vmod_std.c \
+	vmod_std_conversions.c \
 	vmod_std_fileread.c \
-	vmod_std_conversions.c
+	vmod_std_querysort.c
 
 nodist_libvmod_std_la_SOURCES = \
 	vcc_if.c \
 	vcc_if.h
 
 # BUILT_SOURCES is only a hack and dependency tracking does not help for the first build
-vmod_std.lo vmod_std_fileread.lo vmod_std_conversions.lo: vcc_if.h
+$(libvmod_std_la_OBJECTS): vcc_if.h
 
 vcc_if.c vcc_if.h vmod_std.rst vmod_std.man.rst: $(vmodtool) $(vmod_srcdir)/vmod.vcc
 	@PYTHON@ $(vmodtool) $(vmodtoolargs) $(vmod_srcdir)/vmod.vcc
 
-
 EXTRA_DIST = vmod.vcc
 
 CLEANFILES = $(builddir)/vcc_if.c $(builddir)/vcc_if.h \
diff --git a/lib/libvmod_std/vmod_std.c b/lib/libvmod_std/vmod_std.c
index 98ed54d..233c2da 100644
--- a/lib/libvmod_std/vmod_std.c
+++ b/lib/libvmod_std/vmod_std.c
@@ -225,98 +225,3 @@ vmod_timestamp(const struct vrt_ctx *ctx, VCL_STRING label)
 		VSLb_ts_req(ctx->req, label, VTIM_real());
 	}
 }
-
-/*
- * Boltsort
- * Author: Naren Venkataraman of Vimeo Inc.
- * Included here with permission.
- */
-
-#define QS_MAX_PARAM_COUNT	32
-#define QS_EQUALS(a, b)	\
-    ((a) == (b) || ((a) == '\0' && (b) == '&') || ((a) == '&' && (b) == '\0'))
-
-static ssize_t
-param_compare(const char *s, const char *t)
-{
-	for (; QS_EQUALS(*s, *t); s++, t++) {
-		if (*s == '&' || *s == '\0')
-			return (0);
-	}
-	return (*s - *t);
-}
-
-static size_t
-param_copy(char *dst, const char *src)
-{
-	size_t len;
-	len = strcspn(src, "&");
-	memcpy(dst, src, len);
-	return (len);
-}
-
-VCL_STRING __match_proto__(td_std_querysort)
-vmod_querysort(const struct vrt_ctx *ctx, VCL_STRING url)
-{
-	char *param, *params[QS_MAX_PARAM_COUNT];
-	char *p, *r;
-	size_t len;
-	int param_count;
-	int i, n;
-
-	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
-
-	if (url == NULL)
-		return (NULL);
-
-	p = strchr(url, '?');
-	if (p == NULL)
-		return (url);
-
-	param_count = 0;
-	params[param_count++] = ++p;
-	len = p - url;
-
-	while ((p = strchr(p, '&')) != NULL) {
-		param = ++p;
-
-		for (i = 0; i < param_count; i++) {
-			if (param[0] < params[i][0] ||
-			    param_compare(param, params[i]) < 0) {
-				for (n = param_count; n > i; n--)
-					params[n] = params[n - 1];
-				break;
-			}
-		}
-		params[i] = param;
-		param_count++;
-
-		if (param_count == QS_MAX_PARAM_COUNT)
-			return (url);
-	}
-
-	if (param_count == 1)
-		return (url);
-
-	r = WS_Alloc(ctx->ws, strchr(param, '\0') - url + 1);
-	if (r == NULL)
-		return (url);
-
-	p = memcpy(r, url, len);
-	p += len;
-
-	for (i = 0; i < param_count - 1; i++) {
-		if (params[i][0] != '\0' && params[i][0] != '&')
-			break;
-	}
-
-	for (; i < param_count - 1; i++) {
-		p += param_copy(p, params[i]);
-		*p++ = '&';
-	}
-
-	p += param_copy(p, params[i]);
-	*p = '\0';
-
-	return (r);
-}
diff --git a/lib/libvmod_std/vmod_std_querysort.c b/lib/libvmod_std/vmod_std_querysort.c
new file mode 100644
index 0000000..8aac9df
--- /dev/null
+++ b/lib/libvmod_std/vmod_std_querysort.c
@@ -0,0 +1,124 @@
+/*-
+ * Copyright (c) 2010-2014 Varnish Software AS
+ * All rights reserved.
+ *
+ * Author: Naren Venkataraman of Vimeo Inc.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include "config.h"
+
+#include "vrt.h"
+
+#include "cache/cache.h"
+
+#include "vcc_if.h"
+
+#define QS_MAX_PARAM_COUNT	32
+#define QS_EQUALS(a, b)	\
+    ((a) == (b) || ((a) == '\0' && (b) == '&') || ((a) == '&' && (b) == '\0'))
+
+static ssize_t
+param_compare(const char *s, const char *t)
+{
+	for (; QS_EQUALS(*s, *t); s++, t++) {
+		if (*s == '&' || *s == '\0')
+			return (0);
+	}
+	return (*s - *t);
+}
+
+static size_t
+param_copy(char *dst, const char *src)
+{
+	size_t len;
+	len = strcspn(src, "&");
+	memcpy(dst, src, len);
+	return (len);
+}
+
+VCL_STRING __match_proto__(td_std_querysort)
+vmod_querysort(const struct vrt_ctx *ctx, VCL_STRING url)
+{
+	char *param, *params[QS_MAX_PARAM_COUNT];
+	char *p, *r;
+	size_t len;
+	int param_count;
+	int i, n;
+
+	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
+
+	if (url == NULL)
+		return (NULL);
+
+	p = strchr(url, '?');
+	if (p == NULL)
+		return (url);
+
+	param_count = 0;
+	params[param_count++] = ++p;
+	len = p - url;
+
+	while ((p = strchr(p, '&')) != NULL) {
+		param = ++p;
+
+		for (i = 0; i < param_count; i++) {
+			if (param[0] < params[i][0] ||
+			    param_compare(param, params[i]) < 0) {
+				for (n = param_count; n > i; n--)
+					params[n] = params[n - 1];
+				break;
+			}
+		}
+		params[i] = param;
+		param_count++;
+
+		if (param_count == QS_MAX_PARAM_COUNT)
+			return (url);
+	}
+
+	if (param_count == 1)
+		return (url);
+
+	r = WS_Alloc(ctx->ws, strchr(param, '\0') - url + 1);
+	if (r == NULL)
+		return (url);
+
+	p = memcpy(r, url, len);
+	p += len;
+
+	for (i = 0; i < param_count - 1; i++) {
+		if (params[i][0] != '\0' && params[i][0] != '&')
+			break;
+	}
+
+	for (; i < param_count - 1; i++) {
+		p += param_copy(p, params[i]);
+		*p++ = '&';
+	}
+
+	p += param_copy(p, params[i]);
+	*p = '\0';
+
+	return (r);
+}



From fgsch at lodoss.net  Mon Jun 23 23:35:28 2014
From: fgsch at lodoss.net (Federico G. Schwindt)
Date: Tue, 24 Jun 2014 01:35:28 +0200
Subject: [master] 622d1b5 Remove superfluous braces
Message-ID: <E1WzDlo-0001Mr-Dm@project.varnish-software.com>


commit 622d1b56a81e1dd9992e3417e23aa039135a9664
Author: Federico G. Schwindt <fgsch at lodoss.net>
Date:   Mon Jun 23 23:44:02 2014 +0100

    Remove superfluous braces

diff --git a/lib/libvmod_std/vmod_std.c b/lib/libvmod_std/vmod_std.c
index 233c2da..718fb71 100644
--- a/lib/libvmod_std/vmod_std.c
+++ b/lib/libvmod_std/vmod_std.c
@@ -178,13 +178,12 @@ vmod_collect(const struct vrt_ctx *ctx, VCL_HEADER hdr)
 	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
 	if (hdr->where == HDR_REQ)
 		http_CollectHdr(ctx->http_req, hdr->what);
-	else if (hdr->where == HDR_BEREQ) {
+	else if (hdr->where == HDR_BEREQ)
 		http_CollectHdr(ctx->http_bereq, hdr->what);
-	} else if (hdr->where == HDR_BERESP) {
+	else if (hdr->where == HDR_BERESP)
 		http_CollectHdr(ctx->http_beresp, hdr->what);
-	} else if (hdr->where == HDR_RESP) {
+	else if (hdr->where == HDR_RESP)
 		http_CollectHdr(ctx->http_resp, hdr->what);
-	}
 }
 
 VCL_BOOL __match_proto__(td_std_healthy)



From fgsch at lodoss.net  Mon Jun 23 23:35:28 2014
From: fgsch at lodoss.net (Federico G. Schwindt)
Date: Tue, 24 Jun 2014 01:35:28 +0200
Subject: [master] 133e2b2 Use VCL_xxx types and add missing matching prototypes
Message-ID: <E1WzDlo-0001N0-FV@project.varnish-software.com>


commit 133e2b222c0bfbfb98b9b0d9c26937164d062701
Author: Federico G. Schwindt <fgsch at lodoss.net>
Date:   Mon Jun 23 23:44:21 2014 +0100

    Use VCL_xxx types and add missing matching prototypes
    
    While here unify check.

diff --git a/lib/libvmod_std/vmod_std_conversions.c b/lib/libvmod_std/vmod_std_conversions.c
index 7c1ac08..589d708 100644
--- a/lib/libvmod_std/vmod_std_conversions.c
+++ b/lib/libvmod_std/vmod_std_conversions.c
@@ -43,8 +43,8 @@
 #include "vsa.h"
 #include "vcc_if.h"
 
-VCL_DURATION __match_proto__()
-vmod_duration(const struct vrt_ctx *ctx, const char *p, VCL_DURATION d)
+VCL_DURATION __match_proto__(td_std_duration)
+vmod_duration(const struct vrt_ctx *ctx, VCL_STRING p, VCL_DURATION d)
 {
 	char *e;
 	double r;
@@ -100,8 +100,8 @@ vmod_duration(const struct vrt_ctx *ctx, const char *p, VCL_DURATION d)
 	return (r);
 }
 
-VCL_INT __match_proto__()
-vmod_integer(const struct vrt_ctx *ctx, const char *p, VCL_INT i)
+VCL_INT __match_proto__(td_std_integer)
+vmod_integer(const struct vrt_ctx *ctx, VCL_STRING p, VCL_INT i)
 {
 	char *e;
 	long r;
@@ -121,10 +121,7 @@ vmod_integer(const struct vrt_ctx *ctx, const char *p, VCL_INT i)
 
 	r = strtol(p, &e, 0);
 
-	if (e == NULL)
-		return (i);
-
-	if (*e != '\0')
+	if (e == NULL || *e != '\0')
 		return (i);
 
 	return (r);
@@ -169,7 +166,7 @@ vmod_ip(const struct vrt_ctx *ctx, VCL_STRING s, VCL_IP d)
 	return (r);
 }
 
-VCL_REAL __match_proto__()
+VCL_REAL __match_proto__(td_std_real)
 vmod_real(const struct vrt_ctx *ctx, VCL_STRING p, VCL_REAL d)
 {
 	char *e;



From fgsch at lodoss.net  Mon Jun 23 23:35:28 2014
From: fgsch at lodoss.net (Federico G. Schwindt)
Date: Tue, 24 Jun 2014 01:35:28 +0200
Subject: [master] 14da184 Add real2time,
 time2integer and time2real to the std vmod
Message-ID: <E1WzDlo-0001NB-Hl@project.varnish-software.com>


commit 14da1840453b4076ac8b46448c9092a491037d5a
Author: Federico G. Schwindt <fgsch at lodoss.net>
Date:   Tue Jun 24 00:13:55 2014 +0100

    Add real2time, time2integer and time2real to the std vmod
    
    As discussed on the latest VDD.

diff --git a/bin/varnishtest/tests/m00016.vtc b/bin/varnishtest/tests/m00016.vtc
new file mode 100644
index 0000000..ef51f93
--- /dev/null
+++ b/bin/varnishtest/tests/m00016.vtc
@@ -0,0 +1,23 @@
+varnishtest "Test std.real2time, std.time2integer and std.time2real"
+
+server s1 {
+	rxreq
+	txresp
+} -start
+
+varnish v1 -vcl+backend {
+	import ${vmod_std};
+
+	sub vcl_deliver {
+		set resp.http.x-foo = std.integer(req.http.foo, 0);
+		set resp.http.x-bar = std.time2integer(std.real2time(std.real(resp.http.x-foo, 0.0)));
+		set resp.http.x-baz = std.time2real(std.real2time(std.real(resp.http.x-foo, 0.0)));
+	}
+} -start
+
+client c1 {
+	txreq -hdr "foo: 1140618699"
+	rxresp
+	expect resp.http.x-foo == resp.http.x-bar
+	expect resp.http.x-baz == 1140618699.000
+} -run
diff --git a/lib/libvmod_std/vmod.vcc b/lib/libvmod_std/vmod.vcc
index cc3eb43..fbee007 100644
--- a/lib/libvmod_std/vmod.vcc
+++ b/lib/libvmod_std/vmod.vcc
@@ -143,6 +143,27 @@ Description
 Example
 	set req.http.x-real = std.real(req.http.x-foo, 0.0);
 
+$Function TIME real2time(REAL)
+
+Description
+	Converts the real *r* to a time.
+Example
+	set req.http.x-time = std.real2time(1140618699.00);
+
+$Function INT time2integer(TIME)
+
+Description
+	Converts the time *t* to a integer.
+Example
+	set req.http.x-int = std.time2integer(now);
+
+$Function REAL time2real(TIME)
+
+Description
+	Converts the time *t* to a real.
+Example
+	set req.http.x-real = std.time2real(now);
+
 $Function BOOL healthy(BACKEND)
 
 Description
diff --git a/lib/libvmod_std/vmod_std_conversions.c b/lib/libvmod_std/vmod_std_conversions.c
index 589d708..8ca0dd7 100644
--- a/lib/libvmod_std/vmod_std_conversions.c
+++ b/lib/libvmod_std/vmod_std_conversions.c
@@ -195,3 +195,27 @@ vmod_real(const struct vrt_ctx *ctx, VCL_STRING p, VCL_REAL d)
 
 	return (r);
 }
+
+VCL_TIME __match_proto__(td_std_real2time)
+vmod_real2time(const struct vrt_ctx *ctx, VCL_REAL r)
+{
+	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
+
+	return (r);
+}
+
+VCL_INT __match_proto__(td_std_time2integer)
+vmod_time2integer(const struct vrt_ctx *ctx, VCL_TIME t)
+{
+	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
+
+	return (t);
+}
+
+VCL_REAL __match_proto__(td_std_time2real)
+vmod_time2real(const struct vrt_ctx *ctx, VCL_TIME t)
+{
+	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
+
+	return (t);
+}



From lkarsten at varnish-software.com  Tue Jun 24 09:22:22 2014
From: lkarsten at varnish-software.com (Lasse Karstensen)
Date: Tue, 24 Jun 2014 11:22:22 +0200
Subject: [master] 4354e5e Prepare for the 4.0.1 release.
Message-ID: <E1WzMvm-00085K-ES@project.varnish-software.com>


commit 4354e5e04a6af43394bbbdb32dc2654ca4887de8
Author: Lasse Karstensen <lkarsten at varnish-software.com>
Date:   Tue Jun 24 11:16:20 2014 +0200

    Prepare for the 4.0.1 release.

diff --git a/configure.ac b/configure.ac
index eed48bc..118f743 100644
--- a/configure.ac
+++ b/configure.ac
@@ -2,7 +2,7 @@ AC_PREREQ(2.59)
 AC_COPYRIGHT([Copyright (c) 2006 Verdens Gang AS
 Copyright (c) 2006-2014 Varnish Software AS])
 AC_REVISION([$Id$])
-AC_INIT([Varnish], [4.0.1-rc1], [varnish-dev at varnish-cache.org])
+AC_INIT([Varnish], [4.0.1], [varnish-dev at varnish-cache.org])
 AC_CONFIG_SRCDIR(include/miniobj.h)
 AC_CONFIG_HEADERS([config.h])
 AC_CONFIG_MACRO_DIR([m4])
diff --git a/doc/changes.rst b/doc/changes.rst
index c5e1425..d66e7a0 100644
--- a/doc/changes.rst
+++ b/doc/changes.rst
@@ -1,10 +1,13 @@
 
-============================================
-Changes from 4.0.0 to 4.0.1-rc1 (2014-06-20)
-============================================
+========================================
+Changes from 4.0.0 to 4.0.1 (2014-06-24)
+========================================
 
 New since 4.0.0:
 
+- New functions in vmod_std: real2time, time2integer, time2real, real.
+- Chunked requests are now supported. (pass)
+- Add std.querysort() that sorts GET query arguments. (from libvmod-boltsort)
 - Varnish will no longer reply with "200 Not Modified".
 - Backend IMS is now only attempted when last status was 200.
 - Packaging now uses find-provides instead of find-requires. [redhat]
@@ -27,6 +30,9 @@ New since 4.0.0:
 Bugs fixed
 ----------
 
+* 1269_ - Use correct byte counters in varnishncsa when piping a request.
+* 1524_ - Chunked requests should be pipe-able.
+* 1530_ - Expire old object on successful IMS fetch.
 * 1475_ - time-to-first-byte in varnishncsa was potentially dishonest.
 * 1480_ - Porting guide for 4.0 is incomplete.
 * 1482_ - Inherit group memberships of -u specified user.
@@ -53,6 +59,9 @@ Bugs fixed
 * 1519_ - Round-robin director does not support weight. [docs]
 
 
+.. _1269: https://www.varnish-cache.org/trac/ticket/1269
+.. _1524: https://www.varnish-cache.org/trac/ticket/1524
+.. _1530: https://www.varnish-cache.org/trac/ticket/1530
 .. _1475: https://www.varnish-cache.org/trac/ticket/1475
 .. _1480: https://www.varnish-cache.org/trac/ticket/1480
 .. _1482: https://www.varnish-cache.org/trac/ticket/1482
diff --git a/redhat/varnish.spec b/redhat/varnish.spec
index 338aa96..b541d91 100644
--- a/redhat/varnish.spec
+++ b/redhat/varnish.spec
@@ -1,4 +1,4 @@
-%define v_rc rc1
+%define XXXv_rc rc1
 %define vd_rc %{?v_rc:-%{?v_rc}}
 %define    _use_internal_dependency_generator 0
 %define __find_provides %{_builddir}/varnish-%{version}%{?v_rc:-%{?v_rc}}/redhat/find-provides
@@ -14,7 +14,7 @@ URL: http://www.varnish-cache.org/
 Source0: %{name}-%{version}%{?vd_rc}.tar.gz
 #Source0: %{name}-trunk.tar.gz
 BuildRoot: %{_tmppath}/%{name}-%{version}-%{release}-root-%(%{__id_u} -n)
-# To build from git, start with a make dist, see redhat/README.redhat 
+# To build from git, start with a make dist, see redhat/README.redhat
 # You will need at least automake autoconf libtool
 #BuildRequires: automake autoconf libtool
 BuildRequires: ncurses-devel groff pcre-devel pkgconfig libedit-devel jemalloc-devel



From lkarsten at varnish-software.com  Tue Jun 24 09:31:36 2014
From: lkarsten at varnish-software.com (Lasse Karstensen)
Date: Tue, 24 Jun 2014 11:31:36 +0200
Subject: [4.0] 26c2dc6 Ease this up a bit not scare of users
Message-ID: <E1WzN4i-00030t-Ks@project.varnish-software.com>


commit 26c2dc60b23323ace636eb25667d64062039efd3
Author: Lasse Karstensen <lkarsten at varnish-software.com>
Date:   Thu Apr 10 16:29:40 2014 +0200

    Ease this up a bit not scare of users

diff --git a/doc/sphinx/whats-new/upgrading.rst b/doc/sphinx/whats-new/upgrading.rst
index f975179..c2417e6 100644
--- a/doc/sphinx/whats-new/upgrading.rst
+++ b/doc/sphinx/whats-new/upgrading.rst
@@ -7,7 +7,7 @@ Upgrading to Varnish 4
 Changes to VCL
 ==============
 
-Much of the VCL syntax has changed in Varnish 4. We've tried to
+The backend fetch parts of VCL have changed in Varnish 4. We've tried to
 compile a list of changes needed to upgrade here.
 
 Version statement



From apj at mutt.dk  Tue Jun 24 09:31:36 2014
From: apj at mutt.dk (Andreas Plesner)
Date: Tue, 24 Jun 2014 11:31:36 +0200
Subject: [4.0] 77524d9 Add some text about some of the changed parameters,
 fixes #1474 Introduce new section for new parameters
Message-ID: <E1WzN4i-000312-Mq@project.varnish-software.com>


commit 77524d922370f72037ac41d7980be2294c72817a
Author: Andreas Plesner <apj at mutt.dk>
Date:   Thu Apr 10 22:06:17 2014 +0200

    Add some text about some of the changed parameters, fixes #1474
    Introduce new section for new parameters

diff --git a/doc/sphinx/whats-new/upgrading.rst b/doc/sphinx/whats-new/upgrading.rst
index c2417e6..2b23621 100644
--- a/doc/sphinx/whats-new/upgrading.rst
+++ b/doc/sphinx/whats-new/upgrading.rst
@@ -132,14 +132,16 @@ The `remove` keyword is gone
 
 Replaced by `unset`.
 
-Changes to parameters
-=====================
+Changes to existing parameters
+==============================
 
-linger
-~~~~~~
+session_linger
+~~~~~~~~~~~~~~
+`session_linger` has been renamed to `timeout_linger`.
 
 sess_timeout
 ~~~~~~~~~~~~
+`sess_timeout` has been renamed to `timeout_idle`.
 
 sess_workspace
 ~~~~~~~~~~~~~~
@@ -153,8 +155,11 @@ This memory segment has been split into two in 4.0;
 In most cases where you increased `sess_workspace` before, you
 want to increase `workspace_client` now.
 
+New parameters since 3.0
+========================
+
 vcc_allow_inline_c
 ~~~~~~~~~~~~~~~~~~
 
-This parameter is new since 3.0, and prohibits the use of inline
-C code in VCL by default.
+You can now completely disable inline C in your VCL, and it is
+disabled by default.



From apj at mutt.dk  Tue Jun 24 09:31:36 2014
From: apj at mutt.dk (Andreas Plesner)
Date: Tue, 24 Jun 2014 11:31:36 +0200
Subject: [4.0] b241f61 Add info about client.port
Message-ID: <E1WzN4i-00031B-Og@project.varnish-software.com>


commit b241f613acbd78310424baba86fd70d96f6c2063
Author: Andreas Plesner <apj at mutt.dk>
Date:   Fri Apr 11 12:57:46 2014 +0200

    Add info about client.port

diff --git a/doc/sphinx/whats-new/upgrading.rst b/doc/sphinx/whats-new/upgrading.rst
index 2b23621..82298a2 100644
--- a/doc/sphinx/whats-new/upgrading.rst
+++ b/doc/sphinx/whats-new/upgrading.rst
@@ -110,6 +110,13 @@ is reserved for builtin subs.
 req.backend.healthy replaced by std.healthy(req.backend)
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
+client.port replaced by std.port(client.ip)
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+The `client.ip` is now a proper type, which renders as an IP address by
+default. You need to use the `std.port()` function to get the port
+number.
+
 obj is now read-only
 ~~~~~~~~~~~~~~~~~~~~
 



From apj at mutt.dk  Tue Jun 24 09:31:36 2014
From: apj at mutt.dk (Andreas Plesner)
Date: Tue, 24 Jun 2014 11:31:36 +0200
Subject: [4.0] ec13b93 Fix example,
 since req.* isn't available in vcl_backend_response
Message-ID: <E1WzN4i-00031K-Ru@project.varnish-software.com>


commit ec13b93699041fbf721cc0a0b4d0c4a5fe62a635
Author: Andreas Plesner <apj at mutt.dk>
Date:   Fri Apr 11 17:19:04 2014 +0200

    Fix example, since req.* isn't available in vcl_backend_response

diff --git a/doc/sphinx/users-guide/purging.rst b/doc/sphinx/users-guide/purging.rst
index c0e8381..fcd5f5b 100644
--- a/doc/sphinx/users-guide/purging.rst
+++ b/doc/sphinx/users-guide/purging.rst
@@ -122,7 +122,7 @@ object is not available in the `ban lurker` thread.
 You can use the following template to write `ban lurker` friendly bans::
 
   sub vcl_backend_response {
-    set beresp.http.x-url = req.url;
+    set beresp.http.x-url = bereq.url;
   }
 
   sub vcl_deliver {



From fgsch at lodoss.net  Tue Jun 24 09:31:36 2014
From: fgsch at lodoss.net (Federico G. Schwindt)
Date: Tue, 24 Jun 2014 11:31:36 +0200
Subject: [4.0] 6cb0f12 Update, mostly vcl_fetch -> vcl_backend_response
Message-ID: <E1WzN4i-00031T-US@project.varnish-software.com>


commit 6cb0f12a52c5a12b4b082ce7ea285a0a05199c8e
Author: Federico G. Schwindt <fgsch at lodoss.net>
Date:   Fri Apr 11 19:09:01 2014 +0100

    Update, mostly vcl_fetch -> vcl_backend_response

diff --git a/doc/sphinx/glossary/index.rst b/doc/sphinx/glossary/index.rst
index 574156b..2f9250e 100644
--- a/doc/sphinx/glossary/index.rst
+++ b/doc/sphinx/glossary/index.rst
@@ -78,7 +78,7 @@ Varnish Glossary
    backend response
         The response specifically served from a backend to
         varnishd. The backend response may be manipulated in
-        vcl_fetch.
+        vcl_backend_response.
 
    body
 	The bytes that make up the contents of the object, varnishd
diff --git a/doc/sphinx/reference/varnishtest.rst b/doc/sphinx/reference/varnishtest.rst
index bfae5a5..568d4e3 100644
--- a/doc/sphinx/reference/varnishtest.rst
+++ b/doc/sphinx/reference/varnishtest.rst
@@ -82,9 +82,9 @@ An example::
         } -start
 
         varnish v1 -vcl+backend {
-                sub vcl_fetch {
+                sub vcl_backend_response {
                         set beresp.do_esi = true;
-                        if (req.url == "/foo") {
+                        if (bereq.url == "/foo") {
                                 set beresp.ttl = 0s;
                         } else {
                                 set beresp.ttl = 10m;
diff --git a/doc/sphinx/users-guide/compression.rst b/doc/sphinx/users-guide/compression.rst
index fc84f9f..cb95619 100644
--- a/doc/sphinx/users-guide/compression.rst
+++ b/doc/sphinx/users-guide/compression.rst
@@ -29,7 +29,7 @@ content it will be stored in memory in its compressed form. If the
 backend sends content in clear text it will be stored in clear text.
 
 You can make Varnish compress content before storing it in cache in
-`vcl_fetch` by setting 'do_gzip' to true, like this::
+`vcl_backend_response` by setting 'do_gzip' to true, like this::
 
    sub vcl_backend_response {
         if (beresp.http.content-type ~ "text") {
diff --git a/doc/sphinx/users-guide/devicedetection.rst b/doc/sphinx/users-guide/devicedetection.rst
index 5067c4c..2a0cb21 100644
--- a/doc/sphinx/users-guide/devicedetection.rst
+++ b/doc/sphinx/users-guide/devicedetection.rst
@@ -75,8 +75,8 @@ VCL::
     # If the backend does not mention in Vary that it has crafted special
     # content based on the User-Agent (==X-UA-Device), add it. 
     # If your backend does set Vary: User-Agent, you may have to remove that here.
-    sub vcl_fetch {
-        if (req.http.X-UA-Device) {
+    sub vcl_backend_response {
+        if (bereq.http.X-UA-Device) {
             if (!beresp.http.Vary) { # no Vary at all
                 set beresp.http.Vary = "X-UA-Device"; 
             } elseif (beresp.http.Vary !~ "X-UA-Device") { # add to existing Vary
@@ -85,7 +85,7 @@ VCL::
         }
         # comment this out if you don't want the client to know your
         # classification
-        set beresp.http.X-UA-Device = req.http.X-UA-Device;
+        set beresp.http.X-UA-Device = bereq.http.X-UA-Device;
     }
 
     # to keep any caches in the wild from serving wrong content to client #2
@@ -131,15 +131,15 @@ VCL::
     sub vcl_pass { if (req.http.X-UA-Device) { set bereq.http.User-Agent = req.http.X-UA-Device; } }
 
     # standard Vary handling code from previous examples.
-    sub vcl_fetch {
-        if (req.http.X-UA-Device) {
+    sub vcl_backend_response {
+        if (bereq.http.X-UA-Device) {
             if (!beresp.http.Vary) { # no Vary at all
                 set beresp.http.Vary = "X-UA-Device";
             } elseif (beresp.http.Vary !~ "X-UA-Device") { # add to existing Vary
                 set beresp.http.Vary = beresp.http.Vary + ", X-UA-Device";
             }
         }
-        set beresp.http.X-UA-Device = req.http.X-UA-Device;
+        set beresp.http.X-UA-Device = bereq.http.X-UA-Device;
     }
     sub vcl_deliver {
         if ((req.http.X-UA-Device) && (resp.http.Vary)) {
@@ -186,8 +186,8 @@ VCL::
 
     # Handle redirects, otherwise standard Vary handling code from previous
     # examples.
-    sub vcl_fetch {
-        if (req.http.X-UA-Device) {
+    sub vcl_backend_response {
+        if (bereq.http.X-UA-Device) {
             if (!beresp.http.Vary) { # no Vary at all
                 set beresp.http.Vary = "X-UA-Device";
             } elseif (beresp.http.Vary !~ "X-UA-Device") { # add to existing Vary
@@ -203,7 +203,7 @@ VCL::
                 set beresp.http.location = regsub(beresp.http.location, "[?&]devicetype=.*$", "");
             }
         }
-        set beresp.http.X-UA-Device = req.http.X-UA-Device;
+        set beresp.http.X-UA-Device = bereq.http.X-UA-Device;
     }
     sub vcl_deliver {
         if ((req.http.X-UA-Device) && (resp.http.Vary)) {
diff --git a/doc/sphinx/users-guide/esi.rst b/doc/sphinx/users-guide/esi.rst
index 5c7ce72..b81d2ea 100644
--- a/doc/sphinx/users-guide/esi.rst
+++ b/doc/sphinx/users-guide/esi.rst
@@ -50,11 +50,11 @@ Now, lets have an HTML file that has an ESI include statement::
 
 For ESI to work you need to activate ESI processing in VCL, like this::
 
-    sub vcl_fetch {
-    	if (req.url == "/test.html") {
+    sub vcl_backend_response {
+    	if (bereq.url == "/test.html") {
            set beresp.do_esi = true; // Do ESI processing
            set beresp.ttl = 24 h;    // Sets the TTL on the HTML above
-    	} elseif (req.url == "/cgi-bin/date.cgi") {
+    	} elseif (bereq.url == "/cgi-bin/date.cgi") {
            set beresp.ttl = 1m;      // Sets a one minute TTL on
 	       	       	 	     // the included object
         }
diff --git a/doc/sphinx/users-guide/vcl-actions.rst b/doc/sphinx/users-guide/vcl-actions.rst
index 0775afe..d4ede58 100644
--- a/doc/sphinx/users-guide/vcl-actions.rst
+++ b/doc/sphinx/users-guide/vcl-actions.rst
@@ -8,12 +8,12 @@ The most common actions to return are these:
 .. XXX:Maybe a bit more explanation here what is an action and how it is returned? benc
 
 *pass*
- When you return pass the request and subsequent response will be passed to
- and from the backend server. It won't be cached. `pass` can be returned from
- `vcl_recv`
+  When you return pass the request and subsequent response will be passed to
+  and from the backend server. It won't be cached. `pass` can be returned from
+  `vcl_recv`.
 
-*lookup*
-  When you return lookup from `vcl_recv` you tell Varnish to deliver content 
+*hash*
+  When you return hash from `vcl_recv` you tell Varnish to deliver content 
   from cache even if the request othervise indicates that the request 
   should be passed. 
 
@@ -24,18 +24,15 @@ The most common actions to return are these:
   client and the backend connections and Varnish will just sit there
   and shuffle bytes back and forth. Varnish will not look at the data being 
   send back and forth - so your logs will be incomplete. 
-  Beware that with HTTP 1.1 a client can send several requests on the same 
-  connection and so you should instruct Varnish to add a "Connection: close"
-  header before actually returning pipe. 
 
 *deliver*
- Deliver the object to the client. Usually returned from `vcl_backend_response`. 
+  Deliver the object to the client. Usually returned from `vcl_backend_response`. 
 
 *restart*
- Restart processing of the request. You can restart the processing of
- the whole transaction. Changes to the `req` object are retained.
+  Restart processing of the request. You can restart the processing of
+  the whole transaction. Changes to the `req` object are retained.
 
 *retry*
- Retry the request against the backend. This can be called from
- `vcl_backend_response` or `vcl_backend_error` if you don't like the response 
- that the backend delivered.
+  Retry the request against the backend. This can be returned from
+  `vcl_backend_response` or `vcl_backend_error` if you don't like the response 
+  that the backend delivered.
diff --git a/doc/sphinx/users-guide/vcl-built-in-subs.rst b/doc/sphinx/users-guide/vcl-built-in-subs.rst
index 708c05e..0a90a34 100644
--- a/doc/sphinx/users-guide/vcl-built-in-subs.rst
+++ b/doc/sphinx/users-guide/vcl-built-in-subs.rst
@@ -119,7 +119,7 @@ of the following keywords:
 
   fetch
     Retrieve the requested object from the backend. Control will
-    eventually pass to `vcl_fetch`.
+    eventually pass to `vcl_backend_fetch`.
 
 vcl_hash
 ~~~~~~~~
diff --git a/doc/sphinx/users-guide/vcl-example-manipulating-responses.rst b/doc/sphinx/users-guide/vcl-example-manipulating-responses.rst
index 6719524..31e10e1 100644
--- a/doc/sphinx/users-guide/vcl-example-manipulating-responses.rst
+++ b/doc/sphinx/users-guide/vcl-example-manipulating-responses.rst
@@ -6,8 +6,8 @@ Altering the backend response
 Here we override the TTL of a object coming from the backend if it
 matches certain criteria::
 
-  sub vcl_fetch {
-     if (req.url ~ "\.(png|gif|jpg)$") {
+  sub vcl_backend_response {
+     if (bereq.url ~ "\.(png|gif|jpg)$") {
        unset beresp.http.set-cookie;
        set beresp.ttl = 1h;
     }
diff --git a/doc/sphinx/users-guide/vcl-grace.rst b/doc/sphinx/users-guide/vcl-grace.rst
index afc92bd..c3a122c 100644
--- a/doc/sphinx/users-guide/vcl-grace.rst
+++ b/doc/sphinx/users-guide/vcl-grace.rst
@@ -27,7 +27,7 @@ So, in order to serve stale content we must first have some content to
 serve. So to make Varnish keep all objects for 30 minutes beyond their
 TTL use the following VCL::
 
-  sub vcl_fetch {
+  sub vcl_backend_response {
     set beresp.grace = 30m;
   }
 
@@ -45,7 +45,7 @@ minutes if we are unable to serve them? Well, if you have enabled
 backend is sick and if it is we can serve the stale content for a bit
 longer.::
 
-   if (! req.backend.healthy) {
+   if (!std.healthy(backend)) {
       set req.grace = 5m;
    } else {
       set req.grace = 15s;



From fgsch at lodoss.net  Tue Jun 24 09:31:37 2014
From: fgsch at lodoss.net (Federico G. Schwindt)
Date: Tue, 24 Jun 2014 11:31:37 +0200
Subject: [4.0] cf33308 vcl_hash() returns lookup now
Message-ID: <E1WzN4j-00031d-09@project.varnish-software.com>


commit cf333085ae42d8b3f39707a55fca6b51435fec1c
Author: Federico G. Schwindt <fgsch at lodoss.net>
Date:   Fri Apr 11 19:24:51 2014 +0100

    vcl_hash() returns lookup now
    
    While here fmt(1) the text.

diff --git a/doc/sphinx/users-guide/vcl-hashing.rst b/doc/sphinx/users-guide/vcl-hashing.rst
index 83758ad..bdc47b0 100644
--- a/doc/sphinx/users-guide/vcl-hashing.rst
+++ b/doc/sphinx/users-guide/vcl-hashing.rst
@@ -1,25 +1,25 @@
 Hashing
 -------
 
-Internally, when Varnish stores content in the cache it stores the object together with a hash
-key to find the object again. In the default setup this key is
-calculated based on the content of the *Host* header or the IP address
-of the server and the URL.
+Internally, when Varnish stores content in the cache it stores the object
+together with a hash key to find the object again. In the default setup
+this key is calculated based on the content of the *Host* header or the
+IP address of the server and the URL.
 
 Behold the `default vcl`::
 
- sub vcl_hash {
-     hash_data(req.url);
-     if (req.http.host) {
-         hash_data(req.http.host);
-     } else {
-         hash_data(server.ip);
-     }
-     return (hash);
- }
-
-As you can see it first checks in `req.url` then `req.http.host` if it
-exists. It is worth pointing out that Varnish doesn't lowercase the
+    sub vcl_hash {
+        hash_data(req.url);
+        if (req.http.host) {
+            hash_data(req.http.host);
+        } else {
+            hash_data(server.ip);
+        }
+        return (lookup);
+    }
+
+As you can see it first checks in `req.url` then `req.http.host` if
+it exists. It is worth pointing out that Varnish doesn't lowercase the
 hostname or the URL before hashing it so in theory having "Varnish.org/"
 and "varnish.org/" would result in different cache entries. Browsers
 however, tend to lowercase hostnames.
@@ -29,22 +29,20 @@ serve up different content to different clients based on arbitrary
 criteria.
 
 Let's say you want to serve pages in different languages to your users
-based on where their IP address is located. You would need some Vmod
-to get a country code and then put it into the hash. It might look
-like this.
+based on where their IP address is located. You would need some Vmod to
+get a country code and then put it into the hash. It might look like this.
 
 In `vcl_recv`::
 
-  set req.http.X-Country-Code = geoip.lookup(client.ip);
+    set req.http.X-Country-Code = geoip.lookup(client.ip);
 
 And then add a `vcl_hash`::
 
- sub vcl_hash {
-   hash_data(req.http.X-Country-Code);
- }
+    sub vcl_hash {
+        hash_data(req.http.X-Country-Code);
+    }
 
-As the default VCL will take care of adding the host and URL to the
-hash we don't have to do anything else. Be careful calling
-``return(hash)`` as this will abort the execution of the default VCL and
-Varnish can end up returning data based on
-more or less random inputs.
+As the default VCL will take care of adding the host and URL to the hash
+we don't have to do anything else. Be careful calling ``return (lookup)``
+as this will abort the execution of the default VCL and Varnish can end
+up returning data based on more or less random inputs.



From phk at FreeBSD.org  Tue Jun 24 09:31:37 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 24 Jun 2014 11:31:37 +0200
Subject: [4.0] 514b371 Preaching the "Moral License" gospel,
 not just for Varnish
Message-ID: <E1WzN4j-00031m-2L@project.varnish-software.com>


commit 514b371ea87f974db90b82c55f4e3f518a605bc5
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Fri Apr 11 23:55:29 2014 +0000

    Preaching the "Moral License" gospel, not just for Varnish

diff --git a/doc/sphinx/phk/dough.rst b/doc/sphinx/phk/dough.rst
new file mode 100644
index 0000000..08c5645
--- /dev/null
+++ b/doc/sphinx/phk/dough.rst
@@ -0,0 +1,266 @@
+.. _phk_dough:
+
+====================================================
+Raking in the dough on Free and Open Source Software
+====================================================
+
+I'm writing this on the third day after the "Heartbleed" bug in OpenSSL
+devasted internet security, and while I have been very critical of the
+OpenSSL source code since I first saw it, I have nothing but admiration
+for the OpenSSL crew and their effort.
+
+In particular considering what they're paid for it.
+
+Inspired by an article in `Wall Street Journal`_ which tangentially
+touches on the lack of funding for OpenSSL development, I have
+decided to write up my own experiences with funding Open Source
+Software development in some detail.
+
+I've been in the software industry for 30 years now, and I have
+made a living more or less directly from Open Source Software
+for the most recent 15 years.
+
+Sometimes the money came from helping a customer use Open Source
+Software, some times I wrote the Open Source Software for their
+needs and sometimes, as with the `Varnish Moral License`_ I get
+paid to develop and maintain Open Source Software for the greater
+common good.
+
+FreeBSD community funding
+=========================
+
+My first crowd-funding of Free and Open Source Software, was in
+2004, where I `solicited the FreeBSD community`_ for money, so that
+I could devote three to six months of my time to the FreeBSD disk-I/O
+subsystem.
+
+At that time I had spent 10 years as one of the central and key
+FreeBSD developers, so there were no question about my ability
+or suitability for the task at hand.
+
+But in 2004 crowd-funding was not yet "in", and I had to figure
+out how to do it myself.
+
+My parents brought me up to think that finances is a private matter
+but I concluded that the only way you could ask strangers to trow
+money at you, would be to run an open book, where they could see
+what happened to them, so I did open books.
+
+My next dilemma was about my rate, again I had always perceived my
+rate to be a private matter between me and my customers.
+
+My rate is about half of what most people expect -- because I wont
+work for most people: I only work on things I really *care* about.
+
+One of my worries therefore were that publishing my rate would
+undercut friends and colleagues in the FreeBSD project who made a
+living consulting.
+
+But again, there were no way around it, so I published my rate but
+made every attempt to distinguish it from a consulting rate, and
+I never heard any complaints.
+
+And so, having agonized over the exact text and sounded it off on a
+couple of close friends in the FreeBSD project, I threw the proposal
+out there -- and wondered what would happen next.
+
+I had a perfectly safe fall-back plan, you have to when you have
+two kids and a mortgage to feed, but I really had no idea what would
+happen.
+
+Worst case, I'd cause the mother of all `bikesheds`_ get thrown out
+of the FreeBSD project, and be denounced for my "ideological impurity"
+with respect to Free and Open Source Software.
+
+Best case, I expected to get maybe one or two months funded.
+
+The FreeBSD community responded overwhelmingly, my company has never
+sent as many invoices as it did in 2004, and my accountant nearly
+blew a fuse.
+
+And suddenly I found myself in a situation I had never even considered
+how to handle:  How to stop people from sending me money.
+
+I had simply set up a PayPal account, (more on that in a bit), and
+at least at that time, there were no way to prevent people from
+dropping money into it, no matter how much you wanted to stop them.
+
+In the end I managed to yell loud enough and only got overfunded
+a few percent, and I belive that my attempt to deflect the surplus
+to the FreeBSD Foundation gave them a little boost that year.
+
+So about PayPal:  The first thing they did was to shut my account,
+and demand all kinds of papers to be faxed to them, including a
+copy of my passport, despite the fact that Danish law was quite
+clear on that being illegal.  Then, as now, their dispute resolution
+process was less than user-friendly, and in the end it took an
+appeal to a high-ranking officer in PayPal and quite a bit of time
+to actually get the money people had donated.
+
+I swore to myself that next time, if there ever came a next time,
+PayPal would not be involved.  Besides, I found their fees quite
+excessive.
+
+In total I made EUR27K, and it kept my kids fed and my bank
+happy for the six months I worked on it.
+
+And work I did.
+
+I've never had a harsher boss than those six months, and it surprised
+me how much it stressed me, because I felt like I was working on a
+stage, with the entire FreeBSD project in audience, wondering if I
+were going to deliver the goods or not.
+
+As a result, the 187 donors certainly got their moneys worth,
+most of that half year I worked 80 hour weeks, which made me
+decide not to continue, despite many donors indicating that they
+were perfectly willing to fund several more months.
+
+Varnish community funding
+=========================
+
+Five years later, having developed Varnish 1.0 for Norways "Verdens
+Gang" newspaper, I decided to give community funding a go again.
+
+Wiser from experience, I structured the `Varnish Moral License`_
+to tackle the issues which had caused me grief the first time
+around:
+
+Contact first, then send money, not the other way around, and also
+a focus on fewer larger sponsors, rather than people sending me
+EUR10 or USD15 or even, in one case, the EUR1 which happened to
+remain in his PayPal Account.
+
+I ran even more open books this time, on the VML webpages you can
+see how many hours and a one-line description of what I did in them,
+for every single day I've been working under the VML since 2010.
+
+I also decided to be honest with myself and my donors, one hour
+of work was one hour of work -- nobody would benefit from me
+dying from stress.
+
+In practice it doesn't quite work like that, there are plenty of
+thinking in the shower, emails and IRC answers at all hours of the
+day and a lot of "just checking a detail" that happens off the
+clock, because I like my job, and nothing could stop me anyway.
+
+In each of 2010, 2011 and 2013 I worked around 950 hours work on
+Varnish, funded by the community.
+
+In 2012 I only worked 589 hours, because I was building a prototype
+computer cluster to do adaptive optics real-time calculations for
+the ESO `Extremely Large Telescope`_ ("ELT") -- There was no way I
+could say no to that contract :-)
+
+In 2014 I actually have hours available do even more Varnish work,
+and I have done so in the ramp up to the 4.0.0 release, but despite
+my not so subtle hints, the current outlook is still only for 800
+hours to be funded, but I'm crossing my fingers that more sponsors
+will appear now that V4 is released.  (Nudge, nudge, wink, wink,
+he said knowingly! :-)
+
+Why Free and Open Source costs money
+====================================
+
+Varnish is about 90.000 lines of code, the VML brings in about
+EUR90K a year, and that means that Varnish has me working and
+caring about issues big and small.
+
+Not that I am satisfied with our level of effort, we should have
+much better documentation, our wish-list of features is far too
+long and we take too long to close tickets.
+
+But I'm not going to complain, because the Heartbleed vulnerability
+revealed that even though OpenSSL is about three to five times
+larger in terms of code, the OpenSSL Foundation Inc. took in only
+about EUR700K last year, and most of that was for consulting and
+certification, not for "free-range" development and maintenance of
+the OpenSSL source code base.
+
+I really hope that the Heartbleed vulnerability helps bring home
+the message to other communities, that Free and Open Source Software
+does not materialize out of empty space, it is written by people.
+
+People who love what we do, which is why I'm sitting here,
+way past midnight on a friday evening, writing this phamplet.
+
+But software *is* written by people, real people with kids, cars,
+mortgages, leaky roofs, sick pets, infirm parents and all other
+kinds of perfectly normal worries of an adult human being.
+
+The best way to improve the quality of Free and Open Source Software,
+is to make it possible for these people to spend time on it.
+
+They need time to review submissions carefully, time to write and
+run test-cases, time to respond and fix to bug-reports, time to
+code and most of all, time to think about the code.
+
+But it would not even be close to morally defensible to ask these
+people to forego time to play with their kids, so that they instead
+develop and maintain the software that drives other peoples companies.
+
+The right way to go -- the moral way to go -- and by far the most
+productive way to go, is to pay the developers so they can make
+the software they love their living.
+
+How to fund Free and Open Source Software
+=========================================
+
+One way is to hire them, with the understanding that they spend
+some company time on the software.
+
+Experience has shown that these people almost invariably have highly
+desirable brains which employers love to throw at all sorts of
+interesting problems, which tends to erode the "donated" company
+time.
+
+But a lot of Free and Open Source Software has been, and still is
+developed and  maintained this way, with or without written
+agreements or even knowledge of this being the case.
+
+Another way is for software projects to set up foundations to
+collect money and hire developers.  This is a relatively complex
+thing to do, and it will only be availabel for larger projects.
+
+The Apache Foundation "adopts" smaller projects inside their field
+of interest, and I belive that works OK, but I'm not sure if it
+can easily be transplanted to different topics.
+
+The final way is to simply throw money a the developers, the
+way the FreeBSD and Varnish communities have done with me.
+
+It is a far more flexible solution with respect to level of
+engangement, national boundaries etc. etc, but in many ways it
+demands more from both sides of the deal, in particular
+with respect to paperwork, taxes and so on.
+
+Conclusion
+==========
+
+I am obiously biased, I derive a large fraction of my relatively
+modest income from community funding, for which I am the Varnish
+community deeply grateful.
+
+But biased as I may be, I belive that the Varnish community and I
+has shown that a tiny investment goes a long way in Free and Open
+Source Software.
+
+I hope to see that mutual benefit spread to other communities and
+projects, not just to OpenSSL and not just because they found a
+really bad bug the other way, but to any community around any piece
+of software which does serious work for serious companies.
+
+Thanks in advance,
+
+Poul-Henning, 2014-04-11
+
+.. _Wall Street Journal: http://online.wsj.com/news/articles/SB10001424052702303873604579491350251315132
+
+.. _Varnish Moral License: http://phk.freebsd.dk/VML
+
+.. _solicited the FreeBSD community: http://people.freebsd.org/~phk/funding.html
+
+.. _Extremely Large Telescope: http://www.eso.org/public/teles-instr/e-elt/
+
+.. _bikesheds: http://bikeshed.org/
+
diff --git a/doc/sphinx/phk/index.rst b/doc/sphinx/phk/index.rst
index 7eac4e5..40a4830 100644
--- a/doc/sphinx/phk/index.rst
+++ b/doc/sphinx/phk/index.rst
@@ -8,6 +8,7 @@ You may or may not want to know what Poul-Henning thinks.
 .. toctree::
 	:maxdepth: 1
 
+	dough.rst
 	wanton_destruction.rst
 	spdy.rst
 	http20.rst



From phk at FreeBSD.org  Tue Jun 24 09:31:37 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 24 Jun 2014 11:31:37 +0200
Subject: [4.0] e477eb7 git commit is magic: It makes it possible to see
 mistakes in the text.
Message-ID: <E1WzN4j-00031w-3w@project.varnish-software.com>


commit e477eb72cb6bfb2a71abd1020b7fcde8e0265d63
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sat Apr 12 00:16:51 2014 +0000

    git commit is magic:  It makes it possible to see mistakes in the text.

diff --git a/doc/sphinx/phk/dough.rst b/doc/sphinx/phk/dough.rst
index 08c5645..ed9f6be 100644
--- a/doc/sphinx/phk/dough.rst
+++ b/doc/sphinx/phk/dough.rst
@@ -129,7 +129,7 @@ around:
 Contact first, then send money, not the other way around, and also
 a focus on fewer larger sponsors, rather than people sending me
 EUR10 or USD15 or even, in one case, the EUR1 which happened to
-remain in his PayPal Account.
+linger in his PayPal Account.
 
 I ran even more open books this time, on the VML webpages you can
 see how many hours and a one-line description of what I did in them,
@@ -173,9 +173,11 @@ long and we take too long to close tickets.
 But I'm not going to complain, because the Heartbleed vulnerability
 revealed that even though OpenSSL is about three to five times
 larger in terms of code, the OpenSSL Foundation Inc. took in only
-about EUR700K last year, and most of that was for consulting and
-certification, not for "free-range" development and maintenance of
-the OpenSSL source code base.
+about EUR700K last year.
+
+And most of that EUR700K was for consulting and certification, not
+for "free-range" development and maintenance of the OpenSSL source
+code base so badly needs.
 
 I really hope that the Heartbleed vulnerability helps bring home
 the message to other communities, that Free and Open Source Software
@@ -247,7 +249,7 @@ Source Software.
 
 I hope to see that mutual benefit spread to other communities and
 projects, not just to OpenSSL and not just because they found a
-really bad bug the other way, but to any community around any piece
+really bad bug the other day, but to any community around any piece
 of software which does serious work for serious companies.
 
 Thanks in advance,



From phk at FreeBSD.org  Tue Jun 24 09:31:37 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 24 Jun 2014 11:31:37 +0200
Subject: [4.0] 426474c typo
Message-ID: <E1WzN4j-000325-6I@project.varnish-software.com>


commit 426474c067ce0c129a4f249b6b1a7a29527f2bb2
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sat Apr 12 07:59:17 2014 +0000

    typo

diff --git a/doc/sphinx/phk/dough.rst b/doc/sphinx/phk/dough.rst
index ed9f6be..3e71fe6 100644
--- a/doc/sphinx/phk/dough.rst
+++ b/doc/sphinx/phk/dough.rst
@@ -222,7 +222,7 @@ agreements or even knowledge of this being the case.
 
 Another way is for software projects to set up foundations to
 collect money and hire developers.  This is a relatively complex
-thing to do, and it will only be availabel for larger projects.
+thing to do, and it will only be available for larger projects.
 
 The Apache Foundation "adopts" smaller projects inside their field
 of interest, and I belive that works OK, but I'm not sure if it



From martin at varnish-software.com  Tue Jun 24 09:31:37 2014
From: martin at varnish-software.com (Martin Blix Grydeland)
Date: Tue, 24 Jun 2014 11:31:37 +0200
Subject: [4.0] 89870e0 Be more RFC2616 compliant,
 by keeping Date header as given from the backend. Set it on the
 object if missing.
Message-ID: <E1WzN4j-00032E-8H@project.varnish-software.com>


commit 89870e0bbd785964c322e1e453f492d747731c88
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Mon Apr 14 14:40:11 2014 +0200

    Be more RFC2616 compliant, by keeping Date header as given from the
    backend. Set it on the object if missing.
    
    Previously we would add an explicit Date header on each response with
    the time and date of the response, overwriting any Date header on the
    object that came from the backend.
    
    Now we will add a Date header if the backend didn't supply one just
    before vcl_backend_response() is run, and not add any Date headers for
    the replies.
    
    This change means Date headers sent by Varnish is approximately the
    time and date this object was generated by the backend, and not the
    current date and time.

diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c
index 19b8fb8..3091d29 100644
--- a/bin/varnishd/cache/cache_fetch.c
+++ b/bin/varnishd/cache/cache_fetch.c
@@ -249,6 +249,8 @@ static enum fetch_step
 vbf_stp_startfetch(struct worker *wrk, struct busyobj *bo)
 {
 	int i, do_ims;
+	double now;
+	char time_str[VTIM_FORMAT_SIZE];
 
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
@@ -292,7 +294,8 @@ vbf_stp_startfetch(struct worker *wrk, struct busyobj *bo)
 		VSC_C_main->backend_retry++;
 		i = V1F_fetch_hdr(wrk, bo, bo->req);
 	}
-	VSLb_ts_busyobj(bo, "Beresp", W_TIM_real(wrk));
+	now = W_TIM_real(wrk);
+	VSLb_ts_busyobj(bo, "Beresp", now);
 
 	if (i) {
 		AZ(bo->vbc);
@@ -302,6 +305,22 @@ vbf_stp_startfetch(struct worker *wrk, struct busyobj *bo)
 	AN(bo->vbc);
 	http_VSL_log(bo->beresp);
 
+	if (!http_GetHdr(bo->beresp, H_Date, NULL)) {
+		/*
+		 * RFC 2616 14.18 Date: The Date general-header field
+		 * represents the date and time at which the message was
+		 * originated, having the same semantics as orig-date in
+		 * RFC 822. ... A received message that does not have a
+		 * Date header field MUST be assigned one by the recipient
+		 * if the message will be cached by that recipient or
+		 * gatewayed via a protocol which requires a Date.
+		 *
+		 * If we didn't get a Date header, we assign one here.
+		 */
+		VTIM_format(now, time_str);
+		http_PrintfHeader(bo->beresp, "Date: %s", time_str);
+	}
+
 	/*
 	 * These two headers can be spread over multiple actual headers
 	 * and we rely on their content outside of VCL, so collect them
diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c
index 1e9ecd5..c782d44 100644
--- a/bin/varnishd/cache/cache_req_fsm.c
+++ b/bin/varnishd/cache/cache_req_fsm.c
@@ -87,7 +87,6 @@ DOT deliver:deliver:s -> DONE [style=bold,color=blue]
 static enum req_fsm_nxt
 cnt_deliver(struct worker *wrk, struct req *req)
 {
-	char time_str[30];
 	double now;
 
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
@@ -110,10 +109,6 @@ cnt_deliver(struct worker *wrk, struct req *req)
 	http_ClrHeader(req->resp);
 	http_FilterResp(req->obj->http, req->resp, 0);
 
-	http_Unset(req->resp, H_Date);
-	VTIM_format(now, time_str);
-	http_PrintfHeader(req->resp, "Date: %s", time_str);
-
 	if (req->wrk->stats.cache_hit)
 		http_PrintfHeader(req->resp,
 		    "X-Varnish: %u %u", req->vsl->wid & VSL_IDENTMASK,
diff --git a/bin/varnishtest/tests/c00044.vtc b/bin/varnishtest/tests/c00044.vtc
index 2d1d3cc..091e8a8 100644
--- a/bin/varnishtest/tests/c00044.vtc
+++ b/bin/varnishtest/tests/c00044.vtc
@@ -25,6 +25,8 @@ varnish v1 \
 	sub vcl_backend_response {
 		set beresp.do_stream = false;
 		set beresp.storage_hint = "invalid";
+		# Unset Date header to not change the object sizes
+		unset beresp.http.Date;
 	}
 } -start
 
diff --git a/bin/varnishtest/tests/c00045.vtc b/bin/varnishtest/tests/c00045.vtc
index 126f1a9..33b7b9e 100644
--- a/bin/varnishtest/tests/c00045.vtc
+++ b/bin/varnishtest/tests/c00045.vtc
@@ -17,6 +17,8 @@ varnish v1 \
 	sub vcl_backend_response {
 		set beresp.do_stream = false;
 		set beresp.storage_hint = "s0";
+		# Unset Date header to not change the object sizes
+		unset beresp.http.Date;
 	}
 } -start
 
diff --git a/bin/varnishtest/tests/r01140.vtc b/bin/varnishtest/tests/r01140.vtc
index e38e304..11e2a72 100644
--- a/bin/varnishtest/tests/r01140.vtc
+++ b/bin/varnishtest/tests/r01140.vtc
@@ -22,6 +22,8 @@ varnish v1 -arg "-p nuke_limit=0 -p shortlived=0" \
 	-arg "-smalloc,1m" -vcl+backend {
 	sub vcl_backend_response {
 		set beresp.do_stream = false;
+		# Unset Date header to not change the object sizes
+		unset beresp.http.Date;
 	}
 } -start
 
diff --git a/bin/varnishtest/tests/r01284.vtc b/bin/varnishtest/tests/r01284.vtc
index f89b579..dc6336f 100644
--- a/bin/varnishtest/tests/r01284.vtc
+++ b/bin/varnishtest/tests/r01284.vtc
@@ -16,6 +16,8 @@ varnish v1 \
 	sub vcl_backend_response {
 		set beresp.do_stream = false;
 		set beresp.storage_hint = "Transient";
+		# Unset Date header to not change the object sizes
+		unset beresp.http.Date;
 	}
 } -start
 



From martin at varnish-software.com  Tue Jun 24 09:31:37 2014
From: martin at varnish-software.com (Martin Blix Grydeland)
Date: Tue, 24 Jun 2014 11:31:37 +0200
Subject: [4.0] d31fa83 Timestamp "Process" after vcl_deliver has been run.
Message-ID: <E1WzN4j-00032O-A9@project.varnish-software.com>


commit d31fa8397deda3697d68a381d7df48073e41556d
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Mon Apr 14 15:44:00 2014 +0200

    Timestamp "Process" after vcl_deliver has been run.
    
    The EXP_Touch and the Age header will be based off the last timestamp
    taken. This can be slightly inaccurate, but saves us from having to do
    another timestamp for them.
    
    Fixes: #1475

diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c
index c782d44..b6d0500 100644
--- a/bin/varnishd/cache/cache_req_fsm.c
+++ b/bin/varnishd/cache/cache_req_fsm.c
@@ -87,7 +87,6 @@ DOT deliver:deliver:s -> DONE [style=bold,color=blue]
 static enum req_fsm_nxt
 cnt_deliver(struct worker *wrk, struct req *req)
 {
-	double now;
 
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
@@ -99,10 +98,8 @@ cnt_deliver(struct worker *wrk, struct req *req)
 
 	assert(req->obj->objcore->refcnt > 0);
 
-	now = W_TIM_real(wrk);
-	VSLb_ts_req(req, "Process", now);
 	if (req->obj->objcore->exp_flags & OC_EF_EXP)
-		EXP_Touch(req->obj->objcore, now);
+		EXP_Touch(req->obj->objcore, req->t_prev);
 
 	HTTP_Setup(req->resp, req->ws, req->vsl, SLT_RespMethod);
 
@@ -117,8 +114,13 @@ cnt_deliver(struct worker *wrk, struct req *req)
 		http_PrintfHeader(req->resp,
 		    "X-Varnish: %u", req->vsl->wid & VSL_IDENTMASK);
 
+	/* We base Age calculation upon the last timestamp taken during
+	   client request processing. This gives some inaccuracy, but
+	   since Age is only full second resolution that shouldn't
+	   matter. */
+	assert(req->t_prev > req->obj->exp.t_origin);
 	http_PrintfHeader(req->resp, "Age: %.0f",
-	    now - req->obj->exp.t_origin);
+	    req->t_prev - req->obj->exp.t_origin);
 
 	http_SetHeader(req->resp, "Via: 1.1 varnish (v4)");
 
@@ -127,6 +129,7 @@ cnt_deliver(struct worker *wrk, struct req *req)
 		RFC2616_Weaken_Etag(req->resp);
 
 	VCL_deliver_method(req->vcl, wrk, req, NULL, req->http->ws);
+	VSLb_ts_req(req, "Process", W_TIM_real(wrk));
 
 	/* Stop the insanity before it turns "Hotel California" on us */
 	if (req->restarts >= cache_param->max_restarts)



From martin at varnish-software.com  Tue Jun 24 09:31:37 2014
From: martin at varnish-software.com (Martin Blix Grydeland)
Date: Tue, 24 Jun 2014 11:31:37 +0200
Subject: [4.0] 9919320 Use the time we get the content from the backend as the
 origin time of objects.
Message-ID: <E1WzN4j-00032X-DW@project.varnish-software.com>


commit 9919320874d90c1a99e3a7fe9172c63fa8c24159
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Mon Apr 14 17:09:20 2014 +0200

    Use the time we get the content from the backend as the origin time of objects.
    
    Before, the origin time for the objects was defined as the time we
    received the full request from the client that initiated the
    fetch. Now we use the time we received the response headers from the
    backend.

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index c8d8d89..b0740c1 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -532,7 +532,6 @@ struct busyobj {
 	 */
 	unsigned		refcount;
 	int			retries;
-	double			t_fetch;
 	struct req		*req;
 
 	uint8_t			*vary;
@@ -1240,7 +1239,7 @@ int WS_Overflowed(const struct ws *ws);
 void *WS_Printf(struct ws *ws, const char *fmt, ...) __printflike(2, 3);
 
 /* rfc2616.c */
-void RFC2616_Ttl(struct busyobj *);
+void RFC2616_Ttl(struct busyobj *, double now);
 enum body_status RFC2616_Body(struct busyobj *, struct dstat *);
 unsigned RFC2616_Req_Gzip(const struct http *);
 int RFC2616_Do_Cond(const struct req *sp);
diff --git a/bin/varnishd/cache/cache_busyobj.c b/bin/varnishd/cache/cache_busyobj.c
index 22f6441..45e2845 100644
--- a/bin/varnishd/cache/cache_busyobj.c
+++ b/bin/varnishd/cache/cache_busyobj.c
@@ -148,8 +148,6 @@ VBO_GetBusyObj(struct worker *wrk, const struct req *req)
 	bo->vcl = req->vcl;
 	VCL_Ref(bo->vcl);
 
-	bo->t_fetch = req->t_req;
-	assert(!isnan(bo->t_fetch) && bo->t_fetch != 0.);
 	bo->t_first = bo->t_prev = NAN;
 
 	return (bo);
diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c
index 3091d29..2bb5feb 100644
--- a/bin/varnishd/cache/cache_fetch.c
+++ b/bin/varnishd/cache/cache_fetch.c
@@ -347,7 +347,7 @@ vbf_stp_startfetch(struct worker *wrk, struct busyobj *bo)
 	 * What does RFC2616 think about TTL ?
 	 */
 	EXP_Clr(&bo->exp);
-	RFC2616_Ttl(bo);
+	RFC2616_Ttl(bo, now);
 
 	/* private objects have negative TTL */
 	if (bo->fetch_objcore->flags & OC_F_PRIVATE)
@@ -670,7 +670,7 @@ vbf_stp_error(struct worker *wrk, struct busyobj *bo)
 	HTTP_Setup(bo->beresp, bo->ws, bo->vsl, SLT_BerespMethod);
 	http_SetResp(bo->beresp, "HTTP/1.1", 503, "Backend fetch failed");
 
-	bo->exp.t_origin = VTIM_real();
+	bo->exp.t_origin = bo->t_prev;
 	bo->exp.ttl = 0;
 	bo->exp.grace = 0;
 	bo->exp.keep = 0;
diff --git a/bin/varnishd/cache/cache_rfc2616.c b/bin/varnishd/cache/cache_rfc2616.c
index 5334291..bb0a85e 100644
--- a/bin/varnishd/cache/cache_rfc2616.c
+++ b/bin/varnishd/cache/cache_rfc2616.c
@@ -63,7 +63,7 @@
  */
 
 void
-RFC2616_Ttl(struct busyobj *bo)
+RFC2616_Ttl(struct busyobj *bo, double now)
 {
 	unsigned max_age, age;
 	double h_date, h_expires;
@@ -76,8 +76,8 @@ RFC2616_Ttl(struct busyobj *bo)
 
 	hp = bo->beresp;
 
-	assert(bo->t_fetch != 0.0 && !isnan(bo->t_fetch));
-	expp->t_origin = bo->t_fetch;
+	assert(now != 0.0 && !isnan(now));
+	expp->t_origin = now;
 
 	expp->ttl = cache_param->default_ttl;
 	expp->grace = cache_param->default_grace;
@@ -154,16 +154,16 @@ RFC2616_Ttl(struct busyobj *bo)
 		}
 
 		if (h_date == 0 ||
-		    fabs(h_date - bo->t_fetch) < cache_param->clock_skew) {
+		    fabs(h_date - now) < cache_param->clock_skew) {
 			/*
 			 * If we have no Date: header or if it is
 			 * sufficiently close to our clock we will
 			 * trust Expires: relative to our own clock.
 			 */
-			if (h_expires < bo->t_fetch)
+			if (h_expires < now)
 				expp->ttl = 0;
 			else
-				expp->ttl = h_expires - bo->t_fetch;
+				expp->ttl = h_expires - now;
 			break;
 		} else {
 			/*
@@ -179,7 +179,7 @@ RFC2616_Ttl(struct busyobj *bo)
 	/* calculated TTL, Our time, Date, Expires, max-age, age */
 	VSLb(bo->vsl, SLT_TTL,
 	    "RFC %.0f %.0f %.0f %.0f %.0f %.0f %.0f %u",
-	    expp->ttl, -1., -1., bo->t_fetch,
+	    expp->ttl, -1., -1., now,
 	    expp->t_origin, h_date, h_expires, max_age);
 }
 



From martin at varnish-software.com  Tue Jun 24 09:31:37 2014
From: martin at varnish-software.com (Martin Blix Grydeland)
Date: Tue, 24 Jun 2014 11:31:37 +0200
Subject: [4.0] 0a02a39 Add Date and Server headers also on backend error
 objects
Message-ID: <E1WzN4j-00032h-FF@project.varnish-software.com>


commit 0a02a3974f36929bef9212904ae2a1494bbba7c6
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Mon Apr 14 17:50:11 2014 +0200

    Add Date and Server headers also on backend error objects

diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c
index 2bb5feb..be664e5 100644
--- a/bin/varnishd/cache/cache_fetch.c
+++ b/bin/varnishd/cache/cache_fetch.c
@@ -653,11 +653,14 @@ vbf_stp_error(struct worker *wrk, struct busyobj *bo)
 {
 	struct storage *st;
 	ssize_t l;
+	double now;
+	char time_str[VTIM_FORMAT_SIZE];
 
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
 
-	VSLb_ts_busyobj(bo, "Error", W_TIM_real(wrk));
+	now = W_TIM_real(wrk);
+	VSLb_ts_busyobj(bo, "Error", now);
 
 	AN(bo->fetch_objcore->flags & OC_F_BUSY);
 
@@ -669,6 +672,9 @@ vbf_stp_error(struct worker *wrk, struct busyobj *bo)
 
 	HTTP_Setup(bo->beresp, bo->ws, bo->vsl, SLT_BerespMethod);
 	http_SetResp(bo->beresp, "HTTP/1.1", 503, "Backend fetch failed");
+	VTIM_format(now, time_str);
+	http_PrintfHeader(bo->beresp, "Date: %s", time_str);
+	http_SetHeader(bo->beresp, "Server: Varnish");
 
 	bo->exp.t_origin = bo->t_prev;
 	bo->exp.ttl = 0;



From martin at varnish-software.com  Tue Jun 24 09:31:37 2014
From: martin at varnish-software.com (Martin Blix Grydeland)
Date: Tue, 24 Jun 2014 11:31:37 +0200
Subject: [4.0] 1127242 Add test case to make sure we always emit Date headers
Message-ID: <E1WzN4j-00032q-H1@project.varnish-software.com>


commit 1127242299c930330a389c6960bb27f254132b96
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Mon Apr 14 17:50:51 2014 +0200

    Add test case to make sure we always emit Date headers

diff --git a/bin/varnishtest/tests/c00066.vtc b/bin/varnishtest/tests/c00066.vtc
new file mode 100644
index 0000000..eeb1ff9
--- /dev/null
+++ b/bin/varnishtest/tests/c00066.vtc
@@ -0,0 +1,58 @@
+varnishtest "Check that we always deliver Date headers"
+
+server s1 {
+	rxreq
+	txresp
+} -start
+
+varnish v1 -vcl+backend {
+	backend bad { .host = "${bad_ip}"; }
+	sub vcl_recv {
+		if (req.url == "/synth") {
+			return (synth(200, "Synth test"));
+		}
+		if (req.url == "/error") {
+			set req.backend_hint = bad;
+		}
+	}
+	sub vcl_backend_response {
+		set beresp.do_stream = false;
+	}
+} -start
+
+varnish v1 -cliok "param.set connect_timeout 1"
+
+logexpect l1 -v v1 -g request {
+	expect	0 1001	Begin		"^req .* rxreq"
+	expect	* =	ReqURL		"/"
+	expect	* =	RespHeader	"^Date: "
+	expect	* =	End
+
+	expect	* 1003	Begin		"^req .* rxreq"
+	expect	* =	ReqURL		"/synth"
+	expect	* =	RespHeader	"^Date: "
+	expect	* =	End
+
+	expect	* 1004	Begin		"^req .* rxreq"
+	expect	* =	ReqURL		"/error"
+	expect	* =	RespHeader	"^Date: "
+	expect	* =	End
+} -start
+
+client c1 {
+	txreq -url "/"
+	rxresp
+	expect resp.status == 200
+	expect resp.msg == "Ok"
+
+	txreq -url "/synth"
+	rxresp
+	expect resp.status == 200
+	expect resp.msg == "Synth test"
+
+	txreq -url "/error"
+	rxresp
+	expect resp.status == 503
+} -run
+
+logexpect l1 -wait



From fgsch at lodoss.net  Tue Jun 24 09:31:37 2014
From: fgsch at lodoss.net (Federico G. Schwindt)
Date: Tue, 24 Jun 2014 11:31:37 +0200
Subject: [4.0] a9716fe Use bereq.url within vcl_backend_response
Message-ID: <E1WzN4j-00032z-JN@project.varnish-software.com>


commit a9716fe5ba673ed32f4eaeffd17ae65bbe4381e7
Author: Federico G. Schwindt <fgsch at lodoss.net>
Date:   Wed Apr 16 17:03:26 2014 +0100

    Use bereq.url within vcl_backend_response
    
    Pointed out by dharrigan in #varnish.

diff --git a/doc/sphinx/users-guide/increasing-your-hitrate.rst b/doc/sphinx/users-guide/increasing-your-hitrate.rst
index a413b71..231e27b 100644
--- a/doc/sphinx/users-guide/increasing-your-hitrate.rst
+++ b/doc/sphinx/users-guide/increasing-your-hitrate.rst
@@ -273,7 +273,7 @@ You need VCL to identify the objects you want and then you set the
 'beresp.ttl' to whatever you want::
 
     sub vcl_backend_response {
-        if (req.url ~ "^/legacy_broken_cms/") {
+        if (bereq.url ~ "^/legacy_broken_cms/") {
             set beresp.ttl = 5d;
         }
     }



From fgsch at lodoss.net  Tue Jun 24 09:31:37 2014
From: fgsch at lodoss.net (Federico G. Schwindt)
Date: Tue, 24 Jun 2014 11:31:37 +0200
Subject: [4.0] 5e1586a Fix typo in error message
Message-ID: <E1WzN4j-000338-LP@project.varnish-software.com>


commit 5e1586aa4c3f9bf346cac63a018ccc8d30ee0618
Author: Matt Robenolt <matt at ydekproductions.com>
Date:   Thu Apr 17 14:33:38 2014 -0700

    Fix typo in error message

diff --git a/lib/libvarnishtools/vut.c b/lib/libvarnishtools/vut.c
index f465b1f..d943d8f 100644
--- a/lib/libvarnishtools/vut.c
+++ b/lib/libvarnishtools/vut.c
@@ -304,7 +304,7 @@ VUT_Main(void)
 			VUT.vslq = VSLQ_New(VUT.vsl, &c, VUT.g_arg, VUT.q_arg);
 			AN(VUT.vslq);
 			AZ(c);
-			VUT_Error(0, "Log reaquired");
+			VUT_Error(0, "Log reacquired");
 		}
 
 		i = VSLQ_Dispatch(VUT.vslq, VUT.dispatch_f, VUT.dispatch_priv);



From apj at mutt.dk  Tue Jun 24 09:31:37 2014
From: apj at mutt.dk (Andreas Plesner)
Date: Tue, 24 Jun 2014 11:31:37 +0200
Subject: [4.0] e7340a6 Add more changes. Fixes #1480
Message-ID: <E1WzN4j-00033H-O0@project.varnish-software.com>


commit e7340a68ecf908968f4ffbb4ca4be24347348aae
Author: Andreas Plesner <apj at mutt.dk>
Date:   Mon Apr 21 22:11:16 2014 +0200

    Add more changes.
    Fixes #1480

diff --git a/doc/sphinx/whats-new/upgrading.rst b/doc/sphinx/whats-new/upgrading.rst
index 82298a2..788764a 100644
--- a/doc/sphinx/whats-new/upgrading.rst
+++ b/doc/sphinx/whats-new/upgrading.rst
@@ -19,6 +19,12 @@ VCL version number::
 
     vcl 4.0;
 
+req.request is now req.method
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+To align better with RFC naming, `req.request` has been renamed to
+`req.method`.
+
 vcl_fetch is now vcl_backend_response
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
@@ -57,6 +63,13 @@ has been removed, and you should use the hash director directly::
         set req.backend_hint = h.backend(client.identity);
     }
 
+vcl_error is now vcl_backend_error
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+To make a distinction between internally generated errors and
+VCL synthetic responses, `vcl_backend_error` will be called when 
+varnish encounters an error when trying to fetch an object.
+
 error() is now synth()
 ~~~~~~~~~~~~~~~~~~~~~~
 
@@ -79,6 +92,13 @@ The synthetic keyword is now a function::
         return (deliver);
     }
 
+obj in vcl_error replaced by beresp in vcl_backend_error
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+To better represent a the context in which it is called, you
+should now use `beresp.*` vcl_backend_error, where you used to
+use `obj.*` in `vcl_error`.
+
 hit_for_pass objects are created using beresp.uncacheable
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
@@ -92,9 +112,6 @@ Example::
         }
     }
 
-vcl_recv should return(hash) instead of lookup now
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
 req.* not available in vcl_backend_response
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
@@ -110,12 +127,12 @@ is reserved for builtin subs.
 req.backend.healthy replaced by std.healthy(req.backend)
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-client.port replaced by std.port(client.ip)
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+client.port, and server.port replaced by respectively std.port(client.ip) and std.port(server.ip)
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-The `client.ip` is now a proper type, which renders as an IP address by
-default. You need to use the `std.port()` function to get the port
-number.
+`client.ip` and `server.ip` are now proper datatypes, which renders
+as an IP address by default. You need to use the `std.port()`
+function to get the port number.
 
 obj is now read-only
 ~~~~~~~~~~~~~~~~~~~~
@@ -123,6 +140,15 @@ obj is now read-only
 `obj` is now read-only. `obj.hits`, if enabled in VCL, now counts per
 objecthead, not per object. `obj.last_use` has been retired.
 
+Some return values have been replaced
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Apart from the new `synth` return value described above, the
+following has changed:
+
+ - `vcl_recv` must now return `hash` instead of `lookup`
+ - `vcl_hash` must now return `lookup` instead of `hash`
+ - `vcl_pass` must now return `fetch` instead of `pass`
 
 default/builtin VCL changes
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -139,6 +165,8 @@ The `remove` keyword is gone
 
 Replaced by `unset`.
 
+
+
 Changes to existing parameters
 ==============================
 
@@ -170,3 +198,14 @@ vcc_allow_inline_c
 
 You can now completely disable inline C in your VCL, and it is
 disabled by default.
+
+Other changes
+=============
+
+New log filtering
+~~~~~~~~~~~~~~~~~
+
+The logging framework has a new filtering language, which means
+that the -m switch has been replaced with a new -q switch.
+See :ref:`ref-vsl-query` for more information about the new
+query language.



From phk at FreeBSD.org  Tue Jun 24 09:31:37 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 24 Jun 2014 11:31:37 +0200
Subject: [4.0] 203eafe Some stylistic consistency patches from Dridi
 Boukelmoune who made them with Coccinelle.
Message-ID: <E1WzN4j-00033Q-QX@project.varnish-software.com>


commit 203eafef5b9c9f03dd29826128e4800a17e28d39
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Apr 22 08:16:41 2014 +0000

    Some stylistic consistency patches from Dridi Boukelmoune
    who made them with Coccinelle.
    
    Thanks!

diff --git a/bin/varnishd/cache/cache_cli.c b/bin/varnishd/cache/cache_cli.c
index ca9da16..99b781c 100644
--- a/bin/varnishd/cache/cache_cli.c
+++ b/bin/varnishd/cache/cache_cli.c
@@ -214,7 +214,7 @@ ccf_panic(struct cli *cli, const char * const *av, void *priv)
 	(void)cli;
 	(void)av;
 	(void)priv;
-	assert(!strcmp("", "You asked for it"));
+	AZ(strcmp("", "You asked for it"));
 }
 
 /*--------------------------------------------------------------------*/
diff --git a/bin/varnishd/cache/cache_esi_deliver.c b/bin/varnishd/cache/cache_esi_deliver.c
index 5aa85ee..a246985 100644
--- a/bin/varnishd/cache/cache_esi_deliver.c
+++ b/bin/varnishd/cache/cache_esi_deliver.c
@@ -302,7 +302,7 @@ ESI_Deliver(struct req *req)
 		i = VGZ_Gunzip(vgz, &dp, &dl);
 		assert(i == VGZ_OK);
 		assert(VGZ_IbufEmpty(vgz));
-		assert(dl == 0);
+		AZ(dl);
 	}
 
 	st = VTAILQ_FIRST(&req->obj->store);
diff --git a/bin/varnishd/cache/cache_expire.c b/bin/varnishd/cache/cache_expire.c
index 8bd0707..b5ecfbe 100644
--- a/bin/varnishd/cache/cache_expire.c
+++ b/bin/varnishd/cache/cache_expire.c
@@ -98,7 +98,7 @@ exp_when(const struct object *o)
 	CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC);
 
 	when = o->exp.t_origin + o->exp.ttl + o->exp.grace + o->exp.keep;
-	assert(!isnan(when));
+	AZ(isnan(when));
 	return (when);
 }
 
diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c
index be664e5..5a0a9ff 100644
--- a/bin/varnishd/cache/cache_fetch.c
+++ b/bin/varnishd/cache/cache_fetch.c
@@ -801,7 +801,7 @@ vbf_fetch_thread(struct worker *wrk, void *priv)
 	http_Teardown(bo->beresp);
 
 	if (bo->state == BOS_FINISHED) {
-		assert(!(bo->fetch_objcore->flags & OC_F_FAILED));
+		AZ(bo->fetch_objcore->flags & OC_F_FAILED);
 		HSH_Complete(bo->fetch_objcore);
 		VSLb(bo->vsl, SLT_Length, "%zd", bo->fetch_obj->len);
 		{
diff --git a/bin/varnishd/cache/cache_fetch_proc.c b/bin/varnishd/cache/cache_fetch_proc.c
index 8861e1d..429052d 100644
--- a/bin/varnishd/cache/cache_fetch_proc.c
+++ b/bin/varnishd/cache/cache_fetch_proc.c
@@ -230,7 +230,7 @@ VFP_Fetch_Body(struct busyobj *bo, ssize_t est)
 		AZ(bo->failed);
 		vfps = VFP_Suck(bo, st->ptr + st->len, &l);
 		if (l > 0 && vfps != VFP_ERROR) {
-			assert(!VTAILQ_EMPTY(&bo->fetch_obj->store));
+			AZ(VTAILQ_EMPTY(&bo->fetch_obj->store));
 			VBO_extend(bo, l);
 		}
 		if (st->len == st->space)
diff --git a/bin/varnishd/cache/cache_http1_fsm.c b/bin/varnishd/cache/cache_http1_fsm.c
index 9bb5e79..3702827 100644
--- a/bin/varnishd/cache/cache_http1_fsm.c
+++ b/bin/varnishd/cache/cache_http1_fsm.c
@@ -103,7 +103,7 @@ http1_wait(struct sess *sp, struct worker *wrk, struct req *req)
 	AZ(req->vcl);
 	AZ(req->obj);
 	AZ(req->esi_level);
-	assert(!isnan(sp->t_idle));
+	AZ(isnan(sp->t_idle));
 	assert(isnan(req->t_first));
 	assert(isnan(req->t_prev));
 	assert(isnan(req->t_req));
diff --git a/bin/varnishd/cache/cache_lck.c b/bin/varnishd/cache/cache_lck.c
index 54203af..288d452 100644
--- a/bin/varnishd/cache/cache_lck.c
+++ b/bin/varnishd/cache/cache_lck.c
@@ -134,8 +134,8 @@ Lck__Assert(const struct lock *lck, int held)
 		assert(ilck->held);
 		assert(pthread_equal(ilck->owner, pthread_self()));
 	} else {
-		assert(!ilck->held);
-		assert(!pthread_equal(ilck->owner, pthread_self()));
+		AZ(ilck->held);
+		AZ(pthread_equal(ilck->owner, pthread_self()));
 	}
 }
 
diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c
index b6d0500..2518422 100644
--- a/bin/varnishd/cache/cache_req_fsm.c
+++ b/bin/varnishd/cache/cache_req_fsm.c
@@ -690,9 +690,9 @@ cnt_recv(struct worker *wrk, struct req *req)
 	AZ(req->obj);
 	AZ(req->objcore);
 
-	assert(!isnan(req->t_first));
-	assert(!isnan(req->t_prev));
-	assert(!isnan(req->t_req));
+	AZ(isnan(req->t_first));
+	AZ(isnan(req->t_prev));
+	AZ(isnan(req->t_req));
 
 	VSLb(req->vsl, SLT_ReqStart, "%s %s",
 	    req->sp->client_addr_str, req->sp->client_port_str);
diff --git a/bin/varnishd/cache/cache_session.c b/bin/varnishd/cache/cache_session.c
index 0c010c1..c938b98 100644
--- a/bin/varnishd/cache/cache_session.c
+++ b/bin/varnishd/cache/cache_session.c
@@ -303,7 +303,7 @@ SES_Delete(struct sess *sp, enum sess_close reason, double now)
 
 	if (isnan(now))
 		now = VTIM_real();
-	assert(!isnan(sp->t_open));
+	AZ(isnan(sp->t_open));
 
 	VSL(SLT_SessClose, sp->vxid, "%s %.3f",
 	    sess_close_2str(sp->reason, 0), now - sp->t_open);
diff --git a/bin/varnishd/cache/cache_shmlog.c b/bin/varnishd/cache/cache_shmlog.c
index af78bfa..d0733d7 100644
--- a/bin/varnishd/cache/cache_shmlog.c
+++ b/bin/varnishd/cache/cache_shmlog.c
@@ -78,7 +78,7 @@ static inline uint32_t *
 vsl_hdr(enum VSL_tag_e tag, uint32_t *p, unsigned len, uint32_t vxid)
 {
 
-	assert(((uintptr_t)p & 0x3) == 0);
+	AZ((uintptr_t)p & 0x3);
 	assert(tag > SLT__Bogus);
 	assert(tag < SLT__Reserved);
 	AZ(len & ~VSL_LENMASK);
@@ -133,7 +133,7 @@ vsl_get(unsigned len, unsigned records, unsigned flushes)
 		AZ(err);
 	}
 	assert(vsl_ptr < vsl_end);
-	assert(((uintptr_t)vsl_ptr & 0x3) == 0);
+	AZ((uintptr_t)vsl_ptr & 0x3);
 
 	VSC_C_main->shm_writes++;
 	VSC_C_main->shm_flushes += flushes;
@@ -146,7 +146,7 @@ vsl_get(unsigned len, unsigned records, unsigned flushes)
 	p = vsl_ptr;
 	vsl_ptr = VSL_END(vsl_ptr, len);
 	assert(vsl_ptr < vsl_end);
-	assert(((uintptr_t)vsl_ptr & 0x3) == 0);
+	AZ((uintptr_t)vsl_ptr & 0x3);
 
 	*vsl_ptr = VSL_ENDMARKER;
 
diff --git a/bin/varnishd/cache/cache_vcl.c b/bin/varnishd/cache/cache_vcl.c
index 0018095..e6dd179 100644
--- a/bin/varnishd/cache/cache_vcl.c
+++ b/bin/varnishd/cache/cache_vcl.c
@@ -250,7 +250,7 @@ VCL_Nuke(struct vcls *vcl)
 	ASSERT_CLI();
 	assert(vcl != vcl_active);
 	assert(vcl->conf->discard);
-	assert(vcl->conf->busy == 0);
+	AZ(vcl->conf->busy);
 	VTAILQ_REMOVE(&vcl_head, vcl, list);
 	ctx.method = VCL_MET_FINI;
 	ctx.handling = &hand;
@@ -438,7 +438,7 @@ VCL_##func##_method(struct VCL_conf *vcl, struct worker *wrk,		\
 	vcl_call_method(wrk, req, bo, ws, VCL_MET_ ## upper,		\
 	    vcl->func##_func);						\
 	assert((1U << wrk->handling) & bitmap);				\
-	assert(!((1U << wrk->handling) & ~bitmap));			\
+	AZ((1U << wrk->handling) & ~bitmap);			\
 }
 
 #include "tbl/vcl_returns.h"
diff --git a/bin/varnishd/cache/cache_wrw.c b/bin/varnishd/cache/cache_wrw.c
index a418ea0..09c7e26 100644
--- a/bin/varnishd/cache/cache_wrw.c
+++ b/bin/varnishd/cache/cache_wrw.c
@@ -137,7 +137,7 @@ wrw_prune(struct wrw *wrw, ssize_t bytes)
 		}
 		used += wrw->iov[j].iov_len;
 	}
-	assert(wrw->liov == 0);
+	AZ(wrw->liov);
 }
 
 unsigned
diff --git a/bin/varnishd/hash/hash_critbit.c b/bin/varnishd/hash/hash_critbit.c
index 46f7f95..4e14bdd 100644
--- a/bin/varnishd/hash/hash_critbit.c
+++ b/bin/varnishd/hash/hash_critbit.c
@@ -73,7 +73,7 @@ hcb_build_bittbl(void)
 
 	/* Quick asserts for sanity check */
 	assert(hcb_bits(0x34, 0x34) == 8);
-	assert(hcb_bits(0xaa, 0x55) == 0);
+	AZ(hcb_bits(0xaa, 0x55));
 	assert(hcb_bits(0x01, 0x22) == 2);
 	assert(hcb_bits(0x10, 0x0b) == 3);
 }
@@ -132,7 +132,7 @@ static uintptr_t
 hcb_r_node(struct objhead *n)
 {
 
-	assert(!((uintptr_t)n & (HCB_BIT_NODE|HCB_BIT_Y)));
+	AZ((uintptr_t)n & (HCB_BIT_NODE | HCB_BIT_Y));
 	return (HCB_BIT_NODE | (uintptr_t)n);
 }
 
@@ -141,7 +141,7 @@ hcb_l_node(uintptr_t u)
 {
 
 	assert(u & HCB_BIT_NODE);
-	assert(!(u & HCB_BIT_Y));
+	AZ(u & HCB_BIT_Y);
 	return ((struct objhead *)(u & ~HCB_BIT_NODE));
 }
 
@@ -150,7 +150,7 @@ hcb_r_y(struct hcb_y *y)
 {
 
 	CHECK_OBJ_NOTNULL(y, HCB_Y_MAGIC);
-	assert(!((uintptr_t)y & (HCB_BIT_NODE|HCB_BIT_Y)));
+	AZ((uintptr_t)y & (HCB_BIT_NODE | HCB_BIT_Y));
 	return (HCB_BIT_Y | (uintptr_t)y);
 }
 
@@ -158,7 +158,7 @@ static struct hcb_y *
 hcb_l_y(uintptr_t u)
 {
 
-	assert(!(u & HCB_BIT_NODE));
+	AZ(u & HCB_BIT_NODE);
 	assert(u & HCB_BIT_Y);
 	return ((struct hcb_y *)(u & ~HCB_BIT_Y));
 }
@@ -252,7 +252,7 @@ hcb_insert(struct worker *wrk, struct hcb_root *root, const uint8_t *digest,
 	s2 = 1-s2;
 
 	p = &root->origo;
-	assert(*p != 0);
+	AN(*p);
 
 	while(hcb_is_y(*p)) {
 		y = hcb_l_y(*p);
diff --git a/bin/varnishd/mgt/mgt_cli.c b/bin/varnishd/mgt/mgt_cli.c
index f977adb..3d71a14 100644
--- a/bin/varnishd/mgt/mgt_cli.c
+++ b/bin/varnishd/mgt/mgt_cli.c
@@ -116,7 +116,7 @@ mcf_panic(struct cli *cli, const char * const *av, void *priv)
 	(void)cli;
 	(void)av;
 	(void)priv;
-	assert(!strcmp("", "You asked for it"));
+	AZ(strcmp("", "You asked for it"));
 }
 
 static struct cli_proto cli_debug[] = {
diff --git a/bin/varnishd/mgt/mgt_main.c b/bin/varnishd/mgt/mgt_main.c
index 5dc602e..76b702b 100644
--- a/bin/varnishd/mgt/mgt_main.c
+++ b/bin/varnishd/mgt/mgt_main.c
@@ -300,7 +300,7 @@ cli_stdin_close(void *priv)
 	(void)close(0);
 	(void)close(1);
 	(void)close(2);
-	assert(open("/dev/null", O_RDONLY) == 0);
+	AZ(open("/dev/null", O_RDONLY));
 	assert(open("/dev/null", O_WRONLY) == 1);
 	assert(open("/dev/null", O_WRONLY) == 2);
 
diff --git a/bin/varnishd/storage/stevedore.c b/bin/varnishd/storage/stevedore.c
index 30ae453..5e54ed6 100644
--- a/bin/varnishd/storage/stevedore.c
+++ b/bin/varnishd/storage/stevedore.c
@@ -183,8 +183,7 @@ stv_alloc(struct stevedore *stv, size_t size)
 
 		size >>= 1;
 	}
-	if (st != NULL)
-		CHECK_OBJ_NOTNULL(st, STORAGE_MAGIC);
+	CHECK_OBJ_ORNULL(st, STORAGE_MAGIC);
 	return (st);
 }
 
@@ -226,8 +225,7 @@ stv_alloc_obj(struct busyobj *bo, size_t size)
 		    EXP_NukeOne(bo, stv->lru) == -1)
 			break;
 	}
-	if (st != NULL)
-		CHECK_OBJ_NOTNULL(st, STORAGE_MAGIC);
+	CHECK_OBJ_ORNULL(st, STORAGE_MAGIC);
 	return (st);
 }
 
diff --git a/bin/varnishd/storage/stevedore_utils.c b/bin/varnishd/storage/stevedore_utils.c
index 081b93a..a86d57e 100644
--- a/bin/varnishd/storage/stevedore_utils.c
+++ b/bin/varnishd/storage/stevedore_utils.c
@@ -159,7 +159,7 @@ stv_fsspace(int fd, unsigned *bs)
 	/* We use units of the larger of filesystem blocksize and pagesize */
 	if (*bs < bsize)
 		*bs = bsize;
-	xxxassert(*bs % bsize == 0);
+	XXXAZ(*bs % bsize);
 	return (bsize * bavail);
 }
 
@@ -188,7 +188,7 @@ STV_FileSize(int fd, const char *size, unsigned *granularity, const char *ctx)
 
 	bs = *granularity;
 	fssize = stv_fsspace(fd, &bs);
-	xxxassert(bs % *granularity == 0);
+	XXXAZ(bs % *granularity);
 
 	if ((size == NULL || *size == '\0') && st.st_size != 0) {
 		/*
diff --git a/bin/varnishd/storage/storage_file.c b/bin/varnishd/storage/storage_file.c
index ac0b6c4..c48200c 100644
--- a/bin/varnishd/storage/storage_file.c
+++ b/bin/varnishd/storage/storage_file.c
@@ -168,7 +168,7 @@ insfree(struct smf_sc *sc, struct smf *sp)
 	struct smf *sp2;
 	size_t ns;
 
-	assert(sp->alloc == 0);
+	AZ(sp->alloc);
 	assert(sp->flist == NULL);
 	Lck_AssertHeld(&sc->mtx);
 	b = sp->size / sc->pagesize;
@@ -182,7 +182,7 @@ insfree(struct smf_sc *sc, struct smf *sp)
 	ns = b * sc->pagesize;
 	VTAILQ_FOREACH(sp2, sp->flist, status) {
 		assert(sp2->size >= ns);
-		assert(sp2->alloc == 0);
+		AZ(sp2->alloc);
 		assert(sp2->flist == sp->flist);
 		if (sp->offset < sp2->offset)
 			break;
@@ -198,7 +198,7 @@ remfree(const struct smf_sc *sc, struct smf *sp)
 {
 	size_t b;
 
-	assert(sp->alloc == 0);
+	AZ(sp->alloc);
 	assert(sp->flist != NULL);
 	Lck_AssertHeld(&sc->mtx);
 	b = sp->size / sc->pagesize;
@@ -223,7 +223,7 @@ alloc_smf(struct smf_sc *sc, size_t bytes)
 	struct smf *sp, *sp2;
 	size_t b;
 
-	assert(!(bytes % sc->pagesize));
+	AZ(bytes % sc->pagesize);
 	b = bytes / sc->pagesize;
 	if (b >= NBUCKET)
 		b = NBUCKET - 1;
@@ -280,9 +280,9 @@ free_smf(struct smf *sp)
 	struct smf_sc *sc = sp->sc;
 
 	CHECK_OBJ_NOTNULL(sp, SMF_MAGIC);
-	assert(sp->alloc != 0);
+	AN(sp->alloc);
 	assert(sp->size > 0);
-	assert(!(sp->size % sc->pagesize));
+	AZ(sp->size % sc->pagesize);
 	VTAILQ_REMOVE(&sc->used, sp, status);
 	sp->alloc = 0;
 
@@ -324,11 +324,11 @@ trim_smf(struct smf *sp, size_t bytes)
 	struct smf *sp2;
 	struct smf_sc *sc = sp->sc;
 
-	assert(sp->alloc != 0);
+	AN(sp->alloc);
 	assert(bytes > 0);
 	assert(bytes < sp->size);
-	assert(!(bytes % sc->pagesize));
-	assert(!(sp->size % sc->pagesize));
+	AZ(bytes % sc->pagesize);
+	AZ(sp->size % sc->pagesize);
 	CHECK_OBJ_NOTNULL(sp, SMF_MAGIC);
 	sp2 = malloc(sizeof *sp2);
 	XXXAN(sp2);
@@ -353,7 +353,7 @@ new_smf(struct smf_sc *sc, unsigned char *ptr, off_t off, size_t len)
 {
 	struct smf *sp, *sp2;
 
-	assert(!(len % sc->pagesize));
+	AZ(len % sc->pagesize);
 	sp = calloc(sizeof *sp, 1);
 	XXXAN(sp);
 	sp->magic = SMF_MAGIC;
@@ -395,8 +395,8 @@ smf_open_chunk(struct smf_sc *sc, off_t sz, off_t off, off_t *fail, off_t *sum)
 	void *p;
 	off_t h;
 
-	assert(sz != 0);
-	assert(!(sz % sc->pagesize));
+	AN(sz);
+	AZ(sz % sc->pagesize);
 
 	if (*fail < (uintmax_t)sc->pagesize * MINPAGES)
 		return;
diff --git a/bin/varnishd/waiter/cache_waiter_epoll.c b/bin/varnishd/waiter/cache_waiter_epoll.c
index ba41210..ea875e7 100644
--- a/bin/varnishd/waiter/cache_waiter_epoll.c
+++ b/bin/varnishd/waiter/cache_waiter_epoll.c
@@ -124,7 +124,7 @@ vwe_eev(struct vwe *vwe, const struct epoll_event *ep, double now)
 				j++;
 				i -= sizeof ss[0];
 			}
-			assert(i == 0);
+			AZ(i);
 		}
 	} else {
 		CAST_OBJ_NOTNULL(sp, ep->data.ptr, SESS_MAGIC);
diff --git a/bin/varnishd/waiter/cache_waiter_kqueue.c b/bin/varnishd/waiter/cache_waiter_kqueue.c
index fb82e99..68f7fba 100644
--- a/bin/varnishd/waiter/cache_waiter_kqueue.c
+++ b/bin/varnishd/waiter/cache_waiter_kqueue.c
@@ -70,7 +70,7 @@ vwk_kq_flush(struct vwk *vwk)
 	if (vwk->nki == 0)
 		return;
 	i = kevent(vwk->kq, vwk->ki, vwk->nki, NULL, 0, NULL);
-	assert(i == 0);
+	AZ(i);
 	vwk->nki = 0;
 }
 
@@ -108,7 +108,7 @@ vwk_pipe_ev(struct vwk *vwk, const struct kevent *kp)
 		j++;
 		i -= sizeof ss[0];
 	}
-	assert(i == 0);
+	AZ(i);
 }
 
 /*--------------------------------------------------------------------*/
diff --git a/bin/varnishd/waiter/cache_waiter_poll.c b/bin/varnishd/waiter/cache_waiter_poll.c
index d2c29d1..de8fd58 100644
--- a/bin/varnishd/waiter/cache_waiter_poll.c
+++ b/bin/varnishd/waiter/cache_waiter_poll.c
@@ -94,8 +94,8 @@ vwp_poll(struct vwp *vwp, int fd)
 		vwp->hpoll = fd;
 
 	assert(vwp->pollfd[fd].fd == -1);
-	assert(vwp->pollfd[fd].events == 0);
-	assert(vwp->pollfd[fd].revents == 0);
+	AZ(vwp->pollfd[fd].events);
+	AZ(vwp->pollfd[fd].revents);
 
 	vwp->pollfd[fd].fd = fd;
 	vwp->pollfd[fd].events = POLLIN;
@@ -111,7 +111,7 @@ vwp_unpoll(struct vwp *vwp, int fd)
 
 	assert(vwp->pollfd[fd].fd == fd);
 	assert(vwp->pollfd[fd].events == POLLIN);
-	assert(vwp->pollfd[fd].revents == 0);
+	AZ(vwp->pollfd[fd].revents);
 
 	vwp->pollfd[fd].fd = -1;
 	vwp->pollfd[fd].events = 0;
@@ -177,7 +177,7 @@ vwp_main(void *priv)
 			v2--;
 			i = read(vwp->pipes[0], ss, sizeof ss);
 			assert(i >= 0);
-			assert(((unsigned)i % sizeof ss[0]) == 0);
+			AZ((unsigned)i % sizeof ss[0]);
 			for (j = 0; j * sizeof ss[0] < i; j++) {
 				CHECK_OBJ_NOTNULL(ss[j], SESS_MAGIC);
 				assert(ss[j]->fd >= 0);
@@ -185,7 +185,7 @@ vwp_main(void *priv)
 				vwp_poll(vwp, ss[j]->fd);
 			}
 		}
-		assert(v2 == 0);
+		AZ(v2);
 	}
 	NEEDLESS_RETURN(NULL);
 }
diff --git a/bin/varnishreplay/varnishreplay.c b/bin/varnishreplay/varnishreplay.c
index d8f2877..7fc24fe 100644
--- a/bin/varnishreplay/varnishreplay.c
+++ b/bin/varnishreplay/varnishreplay.c
@@ -246,7 +246,7 @@ static struct replay_thread *
 thread_get(int fd, void *(*thread_main)(void *))
 {
 
-	assert(fd != 0);
+	AN(fd);
 	if (fd >= nthreads) {
 		struct replay_thread **newthreads = threads;
 		size_t newnthreads = nthreads;
diff --git a/bin/varnishstat/varnishstat.c b/bin/varnishstat/varnishstat.c
index 7b37f0b..6fa7cc7 100644
--- a/bin/varnishstat/varnishstat.c
+++ b/bin/varnishstat/varnishstat.c
@@ -55,7 +55,7 @@ do_xml_cb(void *priv, const struct VSC_point * const pt)
 	(void)priv;
 	if (pt == NULL)
 		return (0);
-	assert(!strcmp(pt->desc->fmt, "uint64_t"));
+	AZ(strcmp(pt->desc->fmt, "uint64_t"));
 	val = *(const volatile uint64_t*)pt->ptr;
 	sec = pt->section;
 
@@ -100,7 +100,7 @@ do_json_cb(void *priv, const struct VSC_point * const pt)
 		return (0);
 
 	jp = priv;
-	assert(!strcmp(pt->desc->fmt, "uint64_t"));
+	AZ(strcmp(pt->desc->fmt, "uint64_t"));
 	val = *(const volatile uint64_t*)pt->ptr;
 	sec = pt->section;
 
@@ -167,7 +167,7 @@ do_once_cb(void *priv, const struct VSC_point * const pt)
 	if (pt == NULL)
 		return (0);
 	op = priv;
-	assert(!strcmp(pt->desc->fmt, "uint64_t"));
+	AZ(strcmp(pt->desc->fmt, "uint64_t"));
 	val = *(const volatile uint64_t*)pt->ptr;
 	sec = pt->section;
 	i = 0;
diff --git a/bin/varnishstat/varnishstat_curses.c b/bin/varnishstat/varnishstat_curses.c
index 356a92b..373cf09 100644
--- a/bin/varnishstat/varnishstat_curses.c
+++ b/bin/varnishstat/varnishstat_curses.c
@@ -264,7 +264,7 @@ build_pt_list_cb(void *priv, const struct VSC_point *vpt)
 
 	CAST_OBJ_NOTNULL(pt_priv, priv, PT_PRIV_MAGIC);
 
-	assert(!strcmp(vpt->desc->fmt, "uint64_t"));
+	AZ(strcmp(vpt->desc->fmt, "uint64_t"));
 	snprintf(buf, sizeof buf, "%s.%s.%s", vpt->section->type,
 	    vpt->section->ident, vpt->desc->name);
 	buf[sizeof buf - 1] = '\0';
diff --git a/bin/varnishtest/vtc.c b/bin/varnishtest/vtc.c
index 470c931..a31c83f 100644
--- a/bin/varnishtest/vtc.c
+++ b/bin/varnishtest/vtc.c
@@ -390,7 +390,7 @@ cmd_varnishtest(CMD_ARGS)
 
 	if (av == NULL)
 		return;
-	assert(!strcmp(av[0], "varnishtest"));
+	AZ(strcmp(av[0], "varnishtest"));
 
 	vtc_log(vl, 1, "TEST %s", av[1]);
 	AZ(av[2]);
@@ -414,7 +414,7 @@ cmd_shell(CMD_ARGS)
 	AZ(av[2]);
 	vtc_dump(vl, 4, "shell", av[1], -1);
 	r = system(av[1]);
-	assert(WEXITSTATUS(r) == 0);
+	AZ(WEXITSTATUS(r));
 }
 
 /**********************************************************************
diff --git a/bin/varnishtest/vtc_client.c b/bin/varnishtest/vtc_client.c
index 7eb430f..a273776 100644
--- a/bin/varnishtest/vtc_client.c
+++ b/bin/varnishtest/vtc_client.c
@@ -217,7 +217,7 @@ cmd_client(CMD_ARGS)
 		return;
 	}
 
-	assert(!strcmp(av[0], "client"));
+	AZ(strcmp(av[0], "client"));
 	av++;
 
 	VTAILQ_FOREACH(c, &clients, list)
diff --git a/bin/varnishtest/vtc_http.c b/bin/varnishtest/vtc_http.c
index 4388f73..17dba6b 100644
--- a/bin/varnishtest/vtc_http.c
+++ b/bin/varnishtest/vtc_http.c
@@ -225,7 +225,7 @@ cmd_http_expect(CMD_ARGS)
 	(void)cmd;
 	(void)vl;
 	CAST_OBJ_NOTNULL(hp, priv, HTTP_MAGIC);
-	assert(!strcmp(av[0], "expect"));
+	AZ(strcmp(av[0], "expect"));
 	av++;
 
 	AN(av[0]);
@@ -298,13 +298,13 @@ http_splitheader(struct http *hp, int req)
 	hh[n++] = p;
 	while (!vct_islws(*p))
 		p++;
-	assert(!vct_iscrlf(p));
+	AZ(vct_iscrlf(p));
 	*p++ = '\0';
 
 	/* URL/STATUS */
 	while (vct_issp(*p))		/* XXX: H space only */
 		p++;
-	assert(!vct_iscrlf(p));
+	AZ(vct_iscrlf(p));
 	hh[n++] = p;
 	while (!vct_islws(*p))
 		p++;
@@ -524,7 +524,7 @@ cmd_http_rxresp(CMD_ARGS)
 	(void)vl;
 	CAST_OBJ_NOTNULL(hp, priv, HTTP_MAGIC);
 	ONLY_CLIENT(hp, av);
-	assert(!strcmp(av[0], "rxresp"));
+	AZ(strcmp(av[0], "rxresp"));
 	av++;
 
 	for(; *av != NULL; av++)
@@ -554,7 +554,7 @@ cmd_http_rxresphdrs(CMD_ARGS)
 	(void)vl;
 	CAST_OBJ_NOTNULL(hp, priv, HTTP_MAGIC);
 	ONLY_CLIENT(hp, av);
-	assert(!strcmp(av[0], "rxresphdrs"));
+	AZ(strcmp(av[0], "rxresphdrs"));
 	av++;
 
 	for(; *av != NULL; av++)
@@ -767,7 +767,7 @@ cmd_http_txresp(CMD_ARGS)
 	(void)vl;
 	CAST_OBJ_NOTNULL(hp, priv, HTTP_MAGIC);
 	ONLY_SERVER(hp, av);
-	assert(!strcmp(av[0], "txresp"));
+	AZ(strcmp(av[0], "txresp"));
 	av++;
 
 	VSB_clear(hp->vsb);
@@ -812,7 +812,7 @@ cmd_http_rxreq(CMD_ARGS)
 	(void)vl;
 	CAST_OBJ_NOTNULL(hp, priv, HTTP_MAGIC);
 	ONLY_SERVER(hp, av);
-	assert(!strcmp(av[0], "rxreq"));
+	AZ(strcmp(av[0], "rxreq"));
 	av++;
 
 	for(; *av != NULL; av++)
@@ -832,7 +832,7 @@ cmd_http_rxreqhdrs(CMD_ARGS)
 	(void)cmd;
 	(void)vl;
 	CAST_OBJ_NOTNULL(hp, priv, HTTP_MAGIC);
-	assert(!strcmp(av[0], "rxreqhdrs"));
+	AZ(strcmp(av[0], "rxreqhdrs"));
 	av++;
 
 	for(; *av != NULL; av++)
@@ -850,7 +850,7 @@ cmd_http_rxbody(CMD_ARGS)
 	(void)vl;
 	CAST_OBJ_NOTNULL(hp, priv, HTTP_MAGIC);
 	ONLY_SERVER(hp, av);
-	assert(!strcmp(av[0], "rxbody"));
+	AZ(strcmp(av[0], "rxbody"));
 	av++;
 
 	for(; *av != NULL; av++)
@@ -895,7 +895,7 @@ cmd_http_txreq(CMD_ARGS)
 	(void)vl;
 	CAST_OBJ_NOTNULL(hp, priv, HTTP_MAGIC);
 	ONLY_CLIENT(hp, av);
-	assert(!strcmp(av[0], "txreq"));
+	AZ(strcmp(av[0], "txreq"));
 	av++;
 
 	VSB_clear(hp->vsb);
diff --git a/bin/varnishtest/vtc_logexp.c b/bin/varnishtest/vtc_logexp.c
index e6adae7..48c8f3d 100644
--- a/bin/varnishtest/vtc_logexp.c
+++ b/bin/varnishtest/vtc_logexp.c
@@ -489,7 +489,7 @@ cmd_logexp(CMD_ARGS)
 		return;
 	}
 
-	assert(!strcmp(av[0], "logexpect"));
+	AZ(strcmp(av[0], "logexpect"));
 	av++;
 
 	VTAILQ_FOREACH(le, &logexps, list) {
diff --git a/bin/varnishtest/vtc_sema.c b/bin/varnishtest/vtc_sema.c
index fd4c561..2ebfdee 100644
--- a/bin/varnishtest/vtc_sema.c
+++ b/bin/varnishtest/vtc_sema.c
@@ -129,7 +129,7 @@ cmd_sema(CMD_ARGS)
 		return;
 	}
 
-	assert(!strcmp(av[0], "sema"));
+	AZ(strcmp(av[0], "sema"));
 	av++;
 
 	AZ(pthread_mutex_lock(&sema_mtx));
diff --git a/bin/varnishtest/vtc_server.c b/bin/varnishtest/vtc_server.c
index eaa680f..86cf293 100644
--- a/bin/varnishtest/vtc_server.c
+++ b/bin/varnishtest/vtc_server.c
@@ -251,7 +251,7 @@ cmd_server(CMD_ARGS)
 		return;
 	}
 
-	assert(!strcmp(av[0], "server"));
+	AZ(strcmp(av[0], "server"));
 	av++;
 
 	VTAILQ_FOREACH(s, &servers, list)
@@ -278,7 +278,7 @@ cmd_server(CMD_ARGS)
 		if (s->run)
 			server_wait(s);
 
-		assert(s->run == 0);
+		AZ(s->run);
 		if (!strcmp(*av, "-repeat")) {
 			s->repeat = atoi(av[1]);
 			av++;
diff --git a/bin/varnishtest/vtc_varnish.c b/bin/varnishtest/vtc_varnish.c
index 19f51dd..14b94be 100644
--- a/bin/varnishtest/vtc_varnish.c
+++ b/bin/varnishtest/vtc_varnish.c
@@ -104,7 +104,7 @@ varnish_ask_cli(const struct varnish *v, const char *cmd, char **repl)
 		    cmd, i, retval, r);
 		return ((enum VCLI_status_e)retval);
 	}
-	assert(i == 0);
+	AZ(i);
 	vtc_log(v->vl, 3, "CLI RX  %u", retval);
 	vtc_dump(v->vl, 4, "CLI RX", r, -1);
 	if (repl != NULL)
@@ -402,7 +402,7 @@ varnish_launch(struct varnish *v)
 	v->pid = fork();
 	assert(v->pid >= 0);
 	if (v->pid == 0) {
-		assert(dup2(v->fds[0], 0) == 0);
+		AZ(dup2(v->fds[0], 0));
 		assert(dup2(v->fds[3], 1) == 1);
 		assert(dup2(1, 2) == 2);
 		AZ(close(v->fds[0]));
@@ -754,7 +754,7 @@ do_stat_cb(void *priv, const struct VSC_point * const pt)
 	if (strcmp(pt->desc->name, p))
 		return (0);
 
-	assert(!strcmp(pt->desc->fmt, "uint64_t"));
+	AZ(strcmp(pt->desc->fmt, "uint64_t"));
 	sp->val = *(const volatile uint64_t*)pt->ptr;
 	return (1);
 }
@@ -843,7 +843,7 @@ cmd_varnish(CMD_ARGS)
 		return;
 	}
 
-	assert(!strcmp(av[0], "varnish"));
+	AZ(strcmp(av[0], "varnish"));
 	av++;
 
 	VTAILQ_FOREACH(v, &varnishes, list)
diff --git a/lib/libvarnish/binary_heap.c b/lib/libvarnish/binary_heap.c
index 33616b5..491974b 100644
--- a/lib/libvarnish/binary_heap.c
+++ b/lib/libvarnish/binary_heap.c
@@ -215,7 +215,7 @@ binheap_new(void *priv, binheap_cmp_t *cmp_f, binheap_update_t *update_f)
 
 	bh->page_size = (unsigned)getpagesize() / sizeof (void *);
 	bh->page_mask = bh->page_size - 1;
-	assert(!(bh->page_size & bh->page_mask));	/* power of two */
+	AZ(bh->page_size & bh->page_mask);	/* power of two */
 	for (u = 1; (1U << u) != bh->page_size; u++)
 		;
 	bh->page_shift = u;
@@ -350,7 +350,7 @@ chk(const struct binheap *bh)
 
 	for (u = 2; u < bh->next; u++) {
 		v = parent(bh, u);
-		assert(!bh->cmp(bh->priv, A(bh, u), A(bh, v)));
+		AZ(bh->cmp(bh->priv, A(bh, u), A(bh, v)));
 	}
 }
 #endif
@@ -603,7 +603,7 @@ main(int argc, char **argv)
 			v = random() % N;
 			if (ff[v] != NULL) {
 				CHECK_OBJ_NOTNULL(ff[v], FOO_MAGIC);
-				assert(ff[v]->idx != 0);
+				AN(ff[v]->idx);
 				if (ff[v]->key & 1) {
 					binheap_delete(bh, ff[v]->idx);
 					assert(ff[v]->idx == BINHEAP_NOIDX);
@@ -619,7 +619,7 @@ main(int argc, char **argv)
 				ff[v]->key = random() % R;
 				binheap_insert(bh, ff[v]);
 				CHECK_OBJ_NOTNULL(ff[v], FOO_MAGIC);
-				assert(ff[v]->idx != 0);
+				AN(ff[v]->idx);
 			}
 			if (0)
 				chk2(bh);
diff --git a/lib/libvarnish/cli_serve.c b/lib/libvarnish/cli_serve.c
index 3ea59d6..7b7d9d5 100644
--- a/lib/libvarnish/cli_serve.c
+++ b/lib/libvarnish/cli_serve.c
@@ -369,7 +369,7 @@ cls_vlu(void *priv, const char *p)
 		return (0);
 	} else {
 		AN(cfd->argv[cfd->last_idx]);
-		assert(!strcmp(cfd->argv[cfd->last_idx], "<<"));
+		AZ(strcmp(cfd->argv[cfd->last_idx], "<<"));
 		AN(cfd->argv[cfd->last_idx + 1]);
 		if (strcmp(p, cfd->argv[cfd->last_idx + 1])) {
 			VSB_cat(cfd->last_arg, p);
diff --git a/lib/libvarnish/vav.c b/lib/libvarnish/vav.c
index 24b16b7..715e43c 100644
--- a/lib/libvarnish/vav.c
+++ b/lib/libvarnish/vav.c
@@ -90,7 +90,7 @@ VAV_BackSlash(const char *s, char *res)
 		break;
 	case 'x':
 		if (1 == sscanf(s + 1, "x%02x", &u)) {
-			assert(!(u & ~0xff));
+			AZ(u & ~0xff);
 			c = u;	/*lint !e734 loss of precision */
 			r = 4;
 		}
diff --git a/lib/libvarnish/vev.c b/lib/libvarnish/vev.c
index 87d1582..6429e4b 100644
--- a/lib/libvarnish/vev.c
+++ b/lib/libvarnish/vev.c
@@ -257,7 +257,7 @@ vev_add(struct vev_base *evb, struct vev *e)
 		es = &vev_sigs[e->sig];
 		if (es->vev != NULL)
 			return (EBUSY);
-		assert(es->happened == 0);
+		AZ(es->happened);
 		es->vev = e;
 		es->vevb = evb;
 		es->sigact.sa_flags = e->sig_flags;
@@ -299,7 +299,7 @@ vev_add(struct vev_base *evb, struct vev *e)
 
 	if (e->sig > 0) {
 		assert(es != NULL);
-		assert(sigaction(e->sig, &es->sigact, NULL) == 0);
+		AZ(sigaction(e->sig, &es->sigact, NULL));
 	}
 
 	return (0);
@@ -320,7 +320,7 @@ vev_del(struct vev_base *evb, struct vev *e)
 
 	if (e->__binheap_idx != 0)
 		binheap_delete(evb->binheap, e->__binheap_idx);
-	assert(e->__binheap_idx == 0);
+	AZ(e->__binheap_idx);
 
 	if (e->fd >= 0) {
 		DBG(evb, "... pidx = %d\n", e->__poll_idx);
@@ -341,7 +341,7 @@ vev_del(struct vev_base *evb, struct vev *e)
 		es->vevb = NULL;
 		es->sigact.sa_flags = e->sig_flags;
 		es->sigact.sa_handler = SIG_DFL;
-		assert(sigaction(e->sig, &es->sigact, NULL) == 0);
+		AZ(sigaction(e->sig, &es->sigact, NULL));
 		es->happened = 0;
 	}
 
@@ -517,6 +517,6 @@ vev_schedule_one(struct vev_base *evb)
 			free(e);
 		}
 	}
-	assert(i == 0);
+	AZ(i);
 	return (1);
 }
diff --git a/lib/libvarnish/vfil.c b/lib/libvarnish/vfil.c
index 5a5f087..920a06f 100644
--- a/lib/libvarnish/vfil.c
+++ b/lib/libvarnish/vfil.c
@@ -87,7 +87,7 @@ VFIL_readfd(int fd, ssize_t *sz)
 	char *f;
 	int i;
 
-	assert(0 == fstat(fd, &st));
+	AZ(fstat(fd, &st));
 	if (!S_ISREG(st.st_mode))
 		return (NULL);
 	f = malloc(st.st_size + 1);
diff --git a/lib/libvarnish/vsha256.c b/lib/libvarnish/vsha256.c
index a2208cf..6642d9f 100644
--- a/lib/libvarnish/vsha256.c
+++ b/lib/libvarnish/vsha256.c
@@ -329,6 +329,6 @@ SHA256_Test(void)
 		SHA256_Init(&c);
 		SHA256_Update(&c, p->input, strlen(p->input));
 		SHA256_Final(o, &c);
-		assert(!memcmp(o, p->output, 32));
+		AZ(memcmp(o, p->output, 32));
 	}
 }
diff --git a/lib/libvarnish/vtim.c b/lib/libvarnish/vtim.c
index 42c1558..c0efdbd 100644
--- a/lib/libvarnish/vtim.c
+++ b/lib/libvarnish/vtim.c
@@ -73,12 +73,12 @@ VTIM_mono(void)
 #elif  HAVE_CLOCK_GETTIME
 	struct timespec ts;
 
-	assert(clock_gettime(CLOCK_MONOTONIC, &ts) == 0);
+	AZ(clock_gettime(CLOCK_MONOTONIC, &ts));
 	return (ts.tv_sec + 1e-9 * ts.tv_nsec);
 #else
 	struct timeval tv;
 
-	assert(gettimeofday(&tv, NULL) == 0);
+	AZ(gettimeofday(&tv, NULL));
 	return (tv.tv_sec + 1e-6 * tv.tv_usec);
 #endif
 }
@@ -89,12 +89,12 @@ VTIM_real(void)
 #ifdef HAVE_CLOCK_GETTIME
 	struct timespec ts;
 
-	assert(clock_gettime(CLOCK_REALTIME, &ts) == 0);
+	AZ(clock_gettime(CLOCK_REALTIME, &ts));
 	return (ts.tv_sec + 1e-9 * ts.tv_nsec);
 #else
 	struct timeval tv;
 
-	assert(gettimeofday(&tv, NULL) == 0);
+	AZ(gettimeofday(&tv, NULL));
 	return (tv.tv_sec + 1e-6 * tv.tv_usec);
 #endif
 }
@@ -150,7 +150,7 @@ VTIM_parse(const char *p)
 			 * to set the timezone to UTC.  We check that.
 			 */
 			t = mktime(&tm);
-			assert(!strcmp(tzname[0], "UTC"));
+			AZ(strcmp(tzname[0], "UTC"));
 #endif
 			return (t);
 		}
diff --git a/lib/libvcc/vcc_storage.c b/lib/libvcc/vcc_storage.c
index 0ab3f64..89e63b0 100644
--- a/lib/libvcc/vcc_storage.c
+++ b/lib/libvcc/vcc_storage.c
@@ -109,7 +109,7 @@ vcc_Stv_Wildcard(struct vcc *tl, const struct token *t,
 
 	(void)wcsym;
 	assert((t->e - t->b) > strlen(PFX));
-	assert(!memcmp(t->b, PFX, strlen(PFX)));
+	AZ(memcmp(t->b, PFX, strlen(PFX)));
 
 	p = t->b + strlen(PFX);
 	for (q = p; q < t->e && *q != '.'; q++)



From phk at FreeBSD.org  Tue Jun 24 09:31:37 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 24 Jun 2014 11:31:37 +0200
Subject: [4.0] 3599490 One may have gcc or other things restricted,
 so that e.g. only a specific user and/or group may execute it.
 Varnish never inherited the groups of the user that has been specified by
 "-u". initgroups() will make sure that varnish gets all supplementary groups.
Message-ID: <E1WzN4j-00033a-TW@project.varnish-software.com>


commit 3599490aed5524ea19a63cf488a60dc8ddb59365
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Apr 22 08:51:58 2014 +0000

    One may have gcc or other things restricted, so that e.g. only a
    specific user and/or group may execute it. Varnish never inherited
    the groups of the user that has been specified by "-u". initgroups()
    will make sure that varnish gets all supplementary groups.
    
    Submitted by:	Christian Ruppert
    Fixes	#1482

diff --git a/bin/varnishd/mgt/mgt_sandbox.c b/bin/varnishd/mgt/mgt_sandbox.c
index 069d660..83a2aeb 100644
--- a/bin/varnishd/mgt/mgt_sandbox.c
+++ b/bin/varnishd/mgt/mgt_sandbox.c
@@ -48,6 +48,7 @@
 #include <sys/prctl.h>
 #endif
 
+#include <grp.h>
 #include <stdio.h>
 #include <syslog.h>
 #include <unistd.h>
@@ -64,6 +65,7 @@ mgt_sandbox_unix(enum sandbox_e who)
 	(void)who;
 	if (geteuid() == 0) {
 		XXXAZ(setgid(mgt_param.gid));
+		XXXAZ(initgroups(mgt_param.user, mgt_param.gid));
 		XXXAZ(setuid(mgt_param.uid));
 	} else {
 		REPORT0(LOG_INFO, "Not running as root, no priv-sep");



From perbu at varnish-software.com  Tue Jun 24 09:31:38 2014
From: perbu at varnish-software.com (Per Buer)
Date: Tue, 24 Jun 2014 11:31:38 +0200
Subject: [4.0] 35308a3 Missing paren. Spotted by Alexander Kuznetcov
Message-ID: <E1WzN4k-00033j-2y@project.varnish-software.com>


commit 35308a3c975ce3bd3690fe3afa4f25bb78b887e5
Author: Per Buer <perbu at varnish-software.com>
Date:   Tue Apr 22 18:10:50 2014 +0200

    Missing paren. Spotted by Alexander Kuznetcov

diff --git a/doc/sphinx/users-guide/purging.rst b/doc/sphinx/users-guide/purging.rst
index fcd5f5b..0c9b7c9 100644
--- a/doc/sphinx/users-guide/purging.rst
+++ b/doc/sphinx/users-guide/purging.rst
@@ -37,7 +37,7 @@ following VCL in place::
 
 	  if (req.method == "PURGE") {
 		  if (!client.ip ~ purge) {
-			  return(synth(405,"Not allowed.");
+			  return(synth(405,"Not allowed."));
 		  }
                   # jump to hit/miss
 		  return (purge);



From daghf at varnish-software.com  Tue Jun 24 09:31:38 2014
From: daghf at varnish-software.com (Dag Haavi Finstad)
Date: Tue, 24 Jun 2014 11:31:38 +0200
Subject: [4.0] 26546ec varnishtest txresp reason phrase 'Ok' -> 'OK'
Message-ID: <E1WzN4k-00033s-6m@project.varnish-software.com>


commit 26546ec34d101ce0e8bd3ffbeb51e7af5af77f4b
Author: Dag Haavi Finstad <daghf at varnish-software.com>
Date:   Wed Apr 23 16:39:34 2014 +0200

    varnishtest txresp reason phrase 'Ok' -> 'OK'

diff --git a/bin/varnishtest/tests/a00001.vtc b/bin/varnishtest/tests/a00001.vtc
index 04106fe..d842287 100644
--- a/bin/varnishtest/tests/a00001.vtc
+++ b/bin/varnishtest/tests/a00001.vtc
@@ -15,7 +15,7 @@ client c1 -connect ${s1_sock} {
 	rxresp
 	expect resp.proto == HTTP/1.1
 	expect resp.status == 200
-	expect resp.msg == Ok
+	expect resp.msg == OK
 }
 
 client c1 -run
diff --git a/bin/varnishtest/tests/a00003.vtc b/bin/varnishtest/tests/a00003.vtc
index f928a42..810da48 100644
--- a/bin/varnishtest/tests/a00003.vtc
+++ b/bin/varnishtest/tests/a00003.vtc
@@ -32,7 +32,7 @@ client c2 -connect ${s2_sock} {
 	rxresp
 	expect resp.proto == HTTP/1.1
 	expect resp.status == 200
-	expect resp.msg == Ok
+	expect resp.msg == OK
 }
 
 client c1 -start
diff --git a/bin/varnishtest/tests/a00005.vtc b/bin/varnishtest/tests/a00005.vtc
index fcf4e58..6c0d634 100644
--- a/bin/varnishtest/tests/a00005.vtc
+++ b/bin/varnishtest/tests/a00005.vtc
@@ -34,7 +34,7 @@ client c1 -connect ${s2_sock} {
 	rxresp
 	expect resp.proto == HTTP/1.1
 	expect resp.status == 200
-	expect resp.msg == Ok
+	expect resp.msg == OK
 }
 
 client c1 -run
diff --git a/bin/varnishtest/tests/b00007.vtc b/bin/varnishtest/tests/b00007.vtc
index a48fa3f..02babe7 100644
--- a/bin/varnishtest/tests/b00007.vtc
+++ b/bin/varnishtest/tests/b00007.vtc
@@ -3,7 +3,7 @@ varnishtest "Check chunked encoding from backend works"
 server s1 {
 	rxreq
 	expect req.url == "/bar"
-	send "HTTP/1.1 200 Ok\r\n"
+	send "HTTP/1.1 200 OK\r\n"
 	send "Transfer-encoding: chunked\r\n"
 	send "\r\n"
 	send "00000004\r\n1234\r\n"
@@ -12,7 +12,7 @@ server s1 {
 
 	rxreq
 	expect req.url == "/foo"
-	send "HTTP/1.1 200 Ok\r\n"
+	send "HTTP/1.1 200 OK\r\n"
 	send "Transfer-encoding: chunked\r\n"
 	send "\r\n"
 	send "00000004\r\n1234\r\n"
diff --git a/bin/varnishtest/tests/b00011.vtc b/bin/varnishtest/tests/b00011.vtc
index bbd8cb3..82381a8 100644
--- a/bin/varnishtest/tests/b00011.vtc
+++ b/bin/varnishtest/tests/b00011.vtc
@@ -2,7 +2,7 @@ varnishtest "Check HTTP/1.0 EOF transmission"
 
 server s1 {
 	rxreq
-	send "HTTP/1.1 200 Ok\n"
+	send "HTTP/1.1 200 OK\n"
 	send "Connection: close\n"
 	send "\n"
 	send "Body line 1\n"
diff --git a/bin/varnishtest/tests/b00020.vtc b/bin/varnishtest/tests/b00020.vtc
index 20bc72b..1448ed5 100644
--- a/bin/varnishtest/tests/b00020.vtc
+++ b/bin/varnishtest/tests/b00020.vtc
@@ -4,7 +4,7 @@ feature SO_RCVTIMEO_WORKS
 
 server s1 {
 	rxreq
-	send "HTTP/1.1 200 Ok\r\nConnection: close\r\n\r\n"
+	send "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n"
 	delay 1.5
 	# send "Baba\n"
 } -start
@@ -29,7 +29,7 @@ varnish v1 -expect n_objectcore == 0
 
 server s1 -wait {
 	rxreq
-	send "HTTP/1.1 200 Ok\r\nConnection: close\r\n\r\n"
+	send "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n"
 	delay 0.5
 	send "Baba\n"
 	delay 0.5
diff --git a/bin/varnishtest/tests/b00021.vtc b/bin/varnishtest/tests/b00021.vtc
index 5646183..3d4e241 100644
--- a/bin/varnishtest/tests/b00021.vtc
+++ b/bin/varnishtest/tests/b00021.vtc
@@ -4,7 +4,7 @@ feature SO_RCVTIMEO_WORKS
 
 server s1 {
 	rxreq
-	send "HTTP/1.1 200 Ok\r\nConnection: close\r\n\r\n"
+	send "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n"
 	delay 4.0
 	send "Baba\n"
 } -start
@@ -26,7 +26,7 @@ client c1 {
 
 server s1 {
 	rxreq
-	send "HTTP/1.1 200 Ok\r\nConnection: close\r\n\r\n"
+	send "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n"
 	delay 1.0
 	send "Baba\n"
 	delay 1.0
diff --git a/bin/varnishtest/tests/b00022.vtc b/bin/varnishtest/tests/b00022.vtc
index 2201586..c232408 100644
--- a/bin/varnishtest/tests/b00022.vtc
+++ b/bin/varnishtest/tests/b00022.vtc
@@ -4,7 +4,7 @@ feature SO_RCVTIMEO_WORKS
 
 server s1 {
 	rxreq
-	send "HTTP/1.1 200 Ok\r\nConnection: close\r\n\r\n"
+	send "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n"
 	delay 1.5
 	send "Baba\n"
 } -start
@@ -28,7 +28,7 @@ client c1 {
 
 server s1 {
 	rxreq
-	send "HTTP/1.1 200 Ok\r\nConnection: close\r\n\r\n"
+	send "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n"
 	delay 0.5
 	send "Baba\n"
 	delay 0.5
diff --git a/bin/varnishtest/tests/b00027.vtc b/bin/varnishtest/tests/b00027.vtc
index f1907c0..e067475 100644
--- a/bin/varnishtest/tests/b00027.vtc
+++ b/bin/varnishtest/tests/b00027.vtc
@@ -6,7 +6,7 @@ server s1 {
 	rxreq
 	txresp -proto HTTP/1.0 -hdr "Connection: keep-alive"
 	rxreq
-	send "HTTP/1.1 200 Ok\n"
+	send "HTTP/1.1 200 OK\n"
 	send "Transfer-encoding: foobar\n"
 	send "\n"
 } -start
diff --git a/bin/varnishtest/tests/c00013.vtc b/bin/varnishtest/tests/c00013.vtc
index b9578aa..081ed1f 100644
--- a/bin/varnishtest/tests/c00013.vtc
+++ b/bin/varnishtest/tests/c00013.vtc
@@ -3,7 +3,7 @@ varnishtest "Test parking second request on backend delay"
 server s1 {
 	rxreq
 	expect req.url == "/foo"
-	send "HTTP/1.1 200 Ok\r\nConnection: close\r\n\r\n"
+	send "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n"
 	delay .2
 	sema r1 sync 2
 	delay .2
diff --git a/bin/varnishtest/tests/c00014.vtc b/bin/varnishtest/tests/c00014.vtc
index fffbbd7..99f2dd7 100644
--- a/bin/varnishtest/tests/c00014.vtc
+++ b/bin/varnishtest/tests/c00014.vtc
@@ -4,7 +4,7 @@ server s1 {
 	rxreq
 	expect req.url == "/foo"
 	sema r1 sync 2
-	send "HTTP/1.1 200 Ok\r\nContent-Length: 12\r\n\r\n"
+	send "HTTP/1.1 200 OK\r\nContent-Length: 12\r\n\r\n"
 	send "line1\n"
 	send "line2\n"
 
diff --git a/bin/varnishtest/tests/c00066.vtc b/bin/varnishtest/tests/c00066.vtc
index eeb1ff9..18a9b67 100644
--- a/bin/varnishtest/tests/c00066.vtc
+++ b/bin/varnishtest/tests/c00066.vtc
@@ -43,7 +43,7 @@ client c1 {
 	txreq -url "/"
 	rxresp
 	expect resp.status == 200
-	expect resp.msg == "Ok"
+	expect resp.msg == "OK"
 
 	txreq -url "/synth"
 	rxresp
diff --git a/bin/varnishtest/tests/e00007.vtc b/bin/varnishtest/tests/e00007.vtc
index 519759c..a03ef55 100644
--- a/bin/varnishtest/tests/e00007.vtc
+++ b/bin/varnishtest/tests/e00007.vtc
@@ -16,7 +16,7 @@ varnishtest "ESI spanning storage bits"
 server s1 {
 	rxreq
 	expect req.url == "/foo/bar"
-	send "HTTP/1.1 200 Ok\n"
+	send "HTTP/1.1 200 OK\n"
 	send "Connection: close\n"
 	send "\n"
 	send {
diff --git a/bin/varnishtest/tests/e00014.vtc b/bin/varnishtest/tests/e00014.vtc
index 7021c95..ec6d16b 100644
--- a/bin/varnishtest/tests/e00014.vtc
+++ b/bin/varnishtest/tests/e00014.vtc
@@ -3,7 +3,7 @@ varnishtest "Check <esi: detector"
 server s1 {
         rxreq
         expect req.url == "/foo"
-	send "HTTP/1.1 200 Ok\n"
+	send "HTTP/1.1 200 OK\n"
         send "Connection: close\n"
 	send "\n"
         send {           <a>   <esi/>                          }
diff --git a/bin/varnishtest/tests/l00002.vtc b/bin/varnishtest/tests/l00002.vtc
index 8c7bc35..72faa08 100644
--- a/bin/varnishtest/tests/l00002.vtc
+++ b/bin/varnishtest/tests/l00002.vtc
@@ -28,7 +28,7 @@ varnish v1 -vcl+backend {
 # \r\n				  2 bytes
 # Total:			 38 bytes
 # Response:
-# HTTP/1.1 200 Ok\r\n		 17 bytes
+# HTTP/1.1 200 OK\r\n		 17 bytes
 # Content-Length: 1000\r\n	 22 bytes
 # Connection: keep-alive\r\n	 24 bytes
 # Accept-Ranges: bytes\r\n	 22 bytes
@@ -40,7 +40,7 @@ varnish v1 -vcl+backend {
 # \r\n				  2 bytes
 # Total:			 19 bytes
 # Response:
-# HTTP/1.1 200 Ok\r\n		 17 bytes
+# HTTP/1.1 200 OK\r\n		 17 bytes
 # Content-Length: 2000\r\n	 22 bytes
 # Connection: keep-alive\r\n	 24 bytes
 # Accept-Ranges: bytes\r\n	 22 bytes
@@ -52,7 +52,7 @@ varnish v1 -vcl+backend {
 # \r\n				  2 bytes
 # Total:			 19 bytes
 # Response:
-# HTTP/1.1 200 Ok\r\n		 17 bytes
+# HTTP/1.1 200 OK\r\n		 17 bytes
 # Content-Length: 2000\r\n	 22 bytes
 # Connection: keep-alive\r\n	 24 bytes
 # Accept-Ranges: bytes\r\n	 22 bytes
diff --git a/bin/varnishtest/tests/l00003.vtc b/bin/varnishtest/tests/l00003.vtc
index cb03a18..86c9e2a 100644
--- a/bin/varnishtest/tests/l00003.vtc
+++ b/bin/varnishtest/tests/l00003.vtc
@@ -32,7 +32,7 @@ varnish v1 -vcl+backend {
 # Total:				18 bytes
 
 # Reponse:
-# HTTP/1.1 200 Ok\r\n			17 bytes
+# HTTP/1.1 200 OK\r\n			17 bytes
 # Transfer-Encoding: chunked\r\n	28 bytes
 # Connection: keep-alive\r\n		24 bytes
 # \r\n					 2 bytes
diff --git a/bin/varnishtest/tests/l00004.vtc b/bin/varnishtest/tests/l00004.vtc
index 3b07e36..22e5ea3 100644
--- a/bin/varnishtest/tests/l00004.vtc
+++ b/bin/varnishtest/tests/l00004.vtc
@@ -37,7 +37,7 @@ varnish v1 -vcl+backend {
 # asdf					 4 bytes
 
 # resp:
-# HTTP/1.1 200 Ok\r\n			17 bytes
+# HTTP/1.1 200 OK\r\n			17 bytes
 # Content-Length: 4\r\n			19 bytes
 # \r\n					 2 bytes
 # fdsa					 4 bytes
diff --git a/bin/varnishtest/tests/l00005.vtc b/bin/varnishtest/tests/l00005.vtc
index dbbea1b..e7472ff 100644
--- a/bin/varnishtest/tests/l00005.vtc
+++ b/bin/varnishtest/tests/l00005.vtc
@@ -28,7 +28,7 @@ varnish v1 -vcl+backend {
 # \r\n				  2 bytes
 # Total:			 54 bytes
 # Response:
-# HTTP/1.1 200 Ok\r\n		 17 bytes
+# HTTP/1.1 200 OK\r\n		 17 bytes
 # Content-Length: 1000\r\n	 22 bytes
 # \r\n				  2 bytes
 # Total:			 41 bytes
diff --git a/bin/varnishtest/tests/p00007.vtc b/bin/varnishtest/tests/p00007.vtc
index ca6d04d..5fd9e9e 100644
--- a/bin/varnishtest/tests/p00007.vtc
+++ b/bin/varnishtest/tests/p00007.vtc
@@ -3,7 +3,7 @@ varnishtest "test reload of object spanning incomplete segment"
 server s1 {
 	rxreq
 	expect req.url == "/1"
-	send "HTTP/1.1 200 Ok\n"
+	send "HTTP/1.1 200 OK\n"
 	send "Transfer-encoding: chunked\n"
 	send "\n"
 	chunkedlen 32
diff --git a/bin/varnishtest/tests/r00387.vtc b/bin/varnishtest/tests/r00387.vtc
index 5edb18a..57ec219 100644
--- a/bin/varnishtest/tests/r00387.vtc
+++ b/bin/varnishtest/tests/r00387.vtc
@@ -3,7 +3,7 @@ varnishtest "Regression test for #387: too long chunk header"
 server s1 {
 	rxreq
 	non-fatal
-	send "HTTP/1.1 200 Ok\r\n"
+	send "HTTP/1.1 200 OK\r\n"
 	send "Transfer-encoding: chunked\r\n"
 	send "\r\n"
 	send "004\r\n1234\r\n"
diff --git a/bin/varnishtest/tests/r00694.vtc b/bin/varnishtest/tests/r00694.vtc
index da2d9a5..a28f4f1 100644
--- a/bin/varnishtest/tests/r00694.vtc
+++ b/bin/varnishtest/tests/r00694.vtc
@@ -2,7 +2,7 @@ varnishtest "Wrong calculation of last storage segment length"
 
 server s1 {
 	rxreq
-	send "HTTP/1.1 200 Ok\r\n"
+	send "HTTP/1.1 200 OK\r\n"
 	send "Transfer-encoding: chunked\r\n"
 	send "\r\n"
 	# This is chunksize (128k) + 2 to force to chunks to be allocated
diff --git a/bin/varnishtest/tests/r00733.vtc b/bin/varnishtest/tests/r00733.vtc
index 4d6dd05..365f0ec 100644
--- a/bin/varnishtest/tests/r00733.vtc
+++ b/bin/varnishtest/tests/r00733.vtc
@@ -2,7 +2,7 @@ varnishtest "HTTP/1.1 Backend sends no length hint"
 
 server s1 {
 	rxreq
-	send "HTTP/1.1 200 Ok\n"
+	send "HTTP/1.1 200 OK\n"
 	send "\n"
 	send "12345"
 } -start
diff --git a/bin/varnishtest/tests/r01419.vtc b/bin/varnishtest/tests/r01419.vtc
index 5165c63..a47d424 100644
--- a/bin/varnishtest/tests/r01419.vtc
+++ b/bin/varnishtest/tests/r01419.vtc
@@ -2,7 +2,7 @@ varnishtest "Make sure banlurker skips busy objects"
 
 server s1 {
 	rxreq
-	send "HTTP/1.0 200 Ok\r\n"
+	send "HTTP/1.0 200 OK\r\n"
 	sema r1 sync 2
 	send "Foobar: blaf\r\n"
 	send "Content-Length: 10\r\n"
diff --git a/bin/varnishtest/vtc_http.c b/bin/varnishtest/vtc_http.c
index 17dba6b..966f043 100644
--- a/bin/varnishtest/vtc_http.c
+++ b/bin/varnishtest/vtc_http.c
@@ -760,7 +760,7 @@ cmd_http_txresp(CMD_ARGS)
 	struct http *hp;
 	const char *proto = "HTTP/1.1";
 	const char *status = "200";
-	const char *msg = "Ok";
+	const char *msg = "OK";
 	char* body = NULL;
 
 	(void)cmd;



From tfheen at err.no  Tue Jun 24 09:31:38 2014
From: tfheen at err.no (Tollef Fog Heen)
Date: Tue, 24 Jun 2014 11:31:38 +0200
Subject: [4.0] 06ba9c2 Error when no rst2man is found
Message-ID: <E1WzN4k-000343-CQ@project.varnish-software.com>


commit 06ba9c2ae51c5496ae2fc322f4d3543f0894479b
Author: Tollef Fog Heen <tfheen at fastly.com>
Date:   Thu Apr 24 18:27:13 2014 +0200

    Error when no rst2man is found
    
    If one runs configure with --without-rst2man or no rst2man is found,
    the value is "no", not an empty string.  Handle this correctly in
    configure.
    
    Fixes: #1473

diff --git a/configure.ac b/configure.ac
index 4dfb222..82bd906 100644
--- a/configure.ac
+++ b/configure.ac
@@ -55,7 +55,7 @@ AC_ARG_WITH([rst2man],
   AS_HELP_STRING([--with-rst2man=PATH], [Location of rst2man (auto)]),
   [RST2MAN="$withval"],
   AC_CHECK_PROGS(RST2MAN, [rst2man rst2man.py], []))
-if test -z "$RST2MAN"; then
+if test "$RST2MAN" = "no"; then
   AC_MSG_ERROR(
     [rst2man is needed to build Varnish, please install python-docutils.])
 fi



From phk at FreeBSD.org  Tue Jun 24 09:31:38 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 24 Jun 2014 11:31:38 +0200
Subject: [4.0] e99b5cf Only attempt IMS on status=200 objects.
Message-ID: <E1WzN4k-00034D-Hg@project.varnish-software.com>


commit e99b5cfd886ec38a7f883e23ba516063cf4c16f8
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Fri Apr 25 11:42:37 2014 +0000

    Only attempt IMS on status=200 objects.

diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c
index 5a0a9ff..e61a007 100644
--- a/bin/varnishd/cache/cache_fetch.c
+++ b/bin/varnishd/cache/cache_fetch.c
@@ -199,7 +199,7 @@ vbf_stp_mkbereq(const struct worker *wrk, struct busyobj *bo)
 		http_CopyHome(bo->bereq0);
 	}
 
-	if (bo->ims_obj != NULL) {
+	if (bo->ims_obj != NULL && bo->ims_obj->http->status == 200) {
 		if (http_GetHdr(bo->ims_obj->http, H_Last_Modified, &p)) {
 			http_PrintfHeader(bo->bereq0,
 			    "If-Modified-Since: %s", p);



From martin at varnish-software.com  Tue Jun 24 09:31:38 2014
From: martin at varnish-software.com (Martin Blix Grydeland)
Date: Tue, 24 Jun 2014 11:31:38 +0200
Subject: [4.0] e258ca1 Remove negative Age assertion and truncate to zero
 instead.
Message-ID: <E1WzN4k-00034M-Uk@project.varnish-software.com>


commit e258ca15604b210aad3d70960eccd21fce645778
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Thu Apr 24 14:55:06 2014 +0200

    Remove negative Age assertion and truncate to zero instead.
    
    The Age reported on response objects is calculated from the last
    request timestamp taken. For a cache hit that hasn't been on a
    waitinglist, that will be the Start timestamp. This opens a race where
    the requests' last timestamp could be before the objects t_origin.
    
    Truncate the Age to zero rather than assert in that case.
    
    Fixes: #1486

diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c
index 2518422..88cebcd 100644
--- a/bin/varnishd/cache/cache_req_fsm.c
+++ b/bin/varnishd/cache/cache_req_fsm.c
@@ -117,10 +117,12 @@ cnt_deliver(struct worker *wrk, struct req *req)
 	/* We base Age calculation upon the last timestamp taken during
 	   client request processing. This gives some inaccuracy, but
 	   since Age is only full second resolution that shouldn't
-	   matter. */
-	assert(req->t_prev > req->obj->exp.t_origin);
+	   matter. (Last request timestamp could be a Start timestamp
+	   taken before the object entered into cache leading to negative
+	   age. Truncate to zero in that case).
+	*/
 	http_PrintfHeader(req->resp, "Age: %.0f",
-	    req->t_prev - req->obj->exp.t_origin);
+	    fmax(0., req->t_prev - req->obj->exp.t_origin));
 
 	http_SetHeader(req->resp, "Via: 1.1 varnish (v4)");
 



From martin at varnish-software.com  Tue Jun 24 09:31:39 2014
From: martin at varnish-software.com (Martin Blix Grydeland)
Date: Tue, 24 Jun 2014 11:31:39 +0200
Subject: [4.0] e04394c Copy the status code, proto,
 status string and response message on backend IMS.
Message-ID: <E1WzN4l-00034V-HC@project.varnish-software.com>


commit e04394c7205547193d188ca5d18e8361e57c838d
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Fri Apr 25 11:11:13 2014 +0200

    Copy the status code, proto, status string and response message on
    backend IMS.
    
    When revalidating using backend IMS, copy the status code, status
    code, status string and response message from the original object into
    the new revalidated object. This makes sure that none of the 304
    message fields gets applied to the new revalidated object.
    
    Fixes: #1485

diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c
index e61a007..dded999 100644
--- a/bin/varnishd/cache/cache_fetch.c
+++ b/bin/varnishd/cache/cache_fetch.c
@@ -356,9 +356,9 @@ vbf_stp_startfetch(struct worker *wrk, struct busyobj *bo)
 	AZ(bo->do_esi);
 
 	if (bo->ims_obj != NULL && bo->beresp->status == 304) {
-		bo->beresp->status = 200;
 		http_Merge(bo->ims_obj->http, bo->beresp,
 		    bo->ims_obj->changed_gzip);
+		assert(bo->beresp->status == 200);
 		do_ims = 1;
 	} else
 		do_ims = 0;
diff --git a/bin/varnishd/cache/cache_http.c b/bin/varnishd/cache/cache_http.c
index 6898987..7fe6c9f 100644
--- a/bin/varnishd/cache/cache_http.c
+++ b/bin/varnishd/cache/cache_http.c
@@ -637,7 +637,9 @@ http_FilterResp(const struct http *fm, struct http *to, unsigned how)
 }
 
 /*--------------------------------------------------------------------
- * Merge two HTTP headers the "wrong" way.
+ * Merge two HTTP headers the "wrong" way. Used by backend IMS to
+ * merge in the headers of the validated object with the headers of
+ * the 304 response.
  */
 
 void
@@ -646,6 +648,11 @@ http_Merge(const struct http *fm, struct http *to, int not_ce)
 	unsigned u, v;
 	const char *p;
 
+	to->status = fm->status;
+	http_SetH(to, HTTP_HDR_PROTO, fm->hd[HTTP_HDR_PROTO].b);
+	http_SetH(to, HTTP_HDR_STATUS, fm->hd[HTTP_HDR_STATUS].b);
+	http_SetH(to, HTTP_HDR_RESPONSE, fm->hd[HTTP_HDR_RESPONSE].b);
+
 	for (u = HTTP_HDR_FIRST; u < fm->nhd; u++)
 		fm->hdf[u] |= HDF_MARKER;
 	if (not_ce) {
diff --git a/bin/varnishtest/tests/r01485.vtc b/bin/varnishtest/tests/r01485.vtc
new file mode 100644
index 0000000..caf267a
--- /dev/null
+++ b/bin/varnishtest/tests/r01485.vtc
@@ -0,0 +1,32 @@
+varnishtest "#1485: Wrong response reason phrase"
+
+server s1 {
+	rxreq
+	txresp -hdr "Etag: foo"
+
+	rxreq
+	expect req.http.If-None-Match == "foo"
+	txresp -status 304 -msg "Not Modified"
+} -start
+
+varnish v1 -vcl+backend {
+	sub vcl_backend_response {
+		set beresp.ttl = 1ms;
+		set beresp.grace = 0s;
+		set beresp.keep = 1h;
+	}
+} -start
+
+client c1 {
+	txreq
+	rxresp
+	expect resp.status == 200
+	expect resp.msg == "OK"
+
+	delay 0.1
+
+	txreq
+	rxresp
+	expect resp.status == 200
+	expect resp.msg == "OK"
+} -run



From fgsch at lodoss.net  Tue Jun 24 09:31:39 2014
From: fgsch at lodoss.net (Federico G. Schwindt)
Date: Tue, 24 Jun 2014 11:31:39 +0200
Subject: [4.0] 5729e77 Document vcl_purge and replace vcl_erro with vcl_synth
Message-ID: <E1WzN4l-00034e-V8@project.varnish-software.com>


commit 5729e773973fb1b9190bccadc88caba948b4e5f6
Author: Federico G. Schwindt <fgsch at lodoss.net>
Date:   Sun Apr 27 14:09:19 2014 +0100

    Document vcl_purge and replace vcl_erro with vcl_synth

diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c
index 88cebcd..d12f91c 100644
--- a/bin/varnishd/cache/cache_req_fsm.c
+++ b/bin/varnishd/cache/cache_req_fsm.c
@@ -53,7 +53,7 @@ DOT	label="Request received"
 DOT ]
 DOT ESI_REQ [ shape=hexagon ]
 DOT ESI_REQ -> recv
-DOT ERROR [shape=plaintext]
+DOT SYNTH [shape=plaintext]
 DOT RESTART [shape=plaintext]
 DOT acceptor -> recv [style=bold,color=green]
  */
@@ -178,15 +178,15 @@ VSLb(req->vsl, SLT_Debug, "XXX REF %d", req->obj->objcore->refcnt);
 }
 
 /*--------------------------------------------------------------------
- * Emit an error
+ * Emit a synthetic response
  *
-DOT subgraph xcluster_error {
-DOT	vcl_error [
+DOT subgraph xcluster_synth {
+DOT	synth [
 DOT		shape=record
-DOT		label="{{vcl_error()|resp.}|{<del>deliver?|<restart>restart?}}"
+DOT		label="{cnt_synth:|{vcl_synth\{\}|resp.}|{<del>deliver?|<restart>restart?}}"
 DOT	]
-DOT	ERROR -> vcl_error
-DOT	vcl_error:del:s -> deliver [label=deliver]
+DOT	SYNTH -> synth
+DOT	synth:del:s -> deliver [label=deliver]
 DOT }
  */
 
@@ -313,7 +313,7 @@ DOT		label="{<top>cnt_lookup:|hash lookup|{<busy>busy?|<e>exp?|<eb>exp+busy?|<h>
 DOT	]
 DOT	lookup2 [
 DOT		shape=record
-DOT		label="{<top>cnt_lookup:|{vcl_hit\{\}|req.*, obj.*}|{<deliver>deliver?|error?|restart?|<fetch>fetch?|<pass>pass?}}"
+DOT		label="{<top>cnt_lookup:|{vcl_hit\{\}|req.*, obj.*}|{<deliver>deliver?|synth?|restart?|<fetch>fetch?|<pass>pass?}}"
 DOT	]
 DOT }
 DOT lookup:busy:w -> lookup:top:w [label="(waitinglist)"]
@@ -471,7 +471,7 @@ cnt_lookup(struct worker *wrk, struct req *req)
 DOT subgraph xcluster_miss {
 DOT	miss [
 DOT		shape=record
-DOT		label="{cnt_miss:|{vcl_miss\{\}|req.*}|{<fetch>fetch?|<err>error?|<rst>restart?|<pass>pass?}}"
+DOT		label="{cnt_miss:|{vcl_miss\{\}|req.*}|{<fetch>fetch?|<synth>synth?|<rst>restart?|<pass>pass?}}"
 DOT	]
 DOT }
 DOT miss:fetch:s -> fetch [label="fetch",style=bold,color=blue]
@@ -525,7 +525,7 @@ cnt_miss(struct worker *wrk, struct req *req)
 DOT subgraph xcluster_pass {
 DOT	pass [
 DOT		shape=record
-DOT		label="{cnt_pass:|{vcl_pass\{\}|req.*}|{<fetch>fetch?|<err>error?|<rst>restart?}}"
+DOT		label="{cnt_pass:|{vcl_pass\{\}|req.*}|{<fetch>fetch?|<synth>synth?|<rst>restart?}}"
 DOT	]
 DOT }
 DOT pass:fetch:s -> fetch:n [style=bold, color=red]
@@ -573,7 +573,7 @@ cnt_pass(struct worker *wrk, struct req *req)
 DOT subgraph xcluster_pipe {
 DOT	pipe [
 DOT		shape=record
-DOT		label="{cnt_pipe:|filter req.*-\>bereq.*|{vcl_pipe()|req.*, bereq\.*}|{<pipe>pipe?|<error>error?}}"
+DOT		label="{cnt_pipe:|filter req.*-\>bereq.*|{vcl_pipe()|req.*, bereq\.*}|{<pipe>pipe?|<synth>synth?}}"
 DOT	]
 DOT	pipe_do [
 DOT		shape=ellipse
@@ -625,7 +625,7 @@ DOT }
 DOT RESTART -> restart [color=purple]
 DOT restart -> recv [color=purple]
 DOT restart -> err_restart
-DOT err_restart [label="ERROR",shape=plaintext]
+DOT err_restart [label="SYNTH",shape=plaintext]
  */
 
 static enum req_fsm_nxt
@@ -667,7 +667,7 @@ cnt_restart(struct worker *wrk, struct req *req)
 DOT subgraph xcluster_recv {
 DOT	recv [
 DOT		shape=record
-DOT		label="{cnt_recv:|{vcl_recv\{\}|req.*}|{vcl_hash\{\}|req.*}|{<lookup>lookup?|<pass>pass?|<pipe>pipe?|<error>error?|<purge>purge?}}"
+DOT		label="{cnt_recv:|{vcl_recv\{\}|req.*}|{vcl_hash\{\}|req.*}|{<lookup>lookup?|<pass>pass?|<pipe>pipe?|<synth>synth?|<purge>purge?}}"
 DOT	]
 DOT }
 DOT recv:pipe -> pipe [style=bold,color=orange]
@@ -793,6 +793,13 @@ cnt_recv(struct worker *wrk, struct req *req)
  * Find the objhead, purge it and ask VCL if we should fetch or
  * just return.
  * XXX: fetching not implemented yet.
+ *
+DOT subgraph xcluster_purge {
+DOT	purge [
+DOT		shape=record
+DOT		label="{cnt_purge:|{vcl_purge\{\}|req.*}|{<synth>synth?}}"
+DOT	]
+DOT }
  */
 
 static enum req_fsm_nxt



From martin at varnish-software.com  Tue Jun 24 09:31:40 2014
From: martin at varnish-software.com (Martin Blix Grydeland)
Date: Tue, 24 Jun 2014 11:31:40 +0200
Subject: [4.0] 94a3d07 Close race on req->wrk clearing introduced in 6f2e1bcd
Message-ID: <E1WzN4m-00034o-3m@project.varnish-software.com>


commit 94a3d071b66d4f7aa7f94660cad42e4d58f25724
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Mon Apr 28 15:19:15 2014 +0200

    Close race on req->wrk clearing introduced in 6f2e1bcd
    
    The disembarking thread would clear the req->wrk pointer in
    CNT_Request. The req could have already been rescheduled on another
    worker at this point, causing the req->wrk to be cleared while
    processing. Change back to not clearing the pointer for
    REQ_FSM_DISEMBARK (it's already been done under mutex in HSH_Lookup
    anyways).
    
    Fixes: #1488

diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c
index d12f91c..d465ba4 100644
--- a/bin/varnishd/cache/cache_req_fsm.c
+++ b/bin/varnishd/cache/cache_req_fsm.c
@@ -913,8 +913,8 @@ CNT_Request(struct worker *wrk, struct req *req)
 			VTAILQ_REMOVE(&req->body, st, list);
 			STV_free(st);
 		}
+		req->wrk = NULL;
 	}
-	req->wrk = NULL;
 	assert(WRW_IsReleased(wrk));
 	return (nxt);
 }



From fgsch at lodoss.net  Tue Jun 24 09:31:40 2014
From: fgsch at lodoss.net (Federico G. Schwindt)
Date: Tue, 24 Jun 2014 11:31:40 +0200
Subject: [4.0] c8663d5 Sync with reality
Message-ID: <E1WzN4m-00034x-67@project.varnish-software.com>


commit c8663d50d0b420dfa28d1a1b8b6fc0560b6fffd0
Author: Federico G. Schwindt <fgsch at lodoss.net>
Date:   Mon Apr 28 16:33:34 2014 +0100

    Sync with reality
    
    Update board members, URLs, spelling and fmt(1).

diff --git a/doc/sphinx/tutorial/introduction.rst b/doc/sphinx/tutorial/introduction.rst
index 2fbc43a..330764f 100644
--- a/doc/sphinx/tutorial/introduction.rst
+++ b/doc/sphinx/tutorial/introduction.rst
@@ -3,32 +3,32 @@
 The fundamentals of web proxy caching with Varnish
 --------------------------------------------------
 
-Varnish is a caching HTTP reverse proxy. It recieves requests from
-clients and tries to answer them from the cache. If Varnish cannot answer
-the request from the cache it will forward the request to the backend,
+Varnish is a caching HTTP reverse proxy. It receives requests from clients
+and tries to answer them from the cache. If Varnish cannot answer the
+request from the cache it will forward the request to the backend,
 fetch the response, store it in the cache and deliver it to the client.
 
-When Varnish has a cached response ready it is typically delivered in
-a matter of microseconds, two orders of magnitude faster than your
-typical backend server, so you want to make sure to have Varnish answer
-as many of the requests as possible directly from the cache.
+When Varnish has a cached response ready it is typically delivered in a
+matter of microseconds, two orders of magnitude faster than your typical
+backend server, so you want to make sure to have Varnish answer as many
+of the requests as possible directly from the cache.
 
-Varnish decides whether it can store the content or not based on the response
-it gets back from the backend. The backend can instruct Varnish to cache the
-content with the HTTP response header `Cache-Control`. There are a few
-conditions where Varnish will not cache, the most common one being the use of
-cookies. Since cookies indicates a client-specific web object, Varnish will by
-default not cache it.
+Varnish decides whether it can store the content or not based on the
+response it gets back from the backend. The backend can instruct Varnish
+to cache the content with the HTTP response header `Cache-Control`. There
+are a few conditions where Varnish will not cache, the most common one
+being the use of cookies. Since cookies indicates a client-specific web
+object, Varnish will by default not cache it.
 
-This behaviour as most of Varnish functionality can be changed using policies
-written in the Varnish Configuration Language (VCL). See
+This behaviour as most of Varnish functionality can be changed using
+policies written in the Varnish Configuration Language (VCL). See
 :ref:`users-guide-index` for more information on how to do that.
 
 Performance
 ~~~~~~~~~~~
 
-Varnish has a modern architecture and is written with performance in
-mind.  It is usually bound by the speed of the network, effectively
+Varnish has a modern architecture and is written with performance
+in mind.  It is usually bound by the speed of the network, effectively
 turning performance into a non-issue. You get to focus on how your web
 application work and you can allow yourself, to some degree, to care
 less about performance and scalability.
@@ -38,45 +38,44 @@ less about performance and scalability.
 Flexibility
 ~~~~~~~~~~~
 
-One of the key features of Varnish Cache, in addition to its
-performance, is the flexibility of its configuration language,
-VCL. VCL enables you to write policies on how incoming requests should
-be handled. 
+One of the key features of Varnish Cache, in addition to its performance,
+is the flexibility of its configuration language, VCL. VCL enables you
+to write policies on how incoming requests should be handled.
 
 In such a policy you can decide what content you want to serve, from
-where you want to get the content and how the request or response
-should be altered. 
+where you want to get the content and how the request or response should
+be altered.
 
 Supported platforms
 --------------------
 
-Varnish is written to run on modern versions of Linux and FreeBSD and
-the best experience is had on those platforms. Thanks to our
-contributors it also runs on NetBSD, OpenBSD, OS X and various
-Solaris-descendants like Oracle Solaris, OmniOS and SmartOS.
+Varnish is written to run on modern versions of Linux and FreeBSD and the
+best experience is had on those platforms. Thanks to our contributors
+it also runs on NetBSD, OpenBSD, OS X and various Solaris-descendants
+like Oracle Solaris, OmniOS and SmartOS.
 
 About the Varnish development process
 -------------------------------------
 
 Varnish is a community driven project. The development is overseen by
-the Varnish Governing Board which currently consist of Poul-Henning
-Kamp (Architect), Rogier Mulhuijzen (Fastly) and Kristian Lyngst?l
-(Varnish Software).
+the Varnish Governing Board which currently consist of Poul-Henning Kamp
+(Architect), Rogier Mulhuijzen (Fastly) and Lasse Karstensen (Varnish
+Software).
 
-Please see https://www.varnish-cache.org/trac/wiki/Contributing as
-a starting point if you would like to contribute to Varnish.
+Please see https://www.varnish-cache.org/trac/wiki/Contributing as a
+starting point if you would like to contribute to Varnish.
 
 Getting in touch
 ----------------
 
-You can get in touch with us trough many channels. For real time chat
-you can reach us on IRC trough the server irc.linpro.net on the
-#varnish and #varnish-hacking channels.
-The are two mailing lists available. One for user questions and one
-for development discussions. See https://www.varnish-cache.org/mailinglist for
-information and signup.  There is also a web forum on the same site.
+You can get in touch with us through many channels. For real time chat
+you can reach us on IRC trough the server irc.linpro.net on the #varnish
+and #varnish-hacking channels.
+There are two mailing lists available. One for user questions and one
+for development discussions. See https://www.varnish-cache.org/lists
+for information and signup.  There is also a web forum on the same site.
 
 Now that you have a vague idea on what Varnish Cache is, let see if we
 can get it up and running.
 
-.. XXX:The above three paragraphs are repetetive this is already handled in previos chapters. The only new information is Governing Board which could be moved to the introduction and the paragraphs scrapped. benc
+.. XXX:The above three paragraphs are repetitive this is already handled in previous chapters. The only new information is Governing Board which could be moved to the introduction and the paragraphs scrapped. benc



From martin at varnish-software.com  Tue Jun 24 09:31:40 2014
From: martin at varnish-software.com (Martin Blix Grydeland)
Date: Tue, 24 Jun 2014 11:31:40 +0200
Subject: [4.0] 9aa6eef Fix up req.esi to only be available in client context.
Message-ID: <E1WzN4m-000357-7x@project.varnish-software.com>


commit 9aa6eef5b2f37f2197ff70bb41e74705b83f80eb
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Wed Apr 30 15:59:12 2014 +0200

    Fix up req.esi to only be available in client context.
    
    Fixes: #1489
    Spotted by: xcir

diff --git a/lib/libvcc/generate.py b/lib/libvcc/generate.py
index ace50e6..57f22ca 100755
--- a/lib/libvcc/generate.py
+++ b/lib/libvcc/generate.py
@@ -254,8 +254,8 @@ sp_variables = [
 	),
 	('req.esi',
 		'BOOL',
-		( 'recv', 'backend_response', 'deliver', 'synth',),
-		( 'recv', 'backend_response', 'deliver', 'synth',), """
+		( 'client',),
+		( 'client',), """
 		Boolean. Set to false to disable ESI processing
 		regardless of any value in beresp.do_esi. Defaults
 		to true. This variable is subject to change in



From lkarsten at varnish-software.com  Tue Jun 24 09:31:40 2014
From: lkarsten at varnish-software.com (Lasse Karstensen)
Date: Tue, 24 Jun 2014 11:31:40 +0200
Subject: [4.0] 292e70e RPM: find-provides should not call find-requires.
Message-ID: <E1WzN4m-00035G-9f@project.varnish-software.com>


commit 292e70eb6f5bc91299419b63f25b8714ac119f60
Author: Lasse Karstensen <lkarsten at varnish-software.com>
Date:   Fri May 2 11:17:23 2014 +0200

    RPM: find-provides should not call find-requires.
    
    Also remove the libvarnishapi "workaround" we put in place
    earlier.
    
    Found by: Ingvar Hagelund
    
    Fixes: #1484

diff --git a/redhat/find-provides b/redhat/find-provides
index bf3ed59..8c7dce5 100755
--- a/redhat/find-provides
+++ b/redhat/find-provides
@@ -4,8 +4,10 @@
 
 set -x
 
-if [ -x /usr/lib/rpm/find-requires ]; then
-    /usr/lib/rpm/find-requires "$@"
+if [ -x /usr/lib/rpm/redhat/find-provides ]; then
+    /usr/lib/rpm/redhat/find-provides "$@"
+elif [ -x /usr/lib/rpm/find-provides ]; then
+    /usr/lib/rpm/find-provides "$@"
 fi
 
 cd $(dirname $0)/..
diff --git a/redhat/varnish.spec b/redhat/varnish.spec
index c1f02cf..09752b8 100644
--- a/redhat/varnish.spec
+++ b/redhat/varnish.spec
@@ -51,11 +51,6 @@ available on the following web site: http://www.varnish-cache.org/
 Summary: Libraries for %{name}
 Group: System Environment/Libraries
 BuildRequires: ncurses-devel
-Provides: libvarnishapi.so.1
-Provides: libvarnishapi.so.1(LIBVARNISHAPI_1.0)(64bit)
-Provides: libvarnishapi.so.1(LIBVARNISHAPI_1.1)(64bit)
-Provides: libvarnishapi.so.1(LIBVARNISHAPI_1.2)(64bit)
-Provides: libvarnishapi.so.1(LIBVARNISHAPI_1.3)(64bit)
 #Obsoletes: libvarnish1
 
 %description libs



From phk at FreeBSD.org  Tue Jun 24 09:31:40 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 24 Jun 2014 11:31:40 +0200
Subject: [4.0] 278d90f Update the documentation for obj.hits to reflect code
 reality.
Message-ID: <E1WzN4m-00035P-Bl@project.varnish-software.com>


commit 278d90fdc0008a55405550482ccf9bc4d74f18dc
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon May 5 07:00:56 2014 +0000

    Update the documentation for obj.hits to reflect code reality.
    
    In 4.x we changed obj.hits to be read-only, and to count all variants
    under the same hash-key (= same objhdr).
    
    Since we cannot trivially tell if a ban hits all Vary: values or
    just some of them, there is no sane heuristic to fiddle obj.hits
    on bans that would Do The Right Thing.
    
    Fixes #1492

diff --git a/bin/varnishtest/tests/v00039.vtc b/bin/varnishtest/tests/v00039.vtc
new file mode 100644
index 0000000..90bc865
--- /dev/null
+++ b/bin/varnishtest/tests/v00039.vtc
@@ -0,0 +1,58 @@
+varnishtest "obj.hits vs Vary"
+
+server s1 {
+	rxreq
+	txresp -hdr "Vary: bar" -body "foobar"
+	rxreq
+	txresp -hdr "Vary: bar" -body "barf"
+} -start
+
+varnish v1 \
+	-arg "-p ban_lurker_sleep=0.01" \
+	-arg "-p ban_lurker_age=0.01" \
+	-vcl+backend {
+		sub vcl_deliver {
+			set resp.http.hits = obj.hits;
+		}
+	} -start
+
+client c1 {
+	# This is a miss -> hits == 0
+	txreq -url "/" -hdr "Bar: 1"
+	rxresp
+	expect resp.status == 200
+	expect resp.bodylen == 6
+	expect resp.http.hits == 0
+
+	# This is a hit -> hits == 1
+	txreq -url "/" -hdr "Bar: 1"
+	rxresp
+	expect resp.status == 200
+	expect resp.bodylen == 6
+	expect resp.http.hits == 1
+
+	# This is a miss on different vary -> hits still == 1
+	txreq -url "/" -hdr "Bar: 2"
+	rxresp
+	expect resp.status == 200
+	expect resp.bodylen == 4
+	expect resp.http.hits == 1
+
+	# This is a hit -> hits == 2
+	txreq -url "/" -hdr "Bar: 2"
+	rxresp
+	expect resp.status == 200
+	expect resp.bodylen == 4
+	expect resp.http.hits == 2
+
+} -run
+
+# Ban everything on this hash-key
+varnish v1 -cliok "ban obj.http.vary ~ ."
+delay 1
+
+# And run the entire test again to see that obj.hits got reset.
+
+server s1 -start
+
+client c1 -run
diff --git a/lib/libvcc/generate.py b/lib/libvcc/generate.py
index 57f22ca..1e523f1 100755
--- a/lib/libvcc/generate.py
+++ b/lib/libvcc/generate.py
@@ -511,9 +511,12 @@ sp_variables = [
 		'INT',
 		( 'hit', 'deliver',),
 		( ), """
-		The approximate number of times the object has been
-		delivered. A value of 0 indicates a cache miss.
-		This variable is also available in vcl_deliver.
+		The count of cache-hits on this hash-key since it was
+		last instantiated.  This counts cache-hits across all
+		Vary:-ants on this hash-key.
+		The counter will only be reset to zero if/when all objects
+		with this hash-key have disappeared from cache.
+		NB: obj.hits == 0 does *not* indicate a cache miss.
 		"""
 	),
 	('obj.http.',



From perbu at varnish-software.com  Tue Jun 24 09:31:40 2014
From: perbu at varnish-software.com (Per Buer)
Date: Tue, 24 Jun 2014 11:31:40 +0200
Subject: [4.0] 697a1c3 Import std before use
Message-ID: <E1WzN4m-00035Y-DT@project.varnish-software.com>


commit 697a1c373b7f4a294a857fb6e9d773ae5de0e4c2
Author: Per Buer <perbu at varnish-software.com>
Date:   Mon May 5 09:15:04 2014 +0200

    Import std before use

diff --git a/doc/sphinx/whats-new/upgrading.rst b/doc/sphinx/whats-new/upgrading.rst
index 788764a..dbd8a8a 100644
--- a/doc/sphinx/whats-new/upgrading.rst
+++ b/doc/sphinx/whats-new/upgrading.rst
@@ -127,6 +127,8 @@ is reserved for builtin subs.
 req.backend.healthy replaced by std.healthy(req.backend)
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
+Remeber to import the std module if you're not doing so already.
+
 client.port, and server.port replaced by respectively std.port(client.ip) and std.port(server.ip)
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 



From phk at FreeBSD.org  Tue Jun 24 09:31:40 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 24 Jun 2014 11:31:40 +0200
Subject: [4.0] bc2a411 Check pthread_join() with an AZ()
Message-ID: <E1WzN4m-00035h-Fd@project.varnish-software.com>


commit bc2a4111b3e464b4c2b61a6eb7b446b6408d8f8d
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon May 5 08:19:20 2014 +0000

    Check pthread_join() with an AZ()

diff --git a/bin/varnishhist/varnishhist.c b/bin/varnishhist/varnishhist.c
index e8eb1ff..1656f0e 100644
--- a/bin/varnishhist/varnishhist.c
+++ b/bin/varnishhist/varnishhist.c
@@ -463,7 +463,7 @@ main(int argc, char **argv)
 	VUT.dispatch_priv = NULL;
 	VUT_Main();
 	end_of_file = 1;
-	pthread_join(thr, NULL);
+	AZ(pthread_join(thr, NULL));
 	VUT_Fini();
 	exit(0);
 }



From phk at FreeBSD.org  Tue Jun 24 09:31:40 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 24 Jun 2014 11:31:40 +0200
Subject: [4.0] 6eaf812 Type error in sizeof.
Message-ID: <E1WzN4m-00035q-HD@project.varnish-software.com>


commit 6eaf812b4f573cdb2b07d751a1e219ac04fe4515
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon May 5 08:23:54 2014 +0000

    Type error in sizeof.
    
    Spotted by:	Coverity

diff --git a/bin/varnishhist/varnishhist.c b/bin/varnishhist/varnishhist.c
index 1656f0e..7a7693e 100644
--- a/bin/varnishhist/varnishhist.c
+++ b/bin/varnishhist/varnishhist.c
@@ -442,8 +442,8 @@ main(int argc, char **argv)
 
 	hist_range = hist_high - hist_low;
 	hist_buckets = hist_range * HIST_RES;
-	bucket_hit = calloc(sizeof bucket_hit, hist_buckets);
-	bucket_miss = calloc(sizeof bucket_miss, hist_buckets);
+	bucket_hit = calloc(sizeof *bucket_hit, hist_buckets);
+	bucket_miss = calloc(sizeof *bucket_miss, hist_buckets);
 
 	format = malloc(4 * fnum);
 	for (i = 0; i < fnum-1; i++) {



From phk at FreeBSD.org  Tue Jun 24 09:31:40 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 24 Jun 2014 11:31:40 +0200
Subject: [4.0] 7c8451d Check VUT_Arg() return with AN for consistency.
Message-ID: <E1WzN4m-00035z-Iu@project.varnish-software.com>


commit 7c8451dd3f4a70a64d4933399bf355a4af1ea160
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon May 5 08:26:01 2014 +0000

    Check VUT_Arg() return with AN for consistency.

diff --git a/bin/varnishtop/varnishtop.c b/bin/varnishtop/varnishtop.c
index 761c3f3..80b19f2 100644
--- a/bin/varnishtop/varnishtop.c
+++ b/bin/varnishtop/varnishtop.c
@@ -329,7 +329,7 @@ main(int argc, char **argv)
 	while ((o = getopt(argc, argv, vopt_optstring)) != -1) {
 		switch (o) {
 		case '1':
-			VUT_Arg('d', NULL);
+			AN(VUT_Arg('d', NULL));
 			once = 1;
 			break;
 		case 'f':



From phk at FreeBSD.org  Tue Jun 24 09:31:40 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 24 Jun 2014 11:31:40 +0200
Subject: [4.0] 8c36361 Insignificant memory leak.
Message-ID: <E1WzN4m-000368-Kv@project.varnish-software.com>


commit 8c363610f79f0460a55e2b7f1017921574f30450
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon May 5 08:29:42 2014 +0000

    Insignificant memory leak.
    
    Spotted by:	Coverity

diff --git a/lib/libvarnish/vss.c b/lib/libvarnish/vss.c
index 2c1d8d6..85b3d67 100644
--- a/lib/libvarnish/vss.c
+++ b/lib/libvarnish/vss.c
@@ -149,9 +149,12 @@ VSS_resolve(const char *addr, const char *port, struct vss_addr ***vap)
 	if (adp == NULL)
 		ret = getaddrinfo(addr, port, &hints, &res0);
 	else {
-		ptst = strtol(adp,NULL,10);
-		if (ptst < 0 || ptst > 65535)
+		ptst = strtol(adp, NULL, 10);
+		if (ptst < 0 || ptst > 65535) {
+			free(hop);
+			free(adp);
 			return(0);
+		}
 		ret = getaddrinfo(hop, adp, &hints, &res0);
 	}
 



From martin at varnish-software.com  Tue Jun 24 09:31:40 2014
From: martin at varnish-software.com (Martin Blix Grydeland)
Date: Tue, 24 Jun 2014 11:31:40 +0200
Subject: [4.0] 5c12ec5 Remember to signal the thread to exit when decimating
 the flock
Message-ID: <E1WzN4m-00036H-Me@project.varnish-software.com>


commit 5c12ec5f2568b43fc1f4cc68d76e9c9889462595
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Wed Apr 30 13:26:11 2014 +0200

    Remember to signal the thread to exit when decimating the flock
    
    The pool herder didn't signal the thread when decimating the flock,
    causing the thread to be leaked.
    
    Spotted by: xcir
    Fixes: #1490
    
    Also close a couple of other races:
    
    When pthread_cond_timedwait returns ETIMEDOUT, the predicate
    (wrk->task.func == NULL) could still have changed while reacquiring
    the mutex. If so, the signal would've been lost and the thread
    leaked. Changed the idle wait loop in Pool_Work_Thread() to always
    check the predicate before resuming the cond_wait.
    
    The update of the predicate (wrk->task.func) from e.g. Pool_Task() is
    done without holding the mutex. This opens a race on the predicate,
    and the possibility of the worker going back on cond_wait loosing the
    signal. Changed to update the predicate while holding the mutex.

diff --git a/bin/varnishd/cache/cache_pool.c b/bin/varnishd/cache/cache_pool.c
index 75fac47..ce8d526 100644
--- a/bin/varnishd/cache/cache_pool.c
+++ b/bin/varnishd/cache/cache_pool.c
@@ -163,12 +163,12 @@ pool_accept(struct worker *wrk, void *arg)
 		}
 		VTAILQ_REMOVE(&pp->idle_queue, &wrk2->task, list);
 		AZ(wrk2->task.func);
-		Lck_Unlock(&pp->mtx);
 		assert(sizeof *wa2 == WS_Reserve(wrk2->aws, sizeof *wa2));
 		wa2 = (void*)wrk2->aws->f;
 		memcpy(wa2, wa, sizeof *wa);
 		wrk2->task.func = SES_pool_accept_task;
 		wrk2->task.priv = pp->sesspool;
+		Lck_Unlock(&pp->mtx);
 		AZ(pthread_cond_signal(&wrk2->cond));
 
 		/*
@@ -204,9 +204,9 @@ Pool_Task(struct pool *pp, struct pool_task *task, enum pool_how how)
 	if (wrk != NULL) {
 		VTAILQ_REMOVE(&pp->idle_queue, &wrk->task, list);
 		AZ(wrk->task.func);
-		Lck_Unlock(&pp->mtx);
 		wrk->task.func = task->func;
 		wrk->task.priv = task->priv;
+		Lck_Unlock(&pp->mtx);
 		AZ(pthread_cond_signal(&wrk->cond));
 		return (0);
 	}
@@ -237,6 +237,17 @@ Pool_Task(struct pool *pp, struct pool_task *task, enum pool_how how)
 }
 
 /*--------------------------------------------------------------------
+ * Empty function used as a pointer value for the thread exit condition.
+ */
+
+static void
+pool_kiss_of_death(struct worker *wrk, void *priv)
+{
+	(void)wrk;
+	(void)priv;
+}
+
+/*--------------------------------------------------------------------
  * This is the work function for worker threads in the pool.
  */
 
@@ -274,7 +285,6 @@ Pool_Work_Thread(void *priv, struct worker *wrk)
 				wrk->lastused = VTIM_real();
 			wrk->task.func = NULL;
 			wrk->task.priv = wrk;
-			AZ(wrk->task.func);
 			VTAILQ_INSERT_HEAD(&pp->idle_queue, &wrk->task, list);
 			if (!stats_clean)
 				WRK_SumStat(wrk);
@@ -283,12 +293,12 @@ Pool_Work_Thread(void *priv, struct worker *wrk)
 				    wrk->vcl == NULL ?  0 : wrk->lastused+60.);
 				if (i == ETIMEDOUT)
 					VCL_Rel(&wrk->vcl);
-			} while (i);
+			} while (wrk->task.func == NULL);
 			tp = &wrk->task;
 		}
 		Lck_Unlock(&pp->mtx);
 
-		if (tp->func == NULL)
+		if (tp->func == pool_kiss_of_death)
 			break;
 
 		assert(wrk->pool == pp);
@@ -387,23 +397,23 @@ pool_herder(void *priv)
 				CAST_OBJ_NOTNULL(wrk, pt->priv, WORKER_MAGIC);
 
 				if (wrk->lastused < t_idle ||
-				    pp->nthr > cache_param->wthread_max)
+				    pp->nthr > cache_param->wthread_max) {
+					/* Give it a kiss on the cheek... */
 					VTAILQ_REMOVE(&pp->idle_queue,
 					    &wrk->task, list);
-				else
+					wrk->task.func = pool_kiss_of_death;
+					AZ(pthread_cond_signal(&wrk->cond));
+				} else
 					wrk = NULL;
 			}
 			Lck_Unlock(&pp->mtx);
 
-			/* And give it a kiss on the cheek... */
 			if (wrk != NULL) {
 				pp->nthr--;
 				Lck_Lock(&pool_mtx);
 				VSC_C_main->threads--;
 				VSC_C_main->threads_destroyed++;
 				Lck_Unlock(&pool_mtx);
-				wrk->task.func = NULL;
-				wrk->task.priv = NULL;
 				VTIM_sleep(cache_param->wthread_destroy_delay);
 				continue;
 			}
diff --git a/bin/varnishtest/tests/r01490.vtc b/bin/varnishtest/tests/r01490.vtc
new file mode 100644
index 0000000..82ffdb4
--- /dev/null
+++ b/bin/varnishtest/tests/r01490.vtc
@@ -0,0 +1,38 @@
+varnishtest "#1490 - thread destruction"
+
+server s1 {
+} -start
+
+varnish v1 \
+	-arg "-p debug=+syncvsl" \
+	-arg "-p vsl_mask=+WorkThread" \
+	-arg "-p thread_pool_min=2" \
+	-arg "-p thread_pool_max=3" \
+	-arg "-p thread_pools=1" \
+	-vcl+backend {}
+varnish v1 -start
+
+varnish v1 -expect threads == 2
+
+logexpect l1 -v v1 -g raw {
+	expect * 0 WorkThread {^\S+ start$}
+	expect * 0 WorkThread {^\S+ end$}
+} -start
+
+varnish v1 -cliok "param.set thread_pool_min 3"
+
+# Herder thread sleeps 5 seconds. Have to wait longer than that.
+delay 6
+
+varnish v1 -expect threads == 3
+
+varnish v1 -cliok "param.set thread_pool_min 2"
+varnish v1 -cliok "param.set thread_pool_max 2"
+
+# Herder thread sleeps 5 seconds. Have to wait longer than that.
+delay 6
+
+varnish v1 -expect threads == 2
+
+# Use logexpect to see that the thread actually exited
+logexpect l1 -wait



From nils.goroll at uplex.de  Tue Jun 24 09:31:40 2014
From: nils.goroll at uplex.de (Nils Goroll)
Date: Tue, 24 Jun 2014 11:31:40 +0200
Subject: [4.0] 6d71c1f set RST2MAN=no if no rst2man found to make it a hard
 dependency for real
Message-ID: <E1WzN4m-00036R-OF@project.varnish-software.com>


commit 6d71c1f96e6ab2fe682cc5c7779b56d6e9f0b1da
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Mon May 5 12:50:24 2014 +0200

    set RST2MAN=no if no rst2man found to make it a hard dependency for real

diff --git a/configure.ac b/configure.ac
index 82bd906..36cc542 100644
--- a/configure.ac
+++ b/configure.ac
@@ -54,8 +54,8 @@ AC_PROG_MAKE_SET
 AC_ARG_WITH([rst2man],
   AS_HELP_STRING([--with-rst2man=PATH], [Location of rst2man (auto)]),
   [RST2MAN="$withval"],
-  AC_CHECK_PROGS(RST2MAN, [rst2man rst2man.py], []))
-if test "$RST2MAN" = "no"; then
+  AC_CHECK_PROGS(RST2MAN, [rst2man rst2man.py], [no]))
+if test "x$RST2MAN" = "xno"; then
   AC_MSG_ERROR(
     [rst2man is needed to build Varnish, please install python-docutils.])
 fi



From daghf at varnish-software.com  Tue Jun 24 09:31:40 2014
From: daghf at varnish-software.com (Dag Haavi Finstad)
Date: Tue, 24 Jun 2014 11:31:40 +0200
Subject: [4.0] b2a6c79 Fix tiny memory leak in vdir.c
Message-ID: <E1WzN4m-00036b-RY@project.varnish-software.com>


commit b2a6c79f751e0899fff96c80e30ecb0bd9b164e8
Author: Dag Haavi Finstad <daghf at varnish-software.com>
Date:   Mon May 5 13:34:09 2014 +0200

    Fix tiny memory leak in vdir.c

diff --git a/lib/libvmod_directors/vdir.c b/lib/libvmod_directors/vdir.c
index c8a8c84..12268eb 100644
--- a/lib/libvmod_directors/vdir.c
+++ b/lib/libvmod_directors/vdir.c
@@ -88,6 +88,7 @@ vdir_delete(struct vdir **vdp)
 	free(vd->backend);
 	free(vd->weight);
 	AZ(pthread_mutex_destroy(&vd->mtx));
+	free(vd->dir->vcl_name);
 	FREE_OBJ(vd->dir);
 	vbit_destroy(vd->vbm);
 	FREE_OBJ(vd);



From lkarsten at varnish-software.com  Tue Jun 24 09:31:40 2014
From: lkarsten at varnish-software.com (Lasse Karstensen)
Date: Tue, 24 Jun 2014 11:31:40 +0200
Subject: [4.0] 9a2ac5b We accept docfixes on github.
Message-ID: <E1WzN4m-00036l-Td@project.varnish-software.com>


commit 9a2ac5bf18be9de80fcf3d980f56c9c29e97a994
Author: Lasse Karstensen <lkarsten at varnish-software.com>
Date:   Mon May 5 13:48:53 2014 +0200

    We accept docfixes on github.
    
    As discussed on bugwash today. Lowering the bar may help us improve the
    documentation.

diff --git a/CONTRIBUTING b/CONTRIBUTING
index 8199602..94a668f 100644
--- a/CONTRIBUTING
+++ b/CONTRIBUTING
@@ -13,7 +13,6 @@ Official development tree is here:
 
 Patches can be sent to varnish-dev at varnish-cache.org.
 
-There is a READ-ONLY git-mirror on Github.  Please do not send
-contributions there, we need them by email to varnish-dev@ for now.
-
+Documentation fixes and other small items for the documentation team may be
+sent as Github pull requests.
 



From phk at FreeBSD.org  Tue Jun 24 09:31:41 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 24 Jun 2014 11:31:41 +0200
Subject: [4.0] f6ef923 Add a 'y'ear conversion factor to std.duration().
Message-ID: <E1WzN4n-00036u-1E@project.varnish-software.com>


commit f6ef923b2155b323f5182eeae2d4423796cd35b9
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon May 5 11:50:39 2014 +0000

    Add a 'y'ear conversion factor to std.duration().
    
    Submitted by:	github::brianjarita

diff --git a/bin/varnishtest/tests/m00005.vtc b/bin/varnishtest/tests/m00005.vtc
index 53f5ae3..5ef9dde 100644
--- a/bin/varnishtest/tests/m00005.vtc
+++ b/bin/varnishtest/tests/m00005.vtc
@@ -97,4 +97,10 @@ client c1 {
 	expect resp.http.ttl == 1000002.000
 	expect resp.bodylen == 1
 
+	txreq -url "/1"  -hdr "ttl: 1y"
+	rxresp
+	expect resp.status == 200
+	expect resp.http.ttl == 32536001.000
+	expect resp.bodylen == 1
+
 } -run
diff --git a/lib/libvmod_std/vmod_std_conversions.c b/lib/libvmod_std/vmod_std_conversions.c
index b9020d1..f98b5fb 100644
--- a/lib/libvmod_std/vmod_std_conversions.c
+++ b/lib/libvmod_std/vmod_std_conversions.c
@@ -86,6 +86,7 @@ vmod_duration(const struct vrt_ctx *ctx, const char *p, VCL_DURATION d)
 	case 'h': r *= 60.*60.; break;
 	case 'd': r *= 60.*60.*24.; break;
 	case 'w': r *= 60.*60.*24.*7.; break;
+	case 'y': r *= 60.*60.*24.*365.; break;
 	default:
 		return (d);
 	}



From lkarsten at varnish-software.com  Tue Jun 24 09:31:41 2014
From: lkarsten at varnish-software.com (Lasse Karstensen)
Date: Tue, 24 Jun 2014 11:31:41 +0200
Subject: [4.0] 18b687e Add travis build script
Message-ID: <E1WzN4n-00037C-5Q@project.varnish-software.com>


commit 18b687e45d986a18241eee92e8413efe872b259f
Author: Joshua Bussdieker <jbussdieker at gmail.com>
Date:   Fri Aug 2 09:13:10 2013 -0700

    Add travis build script

diff --git a/.travis.yml b/.travis.yml
new file mode 100644
index 0000000..129e43a
--- /dev/null
+++ b/.travis.yml
@@ -0,0 +1,6 @@
+---
+script: 'make check'
+before_install:
+  - sudo apt-get install python-docutils
+  - ./autogen.sh
+  - ./configure



From nils.goroll at uplex.de  Tue Jun 24 09:31:41 2014
From: nils.goroll at uplex.de (Nils Goroll)
Date: Tue, 24 Jun 2014 11:31:41 +0200
Subject: [4.0] b6c3a8e move curses includes to one file
Message-ID: <E1WzN4n-00037R-91@project.varnish-software.com>


commit b6c3a8eb48d61a776bb450fc3929636c8a2c01e5
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Mon May 5 14:29:24 2014 +0200

    move curses includes to one file

diff --git a/bin/varnishhist/varnishhist.c b/bin/varnishhist/varnishhist.c
index 7a7693e..d01595c 100644
--- a/bin/varnishhist/varnishhist.c
+++ b/bin/varnishhist/varnishhist.c
@@ -35,7 +35,6 @@
 
 #include <sys/types.h>
 
-#include <curses.h>
 #include <errno.h>
 #include <limits.h>
 #include <math.h>
@@ -47,6 +46,7 @@
 #include <string.h>
 #include <unistd.h>
 
+#include "vcurses.h"
 #include "vapi/vsl.h"
 #include "vapi/vsm.h"
 #include "vapi/voptget.h"
diff --git a/bin/varnishstat/varnishstat_curses.c b/bin/varnishstat/varnishstat_curses.c
index 373cf09..fd80580 100644
--- a/bin/varnishstat/varnishstat_curses.c
+++ b/bin/varnishstat/varnishstat_curses.c
@@ -53,20 +53,7 @@
 #include "vtim.h"
 
 #include "varnishstat.h"
-
-#if defined HAVE_NCURSESW_CURSES_H
-#  include <ncursesw/curses.h>
-#elif defined HAVE_NCURSESW_H
-#  include <ncursesw.h>
-#elif defined HAVE_NCURSES_CURSES_H
-#  include <ncurses/curses.h>
-#elif defined HAVE_NCURSES_H
-#  include <ncurses.h>
-#elif defined HAVE_CURSES_H
-#  include <curses.h>
-#else
-#  error "SysV or X/Open-compatible Curses header file required"
-#endif
+#include "vcurses.h"
 
 #define LINES_STATUS		3
 #define LINES_BAR_T		1
diff --git a/bin/varnishtop/varnishtop.c b/bin/varnishtop/varnishtop.c
index 80b19f2..2c195ef 100644
--- a/bin/varnishtop/varnishtop.c
+++ b/bin/varnishtop/varnishtop.c
@@ -34,7 +34,6 @@
 #include "config.h"
 
 #include <ctype.h>
-#include <curses.h>
 #include <errno.h>
 #include <limits.h>
 #include <pthread.h>
@@ -45,6 +44,7 @@
 #include <string.h>
 #include <unistd.h>
 
+#include "vcurses.h"
 #include "vapi/vsm.h"
 #include "vapi/vsl.h"
 #include "vapi/voptget.h"
diff --git a/include/Makefile.am b/include/Makefile.am
index 8291a50..50fed78 100644
--- a/include/Makefile.am
+++ b/include/Makefile.am
@@ -50,6 +50,7 @@ nobase_noinst_HEADERS = \
 	vcli_serve.h \
 	vcs_version.h \
 	vct.h \
+	vcurses.h \
 	vend.h \
 	vev.h \
 	vfil.h \
diff --git a/include/vcurses.h b/include/vcurses.h
new file mode 100644
index 0000000..d24b556
--- /dev/null
+++ b/include/vcurses.h
@@ -0,0 +1,41 @@
+/*-
+ * Copyright (c) 2014 Varnish Software AS
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * include the right curses headers
+ */
+
+#if defined HAVE_NCURSESW_CURSES_H
+#  include <ncursesw/curses.h>
+#elif defined HAVE_NCURSESW_H
+#  include <ncursesw.h>
+#elif defined HAVE_NCURSES_CURSES_H
+#  include <ncurses/curses.h>
+#elif defined HAVE_NCURSES_H
+#  include <ncurses.h>
+#elif defined HAVE_CURSES_H
+#  include <curses.h>
+#else
+#  error "SysV or X/Open-compatible Curses header file required"
+#endif



From lkarsten at varnish-software.com  Tue Jun 24 09:31:41 2014
From: lkarsten at varnish-software.com (Lasse Karstensen)
Date: Tue, 24 Jun 2014 11:31:41 +0200
Subject: [4.0] 57a9a71 [travis] Run tests in parallell.
Message-ID: <E1WzN4n-00037b-B8@project.varnish-software.com>


commit 57a9a7136343b2acea27347180dbc0a23989b132
Author: Lasse Karstensen <lkarsten at varnish-software.com>
Date:   Mon May 5 14:48:59 2014 +0200

    [travis] Run tests in parallell.

diff --git a/.travis.yml b/.travis.yml
index 129e43a..f55a37b 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -1,5 +1,5 @@
 ---
-script: 'make check'
+script: 'make -j3 check'
 before_install:
   - sudo apt-get install python-docutils
   - ./autogen.sh



From nils.goroll at uplex.de  Tue Jun 24 09:31:41 2014
From: nils.goroll at uplex.de (Nils Goroll)
Date: Tue, 24 Jun 2014 11:31:41 +0200
Subject: [4.0] 5321145 these .c files live in $(srcdir), not in $(builddir)
Message-ID: <E1WzN4n-00037k-Es@project.varnish-software.com>


commit 5321145882f6edd65bbeae24a7ff23cc6985f57e
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Mon May 5 17:55:39 2014 +0200

    these .c files live in $(srcdir), not in $(builddir)

diff --git a/bin/varnishadm/Makefile.am b/bin/varnishadm/Makefile.am
index f3050eb..ba476d6 100644
--- a/bin/varnishadm/Makefile.am
+++ b/bin/varnishadm/Makefile.am
@@ -6,10 +6,10 @@ bin_PROGRAMS = varnishadm
 
 varnishadm_SOURCES = \
 	varnishadm.c \
-	$(top_builddir)/lib/libvarnish/vas.c \
-	$(top_builddir)/lib/libvarnish/vsa.c \
-	$(top_builddir)/lib/libvarnish/vtcp.c \
-	$(top_builddir)/lib/libvarnish/vss.c
+	$(top_srcdir)/lib/libvarnish/vas.c \
+	$(top_srcdir)/lib/libvarnish/vsa.c \
+	$(top_srcdir)/lib/libvarnish/vtcp.c \
+	$(top_srcdir)/lib/libvarnish/vss.c
 
 varnishadm_CFLAGS = @LIBEDIT_CFLAGS@
 
diff --git a/bin/varnishreplay/Makefile.am b/bin/varnishreplay/Makefile.am
index 72a293d..78fed42 100644
--- a/bin/varnishreplay/Makefile.am
+++ b/bin/varnishreplay/Makefile.am
@@ -6,9 +6,9 @@ bin_PROGRAMS = varnishreplay
 
 varnishreplay_SOURCES = \
 	varnishreplay.c \
-	$(top_builddir)/lib/libvarnish/vas.c \
-	$(top_builddir)/lib/libvarnish/vtcp.c \
-	$(top_builddir)/lib/libvarnish/vss.c
+	$(top_srcdir)/lib/libvarnish/vas.c \
+	$(top_srcdir)/lib/libvarnish/vtcp.c \
+	$(top_srcdir)/lib/libvarnish/vss.c
 
 varnishreplay_LDADD = \
 	$(top_builddir)/lib/libvarnishcompat/libvarnishcompat.la \
diff --git a/bin/varnishstat/Makefile.am b/bin/varnishstat/Makefile.am
index fe17250..5c55bab 100644
--- a/bin/varnishstat/Makefile.am
+++ b/bin/varnishstat/Makefile.am
@@ -9,9 +9,9 @@ varnishstat_SOURCES = \
 	\
 	varnishstat.c \
 	varnishstat_curses.c \
-	$(top_builddir)/lib/libvarnish/vas.c \
-	$(top_builddir)/lib/libvarnish/version.c \
-	$(top_builddir)/lib/libvarnish/vtim.c
+	$(top_srcdir)/lib/libvarnish/vas.c \
+	$(top_srcdir)/lib/libvarnish/version.c \
+	$(top_srcdir)/lib/libvarnish/vtim.c
 
 varnishstat_LDADD = \
 	$(top_builddir)/lib/libvarnishcompat/libvarnishcompat.la \



From nils.goroll at uplex.de  Tue Jun 24 09:31:41 2014
From: nils.goroll at uplex.de (Nils Goroll)
Date: Tue, 24 Jun 2014 11:31:41 +0200
Subject: [4.0] 98a52d4 also include from builddir
Message-ID: <E1WzN4n-00037v-Hi@project.varnish-software.com>


commit 98a52d43782ea1530c03a95eeb30e42e472d982c
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Mon May 5 18:17:43 2014 +0200

    also include from builddir

diff --git a/lib/libvarnish/Makefile.am b/lib/libvarnish/Makefile.am
index 4e09fb9..5503f86 100644
--- a/lib/libvarnish/Makefile.am
+++ b/lib/libvarnish/Makefile.am
@@ -1,4 +1,4 @@
-AM_CPPFLAGS = -I$(top_srcdir)/include @PCRE_CFLAGS@
+AM_CPPFLAGS = -I$(top_srcdir)/include -I$(top_builddir)/include @PCRE_CFLAGS@
 
 AM_LDFLAGS  = $(AM_LT_LDFLAGS)
 



From nils.goroll at uplex.de  Tue Jun 24 09:31:41 2014
From: nils.goroll at uplex.de (Nils Goroll)
Date: Tue, 24 Jun 2014 11:31:41 +0200
Subject: [4.0] b1d777d also include from builddir
Message-ID: <E1WzN4n-000384-Mj@project.varnish-software.com>


commit b1d777da5843cab65c7c3abd150a69e8e6439070
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Mon May 5 18:18:24 2014 +0200

    also include from builddir

diff --git a/lib/libvarnishapi/Makefile.am b/lib/libvarnishapi/Makefile.am
index f0a23fa..1ee44dd 100644
--- a/lib/libvarnishapi/Makefile.am
+++ b/lib/libvarnishapi/Makefile.am
@@ -2,7 +2,7 @@
 
 AM_LDFLAGS  = $(AM_LT_LDFLAGS)
 
-AM_CPPFLAGS = -I$(top_srcdir)/include @PCRE_CFLAGS@
+AM_CPPFLAGS = -I$(top_srcdir)/include -I$(top_builddir)/include @PCRE_CFLAGS@
 
 lib_LTLIBRARIES = libvarnishapi.la
 



From nils.goroll at uplex.de  Tue Jun 24 09:31:41 2014
From: nils.goroll at uplex.de (Nils Goroll)
Date: Tue, 24 Jun 2014 11:31:41 +0200
Subject: [4.0] 97023ef streamline AM_CPPFLAGS to include from the builddir
Message-ID: <E1WzN4n-00038E-Pf@project.varnish-software.com>


commit 97023ef179a09d5680312d6c46607cde55aac7a1
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Mon May 5 18:31:18 2014 +0200

    streamline AM_CPPFLAGS to include from the builddir

diff --git a/bin/varnishadm/Makefile.am b/bin/varnishadm/Makefile.am
index ba476d6..9033e08 100644
--- a/bin/varnishadm/Makefile.am
+++ b/bin/varnishadm/Makefile.am
@@ -1,6 +1,8 @@
 #
 
-AM_CPPFLAGS = -I$(top_srcdir)/include
+AM_CPPFLAGS = \
+	-I$(top_srcdir)/include \
+	-I$(top_builddir)/include
 
 bin_PROGRAMS = varnishadm
 
diff --git a/bin/varnishhist/Makefile.am b/bin/varnishhist/Makefile.am
index 748721b..5904430 100644
--- a/bin/varnishhist/Makefile.am
+++ b/bin/varnishhist/Makefile.am
@@ -1,6 +1,8 @@
 #
 
-AM_CPPFLAGS = -I$(top_srcdir)/include
+AM_CPPFLAGS = \
+	-I$(top_srcdir)/include \
+	-I$(top_builddir)/include
 
 bin_PROGRAMS = varnishhist
 
diff --git a/bin/varnishlog/Makefile.am b/bin/varnishlog/Makefile.am
index 1e6fa79..2df9415 100644
--- a/bin/varnishlog/Makefile.am
+++ b/bin/varnishlog/Makefile.am
@@ -1,6 +1,8 @@
 #
 
-AM_CPPFLAGS = -I$(top_srcdir)/include
+AM_CPPFLAGS = \
+	-I$(top_srcdir)/include \
+	-I$(top_builddir)/include
 
 bin_PROGRAMS = varnishlog
 
diff --git a/bin/varnishncsa/Makefile.am b/bin/varnishncsa/Makefile.am
index 12a3f4d..d63df61 100644
--- a/bin/varnishncsa/Makefile.am
+++ b/bin/varnishncsa/Makefile.am
@@ -1,6 +1,8 @@
 #
 
-AM_CPPFLAGS = -I$(top_srcdir)/include
+AM_CPPFLAGS = \
+	-I$(top_srcdir)/include \
+	-I$(top_builddir)/include
 
 bin_PROGRAMS = varnishncsa
 
diff --git a/bin/varnishreplay/Makefile.am b/bin/varnishreplay/Makefile.am
index 78fed42..38d1c30 100644
--- a/bin/varnishreplay/Makefile.am
+++ b/bin/varnishreplay/Makefile.am
@@ -1,6 +1,8 @@
 #
 
-AM_CPPFLAGS = -I$(top_srcdir)/include
+AM_CPPFLAGS = \
+	-I$(top_srcdir)/include \
+	-I$(top_builddir)/include
 
 bin_PROGRAMS = varnishreplay
 
diff --git a/bin/varnishstat/Makefile.am b/bin/varnishstat/Makefile.am
index 5c55bab..664131d 100644
--- a/bin/varnishstat/Makefile.am
+++ b/bin/varnishstat/Makefile.am
@@ -1,6 +1,8 @@
 #
 
-AM_CPPFLAGS = -I$(top_srcdir)/include
+AM_CPPFLAGS = \
+	-I$(top_srcdir)/include \
+	-I$(top_builddir)/include
 
 bin_PROGRAMS = varnishstat
 
diff --git a/bin/varnishtest/Makefile.am b/bin/varnishtest/Makefile.am
index 5af8068..fe599d4 100644
--- a/bin/varnishtest/Makefile.am
+++ b/bin/varnishtest/Makefile.am
@@ -15,7 +15,10 @@ check-local:
 
 DISTCLEANFILES = _.ok
 
-AM_CPPFLAGS = -I$(top_srcdir)/include -I$(top_srcdir)/lib/libvgz
+AM_CPPFLAGS = \
+	-I$(top_srcdir)/include \
+	-I$(top_builddir)/include \
+	-I$(top_srcdir)/lib/libvgz
 
 bin_PROGRAMS =	varnishtest
 
diff --git a/bin/varnishtop/Makefile.am b/bin/varnishtop/Makefile.am
index b919dba..f48e972 100644
--- a/bin/varnishtop/Makefile.am
+++ b/bin/varnishtop/Makefile.am
@@ -1,6 +1,8 @@
 #
 
-AM_CPPFLAGS = -I$(top_srcdir)/include
+AM_CPPFLAGS = \
+	-I$(top_srcdir)/include \
+	-I$(top_builddir)/include
 
 bin_PROGRAMS = varnishtop
 
diff --git a/lib/libvarnish/Makefile.am b/lib/libvarnish/Makefile.am
index 5503f86..9275fd2 100644
--- a/lib/libvarnish/Makefile.am
+++ b/lib/libvarnish/Makefile.am
@@ -1,4 +1,7 @@
-AM_CPPFLAGS = -I$(top_srcdir)/include -I$(top_builddir)/include @PCRE_CFLAGS@
+AM_CPPFLAGS = \
+	-I$(top_srcdir)/include \
+	-I$(top_builddir)/include \
+	@PCRE_CFLAGS@
 
 AM_LDFLAGS  = $(AM_LT_LDFLAGS)
 
diff --git a/lib/libvarnishapi/Makefile.am b/lib/libvarnishapi/Makefile.am
index 1ee44dd..287472b 100644
--- a/lib/libvarnishapi/Makefile.am
+++ b/lib/libvarnishapi/Makefile.am
@@ -2,7 +2,10 @@
 
 AM_LDFLAGS  = $(AM_LT_LDFLAGS)
 
-AM_CPPFLAGS = -I$(top_srcdir)/include -I$(top_builddir)/include @PCRE_CFLAGS@
+AM_CPPFLAGS = \
+	-I$(top_srcdir)/include \
+	-I$(top_builddir)/include \
+	@PCRE_CFLAGS@
 
 lib_LTLIBRARIES = libvarnishapi.la
 
diff --git a/lib/libvarnishcompat/Makefile.am b/lib/libvarnishcompat/Makefile.am
index 543fbb2..78683c0 100644
--- a/lib/libvarnishcompat/Makefile.am
+++ b/lib/libvarnishcompat/Makefile.am
@@ -1,6 +1,9 @@
 #
 
-AM_CPPFLAGS = -I$(top_srcdir)/include
+AM_CPPFLAGS = \
+	-I$(top_srcdir)/include \
+	-I$(top_builddir)/include
+
 AM_LDFLAGS  = $(AM_LT_LDFLAGS)
 
 pkglib_LTLIBRARIES = libvarnishcompat.la
diff --git a/lib/libvcc/Makefile.am b/lib/libvcc/Makefile.am
index b030dc4..c7e183d 100644
--- a/lib/libvcc/Makefile.am
+++ b/lib/libvcc/Makefile.am
@@ -2,7 +2,9 @@
 
 AM_LDFLAGS  = $(AM_LT_LDFLAGS)
 
-AM_CPPFLAGS = -I$(top_srcdir)/include -I$(top_builddir)/include
+AM_CPPFLAGS = \
+	-I$(top_srcdir)/include \
+	-I$(top_builddir)/include
 
 pkglib_LTLIBRARIES = libvcc.la
 



From phk at FreeBSD.org  Tue Jun 24 09:31:41 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 24 Jun 2014 11:31:41 +0200
Subject: [4.0] cec1281 Don't close and reopen(+bind) the acceptor socket in
 the server instances whenever we give the server instance more work,
 doing so runs into a dark and murky corner somewhere in Stevens and Solaris
 sometimes doesn't like that.
Message-ID: <E1WzN4n-00038O-S6@project.varnish-software.com>


commit cec12811e38df53ea1e9a3307788b7741496c80a
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue May 6 08:27:44 2014 +0000

    Don't close and reopen(+bind) the acceptor socket in the server
    instances whenever we give the server instance more work, doing
    so runs into a dark and murky corner somewhere in Stevens and
    Solaris sometimes doesn't like that.
    
    The trouble is that the initial bind(2) asks the kernel to assign
    a port, but the subsequent insist on reusing that port, and that,
    in some cases, could mean toruble which cannot be allowed.
    
    Add a "-break" instruction to server so that test case c00035 can
    still work.
    
    Spotted by:	Nils Goroll / UPLEX

diff --git a/bin/varnishtest/tests/c00035.vtc b/bin/varnishtest/tests/c00035.vtc
index 83400f1..9463548 100644
--- a/bin/varnishtest/tests/c00035.vtc
+++ b/bin/varnishtest/tests/c00035.vtc
@@ -1,6 +1,6 @@
 varnishtest "Dropping polling of a backend"
 
-server s1 -repeat 1 {
+server s1 -repeat 20 {
 	rxreq
 	txresp
 } -start
@@ -32,7 +32,7 @@ delay 1
 varnish v1 -cliok "vcl.list"
 varnish v1 -cliok "debug.health"
 
-server s1 {
+server s1 -break {
 	rxreq
 	expect req.url == /foo
 	txresp -bodylen 4
diff --git a/bin/varnishtest/vtc_server.c b/bin/varnishtest/vtc_server.c
index 86cf293..396a869 100644
--- a/bin/varnishtest/vtc_server.c
+++ b/bin/varnishtest/vtc_server.c
@@ -187,6 +187,23 @@ server_start(struct server *s)
 }
 
 /**********************************************************************
+ * Force stop the server thread
+ */
+
+static void
+server_break(struct server *s)
+{
+	void *res;
+
+	CHECK_OBJ_NOTNULL(s, SERVER_MAGIC);
+	vtc_log(s->vl, 2, "Breaking for server");
+	(void)pthread_cancel(s->tp);
+	AZ(pthread_join(s->tp, &res));
+	s->tp = 0;
+	s->run = 0;
+}
+
+/**********************************************************************
  * Wait for server thread to stop
  */
 
@@ -202,8 +219,6 @@ server_wait(struct server *s)
 		vtc_log(s->vl, 0, "Server returned \"%p\"",
 		    (char *)res);
 	s->tp = 0;
-	VTCP_close(&s->sock);
-	s->sock = -1;
 	s->run = 0;
 }
 
@@ -246,6 +261,10 @@ cmd_server(CMD_ARGS)
 				(void)pthread_cancel(s->tp);
 				server_wait(s);
 			}
+			if (s->sock >= 0) {
+				VTCP_close(&s->sock);
+				s->sock = -1;
+			}
 			server_delete(s);
 		}
 		return;
@@ -271,6 +290,12 @@ cmd_server(CMD_ARGS)
 			server_wait(s);
 			continue;
 		}
+
+		if (!strcmp(*av, "-break")) {
+			server_break(s);
+			continue;
+		}
+
 		/*
 		 * We do an implict -wait if people muck about with a
 		 * running server.
@@ -285,6 +310,10 @@ cmd_server(CMD_ARGS)
 			continue;
 		}
 		if (!strcmp(*av, "-listen")) {
+			if (s->sock >= 0) {
+				VTCP_close(&s->sock);
+				s->sock = -1;
+			}
 			bprintf(s->listen, "%s", av[1]);
 			AZ(VSS_parse(s->listen, &s->addr, &s->port));
 			av++;



From martin at varnish-software.com  Tue Jun 24 09:31:41 2014
From: martin at varnish-software.com (Martin Blix Grydeland)
Date: Tue, 24 Jun 2014 11:31:41 +0200
Subject: [4.0] 2baa875 Recycle backend connection before setting BOS_FINISHED
 (take two)
Message-ID: <E1WzN4n-00038X-W3@project.varnish-software.com>


commit 2baa8754fe4a0fc3e4efc9865bddc265fae13690
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Mon May 5 14:06:40 2014 +0200

    Recycle backend connection before setting BOS_FINISHED (take two)
    
    This to give predictable backend reuse behavior for varnishtest,
    making the test cases non-racy.
    
    2nd version of this patch. This version will only deal with recyclable
    backend connections before BOS_FINISHED, leaving the slower close path
    to the clean up code that is run after the BOS_FINISHED state.
    
    Fixes: #1491

diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c
index dded999..ac4c478 100644
--- a/bin/varnishd/cache/cache_fetch.c
+++ b/bin/varnishd/cache/cache_fetch.c
@@ -548,6 +548,14 @@ vbf_stp_fetch(struct worker *wrk, struct busyobj *bo)
 		HSH_Unbusy(&wrk->stats, obj->objcore);
 	}
 	VSLb_ts_busyobj(bo, "BerespBody", W_TIM_real(wrk));
+
+	/* Recycle the backend connection before setting BOS_FINISHED to
+	   give predictable backend reuse behavior for varnishtest */
+	if (bo->vbc != NULL && !(bo->should_close)) {
+		VDI_RecycleFd(&bo->vbc, &bo->acct);
+		AZ(bo->vbc);
+	}
+
 	VBO_setstate(bo, BOS_FINISHED);
 	return (F_STP_DONE);
 }
@@ -639,6 +647,14 @@ vbf_stp_condfetch(struct worker *wrk, struct busyobj *bo)
 	assert(al == bo->ims_obj->len);
 	assert(obj->len == al);
 	EXP_Rearm(bo->ims_obj, bo->ims_obj->exp.t_origin, 0, 0, 0);
+
+	/* Recycle the backend connection before setting BOS_FINISHED to
+	   give predictable backend reuse behavior for varnishtest */
+	if (bo->vbc != NULL && !(bo->should_close)) {
+		VDI_RecycleFd(&bo->vbc, &bo->acct);
+		AZ(bo->vbc);
+	}
+
 	VBO_setstate(bo, BOS_FINISHED);
 	VSLb_ts_busyobj(bo, "BerespBody", W_TIM_real(wrk));
 	return (F_STP_DONE);



From martin at varnish-software.com  Tue Jun 24 09:31:42 2014
From: martin at varnish-software.com (Martin Blix Grydeland)
Date: Tue, 24 Jun 2014 11:31:42 +0200
Subject: [4.0] dabd65a Be consistent with when the BerespBody timestamp is
 taken.
Message-ID: <E1WzN4o-00038i-3O@project.varnish-software.com>


commit dabd65a91b8737c0d71311f68464ed33c249e5f0
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Tue May 6 10:43:44 2014 +0200

    Be consistent with when the BerespBody timestamp is taken.

diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c
index ac4c478..f8e2848 100644
--- a/bin/varnishd/cache/cache_fetch.c
+++ b/bin/varnishd/cache/cache_fetch.c
@@ -547,7 +547,6 @@ vbf_stp_fetch(struct worker *wrk, struct busyobj *bo)
 		assert(bo->state == BOS_REQ_DONE);
 		HSH_Unbusy(&wrk->stats, obj->objcore);
 	}
-	VSLb_ts_busyobj(bo, "BerespBody", W_TIM_real(wrk));
 
 	/* Recycle the backend connection before setting BOS_FINISHED to
 	   give predictable backend reuse behavior for varnishtest */
@@ -557,6 +556,7 @@ vbf_stp_fetch(struct worker *wrk, struct busyobj *bo)
 	}
 
 	VBO_setstate(bo, BOS_FINISHED);
+	VSLb_ts_busyobj(bo, "BerespBody", W_TIM_real(wrk));
 	return (F_STP_DONE);
 }
 



From phk at FreeBSD.org  Tue Jun 24 09:31:42 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 24 Jun 2014 11:31:42 +0200
Subject: [4.0] a0c9c7e Add two counters, one counts purge operations,
 the other objects purged.
Message-ID: <E1WzN4o-00038s-6f@project.varnish-software.com>


commit a0c9c7ec12596df022e1546fb21b88acc9c65340
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue May 6 08:56:41 2014 +0000

    Add two counters, one counts purge operations, the other objects purged.
    
    Suggested by:	Steven Engelhardt via github

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index b0740c1..75a64ac 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -1219,6 +1219,7 @@ void WAIT_Write_Session(struct sess *sp, int fd);
 void WRK_Init(void);
 int WRK_TrySumStat(struct worker *w);
 void WRK_SumStat(struct worker *w);
+void WRK_PurgeStat(unsigned nobj);
 void *WRK_thread(void *priv);
 typedef void *bgthread_t(struct worker *, void *priv);
 void WRK_BgThread(pthread_t *thr, const char *name, bgthread_t *func,
diff --git a/bin/varnishd/cache/cache_hash.c b/bin/varnishd/cache/cache_hash.c
index f4bbae0..7aa6d24 100644
--- a/bin/varnishd/cache/cache_hash.c
+++ b/bin/varnishd/cache/cache_hash.c
@@ -602,6 +602,7 @@ HSH_Purge(struct worker *wrk, struct objhead *oh, double ttl, double grace)
 		(void)HSH_DerefObj(&wrk->stats, &o);
 	}
 	WS_Release(wrk->aws, 0);
+	WRK_PurgeStat(nobj);
 }
 
 
diff --git a/bin/varnishd/cache/cache_wrk.c b/bin/varnishd/cache/cache_wrk.c
index ec18df9..6496a3a 100644
--- a/bin/varnishd/cache/cache_wrk.c
+++ b/bin/varnishd/cache/cache_wrk.c
@@ -77,6 +77,19 @@ WRK_TrySumStat(struct worker *w)
 }
 
 /*--------------------------------------------------------------------
+ * Helper function to update stats for purges under lock
+ */
+
+void
+WRK_PurgeStat(unsigned nobj)
+{
+	Lck_Lock(&wstat_mtx);
+	VSC_C_main->n_purges++;
+	VSC_C_main->n_obj_purged += nobj;
+	Lck_Unlock(&wstat_mtx);
+}
+
+/*--------------------------------------------------------------------
  * Create and starte a back-ground thread which as its own worker and
  * session data structures;
  */
diff --git a/include/tbl/vsc_f_main.h b/include/tbl/vsc_f_main.h
index 4040b0d..8f25d39 100644
--- a/include/tbl/vsc_f_main.h
+++ b/include/tbl/vsc_f_main.h
@@ -583,6 +583,17 @@ VSC_F(bans_persisted_fragmentation,	uint64_t, 0, 'g', diag,
 
 /*--------------------------------------------------------------------*/
 
+VSC_F(n_purges,			uint64_t, 0, 'i', info,
+    "Number of purge operations",
+	""
+)
+VSC_F(n_obj_purged,		uint64_t, 0, 'i', info,
+    "number of purged objects",
+	""
+)
+
+/*--------------------------------------------------------------------*/
+
 VSC_F(exp_mailed,			uint64_t, 0, 'c', diag,
     "Number of objects mailed to expiry thread",
 	"Number of objects mailed to expiry thread for handling."



From lkarsten at varnish-software.com  Tue Jun 24 09:31:42 2014
From: lkarsten at varnish-software.com (Lasse Karstensen)
Date: Tue, 24 Jun 2014 11:31:42 +0200
Subject: [4.0] 4a3735e Remove extra whitespace.
Message-ID: <E1WzN4o-000393-9E@project.varnish-software.com>


commit 4a3735ebe28335c36b9336dcaa96690ee7aff631
Author: Lasse Karstensen <lkarsten at varnish-software.com>
Date:   Tue May 6 12:34:54 2014 +0200

    Remove extra whitespace.

diff --git a/redhat/varnish.initrc b/redhat/varnish.initrc
index c2faa25..dbaf673 100755
--- a/redhat/varnish.initrc
+++ b/redhat/varnish.initrc
@@ -53,7 +53,7 @@ start() {
 	# Open files (usually 1024, which is way too small for varnish)
 	ulimit -n ${NFILES:-131072}
 
-	# Varnish wants to lock shared memory log in memory. 
+	# Varnish wants to lock shared memory log in memory.
 	ulimit -l ${MEMLOCK:-82000}
 
 	# Maximum number of threads (default in CentOS is 1024, which



From lkarsten at varnish-software.com  Tue Jun 24 09:31:42 2014
From: lkarsten at varnish-software.com (Lasse Karstensen)
Date: Tue, 24 Jun 2014 11:31:42 +0200
Subject: [4.0] 1bbbc97 Add support for setting coresize from sysconfig.
Message-ID: <E1WzN4o-00039C-Cu@project.varnish-software.com>


commit 1bbbc97deb6a201d36b8ec191f205acd13f3b8bf
Author: Lasse Karstensen <lkarsten at varnish-software.com>
Date:   Tue May 6 12:36:51 2014 +0200

    Add support for setting coresize from sysconfig.
    
    Add support for using this if it is defined by the administrator
    in the /etc/sysconfig/varnish file.
    
    From way back we've had a commented-out DAEMON_COREFILE_LIMIT in
    our /etc/sysconfig/varnish file, which wasn't referenced anywhere.
    
    Suggested by: gaoyongwei/itxx00 on github.

diff --git a/redhat/varnish.initrc b/redhat/varnish.initrc
index dbaf673..0db3ab6 100755
--- a/redhat/varnish.initrc
+++ b/redhat/varnish.initrc
@@ -60,6 +60,12 @@ start() {
 	# is often too small for varnish)
 	ulimit -u ${NPROCS:-unlimited}
 
+	# If defined, set maximum core size.
+	if [ -n?"${DAEMON_COREFILE_LIMIT}" ]
+	then
+		ulimit -c ${DAEMON_COREFILE_LIMIT}
+	fi
+
         # $DAEMON_OPTS is set in /etc/sysconfig/varnish. At least, one
         # has to set up a backend, or /tmp will be used, which is a bad idea.
 	if [ "$DAEMON_OPTS" = "" ]; then



From phk at FreeBSD.org  Tue Jun 24 09:31:42 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 24 Jun 2014 11:31:42 +0200
Subject: [4.0] c0463aa Up the repeat count for slow test-enviroments
Message-ID: <E1WzN4o-00039O-G6@project.varnish-software.com>


commit c0463aa3d91d86e71a628e93657f770ee625b0d3
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue May 6 10:54:02 2014 +0000

    Up the repeat count for slow test-enviroments

diff --git a/bin/varnishtest/tests/c00035.vtc b/bin/varnishtest/tests/c00035.vtc
index 9463548..8b6bbea 100644
--- a/bin/varnishtest/tests/c00035.vtc
+++ b/bin/varnishtest/tests/c00035.vtc
@@ -1,6 +1,6 @@
 varnishtest "Dropping polling of a backend"
 
-server s1 -repeat 20 {
+server s1 -repeat 40 {
 	rxreq
 	txresp
 } -start



From perbu at varnish-software.com  Tue Jun 24 09:31:42 2014
From: perbu at varnish-software.com (Per Buer)
Date: Tue, 24 Jun 2014 11:31:42 +0200
Subject: [4.0] 928943c Update the grace docs to 4.0
Message-ID: <E1WzN4o-00039d-Jd@project.varnish-software.com>


commit 928943c78ca0051cfd8f5baac5fd341530d64a30
Author: Per Buer <perbu at varnish-software.com>
Date:   Tue May 6 14:40:28 2014 +0200

    Update the grace docs to 4.0

diff --git a/doc/sphinx/users-guide/vcl-grace.rst b/doc/sphinx/users-guide/vcl-grace.rst
index c3a122c..16c1d11 100644
--- a/doc/sphinx/users-guide/vcl-grace.rst
+++ b/doc/sphinx/users-guide/vcl-grace.rst
@@ -24,34 +24,47 @@ the objects in cache beyond their TTL and to serve the waiting
 requests somewhat stale content.
 
 So, in order to serve stale content we must first have some content to
-serve. So to make Varnish keep all objects for 30 minutes beyond their
+serve. So to make Varnish keep all objects for 2 minutes beyond their
 TTL use the following VCL::
 
   sub vcl_backend_response {
-    set beresp.grace = 30m;
+    set beresp.grace = 2m;
   }
 
-Varnish still won't serve the stale objects. In order to enable
-Varnish to actually serve the stale object we must enable this on the
-request. Lets us say that we accept serving 15s old object.::
+Now Varnish will be allowed to serve objects that are up to two
+minutes out of date. When it does it will also schedule a refresh of
+the object. This will happen asynchronously and the moment the new
+object is in it will replace the one we've already got.
 
-  sub vcl_recv {
-    set req.grace = 15s;
+You can influence how this logic works by adding code in vcl_hit. The
+default looks like this:::
+
+  sub vcl_hit {
+     if (obj.ttl >= 0s) {
+         // A pure unadultered hit, deliver it
+         return (deliver);
+     }
+     if (obj.ttl + obj.grace > 0s) {
+         // Object is in grace, deliver it
+         // Automatically triggers a background fetch
+         return (deliver);
+     }
+     // fetch & deliver once we get the result
+     return (fetch);
   }
 
-You might wonder why we should keep the objects in the cache for 30
-minutes if we are unable to serve them? Well, if you have enabled
-:ref:`users-guide-advanced_backend_servers-health` you can check if the
-backend is sick and if it is we can serve the stale content for a bit
-longer.::
+The grace logic is pretty obvious here. If you have enabled
+:ref:`users-guide-advanced_backend_servers-health` you can check if
+the backend is sick and only serve graced object then. Replace the
+second if-clause with something like this:::
 
-   if (!std.healthy(backend)) {
-      set req.grace = 5m;
+   if (!std.healthy(backend) && (obj.ttl + obj.grace > 0s) {
+         return (deliver);
    } else {
-      set req.grace = 15s;
+         return (fetch);
    }
 
 So, to sum up, grace mode solves two problems:
  * it serves stale content to avoid request pile-up.
- * it serves stale content if the backend is not healthy.
+ * it serves stale content if you allow it.
 



From perbu at varnish-software.com  Tue Jun 24 09:31:42 2014
From: perbu at varnish-software.com (Per Buer)
Date: Tue, 24 Jun 2014 11:31:42 +0200
Subject: [4.0] 817ecee Syntax fixed
Message-ID: <E1WzN4o-00039k-Lr@project.varnish-software.com>


commit 817eceee934072faa442cd831541d8ff235f8955
Author: Per Buer <perbu at varnish-software.com>
Date:   Tue May 6 16:09:32 2014 +0200

    Syntax fixed

diff --git a/doc/sphinx/users-guide/vcl-grace.rst b/doc/sphinx/users-guide/vcl-grace.rst
index 16c1d11..e226ab6 100644
--- a/doc/sphinx/users-guide/vcl-grace.rst
+++ b/doc/sphinx/users-guide/vcl-grace.rst
@@ -58,7 +58,7 @@ The grace logic is pretty obvious here. If you have enabled
 the backend is sick and only serve graced object then. Replace the
 second if-clause with something like this:::
 
-   if (!std.healthy(backend) && (obj.ttl + obj.grace > 0s) {
+   if (!std.healthy(req.backend_hint) && (obj.ttl + obj.grace > 0s)) {
          return (deliver);
    } else {
          return (fetch);



From fgsch at lodoss.net  Tue Jun 24 09:31:42 2014
From: fgsch at lodoss.net (Federico G. Schwindt)
Date: Tue, 24 Jun 2014 11:31:42 +0200
Subject: [4.0] 0a06bad Fix syntax
Message-ID: <E1WzN4o-00039v-Ue@project.varnish-software.com>


commit 0a06badefd94a3a1a69711b9b01371a7fe3a9371
Author: Federico G. Schwindt <fgsch at lodoss.net>
Date:   Thu May 8 02:26:21 2014 +0100

    Fix syntax
    
    Submitted by:	github::matsuu
    
    While here add missing semicolon.

diff --git a/doc/sphinx/users-guide/vcl-backends.rst b/doc/sphinx/users-guide/vcl-backends.rst
index a51e4f6..be50e1c 100644
--- a/doc/sphinx/users-guide/vcl-backends.rst
+++ b/doc/sphinx/users-guide/vcl-backends.rst
@@ -64,7 +64,7 @@ Now we need tell Varnish where to send the difference URL. Lets look at `vcl_rec
         if (req.url ~ "^/java/") {
             set req.backend_hint = java;
         } else {
-            set req.backend_hint = default.
+            set req.backend_hint = default;
         }
     }
 
@@ -138,7 +138,7 @@ call certain actions in `vcl_init`.::
 
     sub vcl_recv {
         # send all traffic to the bar director:
-        req.backend_hint = bar.backend();
+        set req.backend_hint = bar.backend();
     }
 
 This director is a round-robin director. This means the director will



From phk at FreeBSD.org  Tue Jun 24 09:31:43 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 24 Jun 2014 11:31:43 +0200
Subject: [4.0] 2c2eb9b Fix a race i noticed while pondering #1500: We must
 snapshot the object length while processing the Range header to ensure
 consistent results when streaming from the backend.
Message-ID: <E1WzN4p-0003A5-HC@project.varnish-software.com>


commit 2c2eb9bcf45f75676f91eb6de2a4febd680db716
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon May 12 07:26:18 2014 +0000

    Fix a race i noticed while pondering #1500:  We must snapshot the
    object length while processing the Range header to ensure consistent
    results when streaming from the backend.

diff --git a/bin/varnishd/cache/cache_http1_deliver.c b/bin/varnishd/cache/cache_http1_deliver.c
index 765fd83..0b0e138 100644
--- a/bin/varnishd/cache/cache_http1_deliver.c
+++ b/bin/varnishd/cache/cache_http1_deliver.c
@@ -89,10 +89,18 @@ v1d_range_bytes(struct req *req, enum vdp_action act, const void *ptr,
 static void
 v1d_dorange(struct req *req, const char *r)
 {
-	ssize_t low, high, has_low;
+	ssize_t len, low, high, has_low;
 
 	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
+	CHECK_OBJ_NOTNULL(req->obj, OBJECT_MAGIC);
 	assert(http_GetStatus(req->obj->http) == 200);
+
+	/* We must snapshot the length if we're streaming from the backend */
+	if (req->obj->objcore->busyobj != NULL)
+		len = VBO_waitlen(req->obj->objcore->busyobj, -1);
+	else
+		len = req->obj->len;
+
 	if (strncmp(r, "bytes=", 6))
 		return;
 	r += 6;
@@ -108,7 +116,7 @@ v1d_dorange(struct req *req, const char *r)
 		r++;
 	}
 
-	if (low >= req->obj->len)
+	if (low >= len)
 		return;
 
 	if (*r != '-')
@@ -124,24 +132,24 @@ v1d_dorange(struct req *req, const char *r)
 			r++;
 		}
 		if (!has_low) {
-			low = req->obj->len - high;
+			low = len - high;
 			if (low < 0)
 				low = 0;
-			high = req->obj->len - 1;
+			high = len - 1;
 		}
 	} else
-		high = req->obj->len - 1;
+		high = len - 1;
 	if (*r != '\0')
 		return;
 
-	if (high >= req->obj->len)
-		high = req->obj->len - 1;
+	if (high >= len)
+		high = len - 1;
 
 	if (low > high)
 		return;
 
 	http_PrintfHeader(req->resp, "Content-Range: bytes %jd-%jd/%jd",
-	    (intmax_t)low, (intmax_t)high, (intmax_t)req->obj->len);
+	    (intmax_t)low, (intmax_t)high, (intmax_t)len);
 	http_Unset(req->resp, H_Content_Length);
 	if (req->res_mode & RES_LEN)
 		http_PrintfHeader(req->resp, "Content-Length: %jd",



From phk at FreeBSD.org  Tue Jun 24 09:31:44 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 24 Jun 2014 11:31:44 +0200
Subject: [4.0] 6b27b8c Prefix the C-symbol of VCL objects with "vo_" to avoid
 issues with reserved C keywords.
Message-ID: <E1WzN4q-0003AF-2C@project.varnish-software.com>


commit 6b27b8c35c090df1dd61eead77184ac98e402f1f
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon May 12 07:54:30 2014 +0000

    Prefix the C-symbol of VCL objects with "vo_" to avoid issues with
    reserved C keywords.
    
    Fixes #1498

diff --git a/bin/varnishtest/tests/r01498.vtc b/bin/varnishtest/tests/r01498.vtc
new file mode 100644
index 0000000..d321448
--- /dev/null
+++ b/bin/varnishtest/tests/r01498.vtc
@@ -0,0 +1,18 @@
+varnishtest "backend name VCC crash"
+
+varnish v1 -vcl {
+	vcl 4.0;
+	import ${vmod_directors};
+	backend s1 {
+	     .host = "127.0.0.1";
+	     .port = "80";
+	}
+	sub vcl_init {
+	    new static = directors.random();
+	    static.add_backend(s1, 100.0);
+	}
+
+	sub vcl_backend_fetch {
+		set bereq.backend = static.backend();
+	}
+}
diff --git a/lib/libvcc/vcc_action.c b/lib/libvcc/vcc_action.c
index 84724c1..b06ff3d 100644
--- a/lib/libvcc/vcc_action.c
+++ b/lib/libvcc/vcc_action.c
@@ -200,17 +200,17 @@ parse_new(struct vcc *tl)
 		p++;
 	p += 2;
 
-	Fh(tl, 0, "static %s *%s;\n\n", s_struct, sy1->name);
+	Fh(tl, 0, "static %s *vo_%s;\n\n", s_struct, sy1->name);
 
 	vcc_NextToken(tl);
 
-	bprintf(buf1, ", &%s, \"%s\"", sy1->name, sy1->name);
+	bprintf(buf1, ", &vo_%s, \"%s\"", sy1->name, sy1->name);
 	vcc_Eval_Func(tl, s_init, buf1, "ASDF", s_init + strlen(s_init) + 1);
 	ifp = New_IniFin(tl);
-	VSB_printf(ifp->fin, "\t%s(&%s);", s_fini, sy1->name);
+	VSB_printf(ifp->fin, "\t%s(&vo_%s);", s_fini, sy1->name);
 	ExpectErr(tl, ';');
 
-	bprintf(buf1, ", %s", sy1->name);
+	bprintf(buf1, ", vo_%s", sy1->name);
 	/* Split the methods from the args */
 	while (*p != '\0') {
 		p += strlen(s_obj);



From phk at FreeBSD.org  Tue Jun 24 09:31:44 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 24 Jun 2014 11:31:44 +0200
Subject: [4.0] 8f036a0 Make our Via header RFC compliant.
Message-ID: <E1WzN4q-0003AQ-JH@project.varnish-software.com>


commit 8f036a0b86221d6dcbf29021ccfdae1cbda7d04b
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon May 12 08:14:09 2014 +0000

    Make our Via header RFC compliant.

diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c
index d465ba4..fea943e 100644
--- a/bin/varnishd/cache/cache_req_fsm.c
+++ b/bin/varnishd/cache/cache_req_fsm.c
@@ -124,7 +124,7 @@ cnt_deliver(struct worker *wrk, struct req *req)
 	http_PrintfHeader(req->resp, "Age: %.0f",
 	    fmax(0., req->t_prev - req->obj->exp.t_origin));
 
-	http_SetHeader(req->resp, "Via: 1.1 varnish (v4)");
+	http_SetHeader(req->resp, "Via: 1.1 varnish-v4");
 
 	if (cache_param->http_gzip_support && req->obj->gziped &&
 	    !RFC2616_Req_Gzip(req->http))



From martin at varnish-software.com  Tue Jun 24 09:31:45 2014
From: martin at varnish-software.com (Martin Blix Grydeland)
Date: Tue, 24 Jun 2014 11:31:45 +0200
Subject: [4.0] 36c5d3a Deref the old object after finishing with it in
 cnt_miss()
Message-ID: <E1WzN4r-0003Ab-De@project.varnish-software.com>


commit 36c5d3a7daee2a4088f9408ad92e3eb9c4468463
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Thu May 8 17:43:38 2014 +0200

    Deref the old object after finishing with it in cnt_miss()
    
    Fixes: #1499

diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c
index fea943e..d60c136 100644
--- a/bin/varnishd/cache/cache_req_fsm.c
+++ b/bin/varnishd/cache/cache_req_fsm.c
@@ -498,6 +498,8 @@ cnt_miss(struct worker *wrk, struct req *req)
 		wrk->stats.cache_miss++;
 		VBF_Fetch(wrk, req, req->objcore, o, VBF_NORMAL);
 		req->req_step = R_STP_FETCH;
+		if (o != NULL)
+			(void)HSH_DerefObj(&wrk->stats, &o);
 		return (REQ_FSM_MORE);
 	case VCL_RET_SYNTH:
 		req->req_step = R_STP_SYNTH;
diff --git a/bin/varnishtest/tests/r01499.vtc b/bin/varnishtest/tests/r01499.vtc
new file mode 100644
index 0000000..7b438b2
--- /dev/null
+++ b/bin/varnishtest/tests/r01499.vtc
@@ -0,0 +1,32 @@
+varnishtest "#1499 - objcore ref leak on IMS update"
+
+server s1 {
+	rxreq
+	txresp -hdr "Etag: foo"
+	rxreq
+	expect req.http.if-none-match == "foo"
+	txresp -hdr "X-Resp: 2"
+} -start
+
+varnish v1 -vcl+backend {
+	sub vcl_backend_response {
+		set beresp.ttl = 0.0001s;
+		set beresp.grace = 0s;
+		set beresp.keep = 2s;
+	}
+} -start
+
+client c1 {
+	txreq
+	rxresp
+
+	delay 0.5
+
+	txreq
+	rxresp
+	expect resp.http.x-resp == "2"
+} -run
+
+delay 3
+
+varnish v1 -expect n_object == 0



From phk at FreeBSD.org  Tue Jun 24 09:31:45 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 24 Jun 2014 11:31:45 +0200
Subject: [4.0] 1dc303a The vcl_purge{} method can return synth or restart.
Message-ID: <E1WzN4r-0003Ak-Gv@project.varnish-software.com>


commit 1dc303aac59deb4ec1ed0e9b491f4016aba2bcfb
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon May 12 08:55:50 2014 +0000

    The vcl_purge{} method can return synth or restart.
    
    (Remember to change the "PURGE" to "GET" if you do this.)
    
    Fixes #1493

diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c
index d60c136..9ebad3f 100644
--- a/bin/varnishd/cache/cache_req_fsm.c
+++ b/bin/varnishd/cache/cache_req_fsm.c
@@ -830,7 +830,16 @@ cnt_purge(struct worker *wrk, struct req *req)
 	AZ(HSH_DerefObjCore(&wrk->stats, &boc));
 
 	VCL_purge_method(req->vcl, wrk, req, NULL, req->http->ws);
-	req->req_step = R_STP_SYNTH;
+	switch (wrk->handling) {
+	case VCL_RET_RESTART:
+		req->req_step = R_STP_RESTART;
+		break;
+	case VCL_RET_SYNTH:
+		req->req_step = R_STP_SYNTH;
+		break;
+	default:
+		WRONG("Illegal return from vcl_purge{}");
+	}
 	return (REQ_FSM_MORE);
 }
 
diff --git a/bin/varnishtest/tests/r01493.vtc b/bin/varnishtest/tests/r01493.vtc
new file mode 100644
index 0000000..f27d899
--- /dev/null
+++ b/bin/varnishtest/tests/r01493.vtc
@@ -0,0 +1,24 @@
+varnishtest "restart in vcl_purge"
+
+server s1 {
+	rxreq
+	txresp
+} -start
+
+varnish v1 -vcl+backend {
+	sub vcl_recv {
+		if (req.method == "PURGE") {
+			return (purge);
+		}
+	}
+	sub vcl_purge {
+		set req.method = "GET";
+		return (restart);
+	}
+} -start
+
+client c1 {
+	txreq -req PURGE
+	rxresp
+	expect resp.status == 200
+} -run
diff --git a/lib/libvcc/generate.py b/lib/libvcc/generate.py
index 1e523f1..c3b51ff 100755
--- a/lib/libvcc/generate.py
+++ b/lib/libvcc/generate.py
@@ -100,7 +100,7 @@ returns =(
 	),
 	('purge',
 		"C",
-		('synth', 'fetch',)
+		('synth', 'restart',)
 	),
 	('miss',
 		"C",



From lkarsten at varnish-software.com  Tue Jun 24 09:31:45 2014
From: lkarsten at varnish-software.com (Lasse Karstensen)
Date: Tue, 24 Jun 2014 11:31:45 +0200
Subject: [4.0] c3b1378 Fix typo.
Message-ID: <E1WzN4r-0003Av-Ko@project.varnish-software.com>


commit c3b1378870a85371ca9c24dcca883516b68de617
Author: Lasse Karstensen <lkarsten at varnish-software.com>
Date:   Mon May 12 13:11:35 2014 +0200

    Fix typo.

diff --git a/doc/sphinx/whats-new/upgrading.rst b/doc/sphinx/whats-new/upgrading.rst
index dbd8a8a..3170a85 100644
--- a/doc/sphinx/whats-new/upgrading.rst
+++ b/doc/sphinx/whats-new/upgrading.rst
@@ -127,7 +127,7 @@ is reserved for builtin subs.
 req.backend.healthy replaced by std.healthy(req.backend)
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-Remeber to import the std module if you're not doing so already.
+Remember to import the std module if you're not doing so already.
 
 client.port, and server.port replaced by respectively std.port(client.ip) and std.port(server.ip)
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~



From lkarsten at varnish-software.com  Tue Jun 24 09:31:45 2014
From: lkarsten at varnish-software.com (Lasse Karstensen)
Date: Tue, 24 Jun 2014 11:31:45 +0200
Subject: [4.0] 47cb9b9 Document that obj.hits may behave differently.
Message-ID: <E1WzN4r-0003B5-OQ@project.varnish-software.com>


commit 47cb9b987508d710d9932d6a5ae03ad58c5d5574
Author: Lasse Karstensen <lkarsten at varnish-software.com>
Date:   Mon May 12 13:11:53 2014 +0200

    Document that obj.hits may behave differently.

diff --git a/doc/sphinx/whats-new/upgrading.rst b/doc/sphinx/whats-new/upgrading.rst
index 3170a85..c882093 100644
--- a/doc/sphinx/whats-new/upgrading.rst
+++ b/doc/sphinx/whats-new/upgrading.rst
@@ -142,6 +142,12 @@ obj is now read-only
 `obj` is now read-only. `obj.hits`, if enabled in VCL, now counts per
 objecthead, not per object. `obj.last_use` has been retired.
 
+Note that obj.hits may not be reset in some cases where bans are in use. See
+bug 1492_ for details.
+
+.. _1492: https://www.varnish-cache.org/trac/ticket/1492
+
+
 Some return values have been replaced
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 



From lkarsten at varnish-software.com  Tue Jun 24 09:31:45 2014
From: lkarsten at varnish-software.com (Lasse Karstensen)
Date: Tue, 24 Jun 2014 11:31:45 +0200
Subject: [4.0] 28a9e89 Backtrack advice that bit during testing.
Message-ID: <E1WzN4r-0003BG-Rj@project.varnish-software.com>


commit 28a9e8947a924ca7d0eb1e970b024a71388cc824
Author: Lasse Karstensen <lkarsten at varnish-software.com>
Date:   Mon May 12 13:12:11 2014 +0200

    Backtrack advice that bit during testing.

diff --git a/doc/sphinx/whats-new/upgrading.rst b/doc/sphinx/whats-new/upgrading.rst
index c882093..c8ed011 100644
--- a/doc/sphinx/whats-new/upgrading.rst
+++ b/doc/sphinx/whats-new/upgrading.rst
@@ -192,11 +192,13 @@ sess_workspace
 In 3.0 it was often necessary to increase `sess_workspace` if a
 lot of VMODs, complex header operations or ESI were in use.
 
-This memory segment has been split into two in 4.0;
-`workspace_backend` and `workspace_client`.
+This is no longer necessary, because ESI scratch space happens
+elsewhere in 4.0.
+
+If you are using a lot of VMODs,  you may need to increase
+either `workspace_backend` and `workspace_client` based on where
+your VMOD is doing its work.
 
-In most cases where you increased `sess_workspace` before, you
-want to increase `workspace_client` now.
 
 New parameters since 3.0
 ========================



From phk at FreeBSD.org  Tue Jun 24 09:31:45 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 24 Jun 2014 11:31:45 +0200
Subject: [4.0] 2fbac3f Cope with systems having both <sys/endian.h> and
 <endian.h>
Message-ID: <E1WzN4r-0003BP-Ur@project.varnish-software.com>


commit 2fbac3f212d858a55fe3ad11848fb2bfbee65671
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon May 12 11:29:46 2014 +0000

    Cope with systems having both <sys/endian.h> and <endian.h>
    
    Fixes:	1476

diff --git a/lib/libvarnish/vsha256.c b/lib/libvarnish/vsha256.c
index 6642d9f..90ac3ed 100644
--- a/lib/libvarnish/vsha256.c
+++ b/lib/libvarnish/vsha256.c
@@ -28,18 +28,17 @@
 
 #include "config.h"
 
-#ifdef HAVE_SYS_ENDIAN_H
-#include <sys/types.h>
-#include <sys/endian.h>
-#define VBYTE_ORDER	_BYTE_ORDER
-#define VBIG_ENDIAN	_BIG_ENDIAN
+#if defined(HAVE_SYS_ENDIAN_H)
+#  include <sys/types.h>
+#  include <sys/endian.h>
+#  define VBYTE_ORDER	_BYTE_ORDER
+#  define VBIG_ENDIAN	_BIG_ENDIAN
+#elif defined(HAVE_ENDIAN_H)
+#  include <endian.h>
+#  define VBYTE_ORDER	__BYTE_ORDER
+#  define VBIG_ENDIAN	__BIG_ENDIAN
 #endif
 
-#ifdef HAVE_ENDIAN_H
-#include <endian.h>
-#define VBYTE_ORDER	__BYTE_ORDER
-#define VBIG_ENDIAN	__BIG_ENDIAN
-#endif
 #include <errno.h>
 #include <stdint.h>
 #include <string.h>



From phk at FreeBSD.org  Tue Jun 24 09:31:46 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 24 Jun 2014 11:31:46 +0200
Subject: [4.0] 1af9e04 By default linux::getopt(3) mangles the argv order,
 such that
Message-ID: <E1WzN4s-0003Bb-1L@project.varnish-software.com>


commit 1af9e045df49393eba0951c67b65c924ab128f8f
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon May 12 12:04:27 2014 +0000

    By default linux::getopt(3) mangles the argv order, such that
    
    	varnishadm -n bla param.set foo -bar
    
    gets interpreted as
    
    	varnishadm -n bla -bar param.set foo
    
    This is counterintuitive and invites random script breaking if people
    don't know they have to add an ornamental '--' in front of the CLI command
    in Linux systems.
    
    This commit makes Linux work the same way as every other POSIX system.
    
    Fixes	#1496

diff --git a/bin/varnishadm/varnishadm.c b/bin/varnishadm/varnishadm.c
index 90016a9..4b219de 100644
--- a/bin/varnishadm/varnishadm.c
+++ b/bin/varnishadm/varnishadm.c
@@ -439,7 +439,15 @@ main(int argc, char * const *argv)
 	const char *n_arg = NULL;
 	int opt, sock;
 
-	while ((opt = getopt(argc, argv, "n:S:T:t:")) != -1) {
+	/*
+	 * By default linux::getopt(3) mangles the argv order, such that
+	 * 	varnishadm -n bla param.set foo -bar
+	 * gets interpreted as
+	 * 	varnishadm -n bla -bar param.set foo 
+	 * The '+' stops that from happening
+	 * See #1496
+	 */
+	while ((opt = getopt(argc, argv, "+n:S:T:t:")) != -1) {
 		switch (opt) {
 		case 'n':
 			n_arg = optarg;



From phk at FreeBSD.org  Tue Jun 24 09:31:46 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 24 Jun 2014 11:31:46 +0200
Subject: [4.0] 32d0751 Reset the "must close" flag when we close the backend
 connection before a retry.
Message-ID: <E1WzN4s-0003Bm-4W@project.varnish-software.com>


commit 32d0751bc19f3725e291e46b95ff33e1882a291e
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon May 12 16:00:59 2014 +0000

    Reset the "must close" flag when we close the backend connection
    before a retry.
    
    Fixes #1494
    
    Testcase by:	scoof

diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c
index f8e2848..0f49a3b 100644
--- a/bin/varnishd/cache/cache_fetch.c
+++ b/bin/varnishd/cache/cache_fetch.c
@@ -371,6 +371,7 @@ vbf_stp_startfetch(struct worker *wrk, struct busyobj *bo)
 	if (wrk->handling == VCL_RET_RETRY) {
 		AN (bo->vbc);
 		VDI_CloseFd(&bo->vbc, &bo->acct);
+		bo->should_close = 0;
 		bo->retries++;
 		if (bo->retries <= cache_param->max_retries)
 			return (F_STP_RETRY);
diff --git a/bin/varnishtest/tests/r01494.vtc b/bin/varnishtest/tests/r01494.vtc
new file mode 100644
index 0000000..38880c5
--- /dev/null
+++ b/bin/varnishtest/tests/r01494.vtc
@@ -0,0 +1,27 @@
+varnishtest "Test retry in be_resp w/conn: close"
+
+server s1 {
+	rxreq
+	txresp -hdr "Connection: close" -bodylen 3
+	expect_close
+	accept
+
+	rxreq
+	txresp -hdr "Connection: close" -bodylen 5
+	expect_close
+} -start
+
+varnish v1 -vcl+backend {
+	sub vcl_backend_response {
+		if(bereq.retries == 0) {
+			return(retry);
+		}
+	}
+} -start
+
+client c1 {
+	txreq -url "/"
+	rxresp
+	expect resp.status == 200
+	expect resp.bodylen == 5
+} -run



From phk at FreeBSD.org  Tue Jun 24 09:31:46 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 24 Jun 2014 11:31:46 +0200
Subject: [4.0] 578366e Also zap keep when purging.
Message-ID: <E1WzN4s-0003Bv-6Y@project.varnish-software.com>


commit 578366ecf3fbc1159ea5ed8543f3e81412bbe43d
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue May 13 06:56:04 2014 +0000

    Also zap keep when purging.
    
    Fixes #1139

diff --git a/bin/varnishd/cache/cache_hash.c b/bin/varnishd/cache/cache_hash.c
index 7aa6d24..9acf893 100644
--- a/bin/varnishd/cache/cache_hash.c
+++ b/bin/varnishd/cache/cache_hash.c
@@ -555,7 +555,8 @@ hsh_rush(struct dstat *ds, struct objhead *oh)
  */
 
 void
-HSH_Purge(struct worker *wrk, struct objhead *oh, double ttl, double grace)
+HSH_Purge(struct worker *wrk, struct objhead *oh, double ttl, double grace,
+double keep)
 {
 	struct objcore *oc, **ocp;
 	unsigned spc, nobj, n;
@@ -598,7 +599,7 @@ HSH_Purge(struct worker *wrk, struct objhead *oh, double ttl, double grace)
 		if (o == NULL)
 			continue;
 		CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC);
-		EXP_Rearm(o, now, ttl, grace, NAN);	// XXX: Keep ?
+		EXP_Rearm(o, now, ttl, grace, keep);
 		(void)HSH_DerefObj(&wrk->stats, &o);
 	}
 	WS_Release(wrk->aws, 0);
diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c
index 9ebad3f..86fed3e 100644
--- a/bin/varnishd/cache/cache_req_fsm.c
+++ b/bin/varnishd/cache/cache_req_fsm.c
@@ -825,7 +825,7 @@ cnt_purge(struct worker *wrk, struct req *req)
 	CHECK_OBJ_NOTNULL(boc, OBJCORE_MAGIC);
 	VRY_Finish(req, DISCARD);
 
-	HSH_Purge(wrk, boc->objhead, 0, 0);
+	HSH_Purge(wrk, boc->objhead, 0, 0, 0);
 
 	AZ(HSH_DerefObjCore(&wrk->stats, &boc));
 
diff --git a/bin/varnishd/cache/cache_vrt.c b/bin/varnishd/cache/cache_vrt.c
index c423eb2..4e1bcbf 100644
--- a/bin/varnishd/cache/cache_vrt.c
+++ b/bin/varnishd/cache/cache_vrt.c
@@ -492,10 +492,10 @@ VRT_purge(const struct vrt_ctx *ctx, double ttl, double grace)
 	CHECK_OBJ_NOTNULL(ctx->req->wrk, WORKER_MAGIC);
 	if (ctx->method == VCL_MET_HIT)
 		HSH_Purge(ctx->req->wrk, ctx->req->obj->objcore->objhead,
-		    ttl, grace);
+		    ttl, grace, NAN);
 	else if (ctx->method == VCL_MET_MISS)
 		HSH_Purge(ctx->req->wrk, ctx->req->objcore->objhead,
-		    ttl, grace);
+		    ttl, grace, NAN);
 }
 
 /*--------------------------------------------------------------------
diff --git a/bin/varnishd/hash/hash_slinger.h b/bin/varnishd/hash/hash_slinger.h
index 40d3aef..3c6f815 100644
--- a/bin/varnishd/hash/hash_slinger.h
+++ b/bin/varnishd/hash/hash_slinger.h
@@ -70,7 +70,8 @@ void HSH_Drop(struct worker *, struct object **);
 void HSH_Init(const struct hash_slinger *slinger);
 void HSH_AddString(const struct req *, const char *str);
 void HSH_Insert(struct worker *, const void *hash, struct objcore *);
-void HSH_Purge(struct worker *, struct objhead *, double ttl, double grace);
+void HSH_Purge(struct worker *, struct objhead *, double ttl, double grace,
+    double keep);
 void HSH_config(const char *h_arg);
 struct busyobj *HSH_RefBusy(const struct objcore *oc);
 struct objcore *HSH_Private(struct worker *wrk);



From phk at FreeBSD.org  Tue Jun 24 09:31:46 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 24 Jun 2014 11:31:46 +0200
Subject: [4.0] ee92500 Remove the stand-alone "purge" action from VCL,
 now that we have return(purge) from vcl_recv.
Message-ID: <E1WzN4s-0003C4-8x@project.varnish-software.com>


commit ee9250095af04b74c31ca769a55da3d18a6eeb7d
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue May 13 07:10:11 2014 +0000

    Remove the stand-alone "purge" action from VCL, now that we have
    return(purge) from vcl_recv.
    
    Retain the VRT_purge() for VMOD and inline-C use.

diff --git a/bin/varnishd/cache/cache_vrt.c b/bin/varnishd/cache/cache_vrt.c
index 4e1bcbf..08e3b77 100644
--- a/bin/varnishd/cache/cache_vrt.c
+++ b/bin/varnishd/cache/cache_vrt.c
@@ -480,11 +480,11 @@ VRT_CacheReqBody(const struct vrt_ctx *ctx, long long maxsize)
 }
 
 /*--------------------------------------------------------------------
- * "real" purges
+ * purges
  */
 
 void
-VRT_purge(const struct vrt_ctx *ctx, double ttl, double grace)
+VRT_purge(const struct vrt_ctx *ctx, double ttl, double grace, double keep)
 {
 
 	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
@@ -492,10 +492,10 @@ VRT_purge(const struct vrt_ctx *ctx, double ttl, double grace)
 	CHECK_OBJ_NOTNULL(ctx->req->wrk, WORKER_MAGIC);
 	if (ctx->method == VCL_MET_HIT)
 		HSH_Purge(ctx->req->wrk, ctx->req->obj->objcore->objhead,
-		    ttl, grace, NAN);
+		    ttl, grace, keep);
 	else if (ctx->method == VCL_MET_MISS)
 		HSH_Purge(ctx->req->wrk, ctx->req->objcore->objhead,
-		    ttl, grace, NAN);
+		    ttl, grace, keep);
 }
 
 /*--------------------------------------------------------------------
diff --git a/bin/varnishtest/tests/c00041.vtc b/bin/varnishtest/tests/c00041.vtc
index 2b7efc2..bd6abdb 100644
--- a/bin/varnishtest/tests/c00041.vtc
+++ b/bin/varnishtest/tests/c00041.vtc
@@ -39,8 +39,12 @@ server s1 {
 } -start
 
 varnish v1 -vcl+backend {
-	sub vcl_miss { if (req.http.purge == "yes") { purge; } }
-	sub vcl_hit { if (req.http.purge == "yes") { purge; return(restart);} }
+	sub vcl_recv { if (req.http.purge == "yes") { return(purge); } }
+	sub vcl_purge { if (req.http.restart == "yes") {
+		unset req.http.purge;
+		unset req.http.restart;
+		return(restart);} 
+	}
 } -start
 
 client c1 {
@@ -49,6 +53,7 @@ client c1 {
 	expect resp.status == 200
 	expect resp.http.x-varnish == 1001
 	expect resp.bodylen == 1
+	delay .1
 
 	txreq -url "/1" -hdr "Foo: foo2"
 	rxresp
@@ -62,42 +67,44 @@ client c1 {
 	expect resp.status == 200
 	expect resp.http.x-varnish == "1005 1002"
 	expect resp.bodylen == 1
+	delay .1
 
 	txreq -url "/1" -hdr "Foo: foo2"
 	rxresp
 	expect resp.status == 200
 	expect resp.http.x-varnish == "1006 1004"
 	expect resp.bodylen == 2
+	delay .1
 
 	# Purge on hit
-	txreq -url "/1" -hdr "Foo: foo2" -hdr "purge: yes"
+	txreq -url "/1" -hdr "Foo: foo2" -hdr "purge: yes" -hdr "restart: yes"
 	rxresp
 	expect resp.status == 200
 	expect resp.bodylen == 12
+	delay .1
 
 	txreq -url "/1" -hdr "foo: foo1"
 	rxresp
 	expect resp.status == 200
 	expect resp.bodylen == 11
+	delay .1
 
 	# Purge on miss
-	txreq -url "/1" -hdr "Foo: foo3" -hdr "purge: yes"
+	txreq -url "/1" -hdr "Foo: foo3" -hdr "purge: yes" -hdr "restart: yes"
 	rxresp
 	expect resp.status == 200
 	expect resp.bodylen == 23
+	delay .1
 
 	txreq -url "/1" -hdr "foo: foo1"
 	rxresp
 	expect resp.status == 200
 	expect resp.bodylen == 21
+	delay .1
 
 	txreq -url "/1" -hdr "Foo: foo2"
 	rxresp
 	expect resp.status == 200
 	expect resp.bodylen == 22
+	delay .1
 } -run
-
-varnish v1 -errvcl {'purge': not a valid action in method 'vcl_recv'.} {
-	backend s1 { .host = "${s1_addr}"; }
-	sub vcl_recv { if (req.http.purge == "yes") { purge; } }
-}
diff --git a/include/vrt.h b/include/vrt.h
index 8e5acba..bfbbb2d 100644
--- a/include/vrt.h
+++ b/include/vrt.h
@@ -201,7 +201,7 @@ const char *VRT_regsub(const struct vrt_ctx *, int all, const char *,
     void *, const char *);
 
 void VRT_ban_string(const struct vrt_ctx *, const char *);
-void VRT_purge(const struct vrt_ctx *, double ttl, double grace);
+void VRT_purge(const struct vrt_ctx *, double ttl, double grace, double keep);
 
 void VRT_count(const struct vrt_ctx *, unsigned);
 int VRT_rewrite(const char *, const char *);
diff --git a/lib/libvcc/vcc_action.c b/lib/libvcc/vcc_action.c
index b06ff3d..910a89c 100644
--- a/lib/libvcc/vcc_action.c
+++ b/lib/libvcc/vcc_action.c
@@ -367,16 +367,6 @@ parse_rollback(struct vcc *tl)
 /*--------------------------------------------------------------------*/
 
 static void
-parse_purge(struct vcc *tl)
-{
-
-	vcc_NextToken(tl);
-	Fb(tl, 1, "VRT_purge(ctx, 0, 0);\n");
-}
-
-/*--------------------------------------------------------------------*/
-
-static void
 parse_synthetic(struct vcc *tl)
 {
 	vcc_NextToken(tl);
@@ -409,7 +399,6 @@ static struct action_table {
 	{ "call",		parse_call },
 	{ "hash_data",		parse_hash_data, VCL_MET_HASH },
 	{ "new",		parse_new, VCL_MET_INIT},
-	{ "purge",		parse_purge, VCL_MET_MISS | VCL_MET_HIT },
 	{ "return",		parse_return },
 	{ "rollback",		parse_rollback },
 	{ "set",		parse_set },



From phk at FreeBSD.org  Tue Jun 24 09:31:46 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 24 Jun 2014 11:31:46 +0200
Subject: [4.0] 39a3455 Insert a newline before the actual panic message in
 order to avoid line wrapping
Message-ID: <E1WzN4s-0003CE-B9@project.varnish-software.com>


commit 39a3455f894ed68ff1d6e55721e05aca5c2fab20
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue May 13 07:36:23 2014 +0000

    Insert a newline before the actual panic message in order to
    avoid line wrapping

diff --git a/bin/varnishd/mgt/mgt_child.c b/bin/varnishd/mgt/mgt_child.c
index 3c739f1..1ae0c0e 100644
--- a/bin/varnishd/mgt/mgt_child.c
+++ b/bin/varnishd/mgt/mgt_child.c
@@ -121,7 +121,7 @@ mgt_panic_record(pid_t r)
 	char time_str[30];
 
 	AN(heritage.panic_str[0]);
-	REPORT(LOG_ERR, "Child (%jd) Panic message: %s",
+	REPORT(LOG_ERR, "Child (%jd) Panic message:\n%s",
 	    (intmax_t)r, heritage.panic_str);
 
 	if (child_panic != NULL)



From phk at FreeBSD.org  Tue Jun 24 09:31:46 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 24 Jun 2014 11:31:46 +0200
Subject: [4.0] 20d3eca Move a misplaced assert.
Message-ID: <E1WzN4s-0003CO-Dd@project.varnish-software.com>


commit 20d3ecae26e8b9f522c04b8759e8e5b90df5789a
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue May 13 08:44:32 2014 +0000

    Move a misplaced assert.
    
    Fixes	#1478

diff --git a/bin/varnishd/cache/cache_hash.c b/bin/varnishd/cache/cache_hash.c
index 9acf893..cd48671 100644
--- a/bin/varnishd/cache/cache_hash.c
+++ b/bin/varnishd/cache/cache_hash.c
@@ -458,10 +458,10 @@ HSH_Lookup(struct req *req, struct objcore **ocp, struct objcore **bocp,
 		exp_oc->refcnt++;
 
 		if (!busy_found) {
-			AZ(req->hash_ignore_busy);
 			*bocp = hsh_insert_busyobj(wrk, oh);
 			retval = HSH_EXPBUSY;
 		} else {
+			AZ(req->hash_ignore_busy);
 			retval = HSH_EXP;
 		}
 		if (oh->hits < LONG_MAX)



From lkarsten at varnish-software.com  Tue Jun 24 09:31:46 2014
From: lkarsten at varnish-software.com (Lasse Karstensen)
Date: Tue, 24 Jun 2014 11:31:46 +0200
Subject: [4.0] 284c933 Add documentation for the fallback director.
Message-ID: <E1WzN4s-0003CY-Gd@project.varnish-software.com>


commit 284c933d07337bbdd7530b8fd0745c93e91cc0c5
Author: Lasse Karstensen <lkarsten at varnish-software.com>
Date:   Tue May 13 11:12:38 2014 +0200

    Add documentation for the fallback director.

diff --git a/lib/libvmod_directors/vmod.vcc b/lib/libvmod_directors/vmod.vcc
index 9b5732a..c360837 100644
--- a/lib/libvmod_directors/vmod.vcc
+++ b/lib/libvmod_directors/vmod.vcc
@@ -59,8 +59,9 @@ Note that directors can use other directors as backends.
 $Object round_robin()
 
 Description
-       Create a round robin director. This director will pick backends
-       in a round robin fashion.
+        Create a round robin director.
+
+	This director will pick backends in a round robin fashion.
 Example
        new bar = directors.round_robin();
 
@@ -78,30 +79,37 @@ Description
 Example
        set req.backend_hint = rrdir.backend();
 
+####################################################################
 $Object fallback()
 
 Description
-        Creates a fallback director.
+        Create a fallback director.
 
-#	XXX: Add description of fallback director.
+        A fallback director will try each of the added backends in turn,
+        and return the first one that is healthy.
 
 Example
-        new foo = directors.fallback();
+        new vdir = directors.fallback();
 
 $Method VOID .add_backend(BACKEND)
 
 Description
-       Adds a backend to the director.
+        Add a backend to the director.
+
+        Note that the order in which this is done matters in the for the
+        fallback director.
+
 Example
-       bar.add_backend(backend1);
+       vdir.add_backend(backend1);
 
 $Method BACKEND .backend()
 
 Description
-       Picks a backend from the director.
+       Pick a backend from the director.
 Example
-       set req.backend_hint = rrdir.backend();
+       set req.backend_hint = vdir.backend();
 
+####################################################################
 $Object random()
 
 Description



From lkarsten at varnish-software.com  Tue Jun 24 09:31:46 2014
From: lkarsten at varnish-software.com (Lasse Karstensen)
Date: Tue, 24 Jun 2014 11:31:46 +0200
Subject: [4.0] 7ac9230 Rewrite first paragraph to make it easier to read.
Message-ID: <E1WzN4s-0003Ch-In@project.varnish-software.com>


commit 7ac92307370a6f2ddce8cba892f6a87251b71ac6
Author: Lasse Karstensen <lkarsten at varnish-software.com>
Date:   Tue May 13 11:18:14 2014 +0200

    Rewrite first paragraph to make it easier to read.

diff --git a/lib/libvmod_directors/vmod.vcc b/lib/libvmod_directors/vmod.vcc
index c360837..d18151f 100644
--- a/lib/libvmod_directors/vmod.vcc
+++ b/lib/libvmod_directors/vmod.vcc
@@ -30,9 +30,11 @@ $Module directors 3 Backend traffic directors
 DESCRIPTION
 ===========
 
-Vmod_directors enables load balancing in Varnish. The module serves
-as an example on how one could extend the load balancing capabilities
-of Varnish.
+`vmod_directors` enables backend load balancing in Varnish.
+
+The module implements a set of basic load balancing techniques, and
+also serves as an example on how one could extend the load balancing
+capabilities of Varnish.
 
 To enable load balancing you must import this vmod (directors) in your
 VCL:::



From lkarsten at varnish-software.com  Tue Jun 24 09:31:46 2014
From: lkarsten at varnish-software.com (Lasse Karstensen)
Date: Tue, 24 Jun 2014 11:31:46 +0200
Subject: [4.0] c7c7089 Add dependeny to build both rst files from vcc.
Message-ID: <E1WzN4s-0003Cr-ME@project.varnish-software.com>


commit c7c70895a4cc0213f9ec0f043eb4e2456599a6b0
Author: Lasse Karstensen <lkarsten at varnish-software.com>
Date:   Tue May 13 11:24:06 2014 +0200

    Add dependeny to build both rst files from vcc.
    
    Fix spotty dependency where none or just one of the
    .rst files output by vmodtool.py was known by the Makefile.

diff --git a/lib/libvmod_debug/Makefile.am b/lib/libvmod_debug/Makefile.am
index 89df1b2..f856dde 100644
--- a/lib/libvmod_debug/Makefile.am
+++ b/lib/libvmod_debug/Makefile.am
@@ -24,7 +24,7 @@ nodist_libvmod_debug_la_SOURCES = \
 # BUILT_SOURCES is only a hack and dependency tracking does not help for the first build
 vmod_debug.lo vmod_debug_obj.lo: vcc_if.h
 
-vcc_if.c vcc_if.h: $(vmodtool) $(vmod_srcdir)/vmod.vcc
+vcc_if.c vcc_if.h vmod_debug.rst vmod_debug.man.rst: $(vmodtool) $(vmod_srcdir)/vmod.vcc
 	@PYTHON@ $(vmodtool) $(vmod_srcdir)/vmod.vcc
 
 EXTRA_DIST = vmod.vcc
diff --git a/lib/libvmod_directors/Makefile.am b/lib/libvmod_directors/Makefile.am
index 05912e0..583b1f7 100644
--- a/lib/libvmod_directors/Makefile.am
+++ b/lib/libvmod_directors/Makefile.am
@@ -28,7 +28,7 @@ nodist_libvmod_directors_la_SOURCES = \
 # BUILT_SOURCES is only a hack and dependency tracking does not help for the first build
 vdir.lo fall_back.lo hash.lo random.lo round_robin.lo: vcc_if.h
 
-vcc_if.c vcc_if.h vmod_directors.man.rst: $(vmodtool) $(vmod_srcdir)/vmod.vcc
+vcc_if.c vcc_if.h vmod_directors.rst vmod_directors.man.rst: $(vmodtool) $(vmod_srcdir)/vmod.vcc
 	@PYTHON@ $(vmodtool) $(vmod_srcdir)/vmod.vcc
 
 EXTRA_DIST = vmod.vcc
diff --git a/lib/libvmod_std/Makefile.am b/lib/libvmod_std/Makefile.am
index 2e9832a..ff8466e 100644
--- a/lib/libvmod_std/Makefile.am
+++ b/lib/libvmod_std/Makefile.am
@@ -26,7 +26,7 @@ nodist_libvmod_std_la_SOURCES = \
 # BUILT_SOURCES is only a hack and dependency tracking does not help for the first build
 vmod_std.lo vmod_std_fileread.lo vmod_std_conversions.lo: vcc_if.h
 
-vcc_if.c vcc_if.h vmod_std.man.rst: $(vmodtool) $(vmod_srcdir)/vmod.vcc
+vcc_if.c vcc_if.h vmod_std.rst vmod_std.man.rst: $(vmodtool) $(vmod_srcdir)/vmod.vcc
 	@PYTHON@ $(vmodtool) $(vmod_srcdir)/vmod.vcc
 
 EXTRA_DIST = vmod.vcc



From lkarsten at varnish-software.com  Tue Jun 24 09:31:46 2014
From: lkarsten at varnish-software.com (Lasse Karstensen)
Date: Tue, 24 Jun 2014 11:31:46 +0200
Subject: [4.0] 45ee24a Tiny lint; file is called vmod.vcc.
Message-ID: <E1WzN4s-0003D1-P1@project.varnish-software.com>


commit 45ee24af32f2a3742966efdf9c0aff95783a863a
Author: Lasse Karstensen <lkarsten at varnish-software.com>
Date:   Tue May 13 11:44:31 2014 +0200

    Tiny lint; file is called vmod.vcc.

diff --git a/lib/libvcc/vmodtool.py b/lib/libvcc/vmodtool.py
index be6cff6..b9d7c95 100755
--- a/lib/libvcc/vmodtool.py
+++ b/lib/libvcc/vmodtool.py
@@ -26,7 +26,7 @@
 # OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 # SUCH DAMAGE.
 #
-# Read the vmod.spec file and produce the vmod.h and vmod.c files.
+# Read the vmod.vcc file and produce the vmod.h and vmod.c files.
 #
 # vmod.h contains the prototypes for the published functions, the module
 # C-code should include this file to ensure type-consistency.



From lkarsten at varnish-software.com  Tue Jun 24 09:31:46 2014
From: lkarsten at varnish-software.com (Lasse Karstensen)
Date: Tue, 24 Jun 2014 11:31:46 +0200
Subject: [4.0] 7a2a983 Output correct tool name.
Message-ID: <E1WzN4s-0003DA-Rz@project.varnish-software.com>


commit 7a2a9836424092dbd2807cd81751f1b9de19c26e
Author: Lasse Karstensen <lkarsten at varnish-software.com>
Date:   Tue May 13 11:45:14 2014 +0200

    Output correct tool name.
    
    And avoid hardcoding it, in case we change it in the future
    some time.

diff --git a/lib/libvcc/vmodtool.py b/lib/libvcc/vmodtool.py
index b9d7c95..9dc2c4b 100755
--- a/lib/libvcc/vmodtool.py
+++ b/lib/libvcc/vmodtool.py
@@ -39,6 +39,7 @@
 
 import sys
 import re
+from os.path import basename
 
 if len(sys.argv) == 2:
 	specfile = sys.argv[1]
@@ -69,10 +70,10 @@ def file_header(fo):
         fo.write("""/*
  * NB:  This file is machine generated, DO NOT EDIT!
  *
- * Edit vmod.vcc and run vmod.py instead
+ * Edit vmod.vcc and run %s instead
  */
 
-""")
+""" % basename(__file__))
 
 #######################################################################
 



From lkarsten at varnish-software.com  Tue Jun 24 09:31:46 2014
From: lkarsten at varnish-software.com (Lasse Karstensen)
Date: Tue, 24 Jun 2014 11:31:46 +0200
Subject: [4.0] 2180ad2 Output autogen-msg to .rst; pythonify a bit.
Message-ID: <E1WzN4s-0003DO-UA@project.varnish-software.com>


commit 2180ad2b748996d7550748199db46f4924765bf0
Author: Lasse Karstensen <lkarsten at varnish-software.com>
Date:   Tue May 13 11:46:18 2014 +0200

    Output autogen-msg to .rst; pythonify a bit.
    
    Write a header in generated .rst files that it was generated from the
    .vcc file.
    
    Also softly sprinkled with some python formatting:
    * use a python 2.5 context manager for the filepointer
    * and make the output easier to read by avoiding concatination.

diff --git a/lib/libvcc/vmodtool.py b/lib/libvcc/vmodtool.py
index 9dc2c4b..74fd633 100755
--- a/lib/libvcc/vmodtool.py
+++ b/lib/libvcc/vmodtool.py
@@ -799,16 +799,19 @@ fc.close()
 fh.close()
 
 for suf in ("", ".man"):
-	fr = open("vmod_" + vx[0].nam + suf + ".rst", "w")
-	vx[0].doc_dump(fr, suf)
-
-	if len(copyright) > 0:
-		fr.write("\n")
-		fr.write("COPYRIGHT\n")
-		fr.write("=========\n")
-		fr.write("\n::\n\n")
-		for i in copyright:
-			fr.write("  " + i + "\n")
-		fr.write("\n")
-
-	fr.close()
+	with open("vmod_%s%s.rst" % (vx[0].nam, suf), "w") as fp:
+		fp.write("..\n")
+		fp.write(".. This file was autogenerated by %s. DO NOT EDIT!\n" %
+			 basename(__file__))
+		fp.write("..\n")
+
+		vx[0].doc_dump(fp, suf)
+
+		if len(copyright) > 0:
+			fp.write("\n")
+			fp.write("COPYRIGHT\n")
+			fp.write("=========\n")
+			fp.write("\n::\n\n")
+			for i in copyright:
+				fp.write("  %s\n" % i)
+			fp.write("\n")



From lkarsten at varnish-software.com  Tue Jun 24 09:31:46 2014
From: lkarsten at varnish-software.com (Lasse Karstensen)
Date: Tue, 24 Jun 2014 11:31:46 +0200
Subject: [4.0] fb1e33d Remove text block dividers just added.
Message-ID: <E1WzN4s-0003DZ-W9@project.varnish-software.com>


commit fb1e33d83d70a3f822315815a31bb0f5c9936e6c
Author: Lasse Karstensen <lkarsten at varnish-software.com>
Date:   Tue May 13 11:51:33 2014 +0200

    Remove text block dividers just added.
    
    This made it easier to see where objects started/stopped, but
    vmodtool.py don't believe in this comment format. Especially rst2man
    wasn't impressed, so roll back for now.

diff --git a/lib/libvmod_directors/vmod.vcc b/lib/libvmod_directors/vmod.vcc
index d18151f..f0d9424 100644
--- a/lib/libvmod_directors/vmod.vcc
+++ b/lib/libvmod_directors/vmod.vcc
@@ -81,7 +81,7 @@ Description
 Example
        set req.backend_hint = rrdir.backend();
 
-####################################################################
+
 $Object fallback()
 
 Description
@@ -111,7 +111,7 @@ Description
 Example
        set req.backend_hint = vdir.backend();
 
-####################################################################
+
 $Object random()
 
 Description



From lkarsten at varnish-software.com  Tue Jun 24 09:31:47 2014
From: lkarsten at varnish-software.com (Lasse Karstensen)
Date: Tue, 24 Jun 2014 11:31:47 +0200
Subject: [4.0] 8fb3ea6 Output RST comment with enough linefeeds.
Message-ID: <E1WzN4t-0003Dl-2T@project.varnish-software.com>


commit 8fb3ea686c03fe64bc66da54bbccf4d645343415
Author: Lasse Karstensen <lkarsten at varnish-software.com>
Date:   Tue May 13 13:19:25 2014 +0200

    Output RST comment with enough linefeeds.
    
    Empty RST comments need an extra empty line afterwards, re this output
    from rst2man:
    
        (WARNING/2) Explicit markup ends without a blank line; unexpected unindent

diff --git a/lib/libvcc/vmodtool.py b/lib/libvcc/vmodtool.py
index 74fd633..4d34d07 100755
--- a/lib/libvcc/vmodtool.py
+++ b/lib/libvcc/vmodtool.py
@@ -803,7 +803,7 @@ for suf in ("", ".man"):
 		fp.write("..\n")
 		fp.write(".. This file was autogenerated by %s. DO NOT EDIT!\n" %
 			 basename(__file__))
-		fp.write("..\n")
+		fp.write("..\n\n")
 
 		vx[0].doc_dump(fp, suf)
 



From lkarsten at varnish-software.com  Tue Jun 24 09:31:47 2014
From: lkarsten at varnish-software.com (Lasse Karstensen)
Date: Tue, 24 Jun 2014 11:31:47 +0200
Subject: [4.0] 4141e7a Halt rst2man/rst2html on warnings.
Message-ID: <E1WzN4t-0003Dx-59@project.varnish-software.com>


commit 4141e7a47629ca2dfb6e1adbbc918a29767a7c9a
Author: Lasse Karstensen <lkarsten at varnish-software.com>
Date:   Tue May 13 13:21:49 2014 +0200

    Halt rst2man/rst2html on warnings.
    
    We don't allow compiler warnings in the C code, so I don't see any
    reason why we should allow warnings in the documentation.
    
    Warnings in the documentation are now build errors.

diff --git a/man/Makefile.am b/man/Makefile.am
index e4a0bfe..a3d3077 100644
--- a/man/Makefile.am
+++ b/man/Makefile.am
@@ -25,67 +25,69 @@ dist_man_MANS = \
 
 MAINTAINERCLEANFILES = $(dist_man_MANS)
 
+RST2ANY_FLAGS = --halt=2
+
 varnish-cli.7: $(top_srcdir)/doc/sphinx/reference/varnish-cli.rst
-	${RST2MAN} $(top_srcdir)/doc/sphinx/reference/varnish-cli.rst $@
+	${RST2MAN} $(RST2ANY_FLAGS) $(top_srcdir)/doc/sphinx/reference/varnish-cli.rst $@
 
 varnish-counters.7: vsc2rst
-	./vsc2rst | ${RST2MAN} - $@
+	./vsc2rst | ${RST2MAN} $(RST2ANY_FLAGS) - $@
 
 vcl.7: $(top_srcdir)/doc/sphinx/reference/vcl.rst \
 	$(top_srcdir)/bin/varnishd/builtin.vcl
-	${RST2MAN} $(top_srcdir)/doc/sphinx/reference/vcl.rst $@
+	${RST2MAN} $(RST2ANY_FLAGS) $(top_srcdir)/doc/sphinx/reference/vcl.rst $@
 
 vsl.7: $(top_srcdir)/doc/sphinx/reference/vsl.rst \
 	$(top_srcdir)/lib/libvarnishapi/vsl-tags.rst
-	${RST2MAN} $(top_srcdir)/doc/sphinx/reference/vsl.rst $@
+	${RST2MAN} $(RST2ANY_FLAGS) $(top_srcdir)/doc/sphinx/reference/vsl.rst $@
 
 vsl-query.7: $(top_srcdir)/doc/sphinx/reference/vsl-query.rst
-	${RST2MAN} $(top_srcdir)/doc/sphinx/reference/vsl-query.rst $@
+	${RST2MAN} $(RST2ANY_FLAGS) $(top_srcdir)/doc/sphinx/reference/vsl-query.rst $@
 
 varnishadm.1: $(top_srcdir)/doc/sphinx/reference/varnishadm.rst
-	${RST2MAN} $(top_srcdir)/doc/sphinx/reference/varnishadm.rst $@
+	${RST2MAN} $(RST2ANY_FLAGS) $(top_srcdir)/doc/sphinx/reference/varnishadm.rst $@
 
 varnishd.1: \
 	$(top_srcdir)/doc/sphinx/reference/varnishd.rst \
 	$(top_srcdir)/doc/sphinx/include/params.rst
-	${RST2MAN} $(top_srcdir)/doc/sphinx/reference/varnishd.rst $@
+	${RST2MAN} $(RST2ANY_FLAGS) $(top_srcdir)/doc/sphinx/reference/varnishd.rst $@
 
 varnishncsa.1: \
 	$(top_srcdir)/doc/sphinx/reference/varnishncsa.rst \
 	$(top_srcdir)/doc/sphinx/include/varnishncsa_options.rst \
 	$(top_srcdir)/doc/sphinx/include/varnishncsa_synopsis.rst
-	${RST2MAN} $(top_srcdir)/doc/sphinx/reference/varnishncsa.rst $@
+	${RST2MAN} $(RST2ANY_FLAGS) $(top_srcdir)/doc/sphinx/reference/varnishncsa.rst $@
 
 varnishlog.1: \
 	$(top_srcdir)/doc/sphinx/reference/varnishlog.rst \
 	$(top_srcdir)/doc/sphinx/include/varnishlog_options.rst \
 	$(top_srcdir)/doc/sphinx/include/varnishlog_synopsis.rst
-	${RST2MAN} $(top_srcdir)/doc/sphinx/reference/varnishlog.rst $@
+	${RST2MAN} $(RST2ANY_FLAGS) $(top_srcdir)/doc/sphinx/reference/varnishlog.rst $@
 
 varnishreplay.1: $(top_srcdir)/doc/sphinx/reference/varnishreplay.rst
-	${RST2MAN} $(top_srcdir)/doc/sphinx/reference/varnishreplay.rst $@
+	${RST2MAN} $(RST2ANY_FLAGS) $(top_srcdir)/doc/sphinx/reference/varnishreplay.rst $@
 
 # XXX add _options.rst and _synopsis.rst here when it's been _opt2rst'ed
 varnishstat.1: $(top_srcdir)/doc/sphinx/reference/varnishstat.rst
-	${RST2MAN} $(top_srcdir)/doc/sphinx/reference/varnishstat.rst $@
+	${RST2MAN} $(RST2ANY_FLAGS) $(top_srcdir)/doc/sphinx/reference/varnishstat.rst $@
 
 varnishtest.1: $(top_srcdir)/doc/sphinx/reference/varnishtest.rst
-	${RST2MAN} $(top_srcdir)/doc/sphinx/reference/varnishtest.rst $@
+	${RST2MAN} $(RST2ANY_FLAGS) $(top_srcdir)/doc/sphinx/reference/varnishtest.rst $@
 
 varnishtop.1: \
 	$(top_srcdir)/doc/sphinx/reference/varnishtop.rst \
 	$(top_srcdir)/doc/sphinx/include/varnishtop_options.rst \
 	$(top_srcdir)/doc/sphinx/include/varnishtop_synopsis.rst
-	${RST2MAN} $(top_srcdir)/doc/sphinx/reference/varnishtop.rst $@
+	${RST2MAN} $(RST2ANY_FLAGS) $(top_srcdir)/doc/sphinx/reference/varnishtop.rst $@
 
 varnishhist.1: \
 	$(top_srcdir)/doc/sphinx/reference/varnishhist.rst \
 	$(top_srcdir)/doc/sphinx/include/varnishhist_options.rst \
 	$(top_srcdir)/doc/sphinx/include/varnishhist_synopsis.rst
-	${RST2MAN} $(top_srcdir)/doc/sphinx/reference/varnishhist.rst $@
+	${RST2MAN} $(RST2ANY_FLAGS) $(top_srcdir)/doc/sphinx/reference/varnishhist.rst $@
 
 vmod_std.3: $(top_srcdir)/lib/libvmod_std/vmod_std.man.rst
-	${RST2MAN} $? $@
+	${RST2MAN} $(RST2ANY_FLAGS) $? $@
 
 vmod_directors.3: $(top_srcdir)/lib/libvmod_directors/vmod_directors.man.rst
-	${RST2MAN} $? $@
+	${RST2MAN} $(RST2ANY_FLAGS) $(RST2ANY_FLAGS) $? $@



From lkarsten at varnish-software.com  Tue Jun 24 09:31:47 2014
From: lkarsten at varnish-software.com (Lasse Karstensen)
Date: Tue, 24 Jun 2014 11:31:47 +0200
Subject: [4.0] ebbd828 Delete unreferenced file vcl-intro.rst.
Message-ID: <E1WzN4t-0003EB-7V@project.varnish-software.com>


commit ebbd828839bd4668382d83acc2004b3e370c445f
Author: Lasse Karstensen <lkarsten at varnish-software.com>
Date:   Tue May 13 13:39:38 2014 +0200

    Delete unreferenced file vcl-intro.rst.
    
    Sphinx complains that this file isn't referenced
    anywhere, and a quick glance indicate that the contents
    have been merged into other rst files.

diff --git a/doc/sphinx/users-guide/vcl-intro.rst b/doc/sphinx/users-guide/vcl-intro.rst
deleted file mode 100644
index 095896c..0000000
--- a/doc/sphinx/users-guide/vcl-intro.rst
+++ /dev/null
@@ -1,28 +0,0 @@
-Varnish Configuration Language - VCL
--------------------------------------
-
-Varnish has a great configuration system. Most other systems use
-configuration directives, where you basically turn on and off lots of
-switches. Varnish uses a domain specific language called Varnish
-Configuration Language, or VCL for short. Varnish translates this
-configuration into binary code which is then executed when requests
-arrive.
-
-The VCL files are divided into subroutines. The different subroutines
-are executed at different times. One is executed when we get the
-request, another when files are fetched from the backend server.
-
-Varnish will execute these subroutines of code at different stages of
-its work. At some point you call an action in this subroutine and then
-the execution of that subroutine stops.
-
-If you don't call an action in your subroutine and it reaches the end
-Varnish will execute the built in VCL code. You will see this VCL
-code commented out in default.vcl.
-
-99% of all the changes you'll need to do will be done in two of these
-subroutines. *vcl_recv* and *vcl_backend_response*.
-
-.. _users-guide-vcl_fetch_actions:
-
-



From lkarsten at varnish-software.com  Tue Jun 24 09:31:47 2014
From: lkarsten at varnish-software.com (Lasse Karstensen)
Date: Tue, 24 Jun 2014 11:31:47 +0200
Subject: [4.0] 57a46f0 Put vmod docs back into the documentation.
Message-ID: <E1WzN4t-0003EN-Bf@project.varnish-software.com>


commit 57a46f0229a4fbbb328a9cbbcb76da750e394c66
Author: Lasse Karstensen <lkarsten at varnish-software.com>
Date:   Tue May 13 13:54:15 2014 +0200

    Put vmod docs back into the documentation.
    
    Squashing Sphinx warnings revealed that the vmod_std and
    vmod_directors includes wasn't working.
    
    Move the generated files a bit so Sphinx is happy to load them.
    
    I'm not using include (where relative paths seem to work)
    since that works differently in a TOC listing.

diff --git a/.gitignore b/.gitignore
index ba64e39..b4ae99b 100644
--- a/.gitignore
+++ b/.gitignore
@@ -80,6 +80,7 @@ cscope.*out
 /man/*.3
 /man/*.7
 /doc/sphinx/include
+/doc/sphinx/*/*generated.rst
 /bin/varnish*/varnish*_opt2rst
 /bin/varnishadm/varnishadm
 /bin/varnishd/varnishd
diff --git a/doc/sphinx/Makefile.am b/doc/sphinx/Makefile.am
index ab8d021..0219fac 100644
--- a/doc/sphinx/Makefile.am
+++ b/doc/sphinx/Makefile.am
@@ -213,13 +213,13 @@ include/varnishhist_synopsis.rst: $(top_builddir)/bin/varnishhist/varnishhist_op
 BUILT_SOURCES += include/varnishhist_options.rst \
 	 include/varnishhist_synopsis.rst
 
-include/vmod_std.rst: $(top_builddir)/lib/libvmod_std/vmod_std.rst
+reference/vmod_std.generated.rst: $(top_builddir)/lib/libvmod_std/vmod_std.rst
 	cp $? $@
-BUILT_SOURCES += include/vmod_std.rst
+BUILT_SOURCES += reference/vmod_std.generated.rst
 
-include/vmod_directors.rst: $(top_builddir)/lib/libvmod_directors/vmod_directors.rst
+reference/vmod_directors.generated.rst: $(top_builddir)/lib/libvmod_directors/vmod_directors.rst
 	cp $? $@
-BUILT_SOURCES += include/vmod_directors.rst
+BUILT_SOURCES += reference/vmod_directors.generated.rst
 
 EXTRA_DIST += $(BUILT_SOURCES)
 MAINTAINERCLEANFILES = $(EXTRA_DIST)
diff --git a/doc/sphinx/reference/index.rst b/doc/sphinx/reference/index.rst
index 20ac975..32458a6 100644
--- a/doc/sphinx/reference/index.rst
+++ b/doc/sphinx/reference/index.rst
@@ -21,8 +21,8 @@ The Varnish Reference Manual
 	varnishtop.rst
 	vsm.rst
 	vmod.rst
-	../include/vmod_std.rst
-	../include/vmod_directors.rst
+	vmod_std.generated.rst
+	vmod_directors.generated.rst
 	vsl.rst
 	vsl-query.rst
 



From lkarsten at varnish-software.com  Tue Jun 24 09:31:47 2014
From: lkarsten at varnish-software.com (Lasse Karstensen)
Date: Tue, 24 Jun 2014 11:31:47 +0200
Subject: [4.0] 9acf7dc Increase Sphinx strictness.
Message-ID: <E1WzN4t-0003Ea-Fo@project.varnish-software.com>


commit 9acf7dccf0f768ee265454364fc3a66e8beb5011
Author: Lasse Karstensen <lkarsten at varnish-software.com>
Date:   Tue May 13 13:59:01 2014 +0200

    Increase Sphinx strictness.
    
    Warnings are now considered build errors.
    
    We've been shipping documentation with sections missing
    because we didn't take warnings seriously.
    
    Bonus change: build in parallel, save 82 centiseconds per build!
    (html target on my laptop, ymmv)

diff --git a/doc/sphinx/Makefile.am b/doc/sphinx/Makefile.am
index 0219fac..a5b9718 100644
--- a/doc/sphinx/Makefile.am
+++ b/doc/sphinx/Makefile.am
@@ -3,7 +3,7 @@
 
 # You can set these variables from the command line.
 SPHINXOPTS    =
-SPHINXBUILD   = sphinx-build
+SPHINXBUILD   = sphinx-build -j4 -W -N -n
 PAPER         = a4
 BUILDDIR      = build
 



From lkarsten at varnish-software.com  Tue Jun 24 09:31:47 2014
From: lkarsten at varnish-software.com (Lasse Karstensen)
Date: Tue, 24 Jun 2014 11:31:47 +0200
Subject: [4.0] 57136c2 Disable parallel build due to old sphinx.
Message-ID: <E1WzN4t-0003En-JU@project.varnish-software.com>


commit 57136c2322315ed88e0bacbf6df6759ab8a10287
Author: Lasse Karstensen <lkarsten at varnish-software.com>
Date:   Tue May 13 14:10:41 2014 +0200

    Disable parallel build due to old sphinx.
    
    The sphinx version on git.varnish-cache.org is v1.1.8 which
    hasn't got -j.
    
    In sphinx v1.2.2 (debian jessie) it is in place.
    
    Guess we'll have to live with those extra 820 milliseconds
    per build for now.

diff --git a/doc/sphinx/Makefile.am b/doc/sphinx/Makefile.am
index a5b9718..27f8523 100644
--- a/doc/sphinx/Makefile.am
+++ b/doc/sphinx/Makefile.am
@@ -3,7 +3,7 @@
 
 # You can set these variables from the command line.
 SPHINXOPTS    =
-SPHINXBUILD   = sphinx-build -j4 -W -N -n
+SPHINXBUILD   = sphinx-build -W -N -n
 PAPER         = a4
 BUILDDIR      = build
 



From lkarsten at varnish-software.com  Tue Jun 24 09:31:47 2014
From: lkarsten at varnish-software.com (Lasse Karstensen)
Date: Tue, 24 Jun 2014 11:31:47 +0200
Subject: [4.0] d2358fd Remove remaining references to vcl-intro.rst.
Message-ID: <E1WzN4t-0003Ew-Mf@project.varnish-software.com>


commit d2358fd4b7925129f2003c415e86fc24a45b6694
Author: Lasse Karstensen <lkarsten at varnish-software.com>
Date:   Tue May 13 14:42:41 2014 +0200

    Remove remaining references to vcl-intro.rst.
    
    These hard-code references were forgotten when I
    removed this file in ebbd828.

diff --git a/doc/sphinx/Makefile.am b/doc/sphinx/Makefile.am
index 27f8523..e714383 100644
--- a/doc/sphinx/Makefile.am
+++ b/doc/sphinx/Makefile.am
@@ -166,7 +166,6 @@ EXTRA_DIST = \
 	users-guide/vcl-examples.rst \
 	users-guide/vcl-hashing.rst \
 	users-guide/vcl-inline-c.rst \
-	users-guide/vcl-intro.rst \
 	users-guide/vcl-grace.rst \
 	users-guide/vcl-syntax.rst \
 	users-guide/vcl-variables.rst \
diff --git a/doc/sphinx/Makefile.phk b/doc/sphinx/Makefile.phk
index fbfc35f..7e43fef 100644
--- a/doc/sphinx/Makefile.phk
+++ b/doc/sphinx/Makefile.phk
@@ -176,7 +176,6 @@ EXTRA_DIST = \
 	users-guide/vcl-examples.rst \
 	users-guide/vcl-hashing.rst \
 	users-guide/vcl-inline-c.rst \
-	users-guide/vcl-intro.rst \
 	users-guide/vcl-saint-and-grace.rst \
 	users-guide/vcl-syntax.rst \
 	users-guide/vcl-variables.rst \



From lkarsten at varnish-software.com  Tue Jun 24 09:31:47 2014
From: lkarsten at varnish-software.com (Lasse Karstensen)
Date: Tue, 24 Jun 2014 11:31:47 +0200
Subject: [4.0] 96c0ae4 Add option parser, error handling and strictness.
Message-ID: <E1WzN4t-0003F7-QL@project.varnish-software.com>


commit 96c0ae47a0cd5d5325fae83f3b3cb64b8b5553de
Author: Lasse Karstensen <lkarsten at varnish-software.com>
Date:   Tue May 13 16:30:33 2014 +0200

    Add option parser, error handling and strictness.
    
    This is one of those tiny jobs you start on (add --strict),
    and then it just goes downhill from there.
    
    Main changes:
    * Use optparse to parse command line arguments.
    * With --strict it aborts if a parse or format error is seen.
    * A basic test case for the parser has been added.
    
    No changes were made to the parser or output format.
    
    Adding the test case meant reordering the execution flow.
    
    Since Varnish code standard is using tabs, I've kept this to my
    best ability. Pylint is not happy about this and give the code
    a score of -4 out of 10 points. :-)

diff --git a/lib/libvcc/vmodtool.py b/lib/libvcc/vmodtool.py
index 4d34d07..3319199 100755
--- a/lib/libvcc/vmodtool.py
+++ b/lib/libvcc/vmodtool.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 #-
 # Copyright (c) 2010-2014 Varnish Software AS
 # All rights reserved.
@@ -26,7 +26,7 @@
 # OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 # SUCH DAMAGE.
 #
-# Read the vmod.vcc file and produce the vmod.h and vmod.c files.
+# Read the vmod.vcc file (inputvcc) and produce the vmod.h and vmod.c files.
 #
 # vmod.h contains the prototypes for the published functions, the module
 # C-code should include this file to ensure type-consistency.
@@ -39,12 +39,12 @@
 
 import sys
 import re
-from os.path import basename
+import optparse
+import unittest
+from os import unlink
+from os.path import dirname, basename, realpath, exists
+from pprint import pprint, pformat
 
-if len(sys.argv) == 2:
-	specfile = sys.argv[1]
-else:
-	specfile = "vmod.vcc"
 
 ctypes = {
 	'BACKEND':	"VCL_BACKEND",
@@ -66,8 +66,8 @@ ctypes = {
 
 #######################################################################
 
-def file_header(fo):
-        fo.write("""/*
+def write_file_header(fo):
+	fo.write("""/*
  * NB:  This file is machine generated, DO NOT EDIT!
  *
  * Edit vmod.vcc and run %s instead
@@ -80,13 +80,29 @@ def file_header(fo):
 def is_c_name(s):
 	return None != re.match("^[a-z][a-z0-9_]*$", s)
 
+
+class ParseError(Exception):
+	"An error reading the input file."
+	pass
+
+class FormatError(Exception):
+	"""
+		Raised if the content of the (otherwise well-formed) input file
+		is invalid.
+  """
+	def __init__(self, msg, details):
+		self.msg = msg
+		self.details = details
+		Exception.__init__(self)
+
+
 #######################################################################
 
 class token(object):
-	def __init__(self, ln, ch, str):
+	def __init__(self, ln, ch, tokstr):
 		self.ln = ln
 		self.ch = ch
-		self.str = str
+		self.str = tokstr
 
 	def __repr__(self):
 		return "<@%d \"%s\">" % (self.ln, self.str)
@@ -96,7 +112,7 @@ class token(object):
 class vmod(object):
 	def __init__(self, nam, dnam, sec):
 		if not is_c_name(nam):
-			raise Exception("Module name '%s' is illegal" % nam)
+			raise ParseError("Module name '%s' is illegal", nam)
 		self.nam = nam
 		self.dnam = dnam
 		self.sec = sec
@@ -108,9 +124,9 @@ class vmod(object):
 
 	def set_init(self, nam):
 		if self.init != None:
-			raise Exception("Module %s already has Init" % self.nam)
+			raise ParseError("Module %s already has Init", self.nam)
 		if not is_c_name(nam):
-			raise Exception("Init name '%s' is illegal" % nam)
+			raise ParseError("Init name '%s' is illegal", nam)
 		self.init = nam
 
 	def add_func(self, fn):
@@ -159,7 +175,7 @@ class vmod(object):
 		cs = self.c_struct()
 		fo.write(cs + ';\n')
 
-		vfn = 'Vmod_' + self.nam + '_Func';
+		vfn = 'Vmod_' + self.nam + '_Func'
 
 		fo.write("extern const struct " + vfn + " " + vfn + ';\n')
 		fo.write("const struct " + vfn + " " + vfn + ' =')
@@ -286,8 +302,8 @@ class func(object):
 		#if not is_c_name(nam):
 		#	raise Exception("Func name '%s' is illegal" % nam)
 		if retval not in ctypes:
-			raise Exception(
-			    "Return type '%s' not a valid type" % retval)
+			raise TypeError(
+			    "Return type '%s' not a valid type", retval)
 		self.nam = nam
 		self.cnam = nam.replace(".", "_")
 		self.al = al
@@ -400,6 +416,7 @@ class obj(object):
 		self.fini = None
 		self.methods = list()
 		self.doc_str = []
+		self.st = None
 
 	def fixup(self, modnam):
 		assert self.nam != None
@@ -431,7 +448,7 @@ class obj(object):
 	def c_proto(self, fo):
 		fo.write(self.st + ";\n")
 		self.init.c_proto(fo)
-		self.fini.c_proto(fo, fini = True)
+		self.fini.c_proto(fo, fini=True)
 		for m in self.methods:
 			m.c_proto(fo)
 
@@ -491,7 +508,7 @@ class obj(object):
 #######################################################################
 
 class arg(object):
-	def __init__(self, typ, nam = None, det = None):
+	def __init__(self, typ, nam=None, det=None):
 		self.nam = nam
 		self.typ = typ
 		self.det = det
@@ -512,7 +529,7 @@ class arg(object):
 def parse_enum2(tl):
 	t = tl.get_token()
 	if t.str != "{":
-		raise Exception("expected \"{\"")
+		raise ParseError("expected \"{\"")
 	s = "ENUM\\0"
 	t = None
 	while True:
@@ -527,7 +544,7 @@ def parse_enum2(tl):
 		elif t.str == "}":
 			break
 		else:
-			raise Exception(
+			raise ParseError(
 			    "Expected \"}\" or \",\" not \"%s\"" % t.str)
 	s += "\\0"
 	return arg("ENUM", det=s)
@@ -549,13 +566,13 @@ def parse_module(tl):
 #
 #
 
-def parse_func(tl, rt_type = None, obj=None):
+def parse_func(tl, rt_type=None, obj=None):
 	al = list()
 	if rt_type == None:
 		t = tl.get_token()
 		rt_type = t.str
 	if rt_type not in ctypes:
-		raise Exception(
+		raise TypeError(
 		    "Return type '%s' not a valid type" % rt_type)
 
 	t = tl.get_token()
@@ -563,11 +580,11 @@ def parse_func(tl, rt_type = None, obj=None):
 	if obj != None and fname[0] == "." and is_c_name(fname[1:]):
 		fname = obj + fname
 	elif not is_c_name(fname):
-		raise Exception("Function name '%s' is illegal" % fname)
+		raise ParseError("Function name '%s' is illegal", fname)
 
 	t = tl.get_token()
 	if t.str != "(":
-		raise Exception("Expected \"(\" got \"%s\"", t.str)
+		raise ParseError("Expected \"(\" got \"%s\"", t.str)
 
 	t = None
 	while True:
@@ -582,7 +599,7 @@ def parse_func(tl, rt_type = None, obj=None):
 		elif t.str == ")":
 			break
 		else:
-			raise Exception("ARG? %s" % t.str)
+			raise Exception("ARG? %s", t.str)
 		t = tl.get_token()
 		if is_c_name(t.str):
 			al[-1].nam = t.str
@@ -592,10 +609,10 @@ def parse_func(tl, rt_type = None, obj=None):
 		elif t.str == ")":
 			break
 		else:
-			raise Exception(
-			    "Expceted \")\" or \",\" not \"%s\"" % t.str)
+			raise ParseError(
+			    "Expected \")\" or \",\" not \"%s\"" % t.str)
 	if t.str != ")":
-		raise Exception("End Of Input looking for ')'")
+		raise ParseError("End Of Input looking for ')'")
 	f = func(fname, rt_type, al)
 
 	return f
@@ -612,7 +629,7 @@ def parse_obj(tl):
 
 
 #######################################################################
-# A section of the specfile, starting at a keyword
+# A section of the inputvcc, starting at a keyword
 
 class file_section(object):
 	def __init__(self):
@@ -633,7 +650,7 @@ class file_section(object):
 		return None
 
 	def more_tokens(self):
-		ln,l = self.l.pop(0)
+		ln, l = self.l.pop(0)
 		if l == "":
 			return
 		l = re.sub("[ \t]*#.*$", "", l)
@@ -669,19 +686,23 @@ class file_section(object):
 			vx.append(o)
 		elif t.str == "$Method":
 			if len(vx) != 2:
-				raise Exception("$Method outside $Object")
-			o = parse_func(self, obj = vx[1].nam)
+				raise FormatError("$Method outside $Object", "")
+			o = parse_func(self, obj=vx[1].nam)
 			vx[1].add_method(o)
 		else:
-			raise Exception("Unknown keyword: " + t.str)
+			raise FormatError("Unknown keyword: %s" % t.str, "")
+
 		assert len(self.tl) == 0
-		if o == None:
-			print("Warning:")
-			print("%s description is not included in .rst:" %t0)
-			for ln,i in self.l:
-				print("\t", i)
+		if o is None and len(self.l) > 0:
+			m = "%s description is not included in .rst" % t0
+			details = pformat(self.l)
+			if opts.strict:
+				raise FormatError(m, details)
+			else:
+				print("WARNING: %s:" % m, file=sys.stderr)
+				print(details, file=sys.stderr)
 		else:
-			for ln,i in self.l:
+			for ln, i in self.l:
 				o.doc(i)
 
 #######################################################################
@@ -704,86 +725,104 @@ def polish(l):
 		return True
 	return False
 
-#######################################################################
-# Read the file in
-
-f = open(specfile, "r")
-lines = []
-for i in f:
-	lines.append(i.rstrip())
-f.close()
-ln = 0
 
-#######################################################################
-# First collect the copyright:  All initial lines starting with '#'
+class SimpleTestCase(unittest.TestCase):
+	def test_included_vccs(self):
+		from tempfile import mktemp
+		from glob import glob
+		tmpfile = mktemp()
+		for inputfile in glob(dirname(realpath(__file__)) + "../libvmod_*/vmod.vcc"):
+			runmain(inputvcc, outputname=tmpfile)
+			for suffix in [".c", ".h"]:
+				unlink(tmpfile + suffix)
 
-copyright = []
-while len(lines[0]) > 0 and lines[0][0] == "#":
-	ln += 1
-	copyright.append(lines.pop(0))
-
-if len(copyright) > 0:
-	if copyright[0] == "#-":
-		copyright = [ ]
-	else:
-		while polish(copyright):
-			continue
-
-if False:
-	for i in copyright:
-		print("(C)\t", i)
 
 #######################################################################
-# Break into sections
-
-keywords = {
-	"$Module":	True,
-	"$Function":	True,
-	"$Object":	True,
-	"$Method":	True,
-	"$Init":	True,
-}
-
-sl = []
-sc = file_section()
-sl.append(sc)
-while len(lines) > 0:
-	ln += 1
-	l = lines.pop(0)
-	j = l.split()
-	if len(j) > 0 and j[0] in keywords:
-		sc = file_section()
-		sl.append(sc)
-	sc.add_line(ln,l)
-
-#######################################################################
-# Parse each section
-
-first = True
-
-vx = []
-for i in sl:
-	i.parse(vx)
-	assert len(i.tl) == 0
-
-#######################################################################
-# Parsing done, now process
-#
-
-fc = open("vcc_if.c", "w")
-fh = open("vcc_if.h", "w")
-
-file_header(fc)
-file_header(fh)
-
-fh.write('struct vrt_ctx;\n')
-fh.write('struct VCL_conf;\n')
-fh.write('struct vmod_priv;\n')
-fh.write("\n");
-
-vx[0].c_proto(fh)
-
-fc.write("""#include "config.h"
+def runmain(inputvcc, outputname="vcc_if"):
+	# Read the file in
+	lines = []
+	with open(inputvcc, "r") as fp:
+		for i in fp:
+			lines.append(i.rstrip())
+	ln = 0
+
+	#######################################################################
+	# First collect the copyright:  All initial lines starting with '#'
+
+	copyright = []
+	while len(lines[0]) > 0 and lines[0][0] == "#":
+		ln += 1
+		copyright.append(lines.pop(0))
+
+	if len(copyright) > 0:
+		if copyright[0] == "#-":
+			copyright = []
+		else:
+			while polish(copyright):
+				continue
+
+	if False:
+		for i in copyright:
+			print("(C)\t", i)
+
+	#######################################################################
+	# Break into sections
+
+	keywords = {
+		"$Module":	True,
+		"$Function":	True,
+		"$Object":	True,
+		"$Method":	True,
+		"$Init":	True,
+	}
+
+	sl = []
+	sc = file_section()
+	sl.append(sc)
+	while len(lines) > 0:
+		ln += 1
+		l = lines.pop(0)
+		j = l.split()
+		if len(j) > 0 and j[0] in keywords:
+			sc = file_section()
+			sl.append(sc)
+		sc.add_line(ln, l)
+
+	#######################################################################
+	# Parse each section
+
+	try:
+		vx = []
+		for i in sl:
+			i.parse(vx)
+			assert len(i.tl) == 0
+	except ParseError as e:
+		print("ERROR: Parse error reading \"%s\":" % inputvcc)
+		pprint(str(e))
+		exit(-1)
+	except FormatError as e:
+		print("ERROR: Format error reading \"%s\": %s" % (inputvcc, pformat(e.msg)))
+		print(e.details)
+		exit(-2)
+
+	#######################################################################
+	# Parsing done, now process
+	#
+
+	fc = open("%s.c" % outputname, "w")
+	fh = open("%s.h" % outputname, "w")
+
+	write_file_header(fc)
+	write_file_header(fh)
+
+	fh.write('struct vrt_ctx;\n')
+	fh.write('struct VCL_conf;\n')
+	fh.write('struct vmod_priv;\n')
+	fh.write("\n")
+
+	vx[0].c_proto(fh)
+
+	fc.write("""#include "config.h"
 
 #include "vrt.h"
 #include "vcc_if.h"
@@ -792,26 +831,55 @@ fc.write("""#include "config.h"
 
 """)
 
-vx[0].c_typedefs(fc)
-vx[0].c_vmod(fc)
-
-fc.close()
-fh.close()
-
-for suf in ("", ".man"):
-	with open("vmod_%s%s.rst" % (vx[0].nam, suf), "w") as fp:
-		fp.write("..\n")
-		fp.write(".. This file was autogenerated by %s. DO NOT EDIT!\n" %
-			 basename(__file__))
-		fp.write("..\n\n")
-
-		vx[0].doc_dump(fp, suf)
-
-		if len(copyright) > 0:
-			fp.write("\n")
-			fp.write("COPYRIGHT\n")
-			fp.write("=========\n")
-			fp.write("\n::\n\n")
-			for i in copyright:
-				fp.write("  %s\n" % i)
-			fp.write("\n")
+	vx[0].c_typedefs(fc)
+	vx[0].c_vmod(fc)
+
+	fc.close()
+	fh.close()
+
+	for suf in ("", ".man"):
+		with open("vmod_%s%s.rst" % (vx[0].nam, suf), "w") as fp:
+			fp.write("..\n")
+			fp.write(".. This file was autogenerated by %s. DO NOT EDIT!\n" %
+				 basename(__file__))
+			fp.write("..\n\n")
+
+			vx[0].doc_dump(fp, suf)
+
+			if len(copyright) > 0:
+				fp.write("\n")
+				fp.write("COPYRIGHT\n")
+				fp.write("=========\n")
+				fp.write("\n::\n\n")
+				for i in copyright:
+					fp.write("  %s\n" % i)
+				fp.write("\n")
+
+
+if __name__ == "__main__":
+	usagetext = "Usage: %prog [options] <vmod.vcc>"
+	oparser = optparse.OptionParser(usage=usagetext)
+
+	oparser.add_option('-N', '--strict', action='store_true', default=False,
+			    help="Be strict when parsing input file. (vmod.vcc)")
+	oparser.add_option('', '--runtests', action='store_true', default=False,
+			    dest="runtests", help=optparse.SUPPRESS_HELP)
+	(opts, args) = oparser.parse_args()
+
+	if opts.runtests:
+		del sys.argv[1]  # Pop off --runtests, pass remaining to unittest.
+		unittest.main()
+		exit()
+
+	inputvcc = None
+	if len(args) == 1 and exists(args[0]):
+		inputvcc = args[0]
+	elif exists("vmod.vcc"):
+		if not inputvcc:
+		    inputvcc = "vmod.vcc"
+	else:
+		print("ERROR: No vmod.vcc file supplied or found.", file=sys.stderr)
+		oparser.print_help()
+		exit(-1)
+
+	runmain(inputvcc)



From lkarsten at varnish-software.com  Tue Jun 24 09:31:47 2014
From: lkarsten at varnish-software.com (Lasse Karstensen)
Date: Tue, 24 Jun 2014 11:31:47 +0200
Subject: [4.0] 92c9466 Remove unused description.
Message-ID: <E1WzN4t-0003FH-So@project.varnish-software.com>


commit 92c94665e8d63ff15b97dcd5940c2c9009596f1d
Author: Lasse Karstensen <lkarsten at varnish-software.com>
Date:   Tue May 13 18:04:21 2014 +0200

    Remove unused description.
    
    This description isn't used anywhere, and it doesn't contain
    much useful information.

diff --git a/lib/libvmod_debug/vmod.vcc b/lib/libvmod_debug/vmod.vcc
index 71a98d6..6872775 100644
--- a/lib/libvmod_debug/vmod.vcc
+++ b/lib/libvmod_debug/vmod.vcc
@@ -33,10 +33,8 @@ DESCRIPTION
 This vmod is used to develop, test and debug the various aspects
 of VMOD handling in Varnish.
 
-$Init init_function
-
-Call this whenever a VCL is loaded which imports this vmod.
 
+$Init init_function
 $Function VOID panic(STRING_LIST)
 
 Don't.



From lkarsten at varnish-software.com  Tue Jun 24 09:31:47 2014
From: lkarsten at varnish-software.com (Lasse Karstensen)
Date: Tue, 24 Jun 2014 11:31:47 +0200
Subject: [4.0] 30f6671 Use vmodtool.py in strict mode by default.
Message-ID: <E1WzN4t-0003FQ-Uw@project.varnish-software.com>


commit 30f66716781e8af2c45ad06505fc0ca3823fed5a
Author: Lasse Karstensen <lkarsten at varnish-software.com>
Date:   Tue May 13 18:06:06 2014 +0200

    Use vmodtool.py in strict mode by default.
    
    Use the strict parsing mode (still infant) to be sure
    that our own vmod.vcc files are valid.

diff --git a/lib/libvmod_debug/Makefile.am b/lib/libvmod_debug/Makefile.am
index f856dde..d189102 100644
--- a/lib/libvmod_debug/Makefile.am
+++ b/lib/libvmod_debug/Makefile.am
@@ -9,6 +9,8 @@ AM_CPPFLAGS = \
 vmoddir = $(pkglibdir)/vmods
 vmod_srcdir = $(top_srcdir)/lib/libvmod_debug
 vmodtool = $(top_srcdir)/lib/libvcc/vmodtool.py
+vmodtoolargs = --strict
+
 noinst_LTLIBRARIES = libvmod_debug.la
 
 libvmod_debug_la_LDFLAGS = $(AM_LDFLAGS) -module -export-dynamic -avoid-version -shared -rpath /nowhere
@@ -25,7 +27,7 @@ nodist_libvmod_debug_la_SOURCES = \
 vmod_debug.lo vmod_debug_obj.lo: vcc_if.h
 
 vcc_if.c vcc_if.h vmod_debug.rst vmod_debug.man.rst: $(vmodtool) $(vmod_srcdir)/vmod.vcc
-	@PYTHON@ $(vmodtool) $(vmod_srcdir)/vmod.vcc
+	@PYTHON@ $(vmodtool) $(vmodtoolargs) $(vmod_srcdir)/vmod.vcc
 
 EXTRA_DIST = vmod.vcc
 
diff --git a/lib/libvmod_directors/Makefile.am b/lib/libvmod_directors/Makefile.am
index 583b1f7..64b72a8 100644
--- a/lib/libvmod_directors/Makefile.am
+++ b/lib/libvmod_directors/Makefile.am
@@ -9,6 +9,7 @@ AM_CPPFLAGS = \
 vmoddir = $(pkglibdir)/vmods
 vmod_srcdir = $(top_srcdir)/lib/libvmod_directors
 vmodtool = $(top_srcdir)/lib/libvcc/vmodtool.py
+vmodtoolargs = --strict
 vmod_LTLIBRARIES = libvmod_directors.la
 
 libvmod_directors_la_LDFLAGS = $(AM_LDFLAGS) -module -export-dynamic -avoid-version -shared
@@ -29,7 +30,8 @@ nodist_libvmod_directors_la_SOURCES = \
 vdir.lo fall_back.lo hash.lo random.lo round_robin.lo: vcc_if.h
 
 vcc_if.c vcc_if.h vmod_directors.rst vmod_directors.man.rst: $(vmodtool) $(vmod_srcdir)/vmod.vcc
-	@PYTHON@ $(vmodtool) $(vmod_srcdir)/vmod.vcc
+	@PYTHON@ $(vmodtool) $(vmodtoolargs) $(vmod_srcdir)/vmod.vcc
+
 
 EXTRA_DIST = vmod.vcc
 
diff --git a/lib/libvmod_std/Makefile.am b/lib/libvmod_std/Makefile.am
index ff8466e..c005b85 100644
--- a/lib/libvmod_std/Makefile.am
+++ b/lib/libvmod_std/Makefile.am
@@ -10,6 +10,7 @@ AM_CPPFLAGS = \
 vmoddir = $(pkglibdir)/vmods
 vmod_srcdir = $(top_srcdir)/lib/libvmod_std
 vmodtool = $(top_srcdir)/lib/libvcc/vmodtool.py
+vmodtoolargs = --strict
 vmod_LTLIBRARIES = libvmod_std.la
 
 libvmod_std_la_LDFLAGS = $(AM_LDFLAGS) -module -export-dynamic -avoid-version -shared
@@ -27,7 +28,8 @@ nodist_libvmod_std_la_SOURCES = \
 vmod_std.lo vmod_std_fileread.lo vmod_std_conversions.lo: vcc_if.h
 
 vcc_if.c vcc_if.h vmod_std.rst vmod_std.man.rst: $(vmodtool) $(vmod_srcdir)/vmod.vcc
-	@PYTHON@ $(vmodtool) $(vmod_srcdir)/vmod.vcc
+	@PYTHON@ $(vmodtool) $(vmodtoolargs) $(vmod_srcdir)/vmod.vcc
+
 
 EXTRA_DIST = vmod.vcc
 



From lkarsten at varnish-software.com  Tue Jun 24 09:31:48 2014
From: lkarsten at varnish-software.com (Lasse Karstensen)
Date: Tue, 24 Jun 2014 11:31:48 +0200
Subject: [4.0] bbf9d33 Walk back Python 3 changes, use python2 explicitly.
Message-ID: <E1WzN4u-0003Fa-1Z@project.varnish-software.com>


commit bbf9d33084fef3070b56e207582714baef924fcf
Author: Lasse Karstensen <lkarsten at varnish-software.com>
Date:   Wed May 14 10:15:09 2014 +0200

    Walk back Python 3 changes, use python2 explicitly.
    
    None of our automatic build environments have Python 3 currently.

diff --git a/lib/libvcc/vmodtool.py b/lib/libvcc/vmodtool.py
index 3319199..4cc13e1 100755
--- a/lib/libvcc/vmodtool.py
+++ b/lib/libvcc/vmodtool.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python3
+#!/usr/bin/env python2
 #-
 # Copyright (c) 2010-2014 Varnish Software AS
 # All rights reserved.
@@ -699,8 +699,8 @@ class file_section(object):
 			if opts.strict:
 				raise FormatError(m, details)
 			else:
-				print("WARNING: %s:" % m, file=sys.stderr)
-				print(details, file=sys.stderr)
+				print >>sys.stderr, "WARNING: %s:" % m
+				print >>sys.stderr, details
 		else:
 			for ln, i in self.l:
 				o.doc(i)
@@ -797,12 +797,12 @@ def runmain(inputvcc, outputname="vcc_if"):
 			i.parse(vx)
 			assert len(i.tl) == 0
 	except ParseError as e:
-		print("ERROR: Parse error reading \"%s\":" % inputvcc)
+		print "ERROR: Parse error reading \"%s\":" % inputvcc
 		pprint(str(e))
 		exit(-1)
 	except FormatError as e:
-		print("ERROR: Format error reading \"%s\": %s" % (inputvcc, pformat(e.msg)))
-		print(e.details)
+		print "ERROR: Format error reading \"%s\": %s" % (inputvcc, pformat(e.msg))
+		print e.details
 		exit(-2)
 
 	#######################################################################
@@ -878,7 +878,7 @@ if __name__ == "__main__":
 		if not inputvcc:
 		    inputvcc = "vmod.vcc"
 	else:
-		print("ERROR: No vmod.vcc file supplied or found.", file=sys.stderr)
+		print >>sys.stderr, "ERROR: No vmod.vcc file supplied or found."
 		oparser.print_help()
 		exit(-1)
 



From lkarsten at varnish-software.com  Tue Jun 24 09:31:48 2014
From: lkarsten at varnish-software.com (Lasse Karstensen)
Date: Tue, 24 Jun 2014 11:31:48 +0200
Subject: [4.0] babbaa5 Fix glob that found no input files.
Message-ID: <E1WzN4u-0003Fj-3h@project.varnish-software.com>


commit babbaa5318027849e423ed4ed2d9f62fcc0bd8fd
Author: Lasse Karstensen <lkarsten at varnish-software.com>
Date:   Wed May 14 10:19:34 2014 +0200

    Fix glob that found no input files.

diff --git a/lib/libvcc/vmodtool.py b/lib/libvcc/vmodtool.py
index 4cc13e1..6e875bf 100755
--- a/lib/libvcc/vmodtool.py
+++ b/lib/libvcc/vmodtool.py
@@ -731,8 +731,9 @@ class SimpleTestCase(unittest.TestCase):
 		from tempfile import mktemp
 		from glob import glob
 		tmpfile = mktemp()
-		for inputfile in glob(dirname(realpath(__file__)) + "../libvmod_*/vmod.vcc"):
-			runmain(inputvcc, outputname=tmpfile)
+		bdir = dirname(realpath(__file__))
+		for inputfile in glob(bdir + "/../libvmod_*/vmod.vcc"):
+			runmain(inputfile, outputname=tmpfile)
 			for suffix in [".c", ".h"]:
 				unlink(tmpfile + suffix)
 



From lkarsten at varnish-software.com  Tue Jun 24 09:31:48 2014
From: lkarsten at varnish-software.com (Lasse Karstensen)
Date: Tue, 24 Jun 2014 11:31:48 +0200
Subject: [4.0] 45362c9 Support both python2 and python3.
Message-ID: <E1WzN4u-0003Fs-5m@project.varnish-software.com>


commit 45362c9c1ad0aa2b949d5aa49c2c8fa39d363565
Author: Lasse Karstensen <lkarsten at varnish-software.com>
Date:   Wed May 14 10:26:12 2014 +0200

    Support both python2 and python3.

diff --git a/lib/libvcc/vmodtool.py b/lib/libvcc/vmodtool.py
index 6e875bf..25e0624 100755
--- a/lib/libvcc/vmodtool.py
+++ b/lib/libvcc/vmodtool.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python
 #-
 # Copyright (c) 2010-2014 Varnish Software AS
 # All rights reserved.
@@ -37,6 +37,9 @@
 # as a string, suitable for inclusion in the C-source of the compile VCL
 # program.
 
+# This script should work with both Python 2 and Python 3.
+from __future__ import print_function
+
 import sys
 import re
 import optparse
@@ -45,7 +48,6 @@ from os import unlink
 from os.path import dirname, basename, realpath, exists
 from pprint import pprint, pformat
 
-
 ctypes = {
 	'BACKEND':	"VCL_BACKEND",
 	'BOOL':		"VCL_BOOL",
@@ -89,7 +91,7 @@ class FormatError(Exception):
 	"""
 		Raised if the content of the (otherwise well-formed) input file
 		is invalid.
-  """
+	"""
 	def __init__(self, msg, details):
 		self.msg = msg
 		self.details = details
@@ -699,8 +701,8 @@ class file_section(object):
 			if opts.strict:
 				raise FormatError(m, details)
 			else:
-				print >>sys.stderr, "WARNING: %s:" % m
-				print >>sys.stderr, details
+				print("WARNING: %s:" % m, file=sys.stderr)
+				print(details, file=sys.stderr)
 		else:
 			for ln, i in self.l:
 				o.doc(i)
@@ -798,12 +800,12 @@ def runmain(inputvcc, outputname="vcc_if"):
 			i.parse(vx)
 			assert len(i.tl) == 0
 	except ParseError as e:
-		print "ERROR: Parse error reading \"%s\":" % inputvcc
+		print("ERROR: Parse error reading \"%s\":" % inputvcc)
 		pprint(str(e))
 		exit(-1)
 	except FormatError as e:
-		print "ERROR: Format error reading \"%s\": %s" % (inputvcc, pformat(e.msg))
-		print e.details
+		print("ERROR: Format error reading \"%s\": %s" % (inputvcc, pformat(e.msg)))
+		print(e.details)
 		exit(-2)
 
 	#######################################################################
@@ -879,7 +881,7 @@ if __name__ == "__main__":
 		if not inputvcc:
 		    inputvcc = "vmod.vcc"
 	else:
-		print >>sys.stderr, "ERROR: No vmod.vcc file supplied or found."
+		print("ERROR: No vmod.vcc file supplied or found.", file=sys.stderr)
 		oparser.print_help()
 		exit(-1)
 



From lkarsten at varnish-software.com  Tue Jun 24 09:31:48 2014
From: lkarsten at varnish-software.com (Lasse Karstensen)
Date: Tue, 24 Jun 2014 11:31:48 +0200
Subject: [4.0] 1929299 Load balancing example should use new syntax.
Message-ID: <E1WzN4u-0003G3-7x@project.varnish-software.com>


commit 192929994b4b4d72de3ba5c3085fc196c2bb0c1b
Author: Lasse Karstensen <lkarsten at varnish-software.com>
Date:   Thu May 15 16:34:03 2014 +0200

    Load balancing example should use new syntax.
    
    One of the examples was still referring to the old way of defining
    directors, which I am sure is confusing for the new users this
    document is meant for.
    
    Noticed by: coredump on irc.
    
    Used the occation to do some minor scrubbing of the text.

diff --git a/doc/sphinx/users-guide/vcl-backends.rst b/doc/sphinx/users-guide/vcl-backends.rst
index be50e1c..4a83b75 100644
--- a/doc/sphinx/users-guide/vcl-backends.rst
+++ b/doc/sphinx/users-guide/vcl-backends.rst
@@ -71,7 +71,7 @@ Now we need tell Varnish where to send the difference URL. Lets look at `vcl_rec
 It's quite simple, really. Lets stop and think about this for a
 moment. As you can see you can define how you choose backends based on
 really arbitrary data. You want to send mobile devices to a different
-backend? No problem. ``if (req.User-agent ~ /mobile/) ..`` should do the
+backend? No problem. ``if (req.http.User-agent ~ /mobile/) ..`` should do the
 trick.
 
 
@@ -79,7 +79,7 @@ Backends and virtual hosts in Varnish
 -------------------------------------
 
 Varnish fully supports virtual hosts. They might however work in a somewhat
-counter intuitive fashion since they are never declared
+counter-intuitive fashion since they are never declared
 explicitly. You set up the routing of incoming HTTP requests in
 `vcl_recv`. If you want this routing to be done on the basis of virtual
 hosts you just need to inspect `req.http.host`.
@@ -155,8 +155,8 @@ Checks come into play.
 Health checks
 -------------
 
-Lets set up a director with two backends and health checks. First lets
-define the backends.::
+Lets set up a director with two backends and health checks. First let
+us define the backends.::
 
     backend server1 {
         .host = "server1.example.com";
@@ -204,29 +204,28 @@ initial
     Defaults to the same amount as the threshold.
 
 
-.. XXX: Where and why? benc
-
 Now we define the 'director'::
 
+
     import directors;
 
-    director example_director round-robin {
-        { .backend = server1; }
-        { .backend = server2; }
+    sub vcl_init {
+        new vdir = directors.round_robin();
+        vdir.add_backend(backend1);
+        vdir.add_backend(backend2);
     }
 
 
-You use this director just as you would use any other director or
-backend. Varnish will not send traffic to hosts that are marked as
-unhealthy. Varnish can also serve stale content if all the backends are
+You use this director as a backend_hint for requests, just like you would
+with a simple backend. Varnish will not send traffic to hosts that are marked as
+unhealthy.
+
+Varnish can also serve stale content if all the backends are
 down. See :ref:`users-guide-handling_misbehaving_servers` for more
 information on how to enable this.
 
-Please note that Varnish will keep probes active for all loaded VCLs. Varnish
+Please note that Varnish will keep health probes running for all loaded VCLs. Varnish
 will coalesce probes that seem identical - so be careful not to change the
 probe config if you do a lot of VCL loading. Unloading the VCL will discard the
-probes.
-
-For more information on how to do this please see
-ref:`reference-vcl-director`.
+probes. For more information on how to do this please see ref:`reference-vcl-director`.
 



From lkarsten at varnish-software.com  Tue Jun 24 09:31:48 2014
From: lkarsten at varnish-software.com (Lasse Karstensen)
Date: Tue, 24 Jun 2014 11:31:48 +0200
Subject: [4.0] 52a6e58 Use consistent naming through the example.
Message-ID: <E1WzN4u-0003GC-AW@project.varnish-software.com>


commit 52a6e58600f31378251a250fe1954769587b4fc7
Author: Lasse Karstensen <lkarsten at varnish-software.com>
Date:   Thu May 15 16:47:09 2014 +0200

    Use consistent naming through the example.

diff --git a/doc/sphinx/users-guide/vcl-backends.rst b/doc/sphinx/users-guide/vcl-backends.rst
index 4a83b75..ecb8b31 100644
--- a/doc/sphinx/users-guide/vcl-backends.rst
+++ b/doc/sphinx/users-guide/vcl-backends.rst
@@ -211,12 +211,12 @@ Now we define the 'director'::
 
     sub vcl_init {
         new vdir = directors.round_robin();
-        vdir.add_backend(backend1);
-        vdir.add_backend(backend2);
+        vdir.add_backend(server1);
+        vdir.add_backend(server2);
     }
 
 
-You use this director as a backend_hint for requests, just like you would
+You use this `vdir` director as a backend_hint for requests, just like you would
 with a simple backend. Varnish will not send traffic to hosts that are marked as
 unhealthy.
 



From martin at varnish-software.com  Tue Jun 24 09:31:48 2014
From: martin at varnish-software.com (Martin Blix Grydeland)
Date: Tue, 24 Jun 2014 11:31:48 +0200
Subject: [4.0] bba689d Truncate %D varnishncsa format specifier to an integer
 value.
Message-ID: <E1WzN4u-0003GL-Cu@project.varnish-software.com>


commit bba689d2fa11223f765ac773d36fd7e0b7c63dc3
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Fri May 16 14:23:28 2014 +0200

    Truncate %D varnishncsa format specifier to an integer value.
    
    This follows the format Apache logs in.

diff --git a/bin/varnishncsa/varnishncsa.c b/bin/varnishncsa/varnishncsa.c
index e3f2432..2162118 100644
--- a/bin/varnishncsa/varnishncsa.c
+++ b/bin/varnishncsa/varnishncsa.c
@@ -276,7 +276,7 @@ format_time(const struct format *format)
 
 	switch (format->time_type) {
 	case 'D':
-		AZ(VSB_printf(CTX.vsb, "%f", (t_end - t_start) * 1e6));
+		AZ(VSB_printf(CTX.vsb, "%d", (int)((t_end - t_start) * 1e6)));
 		break;
 	case 't':
 		AN(format->time_fmt);



From apj at mutt.dk  Tue Jun 24 09:31:48 2014
From: apj at mutt.dk (Andreas Plesner)
Date: Tue, 24 Jun 2014 11:31:48 +0200
Subject: [4.0] bc73592 Exclude vcl_var since it is included in vcl.rst.
Message-ID: <E1WzN4u-0003GV-GI@project.varnish-software.com>


commit bc73592130bcbad5f30ad351809f11770320ac2c
Author: Andreas Plesner <apj at mutt.dk>
Date:   Fri May 16 17:51:34 2014 +0200

    Exclude vcl_var since it is included in vcl.rst.

diff --git a/doc/sphinx/conf.py.in b/doc/sphinx/conf.py.in
index 8104bb3..5e411b6 100644
--- a/doc/sphinx/conf.py.in
+++ b/doc/sphinx/conf.py.in
@@ -64,7 +64,7 @@ release = '@VERSION@'
 
 # List of directories, relative to source directory, that shouldn't be searched
 # for source files.
-exclude_patterns = ['build','include/*.rst']
+exclude_patterns = ['build','include/*.rst','reference/vcl_var.rst']
 
 # The reST default role (used for this markup: `text`) to use for all documents.
 #default_role = None



From apj at mutt.dk  Tue Jun 24 09:31:48 2014
From: apj at mutt.dk (Andreas Plesner)
Date: Tue, 24 Jun 2014 11:31:48 +0200
Subject: [4.0] 419d15f Add missing returns, and do a bit of cleanup
Message-ID: <E1WzN4u-0003Gf-J6@project.varnish-software.com>


commit 419d15fadd06470ce46c7e4c9883469ba7f748eb
Author: Andreas Plesner <apj at mutt.dk>
Date:   Fri May 16 17:53:15 2014 +0200

    Add missing returns, and do a bit of cleanup

diff --git a/doc/sphinx/users-guide/vcl-built-in-subs.rst b/doc/sphinx/users-guide/vcl-built-in-subs.rst
index 0a90a34..cb6396e 100644
--- a/doc/sphinx/users-guide/vcl-built-in-subs.rst
+++ b/doc/sphinx/users-guide/vcl-built-in-subs.rst
@@ -22,8 +22,8 @@ yourself doing frequently.
 The `vcl_recv` subroutine may terminate with calling ``return()`` on one
 of the following keywords:
 
-  synth 
-    Return a synthetic object with the specified error code to the
+  synth(status code, reason)
+    Return a synthetic object with the specified status code to the
     client and abandon the request.
 
   pass
@@ -52,8 +52,8 @@ shuffling bytes back and forth.
 The `vcl_pipe` subroutine may terminate with calling ``return()`` with one
 of the following keywords:
 
-  synth(error code, reason)
-    Return the specified error code to the client and abandon the request.
+  synth(status code, reason)
+    Return the specified status code to the client and abandon the request.
 
   pipe
     Proceed with pipe mode.
@@ -69,8 +69,8 @@ submitted over the same client connection are handled normally.
 The `vcl_pass` subroutine may terminate with calling ``return()`` with one
 of the following keywords:
 
-  synth(error code, reason)
-    Return the specified error code to the client and abandon the request.
+  synth(status code, reason)
+    Return the specified status code to the client and abandon the request.
 
   pass
     Proceed with pass mode.
@@ -86,7 +86,7 @@ vcl_hit
 
 Called when a cache lookup is successful. 
 
-.. XXX: missing the "The `vcl_hit` subroutine may terminate with calling ``return()`` with one of the following keywords:" thing. benc
+The `vcl_hit` subroutine may terminate with calling ``return()`` with one of the following keywords:
 
 
   restart
@@ -97,8 +97,8 @@ Called when a cache lookup is successful.
   deliver
     Deliver the object. Control passes to `vcl_deliver`.
 
-  synth(error code, reason)
-    Return the specified error code to the client and abandon the request.
+  synth(status code, reason)
+    Return the specified status code to the client and abandon the request.
 
 
 vcl_miss
@@ -111,8 +111,8 @@ retrieve the document from the backend, and which backend to use.
 The `vcl_miss` subroutine may terminate with calling ``return()`` with one
 of the following keywords:
 
-  synth(error code, reason)
-    Return the specified error code to the client and abandon the request.
+  synth(status code, reason)
+    Return the specified status code to the client and abandon the request.
 
   pass
     Switch to pass mode. Control will eventually pass to `vcl_pass`.
@@ -121,38 +121,51 @@ of the following keywords:
     Retrieve the requested object from the backend. Control will
     eventually pass to `vcl_backend_fetch`.
 
+  restart
+    Restart the transaction. Increases the restart counter. If the number
+    of restarts is higher than *max_restarts* Varnish emits a guru meditation
+    error.
+
+
 vcl_hash
 ~~~~~~~~
 
 Called after `vcl_recv` to create a hash value for the request. This is
 used as a key to look up the object in Varnish.
 
+The `vcl_hash` subroutine may terminate with calling ``return()`` with one
+of the following keywords:
+
   lookup
     Look up the object in cache. Control passes to vcl_miss, vcl_hit
     or vcl_purge.
 
 
-
-
 vcl_purge
 ~~~~~~~~~
 
 Called after the purge has been executed and all its variants have been evited.
 
+The `vcl_purge` subroutine may terminate with calling ``return()`` with one
+of the following keywords:
+
   synth
     Produce a response.
 
+  restart
+    Restart the transaction. Increases the restart counter. If the number
+    of restarts is higher than *max_restarts* Varnish emits a guru meditation
+    error.
+
 
 vcl_deliver
 ~~~~~~~~~~~
 
 Called before a cached object is delivered to the client.
 
-The ``vcl_deliver`` subroutine may terminate calling ``return()`` with one
+The `vcl_deliver` subroutine may terminate with calling ``return()`` with one
 of the following keywords:
 
-.. XXX: Should perhaps be return as above? benc
-
   deliver
     Deliver the object to the client.
 
@@ -168,7 +181,8 @@ vcl_backend_fetch
 Called before sending the backend request. In this subroutine you
 typically alter the request before it gets to the backend.
 
-.. XXX: Missing terminate..keywords sentence? benc
+The `vcl_backend_fetch` subroutine may terminate with calling
+``return()`` with one of the following keywords:
 
   fetch
     Fetch the object from the backend.
@@ -180,71 +194,40 @@ typically alter the request before it gets to the backend.
 vcl_backend_response
 ~~~~~~~~~~~~~~~~~~~~
 
-Called after a response has been successfully retrieved from the
-backend. The response is available as `beresp`. 
-
-.. XXX: beresp comes out of the blue here. maybe a short description? benc
-
-Note that Varnish might
-not be talking to an actual client, so operations that require a
-client to be present are not allowed. Specifically there is no `req
-object` and restarts are not allowed.
+Called after the response headers has been successfully retrieved from
+the backend.
 
-.. XXX: I do not follow sentence above. benc
-
-The `vcl_backend_response` subroutine may terminate with calling ``return()`` with one
-of the following keywords:
+The `vcl_backend_response` subroutine may terminate with calling
+``return()`` with one of the following keywords:
 
   deliver
     Possibly insert the object into the cache, then deliver it to the
     Control will eventually pass to `vcl_deliver`. Caching is dependant
     on 'beresp.cacheable'.
 
-.. XXX:A parameter? that is set how? benc
-    
-
-  error(error code, reason)
-    Return the specified error code to the client and abandon the request.
+  abandon
+    Abandon the backend request and generates an error.
 
   retry
-    Retry the backend transaction. Increases the `retries` counter. If the number
-    of retries is higher than *max_retries* Varnish emits a guru meditation
-    error.
+    Retry the backend transaction. Increases the `retries` counter.
+    If the number of retries is higher than *max_retries* Varnish
+    emits a guru meditation error.
 
 vcl_backend_error
 ~~~~~~~~~~~~~~~~~
 
 This subroutine is called if we fail the backend fetch. 
 
-.. XXX:Missing the terminate return structure? benc
+The `vcl_backend_error` subroutine may terminate with calling ``return()``
+with one of the following keywords:
 
   deliver
     Deliver the error.
 
   retry
-    Retry the backend transaction. Increases the `retries` counter. If the number
-    of retries is higher than *max_retries* Varnish emits a guru meditation
-    error.
-
-
-vcl_backend_error
-~~~~~~~~~~~~~~~~~
-
-.. XXX: Same name as section above? benc
-
-Called when we hit an error, either explicitly or implicitly due to
-backend or internal errors.
-
-The `vcl_backend_error` subroutine may terminate by calling ``return()`` with one of
-the following keywords:
-
-  deliver
-    Deliver the error object to the client.
-
-  retry
-    Retry the backend transaction. Increases the retries counter. If the number
-    of retries is higher than *max_retries* Varnish emits a guru meditation
-    error.
+    Retry the backend transaction. Increases the `retries` counter. If
+    the number of retries is higher than *max_retries* Varnish emits a
+    guru meditation error.
 
 
 vcl_init
@@ -253,9 +236,8 @@ vcl_init
 Called when VCL is loaded, before any requests pass through it.
 Typically used to initialize VMODs.
 
-.. XXX: Missing the terminate return structure? benc
-
-  ``return()`` values:
+The `vcl_init` subroutine may terminate with calling ``return()``
+with one of the following keywords:
 
   ok
     Normal return, VCL continues loading.
@@ -267,13 +249,8 @@ vcl_fini
 Called when VCL is discarded only after all requests have exited the VCL.
 Typically used to clean up VMODs.
 
-
-.. XXX: Missing the terminate return structure? benc
-
-  ``return()`` values:
+The `vcl_fini` subroutine may terminate with calling ``return()``
+with one of the following keywords:
 
   ok
     Normal return, VCL will be discarded.
-
-
-.. XXX: Maybe end here with the detailed flowchart from the book together with a reference to the book? benc



From daghf at varnish-software.com  Tue Jun 24 09:31:48 2014
From: daghf at varnish-software.com (Dag Haavi Finstad)
Date: Tue, 24 Jun 2014 11:31:48 +0200
Subject: [4.0] f599976 req.backend -> req.backend_hint
Message-ID: <E1WzN4u-0003Gp-NJ@project.varnish-software.com>


commit f5999765e77edc182846c99c59286aa443b87bf6
Author: Dag Haavi Finstad <daghf at varnish-software.com>
Date:   Fri May 16 20:37:14 2014 +0200

    req.backend -> req.backend_hint

diff --git a/doc/sphinx/whats-new/upgrading.rst b/doc/sphinx/whats-new/upgrading.rst
index c8ed011..f48548f 100644
--- a/doc/sphinx/whats-new/upgrading.rst
+++ b/doc/sphinx/whats-new/upgrading.rst
@@ -124,8 +124,8 @@ vcl_* reserved
 Any custom-made subs cannot be named 'vcl_*' anymore. This namespace
 is reserved for builtin subs.
 
-req.backend.healthy replaced by std.healthy(req.backend)
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+req.backend.healthy replaced by std.healthy(req.backend_hint)
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 Remember to import the std module if you're not doing so already.
 



From phk at FreeBSD.org  Tue Jun 24 09:31:48 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 24 Jun 2014 11:31:48 +0200
Subject: [4.0] d3b8f34 Forgot to commit the test-case
Message-ID: <E1WzN4u-0003H1-SV@project.varnish-software.com>


commit d3b8f34ba42db8772ac7ec468d6ffbcaa6d97b9b
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon May 19 06:45:31 2014 +0000

    Forgot to commit the test-case

diff --git a/bin/varnishtest/tests/r01478.vtc b/bin/varnishtest/tests/r01478.vtc
new file mode 100644
index 0000000..1df49a6
--- /dev/null
+++ b/bin/varnishtest/tests/r01478.vtc
@@ -0,0 +1,60 @@
+varnishtest "panic due to hash_ignore_busy"
+
+server s1 {
+	rxreq
+	txresp -nolen -hdr "Transfer-Encoding: chunked"
+	chunkedlen 10
+	delay .5
+	sema r1 sync 2
+	delay .5
+	chunkedlen 10
+	delay .5
+	sema r2 sync 2
+	delay .5
+	chunkedlen 10
+	delay .5
+	sema r3 sync 2
+	delay .5
+	chunkedlen 10
+	delay .5
+	sema r4 sync 2
+	delay .5
+	chunkedlen 10
+	delay .5
+	chunkedlen 0
+} -start
+
+varnish v1 -vcl+backend {
+	sub vcl_recv {
+		set req.hash_ignore_busy         = true;
+		return(hash);
+	}
+	sub vcl_backend_response {
+		# we assume streaming for all objects as default:
+		set beresp.do_stream             = true;
+		set beresp.ttl = 2s;
+	}
+} -start
+
+client c1 {
+	txreq -hdr "client: c1"
+	rxresp
+} -start
+
+sema r1 sync 2
+client c2 {
+	txreq -hdr "client: c2"
+	sema r2 sync 2
+	rxresp
+} -start
+
+sema r3 sync 2
+client c3 {
+	txreq -hdr "client: c3"
+	sema r4 sync 2
+	rxresp
+} -start
+
+client c1 -wait
+client c2 -wait
+client c3 -wait



From phk at FreeBSD.org  Tue Jun 24 09:31:49 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 24 Jun 2014 11:31:49 +0200
Subject: [4.0] a1b74f7 Allow ACLs to be unreferenced when vcc_err_unref=off.
Message-ID: <E1WzN4v-0003HC-2T@project.varnish-software.com>


commit a1b74f7b57d8a95abbbc0fa74217a09f8f06a39f
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon May 19 06:45:44 2014 +0000

    Allow ACLs to be unreferenced when vcc_err_unref=off.
    
    Add a reference from the init function to any non-anonymous match_acl_*()
    functions to ensure the compiler does not barf.
    
    Fixes	#1504

diff --git a/bin/varnishtest/tests/r01504.vtc b/bin/varnishtest/tests/r01504.vtc
new file mode 100644
index 0000000..0277c65
--- /dev/null
+++ b/bin/varnishtest/tests/r01504.vtc
@@ -0,0 +1,10 @@
+varnishtest "unreferenced acls"
+
+varnish v1 -arg "-p vcc_err_unref=off" -vcl {
+	backend s1 {
+		.host = "${bad_ip}";
+	}
+	acl foo {
+		"127.0.0.1";
+	}
+}
diff --git a/lib/libvcc/vcc_acl.c b/lib/libvcc/vcc_acl.c
index 1ab670a..997ae3c 100644
--- a/lib/libvcc/vcc_acl.c
+++ b/lib/libvcc/vcc_acl.c
@@ -343,13 +343,14 @@ vcc_acl_entry(struct vcc *tl)
  */
 
 static void
-vcc_acl_emit(const struct vcc *tl, const char *acln, int anon)
+vcc_acl_emit(struct vcc *tl, const char *acln, int anon)
 {
 	struct acl_e *ae;
 	int depth, l, m, i;
 	unsigned at[VRT_ACL_MAXADDR + 1];
 	const char *oc;
 	struct token *t;
+	struct inifin *ifp;
 
 	Fh(tl, 0, "\nstatic int\n");
 	Fh(tl, 0,
@@ -364,6 +365,11 @@ vcc_acl_emit(const struct vcc *tl, const char *acln, int anon)
 	Fh(tl, 0, "\t\tVRT_acl_log(ctx, \"NO_FAM %s\");\n", acln);
 	Fh(tl, 0, "\t\treturn(0);\n");
 	Fh(tl, 0, "\t}\n\n");
+	if (!tl->err_unref && !anon ) {
+		ifp = New_IniFin(tl);
+		VSB_printf(ifp->ini,
+			"\tif (0) match_acl_named_%s(0, 0);\n", acln);
+	}
 	depth = -1;
 	oc = 0;
 	at[0] = 256;



From phk at FreeBSD.org  Tue Jun 24 09:31:49 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 24 Jun 2014 11:31:49 +0200
Subject: [4.0] 75802e2 Add a debug.no_backend() function to simulate a
 director failing to pick a backend.
Message-ID: <E1WzN4v-0003HL-9u@project.varnish-software.com>


commit 75802e279457d209e59bdd3d26aceb59bb214715
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon May 19 07:14:50 2014 +0000

    Add a debug.no_backend() function to simulate a director failing to
    pick a backend.

diff --git a/lib/libvmod_debug/vmod.vcc b/lib/libvmod_debug/vmod.vcc
index 6872775..8667298 100644
--- a/lib/libvmod_debug/vmod.vcc
+++ b/lib/libvmod_debug/vmod.vcc
@@ -59,6 +59,10 @@ $Function STRING blob2hex(BLOB)
 
 Hexdump a blob
 
+$Function BACKEND no_backend()
+
+Fails at backend selection
+
 $Object obj(STRING) 
 
 Test object
diff --git a/lib/libvmod_debug/vmod_debug.c b/lib/libvmod_debug/vmod_debug.c
index 146f46b..b3e8334 100644
--- a/lib/libvmod_debug/vmod_debug.c
+++ b/lib/libvmod_debug/vmod_debug.c
@@ -129,3 +129,12 @@ vmod_blob2hex(const struct vrt_ctx *ctx, VCL_BLOB b)
 	VRT_priv_fini(b);
 	return (s);
 }
+
+VCL_BACKEND
+vmod_no_backend(const struct vrt_ctx *ctx)
+{
+	
+	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
+	return (NULL);
+}
+



From phk at FreeBSD.org  Tue Jun 24 09:31:49 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 24 Jun 2014 11:31:49 +0200
Subject: [4.0] 01f0e71 Directors are allowed to fail to pick a backend.
Message-ID: <E1WzN4v-0003HW-Ip@project.varnish-software.com>


commit 01f0e719f90bba1219426fa25abcd6cf3b68eebd
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon May 19 07:19:17 2014 +0000

    Directors are allowed to fail to pick a backend.
    
    Fixes	#1501

diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c
index 0f49a3b..7ad709d 100644
--- a/bin/varnishd/cache/cache_fetch.c
+++ b/bin/varnishd/cache/cache_fetch.c
@@ -181,7 +181,6 @@ vbf_stp_mkbereq(const struct worker *wrk, struct busyobj *bo)
 	CHECK_OBJ_NOTNULL(bo->req, REQ_MAGIC);
 
 	assert(bo->state == BOS_INVALID);
-	AN(bo->director);
 	AZ(bo->vbc);
 	AZ(bo->should_close);
 	AZ(bo->storage_hint);
@@ -255,7 +254,6 @@ vbf_stp_startfetch(struct worker *wrk, struct busyobj *bo)
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
 
-	AN(bo->director);
 	AZ(bo->vbc);
 	AZ(bo->should_close);
 	AZ(bo->storage_hint);
diff --git a/bin/varnishd/cache/cache_http1_fetch.c b/bin/varnishd/cache/cache_http1_fetch.c
index 4ddacb1..11c712e 100644
--- a/bin/varnishd/cache/cache_http1_fetch.c
+++ b/bin/varnishd/cache/cache_http1_fetch.c
@@ -300,6 +300,10 @@ V1F_fetch_hdr(struct worker *wrk, struct busyobj *bo, struct req *req)
 	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
 	htc = &bo->htc;
 
+	if (bo->director == NULL) {
+		VSLb(bo->vsl, SLT_FetchError, "No backend");
+		return (-1);
+	}
 	AN(bo->director);
 
 	hp = bo->bereq;
diff --git a/bin/varnishd/cache/cache_vrt_var.c b/bin/varnishd/cache/cache_vrt_var.c
index b68031c..65c72db 100644
--- a/bin/varnishd/cache/cache_vrt_var.c
+++ b/bin/varnishd/cache/cache_vrt_var.c
@@ -270,9 +270,8 @@ VRT_r_beresp_backend_name(const struct vrt_ctx *ctx)
 		CHECK_OBJ_NOTNULL(ctx->bo->vbc, VBC_MAGIC);
 		return (ctx->bo->vbc->backend->vcl_name);
 	}
-	if (ctx->bo->director != NULL) {
+	if (ctx->bo->director != NULL)
 		return (ctx->bo->director->vcl_name);
-	}
 	return (NULL);
 }
 
@@ -350,7 +349,6 @@ VRT_l_bereq_backend(const struct vrt_ctx *ctx, struct director *be)
 
 	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
 	CHECK_OBJ_NOTNULL(ctx->bo, BUSYOBJ_MAGIC);
-	AN(ctx->bo->director);
 	ctx->bo->director = be;
 }
 
@@ -360,7 +358,6 @@ VRT_r_bereq_backend(const struct vrt_ctx *ctx)
 
 	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
 	CHECK_OBJ_NOTNULL(ctx->bo, BUSYOBJ_MAGIC);
-	AN(ctx->bo->director);
 	return (ctx->bo->director);
 }
 
diff --git a/bin/varnishtest/tests/r01501.vtc b/bin/varnishtest/tests/r01501.vtc
new file mode 100644
index 0000000..c7ce3b8
--- /dev/null
+++ b/bin/varnishtest/tests/r01501.vtc
@@ -0,0 +1,21 @@
+varnishtest "director fails to pick backend"
+
+server s1 {
+	rxreq
+	txresp
+} -start
+
+varnish v1 -vcl+backend {
+
+	import ${vmod_debug};
+
+	sub vcl_recv {
+		set req.backend_hint = debug.no_backend();
+	}
+} -start
+
+client c1 {
+	txreq
+	rxresp
+	expect resp.status == 503
+} -run



From phk at FreeBSD.org  Tue Jun 24 09:31:49 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 24 Jun 2014 11:31:49 +0200
Subject: [4.0] 7084fb1 White-space fix
Message-ID: <E1WzN4v-0003Hh-Ob@project.varnish-software.com>


commit 7084fb10dca811c86c6c39b5a42cb71c413422bb
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon May 19 07:20:30 2014 +0000

    White-space fix

diff --git a/bin/varnishtest/tests/c00041.vtc b/bin/varnishtest/tests/c00041.vtc
index bd6abdb..c46e95b 100644
--- a/bin/varnishtest/tests/c00041.vtc
+++ b/bin/varnishtest/tests/c00041.vtc
@@ -40,10 +40,12 @@ server s1 {
 
 varnish v1 -vcl+backend {
 	sub vcl_recv { if (req.http.purge == "yes") { return(purge); } }
-	sub vcl_purge { if (req.http.restart == "yes") {
-		unset req.http.purge;
-		unset req.http.restart;
-		return(restart);} 
+	sub vcl_purge {
+		if (req.http.restart == "yes") {
+			unset req.http.purge;
+			unset req.http.restart;
+			return(restart);
+		}
 	}
 } -start
 



From phk at FreeBSD.org  Tue Jun 24 09:31:50 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 24 Jun 2014 11:31:50 +0200
Subject: [4.0] 147e71d White space fixes
Message-ID: <E1WzN4w-0003Hs-7i@project.varnish-software.com>


commit 147e71d7d378173f81a62c3f861e03981b70ed66
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon May 19 07:21:20 2014 +0000

    White space fixes

diff --git a/bin/varnishadm/varnishadm.c b/bin/varnishadm/varnishadm.c
index 4b219de..60f36ba 100644
--- a/bin/varnishadm/varnishadm.c
+++ b/bin/varnishadm/varnishadm.c
@@ -441,9 +441,9 @@ main(int argc, char * const *argv)
 
 	/*
 	 * By default linux::getopt(3) mangles the argv order, such that
-	 * 	varnishadm -n bla param.set foo -bar
+	 *	varnishadm -n bla param.set foo -bar
 	 * gets interpreted as
-	 * 	varnishadm -n bla -bar param.set foo 
+	 *	varnishadm -n bla -bar param.set foo
 	 * The '+' stops that from happening
 	 * See #1496
 	 */
diff --git a/lib/libvmod_debug/vmod_debug.c b/lib/libvmod_debug/vmod_debug.c
index b3e8334..b2f3313 100644
--- a/lib/libvmod_debug/vmod_debug.c
+++ b/lib/libvmod_debug/vmod_debug.c
@@ -133,7 +133,7 @@ vmod_blob2hex(const struct vrt_ctx *ctx, VCL_BLOB b)
 VCL_BACKEND
 vmod_no_backend(const struct vrt_ctx *ctx)
 {
-	
+
 	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
 	return (NULL);
 }



From lkarsten at varnish-software.com  Tue Jun 24 09:31:50 2014
From: lkarsten at varnish-software.com (Lasse Karstensen)
Date: Tue, 24 Jun 2014 11:31:50 +0200
Subject: [4.0] 0993266 Remove duplicate command line arguments.
Message-ID: <E1WzN4w-0003IK-Tp@project.varnish-software.com>


commit 099326633d6ce324dc356ab09e4d9bcf11dcc28a
Author: Lasse Karstensen <lkarsten at varnish-software.com>
Date:   Mon May 19 10:26:49 2014 +0200

    Remove duplicate command line arguments.
    
    We don't need to add the RST2ANY_FLAGS arguments twice.

diff --git a/man/Makefile.am b/man/Makefile.am
index a3d3077..c7dd1bf 100644
--- a/man/Makefile.am
+++ b/man/Makefile.am
@@ -90,4 +90,4 @@ vmod_std.3: $(top_srcdir)/lib/libvmod_std/vmod_std.man.rst
 	${RST2MAN} $(RST2ANY_FLAGS) $? $@
 
 vmod_directors.3: $(top_srcdir)/lib/libvmod_directors/vmod_directors.man.rst
-	${RST2MAN} $(RST2ANY_FLAGS) $(RST2ANY_FLAGS) $? $@
+	${RST2MAN} $(RST2ANY_FLAGS) $? $@



From lkarsten at varnish-software.com  Tue Jun 24 09:31:51 2014
From: lkarsten at varnish-software.com (Lasse Karstensen)
Date: Tue, 24 Jun 2014 11:31:51 +0200
Subject: [4.0] 30f32b6 Copy from source where the file exists.
Message-ID: <E1WzN4x-0003IY-Ec@project.varnish-software.com>


commit 30f32b65ee3135b101c57621a2aa483f651fc73f
Author: Lasse Karstensen <lkarsten at varnish-software.com>
Date:   Mon May 19 11:27:46 2014 +0200

    Copy from source where the file exists.
    
    With parallell builds, building this file has usually not happened yet
    in the build directory.
    
    Suggested by: Martin

diff --git a/doc/sphinx/Makefile.am b/doc/sphinx/Makefile.am
index e714383..dbed031 100644
--- a/doc/sphinx/Makefile.am
+++ b/doc/sphinx/Makefile.am
@@ -212,11 +212,11 @@ include/varnishhist_synopsis.rst: $(top_builddir)/bin/varnishhist/varnishhist_op
 BUILT_SOURCES += include/varnishhist_options.rst \
 	 include/varnishhist_synopsis.rst
 
-reference/vmod_std.generated.rst: $(top_builddir)/lib/libvmod_std/vmod_std.rst
+reference/vmod_std.generated.rst: $(top_srcdir)/lib/libvmod_std/vmod_std.rst
 	cp $? $@
 BUILT_SOURCES += reference/vmod_std.generated.rst
 
-reference/vmod_directors.generated.rst: $(top_builddir)/lib/libvmod_directors/vmod_directors.rst
+reference/vmod_directors.generated.rst: $(top_srcdir)/lib/libvmod_directors/vmod_directors.rst
 	cp $? $@
 BUILT_SOURCES += reference/vmod_directors.generated.rst
 



From lkarsten at varnish-software.com  Tue Jun 24 09:31:51 2014
From: lkarsten at varnish-software.com (Lasse Karstensen)
Date: Tue, 24 Jun 2014 11:31:51 +0200
Subject: [4.0] 4482199 Don't package build/doctrees in tarball.
Message-ID: <E1WzN4x-0003Il-Hm@project.varnish-software.com>


commit 448219905424d04f3a77cb3af64fcca4bfebdc17
Author: Lasse Karstensen <lkarsten at varnish-software.com>
Date:   Mon May 19 12:59:04 2014 +0200

    Don't package build/doctrees in tarball.
    
    These are just intermediate files written by Sphinx to
    build the HTML documentation. We don't need to distribute
    them.

diff --git a/doc/sphinx/Makefile.am b/doc/sphinx/Makefile.am
index dbed031..22dd894 100644
--- a/doc/sphinx/Makefile.am
+++ b/doc/sphinx/Makefile.am
@@ -91,89 +91,21 @@ doctest:
 EXTRA_DIST = \
 	conf.py \
 	index.rst \
-	glossary/index.rst \
-	installation/bugs.rst \
-	installation/help.rst \
-	installation/index.rst \
-	installation/install.rst \
-	installation/platformnotes.rst \
-	installation/prerequisites.rst \
-	phk/autocrap.rst \
-	phk/backends.rst \
-	phk/barriers.rst \
-	phk/gzip.rst \
-	phk/http20.rst \
-	phk/index.rst \
-	phk/ipv6suckage.rst \
-	phk/platforms.rst \
-	phk/spdy.rst \
-	phk/sphinx.rst \
-	phk/ssl.rst \
-	phk/thetoolsweworkwith.rst \
-	phk/thoughts.rst \
-	phk/three-zero.rst \
-	phk/varnish_does_not_hash.rst \
-	phk/vcl_expr.rst \
-	phk/wanton_destruction.rst \
-	reference/index.rst \
-	reference/varnish-cli.rst \
-	reference/varnishadm.rst \
-	reference/varnishd.rst \
-	reference/varnishhist.rst \
-	reference/varnishlog.rst \
-	reference/varnishncsa.rst \
-	reference/varnishstat.rst \
-	reference/varnishtest.rst \
-	reference/varnishtop.rst \
-	reference/vcl.rst \
-	reference/vmod.rst \
-	reference/vsl-query.rst \
-	reference/vsl.rst \
-	reference/vsm.rst \
-	tutorial/backend_servers.rst \
-	tutorial/index.rst \
-	tutorial/introduction.rst \
-	tutorial/now_what.rst \
-	tutorial/peculiarities.rst \
-	tutorial/putting_varnish_on_port_80.rst \
-	tutorial/starting_varnish.rst \
-	users-guide/command-line.rst \
-	users-guide/compression.rst \
-	users-guide/devicedetection.rst \
-	users-guide/esi.rst \
-	users-guide/increasing-your-hitrate.rst \
-	users-guide/index.rst \
-	users-guide/intro.rst \
-	users-guide/operation-logging.rst \
-	users-guide/operation-statistics.rst \
-	users-guide/params.rst \
-	users-guide/performance.rst \
-	users-guide/purging.rst \
-	users-guide/report.rst \
-	users-guide/run_cli.rst \
-	users-guide/run_security.rst \
-	users-guide/running.rst \
-	users-guide/sizing-your-cache.rst \
-	users-guide/storage-backends.rst \
-	users-guide/troubleshooting.rst \
-	users-guide/vcl-actions.rst \
-	users-guide/vcl-backends.rst \
-	users-guide/vcl-built-in-subs.rst \
-	users-guide/vcl-example-acls.rst \
-	users-guide/vcl-example-manipulating-headers.rst \
-	users-guide/vcl-example-manipulating-responses.rst \
-	users-guide/vcl-example-websockets.rst \
-	users-guide/vcl-examples.rst \
-	users-guide/vcl-hashing.rst \
-	users-guide/vcl-inline-c.rst \
-	users-guide/vcl-grace.rst \
-	users-guide/vcl-syntax.rst \
-	users-guide/vcl-variables.rst \
-	users-guide/vcl.rst
+	include \
+	glossary \
+	installation \
+	phk \
+	reference \
+	tutorial \
+	users-guide \
+	whats-new
 
 dist-hook:
 	$(MAKE) html
+	rm -rf $(BUILDDIR)/doctrees
 	cp -r $(BUILDDIR) $(distdir)
+# Remove any extra non-rst files we may have copied over.
+	find $(distdir) ! -type f -a -name \*.rst -ls
 
 distclean-local:
 	rm -rf $(BUILDDIR)



From lkarsten at varnish-software.com  Tue Jun 24 09:31:51 2014
From: lkarsten at varnish-software.com (Lasse Karstensen)
Date: Tue, 24 Jun 2014 11:31:51 +0200
Subject: [4.0] 010aac8 Remove any superflous non rst/html from tarball.
Message-ID: <E1WzN4x-0003Iu-L6@project.varnish-software.com>


commit 010aac8abadfff3ebe227d1011bd8ba8d8acc289
Author: Lasse Karstensen <lkarsten at varnish-software.com>
Date:   Mon May 19 13:28:44 2014 +0200

    Remove any superflous non rst/html from tarball.
    
    When we in the future introduce something new and shiny here, don't
    automatically start putting it into the tarball release.
    
    This may seem a bit backward, the main idea is to avoid hardcoding
    the section/subdirectory names yet another place.

diff --git a/doc/sphinx/Makefile.am b/doc/sphinx/Makefile.am
index 22dd894..f1b9bfc 100644
--- a/doc/sphinx/Makefile.am
+++ b/doc/sphinx/Makefile.am
@@ -105,7 +105,7 @@ dist-hook:
 	rm -rf $(BUILDDIR)/doctrees
 	cp -r $(BUILDDIR) $(distdir)
 # Remove any extra non-rst files we may have copied over.
-	find $(distdir) ! -type f -a -name \*.rst -ls
+	find $(distdir) -mindepth 2 -type f ! -name \*.rst | grep -v '/build/' | xargs rm -f
 
 distclean-local:
 	rm -rf $(BUILDDIR)



From phk at FreeBSD.org  Tue Jun 24 09:31:51 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 24 Jun 2014 11:31:51 +0200
Subject: [4.0] afe89d5 Reduce contention on the wstat_mtx marginally while we
 figure out what the real solution is going to be.
Message-ID: <E1WzN4x-0003J4-P4@project.varnish-software.com>


commit afe89d5520114dc297a3d3c4cec3c4373fefadf3
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue May 20 07:11:57 2014 +0000

    Reduce contention on the wstat_mtx marginally while we figure out
    what the real solution is going to be.

diff --git a/bin/varnishd/cache/cache_wrk.c b/bin/varnishd/cache/cache_wrk.c
index 6496a3a..7f9d410 100644
--- a/bin/varnishd/cache/cache_wrk.c
+++ b/bin/varnishd/cache/cache_wrk.c
@@ -54,7 +54,6 @@ wrk_sumstat(struct worker *w)
 #undef VSC_F
 #undef L0
 #undef L1
-	memset(&w->stats, 0, sizeof w->stats);
 }
 
 void
@@ -64,6 +63,7 @@ WRK_SumStat(struct worker *w)
 	Lck_Lock(&wstat_mtx);
 	wrk_sumstat(w);
 	Lck_Unlock(&wstat_mtx);
+	memset(&w->stats, 0, sizeof w->stats);
 }
 
 int
@@ -73,6 +73,7 @@ WRK_TrySumStat(struct worker *w)
 		return (0);
 	wrk_sumstat(w);
 	Lck_Unlock(&wstat_mtx);
+	memset(&w->stats, 0, sizeof w->stats);
 	return (1);
 }
 



From phk at FreeBSD.org  Tue Jun 24 09:31:51 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 24 Jun 2014 11:31:51 +0200
Subject: [4.0] 504c885 Stabilize a race in this test with some small delay
Message-ID: <E1WzN4x-0003JH-TR@project.varnish-software.com>


commit 504c885847f2d0ed8ae350b11c28fb00b05e21cd
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue May 20 07:48:56 2014 +0000

    Stabilize a race in this test with some small delay

diff --git a/bin/varnishtest/tests/c00020.vtc b/bin/varnishtest/tests/c00020.vtc
index 509d6ee..ec54fe3 100644
--- a/bin/varnishtest/tests/c00020.vtc
+++ b/bin/varnishtest/tests/c00020.vtc
@@ -3,7 +3,7 @@ varnishtest "Test -h critbit a bit"
 server s1 {
         rxreq
 	expect req.url == "/"
-        txresp -hdr "Connection: close" -body "012345\n"
+        txresp -hdr "ID: slash" -hdr "Connection: close" -body "012345\n"
 } -start
 
 varnish v1 -arg "-hcritbit" -vcl+backend { } -start
@@ -13,41 +13,55 @@ client c1 {
         rxresp
         expect resp.status == 200
         expect resp.http.X-Varnish == "1001"
+        expect resp.http.ID == "slash"
 } -run
+delay .1
 
 client c2 {
         txreq -url "/"
         rxresp
         expect resp.status == 200
         expect resp.http.X-Varnish == "1004 1002"
+        expect resp.http.ID == "slash"
 } -run
+delay .1
 
 server s1 {
         rxreq
 	expect req.url == "/foo"
-        txresp -body "012345\n"
+        txresp -hdr "ID: foo" -body "012345\n"
         rxreq
 	expect req.url == "/bar"
-        txresp -body "012345\n"
+        txresp -hdr "ID: bar" -body "012345\n"
 } -start
 
-client c2 {
+client c1 {
         txreq -url "/foo"
         rxresp
         expect resp.status == 200
         expect resp.http.X-Varnish == "1006"
+        expect resp.http.ID == "foo"
+	delay .1
+
         txreq -url "/"
         rxresp
         expect resp.status == 200
         expect resp.http.X-Varnish == "1008 1002"
+        expect resp.http.ID == "slash"
+	delay .1
+
         txreq -url "/bar"
         rxresp
         expect resp.status == 200
         expect resp.http.X-Varnish == "1009"
+        expect resp.http.ID == "bar"
+	delay .1
+
         txreq -url "/foo"
         rxresp
         expect resp.status == 200
         expect resp.http.X-Varnish == "1011 1007"
+        expect resp.http.ID == "foo"
 } -run
 
 varnish v1 -expect sess_conn == 3



From phk at FreeBSD.org  Tue Jun 24 09:31:52 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 24 Jun 2014 11:31:52 +0200
Subject: [4.0] b1ad9cd Move the wthread_stats_rate check into the threadpool,
 so that it also takes account of fetch jobs.
Message-ID: <E1WzN4y-0003JO-3B@project.varnish-software.com>


commit b1ad9cd6488ecbd7d969542372c9f665125a5f8d
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue May 20 08:01:53 2014 +0000

    Move the wthread_stats_rate check into the threadpool, so that it
    also takes account of fetch jobs.

diff --git a/bin/varnishd/cache/cache_http1_fsm.c b/bin/varnishd/cache/cache_http1_fsm.c
index 3702827..9191d1c 100644
--- a/bin/varnishd/cache/cache_http1_fsm.c
+++ b/bin/varnishd/cache/cache_http1_fsm.c
@@ -240,9 +240,6 @@ http1_cleanup(struct sess *sp, struct worker *wrk, struct req *req)
 		return (SESS_DONE_RET_GONE);
 	}
 
-	if (wrk->stats.client_req >= cache_param->wthread_stats_rate)
-		WRK_SumStat(wrk);
-
 	WS_Reset(req->ws, NULL);
 	WS_Reset(wrk->aws, NULL);
 
diff --git a/bin/varnishd/cache/cache_pool.c b/bin/varnishd/cache/cache_pool.c
index ce8d526..fd5a43c 100644
--- a/bin/varnishd/cache/cache_pool.c
+++ b/bin/varnishd/cache/cache_pool.c
@@ -255,13 +255,13 @@ void
 Pool_Work_Thread(void *priv, struct worker *wrk)
 {
 	struct pool *pp;
-	int stats_clean;
+	int stats_dirty;
 	struct pool_task *tp;
 	int i;
 
 	CAST_OBJ_NOTNULL(pp, priv, POOL_MAGIC);
 	wrk->pool = pp;
-	stats_clean = 1;
+	stats_dirty = 0;
 	while (1) {
 		Lck_Lock(&pp->mtx);
 
@@ -286,8 +286,10 @@ Pool_Work_Thread(void *priv, struct worker *wrk)
 			wrk->task.func = NULL;
 			wrk->task.priv = wrk;
 			VTAILQ_INSERT_HEAD(&pp->idle_queue, &wrk->task, list);
-			if (!stats_clean)
+			if (stats_dirty) {
 				WRK_SumStat(wrk);
+				stats_dirty = 0;
+			}
 			do {
 				i = Lck_CondWait(&wrk->cond, &pp->mtx,
 				    wrk->vcl == NULL ?  0 : wrk->lastused+60.);
@@ -303,7 +305,12 @@ Pool_Work_Thread(void *priv, struct worker *wrk)
 
 		assert(wrk->pool == pp);
 		tp->func(wrk, tp->priv);
-		stats_clean = WRK_TrySumStat(wrk);
+
+		if (++stats_dirty >= cache_param->wthread_stats_rate) {
+			WRK_SumStat(wrk);
+			stats_dirty = 0;
+		} else if (WRK_TrySumStat(wrk))
+			stats_dirty = 0;
 	}
 	wrk->pool = NULL;
 }
diff --git a/bin/varnishd/cache/cache_wrk.c b/bin/varnishd/cache/cache_wrk.c
index 7f9d410..2c266ce 100644
--- a/bin/varnishd/cache/cache_wrk.c
+++ b/bin/varnishd/cache/cache_wrk.c
@@ -43,12 +43,12 @@ static struct lock		wstat_mtx;
 /*--------------------------------------------------------------------*/
 
 static void
-wrk_sumstat(struct worker *w)
+wrk_sumstat(const struct dstat *ds)
 {
 
 	Lck_AssertHeld(&wstat_mtx);
 #define L0(n)
-#define L1(n) (VSC_C_main->n += w->stats.n)
+#define L1(n) (VSC_C_main->n += ds->n)
 #define VSC_F(n, t, l, f, v, d, e) L##l(n);
 #include "tbl/vsc_f_main.h"
 #undef VSC_F
@@ -61,7 +61,7 @@ WRK_SumStat(struct worker *w)
 {
 
 	Lck_Lock(&wstat_mtx);
-	wrk_sumstat(w);
+	wrk_sumstat(&w->stats);
 	Lck_Unlock(&wstat_mtx);
 	memset(&w->stats, 0, sizeof w->stats);
 }
@@ -71,7 +71,7 @@ WRK_TrySumStat(struct worker *w)
 {
 	if (Lck_Trylock(&wstat_mtx))
 		return (0);
-	wrk_sumstat(w);
+	wrk_sumstat(&w->stats);
 	Lck_Unlock(&wstat_mtx);
 	memset(&w->stats, 0, sizeof w->stats);
 	return (1);
diff --git a/bin/varnishd/mgt/mgt_pool.c b/bin/varnishd/mgt/mgt_pool.c
index b3294f7..f43f019 100644
--- a/bin/varnishd/mgt/mgt_pool.c
+++ b/bin/varnishd/mgt/mgt_pool.c
@@ -183,8 +183,8 @@ struct parspec WRK_parspec[] = {
 		"0", NULL,
 		"Worker threads accumulate statistics, and dump these into "
 		"the global stats counters if the lock is free when they "
-		"finish a request.\n"
-		"This parameters defines the maximum number of requests "
+		"finish a job (request/fetch etc.)\n"
+		"This parameters defines the maximum number of jobs "
 		"a worker thread may handle, before it is forced to dump "
 		"its accumulated stats into the global counters.",
 		EXPERIMENTAL,



From phk at FreeBSD.org  Tue Jun 24 09:31:52 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 24 Jun 2014 11:31:52 +0200
Subject: [4.0] 46dd961 Move the stat-summing from WRK to Pool where it mostly
 happens.
Message-ID: <E1WzN4y-0003Ja-6Y@project.varnish-software.com>


commit 46dd9617c7d031aa82ca420af95e6579e84d62ad
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue May 20 08:44:05 2014 +0000

    Move the stat-summing from WRK to Pool where it mostly happens.

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index 75a64ac..cab075b 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -1107,6 +1107,8 @@ void Pool_Init(void);
 void Pool_Accept(void);
 void Pool_Work_Thread(void *priv, struct worker *w);
 int Pool_Task(struct pool *pp, struct pool_task *task, enum pool_how how);
+void Pool_Sumstat(struct worker *w);
+void Pool_PurgeStat(unsigned nobj);
 
 #define WRW_IsReleased(w)	((w)->wrw == NULL)
 int WRW_Error(const struct worker *w);
@@ -1216,10 +1218,6 @@ void WAIT_Write_Session(struct sess *sp, int fd);
 
 /* cache_wrk.c */
 
-void WRK_Init(void);
-int WRK_TrySumStat(struct worker *w);
-void WRK_SumStat(struct worker *w);
-void WRK_PurgeStat(unsigned nobj);
 void *WRK_thread(void *priv);
 typedef void *bgthread_t(struct worker *, void *priv);
 void WRK_BgThread(pthread_t *thr, const char *name, bgthread_t *func,
diff --git a/bin/varnishd/cache/cache_expire.c b/bin/varnishd/cache/cache_expire.c
index b5ecfbe..a893ecc 100644
--- a/bin/varnishd/cache/cache_expire.c
+++ b/bin/varnishd/cache/cache_expire.c
@@ -404,7 +404,7 @@ EXP_NukeLRU(struct worker *wrk, struct vsl_log *vsl, struct lru *lru)
 	}
 	Lck_Unlock(&lru->mtx);
 
-	WRK_SumStat(wrk);
+	Pool_Sumstat(wrk);
 }
 
 #endif
@@ -589,7 +589,7 @@ exp_thread(struct worker *wrk, void *priv)
 			tnext = 0;
 		} else if (tnext > t) {
 			VSL_Flush(&ep->vsl, 0);
-			WRK_SumStat(wrk);
+			Pool_Sumstat(wrk);
 			(void)Lck_CondWait(&ep->condvar, &ep->mtx, tnext);
 		}
 		Lck_Unlock(&ep->mtx);
diff --git a/bin/varnishd/cache/cache_hash.c b/bin/varnishd/cache/cache_hash.c
index cd48671..73acf01 100644
--- a/bin/varnishd/cache/cache_hash.c
+++ b/bin/varnishd/cache/cache_hash.c
@@ -603,7 +603,7 @@ double keep)
 		(void)HSH_DerefObj(&wrk->stats, &o);
 	}
 	WS_Release(wrk->aws, 0);
-	WRK_PurgeStat(nobj);
+	Pool_PurgeStat(nobj);
 }
 
 
diff --git a/bin/varnishd/cache/cache_main.c b/bin/varnishd/cache/cache_main.c
index c6a2599..31d167d 100644
--- a/bin/varnishd/cache/cache_main.c
+++ b/bin/varnishd/cache/cache_main.c
@@ -217,7 +217,6 @@ child_main(void)
 	VBO_Init();
 	VBE_InitCfg();
 	VBP_Init();
-	WRK_Init();
 	Pool_Init();
 	Pipe_Init();
 
diff --git a/bin/varnishd/cache/cache_pool.c b/bin/varnishd/cache/cache_pool.c
index fd5a43c..f438b26 100644
--- a/bin/varnishd/cache/cache_pool.c
+++ b/bin/varnishd/cache/cache_pool.c
@@ -78,6 +78,59 @@ static struct lock		pool_mtx;
 static pthread_t		thr_pool_herder;
 static unsigned			pool_accepting = 0;
 
+static struct lock		wstat_mtx;
+
+/*--------------------------------------------------------------------*/
+
+static void
+wrk_sumstat(const struct dstat *ds)
+{
+
+	Lck_AssertHeld(&wstat_mtx);
+#define L0(n)
+#define L1(n) (VSC_C_main->n += ds->n)
+#define VSC_F(n, t, l, f, v, d, e) L##l(n);
+#include "tbl/vsc_f_main.h"
+#undef VSC_F
+#undef L0
+#undef L1
+}
+
+void
+Pool_Sumstat(struct worker *w)
+{
+
+	Lck_Lock(&wstat_mtx);
+	wrk_sumstat(&w->stats);
+	Lck_Unlock(&wstat_mtx);
+	memset(&w->stats, 0, sizeof w->stats);
+}
+
+static int
+Pool_TrySumstat(struct worker *w)
+{
+	if (Lck_Trylock(&wstat_mtx))
+		return (0);
+	wrk_sumstat(&w->stats);
+	Lck_Unlock(&wstat_mtx);
+	memset(&w->stats, 0, sizeof w->stats);
+	return (1);
+}
+
+/*--------------------------------------------------------------------
+ * Helper function to update stats for purges under lock
+ */
+
+void
+Pool_PurgeStat(unsigned nobj)
+{
+	Lck_Lock(&wstat_mtx);
+	VSC_C_main->n_purges++;
+	VSC_C_main->n_obj_purged += nobj;
+	Lck_Unlock(&wstat_mtx);
+}
+
+
 /*--------------------------------------------------------------------
  */
 
@@ -148,7 +201,7 @@ pool_accept(struct worker *wrk, void *arg)
 		if (VCA_Accept(ps->lsock, wa) < 0) {
 			wrk->stats.sess_fail++;
 			/* We're going to pace in vca anyway... */
-			(void)WRK_TrySumStat(wrk);
+			(void)Pool_TrySumstat(wrk);
 			continue;
 		}
 
@@ -287,7 +340,7 @@ Pool_Work_Thread(void *priv, struct worker *wrk)
 			wrk->task.priv = wrk;
 			VTAILQ_INSERT_HEAD(&pp->idle_queue, &wrk->task, list);
 			if (stats_dirty) {
-				WRK_SumStat(wrk);
+				Pool_Sumstat(wrk);
 				stats_dirty = 0;
 			}
 			do {
@@ -307,9 +360,9 @@ Pool_Work_Thread(void *priv, struct worker *wrk)
 		tp->func(wrk, tp->priv);
 
 		if (++stats_dirty >= cache_param->wthread_stats_rate) {
-			WRK_SumStat(wrk);
+			Pool_Sumstat(wrk);
 			stats_dirty = 0;
-		} else if (WRK_TrySumStat(wrk))
+		} else if (Pool_TrySumstat(wrk))
 			stats_dirty = 0;
 	}
 	wrk->pool = NULL;
@@ -531,6 +584,7 @@ void
 Pool_Init(void)
 {
 
+	Lck_New(&wstat_mtx, lck_wstat);
 	Lck_New(&pool_mtx, lck_wq);
 	AZ(pthread_create(&thr_pool_herder, NULL, pool_poolherder, NULL));
 }
diff --git a/bin/varnishd/cache/cache_wrk.c b/bin/varnishd/cache/cache_wrk.c
index 2c266ce..38a662b 100644
--- a/bin/varnishd/cache/cache_wrk.c
+++ b/bin/varnishd/cache/cache_wrk.c
@@ -37,59 +37,6 @@
 #include "cache.h"
 
 #include "hash/hash_slinger.h"
-
-static struct lock		wstat_mtx;
-
-/*--------------------------------------------------------------------*/
-
-static void
-wrk_sumstat(const struct dstat *ds)
-{
-
-	Lck_AssertHeld(&wstat_mtx);
-#define L0(n)
-#define L1(n) (VSC_C_main->n += ds->n)
-#define VSC_F(n, t, l, f, v, d, e) L##l(n);
-#include "tbl/vsc_f_main.h"
-#undef VSC_F
-#undef L0
-#undef L1
-}
-
-void
-WRK_SumStat(struct worker *w)
-{
-
-	Lck_Lock(&wstat_mtx);
-	wrk_sumstat(&w->stats);
-	Lck_Unlock(&wstat_mtx);
-	memset(&w->stats, 0, sizeof w->stats);
-}
-
-int
-WRK_TrySumStat(struct worker *w)
-{
-	if (Lck_Trylock(&wstat_mtx))
-		return (0);
-	wrk_sumstat(&w->stats);
-	Lck_Unlock(&wstat_mtx);
-	memset(&w->stats, 0, sizeof w->stats);
-	return (1);
-}
-
-/*--------------------------------------------------------------------
- * Helper function to update stats for purges under lock
- */
-
-void
-WRK_PurgeStat(unsigned nobj)
-{
-	Lck_Lock(&wstat_mtx);
-	VSC_C_main->n_purges++;
-	VSC_C_main->n_obj_purged += nobj;
-	Lck_Unlock(&wstat_mtx);
-}
-
 /*--------------------------------------------------------------------
  * Create and starte a back-ground thread which as its own worker and
  * session data structures;
@@ -164,7 +111,7 @@ wrk_thread_real(void *priv, unsigned thread_workspace)
 	if (w->nbo != NULL)
 		VBO_Free(&w->nbo);
 	HSH_Cleanup(w);
-	WRK_SumStat(w);
+	Pool_Sumstat(w);
 	return (NULL);
 }
 
@@ -174,9 +121,3 @@ WRK_thread(void *priv)
 
 	return (wrk_thread_real(priv, cache_param->workspace_thread));
 }
-
-void
-WRK_Init(void)
-{
-	Lck_New(&wstat_mtx, lck_wstat);
-}
diff --git a/bin/varnishd/hash/hash_critbit.c b/bin/varnishd/hash/hash_critbit.c
index 4e14bdd..4c7e4b5 100644
--- a/bin/varnishd/hash/hash_critbit.c
+++ b/bin/varnishd/hash/hash_critbit.c
@@ -369,7 +369,7 @@ hcb_cleaner(struct worker *wrk, void *priv)
 		VSTAILQ_CONCAT(&dead_y, &cool_y);
 		VTAILQ_CONCAT(&dead_h, &cool_h, hoh_list);
 		Lck_Unlock(&hcb_mtx);
-		WRK_SumStat(wrk);
+		Pool_Sumstat(wrk);
 		VTIM_sleep(cache_param->critbit_cooloff);
 	}
 	NEEDLESS_RETURN(NULL);
diff --git a/bin/varnishd/storage/storage_persistent_silo.c b/bin/varnishd/storage/storage_persistent_silo.c
index f27ccd9..f66cd52 100644
--- a/bin/varnishd/storage/storage_persistent_silo.c
+++ b/bin/varnishd/storage/storage_persistent_silo.c
@@ -161,7 +161,7 @@ smp_load_seg(struct worker *wrk, const struct smp_sc *sc,
 		EXP_Inject(oc, sg->lru, so->ttl);
 		sg->nobj++;
 	}
-	WRK_SumStat(wrk);
+	Pool_Sumstat(wrk);
 	sg->flags |= SMP_SEG_LOADED;
 }
 



From phk at FreeBSD.org  Tue Jun 24 09:31:52 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 24 Jun 2014 11:31:52 +0200
Subject: [4.0] 38ee783 The mutex protecting summing of stats into the global
 VSC counters is a contention point and scales negatively with number of
 thread-pools.
Message-ID: <E1WzN4y-0003Jo-9q@project.varnish-software.com>


commit 38ee7837bb5129276e4c75d45a554fb56ddbeb37
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue May 20 10:31:48 2014 +0000

    The mutex protecting summing of stats into the global VSC counters
    is a contention point and scales negatively with number of thread-pools.
    
    Sum into a per-pool dstat structure instead, and have the first
    idle thread summ that into the global counters without blocking the
    pool mutex.
    
    Fixes	#1495
    
    (I hope)

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index cab075b..da77ecb 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -248,6 +248,7 @@ struct acct_bereq {
 #define L1(t, n)		t n;
 #define VSC_F(n,t,l,f,v,e,d)	L##l(t, n)
 struct dstat {
+	unsigned		summs;
 #include "tbl/vsc_f_main.h"
 };
 #undef VSC_F
diff --git a/bin/varnishd/cache/cache_pool.c b/bin/varnishd/cache/cache_pool.c
index f438b26..754438a 100644
--- a/bin/varnishd/cache/cache_pool.c
+++ b/bin/varnishd/cache/cache_pool.c
@@ -72,6 +72,8 @@ struct pool {
 	uintmax_t			ndropped;
 	uintmax_t			nqueued;
 	struct sesspool			*sesspool;
+	struct dstat			*a_stat;
+	struct dstat			*b_stat;
 };
 
 static struct lock		pool_mtx;
@@ -80,15 +82,17 @@ static unsigned			pool_accepting = 0;
 
 static struct lock		wstat_mtx;
 
-/*--------------------------------------------------------------------*/
+/*--------------------------------------------------------------------
+ * Summing of stats into global stats counters
+ */
 
 static void
-wrk_sumstat(const struct dstat *ds)
+pool_sumstat(const struct dstat *src)
 {
 
 	Lck_AssertHeld(&wstat_mtx);
 #define L0(n)
-#define L1(n) (VSC_C_main->n += ds->n)
+#define L1(n) (VSC_C_main->n += src->n)
 #define VSC_F(n, t, l, f, v, d, e) L##l(n);
 #include "tbl/vsc_f_main.h"
 #undef VSC_F
@@ -101,7 +105,7 @@ Pool_Sumstat(struct worker *w)
 {
 
 	Lck_Lock(&wstat_mtx);
-	wrk_sumstat(&w->stats);
+	pool_sumstat(&w->stats);
 	Lck_Unlock(&wstat_mtx);
 	memset(&w->stats, 0, sizeof w->stats);
 }
@@ -111,13 +115,32 @@ Pool_TrySumstat(struct worker *w)
 {
 	if (Lck_Trylock(&wstat_mtx))
 		return (0);
-	wrk_sumstat(&w->stats);
+	pool_sumstat(&w->stats);
 	Lck_Unlock(&wstat_mtx);
 	memset(&w->stats, 0, sizeof w->stats);
 	return (1);
 }
 
 /*--------------------------------------------------------------------
+ * Summing of stats into pool counters
+ */
+
+static void
+pool_addstat(struct dstat *dst, struct dstat *src)
+{
+
+	dst->summs++;
+#define L0(n)
+#define L1(n) (dst->n += src->n)
+#define VSC_F(n, t, l, f, v, d, e) L##l(n);
+#include "tbl/vsc_f_main.h"
+#undef VSC_F
+#undef L0
+#undef L1
+	memset(src, 0, sizeof *src);
+}
+
+/*--------------------------------------------------------------------
  * Helper function to update stats for purges under lock
  */
 
@@ -301,6 +324,27 @@ pool_kiss_of_death(struct worker *wrk, void *priv)
 }
 
 /*--------------------------------------------------------------------
+ * Special function to summ stats
+ */
+
+static void __match_proto__(pool_func_t)
+pool_stat_summ(struct worker *wrk, void *priv)
+{
+	struct dstat *src;
+
+	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
+	CHECK_OBJ_NOTNULL(wrk->pool, POOL_MAGIC);
+	VSL(SLT_Debug, 0, "STATSUMM");
+	AN(priv);
+	src = priv;
+	Lck_Lock(&wstat_mtx);
+	pool_sumstat(src);
+	Lck_Unlock(&wstat_mtx);
+	memset(src, 0, sizeof *src);
+	wrk->pool->b_stat = src;
+}
+
+/*--------------------------------------------------------------------
  * This is the work function for worker threads in the pool.
  */
 
@@ -308,13 +352,12 @@ void
 Pool_Work_Thread(void *priv, struct worker *wrk)
 {
 	struct pool *pp;
-	int stats_dirty;
 	struct pool_task *tp;
+	struct pool_task tps;
 	int i;
 
 	CAST_OBJ_NOTNULL(pp, priv, POOL_MAGIC);
 	wrk->pool = pp;
-	stats_dirty = 0;
 	while (1) {
 		Lck_Lock(&pp->mtx);
 
@@ -332,17 +375,26 @@ Pool_Work_Thread(void *priv, struct worker *wrk)
 				VTAILQ_REMOVE(&pp->back_queue, tp, list);
 		}
 
-		if (tp == NULL) {
+		if ((tp == NULL && wrk->stats.summs > 0) ||
+		    (wrk->stats.summs >= cache_param->wthread_stats_rate))
+			pool_addstat(pp->a_stat, &wrk->stats);
+
+		if (tp != NULL) {
+			wrk->stats.summs++;
+		} else if (pp->b_stat != NULL && pp->a_stat->summs) {
+			/* Nothing to do, push pool stats into global pool */
+			tps.func = pool_stat_summ;
+			tps.priv = pp->a_stat;
+			pp->a_stat = pp->b_stat;
+			pp->b_stat = NULL;
+			tp = &tps;
+		} else {
 			/* Nothing to do: To sleep, perchance to dream ... */
 			if (isnan(wrk->lastused))
 				wrk->lastused = VTIM_real();
 			wrk->task.func = NULL;
 			wrk->task.priv = wrk;
 			VTAILQ_INSERT_HEAD(&pp->idle_queue, &wrk->task, list);
-			if (stats_dirty) {
-				Pool_Sumstat(wrk);
-				stats_dirty = 0;
-			}
 			do {
 				i = Lck_CondWait(&wrk->cond, &pp->mtx,
 				    wrk->vcl == NULL ?  0 : wrk->lastused+60.);
@@ -350,6 +402,7 @@ Pool_Work_Thread(void *priv, struct worker *wrk)
 					VCL_Rel(&wrk->vcl);
 			} while (wrk->task.func == NULL);
 			tp = &wrk->task;
+			wrk->stats.summs++;
 		}
 		Lck_Unlock(&pp->mtx);
 
@@ -358,12 +411,6 @@ Pool_Work_Thread(void *priv, struct worker *wrk)
 
 		assert(wrk->pool == pp);
 		tp->func(wrk, tp->priv);
-
-		if (++stats_dirty >= cache_param->wthread_stats_rate) {
-			Pool_Sumstat(wrk);
-			stats_dirty = 0;
-		} else if (Pool_TrySumstat(wrk))
-			stats_dirty = 0;
 	}
 	wrk->pool = NULL;
 }
@@ -507,6 +554,10 @@ pool_mkpool(unsigned pool_no)
 	ALLOC_OBJ(pp, POOL_MAGIC);
 	if (pp == NULL)
 		return (NULL);
+	pp->a_stat = calloc(1, sizeof *pp->a_stat);
+	AN(pp->a_stat);
+	pp->b_stat = calloc(1, sizeof *pp->b_stat);
+	AN(pp->b_stat);
 	Lck_New(&pp->mtx, lck_wq);
 
 	VTAILQ_INIT(&pp->idle_queue);



From phk at FreeBSD.org  Tue Jun 24 09:31:52 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 24 Jun 2014 11:31:52 +0200
Subject: [4.0] b8a53e5 Remove a now pointless assert.
Message-ID: <E1WzN4y-0003KB-G9@project.varnish-software.com>


commit b8a53e57afc6e688c81c3e581cc8ae4226c41f92
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon May 26 08:56:00 2014 +0000

    Remove a now pointless assert.
    
    Pointed out by:	Dridi Boukelmoune

diff --git a/lib/libvcc/vcc_action.c b/lib/libvcc/vcc_action.c
index 745f5b4..080f50e 100644
--- a/lib/libvcc/vcc_action.c
+++ b/lib/libvcc/vcc_action.c
@@ -178,7 +178,6 @@ parse_new(struct vcc *tl)
 		vcc_ErrWhere(tl, tl->t);
 		return;
 	}
-	XXXAZ(sy1);
 
 	sy1 = VCC_AddSymbolTok(tl, tl->t, SYM_NONE);	// XXX: NONE ?
 	XXXAN(sy1);



From phk at FreeBSD.org  Tue Jun 24 09:31:52 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 24 Jun 2014 11:31:52 +0200
Subject: [4.0] 60b5e71 Properly complain if people try to reuse symbols.
Message-ID: <E1WzN4y-0003Jy-DY@project.varnish-software.com>


commit 60b5e71e8a59a53f2575802c30a2f6eb2b7d2336
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon May 26 08:39:50 2014 +0000

    Properly complain if people try to reuse symbols.
    
    Fixes #1510

diff --git a/bin/varnishtest/tests/r01510.vtc b/bin/varnishtest/tests/r01510.vtc
new file mode 100644
index 0000000..e8933be
--- /dev/null
+++ b/bin/varnishtest/tests/r01510.vtc
@@ -0,0 +1,19 @@
+varnishtest "Duplicate object names"
+
+varnish v1 -errvcl {Object name 'first' already used.} {
+	import ${vmod_debug};
+	sub vcl_init {
+		new first = debug.obj("FOO");
+		new first = debug.obj("BAH");
+	}
+} 
+
+varnish v1 -errvcl {Object name 'first' already used.} {
+	import ${vmod_debug};
+
+	backend first { .host = "${bad_ip}"; }
+
+	sub vcl_init {
+		new first = debug.obj("FOO");
+	}
+} 
diff --git a/lib/libvcc/vcc_action.c b/lib/libvcc/vcc_action.c
index 910a89c..745f5b4 100644
--- a/lib/libvcc/vcc_action.c
+++ b/lib/libvcc/vcc_action.c
@@ -164,10 +164,25 @@ parse_new(struct vcc *tl)
 		return;
 	}
 	sy1 = VCC_FindSymbol(tl, tl->t, SYM_NONE);
+	if (sy1 != NULL) {
+		VSB_printf(tl->sb, "Object name '%.*s' already used.\n",
+		    PF(tl->t));
+
+		VSB_printf(tl->sb, "First usage:\n");
+		AN(sy1->def_b);
+		if (sy1->def_e != NULL)
+			vcc_ErrWhere2(tl, sy1->def_b, sy1->def_e);
+		else
+			vcc_ErrWhere(tl, sy1->def_b);
+		VSB_printf(tl->sb, "Redefinition:\n");
+		vcc_ErrWhere(tl, tl->t);
+		return;
+	}
 	XXXAZ(sy1);
 
 	sy1 = VCC_AddSymbolTok(tl, tl->t, SYM_NONE);	// XXX: NONE ?
 	XXXAN(sy1);
+	sy1->def_b = tl->t;
 	vcc_NextToken(tl);
 
 	ExpectErr(tl, '=');
@@ -243,6 +258,7 @@ parse_new(struct vcc *tl)
 		}
 		p += 2;
 	}
+	sy1->def_e = tl->t;
 	/*lint -restore */
 }
 



From daghf at varnish-software.com  Tue Jun 24 09:31:52 2014
From: daghf at varnish-software.com (Dag Haavi Finstad)
Date: Tue, 24 Jun 2014 11:31:52 +0200
Subject: [4.0] 2e48fba Verify that s_resp_bodybytes is indeed incremented the
 correct number of bytes on partial content responses.
Message-ID: <E1WzN4y-0003KL-KW@project.varnish-software.com>


commit 2e48fbaf3b02caaae6b7f51b58e70dd07afc3238
Author: Dag Haavi Finstad <daghf at varnish-software.com>
Date:   Mon May 26 15:06:26 2014 +0200

    Verify that s_resp_bodybytes is indeed incremented the correct number of bytes on partial content responses.

diff --git a/bin/varnishtest/tests/c00034.vtc b/bin/varnishtest/tests/c00034.vtc
index ede05b2..513038c 100644
--- a/bin/varnishtest/tests/c00034.vtc
+++ b/bin/varnishtest/tests/c00034.vtc
@@ -90,3 +90,5 @@ client c1 {
 	expect resp.status == 206
 	expect resp.bodylen == 100
 } -run
+
+varnish v1 -expect s_resp_bodybytes == 1040



From phk at FreeBSD.org  Tue Jun 24 09:31:52 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 24 Jun 2014 11:31:52 +0200
Subject: [4.0] 46b6b56 Add a message about persistent storage being deprecated
 now.
Message-ID: <E1WzN4y-0003KU-NA@project.varnish-software.com>


commit 46b6b56da4fdbea9cb86c2e69fa20e6370447c0d
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon May 26 20:53:02 2014 +0000

    Add a message about persistent storage being deprecated now.

diff --git a/doc/sphinx/phk/index.rst b/doc/sphinx/phk/index.rst
index 40a4830..3cef4af 100644
--- a/doc/sphinx/phk/index.rst
+++ b/doc/sphinx/phk/index.rst
@@ -8,6 +8,7 @@ You may or may not want to know what Poul-Henning thinks.
 .. toctree::
 	:maxdepth: 1
 
+	persistent.rst
 	dough.rst
 	wanton_destruction.rst
 	spdy.rst
diff --git a/doc/sphinx/phk/persistent.rst b/doc/sphinx/phk/persistent.rst
new file mode 100644
index 0000000..33af07c
--- /dev/null
+++ b/doc/sphinx/phk/persistent.rst
@@ -0,0 +1,98 @@
+.. _phk_pesistent:
+
+====================
+A persistent message
+====================
+
+This message is about -spersistent and why you should not use it,
+even though it is still present in Varnish 4.x.
+
+TL;DR:
+------
+
+Under narrow and ill defined circumstances, -spersistent works well,
+but in general it is more trouble than it is worth for you to run
+it, and we don't presently have the development resources to fix that.
+
+If you think you have these circumstances, you need to specify
+
+	-sdeprecated_persistence
+
+in order to use it.
+
+The long story
+--------------
+
+When we added -spersistent, to Varnish, it was in response to, and
+sponsored by a specific set of customers who really wanted this.
+
+A persistent storage module is an entirely different kettle of vax
+than a non-persistent module, because of all the ugly consistency
+issues it raises.
+
+Let me give you an example.
+
+Imagine a cluster of some Varnish servers on which bans are used.
+
+Without persistent storage, if one of them goes down and comes back
+up, all the old cached objects are gone, and so are, by definition
+all the banned objects.
+
+With persistent storage, we not only have to store the still live
+bans with the cached objects, and keep the two painfully in sync,
+so the bans gets revived with the objects, we also have to worry
+about missing bans duing the downtime, since those might ban objects
+we will recover on startup.
+
+Ouch:  Straight into database/filesystem consistency territory.
+
+But we knew that, and I thought I had a good strategy to deal with
+this.
+
+And in a sense I did.
+
+Varnish has the advantage over databases and filesystems that we
+can actually loose objects without it being a catastrophy.  It would
+be better if we didn't, but we can simply ditch stuff which doesn't
+look consistent and we'll be safe.
+
+The strategy was to do a "Log Structured Filesystem", a once promising
+concept which soon proved very troublesome to implement well.
+
+Interestingly, today the ARM chip in your SSD most likely implements
+a LFS for wear-levelling, but with a vastly reduced feature set:
+All "files" are one sector long, filenames are integers and there
+are no subdirectories or rename operations.  On the other hand,
+there is extra book-keeping about the state of the flash array.
+
+A LFS consists of two major components:  The bit that reads and
+writes, which is pretty trivial, and the bit which makes space
+available which isn't.
+
+Initially we didn't even do the second part, because in varnish
+objects expire, and provided they do so fast enough, the space will
+magically make itself available.  This worked well enough for our
+initial users, and they only used bans sporadically so that was
+cool too.
+
+In other words, a classic 20% effort, 80% benefit.
+
+Unfortunately we have not been able to find time and money for the
+other 80% effort which gives the last 20% benefit, and therefor
+-spersistent has ended up in limbo.
+
+Today we decided to officially deprecate -spersistent, and start
+warning people against using it, but we will leave it in the source
+code for now, in order to keep the interfaces necessary for a
+persistent storage working, in the hope that we will get to use
+them again later.
+
+So you can still use persistent storage, if you really want to,
+and if you know what you're doing, by using:
+
+	-sdeprecated_persistent
+
+You've been warned.
+
+
+Poul-Henning, 2014-05-26



From phk at FreeBSD.org  Tue Jun 24 09:31:52 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 24 Jun 2014 11:31:52 +0200
Subject: [4.0] 3b4efd0 Rename -spersistent to make it clear that it is
 deprecated.
Message-ID: <E1WzN4y-0003Kf-Qr@project.varnish-software.com>


commit 3b4efd06d2a687c3a32e561cf5ec0b64f3e8cd30
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon May 26 22:22:27 2014 +0000

    Rename -spersistent to make it clear that it is deprecated.

diff --git a/bin/varnishd/storage/stevedore_mgt.c b/bin/varnishd/storage/stevedore_mgt.c
index 53bfb07..7a38171 100644
--- a/bin/varnishd/storage/stevedore_mgt.c
+++ b/bin/varnishd/storage/stevedore_mgt.c
@@ -74,17 +74,44 @@ struct cli_proto cli_stv[] = {
 	    0, 0, "", stv_cli_list },
 	{ NULL}
 };
+
+/*--------------------------------------------------------------------
+ */
+
+static void
+smp_fake_init(struct stevedore *parent, int ac, char * const *av)
+{
+
+	(void)parent;
+	(void)ac;
+	(void)av;
+	ARGV_ERR(
+	    "-spersistent has been deprecated, please see:\n"
+	    "  https://www.varnish-cache.org/docs/trunk/phk/persistent.html\n"
+	    "for details.\n"
+	);
+}
+
+
+static const struct stevedore smp_fake_stevedore = {
+	.magic = STEVEDORE_MAGIC,
+	.name = "deprecated_persistent",
+	.init = smp_fake_init,
+};
+
+
 /*--------------------------------------------------------------------
  * Parse a stevedore argument on the form:
  *	[ name '=' ] strategy [ ',' arg ] *
  */
 
 static const struct choice STV_choice[] = {
-	{ "file",	&smf_stevedore },
-	{ "malloc",	&sma_stevedore },
-	{ "persistent",	&smp_stevedore },
+	{ "file",			&smf_stevedore },
+	{ "malloc",			&sma_stevedore },
+	{ "deprecated_persistent",	&smp_stevedore },
+	{ "persistent",			&smp_fake_stevedore },
 #ifdef HAVE_LIBUMEM
-	{ "umem",	&smu_stevedore },
+	{ "umem",			&smu_stevedore },
 #endif
 	{ NULL,		NULL }
 };
@@ -133,9 +160,6 @@ STV_Config(const char *spec)
 
 	*stv = *stv2;
 	AN(stv->name);
-	AN(stv->alloc);
-	if (stv->allocobj == NULL)
-		stv->allocobj = stv_default_allocobj;
 
 	if (p == NULL)
 		bprintf(stv->ident, "s%u", seq++);
@@ -158,6 +182,10 @@ STV_Config(const char *spec)
 	else if (ac != 0)
 		ARGV_ERR("(-s%s) too many arguments\n", stv->name);
 
+	AN(stv->alloc);
+	if (stv->allocobj == NULL)
+		stv->allocobj = stv_default_allocobj;
+
 	if (!strcmp(stv->ident, TRANSIENT_STORAGE)) {
 		stv->transient = 1;
 		AZ(stv_transient);
diff --git a/bin/varnishd/storage/storage_persistent.c b/bin/varnishd/storage/storage_persistent.c
index d7506f0..6633799 100644
--- a/bin/varnishd/storage/storage_persistent.c
+++ b/bin/varnishd/storage/storage_persistent.c
@@ -588,7 +588,7 @@ smp_free(struct storage *st)
 
 const struct stevedore smp_stevedore = {
 	.magic	=	STEVEDORE_MAGIC,
-	.name	=	"persistent",
+	.name	=	"deprecated_persistent",
 	.init	=	smp_mgt_init,
 	.open	=	smp_open,
 	.close	=	smp_close,
diff --git a/bin/varnishtest/tests/a00009.vtc b/bin/varnishtest/tests/a00009.vtc
index 34a56f0..73021ba 100644
--- a/bin/varnishtest/tests/a00009.vtc
+++ b/bin/varnishtest/tests/a00009.vtc
@@ -1,5 +1,6 @@
-varnishtest "Code coverage of VCL compiler and RSTdump"
+varnishtest "Code coverage of VCL compiler and RSTdump etc"
 
 shell "${varnishd} -b 127.0.0.1:80 -C -n ${tmpdir} > /dev/null 2>&1"
 shell "${varnishd} -x dumprstparam > /dev/null 2>&1"
 shell "${varnishd} -x dumprstvsl > /dev/null 2>&1"
+shell "! ${varnishd} -spersistent > /dev/null 2>&1"
diff --git a/bin/varnishtest/tests/p00000.vtc b/bin/varnishtest/tests/p00000.vtc
index ab59ccf..eb9d29d 100644
--- a/bin/varnishtest/tests/p00000.vtc
+++ b/bin/varnishtest/tests/p00000.vtc
@@ -9,7 +9,7 @@ shell "rm -f ${tmpdir}/_.per"
 
 varnish v1 \
 	-arg "-pfeature=+wait_silo" \
-	-arg "-spersistent,${tmpdir}/_.per,10m" \
+	-arg "-sdeprecated_persistent,${tmpdir}/_.per,10m" \
 	-vcl+backend { } -start
 
 varnish v1 -stop
diff --git a/bin/varnishtest/tests/p00002.vtc b/bin/varnishtest/tests/p00002.vtc
index 1e04ebc..e60c8a6 100644
--- a/bin/varnishtest/tests/p00002.vtc
+++ b/bin/varnishtest/tests/p00002.vtc
@@ -10,8 +10,8 @@ server s1 {
 varnish v1 \
 	-arg "-pfeature=+wait_silo" \
 	-arg "-pban_lurker_sleep=0" \
-	-arg "-spersistent,${tmpdir}/_.per1,10m" \
-	-arg "-spersistent,${tmpdir}/_.per2,10m" \
+	-arg "-sdeprecated_persistent,${tmpdir}/_.per1,10m" \
+	-arg "-sdeprecated_persistent,${tmpdir}/_.per2,10m" \
 	-vcl+backend { } -start
 
 client c1 {
diff --git a/bin/varnishtest/tests/p00003.vtc b/bin/varnishtest/tests/p00003.vtc
index 0e8eec7..98984df 100644
--- a/bin/varnishtest/tests/p00003.vtc
+++ b/bin/varnishtest/tests/p00003.vtc
@@ -9,7 +9,7 @@ server s1 {
 
 varnish v1 \
 	-arg "-pfeature=+wait_silo" \
-	-arg "-spersistent,${tmpdir}/_.per,10m" \
+	-arg "-sdeprecated_persistent,${tmpdir}/_.per,10m" \
 	-arg "-pban_lurker_sleep=0" \
 	-vcl+backend { } -start
 
diff --git a/bin/varnishtest/tests/p00004.vtc b/bin/varnishtest/tests/p00004.vtc
index c3aea98..a908913 100644
--- a/bin/varnishtest/tests/p00004.vtc
+++ b/bin/varnishtest/tests/p00004.vtc
@@ -11,7 +11,7 @@ server s1 {
 
 varnish v1 \
 	-arg "-pfeature=+wait_silo" \
-	-arg "-spersistent,${tmpdir}/_.per,10m" \
+	-arg "-sdeprecated_persistent,${tmpdir}/_.per,10m" \
 	-arg "-pban_lurker_sleep=0" \
 	-vcl+backend { } -start
 
diff --git a/bin/varnishtest/tests/p00005.vtc b/bin/varnishtest/tests/p00005.vtc
index 1ac8e08..55bc6c9 100644
--- a/bin/varnishtest/tests/p00005.vtc
+++ b/bin/varnishtest/tests/p00005.vtc
@@ -8,7 +8,7 @@ server s1 {
 } -start
 
 varnish v1 \
-	-arg "-spersistent,${tmpdir}/_.per,10m" \
+	-arg "-sdeprecated_persistent,${tmpdir}/_.per,10m" \
 	-arg "-pban_lurker_sleep=0" \
 	-arg "-pshortlived=0" \
 	-vcl+backend {
diff --git a/bin/varnishtest/tests/p00006.vtc b/bin/varnishtest/tests/p00006.vtc
index 96f4d17..fb5fcca 100644
--- a/bin/varnishtest/tests/p00006.vtc
+++ b/bin/varnishtest/tests/p00006.vtc
@@ -11,7 +11,7 @@ server s1 {
 
 
 varnish v1 \
-	-arg "-spersistent,${tmpdir}/_.per,10m" \
+	-arg "-sdeprecated_persistent,${tmpdir}/_.per,10m" \
 	-arg "-pban_lurker_sleep=0" \
 	-vcl+backend { } -start
 
diff --git a/bin/varnishtest/tests/p00007.vtc b/bin/varnishtest/tests/p00007.vtc
index 5fd9e9e..1daf4e5 100644
--- a/bin/varnishtest/tests/p00007.vtc
+++ b/bin/varnishtest/tests/p00007.vtc
@@ -24,7 +24,7 @@ server s1 {
 	txresp -bodylen 48
 } -start
 
-varnish v1 -arg "-spersistent,${tmpdir}/_.per,10m" \
+varnish v1 -arg "-sdeprecated_persistent,${tmpdir}/_.per,10m" \
 	-vcl+backend {} -start
 
 varnish v1 -cliok "debug.fragfetch 32"
diff --git a/bin/varnishtest/tests/p00008.vtc b/bin/varnishtest/tests/p00008.vtc
index b839a0c..c7a144d 100644
--- a/bin/varnishtest/tests/p00008.vtc
+++ b/bin/varnishtest/tests/p00008.vtc
@@ -16,8 +16,8 @@ server s1 {
 varnish v1 \
 	-arg "-pfeature=+wait_silo" \
 	-arg "-pban_lurker_sleep=0" \
-	-arg "-sper1=persistent,${tmpdir}/_.per1,10m" \
-	-arg "-sper2=persistent,${tmpdir}/_.per2,10m" \
+	-arg "-sper1=deprecated_persistent,${tmpdir}/_.per1,10m" \
+	-arg "-sper2=deprecated_persistent,${tmpdir}/_.per2,10m" \
 	-vcl+backend {
 		sub vcl_backend_response {
 			set beresp.storage_hint = "per1";
@@ -45,7 +45,7 @@ server s1 -wait
 varnish v2 \
 	-arg "-pfeature=+wait_silo" \
 	-arg "-pban_lurker_sleep=0" \
-	-arg "-spersistent,${tmpdir}/_.per1,10m" \
+	-arg "-sdeprecated_persistent,${tmpdir}/_.per1,10m" \
 	-vcl+backend { } -start
 varnish v2 -cliok "ban obj.http.x-foo == foo"
 varnish v2 -cliok "ban.list"
@@ -56,8 +56,8 @@ varnish v2 -stop
 varnish v3 \
 	-arg "-pfeature=+wait_silo" \
 	-arg "-pban_lurker_sleep=0" \
-	-arg "-spersistent,${tmpdir}/_.per1,10m" \
-	-arg "-spersistent,${tmpdir}/_.per2,10m" \
+	-arg "-sdeprecated_persistent,${tmpdir}/_.per1,10m" \
+	-arg "-sdeprecated_persistent,${tmpdir}/_.per2,10m" \
 	-vcl+backend { } -start
 varnish v3 -cliok "ban.list"
 varnish v3 -stop
@@ -72,7 +72,7 @@ server s1 {
 varnish v4 \
 	-arg "-pfeature=+wait_silo" \
 	-arg "-pban_lurker_sleep=0" \
-	-arg "-spersistent,${tmpdir}/_.per2,10m" \
+	-arg "-sdeprecated_persistent,${tmpdir}/_.per2,10m" \
 	-vcl+backend { } -start
 client c1 -connect ${v4_sock} {
 	txreq -url "/silo2"
diff --git a/bin/varnishtest/tests/p00009.vtc b/bin/varnishtest/tests/p00009.vtc
index 66b1e2f..6ec15d4 100644
--- a/bin/varnishtest/tests/p00009.vtc
+++ b/bin/varnishtest/tests/p00009.vtc
@@ -14,8 +14,8 @@ server s1 {
 varnish v1 \
 	-arg "-pfeature=+wait_silo" \
 	-arg "-pban_lurker_sleep=0" \
-	-arg "-sper1=persistent,${tmpdir}/_.per1,10m" \
-	-arg "-sper2=persistent,${tmpdir}/_.per2,10m" \
+	-arg "-sper1=deprecated_persistent,${tmpdir}/_.per1,10m" \
+	-arg "-sper2=deprecated_persistent,${tmpdir}/_.per2,10m" \
 	-vcl+backend {
 	}
 varnish v1 -start
diff --git a/bin/varnishtest/tests/r00915.vtc b/bin/varnishtest/tests/r00915.vtc
index 607d6db..3e1528c 100644
--- a/bin/varnishtest/tests/r00915.vtc
+++ b/bin/varnishtest/tests/r00915.vtc
@@ -9,7 +9,7 @@ shell "rm -f ${tmpdir}/_.per"
 
 varnish v1 \
 	-arg "-pfeature=+wait_silo" \
-	-arg "-spersistent,${tmpdir}/_.per,10m" \
+	-arg "-sdeprecated_persistent,${tmpdir}/_.per,10m" \
 	-vcl+backend {
 
 	sub vcl_backend_response {
diff --git a/bin/varnishtest/tests/r00962.vtc b/bin/varnishtest/tests/r00962.vtc
index 9cae4cf..6d750cb 100644
--- a/bin/varnishtest/tests/r00962.vtc
+++ b/bin/varnishtest/tests/r00962.vtc
@@ -12,8 +12,8 @@ shell "rm -f ${tmpdir}/_.per?"
 
 varnish v1 \
 	-arg "-pfeature=+wait_silo" \
-	-arg "-spersistent,${tmpdir}/_.per1,10m" \
-	-arg "-spersistent,${tmpdir}/_.per2,10m" \
+	-arg "-sdeprecated_persistent,${tmpdir}/_.per1,10m" \
+	-arg "-sdeprecated_persistent,${tmpdir}/_.per2,10m" \
 	-vcl+backend {
 	sub vcl_backend_response {
 		set beresp.storage = "s0";
@@ -44,8 +44,8 @@ server s1 {
 
 varnish v2 \
 	-arg "-pfeature=+wait_silo" \
-	-arg "-spersistent,${tmpdir}/_.per2,10m" \
-	-arg "-spersistent,${tmpdir}/_.per1,10m" \
+	-arg "-sdeprecated_persistent,${tmpdir}/_.per2,10m" \
+	-arg "-sdeprecated_persistent,${tmpdir}/_.per1,10m" \
 	-vcl+backend { } -start
 
 client c1 -connect ${v2_sock} {
diff --git a/bin/varnishtest/tests/r01225.vtc b/bin/varnishtest/tests/r01225.vtc
index 23d0253..85cc418 100644
--- a/bin/varnishtest/tests/r01225.vtc
+++ b/bin/varnishtest/tests/r01225.vtc
@@ -10,7 +10,7 @@ server s1 {
 
 varnish v1 \
 	-arg "-pfeature=+wait_silo" \
-	-arg "-spersistent,${tmpdir}/_.per,10m" \
+	-arg "-sdeprecated_persistent,${tmpdir}/_.per,10m" \
 	-arg "-pban_lurker_sleep=0.01" \
 	-vcl+backend { } -start
 
diff --git a/bin/varnishtest/tests/r01266.vtc b/bin/varnishtest/tests/r01266.vtc
index 76352c3..bf4bf42 100644
--- a/bin/varnishtest/tests/r01266.vtc
+++ b/bin/varnishtest/tests/r01266.vtc
@@ -12,7 +12,7 @@ server s1 {
 varnish v1 \
 	-arg "-pfeature=+wait_silo" \
 	-arg "-pban_lurker_sleep=0.01" \
-	-arg "-sper1=persistent,${tmpdir}/_.per1,10m" \
+	-arg "-sper1=deprecated_persistent,${tmpdir}/_.per1,10m" \
 	-vcl+backend {
 	}
 varnish v1 -start



From phk at FreeBSD.org  Tue Jun 24 09:31:52 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 24 Jun 2014 11:31:52 +0200
Subject: [4.0] 5dfd0c0 Minor cleanup of how we write the warning at the top of
 generated files.
Message-ID: <E1WzN4y-0003Kp-TQ@project.varnish-software.com>


commit 5dfd0c099569282164e4f03c64e172205cb2c17d
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue May 27 10:18:22 2014 +0000

    Minor cleanup of how we write the warning at the top of generated files.

diff --git a/lib/libvcc/vmodtool.py b/lib/libvcc/vmodtool.py
index 25e0624..42e2fd3 100755
--- a/lib/libvcc/vmodtool.py
+++ b/lib/libvcc/vmodtool.py
@@ -68,14 +68,18 @@ ctypes = {
 
 #######################################################################
 
-def write_file_header(fo):
-	fo.write("""/*
- * NB:  This file is machine generated, DO NOT EDIT!
- *
- * Edit vmod.vcc and run %s instead
- */
+def write_file_warning(fo, a, b, c):
+	fo.write(a + "\n")
+	fo.write(b + "NB:  This file is machine generated, DO NOT EDIT!\n")
+	fo.write(b + "\n")
+	fo.write(b + "Edit vmod.vcc and run make instead\n")
+	fo.write(c + "\n\n")
 
-""" % basename(__file__))
+def write_c_file_warning(fo):
+	write_file_warning(fo, "/*", " * ", " */")
+
+def write_rst_file_warning(fo):
+	write_file_warning(fo, "..", ".. ", "..")
 
 #######################################################################
 
@@ -815,8 +819,8 @@ def runmain(inputvcc, outputname="vcc_if"):
 	fc = open("%s.c" % outputname, "w")
 	fh = open("%s.h" % outputname, "w")
 
-	write_file_header(fc)
-	write_file_header(fh)
+	write_c_file_warning(fc)
+	write_c_file_warning(fh)
 
 	fh.write('struct vrt_ctx;\n')
 	fh.write('struct VCL_conf;\n')
@@ -841,22 +845,19 @@ def runmain(inputvcc, outputname="vcc_if"):
 	fh.close()
 
 	for suf in ("", ".man"):
-		with open("vmod_%s%s.rst" % (vx[0].nam, suf), "w") as fp:
-			fp.write("..\n")
-			fp.write(".. This file was autogenerated by %s. DO NOT EDIT!\n" %
-				 basename(__file__))
-			fp.write("..\n\n")
-
-			vx[0].doc_dump(fp, suf)
-
-			if len(copyright) > 0:
-				fp.write("\n")
-				fp.write("COPYRIGHT\n")
-				fp.write("=========\n")
-				fp.write("\n::\n\n")
-				for i in copyright:
-					fp.write("  %s\n" % i)
-				fp.write("\n")
+		fp = open("vmod_%s%s.rst" % (vx[0].nam, suf), "w")
+		write_rst_file_warning(fp)
+
+		vx[0].doc_dump(fp, suf)
+
+		if len(copyright) > 0:
+			fp.write("\n")
+			fp.write("COPYRIGHT\n")
+			fp.write("=========\n")
+			fp.write("\n::\n\n")
+			for i in copyright:
+				fp.write("  %s\n" % i)
+			fp.write("\n")
 
 
 if __name__ == "__main__":



From martin at varnish-software.com  Tue Jun 24 09:31:53 2014
From: martin at varnish-software.com (Martin Blix Grydeland)
Date: Tue, 24 Jun 2014 11:31:53 +0200
Subject: [4.0] 4a18350 Added %I and %O formatters to varnishncsa
Message-ID: <E1WzN4z-0003Ky-1A@project.varnish-software.com>


commit 4a18350b045b945dfc0b22eab2585c8c8e11ba92
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Tue May 27 12:37:10 2014 +0200

    Added %I and %O formatters to varnishncsa
    
    This adds %I and %O formatters to varnishncsa, which are total bytes
    received and total bytes sent. These are similar to the same
    formatters in apache (with mod_logio). Useful for bandwidth accounting.
    
    Patch by: David Robertson

diff --git a/bin/varnishncsa/varnishncsa.c b/bin/varnishncsa/varnishncsa.c
index 2162118..5aa0361 100644
--- a/bin/varnishncsa/varnishncsa.c
+++ b/bin/varnishncsa/varnishncsa.c
@@ -45,6 +45,8 @@
  *	%b		Length of reply body, or "-"
  *	%{Referer}i	Contents of "Referer" request header
  *	%{User-agent}i	Contents of "User-agent" request header
+ *  %I		Total bytes recieved from client
+ *  %O		Total bytes sent to client
  *
  * Actually, we cheat a little and replace "%r" with something close to
  * "%m http://%{Host}i%U%q %H", where the additional fields are:
@@ -95,6 +97,8 @@ enum e_frag {
 	F_h,			/* %h Host name / IP Address */
 	F_m,			/* %m Method */
 	F_s,			/* %s Status */
+	F_I,			/* %I Bytes recieved */
+	F_O,			/* %O Bytes sent */
 	F_tstart,		/* Time start */
 	F_tend,			/* Time end */
 	F_ttfb,			/* %{Varnish:time_firstbyte}x */
@@ -574,6 +578,12 @@ parse_format(const char *format)
 		case 'U':	/* URL */
 			addf_fragment(&CTX.frag[F_U], "-");
 			break;
+		case 'I':	/* Bytes recieved */
+			addf_fragment(&CTX.frag[F_I], "-");
+			break;
+		case 'O':	/* Bytes sent */
+			addf_fragment(&CTX.frag[F_O], "-");
+			break;
 		case '{':
 			p++;
 			q = p;
@@ -783,6 +793,8 @@ dispatch_f(struct VSL_data *vsl, struct VSL_transaction * const pt[],
 				break;
 			case SLT_ReqAcct:
 				frag_fields(b, e, 5, &CTX.frag[F_b], 0, NULL);
+				frag_fields(b, e, 3, &CTX.frag[F_I], 0, NULL);
+				frag_fields(b, e, 6, &CTX.frag[F_O], 0, NULL);
 				break;
 			case SLT_Timestamp:
 				if (isprefix(b, "Start:", e, &p)) {



From martin at varnish-software.com  Tue Jun 24 09:31:53 2014
From: martin at varnish-software.com (Martin Blix Grydeland)
Date: Tue, 24 Jun 2014 11:31:53 +0200
Subject: [4.0] d4baf14 Document %I and %O ncsa format specifiers
Message-ID: <E1WzN4z-0003L9-4S@project.varnish-software.com>


commit d4baf146d1f412d6ab36c21cd7ca45b5357fe77b
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Tue May 27 13:01:45 2014 +0200

    Document %I and %O ncsa format specifiers

diff --git a/doc/sphinx/reference/varnishncsa.rst b/doc/sphinx/reference/varnishncsa.rst
index 77086ac..ffdb7f4 100644
--- a/doc/sphinx/reference/varnishncsa.rst
+++ b/doc/sphinx/reference/varnishncsa.rst
@@ -57,6 +57,9 @@ Supported formatters are:
 %h
   Remote host. Defaults to '-' if not known.
 
+%I
+  Total bytes received from client.
+
 %{X}i
   The contents of request header X.
 
@@ -72,6 +75,9 @@ Supported formatters are:
 %{X}o
   The contents of response header X.
 
+%O
+  Total bytes sent to client.
+
 %r
   The first line of the request. Synthesized from other fields, so it
   may not be the request verbatim.



From martin at varnish-software.com  Tue Jun 24 09:31:53 2014
From: martin at varnish-software.com (Martin Blix Grydeland)
Date: Tue, 24 Jun 2014 11:31:53 +0200
Subject: [4.0] f53ff46 Only parse ReqAcct once
Message-ID: <E1WzN4z-0003LL-8E@project.varnish-software.com>


commit f53ff463561ca6e4c79a820852550453fb0dac5b
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Tue May 27 13:02:55 2014 +0200

    Only parse ReqAcct once

diff --git a/bin/varnishncsa/varnishncsa.c b/bin/varnishncsa/varnishncsa.c
index 5aa0361..16e9e30 100644
--- a/bin/varnishncsa/varnishncsa.c
+++ b/bin/varnishncsa/varnishncsa.c
@@ -792,9 +792,11 @@ dispatch_f(struct VSL_data *vsl, struct VSL_transaction * const pt[],
 				frag_line(b, e, &CTX.frag[F_s]);
 				break;
 			case SLT_ReqAcct:
-				frag_fields(b, e, 5, &CTX.frag[F_b], 0, NULL);
-				frag_fields(b, e, 3, &CTX.frag[F_I], 0, NULL);
-				frag_fields(b, e, 6, &CTX.frag[F_O], 0, NULL);
+				frag_fields(b, e,
+				    3, &CTX.frag[F_I],
+				    5, &CTX.frag[F_b],
+				    6, &CTX.frag[F_O],
+				    0, NULL);
 				break;
 			case SLT_Timestamp:
 				if (isprefix(b, "Start:", e, &p)) {



From martin at varnish-software.com  Tue Jun 24 09:31:53 2014
From: martin at varnish-software.com (Martin Blix Grydeland)
Date: Tue, 24 Jun 2014 11:31:53 +0200
Subject: [4.0] f2cf2ea Keep the list of ncsa format specifiers in one location
 only
Message-ID: <E1WzN4z-0003LS-Af@project.varnish-software.com>


commit f2cf2eaf5c3ea5a304454b9d265028024f4889b9
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Tue May 27 13:03:52 2014 +0200

    Keep the list of ncsa format specifiers in one location only
    
    Other minor cleanups

diff --git a/bin/varnishncsa/varnishncsa.c b/bin/varnishncsa/varnishncsa.c
index 16e9e30..2ff1fc3 100644
--- a/bin/varnishncsa/varnishncsa.c
+++ b/bin/varnishncsa/varnishncsa.c
@@ -30,32 +30,12 @@
  * SUCH DAMAGE.
  *
  * Obtain log data from the shared memory log, order it by session ID, and
- * display it in Apache / NCSA combined log format:
+ * display it in Apache / NCSA combined log format.
  *
- *	%h %l %u %t "%r" %s %b "%{Referer}i" "%{User-agent}i"
+ * See doc/sphinx/reference/varnishncsa.rst for the supported format
+ * specifiers.
  *
- * where the fields are defined as follows:
- *
- *	%h		Client host name or IP address (always the latter)
- *	%l		Client user ID as reported by identd (always "-")
- *	%u		User ID if using HTTP authentication, or "-"
- *	%t		Date and time of request
- *	%r		Request line
- *	%s		Status code
- *	%b		Length of reply body, or "-"
- *	%{Referer}i	Contents of "Referer" request header
- *	%{User-agent}i	Contents of "User-agent" request header
- *  %I		Total bytes recieved from client
- *  %O		Total bytes sent to client
- *
- * Actually, we cheat a little and replace "%r" with something close to
- * "%m http://%{Host}i%U%q %H", where the additional fields are:
- *
- *	%m		Request method
- *	%{Host}i	Contents of "Host" request header
- *	%U		URL path
- *	%q		Query string
- *	%H		Protocol version
+ * Note: %r is "%m http://%{Host}i%U%q %H"
  *
  */
 
@@ -551,12 +531,18 @@ parse_format(const char *format)
 		case 'H':	/* Protocol */
 			addf_fragment(&CTX.frag[F_H], "HTTP/1.0");
 			break;
+		case 'I':	/* Bytes recieved */
+			addf_fragment(&CTX.frag[F_I], "-");
+			break;
 		case 'l':	/* Client user ID (identd) always '-' */
 			AZ(VSB_putc(vsb, '-'));
 			break;
 		case 'm':	/* Method */
 			addf_fragment(&CTX.frag[F_m], "-");
 			break;
+		case 'O':	/* Bytes sent */
+			addf_fragment(&CTX.frag[F_O], "-");
+			break;
 		case 'q':	/* Query string */
 			addf_fragment(&CTX.frag[F_q], "");
 			break;
@@ -572,18 +558,12 @@ parse_format(const char *format)
 		case 'T':	/* Int request time */
 			addf_time(*p, NULL, NULL);
 			break;
-		case 'u':
+		case 'u':	/* Remote user from auth */
 			addf_auth("-");
 			break;
 		case 'U':	/* URL */
 			addf_fragment(&CTX.frag[F_U], "-");
 			break;
-		case 'I':	/* Bytes recieved */
-			addf_fragment(&CTX.frag[F_I], "-");
-			break;
-		case 'O':	/* Bytes sent */
-			addf_fragment(&CTX.frag[F_O], "-");
-			break;
 		case '{':
 			p++;
 			q = p;
diff --git a/doc/sphinx/reference/varnishncsa.rst b/doc/sphinx/reference/varnishncsa.rst
index ffdb7f4..85906fc 100644
--- a/doc/sphinx/reference/varnishncsa.rst
+++ b/doc/sphinx/reference/varnishncsa.rst
@@ -69,15 +69,15 @@ Supported formatters are:
 %m
    Request method. Defaults to '-' if not known.
 
-%q
-  The query string, if no query string exists, an empty string.
-
 %{X}o
   The contents of response header X.
 
 %O
   Total bytes sent to client.
 
+%q
+  The query string, if no query string exists, an empty string.
+
 %r
   The first line of the request. Synthesized from other fields, so it
   may not be the request verbatim.



From phk at FreeBSD.org  Tue Jun 24 09:31:53 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 24 Jun 2014 11:31:53 +0200
Subject: [4.0] 9c3a747 Beautify the output of the vmodtool to make it (also)
 human readable.
Message-ID: <E1WzN4z-0003Lb-Dn@project.varnish-software.com>


commit 9c3a747d2ae9352458337316e90c06693e960832
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue May 27 14:10:29 2014 +0000

    Beautify the output of the vmodtool to make it (also) human readable.

diff --git a/lib/libvcc/vmodtool.py b/lib/libvcc/vmodtool.py
index 42e2fd3..ea01391 100755
--- a/lib/libvcc/vmodtool.py
+++ b/lib/libvcc/vmodtool.py
@@ -83,6 +83,27 @@ def write_rst_file_warning(fo):
 
 #######################################################################
 
+def lwrap(s, w=72):
+	"""
+	Wrap a c-prototype like string into a number of lines
+	"""
+	l = []
+	p=""
+	while len(s) > w:
+		y = s[:w].rfind(',')
+		if y == -1:
+			y = s[:w].rfind('(')
+		if y == -1:
+			break
+		l.append(p + s[:y + 1])
+		s = s[y + 1:].lstrip()
+		p = "    "
+	if len(s) > 0:
+		l.append(p + s)
+	return l
+		
+#######################################################################
+
 def is_c_name(s):
 	return None != re.match("^[a-z][a-z0-9_]*$", s)
 
@@ -145,11 +166,15 @@ class vmod(object):
 
 	def c_proto(self, fo):
 		for o in self.objs:
+			fo.write("/* Object %s */\n" % o.nam)
 			o.fixup(self.nam)
 			o.c_proto(fo)
 			fo.write("\n")
+		if len(self.funcs) > 0:
+			fo.write("/* Functions */\n")
 		for f in self.funcs:
-			f.c_proto(fo)
+			for i in lwrap(f.c_proto()):
+				fo.write(i + "\n")
 		if self.init != None:
 			fo.write("\n")
 			fo.write("int " + self.init)
@@ -162,7 +187,8 @@ class vmod(object):
 			for t in o.c_typedefs(self.nam):
 				l.append(t)
 			l.append("")
-		l.append("/* Functions */")
+		if len(self.funcs) > 0:
+			l.append("/* Functions */")
 		for f in self.funcs:
 			l.append(f.c_typedef(self.nam))
 		l.append("")
@@ -170,7 +196,8 @@ class vmod(object):
 
 	def c_typedefs(self, fo):
 		for i in self.c_typedefs_():
-			fo.write(i + "\n")
+			for j in lwrap(i):
+				fo.write(j + "\n")
 
 	def c_vmod(self, fo):
 		fo.write('extern const char Vmod_' + self.nam + '_Name[];\n')
@@ -197,7 +224,8 @@ class vmod(object):
 		fo.write("extern const char Vmod_" + self.nam + "_Proto[];\n")
 		fo.write("const char Vmod_" + self.nam + "_Proto[] =\n")
 		for t in self.c_typedefs_():
-			fo.write('\t"' + t + '\\n"\n')
+			for i in lwrap(t, w=64):
+				fo.write('\t"' + i + '\\n"\n')
 		fo.write('\t"\\n"\n')
 		for i in (cs + ";").split("\n"):
 			fo.write('\n\t"' + i + '\\n"')
@@ -249,14 +277,15 @@ class vmod(object):
 		s = "extern " + s + ";\n" + s + " = {\n"
 
 		for o in self.objs:
-			s += o.c_strspec(self.nam)
+			s += o.c_strspec(self.nam) + ",\n\n"
 
-		s += "\n\t/* Functions */\n"
+		if len(self.funcs) > 0:
+			s += "\t/* Functions */\n"
 		for f in self.funcs:
-			s += '\t"' + f.c_strspec(self.nam) + '",\n'
+			s += f.c_strspec(self.nam) + ',\n\n'
 
-		s += "\n\t/* Init/Fini */\n"
 		if self.init != None:
+			s += "\t/* Init/Fini */\n"
 			s += '\t"INIT\\0Vmod_' + self.nam + '_Func._init",\n'
 
 		s += "\t0\n"
@@ -323,22 +352,22 @@ class func(object):
 	def set_pfx(self, s):
 		self.pfx = s
 
-	def c_proto(self, fo, fini=False):
-		fo.write(ctypes[self.retval])
-		fo.write(" vmod_" + self.cnam + "(")
+	def c_proto(self, fini=False):
+		s = ctypes[self.retval] + " vmod_" + self.cnam + "("
 		p = ""
 		if not fini:
-			fo.write("const struct vrt_ctx *")
+			s += "const struct vrt_ctx *"
 			p = ", "
 		if self.pfx != None:
-			fo.write(p + self.pfx)
+			s += p + self.pfx
 			p = ", "
 		for a in self.al:
-			fo.write(p + ctypes[a.typ])
+			s += p + ctypes[a.typ]
 			p = ", "
 			if a.nam != None:
-				fo.write(" " + a.nam)
-		fo.write(");\n")
+				s += " " + a.nam
+		s += ");"
+		return s
 
 	def c_typedef(self, modname, fini=False):
 		s = "typedef "
@@ -359,21 +388,25 @@ class func(object):
 
 	def c_struct(self, modname):
 		s = '\ttd_' + modname + "_" + self.cnam
-		while len(s.expandtabs()) < 40:
-			s += "\t"
+		if len(s.expandtabs()) >= 40:
+			s += "\n\t\t\t\t\t"
+		else:
+			while len(s.expandtabs()) < 40:
+				s += "\t"
 		s += "*" + self.cnam + ";\n"
 		return s
 
 	def c_initializer(self):
 		return "\tvmod_" + self.cnam + ",\n"
 
-	def c_strspec(self, modnam):
-		s = modnam + "." + self.nam
-		s += "\\0"
-		s += "Vmod_" + modnam + "_Func." + self.cnam + "\\0"
-		s += self.retval + "\\0"
+	def c_strspec(self, modnam, pfx = "\t"):
+		s = pfx + '"' + modnam + "." + self.nam + '\\0"\n'
+		s += pfx + '"'
+		s += "Vmod_" + modnam + "_Func." + self.cnam + '\\0"\n'
+		s += pfx + '    "' + self.retval + '\\0"\n'
 		for a in self.al:
-			s += a.c_strspec()
+			s += pfx + '\t"' + a.c_strspec() + '"\n'
+		s += pfx + '"\\0"'
 		return s
 
 	def doc(self, l):
@@ -453,10 +486,13 @@ class obj(object):
 
 	def c_proto(self, fo):
 		fo.write(self.st + ";\n")
-		self.init.c_proto(fo)
-		self.fini.c_proto(fo, fini=True)
+		l = []
+		l += lwrap(self.init.c_proto())
+		l += lwrap(self.fini.c_proto(fini=True))
 		for m in self.methods:
-			m.c_proto(fo)
+			l += lwrap(m.c_proto())
+		for i in l:
+			fo.write(i + "\n")
 
 	def c_struct(self, modnam):
 		s = "\t/* Object " + self.nam + " */\n"
@@ -477,12 +513,12 @@ class obj(object):
 	def c_strspec(self, modnam):
 		s = "\t/* Object " + self.nam + " */\n"
 		s += '\t"OBJ\\0"\n'
-		s += '\t\t"' + self.init.c_strspec(modnam) + '\\0"\n'
+		s += self.init.c_strspec(modnam, pfx="\t\t") + '\n'
 		s += '\t\t"' + self.st + '\\0"\n'
-		s += '\t\t"' + self.fini.c_strspec(modnam) + '\\0"\n'
+		s += self.fini.c_strspec(modnam, pfx="\t\t") + '\n'
 		for m in self.methods:
-			s += '\t\t"' + m.c_strspec(modnam) + '\\0"\n'
-		s += '\t\t"\\0",\n'
+			s += m.c_strspec(modnam, pfx="\t\t") + '\n'
+		s += '\t"\\0"'
 		return s
 
 	def doc(self, l):



From phk at FreeBSD.org  Tue Jun 24 09:31:53 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 24 Jun 2014 11:31:53 +0200
Subject: [4.0] 046d8d1 Some more style polishing of vmodtool
Message-ID: <E1WzN4z-0003Lm-HL@project.varnish-software.com>


commit 046d8d1c6514e7987604e8ee28d9ec860938a4b0
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Jun 2 08:15:36 2014 +0000

    Some more style polishing of vmodtool

diff --git a/lib/libvcc/vmodtool.py b/lib/libvcc/vmodtool.py
index ea01391..3299074 100755
--- a/lib/libvcc/vmodtool.py
+++ b/lib/libvcc/vmodtool.py
@@ -25,17 +25,13 @@
 # LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
 # OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 # SUCH DAMAGE.
-#
-# Read the vmod.vcc file (inputvcc) and produce the vmod.h and vmod.c files.
-#
-# vmod.h contains the prototypes for the published functions, the module
-# C-code should include this file to ensure type-consistency.
-#
-# vmod.c contains the symbols which VCC and varnishd will use to access
-# the module:  A structure of properly typed function pointers, the
-# size of this structure in bytes, and the definition of the structure
-# as a string, suitable for inclusion in the C-source of the compile VCL
-# program.
+
+"""
+Read the vmod.vcc file (inputvcc) and produce:
+	vmod_if.h -- Prototypes for the implementation
+	vmod_if.c -- Magic glue & datastructures to make things a VMOD.
+	vmod_${name}.rst -- Extracted documentation
+"""
 
 # This script should work with both Python 2 and Python 3.
 from __future__ import print_function
@@ -45,7 +41,7 @@ import re
 import optparse
 import unittest
 from os import unlink
-from os.path import dirname, basename, realpath, exists
+from os.path import dirname, realpath, exists
 from pprint import pprint, pformat
 
 ctypes = {
@@ -88,7 +84,7 @@ def lwrap(s, w=72):
 	Wrap a c-prototype like string into a number of lines
 	"""
 	l = []
-	p=""
+	p = ""
 	while len(s) > w:
 		y = s[:w].rfind(',')
 		if y == -1:
@@ -101,7 +97,7 @@ def lwrap(s, w=72):
 	if len(s) > 0:
 		l.append(p + s)
 	return l
-		
+
 #######################################################################
 
 def is_c_name(s):
@@ -125,7 +121,7 @@ class FormatError(Exception):
 
 #######################################################################
 
-class token(object):
+class Token(object):
 	def __init__(self, ln, ch, tokstr):
 		self.ln = ln
 		self.ch = ch
@@ -136,7 +132,7 @@ class token(object):
 
 #######################################################################
 
-class vmod(object):
+class Vmod(object):
 	def __init__(self, nam, dnam, sec):
 		if not is_c_name(nam):
 			raise ParseError("Module name '%s' is illegal", nam)
@@ -160,9 +156,9 @@ class vmod(object):
 		self.funcs.append(fn)
 		self.doc_order.append(fn)
 
-	def add_obj(self, obj):
-		self.objs.append(obj)
-		self.doc_order.append(obj)
+	def add_obj(self, o):
+		self.objs.append(o)
+		self.doc_order.append(o)
 
 	def c_proto(self, fo):
 		for o in self.objs:
@@ -332,7 +328,7 @@ class vmod(object):
 
 #######################################################################
 
-class func(object):
+class Func(object):
 	def __init__(self, nam, retval, al):
 		#if not is_c_name(nam):
 		#	raise Exception("Func name '%s' is illegal" % nam)
@@ -399,7 +395,7 @@ class func(object):
 	def c_initializer(self):
 		return "\tvmod_" + self.cnam + ",\n"
 
-	def c_strspec(self, modnam, pfx = "\t"):
+	def c_strspec(self, modnam, pfx="\t"):
 		s = pfx + '"' + modnam + "." + self.nam + '\\0"\n'
 		s += pfx + '"'
 		s += "Vmod_" + modnam + "_Func." + self.cnam + '\\0"\n'
@@ -448,7 +444,7 @@ class func(object):
 
 #######################################################################
 
-class obj(object):
+class Obj(object):
 	def __init__(self, nam):
 		self.nam = nam
 		self.init = None
@@ -467,7 +463,7 @@ class obj(object):
 
 	def set_init(self, f):
 		self.init = f
-		self.fini = func(f.nam, "VOID", [])
+		self.fini = Func(f.nam, "VOID", [])
 		self.init.cnam += "__init"
 		self.fini.cnam += "__fini"
 
@@ -549,7 +545,7 @@ class obj(object):
 
 #######################################################################
 
-class arg(object):
+class Arg(object):
 	def __init__(self, typ, nam=None, det=None):
 		self.nam = nam
 		self.typ = typ
@@ -589,7 +585,7 @@ def parse_enum2(tl):
 			raise ParseError(
 			    "Expected \"}\" or \",\" not \"%s\"" % t.str)
 	s += "\\0"
-	return arg("ENUM", det=s)
+	return Arg("ENUM", det=s)
 
 #######################################################################
 #
@@ -602,13 +598,13 @@ def parse_module(tl):
 	while len(tl.tl) > 0:
 		s += " " + tl.get_token().str
 	dnm = s[1:]
-	return vmod(nm, dnm, sec)
+	return Vmod(nm, dnm, sec)
 
 #######################################################################
 #
 #
 
-def parse_func(tl, rt_type=None, obj=None):
+def parse_func(tl, rt_type=None, pobj=None):
 	al = list()
 	if rt_type == None:
 		t = tl.get_token()
@@ -619,8 +615,8 @@ def parse_func(tl, rt_type=None, obj=None):
 
 	t = tl.get_token()
 	fname = t.str
-	if obj != None and fname[0] == "." and is_c_name(fname[1:]):
-		fname = obj + fname
+	if pobj != None and fname[0] == "." and is_c_name(fname[1:]):
+		fname = pobj + fname
 	elif not is_c_name(fname):
 		raise ParseError("Function name '%s' is illegal", fname)
 
@@ -637,7 +633,7 @@ def parse_func(tl, rt_type=None, obj=None):
 		if t.str == "ENUM":
 			al.append(parse_enum2(tl))
 		elif t.str in ctypes:
-			al.append(arg(t.str))
+			al.append(Arg(t.str))
 		elif t.str == ")":
 			break
 		else:
@@ -655,7 +651,7 @@ def parse_func(tl, rt_type=None, obj=None):
 			    "Expected \")\" or \",\" not \"%s\"" % t.str)
 	if t.str != ")":
 		raise ParseError("End Of Input looking for ')'")
-	f = func(fname, rt_type, al)
+	f = Func(fname, rt_type, al)
 
 	return f
 
@@ -665,7 +661,7 @@ def parse_func(tl, rt_type=None, obj=None):
 
 def parse_obj(tl):
 	f = parse_func(tl, "VOID")
-	o = obj(f.nam)
+	o = Obj(f.nam)
 	o.set_init(f)
 	return o
 
@@ -673,7 +669,7 @@ def parse_obj(tl):
 #######################################################################
 # A section of the inputvcc, starting at a keyword
 
-class file_section(object):
+class FileSection(object):
 	def __init__(self):
 		self.l = []
 		self.tl = []
@@ -701,7 +697,7 @@ class file_section(object):
 		if l == "":
 			return
 		for j in l.split():
-			self.tl.append(token(ln, 0, j))
+			self.tl.append(Token(ln, 0, j))
 
 	def parse(self, vx):
 		t = self.get_token()
@@ -729,7 +725,7 @@ class file_section(object):
 		elif t.str == "$Method":
 			if len(vx) != 2:
 				raise FormatError("$Method outside $Object", "")
-			o = parse_func(self, obj=vx[1].nam)
+			o = parse_func(self, pobj=vx[1].nam)
 			vx[1].add_method(o)
 		else:
 			raise FormatError("Unknown keyword: %s" % t.str, "")
@@ -792,20 +788,20 @@ def runmain(inputvcc, outputname="vcc_if"):
 	#######################################################################
 	# First collect the copyright:  All initial lines starting with '#'
 
-	copyright = []
+	copy_right = []
 	while len(lines[0]) > 0 and lines[0][0] == "#":
 		ln += 1
-		copyright.append(lines.pop(0))
+		copy_right.append(lines.pop(0))
 
-	if len(copyright) > 0:
-		if copyright[0] == "#-":
-			copyright = []
+	if len(copy_right) > 0:
+		if copy_right[0] == "#-":
+			copy_right = []
 		else:
-			while polish(copyright):
+			while polish(copy_right):
 				continue
 
 	if False:
-		for i in copyright:
+		for i in copy_right:
 			print("(C)\t", i)
 
 	#######################################################################
@@ -820,14 +816,14 @@ def runmain(inputvcc, outputname="vcc_if"):
 	}
 
 	sl = []
-	sc = file_section()
+	sc = FileSection()
 	sl.append(sc)
 	while len(lines) > 0:
 		ln += 1
 		l = lines.pop(0)
 		j = l.split()
 		if len(j) > 0 and j[0] in keywords:
-			sc = file_section()
+			sc = FileSection()
 			sl.append(sc)
 		sc.add_line(ln, l)
 
@@ -844,7 +840,8 @@ def runmain(inputvcc, outputname="vcc_if"):
 		pprint(str(e))
 		exit(-1)
 	except FormatError as e:
-		print("ERROR: Format error reading \"%s\": %s" % (inputvcc, pformat(e.msg)))
+		print("ERROR: Format error reading \"%s\": %s" %
+		    (inputvcc, pformat(e.msg)))
 		print(e.details)
 		exit(-2)
 
@@ -865,14 +862,11 @@ def runmain(inputvcc, outputname="vcc_if"):
 
 	vx[0].c_proto(fh)
 
-	fc.write("""#include "config.h"
-
-#include "vrt.h"
-#include "vcc_if.h"
-#include "vmod_abi.h"
-
-
-""")
+	fc.write('#include "config.h"\n')
+	fc.write('#include "vrt.h"\n')
+	fc.write('#include "vcc_if.h"\n')
+	fc.write('#include "vmod_abi.h"\n')
+	fc.write('\n')
 
 	vx[0].c_typedefs(fc)
 	vx[0].c_vmod(fc)
@@ -886,12 +880,12 @@ def runmain(inputvcc, outputname="vcc_if"):
 
 		vx[0].doc_dump(fp, suf)
 
-		if len(copyright) > 0:
+		if len(copy_right) > 0:
 			fp.write("\n")
 			fp.write("COPYRIGHT\n")
 			fp.write("=========\n")
 			fp.write("\n::\n\n")
-			for i in copyright:
+			for i in copy_right:
 				fp.write("  %s\n" % i)
 			fp.write("\n")
 
@@ -901,25 +895,27 @@ if __name__ == "__main__":
 	oparser = optparse.OptionParser(usage=usagetext)
 
 	oparser.add_option('-N', '--strict', action='store_true', default=False,
-			    help="Be strict when parsing input file. (vmod.vcc)")
+	    help="Be strict when parsing input file. (vmod.vcc)")
 	oparser.add_option('', '--runtests', action='store_true', default=False,
-			    dest="runtests", help=optparse.SUPPRESS_HELP)
+	    dest="runtests", help=optparse.SUPPRESS_HELP)
 	(opts, args) = oparser.parse_args()
 
 	if opts.runtests:
-		del sys.argv[1]  # Pop off --runtests, pass remaining to unittest.
+		# Pop off --runtests, pass remaining to unittest.
+		del sys.argv[1]
 		unittest.main()
 		exit()
 
-	inputvcc = None
+	i_vcc = None
 	if len(args) == 1 and exists(args[0]):
-		inputvcc = args[0]
+		i_vcc = args[0]
 	elif exists("vmod.vcc"):
-		if not inputvcc:
-		    inputvcc = "vmod.vcc"
+		if not i_vcc:
+			i_vcc = "vmod.vcc"
 	else:
-		print("ERROR: No vmod.vcc file supplied or found.", file=sys.stderr)
+		print("ERROR: No vmod.vcc file supplied or found.",
+		    file=sys.stderr)
 		oparser.print_help()
 		exit(-1)
 
-	runmain(inputvcc)
+	runmain(i_vcc)



From phk at FreeBSD.org  Tue Jun 24 09:31:53 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 24 Jun 2014 11:31:53 +0200
Subject: [4.0] 5f67a2e Remove some unused forward struct declarations
Message-ID: <E1WzN4z-0003Ly-Jo@project.varnish-software.com>


commit 5f67a2e41562ea78ef8c0fd270defcc3dca050ca
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Jun 2 08:25:56 2014 +0000

    Remove some unused forward struct declarations

diff --git a/include/vrt.h b/include/vrt.h
index bfbbb2d..5884b00 100644
--- a/include/vrt.h
+++ b/include/vrt.h
@@ -33,15 +33,12 @@
 
 struct req;
 struct busyobj;
-struct worker;
 struct vsl_log;
 struct http;
 struct ws;
-struct vsb;
 struct cli;
 struct director;
 struct VCL_conf;
-struct sockaddr_storage;
 struct suckaddr;
 
 /***********************************************************************



From phk at FreeBSD.org  Tue Jun 24 09:31:53 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 24 Jun 2014 11:31:53 +0200
Subject: [4.0] 51a5d55 Beautify the compiled VCL a bit
Message-ID: <E1WzN4z-0003M7-NI@project.varnish-software.com>


commit 51a5d55314401d142ccd5c1cae0a9262d4dabefa
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Jun 2 08:56:55 2014 +0000

    Beautify the compiled VCL a bit

diff --git a/lib/libvcc/generate.py b/lib/libvcc/generate.py
index c3b51ff..75649db 100755
--- a/lib/libvcc/generate.py
+++ b/lib/libvcc/generate.py
@@ -726,7 +726,7 @@ def emit_file(fo, fd, bn):
 	x = 0
 	l = 0
 	fo.write("\n\t/* %s */\n\n" % fn)
-	fo.write('\tVSB_cat(sb, "/* ---===### %s ###===--- */\\n");\n' % bn)
+	fo.write('\tVSB_cat(sb, "/* ---===### %s ###===--- */\\n\\n");\n' % bn)
 	for c in fc:
 		if l == 0:
 			fo.write("\tVSB_cat(sb, \"")
@@ -764,9 +764,10 @@ def emit_file(fo, fd, bn):
 			fo.write("\"\n")
 			x = 0
 	if x != 0:
-		fo.write("\"")
+		fo.write("\"\n")
 	if l != 0:
 		fo.write("\t);\n")
+	fo.write('\tVSB_cat(sb, "\\n");\n')
 
 #######################################################################
 
diff --git a/lib/libvcc/vcc_compile.c b/lib/libvcc/vcc_compile.c
index 81a7620..31212aa 100644
--- a/lib/libvcc/vcc_compile.c
+++ b/lib/libvcc/vcc_compile.c
@@ -597,7 +597,7 @@ vcc_CompileSource(const struct vcc *tl0, struct vsb *sb, struct source *sp)
 	sym->wildcard = vcc_Stv_Wildcard;
 
 	vcl_output_lang_h(tl->fh);
-	Fh(tl, 0, "\n/* ---===### VCC generated code ###===---*/\n");
+	Fh(tl, 0, "/* ---===### VCC generated code ###===---*/\n");
 	Fh(tl, 0, "\nextern const struct VCL_conf VCL_conf;\n");
 
 	/* Macro for accessing directors */
diff --git a/lib/libvcc/vcc_vmod.c b/lib/libvcc/vcc_vmod.c
index 044cd24..1723007 100644
--- a/lib/libvcc/vcc_vmod.c
+++ b/lib/libvcc/vcc_vmod.c
@@ -104,8 +104,6 @@ vcc_ParseImport(struct vcc *tl)
 		bprintf(fn, "%s/libvmod_%.*s.so", tl->vmod_dir, PF(mod));
 	}
 
-	Fh(tl, 0, "static void *VGC_vmod_%.*s;\n", PF(mod));
-
 	ifp = New_IniFin(tl);
 
 	VSB_printf(ifp->ini, "\tif (VRT_Vmod_Init(&VGC_vmod_%.*s,\n", PF(mod));
@@ -177,7 +175,6 @@ vcc_ParseImport(struct vcc *tl)
 		vcc_ErrWhere(tl, mod);
 		return;
 	}
-	Fh(tl, 0, "static struct vmod_priv vmod_priv_%.*s;\n", PF(mod));
 	for (; *spec != NULL; spec++) {
 		p = *spec;
 		if (!strcmp(p, "OBJ")) {
@@ -207,5 +204,10 @@ vcc_ParseImport(struct vcc *tl)
 				sym->kind = SYM_PROC;
 		}
 	}
+
+	Fh(tl, 0, "\n/* --- BEGIN VMOD %.*s --- */\n\n", PF(mod));
+	Fh(tl, 0, "static void *VGC_vmod_%.*s;\n", PF(mod));
+	Fh(tl, 0, "static struct vmod_priv vmod_priv_%.*s;\n", PF(mod));
 	Fh(tl, 0, "\n%s\n", proto);
+	Fh(tl, 0, "\n/* --- END VMOD %.*s --- */\n\n", PF(mod));
 }



From martin at varnish-software.com  Tue Jun 24 09:31:53 2014
From: martin at varnish-software.com (Martin Blix Grydeland)
Date: Tue, 24 Jun 2014 11:31:53 +0200
Subject: [4.0] aadf302 Document the storage.<name>.* VCL variables.
Message-ID: <E1WzN4z-0003MJ-QH@project.varnish-software.com>


commit aadf302e4b7d856070dbf92ab1e7a6571003812d
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Mon Jun 2 16:01:03 2014 +0200

    Document the storage.<name>.* VCL variables.
    
    Fixes: #1514

diff --git a/lib/libvcc/generate.py b/lib/libvcc/generate.py
index 75649db..a1fd2d7 100755
--- a/lib/libvcc/generate.py
+++ b/lib/libvcc/generate.py
@@ -596,9 +596,18 @@ aliases = [
 ]
 
 stv_variables = (
-	('free_space',	'BYTES',	"0."),
-	('used_space',	'BYTES',	"0."),
-	('happy',	'BOOL',		"0"),
+	('free_space',	'BYTES',	"0.", 'storage.<name>.free_space', """
+        Free space available in the named stevedore. Only available for
+        the malloc stevedore.
+        """),
+	('used_space',	'BYTES',	"0.", 'storage.<name>.used_space', """
+        Used space in the named stevedore. Only available for the malloc
+        stevedore.
+        """),
+	('happy',	'BOOL',		"0", 'storage.<name>.happy', """
+        Health status for the named stevedore. Not available in any of the
+        current stevedores.
+        """),
 )
 
 #######################################################################
@@ -1175,4 +1184,15 @@ for i in l:
 	for j in i[4].split("\n"):
 		fp_vclvar.write("\t%s\n" % j.strip())
 
+hdr="storage"
+fp_vclvar.write("\n" + hdr + "\n");
+fp_vclvar.write("~" * len(hdr) + "\n");
+for i in stv_variables:
+        fp_vclvar.write("\n" + i[3] + "\n\n")
+        fp_vclvar.write("\tType: " + i[1] + "\n\n")
+        fp_vclvar.write("\tReadable from: client, backend\n\n")
+        for j in i[4].split("\n"):
+                fp_vclvar.write("\t%s\n" % j.strip())
+
+
 fp_vclvar.close()



From phk at FreeBSD.org  Tue Jun 24 09:31:53 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 24 Jun 2014 11:31:53 +0200
Subject: [4.0] b25cfdc Simplify http_Teardown() a bit.
Message-ID: <E1WzN4z-0003MS-TE@project.varnish-software.com>


commit b25cfdc974aec31cd82d9ff3238fad8b25458c08
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Jun 3 09:28:01 2014 +0000

    Simplify http_Teardown() a bit.

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index da77ecb..a82ea69 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -186,16 +186,19 @@ struct http {
 	unsigned		magic;
 #define HTTP_MAGIC		0x6428b5c9
 
-	enum VSL_tag_e		logtag;		/* Must be SLT_*Method */
-	struct vsl_log		*vsl;
-
-	struct ws		*ws;
+	uint16_t		shd;		/* Size of hd space */
 	txt			*hd;
 	unsigned char		*hdf;
 #define HDF_FILTER		(1 << 0)	/* Filtered by Connection */
 #define HDF_MARKER		(1 << 1)	/* Marker bit */
-	uint16_t		shd;		/* Size of hd space */
+
+	/* NB: ->nhd and below zeroed/initialized by http_Teardown */
 	uint16_t		nhd;		/* Next free hd */
+
+	enum VSL_tag_e		logtag;		/* Must be SLT_*Method */
+	struct vsl_log		*vsl;
+
+	struct ws		*ws;
 	uint16_t		status;
 	uint8_t			protover;
 	uint8_t			conds;		/* If-* headers present */
diff --git a/bin/varnishd/cache/cache_http.c b/bin/varnishd/cache/cache_http.c
index 7fe6c9f..88ff577 100644
--- a/bin/varnishd/cache/cache_http.c
+++ b/bin/varnishd/cache/cache_http.c
@@ -32,6 +32,7 @@
 #include "config.h"
 
 #include <stdio.h>
+#include <stddef.h>
 
 #include "cache.h"
 
@@ -155,22 +156,13 @@ HTTP_Setup(struct http *hp, struct ws *ws, struct vsl_log *vsl,
 void
 http_Teardown(struct http *hp)
 {
-	uint16_t shd;
-	txt *hd;
-	unsigned char *hdf;
-
-	/* XXX: This is not elegant, is it efficient ? */
-	shd = hp->shd;
-	hd = hp->hd;
-	hdf = hp->hdf;
-	memset(hp, 0, sizeof *hp);
-	memset(hd, 0, sizeof *hd * shd);
-	memset(hdf, 0, sizeof *hdf * shd);
-	hp->magic = HTTP_MAGIC;
+
+	CHECK_OBJ_NOTNULL(hp, HTTP_MAGIC);
+	AN(hp->shd);
+	memset(&hp->nhd, 0, sizeof *hp - offsetof(struct http, nhd));
+	memset(hp->hd, 0, sizeof *hp->hd * hp->shd);
+	memset(hp->hdf, 0, sizeof *hp->hdf * hp->shd);
 	hp->nhd = HTTP_HDR_FIRST;
-	hp->shd = shd;
-	hp->hd = hd;
-	hp->hdf = hdf;
 }
 
 /*--------------------------------------------------------------------*/



From phk at FreeBSD.org  Tue Jun 24 09:31:54 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 24 Jun 2014 11:31:54 +0200
Subject: [4.0] 47d66bb Introduce http->failed to mark struct http's suffering
 from trouble.
Message-ID: <E1WzN50-0003Me-1C@project.varnish-software.com>


commit 47d66bb472669e4def4c180cd349e0552d3b4451
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Jun 3 11:55:18 2014 +0000

    Introduce http->failed to mark struct http's suffering from trouble.
    
    Usually, (probably always) this means failure to get workspace for
    modifications of the http.
    
    The intent is that this flag will have the same "latching" behaviour
    as error handling in VSB's:  Once set, it stays set and nobody
    reads or writes the struct http any more.
    
    Setting the flag causes a SLT_Error message.
    
    Rewrite http_CollectHdr() to respect failed and optimize it
    slightly while were here.

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index a82ea69..979eedd 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -202,6 +202,7 @@ struct http {
 	uint16_t		status;
 	uint8_t			protover;
 	uint8_t			conds;		/* If-* headers present */
+	uint8_t			failed;		/* usually: ws-alloc failed */
 };
 
 /*--------------------------------------------------------------------
diff --git a/bin/varnishd/cache/cache_http.c b/bin/varnishd/cache/cache_http.c
index 88ff577..49522cd 100644
--- a/bin/varnishd/cache/cache_http.c
+++ b/bin/varnishd/cache/cache_http.c
@@ -93,6 +93,15 @@ http_VSL_log(const struct http *hp)
 }
 
 /*--------------------------------------------------------------------*/
+
+static void
+http_fail(struct http *hp)
+{
+	VSLb(hp->vsl, SLT_Error, "out of workspace");
+	hp->failed = 1;
+}
+
+/*--------------------------------------------------------------------*/
 /* List of canonical HTTP response code names from RFC2616 */
 
 static struct http_msg {
@@ -146,12 +155,17 @@ HTTP_Setup(struct http *hp, struct ws *ws, struct vsl_log *vsl,
     enum VSL_tag_e  whence)
 {
 	http_Teardown(hp);
+	hp->nhd = HTTP_HDR_FIRST;
 	hp->logtag = whence;
 	hp->ws = ws;
 	hp->vsl = vsl;
 }
 
-/*--------------------------------------------------------------------*/
+/*--------------------------------------------------------------------
+ * http_Teardown() is a safety feature, we use it to zap all http
+ * structs once we're done with them, to minimize the risk that
+ * old stale pointers exist to no longer valid stuff.
+ */
 
 void
 http_Teardown(struct http *hp)
@@ -162,7 +176,6 @@ http_Teardown(struct http *hp)
 	memset(&hp->nhd, 0, sizeof *hp - offsetof(struct http, nhd));
 	memset(hp->hd, 0, sizeof *hp->hd * hp->shd);
 	memset(hp->hdf, 0, sizeof *hp->hdf * hp->shd);
-	hp->nhd = HTTP_HDR_FIRST;
 }
 
 /*--------------------------------------------------------------------*/
@@ -181,6 +194,26 @@ http_IsHdr(const txt *hh, const char *hdr)
 	return (!strncasecmp(hdr, hh->b, l));
 }
 
+/*--------------------------------------------------------------------*/
+
+static unsigned
+http_findhdr(const struct http *hp, unsigned l, const char *hdr)
+{
+	unsigned u;
+
+	for (u = HTTP_HDR_FIRST; u < hp->nhd; u++) {
+		Tcheck(hp->hd[u]);
+		if (hp->hd[u].e < hp->hd[u].b + l + 1)
+			continue;
+		if (hp->hd[u].b[l] != ':')
+			continue;
+		if (strncasecmp(hdr, hp->hd[u].b, l))
+			continue;
+		return (u);
+	}
+	return (0);
+}
+
 /*--------------------------------------------------------------------
  * This function collapses multiple headerlines of the same name.
  * The lines are joined with a comma, according to [rfc2616, 4.2bot, p32]
@@ -189,57 +222,60 @@ http_IsHdr(const txt *hh, const char *hdr)
 void
 http_CollectHdr(struct http *hp, const char *hdr)
 {
-	unsigned u, v, ml, f = 0, x;
+	unsigned u, l, ml, f, x, d;
 	char *b = NULL, *e = NULL;
 
-	for (u = HTTP_HDR_FIRST; u < hp->nhd; u++) {
-		while (u < hp->nhd && http_IsHdr(&hp->hd[u], hdr)) {
-			Tcheck(hp->hd[u]);
-			if (f == 0) {
-				/* Found first header, just record the fact */
-				f = u;
-				break;
-			}
-			if (b == NULL) {
-				/* Found second header, start our collection */
-				ml = WS_Reserve(hp->ws, 0);
-				b = hp->ws->f;
-				e = b + ml;
-				x = Tlen(hp->hd[f]);
-				if (b + x < e) {
-					memcpy(b, hp->hd[f].b, x);
-					b += x;
-				} else
-					b = e;
-			}
+	if (hp->failed)
+		return;
+	l = hdr[0];
+	assert(l == strlen(hdr + 1));
+	assert(hdr[l] == ':');
+	f = http_findhdr(hp, l - 1, hdr + 1);
+	if (f == 0)
+		return;
 
-			AN(b);
-			AN(e);
-
-			/* Append the Nth header we found */
-			if (b < e)
-				*b++ = ',';
-			x = Tlen(hp->hd[u]) - *hdr;
-			if (b + x < e) {
-				memcpy(b, hp->hd[u].b + *hdr, x);
-				b += x;
-			} else
-				b = e;
-
-			/* Shift remaining headers up one slot */
-			for (v = u; v < hp->nhd - 1; v++)
-				hp->hd[v] = hp->hd[v + 1];
-			hp->nhd--;
+	for (d = u = f + 1; u < hp->nhd; u++) {
+		Tcheck(hp->hd[u]);
+		if (!http_IsHdr(&hp->hd[u], hdr)) {
+			if (d != u)
+				hp->hd[d] = hp->hd[u];
+			d++;
+			continue;
+		}
+		if (b == NULL) {
+			/* Found second header, start our collection */
+			ml = WS_Reserve(hp->ws, 0);
+			b = hp->ws->f;
+			e = b + ml;
+			x = Tlen(hp->hd[f]);
+			if (b + x >= e) {
+				http_fail(hp);
+				WS_Release(hp->ws, 0);
+				return;
+			}
+			memcpy(b, hp->hd[f].b, x);
+			b += x;
 		}
 
+		AN(b);
+		AN(e);
+
+		/* Append the Nth header we found */
+		if (b < e)
+			*b++ = ',';
+		x = Tlen(hp->hd[u]) - l;
+		if (b + x >= e) {
+			http_fail(hp);
+			WS_Release(hp->ws, 0);
+			return;
+		}
+		memcpy(b, hp->hd[u].b + *hdr, x);
+		b += x;
 	}
 	if (b == NULL)
 		return;
+	hp->nhd = (uint16_t)d;
 	AN(e);
-	if (b >= e) {
-		WS_Release(hp->ws, 0);
-		return;
-	}
 	*b = '\0';
 	hp->hd[f].b = hp->ws->f;
 	hp->hd[f].e = b;
@@ -248,24 +284,6 @@ http_CollectHdr(struct http *hp, const char *hdr)
 
 /*--------------------------------------------------------------------*/
 
-static unsigned
-http_findhdr(const struct http *hp, unsigned l, const char *hdr)
-{
-	unsigned u;
-
-	for (u = HTTP_HDR_FIRST; u < hp->nhd; u++) {
-		Tcheck(hp->hd[u]);
-		if (hp->hd[u].e < hp->hd[u].b + l + 1)
-			continue;
-		if (hp->hd[u].b[l] != ':')
-			continue;
-		if (strncasecmp(hdr, hp->hd[u].b, l))
-			continue;
-		return (u);
-	}
-	return (0);
-}
-
 int
 http_GetHdr(const struct http *hp, const char *hdr, char **ptr)
 {



From phk at FreeBSD.org  Tue Jun 24 09:31:54 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 24 Jun 2014 11:31:54 +0200
Subject: [4.0] 48bac01 More http->failed infusion and associated cleanups.
Message-ID: <E1WzN50-0003Mr-4t@project.varnish-software.com>


commit 48bac014d3f51b72505afdde661a8e24a307564e
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Jun 3 12:42:06 2014 +0000

    More http->failed infusion and associated cleanups.

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index 979eedd..1c2732b 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -987,15 +987,14 @@ struct http *HTTP_create(void *p, uint16_t nhttp);
 const char *http_StatusMessage(unsigned);
 unsigned http_EstimateWS(const struct http *fm, unsigned how, uint16_t *nhd);
 void HTTP_Init(void);
-void http_ClrHeader(struct http *to);
 void http_SetResp(struct http *to, const char *proto, uint16_t status,
     const char *response);
 void http_FilterReq(struct http *to, const struct http *fm, unsigned how);
 void http_FilterResp(const struct http *fm, struct http *to, unsigned how);
-void http_PutProtocol(const struct http *to, const char *protocol);
+void http_PutProtocol(struct http *to, const char *protocol);
 void http_PutStatus(struct http *to, uint16_t status);
 void http_ForceHeader(struct http *to, const char *hdr, const char *val);
-void http_PutResponse(const struct http *to, const char *response);
+void http_PutResponse(struct http *to, const char *response);
 void http_PrintfHeader(struct http *to, const char *fmt, ...)
     __printflike(2, 3);
 void http_SetHeader(struct http *to, const char *hdr);
@@ -1014,7 +1013,7 @@ const char *http_GetReq(const struct http *hp);
 int http_HdrIs(const struct http *hp, const char *hdr, const char *val);
 int http_IsHdr(const txt *hh, const char *hdr);
 enum sess_close http_DoConnection(const struct http *);
-void http_CopyHome(const struct http *hp);
+void http_CopyHome(struct http *hp);
 void http_Unset(struct http *hp, const char *hdr);
 void http_CollectHdr(struct http *hp, const char *hdr);
 void http_VSL_log(const struct http *hp);
diff --git a/bin/varnishd/cache/cache_http.c b/bin/varnishd/cache/cache_http.c
index 49522cd..032ac79 100644
--- a/bin/varnishd/cache/cache_http.c
+++ b/bin/varnishd/cache/cache_http.c
@@ -97,6 +97,8 @@ http_VSL_log(const struct http *hp)
 static void
 http_fail(struct http *hp)
 {
+
+	VSC_C_main->losthdr++;
 	VSLb(hp->vsl, SLT_Error, "out of workspace");
 	hp->failed = 1;
 }
@@ -180,6 +182,19 @@ http_Teardown(struct http *hp)
 
 /*--------------------------------------------------------------------*/
 
+void
+HTTP_Copy(struct http *to, const struct http * const fm)
+{
+
+	assert(fm->nhd <= to->shd);
+	memcpy(&to->nhd, &fm->nhd, sizeof *to - offsetof(struct http, nhd));
+	memcpy(to->hd, fm->hd, fm->nhd * sizeof *to->hd);
+	memcpy(to->hdf, fm->hdf, fm->nhd * sizeof *to->hdf);
+}
+
+
+/*--------------------------------------------------------------------*/
+
 int
 http_IsHdr(const txt *hh, const char *hdr)
 {
@@ -237,8 +252,10 @@ http_CollectHdr(struct http *hp, const char *hdr)
 	for (d = u = f + 1; u < hp->nhd; u++) {
 		Tcheck(hp->hd[u]);
 		if (!http_IsHdr(&hp->hd[u], hdr)) {
-			if (d != u)
+			if (d != u) {
 				hp->hd[d] = hp->hd[u];
+				hp->hdf[d] = hp->hdf[u];
+			}
 			d++;
 			continue;
 		}
@@ -250,6 +267,7 @@ http_CollectHdr(struct http *hp, const char *hdr)
 			x = Tlen(hp->hd[f]);
 			if (b + x >= e) {
 				http_fail(hp);
+				VSLb(hp->vsl, SLT_LostHeader, "%s", hdr + 1);
 				WS_Release(hp->ws, 0);
 				return;
 			}
@@ -266,6 +284,7 @@ http_CollectHdr(struct http *hp, const char *hdr)
 		x = Tlen(hp->hd[u]) - l;
 		if (b + x >= e) {
 			http_fail(hp);
+			VSLb(hp->vsl, SLT_LostHeader, "%s", hdr + 1);
 			WS_Release(hp->ws, 0);
 			return;
 		}
@@ -520,17 +539,6 @@ http_SetH(const struct http *to, unsigned n, const char *fm)
 	http_VSLH(to, n);
 }
 
-static void
-http_linkh(const struct http *to, const struct http *fm, unsigned n)
-{
-
-	assert(n < HTTP_HDR_FIRST);
-	Tcheck(fm->hd[n]);
-	to->hd[n] = fm->hd[n];
-	to->hdf[n] = fm->hdf[n];
-	http_VSLH(to, n);
-}
-
 void
 http_ForceGet(const struct http *to)
 {
@@ -596,26 +604,35 @@ http_filterfields(struct http *to, const struct http *fm, unsigned how)
 			continue;
 		if (fm->hdf[u] & HDF_FILTER)
 			continue;
+		Tcheck(fm->hd[u]);
 #define HTTPH(a, b, c) \
 		if (((c) & how) && http_IsHdr(&fm->hd[u], (b))) \
 			continue;
 #include "tbl/http_headers.h"
 #undef HTTPH
-		Tcheck(fm->hd[u]);
-		if (to->nhd < to->shd) {
-			to->hd[to->nhd] = fm->hd[u];
-			to->hdf[to->nhd] = 0;
-			http_VSLH(to, to->nhd);
-			to->nhd++;
-		} else  {
-			VSC_C_main->losthdr++;
-			VSLbt(to->vsl, SLT_LostHeader, fm->hd[u]);
-		}
+		assert (to->nhd < to->shd);
+		to->hd[to->nhd] = fm->hd[u];
+		to->hdf[to->nhd] = 0;
+		http_VSLH(to, to->nhd);
+		to->nhd++;
 	}
 }
 
 /*--------------------------------------------------------------------*/
 
+static void
+http_linkh(const struct http *to, const struct http *fm, unsigned n)
+{
+
+	assert(n < HTTP_HDR_FIRST);
+	Tcheck(fm->hd[n]);
+	to->hd[n] = fm->hd[n];
+	to->hdf[n] = fm->hdf[n];
+	http_VSLH(to, n);
+}
+
+/*--------------------------------------------------------------------*/
+
 void
 http_FilterReq(struct http *to, const struct http *fm, unsigned how)
 {
@@ -689,7 +706,7 @@ http_Merge(const struct http *fm, struct http *to, int not_ce)
  */
 
 void
-http_CopyHome(const struct http *hp)
+http_CopyHome(struct http *hp)
 {
 	unsigned u, l;
 	char *p;
@@ -697,48 +714,31 @@ http_CopyHome(const struct http *hp)
 	for (u = 0; u < hp->nhd; u++) {
 		if (hp->hd[u].b == NULL)
 			continue;
-		if (hp->hd[u].b >= hp->ws->s && hp->hd[u].e <= hp->ws->e) {
+		if (hp->hd[u].b >= hp->ws->s && hp->hd[u].e <= hp->ws->e)
 			continue;
-		}
+
 		l = Tlen(hp->hd[u]);
 		p = WS_Copy(hp->ws, hp->hd[u].b, l + 1L);
-		if (p != NULL) {
-			hp->hd[u].b = p;
-			hp->hd[u].e = p + l;
-		} else {
-			/* XXX This leaves a slot empty */
-			VSC_C_main->losthdr++;
-			VSLbt(hp->vsl, SLT_LostHeader, hp->hd[u]);
-			hp->hd[u].b = NULL;
-			hp->hd[u].e = NULL;
+		if (p == NULL) {
+			http_fail(hp);
+			VSLb(hp->vsl, SLT_LostHeader, "%s", hp->hd[u].b);
+			return;
 		}
+		hp->hd[u].b = p;
+		hp->hd[u].e = p + l;
 	}
 }
 
 /*--------------------------------------------------------------------*/
 
 void
-http_ClrHeader(struct http *to)
-{
-
-	CHECK_OBJ_NOTNULL(to, HTTP_MAGIC);
-	to->nhd = HTTP_HDR_FIRST;
-	to->status = 0;
-	to->protover = 0;
-	to->conds = 0;
-	memset(to->hd, 0, sizeof *to->hd * to->shd);
-}
-
-/*--------------------------------------------------------------------*/
-
-void
 http_SetHeader(struct http *to, const char *hdr)
 {
 
 	CHECK_OBJ_NOTNULL(to, HTTP_MAGIC);
 	if (to->nhd >= to->shd) {
-		VSC_C_main->losthdr++;
 		VSLb(to->vsl, SLT_LostHeader, "%s", hdr);
+		http_fail(to);
 		return;
 	}
 	http_SetH(to, to->nhd++, hdr);
@@ -760,32 +760,29 @@ http_ForceHeader(struct http *to, const char *hdr, const char *val)
 /*--------------------------------------------------------------------*/
 
 static void
-http_PutField(const struct http *to, int field, const char *string)
+http_PutField(struct http *to, int field, const char *string)
 {
 	char *p;
 
 	CHECK_OBJ_NOTNULL(to, HTTP_MAGIC);
 	p = WS_Copy(to->ws, string, -1);
 	if (p == NULL) {
+		http_fail(to);
 		VSLb(to->vsl, SLT_LostHeader, "%s", string);
-		to->hd[field].b = NULL;
-		to->hd[field].e = NULL;
-		to->hdf[field] = 0;
-	} else {
-		to->hd[field].b = p;
-		to->hd[field].e = strchr(p, '\0');
-		to->hdf[field] = 0;
-		http_VSLH(to, field);
+		return;
 	}
+	to->hd[field].b = p;
+	to->hd[field].e = strchr(p, '\0');
+	to->hdf[field] = 0;
+	http_VSLH(to, field);
 }
 
 void
-http_PutProtocol(const struct http *to, const char *protocol)
+http_PutProtocol(struct http *to, const char *protocol)
 {
 
+	AN(protocol);
 	http_PutField(to, HTTP_HDR_PROTO, protocol);
-	if (to->hd[HTTP_HDR_PROTO].b == NULL)
-		http_SetH(to, HTTP_HDR_PROTO, "HTTP/1.1");
 	Tcheck(to->hd[HTTP_HDR_PROTO]);
 }
 
@@ -801,12 +798,11 @@ http_PutStatus(struct http *to, uint16_t status)
 }
 
 void
-http_PutResponse(const struct http *to, const char *response)
+http_PutResponse(struct http *to, const char *response)
 {
 
+	AN(response);
 	http_PutField(to, HTTP_HDR_RESPONSE, response);
-	if (to->hd[HTTP_HDR_RESPONSE].b == NULL)
-		http_SetH(to, HTTP_HDR_RESPONSE, "Lost Response");
 	Tcheck(to->hd[HTTP_HDR_RESPONSE]);
 }
 
@@ -822,17 +818,17 @@ http_PrintfHeader(struct http *to, const char *fmt, ...)
 	n = vsnprintf(to->ws->f, l, fmt, ap);
 	va_end(ap);
 	if (n + 1 >= l || to->nhd >= to->shd) {
-		VSC_C_main->losthdr++;
+		http_fail(to);
 		VSLb(to->vsl, SLT_LostHeader, "%s", to->ws->f);
 		WS_Release(to->ws, 0);
-	} else {
-		to->hd[to->nhd].b = to->ws->f;
-		to->hd[to->nhd].e = to->ws->f + n;
-		to->hdf[to->nhd] = 0;
-		WS_Release(to->ws, n + 1);
-		http_VSLH(to, to->nhd);
-		to->nhd++;
-	}
+		return;
+	} 
+	to->hd[to->nhd].b = to->ws->f;
+	to->hd[to->nhd].e = to->ws->f + n;
+	to->hdf[to->nhd] = 0;
+	WS_Release(to->ws, n + 1);
+	http_VSLH(to, to->nhd);
+	to->nhd++;
 }
 
 /*--------------------------------------------------------------------*/
@@ -861,22 +857,6 @@ http_Unset(struct http *hp, const char *hdr)
 /*--------------------------------------------------------------------*/
 
 void
-HTTP_Copy(struct http *to, const struct http * const fm)
-{
-
-	to->conds = fm->conds;
-	to->logtag = fm->logtag;
-	to->status = fm->status;
-	to->protover = fm->protover;
-	to->nhd = fm->nhd;
-	assert(fm->nhd <= to->shd);
-	memcpy(to->hd, fm->hd, fm->nhd * sizeof *to->hd);
-	memcpy(to->hdf, fm->hdf, fm->nhd * sizeof *to->hdf);
-}
-
-/*--------------------------------------------------------------------*/
-
-void
 HTTP_Init(void)
 {
 
diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c
index 86fed3e..0eed42b 100644
--- a/bin/varnishd/cache/cache_req_fsm.c
+++ b/bin/varnishd/cache/cache_req_fsm.c
@@ -102,8 +102,6 @@ cnt_deliver(struct worker *wrk, struct req *req)
 		EXP_Touch(req->obj->objcore, req->t_prev);
 
 	HTTP_Setup(req->resp, req->ws, req->vsl, SLT_RespMethod);
-
-	http_ClrHeader(req->resp);
 	http_FilterResp(req->obj->http, req->resp, 0);
 
 	if (req->wrk->stats.cache_hit)
@@ -202,8 +200,6 @@ cnt_synth(struct worker *wrk, struct req *req)
 
 	wrk->stats.s_synth++;
 
-	HTTP_Setup(req->resp, req->ws, req->vsl, SLT_RespMethod);
-	h = req->resp;
 
 	now = W_TIM_real(wrk);
 	VSLb_ts_req(req, "Process", now);
@@ -211,7 +207,8 @@ cnt_synth(struct worker *wrk, struct req *req)
 	if (req->err_code < 100 || req->err_code > 999)
 		req->err_code = 501;
 
-	http_ClrHeader(h);
+	HTTP_Setup(req->resp, req->ws, req->vsl, SLT_RespMethod);
+	h = req->resp;
 	http_PutProtocol(h, "HTTP/1.1");
 	http_PutStatus(h, req->err_code);
 	VTIM_format(now, date);
@@ -235,7 +232,7 @@ cnt_synth(struct worker *wrk, struct req *req)
 	AZ(VSB_finish(req->synth_body));
 
 	if (wrk->handling == VCL_RET_RESTART) {
-		http_ClrHeader(h);
+		HTTP_Setup(h, req->ws, req->vsl, SLT_RespMethod);
 		VSB_delete(req->synth_body);
 		req->synth_body = NULL;
 		req->req_step = R_STP_RESTART;



From phk at FreeBSD.org  Tue Jun 24 09:31:54 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 24 Jun 2014 11:31:54 +0200
Subject: [4.0] 22b8488 Cut the thicket for setting a HTTP response's first
 line down to just one function which does what you'd expect.
Message-ID: <E1WzN50-0003N4-7m@project.varnish-software.com>


commit 22b8488592d5b76af6ac0636e070861be3e892d9
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Jun 4 07:42:25 2014 +0000

    Cut the thicket for setting a HTTP response's first line down to just
    one function which does what you'd expect.
    
    Start allowing internal VCL use of *resp.status > 999.  The idea
    is that we ignore the higher bits but it still has to end up >=
    100, so 10404 becomes 404 and 20099 is illegal.   (This change
    is not yet complete it will panic if you use it yet.)
    
    More http->failed stuff.

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index 1c2732b..2429ccd 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -987,14 +987,11 @@ struct http *HTTP_create(void *p, uint16_t nhttp);
 const char *http_StatusMessage(unsigned);
 unsigned http_EstimateWS(const struct http *fm, unsigned how, uint16_t *nhd);
 void HTTP_Init(void);
-void http_SetResp(struct http *to, const char *proto, uint16_t status,
+void http_PutResponse(struct http *to, const char *proto, uint16_t status,
     const char *response);
 void http_FilterReq(struct http *to, const struct http *fm, unsigned how);
 void http_FilterResp(const struct http *fm, struct http *to, unsigned how);
-void http_PutProtocol(struct http *to, const char *protocol);
-void http_PutStatus(struct http *to, uint16_t status);
 void http_ForceHeader(struct http *to, const char *hdr, const char *val);
-void http_PutResponse(struct http *to, const char *response);
 void http_PrintfHeader(struct http *to, const char *fmt, ...)
     __printflike(2, 3);
 void http_SetHeader(struct http *to, const char *hdr);
diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c
index 7ad709d..450560c 100644
--- a/bin/varnishd/cache/cache_fetch.c
+++ b/bin/varnishd/cache/cache_fetch.c
@@ -686,7 +686,7 @@ vbf_stp_error(struct worker *wrk, struct busyobj *bo)
 	// XXX: reset all beresp flags ?
 
 	HTTP_Setup(bo->beresp, bo->ws, bo->vsl, SLT_BerespMethod);
-	http_SetResp(bo->beresp, "HTTP/1.1", 503, "Backend fetch failed");
+	http_PutResponse(bo->beresp, "HTTP/1.1", 503, "Backend fetch failed");
 	VTIM_format(now, time_str);
 	http_PrintfHeader(bo->beresp, "Date: %s", time_str);
 	http_SetHeader(bo->beresp, "Server: Varnish");
diff --git a/bin/varnishd/cache/cache_http.c b/bin/varnishd/cache/cache_http.c
index 032ac79..d4296cd 100644
--- a/bin/varnishd/cache/cache_http.c
+++ b/bin/varnishd/cache/cache_http.c
@@ -120,11 +120,12 @@ http_StatusMessage(unsigned status)
 {
 	struct http_msg *mp;
 
-	assert(status >= 100 && status <= 999);
+	status %= 1000;
+	assert(status >= 100);
 	for (mp = http_msg; mp->nbr != 0 && mp->nbr <= status; mp++)
 		if (mp->nbr == status)
 			return (mp->txt);
-	return ("Unknown Error");
+	return ("Unknown HTTP Status");
 }
 
 /*--------------------------------------------------------------------*/
@@ -527,6 +528,25 @@ http_GetReq(const struct http *hp)
 
 /*--------------------------------------------------------------------*/
 
+static void
+http_PutField(struct http *to, int field, const char *string)
+{
+	char *p;
+
+	CHECK_OBJ_NOTNULL(to, HTTP_MAGIC);
+	p = WS_Copy(to->ws, string, -1);
+	if (p == NULL) {
+		http_fail(to);
+		VSLb(to->vsl, SLT_LostHeader, "%s", string);
+		return;
+	}
+	to->hd[field].b = p;
+	to->hd[field].e = strchr(p, '\0');
+	to->hdf[field] = 0;
+	http_VSLH(to, field);
+}
+/*--------------------------------------------------------------------*/
+
 void
 http_SetH(const struct http *to, unsigned n, const char *fm)
 {
@@ -547,14 +567,24 @@ http_ForceGet(const struct http *to)
 }
 
 void
-http_SetResp(struct http *to, const char *proto, uint16_t status,
+http_PutResponse(struct http *to, const char *proto, uint16_t status,
     const char *response)
 {
+	char buf[4];
 
 	CHECK_OBJ_NOTNULL(to, HTTP_MAGIC);
 	http_SetH(to, HTTP_HDR_PROTO, proto);
-	assert(status >= 100 && status <= 999);
-	http_PutStatus(to, status);
+	/*
+	 * We allow people to use top digits for internal VCL
+	 * signalling, strip them here.
+	 */
+	status %= 1000;
+	assert(status >= 100);
+	to->status = status;
+	bprintf(buf, "%03d", status % 1000);
+	http_PutField(to, HTTP_HDR_STATUS, buf);
+	if (response == NULL)
+		response = http_StatusMessage(status);
 	http_SetH(to, HTTP_HDR_RESPONSE, response);
 }
 
@@ -757,55 +787,6 @@ http_ForceHeader(struct http *to, const char *hdr, const char *val)
 	http_PrintfHeader(to, "%s %s", hdr + 1, val);
 }
 
-/*--------------------------------------------------------------------*/
-
-static void
-http_PutField(struct http *to, int field, const char *string)
-{
-	char *p;
-
-	CHECK_OBJ_NOTNULL(to, HTTP_MAGIC);
-	p = WS_Copy(to->ws, string, -1);
-	if (p == NULL) {
-		http_fail(to);
-		VSLb(to->vsl, SLT_LostHeader, "%s", string);
-		return;
-	}
-	to->hd[field].b = p;
-	to->hd[field].e = strchr(p, '\0');
-	to->hdf[field] = 0;
-	http_VSLH(to, field);
-}
-
-void
-http_PutProtocol(struct http *to, const char *protocol)
-{
-
-	AN(protocol);
-	http_PutField(to, HTTP_HDR_PROTO, protocol);
-	Tcheck(to->hd[HTTP_HDR_PROTO]);
-}
-
-void
-http_PutStatus(struct http *to, uint16_t status)
-{
-	char buf[4];
-
-	assert(status >= 100 && status <= 999);
-	to->status = status;
-	bprintf(buf, "%03d", status % 1000);
-	http_PutField(to, HTTP_HDR_STATUS, buf);
-}
-
-void
-http_PutResponse(struct http *to, const char *response)
-{
-
-	AN(response);
-	http_PutField(to, HTTP_HDR_RESPONSE, response);
-	Tcheck(to->hd[HTTP_HDR_RESPONSE]);
-}
-
 void
 http_PrintfHeader(struct http *to, const char *fmt, ...)
 {
@@ -822,7 +803,7 @@ http_PrintfHeader(struct http *to, const char *fmt, ...)
 		VSLb(to->vsl, SLT_LostHeader, "%s", to->ws->f);
 		WS_Release(to->ws, 0);
 		return;
-	} 
+	}
 	to->hd[to->nhd].b = to->ws->f;
 	to->hd[to->nhd].e = to->ws->f + n;
 	to->hdf[to->nhd] = 0;
diff --git a/bin/varnishd/cache/cache_http1_deliver.c b/bin/varnishd/cache/cache_http1_deliver.c
index 0b0e138..06b4a32 100644
--- a/bin/varnishd/cache/cache_http1_deliver.c
+++ b/bin/varnishd/cache/cache_http1_deliver.c
@@ -154,7 +154,7 @@ v1d_dorange(struct req *req, const char *r)
 	if (req->res_mode & RES_LEN)
 		http_PrintfHeader(req->resp, "Content-Length: %jd",
 		    (intmax_t)(1 + high - low));
-	http_SetResp(req->resp, "HTTP/1.1", 206, "Partial Content");
+	http_PutResponse(req->resp, "HTTP/1.1", 206, NULL);
 
 	req->range_off = 0;
 	req->range_low = low;
diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c
index 0eed42b..d14f398 100644
--- a/bin/varnishd/cache/cache_req_fsm.c
+++ b/bin/varnishd/cache/cache_req_fsm.c
@@ -149,7 +149,7 @@ cnt_deliver(struct worker *wrk, struct req *req)
 	    && req->esi_level == 0
 	    && http_GetStatus(req->obj->http) == 200
 	    && req->http->conds && RFC2616_Do_Cond(req))
-		http_SetResp(req->resp, "HTTP/1.1", 304, "Not Modified");
+		http_PutResponse(req->resp, "HTTP/1.1", 304, NULL);
 
 	V1D_Deliver(req);
 	VSLb_ts_req(req, "Resp", W_TIM_real(wrk));
@@ -209,17 +209,12 @@ cnt_synth(struct worker *wrk, struct req *req)
 
 	HTTP_Setup(req->resp, req->ws, req->vsl, SLT_RespMethod);
 	h = req->resp;
-	http_PutProtocol(h, "HTTP/1.1");
-	http_PutStatus(h, req->err_code);
 	VTIM_format(now, date);
 	http_PrintfHeader(h, "Date: %s", date);
 	http_SetHeader(h, "Server: Varnish");
 	http_PrintfHeader(req->resp,
 	    "X-Varnish: %u", req->vsl->wid & VSL_IDENTMASK);
-	if (req->err_reason != NULL)
-		http_PutResponse(h, req->err_reason);
-	else
-		http_PutResponse(h, http_StatusMessage(req->err_code));
+	http_PutResponse(h, "HTTP/1.1", req->err_code, req->err_reason);
 
 	AZ(req->synth_body);
 	req->synth_body = VSB_new_auto();
diff --git a/bin/varnishd/cache/cache_vrt_var.c b/bin/varnishd/cache/cache_vrt_var.c
index 65c72db..d2e6647 100644
--- a/bin/varnishd/cache/cache_vrt_var.c
+++ b/bin/varnishd/cache/cache_vrt_var.c
@@ -48,19 +48,20 @@ static char vrt_hostname[255] = "";
 /*--------------------------------------------------------------------*/
 
 static void
-vrt_do_string(const struct http *hp, int fld,
+vrt_do_string(struct http *hp, int fld,
     const char *err, const char *p, va_list ap)
 {
 	const char *b;
 
-	AN(hp);
+	CHECK_OBJ_NOTNULL(hp, HTTP_MAGIC);
+
 	b = VRT_String(hp->ws, NULL, p, ap);
 	if (b == NULL || *b == '\0') {
 		VSLb(hp->vsl, SLT_LostHeader, "%s", err);
+		hp->failed = 1;
 	} else {
 		http_SetH(hp, fld, b);
 	}
-	va_end(ap);
 }
 
 #define VRT_HDR_L(obj, hdr, fld)					\
@@ -95,8 +96,15 @@ VRT_l_##obj##_status(const struct vrt_ctx *ctx, long num)		\
 									\
 	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);				\
 	CHECK_OBJ_NOTNULL(ctx->http_##obj, HTTP_MAGIC);			\
-	assert(num >= 100 && num <= 999);				\
-	ctx->http_##obj->status = (uint16_t)num;			\
+	if (num > 65535) {						\
+		VSLb(ctx->vsl, SLT_VCL_Error, "%s.status > 65535", #obj); \
+		ctx->http_##obj->failed = 1;				\
+	} else if ((num % 1000) < 100) {				\
+		VSLb(ctx->vsl, SLT_VCL_Error, "illegal %s.status (..0##)", \
+		    #obj);						\
+		ctx->http_##obj->failed = 1;				\
+	} else								\
+		ctx->http_##obj->status = (uint16_t)num;		\
 }
 
 #define VRT_STATUS_R(obj)						\



From phk at FreeBSD.org  Tue Jun 24 09:31:54 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 24 Jun 2014 11:31:54 +0200
Subject: [4.0] d5affe5 Having wasted half an hour on it (again) it's time to
 kill the "response/reason" confusion for good.
Message-ID: <E1WzN50-0003NM-B3@project.varnish-software.com>


commit d5affe570d38cee48dcbdca911aed28399a4771d
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Jun 4 08:05:48 2014 +0000

    Having wasted half an hour on it (again) it's time to kill the
    "response/reason" confusion for good.

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index 2429ccd..681c857 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -984,7 +984,7 @@ void VGZ_WrwFlush(struct req *, struct vgz *vg);
 unsigned HTTP_estimate(unsigned nhttp);
 void HTTP_Copy(struct http *to, const struct http * const fm);
 struct http *HTTP_create(void *p, uint16_t nhttp);
-const char *http_StatusMessage(unsigned);
+const char *http_Status2Reason(unsigned);
 unsigned http_EstimateWS(const struct http *fm, unsigned how, uint16_t *nhd);
 void HTTP_Init(void);
 void http_PutResponse(struct http *to, const char *proto, uint16_t status,
diff --git a/bin/varnishd/cache/cache_http.c b/bin/varnishd/cache/cache_http.c
index d4296cd..1b17cac 100644
--- a/bin/varnishd/cache/cache_http.c
+++ b/bin/varnishd/cache/cache_http.c
@@ -116,7 +116,7 @@ static struct http_msg {
 };
 
 const char *
-http_StatusMessage(unsigned status)
+http_Status2Reason(unsigned status)
 {
 	struct http_msg *mp;
 
@@ -568,7 +568,7 @@ http_ForceGet(const struct http *to)
 
 void
 http_PutResponse(struct http *to, const char *proto, uint16_t status,
-    const char *response)
+    const char *reason)
 {
 	char buf[4];
 
@@ -583,9 +583,9 @@ http_PutResponse(struct http *to, const char *proto, uint16_t status,
 	to->status = status;
 	bprintf(buf, "%03d", status % 1000);
 	http_PutField(to, HTTP_HDR_STATUS, buf);
-	if (response == NULL)
-		response = http_StatusMessage(status);
-	http_SetH(to, HTTP_HDR_RESPONSE, response);
+	if (reason == NULL)
+		reason = http_Status2Reason(status);
+	http_SetH(to, HTTP_HDR_REASON, reason);
 }
 
 /*--------------------------------------------------------------------
@@ -689,7 +689,7 @@ http_FilterResp(const struct http *fm, struct http *to, unsigned how)
 	http_SetH(to, HTTP_HDR_PROTO, "HTTP/1.1");
 	to->status = fm->status;
 	http_linkh(to, fm, HTTP_HDR_STATUS);
-	http_linkh(to, fm, HTTP_HDR_RESPONSE);
+	http_linkh(to, fm, HTTP_HDR_REASON);
 	http_filterfields(to, fm, how);
 }
 
@@ -708,7 +708,7 @@ http_Merge(const struct http *fm, struct http *to, int not_ce)
 	to->status = fm->status;
 	http_SetH(to, HTTP_HDR_PROTO, fm->hd[HTTP_HDR_PROTO].b);
 	http_SetH(to, HTTP_HDR_STATUS, fm->hd[HTTP_HDR_STATUS].b);
-	http_SetH(to, HTTP_HDR_RESPONSE, fm->hd[HTTP_HDR_RESPONSE].b);
+	http_SetH(to, HTTP_HDR_REASON, fm->hd[HTTP_HDR_REASON].b);
 
 	for (u = HTTP_HDR_FIRST; u < fm->nhd; u++)
 		fm->hdf[u] |= HDF_MARKER;
diff --git a/bin/varnishd/cache/cache_http1_proto.c b/bin/varnishd/cache/cache_http1_proto.c
index a896546..4daa814 100644
--- a/bin/varnishd/cache/cache_http1_proto.c
+++ b/bin/varnishd/cache/cache_http1_proto.c
@@ -310,7 +310,7 @@ htc_splitline(struct http *hp, const struct http_conn *htc, int req)
 	} else {
 		h1 = HTTP_HDR_PROTO;
 		h2 = HTTP_HDR_STATUS;
-		h3 = HTTP_HDR_RESPONSE;
+		h3 = HTTP_HDR_REASON;
 	}
 
 	/* Skip leading LWS */
@@ -493,13 +493,13 @@ HTTP1_DissectResponse(struct http *hp, const struct http_conn *htc)
 	} else
 		htc_proto_ver(hp);
 
-	if (hp->hd[HTTP_HDR_RESPONSE].b == NULL ||
-	    !Tlen(hp->hd[HTTP_HDR_RESPONSE])) {
+	if (hp->hd[HTTP_HDR_REASON].b == NULL ||
+	    !Tlen(hp->hd[HTTP_HDR_REASON])) {
 		/* Backend didn't send a response string, use the standard */
-		hp->hd[HTTP_HDR_RESPONSE].b =
-		    TRUST_ME(http_StatusMessage(hp->status));
-		hp->hd[HTTP_HDR_RESPONSE].e =
-		    strchr(hp->hd[HTTP_HDR_RESPONSE].b, '\0');
+		hp->hd[HTTP_HDR_REASON].b =
+		    TRUST_ME(http_Status2Reason(hp->status));
+		hp->hd[HTTP_HDR_REASON].e =
+		    strchr(hp->hd[HTTP_HDR_REASON].b, '\0');
 	}
 	return (retval);
 }
@@ -523,7 +523,7 @@ HTTP1_Write(const struct worker *w, const struct http *hp, int resp)
 
 		l += WRW_WriteH(w, &hp->hd[HTTP_HDR_STATUS], " ");
 
-		l += WRW_WriteH(w, &hp->hd[HTTP_HDR_RESPONSE], "\r\n");
+		l += WRW_WriteH(w, &hp->hd[HTTP_HDR_REASON], "\r\n");
 	} else {
 		AN(hp->hd[HTTP_HDR_URL].b);
 		l = WRW_WriteH(w, &hp->hd[HTTP_HDR_METHOD], " ");
diff --git a/bin/varnishd/cache/cache_vrt.c b/bin/varnishd/cache/cache_vrt.c
index 08e3b77..4038693 100644
--- a/bin/varnishd/cache/cache_vrt.c
+++ b/bin/varnishd/cache/cache_vrt.c
@@ -63,7 +63,7 @@ VRT_error(const struct vrt_ctx *ctx, unsigned code, const char *reason)
 		code = 503;
 	ctx->req->err_code = (uint16_t)code;
 	ctx->req->err_reason =
-	    reason ? reason : http_StatusMessage(ctx->req->err_code);
+	    reason ? reason : http_Status2Reason(ctx->req->err_code);
 }
 
 /*--------------------------------------------------------------------*/
diff --git a/bin/varnishd/cache/cache_vrt_var.c b/bin/varnishd/cache/cache_vrt_var.c
index d2e6647..293b029 100644
--- a/bin/varnishd/cache/cache_vrt_var.c
+++ b/bin/varnishd/cache/cache_vrt_var.c
@@ -122,11 +122,11 @@ VRT_HDR_LR(req,    url,		HTTP_HDR_URL)
 VRT_HDR_LR(req,    proto,	HTTP_HDR_PROTO)
 
 VRT_HDR_R(obj,    proto,	HTTP_HDR_PROTO)
-VRT_HDR_R(obj,    reason,	HTTP_HDR_RESPONSE)
+VRT_HDR_R(obj,    reason,	HTTP_HDR_REASON)
 VRT_STATUS_R(obj)
 
 VRT_HDR_LR(resp,   proto,	HTTP_HDR_PROTO)
-VRT_HDR_LR(resp,   reason,	HTTP_HDR_RESPONSE)
+VRT_HDR_LR(resp,   reason,	HTTP_HDR_REASON)
 VRT_STATUS_L(resp)
 VRT_STATUS_R(resp)
 
@@ -134,7 +134,7 @@ VRT_HDR_LR(bereq,  method,	HTTP_HDR_METHOD)
 VRT_HDR_LR(bereq,  url,		HTTP_HDR_URL)
 VRT_HDR_LR(bereq,  proto,	HTTP_HDR_PROTO)
 VRT_HDR_LR(beresp, proto,	HTTP_HDR_PROTO)
-VRT_HDR_LR(beresp, reason,	HTTP_HDR_RESPONSE)
+VRT_HDR_LR(beresp, reason,	HTTP_HDR_REASON)
 VRT_STATUS_L(beresp)
 VRT_STATUS_R(beresp)
 
diff --git a/include/tbl/vsl_tags_http.h b/include/tbl/vsl_tags_http.h
index 3a3ae50..ab09d70 100644
--- a/include/tbl/vsl_tags_http.h
+++ b/include/tbl/vsl_tags_http.h
@@ -55,7 +55,7 @@ SLTH(Protocol,	HTTP_HDR_PROTO,		1, 1, "protocol",
 SLTH(Status,	HTTP_HDR_STATUS,	0, 1, "status",
 	"The HTTP status code received.\n\n"
 )
-SLTH(Response,	HTTP_HDR_RESPONSE,	0, 1, "response",
+SLTH(Reason,	HTTP_HDR_REASON,	0, 1, "response",
 	"The HTTP response string received.\n\n"
 )
 SLTH(Header,	HTTP_HDR_FIRST,		1, 1, "header",



From phk at FreeBSD.org  Tue Jun 24 09:31:54 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 24 Jun 2014 11:31:54 +0200
Subject: [4.0] 9f1222e More http_ cleanup:
Message-ID: <E1WzN50-0003NY-DR@project.varnish-software.com>


commit 9f1222ed48c3105ca395d88dc2d75ef77223da81
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Jun 4 08:51:10 2014 +0000

    More http_ cleanup:
    
    Move policy decisions out where they belong and let the http_Filter
    function send the first like through unmolested.
    
    Rather than value specific functions, make a generic
    http_ForceField() function, because the name is cool.

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index 681c857..22e0362 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -996,7 +996,7 @@ void http_PrintfHeader(struct http *to, const char *fmt, ...)
     __printflike(2, 3);
 void http_SetHeader(struct http *to, const char *hdr);
 void http_SetH(const struct http *to, unsigned n, const char *fm);
-void http_ForceGet(const struct http *to);
+void http_ForceField(const struct http *to, unsigned n, const char *t);
 void HTTP_Setup(struct http *, struct ws *, struct vsl_log *, enum VSL_tag_e);
 void http_Teardown(struct http *ht);
 int http_GetHdr(const struct http *hp, const char *hdr, char **ptr);
diff --git a/bin/varnishd/cache/cache_esi_deliver.c b/bin/varnishd/cache/cache_esi_deliver.c
index a246985..ee1b0e5 100644
--- a/bin/varnishd/cache/cache_esi_deliver.c
+++ b/bin/varnishd/cache/cache_esi_deliver.c
@@ -79,8 +79,12 @@ ved_include(struct req *preq, const char *src, const char *host)
 		http_SetHeader(req->http0, host);
 	}
 
-	http_ForceGet(req->http0);
+	http_ForceField(req->http0, HTTP_HDR_METHOD, "GET");
+	http_ForceField(req->http0, HTTP_HDR_PROTO, "HTTP/1.1");
+
+	/* Don't allow Conditions, we can't use a 304 */
 	http_Unset(req->http0, H_If_Modified_Since);
+	http_Unset(req->http0, H_If_None_Match);
 
 	/* Client content already taken care of */
 	http_Unset(req->http0, H_Content_Length);
diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c
index 450560c..c175e74 100644
--- a/bin/varnishd/cache/cache_fetch.c
+++ b/bin/varnishd/cache/cache_fetch.c
@@ -190,7 +190,8 @@ vbf_stp_mkbereq(const struct worker *wrk, struct busyobj *bo)
 	    bo->do_pass ? HTTPH_R_PASS : HTTPH_R_FETCH);
 
 	if (!bo->do_pass) {
-		http_ForceGet(bo->bereq0);
+		http_ForceField(bo->bereq0, HTTP_HDR_METHOD, "GET");
+		http_ForceField(bo->bereq0, HTTP_HDR_PROTO, "HTTP/1.1");
 		if (cache_param->http_gzip_support)
 			http_ForceHeader(bo->bereq0, H_Accept_Encoding, "gzip");
 		AN(bo->req);
diff --git a/bin/varnishd/cache/cache_http.c b/bin/varnishd/cache/cache_http.c
index 1b17cac..1d1f872 100644
--- a/bin/varnishd/cache/cache_http.c
+++ b/bin/varnishd/cache/cache_http.c
@@ -545,6 +545,7 @@ http_PutField(struct http *to, int field, const char *string)
 	to->hdf[field] = 0;
 	http_VSLH(to, field);
 }
+
 /*--------------------------------------------------------------------*/
 
 void
@@ -559,13 +560,22 @@ http_SetH(const struct http *to, unsigned n, const char *fm)
 	http_VSLH(to, n);
 }
 
+/*--------------------------------------------------------------------
+ * Force a particular header field to a particular value
+ */
+
 void
-http_ForceGet(const struct http *to)
+http_ForceField(const struct http *to, unsigned n, const char *t)
 {
-	if (strcmp(http_GetReq(to), "GET"))
-		http_SetH(to, HTTP_HDR_METHOD, "GET");
+	CHECK_OBJ_NOTNULL(to, HTTP_MAGIC);
+	assert(n < HTTP_HDR_FIRST);
+	AN(t);
+	if (to->hd[n].b == NULL || strcmp(to->hd[n].b, t))
+		http_SetH(to, n, t);
 }
 
+/*--------------------------------------------------------------------*/
+
 void
 http_PutResponse(struct http *to, const char *proto, uint16_t status,
     const char *reason)
@@ -671,10 +681,7 @@ http_FilterReq(struct http *to, const struct http *fm, unsigned how)
 
 	http_linkh(to, fm, HTTP_HDR_METHOD);
 	http_linkh(to, fm, HTTP_HDR_URL);
-	if (how == HTTPH_R_FETCH)
-		http_SetH(to, HTTP_HDR_PROTO, "HTTP/1.1");
-	else
-		http_linkh(to, fm, HTTP_HDR_PROTO);
+	http_linkh(to, fm, HTTP_HDR_PROTO);
 	http_filterfields(to, fm, how);
 }
 
@@ -686,8 +693,8 @@ http_FilterResp(const struct http *fm, struct http *to, unsigned how)
 
 	CHECK_OBJ_NOTNULL(fm, HTTP_MAGIC);
 	CHECK_OBJ_NOTNULL(to, HTTP_MAGIC);
-	http_SetH(to, HTTP_HDR_PROTO, "HTTP/1.1");
 	to->status = fm->status;
+	http_linkh(to, fm, HTTP_HDR_PROTO);
 	http_linkh(to, fm, HTTP_HDR_STATUS);
 	http_linkh(to, fm, HTTP_HDR_REASON);
 	http_filterfields(to, fm, how);
@@ -706,9 +713,9 @@ http_Merge(const struct http *fm, struct http *to, int not_ce)
 	const char *p;
 
 	to->status = fm->status;
-	http_SetH(to, HTTP_HDR_PROTO, fm->hd[HTTP_HDR_PROTO].b);
-	http_SetH(to, HTTP_HDR_STATUS, fm->hd[HTTP_HDR_STATUS].b);
-	http_SetH(to, HTTP_HDR_REASON, fm->hd[HTTP_HDR_REASON].b);
+	http_linkh(to, fm, HTTP_HDR_PROTO);
+	http_linkh(to, fm, HTTP_HDR_STATUS);
+	http_linkh(to, fm, HTTP_HDR_REASON);
 
 	for (u = HTTP_HDR_FIRST; u < fm->nhd; u++)
 		fm->hdf[u] |= HDF_MARKER;
diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c
index d14f398..4c8ca8e 100644
--- a/bin/varnishd/cache/cache_req_fsm.c
+++ b/bin/varnishd/cache/cache_req_fsm.c
@@ -103,6 +103,7 @@ cnt_deliver(struct worker *wrk, struct req *req)
 
 	HTTP_Setup(req->resp, req->ws, req->vsl, SLT_RespMethod);
 	http_FilterResp(req->obj->http, req->resp, 0);
+	http_ForceField(req->resp, HTTP_HDR_PROTO, "HTTP/1.1");
 
 	if (req->wrk->stats.cache_hit)
 		http_PrintfHeader(req->resp,



From phk at FreeBSD.org  Tue Jun 24 09:31:54 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 24 Jun 2014 11:31:54 +0200
Subject: [4.0] 1cf2d3c Reduce default thread pool size to 10 threads,
 no need to stress the pthread implementation in *every* test case.
Message-ID: <E1WzN50-0003Ni-Go@project.varnish-software.com>


commit 1cf2d3c4dbd9923d41dde296252a51b727b74188
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Jun 4 12:20:54 2014 +0000

    Reduce default thread pool size to 10 threads, no need to stress
    the pthread implementation in *every* test case.

diff --git a/bin/varnishtest/vtc_varnish.c b/bin/varnishtest/vtc_varnish.c
index 14b94be..c559810 100644
--- a/bin/varnishtest/vtc_varnish.c
+++ b/bin/varnishtest/vtc_varnish.c
@@ -386,6 +386,7 @@ varnish_launch(struct varnish *v)
 	VSB_printf(vsb, " -p auto_restart=off");
 	VSB_printf(vsb, " -p syslog_cli_traffic=off");
 	VSB_printf(vsb, " -p sigsegv_handler=on");
+	VSB_printf(vsb, " -p thread_pool_min=10");
 	VSB_printf(vsb, " -a '%s'", "127.0.0.1:0");
 	VSB_printf(vsb, " -M '%s %s'", abuf, pbuf);
 	VSB_printf(vsb, " -P %s/varnishd.pid", v->workdir);



From phk at FreeBSD.org  Tue Jun 24 09:31:54 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 24 Jun 2014 11:31:54 +0200
Subject: [4.0] e51f32d More struct http polishing
Message-ID: <E1WzN50-0003Ns-Iv@project.varnish-software.com>


commit e51f32d1db9e8c1ec813ef2c7fdf825979e3435c
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Jun 4 12:21:38 2014 +0000

    More struct http polishing

diff --git a/bin/varnishd/cache/cache_vrt_var.c b/bin/varnishd/cache/cache_vrt_var.c
index 293b029..3ce5817 100644
--- a/bin/varnishd/cache/cache_vrt_var.c
+++ b/bin/varnishd/cache/cache_vrt_var.c
@@ -45,7 +45,9 @@
 
 static char vrt_hostname[255] = "";
 
-/*--------------------------------------------------------------------*/
+/*--------------------------------------------------------------------
+ * VRT variables relating to first line of HTTP/1.1 req/resp
+ */
 
 static void
 vrt_do_string(struct http *hp, int fld,
@@ -59,9 +61,9 @@ vrt_do_string(struct http *hp, int fld,
 	if (b == NULL || *b == '\0') {
 		VSLb(hp->vsl, SLT_LostHeader, "%s", err);
 		hp->failed = 1;
-	} else {
-		http_SetH(hp, fld, b);
+		return;
 	}
+	http_SetH(hp, fld, b);
 }
 
 #define VRT_HDR_L(obj, hdr, fld)					\
@@ -138,7 +140,9 @@ VRT_HDR_LR(beresp, reason,	HTTP_HDR_REASON)
 VRT_STATUS_L(beresp)
 VRT_STATUS_R(beresp)
 
-/*--------------------------------------------------------------------*/
+/*--------------------------------------------------------------------
+ * bool-fields (.do_*)
+ */
 
 #define VBERESPW0(field)
 #define VBERESPW1(field)						\
@@ -235,9 +239,13 @@ VRT_l_client_identity(const struct vrt_ctx *ctx, const char *str, ...)
 	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
 	CHECK_OBJ_NOTNULL(ctx->req, REQ_MAGIC);
 	va_start(ap, str);
-	// XXX ?
 	b = VRT_String(ctx->req->http->ws, NULL, str, ap);
 	va_end(ap);
+	if (b == NULL) {
+		VSLb(ctx->vsl, SLT_LostHeader, "client.identity");
+		ctx->req->http->failed = 1;
+		return;
+	}
 	ctx->req->client_identity = b;
 }
 
@@ -296,6 +304,8 @@ VRT_r_beresp_backend_ip(const struct vrt_ctx *ctx)
 		return (NULL);
 }
 
+/*--------------------------------------------------------------------*/
+
 const char *
 VRT_r_beresp_storage_hint(const struct vrt_ctx *ctx)
 {
@@ -318,6 +328,11 @@ VRT_l_beresp_storage_hint(const struct vrt_ctx *ctx, const char *str, ...)
 	va_start(ap, str);
 	b = VRT_String(ctx->bo->ws, NULL, str, ap);	// XXX: ctx->ws ?
 	va_end(ap);
+	if (b == NULL) {
+		VSLb(ctx->vsl, SLT_LostHeader, "storage.hint");
+		ctx->bo->beresp->failed = 1;
+		return;
+	}
 	ctx->bo->storage_hint = b;
 }
 
@@ -414,7 +429,6 @@ VRT_r_req_can_gzip(const struct vrt_ctx *ctx)
 	return (RFC2616_Req_Gzip(ctx->req->http));	// XXX ?
 }
 
-
 /*--------------------------------------------------------------------*/
 
 long



From lkarsten at varnish-software.com  Tue Jun 24 09:31:54 2014
From: lkarsten at varnish-software.com (Lasse Karstensen)
Date: Tue, 24 Jun 2014 11:31:54 +0200
Subject: [4.0] 80e9e0c Add version requirement for docutils.
Message-ID: <E1WzN50-0003O1-Lr@project.varnish-software.com>


commit 80e9e0cbf9e7aea064ca566a177669ae7b84fe09
Author: Lasse Karstensen <lkarsten at varnish-software.com>
Date:   Thu Jun 5 16:11:18 2014 +0200

    Add version requirement for docutils.
    
    The rst2man utility exists only in "newer" versions of python-docutils.
    
    Require at least 0.7, released July 2010.

diff --git a/redhat/varnish.spec b/redhat/varnish.spec
index 09752b8..82c17a7 100644
--- a/redhat/varnish.spec
+++ b/redhat/varnish.spec
@@ -17,7 +17,8 @@ BuildRoot: %{_tmppath}/%{name}-%{version}-%{release}-root-%(%{__id_u} -n)
 # To build from git, start with a make dist, see redhat/README.redhat 
 # You will need at least automake autoconf libtool
 #BuildRequires: automake autoconf libtool
-BuildRequires: ncurses-devel groff pcre-devel pkgconfig python-docutils libedit-devel jemalloc-devel
+BuildRequires: ncurses-devel groff pcre-devel pkgconfig libedit-devel jemalloc-devel
+BuildRequires: python-docutils <= 0.7
 Requires: varnish-libs = %{version}-%{release}
 Requires: logrotate
 Requires: ncurses



From lkarsten at varnish-software.com  Tue Jun 24 09:31:54 2014
From: lkarsten at varnish-software.com (Lasse Karstensen)
Date: Tue, 24 Jun 2014 11:31:54 +0200
Subject: [4.0] b356206 Correct wrong comparator from previous commit.
Message-ID: <E1WzN50-0003OE-OU@project.varnish-software.com>


commit b356206814a7b5f403ef65e1beb9cc381c3f2ab3
Author: Lasse Karstensen <lkarsten at varnish-software.com>
Date:   Thu Jun 5 16:17:28 2014 +0200

    Correct wrong comparator from previous commit.

diff --git a/redhat/varnish.spec b/redhat/varnish.spec
index 82c17a7..b85217c 100644
--- a/redhat/varnish.spec
+++ b/redhat/varnish.spec
@@ -18,7 +18,7 @@ BuildRoot: %{_tmppath}/%{name}-%{version}-%{release}-root-%(%{__id_u} -n)
 # You will need at least automake autoconf libtool
 #BuildRequires: automake autoconf libtool
 BuildRequires: ncurses-devel groff pcre-devel pkgconfig libedit-devel jemalloc-devel
-BuildRequires: python-docutils <= 0.7
+BuildRequires: python-docutils >= 0.7
 Requires: varnish-libs = %{version}-%{release}
 Requires: logrotate
 Requires: ncurses



From phk at FreeBSD.org  Tue Jun 24 09:31:54 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 24 Jun 2014 11:31:54 +0200
Subject: [4.0] 6960078 Minor polishing in http space,
 to get them out of substantial change to follow shortly.
Message-ID: <E1WzN50-0003OP-RZ@project.varnish-software.com>


commit 69600783b911cb6b15f3fb07f6dbd595697a0933
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Jun 10 07:19:41 2014 +0000

    Minor polishing in http space, to get them out of substantial change
    to follow shortly.

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index 22e0362..732bf9f 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -89,15 +89,10 @@ enum sess_close {
 #undef SESS_CLOSE
 };
 
-/*--------------------------------------------------------------------*/
-
-/*
- * NB: HDR_STATUS is only used in cache_http.c, everybody else uses the
- * http->status integer field.
+/*--------------------------------------------------------------------
+ * Indicies into http->hd[]
  */
-
 enum {
-	/* Fields from the first line of HTTP proto */
 #define SLTH(tag, ind, req, resp, sdesc, ldesc)	ind,
 #include "tbl/vsl_tags_http.h"
 #undef SLTH
diff --git a/bin/varnishd/cache/cache_http.c b/bin/varnishd/cache/cache_http.c
index 1d1f872..8411e1f 100644
--- a/bin/varnishd/cache/cache_http.c
+++ b/bin/varnishd/cache/cache_http.c
@@ -193,6 +193,39 @@ HTTP_Copy(struct http *to, const struct http * const fm)
 	memcpy(to->hdf, fm->hdf, fm->nhd * sizeof *to->hdf);
 }
 
+/*--------------------------------------------------------------------*/
+
+void
+http_SetH(const struct http *to, unsigned n, const char *fm)
+{
+
+	assert(n < to->shd);
+	AN(fm);
+	to->hd[n].b = TRUST_ME(fm);
+	to->hd[n].e = strchr(to->hd[n].b, '\0');
+	to->hdf[n] = 0;
+	http_VSLH(to, n);
+}
+
+/*--------------------------------------------------------------------*/
+
+static void
+http_PutField(struct http *to, int field, const char *string)
+{
+	char *p;
+
+	CHECK_OBJ_NOTNULL(to, HTTP_MAGIC);
+	p = WS_Copy(to->ws, string, -1);
+	if (p == NULL) {
+		http_fail(to);
+		VSLb(to->vsl, SLT_LostHeader, "%s", string);
+		return;
+	}
+	to->hd[field].b = p;
+	to->hd[field].e = strchr(p, '\0');
+	to->hdf[field] = 0;
+	http_VSLH(to, field);
+}
 
 /*--------------------------------------------------------------------*/
 
@@ -515,6 +548,7 @@ uint16_t
 http_GetStatus(const struct http *hp)
 {
 
+	CHECK_OBJ_NOTNULL(hp, HTTP_MAGIC);
 	return (hp->status);
 }
 
@@ -522,44 +556,11 @@ const char *
 http_GetReq(const struct http *hp)
 {
 
+	CHECK_OBJ_NOTNULL(hp, HTTP_MAGIC);
 	Tcheck(hp->hd[HTTP_HDR_METHOD]);
 	return (hp->hd[HTTP_HDR_METHOD].b);
 }
 
-/*--------------------------------------------------------------------*/
-
-static void
-http_PutField(struct http *to, int field, const char *string)
-{
-	char *p;
-
-	CHECK_OBJ_NOTNULL(to, HTTP_MAGIC);
-	p = WS_Copy(to->ws, string, -1);
-	if (p == NULL) {
-		http_fail(to);
-		VSLb(to->vsl, SLT_LostHeader, "%s", string);
-		return;
-	}
-	to->hd[field].b = p;
-	to->hd[field].e = strchr(p, '\0');
-	to->hdf[field] = 0;
-	http_VSLH(to, field);
-}
-
-/*--------------------------------------------------------------------*/
-
-void
-http_SetH(const struct http *to, unsigned n, const char *fm)
-{
-
-	assert(n < to->shd);
-	AN(fm);
-	to->hd[n].b = TRUST_ME(fm);
-	to->hd[n].e = strchr(to->hd[n].b, '\0');
-	to->hdf[n] = 0;
-	http_VSLH(to, n);
-}
-
 /*--------------------------------------------------------------------
  * Force a particular header field to a particular value
  */
diff --git a/bin/varnishd/cache/cache_http1_proto.c b/bin/varnishd/cache/cache_http1_proto.c
index 4daa814..66323a9 100644
--- a/bin/varnishd/cache/cache_http1_proto.c
+++ b/bin/varnishd/cache/cache_http1_proto.c
@@ -312,6 +312,9 @@ htc_splitline(struct http *hp, const struct http_conn *htc, int req)
 		h2 = HTTP_HDR_STATUS;
 		h3 = HTTP_HDR_REASON;
 	}
+	AZ(hp->hd[h1].b);
+	AZ(hp->hd[h2].b);
+	AZ(hp->hd[h3].b);
 
 	/* Skip leading LWS */
 	for (p = htc->rxbuf.b ; vct_islws(*p); p++)
@@ -339,7 +342,6 @@ htc_splitline(struct http *hp, const struct http_conn *htc, int req)
 			return (400);
 	}
 	hp->hd[h2].e = p;
-
 	if (!Tlen(hp->hd[h2]))
 		return (400);
 
@@ -364,9 +366,7 @@ htc_splitline(struct http *hp, const struct http_conn *htc, int req)
 
 	*hp->hd[h1].e = '\0';
 	*hp->hd[h2].e = '\0';
-
-	if (hp->hd[h3].e != NULL)
-		*hp->hd[h3].e = '\0';
+	*hp->hd[h3].e = '\0';
 
 	return (htc_dissect_hdrs(hp, p, htc));
 }



From phk at FreeBSD.org  Tue Jun 24 09:31:54 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 24 Jun 2014 11:31:54 +0200
Subject: [4.0] bb603b3 Introduce a dedicated http_SetStatus() function.
Message-ID: <E1WzN50-0003Ob-UB@project.varnish-software.com>


commit bb603b3d1f9c76906bea167cab1c6cadd4f0df54
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Jun 10 07:29:44 2014 +0000

    Introduce a dedicated http_SetStatus() function.

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index 732bf9f..a79b224 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -1001,6 +1001,7 @@ int http_GetHdrField(const struct http *hp, const char *hdr,
     const char *field, char **ptr);
 double http_GetHdrQ(const struct http *hp, const char *hdr, const char *field);
 uint16_t http_GetStatus(const struct http *hp);
+void http_SetStatus(struct http *to, uint16_t status);
 const char *http_GetReq(const struct http *hp);
 int http_HdrIs(const struct http *hp, const char *hdr, const char *val);
 int http_IsHdr(const txt *hh, const char *hdr);
diff --git a/bin/varnishd/cache/cache_http.c b/bin/varnishd/cache/cache_http.c
index 8411e1f..8313007 100644
--- a/bin/varnishd/cache/cache_http.c
+++ b/bin/varnishd/cache/cache_http.c
@@ -552,6 +552,27 @@ http_GetStatus(const struct http *hp)
 	return (hp->status);
 }
 
+/*--------------------------------------------------------------------*/
+
+void
+http_SetStatus(struct http *to, uint16_t status)
+{
+	char buf[4];
+
+	CHECK_OBJ_NOTNULL(to, HTTP_MAGIC);
+	/*
+	 * We allow people to use top digits for internal VCL
+	 * signalling, but strip them from the ASCII version.
+	 */
+	to->status = status;
+	status %= 1000;
+	assert(status >= 100);
+	bprintf(buf, "%03d", status);
+	http_PutField(to, HTTP_HDR_STATUS, buf);
+}
+
+/*--------------------------------------------------------------------*/
+
 const char *
 http_GetReq(const struct http *hp)
 {
@@ -581,19 +602,11 @@ void
 http_PutResponse(struct http *to, const char *proto, uint16_t status,
     const char *reason)
 {
-	char buf[4];
 
 	CHECK_OBJ_NOTNULL(to, HTTP_MAGIC);
-	http_SetH(to, HTTP_HDR_PROTO, proto);
-	/*
-	 * We allow people to use top digits for internal VCL
-	 * signalling, strip them here.
-	 */
-	status %= 1000;
-	assert(status >= 100);
-	to->status = status;
-	bprintf(buf, "%03d", status % 1000);
-	http_PutField(to, HTTP_HDR_STATUS, buf);
+	if (proto != NULL)
+		http_SetH(to, HTTP_HDR_PROTO, proto);
+	http_SetStatus(to, status);
 	if (reason == NULL)
 		reason = http_Status2Reason(status);
 	http_SetH(to, HTTP_HDR_REASON, reason);
diff --git a/bin/varnishd/cache/cache_vrt_var.c b/bin/varnishd/cache/cache_vrt_var.c
index 3ce5817..b74e101 100644
--- a/bin/varnishd/cache/cache_vrt_var.c
+++ b/bin/varnishd/cache/cache_vrt_var.c
@@ -105,8 +105,9 @@ VRT_l_##obj##_status(const struct vrt_ctx *ctx, long num)		\
 		VSLb(ctx->vsl, SLT_VCL_Error, "illegal %s.status (..0##)", \
 		    #obj);						\
 		ctx->http_##obj->failed = 1;				\
-	} else								\
-		ctx->http_##obj->status = (uint16_t)num;		\
+	} else {							\
+		http_SetStatus(ctx->http_##obj, (uint16_t)num);		\
+	}								\
 }
 
 #define VRT_STATUS_R(obj)						\



From phk at FreeBSD.org  Tue Jun 24 09:31:55 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 24 Jun 2014 11:31:55 +0200
Subject: [4.0] dbd98ce Polish up HTTP1_DissectResponse()
Message-ID: <E1WzN51-0003Ol-2b@project.varnish-software.com>


commit dbd98ce246e834936a044b2d04085c384a444ce2
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Jun 10 07:50:43 2014 +0000

    Polish up HTTP1_DissectResponse()

diff --git a/bin/varnishd/cache/cache_http1_proto.c b/bin/varnishd/cache/cache_http1_proto.c
index 66323a9..7316f0b 100644
--- a/bin/varnishd/cache/cache_http1_proto.c
+++ b/bin/varnishd/cache/cache_http1_proto.c
@@ -468,8 +468,11 @@ HTTP1_DissectResponse(struct http *hp, const struct http_conn *htc)
 	if (htc_splitline(hp, htc, 0))
 		retval = 503;
 
-	if (retval == 0 && memcmp(hp->hd[HTTP_HDR_PROTO].b, "HTTP/1.", 7))
-		retval = 503;
+	if (retval == 0) {
+		htc_proto_ver(hp);
+		if (hp->protover != 10 && hp->protover != 11)
+			retval = 503;
+	}
 
 	if (retval == 0 && Tlen(hp->hd[HTTP_HDR_STATUS]) != 3)
 		retval = 503;
@@ -489,18 +492,15 @@ HTTP1_DissectResponse(struct http *hp, const struct http_conn *htc)
 	if (retval != 0) {
 		VSLbt(hp->vsl, SLT_HttpGarbage, htc->rxbuf);
 		assert(retval >= 100 && retval <= 999);
+		assert(retval == 503);
 		hp->status = retval;
-	} else
-		htc_proto_ver(hp);
+		http_SetH(hp, HTTP_HDR_STATUS, "503");
+	}
 
 	if (hp->hd[HTTP_HDR_REASON].b == NULL ||
-	    !Tlen(hp->hd[HTTP_HDR_REASON])) {
-		/* Backend didn't send a response string, use the standard */
-		hp->hd[HTTP_HDR_REASON].b =
-		    TRUST_ME(http_Status2Reason(hp->status));
-		hp->hd[HTTP_HDR_REASON].e =
-		    strchr(hp->hd[HTTP_HDR_REASON].b, '\0');
-	}
+	    !Tlen(hp->hd[HTTP_HDR_REASON]))
+		http_SetH(hp, HTTP_HDR_REASON, http_Status2Reason(hp->status));
+
 	return (retval);
 }
 



From phk at FreeBSD.org  Tue Jun 24 09:31:55 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 24 Jun 2014 11:31:55 +0200
Subject: [4.0] e99e8fb Always set the reason headerfield when the status field
 is set, to keep them in sync.  If you want a custom reason field,
 set status first, then reason.
Message-ID: <E1WzN51-0003Ox-5J@project.varnish-software.com>


commit e99e8fbaa85aafa36abaa0aa9f508e98fd93dfc8
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Jun 10 08:31:16 2014 +0000

    Always set the reason headerfield when the status field is set, to
    keep them in sync.  If you want a custom reason field, set status
    first, then reason.
    
    With this in place, HTTP1_Write() doesn't need to worry about them
    being OK, and we can polish that code a fair bit.
    
    Turn a couple of boolean arguments into more readable magics.

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index a79b224..38e597b 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -857,6 +857,8 @@ void HTTP1_Session(struct worker *, struct req *);
 int HTTP1_DiscardReqBody(struct req *req);
 int HTTP1_CacheReqBody(struct req *req, ssize_t maxsize);
 int HTTP1_IterateReqBody(struct req *req, req_body_iter_f *func, void *priv);
+extern const int HTTP1_Req[3];
+extern const int HTTP1_Resp[3];
 
 /* cache_http1_deliver.c */
 unsigned V1D_FlushReleaseAcct(struct req *req);
@@ -1030,7 +1032,7 @@ ssize_t HTTP1_Read(struct http_conn *htc, void *d, size_t len);
 enum htc_status_e HTTP1_Complete(struct http_conn *htc);
 uint16_t HTTP1_DissectRequest(struct req *);
 uint16_t HTTP1_DissectResponse(struct http *sp, const struct http_conn *htc);
-unsigned HTTP1_Write(const struct worker *w, const struct http *hp, int resp);
+unsigned HTTP1_Write(const struct worker *w, const struct http *hp, const int*);
 
 #define HTTPH(a, b, c) extern char b[];
 #include "tbl/http_headers.h"
diff --git a/bin/varnishd/cache/cache_http.c b/bin/varnishd/cache/cache_http.c
index 8313007..159751b 100644
--- a/bin/varnishd/cache/cache_http.c
+++ b/bin/varnishd/cache/cache_http.c
@@ -552,7 +552,9 @@ http_GetStatus(const struct http *hp)
 	return (hp->status);
 }
 
-/*--------------------------------------------------------------------*/
+/*--------------------------------------------------------------------
+ * Setting the status will also set the Reason appropriately
+ */
 
 void
 http_SetStatus(struct http *to, uint16_t status)
@@ -569,6 +571,7 @@ http_SetStatus(struct http *to, uint16_t status)
 	assert(status >= 100);
 	bprintf(buf, "%03d", status);
 	http_PutField(to, HTTP_HDR_STATUS, buf);
+	http_SetH(to, HTTP_HDR_REASON, http_Status2Reason(status));
 }
 
 /*--------------------------------------------------------------------*/
diff --git a/bin/varnishd/cache/cache_http1_deliver.c b/bin/varnishd/cache/cache_http1_deliver.c
index 06b4a32..8778303 100644
--- a/bin/varnishd/cache/cache_http1_deliver.c
+++ b/bin/varnishd/cache/cache_http1_deliver.c
@@ -322,7 +322,8 @@ V1D_Deliver(struct req *req)
 	 * Send HTTP protocol header, unless interior ESI object
 	 */
 	if (!(req->res_mode & RES_ESI_CHILD))
-		req->resp_hdrbytes += HTTP1_Write(req->wrk, req->resp, 1);
+		req->resp_hdrbytes +=
+		    HTTP1_Write(req->wrk, req->resp, HTTP1_Resp);
 
 	if (req->res_mode & RES_CHUNKED)
 		WRW_Chunked(req->wrk);
@@ -429,7 +430,8 @@ V1D_Deliver_Synth(struct req *req)
 	 * Send HTTP protocol header, unless interior ESI object
 	 */
 	if (!(req->res_mode & RES_ESI_CHILD))
-		req->resp_hdrbytes += HTTP1_Write(req->wrk, req->resp, 1);
+		req->resp_hdrbytes +=
+		    HTTP1_Write(req->wrk, req->resp, HTTP1_Resp);
 
 	if (req->res_mode & RES_CHUNKED)
 		WRW_Chunked(req->wrk);
diff --git a/bin/varnishd/cache/cache_http1_fetch.c b/bin/varnishd/cache/cache_http1_fetch.c
index 11c712e..1c3ef53 100644
--- a/bin/varnishd/cache/cache_http1_fetch.c
+++ b/bin/varnishd/cache/cache_http1_fetch.c
@@ -327,7 +327,7 @@ V1F_fetch_hdr(struct worker *wrk, struct busyobj *bo, struct req *req)
 
 	(void)VTCP_blocking(vc->fd);	/* XXX: we should timeout instead */
 	WRW_Reserve(wrk, &vc->fd, bo->vsl, bo->t_prev);
-	hdrbytes = HTTP1_Write(wrk, hp, 0);
+	hdrbytes = HTTP1_Write(wrk, hp, HTTP1_Req);
 
 	/* Deal with any message-body the request might (still) have */
 	i = 0;
diff --git a/bin/varnishd/cache/cache_http1_proto.c b/bin/varnishd/cache/cache_http1_proto.c
index 7316f0b..7f7852c 100644
--- a/bin/varnishd/cache/cache_http1_proto.c
+++ b/bin/varnishd/cache/cache_http1_proto.c
@@ -49,6 +49,14 @@
 
 #include "vct.h"
 
+const int HTTP1_Req[3] = {
+	HTTP_HDR_METHOD, HTTP_HDR_URL, HTTP_HDR_PROTO
+};
+
+const int HTTP1_Resp[3] = {
+	HTTP_HDR_PROTO, HTTP_HDR_STATUS, HTTP_HDR_REASON
+};
+
 /*--------------------------------------------------------------------*/
 
 void
@@ -294,55 +302,46 @@ htc_dissect_hdrs(struct http *hp, char *p, const struct http_conn *htc)
  */
 
 static uint16_t
-htc_splitline(struct http *hp, const struct http_conn *htc, int req)
+htc_splitline(struct http *hp, const struct http_conn *htc, const int *hf)
 {
 	char *p;
-	int h1, h2, h3;
 
+	assert(hf == HTTP1_Req || hf == HTTP1_Resp);
 	CHECK_OBJ_NOTNULL(htc, HTTP_CONN_MAGIC);
 	CHECK_OBJ_NOTNULL(hp, HTTP_MAGIC);
 	Tcheck(htc->rxbuf);
 
-	if (req) {
-		h1 = HTTP_HDR_METHOD;
-		h2 = HTTP_HDR_URL;
-		h3 = HTTP_HDR_PROTO;
-	} else {
-		h1 = HTTP_HDR_PROTO;
-		h2 = HTTP_HDR_STATUS;
-		h3 = HTTP_HDR_REASON;
-	}
-	AZ(hp->hd[h1].b);
-	AZ(hp->hd[h2].b);
-	AZ(hp->hd[h3].b);
+	AZ(hp->hd[hf[0]].b);
+	AZ(hp->hd[hf[1]].b);
+	AZ(hp->hd[hf[2]].b);
 
 	/* Skip leading LWS */
 	for (p = htc->rxbuf.b ; vct_islws(*p); p++)
 		continue;
-	hp->hd[h1].b = p;
+	hp->hd[hf[0]].b = p;
 
 	/* First field cannot contain SP or CTL */
 	for (; !vct_issp(*p); p++) {
 		if (vct_isctl(*p))
 			return (400);
 	}
-	hp->hd[h1].e = p;
-	assert(Tlen(hp->hd[h1]));
+	hp->hd[hf[0]].e = p;
+	assert(Tlen(hp->hd[hf[0]]));
 
 	/* Skip SP */
 	for (; vct_issp(*p); p++) {
 		if (vct_isctl(*p))
 			return (400);
 	}
-	hp->hd[h2].b = p;
+	hp->hd[hf[1]].b = p;
 
 	/* Second field cannot contain LWS or CTL */
 	for (; !vct_islws(*p); p++) {
 		if (vct_isctl(*p))
 			return (400);
 	}
-	hp->hd[h2].e = p;
-	if (!Tlen(hp->hd[h2]))
+	hp->hd[hf[1]].e = p;
+	if (!Tlen(hp->hd[hf[1]]))
 		return (400);
 
 	/* Skip SP */
@@ -350,23 +349,23 @@ htc_splitline(struct http *hp, const struct http_conn *htc, int req)
 		if (vct_isctl(*p))
 			return (400);
 	}
-	hp->hd[h3].b = p;
+	hp->hd[hf[2]].b = p;
 
 	/* Third field is optional and cannot contain CTL except TAB */
 	for (; !vct_iscrlf(p); p++) {
 		if (vct_isctl(*p) && !vct_issp(*p)) {
-			hp->hd[h3].b = NULL;
+			hp->hd[hf[2]].b = NULL;
 			return (400);
 		}
 	}
-	hp->hd[h3].e = p;
+	hp->hd[hf[2]].e = p;
 
 	/* Skip CRLF */
 	p += vct_skipcrlf(p);
 
-	*hp->hd[h1].e = '\0';
-	*hp->hd[h2].e = '\0';
-	*hp->hd[h3].e = '\0';
+	*hp->hd[hf[0]].e = '\0';
+	*hp->hd[hf[1]].e = '\0';
+	*hp->hd[hf[2]].e = '\0';
 
 	return (htc_dissect_hdrs(hp, p, htc));
 }
@@ -426,7 +425,7 @@ HTTP1_DissectRequest(struct req *req)
 	hp = req->http;
 	CHECK_OBJ_NOTNULL(hp, HTTP_MAGIC);
 
-	retval = htc_splitline(hp, htc, 1);
+	retval = htc_splitline(hp, htc, HTTP1_Req);
 	if (retval != 0) {
 		VSLbt(req->vsl, SLT_HttpGarbage, htc->rxbuf);
 		return (retval);
@@ -465,7 +464,7 @@ HTTP1_DissectResponse(struct http *hp, const struct http_conn *htc)
 	CHECK_OBJ_NOTNULL(htc, HTTP_CONN_MAGIC);
 	CHECK_OBJ_NOTNULL(hp, HTTP_MAGIC);
 
-	if (htc_splitline(hp, htc, 0))
+	if (htc_splitline(hp, htc, HTTP1_Resp))
 		retval = 503;
 
 	if (retval == 0) {
@@ -495,6 +494,7 @@ HTTP1_DissectResponse(struct http *hp, const struct http_conn *htc)
 		assert(retval == 503);
 		hp->status = retval;
 		http_SetH(hp, HTTP_HDR_STATUS, "503");
+		http_SetH(hp, HTTP_HDR_REASON, http_Status2Reason(retval));
 	}
 
 	if (hp->hd[HTTP_HDR_REASON].b == NULL ||
@@ -507,36 +507,20 @@ HTTP1_DissectResponse(struct http *hp, const struct http_conn *htc)
 /*--------------------------------------------------------------------*/
 
 unsigned
-HTTP1_Write(const struct worker *w, const struct http *hp, int resp)
+HTTP1_Write(const struct worker *w, const struct http *hp, const int *hf)
 {
 	unsigned u, l;
 
-	if (resp) {
-		l = WRW_WriteH(w, &hp->hd[HTTP_HDR_PROTO], " ");
-
-		hp->hd[HTTP_HDR_STATUS].b = WS_Alloc(hp->ws, 4);
-		AN(hp->hd[HTTP_HDR_STATUS].b);
+	assert(hf == HTTP1_Req || hf == HTTP1_Resp);
+	AN(hp->hd[hf[0]].b);
+	AN(hp->hd[hf[1]].b);
+	AN(hp->hd[hf[2]].b);
+	l = WRW_WriteH(w, &hp->hd[hf[0]], " ");
+	l += WRW_WriteH(w, &hp->hd[hf[1]], " ");
+	l += WRW_WriteH(w, &hp->hd[hf[2]], "\r\n");
 
-		assert(hp->status >= 100 && hp->status <= 999);
-		sprintf(hp->hd[HTTP_HDR_STATUS].b, "%3d", hp->status);
-		hp->hd[HTTP_HDR_STATUS].e = hp->hd[HTTP_HDR_STATUS].b + 3;
-
-		l += WRW_WriteH(w, &hp->hd[HTTP_HDR_STATUS], " ");
-
-		l += WRW_WriteH(w, &hp->hd[HTTP_HDR_REASON], "\r\n");
-	} else {
-		AN(hp->hd[HTTP_HDR_URL].b);
-		l = WRW_WriteH(w, &hp->hd[HTTP_HDR_METHOD], " ");
-		l += WRW_WriteH(w, &hp->hd[HTTP_HDR_URL], " ");
-		l += WRW_WriteH(w, &hp->hd[HTTP_HDR_PROTO], "\r\n");
-	}
-	for (u = HTTP_HDR_FIRST; u < hp->nhd; u++) {
-		if (hp->hd[u].b == NULL)
-			continue;
-		AN(hp->hd[u].b);
-		AN(hp->hd[u].e);
+	for (u = HTTP_HDR_FIRST; u < hp->nhd; u++)
 		l += WRW_WriteH(w, &hp->hd[u], "\r\n");
-	}
 	l += WRW_Write(w, "\r\n", -1);
 	return (l);
 }
diff --git a/bin/varnishd/cache/cache_pipe.c b/bin/varnishd/cache/cache_pipe.c
index bb666a1..27ea964 100644
--- a/bin/varnishd/cache/cache_pipe.c
+++ b/bin/varnishd/cache/cache_pipe.c
@@ -124,7 +124,7 @@ PipeRequest(struct req *req, struct busyobj *bo)
 	(void)VTCP_blocking(vc->fd);
 
 	WRW_Reserve(wrk, &vc->fd, bo->vsl, req->t_req);
-	hdrbytes = HTTP1_Write(wrk, bo->bereq, 0);
+	hdrbytes = HTTP1_Write(wrk, bo->bereq, HTTP1_Req);
 
 	if (req->htc->pipeline.b != NULL)
 		(void)WRW_Write(wrk, req->htc->pipeline.b,



From daghf at varnish-software.com  Tue Jun 24 09:31:55 2014
From: daghf at varnish-software.com (Dag Haavi Finstad)
Date: Tue, 24 Jun 2014 11:31:55 +0200
Subject: [4.0] 8a1a59a Docfix an erroneous reference to r-r director having
 weight.
Message-ID: <E1WzN51-0003PC-9z@project.varnish-software.com>


commit 8a1a59ab1547366bae942c087eef974aaba6fb5e
Author: Dag Haavi Finstad <daghf at varnish-software.com>
Date:   Tue Jun 10 12:10:32 2014 +0200

    Docfix an erroneous reference to r-r director having weight.
    
    Fixes: #1519

diff --git a/lib/libvmod_directors/vmod.vcc b/lib/libvmod_directors/vmod.vcc
index f0d9424..cc0bb6c 100644
--- a/lib/libvmod_directors/vmod.vcc
+++ b/lib/libvmod_directors/vmod.vcc
@@ -153,7 +153,7 @@ $Method VOID .add_backend(BACKEND, REAL)
 Description
 	Adds a backend to the director with a certain weight.
 
-	Weight is used as in the round_robin director. Recommended value is
+	Weight is used as in the random director. Recommended value is
 	1.0 unless you have special needs.
 
 Example



From lkarsten at varnish-software.com  Tue Jun 24 09:31:55 2014
From: lkarsten at varnish-software.com (Lasse Karstensen)
Date: Tue, 24 Jun 2014 11:31:55 +0200
Subject: [4.0] c0098d3 Fix minor double wording.
Message-ID: <E1WzN51-0003PI-Dj@project.varnish-software.com>


commit c0098d30058144c67243143fab16aaa8df386d90
Author: Lasse Karstensen <lkarsten at varnish-software.com>
Date:   Tue Jun 10 12:04:28 2014 +0200

    Fix minor double wording.

diff --git a/lib/libvmod_directors/vmod.vcc b/lib/libvmod_directors/vmod.vcc
index cc0bb6c..f7f3330 100644
--- a/lib/libvmod_directors/vmod.vcc
+++ b/lib/libvmod_directors/vmod.vcc
@@ -98,8 +98,8 @@ $Method VOID .add_backend(BACKEND)
 Description
         Add a backend to the director.
 
-        Note that the order in which this is done matters in the for the
-        fallback director.
+	Note that the order in which this is done matters for the fallback
+	director.
 
 Example
        vdir.add_backend(backend1);



From lkarsten at varnish-software.com  Tue Jun 24 09:31:55 2014
From: lkarsten at varnish-software.com (Lasse Karstensen)
Date: Tue, 24 Jun 2014 11:31:55 +0200
Subject: [4.0] ae54c1b Name directors consistently for clarity.
Message-ID: <E1WzN51-0003PT-Ge@project.varnish-software.com>


commit ae54c1b0b275d96404c41a25e200c806a6e75c9c
Author: Lasse Karstensen <lkarsten at varnish-software.com>
Date:   Tue Jun 10 12:42:21 2014 +0200

    Name directors consistently for clarity.
    
    Using multiple names for the instanciateed directors in this document
    just makes it more confusing, since .add_backend() and .backend() is
    mostly identical for all of them.

diff --git a/lib/libvmod_directors/vmod.vcc b/lib/libvmod_directors/vmod.vcc
index f7f3330..4900d4e 100644
--- a/lib/libvmod_directors/vmod.vcc
+++ b/lib/libvmod_directors/vmod.vcc
@@ -47,9 +47,9 @@ want to emulate the previous behavior of Varnish 3.0 you can just
 initialize the directors in vcl_init, like this:::
 
     sub vcl_init {
-        new bar = directors.round_robin();
-        bar.add_backend(server1);
-        bar.add_backend(server2);
+        new vdir = directors.round_robin();
+        vdir.add_backend(backend1);
+        vdir.add_backend(backend2);
     }
 
 As you can see there is nothing keeping you from manipulating the
@@ -65,21 +65,22 @@ Description
 
 	This director will pick backends in a round robin fashion.
 Example
-       new bar = directors.round_robin();
+	new vdir = directors.round_robin();
 
 $Method VOID .add_backend(BACKEND)
 
 Description
        Adds a backend to the director.
 Example
-       rrdir.add_backend(backend1);
+       vdir.add_backend(backend1);
+       vdir.add_backend(backend2);
 
 $Method BACKEND .backend()
 
 Description
        Picks a backend from the director.
 Example
-       set req.backend_hint = rrdir.backend();
+       set req.backend_hint = vdir.backend();
 
 
 $Object fallback()
@@ -122,21 +123,24 @@ Description
        more or less 1% of the traffic of a backend in the same
        director with a weight of 100.
 Example
-       new rand_dir = directors.random();
+	new vdir = directors.random();
 
 $Method VOID .add_backend(BACKEND, REAL)
 
 Description
        Adds a backend to the director with weight.
 Example
-       bar.add_backend(backend1, 3.14);
+	# 2/3 to backend1, 1/3 to backend2.
+	vdir.add_backend(backend1, 10);
+	vdir.add_backend(backend2, 5);
+
 
 $Method BACKEND .backend()
 
 Description
        Picks a backend from the director.
 Example
-       set req.backend_hint = rrdir.backend();
+	set req.backend_hint = vdir.backend();
 
 $Object hash()
 



From lkarsten at varnish-software.com  Tue Jun 24 09:31:55 2014
From: lkarsten at varnish-software.com (Lasse Karstensen)
Date: Tue, 24 Jun 2014 11:31:55 +0200
Subject: [4.0] f49967a Rewrite director descriptions.
Message-ID: <E1WzN51-0003Pc-JR@project.varnish-software.com>


commit f49967a722206d089e920f5bcc3a29a69d71b765
Author: Lasse Karstensen <lkarsten at varnish-software.com>
Date:   Tue Jun 10 12:55:03 2014 +0200

    Rewrite director descriptions.
    
    Add a bit more detail while cleaning up the language.
    
    Reformat for tabs instead of 7 spaces.

diff --git a/lib/libvmod_directors/vmod.vcc b/lib/libvmod_directors/vmod.vcc
index 4900d4e..94b9349 100644
--- a/lib/libvmod_directors/vmod.vcc
+++ b/lib/libvmod_directors/vmod.vcc
@@ -70,7 +70,7 @@ Example
 $Method VOID .add_backend(BACKEND)
 
 Description
-       Adds a backend to the director.
+       Add a backend to the round-robin director.
 Example
        vdir.add_backend(backend1);
        vdir.add_backend(backend2);
@@ -78,7 +78,7 @@ Example
 $Method BACKEND .backend()
 
 Description
-       Picks a backend from the director.
+       Pick a backend from the director.
 Example
        set req.backend_hint = vdir.backend();
 
@@ -103,7 +103,8 @@ Description
 	director.
 
 Example
-       vdir.add_backend(backend1);
+	vdir.add_backend(backend1);
+	vdir.add_backend(backend2);
 
 $Method BACKEND .backend()
 
@@ -116,59 +117,69 @@ Example
 $Object random()
 
 Description
-       Adds a random director. This director chooses backend based on
-       a random number. As you add backends to the director each
-       backends gets a weight, which is used to when requests are
-       being distributed. So, a backend with a weight of 1 would get
-       more or less 1% of the traffic of a backend in the same
-       director with a weight of 100.
+	Create a random backend director.
+
+	The random director distributes load over the backends using
+	a weighted random probability distribution.
+
 Example
 	new vdir = directors.random();
 
 $Method VOID .add_backend(BACKEND, REAL)
 
 Description
-       Adds a backend to the director with weight.
+	Add a backend to the director with a given weight.
+
+	Each backend backend will receive approximately
+	100 * (weight / (sum(all_added_weights))) per cent of the traffic sent
+	to this backend.
+
 Example
-	# 2/3 to backend1, 1/3 to backend2.
 	vdir.add_backend(backend1, 10);
 	vdir.add_backend(backend2, 5);
+	# 2/3 to backend1, 1/3 to backend2.
 
 
 $Method BACKEND .backend()
 
 Description
-       Picks a backend from the director.
+	Pick a backend from the director.
 Example
 	set req.backend_hint = vdir.backend();
 
 $Object hash()
 
 Description
-       Creates a hash director. The hash director chooses the backend
-       based on hashing an arbitrary string. If you provide it with a
-       session cookie, you'll have the client connecting to the same
-       backend every time.
+	Create a hashing backend director.
+
+	The director chooses the backend server by computing a hash/digest of
+	the string given to .backend().
+
+	Commonly used with ``client.identity`` or a session cookie to get
+	sticky sessions.
+
 Example
-       new hdir = directors.hash();
+	new vdir = directors.hash();
 
 $Method VOID .add_backend(BACKEND, REAL)
 
 Description
-	Adds a backend to the director with a certain weight.
+	Add a backend to the director with a certain weight.
 
 	Weight is used as in the random director. Recommended value is
 	1.0 unless you have special needs.
 
 Example
-       hdir.add_backend(backend1, 1.0);
+	vdir.add_backend(backend1, 1.0);
+	vdir.add_backend(backend2, 1.0);
+
 
 $Method BACKEND .backend(STRING_LIST)
 
 Description
-       Picks a backend from the director. Use the string or list of
-       strings provided to pick the backend.
+	Pick a backend from the backend director.
+
+	Use the string or list of strings provided to pick the backend.
 Example
-       # pick a backend based on the cookie header from the client
-       set req.backend_hint = hdir.backend(req.http.cookie);
+	set req.backend_hint = vdir.backend(req.http.cookie);  # pick a backend based on the cookie header from the client
 



From phk at FreeBSD.org  Tue Jun 24 09:31:55 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 24 Jun 2014 11:31:55 +0200
Subject: [4.0] 8061a7c Rememeber to supress body if we do 304 conditional
 response.
Message-ID: <E1WzN51-0003Pm-NX@project.varnish-software.com>


commit 8061a7c804be67b2b93bf8d7b6de7f5988452c48
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Jun 10 11:15:32 2014 +0000

    Rememeber to supress body if we do 304 conditional response.
    
    Fixes	#1518

diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c
index 4c8ca8e..c586ca4 100644
--- a/bin/varnishd/cache/cache_req_fsm.c
+++ b/bin/varnishd/cache/cache_req_fsm.c
@@ -149,8 +149,10 @@ cnt_deliver(struct worker *wrk, struct req *req)
 	if (!(req->obj->objcore->flags & OC_F_PASS)
 	    && req->esi_level == 0
 	    && http_GetStatus(req->obj->http) == 200
-	    && req->http->conds && RFC2616_Do_Cond(req))
+	    && req->http->conds && RFC2616_Do_Cond(req)) {
 		http_PutResponse(req->resp, "HTTP/1.1", 304, NULL);
+		req->wantbody = 0;
+	}
 
 	V1D_Deliver(req);
 	VSLb_ts_req(req, "Resp", W_TIM_real(wrk));



From phk at FreeBSD.org  Tue Jun 24 09:31:55 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 24 Jun 2014 11:31:55 +0200
Subject: [4.0] 7a6a3d7 Merge branch 'master' of
 git.varnish-cache.org:varnish-cache
Message-ID: <E1WzN51-0003Px-Qj@project.varnish-software.com>


commit 7a6a3d71f70e92c82c860c29e2ff645b7cced5bc
Merge: 8061a7c f49967a
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Jun 10 11:15:57 2014 +0000

    Merge branch 'master' of git.varnish-cache.org:varnish-cache



From daghf at varnish-software.com  Tue Jun 24 09:31:55 2014
From: daghf at varnish-software.com (Dag Haavi Finstad)
Date: Tue, 24 Jun 2014 11:31:55 +0200
Subject: [4.0] baad5e7 Clean up mis-pasted(?) information in description of
 .threshold.
Message-ID: <E1WzN51-0003Q6-Ua@project.varnish-software.com>


commit baad5e714c31cb5dd5b15d8b87aa282b90edfc29
Author: Dag Haavi Finstad <daghf at varnish-software.com>
Date:   Thu Jun 12 15:50:37 2014 +0200

    Clean up mis-pasted(?) information in description of .threshold.

diff --git a/doc/sphinx/reference/vcl.rst b/doc/sphinx/reference/vcl.rst
index 1268920..9ae2aa7 100644
--- a/doc/sphinx/reference/vcl.rst
+++ b/doc/sphinx/reference/vcl.rst
@@ -252,11 +252,7 @@ There are no mandatory options. These are the options you can set:
 
   threshold
     How many of the polls in .window must have succeeded for us to
-    consider the backend healthy. If this is set to more than or equal
-    to the threshold, the backend starts as healthy. Defaults to the
-    value of threshold - 1. In this case, the backend starts as sick
-    and requires one poll to pass to become healthy. Defaults to
-    threshold - 1.
+    consider the backend healthy. Defaults to 3.
 
 
 Access Control List (ACL)



From nils.goroll at uplex.de  Tue Jun 24 09:31:56 2014
From: nils.goroll at uplex.de (Nils Goroll)
Date: Tue, 24 Jun 2014 11:31:56 +0200
Subject: [4.0] dcccd5a add a commercial support reference
Message-ID: <E1WzN52-0003QG-0Y@project.varnish-software.com>


commit dcccd5a010a8c912a614c9fd518f0c508ec376c6
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Tue Jun 17 11:45:26 2014 +0200

    add a commercial support reference

diff --git a/doc/sphinx/installation/help.rst b/doc/sphinx/installation/help.rst
index 66433de..897324a 100644
--- a/doc/sphinx/installation/help.rst
+++ b/doc/sphinx/installation/help.rst
@@ -97,6 +97,8 @@ an email to phk at FreeBSD.org.
 	Varnish Software
 	sales at varnish-software.com
 
+	UPLEX
+	info at uplex.de
 
 .. _mailman: http://lists.varnish-cache.org/mailman/listinfo
 .. _pastebin: http://gist.github.com/



From martin at varnish-software.com  Tue Jun 24 09:31:56 2014
From: martin at varnish-software.com (Martin Blix Grydeland)
Date: Tue, 24 Jun 2014 11:31:56 +0200
Subject: [4.0] a318399 Add license to vxp_test.c
Message-ID: <E1WzN52-0003QQ-3Z@project.varnish-software.com>


commit a318399f83d4e8be8bd82058d02657818ef82969
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Tue Jun 17 12:15:02 2014 +0200

    Add license to vxp_test.c

diff --git a/lib/libvarnishapi/vxp_test.c b/lib/libvarnishapi/vxp_test.c
index 89437f1..a4abf4a 100644
--- a/lib/libvarnishapi/vxp_test.c
+++ b/lib/libvarnishapi/vxp_test.c
@@ -1,3 +1,31 @@
+/*-
+ * Copyright (c) 2014 Varnish Software AS
+ * All rights reserved.
+ *
+ * Author: Martin Blix Grydeland <martin at varnish-software.com>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
 #include <stdlib.h>
 #include <stdio.h>
 #include <errno.h>



From fgsch at lodoss.net  Tue Jun 24 09:31:56 2014
From: fgsch at lodoss.net (Federico G. Schwindt)
Date: Tue, 24 Jun 2014 11:31:56 +0200
Subject: [4.0] e54c8aa Document y (years) time unit
Message-ID: <E1WzN52-0003QZ-5U@project.varnish-software.com>


commit e54c8aaee5c7215954762dc3b2aeca09aefd26ad
Author: Federico G. Schwindt <fgsch at lodoss.net>
Date:   Tue Jun 17 12:52:52 2014 +0100

    Document y (years) time unit

diff --git a/lib/libvmod_std/vmod.vcc b/lib/libvmod_std/vmod.vcc
index b1b7010..7a22156 100644
--- a/lib/libvmod_std/vmod.vcc
+++ b/lib/libvmod_std/vmod.vcc
@@ -113,7 +113,7 @@ $Function DURATION duration(STRING, DURATION)
 Description
 	Converts the string *s* to seconds. *s* must be quantified
 	with ms (milliseconds), s (seconds), m (minutes), h (hours),
-	d (days) or w (weeks) units. If *s* fails to parse,
+	d (days), w (weeks) or y (years) units. If *s* fails to parse,
 	*fallback* will be returned.
 Example
 	set beresp.ttl = std.duration("1w", 3600s);



From fgsch at lodoss.net  Tue Jun 24 09:31:56 2014
From: fgsch at lodoss.net (Federico G. Schwindt)
Date: Tue, 24 Jun 2014 11:31:56 +0200
Subject: [4.0] 0913f09 Add y (years) time unit support to VCL
Message-ID: <E1WzN52-0003Qi-8H@project.varnish-software.com>


commit 0913f097cdd55ae2b49e090c04f355d0dd9b1664
Author: Federico G. Schwindt <fgsch at lodoss.net>
Date:   Tue Jun 17 13:37:27 2014 +0100

    Add y (years) time unit support to VCL
    
    Follows the change in the std vmod a while ago.

diff --git a/bin/varnishtest/tests/v00016.vtc b/bin/varnishtest/tests/v00016.vtc
index 1107f86..dfe69ae 100644
--- a/bin/varnishtest/tests/v00016.vtc
+++ b/bin/varnishtest/tests/v00016.vtc
@@ -73,7 +73,7 @@ varnish v1 -vcl {
 
 }
 
-varnish v1 -errvcl {Unknown time unit 'k'.  Legal are 'ms', 's', 'm', 'h', 'd' and 'w'} {
+varnish v1 -errvcl {Unknown time unit 'k'.  Legal are 'ms', 's', 'm', 'h', 'd', 'w' and 'y'} {
 	backend b { .host = "127.0.0.1"; }
 	sub vcl_backend_response { set beresp.ttl = 1. k; }
 }
diff --git a/doc/sphinx/reference/vcl.rst b/doc/sphinx/reference/vcl.rst
index 9ae2aa7..3f8a831 100644
--- a/doc/sphinx/reference/vcl.rst
+++ b/doc/sphinx/reference/vcl.rst
@@ -115,6 +115,8 @@ so 1.5w is allowed.
   w
     weeks
 
+  y
+    years
 
 Integers
 --------
diff --git a/lib/libvcc/vcc_expr.c b/lib/libvcc/vcc_expr.c
index c143cf1..8e4e234 100644
--- a/lib/libvcc/vcc_expr.c
+++ b/lib/libvcc/vcc_expr.c
@@ -74,10 +74,12 @@ vcc_TimeUnit(struct vcc *tl)
 		sc = 60.0 * 60.0 * 24.0;
 	else if (vcc_IdIs(tl->t, "w"))
 		sc = 60.0 * 60.0 * 24.0 * 7.0;
+	else if (vcc_IdIs(tl->t, "y"))
+		sc = 60.0 * 60.0 * 24.0 * 365.0;
 	else {
 		VSB_printf(tl->sb, "Unknown time unit ");
 		vcc_ErrToken(tl, tl->t);
-		VSB_printf(tl->sb, ".  Legal are 'ms', 's', 'm', 'h', 'd' and 'w'\n");
+		VSB_printf(tl->sb, ".  Legal are 'ms', 's', 'm', 'h', 'd', 'w' and 'y'\n");
 		vcc_ErrWhere(tl, tl->t);
 		return (1.0);
 	}



From phk at FreeBSD.org  Tue Jun 24 09:31:56 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 24 Jun 2014 11:31:56 +0200
Subject: [4.0] 5f58e96 White space nits
Message-ID: <E1WzN52-0003Qt-CP@project.varnish-software.com>


commit 5f58e9621e9f5fa915ebc4c2cdc74a558f4c76cd
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu Jun 19 07:56:10 2014 +0000

    White space nits

diff --git a/bin/varnishtest/tests/r01510.vtc b/bin/varnishtest/tests/r01510.vtc
index e8933be..47b87cb 100644
--- a/bin/varnishtest/tests/r01510.vtc
+++ b/bin/varnishtest/tests/r01510.vtc
@@ -6,7 +6,7 @@ varnish v1 -errvcl {Object name 'first' already used.} {
 		new first = debug.obj("FOO");
 		new first = debug.obj("BAH");
 	}
-} 
+}
 
 varnish v1 -errvcl {Object name 'first' already used.} {
 	import ${vmod_debug};
@@ -16,4 +16,4 @@ varnish v1 -errvcl {Object name 'first' already used.} {
 	sub vcl_init {
 		new first = debug.obj("FOO");
 	}
-} 
+}
diff --git a/lib/libvcc/vmodtool.py b/lib/libvcc/vmodtool.py
index 3299074..315b46b 100755
--- a/lib/libvcc/vmodtool.py
+++ b/lib/libvcc/vmodtool.py
@@ -66,16 +66,16 @@ ctypes = {
 
 def write_file_warning(fo, a, b, c):
 	fo.write(a + "\n")
-	fo.write(b + "NB:  This file is machine generated, DO NOT EDIT!\n")
+	fo.write(b + " NB:  This file is machine generated, DO NOT EDIT!\n")
 	fo.write(b + "\n")
-	fo.write(b + "Edit vmod.vcc and run make instead\n")
+	fo.write(b + " Edit vmod.vcc and run make instead\n")
 	fo.write(c + "\n\n")
 
 def write_c_file_warning(fo):
-	write_file_warning(fo, "/*", " * ", " */")
+	write_file_warning(fo, "/*", " *", " */")
 
 def write_rst_file_warning(fo):
-	write_file_warning(fo, "..", ".. ", "..")
+	write_file_warning(fo, "..", "..", "..")
 
 #######################################################################
 



From phk at FreeBSD.org  Tue Jun 24 09:31:56 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 24 Jun 2014 11:31:56 +0200
Subject: [4.0] e5c7502 Make sure attempts to talk (draft-)HTTP/2.0 don't
 confuse anybody.
Message-ID: <E1WzN52-0003R2-G4@project.varnish-software.com>


commit e5c7502defb50e8e0b5113b583c1a7e7a1365d1b
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu Jun 19 07:56:37 2014 +0000

    Make sure attempts to talk (draft-)HTTP/2.0 don't confuse anybody.

diff --git a/bin/varnishd/builtin.vcl b/bin/varnishd/builtin.vcl
index f9d3527..30fcaea 100644
--- a/bin/varnishd/builtin.vcl
+++ b/bin/varnishd/builtin.vcl
@@ -45,6 +45,10 @@ vcl 4.0;
 # Client side
 
 sub vcl_recv {
+    if (req.method == "PRI") {
+	/* We do not support SPDY or HTTP/2.0 */
+	return (synth(405));
+    }
     if (req.method != "GET" &&
       req.method != "HEAD" &&
       req.method != "PUT" &&
diff --git a/include/tbl/http_headers.h b/include/tbl/http_headers.h
index 1db7784..26443ca 100644
--- a/include/tbl/http_headers.h
+++ b/include/tbl/http_headers.h
@@ -69,6 +69,7 @@ HTTPH("Expect",			H_Expect,		0					  )	/* RFC2616 14.20 */
 HTTPH("Expires",		H_Expires,		0					  )	/* RFC2616 14.21 */
 HTTPH("From",			H_From,			0					  )	/* RFC2616 14.22 */
 HTTPH("Host",			H_Host,			0					  )	/* RFC2616 14.23 */
+HTTPH("HTTP2-Settings",		H_HTTP2_Settings,	HTTPH_R_PASS | HTTPH_R_FETCH | HTTPH_A_INS)	/* draft-ietf-httpbis-http2-12.txt */
 HTTPH("If-Match",		H_If_Match,			       HTTPH_R_FETCH		  )	/* RFC2616 14.24 */
 HTTPH("If-Modified-Since",	H_If_Modified_Since,		       HTTPH_R_FETCH		  )	/* RFC2616 14.25 */
 HTTPH("If-None-Match",		H_If_None_Match,		       HTTPH_R_FETCH		  )	/* RFC2616 14.26 */



From phk at FreeBSD.org  Tue Jun 24 09:31:56 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 24 Jun 2014 11:31:56 +0200
Subject: [4.0] ee92182 Add VRT_MAJOR_VERSION and VRT_MINOR_VERSION as
 discussed on the Stockholm VDD.
Message-ID: <E1WzN52-0003RC-ID@project.varnish-software.com>


commit ee92182fe25177c392c3be2d1c3955096d10d2e6
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu Jun 19 09:51:19 2014 +0000

    Add VRT_MAJOR_VERSION and VRT_MINOR_VERSION as discussed on the Stockholm VDD.
    
    Renovate and simplify the compile-time binding between VMODs, VCC
    and varnishd.

diff --git a/bin/varnishd/cache/cache_vrt_vmod.c b/bin/varnishd/cache/cache_vrt_vmod.c
index 70b8d95..a2dc7c0 100644
--- a/bin/varnishd/cache/cache_vrt_vmod.c
+++ b/bin/varnishd/cache/cache_vrt_vmod.c
@@ -39,7 +39,6 @@
 
 #include "vcli_priv.h"
 #include "vrt.h"
-#include "vmod_abi.h"
 
 /*--------------------------------------------------------------------
  * Modules stuff
@@ -64,10 +63,10 @@ static VTAILQ_HEAD(,vmod)	vmods = VTAILQ_HEAD_INITIALIZER(vmods);
 
 int
 VRT_Vmod_Init(void **hdl, void *ptr, int len, const char *nm,
-    const char *path, struct cli *cli)
+    const char *path, const char *file_id, struct cli *cli)
 {
 	struct vmod *v;
-	void *x, *y, *z, *w;
+	const struct vmod_data *d;
 	char buf[256];
 	void *dlhdl;
 
@@ -90,50 +89,37 @@ VRT_Vmod_Init(void **hdl, void *ptr, int len, const char *nm,
 
 		v->hdl = dlhdl;
 
-		bprintf(buf, "Vmod_%s_Name", nm);
-		x = dlsym(v->hdl, buf);
-		bprintf(buf, "Vmod_%s_Len", nm);
-		y = dlsym(v->hdl, buf);
-		bprintf(buf, "Vmod_%s_Func", nm);
-		z = dlsym(v->hdl, buf);
-		bprintf(buf, "Vmod_%s_ABI", nm);
-		w = dlsym(v->hdl, buf);
-		if (x == NULL || y == NULL || z == NULL || w == NULL) {
+		bprintf(buf, "Vmod_%s_Data", nm);
+		d = dlsym(v->hdl, buf);
+		if (d == NULL ||
+		    d->file_id == NULL ||
+		    strcmp(d->file_id, file_id)) {
 			VCLI_Out(cli, "Loading VMOD %s from %s:\n", nm, path);
-			VCLI_Out(cli, "VMOD symbols not found\n");
-			VCLI_Out(cli, "Check relative pathnames.\n");
+			VCLI_Out(cli,
+			    "This is no longer the same file seen by"
+			    " the VCL-compiler.\n");
 			(void)dlclose(v->hdl);
 			FREE_OBJ(v);
 			return (1);
 		}
-		AN(x);
-		AN(y);
-		AN(z);
-		AN(w);
-		if (strcmp(x, nm)) {
+		if (d->vrt_major != VRT_MAJOR_VERSION ||
+		    d->vrt_minor > VRT_MINOR_VERSION ||
+		    d->name == NULL ||
+		    strcmp(d->name, nm) ||
+		    d->func == NULL ||
+		    d->func_len <= 0 ||
+		    d->proto == NULL ||
+		    d->spec == NULL ||
+		    d->abi == NULL) {
 			VCLI_Out(cli, "Loading VMOD %s from %s:\n", nm, path);
-			VCLI_Out(cli, "File contain wrong VMOD (\"%s\")\n",
-			    (char *) x);
-			VCLI_Out(cli, "Check relative pathnames ?.\n");
+			VCLI_Out(cli, "VMOD data is mangled.\n");
 			(void)dlclose(v->hdl);
 			FREE_OBJ(v);
 			return (1);
 		}
 
-		if (strcmp(w, VMOD_ABI_Version)) {
-			VCLI_Out(cli, "Loading VMOD %s from %s:\n", nm, path);
-			VCLI_Out(cli, "VMOD ABI (%s)", (char*)w);
-			VCLI_Out(cli, " incompatible with varnish ABI (%s)\n",
-			    VMOD_ABI_Version);
-			(void)dlclose(v->hdl);
-			FREE_OBJ(v);
-			return (1);
-		}
-
-		// XXX: Check w for ABI version compatibility
-
-		v->funclen = *(const int *)y;
-		v->funcs = z;
+		v->funclen = d->func_len;
+		v->funcs = d->func;
 
 		REPLACE(v->nm, nm);
 		REPLACE(v->path, path);
diff --git a/bin/varnishd/flint.lnt b/bin/varnishd/flint.lnt
index 0c8c7d6..9ad0543 100644
--- a/bin/varnishd/flint.lnt
+++ b/bin/varnishd/flint.lnt
@@ -58,18 +58,8 @@
 
 // Stuff in VMODs which is used through dl*(3) functions
 -esym(754, Vmod_*_Func::*)
--esym(765, Vmod_*_Func)
--esym(552, Vmod_*_Func)
--esym(765, Vmod_*_Len)
--esym(714, Vmod_*_Len)
--esym(765, Vmod_*_Name)
--esym(714, Vmod_*_Name)
--esym(765, Vmod_*_Proto)
--esym(714, Vmod_*_Proto)
--esym(765, Vmod_*_Spec)
--esym(714, Vmod_*_Spec)
--esym(765, Vmod_*_ABI)
--esym(714, Vmod_*_ABI)
+-esym(714, Vmod_*_Data)
+-esym(765, Vmod_*_Data)
 
 
 //-sem (pthread_mutex_lock, thread_lock)
diff --git a/include/vrt.h b/include/vrt.h
index 5884b00..066d6c8 100644
--- a/include/vrt.h
+++ b/include/vrt.h
@@ -26,11 +26,26 @@
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
  *
- * Runtime support for compiled VCL programs.
+ * Runtime support for compiled VCL programs and VMODs.
  *
- * XXX: When this file is changed, lib/libvcc/generate.py *MUST* be rerun.
+ * NB: When this file is changed, lib/libvcc/generate.py *MUST* be rerun.
  */
 
+/***********************************************************************
+ * Major and minor VRT API versions.
+ *
+ * Whenever something is added, increment MINOR version
+ * Whenever something is deleted or changed in a way which is not
+ * binary/load-time compatible, increment MAJOR version
+ */
+
+#define VRT_MAJOR_VERSION	1U
+
+#define VRT_MINOR_VERSION	1U
+
+
+/***********************************************************************/
+
 struct req;
 struct busyobj;
 struct vsl_log;
@@ -89,6 +104,22 @@ struct vrt_ctx {
 
 /***********************************************************************/
 
+struct vmod_data {
+	/* The version/id fields must be first, they protect the rest */
+	unsigned			vrt_major;
+	unsigned			vrt_minor;
+	const char			*file_id;
+
+	const char			*name;
+	const void			*func;
+	int				func_len;
+	const char			*proto;
+	const char			* const *spec;
+	const char			*abi;
+};
+
+/***********************************************************************/
+
 enum gethdr_e { HDR_REQ, HDR_RESP, HDR_OBJ, HDR_BEREQ, HDR_BERESP };
 
 struct gethdr_s {
@@ -230,7 +261,7 @@ int VRT_VSA_GetPtr(const struct suckaddr *sua, const unsigned char ** dst);
 
 /* VMOD/Modules related */
 int VRT_Vmod_Init(void **hdl, void *ptr, int len, const char *nm,
-    const char *path, struct cli *cli);
+    const char *path, const char *file_id, struct cli *cli);
 void VRT_Vmod_Fini(void **hdl);
 
 struct vmod_priv;
diff --git a/lib/libvcc/vcc_vmod.c b/lib/libvcc/vcc_vmod.c
index 1723007..4f139de 100644
--- a/lib/libvcc/vcc_vmod.c
+++ b/lib/libvcc/vcc_vmod.c
@@ -35,6 +35,7 @@
 #include "vcc_compile.h"
 
 #include "vmod_abi.h"
+#include "vrt.h"
 
 void
 vcc_ParseImport(struct vcc *tl)
@@ -44,14 +45,12 @@ vcc_ParseImport(struct vcc *tl)
 	char buf[256];
 	struct token *mod, *t1;
 	struct inifin *ifp;
-	const char *modname;
-	const char *proto;
-	const char *abi;
-	const char **spec;
+	const char * const *spec;
 	struct symbol *sym;
 	const struct symbol *osym;
 	const char *p;
 	// int *modlen;
+	const struct vmod_data *vmd;
 
 	t1 = tl->t;
 	SkipToken(tl, ID);		/* "import" */
@@ -62,7 +61,7 @@ vcc_ParseImport(struct vcc *tl)
 
 	osym = VCC_FindSymbol(tl, mod, SYM_NONE);
 	if (osym != NULL && osym->kind != SYM_VMOD) {
-		VSB_printf(tl->sb, "Module %.*s conflics with other symbol.\n",
+		VSB_printf(tl->sb, "Module %.*s conflicts with other symbol.\n",
 		    PF(mod));
 		vcc_ErrWhere2(tl, t1, tl->t);
 		return;
@@ -104,77 +103,88 @@ vcc_ParseImport(struct vcc *tl)
 		bprintf(fn, "%s/libvmod_%.*s.so", tl->vmod_dir, PF(mod));
 	}
 
-	ifp = New_IniFin(tl);
-
-	VSB_printf(ifp->ini, "\tif (VRT_Vmod_Init(&VGC_vmod_%.*s,\n", PF(mod));
-	VSB_printf(ifp->ini, "\t    &Vmod_%.*s_Func,\n", PF(mod));
-	VSB_printf(ifp->ini, "\t    sizeof(Vmod_%.*s_Func),\n", PF(mod));
-	VSB_printf(ifp->ini, "\t    \"%.*s\",\n", PF(mod));
-	VSB_printf(ifp->ini, "\t    ");
-	EncString(ifp->ini, fn, NULL, 0);
-	VSB_printf(ifp->ini, ",\n\t    ");
-	VSB_printf(ifp->ini, "cli))\n");
-	VSB_printf(ifp->ini, "\t\treturn(1);");
-
-	/* XXX: zero the function pointer structure ?*/
-	VSB_printf(ifp->fin, "\tVRT_priv_fini(&vmod_priv_%.*s);", PF(mod));
-	VSB_printf(ifp->fin, "\n\tVRT_Vmod_Fini(&VGC_vmod_%.*s);", PF(mod));
-
-	ifp = NULL;
-
 	SkipToken(tl, ';');
 
 	hdl = dlopen(fn, RTLD_NOW | RTLD_LOCAL);
 	if (hdl == NULL) {
-		VSB_printf(tl->sb, "Could not load module %.*s\n\t%s\n\t%s\n",
-		    PF(mod), fn, dlerror());
+		VSB_printf(tl->sb, "Could not load VMOD %.*s\n", PF(mod));
+		VSB_printf(tl->sb, "\tFile name: %s\n", fn);
+		VSB_printf(tl->sb, "\tdlerror:: %s\n", dlerror());
 		vcc_ErrWhere(tl, mod);
 		return;
 	}
 
-	bprintf(buf, "Vmod_%.*s_Name", PF(mod));
-	modname = dlsym(hdl, buf);
-	if (modname == NULL) {
-		VSB_printf(tl->sb, "Could not load module %.*s\n\t%s\n\t%s\n",
-		    PF(mod), fn, "Symbol Vmod_Name not found");
+	bprintf(buf, "Vmod_%.*s_Data", PF(mod));
+	vmd = dlsym(hdl, buf);
+	if (vmd == NULL) {
+		VSB_printf(tl->sb, "Malformed VMOD %.*s\n", PF(mod));
+		VSB_printf(tl->sb, "\tFile name: %s\n", fn);
+		VSB_printf(tl->sb, "\t(no Vmod_Data symbol)\n");
 		vcc_ErrWhere(tl, mod);
 		return;
 	}
-	if (!vcc_IdIs(mod, modname)) {
-		VSB_printf(tl->sb, "Could not load module %.*s\n\t%s\n",
-		    PF(mod), fn);
-		VSB_printf(tl->sb, "\tModule has wrong name: <%s>\n", modname);
+	if (vmd->vrt_major != VRT_MAJOR_VERSION ||
+	    vmd->vrt_minor > VRT_MINOR_VERSION) {
+		VSB_printf(tl->sb, "Incompatible VMOD %.*s\n", PF(mod));
+		VSB_printf(tl->sb, "\tFile name: %s\n", fn);
+		VSB_printf(tl->sb, "\tVMOD version %u.%u\n",
+		    vmd->vrt_major, vmd->vrt_minor);
+		VSB_printf(tl->sb, "\tvarnishd version %u.%u\n",
+		    VRT_MAJOR_VERSION, VRT_MINOR_VERSION);
 		vcc_ErrWhere(tl, mod);
 		return;
 	}
-
-	bprintf(buf, "Vmod_%.*s_ABI", PF(mod));
-	abi = dlsym(hdl, buf);
-	if (abi == NULL || strcmp(abi, VMOD_ABI_Version) != 0) {
-		VSB_printf(tl->sb, "Could not load module %.*s\n\t%s\n",
-		    PF(mod), fn);
-		VSB_printf(tl->sb, "\tABI mismatch, expected <%s>, got <%s>\n",
-			   VMOD_ABI_Version, abi);
+	if (vmd->name == NULL ||
+	    vmd->func == NULL ||
+	    vmd->func_len <= 0 ||
+	    vmd->proto == NULL ||
+	    vmd->abi == NULL) {
+		VSB_printf(tl->sb, "Mangled VMOD %.*s\n", PF(mod));
+		VSB_printf(tl->sb, "\tFile name: %s\n", fn);
+		VSB_printf(tl->sb, "\tInconsistent metadata\n");
 		vcc_ErrWhere(tl, mod);
 		return;
 	}
 
-	bprintf(buf, "Vmod_%.*s_Proto", PF(mod));
-	proto = dlsym(hdl, buf);
-	if (proto == NULL) {
-		VSB_printf(tl->sb, "Could not load module %.*s\n\t%s\n\t%s\n",
-		    PF(mod), fn, "Symbol Vmod_Proto not found");
+	if (!vcc_IdIs(mod, vmd->name)) {
+		VSB_printf(tl->sb, "Wrong VMOD file %.*s\n", PF(mod));
+		VSB_printf(tl->sb, "\tFile name: %s\n", fn);
+		VSB_printf(tl->sb, "\tContains vmod \"%s\"\n", vmd->name);
 		vcc_ErrWhere(tl, mod);
 		return;
 	}
-	bprintf(buf, "Vmod_%.*s_Spec", PF(mod));
-	spec = dlsym(hdl, buf);
-	if (spec == NULL) {
-		VSB_printf(tl->sb, "Could not load module %.*s\n\t%s\n\t%s\n",
-		    PF(mod), fn, "Symbol Vmod_Spec not found");
+
+	if (strcmp(vmd->abi, VMOD_ABI_Version) != 0) {
+		VSB_printf(tl->sb, "Incompatible VMOD %.*s\n", PF(mod));
+		VSB_printf(tl->sb, "\tFile name: %s\n", fn);
+		VSB_printf(tl->sb, "\tABI mismatch, expected <%s>, got <%s>\n",
+			   VMOD_ABI_Version, vmd->abi);
 		vcc_ErrWhere(tl, mod);
 		return;
 	}
+
+	ifp = New_IniFin(tl);
+
+	VSB_printf(ifp->ini, "\tif (VRT_Vmod_Init(&VGC_vmod_%.*s,\n", PF(mod));
+	VSB_printf(ifp->ini, "\t    &Vmod_%.*s_Func,\n", PF(mod));
+	VSB_printf(ifp->ini, "\t    sizeof(Vmod_%.*s_Func),\n", PF(mod));
+	VSB_printf(ifp->ini, "\t    \"%.*s\",\n", PF(mod));
+	VSB_printf(ifp->ini, "\t    ");
+	EncString(ifp->ini, fn, NULL, 0);
+	VSB_printf(ifp->ini, ",\n");
+	AN(vmd);
+	AN(vmd->file_id);
+	VSB_printf(ifp->ini, "\t    \"%s\",\n", vmd->file_id);
+	VSB_printf(ifp->ini, "\t    cli))\n");
+	VSB_printf(ifp->ini, "\t\treturn(1);");
+
+	/* XXX: zero the function pointer structure ?*/
+	VSB_printf(ifp->fin, "\tVRT_priv_fini(&vmod_priv_%.*s);", PF(mod));
+	VSB_printf(ifp->fin, "\n\tVRT_Vmod_Fini(&VGC_vmod_%.*s);", PF(mod));
+
+	ifp = NULL;
+
+	spec = vmd->spec;
 	for (; *spec != NULL; spec++) {
 		p = *spec;
 		if (!strcmp(p, "OBJ")) {
@@ -208,6 +218,6 @@ vcc_ParseImport(struct vcc *tl)
 	Fh(tl, 0, "\n/* --- BEGIN VMOD %.*s --- */\n\n", PF(mod));
 	Fh(tl, 0, "static void *VGC_vmod_%.*s;\n", PF(mod));
 	Fh(tl, 0, "static struct vmod_priv vmod_priv_%.*s;\n", PF(mod));
-	Fh(tl, 0, "\n%s\n", proto);
+	Fh(tl, 0, "\n%s\n", vmd->proto);
 	Fh(tl, 0, "\n/* --- END VMOD %.*s --- */\n\n", PF(mod));
 }
diff --git a/lib/libvcc/vmodtool.py b/lib/libvcc/vmodtool.py
index 315b46b..21c6c6d 100755
--- a/lib/libvcc/vmodtool.py
+++ b/lib/libvcc/vmodtool.py
@@ -40,6 +40,7 @@ import sys
 import re
 import optparse
 import unittest
+import random
 from os import unlink
 from os.path import dirname, realpath, exists
 from pprint import pprint, pformat
@@ -196,29 +197,17 @@ class Vmod(object):
 				fo.write(j + "\n")
 
 	def c_vmod(self, fo):
-		fo.write('extern const char Vmod_' + self.nam + '_Name[];\n')
-		fo.write('const char Vmod_' + self.nam + '_Name[] =')
-		fo.write(' \"' + self.nam + '";\n')
-		fo.write("\n")
 
 		cs = self.c_struct()
 		fo.write(cs + ';\n')
 
-		vfn = 'Vmod_' + self.nam + '_Func'
+		vfn = 'Vmod_%s_Func' % self.nam
 
-		fo.write("extern const struct " + vfn + " " + vfn + ';\n')
-		fo.write("const struct " + vfn + " " + vfn + ' =')
+		fo.write("\nstatic const struct %s Vmod_Func =" % vfn)
 		fo.write(self.c_initializer())
 		fo.write("\n")
 
-		fo.write("\n")
-		fo.write("extern const int Vmod_" + self.nam + '_Len;\n')
-		fo.write("const int Vmod_" + self.nam + '_Len =')
-		fo.write(" sizeof(Vmod_" + self.nam + "_Func);\n")
-		fo.write("\n")
-
-		fo.write("extern const char Vmod_" + self.nam + "_Proto[];\n")
-		fo.write("const char Vmod_" + self.nam + "_Proto[] =\n")
+		fo.write("\nstatic const char Vmod_Proto[] =\n")
 		for t in self.c_typedefs_():
 			for i in lwrap(t, w=64):
 				fo.write('\t"' + i + '\\n"\n')
@@ -230,12 +219,28 @@ class Vmod(object):
 		fo.write(self.c_strspec())
 
 		fo.write("\n")
-		fo.write('extern const char Vmod_' + self.nam + '_ABI[];\n')
-		fo.write('const char Vmod_' + self.nam + '_ABI[] =')
-		fo.write(' VMOD_ABI_Version;\n')
-		#fo.write("\n")
-		#fo.write('const void * const Vmod_' + self.nam + '_Id =')
-		#fo.write(' &Vmod_' + self.nam + '_Id;\n')
+
+		nm = "Vmod_" + self.nam + "_Data"
+		fo.write("extern const struct vmod_data " + nm + ";\n\n")
+		fo.write("const struct vmod_data " + nm + " = {\n")
+		fo.write("\t.vrt_major = VRT_MAJOR_VERSION,\n");
+		fo.write("\t.vrt_minor = VRT_MINOR_VERSION,\n");
+		fo.write("\t.name = \"%s\",\n" % self.nam)
+		fo.write("\t.func = &Vmod_Func,\n")
+		fo.write("\t.func_len = sizeof(Vmod_Func),\n")
+		fo.write("\t.proto = Vmod_Proto,\n")
+		fo.write("\t.spec = Vmod_Spec,\n")
+		fo.write("\t.abi = VMOD_ABI_Version,\n")
+
+		# NB: Sort of hackish:
+		# Fill file_id with random stuff, so we can tell if
+		# VCC and VRT_Vmod_Init() dlopens the same file
+		#
+		fo.write("\t.file_id = \"")
+		for i in range(32):
+			fo.write("%c" % random.randint(0x23,0x5b))
+		fo.write("\",\n")
+		fo.write("};\n")
 
 	def c_initializer(self):
 		s = '{\n'
@@ -269,8 +274,7 @@ class Vmod(object):
 		return s
 
 	def c_strspec(self):
-		s = "const char * const Vmod_" + self.nam + "_Spec[]"
-		s = "extern " + s + ";\n" + s + " = {\n"
+		s = "static const char * const Vmod_Spec[] = {\n"
 
 		for o in self.objs:
 			s += o.c_strspec(self.nam) + ",\n\n"



From lkarsten at varnish-software.com  Tue Jun 24 09:31:56 2014
From: lkarsten at varnish-software.com (Lasse Karstensen)
Date: Tue, 24 Jun 2014 11:31:56 +0200
Subject: [4.0] 98923ba Fix use of backend where it should say director.
Message-ID: <E1WzN52-0003RM-Jx@project.varnish-software.com>


commit 98923ba4b07bd5dbb0349e38e8d2ad711a2e67a4
Author: Lasse Karstensen <lkarsten at varnish-software.com>
Date:   Tue Jun 10 13:33:50 2014 +0200

    Fix use of backend where it should say director.

diff --git a/lib/libvmod_directors/vmod.vcc b/lib/libvmod_directors/vmod.vcc
index 94b9349..bbfa0eb 100644
--- a/lib/libvmod_directors/vmod.vcc
+++ b/lib/libvmod_directors/vmod.vcc
@@ -132,7 +132,7 @@ Description
 
 	Each backend backend will receive approximately
 	100 * (weight / (sum(all_added_weights))) per cent of the traffic sent
-	to this backend.
+	to this director.
 
 Example
 	vdir.add_backend(backend1, 10);



From lkarsten at varnish-software.com  Tue Jun 24 09:31:56 2014
From: lkarsten at varnish-software.com (Lasse Karstensen)
Date: Tue, 24 Jun 2014 11:31:56 +0200
Subject: [4.0] 5ef67f5 RST warnings in changes.rst is a hard error.
Message-ID: <E1WzN52-0003Rh-Oo@project.varnish-software.com>


commit 5ef67f5ba14b99a9e08651aa83abfe99219df532
Author: Lasse Karstensen <lkarsten at varnish-software.com>
Date:   Fri Jun 20 16:26:27 2014 +0200

    RST warnings in changes.rst is a hard error.

diff --git a/doc/Makefile.am b/doc/Makefile.am
index cfd0fb5..5da0749 100644
--- a/doc/Makefile.am
+++ b/doc/Makefile.am
@@ -1,10 +1,12 @@
 #
+#
+RST2ANY_FLAGS = --halt=2
 
 EXTRA_DIST = changes.rst changes.html
 
 changes.html: changes.rst
 if HAVE_RST2HTML
-	${RST2HTML} $? $@
+	${RST2HTML} ${RST2ANY_FLAGS} $? $@
 else
 	@echo "========================================"
 	@echo "You need rst2html installed to make dist"



From lkarsten at varnish-software.com  Tue Jun 24 09:31:56 2014
From: lkarsten at varnish-software.com (Lasse Karstensen)
Date: Tue, 24 Jun 2014 11:31:56 +0200
Subject: [4.0] 96d0ba1 Write changelog and do groundwork for 4.0.1-rc1.
Message-ID: <E1WzN52-0003RV-M3@project.varnish-software.com>


commit 96d0ba1df5d7002927caf358e14245e40251520e
Author: Lasse Karstensen <lkarsten at varnish-software.com>
Date:   Fri Jun 20 16:19:12 2014 +0200

    Write changelog and do groundwork for 4.0.1-rc1.
    
    We still have a few bugs left to fix, but start out with this so the
    Jenkins machinery can be fine tuned in advance.

diff --git a/configure.ac b/configure.ac
index 36cc542..eed48bc 100644
--- a/configure.ac
+++ b/configure.ac
@@ -2,7 +2,7 @@ AC_PREREQ(2.59)
 AC_COPYRIGHT([Copyright (c) 2006 Verdens Gang AS
 Copyright (c) 2006-2014 Varnish Software AS])
 AC_REVISION([$Id$])
-AC_INIT([Varnish], [4.0.0], [varnish-dev at varnish-cache.org])
+AC_INIT([Varnish], [4.0.1-rc1], [varnish-dev at varnish-cache.org])
 AC_CONFIG_SRCDIR(include/miniobj.h)
 AC_CONFIG_HEADERS([config.h])
 AC_CONFIG_MACRO_DIR([m4])
diff --git a/doc/changes.rst b/doc/changes.rst
index 837266d..2ee4aeb 100644
--- a/doc/changes.rst
+++ b/doc/changes.rst
@@ -1,3 +1,84 @@
+
+============================================
+Changes from 4.0.0 to 4.0.1-rc1 (2014-06-20)
+============================================
+
+New since 4.0.0:
+
+- Backend IMS is now only attempted when last status was 200.
+- Packaging now uses find-provides instead of find-requires. [redhat]
+- Two new counters: n_purges and n_obj_purged.
+- Core size can now be set from /etc/sysconfig/varnish [redhat]
+- Via header set is now RFC compliant.
+- Removed "purge" keyword in VCL. Use return(purge) instead.
+- fallback director is now documented.
+- %D format flag in varnishncsa is now truncated to an integer value.
+- persistent storage backend is now deprecated.
+  https://www.varnish-cache.org/docs/trunk/phk/persistent.html
+- Added format flags %I (total bytes received) and %O (total bytes sent) for
+  varnishncsa.
+- python-docutils >= 0.7 is now required.
+- Support year (y) as a duration in VCL.
+- VMOD ABI requirements are relaxed, a VMOD no longer have to be run on the
+  same git revision as it was compiled for. Replaced by a major/minor ABI counter.
+
+
+Bugs fixed
+----------
+
+* 1475_ - time-to-first-byte in varnishncsa was potentially dishonest.
+* 1480_ - Porting guide for 4.0 is incomplete.
+* 1482_ - Inherit group memberships of -u specified user.
+* 1473_ - Fail correctly in configure when rst2man is not found.
+* 1486_ - Truncate negative Age values to zero.
+* 1488_ - Don't panic on high request rates.
+* 1489_ - req.esi should only be available in client threads.
+* 1490_ - Fix thread leak when reducing number of threads.
+* 1491_ - Reorder backend connection close procedure to help test cases.
+* 1498_ - Prefix translated VCL names to avoid name clashes.
+* 1499_ - Don't leak an objcore when HSH_Lookup returns expired object.
+* 1493_ - vcl_purge can return synth or restart.
+* 1476_ - Cope with systems having sys/endian.h and endian.h.
+* 1496_ - varnishadm should be consistent in argv ordering.
+* 1494_ - Don't panic on VCL-initiated retry after a backend 500 error.
+* 1139_ - Also reset keep (for IMS) time when purging.
+* 1478_ - Avoid panic when delivering an object that expires during delivery.
+* 1504_ - ACLs can be unreferenced with vcc_err_unref=off set.
+* 1501_ - Handle that a director couldn't pick a backend.
+* 1495_ - Reduce WRK_SumStat contention.
+* 1510_ - Complain on symbol reuse in VCL.
+* 1514_ - Document storage.NAME.free_space and .used_space [docs]
+* 1518_ - Suppress body on 304 response when using ESI.
+* 1519_ - Round-robin director does not support weight. [docs]
+
+
+.. _1475: https://www.varnish-cache.org/trac/ticket/1475
+.. _1480: https://www.varnish-cache.org/trac/ticket/1480
+.. _1482: https://www.varnish-cache.org/trac/ticket/1482
+.. _1473: https://www.varnish-cache.org/trac/ticket/1473
+.. _1486: https://www.varnish-cache.org/trac/ticket/1486
+.. _1488: https://www.varnish-cache.org/trac/ticket/1488
+.. _1489: https://www.varnish-cache.org/trac/ticket/1489
+.. _1490: https://www.varnish-cache.org/trac/ticket/1490
+.. _1491: https://www.varnish-cache.org/trac/ticket/1491
+.. _1498: https://www.varnish-cache.org/trac/ticket/1498
+.. _1499: https://www.varnish-cache.org/trac/ticket/1499
+.. _1493: https://www.varnish-cache.org/trac/ticket/1493
+.. _1476: https://www.varnish-cache.org/trac/ticket/1476
+.. _1496: https://www.varnish-cache.org/trac/ticket/1496
+.. _1494: https://www.varnish-cache.org/trac/ticket/1494
+.. _1139: https://www.varnish-cache.org/trac/ticket/1139
+.. _1478: https://www.varnish-cache.org/trac/ticket/1478
+.. _1504: https://www.varnish-cache.org/trac/ticket/1504
+.. _1501: https://www.varnish-cache.org/trac/ticket/1501
+.. _1495: https://www.varnish-cache.org/trac/ticket/1495
+.. _1510: https://www.varnish-cache.org/trac/ticket/1510
+.. _1514: https://www.varnish-cache.org/trac/ticket/1514
+.. _1518: https://www.varnish-cache.org/trac/ticket/1518
+.. _1519: https://www.varnish-cache.org/trac/ticket/1519
+
+
+
 ==============================================
 Changes from 4.0.0 beta1 to 4.0.0 (2014-04-10)
 ==============================================
diff --git a/redhat/varnish.spec b/redhat/varnish.spec
index b85217c..ec81229 100644
--- a/redhat/varnish.spec
+++ b/redhat/varnish.spec
@@ -1,10 +1,10 @@
-%define v_rc beta1
+%define v_rc rc1
 %define vd_rc %{?v_rc:-%{?v_rc}}
 %define    _use_internal_dependency_generator 0
 %define __find_provides %{_builddir}/varnish-%{version}%{?v_rc:-%{?v_rc}}/redhat/find-provides
 Summary: High-performance HTTP accelerator
 Name: varnish
-Version: 4.0.0
+Version: 4.0.1
 #Release: 0.20140328%{?v_rc}%{?dist}
 Release: 1%{?v_rc}%{?dist}
 License: BSD



From lkarsten at varnish-software.com  Tue Jun 24 09:31:56 2014
From: lkarsten at varnish-software.com (Lasse Karstensen)
Date: Tue, 24 Jun 2014 11:31:56 +0200
Subject: [4.0] 228055e python-docutils v0.6 will have to do.
Message-ID: <E1WzN52-0003Ru-Rc@project.varnish-software.com>


commit 228055ea74aa509b4379025317b560066714a839
Author: Lasse Karstensen <lkarsten at varnish-software.com>
Date:   Fri Jun 20 17:03:25 2014 +0200

    python-docutils v0.6 will have to do.
    
    Seems like requiring a version from 2010 was way optimistic.
    
    EL6 has v0.6 which was the first version rst2man appeared in.

diff --git a/doc/changes.rst b/doc/changes.rst
index 2ee4aeb..a66bda1 100644
--- a/doc/changes.rst
+++ b/doc/changes.rst
@@ -17,7 +17,7 @@ New since 4.0.0:
   https://www.varnish-cache.org/docs/trunk/phk/persistent.html
 - Added format flags %I (total bytes received) and %O (total bytes sent) for
   varnishncsa.
-- python-docutils >= 0.7 is now required.
+- python-docutils >= 0.6 is now required.
 - Support year (y) as a duration in VCL.
 - VMOD ABI requirements are relaxed, a VMOD no longer have to be run on the
   same git revision as it was compiled for. Replaced by a major/minor ABI counter.
diff --git a/redhat/varnish.spec b/redhat/varnish.spec
index ec81229..b50c859 100644
--- a/redhat/varnish.spec
+++ b/redhat/varnish.spec
@@ -18,7 +18,7 @@ BuildRoot: %{_tmppath}/%{name}-%{version}-%{release}-root-%(%{__id_u} -n)
 # You will need at least automake autoconf libtool
 #BuildRequires: automake autoconf libtool
 BuildRequires: ncurses-devel groff pcre-devel pkgconfig libedit-devel jemalloc-devel
-BuildRequires: python-docutils >= 0.7
+BuildRequires: python-docutils >= 0.6
 Requires: varnish-libs = %{version}-%{release}
 Requires: logrotate
 Requires: ncurses



From lkarsten at varnish-software.com  Tue Jun 24 09:31:56 2014
From: lkarsten at varnish-software.com (Lasse Karstensen)
Date: Tue, 24 Jun 2014 11:31:56 +0200
Subject: [4.0] 78b34d9 Forgot to mention the 200 Not Modified fix.
Message-ID: <E1WzN52-0003S0-Ts@project.varnish-software.com>


commit 78b34d9eb05b4afa98daa8bf6851a1ff98913e9b
Author: Lasse Karstensen <lkarsten at varnish-software.com>
Date:   Fri Jun 20 17:08:21 2014 +0200

    Forgot to mention the 200 Not Modified fix.
    
    Reminded by: daghf

diff --git a/doc/changes.rst b/doc/changes.rst
index a66bda1..c5e1425 100644
--- a/doc/changes.rst
+++ b/doc/changes.rst
@@ -5,6 +5,7 @@ Changes from 4.0.0 to 4.0.1-rc1 (2014-06-20)
 
 New since 4.0.0:
 
+- Varnish will no longer reply with "200 Not Modified".
 - Backend IMS is now only attempted when last status was 200.
 - Packaging now uses find-provides instead of find-requires. [redhat]
 - Two new counters: n_purges and n_obj_purged.



From perbu at varnish-software.com  Tue Jun 24 09:31:56 2014
From: perbu at varnish-software.com (Per Buer)
Date: Tue, 24 Jun 2014 11:31:56 +0200
Subject: [4.0] 489460a Ported the boltsort vmod to 4.0 and included it in the
 std vmod as discussed on the latest vdd.
Message-ID: <E1WzN52-0003S9-Ve@project.varnish-software.com>


commit 489460a7ab218f1dc4303b0baeec1bdfa9b4da38
Author: Per Buer <perbu at varnish-software.com>
Date:   Sat Jun 21 18:21:04 2014 +0200

    Ported the boltsort vmod to 4.0 and included it
    in the std vmod as discussed on the latest vdd.
    
    Renamed it to std.querysort. Two tests where
    included. They both pass on my machine.

diff --git a/bin/varnishtest/tests/m00014.vtc b/bin/varnishtest/tests/m00014.vtc
new file mode 100644
index 0000000..db2a62e
--- /dev/null
+++ b/bin/varnishtest/tests/m00014.vtc
@@ -0,0 +1,68 @@
+varnishtest "Test querysort in std vmod"
+
+server s1 {
+
+    rxreq
+    txresp
+
+    rxreq
+    txresp
+
+    rxreq
+    txresp
+
+    rxreq
+    txresp
+
+    rxreq
+    txresp
+
+    rxreq
+    txresp
+
+    rxreq
+    txresp
+
+    rxreq
+    txresp
+
+} -start
+
+varnish v1 -vcl+backend {
+
+    import ${vmod_std};
+    sub vcl_deliver {
+        set resp.http.naren = std.querysort(req.url);
+    }
+
+} -start
+
+client c1 {
+
+    txreq -url "/video/44505073?title=0&byline=0&portrait=0&color=51a516"
+    rxresp
+    expect resp.http.naren == "/video/44505073?byline=0&color=51a516&portrait=0&title=0"
+
+    txreq -url "/video/44505073?byline=0&&&&&"
+    rxresp
+    expect resp.http.naren == "/video/44505073?byline=0"
+
+    txreq -url "/video/2?&"
+    rxresp
+    expect resp.http.naren == "/video/2?"
+
+    txreq -url "/video/2"
+    rxresp
+    expect resp.http.naren == "/video/2"
+
+    txreq -url "/video/2?cod=cape&cape=cod"
+    rxresp
+    expect resp.http.naren == "/video/2?cape=cod&cod=cape"
+
+    txreq -url "/"
+    rxresp
+    expect resp.http.naren == "/"
+
+}
+
+client c1 -run
diff --git a/bin/varnishtest/tests/m00015.vtc b/bin/varnishtest/tests/m00015.vtc
new file mode 100644
index 0000000..47109c8
--- /dev/null
+++ b/bin/varnishtest/tests/m00015.vtc
@@ -0,0 +1,75 @@
+varnishtest "Test querysort of req.url in vcl_hash"
+
+server s1 {
+
+    rxreq
+    txresp
+    rxreq
+    txresp
+
+} -start
+
+varnish v1 -vcl+backend {
+
+    import ${vmod_std};
+
+    sub vcl_hash {
+        set req.url = std.querysort(req.url);
+    }
+
+    sub vcl_deliver {
+        if (obj.hits > 0) {
+            set resp.http.X-Cache = "HIT";
+        }
+        else {
+            set resp.http.X-Cache = "MISS";
+        }
+    }
+
+} -start
+
+client c1 {
+
+    txreq -url "/video/47013255?title=0&byline=0&portrait=0&autoplay=1"
+    rxresp
+    expect resp.status == 200
+    expect resp.http.X-Cache == "MISS"
+
+    txreq -url "/video/47013255?title=0&byline=0&portrait=0&autoplay=1"
+    rxresp
+    expect resp.status == 200
+    expect resp.http.X-Cache == "HIT"
+
+}
+
+client c2 {
+
+    txreq -url "/video/47013255?autoplay=1&title=0&byline=0&portrait=0"
+    rxresp
+    expect resp.status == 200
+    expect resp.http.X-Cache == "HIT"
+
+    txreq -url "autoplay=1&title=0&byline=0&portrait=0&&&&"
+    rxresp
+    expect resp.status == 200
+    expect resp.http.X-Cache == "HIT"
+
+}
+
+client c2 {
+
+    txreq -url "/video/47013255?autoplay=1&title=0&byline=0&portrait=0"
+    rxresp
+    expect resp.status == 200
+    expect resp.http.X-Cache == "HIT"
+
+    txreq -url "/video/47013255?autoplay=1&title=0&byline=0&portrait=0&&&&"
+    rxresp
+    expect resp.status == 200
+    expect resp.http.X-Cache == "HIT"
+
+}
+
+
+client c1 -run
+client c2 -run
diff --git a/lib/libvmod_std/vmod.vcc b/lib/libvmod_std/vmod.vcc
index 7a22156..0be272e 100644
--- a/lib/libvmod_std/vmod.vcc
+++ b/lib/libvmod_std/vmod.vcc
@@ -155,6 +155,16 @@ Description
 Example
 	std.timestamp("curl-request");
 
+$Function STRING querysort(STRING)
+
+Description
+        Sorts the querystring for cache normalization purposes.
+
+Example
+        set req.url = std.querysort(req.url);
+
+
+
 SEE ALSO
 ========
 
@@ -173,3 +183,4 @@ COPYRIGHT
 
 This document is licensed under the same licence as Varnish
 itself. See LICENCE for details.
+
diff --git a/lib/libvmod_std/vmod_std.c b/lib/libvmod_std/vmod_std.c
index 233c2da..bedbd7d 100644
--- a/lib/libvmod_std/vmod_std.c
+++ b/lib/libvmod_std/vmod_std.c
@@ -225,3 +225,140 @@ vmod_timestamp(const struct vrt_ctx *ctx, VCL_STRING label)
 		VSLb_ts_req(ctx->req, label, VTIM_real());
 	}
 }
+
+
+/* Boltsort
+   Author: Naren Venkataraman of Vimeo Inc.
+
+   Included here with permission.
+*/
+
+
+#define QS_MAX_PARAM_COUNT 32
+#define QS_EQUALS(c, h) ((c == h) || (c == '\0' && h == '&') || (c == '&' && h == '\0'))
+#define QS_ENDS(s) (s == '&' || s == '\0') 
+
+static const char QS_TERMINATORS[2] = {'\0', '&'};
+
+//since we dont store param length, we have to evaluate everytime
+static inline int param_compare (char *s, char *t)
+{
+
+    for ( ;QS_EQUALS(*s, *t); s++, t++) {
+        if (QS_ENDS(*s)) {
+            return 0;
+        }
+    }
+    return *s - *t;
+
+}
+
+//end of param is either first occurance of & or '\0'
+static inline int param_copy(char *dst, char *src, char *last_param)
+{
+
+    int len = strchr(src, QS_TERMINATORS[(src != last_param)]) - src;
+    memcpy(dst, src, len);
+    return len;
+
+}
+
+//Varnish vmod requires this
+int init_function(struct vmod_priv *priv, const struct VCL_conf *conf)
+{
+    return 0;
+
+}
+
+//sort query string
+VCL_STRING vmod_querysort(const struct vrt_ctx * ctx, VCL_STRING url)
+{
+
+    if (url == NULL) {
+        return NULL;
+    }
+
+    int qs_index = 0;
+    int param_count = 0;
+
+    char *dst_url = NULL;
+    char *qs = NULL;
+
+    //To avoid 1 pass for count calculations, assuming MAX_PARAM_COUNT as max
+    char* params[QS_MAX_PARAM_COUNT];
+
+    int i, p;
+    char *param = NULL;
+
+    qs = strchr(url, '?');
+    if(!qs) {
+        return url;
+    }
+
+    //add first param and increment count
+    params[param_count++] = ++qs;
+    qs_index = qs - url;
+
+    //Continue to find query string
+    while((qs = strchr(qs, '&')) != NULL) {
+        param = ++qs;
+
+        for(p = 0; p < param_count; p++) {
+            //if incoming param is < param at position then place it at p and then move up rest
+            if(param[0] < params[p][0] || param_compare(param, params[p]) < 0) {
+                for(i = param_count; i > p; i--) {
+                    params[i] = params[i-1];
+                }
+                break;
+            }
+        }
+        params[p] = param;
+        param_count++;
+
+        //if it exceed max params return as is
+        if (param_count == QS_MAX_PARAM_COUNT) {
+            return url;
+        }
+    }
+
+    //there is nothing after & 
+    //eg: http://127.0.0.1/?me=1&
+    if (param_count == 1) {
+        return url;
+    }
+
+    //allocate space for sorted url
+    //    struct ws *ws = sp->wrk->ws; 
+    struct ws *ws = ctx->ws;
+    dst_url = WS_Alloc(ws, strchr(param, '\0') - url + 1);
+    WS_Assert(ws);
+
+    //if alloc fails return as is
+    if(dst_url == NULL) {
+        return url;
+    }
+
+    //copy data before query string
+    char* cp = memcpy(dst_url, url, qs_index) + qs_index;
+
+    //get rid of all empty params /test/?a&&&
+    for(p = 0; p < param_count - 1; p++) { 
+        if (params[p][0] != '\0' && params[p][0] != '&') {
+            break;
+        }
+    }
+
+    //copy sorted params
+    for(; p < param_count - 1; p++) {
+        //copy and increment
+        cp += param_copy(cp, params[p], param);
+        *cp++ = '&';
+    }
+
+    //copy the last param
+    cp += param_copy(cp, params[p], param);
+    *cp = '\0';
+
+    return dst_url;
+
+}



From fgsch at lodoss.net  Tue Jun 24 09:31:57 2014
From: fgsch at lodoss.net (Federico G. Schwindt)
Date: Tue, 24 Jun 2014 11:31:57 +0200
Subject: [4.0] ed8b091 Reorganise querysort a bit
Message-ID: <E1WzN53-0003SM-2A@project.varnish-software.com>


commit ed8b09192612cf22e85be226b966994e96cbdb72
Author: Federico G. Schwindt <fgsch at lodoss.net>
Date:   Sun Jun 22 09:31:02 2014 +0100

    Reorganise querysort a bit
    
    Also kill superflous function and test. Is this a candidate for its
    own file?

diff --git a/bin/varnishtest/tests/m00014.vtc b/bin/varnishtest/tests/m00014.vtc
index db2a62e..260280f 100644
--- a/bin/varnishtest/tests/m00014.vtc
+++ b/bin/varnishtest/tests/m00014.vtc
@@ -1,68 +1,36 @@
-varnishtest "Test querysort in std vmod"
-
-server s1 {
-
-    rxreq
-    txresp
-
-    rxreq
-    txresp
-
-    rxreq
-    txresp
-
-    rxreq
-    txresp
-
-    rxreq
-    txresp
-
-    rxreq
-    txresp
-
-    rxreq
-    txresp
-
-    rxreq
-    txresp
+varnishtest "Test std.querysort"
 
+server s1 -repeat 5 {
+	rxreq
+	txresp
 } -start
 
 varnish v1 -vcl+backend {
+	import ${vmod_std};
 
-    import ${vmod_std};
-    sub vcl_deliver {
-        set resp.http.naren = std.querysort(req.url);
-    }
-
+	sub vcl_deliver {
+		set resp.http.url = std.querysort(req.url);
+	}
 } -start
 
 client c1 {
-
-    txreq -url "/video/44505073?title=0&byline=0&portrait=0&color=51a516"
-    rxresp
-    expect resp.http.naren == "/video/44505073?byline=0&color=51a516&portrait=0&title=0"
-
-    txreq -url "/video/44505073?byline=0&&&&&"
-    rxresp
-    expect resp.http.naren == "/video/44505073?byline=0"
-
-    txreq -url "/video/2?&"
-    rxresp
-    expect resp.http.naren == "/video/2?"
-
-    txreq -url "/video/2"
-    rxresp
-    expect resp.http.naren == "/video/2"
-
-    txreq -url "/video/2?cod=cape&cape=cod"
-    rxresp
-    expect resp.http.naren == "/video/2?cape=cod&cod=cape"
-
-    txreq -url "/"
-    rxresp
-    expect resp.http.naren == "/"
-
-}
-
-client c1 -run
+	txreq -url "/foo/bar?t=0&b=0&p=0&c=5"
+	rxresp
+	expect resp.http.url == "/foo/bar?b=0&c=5&p=0&t=0"
+
+	txreq -url "/foo/bar?coa=0&co=0"
+	rxresp
+	expect resp.http.url == "/foo/bar?co=0&coa=0"
+
+	txreq -url "/foo/bar?a=0&&&&&"
+	rxresp
+	expect resp.http.url == "/foo/bar?a=0"
+
+	txreq -url "/foo/bar?&"
+	rxresp
+	expect resp.http.url == "/foo/bar?"
+
+	txreq -url "/foo/bar"
+	rxresp
+	expect resp.http.url == "/foo/bar"
+} -run
diff --git a/bin/varnishtest/tests/m00015.vtc b/bin/varnishtest/tests/m00015.vtc
deleted file mode 100644
index 47109c8..0000000
--- a/bin/varnishtest/tests/m00015.vtc
+++ /dev/null
@@ -1,75 +0,0 @@
-varnishtest "Test querysort of req.url in vcl_hash"
-
-server s1 {
-
-    rxreq
-    txresp
-    rxreq
-    txresp
-
-} -start
-
-varnish v1 -vcl+backend {
-
-    import ${vmod_std};
-
-    sub vcl_hash {
-        set req.url = std.querysort(req.url);
-    }
-
-    sub vcl_deliver {
-        if (obj.hits > 0) {
-            set resp.http.X-Cache = "HIT";
-        }
-        else {
-            set resp.http.X-Cache = "MISS";
-        }
-    }
-
-} -start
-
-client c1 {
-
-    txreq -url "/video/47013255?title=0&byline=0&portrait=0&autoplay=1"
-    rxresp
-    expect resp.status == 200
-    expect resp.http.X-Cache == "MISS"
-
-    txreq -url "/video/47013255?title=0&byline=0&portrait=0&autoplay=1"
-    rxresp
-    expect resp.status == 200
-    expect resp.http.X-Cache == "HIT"
-
-}
-
-client c2 {
-
-    txreq -url "/video/47013255?autoplay=1&title=0&byline=0&portrait=0"
-    rxresp
-    expect resp.status == 200
-    expect resp.http.X-Cache == "HIT"
-
-    txreq -url "autoplay=1&title=0&byline=0&portrait=0&&&&"
-    rxresp
-    expect resp.status == 200
-    expect resp.http.X-Cache == "HIT"
-
-}
-
-client c2 {
-
-    txreq -url "/video/47013255?autoplay=1&title=0&byline=0&portrait=0"
-    rxresp
-    expect resp.status == 200
-    expect resp.http.X-Cache == "HIT"
-
-    txreq -url "/video/47013255?autoplay=1&title=0&byline=0&portrait=0&&&&"
-    rxresp
-    expect resp.status == 200
-    expect resp.http.X-Cache == "HIT"
-
-}
-
-
-client c1 -run
-client c2 -run
diff --git a/lib/libvmod_std/vmod_std.c b/lib/libvmod_std/vmod_std.c
index bedbd7d..60852b6 100644
--- a/lib/libvmod_std/vmod_std.c
+++ b/lib/libvmod_std/vmod_std.c
@@ -226,139 +226,97 @@ vmod_timestamp(const struct vrt_ctx *ctx, VCL_STRING label)
 	}
 }
 
+/*
+ * Boltsort
+ * Author: Naren Venkataraman of Vimeo Inc.
+ * Included here with permission.
+ */
 
-/* Boltsort
-   Author: Naren Venkataraman of Vimeo Inc.
+#define QS_MAX_PARAM_COUNT	32
+#define QS_EQUALS(a, b)	\
+    ((a) == (b) || ((a) == '\0' && (b) == '&') || ((a) == '&' && (b) == '\0'))
 
-   Included here with permission.
-*/
+static ssize_t
+param_compare(char *s, char *t)
+{
+	for (; QS_EQUALS(*s, *t); s++, t++) {
+		if (*s == '&' || s == '\0')
+			return (0);
+	}
+	return (*s - *t);
+}
 
+static size_t
+param_copy(char *dst, char *src)
+{
+	size_t len;
+	len = strcspn(src, "&");
+	memcpy(dst, src, len);
+	return (len);
+}
 
-#define QS_MAX_PARAM_COUNT 32
-#define QS_EQUALS(c, h) ((c == h) || (c == '\0' && h == '&') || (c == '&' && h == '\0'))
-#define QS_ENDS(s) (s == '&' || s == '\0') 
+VCL_STRING __match_proto__(td_std_querysort)
+vmod_querysort(const struct vrt_ctx *ctx, VCL_STRING url)
+{
+	char *param, *params[QS_MAX_PARAM_COUNT];
+	char *p, *r;
+	size_t len;
+	int param_count;
+	int i, n;
 
-static const char QS_TERMINATORS[2] = {'\0', '&'};
+	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
 
-//since we dont store param length, we have to evaluate everytime
-static inline int param_compare (char *s, char *t)
-{
+	if (url == NULL)
+		return (NULL);
 
-    for ( ;QS_EQUALS(*s, *t); s++, t++) {
-        if (QS_ENDS(*s)) {
-            return 0;
-        }
-    }
-    return *s - *t;
+	p = strchr(url, '?');
+	if (p == NULL)
+		return (url);
 
-}
+	param_count = 0;
+	params[param_count++] = ++p;
+	len = p - url;
 
-//end of param is either first occurance of & or '\0'
-static inline int param_copy(char *dst, char *src, char *last_param)
-{
+	while ((p = strchr(p, '&')) != NULL) {
+		param = ++p;
 
-    int len = strchr(src, QS_TERMINATORS[(src != last_param)]) - src;
-    memcpy(dst, src, len);
-    return len;
+		for (i = 0; i < param_count; i++) {
+			if (param[0] < params[i][0] ||
+			    param_compare(param, params[i]) < 0) {
+				for (n = param_count; n > i; n--)
+					params[n] = params[n - 1];
+				break;
+			}
+		}
+		params[i] = param;
+		param_count++;
 
-}
+		if (param_count == QS_MAX_PARAM_COUNT)
+			return (url);
+	}
 
-//Varnish vmod requires this
-int init_function(struct vmod_priv *priv, const struct VCL_conf *conf)
-{
-    return 0;
+	if (param_count == 1)
+		return (url);
 
-}
+	r = WS_Alloc(ctx->ws, strchr(param, '\0') - url + 1);
+	if (r == NULL)
+		return (url);
 
-//sort query string
-VCL_STRING vmod_querysort(const struct vrt_ctx * ctx, VCL_STRING url)
-{
+	p = memcpy(r, url, len);
+	p += len;
+
+	for (i = 0; i < param_count - 1; i++) {
+		if (params[i][0] != '\0' && params[i][0] != '&')
+			break;
+	}
+
+	for (; i < param_count - 1; i++) {
+		p += param_copy(p, params[i]);
+		*p++ = '&';
+	}
 
-    if (url == NULL) {
-        return NULL;
-    }
-
-    int qs_index = 0;
-    int param_count = 0;
-
-    char *dst_url = NULL;
-    char *qs = NULL;
-
-    //To avoid 1 pass for count calculations, assuming MAX_PARAM_COUNT as max
-    char* params[QS_MAX_PARAM_COUNT];
-
-    int i, p;
-    char *param = NULL;
-
-    qs = strchr(url, '?');
-    if(!qs) {
-        return url;
-    }
-
-    //add first param and increment count
-    params[param_count++] = ++qs;
-    qs_index = qs - url;
-
-    //Continue to find query string
-    while((qs = strchr(qs, '&')) != NULL) {
-        param = ++qs;
-
-        for(p = 0; p < param_count; p++) {
-            //if incoming param is < param at position then place it at p and then move up rest
-            if(param[0] < params[p][0] || param_compare(param, params[p]) < 0) {
-                for(i = param_count; i > p; i--) {
-                    params[i] = params[i-1];
-                }
-                break;
-            }
-        }
-        params[p] = param;
-        param_count++;
-
-        //if it exceed max params return as is
-        if (param_count == QS_MAX_PARAM_COUNT) {
-            return url;
-        }
-    }
-
-    //there is nothing after & 
-    //eg: http://127.0.0.1/?me=1&
-    if (param_count == 1) {
-        return url;
-    }
-
-    //allocate space for sorted url
-    //    struct ws *ws = sp->wrk->ws; 
-    struct ws *ws = ctx->ws;
-    dst_url = WS_Alloc(ws, strchr(param, '\0') - url + 1);
-    WS_Assert(ws);
-
-    //if alloc fails return as is
-    if(dst_url == NULL) {
-        return url;
-    }
-
-    //copy data before query string
-    char* cp = memcpy(dst_url, url, qs_index) + qs_index;
-
-    //get rid of all empty params /test/?a&&&
-    for(p = 0; p < param_count - 1; p++) { 
-        if (params[p][0] != '\0' && params[p][0] != '&') {
-            break;
-        }
-    }
-
-    //copy sorted params
-    for(; p < param_count - 1; p++) {
-        //copy and increment
-        cp += param_copy(cp, params[p], param);
-        *cp++ = '&';
-    }
-
-    //copy the last param
-    cp += param_copy(cp, params[p], param);
-    *cp = '\0';
-
-    return dst_url;
+	p += param_copy(p, params[i]);
+	*p = '\0';
 
+	return (r);
 }



From fgsch at lodoss.net  Tue Jun 24 09:31:57 2014
From: fgsch at lodoss.net (Federico G. Schwindt)
Date: Tue, 24 Jun 2014 11:31:57 +0200
Subject: [4.0] b61800b Add a string to real conversion function
Message-ID: <E1WzN53-0003SX-3Z@project.varnish-software.com>


commit b61800b639eb853faeebb3587592eb37006df199
Author: Federico G. Schwindt <fgsch at lodoss.net>
Date:   Sun Jun 22 09:49:35 2014 +0100

    Add a string to real conversion function
    
    As discussed on the latest vdd.

diff --git a/bin/varnishtest/tests/m00015.vtc b/bin/varnishtest/tests/m00015.vtc
new file mode 100644
index 0000000..077bbaf
--- /dev/null
+++ b/bin/varnishtest/tests/m00015.vtc
@@ -0,0 +1,23 @@
+varnishtest "Test vmod_std.real conversion"
+
+server s1 {
+	rxreq
+	txresp
+} -start
+
+varnish v1 -vcl+backend {
+	import ${vmod_std};
+
+	sub vcl_deliver {
+		set resp.http.x-real = std.real(req.http.foo, 0.0);
+		set resp.http.x-fallback = std.real(req.http.bar, 0.0);
+	}
+} -start
+
+client c1 {
+	txreq -hdr "foo: 1.00"
+	rxresp
+	expect resp.status == 200
+	expect resp.http.x-real == 1.000
+	expect resp.http.x-fallback == 0.000
+} -run
diff --git a/lib/libvmod_std/vmod.vcc b/lib/libvmod_std/vmod.vcc
index 0be272e..cc3eb43 100644
--- a/lib/libvmod_std/vmod.vcc
+++ b/lib/libvmod_std/vmod.vcc
@@ -135,6 +135,14 @@ Description
 Example
 	if (std.ip(req.http.X-forwarded-for, "0.0.0.0") ~ my_acl) { ... }
 
+$Function REAL real(STRING, REAL)
+
+Description
+	Converts the string *s* to a real.  If *s* fails to parse,
+	*fallback* will be returned.
+Example
+	set req.http.x-real = std.real(req.http.x-foo, 0.0);
+
 $Function BOOL healthy(BACKEND)
 
 Description
diff --git a/lib/libvmod_std/vmod_std_conversions.c b/lib/libvmod_std/vmod_std_conversions.c
index f98b5fb..7c1ac08 100644
--- a/lib/libvmod_std/vmod_std_conversions.c
+++ b/lib/libvmod_std/vmod_std_conversions.c
@@ -168,3 +168,33 @@ vmod_ip(const struct vrt_ctx *ctx, VCL_STRING s, VCL_IP d)
 		freeaddrinfo(res0);
 	return (r);
 }
+
+VCL_REAL __match_proto__()
+vmod_real(const struct vrt_ctx *ctx, VCL_STRING p, VCL_REAL d)
+{
+	char *e;
+	double r;
+
+	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
+
+	if (p == NULL)
+		return (d);
+
+	while (isspace(*p))
+		p++;
+
+	if (*p != '+' && *p != '-' && !isdigit(*p))
+		return (d);
+
+	e = NULL;
+
+	r = strtod(p, &e);
+
+	if (!isfinite(r))
+		return (d);
+
+	if (e == NULL || *e != '\0')
+		return (d);
+
+	return (r);
+}



From phk at FreeBSD.org  Tue Jun 24 09:31:57 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 24 Jun 2014 11:31:57 +0200
Subject: [4.0] 07d0bc4 Add a couple of 'const' and a missing star
Message-ID: <E1WzN53-0003Sg-4k@project.varnish-software.com>


commit 07d0bc40e4a1d5d3e8754cafa65cf86ecfdb3de2
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Jun 23 06:57:47 2014 +0000

    Add a couple of 'const' and a missing star

diff --git a/lib/libvmod_std/vmod_std.c b/lib/libvmod_std/vmod_std.c
index 60852b6..98ed54d 100644
--- a/lib/libvmod_std/vmod_std.c
+++ b/lib/libvmod_std/vmod_std.c
@@ -237,17 +237,17 @@ vmod_timestamp(const struct vrt_ctx *ctx, VCL_STRING label)
     ((a) == (b) || ((a) == '\0' && (b) == '&') || ((a) == '&' && (b) == '\0'))
 
 static ssize_t
-param_compare(char *s, char *t)
+param_compare(const char *s, const char *t)
 {
 	for (; QS_EQUALS(*s, *t); s++, t++) {
-		if (*s == '&' || s == '\0')
+		if (*s == '&' || *s == '\0')
 			return (0);
 	}
 	return (*s - *t);
 }
 
 static size_t
-param_copy(char *dst, char *src)
+param_copy(char *dst, const char *src)
 {
 	size_t len;
 	len = strcspn(src, "&");



From phk at FreeBSD.org  Tue Jun 24 09:31:57 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 24 Jun 2014 11:31:57 +0200
Subject: [4.0] b61c230 Always expire the ims-candidate object on a successful
 fetch
Message-ID: <E1WzN53-0003Sp-63@project.varnish-software.com>


commit b61c2305462c9428225a9531e664907db408fd7f
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Jun 23 07:36:23 2014 +0000

    Always expire the ims-candidate object on a successful fetch
    
    Fixes	#1530

diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c
index c175e74..a768170 100644
--- a/bin/varnishd/cache/cache_fetch.c
+++ b/bin/varnishd/cache/cache_fetch.c
@@ -557,6 +557,8 @@ vbf_stp_fetch(struct worker *wrk, struct busyobj *bo)
 
 	VBO_setstate(bo, BOS_FINISHED);
 	VSLb_ts_busyobj(bo, "BerespBody", W_TIM_real(wrk));
+	if (bo->ims_obj != NULL)
+		EXP_Rearm(bo->ims_obj, bo->ims_obj->exp.t_origin, 0, 0, 0);
 	return (F_STP_DONE);
 }
 



From phk at FreeBSD.org  Tue Jun 24 09:31:57 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 24 Jun 2014 11:31:57 +0200
Subject: [4.0] affd96d Generalize v1f_pull_chunked() into HTTP1_Chunked()
 which can also be used on requests.
Message-ID: <E1WzN53-0003Sy-7w@project.varnish-software.com>


commit affd96d46f7be3e11b2db6747b13271ea9f63246
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Jun 23 09:13:25 2014 +0000

    Generalize v1f_pull_chunked() into HTTP1_Chunked() which can also be
    used on requests.

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index 38e597b..2b10f59 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -859,6 +859,14 @@ int HTTP1_CacheReqBody(struct req *req, ssize_t maxsize);
 int HTTP1_IterateReqBody(struct req *req, req_body_iter_f *func, void *priv);
 extern const int HTTP1_Req[3];
 extern const int HTTP1_Resp[3];
+enum http1_chunked_ret {
+	H1CR_ERROR,
+	H1CR_MORE,
+	H1CR_END,
+};
+enum http1_chunked_ret
+HTTP1_Chunked(struct http_conn *htc, intptr_t *priv, const char **error,
+    int64_t *statp, void *ptr, ssize_t *lp);
 
 /* cache_http1_deliver.c */
 unsigned V1D_FlushReleaseAcct(struct req *req);
diff --git a/bin/varnishd/cache/cache_http1_fetch.c b/bin/varnishd/cache/cache_http1_fetch.c
index 1c3ef53..aa009f2 100644
--- a/bin/varnishd/cache/cache_http1_fetch.c
+++ b/bin/varnishd/cache/cache_http1_fetch.c
@@ -40,7 +40,6 @@
 
 #include "cache_backend.h"
 #include "vcli_priv.h"
-#include "vct.h"
 #include "vtcp.h"
 #include "vtim.h"
 
@@ -110,10 +109,7 @@ v1f_pull_straight(struct busyobj *bo, void *p, ssize_t *lp, intptr_t *priv)
 static enum vfp_status __match_proto__(vfp_pull_f)
 v1f_pull_chunked(struct busyobj *bo, void *p, ssize_t *lp, intptr_t *priv)
 {
-	int i;
-	char buf[20];		/* XXX: 20 is arbitrary */
-	unsigned u;
-	ssize_t cl, l, lr;
+	const char *err;
 
 	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
 	if (p == vfp_init)
@@ -123,77 +119,18 @@ v1f_pull_chunked(struct busyobj *bo, void *p, ssize_t *lp, intptr_t *priv)
 	AN(p);
 	AN(lp);
 	AN(priv);
-	l = *lp;
-	*lp = 0;
-	if (*priv == -1) {
-		/* Skip leading whitespace */
-		do {
-			lr = HTTP1_Read(&bo->htc, buf, 1);
-			if (lr <= 0)
-				return (VFP_Error(bo, "chunked read err"));
-			bo->acct.beresp_bodybytes += lr;
-		} while (vct_islws(buf[0]));
-
-		if (!vct_ishex(buf[0]))
-			 return (VFP_Error(bo, "chunked header non-hex"));
-
-		/* Collect hex digits, skipping leading zeros */
-		for (u = 1; u < sizeof buf; u++) {
-			do {
-				lr = HTTP1_Read(&bo->htc, buf + u, 1);
-				if (lr <= 0)
-					return (VFP_Error(bo,
-					    "chunked read err"));
-				bo->acct.beresp_bodybytes += lr;
-			} while (u == 1 && buf[0] == '0' && buf[u] == '0');
-			if (!vct_ishex(buf[u]))
-				break;
-		}
-
-		if (u >= sizeof buf)
-			return (VFP_Error(bo,"chunked header too long"));
-
-		/* Skip trailing white space */
-		while(vct_islws(buf[u]) && buf[u] != '\n') {
-			lr = HTTP1_Read(&bo->htc, buf + u, 1);
-			if (lr <= 0)
-				return (VFP_Error(bo, "chunked read err"));
-			bo->acct.beresp_bodybytes += lr;
-		}
 
-		if (buf[u] != '\n')
-			return (VFP_Error(bo,"chunked header no NL"));
-
-		buf[u] = '\0';
-
-		cl = vbf_fetch_number(buf, 16);
-		if (cl < 0)
-			return (VFP_Error(bo,"chunked header number syntax"));
-		*priv = cl;
-	}
-	if (*priv > 0) {
-		if (*priv < l)
-			l = *priv;
-		lr = HTTP1_Read(&bo->htc, p, l);
-		if (lr <= 0)
-			return (VFP_Error(bo, "straight insufficient bytes"));
-		bo->acct.beresp_bodybytes += lr;
-		*lp = lr;
-		*priv -= lr;
-		if (*priv == 0)
-			*priv = -1;
+	switch (HTTP1_Chunked(&bo->htc, priv, &err,
+	    &bo->acct.beresp_bodybytes, p, lp)) {
+	case H1CR_ERROR:
+		return (VFP_Error(bo, "%s", err));
+	case H1CR_MORE:
 		return (VFP_OK);
+	case H1CR_END:
+		return (VFP_END);
+	default:
+		WRONG("invalid HTTP1_Chunked return");
 	}
-	AZ(*priv);
-	i = HTTP1_Read(&bo->htc, buf, 1);
-	if (i <= 0)
-		return (VFP_Error(bo, "chunked read err"));
-	bo->acct.beresp_bodybytes += i;
-	if (buf[0] == '\r' && HTTP1_Read(&bo->htc, buf, 1) <= 0)
-		return (VFP_Error(bo, "chunked read err"));
-	if (buf[0] != '\n')
-		return (VFP_Error(bo,"chunked tail no NL"));
-	return (VFP_END);
 }
 
 /*--------------------------------------------------------------------*/
diff --git a/bin/varnishd/cache/cache_http1_proto.c b/bin/varnishd/cache/cache_http1_proto.c
index 7f7852c..0dc5fe8 100644
--- a/bin/varnishd/cache/cache_http1_proto.c
+++ b/bin/varnishd/cache/cache_http1_proto.c
@@ -45,6 +45,8 @@
 
 #include "config.h"
 
+#include <inttypes.h>
+
 #include "cache.h"
 
 #include "vct.h"
@@ -524,3 +526,106 @@ HTTP1_Write(const struct worker *w, const struct http *hp, const int *hf)
 	l += WRW_Write(w, "\r\n", -1);
 	return (l);
 }
+
+/*--------------------------------------------------------------------
+ * Read a chunked body.
+ *
+ * XXX: Reading one byte at a time is pretty pessimal.
+ */
+
+enum http1_chunked_ret
+HTTP1_Chunked(struct http_conn *htc, intptr_t *priv, const char **error,
+    int64_t *statp, void *ptr, ssize_t *lp)
+{
+	int i;
+	char buf[20];		/* XXX: 20 is arbitrary */
+	char *q;
+	unsigned u;
+	uintmax_t cll;
+	ssize_t cl, l, lr;
+
+#define  ERR(x) do { *error = (x); return (H1CR_ERROR); } while (0)
+
+	CHECK_OBJ_NOTNULL(htc, HTTP_CONN_MAGIC);
+	AN(priv);
+	AN(error);
+	AN(statp);
+	AN(ptr);
+	AN(lp);
+	l = *lp;
+	*lp = 0;
+	if (*priv == -1) {
+		/* Skip leading whitespace */
+		do {
+			lr = HTTP1_Read(htc, buf, 1);
+			if (lr <= 0)
+				ERR("chunked read err");
+			*statp += lr;
+		} while (vct_islws(buf[0]));
+
+		if (!vct_ishex(buf[0]))
+			 ERR("chunked header non-hex");
+
+		/* Collect hex digits, skipping leading zeros */
+		for (u = 1; u < sizeof buf; u++) {
+			do {
+				lr = HTTP1_Read(htc, buf + u, 1);
+				if (lr <= 0)
+					ERR("chunked read err");
+				*statp += lr;
+			} while (u == 1 && buf[0] == '0' && buf[u] == '0');
+			if (!vct_ishex(buf[u]))
+				break;
+		}
+
+		if (u >= sizeof buf)
+			ERR("chunked header too long");
+
+		/* Skip trailing white space */
+		while(vct_islws(buf[u]) && buf[u] != '\n') {
+			lr = HTTP1_Read(htc, buf + u, 1);
+			if (lr <= 0)
+				ERR("chunked read err");
+			*statp += lr;
+		}
+
+		if (buf[u] != '\n')
+			ERR("chunked header no NL");
+
+		buf[u] = '\0';
+
+		cll = strtoumax(buf, &q, 16);
+		if (q == NULL || *q != '\0')
+			ERR("chunked header number syntax");
+		cl = (ssize_t)cll;
+		if((uintmax_t)cl != cll)
+			ERR("bogusly large chunk size");
+
+		*priv = cl;
+	}
+	if (*priv > 0) {
+		if (*priv < l)
+			l = *priv;
+		lr = HTTP1_Read(htc, ptr, l);
+		if (lr <= 0)
+			ERR("straight insufficient bytes");
+		*statp += lr;
+		*lp = lr;
+		*priv -= lr;
+		if (*priv == 0)
+			*priv = -1;
+		return (H1CR_MORE);
+	}
+	AZ(*priv);
+	i = HTTP1_Read(htc, buf, 1);
+	if (i <= 0)
+		ERR("chunked read err");
+	*statp += i;
+	if (buf[0] == '\r' && HTTP1_Read(htc, buf, 1) <= 0)
+		ERR("chunked read err");
+	if (buf[0] != '\n')
+		ERR("chunked tail no NL");
+	return (H1CR_END);
+#undef ERR
+}
+



From phk at FreeBSD.org  Tue Jun 24 09:31:57 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 24 Jun 2014 11:31:57 +0200
Subject: [4.0] b4b628f Implement proper handling of chunked encoding of
 req.body.
Message-ID: <E1WzN53-0003T8-9H@project.varnish-software.com>


commit b4b628fa01591d50b2ce09e570ecd5470477cc65
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Jun 23 09:53:21 2014 +0000

    Implement proper handling of chunked encoding of req.body.
    
    Fixes	#1524

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index 2b10f59..dc7e6f1 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -236,7 +236,7 @@ struct acct_req {
 /*--------------------------------------------------------------------*/
 
 struct acct_bereq {
-#define ACCT(foo)	ssize_t		foo;
+#define ACCT(foo)	uint64_t	foo;
 #include "tbl/acct_fields_bereq.h"
 #undef ACCT
 };
@@ -665,6 +665,7 @@ struct req {
 
 	unsigned char		wantbody;
 	uint64_t		req_bodybytes;	/* Parsed req bodybytes */
+	intptr_t		chunk_ctr;	/* Parsed req bodybytes */
 
 	uint64_t		resp_hdrbytes;	/* Scheduled resp hdrbytes */
 	uint64_t		resp_bodybytes; /* Scheduled resp bodybytes */
@@ -866,7 +867,7 @@ enum http1_chunked_ret {
 };
 enum http1_chunked_ret
 HTTP1_Chunked(struct http_conn *htc, intptr_t *priv, const char **error,
-    int64_t *statp, void *ptr, ssize_t *lp);
+    uint64_t *statp, void *ptr, ssize_t *lp);
 
 /* cache_http1_deliver.c */
 unsigned V1D_FlushReleaseAcct(struct req *req);
@@ -1123,7 +1124,7 @@ void WRW_Chunked(const struct worker *w);
 void WRW_EndChunk(const struct worker *w);
 void WRW_Reserve(struct worker *w, int *fd, struct vsl_log *, double t0);
 unsigned WRW_Flush(const struct worker *w);
-unsigned WRW_FlushRelease(struct worker *w, ssize_t *pacc);
+unsigned WRW_FlushRelease(struct worker *w, uint64_t *pacc);
 unsigned WRW_Write(const struct worker *w, const void *ptr, int len);
 unsigned WRW_WriteH(const struct worker *w, const txt *hh, const char *suf);
 
diff --git a/bin/varnishd/cache/cache_http1_deliver.c b/bin/varnishd/cache/cache_http1_deliver.c
index 8778303..1bdcef7 100644
--- a/bin/varnishd/cache/cache_http1_deliver.c
+++ b/bin/varnishd/cache/cache_http1_deliver.c
@@ -211,7 +211,7 @@ unsigned
 V1D_FlushReleaseAcct(struct req *req)
 {
 	unsigned u;
-	ssize_t txcnt = 0, hdrbytes;
+	uint64_t txcnt = 0, hdrbytes;
 
 	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
 	CHECK_OBJ_NOTNULL(req->wrk, WORKER_MAGIC);
diff --git a/bin/varnishd/cache/cache_http1_fsm.c b/bin/varnishd/cache/cache_http1_fsm.c
index 9191d1c..dac34e0 100644
--- a/bin/varnishd/cache/cache_http1_fsm.c
+++ b/bin/varnishd/cache/cache_http1_fsm.c
@@ -285,6 +285,8 @@ http1_req_body_status(struct req *req)
 		req->h1.bytes_yet = req->req_bodybytes - req->h1.bytes_done;
 		return (REQ_BODY_PRESENT);
 	}
+	if (http_HdrIs(req->http, H_Transfer_Encoding, "chunked"))
+		return (REQ_BODY_CHUNKED);
 	if (http_GetHdr(req->http, H_Transfer_Encoding, NULL))
 		return (REQ_BODY_FAIL);
 	return (REQ_BODY_NONE);
@@ -480,28 +482,46 @@ HTTP1_Session(struct worker *wrk, struct req *req)
 }
 
 static ssize_t
-http1_iter_req_body(struct req *req, void *buf, ssize_t len)
+http1_iter_req_body(struct req *req, enum req_body_state_e bs,
+    void *buf, ssize_t len)
 {
+	const char *err;
+
 	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
 
-	AN(req->req_bodybytes);
 	AN(len);
 	AN(buf);
-	if (len > req->req_bodybytes - req->h1.bytes_done)
-		len = req->req_bodybytes - req->h1.bytes_done;
-	if (len == 0) {
-		req->req_body_status = REQ_BODY_DONE;
-		return (0);
-	}
-	len = HTTP1_Read(req->htc, buf, len);
-	if (len <= 0) {
-		req->req_body_status = REQ_BODY_FAIL;
-		return (-1);
-	}
-	req->h1.bytes_done += len;
-	req->h1.bytes_yet = req->req_bodybytes - req->h1.bytes_done;
-	req->acct.req_bodybytes += len;
-	return (len);
+	if (bs == REQ_BODY_PRESENT) {
+		AN(req->req_bodybytes);
+		if (len > req->req_bodybytes - req->h1.bytes_done)
+			len = req->req_bodybytes - req->h1.bytes_done;
+		if (len == 0) {
+			req->req_body_status = REQ_BODY_DONE;
+			return (0);
+		}
+		len = HTTP1_Read(req->htc, buf, len);
+		if (len <= 0) {
+			req->req_body_status = REQ_BODY_FAIL;
+			return (-1);
+		}
+		req->h1.bytes_done += len;
+		req->h1.bytes_yet = req->req_bodybytes - req->h1.bytes_done;
+		req->acct.req_bodybytes += len;
+		return (len);
+	} else if (bs == REQ_BODY_CHUNKED) {
+		switch (HTTP1_Chunked(req->htc, &req->chunk_ctr, &err,
+		    &req->acct.req_bodybytes, buf, &len)) {
+		case H1CR_ERROR:
+			return (-1);
+		case H1CR_MORE:
+			return (len);
+		case H1CR_END:
+			return (0);
+		default:
+			WRONG("invalid HTTP1_Chunked return");
+		}
+	} else
+		WRONG("Illegal req_bodystatus");
 }
 
 /*----------------------------------------------------------------------
@@ -516,6 +536,7 @@ HTTP1_IterateReqBody(struct req *req, req_body_iter_f *func, void *priv)
 {
 	char buf[8192];
 	struct storage *st;
+	enum req_body_state_e bs;
 	ssize_t l;
 	int i;
 
@@ -533,6 +554,7 @@ HTTP1_IterateReqBody(struct req *req, req_body_iter_f *func, void *priv)
 	case REQ_BODY_NONE:
 		return (0);
 	case REQ_BODY_PRESENT:
+	case REQ_BODY_CHUNKED:
 		break;
 	case REQ_BODY_DONE:
 	case REQ_BODY_TAKEN:
@@ -543,7 +565,9 @@ HTTP1_IterateReqBody(struct req *req, req_body_iter_f *func, void *priv)
 		WRONG("Wrong req_body_status in HTTP1_IterateReqBody()");
 	}
 	Lck_Lock(&req->sp->mtx);
-	if (req->req_body_status == REQ_BODY_PRESENT) {
+	bs = req->req_body_status;
+	if (req->req_body_status == REQ_BODY_PRESENT ||
+	    req->req_body_status == REQ_BODY_CHUNKED) {
 		req->req_body_status = REQ_BODY_TAKEN;
 		i = 0;
 	} else
@@ -556,7 +580,7 @@ HTTP1_IterateReqBody(struct req *req, req_body_iter_f *func, void *priv)
 	}
 
 	do {
-		l = http1_iter_req_body(req, buf, sizeof buf);
+		l = http1_iter_req_body(req, bs, buf, sizeof buf);
 		if (l < 0) {
 			req->doclose = SC_RX_BODY;
 			break;
@@ -626,6 +650,7 @@ HTTP1_CacheReqBody(struct req *req, ssize_t maxsize)
 		return (-1);
 	case REQ_BODY_NONE:
 		return (0);
+	case REQ_BODY_CHUNKED:
 	case REQ_BODY_PRESENT:
 		break;
 	default:
@@ -653,7 +678,8 @@ HTTP1_CacheReqBody(struct req *req, ssize_t maxsize)
 		}
 
 		l = st->space - st->len;
-		l = http1_iter_req_body(req, st->ptr + st->len, l);
+		l = http1_iter_req_body(req, req->req_body_status,
+		    st->ptr + st->len, l);
 		if (l < 0) {
 			req->doclose = SC_RX_BODY;
 			break;
diff --git a/bin/varnishd/cache/cache_http1_proto.c b/bin/varnishd/cache/cache_http1_proto.c
index 0dc5fe8..a3e2005 100644
--- a/bin/varnishd/cache/cache_http1_proto.c
+++ b/bin/varnishd/cache/cache_http1_proto.c
@@ -535,7 +535,7 @@ HTTP1_Write(const struct worker *w, const struct http *hp, const int *hf)
 
 enum http1_chunked_ret
 HTTP1_Chunked(struct http_conn *htc, intptr_t *priv, const char **error,
-    int64_t *statp, void *ptr, ssize_t *lp)
+    uint64_t *statp, void *ptr, ssize_t *lp)
 {
 	int i;
 	char buf[20];		/* XXX: 20 is arbitrary */
diff --git a/bin/varnishd/cache/cache_pipe.c b/bin/varnishd/cache/cache_pipe.c
index 27ea964..36ef425 100644
--- a/bin/varnishd/cache/cache_pipe.c
+++ b/bin/varnishd/cache/cache_pipe.c
@@ -43,14 +43,14 @@
 static struct lock pipestat_mtx;
 
 struct acct_pipe {
-	ssize_t		req;
-	ssize_t		bereq;
-	ssize_t		in;
-	ssize_t		out;
+	uint64_t	req;
+	uint64_t	bereq;
+	uint64_t	in;
+	uint64_t	out;
 };
 
 static int
-rdf(int fd0, int fd1, ssize_t *pcnt)
+rdf(int fd0, int fd1, uint64_t *pcnt)
 {
 	int i, j;
 	char buf[BUFSIZ], *p;
diff --git a/bin/varnishd/cache/cache_wrw.c b/bin/varnishd/cache/cache_wrw.c
index 09c7e26..cc02bbf 100644
--- a/bin/varnishd/cache/cache_wrw.c
+++ b/bin/varnishd/cache/cache_wrw.c
@@ -102,7 +102,7 @@ WRW_Reserve(struct worker *wrk, int *fd, struct vsl_log *vsl, double t0)
 }
 
 static void
-wrw_release(struct worker *wrk, ssize_t *pacc)
+wrw_release(struct worker *wrk, uint64_t *pacc)
 {
 	struct wrw *wrw;
 
@@ -219,7 +219,7 @@ WRW_Flush(const struct worker *wrk)
 }
 
 unsigned
-WRW_FlushRelease(struct worker *wrk, ssize_t *pacc)
+WRW_FlushRelease(struct worker *wrk, uint64_t *pacc)
 {
 	unsigned u;
 
diff --git a/bin/varnishtest/tests/c00067.vtc b/bin/varnishtest/tests/c00067.vtc
new file mode 100644
index 0000000..b57f904
--- /dev/null
+++ b/bin/varnishtest/tests/c00067.vtc
@@ -0,0 +1,24 @@
+varnishtest "chunked req.body"
+
+server s1 {
+	rxreq
+	expect req.bodylen == 106
+	txresp -body "ABCD"
+} -start
+
+varnish v1 -vcl+backend {
+} -start
+
+client c1 {
+	txreq -req POST -nolen -hdr "Transfer-encoding: chunked"
+	chunked {BLA}
+	delay .2
+	chunkedlen 100
+	delay .2
+	chunked {FOO}
+	delay .2
+	chunkedlen 0
+	rxresp
+	expect resp.status == 200
+	expect resp.bodylen == 4
+} -run
diff --git a/bin/varnishtest/tests/r01524.vtc b/bin/varnishtest/tests/r01524.vtc
new file mode 100644
index 0000000..a845da3
--- /dev/null
+++ b/bin/varnishtest/tests/r01524.vtc
@@ -0,0 +1,19 @@
+varnishtest "pipe of chunked request"
+
+server s1 {
+	rxreq
+	expect req.bodylen == 3
+	txresp -body "ABCD"
+} -start
+
+varnish v1 -vcl+backend {
+} -start
+
+client c1 {
+	txreq -req MAGIC -nolen -hdr "Transfer-encoding: chunked"
+	chunked {BLA}
+	chunkedlen 0
+	rxresp
+	expect resp.status == 200
+	expect resp.bodylen == 4
+} -run



From phk at FreeBSD.org  Tue Jun 24 09:31:57 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 24 Jun 2014 11:31:57 +0200
Subject: [4.0] 0e9bd49 Opps, forgot to commit from top of tree
Message-ID: <E1WzN53-0003TI-CG@project.varnish-software.com>


commit 0e9bd4998cd8f8abda85e45d697af16d8dbbe125
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Jun 23 10:16:14 2014 +0000

    Opps, forgot to commit from top of tree

diff --git a/include/tbl/req_body.h b/include/tbl/req_body.h
index a649bf2..83744cc 100644
--- a/include/tbl/req_body.h
+++ b/include/tbl/req_body.h
@@ -31,6 +31,7 @@
 
 REQ_BODY(INIT)
 REQ_BODY(PRESENT)
+REQ_BODY(CHUNKED)
 REQ_BODY(TAKEN)
 REQ_BODY(CACHED)
 REQ_BODY(DONE)



From lkarsten at varnish-software.com  Tue Jun 24 09:31:57 2014
From: lkarsten at varnish-software.com (Lasse Karstensen)
Date: Tue, 24 Jun 2014 11:31:57 +0200
Subject: [4.0] df2446b Add libedit dependency.
Message-ID: <E1WzN53-0003TR-DW@project.varnish-software.com>


commit df2446b0f6b7be06f775244746ae16ad6b2e50dd
Author: Lasse Karstensen <lkarsten at varnish-software.com>
Date:   Mon Jun 23 12:24:19 2014 +0200

    Add libedit dependency.
    
    Fixes: 	#1509

diff --git a/redhat/varnish.spec b/redhat/varnish.spec
index b50c859..338aa96 100644
--- a/redhat/varnish.spec
+++ b/redhat/varnish.spec
@@ -23,6 +23,7 @@ Requires: varnish-libs = %{version}-%{release}
 Requires: logrotate
 Requires: ncurses
 Requires: pcre
+Requires: libedit
 Requires: jemalloc
 Requires(pre): shadow-utils
 Requires(post): /sbin/chkconfig, /usr/bin/uuidgen



From phk at FreeBSD.org  Tue Jun 24 09:31:57 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 24 Jun 2014 11:31:57 +0200
Subject: [4.0] ac94f16 Now really make chunked req.body work with pass.
Message-ID: <E1WzN53-0003Ta-Fb@project.varnish-software.com>


commit ac94f1662761f9d9b7e3f527a08476a5cddb7dfc
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Jun 23 12:29:33 2014 +0000

    Now really make chunked req.body work with pass.
    
    Please test this ASAP (before 4.0.1)

diff --git a/bin/varnishd/builtin.vcl b/bin/varnishd/builtin.vcl
index 30fcaea..cfb2238 100644
--- a/bin/varnishd/builtin.vcl
+++ b/bin/varnishd/builtin.vcl
@@ -60,12 +60,6 @@ sub vcl_recv {
         return (pipe);
     }
 
-    /* We don't support chunked uploads, except when piping. */
-    if ((req.method == "POST" || req.method == "PUT") &&
-      req.http.transfer-encoding ~ "chunked") {
-        return(pipe);
-    }
-
     if (req.method != "GET" && req.method != "HEAD") {
         /* We only deal with GET and HEAD by default */
         return (pass);
diff --git a/bin/varnishd/cache/cache_http1_fetch.c b/bin/varnishd/cache/cache_http1_fetch.c
index aa009f2..b6d72dd 100644
--- a/bin/varnishd/cache/cache_http1_fetch.c
+++ b/bin/varnishd/cache/cache_http1_fetch.c
@@ -192,6 +192,31 @@ V1F_Setup_Fetch(struct busyobj *bo)
 }
 
 /*--------------------------------------------------------------------
+ * Pass the request body to the backend with chunks
+ */
+
+static int __match_proto__(req_body_iter_f)
+vbf_iter_req_body_chunked(struct req *req, void *priv, void *ptr, size_t l)
+{
+	struct worker *wrk;
+	char buf[20];
+
+	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
+	CAST_OBJ_NOTNULL(wrk, priv, WORKER_MAGIC);
+
+	if (l > 0) {
+		bprintf(buf, "%jx\r\n", (uintmax_t)l);
+		VSLb(req->vsl, SLT_Debug, "WWWW: %s", buf);
+		(void)WRW_Write(wrk, buf, strlen(buf));
+		(void)WRW_Write(wrk, ptr, l);
+		(void)WRW_Write(wrk, "\r\n", 2);
+		if (WRW_Flush(wrk))
+			return (-1);
+	}
+	return (0);
+}
+
+/*--------------------------------------------------------------------
  * Pass the request body to the backend
  */
 
@@ -231,6 +256,7 @@ V1F_fetch_hdr(struct worker *wrk, struct busyobj *bo, struct req *req)
 	int i, j, first;
 	struct http_conn *htc;
 	ssize_t hdrbytes;
+	int do_chunked = 0;
 
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 	CHECK_OBJ_ORNULL(req, REQ_MAGIC);
@@ -262,6 +288,11 @@ V1F_fetch_hdr(struct worker *wrk, struct busyobj *bo, struct req *req)
 	if (!http_GetHdr(bo->bereq, H_Host, NULL))
 		VDI_AddHostHeader(bo->bereq, vc);
 
+	if (req != NULL && req->req_body_status == REQ_BODY_CHUNKED) {
+		http_PrintfHeader(hp, "Transfer-Encoding: chunked");
+		do_chunked = 1;
+	}
+
 	(void)VTCP_blocking(vc->fd);	/* XXX: we should timeout instead */
 	WRW_Reserve(wrk, &vc->fd, bo->vsl, bo->t_prev);
 	hdrbytes = HTTP1_Write(wrk, hp, HTTP1_Req);
@@ -270,10 +301,16 @@ V1F_fetch_hdr(struct worker *wrk, struct busyobj *bo, struct req *req)
 	i = 0;
 
 	if (req != NULL) {
-		i = HTTP1_IterateReqBody(req, vbf_iter_req_body, wrk);
-		if (req->req_body_status == REQ_BODY_DONE)
+		if (do_chunked) {
+			i = HTTP1_IterateReqBody(req,
+			    vbf_iter_req_body_chunked, wrk);
+			(void)WRW_Write(wrk, "0\r\n\r\n", 5);
+		} else {
+			i = HTTP1_IterateReqBody(req, vbf_iter_req_body, wrk);
+		}
+		if (req->req_body_status == REQ_BODY_DONE) {
 			retry = -1;
-		if (req->req_body_status == REQ_BODY_FAIL) {
+		} else if (req->req_body_status == REQ_BODY_FAIL) {
 			VSLb(bo->vsl, SLT_FetchError,
 			    "req.body read error: %d (%s)",
 			    errno, strerror(errno));
diff --git a/bin/varnishd/cache/cache_http1_fsm.c b/bin/varnishd/cache/cache_http1_fsm.c
index dac34e0..454ec28 100644
--- a/bin/varnishd/cache/cache_http1_fsm.c
+++ b/bin/varnishd/cache/cache_http1_fsm.c
@@ -285,8 +285,10 @@ http1_req_body_status(struct req *req)
 		req->h1.bytes_yet = req->req_bodybytes - req->h1.bytes_done;
 		return (REQ_BODY_PRESENT);
 	}
-	if (http_HdrIs(req->http, H_Transfer_Encoding, "chunked"))
+	if (http_HdrIs(req->http, H_Transfer_Encoding, "chunked")) {
+		req->chunk_ctr = -1;
 		return (REQ_BODY_CHUNKED);
+	}
 	if (http_GetHdr(req->http, H_Transfer_Encoding, NULL))
 		return (REQ_BODY_FAIL);
 	return (REQ_BODY_NONE);
@@ -512,6 +514,7 @@ http1_iter_req_body(struct req *req, enum req_body_state_e bs,
 		switch (HTTP1_Chunked(req->htc, &req->chunk_ctr, &err,
 		    &req->acct.req_bodybytes, buf, &len)) {
 		case H1CR_ERROR:
+			VSLb(req->vsl, SLT_Debug, "CHUNKERR: %s", err);
 			return (-1);
 		case H1CR_MORE:
 			return (len);
@@ -639,7 +642,7 @@ int
 HTTP1_CacheReqBody(struct req *req, ssize_t maxsize)
 {
 	struct storage *st;
-	ssize_t l;
+	ssize_t l, l2;
 
 	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
 
@@ -661,6 +664,7 @@ HTTP1_CacheReqBody(struct req *req, ssize_t maxsize)
 		req->req_body_status = REQ_BODY_FAIL;
 		return (-1);
 	}
+	l2 = 0;
 
 	st = NULL;
 	do {
@@ -690,13 +694,28 @@ HTTP1_CacheReqBody(struct req *req, ssize_t maxsize)
 			break;
 		}
 		if (l > 0) {
+			l2 += l;
 			st->len += l;
 			if (st->space == st->len)
 				st = NULL;
 		}
 	} while (l > 0);
-	if (l == 0)
+	if (l == 0) {
+		req->req_bodybytes = l2;
+		/* We must update also the "pristine" req.* copy */
+
+		http_Unset(req->http0, H_Content_Length);
+		http_Unset(req->http0, H_Transfer_Encoding);
+		http_PrintfHeader(req->http0, "Content-Length: %ju",
+		    req->req_bodybytes);
+
+		http_Unset(req->http, H_Content_Length);
+		http_Unset(req->http, H_Transfer_Encoding);
+		http_PrintfHeader(req->http, "Content-Length: %ju",
+		    req->req_bodybytes);
+
 		req->req_body_status = REQ_BODY_CACHED;
+	}
 	VSLb_ts_req(req, "ReqBody", VTIM_real());
 	return (l);
 }
diff --git a/bin/varnishd/cache/cache_http1_proto.c b/bin/varnishd/cache/cache_http1_proto.c
index a3e2005..f0a1abc 100644
--- a/bin/varnishd/cache/cache_http1_proto.c
+++ b/bin/varnishd/cache/cache_http1_proto.c
@@ -196,7 +196,7 @@ HTTP1_Read(struct http_conn *htc, void *d, size_t len)
 {
 	size_t l;
 	unsigned char *p;
-	ssize_t i;
+	ssize_t i = 0;
 
 	CHECK_OBJ_NOTNULL(htc, HTTP_CONN_MAGIC);
 	l = 0;
@@ -212,12 +212,12 @@ HTTP1_Read(struct http_conn *htc, void *d, size_t len)
 		if (htc->pipeline.b == htc->pipeline.e)
 			htc->pipeline.b = htc->pipeline.e = NULL;
 	}
-	if (len == 0)
-		return (l);
-	i = read(htc->fd, p, len);
-	if (i < 0) {
-		VSLb(htc->vsl, SLT_FetchError, "%s", strerror(errno));
-		return (i);
+	if (len > 0) {
+		i = read(htc->fd, p, len);
+		if (i < 0) {
+			VSLb(htc->vsl, SLT_FetchError, "%s", strerror(errno));
+			return (i);
+		}
 	}
 	return (i + l);
 }
diff --git a/bin/varnishtest/tests/c00055.vtc b/bin/varnishtest/tests/c00055.vtc
index e02bc47..0344d02 100644
--- a/bin/varnishtest/tests/c00055.vtc
+++ b/bin/varnishtest/tests/c00055.vtc
@@ -25,7 +25,7 @@ varnish v1 -cliok "param.set vcc_allow_inline_c true" -vcl+backend {
 varnish v1 -cliok "param.set debug +syncvsl"
 
 client c1 {
-	txreq -body "FOO"
+	txreq -req "POST" -body "FOO"
 	rxresp
 	expect resp.http.Foo == "Foo"
 	expect resp.bodylen == 2
diff --git a/bin/varnishtest/tests/c00067.vtc b/bin/varnishtest/tests/c00067.vtc
index b57f904..73bcb34 100644
--- a/bin/varnishtest/tests/c00067.vtc
+++ b/bin/varnishtest/tests/c00067.vtc
@@ -4,11 +4,16 @@ server s1 {
 	rxreq
 	expect req.bodylen == 106
 	txresp -body "ABCD"
+	rxreq
+	expect req.bodylen == 108
+	txresp -body "ABCDE"
 } -start
 
 varnish v1 -vcl+backend {
 } -start
 
+varnish v1 -cliok "param.set debug +syncvsl"
+
 client c1 {
 	txreq -req POST -nolen -hdr "Transfer-encoding: chunked"
 	chunked {BLA}
@@ -22,3 +27,25 @@ client c1 {
 	expect resp.status == 200
 	expect resp.bodylen == 4
 } -run
+
+delay .2
+
+varnish v1 -cliok "param.set vcc_allow_inline_c true" -vcl+backend {
+	sub vcl_recv {
+		C{ VRT_CacheReqBody(ctx, 1000); }C
+	}
+} 
+
+client c1 {
+	txreq -req POST -nolen -hdr "Transfer-encoding: chunked"
+	chunked {BLAS}
+	delay .2
+	chunkedlen 100
+	delay .2
+	chunked {TFOO}
+	delay .2
+	chunkedlen 0
+	rxresp
+	expect resp.status == 200
+	expect resp.bodylen == 5
+} -run
diff --git a/bin/varnishtest/vtc_http.c b/bin/varnishtest/vtc_http.c
index 966f043..28b7340 100644
--- a/bin/varnishtest/vtc_http.c
+++ b/bin/varnishtest/vtc_http.c
@@ -458,7 +458,7 @@ http_swallow_body(struct http *hp, char * const *hh, int body)
 		return;
 	}
 	p = http_find_header(hh, "transfer-encoding");
-	if (p != NULL && !strcmp(p, "chunked")) {
+	if (p != NULL && !strcasecmp(p, "chunked")) {
 		while (http_rxchunk(hp) != 0)
 			continue;
 		vtc_dump(hp->vl, 4, "body", hp->body, ll);



From martin at varnish-software.com  Tue Jun 24 09:31:57 2014
From: martin at varnish-software.com (Martin Blix Grydeland)
Date: Tue, 24 Jun 2014 11:31:57 +0200
Subject: [4.0] 542e5c7 Treat missing t_end in varnishncsa as being the same as
 t_start
Message-ID: <E1WzN53-0003Tk-I3@project.varnish-software.com>


commit 542e5c70fa394a8c6ddf13d3be66536bfd92f957
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Mon Jun 23 15:09:46 2014 +0200

    Treat missing t_end in varnishncsa as being the same as t_start
    
    This makes varnishncsa continue to log if the end timestamp should be missing for some reason.

diff --git a/bin/varnishncsa/varnishncsa.c b/bin/varnishncsa/varnishncsa.c
index 2ff1fc3..d26808e 100644
--- a/bin/varnishncsa/varnishncsa.c
+++ b/bin/varnishncsa/varnishncsa.c
@@ -50,6 +50,7 @@
 #include <stdint.h>
 #include <ctype.h>
 #include <time.h>
+#include <math.h>
 
 #include "base64.h"
 #include "vapi/vsm.h"
@@ -243,20 +244,27 @@ format_time(const struct format *format)
 	struct tm tm;
 
 	CHECK_OBJ_NOTNULL(format, FORMAT_MAGIC);
-	if (CTX.frag[F_tstart].gen != CTX.gen ||
-	    CTX.frag[F_tend].gen != CTX.gen) {
+	if (CTX.frag[F_tstart].gen == CTX.gen) {
+		t_start = strtod(CTX.frag[F_tstart].b, &p);
+		if (p != CTX.frag[F_tstart].e)
+			t_start = NAN;
+	} else
+		t_start = NAN;
+	if (isnan(t_start)) {
+		/* Missing t_start is a no go */
 		if (format->string == NULL)
 			return (-1);
 		AZ(VSB_cat(CTX.vsb, format->string));
 		return (0);
 	}
 
-	t_start = strtod(CTX.frag[F_tstart].b, &p);
-	if (p != CTX.frag[F_tstart].e)
-		return (-1);
-	t_end = strtod(CTX.frag[F_tend].b, &p);
-	if (p != CTX.frag[F_tend].e)
-		return (-1);
+	/* Missing t_end defaults to t_start */
+	if (CTX.frag[F_tend].gen == CTX.gen) {
+		t_end = strtod(CTX.frag[F_tend].b, &p);
+		if (p != CTX.frag[F_tend].e)
+			t_end = t_start;
+	} else
+		t_end = t_start;
 
 	switch (format->time_type) {
 	case 'D':



From apj at mutt.dk  Tue Jun 24 09:31:57 2014
From: apj at mutt.dk (Andreas Plesner)
Date: Tue, 24 Jun 2014 11:31:57 +0200
Subject: [4.0] 0b1fa10 Add varnishncsa logging of pipe requests
Message-ID: <E1WzN53-0003Tt-KH@project.varnish-software.com>


commit 0b1fa10ab261b6c0301d5b20fd13193a739bd978
Author: Andreas Plesner <apj at mutt.dk>
Date:   Mon Jun 23 15:32:11 2014 +0200

    Add varnishncsa logging of pipe requests
    
    Fixes #1269

diff --git a/bin/varnishncsa/varnishncsa.c b/bin/varnishncsa/varnishncsa.c
index d26808e..97cc5ea 100644
--- a/bin/varnishncsa/varnishncsa.c
+++ b/bin/varnishncsa/varnishncsa.c
@@ -558,7 +558,7 @@ parse_format(const char *format)
 			addf_requestline();
 			break;
 		case 's':	/* Status code */
-			addf_fragment(&CTX.frag[F_s], "");
+			addf_fragment(&CTX.frag[F_s], "-");
 			break;
 		case 't':	/* strftime */
 			addf_time(*p, TIME_FMT, NULL);
@@ -791,11 +791,13 @@ dispatch_f(struct VSL_data *vsl, struct VSL_transaction * const pt[],
 					frag_fields(p, e, 1,
 					    &CTX.frag[F_tstart], 0, NULL);
 
-				} else if (isprefix(b, "Resp:", e, &p)) {
+				} else if (isprefix(b, "Resp:", e, &p) ||
+					isprefix(b, "PipeSess:", e, &p)) {
 					frag_fields(p, e, 1,
 					    &CTX.frag[F_tend], 0, NULL);
 
-				} else if (isprefix(b, "Process:", e, &p)) {
+				} else if (isprefix(b, "Process:", e, &p) ||
+					isprefix(b, "Pipe:", e, &p)) {
 					frag_fields(p, e, 2,
 					    &CTX.frag[F_ttfb], 0, NULL);
 				}
@@ -826,14 +828,15 @@ dispatch_f(struct VSL_data *vsl, struct VSL_transaction * const pt[],
 					   wrong */
 					CTX.hitmiss = "miss";
 					CTX.handling = "error";
-				} else if (!strcasecmp(b, "pipe")) {
-					CTX.hitmiss = "miss";
-					CTX.handling = "pipe";
 				}
 				break;
 			case SLT_VCL_return:
-				if (!strcasecmp(b, "restart"))
+				if (!strcasecmp(b, "restart")) {
 					skip = 1;
+				} else if (!strcasecmp(b, "pipe")) {
+					CTX.hitmiss = "miss";
+					CTX.handling = "pipe";
+				}
 				break;
 			default:
 				break;



From lkarsten at varnish-software.com  Tue Jun 24 09:31:57 2014
From: lkarsten at varnish-software.com (Lasse Karstensen)
Date: Tue, 24 Jun 2014 11:31:57 +0200
Subject: [4.0] 8ae27a9 Add Begin and Link VSL records for piped requests.
Message-ID: <E1WzN53-0003U2-MN@project.varnish-software.com>


commit 8ae27a9c73bbbd7ef360a5c241cf98d69dbd89a5
Author: Lasse Karstensen <lkarsten at varnish-software.com>
Date:   Mon Jun 23 15:43:28 2014 +0200

    Add Begin and Link VSL records for piped requests.
    
    This allows varnishlog and the other tools to group it automatically
    with the client request.

diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c
index c586ca4..401ebf1 100644
--- a/bin/varnishd/cache/cache_req_fsm.c
+++ b/bin/varnishd/cache/cache_req_fsm.c
@@ -593,6 +593,7 @@ cnt_pipe(struct worker *wrk, struct req *req)
 	wrk->stats.s_pipe++;
 	bo = VBO_GetBusyObj(wrk, req);
 	HTTP_Setup(bo->bereq, bo->ws, bo->vsl, SLT_BereqMethod);
+	VSLb(bo->vsl, SLT_Begin, "bereq %u pipe", req->vsl->wid & VSL_IDENTMASK);
 	http_FilterReq(bo->bereq, req->http, 0);	// XXX: 0 ?
 	http_PrintfHeader(bo->bereq,
 	    "X-Varnish: %u", req->vsl->wid & VSL_IDENTMASK);
@@ -604,6 +605,7 @@ cnt_pipe(struct worker *wrk, struct req *req)
 		INCOMPL();
 	assert(wrk->handling == VCL_RET_PIPE);
 
+	VSLb(req->vsl, SLT_Link, "bereq %u pipe", bo->vsl->wid & VSL_IDENTMASK);
 	PipeRequest(req, bo);
 	assert(WRW_IsReleased(wrk));
 	http_Teardown(bo->bereq);
diff --git a/include/vapi/vsl.h b/include/vapi/vsl.h
index a6e0839..45c1aaf 100644
--- a/include/vapi/vsl.h
+++ b/include/vapi/vsl.h
@@ -103,6 +103,7 @@ enum VSL_reason_e {
 	VSL_r_pass,
 	VSL_r_fetch,
 	VSL_r_bgfetch,
+	VSL_r_pipe,
 	VSL_r__MAX,
 };
 
diff --git a/lib/libvarnishapi/vsl_dispatch.c b/lib/libvarnishapi/vsl_dispatch.c
index 0502c36..7f393b3 100644
--- a/lib/libvarnishapi/vsl_dispatch.c
+++ b/lib/libvarnishapi/vsl_dispatch.c
@@ -65,6 +65,7 @@ static const char * const vsl_r_names[VSL_r__MAX] = {
 	[VSL_r_pass]	= "pass",
 	[VSL_r_fetch]	= "fetch",
 	[VSL_r_bgfetch]	= "bgfetch",
+	[VSL_r_pipe]	= "pipe",
 };
 
 struct vtx;



From martin at varnish-software.com  Tue Jun 24 09:31:57 2014
From: martin at varnish-software.com (Martin Blix Grydeland)
Date: Tue, 24 Jun 2014 11:31:57 +0200
Subject: [4.0] 44b2c1d Parse byte counters for piped requests in varnishncsa
Message-ID: <E1WzN53-0003UB-OK@project.varnish-software.com>


commit 44b2c1d0d210faa29158e0ef774676724e47ac4b
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Mon Jun 23 16:54:00 2014 +0200

    Parse byte counters for piped requests in varnishncsa

diff --git a/bin/varnishncsa/varnishncsa.c b/bin/varnishncsa/varnishncsa.c
index 97cc5ea..685d869 100644
--- a/bin/varnishncsa/varnishncsa.c
+++ b/bin/varnishncsa/varnishncsa.c
@@ -74,7 +74,7 @@ enum e_frag {
 	F_H,			/* %H Proto */
 	F_U,			/* %U URL path */
 	F_q,			/* %q Query string */
-	F_b,			/* %b Bytes */
+	F_b,			/* %b Body bytes sent */
 	F_h,			/* %h Host name / IP Address */
 	F_m,			/* %m Method */
 	F_s,			/* %s Status */
@@ -527,7 +527,7 @@ parse_format(const char *format)
 
 		p++;
 		switch (*p) {
-		case 'b':	/* Bytes */
+		case 'b':	/* Body bytes sent */
 			addf_fragment(&CTX.frag[F_b], "-");
 			break;
 		case 'D':	/* Float request time */
@@ -760,6 +760,12 @@ dispatch_f(struct VSL_data *vsl, struct VSL_transaction * const pt[],
 				e--;
 
 			switch (tag) {
+			case SLT_PipeAcct:
+				frag_fields(b, e,
+				    3, &CTX.frag[F_I],
+				    4, &CTX.frag[F_O],
+				    0, NULL);
+				break;
 			case SLT_ReqStart:
 				frag_fields(b, e, 1, &CTX.frag[F_h], 0, NULL);
 				break;



From fgsch at lodoss.net  Tue Jun 24 09:31:57 2014
From: fgsch at lodoss.net (Federico G. Schwindt)
Date: Tue, 24 Jun 2014 11:31:57 +0200
Subject: [4.0] 9e5b06f Move querysort into its own file
Message-ID: <E1WzN53-0003UK-Qu@project.varnish-software.com>


commit 9e5b06f94dab9c52089587724ae42d26d5fb6a30
Author: Federico G. Schwindt <fgsch at lodoss.net>
Date:   Mon Jun 23 23:33:19 2014 +0100

    Move querysort into its own file

diff --git a/lib/libvmod_std/Makefile.am b/lib/libvmod_std/Makefile.am
index c005b85..47dd5dc 100644
--- a/lib/libvmod_std/Makefile.am
+++ b/lib/libvmod_std/Makefile.am
@@ -17,20 +17,20 @@ libvmod_std_la_LDFLAGS = $(AM_LDFLAGS) -module -export-dynamic -avoid-version -s
 
 libvmod_std_la_SOURCES = \
 	vmod_std.c \
+	vmod_std_conversions.c \
 	vmod_std_fileread.c \
-	vmod_std_conversions.c
+	vmod_std_querysort.c
 
 nodist_libvmod_std_la_SOURCES = \
 	vcc_if.c \
 	vcc_if.h
 
 # BUILT_SOURCES is only a hack and dependency tracking does not help for the first build
-vmod_std.lo vmod_std_fileread.lo vmod_std_conversions.lo: vcc_if.h
+$(libvmod_std_la_OBJECTS): vcc_if.h
 
 vcc_if.c vcc_if.h vmod_std.rst vmod_std.man.rst: $(vmodtool) $(vmod_srcdir)/vmod.vcc
 	@PYTHON@ $(vmodtool) $(vmodtoolargs) $(vmod_srcdir)/vmod.vcc
 
-
 EXTRA_DIST = vmod.vcc
 
 CLEANFILES = $(builddir)/vcc_if.c $(builddir)/vcc_if.h \
diff --git a/lib/libvmod_std/vmod_std.c b/lib/libvmod_std/vmod_std.c
index 98ed54d..233c2da 100644
--- a/lib/libvmod_std/vmod_std.c
+++ b/lib/libvmod_std/vmod_std.c
@@ -225,98 +225,3 @@ vmod_timestamp(const struct vrt_ctx *ctx, VCL_STRING label)
 		VSLb_ts_req(ctx->req, label, VTIM_real());
 	}
 }
-
-/*
- * Boltsort
- * Author: Naren Venkataraman of Vimeo Inc.
- * Included here with permission.
- */
-
-#define QS_MAX_PARAM_COUNT	32
-#define QS_EQUALS(a, b)	\
-    ((a) == (b) || ((a) == '\0' && (b) == '&') || ((a) == '&' && (b) == '\0'))
-
-static ssize_t
-param_compare(const char *s, const char *t)
-{
-	for (; QS_EQUALS(*s, *t); s++, t++) {
-		if (*s == '&' || *s == '\0')
-			return (0);
-	}
-	return (*s - *t);
-}
-
-static size_t
-param_copy(char *dst, const char *src)
-{
-	size_t len;
-	len = strcspn(src, "&");
-	memcpy(dst, src, len);
-	return (len);
-}
-
-VCL_STRING __match_proto__(td_std_querysort)
-vmod_querysort(const struct vrt_ctx *ctx, VCL_STRING url)
-{
-	char *param, *params[QS_MAX_PARAM_COUNT];
-	char *p, *r;
-	size_t len;
-	int param_count;
-	int i, n;
-
-	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
-
-	if (url == NULL)
-		return (NULL);
-
-	p = strchr(url, '?');
-	if (p == NULL)
-		return (url);
-
-	param_count = 0;
-	params[param_count++] = ++p;
-	len = p - url;
-
-	while ((p = strchr(p, '&')) != NULL) {
-		param = ++p;
-
-		for (i = 0; i < param_count; i++) {
-			if (param[0] < params[i][0] ||
-			    param_compare(param, params[i]) < 0) {
-				for (n = param_count; n > i; n--)
-					params[n] = params[n - 1];
-				break;
-			}
-		}
-		params[i] = param;
-		param_count++;
-
-		if (param_count == QS_MAX_PARAM_COUNT)
-			return (url);
-	}
-
-	if (param_count == 1)
-		return (url);
-
-	r = WS_Alloc(ctx->ws, strchr(param, '\0') - url + 1);
-	if (r == NULL)
-		return (url);
-
-	p = memcpy(r, url, len);
-	p += len;
-
-	for (i = 0; i < param_count - 1; i++) {
-		if (params[i][0] != '\0' && params[i][0] != '&')
-			break;
-	}
-
-	for (; i < param_count - 1; i++) {
-		p += param_copy(p, params[i]);
-		*p++ = '&';
-	}
-
-	p += param_copy(p, params[i]);
-	*p = '\0';
-
-	return (r);
-}
diff --git a/lib/libvmod_std/vmod_std_querysort.c b/lib/libvmod_std/vmod_std_querysort.c
new file mode 100644
index 0000000..8aac9df
--- /dev/null
+++ b/lib/libvmod_std/vmod_std_querysort.c
@@ -0,0 +1,124 @@
+/*-
+ * Copyright (c) 2010-2014 Varnish Software AS
+ * All rights reserved.
+ *
+ * Author: Naren Venkataraman of Vimeo Inc.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include "config.h"
+
+#include "vrt.h"
+
+#include "cache/cache.h"
+
+#include "vcc_if.h"
+
+#define QS_MAX_PARAM_COUNT	32
+#define QS_EQUALS(a, b)	\
+    ((a) == (b) || ((a) == '\0' && (b) == '&') || ((a) == '&' && (b) == '\0'))
+
+static ssize_t
+param_compare(const char *s, const char *t)
+{
+	for (; QS_EQUALS(*s, *t); s++, t++) {
+		if (*s == '&' || *s == '\0')
+			return (0);
+	}
+	return (*s - *t);
+}
+
+static size_t
+param_copy(char *dst, const char *src)
+{
+	size_t len;
+	len = strcspn(src, "&");
+	memcpy(dst, src, len);
+	return (len);
+}
+
+VCL_STRING __match_proto__(td_std_querysort)
+vmod_querysort(const struct vrt_ctx *ctx, VCL_STRING url)
+{
+	char *param, *params[QS_MAX_PARAM_COUNT];
+	char *p, *r;
+	size_t len;
+	int param_count;
+	int i, n;
+
+	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
+
+	if (url == NULL)
+		return (NULL);
+
+	p = strchr(url, '?');
+	if (p == NULL)
+		return (url);
+
+	param_count = 0;
+	params[param_count++] = ++p;
+	len = p - url;
+
+	while ((p = strchr(p, '&')) != NULL) {
+		param = ++p;
+
+		for (i = 0; i < param_count; i++) {
+			if (param[0] < params[i][0] ||
+			    param_compare(param, params[i]) < 0) {
+				for (n = param_count; n > i; n--)
+					params[n] = params[n - 1];
+				break;
+			}
+		}
+		params[i] = param;
+		param_count++;
+
+		if (param_count == QS_MAX_PARAM_COUNT)
+			return (url);
+	}
+
+	if (param_count == 1)
+		return (url);
+
+	r = WS_Alloc(ctx->ws, strchr(param, '\0') - url + 1);
+	if (r == NULL)
+		return (url);
+
+	p = memcpy(r, url, len);
+	p += len;
+
+	for (i = 0; i < param_count - 1; i++) {
+		if (params[i][0] != '\0' && params[i][0] != '&')
+			break;
+	}
+
+	for (; i < param_count - 1; i++) {
+		p += param_copy(p, params[i]);
+		*p++ = '&';
+	}
+
+	p += param_copy(p, params[i]);
+	*p = '\0';
+
+	return (r);
+}



From fgsch at lodoss.net  Tue Jun 24 09:31:57 2014
From: fgsch at lodoss.net (Federico G. Schwindt)
Date: Tue, 24 Jun 2014 11:31:57 +0200
Subject: [4.0] 622d1b5 Remove superfluous braces
Message-ID: <E1WzN53-0003UU-SU@project.varnish-software.com>


commit 622d1b56a81e1dd9992e3417e23aa039135a9664
Author: Federico G. Schwindt <fgsch at lodoss.net>
Date:   Mon Jun 23 23:44:02 2014 +0100

    Remove superfluous braces

diff --git a/lib/libvmod_std/vmod_std.c b/lib/libvmod_std/vmod_std.c
index 233c2da..718fb71 100644
--- a/lib/libvmod_std/vmod_std.c
+++ b/lib/libvmod_std/vmod_std.c
@@ -178,13 +178,12 @@ vmod_collect(const struct vrt_ctx *ctx, VCL_HEADER hdr)
 	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
 	if (hdr->where == HDR_REQ)
 		http_CollectHdr(ctx->http_req, hdr->what);
-	else if (hdr->where == HDR_BEREQ) {
+	else if (hdr->where == HDR_BEREQ)
 		http_CollectHdr(ctx->http_bereq, hdr->what);
-	} else if (hdr->where == HDR_BERESP) {
+	else if (hdr->where == HDR_BERESP)
 		http_CollectHdr(ctx->http_beresp, hdr->what);
-	} else if (hdr->where == HDR_RESP) {
+	else if (hdr->where == HDR_RESP)
 		http_CollectHdr(ctx->http_resp, hdr->what);
-	}
 }
 
 VCL_BOOL __match_proto__(td_std_healthy)



From fgsch at lodoss.net  Tue Jun 24 09:31:57 2014
From: fgsch at lodoss.net (Federico G. Schwindt)
Date: Tue, 24 Jun 2014 11:31:57 +0200
Subject: [4.0] 133e2b2 Use VCL_xxx types and add missing matching prototypes
Message-ID: <E1WzN53-0003Ud-VW@project.varnish-software.com>


commit 133e2b222c0bfbfb98b9b0d9c26937164d062701
Author: Federico G. Schwindt <fgsch at lodoss.net>
Date:   Mon Jun 23 23:44:21 2014 +0100

    Use VCL_xxx types and add missing matching prototypes
    
    While here unify check.

diff --git a/lib/libvmod_std/vmod_std_conversions.c b/lib/libvmod_std/vmod_std_conversions.c
index 7c1ac08..589d708 100644
--- a/lib/libvmod_std/vmod_std_conversions.c
+++ b/lib/libvmod_std/vmod_std_conversions.c
@@ -43,8 +43,8 @@
 #include "vsa.h"
 #include "vcc_if.h"
 
-VCL_DURATION __match_proto__()
-vmod_duration(const struct vrt_ctx *ctx, const char *p, VCL_DURATION d)
+VCL_DURATION __match_proto__(td_std_duration)
+vmod_duration(const struct vrt_ctx *ctx, VCL_STRING p, VCL_DURATION d)
 {
 	char *e;
 	double r;
@@ -100,8 +100,8 @@ vmod_duration(const struct vrt_ctx *ctx, const char *p, VCL_DURATION d)
 	return (r);
 }
 
-VCL_INT __match_proto__()
-vmod_integer(const struct vrt_ctx *ctx, const char *p, VCL_INT i)
+VCL_INT __match_proto__(td_std_integer)
+vmod_integer(const struct vrt_ctx *ctx, VCL_STRING p, VCL_INT i)
 {
 	char *e;
 	long r;
@@ -121,10 +121,7 @@ vmod_integer(const struct vrt_ctx *ctx, const char *p, VCL_INT i)
 
 	r = strtol(p, &e, 0);
 
-	if (e == NULL)
-		return (i);
-
-	if (*e != '\0')
+	if (e == NULL || *e != '\0')
 		return (i);
 
 	return (r);
@@ -169,7 +166,7 @@ vmod_ip(const struct vrt_ctx *ctx, VCL_STRING s, VCL_IP d)
 	return (r);
 }
 
-VCL_REAL __match_proto__()
+VCL_REAL __match_proto__(td_std_real)
 vmod_real(const struct vrt_ctx *ctx, VCL_STRING p, VCL_REAL d)
 {
 	char *e;



From fgsch at lodoss.net  Tue Jun 24 09:31:58 2014
From: fgsch at lodoss.net (Federico G. Schwindt)
Date: Tue, 24 Jun 2014 11:31:58 +0200
Subject: [4.0] 14da184 Add real2time,
 time2integer and time2real to the std vmod
Message-ID: <E1WzN54-0003Um-1A@project.varnish-software.com>


commit 14da1840453b4076ac8b46448c9092a491037d5a
Author: Federico G. Schwindt <fgsch at lodoss.net>
Date:   Tue Jun 24 00:13:55 2014 +0100

    Add real2time, time2integer and time2real to the std vmod
    
    As discussed on the latest VDD.

diff --git a/bin/varnishtest/tests/m00016.vtc b/bin/varnishtest/tests/m00016.vtc
new file mode 100644
index 0000000..ef51f93
--- /dev/null
+++ b/bin/varnishtest/tests/m00016.vtc
@@ -0,0 +1,23 @@
+varnishtest "Test std.real2time, std.time2integer and std.time2real"
+
+server s1 {
+	rxreq
+	txresp
+} -start
+
+varnish v1 -vcl+backend {
+	import ${vmod_std};
+
+	sub vcl_deliver {
+		set resp.http.x-foo = std.integer(req.http.foo, 0);
+		set resp.http.x-bar = std.time2integer(std.real2time(std.real(resp.http.x-foo, 0.0)));
+		set resp.http.x-baz = std.time2real(std.real2time(std.real(resp.http.x-foo, 0.0)));
+	}
+} -start
+
+client c1 {
+	txreq -hdr "foo: 1140618699"
+	rxresp
+	expect resp.http.x-foo == resp.http.x-bar
+	expect resp.http.x-baz == 1140618699.000
+} -run
diff --git a/lib/libvmod_std/vmod.vcc b/lib/libvmod_std/vmod.vcc
index cc3eb43..fbee007 100644
--- a/lib/libvmod_std/vmod.vcc
+++ b/lib/libvmod_std/vmod.vcc
@@ -143,6 +143,27 @@ Description
 Example
 	set req.http.x-real = std.real(req.http.x-foo, 0.0);
 
+$Function TIME real2time(REAL)
+
+Description
+	Converts the real *r* to a time.
+Example
+	set req.http.x-time = std.real2time(1140618699.00);
+
+$Function INT time2integer(TIME)
+
+Description
+	Converts the time *t* to a integer.
+Example
+	set req.http.x-int = std.time2integer(now);
+
+$Function REAL time2real(TIME)
+
+Description
+	Converts the time *t* to a real.
+Example
+	set req.http.x-real = std.time2real(now);
+
 $Function BOOL healthy(BACKEND)
 
 Description
diff --git a/lib/libvmod_std/vmod_std_conversions.c b/lib/libvmod_std/vmod_std_conversions.c
index 589d708..8ca0dd7 100644
--- a/lib/libvmod_std/vmod_std_conversions.c
+++ b/lib/libvmod_std/vmod_std_conversions.c
@@ -195,3 +195,27 @@ vmod_real(const struct vrt_ctx *ctx, VCL_STRING p, VCL_REAL d)
 
 	return (r);
 }
+
+VCL_TIME __match_proto__(td_std_real2time)
+vmod_real2time(const struct vrt_ctx *ctx, VCL_REAL r)
+{
+	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
+
+	return (r);
+}
+
+VCL_INT __match_proto__(td_std_time2integer)
+vmod_time2integer(const struct vrt_ctx *ctx, VCL_TIME t)
+{
+	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
+
+	return (t);
+}
+
+VCL_REAL __match_proto__(td_std_time2real)
+vmod_time2real(const struct vrt_ctx *ctx, VCL_TIME t)
+{
+	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
+
+	return (t);
+}



From lkarsten at varnish-software.com  Tue Jun 24 09:31:58 2014
From: lkarsten at varnish-software.com (Lasse Karstensen)
Date: Tue, 24 Jun 2014 11:31:58 +0200
Subject: [4.0] 4354e5e Prepare for the 4.0.1 release.
Message-ID: <E1WzN54-0003Uv-3J@project.varnish-software.com>


commit 4354e5e04a6af43394bbbdb32dc2654ca4887de8
Author: Lasse Karstensen <lkarsten at varnish-software.com>
Date:   Tue Jun 24 11:16:20 2014 +0200

    Prepare for the 4.0.1 release.

diff --git a/configure.ac b/configure.ac
index eed48bc..118f743 100644
--- a/configure.ac
+++ b/configure.ac
@@ -2,7 +2,7 @@ AC_PREREQ(2.59)
 AC_COPYRIGHT([Copyright (c) 2006 Verdens Gang AS
 Copyright (c) 2006-2014 Varnish Software AS])
 AC_REVISION([$Id$])
-AC_INIT([Varnish], [4.0.1-rc1], [varnish-dev at varnish-cache.org])
+AC_INIT([Varnish], [4.0.1], [varnish-dev at varnish-cache.org])
 AC_CONFIG_SRCDIR(include/miniobj.h)
 AC_CONFIG_HEADERS([config.h])
 AC_CONFIG_MACRO_DIR([m4])
diff --git a/doc/changes.rst b/doc/changes.rst
index c5e1425..d66e7a0 100644
--- a/doc/changes.rst
+++ b/doc/changes.rst
@@ -1,10 +1,13 @@
 
-============================================
-Changes from 4.0.0 to 4.0.1-rc1 (2014-06-20)
-============================================
+========================================
+Changes from 4.0.0 to 4.0.1 (2014-06-24)
+========================================
 
 New since 4.0.0:
 
+- New functions in vmod_std: real2time, time2integer, time2real, real.
+- Chunked requests are now supported. (pass)
+- Add std.querysort() that sorts GET query arguments. (from libvmod-boltsort)
 - Varnish will no longer reply with "200 Not Modified".
 - Backend IMS is now only attempted when last status was 200.
 - Packaging now uses find-provides instead of find-requires. [redhat]
@@ -27,6 +30,9 @@ New since 4.0.0:
 Bugs fixed
 ----------
 
+* 1269_ - Use correct byte counters in varnishncsa when piping a request.
+* 1524_ - Chunked requests should be pipe-able.
+* 1530_ - Expire old object on successful IMS fetch.
 * 1475_ - time-to-first-byte in varnishncsa was potentially dishonest.
 * 1480_ - Porting guide for 4.0 is incomplete.
 * 1482_ - Inherit group memberships of -u specified user.
@@ -53,6 +59,9 @@ Bugs fixed
 * 1519_ - Round-robin director does not support weight. [docs]
 
 
+.. _1269: https://www.varnish-cache.org/trac/ticket/1269
+.. _1524: https://www.varnish-cache.org/trac/ticket/1524
+.. _1530: https://www.varnish-cache.org/trac/ticket/1530
 .. _1475: https://www.varnish-cache.org/trac/ticket/1475
 .. _1480: https://www.varnish-cache.org/trac/ticket/1480
 .. _1482: https://www.varnish-cache.org/trac/ticket/1482
diff --git a/redhat/varnish.spec b/redhat/varnish.spec
index 338aa96..b541d91 100644
--- a/redhat/varnish.spec
+++ b/redhat/varnish.spec
@@ -1,4 +1,4 @@
-%define v_rc rc1
+%define XXXv_rc rc1
 %define vd_rc %{?v_rc:-%{?v_rc}}
 %define    _use_internal_dependency_generator 0
 %define __find_provides %{_builddir}/varnish-%{version}%{?v_rc:-%{?v_rc}}/redhat/find-provides
@@ -14,7 +14,7 @@ URL: http://www.varnish-cache.org/
 Source0: %{name}-%{version}%{?vd_rc}.tar.gz
 #Source0: %{name}-trunk.tar.gz
 BuildRoot: %{_tmppath}/%{name}-%{version}-%{release}-root-%(%{__id_u} -n)
-# To build from git, start with a make dist, see redhat/README.redhat 
+# To build from git, start with a make dist, see redhat/README.redhat
 # You will need at least automake autoconf libtool
 #BuildRequires: automake autoconf libtool
 BuildRequires: ncurses-devel groff pcre-devel pkgconfig libedit-devel jemalloc-devel



From phk at FreeBSD.org  Tue Jun 24 10:53:22 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 24 Jun 2014 12:53:22 +0200
Subject: [master] 2d156c6 Expose WS_MarkOverflow()
Message-ID: <E1WzOLq-0006tE-Qp@project.varnish-software.com>


commit 2d156c6c6fdbc0eb97c11ec03badc7eb7f459438
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Jun 24 10:38:22 2014 +0000

    Expose WS_MarkOverflow()

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index dc7e6f1..020157a 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -1235,6 +1235,7 @@ void WRK_BgThread(pthread_t *thr, const char *name, bgthread_t *func,
 
 void WS_Init(struct ws *ws, const char *id, void *space, unsigned len);
 unsigned WS_Reserve(struct ws *ws, unsigned bytes);
+void WS_MarkOverflow(struct ws *ws);
 void WS_Release(struct ws *ws, unsigned bytes);
 void WS_ReleaseP(struct ws *ws, char *ptr);
 void WS_Assert(const struct ws *ws);
diff --git a/bin/varnishd/cache/cache_ws.c b/bin/varnishd/cache/cache_ws.c
index 8ef7aca..ccc887c 100644
--- a/bin/varnishd/cache/cache_ws.c
+++ b/bin/varnishd/cache/cache_ws.c
@@ -85,7 +85,7 @@ WS_Init(struct ws *ws, const char *id, void *space, unsigned len)
 }
 
 
-static void
+void
 WS_MarkOverflow(struct ws *ws)
 {
 	CHECK_OBJ_NOTNULL(ws, WS_MAGIC);



From phk at FreeBSD.org  Tue Jun 24 10:53:22 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 24 Jun 2014 12:53:22 +0200
Subject: [master] 7a03bef White space nit
Message-ID: <E1WzOLq-0006tK-SV@project.varnish-software.com>


commit 7a03bef9cae2c172d73a2bbe3ce11900a6b7a0a0
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Jun 24 10:44:03 2014 +0000

    White space nit

diff --git a/bin/varnishtest/tests/c00067.vtc b/bin/varnishtest/tests/c00067.vtc
index 73bcb34..d6d4c02 100644
--- a/bin/varnishtest/tests/c00067.vtc
+++ b/bin/varnishtest/tests/c00067.vtc
@@ -34,7 +34,7 @@ varnish v1 -cliok "param.set vcc_allow_inline_c true" -vcl+backend {
 	sub vcl_recv {
 		C{ VRT_CacheReqBody(ctx, 1000); }C
 	}
-} 
+}
 
 client c1 {
 	txreq -req POST -nolen -hdr "Transfer-encoding: chunked"



From phk at FreeBSD.org  Tue Jun 24 10:53:22 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 24 Jun 2014 12:53:22 +0200
Subject: [master] 2afa113 A couple more corner cases
Message-ID: <E1WzOLq-0006tT-U5@project.varnish-software.com>


commit 2afa113483043f6f25cd98866bbeed9cec78038e
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Jun 24 10:44:15 2014 +0000

    A couple more corner cases

diff --git a/bin/varnishtest/tests/m00014.vtc b/bin/varnishtest/tests/m00014.vtc
index 260280f..28c4c79 100644
--- a/bin/varnishtest/tests/m00014.vtc
+++ b/bin/varnishtest/tests/m00014.vtc
@@ -1,6 +1,6 @@
 varnishtest "Test std.querysort"
 
-server s1 -repeat 5 {
+server s1 -repeat 6 {
 	rxreq
 	txresp
 } -start
@@ -18,18 +18,30 @@ client c1 {
 	rxresp
 	expect resp.http.url == "/foo/bar?b=0&c=5&p=0&t=0"
 
+	delay .1
+
 	txreq -url "/foo/bar?coa=0&co=0"
 	rxresp
 	expect resp.http.url == "/foo/bar?co=0&coa=0"
 
+	delay .1
+
 	txreq -url "/foo/bar?a=0&&&&&"
 	rxresp
 	expect resp.http.url == "/foo/bar?a=0"
 
+	txreq -url "/foo/bar?&a=0&&&&&z&w&x&"
+	rxresp
+	expect resp.http.url == "/foo/bar?a=0&w&x&z"
+
+	delay .1
+
 	txreq -url "/foo/bar?&"
 	rxresp
 	expect resp.http.url == "/foo/bar?"
 
+	delay .1
+
 	txreq -url "/foo/bar"
 	rxresp
 	expect resp.http.url == "/foo/bar"



From phk at FreeBSD.org  Tue Jun 24 10:53:22 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 24 Jun 2014 12:53:22 +0200
Subject: [master] c2c3156 Explicitly round down for timestamps
Message-ID: <E1WzOLq-0006tc-Vf@project.varnish-software.com>


commit c2c3156ef330d1e53e60ee4a7e3142170070dfc2
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Jun 24 10:44:30 2014 +0000

    Explicitly round down for timestamps

diff --git a/lib/libvmod_std/vmod_std_conversions.c b/lib/libvmod_std/vmod_std_conversions.c
index 8ca0dd7..cdcd3cb 100644
--- a/lib/libvmod_std/vmod_std_conversions.c
+++ b/lib/libvmod_std/vmod_std_conversions.c
@@ -209,7 +209,7 @@ vmod_time2integer(const struct vrt_ctx *ctx, VCL_TIME t)
 {
 	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
 
-	return (t);
+	return ((long)floor(t));
 }
 
 VCL_REAL __match_proto__(td_std_time2real)



From phk at FreeBSD.org  Tue Jun 24 10:53:23 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 24 Jun 2014 12:53:23 +0200
Subject: [master] 6547a45 I know it's the OCD talking here,
 but neither a fixed 32 param limit or an string-insertion-sort feels
 right for me.
Message-ID: <E1WzOLr-0006tl-19@project.varnish-software.com>


commit 6547a456e793a4437f0004654c3fdf9ca09e3559
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Jun 24 10:44:42 2014 +0000

    I know it's the OCD talking here, but neither a fixed 32 param limit
    or an string-insertion-sort feels right for me.
    
    Using reserved workspace to tracking the params and qsort(3) to sort them
    solves both problems.

diff --git a/lib/libvmod_std/vmod_std_querysort.c b/lib/libvmod_std/vmod_std_querysort.c
index 8aac9df..8f7f93b 100644
--- a/lib/libvmod_std/vmod_std_querysort.c
+++ b/lib/libvmod_std/vmod_std_querysort.c
@@ -28,97 +28,105 @@
 
 #include "config.h"
 
+#include <stdlib.h>
+
 #include "vrt.h"
 
 #include "cache/cache.h"
 
 #include "vcc_if.h"
 
-#define QS_MAX_PARAM_COUNT	32
-#define QS_EQUALS(a, b)	\
-    ((a) == (b) || ((a) == '\0' && (b) == '&') || ((a) == '&' && (b) == '\0'))
-
-static ssize_t
-param_compare(const char *s, const char *t)
+static int
+compa(const void *a, const void *b)
 {
-	for (; QS_EQUALS(*s, *t); s++, t++) {
-		if (*s == '&' || *s == '\0')
-			return (0);
-	}
-	return (*s - *t);
-}
-
-static size_t
-param_copy(char *dst, const char *src)
-{
-	size_t len;
-	len = strcspn(src, "&");
-	memcpy(dst, src, len);
-	return (len);
+	const char * const *pa = a;
+	const char * const *pb = b;
+	const char *a1, *b1;
+
+	for(a1 = pa[0], b1 = pb[0]; a1 < pa[1] && b1 < pb[1]; a1++, b1++)
+		if (*a1 != *b1)
+			return (*a1 - *b1);
+	return (0);
 }
 
 VCL_STRING __match_proto__(td_std_querysort)
 vmod_querysort(const struct vrt_ctx *ctx, VCL_STRING url)
 {
-	char *param, *params[QS_MAX_PARAM_COUNT];
+	const char *cq, *cu;
 	char *p, *r;
-	size_t len;
-	int param_count;
-	int i, n;
+	const char **pp;
+	const char **pe;
+	int np;
+	int i;
 
 	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
 
 	if (url == NULL)
 		return (NULL);
 
-	p = strchr(url, '?');
-	if (p == NULL)
+	/* Split :query from :url */
+	cu = strchr(url, '?');
+	if (cu == NULL)
 		return (url);
 
-	param_count = 0;
-	params[param_count++] = ++p;
-	len = p - url;
-
-	while ((p = strchr(p, '&')) != NULL) {
-		param = ++p;
-
-		for (i = 0; i < param_count; i++) {
-			if (param[0] < params[i][0] ||
-			    param_compare(param, params[i]) < 0) {
-				for (n = param_count; n > i; n--)
-					params[n] = params[n - 1];
-				break;
-			}
-		}
-		params[i] = param;
-		param_count++;
-
-		if (param_count == QS_MAX_PARAM_COUNT)
-			return (url);
-	}
-
-	if (param_count == 1)
+	/* Spot single-param queries */
+	cq = strchr(cu, '&');
+	if (cq == NULL)
 		return (url);
 
-	r = WS_Alloc(ctx->ws, strchr(param, '\0') - url + 1);
+	r = WS_Copy(ctx->ws, url, -1);
 	if (r == NULL)
 		return (url);
 
-	p = memcpy(r, url, len);
-	p += len;
+	(void)WS_Reserve(ctx->ws, 0);
+	/* We trust cache_ws.c to align sensibly */
+	pp = (const char**)(void*)(ctx->ws->f);
+	pe = (const char**)(void*)(ctx->ws->e);
 
-	for (i = 0; i < param_count - 1; i++) {
-		if (params[i][0] != '\0' && params[i][0] != '&')
-			break;
+	if (pp + 4 > pe) {
+		WS_Release(ctx->ws, 0);
+		WS_MarkOverflow(ctx->ws);
+		return (url);
 	}
 
-	for (; i < param_count - 1; i++) {
-		p += param_copy(p, params[i]);
-		*p++ = '&';
+	/* Collect params as pointer pairs */
+	np = 0;
+	pp[np++] = 1 + cu;
+	for (cq = 1 + cu; *cq != '\0'; cq++) {
+		if (*cq == '&') {
+			if (pp + 3 > pe) {
+				WS_Release(ctx->ws, 0);
+				WS_MarkOverflow(ctx->ws);
+				return (url);
+			}
+			pp[np++] = cq;
+			/* Skip trivially empty params */
+			while(cq[1] == '&')
+				cq++;
+			pp[np++] = cq + 1;
+		}
+	}
+	pp[np++] = cq;
+	assert(!(np & 1));
+
+	qsort(pp, np / 2, sizeof(*pp) * 2, compa);
+
+	/* Emit sorted params */
+	p = 1 + r + (cu - url);
+	cq = "";
+	for (i = 0; i < np; i += 2) {
+		/* Ignore any edge-case zero length params */
+		if (pp[i + 1] == pp[i])
+			continue;
+		assert(pp[i + 1] > pp[i]);
+		if (*cq)
+			*p++ = *cq;
+		memcpy(p, pp[i], pp[i + 1] - pp[i]);
+		p += pp[i + 1] - pp[i];
+		cq = "&";
 	}
-
-	p += param_copy(p, params[i]);
 	*p = '\0';
 
+	WS_Release(ctx->ws, 0);
 	return (r);
 }



From apj at mutt.dk  Tue Jun 24 10:56:57 2014
From: apj at mutt.dk (Andreas Plesner)
Date: Tue, 24 Jun 2014 12:56:57 +0200
Subject: [master] a9d7463 Remove comment which breaks markup
Message-ID: <E1WzOPJ-0007QP-Jx@project.varnish-software.com>


commit a9d746324385f55f080e7a6fc9e400c412cac40f
Author: Andreas Plesner <apj at mutt.dk>
Date:   Tue Jun 24 12:56:42 2014 +0200

    Remove comment which breaks markup

diff --git a/doc/sphinx/installation/help.rst b/doc/sphinx/installation/help.rst
index 897324a..2d04c5f 100644
--- a/doc/sphinx/installation/help.rst
+++ b/doc/sphinx/installation/help.rst
@@ -92,8 +92,6 @@ The following companies offer commercial Varnish support, and are listed
 here for your convenience.  If you want your company listed here, drop
 an email to phk at FreeBSD.org.
 
-.. XXX: Should we perhaps enhance this to explain Varnish Plus? benc
-
 	Varnish Software
 	sales at varnish-software.com
 



From lkarsten at varnish-software.com  Tue Jun 24 15:44:43 2014
From: lkarsten at varnish-software.com (Lasse Karstensen)
Date: Tue, 24 Jun 2014 17:44:43 +0200
Subject: [3.0] 4d16476 Prepare for 3.0.6-rc1 release.
Message-ID: <E1WzStn-0005vq-Lk@project.varnish-software.com>


commit 4d1647681aa47b232717004dd2d4c61c00f4071c
Author: Lasse Karstensen <lkarsten at varnish-software.com>
Date:   Tue Jun 24 17:39:41 2014 +0200

    Prepare for 3.0.6-rc1 release.

diff --git a/configure.ac b/configure.ac
index 83cbb47..427b230 100644
--- a/configure.ac
+++ b/configure.ac
@@ -1,8 +1,8 @@
 AC_PREREQ(2.59)
 AC_COPYRIGHT([Copyright (c) 2006 Verdens Gang AS
-Copyright (c) 2006-2011 Varnish Software AS])
+Copyright (c) 2006-2014 Varnish Software AS])
 AC_REVISION([$Id$])
-AC_INIT([Varnish], [3.0.5], [varnish-dev at varnish-cache.org])
+AC_INIT([Varnish], [3.0.6rc1], [varnish-dev at varnish-cache.org])
 AC_CONFIG_SRCDIR(include/varnishapi.h)
 AM_CONFIG_HEADER(config.h)
 
diff --git a/doc/changes.rst b/doc/changes.rst
index e2398d1..10fc995 100644
--- a/doc/changes.rst
+++ b/doc/changes.rst
@@ -1,6 +1,30 @@
-================================
-Changes from 3.0.5 rc 1 to 3.0.5
-================================
+============================================
+Changes from 3.0.5 to 3.0.6-rc1 (2014-06-24)
+============================================
+
+- Document storage.<name>.* VCL variables. Bug 1514_.
+- Round up http_max_hdr to a multiple of 4. Bug 1327_.
+- Avoid negative ReqEnd timestamps with ESI. Bug 1297_.
+- %D format for varnishncsa is now an integer (as documented)
+- Fix compile errors with clang.
+- Clear objectcore flags earlier in ban lurker. Bug 1470_.
+- Patch embedded jemalloc to avoid segfault. Bug 1448_.
+- Allow backend names to start with if, include or else. Bug 1439_.
+- Stop handling gzip after gzip body end. Bug 1086_.
+- Document %D and %T for varnishncsa.
+
+.. _1514: http://varnish-cache.org/trac/ticket/1514
+.. _1327: http://varnish-cache.org/trac/ticket/1327
+.. _1297: http://varnish-cache.org/trac/ticket/1297
+.. _1470: http://varnish-cache.org/trac/ticket/1470
+.. _1448: http://varnish-cache.org/trac/ticket/1448
+.. _1439: http://varnish-cache.org/trac/ticket/1439
+.. _1086: http://varnish-cache.org/trac/ticket/1086
+
+
+=============================================
+Changes from 3.0.5 rc 1 to 3.0.5 (2013-12-02)
+=============================================
 
 varnishd
 --------
diff --git a/redhat/varnish.spec b/redhat/varnish.spec
index a8521ad..c10737d 100644
--- a/redhat/varnish.spec
+++ b/redhat/varnish.spec
@@ -2,11 +2,11 @@
 %define __find_provides %{_builddir}/varnish-%{version}%{?v_rc:-%{?v_rc}}/redhat/find-provides
 Summary: High-performance HTTP accelerator
 Name: varnish
-Version: 3.0.5
+Version: 3.0.6
 Release: 1%{?dist}
 License: BSD
 Group: System Environment/Daemons
-URL: http://www.varnish-cache.org/
+URL: https://www.varnish-cache.org/
 #Source0: http://repo.varnish-cache.org/source/%{name}-%{version}.tar.gz
 Source0: %{name}-%{version}%{?v_rc:-%{v_rc}}.tar.gz
 BuildRoot: %{_tmppath}/%{name}-%{version}-%{release}-root-%(%{__id_u} -n)



From lkarsten at varnish-software.com  Tue Jun 24 15:59:38 2014
From: lkarsten at varnish-software.com (Lasse Karstensen)
Date: Tue, 24 Jun 2014 17:59:38 +0200
Subject: [3.0] e1be153 Use forgotten rc1 tag also in spec file.
Message-ID: <E1WzT8E-0001XI-MD@project.varnish-software.com>


commit e1be153bb8db403deefbf845c3dfbbe20cdd8654
Author: Lasse Karstensen <lkarsten at varnish-software.com>
Date:   Tue Jun 24 17:58:53 2014 +0200

    Use forgotten rc1 tag also in spec file.
    
    This is why it sucks to have the redhat/ directory inside
    the source distribution.

diff --git a/redhat/varnish.spec b/redhat/varnish.spec
index c10737d..39f0141 100644
--- a/redhat/varnish.spec
+++ b/redhat/varnish.spec
@@ -2,7 +2,7 @@
 %define __find_provides %{_builddir}/varnish-%{version}%{?v_rc:-%{?v_rc}}/redhat/find-provides
 Summary: High-performance HTTP accelerator
 Name: varnish
-Version: 3.0.6
+Version: 3.0.6rc1
 Release: 1%{?dist}
 License: BSD
 Group: System Environment/Daemons



From phk at FreeBSD.org  Wed Jun 25 08:25:48 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Wed, 25 Jun 2014 10:25:48 +0200
Subject: [master] bcdc20c Message 850 is not helpful
Message-ID: <E1WziWa-0002SU-Qi@project.varnish-software.com>


commit bcdc20ce259a886f137eff1960dede2550ec2a85
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Jun 25 08:25:24 2014 +0000

    Message 850 is not helpful

diff --git a/bin/varnishd/flint.lnt b/bin/varnishd/flint.lnt
index 9ad0543..be84816 100644
--- a/bin/varnishd/flint.lnt
+++ b/bin/varnishd/flint.lnt
@@ -122,6 +122,9 @@
 -e728	// Symbol ... not explicitly initialized
 -e716	// while(1) ... 
 -e785	// Too few initializers for aggregate 
+-e850	// for loop index variable '___' whose type category is '___' 
+	// is modified in body of the for loop that began at '___'
+
 
 -esym(765, vcc_ProcAction) // could be made static
 -esym(759, vcc_ProcAction) // could be moved to module



From phk at FreeBSD.org  Wed Jun 25 09:11:10 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Wed, 25 Jun 2014 11:11:10 +0200
Subject: [master] 62a2da0 Mark failures on the WS rather than on struct http,
 there is no advantage from having two mechanisms.
Message-ID: <E1WzjEU-0005D1-D4@project.varnish-software.com>


commit 62a2da09c27d5d57978ffbfda011fa7a9ceb068f
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Jun 25 09:09:20 2014 +0000

    Mark failures on the WS rather than on struct http, there is no
    advantage from having two mechanisms.

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index 020157a..410aa7e 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -197,7 +197,6 @@ struct http {
 	uint16_t		status;
 	uint8_t			protover;
 	uint8_t			conds;		/* If-* headers present */
-	uint8_t			failed;		/* usually: ws-alloc failed */
 };
 
 /*--------------------------------------------------------------------
@@ -1017,7 +1016,7 @@ const char *http_GetReq(const struct http *hp);
 int http_HdrIs(const struct http *hp, const char *hdr, const char *val);
 int http_IsHdr(const txt *hh, const char *hdr);
 enum sess_close http_DoConnection(const struct http *);
-void http_CopyHome(struct http *hp);
+void http_CopyHome(const struct http *hp);
 void http_Unset(struct http *hp, const char *hdr);
 void http_CollectHdr(struct http *hp, const char *hdr);
 void http_VSL_log(const struct http *hp);
diff --git a/bin/varnishd/cache/cache_http.c b/bin/varnishd/cache/cache_http.c
index 159751b..2fc70be 100644
--- a/bin/varnishd/cache/cache_http.c
+++ b/bin/varnishd/cache/cache_http.c
@@ -95,12 +95,12 @@ http_VSL_log(const struct http *hp)
 /*--------------------------------------------------------------------*/
 
 static void
-http_fail(struct http *hp)
+http_fail(const struct http *hp)
 {
 
 	VSC_C_main->losthdr++;
-	VSLb(hp->vsl, SLT_Error, "out of workspace");
-	hp->failed = 1;
+	VSLb(hp->vsl, SLT_Error, "out of workspace (%s)", hp->ws->id);
+	WS_MarkOverflow(hp->ws);
 }
 
 /*--------------------------------------------------------------------*/
@@ -210,7 +210,7 @@ http_SetH(const struct http *to, unsigned n, const char *fm)
 /*--------------------------------------------------------------------*/
 
 static void
-http_PutField(struct http *to, int field, const char *string)
+http_PutField(const struct http *to, int field, const char *string)
 {
 	char *p;
 
@@ -274,7 +274,8 @@ http_CollectHdr(struct http *hp, const char *hdr)
 	unsigned u, l, ml, f, x, d;
 	char *b = NULL, *e = NULL;
 
-	if (hp->failed)
+	CHECK_OBJ_NOTNULL(hp, HTTP_MAGIC);
+	if (WS_Overflowed(hp->ws))
 		return;
 	l = hdr[0];
 	assert(l == strlen(hdr + 1));
@@ -760,7 +761,7 @@ http_Merge(const struct http *fm, struct http *to, int not_ce)
  */
 
 void
-http_CopyHome(struct http *hp)
+http_CopyHome(const struct http *hp)
 {
 	unsigned u, l;
 	char *p;
diff --git a/bin/varnishd/cache/cache_vrt_var.c b/bin/varnishd/cache/cache_vrt_var.c
index b74e101..c11f2d9 100644
--- a/bin/varnishd/cache/cache_vrt_var.c
+++ b/bin/varnishd/cache/cache_vrt_var.c
@@ -50,7 +50,7 @@ static char vrt_hostname[255] = "";
  */
 
 static void
-vrt_do_string(struct http *hp, int fld,
+vrt_do_string(const struct http *hp, int fld,
     const char *err, const char *p, va_list ap)
 {
 	const char *b;
@@ -60,7 +60,7 @@ vrt_do_string(struct http *hp, int fld,
 	b = VRT_String(hp->ws, NULL, p, ap);
 	if (b == NULL || *b == '\0') {
 		VSLb(hp->vsl, SLT_LostHeader, "%s", err);
-		hp->failed = 1;
+		WS_MarkOverflow(hp->ws);
 		return;
 	}
 	http_SetH(hp, fld, b);
@@ -100,11 +100,11 @@ VRT_l_##obj##_status(const struct vrt_ctx *ctx, long num)		\
 	CHECK_OBJ_NOTNULL(ctx->http_##obj, HTTP_MAGIC);			\
 	if (num > 65535) {						\
 		VSLb(ctx->vsl, SLT_VCL_Error, "%s.status > 65535", #obj); \
-		ctx->http_##obj->failed = 1;				\
+		WS_MarkOverflow(ctx->http_##obj->ws);			\
 	} else if ((num % 1000) < 100) {				\
 		VSLb(ctx->vsl, SLT_VCL_Error, "illegal %s.status (..0##)", \
 		    #obj);						\
-		ctx->http_##obj->failed = 1;				\
+		WS_MarkOverflow(ctx->http_##obj->ws);			\
 	} else {							\
 		http_SetStatus(ctx->http_##obj, (uint16_t)num);		\
 	}								\
@@ -244,7 +244,7 @@ VRT_l_client_identity(const struct vrt_ctx *ctx, const char *str, ...)
 	va_end(ap);
 	if (b == NULL) {
 		VSLb(ctx->vsl, SLT_LostHeader, "client.identity");
-		ctx->req->http->failed = 1;
+		WS_MarkOverflow(ctx->req->http->ws);
 		return;
 	}
 	ctx->req->client_identity = b;
@@ -331,7 +331,7 @@ VRT_l_beresp_storage_hint(const struct vrt_ctx *ctx, const char *str, ...)
 	va_end(ap);
 	if (b == NULL) {
 		VSLb(ctx->vsl, SLT_LostHeader, "storage.hint");
-		ctx->bo->beresp->failed = 1;
+		WS_MarkOverflow(ctx->bo->beresp->ws);
 		return;
 	}
 	ctx->bo->storage_hint = b;



From lkarsten at varnish-software.com  Wed Jun 25 09:24:02 2014
From: lkarsten at varnish-software.com (Lasse Karstensen)
Date: Wed, 25 Jun 2014 11:24:02 +0200
Subject: [3.0] 7b410fd Improve description of bug #1327.
Message-ID: <E1WzjQw-0005s5-N2@project.varnish-software.com>


commit 7b410fda9ca3a8f78057866fe023346cc1770437
Author: Lasse Karstensen <lkarsten at varnish-software.com>
Date:   Wed Jun 25 11:23:05 2014 +0200

    Improve description of bug #1327.
    
    Reviewed by:	scoof

diff --git a/doc/changes.rst b/doc/changes.rst
index 10fc995..1f0aa34 100644
--- a/doc/changes.rst
+++ b/doc/changes.rst
@@ -3,7 +3,7 @@ Changes from 3.0.5 to 3.0.6-rc1 (2014-06-24)
 ============================================
 
 - Document storage.<name>.* VCL variables. Bug 1514_.
-- Round up http_max_hdr to a multiple of 4. Bug 1327_.
+- Fix memory alignment panic when http_max_hdr is not a multiple of 4. Bug 1327_.
 - Avoid negative ReqEnd timestamps with ESI. Bug 1297_.
 - %D format for varnishncsa is now an integer (as documented)
 - Fix compile errors with clang.



From phk at FreeBSD.org  Wed Jun 25 20:29:20 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Wed, 25 Jun 2014 22:29:20 +0200
Subject: [master] 6868b64 Fix whitespace in compiled vcl
Message-ID: <E1Wztom-0000xw-N9@project.varnish-software.com>


commit 6868b6430758ac9991d19a03a512aa5d9f710d2b
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Jun 25 10:02:36 2014 +0000

    Fix whitespace in compiled vcl

diff --git a/lib/libvcc/vcc_action.c b/lib/libvcc/vcc_action.c
index 080f50e..9b1d22d 100644
--- a/lib/libvcc/vcc_action.c
+++ b/lib/libvcc/vcc_action.c
@@ -291,10 +291,10 @@ parse_hash_data(struct vcc *tl)
 	vcc_NextToken(tl);
 	SkipToken(tl, '(');
 
-	Fb(tl, 1, "VRT_hashdata(ctx, ");
+	Fb(tl, 1, "VRT_hashdata(ctx,\n  ");
 	vcc_Expr(tl, STRING_LIST);
 	ERRCHK(tl);
-	Fb(tl, 0, ");\n");
+	Fb(tl, 1, ");\n");
 	SkipToken(tl, ')');
 }
 



From phk at FreeBSD.org  Wed Jun 25 20:29:20 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Wed, 25 Jun 2014 22:29:20 +0200
Subject: [master] 1d4ca91 Disassociate obj->http from bo->ws once the object
 is setup
Message-ID: <E1Wztom-0000y4-P4@project.varnish-software.com>


commit 1d4ca91c7cbc36a7c9a9dab939397a30f478c36c
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Jun 25 10:17:22 2014 +0000

    Disassociate obj->http from bo->ws once the object is setup

diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c
index a768170..cd2b40d 100644
--- a/bin/varnishd/cache/cache_fetch.c
+++ b/bin/varnishd/cache/cache_fetch.c
@@ -164,6 +164,9 @@ vbf_beresp2obj(struct busyobj *bo)
 	else
 		obj->last_modified = floor(bo->exp.t_origin);
 
+	/* Disassociate the obj from the bo's workspace */
+	hp2->ws = NULL;
+
 	return (0);
 }
 



From phk at FreeBSD.org  Wed Jun 25 20:29:20 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Wed, 25 Jun 2014 22:29:20 +0200
Subject: [master] 34b1297 Enforce that VCL/VMODs cannot make permanant
 allocations from wrk->aws.
Message-ID: <E1Wztom-0000yE-QZ@project.varnish-software.com>


commit 34b12971e96646eb64c8be857395c7b2bc944874
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Jun 25 10:17:48 2014 +0000

    Enforce that VCL/VMODs cannot make permanant allocations from wrk->aws.
    
    Reserving and returning space from wrk->aws is OK.

diff --git a/bin/varnishd/cache/cache_vcl.c b/bin/varnishd/cache/cache_vcl.c
index e6dd179..7e73cfb 100644
--- a/bin/varnishd/cache/cache_vcl.c
+++ b/bin/varnishd/cache/cache_vcl.c
@@ -391,11 +391,9 @@ vcl_call_method(struct worker *wrk, struct req *req, struct busyobj *bo,
 	memset(&ctx, 0, sizeof ctx);
 	ctx.magic = VRT_CTX_MAGIC;
 	if (req != NULL) {
-		// AZ(bo);
 		CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
 		CHECK_OBJ_NOTNULL(req->sp, SESS_MAGIC);
 		vsl = req->vsl;
-		ctx.vsl = vsl;
 		ctx.vcl = req->vcl;
 		ctx.http_req = req->http;
 		ctx.http_resp = req->resp;
@@ -404,16 +402,15 @@ vcl_call_method(struct worker *wrk, struct req *req, struct busyobj *bo,
 			ctx.http_obj = req->obj->http;
 	}
 	if (bo != NULL) {
-		// AZ(req);
 		CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
 		vsl = bo->vsl;
-		ctx.vsl = vsl;
 		ctx.vcl = bo->vcl;
 		ctx.http_bereq = bo->bereq;
 		ctx.http_beresp = bo->beresp;
 		ctx.bo = bo;
 	}
 	ctx.ws = ws;
+	ctx.vsl = vsl;
 	ctx.method = method;
 	ctx.handling = &wrk->handling;
 	aws = WS_Snapshot(wrk->aws);
@@ -424,7 +421,12 @@ vcl_call_method(struct worker *wrk, struct req *req, struct busyobj *bo,
 	(void)func(&ctx);
 	VSLb(vsl, SLT_VCL_return, "%s", VCL_Return_Name(wrk->handling));
 	wrk->cur_method = 0;
-	WS_Reset(wrk->aws, aws);
+
+	/*
+	 * VCL/Vmods are not allowed to make permanent allocations from
+	 * wrk->aws, but they can reserve and return from it.
+	 */
+	assert(aws == WS_Snapshot(wrk->aws));
 }
 
 #define VCL_MET_MAC(func, upper, bitmap)				\
@@ -437,8 +439,7 @@ VCL_##func##_method(struct VCL_conf *vcl, struct worker *wrk,		\
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);				\
 	vcl_call_method(wrk, req, bo, ws, VCL_MET_ ## upper,		\
 	    vcl->func##_func);						\
-	assert((1U << wrk->handling) & bitmap);				\
-	AZ((1U << wrk->handling) & ~bitmap);			\
+	AN((1U << wrk->handling) & bitmap);				\
 }
 
 #include "tbl/vcl_returns.h"



From phk at FreeBSD.org  Wed Jun 25 20:29:20 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Wed, 25 Jun 2014 22:29:20 +0200
Subject: [master] 2e1f2a9 Nitpicking
Message-ID: <E1Wztom-0000yN-Su@project.varnish-software.com>


commit 2e1f2a9671b4238ff14e5132c818e3e00581f5d3
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Jun 25 10:22:21 2014 +0000

    Nitpicking

diff --git a/bin/varnishd/cache/cache_http.c b/bin/varnishd/cache/cache_http.c
index 2fc70be..ec1afae 100644
--- a/bin/varnishd/cache/cache_http.c
+++ b/bin/varnishd/cache/cache_http.c
@@ -639,7 +639,7 @@ http_EstimateWS(const struct http *fm, unsigned how, uint16_t *nhd)
 			continue;
 #include "tbl/http_headers.h"
 #undef HTTPH
-		l += PRNDUP(Tlen(fm->hd[u]) + 1);
+		l += PRNDUP(Tlen(fm->hd[u]) + 1L);
 		(*nhd)++;
 		// fm->hdf[u] |= HDF_COPY;
 	}



From fgsch at lodoss.net  Wed Jun 25 21:31:24 2014
From: fgsch at lodoss.net (Federico G. Schwindt)
Date: Wed, 25 Jun 2014 23:31:24 +0200
Subject: [master] 3802a63 Update source files
Message-ID: <E1Wzumq-00087p-Ld@project.varnish-software.com>


commit 3802a63db3df34b54223b2cb165f977f9bf86e41
Author: Federico G. Schwindt <fgsch at lodoss.net>
Date:   Wed Jun 25 11:34:53 2014 +0100

    Update source files

diff --git a/lib/libvmod_std/Makefile.phk b/lib/libvmod_std/Makefile.phk
index 1b3e34b..376a6f0 100644
--- a/lib/libvmod_std/Makefile.phk
+++ b/lib/libvmod_std/Makefile.phk
@@ -2,6 +2,7 @@
 VMOD_SRC += vmod_std.c
 VMOD_SRC += vmod_std_conversions.c
 VMOD_SRC += vmod_std_fileread.c
+VMOD_SRC += vmod_std_querysort.c
 
 TOPDIR= $(CURDIR)/../..
 include $(TOPDIR)/Makefile.inc.phk



From phk at FreeBSD.org  Thu Jun 26 07:06:23 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Thu, 26 Jun 2014 09:06:23 +0200
Subject: [master] 5c72666 Remove commented out EXP_NukeLRU(). We'll revisit
 this issue when we know what we're doing.
Message-ID: <E1X03lH-0006eR-Ap@project.varnish-software.com>


commit 5c7266623dcf4f23a2369c5faa32306e52cbeec0
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu Jun 26 07:05:59 2014 +0000

    Remove commented out EXP_NukeLRU().  We'll revisit this issue when
    we know what we're doing.

diff --git a/bin/varnishd/cache/cache_expire.c b/bin/varnishd/cache/cache_expire.c
index a893ecc..5f01f65 100644
--- a/bin/varnishd/cache/cache_expire.c
+++ b/bin/varnishd/cache/cache_expire.c
@@ -343,74 +343,6 @@ EXP_NukeOne(struct busyobj *bo, struct lru *lru)
 }
 
 /*--------------------------------------------------------------------
- * Nukes an entire LRU
- */
-
-#if 0		// Not yet
-
-#define NUKEBUF 10	/* XXX: Randomly chosen to be bigger than one */
-
-void
-EXP_NukeLRU(struct worker *wrk, struct vsl_log *vsl, struct lru *lru)
-{
-	struct objcore *oc;
-	struct objcore *oc_array[NUKEBUF];
-	struct object *o;
-	int i, n;
-	double t;
-
-	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
-	CHECK_OBJ_NOTNULL(lru, LRU_MAGIC);
-
-	memset(oc_array, 0, sizeof oc_array);
-
-	t = VTIM_real();
-	Lck_Lock(&lru->mtx);
-	while (!VTAILQ_EMPTY(&lru->lru_head)) {
-		Lck_Lock(&exphdl->mtx);
-		n = 0;
-		while (n < NUKEBUF) {
-			oc = VTAILQ_FIRST(&lru->lru_head);
-			if (oc == NULL)
-				break;
-			CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
-			assert(oc_getlru(oc) == lru);
-
-			/* Remove from the LRU and binheap */
-			VTAILQ_REMOVE(&lru->lru_head, oc, lru_list);
-			assert(oc->timer_idx != BINHEAP_NOIDX);
-			binheap_delete(exphdl->heap, oc->timer_idx);
-			assert(oc->timer_idx == BINHEAP_NOIDX);
-
-			oc_array[n++] = oc;
-			VSC_C_main->n_lru_nuked++;
-		}
-		assert(n > 0);
-		Lck_Unlock(&exphdl->mtx);
-		Lck_Unlock(&lru->mtx);
-
-		for (i = 0; i < n; i++) {
-			oc = oc_array[i];
-			o = oc_getobj(&wrk->stats, oc);
-			/* XXX: Not documented in vsl_tags.h */
-			VSLb(vsl, SLT_ExpKill, "x=%u t=%.0f LRU",
-			    oc_getxid(&wrk->stats, oc) & VSL_IDENTMASK,
-			    EXP_Ttl(NULL, o) - t);
-			o->exp.ttl = 0.0;
-			(void)HSH_DerefObjCore(&wrk->stats, &oc);
-		}
-
-		Lck_Lock(&lru->mtx);
-	}
-	Lck_Unlock(&lru->mtx);
-
-	Pool_Sumstat(wrk);
-}
-
-#endif
-
-
-/*--------------------------------------------------------------------
  * Handle stuff in the inbox
  */
 



From phk at FreeBSD.org  Thu Jun 26 07:26:41 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Thu, 26 Jun 2014 09:26:41 +0200
Subject: [master] 9f9c512 Fold include/persistent.h into
 storage/storage_persistent.h now that it clearly isn't going to be a much
 used public interface.
Message-ID: <E1X044v-0000GG-Dz@project.varnish-software.com>


commit 9f9c512751a8ae747a90b6ee4428ad963b1f4fce
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu Jun 26 07:25:24 2014 +0000

    Fold include/persistent.h into storage/storage_persistent.h now that it
    clearly isn't going to be a much used public interface.

diff --git a/bin/varnishd/storage/storage_persistent.c b/bin/varnishd/storage/storage_persistent.c
index 6633799..694d7fe 100644
--- a/bin/varnishd/storage/storage_persistent.c
+++ b/bin/varnishd/storage/storage_persistent.c
@@ -52,7 +52,6 @@
 #include "vsha256.h"
 #include "vtim.h"
 
-#include "persistent.h"
 #include "storage/storage_persistent.h"
 
 /*--------------------------------------------------------------------*/
diff --git a/bin/varnishd/storage/storage_persistent.h b/bin/varnishd/storage/storage_persistent.h
index cb2efb3..729570e 100644
--- a/bin/varnishd/storage/storage_persistent.h
+++ b/bin/varnishd/storage/storage_persistent.h
@@ -33,6 +33,122 @@
  * XXX: Do we ever free the LRU-lists ?
  */
 
+/*
+ *
+ * Overall layout:
+ *
+ *	struct smp_ident;		Identification and geometry
+ *	sha256[...]			checksum of same
+ *
+ *	struct smp_sign;
+ *	banspace_1;			First ban-space
+ *	sha256[...]			checksum of same
+ *
+ *	struct smp_sign;
+ *	banspace_2;			Second ban-space
+ *	sha256[...]			checksum of same
+ *
+ *	struct smp_sign;
+ *	struct smp_segment_1[N];	First Segment table
+ *	sha256[...]			checksum of same
+ *
+ *	struct smp_sign;
+ *	struct smp_segment_2[N];	Second Segment table
+ *	sha256[...]			checksum of same
+ *
+ *	N segments {
+ *		struct smp_sign;
+ *		struct smp_object[M]	Objects in segment
+ *		sha256[...]		checksum of same
+ *		objspace
+ *	}
+ *
+ */
+
+/*
+ * The identblock is located in the first sector of the storage space.
+ * This is written once and not subsequently modified in normal operation.
+ * It is immediately followed by a SHA256sum of the structure, as stored.
+ */
+
+struct smp_ident {
+	char			ident[32];	/* Human readable ident
+						 * so people and programs
+						 * can tell what the file
+						 * or device contains.
+						 */
+
+	uint32_t		byte_order;	/* 0x12345678 */
+
+	uint32_t		size;		/* sizeof(struct smp_ident) */
+
+	uint32_t		major_version;
+
+	uint32_t		unique;
+
+	uint32_t		align;		/* alignment in silo */
+
+	uint32_t		granularity;	/* smallest ... in bytes */
+
+	uint64_t		mediasize;	/* ... in bytes */
+
+	uint64_t		stuff[6];	/* pointers to stuff */
+#define	SMP_BAN1_STUFF		0
+#define	SMP_BAN2_STUFF		1
+#define	SMP_SEG1_STUFF		2
+#define	SMP_SEG2_STUFF		3
+#define	SMP_SPC_STUFF		4
+#define	SMP_END_STUFF		5
+};
+
+/*
+ * The size of smp_ident should be fixed and constant across all platforms.
+ * We enforce that with the following #define and an assert in smp_init()
+ */
+#define SMP_IDENT_SIZE		112
+
+#define SMP_IDENT_STRING	"Varnish Persistent Storage Silo"
+
+/*
+ * This is used to sign various bits on the disk.
+ */
+
+struct smp_sign {
+	char			ident[8];
+	uint32_t		unique;
+	uint64_t		mapped;
+	/* The length field is the length of the signed data only
+	 * (does not include struct smp_sign) */
+	uint64_t		length;		/* NB: Must be last */
+};
+
+#define SMP_SIGN_SPACE		(sizeof(struct smp_sign) + SHA256_LEN)
+
+/*
+ * A segment pointer.
+ */
+
+struct smp_segptr {
+	uint64_t		offset;		/* rel to silo */
+	uint64_t		length;		/* rel to offset */
+	uint64_t		objlist;	/* rel to silo */
+	uint32_t		lobjlist;	/* len of objlist */
+};
+
+/*
+ * An object descriptor
+ *
+ * A positive ttl is obj.ttl with obj.grace being NAN
+ * A negative ttl is - (obj.ttl + obj.grace)
+ */
+
+struct smp_object {
+	uint8_t			hash[32];	/* really: DIGEST_LEN */
+	double			ttl;
+	double			ban;
+	uint64_t		ptr;		/* rel to silo */
+};
+
 #define ASSERT_SILO_THREAD(sc) \
     do {assert(pthread_equal(pthread_self(), (sc)->thread));} while (0)
 
diff --git a/bin/varnishd/storage/storage_persistent_mgt.c b/bin/varnishd/storage/storage_persistent_mgt.c
index c5b1dfd..f99a6ae 100644
--- a/bin/varnishd/storage/storage_persistent_mgt.c
+++ b/bin/varnishd/storage/storage_persistent_mgt.c
@@ -46,7 +46,6 @@
 
 #include "vsha256.h"
 
-#include "persistent.h"
 #include "storage/storage_persistent.h"
 
 #ifndef MAP_NOCORE
@@ -145,7 +144,7 @@ smp_mgt_init(struct stevedore *parent, int ac, char * const *av)
 	SIZOF(struct smp_object);
 #undef SIZOF
 
-	/* See comments in persistent.h */
+	/* See comments in storage_persistent.h */
 	assert(sizeof(struct smp_ident) == SMP_IDENT_SIZE);
 
 	/* Allocate softc */
diff --git a/bin/varnishd/storage/storage_persistent_silo.c b/bin/varnishd/storage/storage_persistent_silo.c
index f66cd52..ccd9991 100644
--- a/bin/varnishd/storage/storage_persistent_silo.c
+++ b/bin/varnishd/storage/storage_persistent_silo.c
@@ -45,7 +45,6 @@
 #include "vsha256.h"
 #include "vtim.h"
 
-#include "persistent.h"
 #include "storage/storage_persistent.h"
 
 /*--------------------------------------------------------------------
diff --git a/bin/varnishd/storage/storage_persistent_subr.c b/bin/varnishd/storage/storage_persistent_subr.c
index bfe6b40..50e36c2 100644
--- a/bin/varnishd/storage/storage_persistent_subr.c
+++ b/bin/varnishd/storage/storage_persistent_subr.c
@@ -46,7 +46,6 @@
 
 #include "vsha256.h"
 
-#include "persistent.h"
 #include "storage/storage_persistent.h"
 
 /*--------------------------------------------------------------------
diff --git a/include/Makefile.am b/include/Makefile.am
index 50fed78..2091ab5 100644
--- a/include/Makefile.am
+++ b/include/Makefile.am
@@ -44,7 +44,6 @@ nobase_noinst_HEADERS = \
 	compat/execinfo.h \
 	flopen.h \
 	libvcc.h \
-	persistent.h \
 	vcli_common.h \
 	vcli_priv.h \
 	vcli_serve.h \
diff --git a/include/persistent.h b/include/persistent.h
deleted file mode 100644
index f0cb7f5..0000000
--- a/include/persistent.h
+++ /dev/null
@@ -1,144 +0,0 @@
-/*-
- * Copyright (c) 2008-2011 Varnish Software AS
- * All rights reserved.
- *
- * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- *
- */
-
-/*
- *
- * Overall layout:
- *
- *	struct smp_ident;		Identification and geometry
- *	sha256[...]			checksum of same
- *
- *	struct smp_sign;
- *	banspace_1;			First ban-space
- *	sha256[...]			checksum of same
- *
- *	struct smp_sign;
- *	banspace_2;			Second ban-space
- *	sha256[...]			checksum of same
- *
- *	struct smp_sign;
- *	struct smp_segment_1[N];	First Segment table
- *	sha256[...]			checksum of same
- *
- *	struct smp_sign;
- *	struct smp_segment_2[N];	Second Segment table
- *	sha256[...]			checksum of same
- *
- *	N segments {
- *		struct smp_sign;
- *		struct smp_object[M]	Objects in segment
- *		sha256[...]		checksum of same
- *		objspace
- *	}
- *
- */
-
-/*
- * The identblock is located in the first sector of the storage space.
- * This is written once and not subsequently modified in normal operation.
- * It is immediately followed by a SHA256sum of the structure, as stored.
- */
-
-struct smp_ident {
-	char			ident[32];	/* Human readable ident
-						 * so people and programs
-						 * can tell what the file
-						 * or device contains.
-						 */
-
-	uint32_t		byte_order;	/* 0x12345678 */
-
-	uint32_t		size;		/* sizeof(struct smp_ident) */
-
-	uint32_t		major_version;
-
-	uint32_t		unique;
-
-	uint32_t		align;		/* alignment in silo */
-
-	uint32_t		granularity;	/* smallest ... in bytes */
-
-	uint64_t		mediasize;	/* ... in bytes */
-
-	uint64_t		stuff[6];	/* pointers to stuff */
-#define	SMP_BAN1_STUFF		0
-#define	SMP_BAN2_STUFF		1
-#define	SMP_SEG1_STUFF		2
-#define	SMP_SEG2_STUFF		3
-#define	SMP_SPC_STUFF		4
-#define	SMP_END_STUFF		5
-};
-
-/*
- * The size of smp_ident should be fixed and constant across all platforms.
- * We enforce that with the following #define and an assert in smp_init()
- */
-#define SMP_IDENT_SIZE		112
-
-#define SMP_IDENT_STRING	"Varnish Persistent Storage Silo"
-
-/*
- * This is used to sign various bits on the disk.
- */
-
-struct smp_sign {
-	char			ident[8];
-	uint32_t		unique;
-	uint64_t		mapped;
-	/* The length field is the length of the signed data only
-	 * (does not include struct smp_sign) */
-	uint64_t		length;		/* NB: Must be last */
-};
-
-#define SMP_SIGN_SPACE		(sizeof(struct smp_sign) + SHA256_LEN)
-
-/*
- * A segment pointer.
- */
-
-struct smp_segptr {
-	uint64_t		offset;		/* rel to silo */
-	uint64_t		length;		/* rel to offset */
-	uint64_t		objlist;	/* rel to silo */
-	uint32_t		lobjlist;	/* len of objlist */
-};
-
-/*
- * An object descriptor
- *
- * A positive ttl is obj.ttl with obj.grace being NAN
- * A negative ttl is - (obj.ttl + obj.grace)
- */
-
-struct smp_object {
-	uint8_t			hash[32];	/* really: DIGEST_LEN */
-	double			ttl;
-	double			ban;
-	uint64_t		ptr;		/* rel to silo */
-};



From phk at FreeBSD.org  Thu Jun 26 07:55:43 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Thu, 26 Jun 2014 09:55:43 +0200
Subject: [master] 74889dd Make persistent save the entire struct exp rather
 than only the deadline, in preparation for moving exp from obj to objcore
Message-ID: <E1X04X1-0003DE-Pp@project.varnish-software.com>


commit 74889dda9dbb3e6f4a9098ebbfd91badbff34f0a
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu Jun 26 07:55:03 2014 +0000

    Make persistent save the entire struct exp rather than only the deadline,
    in preparation for moving exp from obj to objcore

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index 410aa7e..baca07c 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -928,6 +928,7 @@ extern pthread_t cli_thread;
 void EXP_Clr(struct exp *e);
 
 double EXP_Ttl(const struct req *, const struct object*);
+double EXP_When(const struct exp *exp);
 void EXP_Insert(struct objcore *oc);
 void EXP_Inject(struct objcore *oc, struct lru *lru, double when);
 void EXP_Init(void);
diff --git a/bin/varnishd/cache/cache_expire.c b/bin/varnishd/cache/cache_expire.c
index 5f01f65..9ae6558 100644
--- a/bin/varnishd/cache/cache_expire.c
+++ b/bin/varnishd/cache/cache_expire.c
@@ -87,17 +87,17 @@ EXP_Ttl(const struct req *req, const struct object *o)
 }
 
 /*--------------------------------------------------------------------
- * Calculate when we should wake up for this object
+ * Calculate when this object is no longer useful
  */
 
-static double
-exp_when(const struct object *o)
+double
+EXP_When(const struct exp *e)
 {
 	double when;
 
-	CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC);
-
-	when = o->exp.t_origin + o->exp.ttl + o->exp.grace + o->exp.keep;
+	if (e->t_origin == 0)
+		return (0.);
+	when = e->t_origin + e->ttl + e->grace + e->keep;
 	AZ(isnan(when));
 	return (when);
 }
@@ -243,7 +243,7 @@ EXP_Rearm(struct object *o, double now, double ttl, double grace, double keep)
 	if (!isnan(keep))
 		o->exp.keep = keep;
 
-	when = exp_when(o);
+	when = EXP_When(&o->exp);
 
 	VSL(SLT_ExpKill, 0, "EXP_Rearm p=%p E=%.9f e=%.9f f=0x%x", oc,
 	    oc->timer_when, when, oc->flags);
@@ -391,7 +391,7 @@ exp_inbox(struct exp_priv *ep, struct objcore *oc, double now)
 	if (flags & OC_EF_MOVE) {
 		o = oc_getobj(&ep->wrk->stats, oc);
 		CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC);
-		oc->timer_when = exp_when(o);
+		oc->timer_when = EXP_When(&o->exp);
 		oc_updatemeta(oc);
 	}
 
diff --git a/bin/varnishd/storage/storage_persistent.c b/bin/varnishd/storage/storage_persistent.c
index 694d7fe..f1680a3 100644
--- a/bin/varnishd/storage/storage_persistent.c
+++ b/bin/varnishd/storage/storage_persistent.c
@@ -473,7 +473,7 @@ smp_allocx(struct stevedore *st, size_t min_size, size_t max_size,
 			sc->next_top -= sizeof(**so);
 			*so = (void*)(sc->base + sc->next_top);
 			/* Render this smp_object mostly harmless */
-			(*so)->ttl = 0.;
+			EXP_Clr(&(*so)->exp);
 			(*so)->ban = 0.;
 			(*so)->ptr = 0;
 			sg->objs = *so;
@@ -545,7 +545,7 @@ smp_allocobj(struct stevedore *stv, struct busyobj *bo,
 	/* We have to do this somewhere, might as well be here... */
 	assert(sizeof so->hash == DIGEST_LEN);
 	memcpy(so->hash, oc->objhead->digest, DIGEST_LEN);
-	so->ttl = oc->timer_when;
+	so->exp = bo->exp;
 	so->ptr = (uint8_t*)o - sc->base;
 	so->ban = BAN_Time(oc->ban);
 
diff --git a/bin/varnishd/storage/storage_persistent.h b/bin/varnishd/storage/storage_persistent.h
index 729570e..2d46297 100644
--- a/bin/varnishd/storage/storage_persistent.h
+++ b/bin/varnishd/storage/storage_persistent.h
@@ -144,7 +144,7 @@ struct smp_segptr {
 
 struct smp_object {
 	uint8_t			hash[32];	/* really: DIGEST_LEN */
-	double			ttl;
+	struct exp		exp;
 	double			ban;
 	uint64_t		ptr;		/* rel to silo */
 };
diff --git a/bin/varnishd/storage/storage_persistent_silo.c b/bin/varnishd/storage/storage_persistent_silo.c
index ccd9991..4ceaa6a 100644
--- a/bin/varnishd/storage/storage_persistent_silo.c
+++ b/bin/varnishd/storage/storage_persistent_silo.c
@@ -148,7 +148,7 @@ smp_load_seg(struct worker *wrk, const struct smp_sc *sc,
 	/* Clear the bogus "hold" count */
 	sg->nobj = 0;
 	for (;no > 0; so++,no--) {
-		if (so->ttl == 0 || so->ttl < t_now)
+		if (EXP_When(&so->exp) < t_now)
 			continue;
 		ALLOC_OBJ(oc, OBJCORE_MAGIC);
 		AN(oc);
@@ -157,7 +157,7 @@ smp_load_seg(struct worker *wrk, const struct smp_sc *sc,
 		smp_init_oc(oc, sg, no);
 		oc->ban = BAN_RefBan(oc, so->ban, sc->tailban);
 		HSH_Insert(wrk, so->hash, oc);
-		EXP_Inject(oc, sg->lru, so->ttl);
+		EXP_Inject(oc, sg->lru, EXP_When(&so->exp));
 		sg->nobj++;
 	}
 	Pool_Sumstat(wrk);
@@ -463,7 +463,7 @@ smp_oc_getobj(struct dstat *ds, struct objcore *oc)
 
 		if(bad) {
 			o->exp.ttl = -1;
-			so->ttl = 0;
+			EXP_Clr(&so->exp);
 		}
 
 		sg->nfixed++;
@@ -495,11 +495,11 @@ smp_oc_updatemeta(struct objcore *oc)
 		/* Lock necessary, we might race close_seg */
 		Lck_Lock(&sg->sc->mtx);
 		so->ban = BAN_Time(oc->ban);
-		so->ttl = oc->timer_when;
+		so->exp = o->exp;;
 		Lck_Unlock(&sg->sc->mtx);
 	} else {
 		so->ban = BAN_Time(oc->ban);
-		so->ttl = oc->timer_when;
+		so->exp = o->exp;;
 	}
 }
 
@@ -515,7 +515,7 @@ smp_oc_freeobj(struct objcore *oc)
 	so = smp_find_so(sg, oc->priv2);
 
 	Lck_Lock(&sg->sc->mtx);
-	so->ttl = 0;
+	EXP_Clr(&so->exp);
 	so->ptr = 0;
 
 	assert(sg->nobj > 0);



From phk at FreeBSD.org  Thu Jun 26 15:03:03 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Thu, 26 Jun 2014 17:03:03 +0200
Subject: [master] d299d76 Cherry-picking from Martins obj/objcore/exp patch:
Message-ID: <E1X0BCZ-0007s4-7J@project.varnish-software.com>


commit d299d76867e57c19f2d86115f4335a19374339f4
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu Jun 26 09:24:00 2014 +0000

    Cherry-picking from Martins obj/objcore/exp patch:
    
    move n_object accounting responsibility to stevedores.
    
    Fix n_object accounting detail in persistent.

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index baca07c..3721d06 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -403,7 +403,7 @@ struct storage {
 typedef struct object *getobj_f(struct dstat *ds, struct objcore *oc);
 typedef unsigned getxid_f(struct dstat *ds, struct objcore *oc);
 typedef void updatemeta_f(struct objcore *oc);
-typedef void freeobj_f(struct objcore *oc);
+typedef void freeobj_f(struct dstat *ds, struct objcore *oc);
 typedef struct lru *getlru_f(const struct objcore *oc);
 
 struct objcore_methods {
@@ -479,13 +479,14 @@ oc_updatemeta(struct objcore *oc)
 }
 
 static inline void
-oc_freeobj(struct objcore *oc)
+oc_freeobj(struct dstat *ds, struct objcore *oc)
 {
 
+	AN(ds);
 	CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
 	AN(oc->methods);
 	AN(oc->methods->freeobj);
-	oc->methods->freeobj(oc);
+	oc->methods->freeobj(ds, oc);
 }
 
 static inline struct lru *
diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c
index cd2b40d..1a34043 100644
--- a/bin/varnishd/cache/cache_fetch.c
+++ b/bin/varnishd/cache/cache_fetch.c
@@ -534,9 +534,8 @@ vbf_stp_fetch(struct worker *wrk, struct busyobj *bo)
 	if (bo->failed && !bo->do_stream) {
 		assert(bo->state < BOS_STREAM);
 		if (bo->fetch_obj != NULL) {
-			oc_freeobj(bo->fetch_objcore);
+			oc_freeobj(bo->stats, bo->fetch_objcore);
 			bo->fetch_obj = NULL;
-			bo->stats->n_object--;
 		}
 		return (F_STP_ERROR);
 	}
diff --git a/bin/varnishd/cache/cache_hash.c b/bin/varnishd/cache/cache_hash.c
index 73acf01..66fe0d5 100644
--- a/bin/varnishd/cache/cache_hash.c
+++ b/bin/varnishd/cache/cache_hash.c
@@ -804,10 +804,8 @@ HSH_DerefObjCore(struct dstat *ds, struct objcore **ocp)
 	BAN_DestroyObj(oc);
 	AZ(oc->ban);
 
-	if (oc->methods != NULL) {
-		oc_freeobj(oc);
-		ds->n_object--;
-	}
+	if (oc->methods != NULL)
+		oc_freeobj(ds, oc);
 	FREE_OBJ(oc);
 
 	ds->n_objectcore--;
diff --git a/bin/varnishd/storage/stevedore.c b/bin/varnishd/storage/stevedore.c
index 5e54ed6..cf75a48 100644
--- a/bin/varnishd/storage/stevedore.c
+++ b/bin/varnishd/storage/stevedore.c
@@ -69,11 +69,12 @@ default_oc_getobj(struct dstat *ds, struct objcore *oc)
 	return (o);
 }
 
-static void
-default_oc_freeobj(struct objcore *oc)
+static void __match_proto__(freeobj_f)
+default_oc_freeobj(struct dstat *ds, struct objcore *oc)
 {
 	struct object *o;
 
+	AN(ds);
 	CAST_OBJ_NOTNULL(o, oc->priv, OBJECT_MAGIC);
 	oc->priv = NULL;
 	oc->methods = NULL;
@@ -81,9 +82,11 @@ default_oc_freeobj(struct objcore *oc)
 
 	STV_Freestore(o);
 	STV_free(o->objstore);
+
+	ds->n_object--;
 }
 
-static struct lru *
+static struct lru * __match_proto__(getlru_f)
 default_oc_getlru(const struct objcore *oc)
 {
 	struct stevedore *stv;
@@ -285,7 +288,6 @@ STV_MkObject(struct stevedore *stv, struct busyobj *bo,
 	o->http->magic = HTTP_MAGIC;
 	o->exp = bo->exp;
 	VTAILQ_INIT(&o->store);
-	bo->stats->n_object++;
 
 	o->objcore = bo->fetch_objcore;
 
@@ -321,6 +323,7 @@ stv_default_allocobj(struct stevedore *stv, struct busyobj *bo,
 	o = STV_MkObject(stv, bo, st->ptr, ltot, soc);
 	CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC);
 	o->objstore = st;
+	bo->stats->n_object++;
 	return (o);
 }
 
diff --git a/bin/varnishd/storage/storage_persistent.c b/bin/varnishd/storage/storage_persistent.c
index f1680a3..fcff5be 100644
--- a/bin/varnishd/storage/storage_persistent.c
+++ b/bin/varnishd/storage/storage_persistent.c
@@ -534,6 +534,7 @@ smp_allocobj(struct stevedore *stv, struct busyobj *bo,
 	o = STV_MkObject(stv, bo, st->ptr, ltot, soc);
 	CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC);
 	o->objstore = st;
+	bo->stats->n_object++;
 
 	oc = o->objcore;
 	CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
diff --git a/bin/varnishd/storage/storage_persistent_silo.c b/bin/varnishd/storage/storage_persistent_silo.c
index 4ceaa6a..5627f3f 100644
--- a/bin/varnishd/storage/storage_persistent_silo.c
+++ b/bin/varnishd/storage/storage_persistent_silo.c
@@ -503,12 +503,13 @@ smp_oc_updatemeta(struct objcore *oc)
 	}
 }
 
-static void __match_proto__()
-smp_oc_freeobj(struct objcore *oc)
+static void __match_proto__(freeobj_f)
+smp_oc_freeobj(struct dstat *ds, struct objcore *oc)
 {
 	struct smp_seg *sg;
 	struct smp_object *so;
 
+	AN(ds);
 	CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
 
 	CAST_OBJ_NOTNULL(sg, oc->priv, SMP_SEG_MAGIC);
@@ -519,9 +520,14 @@ smp_oc_freeobj(struct objcore *oc)
 	so->ptr = 0;
 
 	assert(sg->nobj > 0);
-	assert(sg->nfixed > 0);
 	sg->nobj--;
-	sg->nfixed--;
+	if (oc->flags & OC_F_NEEDFIXUP) {
+		ds->n_vampireobject--;
+	} else {
+		assert(sg->nfixed > 0);
+		sg->nfixed--;
+		ds->n_object--;
+	}
 
 	Lck_Unlock(&sg->sc->mtx);
 }
@@ -530,7 +536,7 @@ smp_oc_freeobj(struct objcore *oc)
  * Find the per-segment lru list for this object
  */
 
-static struct lru *
+static struct lru * __match_proto__(getlru_f)
 smp_oc_getlru(const struct objcore *oc)
 {
 	struct smp_seg *sg;



From phk at FreeBSD.org  Thu Jun 26 15:03:03 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Thu, 26 Jun 2014 17:03:03 +0200
Subject: [master] 8070f32 Make EXP_Ttl() take a pointer to struct exp rather
 than struct object
Message-ID: <E1X0BCZ-0007sB-9R@project.varnish-software.com>


commit 8070f32925bd01fd8543f0826d5ff477883c53b8
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu Jun 26 10:32:53 2014 +0000

    Make EXP_Ttl() take a pointer to struct exp rather than struct object

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index 3721d06..60d7b26 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -928,7 +928,7 @@ extern pthread_t cli_thread;
 /* cache_expiry.c */
 void EXP_Clr(struct exp *e);
 
-double EXP_Ttl(const struct req *, const struct object*);
+double EXP_Ttl(const struct req *, const struct exp*);
 double EXP_When(const struct exp *exp);
 void EXP_Insert(struct objcore *oc);
 void EXP_Inject(struct objcore *oc, struct lru *lru, double when);
diff --git a/bin/varnishd/cache/cache_expire.c b/bin/varnishd/cache/cache_expire.c
index 9ae6558..bbb19a3 100644
--- a/bin/varnishd/cache/cache_expire.c
+++ b/bin/varnishd/cache/cache_expire.c
@@ -76,14 +76,14 @@ EXP_Clr(struct exp *e)
  */
 
 double
-EXP_Ttl(const struct req *req, const struct object *o)
+EXP_Ttl(const struct req *req, const struct exp *e)
 {
 	double r;
 
-	r = o->exp.ttl;
+	r = e->ttl;
 	if (req != NULL && req->d_ttl > 0. && req->d_ttl < r)
 		r = req->d_ttl;
-	return (o->exp.t_origin + r);
+	return (e->t_origin + r);
 }
 
 /*--------------------------------------------------------------------
@@ -468,7 +468,7 @@ exp_expire(struct exp_priv *ep, double now)
 	CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC);
 	VSLb(&ep->vsl, SLT_ExpKill, "EXP_Expired x=%u t=%.0f",
 	    oc_getxid(&ep->wrk->stats, oc) & VSL_IDENTMASK,
-	    EXP_Ttl(NULL, o) - now);
+	    EXP_Ttl(NULL, &o->exp) - now);
 	(void)HSH_DerefObjCore(&ep->wrk->stats, &oc);
 	return (0);
 }
diff --git a/bin/varnishd/cache/cache_hash.c b/bin/varnishd/cache/cache_hash.c
index 66fe0d5..d6e404c 100644
--- a/bin/varnishd/cache/cache_hash.c
+++ b/bin/varnishd/cache/cache_hash.c
@@ -430,7 +430,7 @@ HSH_Lookup(struct req *req, struct objcore **ocp, struct objcore **bocp,
 		if (o->vary != NULL && !VRY_Match(req, o->vary))
 			continue;
 
-		if (EXP_Ttl(req, o) >= req->t_req) {
+		if (EXP_Ttl(req, &o->exp) >= req->t_req) {
 			/* If still valid, use it */
 			assert(oh->refcnt > 1);
 			assert(oc->objhead == oh);



From phk at FreeBSD.org  Thu Jun 26 15:03:03 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Thu, 26 Jun 2014 17:03:03 +0200
Subject: [master] e654cc8 Move busyobj->exp to bo->fetch_objcore->exp
Message-ID: <E1X0BCZ-0007sK-Bk@project.varnish-software.com>


commit e654cc8a7b6b9103201be957a36931a39fea610a
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu Jun 26 10:54:55 2014 +0000

    Move busyobj->exp to bo->fetch_objcore->exp

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index 60d7b26..9854cc8 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -425,6 +425,8 @@ struct objcore {
 	struct busyobj		*busyobj;
 	double			timer_when;
 
+	struct exp		exp;
+
 	uint16_t		flags;
 #define OC_F_BUSY		(1<<1)
 #define OC_F_PASS		(1<<2)
@@ -553,7 +555,6 @@ struct busyobj {
 	struct objcore		*fetch_objcore;
 	struct object		*fetch_obj;
 
-	struct exp		exp;
 	struct http_conn	htc;
 
 	struct pool_task	fetch_task;
diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c
index 1a34043..1b824d5 100644
--- a/bin/varnishd/cache/cache_fetch.c
+++ b/bin/varnishd/cache/cache_fetch.c
@@ -49,13 +49,15 @@ static struct object *
 vbf_allocobj(struct busyobj *bo, unsigned l, uint16_t nhttp)
 {
 	struct object *obj;
+	struct objcore *oc;
 	const char *storage_hint;
 	double lifetime;
 
 	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
-	CHECK_OBJ_NOTNULL(bo->fetch_objcore, OBJCORE_MAGIC);
+	oc = bo->fetch_objcore;
+	CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
 
-	lifetime = bo->exp.ttl + bo->exp.grace + bo->exp.keep;
+	lifetime = oc->exp.ttl + oc->exp.grace + oc->exp.keep;
 
 	if (bo->uncacheable || lifetime < cache_param->shortlived)
 		storage_hint = TRANSIENT_STORAGE;
@@ -77,10 +79,10 @@ vbf_allocobj(struct busyobj *bo, unsigned l, uint16_t nhttp)
 	 * on Transient storage.
 	 */
 
-	if (bo->exp.ttl > cache_param->shortlived)
-		bo->exp.ttl = cache_param->shortlived;
-	bo->exp.grace = 0.0;
-	bo->exp.keep = 0.0;
+	if (oc->exp.ttl > cache_param->shortlived)
+		oc->exp.ttl = cache_param->shortlived;
+	oc->exp.grace = 0.0;
+	oc->exp.keep = 0.0;
 	obj = STV_NewObject(bo, TRANSIENT_STORAGE, l, nhttp);
 	return (obj);
 }
@@ -162,7 +164,7 @@ vbf_beresp2obj(struct busyobj *bo)
 	if (http_GetHdr(hp, H_Last_Modified, &b))
 		obj->last_modified = VTIM_parse(b);
 	else
-		obj->last_modified = floor(bo->exp.t_origin);
+		obj->last_modified = floor(bo->fetch_objcore->exp.t_origin);
 
 	/* Disassociate the obj from the bo's workspace */
 	hp2->ws = NULL;
@@ -348,12 +350,12 @@ vbf_stp_startfetch(struct worker *wrk, struct busyobj *bo)
 	/*
 	 * What does RFC2616 think about TTL ?
 	 */
-	EXP_Clr(&bo->exp);
+	EXP_Clr(&bo->fetch_objcore->exp);
 	RFC2616_Ttl(bo, now);
 
 	/* private objects have negative TTL */
 	if (bo->fetch_objcore->flags & OC_F_PRIVATE)
-		bo->exp.ttl = -1.;
+		bo->fetch_objcore->exp.ttl = -1.;
 
 	AZ(bo->do_esi);
 
@@ -696,10 +698,10 @@ vbf_stp_error(struct worker *wrk, struct busyobj *bo)
 	http_PrintfHeader(bo->beresp, "Date: %s", time_str);
 	http_SetHeader(bo->beresp, "Server: Varnish");
 
-	bo->exp.t_origin = bo->t_prev;
-	bo->exp.ttl = 0;
-	bo->exp.grace = 0;
-	bo->exp.keep = 0;
+	bo->fetch_objcore->exp.t_origin = bo->t_prev;
+	bo->fetch_objcore->exp.ttl = 0;
+	bo->fetch_objcore->exp.grace = 0;
+	bo->fetch_objcore->exp.keep = 0;
 
 	VCL_backend_error_method(bo->vcl, wrk, NULL, bo, bo->bereq->ws);
 
diff --git a/bin/varnishd/cache/cache_rfc2616.c b/bin/varnishd/cache/cache_rfc2616.c
index bb0a85e..8ada968 100644
--- a/bin/varnishd/cache/cache_rfc2616.c
+++ b/bin/varnishd/cache/cache_rfc2616.c
@@ -72,7 +72,7 @@ RFC2616_Ttl(struct busyobj *bo, double now)
 	struct exp *expp;
 
 	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
-	expp = &bo->exp;
+	expp = &bo->fetch_objcore->exp;
 
 	hp = bo->beresp;
 
diff --git a/bin/varnishd/cache/cache_vrt_var.c b/bin/varnishd/cache/cache_vrt_var.c
index c11f2d9..9533f75 100644
--- a/bin/varnishd/cache/cache_vrt_var.c
+++ b/bin/varnishd/cache/cache_vrt_var.c
@@ -486,12 +486,12 @@ VRT_DO_EXP_R(obj, ctx->req->obj->exp, ttl,
 VRT_DO_EXP_R(obj, ctx->req->obj->exp, grace, 0)
 VRT_DO_EXP_R(obj, ctx->req->obj->exp, keep, 0)
 
-VRT_DO_EXP_L(beresp, ctx->bo->exp, ttl)
-VRT_DO_EXP_R(beresp, ctx->bo->exp, ttl, 0)
-VRT_DO_EXP_L(beresp, ctx->bo->exp, grace)
-VRT_DO_EXP_R(beresp, ctx->bo->exp, grace, 0)
-VRT_DO_EXP_L(beresp, ctx->bo->exp, keep)
-VRT_DO_EXP_R(beresp, ctx->bo->exp, keep, 0)
+VRT_DO_EXP_L(beresp, ctx->bo->fetch_objcore->exp, ttl)
+VRT_DO_EXP_R(beresp, ctx->bo->fetch_objcore->exp, ttl, 0)
+VRT_DO_EXP_L(beresp, ctx->bo->fetch_objcore->exp, grace)
+VRT_DO_EXP_R(beresp, ctx->bo->fetch_objcore->exp, grace, 0)
+VRT_DO_EXP_L(beresp, ctx->bo->fetch_objcore->exp, keep)
+VRT_DO_EXP_R(beresp, ctx->bo->fetch_objcore->exp, keep, 0)
 
 /*--------------------------------------------------------------------
  * [be]req.xid
diff --git a/bin/varnishd/storage/stevedore.c b/bin/varnishd/storage/stevedore.c
index cf75a48..ce9c007 100644
--- a/bin/varnishd/storage/stevedore.c
+++ b/bin/varnishd/storage/stevedore.c
@@ -286,7 +286,7 @@ STV_MkObject(struct stevedore *stv, struct busyobj *bo,
 
 	HTTP_Setup(o->http, bo->ws_o, bo->vsl, SLT_ObjMethod);
 	o->http->magic = HTTP_MAGIC;
-	o->exp = bo->exp;
+	o->exp = bo->fetch_objcore->exp;
 	VTAILQ_INIT(&o->store);
 
 	o->objcore = bo->fetch_objcore;
@@ -344,6 +344,7 @@ STV_NewObject(struct busyobj *bo, const char *hint,
 	int i;
 
 	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
+	CHECK_OBJ_NOTNULL(bo->fetch_objcore, OBJCORE_MAGIC);
 	assert(wsl > 0);
 	wsl = PRNDUP(wsl);
 
diff --git a/bin/varnishd/storage/storage_persistent.c b/bin/varnishd/storage/storage_persistent.c
index fcff5be..008afe7 100644
--- a/bin/varnishd/storage/storage_persistent.c
+++ b/bin/varnishd/storage/storage_persistent.c
@@ -517,10 +517,13 @@ smp_allocobj(struct stevedore *stv, struct busyobj *bo,
 	unsigned objidx;
 
 	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
-	if (bo->fetch_objcore == NULL)
-		return (NULL);		/* from cnt_error */
 	CAST_OBJ_NOTNULL(sc, stv->priv, SMP_SC_MAGIC);
-	AN((bo->exp.ttl + bo->exp.grace + bo->exp.keep) > 0.);
+
+	/* Don't entertain already dead objects */
+	if ((bo->fetch_objcore->exp.ttl +
+	     bo->fetch_objcore->exp.grace +
+	     bo->fetch_objcore->exp.keep) <= 0.)
+		return (NULL);
 
 	ltot = IRNUP(sc, ltot);
 
@@ -546,7 +549,7 @@ smp_allocobj(struct stevedore *stv, struct busyobj *bo,
 	/* We have to do this somewhere, might as well be here... */
 	assert(sizeof so->hash == DIGEST_LEN);
 	memcpy(so->hash, oc->objhead->digest, DIGEST_LEN);
-	so->exp = bo->exp;
+	so->exp = bo->fetch_objcore->exp;
 	so->ptr = (uint8_t*)o - sc->base;
 	so->ban = BAN_Time(oc->ban);
 



From phk at FreeBSD.org  Thu Jun 26 15:03:03 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Thu, 26 Jun 2014 17:03:03 +0200
Subject: [master] 1a750f2 Divorce EXP_Inject() and EXP_Insert() -- they're
 quite different.
Message-ID: <E1X0BCZ-0007sU-DW@project.varnish-software.com>


commit 1a750f28818fe9e862ee8be887d8a485c5a565df
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu Jun 26 11:05:12 2014 +0000

    Divorce EXP_Inject() and EXP_Insert() -- they're quite different.

diff --git a/bin/varnishd/cache/cache_expire.c b/bin/varnishd/cache/cache_expire.c
index bbb19a3..ae5ebc7 100644
--- a/bin/varnishd/cache/cache_expire.c
+++ b/bin/varnishd/cache/cache_expire.c
@@ -137,19 +137,13 @@ EXP_Inject(struct objcore *oc, struct lru *lru, double when)
 
 	AZ(oc->exp_flags & (OC_EF_OFFLRU | OC_EF_INSERT | OC_EF_MOVE));
 	AZ(oc->exp_flags & OC_EF_DYING);
-	// AN(oc->flags & OC_F_BUSY);
-
-	if (lru == NULL)
-		lru = oc_getlru(oc);
+	AZ(oc->flags & OC_F_BUSY);
 	CHECK_OBJ_NOTNULL(lru, LRU_MAGIC);
 
 	Lck_Lock(&lru->mtx);
 	lru->n_objcore++;
 	oc->exp_flags |= OC_EF_OFFLRU | OC_EF_INSERT | OC_EF_EXP;
-	if (when < 0)
-		oc->exp_flags |= OC_EF_MOVE;
-	else
-		oc->timer_when = when;
+	oc->timer_when = when;
 	Lck_Unlock(&lru->mtx);
 
 	exp_mail_it(oc);
@@ -165,10 +159,25 @@ EXP_Inject(struct objcore *oc, struct lru *lru, double when)
 void
 EXP_Insert(struct objcore *oc)
 {
+	struct lru *lru;
 
 	CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
 	HSH_Ref(oc);
-	EXP_Inject(oc, NULL, -1);
+
+	AZ(oc->exp_flags & (OC_EF_OFFLRU | OC_EF_INSERT | OC_EF_MOVE));
+	AZ(oc->exp_flags & OC_EF_DYING);
+	AN(oc->flags & OC_F_BUSY);
+
+	lru = oc_getlru(oc);
+	CHECK_OBJ_NOTNULL(lru, LRU_MAGIC);
+
+	Lck_Lock(&lru->mtx);
+	lru->n_objcore++;
+	oc->exp_flags |= OC_EF_OFFLRU | OC_EF_INSERT | OC_EF_EXP;
+	oc->exp_flags |= OC_EF_MOVE;
+	Lck_Unlock(&lru->mtx);
+
+	exp_mail_it(oc);
 }
 
 /*--------------------------------------------------------------------



From phk at FreeBSD.org  Thu Jun 26 15:03:03 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Thu, 26 Jun 2014 17:03:03 +0200
Subject: [master] bb4d86c Eliminate obj->exp and use the one we have on
 objcore instead.
Message-ID: <E1X0BCZ-0007sd-HP@project.varnish-software.com>


commit bb4d86cb62d5b2ea10b00d8bc0089ec068899594
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu Jun 26 15:02:07 2014 +0000

    Eliminate obj->exp and use the one we have on objcore instead.
    
    Essentially identical to patch by Martin

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index 9854cc8..554dc8f 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -613,8 +613,6 @@ struct object {
 
 	ssize_t			len;
 
-	struct exp		exp;
-
 	/* VCL only variables */
 	double			last_modified;
 
@@ -830,7 +828,7 @@ void BAN_Init(void);
 void BAN_Shutdown(void);
 void BAN_NewObjCore(struct objcore *oc);
 void BAN_DestroyObj(struct objcore *oc);
-int BAN_CheckObject(struct object *o, struct req *sp);
+int BAN_CheckObject(const struct object *o, struct req *sp);
 void BAN_Reload(const uint8_t *ban, unsigned len);
 struct ban *BAN_TailRef(void);
 void BAN_Compile(void);
@@ -932,9 +930,9 @@ void EXP_Clr(struct exp *e);
 double EXP_Ttl(const struct req *, const struct exp*);
 double EXP_When(const struct exp *exp);
 void EXP_Insert(struct objcore *oc);
-void EXP_Inject(struct objcore *oc, struct lru *lru, double when);
+void EXP_Inject(struct objcore *oc, struct lru *lru);
 void EXP_Init(void);
-void EXP_Rearm(struct object *o, double now, double ttl, double grace,
+void EXP_Rearm(struct objcore *, double now, double ttl, double grace,
     double keep);
 void EXP_Touch(struct objcore *oc, double now);
 int EXP_NukeOne(struct busyobj *, struct lru *lru);
diff --git a/bin/varnishd/cache/cache_ban.c b/bin/varnishd/cache/cache_ban.c
index 99f2914..7bd3378 100644
--- a/bin/varnishd/cache/cache_ban.c
+++ b/bin/varnishd/cache/cache_ban.c
@@ -900,7 +900,7 @@ ban_evaluate(const uint8_t *bs, const struct http *objhttp,
  */
 
 static int
-ban_check_object(struct object *o, struct vsl_log *vsl,
+ban_check_object(const struct object *o, struct vsl_log *vsl,
     const struct http *req_http)
 {
 	struct ban *b;
@@ -964,13 +964,13 @@ ban_check_object(struct object *o, struct vsl_log *vsl,
 		oc->ban = NULL;
 		VSLb(vsl, SLT_ExpBan, "%u banned lookup", o->vxid);
 		VSC_C_main->bans_obj_killed++;
-		EXP_Rearm(o, o->exp.t_origin, 0, 0, 0);	// XXX fake now
+		EXP_Rearm(oc, oc->exp.t_origin, 0, 0, 0);	// XXX fake now
 		return (1);
 	}
 }
 
 int
-BAN_CheckObject(struct object *o, struct req *req)
+BAN_CheckObject(const struct object *o, struct req *req)
 {
 
 	return (ban_check_object(o, req->vsl, req->http) > 0);
@@ -1105,7 +1105,7 @@ ban_lurker_test_ban(struct worker *wrk, struct vsl_log *vsl, struct ban *bt,
 		}
 		if (i) {
 			VSLb(vsl, SLT_ExpBan, "%u banned by lurker", o->vxid);
-			EXP_Rearm(o, o->exp.t_origin, 0, 0, 0);	// XXX fake now
+			EXP_Rearm(oc, oc->exp.t_origin, 0, 0, 0); // XXX fake now
 			VSC_C_main->bans_lurker_obj_killed++;
 		}
 		(void)HSH_DerefObjCore(&wrk->stats, &oc);
diff --git a/bin/varnishd/cache/cache_expire.c b/bin/varnishd/cache/cache_expire.c
index ae5ebc7..e03df79 100644
--- a/bin/varnishd/cache/cache_expire.c
+++ b/bin/varnishd/cache/cache_expire.c
@@ -130,7 +130,7 @@ exp_mail_it(struct objcore *oc)
  */
 
 void
-EXP_Inject(struct objcore *oc, struct lru *lru, double when)
+EXP_Inject(struct objcore *oc, struct lru *lru)
 {
 
 	CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
@@ -143,7 +143,7 @@ EXP_Inject(struct objcore *oc, struct lru *lru, double when)
 	Lck_Lock(&lru->mtx);
 	lru->n_objcore++;
 	oc->exp_flags |= OC_EF_OFFLRU | OC_EF_INSERT | OC_EF_EXP;
-	oc->timer_when = when;
+	oc->timer_when = EXP_When(&oc->exp);
 	Lck_Unlock(&lru->mtx);
 
 	exp_mail_it(oc);
@@ -232,32 +232,29 @@ EXP_Touch(struct objcore *oc, double now)
  */
 
 void
-EXP_Rearm(struct object *o, double now, double ttl, double grace, double keep)
+EXP_Rearm(struct objcore *oc, double now, double ttl, double grace, double keep)
 {
-	struct objcore *oc;
 	struct lru *lru;
 	double when;
 
-	CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC);
-	oc = o->objcore;
 	CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
 	assert(oc->refcnt > 0);
 
 	AN(oc->exp_flags & OC_EF_EXP);
 
 	if (!isnan(ttl))
-		o->exp.ttl = now + ttl - o->exp.t_origin;
+		oc->exp.ttl = now + ttl - oc->exp.t_origin;
 	if (!isnan(grace))
-		o->exp.grace = grace;
+		oc->exp.grace = grace;
 	if (!isnan(keep))
-		o->exp.keep = keep;
+		oc->exp.keep = keep;
 
-	when = EXP_When(&o->exp);
+	when = EXP_When(&oc->exp);
 
 	VSL(SLT_ExpKill, 0, "EXP_Rearm p=%p E=%.9f e=%.9f f=0x%x", oc,
 	    oc->timer_when, when, oc->flags);
 
-	if (when > o->exp.t_origin && when > oc->timer_when)
+	if (when > oc->exp.t_origin && when > oc->timer_when)
 		return;
 
 	lru = oc_getlru(oc);
@@ -400,7 +397,7 @@ exp_inbox(struct exp_priv *ep, struct objcore *oc, double now)
 	if (flags & OC_EF_MOVE) {
 		o = oc_getobj(&ep->wrk->stats, oc);
 		CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC);
-		oc->timer_when = EXP_When(&o->exp);
+		oc->timer_when = EXP_When(&oc->exp);
 		oc_updatemeta(oc);
 	}
 
@@ -477,7 +474,7 @@ exp_expire(struct exp_priv *ep, double now)
 	CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC);
 	VSLb(&ep->vsl, SLT_ExpKill, "EXP_Expired x=%u t=%.0f",
 	    oc_getxid(&ep->wrk->stats, oc) & VSL_IDENTMASK,
-	    EXP_Ttl(NULL, &o->exp) - now);
+	    EXP_Ttl(NULL, &oc->exp) - now);
 	(void)HSH_DerefObjCore(&ep->wrk->stats, &oc);
 	return (0);
 }
diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c
index 1b824d5..8c9acbc 100644
--- a/bin/varnishd/cache/cache_fetch.c
+++ b/bin/varnishd/cache/cache_fetch.c
@@ -562,7 +562,8 @@ vbf_stp_fetch(struct worker *wrk, struct busyobj *bo)
 	VBO_setstate(bo, BOS_FINISHED);
 	VSLb_ts_busyobj(bo, "BerespBody", W_TIM_real(wrk));
 	if (bo->ims_obj != NULL)
-		EXP_Rearm(bo->ims_obj, bo->ims_obj->exp.t_origin, 0, 0, 0);
+		EXP_Rearm(bo->ims_obj->objcore,
+		    bo->ims_obj->objcore->exp.t_origin, 0, 0, 0);
 	return (F_STP_DONE);
 }
 
@@ -652,7 +653,8 @@ vbf_stp_condfetch(struct worker *wrk, struct busyobj *bo)
 
 	assert(al == bo->ims_obj->len);
 	assert(obj->len == al);
-	EXP_Rearm(bo->ims_obj, bo->ims_obj->exp.t_origin, 0, 0, 0);
+	EXP_Rearm(bo->ims_obj->objcore,
+	    bo->ims_obj->objcore->exp.t_origin, 0, 0, 0);
 
 	/* Recycle the backend connection before setting BOS_FINISHED to
 	   give predictable backend reuse behavior for varnishtest */
@@ -757,7 +759,8 @@ vbf_stp_fail(struct worker *wrk, struct busyobj *bo)
 	if (bo->fetch_objcore->exp_flags & OC_EF_EXP) {
 		/* Already unbusied - expire it */
 		AN(bo->fetch_obj);
-		EXP_Rearm(bo->fetch_obj, bo->fetch_obj->exp.t_origin, 0, 0, 0);
+		EXP_Rearm(bo->fetch_objcore,
+		    bo->fetch_objcore->exp.t_origin, 0, 0, 0);
 	}
 	wrk->stats.fetch_failed++;
 	VBO_setstate(bo, BOS_FAILED);
diff --git a/bin/varnishd/cache/cache_hash.c b/bin/varnishd/cache/cache_hash.c
index d6e404c..f03d901 100644
--- a/bin/varnishd/cache/cache_hash.c
+++ b/bin/varnishd/cache/cache_hash.c
@@ -420,17 +420,18 @@ HSH_Lookup(struct req *req, struct objcore **ocp, struct objcore **bocp,
 			continue;
 		}
 
+		if (oc->exp.ttl <= 0.)
+			continue;
+
 		o = oc_getobj(&wrk->stats, oc);
 		CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC);
 
-		if (o->exp.ttl <= 0.)
-			continue;
 		if (BAN_CheckObject(o, req))
 			continue;
 		if (o->vary != NULL && !VRY_Match(req, o->vary))
 			continue;
 
-		if (EXP_Ttl(req, &o->exp) >= req->t_req) {
+		if (EXP_Ttl(req, &oc->exp) >= req->t_req) {
 			/* If still valid, use it */
 			assert(oh->refcnt > 1);
 			assert(oc->objhead == oh);
@@ -442,12 +443,12 @@ HSH_Lookup(struct req *req, struct objcore **ocp, struct objcore **bocp,
 			*ocp = oc;
 			return (HSH_HIT);
 		}
-		if (o->exp.t_origin > exp_t_origin &&
+		if (oc->exp.t_origin > exp_t_origin &&
 		    !(oc->flags & OC_F_PASS)) {
 			/* record the newest object */
 			exp_oc = oc;
 			exp_o = o;
-			exp_t_origin = o->exp.t_origin;
+			exp_t_origin = oc->exp.t_origin;
 		}
 	}
 
@@ -560,7 +561,6 @@ double keep)
 {
 	struct objcore *oc, **ocp;
 	unsigned spc, nobj, n;
-	struct object *o;
 	double now;
 
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
@@ -595,12 +595,8 @@ double keep)
 	for (n = 0; n < nobj; n++) {
 		oc = ocp[n];
 		CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
-		o = oc_getobj(&wrk->stats, oc);
-		if (o == NULL)
-			continue;
-		CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC);
-		EXP_Rearm(o, now, ttl, grace, keep);
-		(void)HSH_DerefObj(&wrk->stats, &o);
+		EXP_Rearm(oc, now, ttl, grace, keep);
+		(void)HSH_DerefObjCore(&wrk->stats, &oc);
 	}
 	WS_Release(wrk->aws, 0);
 	Pool_PurgeStat(nobj);
@@ -618,7 +614,7 @@ HSH_Drop(struct worker *wrk, struct object **oo)
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 	AN(oo);
 	CHECK_OBJ_NOTNULL(*oo, OBJECT_MAGIC);
-	(*oo)->exp.ttl = -1.;
+	(*oo)->objcore->exp.ttl = -1.;
 	AZ(HSH_DerefObj(&wrk->stats, oo));
 }
 
diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c
index 401ebf1..bc5ffc2 100644
--- a/bin/varnishd/cache/cache_req_fsm.c
+++ b/bin/varnishd/cache/cache_req_fsm.c
@@ -121,7 +121,7 @@ cnt_deliver(struct worker *wrk, struct req *req)
 	   age. Truncate to zero in that case).
 	*/
 	http_PrintfHeader(req->resp, "Age: %.0f",
-	    fmax(0., req->t_prev - req->obj->exp.t_origin));
+	    fmax(0., req->t_prev - req->obj->objcore->exp.t_origin));
 
 	http_SetHeader(req->resp, "Via: 1.1 varnish-v4");
 
diff --git a/bin/varnishd/cache/cache_vrt_var.c b/bin/varnishd/cache/cache_vrt_var.c
index 9533f75..28b131f 100644
--- a/bin/varnishd/cache/cache_vrt_var.c
+++ b/bin/varnishd/cache/cache_vrt_var.c
@@ -481,10 +481,10 @@ VRT_r_##which##_##fld(const struct vrt_ctx *ctx)		\
 	return(0.0);						\
 }
 
-VRT_DO_EXP_R(obj, ctx->req->obj->exp, ttl,
-   (ctx->req->t_req - ctx->req->obj->exp.t_origin))
-VRT_DO_EXP_R(obj, ctx->req->obj->exp, grace, 0)
-VRT_DO_EXP_R(obj, ctx->req->obj->exp, keep, 0)
+VRT_DO_EXP_R(obj, ctx->req->obj->objcore->exp, ttl,
+   (ctx->req->t_req - ctx->req->obj->objcore->exp.t_origin))
+VRT_DO_EXP_R(obj, ctx->req->obj->objcore->exp, grace, 0)
+VRT_DO_EXP_R(obj, ctx->req->obj->objcore->exp, keep, 0)
 
 VRT_DO_EXP_L(beresp, ctx->bo->fetch_objcore->exp, ttl)
 VRT_DO_EXP_R(beresp, ctx->bo->fetch_objcore->exp, ttl, 0)
diff --git a/bin/varnishd/storage/stevedore.c b/bin/varnishd/storage/stevedore.c
index ce9c007..413f228 100644
--- a/bin/varnishd/storage/stevedore.c
+++ b/bin/varnishd/storage/stevedore.c
@@ -286,7 +286,6 @@ STV_MkObject(struct stevedore *stv, struct busyobj *bo,
 
 	HTTP_Setup(o->http, bo->ws_o, bo->vsl, SLT_ObjMethod);
 	o->http->magic = HTTP_MAGIC;
-	o->exp = bo->fetch_objcore->exp;
 	VTAILQ_INIT(&o->store);
 
 	o->objcore = bo->fetch_objcore;
diff --git a/bin/varnishd/storage/storage_persistent_silo.c b/bin/varnishd/storage/storage_persistent_silo.c
index 5627f3f..e470b1a 100644
--- a/bin/varnishd/storage/storage_persistent_silo.c
+++ b/bin/varnishd/storage/storage_persistent_silo.c
@@ -157,7 +157,8 @@ smp_load_seg(struct worker *wrk, const struct smp_sc *sc,
 		smp_init_oc(oc, sg, no);
 		oc->ban = BAN_RefBan(oc, so->ban, sc->tailban);
 		HSH_Insert(wrk, so->hash, oc);
-		EXP_Inject(oc, sg->lru, EXP_When(&so->exp));
+		oc->exp = so->exp;
+		EXP_Inject(oc, sg->lru);
 		sg->nobj++;
 	}
 	Pool_Sumstat(wrk);
@@ -462,7 +463,7 @@ smp_oc_getobj(struct dstat *ds, struct objcore *oc)
 			bad |= 0x100;
 
 		if(bad) {
-			o->exp.ttl = -1;
+			EXP_Clr(&oc->exp);
 			EXP_Clr(&so->exp);
 		}
 
@@ -472,7 +473,7 @@ smp_oc_getobj(struct dstat *ds, struct objcore *oc)
 		oc->flags &= ~OC_F_NEEDFIXUP;
 	}
 	Lck_Unlock(&sg->sc->mtx);
-	EXP_Rearm(o, NAN, NAN, NAN, NAN);	// XXX: Shouldn't be needed
+	EXP_Rearm(oc, NAN, NAN, NAN, NAN);	// XXX: Shouldn't be needed
 	return (o);
 }
 
@@ -495,11 +496,11 @@ smp_oc_updatemeta(struct objcore *oc)
 		/* Lock necessary, we might race close_seg */
 		Lck_Lock(&sg->sc->mtx);
 		so->ban = BAN_Time(oc->ban);
-		so->exp = o->exp;;
+		so->exp = oc->exp;
 		Lck_Unlock(&sg->sc->mtx);
 	} else {
 		so->ban = BAN_Time(oc->ban);
-		so->exp = o->exp;;
+		so->exp = oc->exp;
 	}
 }
 
diff --git a/bin/varnishtest/tests/p00000.vtc b/bin/varnishtest/tests/p00000.vtc
index eb9d29d..9e68fcd 100644
--- a/bin/varnishtest/tests/p00000.vtc
+++ b/bin/varnishtest/tests/p00000.vtc
@@ -3,6 +3,9 @@ varnishtest "Test Basic persistence"
 server s1 {
 	rxreq
 	txresp
+	accept
+	rxreq
+	txresp -status 700
 } -start
 
 shell "rm -f ${tmpdir}/_.per"



From fgsch at lodoss.net  Thu Jun 26 17:48:03 2014
From: fgsch at lodoss.net (Federico G. Schwindt)
Date: Thu, 26 Jun 2014 19:48:03 +0200
Subject: [master] 3ae6077 Spelling
Message-ID: <E1X0DmF-000679-TF@project.varnish-software.com>


commit 3ae607753919fa07bb5f95099a461accecf0825a
Author: Federico G. Schwindt <fgsch at lodoss.net>
Date:   Thu Jun 26 18:30:13 2014 +0100

    Spelling

diff --git a/doc/sphinx/tutorial/peculiarities.rst b/doc/sphinx/tutorial/peculiarities.rst
index 45f2f25..085edfd 100644
--- a/doc/sphinx/tutorial/peculiarities.rst
+++ b/doc/sphinx/tutorial/peculiarities.rst
@@ -3,7 +3,7 @@ Peculiarities
 -------------
 
 There are a couple of things that are different with Varnish Cache, as
-opposed to other programs. One thing you've already seen - VCL. In this section we provide a very quick tour of other pecularities you need to know about to get the most out of Varnish.
+opposed to other programs. One thing you've already seen - VCL. In this section we provide a very quick tour of other peculiarities you need to know about to get the most out of Varnish.
 
 Configuration
 ~~~~~~~~~~~~~
diff --git a/doc/sphinx/users-guide/command-line.rst b/doc/sphinx/users-guide/command-line.rst
index aa8c5be..854fb32 100644
--- a/doc/sphinx/users-guide/command-line.rst
+++ b/doc/sphinx/users-guide/command-line.rst
@@ -36,7 +36,7 @@ Here are some examples::
 	-a '[fe80::1]:80'
 	-a '0.0.0.0:8080,[::]:8081'
 
-.. XXX:brief explanation of some of the more comples examples perhaps? benc
+.. XXX:brief explanation of some of the more complex examples perhaps? benc
 
 If your webserver runs on the same machine, you will have to move
 it to another port number first.
diff --git a/doc/sphinx/users-guide/compression.rst b/doc/sphinx/users-guide/compression.rst
index cb95619..bff1ebb 100644
--- a/doc/sphinx/users-guide/compression.rst
+++ b/doc/sphinx/users-guide/compression.rst
@@ -9,7 +9,7 @@ encoding. *Before* 3.0, Varnish would never compress objects.
 In Varnish 4.0 compression defaults to "on", meaning that it tries to
 be smart and do the sensible thing.
 
-.. XXX:Heavy refactoring to VArnish 4 above. benc
+.. XXX:Heavy refactoring to Varnish 4 above. benc
 
 If you don't want Varnish tampering with the encoding you can disable
 compression all together by setting the parameter 'http_gzip_support' to
@@ -40,7 +40,7 @@ You can make Varnish compress content before storing it in cache in
 Please make sure that you don't try to compress content that is
 uncompressable, like jpgs, gifs and mp3. You'll only waste CPU
 cycles. You can also uncompress objects before storing it in memory by
-setting 'do_gunzip' to true but that will ususally not be the most sensible thing to do.
+setting 'do_gunzip' to true but that will usually not be the most sensible thing to do.
 Generally, Varnish doesn't use much CPU so it might make more sense to
 have Varnish spend CPU cycles compressing content than doing it in
 your web- or application servers, which are more likely to be
@@ -71,5 +71,5 @@ the page while delivering it.
 A random outburst
 ~~~~~~~~~~~~~~~~~
 
-Poul-Henning Kamp has written :ref:`phk_gzip` which talks abit more about how the
+Poul-Henning Kamp has written :ref:`phk_gzip` which talks a bit more about how the
 implementation works. 
diff --git a/doc/sphinx/users-guide/devicedetection.rst b/doc/sphinx/users-guide/devicedetection.rst
index 2a0cb21..9138816 100644
--- a/doc/sphinx/users-guide/devicedetection.rst
+++ b/doc/sphinx/users-guide/devicedetection.rst
@@ -68,7 +68,7 @@ VCL::
     }
     # req.http.X-UA-Device is copied by Varnish into bereq.http.X-UA-Device
 
-    # so, this is a bit conterintuitive. The backend creates content based on
+    # so, this is a bit counterintuitive. The backend creates content based on
     # the normalized User-Agent, but we use Vary on X-UA-Device so Varnish will
     # use the same cached object for all U-As that map to the same X-UA-Device.
     #
@@ -101,7 +101,7 @@ VCL::
 Example 2: Normalize the User-Agent string
 ''''''''''''''''''''''''''''''''''''''''''
 
-Another way of signaling the device type is to override or normalize the
+Another way of signalling the device type is to override or normalize the
 'User-Agent' header sent to the backend.
 
 For example::
diff --git a/doc/sphinx/users-guide/intro.rst b/doc/sphinx/users-guide/intro.rst
index f9c5587..927292a 100644
--- a/doc/sphinx/users-guide/intro.rst
+++ b/doc/sphinx/users-guide/intro.rst
@@ -65,7 +65,7 @@ simple to do simple things with::
 	}
 
 The CLI interface allows you to compile and load new VCL programs
-at any time, and you can switch betweem the loaded VCL programs
+at any time, and you can switch between the loaded VCL programs
 instantly, without restarting the child process and without missing
 a single HTTP request.
 
diff --git a/doc/sphinx/users-guide/operation-logging.rst b/doc/sphinx/users-guide/operation-logging.rst
index 4bfdca0..4dfa40f 100644
--- a/doc/sphinx/users-guide/operation-logging.rst
+++ b/doc/sphinx/users-guide/operation-logging.rst
@@ -51,7 +51,7 @@ session. Lines with the same number are coming from the same session
 and are being handled by the same thread. The second column is the
 *tag* of the log message. All log entries are tagged with a tag
 indicating what sort of activity is being logged. Tags starting with
-'Rx' indicate Varnish is recieving data and 'Tx' indicates sending data.
+'Rx' indicate Varnish is receiving data and 'Tx' indicates sending data.
 
 The third column tell us whether this is is data coming or going to
 the client ('c') or to/from the backend ('b'). The forth column is the



From fgsch at lodoss.net  Thu Jun 26 17:48:03 2014
From: fgsch at lodoss.net (Federico G. Schwindt)
Date: Thu, 26 Jun 2014 19:48:03 +0200
Subject: [master] 9074379 Sync with reality
Message-ID: <E1X0DmF-00067G-V6@project.varnish-software.com>


commit 90743796609c2a65624b3491ee49ce820b8b12bd
Author: Federico G. Schwindt <fgsch at lodoss.net>
Date:   Thu Jun 26 18:47:23 2014 +0100

    Sync with reality

diff --git a/doc/sphinx/reference/vcl.rst b/doc/sphinx/reference/vcl.rst
index 3f8a831..743689e 100644
--- a/doc/sphinx/reference/vcl.rst
+++ b/doc/sphinx/reference/vcl.rst
@@ -364,10 +364,6 @@ hash_data(input)
 new()
   Instanciate a new VCL object. Available in vcl_init.
 
-purge()
-  Invalidate all variants of the current object using purge. Available in 
-  vcl_miss and vcl_hit.
-
 return()
   End execution of the current VCL subroutine, and continue to the next step
   in the request handling state machine.
diff --git a/doc/sphinx/users-guide/purging.rst b/doc/sphinx/users-guide/purging.rst
index 0c9b7c9..17b02ee 100644
--- a/doc/sphinx/users-guide/purging.rst
+++ b/doc/sphinx/users-guide/purging.rst
@@ -39,7 +39,6 @@ following VCL in place::
 		  if (!client.ip ~ purge) {
 			  return(synth(405,"Not allowed."));
 		  }
-                  # jump to hit/miss
 		  return (purge);
 	  }
   }



From martin at varnish-software.com  Fri Jun 27 13:13:14 2014
From: martin at varnish-software.com (Martin Blix Grydeland)
Date: Fri, 27 Jun 2014 15:13:14 +0200
Subject: [master] 6498384 Fix memory leak on ESI
Message-ID: <E1X0Vxq-0001Us-FI@project.varnish-software.com>


commit 6498384f1b410f63c914c4e6ab2168e6b4ab5288
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Fri Jun 27 15:08:32 2014 +0200

    Fix memory leak on ESI

diff --git a/bin/varnishd/cache/cache_esi_fetch.c b/bin/varnishd/cache/cache_esi_fetch.c
index 3eb4c49..41bea3b 100644
--- a/bin/varnishd/cache/cache_esi_fetch.c
+++ b/bin/varnishd/cache/cache_esi_fetch.c
@@ -136,6 +136,7 @@ vfp_esi_end(struct busyobj *bo, struct vef_priv *vef, enum vfp_status retval)
 			    "ESI+Gzip Failed at the very end");
 	}
 	if (vef->ibuf != NULL)
+		free(vef->ibuf);
 	FREE_OBJ(vef);
 	return (retval);
 }



From fgsch at lodoss.net  Sun Jun 29 10:24:50 2014
From: fgsch at lodoss.net (Federico G. Schwindt)
Date: Sun, 29 Jun 2014 12:24:50 +0200
Subject: [master] 939f71d Sync with reality
Message-ID: <E1X1CHy-0006yY-W6@project.varnish-software.com>


commit 939f71d361386174eefcd3467df5e70123a01aee
Author: Federico G. Schwindt <fgsch at lodoss.net>
Date:   Sun Jun 29 11:23:03 2014 +0100

    Sync with reality
    
    beresp.cacheable is gone.

diff --git a/doc/sphinx/users-guide/vcl-built-in-subs.rst b/doc/sphinx/users-guide/vcl-built-in-subs.rst
index cb6396e..5280eb4 100644
--- a/doc/sphinx/users-guide/vcl-built-in-subs.rst
+++ b/doc/sphinx/users-guide/vcl-built-in-subs.rst
@@ -202,8 +202,7 @@ The `vcl_backend_response` subroutine may terminate with calling
 
   deliver
     Possibly insert the object into the cache, then deliver it to the
-    Control will eventually pass to `vcl_deliver`. Caching is dependant
-    on 'beresp.cacheable'.
+    Control will eventually pass to `vcl_deliver`.
 
   abandon
     Abandon the backend request and generates an error.



From phk at FreeBSD.org  Mon Jun 30 06:17:16 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Mon, 30 Jun 2014 08:17:16 +0200
Subject: [master] 37916dc Tag objcore with the stevedore which manages its
 storage
Message-ID: <E1X1Utw-0005eC-2r@project.varnish-software.com>


commit 37916dcc9c8a1fb60b44a0f46daf5efa10123132
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Jun 30 06:17:02 2014 +0000

    Tag objcore with the stevedore which manages its storage

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index 554dc8f..c4c20a1 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -418,6 +418,7 @@ struct objcore {
 	unsigned		magic;
 #define OBJCORE_MAGIC		0x4d301302
 	int			refcnt;
+	struct stevedore	*stevedore;
 	struct objcore_methods	*methods;
 	void			*priv;
 	uintptr_t		priv2;
diff --git a/bin/varnishd/storage/stevedore.c b/bin/varnishd/storage/stevedore.c
index 413f228..35f362a 100644
--- a/bin/varnishd/storage/stevedore.c
+++ b/bin/varnishd/storage/stevedore.c
@@ -290,6 +290,7 @@ STV_MkObject(struct stevedore *stv, struct busyobj *bo,
 
 	o->objcore = bo->fetch_objcore;
 
+	o->objcore->stevedore = stv;
 	o->objcore->methods = &default_oc_methods;
 	o->objcore->priv = o;
 	o->objcore->priv2 = (uintptr_t)stv;
@@ -382,6 +383,8 @@ STV_NewObject(struct busyobj *bo, const char *hint,
 
 	CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC);
 	CHECK_OBJ_NOTNULL(o->objstore, STORAGE_MAGIC);
+	CHECK_OBJ_NOTNULL(o->objcore, OBJCORE_MAGIC);
+	assert(o->objcore->stevedore == stv);
 	return (o);
 }
 



From phk at FreeBSD.org  Mon Jun 30 08:14:35 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Mon, 30 Jun 2014 10:14:35 +0200
Subject: [master] 96bdcca Add oc_methods to struct stevedore,
 and pull them from there.
Message-ID: <E1X1WjT-0007sr-4M@project.varnish-software.com>


commit 96bdccae881d2d57136674c1e9920ed581c12f4b
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Jun 30 07:48:55 2014 +0000

    Add oc_methods to struct stevedore, and pull them from there.

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index c4c20a1..55377d2 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -418,8 +418,8 @@ struct objcore {
 	unsigned		magic;
 #define OBJCORE_MAGIC		0x4d301302
 	int			refcnt;
-	struct stevedore	*stevedore;
-	struct objcore_methods	*methods;
+	const struct stevedore	*stevedore;
+	const struct objcore_methods	*methods;
 	void			*priv;
 	uintptr_t		priv2;
 	struct objhead		*objhead;
@@ -1100,6 +1100,7 @@ enum objiter_status {
 struct objiter *ObjIterBegin(struct worker *, struct object *);
 enum objiter_status ObjIter(struct objiter *, void **, ssize_t *);
 void ObjIterEnd(struct objiter **);
+void ObjTrimStore(struct objcore *oc, struct dstat *ds);
 
 /* cache_panic.c */
 void PAN_Init(void);
diff --git a/bin/varnishd/cache/cache_fetch_proc.c b/bin/varnishd/cache/cache_fetch_proc.c
index 429052d..0666c45 100644
--- a/bin/varnishd/cache/cache_fetch_proc.c
+++ b/bin/varnishd/cache/cache_fetch_proc.c
@@ -245,21 +245,8 @@ VFP_Fetch_Body(struct busyobj *bo, ssize_t est)
 
 	vfp_suck_fini(bo);
 
-	/*
-	 * Trim or delete the last segment, if any
-	 */
-
-	st = VTAILQ_LAST(&bo->fetch_obj->store, storagehead);
-	/* XXX: Temporary:  Only trim if we are not streaming */
-	if (st != NULL && !bo->do_stream) {
-		/* None of this this is safe under streaming */
-		if (st->len == 0) {
-			VTAILQ_REMOVE(&bo->fetch_obj->store, st, list);
-			STV_free(st);
-		} else if (st->len < st->space) {
-			STV_trim(st, st->len, 1);
-		}
-	}
+	if (!bo->do_stream)
+		ObjTrimStore(bo->fetch_objcore, bo->stats);
 }
 
 void
diff --git a/bin/varnishd/cache/cache_obj.c b/bin/varnishd/cache/cache_obj.c
index a92d29f..038a751 100644
--- a/bin/varnishd/cache/cache_obj.c
+++ b/bin/varnishd/cache/cache_obj.c
@@ -31,6 +31,7 @@
 #include <stdlib.h>
 
 #include "cache.h"
+#include "storage/storage.h"
 #include "hash/hash_slinger.h"
 
 struct objiter {
@@ -133,3 +134,26 @@ ObjIterEnd(struct objiter **oi)
 	FREE_OBJ((*oi));
 	*oi = NULL;
 }
+
+void
+ObjTrimStore(struct objcore *oc, struct dstat *ds)
+{
+	const struct stevedore *stv;
+	struct storage *st;
+	struct object *o;
+
+	CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
+	stv = oc->stevedore;
+	CHECK_OBJ_NOTNULL(stv, STEVEDORE_MAGIC);
+	o = oc_getobj(ds, oc);
+	CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC);
+	st = VTAILQ_LAST(&o->store, storagehead);
+	if (st == NULL)
+		return;
+	if (st->len == 0) {
+		VTAILQ_REMOVE(&o->store, st, list);
+		STV_free(st);
+	} else if (st->len < st->space) {
+		STV_trim(st, st->len, 1);
+	}
+}
diff --git a/bin/varnishd/cache/cache_panic.c b/bin/varnishd/cache/cache_panic.c
index 575d049..7294439 100644
--- a/bin/varnishd/cache/cache_panic.c
+++ b/bin/varnishd/cache/cache_panic.c
@@ -42,7 +42,7 @@
 #include "common/heritage.h"
 
 #include "cache_backend.h"
-#include "waiter/waiter.h"
+#include "storage/storage.h"
 #include "vcl.h"
 
 /*
@@ -235,6 +235,14 @@ pan_objcore(const char *typ, const struct objcore *oc)
 	VSB_printf(pan_vsp, "    refcnt = %d\n", oc->refcnt);
 	VSB_printf(pan_vsp, "    flags = 0x%x\n", oc->flags);
 	VSB_printf(pan_vsp, "    objhead = %p\n", oc->objhead);
+	VSB_printf(pan_vsp, "    stevedore = %p", oc->stevedore);
+	if (oc->stevedore != NULL) {
+		VSB_printf(pan_vsp, " (%s", oc->stevedore->name);
+		if (strlen(oc->stevedore->ident))
+			VSB_printf(pan_vsp, " %s", oc->stevedore->ident);
+		VSB_printf(pan_vsp, ")");
+	}
+	VSB_printf(pan_vsp, "\n");
 	VSB_printf(pan_vsp, "  }\n");
 }
 
diff --git a/bin/varnishd/storage/stevedore.c b/bin/varnishd/storage/stevedore.c
index 35f362a..a20c58b 100644
--- a/bin/varnishd/storage/stevedore.c
+++ b/bin/varnishd/storage/stevedore.c
@@ -95,14 +95,13 @@ default_oc_getlru(const struct objcore *oc)
 	return (stv->lru);
 }
 
-static struct objcore_methods default_oc_methods = {
+const struct objcore_methods default_oc_methods = {
 	.getobj = default_oc_getobj,
 	.getxid = default_oc_getxid,
 	.freeobj = default_oc_freeobj,
 	.getlru = default_oc_getlru,
 };
 
-
 /*--------------------------------------------------------------------
  */
 
@@ -291,7 +290,8 @@ STV_MkObject(struct stevedore *stv, struct busyobj *bo,
 	o->objcore = bo->fetch_objcore;
 
 	o->objcore->stevedore = stv;
-	o->objcore->methods = &default_oc_methods;
+	AN(stv->methods);
+	o->objcore->methods = stv->methods;
 	o->objcore->priv = o;
 	o->objcore->priv2 = (uintptr_t)stv;
 	VSLb(bo->vsl, SLT_Storage, "%s %s", stv->name, stv->ident);
@@ -385,6 +385,8 @@ STV_NewObject(struct busyobj *bo, const char *hint,
 	CHECK_OBJ_NOTNULL(o->objstore, STORAGE_MAGIC);
 	CHECK_OBJ_NOTNULL(o->objcore, OBJCORE_MAGIC);
 	assert(o->objcore->stevedore == stv);
+	AN(stv->methods);
+	AN(o->objcore->methods);
 	return (o);
 }
 
diff --git a/bin/varnishd/storage/storage.h b/bin/varnishd/storage/storage.h
index 47fabad..29531b0 100644
--- a/bin/varnishd/storage/storage.h
+++ b/bin/varnishd/storage/storage.h
@@ -60,6 +60,8 @@ typedef void storage_banexport_f(const struct stevedore *, const uint8_t *bans,
 
 extern storage_allocobj_f stv_default_allocobj;
 
+extern const struct objcore_methods default_oc_methods;
+
 /*--------------------------------------------------------------------*/
 
 struct stevedore {
@@ -78,6 +80,9 @@ struct stevedore {
 	storage_baninfo_f	*baninfo;	/* --//-- */
 	storage_banexport_f	*banexport;	/* --//-- */
 
+	const struct objcore_methods
+				*methods;
+
 	struct lru		*lru;
 
 #define VRTSTVVAR(nm, vtype, ctype, dval) storage_var_##ctype *var_##nm;
diff --git a/bin/varnishd/storage/storage_file.c b/bin/varnishd/storage/storage_file.c
index c48200c..3d0e006 100644
--- a/bin/varnishd/storage/storage_file.c
+++ b/bin/varnishd/storage/storage_file.c
@@ -541,13 +541,14 @@ smf_free(struct storage *s)
 /*--------------------------------------------------------------------*/
 
 const struct stevedore smf_stevedore = {
-	.magic	=	STEVEDORE_MAGIC,
-	.name	=	"file",
-	.init	=	smf_init,
-	.open	=	smf_open,
-	.alloc	=	smf_alloc,
-	.trim	=	smf_trim,
-	.free	=	smf_free,
+	.magic		=	STEVEDORE_MAGIC,
+	.name		=	"file",
+	.init		=	smf_init,
+	.open		=	smf_open,
+	.alloc		=	smf_alloc,
+	.trim		=	smf_trim,
+	.free		=	smf_free,
+	.methods	=	&default_oc_methods,
 };
 
 #ifdef INCLUDE_TEST_DRIVER
diff --git a/bin/varnishd/storage/storage_malloc.c b/bin/varnishd/storage/storage_malloc.c
index adb6cd2..9549281 100644
--- a/bin/varnishd/storage/storage_malloc.c
+++ b/bin/varnishd/storage/storage_malloc.c
@@ -245,13 +245,14 @@ sma_open(const struct stevedore *st)
 }
 
 const struct stevedore sma_stevedore = {
-	.magic	=	STEVEDORE_MAGIC,
-	.name	=	"malloc",
-	.init	=	sma_init,
-	.open	=	sma_open,
-	.alloc	=	sma_alloc,
-	.free	=	sma_free,
-	.trim	=	sma_trim,
+	.magic		=	STEVEDORE_MAGIC,
+	.name		=	"malloc",
+	.init		=	sma_init,
+	.open		=	sma_open,
+	.alloc		=	sma_alloc,
+	.free		=	sma_free,
+	.trim		=	sma_trim,
+	.methods	=	&default_oc_methods,
 	.var_free_space =	sma_free_space,
 	.var_used_space =	sma_used_space,
 };
diff --git a/bin/varnishd/storage/storage_persistent.c b/bin/varnishd/storage/storage_persistent.c
index 008afe7..a22e126 100644
--- a/bin/varnishd/storage/storage_persistent.c
+++ b/bin/varnishd/storage/storage_persistent.c
@@ -535,6 +535,8 @@ smp_allocobj(struct stevedore *stv, struct busyobj *bo,
 	ltot = st->len = st->space;
 
 	o = STV_MkObject(stv, bo, st->ptr, ltot, soc);
+	AN(bo->fetch_objcore->stevedore);
+	assert(bo->fetch_objcore->stevedore == stv);
 	CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC);
 	o->objstore = st;
 	bo->stats->n_object++;
@@ -601,6 +603,7 @@ const struct stevedore smp_stevedore = {
 	.signal_close = smp_signal_close,
 	.baninfo =	smp_baninfo,
 	.banexport =	smp_banexport,
+	.methods =	&smp_oc_methods,
 };
 
 /*--------------------------------------------------------------------
diff --git a/bin/varnishd/storage/storage_persistent.h b/bin/varnishd/storage/storage_persistent.h
index 2d46297..7d423e1 100644
--- a/bin/varnishd/storage/storage_persistent.h
+++ b/bin/varnishd/storage/storage_persistent.h
@@ -310,6 +310,7 @@ void smp_new_seg(struct smp_sc *sc);
 void smp_close_seg(struct smp_sc *sc, struct smp_seg *sg);
 void smp_init_oc(struct objcore *oc, struct smp_seg *sg, unsigned objidx);
 void smp_save_segs(struct smp_sc *sc);
+extern const struct objcore_methods smp_oc_methods;
 
 /* storage_persistent_subr.c */
 
diff --git a/bin/varnishd/storage/storage_persistent_silo.c b/bin/varnishd/storage/storage_persistent_silo.c
index e470b1a..c0180d8 100644
--- a/bin/varnishd/storage/storage_persistent_silo.c
+++ b/bin/varnishd/storage/storage_persistent_silo.c
@@ -154,6 +154,8 @@ smp_load_seg(struct worker *wrk, const struct smp_sc *sc,
 		AN(oc);
 		oc->flags |= OC_F_NEEDFIXUP;
 		oc->flags &= ~OC_F_BUSY;
+		oc->stevedore = sc->parent;
+		oc->methods = sc->parent->methods;
 		smp_init_oc(oc, sg, no);
 		oc->ban = BAN_RefBan(oc, so->ban, sc->tailban);
 		HSH_Insert(wrk, so->hash, oc);
@@ -546,7 +548,7 @@ smp_oc_getlru(const struct objcore *oc)
 	return (sg->lru);
 }
 
-static struct objcore_methods smp_oc_methods = {
+const struct objcore_methods smp_oc_methods = {
 	.getxid =		smp_oc_getxid,
 	.getobj =		smp_oc_getobj,
 	.updatemeta =		smp_oc_updatemeta,
@@ -562,5 +564,4 @@ smp_init_oc(struct objcore *oc, struct smp_seg *sg, unsigned objidx)
 
 	oc->priv = sg;
 	oc->priv2 = objidx;
-	oc->methods = &smp_oc_methods;
 }
diff --git a/bin/varnishd/storage/storage_umem.c b/bin/varnishd/storage/storage_umem.c
index 52d238d..1e93598 100644
--- a/bin/varnishd/storage/storage_umem.c
+++ b/bin/varnishd/storage/storage_umem.c
@@ -154,13 +154,14 @@ smu_open(const struct stevedore *st)
 }
 
 const struct stevedore smu_stevedore = {
-	.magic	=	STEVEDORE_MAGIC,
-	.name	=	"umem",
-	.init	=	smu_init,
-	.open	=	smu_open,
-	.alloc	=	smu_alloc,
-	.free	=	smu_free,
-	.trim	=	smu_trim,
+	.magic		=	STEVEDORE_MAGIC,
+	.name		=	"umem",
+	.init		=	smu_init,
+	.open		=	smu_open,
+	.alloc		=	smu_alloc,
+	.free		=	smu_free,
+	.trim		=	smu_trim,
+	.methods	=	&default_oc_methods,
 };
 
 #endif /* HAVE_UMEM_H */



From phk at FreeBSD.org  Mon Jun 30 08:14:35 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Mon, 30 Jun 2014 10:14:35 +0200
Subject: [master] 3bfe3fe De-inline the oc_*() methods by moving them into
 cache_obj.c.
Message-ID: <E1X1WjT-0007sy-6S@project.varnish-software.com>


commit 3bfe3fe793e775d3352de91618ba4da122a7dfa8
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Jun 30 08:14:04 2014 +0000

    De-inline the oc_*() methods by moving them into cache_obj.c.
    
    Rename suitably and always put the objcore argument first.

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index 55377d2..7ecaf8f 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -450,58 +450,6 @@ struct objcore {
 	struct ban		*ban;
 };
 
-static inline unsigned
-oc_getxid(struct dstat *ds, struct objcore *oc)
-{
-	CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
-
-	AN(oc->methods);
-	AN(oc->methods->getxid);
-	return (oc->methods->getxid(ds, oc));
-}
-
-static inline struct object *
-oc_getobj(struct dstat *ds, struct objcore *oc)
-{
-
-	CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
-	// AZ(oc->flags & OC_F_BUSY);
-	AN(oc->methods);
-	AN(oc->methods->getobj);
-	return (oc->methods->getobj(ds, oc));
-}
-
-static inline void
-oc_updatemeta(struct objcore *oc)
-{
-
-	CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
-	AN(oc->methods);
-	if (oc->methods->updatemeta != NULL)
-		oc->methods->updatemeta(oc);
-}
-
-static inline void
-oc_freeobj(struct dstat *ds, struct objcore *oc)
-{
-
-	AN(ds);
-	CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
-	AN(oc->methods);
-	AN(oc->methods->freeobj);
-	oc->methods->freeobj(ds, oc);
-}
-
-static inline struct lru *
-oc_getlru(const struct objcore *oc)
-{
-
-	CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
-	AN(oc->methods);
-	AN(oc->methods->getlru);
-	return (oc->methods->getlru(oc));
-}
-
 /* Busy Object structure ---------------------------------------------
  *
  * The busyobj structure captures the aspects of an object related to,
@@ -1100,7 +1048,12 @@ enum objiter_status {
 struct objiter *ObjIterBegin(struct worker *, struct object *);
 enum objiter_status ObjIter(struct objiter *, void **, ssize_t *);
 void ObjIterEnd(struct objiter **);
-void ObjTrimStore(struct objcore *oc, struct dstat *ds);
+void ObjTrimStore(struct objcore *, struct dstat *);
+unsigned ObjGetXID(struct objcore *, struct dstat *);
+struct object *ObjGetObj(struct objcore *, struct dstat *);
+void ObjUpdateMeta(struct objcore *);
+void ObjFreeObj(struct objcore *, struct dstat *);
+struct lru *ObjGetLRU(const struct objcore *);
 
 /* cache_panic.c */
 void PAN_Init(void);
diff --git a/bin/varnishd/cache/cache_ban.c b/bin/varnishd/cache/cache_ban.c
index 7bd3378..7388acb 100644
--- a/bin/varnishd/cache/cache_ban.c
+++ b/bin/varnishd/cache/cache_ban.c
@@ -958,7 +958,7 @@ ban_check_object(const struct object *o, struct vsl_log *vsl,
 
 	if (b == oc->ban) {	/* not banned */
 		oc->ban = b0;
-		oc_updatemeta(oc);
+		ObjUpdateMeta(oc);
 		return (0);
 	} else {
 		oc->ban = NULL;
@@ -1087,7 +1087,7 @@ ban_lurker_test_ban(struct worker *wrk, struct vsl_log *vsl, struct ban *bt,
 		oc = ban_lurker_getfirst(vsl, bt);
 		if (oc == NULL)
 			return;
-		o = oc_getobj(&wrk->stats, oc);
+		o = ObjGetObj(oc, &wrk->stats);
 		CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC);
 		i = 0;
 		VTAILQ_FOREACH_REVERSE_SAFE(bl, obans, banhead_s, l_list, bln) {
diff --git a/bin/varnishd/cache/cache_expire.c b/bin/varnishd/cache/cache_expire.c
index e03df79..dd3ed1a 100644
--- a/bin/varnishd/cache/cache_expire.c
+++ b/bin/varnishd/cache/cache_expire.c
@@ -168,7 +168,7 @@ EXP_Insert(struct objcore *oc)
 	AZ(oc->exp_flags & OC_EF_DYING);
 	AN(oc->flags & OC_F_BUSY);
 
-	lru = oc_getlru(oc);
+	lru = ObjGetLRU(oc);
 	CHECK_OBJ_NOTNULL(lru, LRU_MAGIC);
 
 	Lck_Lock(&lru->mtx);
@@ -201,7 +201,7 @@ EXP_Touch(struct objcore *oc, double now)
 	if (now - oc->last_lru < cache_param->lru_interval)
 		return;
 
-	lru = oc_getlru(oc);
+	lru = ObjGetLRU(oc);
 	CHECK_OBJ_NOTNULL(lru, LRU_MAGIC);
 
 	/*
@@ -257,7 +257,7 @@ EXP_Rearm(struct objcore *oc, double now, double ttl, double grace, double keep)
 	if (when > oc->exp.t_origin && when > oc->timer_when)
 		return;
 
-	lru = oc_getlru(oc);
+	lru = ObjGetLRU(oc);
 	CHECK_OBJ_NOTNULL(lru, LRU_MAGIC);
 
 	Lck_Lock(&lru->mtx);
@@ -334,14 +334,14 @@ EXP_NukeOne(struct busyobj *bo, struct lru *lru)
 	}
 
 	/* XXX: We could grab and return one storage segment to our caller */
-	o = oc_getobj(bo->stats, oc);
+	o = ObjGetObj(oc, bo->stats);
 	CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC);
 	STV_Freestore(o);
 
 	exp_mail_it(oc);
 
 	VSLb(bo->vsl, SLT_ExpKill, "LRU x=%u",
-	    oc_getxid(bo->stats, oc) & VSL_IDENTMASK);
+	    ObjGetXID(oc, bo->stats) & VSL_IDENTMASK);
 	AN(bo->stats);
 	AN(oc);
 	(void)HSH_DerefObjCore(bo->stats, &oc);
@@ -367,7 +367,7 @@ exp_inbox(struct exp_priv *ep, struct objcore *oc, double now)
 
 	// AZ(oc->flags & OC_F_BUSY);
 
-	lru = oc_getlru(oc);
+	lru = ObjGetLRU(oc);
 	CHECK_OBJ_NOTNULL(lru, LRU_MAGIC);
 
 	/* Evacuate our action-flags, and put it back on the LRU list */
@@ -395,10 +395,10 @@ exp_inbox(struct exp_priv *ep, struct objcore *oc, double now)
 	}
 
 	if (flags & OC_EF_MOVE) {
-		o = oc_getobj(&ep->wrk->stats, oc);
+		o = ObjGetObj(oc, &ep->wrk->stats);
 		CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC);
 		oc->timer_when = EXP_When(&oc->exp);
-		oc_updatemeta(oc);
+		ObjUpdateMeta(oc);
 	}
 
 	VSLb(&ep->vsl, SLT_ExpKill, "EXP_When p=%p e=%.9f f=0x%x", oc,
@@ -448,7 +448,7 @@ exp_expire(struct exp_priv *ep, double now)
 
 	VSC_C_main->n_expired++;
 
-	lru = oc_getlru(oc);
+	lru = ObjGetLRU(oc);
 	CHECK_OBJ_NOTNULL(lru, LRU_MAGIC);
 	Lck_Lock(&lru->mtx);
 	// AZ(oc->flags & OC_F_BUSY);
@@ -470,10 +470,10 @@ exp_expire(struct exp_priv *ep, double now)
 	assert(oc->timer_idx == BINHEAP_NOIDX);
 
 	CHECK_OBJ_NOTNULL(oc->objhead, OBJHEAD_MAGIC);
-	o = oc_getobj(&ep->wrk->stats, oc);
+	o = ObjGetObj(oc, &ep->wrk->stats);
 	CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC);
 	VSLb(&ep->vsl, SLT_ExpKill, "EXP_Expired x=%u t=%.0f",
-	    oc_getxid(&ep->wrk->stats, oc) & VSL_IDENTMASK,
+	    ObjGetXID(oc, &ep->wrk->stats) & VSL_IDENTMASK,
 	    EXP_Ttl(NULL, &oc->exp) - now);
 	(void)HSH_DerefObjCore(&ep->wrk->stats, &oc);
 	return (0);
diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c
index 8c9acbc..0980c64 100644
--- a/bin/varnishd/cache/cache_fetch.c
+++ b/bin/varnishd/cache/cache_fetch.c
@@ -536,7 +536,7 @@ vbf_stp_fetch(struct worker *wrk, struct busyobj *bo)
 	if (bo->failed && !bo->do_stream) {
 		assert(bo->state < BOS_STREAM);
 		if (bo->fetch_obj != NULL) {
-			oc_freeobj(bo->stats, bo->fetch_objcore);
+			ObjFreeObj(bo->fetch_objcore, bo->stats);
 			bo->fetch_obj = NULL;
 		}
 		return (F_STP_ERROR);
diff --git a/bin/varnishd/cache/cache_hash.c b/bin/varnishd/cache/cache_hash.c
index f03d901..70796e7 100644
--- a/bin/varnishd/cache/cache_hash.c
+++ b/bin/varnishd/cache/cache_hash.c
@@ -423,7 +423,7 @@ HSH_Lookup(struct req *req, struct objcore **ocp, struct objcore **bocp,
 		if (oc->exp.ttl <= 0.)
 			continue;
 
-		o = oc_getobj(&wrk->stats, oc);
+		o = ObjGetObj(oc, &wrk->stats);
 		CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC);
 
 		if (BAN_CheckObject(o, req))
@@ -801,7 +801,7 @@ HSH_DerefObjCore(struct dstat *ds, struct objcore **ocp)
 	AZ(oc->ban);
 
 	if (oc->methods != NULL)
-		oc_freeobj(ds, oc);
+		ObjFreeObj(oc, ds);
 	FREE_OBJ(oc);
 
 	ds->n_objectcore--;
diff --git a/bin/varnishd/cache/cache_obj.c b/bin/varnishd/cache/cache_obj.c
index 038a751..038255d 100644
--- a/bin/varnishd/cache/cache_obj.c
+++ b/bin/varnishd/cache/cache_obj.c
@@ -145,7 +145,7 @@ ObjTrimStore(struct objcore *oc, struct dstat *ds)
 	CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
 	stv = oc->stevedore;
 	CHECK_OBJ_NOTNULL(stv, STEVEDORE_MAGIC);
-	o = oc_getobj(ds, oc);
+	o = ObjGetObj(oc, ds);
 	CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC);
 	st = VTAILQ_LAST(&o->store, storagehead);
 	if (st == NULL)
@@ -157,3 +157,55 @@ ObjTrimStore(struct objcore *oc, struct dstat *ds)
 		STV_trim(st, st->len, 1);
 	}
 }
+
+unsigned
+ObjGetXID(struct objcore *oc, struct dstat *ds)
+{
+	CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
+
+	AN(oc->methods);
+	AN(oc->methods->getxid);
+	return (oc->methods->getxid(ds, oc));
+}
+
+struct object *
+ObjGetObj(struct objcore *oc, struct dstat *ds)
+{
+
+	CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
+	// AZ(oc->flags & OC_F_BUSY);
+	AN(oc->methods);
+	AN(oc->methods->getobj);
+	return (oc->methods->getobj(ds, oc));
+}
+
+void
+ObjUpdateMeta(struct objcore *oc)
+{
+
+	CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
+	AN(oc->methods);
+	if (oc->methods->updatemeta != NULL)
+		oc->methods->updatemeta(oc);
+}
+
+void
+ObjFreeObj(struct objcore *oc, struct dstat *ds)
+{
+
+	AN(ds);
+	CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
+	AN(oc->methods);
+	AN(oc->methods->freeobj);
+	oc->methods->freeobj(ds, oc);
+}
+
+struct lru *
+ObjGetLRU(const struct objcore *oc)
+{
+
+	CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
+	AN(oc->methods);
+	AN(oc->methods->getlru);
+	return (oc->methods->getlru(oc));
+}
diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c
index bc5ffc2..f987568 100644
--- a/bin/varnishd/cache/cache_req_fsm.c
+++ b/bin/varnishd/cache/cache_req_fsm.c
@@ -287,7 +287,7 @@ cnt_fetch(struct worker *wrk, struct req *req)
 		return (REQ_FSM_MORE);
 	}
 
-	req->obj = oc_getobj(&wrk->stats, req->objcore);
+	req->obj = ObjGetObj(req->objcore, &wrk->stats);
 	req->objcore = NULL;
 	req->err_code = http_GetStatus(req->obj->http);
 	req->req_step = R_STP_DELIVER;
@@ -381,7 +381,7 @@ cnt_lookup(struct worker *wrk, struct req *req)
 	AZ(oc->flags & OC_F_BUSY);
 	AZ(req->objcore);
 
-	o = oc_getobj(&wrk->stats, oc);
+	o = ObjGetObj(oc, &wrk->stats);
 	CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC);
 	req->obj = o;
 
diff --git a/bin/varnishd/storage/stevedore.c b/bin/varnishd/storage/stevedore.c
index a20c58b..7ddee95 100644
--- a/bin/varnishd/storage/stevedore.c
+++ b/bin/varnishd/storage/stevedore.c
@@ -53,7 +53,7 @@ default_oc_getxid(struct dstat *ds, struct objcore *oc)
 {
 	struct object *o;
 
-	o = oc_getobj(ds, oc);
+	o = ObjGetObj(oc, ds);
 	return (o->vxid);
 }
 



From phk at FreeBSD.org  Mon Jun 30 08:29:25 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Mon, 30 Jun 2014 10:29:25 +0200
Subject: [master] 2a7cd48 Eliminate oc->methods,
 access is now through the stevedore
Message-ID: <E1X1Wxp-00014F-A5@project.varnish-software.com>


commit 2a7cd48e79a769141c49c4b2a6c46178fbebded1
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Jun 30 08:29:03 2014 +0000

    Eliminate oc->methods, access is now through the stevedore

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index 7ecaf8f..ad4c3ab 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -419,7 +419,6 @@ struct objcore {
 #define OBJCORE_MAGIC		0x4d301302
 	int			refcnt;
 	const struct stevedore	*stevedore;
-	const struct objcore_methods	*methods;
 	void			*priv;
 	uintptr_t		priv2;
 	struct objhead		*objhead;
diff --git a/bin/varnishd/cache/cache_hash.c b/bin/varnishd/cache/cache_hash.c
index 70796e7..c626117 100644
--- a/bin/varnishd/cache/cache_hash.c
+++ b/bin/varnishd/cache/cache_hash.c
@@ -636,7 +636,7 @@ HSH_Fail(struct objcore *oc)
 	 * will not consider this oc, or an object hung of the oc
 	 * so that it can consider it.
 	 */
-	assert((oc->flags & OC_F_BUSY) || (oc->methods != NULL));
+	assert((oc->flags & OC_F_BUSY) || (oc->stevedore != NULL));
 
 	Lck_Lock(&oh->mtx);
 	oc->flags |= OC_F_FAILED;
@@ -675,7 +675,7 @@ HSH_Unbusy(struct dstat *ds, struct objcore *oc)
 	oh = oc->objhead;
 	CHECK_OBJ(oh, OBJHEAD_MAGIC);
 
-	AN(oc->methods);
+	AN(oc->stevedore);
 	AN(oc->flags & OC_F_BUSY);
 	assert(oh->refcnt > 0);
 
@@ -800,7 +800,7 @@ HSH_DerefObjCore(struct dstat *ds, struct objcore **ocp)
 	BAN_DestroyObj(oc);
 	AZ(oc->ban);
 
-	if (oc->methods != NULL)
+	if (oc->stevedore != NULL)
 		ObjFreeObj(oc, ds);
 	FREE_OBJ(oc);
 
diff --git a/bin/varnishd/cache/cache_obj.c b/bin/varnishd/cache/cache_obj.c
index 038255d..adb35f6 100644
--- a/bin/varnishd/cache/cache_obj.c
+++ b/bin/varnishd/cache/cache_obj.c
@@ -135,6 +135,16 @@ ObjIterEnd(struct objiter **oi)
 	*oi = NULL;
 }
 
+static const struct objcore_methods *
+obj_getmethods(const struct objcore *oc)
+{
+
+	CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
+	CHECK_OBJ_NOTNULL(oc->stevedore, STEVEDORE_MAGIC);
+	AN(oc->stevedore->methods);
+	return (oc->stevedore->methods);
+}
+
 void
 ObjTrimStore(struct objcore *oc, struct dstat *ds)
 {
@@ -143,6 +153,7 @@ ObjTrimStore(struct objcore *oc, struct dstat *ds)
 	struct object *o;
 
 	CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
+	AN(ds);
 	stv = oc->stevedore;
 	CHECK_OBJ_NOTNULL(stv, STEVEDORE_MAGIC);
 	o = ObjGetObj(oc, ds);
@@ -161,51 +172,49 @@ ObjTrimStore(struct objcore *oc, struct dstat *ds)
 unsigned
 ObjGetXID(struct objcore *oc, struct dstat *ds)
 {
-	CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
+	const struct objcore_methods *m = obj_getmethods(oc);
 
-	AN(oc->methods);
-	AN(oc->methods->getxid);
-	return (oc->methods->getxid(ds, oc));
+	AN(ds);
+	AN(m->getxid);
+	return (m->getxid(ds, oc));
 }
 
 struct object *
 ObjGetObj(struct objcore *oc, struct dstat *ds)
 {
+	const struct objcore_methods *m = obj_getmethods(oc);
 
-	CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
-	// AZ(oc->flags & OC_F_BUSY);
-	AN(oc->methods);
-	AN(oc->methods->getobj);
-	return (oc->methods->getobj(ds, oc));
+	AN(ds);
+	AN(m->getobj);
+	return (m->getobj(ds, oc));
 }
 
 void
 ObjUpdateMeta(struct objcore *oc)
 {
+	const struct objcore_methods *m = obj_getmethods(oc);
 
-	CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
-	AN(oc->methods);
-	if (oc->methods->updatemeta != NULL)
-		oc->methods->updatemeta(oc);
+	if (m->updatemeta != NULL)
+		m->updatemeta(oc);
 }
 
 void
 ObjFreeObj(struct objcore *oc, struct dstat *ds)
 {
+	const struct objcore_methods *m = obj_getmethods(oc);
 
 	AN(ds);
 	CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
-	AN(oc->methods);
-	AN(oc->methods->freeobj);
-	oc->methods->freeobj(ds, oc);
+	AN(m->freeobj);
+	m->freeobj(ds, oc);
 }
 
 struct lru *
 ObjGetLRU(const struct objcore *oc)
 {
+	const struct objcore_methods *m = obj_getmethods(oc);
 
 	CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
-	AN(oc->methods);
-	AN(oc->methods->getlru);
-	return (oc->methods->getlru(oc));
+	AN(m->getlru);
+	return (m->getlru(oc));
 }
diff --git a/bin/varnishd/storage/stevedore.c b/bin/varnishd/storage/stevedore.c
index 7ddee95..c9520f8 100644
--- a/bin/varnishd/storage/stevedore.c
+++ b/bin/varnishd/storage/stevedore.c
@@ -77,7 +77,7 @@ default_oc_freeobj(struct dstat *ds, struct objcore *oc)
 	AN(ds);
 	CAST_OBJ_NOTNULL(o, oc->priv, OBJECT_MAGIC);
 	oc->priv = NULL;
-	oc->methods = NULL;
+	oc->stevedore = NULL;
 	o->magic = 0;
 
 	STV_Freestore(o);
@@ -291,7 +291,6 @@ STV_MkObject(struct stevedore *stv, struct busyobj *bo,
 
 	o->objcore->stevedore = stv;
 	AN(stv->methods);
-	o->objcore->methods = stv->methods;
 	o->objcore->priv = o;
 	o->objcore->priv2 = (uintptr_t)stv;
 	VSLb(bo->vsl, SLT_Storage, "%s %s", stv->name, stv->ident);
@@ -386,7 +385,6 @@ STV_NewObject(struct busyobj *bo, const char *hint,
 	CHECK_OBJ_NOTNULL(o->objcore, OBJCORE_MAGIC);
 	assert(o->objcore->stevedore == stv);
 	AN(stv->methods);
-	AN(o->objcore->methods);
 	return (o);
 }
 
diff --git a/bin/varnishd/storage/storage_persistent_silo.c b/bin/varnishd/storage/storage_persistent_silo.c
index c0180d8..5119467 100644
--- a/bin/varnishd/storage/storage_persistent_silo.c
+++ b/bin/varnishd/storage/storage_persistent_silo.c
@@ -155,7 +155,6 @@ smp_load_seg(struct worker *wrk, const struct smp_sc *sc,
 		oc->flags |= OC_F_NEEDFIXUP;
 		oc->flags &= ~OC_F_BUSY;
 		oc->stevedore = sc->parent;
-		oc->methods = sc->parent->methods;
 		smp_init_oc(oc, sg, no);
 		oc->ban = BAN_RefBan(oc, so->ban, sc->tailban);
 		HSH_Insert(wrk, so->hash, oc);
@@ -420,7 +419,7 @@ smp_oc_getobj(struct dstat *ds, struct objcore *oc)
 	int bad;
 
 	/* Some calls are direct, but they should match anyway */
-	assert(oc->methods->getobj == smp_oc_getobj);
+	assert(oc->stevedore->methods->getobj == smp_oc_getobj);
 
 	CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
 	if (ds == NULL)



From fgsch at lodoss.net  Mon Jun 30 11:56:41 2014
From: fgsch at lodoss.net (Federico G. Schwindt)
Date: Mon, 30 Jun 2014 13:56:41 +0200
Subject: [master] 1919eea Spelling
Message-ID: <E1X1aCP-0003vH-Bo@project.varnish-software.com>


commit 1919eeaac90db027602a4204f1bd9784c1804bbf
Author: Federico G. Schwindt <fgsch at lodoss.net>
Date:   Mon Jun 30 12:56:01 2014 +0100

    Spelling

diff --git a/include/tbl/vsc_f_main.h b/include/tbl/vsc_f_main.h
index 8f25d39..a88a12d 100644
--- a/include/tbl/vsc_f_main.h
+++ b/include/tbl/vsc_f_main.h
@@ -271,7 +271,7 @@ VSC_F(threads_created,		uint64_t, 0, 'c', info,
 )
 
 VSC_F(threads_destroyed,	uint64_t, 0, 'c', info,
-    "Threads destoryed",
+    "Threads destroyed",
 	"Total number of threads destroyed in all pools."
 )
 



From fgsch at lodoss.net  Mon Jun 30 12:13:15 2014
From: fgsch at lodoss.net (Federico G. Schwindt)
Date: Mon, 30 Jun 2014 14:13:15 +0200
Subject: [master] 161b42b Workaround a problem in recent libedit versions
Message-ID: <E1X1aSR-0006DP-J3@project.varnish-software.com>


commit 161b42bb9b407e0cb4a8e86c80465ffdbfadae4c
Author: Federico G. Schwindt <fgsch at lodoss.net>
Date:   Mon Jun 30 13:08:42 2014 +0100

    Workaround a problem in recent libedit versions
    
    Reinstall the readline handler when we are called in order to clear the
    input line. Tested in a variety of versions by scn and myself.
    
    Fixes #1531

diff --git a/bin/varnishadm/varnishadm.c b/bin/varnishadm/varnishadm.c
index 60f36ba..0c2ecc1 100644
--- a/bin/varnishadm/varnishadm.c
+++ b/bin/varnishadm/varnishadm.c
@@ -184,6 +184,7 @@ static void send_line(char *l)
 		cli_write(_line_sock, l);
 		cli_write(_line_sock, "\n");
 		add_history(l);
+		rl_callback_handler_install("varnish> ", send_line);
 	} else {
 		RL_EXIT(0);
 	}



From fgsch at lodoss.net  Mon Jun 30 12:23:26 2014
From: fgsch at lodoss.net (Federico G. Schwindt)
Date: Mon, 30 Jun 2014 14:23:26 +0200
Subject: [master] 5e1e35a Fix incorrect representation when using reals
Message-ID: <E1X1acI-0007G8-K4@project.varnish-software.com>


commit 5e1e35a5ff5f984e2a8a85c9b1d6b5e44b5caa07
Author: Federico G. Schwindt <fgsch at lodoss.net>
Date:   Mon Jun 30 13:18:11 2014 +0100

    Fix incorrect representation when using reals
    
    Fixes #1532

diff --git a/bin/varnishtest/tests/r01532.vtc b/bin/varnishtest/tests/r01532.vtc
new file mode 100644
index 0000000..c189f55
--- /dev/null
+++ b/bin/varnishtest/tests/r01532.vtc
@@ -0,0 +1,20 @@
+varnishtest "Incorrect representation when using reals - #1532"
+
+server s1 {
+	rxreq
+	txresp
+} -start
+
+varnish v1 -vcl+backend {
+	import ${vmod_std};
+
+	sub vcl_deliver {
+		set resp.http.x-foo = std.real2time(1140618699.00);
+	}
+} -start
+
+client c1 {
+	txreq
+	rxresp
+	expect resp.http.x-foo == "Wed, 22 Feb 2006 14:31:39 GMT"
+} -run
diff --git a/lib/libvcc/vcc_expr.c b/lib/libvcc/vcc_expr.c
index 8e4e234..0d39073 100644
--- a/lib/libvcc/vcc_expr.c
+++ b/lib/libvcc/vcc_expr.c
@@ -749,7 +749,7 @@ vcc_expr4(struct vcc *tl, struct expr **e, enum var_type fmt)
 			e1 = vcc_mk_expr(BYTES, "%.1f", d);
 			ERRCHK(tl);
 		} else if (fmt == REAL) {
-			e1 = vcc_mk_expr(REAL, "%g", vcc_DoubleVal(tl));
+			e1 = vcc_mk_expr(REAL, "%f", vcc_DoubleVal(tl));
 			ERRCHK(tl);
 		} else {
 			e1 = vcc_mk_expr(INT, "%.*s", PF(tl->t));



From fgsch at lodoss.net  Mon Jun 30 12:32:24 2014
From: fgsch at lodoss.net (Federico G. Schwindt)
Date: Mon, 30 Jun 2014 14:32:24 +0200
Subject: [3.0] 2c0f1ba Workaround a problem in recent libedit versions
Message-ID: <E1X1aky-0008HZ-F4@project.varnish-software.com>


commit 2c0f1ba2cba0e888ae089394fca5c4b8e0ddd528
Author: Federico G. Schwindt <fgsch at lodoss.net>
Date:   Mon Jun 30 13:08:42 2014 +0100

    Workaround a problem in recent libedit versions
    
    Reinstall the readline handler when we are called in order to clear the
    input line. Tested in a variety of versions by scn and myself.
    
    Fixes #1531

diff --git a/bin/varnishadm/varnishadm.c b/bin/varnishadm/varnishadm.c
index 005c62f..8a4ce45 100644
--- a/bin/varnishadm/varnishadm.c
+++ b/bin/varnishadm/varnishadm.c
@@ -177,6 +177,7 @@ static void send_line(char *l)
 		cli_write(_line_sock, l);
 		cli_write(_line_sock, "\n");
 		add_history(l);
+		rl_callback_handler_install("varnish> ", send_line);
 	} else {
 		RL_EXIT(0);
 	}



From phk at FreeBSD.org  Mon Jun 30 19:37:13 2014
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Mon, 30 Jun 2014 21:37:13 +0200
Subject: [master] b2b6e32 Fix a race spotted by Nils: We have to hold a ref to
 the busyobj if we want to examine it in any detail.
Message-ID: <E1X1hO5-0005nW-OG@project.varnish-software.com>


commit b2b6e329da78e465720bec9a26a905e8a77cab92
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Jun 30 19:34:47 2014 +0000

    Fix a race spotted by Nils:  We have to hold a ref to the
    busyobj if we want to examine it in any detail.
    
    Hold that ref over all of V1D_Deliver() (different from Nils
    patch) to make sure we have a consistent view on streaming
    vs. no streaming throughout.
    
    Also use the bo to clean up a hack related to ESI includes.
    
    Diagnosed by:	Nils Goroll

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index ad4c3ab..038510d 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -818,7 +818,7 @@ HTTP1_Chunked(struct http_conn *htc, intptr_t *priv, const char **error,
 
 /* cache_http1_deliver.c */
 unsigned V1D_FlushReleaseAcct(struct req *req);
-void V1D_Deliver(struct req *);
+void V1D_Deliver(struct req *, struct busyobj *);
 void V1D_Deliver_Synth(struct req *req);
 
 
diff --git a/bin/varnishd/cache/cache_http1_deliver.c b/bin/varnishd/cache/cache_http1_deliver.c
index 1bdcef7..73aea69 100644
--- a/bin/varnishd/cache/cache_http1_deliver.c
+++ b/bin/varnishd/cache/cache_http1_deliver.c
@@ -87,7 +87,7 @@ v1d_range_bytes(struct req *req, enum vdp_action act, const void *ptr,
 /*--------------------------------------------------------------------*/
 
 static void
-v1d_dorange(struct req *req, const char *r)
+v1d_dorange(struct req *req, struct busyobj *bo, const char *r)
 {
 	ssize_t len, low, high, has_low;
 
@@ -96,8 +96,8 @@ v1d_dorange(struct req *req, const char *r)
 	assert(http_GetStatus(req->obj->http) == 200);
 
 	/* We must snapshot the length if we're streaming from the backend */
-	if (req->obj->objcore->busyobj != NULL)
-		len = VBO_waitlen(req->obj->objcore->busyobj, -1);
+	if (bo != NULL)
+		len = VBO_waitlen(bo, -1);
 	else
 		len = req->obj->len;
 
@@ -228,10 +228,11 @@ V1D_FlushReleaseAcct(struct req *req)
 }
 
 void
-V1D_Deliver(struct req *req)
+V1D_Deliver(struct req *req, struct busyobj *bo)
 {
 	char *r;
 	enum objiter_status ois;
+	ssize_t l;
 
 	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
 	CHECK_OBJ_NOTNULL(req->obj, OBJECT_MAGIC);
@@ -247,7 +248,7 @@ V1D_Deliver(struct req *req)
 		req->res_mode &= ~RES_LEN;
 		http_Unset(req->resp, H_Content_Length);
 		req->wantbody = 0;
-	} else if (req->obj->objcore->busyobj == NULL) {
+	} else if (bo == NULL) {
 		/* XXX: Not happy with this convoluted test */
 		req->res_mode |= RES_LEN;
 		if (!(req->obj->objcore->flags & OC_F_PASS) ||
@@ -310,7 +311,7 @@ V1D_Deliver(struct req *req)
 	    http_GetStatus(req->resp) == 200) {
 		http_SetHeader(req->resp, "Accept-Ranges: bytes");
 		if (http_GetHdr(req->http, H_Range, &r))
-			v1d_dorange(req, r);
+			v1d_dorange(req, bo, r);
 	}
 
 	if (req->res_mode & RES_ESI)
@@ -334,8 +335,11 @@ V1D_Deliver(struct req *req)
 	} else if (req->res_mode & RES_ESI) {
 		ESI_Deliver(req);
 	} else if (req->res_mode & RES_ESI_CHILD && req->gzip_resp) {
-		while (req->obj->objcore->busyobj)
-			(void)usleep(10000);
+		l = -1;
+		while (req->obj->objcore->busyobj) {
+			assert(bo != NULL);
+			l = VBO_waitlen(bo, l);
+		}
 		ESI_DeliverChild(req);
 	} else if (req->res_mode & RES_GUNZIP ||
 	    (req->res_mode & RES_ESI_CHILD &&
@@ -421,7 +425,7 @@ V1D_Deliver_Synth(struct req *req)
 	    http_GetStatus(req->resp) == 200) {
 		http_SetHeader(req->resp, "Accept-Ranges: bytes");
 		if (http_GetHdr(req->http, H_Range, &r))
-			v1d_dorange(req, r);
+			v1d_dorange(req, NULL, r);
 	}
 
 	WRW_Reserve(req->wrk, &req->sp->fd, req->vsl, req->t_prev);
@@ -441,8 +445,6 @@ V1D_Deliver_Synth(struct req *req)
 #if 0
 	XXX: Missing pretend GZIP for esi-children
 	} else if (req->res_mode & RES_ESI_CHILD && req->gzip_resp) {
-		while (req->obj->objcore->busyobj)
-			(void)usleep(10000);
 		ESI_DeliverChild(req);
 #endif
 	} else {
diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c
index f987568..161c5e8 100644
--- a/bin/varnishd/cache/cache_req_fsm.c
+++ b/bin/varnishd/cache/cache_req_fsm.c
@@ -87,6 +87,7 @@ DOT deliver:deliver:s -> DONE [style=bold,color=blue]
 static enum req_fsm_nxt
 cnt_deliver(struct worker *wrk, struct req *req)
 {
+	struct busyobj *bo;
 
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
@@ -154,7 +155,12 @@ cnt_deliver(struct worker *wrk, struct req *req)
 		req->wantbody = 0;
 	}
 
-	V1D_Deliver(req);
+	/* Grab a ref to the bo if there is one, and hand it down */
+	bo = HSH_RefBusy(req->obj->objcore);
+	V1D_Deliver(req, bo);
+	if (bo != NULL)
+		VBO_DerefBusyObj(req->wrk, &bo);
+
 	VSLb_ts_req(req, "Resp", W_TIM_real(wrk));
 
 	if (http_HdrIs(req->resp, H_Connection, "close"))